Revert "pfBlockerNG Beta v0.99"

This reverts commit b3ce3bde07750e25fabca14faf18c0e5f0eb74dc.

1 files changed, 0 insertions, 928 deletions

diff --git a/config/pfblockerng/pfblockerng.sh b/config/pfblockerng/pfblockerng.sh
deleted file mode 100644
index 335df167..00000000
--- a/config/pfblockerng/pfblockerng.sh
+++ /dev/null
@@ -1,928 +0,0 @@
-#!/bin/sh
-# pfBlockerNG IP Reputation Script - By BBcan177@gmail.com - 04-12-14
-# Copyright (C) 2014 BBcan177@gmail.com
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License Version 2 as
-# published by the Free Software Foundation.  You may not use, modify or
-# distribute this program under any other version of the GNU General
-# Public License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-
-now=$(/bin/date +%m/%d/%y' '%T)
-mtype=$(/usr/bin/uname -m);
-pfs_version="$(cut -c 1-3 /etc/version)"
-
-# Application Paths
-pathgrepcidr="/usr/pbi/pfblockerng-$mtype/bin/grepcidr"
-pathgeoip="/usr/pbi/pfblockerng-$mtype/bin/geoiplookup"
-
-pathtar=/usr/bin/tar
-pathgunzip=/usr/bin/gunzip
-pathpfctl=/sbin/pfctl
-
-# Script Arguments
-alias=$2
-max=$3
-dedup=$4
-cc=$(echo $5 | sed 's/,/, /g')
-ccwhite=$(echo $6 | tr '[A-Z]' '[a-z]')
-ccblack=$(echo $7 | tr '[A-Z]' '[a-z]')
-etblock=$(echo $8 | sed 's/,/, /g')
-etmatch=$(echo $9 | sed 's/,/, /g')
-
-# File Locations
-pathgeoipdat=/var/db/pfblockerng/GeoIP.dat
-pfbsuppression=/var/db/pfblockerng/pfbsuppression.txt
-masterfile=/var/db/pfblockerng/masterfile
-mastercat=/var/db/pfblockerng/mastercat
-geoiplog=/var/log/pfblockerng/geoip.log
-errorlog=/var/log/pfblockerng/error.log
-
-# Folder Locations
-etdir=/var/db/pfblockerng/ET
-tmpxlsx=/tmp/xlsx/
-
-pfbdeny=/var/db/pfblockerng/deny/
-pfborig=/var/db/pfblockerng/original/
-pfbmatch=/var/db/pfblockerng/match/
-pfbpermit=/var/db/pfblockerng/permit/
-pfbnative=/var/db/pfblockerng/native/
-pfsense_alias_dir=/var/db/aliastables/
-
-# Store "Match" d-dedups in matchdedup.txt file
-matchdedup=matchdedup.txt
-
-tempfile=/tmp/pfbtempfile
-tempfile2=/tmp/pfbtempfile2
-dupfile=/tmp//pfbduptemp
-dedupfile=/tmp/pfbdeduptemp
-addfile=/tmp/pfBaddfile
-syncfile=/tmp/pfbsyncfile
-matchfile=/tmp/pfbmatchfile
-tempmatchfile=/tmp/pfbtempmatchfile
-
-if [ ! -f $masterfile ]; then touch $masterfile; fi
-if [ ! -f $mastercat ]; then touch $mastercat; fi
-if [ ! -f $tempfile ]; then touch $tempfile; fi
-if [ ! -f $tempfile2 ]; then touch $tempfile2; fi
-if [ ! -f $dupfile ]; then touch $dupfile; fi
-if [ ! -f $dedupfile ]; then touch $dedupfile; fi
-if [ ! -f $addfile ]; then touch $addfile; fi
-if [ ! -f $syncfile ]; then touch $syncfile; fi
-if [ ! -f $matchfile ]; then touch $matchfile; fi
-if [ ! -f $tempmatchfile ]; then touch $tempmatchfile; fi
-if [ ! -d $pfbmatch ]; then mkdir $pfbmatch; fi
-if [ ! -d $etdir ]; then mkdir $etdir; fi
-if [ ! -d $tmpxlsx ]; then mkdir $tmpxlsx; fi
-
-##########
-# Process to condense an IP range if a "Max" amount of IP addresses are found in a /24 range per Alias Group.
-process24() {
-
-if [ ! -x $pathgeoip ]; then 
-	echo "Process24 - Application [ GeoIP ] Not found. Can't proceed."
-	echo "Process24 - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog
-	exit
-fi
-
-# Download MaxMind GeoIP.dat Binary on first Install.
-if [ ! -f $pathgeoipdat ]; then
-	echo "Downloading [ MaxMind GeoIP.dat ] [ $now ]" >> $geoiplog
-	/usr/local/pkg/pfblockerng/geoipupdate.sh bu
-fi
-# Exit if GeoIP.dat is not found.
-if [ ! -f $pathgeoipdat ]; then 
-	echo "Process24 - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed."
-	echo "Process24 - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >>  $errorlog
-	exit
-fi
-
-count=$(grep -c ^ $pfbdeny$alias".txt")
-echo; echo "Original File Count [ $count ]"
-
-grep -Ev "^(#|$)" $pfbdeny$alias".txt" | sort | uniq > $tempfile
-> $dupfile; > $tempfile2; > $matchfile; > $tempmatchfile
-data="$(cut -d '.' -f 1-3 $tempfile | awk -v max="$max" '{a[$0]++}END{for(i in a){if(a[i] > max){print i}}}')"
-count=$(echo "$data" | grep -c ^); mcount=0; dcount=0; safe=0
-if [ "$data" == "" ]; then count=0; fi
-matchoutfile="match"$header".txt"
-# Classify Repeat Offenders by Country Code
-if [ -f $pathgeoipdat ]; then
-	for ip in $data; do
-		ccheck=$($pathgeoip -f $pathgeoipdat "$ip.1" | cut -c 24-25)
-		case "$cc" in
-			*$ccheck*)
-				safe=$(($safe + 1))
-				if [ "$ccwhite" == "match" -o "$ccblack" == "match" ]; then
-					echo "$ip." >> $matchfile
-				fi
-				;;
-			*)
-				echo "$ip." >> $dupfile
-				;;
-		esac
-	done
-else
-	echo; echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping p24 Process"; echo
-	echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping p24 Process [ $now ]" >> $errorlog
-fi
-# Collect Match File Details
-if [ -s "$matchfile" -a ! "$dedup" == "on" -a "$ccwhite" == "match" ]; then
-	mon=$(sed -e 's/^/^/' -e 's/\./\\\./g' $matchfile)
-	for ip in $mon; do
-	grep $ip $tempfile >> $tempfile2
-	done
-	mcount=$(grep -c ^ $tempfile2)
-	if [ "$ccwhite" == "match" ]; then
-		sed 's/$/0\/24/' $matchfile >> $tempmatchfile
-	sed 's/^/\!/' $tempfile2 >> $tempmatchfile
-	fi
-fi
-
-# If no Matches found remove previous Matchoutfile if exists.
-if [ ! -s "$tempmatchfile" -a -f $matchoutfile ]; then rm -r $matchoutfile; fi
-# Move Match File to the Match Folder by Individual Blocklist Name
-if [ -s "$tempmatchfile" ]; then mv -f $tempmatchfile $pfbmatch$matchoutfile; fi
-
-# Find Repeat Offenders in each individual Blocklist Outfile
-if [ -s "$dupfile" ]; then
-	> $tempfile2
-	dup=$(sed -e 's/^/^/' -e 's/\./\\\./g' $dupfile)
-	for ip in $dup; do
-		grep $ip $tempfile >> $tempfile2
-	done
-	dcount=$(grep -c ^ $tempfile2)
-	if [ "$ccblack" == "block" ]; then
-		awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile2 $tempfile > $pfbdeny$alias".txt"
-		sed 's/$/0\/24/' $dupfile >> $pfbdeny$alias".txt"
-	elif [ "$ccblack" == "match" ]; then
-		sed 's/$/0\/24/' $dupfile >> $tempmatchfile
-		sed 's/^/\!/' $tempfile2 >> $tempmatchfile
-	else
-		:
-	fi
-fi
-if [ "$count" == "0" -a "$safe" == "0" ]; then echo; echo " Process /24 Stats [ $alias ] [ $now ] "; echo "------------------------------------------------"; fi
-if [ "$count" == "0" ]; then echo "Found [ 0 ] IP range(s) over the threshold of [ $max ] p24 - CC Blacklist"; fi
-if [ "$safe" == "0" ]; then echo "Found [ 0 ] IP range(s) over the threshold of [ $max ] p24 - CC Whitelist"; fi
- 
-if [ -s "$dupfile" -o -s "$matchfile" ]; then
-echo
-echo " Process /24 Stats [ $alias ] [ $now ]"
-echo "--------------------------------------------------------"
-echo "Found [ $count ] IP range(s) over the threshold of [ $max ] on the CC Blacklist"
-echo "Found [ $safe ] IP range(s) over the threshold of [ $max ] on the CC Whitelist"
-echo
-echo "Found [ $dcount ] CC Blacklisted IP Address(es) are being set to [ $ccblack ]"
-# Skip Match Process if dedup=yes as it will create duplicates
-if [ "$dedup" == "on" ]; then mcount=Skipped; fi
-echo "Found [ $mcount ] CC Whitelisted IP Address(es) are being set to [ $ccwhite ]"
-if [ "$ccblack" == "block" ]; then
-	echo; echo "Removed the following IP Ranges"
-	cat $dupfile | tr '\n' '|'; echo
-else
-	echo "Skipped, CCBlack set to [ $ccblack ]"
-fi
-sort $pfbdeny$alias".txt" | uniq > $tempfile; mv -f $tempfile $pfbdeny$alias".txt"
-echo "-------------------------------------------------------"
-cocount=$(grep -cv "^1\.1\.1\.1" $pfbdeny$alias".txt")
-echo "Post /24 Count   [ $cocount ]"; echo
-fi
-}
-
-
-##########
-process255() {
-# Remove IPs if exists over 255 IPs in a Range and replace with a single /24 Block
-cp $pfbdeny$alias".txt" $tempfile; > $dedupfile
-
-data255="$(cut -d '.' -f 1-3 $tempfile | awk '{a[$0]++}END{for(i in a){if(a[i] > 255){print i}}}')"
-if [ ! -z "$data255" ]; then
-	for ip in $data255; do
-		ii=$(echo "^$ip" | sed 's/\./\\\./g')
-		grep $ii $tempfile >> $dedupfile
-	done
-	awk 'FNR==NR{a[$0];next}!($0 in a)' $dedupfile $tempfile > $pfbdeny$alias".txt"
-	for ip in $data255; do echo $ip"0/24" >> $pfbdeny$alias".txt"; done
-fi
-}
-
-
-##########
-continent() {
-
-dupcheck=yes
-# Check if Masterfile is Empty
-hcheck=$(grep -c ^ $masterfile); if [ "$hcheck" -eq "0" ]; then dupcheck=no; fi
-# Check if Alias exists in Masterfile
-lcheck=$(grep -m 1 "$alias " $masterfile ); if [ "$lcheck" == "" ]; then dupcheck=no; fi
-
-if [ "$dupcheck" == "yes" ]; then
-	# Grep Alias with a trailing Space character
-	grep "$alias[[:space:]]" $masterfile > $tempfile
-	awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
-	cut -d' ' -f2 $masterfile > $mastercat
-fi
-
-grep -Ev "^(#|$)" $pfbdeny$alias".txt" | sort | uniq > $tempfile
-
-if [ ! "$hcheck" -eq "0" ]; then
-	$pathgrepcidr -vf $mastercat $pfbdeny$alias".txt" > $tempfile; mv -f $tempfile $pfbdeny$alias".txt"
-fi
-
-sed -e 's/^/'$alias' /' $pfbdeny$alias".txt" >> $masterfile
-cut -d' ' -f2 $masterfile > $mastercat
-
-countg=$(grep -c ^ $pfborig$alias".orig")
-countm=$(grep -c "$alias " $masterfile); counto=$(grep -c ^ $pfbdeny$alias".txt")
-if [ "$countm" == "$counto" ]; then sanity="Passed"; else sanity=" ==> FAILED <== "; fi
-echo "----------------------------------------------------------"
-echo; echo " Post Duplication count [ $now ]"
-echo "----------------------------------------------------------"
-printf "%-10s %-10s %-10s %-30s\n" "Original" "Masterfile" "Outfile" "Sanity Check"
-echo "----------------------------------------------------------"
-printf "%-10s %-10s %-10s %-30s\n" "$countg" "$countm" "$counto" " [ $sanity ]"
-echo "----------------------------------------------------------"
-}
-
-
-##########
-# Process to remove Suppressed Entries and RFC 1918 and Misc IPs on each downloaded Blocklist
-suppress() {
-
-if [ ! -x $pathgrepcidr ]; then
-	echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]"
-	echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" >> errorlog
-	exit
-fi
-
-if [ -e "$pfbsuppression" ] && [ -s "$pfbsuppression" ]; then
-	# Find '/24' Blocked IPs that are single addresses in the Suppressed IP Address List.
-	# These '/24' Are converted to single Addresses excluding the Suppressed IPs.
-	data="$(cat $pfbsuppression)"
-	if [ ! -z "$data" -a ! -z "$cc" ]; then
-		# Loop thru each Updated List to remove Suppression and RFC1918 Addresses
-		if [ "$cc" == "suppressheader" ]; then
-			echo "===[ Suppression Stats ]========================================"; echo
-			printf "%-20s %-10s %-10s %-10s %-10s\n" "List" "Pre" "RFC1918" "Suppress" "Masterfile"
-			echo "----------------------------------------------------------------"
-			exit
-		fi
-
-		for i in $cc; do
-			counter=0
-			> $dupfile
-			alias=$(echo "${i%|*}")
-			pfbfolder=$(echo "${i#*|}")
-
-			if [ ! "$alias" == "" ]; then
-				# Count (PRE)	
-				countg=$(grep -c ^ $pfbfolder$alias".txt")
-
-				grep -Ev "^(192\.168|10\.|172\.1[6789]\.|172\.2[0-9]\.|172\.3[01]\.|127\.0\.0\.1|0\.0\.0\.0|#|$)" $pfbfolder$alias".txt" |
-					sort | uniq > $tempfile	
-				# Count (Post RFC1918)
-				countm=$(grep -c ^ $tempfile)
-
-				for ip in $data; do
-					found=""; ddcheck="";
-					iptrim=$(echo $ip | cut -d '.' -f 1-3)
-					mask=$(echo $ip | cut -d"/" -f2)
-					found=$(grep -m1 $iptrim".0/24" $tempfile)
-					# If a Suppression is '/32' and a Blocklist has a full '/24' Block execute the following.
-					if [ ! "$found" == "" -a "$mask" == "32" ]; then
-						echo " Suppression $alias: $iptrim.0/24"
-						octet4=$(echo $ip | cut -d '.' -f 4 | sed 's/\/.*//')
-						dcheck=$(grep $iptrim".0/24" $dupfile)
-						if [ "$dcheck" == "" ]; then
-							echo $iptrim".0" >> $tempfile
-							echo $iptrim".0/24" >> $dupfile
-							counter=$(($counter + 1))
-							# Add Individual IP addresses from Range excluding Suppressed IP
-							for i in $(/usr/bin/jot 255); do
-								if [ "$i" != "$octet4" ]; then
-									echo $iptrim"."$i >> $tempfile
-									counter=$(($counter + 1))
-								fi
-							done
-						fi
-					fi
-				done
-				if [ -s $dupfile ]; then
-					# Remove '/24' Suppressed Ranges
-					awk 'FNR==NR{a[$0];next}!($0 in a)' $dupfile $tempfile > $tempfile2; mv -f $tempfile2 $tempfile
-				fi
-				# Remove All other Suppressions from Lists
-				$pathgrepcidr -vf $pfbsuppression $tempfile > $pfbfolder$alias".txt"
-				# Update Masterfiles. Don't execute if Duplication Process is Disabled
-				if [ "$dedup" == "x" ]; then
-					# Dont execute if Alias doesnt exist in Masterfile
-					lcheck=$(grep -m1 "$alias " $masterfile)
-					if [ ! "$lcheck" == "" ]; then
-						# Replace Masterfile with changes to List.
-						grep "$alias[[:space:]]" $masterfile > $tempfile
-						awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
-						sed -e 's/^/'$alias' /' $pfbfolder$alias".txt" >> $masterfile
-						cut -d' ' -f2 $masterfile > $mastercat
-					fi
-				fi
-				countk=$(grep -c ^ $masterfile)
-				countx=$(grep -c ^ $pfbfolder$alias".txt")
-				counto=$(($countx - $counter))
-				printf "%-20s %-10s %-10s %-10s %-10s\n" "$alias" "$countg" "$countm" "$counto" "$countk"
-			fi
-		done
-	fi
-else
-	if [ "$cc" == "suppressheader" ]; then
-		echo "===[ Suppression Stats ]========================================"; echo
-		printf "%-20s %-10s %-10s %-10s %-10s\n" "List" "Pre" "RFC1918" "Suppress" "Masterfile"
-		echo "----------------------------------------------------------------"
-		exit
-	fi
-	for i in $cc; do
-		alias=$(echo "${i%|*}")
-		pfbfolder=$(echo "${i#*|}")
-
-		if [ ! "$alias" == "" ]; then
-			countg=$(grep -c ^ $pfbfolder$alias".txt")
-			grep -Ev "^(192\.168|10\.|172\.1[6789]\.|172\.2[0-9]\.|172\.3[01]\.|127\.0\.0\.1|0\.0\.0\.0|#|$)" $pfbfolder$alias".txt" |
-				sort | uniq > $tempfile; mv -f $tempfile $pfbfolder$alias".txt"
-			countx=$(grep -c ^ $pfbfolder$alias".txt")
-			# Update Masterfiles. Don't execute if Duplication Process is Disabled or if No Suppression Changes Found
-			if [ "$dedup" == "x" -a "$countg" != "$countx" ]; then
-				# Dont execute if Alias doesnt exist in Masterfile
-				lcheck=$(grep -m1 "$alias " $masterfile)
-				if [ ! "$lcheck" == "" ]; then
-					# Replace Masterfile with changes to List.
-					grep "$alias[[:space:]]" $masterfile > $tempfile
-					awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
-					sed -e 's/^/'$alias' /' $pfbfolder$alias".txt" >> $masterfile
-					cut -d' ' -f2 $masterfile > $mastercat
-				fi
-			fi
-			countm=$(grep -c ^ $pfbfolder$alias".txt")
-			counto=" - "
-			countk=$(grep -c ^ $masterfile)
-			printf "%-20s %-10s %-10s %-10s %-10s\n" "$alias" "$countg" "$countm" "$counto" "$countk"
-		fi
-	done
-fi
-}
-
-
-##########
-# Process to remove Duplicate Entries on each downloaded Blocklist Individually
-duplicate() {
-
-if [ ! -x $pathgrepcidr ]; then
-	echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]"
-	echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" >> errorlog
-	exit
-fi
-
-dupcheck=yes
-# Check if Masterfile is Empty
-hcheck=$(grep -cv "^$" $masterfile); if [ "$hcheck" -eq "0" ]; then dupcheck=no; fi
-# Check if Alias exists in Masterfile
-lcheck=$(grep -m1 "$alias " $masterfile); if [ "$lcheck" == "" ]; then dupcheck=no; fi
-
-if [ "$dupcheck" == "yes" ]; then
-	# Grep Alias with a trailing Space character
-	grep "$alias[[:space:]]" $masterfile > $tempfile
-	awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
-	cut -d' ' -f2 $masterfile > $mastercat
-fi
-
-grep -Ev "^(#|$)" $pfbdeny$alias".txt" | sort | uniq > $tempfile; mv -f $tempfile $pfbdeny$alias".txt"
-
-if [ ! "$hcheck" -eq "0" ]; then
-	$pathgrepcidr -vf $mastercat $pfbdeny$alias".txt" > $tempfile; mv -f $tempfile $pfbdeny$alias".txt"
-fi
-
-sed -e 's/^/'$alias' /' $pfbdeny$alias".txt" >> $masterfile
-cut -d' ' -f2 $masterfile > $mastercat
-
-countg=$(grep -c ^ $pfborig$alias".orig")
-countm=$(grep -c "$alias " $masterfile); counto=$(grep -c ^ $pfbdeny$alias".txt")
-if [ "$countm" == "$counto" ]; then sanity="Passed"; else sanity=" ==> FAILED <== "; fi
-echo "----------------------------------------------------------"
-printf "%-10s %-10s %-10s %-30s\n" "Original" "Masterfile" "Outfile" " [ Post Duplication count ]"
-echo "----------------------------------------------------------"
-printf "%-10s %-10s %-10s %-30s\n" "$countg" "$countm" "$counto" " [ $sanity ]"
-echo "----------------------------------------------------------"
-}
-
-
-##########
-# De-Duplication utilizing MaxMind GeoIP Country Code Whitelisting ("dmax" variable)
-deduplication() {
-
-if [ ! -x $pathgeoip ]; then
-	echo "d-duplication - Application [ GeoIP ] Not found. Can't proceed."
-	echo "d-duplication - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog
-	exit
-fi
-
-# Download MaxMind GeoIP.dat on first Install.
-if [ ! -f $pathgeoipdat ]; then 
-	echo "Downloading [ MaxMind GeoIP.dat ] [ $now ]" >> $geoiplog
-	/usr/local/pkg/pfblockerng/geoipupdate.sh bu
-fi
-
-# Exit if GeoIP.dat is not found
-if [ ! -f $pathgeoipdat ]; then
-	echo "d-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed."
-	echo "d-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog
-	exit
-fi
-
-> $tempfile; > $tempfile2; > $dupfile; > $addfile; > $dedupfile; > $matchfile; > $tempmatchfile; count=0; dcount=0; mcount=0; mmcount=0
-echo; echo "Querying for Repeat Offenders"
-data="$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | cut -d '.' -f 1-3 $pfbdeny*.txt |
-	awk -v max="$max" '{a[$0]++}END{for(i in a){if(a[i] > max){print i}}}' | grep -v "^1\.1\.1")"
-count=$(echo "$data" | grep -c ^)
-if [ "$data" == "" ]; then count=0; fi
-safe=0
-# Classify Repeat Offenders by Country Code
-if [ -f $pathgeoipdat ]; then
-	echo "Classifying Repeat Offenders by GeoIP"
-	for ip in $data; do
-		ccheck=$($pathgeoip -f $pathgeoipdat "$ip.1" | cut -c 24-25)
-		case "$cc" in
-			*$ccheck*)
-				safe=$(($safe + 1))
-				if [ "$ccwhite" == "match" -o "$ccblack" == "match" ]; then
-					echo "$ip." >> $matchfile
-				fi
-				;;
-			*)
-				echo "$ip." >> $dupfile
-				;;
-		esac
-	done
-else
-	echo; echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping d-dedup Process"; echo
-	echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping d-dedup Process [ $now ]" >> $errorlog
-fi
-if [ -s "$matchfile" -a "$ccwhite" == "match" ]; then
-	echo "Processing [ Match ] IPs"
-	match=$(sed -e 's/^/^/' -e 's/\./\\\./g' $matchfile)
-	for mfile in $match; do
-		grep $mfile $pfbdeny*.txt >> $tempfile
-	done
-	sed 's/$/0\/24/' $matchfile >> $tempmatchfile
-	sed -e 's/.*://' -e 's/^/\!/' $tempfile >> $tempmatchfile
-	mv -f $tempmatchfile $pfbmatch$matchdedup
-	mcount=$(grep -c ^ $tempfile)
-	mmcount=$(($mcount + $mmcount))
-fi
-# Find Repeat Offenders in each individual Blocklist Outfile
-if [ -s "$dupfile" ]; then
-	echo "Processing [ Block ] IPs"
-	dup=$(cat $dupfile)
-	for ip in $dup; do
-		pcount=1; ii=$(echo "^$ip" | sed 's/\./\\\./g')
-		list=$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | xargs grep -al $ii)
-		for blfile in $list; do
-			header=$(echo "${blfile##*/}" | cut -d '.' -f1)
-			grep $ii $blfile > $tempfile
-			if [ "$ccblack" == "block" ]; then
-				awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $blfile > $tempfile2; mv -f $tempfile2 $blfile
-				if [ "$pcount" -eq "1" ]; then
-					echo $ip"0/24" >> $blfile
-					echo $header" "$ip >> $dedupfile
-					echo $header" "$ip"0/24" >> $addfile
-					pcount=2
-				else
-					echo $header" "$ip >> $dedupfile
-				fi
-			else	
-				if [ "$pcount" -eq "1" ]; then
-					matchoutfile="match"$header".txt"
-					echo $ip"0/24" >> $pfbmatch$matchoutfile
-					sed 's/^/\!/' $tempfile >> $pfbmatch$matchoutfile
-					mcount=$(grep -c ^ $pfbmatch$matchoutfile)
-					mmcount=$(($mcount + $mmcount))
-					pcount=2
-				fi
-			fi
-		done
-	done
-	# Remove Repeat Offenders in Masterfiles
-	if [ -s "$dedupfile" ]; then
-		echo "Removing   [ Block ] IPs"
-		> $tempfile; > $tempfile2
-		sed 's/\./\\\./g' $dedupfile > $tempfile2
-		while IFS=' ' read -r ips; do grep "$ips" $masterfile >> $tempfile; done < $tempfile2
-		dcount=$(grep -c ^ $tempfile)
-		awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
-		cat $addfile >> $masterfile
-		cut -d' ' -f2 $masterfile > $mastercat
-	fi
-fi
-
-echo; echo "d-Duplication Process  [ $now ]"; echo "------------------------------------------------"
-echo; echo "Found [ $count ] IP range(s) over the threshold of dmax= [ $max ]"
-echo "Found [ $safe ] IP range(s) classified as Whitelisted"
-echo; echo "Found [ $dcount ] CC Blacklisted IP Address(es) are being set to [ $ccblack ]"
-echo "Found [ $mmcount ] CC Whitelisted IP Address(es) are being set to [ $ccwhite ]"; echo
-if [ -s "$addfile" ]; then
-        echo; echo "Removed the following IP Ranges"
-        sed -e 's/^.* //' -e 's/0\/24//' $addfile | tr '\n' '|'; echo
-fi
-count=$(grep -c ^ $masterfile)
-echo " [ Post d-Deduplication count ]  [ $count ]"; echo
-
-# Write "1.1.1.1" to empty Final Blocklist Files
-emptyfiles=$(find $pfbdeny -size 0)
-for i in $emptyfiles; do echo "1.1.1.1" > $i; done
-}
-
-
-##########
-# Process to perform a final De-Duplication on all of the BlockLists (Excluding Country Whitelist) ("pmax" variable).
-pdeduplication(){
-
-if [ ! -x $pathgeoip ]; then
-	echo "p-duplication - Application [ GeoIP ] Not found. Can't proceed."
-	echo "p-duplication - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog
-	exit
-fi
-
-# Download MaxMind GeoIP.dat on first Install.
-if [ ! -f $pathgeoipdat ]; then
-	echo "Downloading [ MaxMind GeoIP.dat ] [ $now ]" >> $geoiplog
-	/usr/local/pkg/pfblockerng/geoipupdate.sh bu
-fi
-# Exit if GeoIP.dat is not found.
-if [ ! -f $pathgeoipdat ]; then
-	echo "p-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed."
-	echo "p-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog
-	exit
-fi
-
-> $tempfile; > $tempfile2; > $dupfile; > $addfile; > $dedupfile; count=0; dcount=0
-echo; echo "====================================================================="
-echo; echo "Querying for Repeat Offenders"
-data="$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | cut -d '.' -f 1-3 $pfbdeny*.txt |
-	awk -v max="$max" '{a[$0]++}END{for(i in a){if(a[i] > max){print i}}}' | grep -v "^1\.1\.1")"
-count=$(echo "$data" | grep -c ^)
-if [ "$data" == "" ]; then count=0; fi
-# Find Repeat Offenders in each individual Blocklist Outfile
-echo "Processing [ Block ] IPs"
-for ip in $data; do
-	pcount=1; ii=$(echo "^$ip." | sed 's/\./\\\./g')
-	list=$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | xargs grep -al $ii)
-	for blfile in $list; do
-		header=$(echo "${blfile##*/}" | cut -d '.' -f1)
-		grep $ii $blfile > $tempfile
-		awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $blfile > $tempfile2; mv -f $tempfile2 $blfile
-		if [ "$pcount" -eq "1" ]; then
-			echo $ip".0/24" >> $blfile
-			echo $header" $ip." >> $dedupfile
-			echo $header" "$ip".0/24" >> $addfile
-			pcount=2
-		else
-			echo $header" $ip." >> $dedupfile
-		fi
-	done
-done
-# Remove Repeat Offenders in Masterfile
-if [ -s "$dedupfile" ]; then
-	echo "Removing   [ Block ] IPs"
-	> $tempfile; > $tempfile2
-	sed 's/\./\\\./g' $dedupfile > $tempfile2
-	while IFS=' ' read -r ips; do grep "$ips" $masterfile >> $tempfile; done < $tempfile2
-	dcount=$(grep -c ^ $tempfile)
-	awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
-	cat $addfile >> $masterfile
-	cut -d' ' -f2 $masterfile > $mastercat
-fi
-
-echo; echo "p-Duplication Process  [ $now ]"; echo "------------------------------------------------"
-echo "Found [ $dcount ] IP Address(es) are being set to [ block ]"
-if [ -s "$addfile" ]; then
-	echo; echo "Removed the following IP Ranges"
-	sed -e 's/^.* //' -e 's/0\/24//' $addfile | tr '\n' '|'; echo
-fi
-count=$(grep -c ^ $masterfile)
-echo; echo " [ Post p-Deduplication count ]  [ $count ]"
-
-# Write "1.1.1.1" to empty Final Blocklist Files
-emptyfiles=$(find $pfbdeny -size 0)
-for i in $emptyfiles; do echo "1.1.1.1" > $i; done
-}
-
-
-##########
-# Process to Split ET Pro IPREP into Category Files and Compile selected Blocked categories into Outfile
-processet() {
-
-if [ ! -x $pathgunzip ]; then
-	echo "Application [ Gunzip ] Not found, Can't proceed."
-	echo "Application [ Gunzip ] Not found, Can't proceed. [ $now ]" >> $errorlog
-	exit
-fi
-
-if [ -s $pfborig$alias".gz" ]; then
-	evar="ET_*"
-	# Remove Previous ET IPRep Files
-	[ -d $etdir ] && [ "$(ls -A $etdir)" ] && rm -r $etdir/$evar
-	> $tempfile; > $tempfile2
-
-	$pathgunzip -c $pfborig$alias".gz" > $pfborig$alias".raw"
-
-	# ET CSV Format (IP, Category, Score)
-	echo; echo "Processing [ $alias ]"
-	while IFS="," read a b c; do
-		# Some ET Categories are not in use (For Future Use)
-		case "$b" in
-			1)  echo $a >> $etdir/ET_Cnc;;
-			2)  echo $a >> $etdir/ET_Bot;;
-			3)  echo $a >> $etdir/ET_Spam;;
-			4)  echo $a >> $etdir/ET_Drop;;
-			5)  echo $a >> $etdir/ET_Spywarecnc;;
-			6)  echo $a >> $etdir/ET_Onlinegaming;;
-			7)  echo $a >> $etdir/ET_Drivebysrc;;
-			8)  echo $a >> $etdir/ET_Cat8;;
-			9)  echo $a >> $etdir/ET_Chatserver;;
-			10) echo $a >> $etdir/ET_Tornode;;
-			11) echo $a >> $etdir/ET_Cat11;;
-			12) echo $a >> $etdir/ET_Cat12;;
-			13) echo $a >> $etdir/ET_Compromised;;
-			14) echo $a >> $etdir/ET_Cat14;;
-			15) echo $a >> $etdir/ET_P2P;;
-			16) echo $a >> $etdir/ET_Proxy;;
-			17) echo $a >> $etdir/ET_Ipcheck;;
-			18) echo $a >> $etdir/ET_Cat18;;
-			19) echo $a >> $etdir/ET_Utility;;
-			20) echo $a >> $etdir/ET_DDos;;
-			21) echo $a >> $etdir/ET_Scanner;;
-			22) echo $a >> $etdir/ET_Cat22;;
-			23) echo $a >> $etdir/ET_Brute;;
-			24) echo $a >> $etdir/ET_Fakeav;;
-			25) echo $a >> $etdir/ET_Dyndns;;
-			26) echo $a >> $etdir/ET_Undesireable;;
-			27) echo $a >> $etdir/ET_Abusedtld;;
-			28) echo $a >> $etdir/ET_Selfsignedssl;;
-			29) echo $a >> $etdir/ET_Blackhole;;
-			30) echo $a >> $etdir/ET_RAS;;
-			31) echo $a >> $etdir/ET_P2Pcnc;;
-			32) echo $a >> $etdir/ET_Sharedhosting;;
-			33) echo $a >> $etdir/ET_Parking;;
-			34) echo $a >> $etdir/ET_VPN;;
-			35) echo $a >> $etdir/ET_Exesource;;
-			36) echo $a >> $etdir/ET_Cat36;;
-			37) echo $a >> $etdir/ET_Mobilecnc;;
-			38) echo $a >> $etdir/ET_Mobilespyware;;
-			39) echo $a >> $etdir/ET_Skypenode;;
-			40) echo $a >> $etdir/ET_Bitcoin;;
-			41) echo $a >> $etdir/ET_DDosattack;;
-			*)  echo $a >> $etdir/ET_Unknown;;
-		esac
-	done <"$pfborig$alias.raw"
-	data=$(ls $etdir)
-	echo "Compiling ET IP IQRisk REP Lists based upon User Selected Categories"
-	printf "%-10s %-25s\n" "  Action" "Category"
-	echo "-------------------------------------------"
-
-	for list in $data; do
-		case "$etblock" in
-			*$list*)
-				printf "%-10s %-25s\n" "  Block: " "$list"
-				cat $etdir/$list >> $tempfile
-				;;
-		esac
-		case "$etmatch" in
-			*$list*)
-				printf "%-10s %-25s\n" "  Match: " "$list"
-				cat $etdir/$list >> $tempfile2
-				;;
-		esac
-	done
-	echo "-------------------------------------------"
-
-	if [ -f $tempfile ]; then mv -f $tempfile $pfborig$alias".orig"; fi
-	if [ "$etmatch" != "x" ]; then mv -f $tempfile2 $pfbmatch/ETMatch.txt; fi
-	cicount=$(cat $etdir/$evar | grep -cv '^#\|^$'); cocount=$(grep -cv "^1\.1\.1\.1" $pfborig$alias".orig")
-	echo; echo "ET Folder count [ $cicount ]  Outfile count [ $cocount ]"
-else
-	echo; echo "No ET .GZ File Found!"
-fi
-}
-
-# Process to extract IP addresses from XLSX Files
-processxlsx() {
-
-if [ ! -x $pathtar ]; then
-	echo "Application [ TAR ] Not found, Can't proceed."
-	echo "Application [ TAR ] Not found, Can't proceed. [ $now ]" >> $errorlog
-	exit
-fi
-
-if [ -s $pfborig$alias".zip" ]; then
-
-	$pathtar -xf $pfborig$alias".zip" -C $tmpxlsx
-	$pathtar -xOf $tmpxlsx*.[xX][lL][sS][xX] xl/sharedStrings.xml |
-		grep -aoEw "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" | sort | uniq > $pfborig$alias".orig"
-	rm -r $tmpxlsx*
-
-	cocount=$(grep -cv "^1\.1\.1\.1" $pfborig$alias".orig")
-	echo; echo "Download file count [ ZIP file ]  Outfile count [ $cocount ]"
-else
-	echo "XLSX Download File Missing"
-	echo " [ $alias ] XLSX Download File Missing [ $now ]" >> $errorlog
-fi
-}
-
-closingprocess() {
-
-# Write "1.1.1.1" to empty Final Blocklist Files
-emptyfiles=$(find $pfbdeny -size 0)
-for i in $emptyfiles; do echo "1.1.1.1" > $i; done
-
-if [ -d "$pfborig" ] && [ "$(ls -A $pfborig)" ]; then
-	fcount=$(find $pfborig*.orig | xargs cat | grep -cv '^#\|^$')
-else
-	fcount=0
-fi
-
-if [ "$alias" == "on" ]; then
-	sort -o $masterfile $masterfile
-	sort -t . -k 1,1n -k 2,2n -k 3,3n -k 4,4n $mastercat > $tempfile; mv -f $tempfile $mastercat
-
-	echo; echo; echo "===[ FINAL Processing ]====================================="; echo
-	echo "   [ Original count   ]  [ $fcount ]"
-	count=$(grep -c ^ $masterfile)
-	echo; echo "   [ Processed Count  ]  [ $count ]"; echo
-
-	s1=$(grep -cv "1\.1\.1\.1" $masterfile)
-	s2=$(find $pfbdeny ! -name "*_v6.txt" -type f | xargs cat | grep -cv "^1\.1\.1\.1")
-	s3=$(sort $mastercat | uniq -d | tail -30)
-	s4=$(find $pfbdeny ! -name "*_v6.txt" -type f | xargs cat | sort | uniq -d | tail -30 | grep -v "^1\.1\.1\.1")
-
-	if [ -d "$pfbpermit" ] && [ "$(ls -A $pfbpermit)" ]; then
-		echo; echo "===[ Permit List IP Counts ]========================="; echo
-		wc -l $pfbpermit* | sort -n -r
-	fi
-	if [ -d "$pfbmatch" ] && [ "$(ls -A $pfbmatch)" ]; then
-		echo; echo "===[ Match List IP Counts ]=========================="; echo
-		wc -l $pfbmatch* | sort -n -r
-	fi
-	if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then
-		echo; echo "===[ Deny List IP Counts ]==========================="; echo
-		wc -l $pfbdeny* | sort -n -r
-	fi
-	if [ -d "$pfbnative" ] && [ "$(ls -A $pfbnative)" ]; then
-		echo; echo "===[ Native List IP Counts ] ==================================="; echo
-		wc -l $pfbnative* | sort -n -r
-	fi
-	if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then
-		emptylists=$(grep "1\.1\.1\.1" $pfbdeny* | sed -e 's/^.*[a-zA-Z]\///' -e 's/\.txt:1.1.1.1/ /')
-		if [ ! -z "$emptylists" ]; then 
-			echo; echo "====================[ Empty Lists w/1.1.1.1 ]=================="; echo
-			for list in $emptylists; do
-				echo $list
-			done
-		fi
-	fi
-	if [ -d "$pfborig" ] && [ "$(ls -A $pfborig)" ]; then
-		echo; echo "====================[ Last Updated List Summary ]=============="; echo
-		ls -lahtr $pfborig* | sed -e 's/\/.*\// /' -e 's/.orig//' | awk -v OFS='\t' '{print $6" "$7,$8,$9}'
-	fi
-	echo "==============================================================="; echo
-	echo "Sanity Check (Not Including IPv6)  ** These two Counts should Match! **"
-	echo "------------"
-	echo "Masterfile Count    [ $s1 ]"
-	echo "Deny folder Count   [ $s2 ]"; echo
-	echo "Duplication Sanity Check (Pass=No IPs reported)"
-	echo "------------------------"
-	echo "Masterfile/Deny Folder Uniq check"
-	if [ ! -z "$s3" ]; then echo $s3; fi
-	echo "Deny Folder/Masterfile Uniq check"
-	if [ ! -z "$s4" ]; then echo $s4; fi
-	echo; echo "Sync Check (Pass=No IPs reported)"
-	echo "----------"
-else
-	echo; echo "===[ FINAL Processing ]============================================="; echo
-	echo "   [ Original count   ]  [ $fcount ]"
-	if [ -d "$pfbpermit" ] && [ "$(ls -A $pfbpermit)" ]; then
-		echo; echo "===[ Permit List IP Counts ]========================="; echo
-		wc -l $pfbpermit* | sort -n -r
-	fi
-	if [ -d "$pfbmatch" ] && [ "$(ls -A $pfbmatch)" ]; then
-		echo; echo "===[ Match List IP Counts ]=========================="; echo
-		wc -l $pfbmatch* | sort -n -r
-	fi
-	if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then
-		echo; echo "===[ Deny List IP Counts ]==========================="; echo
-		wc -l $pfbdeny* | sort -n -r
-	fi
-	if [ -d "$pfbnative" ] && [ "$(ls -A $pfbnative)" ]; then
-		echo; echo "===[ Native List IP Counts ] ==================================="; echo
-		wc -l $pfbnative* | sort -n -r
-	fi
-	if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then
-		emptylists=$(grep "1\.1\.1\.1" $pfbdeny* | sed -e 's/^.*[a-zA-Z]\///' -e 's/\.txt:1.1.1.1/ /')
-		if [ ! -z "$emptylists" ]; then
-			echo; echo "====================[ Empty Lists w/1.1.1.1 ]=================="; echo
-			for list in $emptylists; do
-				echo $list
-			done
-		fi
-	fi
-	if [ -d "$pfborig" ] && [ "$(ls -A $pfborig)" ]; then
-		echo; echo "====================[ Last Updated List Summary ]=============="; echo
-		ls -lahtr $pfborig* | sed -e 's/\/.*\// /' -e 's/.orig//' | awk -v OFS='\t' '{print $6" "$7,$8,$9}'
-		echo "==============================================================="
-	fi
-fi
-
-echo; echo "IPv4 Alias Table IP Total"; echo "-----------------------------"
-find $pfsense_alias_dir ! -name "*_v6.txt" -type f | xargs cat | grep -c ^
-
-echo; echo "IPv6 Alias Table IP Total"; echo "-----------------------------"
-find $pfsense_alias_dir -name "*_v6.txt" -type f | xargs cat | grep -c ^
-
-echo; echo "Alias Table IP Counts"; echo "-----------------------------"
-wc -l $pfsense_alias_dir*.txt | sort -n -r
-
-echo; echo "pfSense Table Stats"; echo "-------------------"
-$pathpfctl -s memory | grep "table-entries"
-pfctlcount=$($pathpfctl -vvsTables | awk '/Addresses/ {s+=$2}; END {print s}')
-echo "Table Usage Count       " $pfctlcount
-}
-
-remove() {
-# Remove Lists from Masterfiles and Delete Associated Files
-echo
-for i in $cc; do
-	header=$(echo "${i%*,}")
-	if [ ! "$header" == "" ]; then
-		# Make sure that Alias Exists in Masterfile before removal.
-		masterchk=$(grep -m1 "$header[[:space:]]" $masterfile)
-		if [ ! -z "$masterchk" ]; then
-			# Grep Header with a Trailing Space character
-			grep "$header[[:space:]]" $masterfile > $tempfile
-			awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
-			cut -d' ' -f2 $masterfile > $mastercat
-		fi
-		rm -rf $pfborig$header*; rm -rf $pfbdeny$header*; rm -rf $pfbmatch$header*; rm -rf $pfbpermit$header*
-		echo "The Following list has been REMOVED [ $header ]"
-	fi
-	echo
-done
-
-# Delete Masterfiles if they are empty
-emptychk=$(find $masterfile -size 0)
-if [ ! "$emptychk" == "" ]; then
-	rm -r $masterfile; rm -r $mastercat
-fi
-}
-
-
-##########
-# CALL APPROPRIATE PROCESSES using Script Argument $1
-case $1 in
-	continent)
-		continent
-		;;
-	duplicate)
-		process255
-		duplicate
-		;;
-	suppress)
-		suppress
-		;;
-	p24)
-		process24
-		;;
-	dedup)
-		deduplication
-		;;
-	pdup)
-		pdeduplication
-		;;
-	et)
-		processet
-		;;
-	xlsx)
-		processxlsx
-		;;
-	closing)
-		closingprocess
-		;;
-	remove)
-		remove
-		;;
-	*)
-		exit
-		;;
-esac
-exit
\ No newline at end of file