Merge pull request #743 from BBcan177/pfBlockerNG_Beta_0.99

1 files changed, 927 insertions, 0 deletions

diff --git a/config/pfblockerng/pfblockerng.sh b/config/pfblockerng/pfblockerng.sh
new file mode 100644
index 00000000..c09d52e1
--- /dev/null
+++ b/config/pfblockerng/pfblockerng.sh
@@ -0,0 +1,927 @@
+#!/bin/sh
+# pfBlockerNG IP Reputation Script - By BBcan177@gmail.com - 04-12-14
+# Copyright (C) 2014 BBcan177@gmail.com
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License Version 2 as
+# published by the Free Software Foundation.  You may not use, modify or
+# distribute this program under any other version of the GNU General
+# Public License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+now=$(/bin/date +%m/%d/%y' '%T)
+mtype=$(/usr/bin/uname -m);
+
+# Application Paths
+pathgrepcidr="/usr/pbi/pfblockerng-$mtype/bin/grepcidr"
+pathgeoip="/usr/pbi/pfblockerng-$mtype/bin/geoiplookup"
+
+pathtar=/usr/bin/tar
+pathgunzip=/usr/bin/gunzip
+pathpfctl=/sbin/pfctl
+
+# Script Arguments
+alias=$2
+max=$3
+dedup=$4
+cc=$(echo $5 | sed 's/,/, /g')
+ccwhite=$(echo $6 | tr '[A-Z]' '[a-z]')
+ccblack=$(echo $7 | tr '[A-Z]' '[a-z]')
+etblock=$(echo $8 | sed 's/,/, /g')
+etmatch=$(echo $9 | sed 's/,/, /g')
+
+# File Locations
+pathgeoipdat=/var/db/pfblockerng/GeoIP.dat
+pfbsuppression=/var/db/pfblockerng/pfbsuppression.txt
+masterfile=/var/db/pfblockerng/masterfile
+mastercat=/var/db/pfblockerng/mastercat
+geoiplog=/var/log/pfblockerng/geoip.log
+errorlog=/var/log/pfblockerng/error.log
+
+# Folder Locations
+etdir=/var/db/pfblockerng/ET
+tmpxlsx=/tmp/xlsx/
+
+pfbdeny=/var/db/pfblockerng/deny/
+pfborig=/var/db/pfblockerng/original/
+pfbmatch=/var/db/pfblockerng/match/
+pfbpermit=/var/db/pfblockerng/permit/
+pfbnative=/var/db/pfblockerng/native/
+pfsense_alias_dir=/var/db/aliastables/
+
+# Store "Match" d-dedups in matchdedup.txt file
+matchdedup=matchdedup.txt
+
+tempfile=/tmp/pfbtempfile
+tempfile2=/tmp/pfbtempfile2
+dupfile=/tmp//pfbduptemp
+dedupfile=/tmp/pfbdeduptemp
+addfile=/tmp/pfBaddfile
+syncfile=/tmp/pfbsyncfile
+matchfile=/tmp/pfbmatchfile
+tempmatchfile=/tmp/pfbtempmatchfile
+
+if [ ! -f $masterfile ]; then touch $masterfile; fi
+if [ ! -f $mastercat ]; then touch $mastercat; fi
+if [ ! -f $tempfile ]; then touch $tempfile; fi
+if [ ! -f $tempfile2 ]; then touch $tempfile2; fi
+if [ ! -f $dupfile ]; then touch $dupfile; fi
+if [ ! -f $dedupfile ]; then touch $dedupfile; fi
+if [ ! -f $addfile ]; then touch $addfile; fi
+if [ ! -f $syncfile ]; then touch $syncfile; fi
+if [ ! -f $matchfile ]; then touch $matchfile; fi
+if [ ! -f $tempmatchfile ]; then touch $tempmatchfile; fi
+if [ ! -d $pfbmatch ]; then mkdir $pfbmatch; fi
+if [ ! -d $etdir ]; then mkdir $etdir; fi
+if [ ! -d $tmpxlsx ]; then mkdir $tmpxlsx; fi
+
+##########
+# Process to condense an IP range if a "Max" amount of IP addresses are found in a /24 range per Alias Group.
+process24() {
+
+if [ ! -x $pathgeoip ]; then 
+	echo "Process24 - Application [ GeoIP ] Not found. Can't proceed."
+	echo "Process24 - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog
+	exit
+fi
+
+# Download MaxMind GeoIP.dat Binary on first Install.
+if [ ! -f $pathgeoipdat ]; then
+	echo "Downloading [ MaxMind GeoIP.dat ] [ $now ]" >> $geoiplog
+	/usr/local/pkg/pfblockerng/geoipupdate.sh bu
+fi
+# Exit if GeoIP.dat is not found.
+if [ ! -f $pathgeoipdat ]; then 
+	echo "Process24 - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed."
+	echo "Process24 - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >>  $errorlog
+	exit
+fi
+
+count=$(grep -c ^ $pfbdeny$alias".txt")
+echo; echo "Original File Count [ $count ]"
+
+grep -Ev "^(#|$)" $pfbdeny$alias".txt" | sort | uniq > $tempfile
+> $dupfile; > $tempfile2; > $matchfile; > $tempmatchfile
+data="$(cut -d '.' -f 1-3 $tempfile | awk -v max="$max" '{a[$0]++}END{for(i in a){if(a[i] > max){print i}}}')"
+count=$(echo "$data" | grep -c ^); mcount=0; dcount=0; safe=0
+if [ "$data" == "" ]; then count=0; fi
+matchoutfile="match"$header".txt"
+# Classify Repeat Offenders by Country Code
+if [ -f $pathgeoipdat ]; then
+	for ip in $data; do
+		ccheck=$($pathgeoip -f $pathgeoipdat "$ip.1" | cut -c 24-25)
+		case "$cc" in
+			*$ccheck*)
+				safe=$(($safe + 1))
+				if [ "$ccwhite" == "match" -o "$ccblack" == "match" ]; then
+					echo "$ip." >> $matchfile
+				fi
+				;;
+			*)
+				echo "$ip." >> $dupfile
+				;;
+		esac
+	done
+else
+	echo; echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping p24 Process"; echo
+	echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping p24 Process [ $now ]" >> $errorlog
+fi
+# Collect Match File Details
+if [ -s "$matchfile" -a ! "$dedup" == "on" -a "$ccwhite" == "match" ]; then
+	mon=$(sed -e 's/^/^/' -e 's/\./\\\./g' $matchfile)
+	for ip in $mon; do
+	grep $ip $tempfile >> $tempfile2
+	done
+	mcount=$(grep -c ^ $tempfile2)
+	if [ "$ccwhite" == "match" ]; then
+		sed 's/$/0\/24/' $matchfile >> $tempmatchfile
+	sed 's/^/\!/' $tempfile2 >> $tempmatchfile
+	fi
+fi
+
+# If no Matches found remove previous Matchoutfile if exists.
+if [ ! -s "$tempmatchfile" -a -f $matchoutfile ]; then rm -r $matchoutfile; fi
+# Move Match File to the Match Folder by Individual Blocklist Name
+if [ -s "$tempmatchfile" ]; then mv -f $tempmatchfile $pfbmatch$matchoutfile; fi
+
+# Find Repeat Offenders in each individual Blocklist Outfile
+if [ -s "$dupfile" ]; then
+	> $tempfile2
+	dup=$(sed -e 's/^/^/' -e 's/\./\\\./g' $dupfile)
+	for ip in $dup; do
+		grep $ip $tempfile >> $tempfile2
+	done
+	dcount=$(grep -c ^ $tempfile2)
+	if [ "$ccblack" == "block" ]; then
+		awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile2 $tempfile > $pfbdeny$alias".txt"
+		sed 's/$/0\/24/' $dupfile >> $pfbdeny$alias".txt"
+	elif [ "$ccblack" == "match" ]; then
+		sed 's/$/0\/24/' $dupfile >> $tempmatchfile
+		sed 's/^/\!/' $tempfile2 >> $tempmatchfile
+	else
+		:
+	fi
+fi
+if [ "$count" == "0" -a "$safe" == "0" ]; then echo; echo " Process /24 Stats [ $alias ] [ $now ] "; echo "------------------------------------------------"; fi
+if [ "$count" == "0" ]; then echo "Found [ 0 ] IP range(s) over the threshold of [ $max ] p24 - CC Blacklist"; fi
+if [ "$safe" == "0" ]; then echo "Found [ 0 ] IP range(s) over the threshold of [ $max ] p24 - CC Whitelist"; fi
+ 
+if [ -s "$dupfile" -o -s "$matchfile" ]; then
+echo
+echo " Process /24 Stats [ $alias ] [ $now ]"
+echo "--------------------------------------------------------"
+echo "Found [ $count ] IP range(s) over the threshold of [ $max ] on the CC Blacklist"
+echo "Found [ $safe ] IP range(s) over the threshold of [ $max ] on the CC Whitelist"
+echo
+echo "Found [ $dcount ] CC Blacklisted IP Address(es) are being set to [ $ccblack ]"
+# Skip Match Process if dedup=yes as it will create duplicates
+if [ "$dedup" == "on" ]; then mcount=Skipped; fi
+echo "Found [ $mcount ] CC Whitelisted IP Address(es) are being set to [ $ccwhite ]"
+if [ "$ccblack" == "block" ]; then
+	echo; echo "Removed the following IP Ranges"
+	cat $dupfile | tr '\n' '|'; echo
+else
+	echo "Skipped, CCBlack set to [ $ccblack ]"
+fi
+sort $pfbdeny$alias".txt" | uniq > $tempfile; mv -f $tempfile $pfbdeny$alias".txt"
+echo "-------------------------------------------------------"
+cocount=$(grep -cv "^1\.1\.1\.1" $pfbdeny$alias".txt")
+echo "Post /24 Count   [ $cocount ]"; echo
+fi
+}
+
+
+##########
+process255() {
+# Remove IPs if exists over 255 IPs in a Range and replace with a single /24 Block
+cp $pfbdeny$alias".txt" $tempfile; > $dedupfile
+
+data255="$(cut -d '.' -f 1-3 $tempfile | awk '{a[$0]++}END{for(i in a){if(a[i] > 255){print i}}}')"
+if [ ! -z "$data255" ]; then
+	for ip in $data255; do
+		ii=$(echo "^$ip" | sed 's/\./\\\./g')
+		grep $ii $tempfile >> $dedupfile
+	done
+	awk 'FNR==NR{a[$0];next}!($0 in a)' $dedupfile $tempfile > $pfbdeny$alias".txt"
+	for ip in $data255; do echo $ip"0/24" >> $pfbdeny$alias".txt"; done
+fi
+}
+
+
+##########
+continent() {
+
+dupcheck=yes
+# Check if Masterfile is Empty
+hcheck=$(grep -c ^ $masterfile); if [ "$hcheck" -eq "0" ]; then dupcheck=no; fi
+# Check if Alias exists in Masterfile
+lcheck=$(grep -m 1 "$alias " $masterfile ); if [ "$lcheck" == "" ]; then dupcheck=no; fi
+
+if [ "$dupcheck" == "yes" ]; then
+	# Grep Alias with a trailing Space character
+	grep "$alias[[:space:]]" $masterfile > $tempfile
+	awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
+	cut -d' ' -f2 $masterfile > $mastercat
+fi
+
+grep -Ev "^(#|$)" $pfbdeny$alias".txt" | sort | uniq > $tempfile
+
+if [ ! "$hcheck" -eq "0" ]; then
+	$pathgrepcidr -vf $mastercat $pfbdeny$alias".txt" > $tempfile; mv -f $tempfile $pfbdeny$alias".txt"
+fi
+
+sed -e 's/^/'$alias' /' $pfbdeny$alias".txt" >> $masterfile
+cut -d' ' -f2 $masterfile > $mastercat
+
+countg=$(grep -c ^ $pfborig$alias".orig")
+countm=$(grep -c "$alias " $masterfile); counto=$(grep -c ^ $pfbdeny$alias".txt")
+if [ "$countm" == "$counto" ]; then sanity="Passed"; else sanity=" ==> FAILED <== "; fi
+echo "----------------------------------------------------------"
+echo; echo " Post Duplication count [ $now ]"
+echo "----------------------------------------------------------"
+printf "%-10s %-10s %-10s %-30s\n" "Original" "Masterfile" "Outfile" "Sanity Check"
+echo "----------------------------------------------------------"
+printf "%-10s %-10s %-10s %-30s\n" "$countg" "$countm" "$counto" " [ $sanity ]"
+echo "----------------------------------------------------------"
+}
+
+
+##########
+# Process to remove Suppressed Entries and RFC 1918 and Misc IPs on each downloaded Blocklist
+suppress() {
+
+if [ ! -x $pathgrepcidr ]; then
+	echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]"
+	echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" >> errorlog
+	exit
+fi
+
+if [ -e "$pfbsuppression" ] && [ -s "$pfbsuppression" ]; then
+	# Find '/24' Blocked IPs that are single addresses in the Suppressed IP Address List.
+	# These '/24' Are converted to single Addresses excluding the Suppressed IPs.
+	data="$(cat $pfbsuppression)"
+	if [ ! -z "$data" -a ! -z "$cc" ]; then
+		# Loop thru each Updated List to remove Suppression and RFC1918 Addresses
+		if [ "$cc" == "suppressheader" ]; then
+			echo; echo "===[ Suppression Stats ]========================================"; echo
+			printf "%-20s %-10s %-10s %-10s %-10s\n" "List" "Pre" "RFC1918" "Suppress" "Masterfile"
+			echo "----------------------------------------------------------------"
+			exit
+		fi
+
+		for i in $cc; do
+			counter=0
+			> $dupfile
+			alias=$(echo "${i%|*}")
+			pfbfolder=$(echo "${i#*|}")
+
+			if [ ! "$alias" == "" ]; then
+				# Count (PRE)	
+				countg=$(grep -c ^ $pfbfolder$alias".txt")
+
+				grep -Ev "^(192\.168|10\.|172\.1[6789]\.|172\.2[0-9]\.|172\.3[01]\.|#|$)" $pfbfolder$alias".txt" |
+					sort | uniq > $tempfile	
+				# Count (Post RFC1918)
+				countm=$(grep -c ^ $tempfile)
+
+				for ip in $data; do
+					found=""; ddcheck="";
+					iptrim=$(echo $ip | cut -d '.' -f 1-3)
+					mask=$(echo $ip | cut -d"/" -f2)
+					found=$(grep -m1 $iptrim".0/24" $tempfile)
+					# If a Suppression is '/32' and a Blocklist has a full '/24' Block execute the following.
+					if [ ! "$found" == "" -a "$mask" == "32" ]; then
+						echo " Suppression $alias: $iptrim.0/24"
+						octet4=$(echo $ip | cut -d '.' -f 4 | sed 's/\/.*//')
+						dcheck=$(grep $iptrim".0/24" $dupfile)
+						if [ "$dcheck" == "" ]; then
+							echo $iptrim".0" >> $tempfile
+							echo $iptrim".0/24" >> $dupfile
+							counter=$(($counter + 1))
+							# Add Individual IP addresses from Range excluding Suppressed IP
+							for i in $(/usr/bin/jot 255); do
+								if [ "$i" != "$octet4" ]; then
+									echo $iptrim"."$i >> $tempfile
+									counter=$(($counter + 1))
+								fi
+							done
+						fi
+					fi
+				done
+				if [ -s $dupfile ]; then
+					# Remove '/24' Suppressed Ranges
+					awk 'FNR==NR{a[$0];next}!($0 in a)' $dupfile $tempfile > $tempfile2; mv -f $tempfile2 $tempfile
+				fi
+				# Remove All other Suppressions from Lists
+				$pathgrepcidr -vf $pfbsuppression $tempfile > $pfbfolder$alias".txt"
+				# Update Masterfiles. Don't execute if Duplication Process is Disabled
+				if [ "$dedup" == "x" ]; then
+					# Dont execute if Alias doesnt exist in Masterfile
+					lcheck=$(grep -m1 "$alias " $masterfile)
+					if [ ! "$lcheck" == "" ]; then
+						# Replace Masterfile with changes to List.
+						grep "$alias[[:space:]]" $masterfile > $tempfile
+						awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
+						sed -e 's/^/'$alias' /' $pfbfolder$alias".txt" >> $masterfile
+						cut -d' ' -f2 $masterfile > $mastercat
+					fi
+				fi
+				countk=$(grep -c ^ $masterfile)
+				countx=$(grep -c ^ $pfbfolder$alias".txt")
+				counto=$(($countx - $counter))
+				printf "%-20s %-10s %-10s %-10s %-10s\n" "$alias" "$countg" "$countm" "$counto" "$countk"
+			fi
+		done
+	fi
+else
+	if [ "$cc" == "suppressheader" ]; then
+		echo "===[ Suppression Stats ]========================================"; echo
+		printf "%-20s %-10s %-10s %-10s %-10s\n" "List" "Pre" "RFC1918" "Suppress" "Masterfile"
+		echo "----------------------------------------------------------------"
+		exit
+	fi
+	for i in $cc; do
+		alias=$(echo "${i%|*}")
+		pfbfolder=$(echo "${i#*|}")
+
+		if [ ! "$alias" == "" ]; then
+			countg=$(grep -c ^ $pfbfolder$alias".txt")
+			grep -Ev "^(192\.168|10\.|172\.1[6789]\.|172\.2[0-9]\.|172\.3[01]\.|#|$)" $pfbfolder$alias".txt" |
+				sort | uniq > $tempfile; mv -f $tempfile $pfbfolder$alias".txt"
+			countx=$(grep -c ^ $pfbfolder$alias".txt")
+			# Update Masterfiles. Don't execute if Duplication Process is Disabled or if No Suppression Changes Found
+			if [ "$dedup" == "x" -a "$countg" != "$countx" ]; then
+				# Dont execute if Alias doesnt exist in Masterfile
+				lcheck=$(grep -m1 "$alias " $masterfile)
+				if [ ! "$lcheck" == "" ]; then
+					# Replace Masterfile with changes to List.
+					grep "$alias[[:space:]]" $masterfile > $tempfile
+					awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
+					sed -e 's/^/'$alias' /' $pfbfolder$alias".txt" >> $masterfile
+					cut -d' ' -f2 $masterfile > $mastercat
+				fi
+			fi
+			countm=$(grep -c ^ $pfbfolder$alias".txt")
+			counto=" - "
+			countk=$(grep -c ^ $masterfile)
+			printf "%-20s %-10s %-10s %-10s %-10s\n" "$alias" "$countg" "$countm" "$counto" "$countk"
+		fi
+	done
+fi
+}
+
+
+##########
+# Process to remove Duplicate Entries on each downloaded Blocklist Individually
+duplicate() {
+
+if [ ! -x $pathgrepcidr ]; then
+	echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]"
+	echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" >> errorlog
+	exit
+fi
+
+dupcheck=yes
+# Check if Masterfile is Empty
+hcheck=$(grep -cv "^$" $masterfile); if [ "$hcheck" -eq "0" ]; then dupcheck=no; fi
+# Check if Alias exists in Masterfile
+lcheck=$(grep -m1 "$alias " $masterfile); if [ "$lcheck" == "" ]; then dupcheck=no; fi
+
+if [ "$dupcheck" == "yes" ]; then
+	# Grep Alias with a trailing Space character
+	grep "$alias[[:space:]]" $masterfile > $tempfile
+	awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
+	cut -d' ' -f2 $masterfile > $mastercat
+fi
+
+grep -Ev "^(#|$)" $pfbdeny$alias".txt" | sort | uniq > $tempfile; mv -f $tempfile $pfbdeny$alias".txt"
+
+if [ ! "$hcheck" -eq "0" ]; then
+	$pathgrepcidr -vf $mastercat $pfbdeny$alias".txt" > $tempfile; mv -f $tempfile $pfbdeny$alias".txt"
+fi
+
+sed -e 's/^/'$alias' /' $pfbdeny$alias".txt" >> $masterfile
+cut -d' ' -f2 $masterfile > $mastercat
+
+countg=$(grep -c ^ $pfborig$alias".orig")
+countm=$(grep -c "$alias " $masterfile); counto=$(grep -c ^ $pfbdeny$alias".txt")
+if [ "$countm" == "$counto" ]; then sanity="Passed"; else sanity=" ==> FAILED <== "; fi
+echo "----------------------------------------------------------"
+printf "%-10s %-10s %-10s %-30s\n" "Original" "Masterfile" "Outfile" " [ Post Duplication count ]"
+echo "----------------------------------------------------------"
+printf "%-10s %-10s %-10s %-30s\n" "$countg" "$countm" "$counto" " [ $sanity ]"
+echo "----------------------------------------------------------"
+}
+
+
+##########
+# De-Duplication utilizing MaxMind GeoIP Country Code Whitelisting ("dmax" variable)
+deduplication() {
+
+if [ ! -x $pathgeoip ]; then
+	echo "d-duplication - Application [ GeoIP ] Not found. Can't proceed."
+	echo "d-duplication - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog
+	exit
+fi
+
+# Download MaxMind GeoIP.dat on first Install.
+if [ ! -f $pathgeoipdat ]; then 
+	echo "Downloading [ MaxMind GeoIP.dat ] [ $now ]" >> $geoiplog
+	/usr/local/pkg/pfblockerng/geoipupdate.sh bu
+fi
+
+# Exit if GeoIP.dat is not found
+if [ ! -f $pathgeoipdat ]; then
+	echo "d-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed."
+	echo "d-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog
+	exit
+fi
+
+> $tempfile; > $tempfile2; > $dupfile; > $addfile; > $dedupfile; > $matchfile; > $tempmatchfile; count=0; dcount=0; mcount=0; mmcount=0
+echo; echo "Querying for Repeat Offenders"
+data="$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | cut -d '.' -f 1-3 $pfbdeny*.txt |
+	awk -v max="$max" '{a[$0]++}END{for(i in a){if(a[i] > max){print i}}}' | grep -v "^1\.1\.1")"
+count=$(echo "$data" | grep -c ^)
+if [ "$data" == "" ]; then count=0; fi
+safe=0
+# Classify Repeat Offenders by Country Code
+if [ -f $pathgeoipdat ]; then
+	echo "Classifying Repeat Offenders by GeoIP"
+	for ip in $data; do
+		ccheck=$($pathgeoip -f $pathgeoipdat "$ip.1" | cut -c 24-25)
+		case "$cc" in
+			*$ccheck*)
+				safe=$(($safe + 1))
+				if [ "$ccwhite" == "match" -o "$ccblack" == "match" ]; then
+					echo "$ip." >> $matchfile
+				fi
+				;;
+			*)
+				echo "$ip." >> $dupfile
+				;;
+		esac
+	done
+else
+	echo; echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping d-dedup Process"; echo
+	echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping d-dedup Process [ $now ]" >> $errorlog
+fi
+if [ -s "$matchfile" -a "$ccwhite" == "match" ]; then
+	echo "Processing [ Match ] IPs"
+	match=$(sed -e 's/^/^/' -e 's/\./\\\./g' $matchfile)
+	for mfile in $match; do
+		grep $mfile $pfbdeny*.txt >> $tempfile
+	done
+	sed 's/$/0\/24/' $matchfile >> $tempmatchfile
+	sed -e 's/.*://' -e 's/^/\!/' $tempfile >> $tempmatchfile
+	mv -f $tempmatchfile $pfbmatch$matchdedup
+	mcount=$(grep -c ^ $tempfile)
+	mmcount=$(($mcount + $mmcount))
+fi
+# Find Repeat Offenders in each individual Blocklist Outfile
+if [ -s "$dupfile" ]; then
+	echo "Processing [ Block ] IPs"
+	dup=$(cat $dupfile)
+	for ip in $dup; do
+		pcount=1; ii=$(echo "^$ip" | sed 's/\./\\\./g')
+		list=$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | xargs grep -al $ii)
+		for blfile in $list; do
+			header=$(echo "${blfile##*/}" | cut -d '.' -f1)
+			grep $ii $blfile > $tempfile
+			if [ "$ccblack" == "block" ]; then
+				awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $blfile > $tempfile2; mv -f $tempfile2 $blfile
+				if [ "$pcount" -eq "1" ]; then
+					echo $ip"0/24" >> $blfile
+					echo $header" "$ip >> $dedupfile
+					echo $header" "$ip"0/24" >> $addfile
+					pcount=2
+				else
+					echo $header" "$ip >> $dedupfile
+				fi
+			else	
+				if [ "$pcount" -eq "1" ]; then
+					matchoutfile="match"$header".txt"
+					echo $ip"0/24" >> $pfbmatch$matchoutfile
+					sed 's/^/\!/' $tempfile >> $pfbmatch$matchoutfile
+					mcount=$(grep -c ^ $pfbmatch$matchoutfile)
+					mmcount=$(($mcount + $mmcount))
+					pcount=2
+				fi
+			fi
+		done
+	done
+	# Remove Repeat Offenders in Masterfiles
+	if [ -s "$dedupfile" ]; then
+		echo "Removing   [ Block ] IPs"
+		> $tempfile; > $tempfile2
+		sed 's/\./\\\./g' $dedupfile > $tempfile2
+		while IFS=' ' read -r ips; do grep "$ips" $masterfile >> $tempfile; done < $tempfile2
+		dcount=$(grep -c ^ $tempfile)
+		awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
+		cat $addfile >> $masterfile
+		cut -d' ' -f2 $masterfile > $mastercat
+	fi
+fi
+
+echo; echo "d-Duplication Process  [ $now ]"; echo "------------------------------------------------"
+echo; echo "Found [ $count ] IP range(s) over the threshold of dmax= [ $max ]"
+echo "Found [ $safe ] IP range(s) classified as Whitelisted"
+echo; echo "Found [ $dcount ] CC Blacklisted IP Address(es) are being set to [ $ccblack ]"
+echo "Found [ $mmcount ] CC Whitelisted IP Address(es) are being set to [ $ccwhite ]"; echo
+if [ -s "$addfile" ]; then
+        echo; echo "Removed the following IP Ranges"
+        sed -e 's/^.* //' -e 's/0\/24//' $addfile | tr '\n' '|'; echo
+fi
+count=$(grep -c ^ $masterfile)
+echo " [ Post d-Deduplication count ]  [ $count ]"; echo
+
+# Write "1.1.1.1" to empty Final Blocklist Files
+emptyfiles=$(find $pfbdeny -size 0)
+for i in $emptyfiles; do echo "1.1.1.1" > $i; done
+}
+
+
+##########
+# Process to perform a final De-Duplication on all of the BlockLists (Excluding Country Whitelist) ("pmax" variable).
+pdeduplication(){
+
+if [ ! -x $pathgeoip ]; then
+	echo "p-duplication - Application [ GeoIP ] Not found. Can't proceed."
+	echo "p-duplication - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog
+	exit
+fi
+
+# Download MaxMind GeoIP.dat on first Install.
+if [ ! -f $pathgeoipdat ]; then
+	echo "Downloading [ MaxMind GeoIP.dat ] [ $now ]" >> $geoiplog
+	/usr/local/pkg/pfblockerng/geoipupdate.sh bu
+fi
+# Exit if GeoIP.dat is not found.
+if [ ! -f $pathgeoipdat ]; then
+	echo "p-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed."
+	echo "p-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog
+	exit
+fi
+
+> $tempfile; > $tempfile2; > $dupfile; > $addfile; > $dedupfile; count=0; dcount=0
+echo; echo "====================================================================="
+echo; echo "Querying for Repeat Offenders"
+data="$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | cut -d '.' -f 1-3 $pfbdeny*.txt |
+	awk -v max="$max" '{a[$0]++}END{for(i in a){if(a[i] > max){print i}}}' | grep -v "^1\.1\.1")"
+count=$(echo "$data" | grep -c ^)
+if [ "$data" == "" ]; then count=0; fi
+# Find Repeat Offenders in each individual Blocklist Outfile
+echo "Processing [ Block ] IPs"
+for ip in $data; do
+	pcount=1; ii=$(echo "^$ip." | sed 's/\./\\\./g')
+	list=$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | xargs grep -al $ii)
+	for blfile in $list; do
+		header=$(echo "${blfile##*/}" | cut -d '.' -f1)
+		grep $ii $blfile > $tempfile
+		awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $blfile > $tempfile2; mv -f $tempfile2 $blfile
+		if [ "$pcount" -eq "1" ]; then
+			echo $ip".0/24" >> $blfile
+			echo $header" $ip." >> $dedupfile
+			echo $header" "$ip".0/24" >> $addfile
+			pcount=2
+		else
+			echo $header" $ip." >> $dedupfile
+		fi
+	done
+done
+# Remove Repeat Offenders in Masterfile
+if [ -s "$dedupfile" ]; then
+	echo "Removing   [ Block ] IPs"
+	> $tempfile; > $tempfile2
+	sed 's/\./\\\./g' $dedupfile > $tempfile2
+	while IFS=' ' read -r ips; do grep "$ips" $masterfile >> $tempfile; done < $tempfile2
+	dcount=$(grep -c ^ $tempfile)
+	awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
+	cat $addfile >> $masterfile
+	cut -d' ' -f2 $masterfile > $mastercat
+fi
+
+echo; echo "p-Duplication Process  [ $now ]"; echo "------------------------------------------------"
+echo "Found [ $dcount ] IP Address(es) are being set to [ block ]"
+if [ -s "$addfile" ]; then
+	echo; echo "Removed the following IP Ranges"
+	sed -e 's/^.* //' -e 's/0\/24//' $addfile | tr '\n' '|'; echo
+fi
+count=$(grep -c ^ $masterfile)
+echo; echo " [ Post p-Deduplication count ]  [ $count ]"
+
+# Write "1.1.1.1" to empty Final Blocklist Files
+emptyfiles=$(find $pfbdeny -size 0)
+for i in $emptyfiles; do echo "1.1.1.1" > $i; done
+}
+
+
+##########
+# Process to Split ET Pro IPREP into Category Files and Compile selected Blocked categories into Outfile
+processet() {
+
+if [ ! -x $pathgunzip ]; then
+	echo "Application [ Gunzip ] Not found, Can't proceed."
+	echo "Application [ Gunzip ] Not found, Can't proceed. [ $now ]" >> $errorlog
+	exit
+fi
+
+if [ -s $pfborig$alias".gz" ]; then
+	evar="ET_*"
+	# Remove Previous ET IPRep Files
+	[ -d $etdir ] && [ "$(ls -A $etdir)" ] && rm -r $etdir/$evar
+	> $tempfile; > $tempfile2
+
+	$pathgunzip -c $pfborig$alias".gz" > $pfborig$alias".raw"
+
+	# ET CSV Format (IP, Category, Score)
+	echo; echo "Processing [ $alias ]"
+	while IFS="," read a b c; do
+		# Some ET Categories are not in use (For Future Use)
+		case "$b" in
+			1)  echo $a >> $etdir/ET_Cnc;;
+			2)  echo $a >> $etdir/ET_Bot;;
+			3)  echo $a >> $etdir/ET_Spam;;
+			4)  echo $a >> $etdir/ET_Drop;;
+			5)  echo $a >> $etdir/ET_Spywarecnc;;
+			6)  echo $a >> $etdir/ET_Onlinegaming;;
+			7)  echo $a >> $etdir/ET_Drivebysrc;;
+			8)  echo $a >> $etdir/ET_Cat8;;
+			9)  echo $a >> $etdir/ET_Chatserver;;
+			10) echo $a >> $etdir/ET_Tornode;;
+			11) echo $a >> $etdir/ET_Cat11;;
+			12) echo $a >> $etdir/ET_Cat12;;
+			13) echo $a >> $etdir/ET_Compromised;;
+			14) echo $a >> $etdir/ET_Cat14;;
+			15) echo $a >> $etdir/ET_P2P;;
+			16) echo $a >> $etdir/ET_Proxy;;
+			17) echo $a >> $etdir/ET_Ipcheck;;
+			18) echo $a >> $etdir/ET_Cat18;;
+			19) echo $a >> $etdir/ET_Utility;;
+			20) echo $a >> $etdir/ET_DDos;;
+			21) echo $a >> $etdir/ET_Scanner;;
+			22) echo $a >> $etdir/ET_Cat22;;
+			23) echo $a >> $etdir/ET_Brute;;
+			24) echo $a >> $etdir/ET_Fakeav;;
+			25) echo $a >> $etdir/ET_Dyndns;;
+			26) echo $a >> $etdir/ET_Undesireable;;
+			27) echo $a >> $etdir/ET_Abusedtld;;
+			28) echo $a >> $etdir/ET_Selfsignedssl;;
+			29) echo $a >> $etdir/ET_Blackhole;;
+			30) echo $a >> $etdir/ET_RAS;;
+			31) echo $a >> $etdir/ET_P2Pcnc;;
+			32) echo $a >> $etdir/ET_Sharedhosting;;
+			33) echo $a >> $etdir/ET_Parking;;
+			34) echo $a >> $etdir/ET_VPN;;
+			35) echo $a >> $etdir/ET_Exesource;;
+			36) echo $a >> $etdir/ET_Cat36;;
+			37) echo $a >> $etdir/ET_Mobilecnc;;
+			38) echo $a >> $etdir/ET_Mobilespyware;;
+			39) echo $a >> $etdir/ET_Skypenode;;
+			40) echo $a >> $etdir/ET_Bitcoin;;
+			41) echo $a >> $etdir/ET_DDosattack;;
+			*)  echo $a >> $etdir/ET_Unknown;;
+		esac
+	done <"$pfborig$alias.raw"
+	data=$(ls $etdir)
+	echo "Compiling ET IP IQRisk REP Lists based upon User Selected Categories"
+	printf "%-10s %-25s\n" "  Action" "Category"
+	echo "-------------------------------------------"
+
+	for list in $data; do
+		case "$etblock" in
+			*$list*)
+				printf "%-10s %-25s\n" "  Block: " "$list"
+				cat $etdir/$list >> $tempfile
+				;;
+		esac
+		case "$etmatch" in
+			*$list*)
+				printf "%-10s %-25s\n" "  Match: " "$list"
+				cat $etdir/$list >> $tempfile2
+				;;
+		esac
+	done
+	echo "-------------------------------------------"
+
+	if [ -f $tempfile ]; then mv -f $tempfile $pfborig$alias".orig"; fi
+	if [ "$etmatch" != "x" ]; then mv -f $tempfile2 $pfbmatch/ETMatch.txt; fi
+	cicount=$(cat $etdir/$evar | grep -cv '^#\|^$'); cocount=$(grep -cv "^1\.1\.1\.1" $pfborig$alias".orig")
+	echo; echo "ET Folder count [ $cicount ]  Outfile count [ $cocount ]"
+else
+	echo; echo "No ET .GZ File Found!"
+fi
+}
+
+# Process to extract IP addresses from XLSX Files
+processxlsx() {
+
+if [ ! -x $pathtar ]; then
+	echo "Application [ TAR ] Not found, Can't proceed."
+	echo "Application [ TAR ] Not found, Can't proceed. [ $now ]" >> $errorlog
+	exit
+fi
+
+if [ -s $pfborig$alias".zip" ]; then
+
+	$pathtar -xf $pfborig$alias".zip" -C $tmpxlsx
+	$pathtar -xOf $tmpxlsx*.[xX][lL][sS][xX] xl/sharedStrings.xml |
+		grep -aoEw "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" | sort | uniq > $pfborig$alias".orig"
+	rm -r $tmpxlsx*
+
+	cocount=$(grep -cv "^1\.1\.1\.1" $pfborig$alias".orig")
+	echo; echo "Download file count [ ZIP file ]  Outfile count [ $cocount ]"
+else
+	echo "XLSX Download File Missing"
+	echo " [ $alias ] XLSX Download File Missing [ $now ]" >> $errorlog
+fi
+}
+
+closingprocess() {
+
+# Write "1.1.1.1" to empty Final Blocklist Files
+emptyfiles=$(find $pfbdeny -size 0)
+for i in $emptyfiles; do echo "1.1.1.1" > $i; done
+
+if [ -d "$pfborig" ] && [ "$(ls -A $pfborig)" ]; then
+	fcount=$(find $pfborig*.orig | xargs cat | grep -cv '^#\|^$')
+else
+	fcount=0
+fi
+
+if [ "$alias" == "on" ]; then
+	sort -o $masterfile $masterfile
+	sort -t . -k 1,1n -k 2,2n -k 3,3n -k 4,4n $mastercat > $tempfile; mv -f $tempfile $mastercat
+
+	echo; echo; echo "===[ FINAL Processing ]====================================="; echo
+	echo "   [ Original count   ]  [ $fcount ]"
+	count=$(grep -c ^ $masterfile)
+	echo; echo "   [ Processed Count  ]  [ $count ]"; echo
+
+	s1=$(grep -cv "1\.1\.1\.1" $masterfile)
+	s2=$(find $pfbdeny ! -name "*_v6.txt" -type f | xargs cat | grep -cv "^1\.1\.1\.1")
+	s3=$(sort $mastercat | uniq -d | tail -30)
+	s4=$(find $pfbdeny ! -name "*_v6.txt" -type f | xargs cat | sort | uniq -d | tail -30 | grep -v "^1\.1\.1\.1")
+
+	if [ -d "$pfbpermit" ] && [ "$(ls -A $pfbpermit)" ]; then
+		echo; echo "===[ Permit List IP Counts ]========================="; echo
+		wc -l $pfbpermit* | sort -n -r
+	fi
+	if [ -d "$pfbmatch" ] && [ "$(ls -A $pfbmatch)" ]; then
+		echo; echo "===[ Match List IP Counts ]=========================="; echo
+		wc -l $pfbmatch* | sort -n -r
+	fi
+	if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then
+		echo; echo "===[ Deny List IP Counts ]==========================="; echo
+		wc -l $pfbdeny* | sort -n -r
+	fi
+	if [ -d "$pfbnative" ] && [ "$(ls -A $pfbnative)" ]; then
+		echo; echo "===[ Native List IP Counts ] ==================================="; echo
+		wc -l $pfbnative* | sort -n -r
+	fi
+	if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then
+		emptylists=$(grep "1\.1\.1\.1" $pfbdeny* | sed -e 's/^.*[a-zA-Z]\///' -e 's/\.txt:1.1.1.1/ /')
+		if [ ! -z "$emptylists" ]; then 
+			echo; echo "====================[ Empty Lists w/1.1.1.1 ]=================="; echo
+			for list in $emptylists; do
+				echo $list
+			done
+		fi
+	fi
+	if [ -d "$pfborig" ] && [ "$(ls -A $pfborig)" ]; then
+		echo; echo "====================[ Last Updated List Summary ]=============="; echo
+		ls -lahtr $pfborig* | sed -e 's/\/.*\// /' -e 's/.orig//' | awk -v OFS='\t' '{print $6" "$7,$8,$9}'
+	fi
+	echo "==============================================================="; echo
+	echo "Sanity Check (Not Including IPv6)  ** These two Counts should Match! **"
+	echo "------------"
+	echo "Masterfile Count    [ $s1 ]"
+	echo "Deny folder Count   [ $s2 ]"; echo
+	echo "Duplication Sanity Check (Pass=No IPs reported)"
+	echo "------------------------"
+	echo "Masterfile/Deny Folder Uniq check"
+	if [ ! -z "$s3" ]; then echo $s3; fi
+	echo "Deny Folder/Masterfile Uniq check"
+	if [ ! -z "$s4" ]; then echo $s4; fi
+	echo; echo "Sync Check (Pass=No IPs reported)"
+	echo "----------"
+else
+	echo; echo "===[ FINAL Processing ]============================================="; echo
+	echo "   [ Original count   ]  [ $fcount ]"
+	if [ -d "$pfbpermit" ] && [ "$(ls -A $pfbpermit)" ]; then
+		echo; echo "===[ Permit List IP Counts ]========================="; echo
+		wc -l $pfbpermit* | sort -n -r
+	fi
+	if [ -d "$pfbmatch" ] && [ "$(ls -A $pfbmatch)" ]; then
+		echo; echo "===[ Match List IP Counts ]=========================="; echo
+		wc -l $pfbmatch* | sort -n -r
+	fi
+	if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then
+		echo; echo "===[ Deny List IP Counts ]==========================="; echo
+		wc -l $pfbdeny* | sort -n -r
+	fi
+	if [ -d "$pfbnative" ] && [ "$(ls -A $pfbnative)" ]; then
+		echo; echo "===[ Native List IP Counts ] ==================================="; echo
+		wc -l $pfbnative* | sort -n -r
+	fi
+	if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then
+		emptylists=$(grep "1\.1\.1\.1" $pfbdeny* | sed -e 's/^.*[a-zA-Z]\///' -e 's/\.txt:1.1.1.1/ /')
+		if [ ! -z "$emptylists" ]; then
+			echo; echo "====================[ Empty Lists w/1.1.1.1 ]=================="; echo
+			for list in $emptylists; do
+				echo $list
+			done
+		fi
+	fi
+	if [ -d "$pfborig" ] && [ "$(ls -A $pfborig)" ]; then
+		echo; echo "====================[ Last Updated List Summary ]=============="; echo
+		ls -lahtr $pfborig* | sed -e 's/\/.*\// /' -e 's/.orig//' | awk -v OFS='\t' '{print $6" "$7,$8,$9}'
+		echo "==============================================================="
+	fi
+fi
+
+echo; echo "IPv4 Alias Table IP Total"; echo "-----------------------------"
+find $pfsense_alias_dir ! -name "*_v6.txt" -type f | xargs cat | grep -c ^
+
+echo; echo "IPv6 Alias Table IP Total"; echo "-----------------------------"
+find $pfsense_alias_dir -name "*_v6.txt" -type f | xargs cat | grep -c ^
+
+echo; echo "Alias Table IP Counts"; echo "-----------------------------"
+wc -l $pfsense_alias_dir*.txt | sort -n -r
+
+echo; echo "pfSense Table Stats"; echo "-------------------"
+$pathpfctl -s memory | grep "table-entries"
+pfctlcount=$($pathpfctl -vvsTables | awk '/Addresses/ {s+=$2}; END {print s}')
+echo "Table Usage Count       " $pfctlcount
+}
+
+remove() {
+# Remove Lists from Masterfiles and Delete Associated Files
+echo
+for i in $cc; do
+	header=$(echo "${i%*,}")
+	if [ ! "$header" == "" ]; then
+		# Make sure that Alias Exists in Masterfile before removal.
+		masterchk=$(grep -m1 "$header[[:space:]]" $masterfile)
+		if [ ! -z "$masterchk" ]; then
+			# Grep Header with a Trailing Space character
+			grep "$header[[:space:]]" $masterfile > $tempfile
+			awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
+			cut -d' ' -f2 $masterfile > $mastercat
+		fi
+		rm -rf $pfborig$header*; rm -rf $pfbdeny$header*; rm -rf $pfbmatch$header*; rm -rf $pfbpermit$header*; rm -rf $pfbnative$header*
+		echo "The Following list has been REMOVED [ $header ]"
+	fi
+	echo
+done
+
+# Delete Masterfiles if they are empty
+emptychk=$(find $masterfile -size 0)
+if [ ! "$emptychk" == "" ]; then
+	rm -r $masterfile; rm -r $mastercat
+fi
+}
+
+
+##########
+# CALL APPROPRIATE PROCESSES using Script Argument $1
+case $1 in
+	continent)
+		continent
+		;;
+	duplicate)
+		process255
+		duplicate
+		;;
+	suppress)
+		suppress
+		;;
+	p24)
+		process24
+		;;
+	dedup)
+		deduplication
+		;;
+	pdup)
+		pdeduplication
+		;;
+	et)
+		processet
+		;;
+	xlsx)
+		processxlsx
+		;;
+	closing)
+		closingprocess
+		;;
+	remove)
+		remove
+		;;
+	*)
+		exit
+		;;
+esac
+exit
\ No newline at end of file