aboutsummaryrefslogtreecommitdiffstats
path: root/config/pfblockerng/pfblockerng.sh
diff options
context:
space:
mode:
Diffstat (limited to 'config/pfblockerng/pfblockerng.sh')
-rw-r--r--config/pfblockerng/pfblockerng.sh927
1 files changed, 927 insertions, 0 deletions
diff --git a/config/pfblockerng/pfblockerng.sh b/config/pfblockerng/pfblockerng.sh
new file mode 100644
index 00000000..c09d52e1
--- /dev/null
+++ b/config/pfblockerng/pfblockerng.sh
@@ -0,0 +1,927 @@
+#!/bin/sh
+# pfBlockerNG IP Reputation Script - By BBcan177@gmail.com - 04-12-14
+# Copyright (C) 2014 BBcan177@gmail.com
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License Version 2 as
+# published by the Free Software Foundation. You may not use, modify or
+# distribute this program under any other version of the GNU General
+# Public License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+now=$(/bin/date +%m/%d/%y' '%T)
+mtype=$(/usr/bin/uname -m);
+
+# Application Paths
+pathgrepcidr="/usr/pbi/pfblockerng-$mtype/bin/grepcidr"
+pathgeoip="/usr/pbi/pfblockerng-$mtype/bin/geoiplookup"
+
+pathtar=/usr/bin/tar
+pathgunzip=/usr/bin/gunzip
+pathpfctl=/sbin/pfctl
+
+# Script Arguments
+alias=$2
+max=$3
+dedup=$4
+cc=$(echo $5 | sed 's/,/, /g')
+ccwhite=$(echo $6 | tr '[A-Z]' '[a-z]')
+ccblack=$(echo $7 | tr '[A-Z]' '[a-z]')
+etblock=$(echo $8 | sed 's/,/, /g')
+etmatch=$(echo $9 | sed 's/,/, /g')
+
+# File Locations
+pathgeoipdat=/var/db/pfblockerng/GeoIP.dat
+pfbsuppression=/var/db/pfblockerng/pfbsuppression.txt
+masterfile=/var/db/pfblockerng/masterfile
+mastercat=/var/db/pfblockerng/mastercat
+geoiplog=/var/log/pfblockerng/geoip.log
+errorlog=/var/log/pfblockerng/error.log
+
+# Folder Locations
+etdir=/var/db/pfblockerng/ET
+tmpxlsx=/tmp/xlsx/
+
+pfbdeny=/var/db/pfblockerng/deny/
+pfborig=/var/db/pfblockerng/original/
+pfbmatch=/var/db/pfblockerng/match/
+pfbpermit=/var/db/pfblockerng/permit/
+pfbnative=/var/db/pfblockerng/native/
+pfsense_alias_dir=/var/db/aliastables/
+
+# Store "Match" d-dedups in matchdedup.txt file
+matchdedup=matchdedup.txt
+
+tempfile=/tmp/pfbtempfile
+tempfile2=/tmp/pfbtempfile2
+dupfile=/tmp//pfbduptemp
+dedupfile=/tmp/pfbdeduptemp
+addfile=/tmp/pfBaddfile
+syncfile=/tmp/pfbsyncfile
+matchfile=/tmp/pfbmatchfile
+tempmatchfile=/tmp/pfbtempmatchfile
+
+if [ ! -f $masterfile ]; then touch $masterfile; fi
+if [ ! -f $mastercat ]; then touch $mastercat; fi
+if [ ! -f $tempfile ]; then touch $tempfile; fi
+if [ ! -f $tempfile2 ]; then touch $tempfile2; fi
+if [ ! -f $dupfile ]; then touch $dupfile; fi
+if [ ! -f $dedupfile ]; then touch $dedupfile; fi
+if [ ! -f $addfile ]; then touch $addfile; fi
+if [ ! -f $syncfile ]; then touch $syncfile; fi
+if [ ! -f $matchfile ]; then touch $matchfile; fi
+if [ ! -f $tempmatchfile ]; then touch $tempmatchfile; fi
+if [ ! -d $pfbmatch ]; then mkdir $pfbmatch; fi
+if [ ! -d $etdir ]; then mkdir $etdir; fi
+if [ ! -d $tmpxlsx ]; then mkdir $tmpxlsx; fi
+
+##########
+# Process to condense an IP range if a "Max" amount of IP addresses are found in a /24 range per Alias Group.
+process24() {
+
+if [ ! -x $pathgeoip ]; then
+ echo "Process24 - Application [ GeoIP ] Not found. Can't proceed."
+ echo "Process24 - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog
+ exit
+fi
+
+# Download MaxMind GeoIP.dat Binary on first Install.
+if [ ! -f $pathgeoipdat ]; then
+ echo "Downloading [ MaxMind GeoIP.dat ] [ $now ]" >> $geoiplog
+ /usr/local/pkg/pfblockerng/geoipupdate.sh bu
+fi
+# Exit if GeoIP.dat is not found.
+if [ ! -f $pathgeoipdat ]; then
+ echo "Process24 - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed."
+ echo "Process24 - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog
+ exit
+fi
+
+count=$(grep -c ^ $pfbdeny$alias".txt")
+echo; echo "Original File Count [ $count ]"
+
+grep -Ev "^(#|$)" $pfbdeny$alias".txt" | sort | uniq > $tempfile
+> $dupfile; > $tempfile2; > $matchfile; > $tempmatchfile
+data="$(cut -d '.' -f 1-3 $tempfile | awk -v max="$max" '{a[$0]++}END{for(i in a){if(a[i] > max){print i}}}')"
+count=$(echo "$data" | grep -c ^); mcount=0; dcount=0; safe=0
+if [ "$data" == "" ]; then count=0; fi
+matchoutfile="match"$header".txt"
+# Classify Repeat Offenders by Country Code
+if [ -f $pathgeoipdat ]; then
+ for ip in $data; do
+ ccheck=$($pathgeoip -f $pathgeoipdat "$ip.1" | cut -c 24-25)
+ case "$cc" in
+ *$ccheck*)
+ safe=$(($safe + 1))
+ if [ "$ccwhite" == "match" -o "$ccblack" == "match" ]; then
+ echo "$ip." >> $matchfile
+ fi
+ ;;
+ *)
+ echo "$ip." >> $dupfile
+ ;;
+ esac
+ done
+else
+ echo; echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping p24 Process"; echo
+ echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping p24 Process [ $now ]" >> $errorlog
+fi
+# Collect Match File Details
+if [ -s "$matchfile" -a ! "$dedup" == "on" -a "$ccwhite" == "match" ]; then
+ mon=$(sed -e 's/^/^/' -e 's/\./\\\./g' $matchfile)
+ for ip in $mon; do
+ grep $ip $tempfile >> $tempfile2
+ done
+ mcount=$(grep -c ^ $tempfile2)
+ if [ "$ccwhite" == "match" ]; then
+ sed 's/$/0\/24/' $matchfile >> $tempmatchfile
+ sed 's/^/\!/' $tempfile2 >> $tempmatchfile
+ fi
+fi
+
+# If no Matches found remove previous Matchoutfile if exists.
+if [ ! -s "$tempmatchfile" -a -f $matchoutfile ]; then rm -r $matchoutfile; fi
+# Move Match File to the Match Folder by Individual Blocklist Name
+if [ -s "$tempmatchfile" ]; then mv -f $tempmatchfile $pfbmatch$matchoutfile; fi
+
+# Find Repeat Offenders in each individual Blocklist Outfile
+if [ -s "$dupfile" ]; then
+ > $tempfile2
+ dup=$(sed -e 's/^/^/' -e 's/\./\\\./g' $dupfile)
+ for ip in $dup; do
+ grep $ip $tempfile >> $tempfile2
+ done
+ dcount=$(grep -c ^ $tempfile2)
+ if [ "$ccblack" == "block" ]; then
+ awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile2 $tempfile > $pfbdeny$alias".txt"
+ sed 's/$/0\/24/' $dupfile >> $pfbdeny$alias".txt"
+ elif [ "$ccblack" == "match" ]; then
+ sed 's/$/0\/24/' $dupfile >> $tempmatchfile
+ sed 's/^/\!/' $tempfile2 >> $tempmatchfile
+ else
+ :
+ fi
+fi
+if [ "$count" == "0" -a "$safe" == "0" ]; then echo; echo " Process /24 Stats [ $alias ] [ $now ] "; echo "------------------------------------------------"; fi
+if [ "$count" == "0" ]; then echo "Found [ 0 ] IP range(s) over the threshold of [ $max ] p24 - CC Blacklist"; fi
+if [ "$safe" == "0" ]; then echo "Found [ 0 ] IP range(s) over the threshold of [ $max ] p24 - CC Whitelist"; fi
+
+if [ -s "$dupfile" -o -s "$matchfile" ]; then
+echo
+echo " Process /24 Stats [ $alias ] [ $now ]"
+echo "--------------------------------------------------------"
+echo "Found [ $count ] IP range(s) over the threshold of [ $max ] on the CC Blacklist"
+echo "Found [ $safe ] IP range(s) over the threshold of [ $max ] on the CC Whitelist"
+echo
+echo "Found [ $dcount ] CC Blacklisted IP Address(es) are being set to [ $ccblack ]"
+# Skip Match Process if dedup=yes as it will create duplicates
+if [ "$dedup" == "on" ]; then mcount=Skipped; fi
+echo "Found [ $mcount ] CC Whitelisted IP Address(es) are being set to [ $ccwhite ]"
+if [ "$ccblack" == "block" ]; then
+ echo; echo "Removed the following IP Ranges"
+ cat $dupfile | tr '\n' '|'; echo
+else
+ echo "Skipped, CCBlack set to [ $ccblack ]"
+fi
+sort $pfbdeny$alias".txt" | uniq > $tempfile; mv -f $tempfile $pfbdeny$alias".txt"
+echo "-------------------------------------------------------"
+cocount=$(grep -cv "^1\.1\.1\.1" $pfbdeny$alias".txt")
+echo "Post /24 Count [ $cocount ]"; echo
+fi
+}
+
+
+##########
+process255() {
+# Remove IPs if exists over 255 IPs in a Range and replace with a single /24 Block
+cp $pfbdeny$alias".txt" $tempfile; > $dedupfile
+
+data255="$(cut -d '.' -f 1-3 $tempfile | awk '{a[$0]++}END{for(i in a){if(a[i] > 255){print i}}}')"
+if [ ! -z "$data255" ]; then
+ for ip in $data255; do
+ ii=$(echo "^$ip" | sed 's/\./\\\./g')
+ grep $ii $tempfile >> $dedupfile
+ done
+ awk 'FNR==NR{a[$0];next}!($0 in a)' $dedupfile $tempfile > $pfbdeny$alias".txt"
+ for ip in $data255; do echo $ip"0/24" >> $pfbdeny$alias".txt"; done
+fi
+}
+
+
+##########
+continent() {
+
+dupcheck=yes
+# Check if Masterfile is Empty
+hcheck=$(grep -c ^ $masterfile); if [ "$hcheck" -eq "0" ]; then dupcheck=no; fi
+# Check if Alias exists in Masterfile
+lcheck=$(grep -m 1 "$alias " $masterfile ); if [ "$lcheck" == "" ]; then dupcheck=no; fi
+
+if [ "$dupcheck" == "yes" ]; then
+ # Grep Alias with a trailing Space character
+ grep "$alias[[:space:]]" $masterfile > $tempfile
+ awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
+ cut -d' ' -f2 $masterfile > $mastercat
+fi
+
+grep -Ev "^(#|$)" $pfbdeny$alias".txt" | sort | uniq > $tempfile
+
+if [ ! "$hcheck" -eq "0" ]; then
+ $pathgrepcidr -vf $mastercat $pfbdeny$alias".txt" > $tempfile; mv -f $tempfile $pfbdeny$alias".txt"
+fi
+
+sed -e 's/^/'$alias' /' $pfbdeny$alias".txt" >> $masterfile
+cut -d' ' -f2 $masterfile > $mastercat
+
+countg=$(grep -c ^ $pfborig$alias".orig")
+countm=$(grep -c "$alias " $masterfile); counto=$(grep -c ^ $pfbdeny$alias".txt")
+if [ "$countm" == "$counto" ]; then sanity="Passed"; else sanity=" ==> FAILED <== "; fi
+echo "----------------------------------------------------------"
+echo; echo " Post Duplication count [ $now ]"
+echo "----------------------------------------------------------"
+printf "%-10s %-10s %-10s %-30s\n" "Original" "Masterfile" "Outfile" "Sanity Check"
+echo "----------------------------------------------------------"
+printf "%-10s %-10s %-10s %-30s\n" "$countg" "$countm" "$counto" " [ $sanity ]"
+echo "----------------------------------------------------------"
+}
+
+
+##########
+# Process to remove Suppressed Entries and RFC 1918 and Misc IPs on each downloaded Blocklist
+suppress() {
+
+if [ ! -x $pathgrepcidr ]; then
+ echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]"
+ echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" >> errorlog
+ exit
+fi
+
+if [ -e "$pfbsuppression" ] && [ -s "$pfbsuppression" ]; then
+ # Find '/24' Blocked IPs that are single addresses in the Suppressed IP Address List.
+ # These '/24' Are converted to single Addresses excluding the Suppressed IPs.
+ data="$(cat $pfbsuppression)"
+ if [ ! -z "$data" -a ! -z "$cc" ]; then
+ # Loop thru each Updated List to remove Suppression and RFC1918 Addresses
+ if [ "$cc" == "suppressheader" ]; then
+ echo; echo "===[ Suppression Stats ]========================================"; echo
+ printf "%-20s %-10s %-10s %-10s %-10s\n" "List" "Pre" "RFC1918" "Suppress" "Masterfile"
+ echo "----------------------------------------------------------------"
+ exit
+ fi
+
+ for i in $cc; do
+ counter=0
+ > $dupfile
+ alias=$(echo "${i%|*}")
+ pfbfolder=$(echo "${i#*|}")
+
+ if [ ! "$alias" == "" ]; then
+ # Count (PRE)
+ countg=$(grep -c ^ $pfbfolder$alias".txt")
+
+ grep -Ev "^(192\.168|10\.|172\.1[6789]\.|172\.2[0-9]\.|172\.3[01]\.|#|$)" $pfbfolder$alias".txt" |
+ sort | uniq > $tempfile
+ # Count (Post RFC1918)
+ countm=$(grep -c ^ $tempfile)
+
+ for ip in $data; do
+ found=""; ddcheck="";
+ iptrim=$(echo $ip | cut -d '.' -f 1-3)
+ mask=$(echo $ip | cut -d"/" -f2)
+ found=$(grep -m1 $iptrim".0/24" $tempfile)
+ # If a Suppression is '/32' and a Blocklist has a full '/24' Block execute the following.
+ if [ ! "$found" == "" -a "$mask" == "32" ]; then
+ echo " Suppression $alias: $iptrim.0/24"
+ octet4=$(echo $ip | cut -d '.' -f 4 | sed 's/\/.*//')
+ dcheck=$(grep $iptrim".0/24" $dupfile)
+ if [ "$dcheck" == "" ]; then
+ echo $iptrim".0" >> $tempfile
+ echo $iptrim".0/24" >> $dupfile
+ counter=$(($counter + 1))
+ # Add Individual IP addresses from Range excluding Suppressed IP
+ for i in $(/usr/bin/jot 255); do
+ if [ "$i" != "$octet4" ]; then
+ echo $iptrim"."$i >> $tempfile
+ counter=$(($counter + 1))
+ fi
+ done
+ fi
+ fi
+ done
+ if [ -s $dupfile ]; then
+ # Remove '/24' Suppressed Ranges
+ awk 'FNR==NR{a[$0];next}!($0 in a)' $dupfile $tempfile > $tempfile2; mv -f $tempfile2 $tempfile
+ fi
+ # Remove All other Suppressions from Lists
+ $pathgrepcidr -vf $pfbsuppression $tempfile > $pfbfolder$alias".txt"
+ # Update Masterfiles. Don't execute if Duplication Process is Disabled
+ if [ "$dedup" == "x" ]; then
+ # Dont execute if Alias doesnt exist in Masterfile
+ lcheck=$(grep -m1 "$alias " $masterfile)
+ if [ ! "$lcheck" == "" ]; then
+ # Replace Masterfile with changes to List.
+ grep "$alias[[:space:]]" $masterfile > $tempfile
+ awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
+ sed -e 's/^/'$alias' /' $pfbfolder$alias".txt" >> $masterfile
+ cut -d' ' -f2 $masterfile > $mastercat
+ fi
+ fi
+ countk=$(grep -c ^ $masterfile)
+ countx=$(grep -c ^ $pfbfolder$alias".txt")
+ counto=$(($countx - $counter))
+ printf "%-20s %-10s %-10s %-10s %-10s\n" "$alias" "$countg" "$countm" "$counto" "$countk"
+ fi
+ done
+ fi
+else
+ if [ "$cc" == "suppressheader" ]; then
+ echo "===[ Suppression Stats ]========================================"; echo
+ printf "%-20s %-10s %-10s %-10s %-10s\n" "List" "Pre" "RFC1918" "Suppress" "Masterfile"
+ echo "----------------------------------------------------------------"
+ exit
+ fi
+ for i in $cc; do
+ alias=$(echo "${i%|*}")
+ pfbfolder=$(echo "${i#*|}")
+
+ if [ ! "$alias" == "" ]; then
+ countg=$(grep -c ^ $pfbfolder$alias".txt")
+ grep -Ev "^(192\.168|10\.|172\.1[6789]\.|172\.2[0-9]\.|172\.3[01]\.|#|$)" $pfbfolder$alias".txt" |
+ sort | uniq > $tempfile; mv -f $tempfile $pfbfolder$alias".txt"
+ countx=$(grep -c ^ $pfbfolder$alias".txt")
+ # Update Masterfiles. Don't execute if Duplication Process is Disabled or if No Suppression Changes Found
+ if [ "$dedup" == "x" -a "$countg" != "$countx" ]; then
+ # Dont execute if Alias doesnt exist in Masterfile
+ lcheck=$(grep -m1 "$alias " $masterfile)
+ if [ ! "$lcheck" == "" ]; then
+ # Replace Masterfile with changes to List.
+ grep "$alias[[:space:]]" $masterfile > $tempfile
+ awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
+ sed -e 's/^/'$alias' /' $pfbfolder$alias".txt" >> $masterfile
+ cut -d' ' -f2 $masterfile > $mastercat
+ fi
+ fi
+ countm=$(grep -c ^ $pfbfolder$alias".txt")
+ counto=" - "
+ countk=$(grep -c ^ $masterfile)
+ printf "%-20s %-10s %-10s %-10s %-10s\n" "$alias" "$countg" "$countm" "$counto" "$countk"
+ fi
+ done
+fi
+}
+
+
+##########
+# Process to remove Duplicate Entries on each downloaded Blocklist Individually
+duplicate() {
+
+if [ ! -x $pathgrepcidr ]; then
+ echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]"
+ echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" >> errorlog
+ exit
+fi
+
+dupcheck=yes
+# Check if Masterfile is Empty
+hcheck=$(grep -cv "^$" $masterfile); if [ "$hcheck" -eq "0" ]; then dupcheck=no; fi
+# Check if Alias exists in Masterfile
+lcheck=$(grep -m1 "$alias " $masterfile); if [ "$lcheck" == "" ]; then dupcheck=no; fi
+
+if [ "$dupcheck" == "yes" ]; then
+ # Grep Alias with a trailing Space character
+ grep "$alias[[:space:]]" $masterfile > $tempfile
+ awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
+ cut -d' ' -f2 $masterfile > $mastercat
+fi
+
+grep -Ev "^(#|$)" $pfbdeny$alias".txt" | sort | uniq > $tempfile; mv -f $tempfile $pfbdeny$alias".txt"
+
+if [ ! "$hcheck" -eq "0" ]; then
+ $pathgrepcidr -vf $mastercat $pfbdeny$alias".txt" > $tempfile; mv -f $tempfile $pfbdeny$alias".txt"
+fi
+
+sed -e 's/^/'$alias' /' $pfbdeny$alias".txt" >> $masterfile
+cut -d' ' -f2 $masterfile > $mastercat
+
+countg=$(grep -c ^ $pfborig$alias".orig")
+countm=$(grep -c "$alias " $masterfile); counto=$(grep -c ^ $pfbdeny$alias".txt")
+if [ "$countm" == "$counto" ]; then sanity="Passed"; else sanity=" ==> FAILED <== "; fi
+echo "----------------------------------------------------------"
+printf "%-10s %-10s %-10s %-30s\n" "Original" "Masterfile" "Outfile" " [ Post Duplication count ]"
+echo "----------------------------------------------------------"
+printf "%-10s %-10s %-10s %-30s\n" "$countg" "$countm" "$counto" " [ $sanity ]"
+echo "----------------------------------------------------------"
+}
+
+
+##########
+# De-Duplication utilizing MaxMind GeoIP Country Code Whitelisting ("dmax" variable)
+deduplication() {
+
+if [ ! -x $pathgeoip ]; then
+ echo "d-duplication - Application [ GeoIP ] Not found. Can't proceed."
+ echo "d-duplication - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog
+ exit
+fi
+
+# Download MaxMind GeoIP.dat on first Install.
+if [ ! -f $pathgeoipdat ]; then
+ echo "Downloading [ MaxMind GeoIP.dat ] [ $now ]" >> $geoiplog
+ /usr/local/pkg/pfblockerng/geoipupdate.sh bu
+fi
+
+# Exit if GeoIP.dat is not found
+if [ ! -f $pathgeoipdat ]; then
+ echo "d-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed."
+ echo "d-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog
+ exit
+fi
+
+> $tempfile; > $tempfile2; > $dupfile; > $addfile; > $dedupfile; > $matchfile; > $tempmatchfile; count=0; dcount=0; mcount=0; mmcount=0
+echo; echo "Querying for Repeat Offenders"
+data="$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | cut -d '.' -f 1-3 $pfbdeny*.txt |
+ awk -v max="$max" '{a[$0]++}END{for(i in a){if(a[i] > max){print i}}}' | grep -v "^1\.1\.1")"
+count=$(echo "$data" | grep -c ^)
+if [ "$data" == "" ]; then count=0; fi
+safe=0
+# Classify Repeat Offenders by Country Code
+if [ -f $pathgeoipdat ]; then
+ echo "Classifying Repeat Offenders by GeoIP"
+ for ip in $data; do
+ ccheck=$($pathgeoip -f $pathgeoipdat "$ip.1" | cut -c 24-25)
+ case "$cc" in
+ *$ccheck*)
+ safe=$(($safe + 1))
+ if [ "$ccwhite" == "match" -o "$ccblack" == "match" ]; then
+ echo "$ip." >> $matchfile
+ fi
+ ;;
+ *)
+ echo "$ip." >> $dupfile
+ ;;
+ esac
+ done
+else
+ echo; echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping d-dedup Process"; echo
+ echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping d-dedup Process [ $now ]" >> $errorlog
+fi
+if [ -s "$matchfile" -a "$ccwhite" == "match" ]; then
+ echo "Processing [ Match ] IPs"
+ match=$(sed -e 's/^/^/' -e 's/\./\\\./g' $matchfile)
+ for mfile in $match; do
+ grep $mfile $pfbdeny*.txt >> $tempfile
+ done
+ sed 's/$/0\/24/' $matchfile >> $tempmatchfile
+ sed -e 's/.*://' -e 's/^/\!/' $tempfile >> $tempmatchfile
+ mv -f $tempmatchfile $pfbmatch$matchdedup
+ mcount=$(grep -c ^ $tempfile)
+ mmcount=$(($mcount + $mmcount))
+fi
+# Find Repeat Offenders in each individual Blocklist Outfile
+if [ -s "$dupfile" ]; then
+ echo "Processing [ Block ] IPs"
+ dup=$(cat $dupfile)
+ for ip in $dup; do
+ pcount=1; ii=$(echo "^$ip" | sed 's/\./\\\./g')
+ list=$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | xargs grep -al $ii)
+ for blfile in $list; do
+ header=$(echo "${blfile##*/}" | cut -d '.' -f1)
+ grep $ii $blfile > $tempfile
+ if [ "$ccblack" == "block" ]; then
+ awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $blfile > $tempfile2; mv -f $tempfile2 $blfile
+ if [ "$pcount" -eq "1" ]; then
+ echo $ip"0/24" >> $blfile
+ echo $header" "$ip >> $dedupfile
+ echo $header" "$ip"0/24" >> $addfile
+ pcount=2
+ else
+ echo $header" "$ip >> $dedupfile
+ fi
+ else
+ if [ "$pcount" -eq "1" ]; then
+ matchoutfile="match"$header".txt"
+ echo $ip"0/24" >> $pfbmatch$matchoutfile
+ sed 's/^/\!/' $tempfile >> $pfbmatch$matchoutfile
+ mcount=$(grep -c ^ $pfbmatch$matchoutfile)
+ mmcount=$(($mcount + $mmcount))
+ pcount=2
+ fi
+ fi
+ done
+ done
+ # Remove Repeat Offenders in Masterfiles
+ if [ -s "$dedupfile" ]; then
+ echo "Removing [ Block ] IPs"
+ > $tempfile; > $tempfile2
+ sed 's/\./\\\./g' $dedupfile > $tempfile2
+ while IFS=' ' read -r ips; do grep "$ips" $masterfile >> $tempfile; done < $tempfile2
+ dcount=$(grep -c ^ $tempfile)
+ awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
+ cat $addfile >> $masterfile
+ cut -d' ' -f2 $masterfile > $mastercat
+ fi
+fi
+
+echo; echo "d-Duplication Process [ $now ]"; echo "------------------------------------------------"
+echo; echo "Found [ $count ] IP range(s) over the threshold of dmax= [ $max ]"
+echo "Found [ $safe ] IP range(s) classified as Whitelisted"
+echo; echo "Found [ $dcount ] CC Blacklisted IP Address(es) are being set to [ $ccblack ]"
+echo "Found [ $mmcount ] CC Whitelisted IP Address(es) are being set to [ $ccwhite ]"; echo
+if [ -s "$addfile" ]; then
+ echo; echo "Removed the following IP Ranges"
+ sed -e 's/^.* //' -e 's/0\/24//' $addfile | tr '\n' '|'; echo
+fi
+count=$(grep -c ^ $masterfile)
+echo " [ Post d-Deduplication count ] [ $count ]"; echo
+
+# Write "1.1.1.1" to empty Final Blocklist Files
+emptyfiles=$(find $pfbdeny -size 0)
+for i in $emptyfiles; do echo "1.1.1.1" > $i; done
+}
+
+
+##########
+# Process to perform a final De-Duplication on all of the BlockLists (Excluding Country Whitelist) ("pmax" variable).
+pdeduplication(){
+
+if [ ! -x $pathgeoip ]; then
+ echo "p-duplication - Application [ GeoIP ] Not found. Can't proceed."
+ echo "p-duplication - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog
+ exit
+fi
+
+# Download MaxMind GeoIP.dat on first Install.
+if [ ! -f $pathgeoipdat ]; then
+ echo "Downloading [ MaxMind GeoIP.dat ] [ $now ]" >> $geoiplog
+ /usr/local/pkg/pfblockerng/geoipupdate.sh bu
+fi
+# Exit if GeoIP.dat is not found.
+if [ ! -f $pathgeoipdat ]; then
+ echo "p-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed."
+ echo "p-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog
+ exit
+fi
+
+> $tempfile; > $tempfile2; > $dupfile; > $addfile; > $dedupfile; count=0; dcount=0
+echo; echo "====================================================================="
+echo; echo "Querying for Repeat Offenders"
+data="$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | cut -d '.' -f 1-3 $pfbdeny*.txt |
+ awk -v max="$max" '{a[$0]++}END{for(i in a){if(a[i] > max){print i}}}' | grep -v "^1\.1\.1")"
+count=$(echo "$data" | grep -c ^)
+if [ "$data" == "" ]; then count=0; fi
+# Find Repeat Offenders in each individual Blocklist Outfile
+echo "Processing [ Block ] IPs"
+for ip in $data; do
+ pcount=1; ii=$(echo "^$ip." | sed 's/\./\\\./g')
+ list=$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | xargs grep -al $ii)
+ for blfile in $list; do
+ header=$(echo "${blfile##*/}" | cut -d '.' -f1)
+ grep $ii $blfile > $tempfile
+ awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $blfile > $tempfile2; mv -f $tempfile2 $blfile
+ if [ "$pcount" -eq "1" ]; then
+ echo $ip".0/24" >> $blfile
+ echo $header" $ip." >> $dedupfile
+ echo $header" "$ip".0/24" >> $addfile
+ pcount=2
+ else
+ echo $header" $ip." >> $dedupfile
+ fi
+ done
+done
+# Remove Repeat Offenders in Masterfile
+if [ -s "$dedupfile" ]; then
+ echo "Removing [ Block ] IPs"
+ > $tempfile; > $tempfile2
+ sed 's/\./\\\./g' $dedupfile > $tempfile2
+ while IFS=' ' read -r ips; do grep "$ips" $masterfile >> $tempfile; done < $tempfile2
+ dcount=$(grep -c ^ $tempfile)
+ awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
+ cat $addfile >> $masterfile
+ cut -d' ' -f2 $masterfile > $mastercat
+fi
+
+echo; echo "p-Duplication Process [ $now ]"; echo "------------------------------------------------"
+echo "Found [ $dcount ] IP Address(es) are being set to [ block ]"
+if [ -s "$addfile" ]; then
+ echo; echo "Removed the following IP Ranges"
+ sed -e 's/^.* //' -e 's/0\/24//' $addfile | tr '\n' '|'; echo
+fi
+count=$(grep -c ^ $masterfile)
+echo; echo " [ Post p-Deduplication count ] [ $count ]"
+
+# Write "1.1.1.1" to empty Final Blocklist Files
+emptyfiles=$(find $pfbdeny -size 0)
+for i in $emptyfiles; do echo "1.1.1.1" > $i; done
+}
+
+
+##########
+# Process to Split ET Pro IPREP into Category Files and Compile selected Blocked categories into Outfile
+processet() {
+
+if [ ! -x $pathgunzip ]; then
+ echo "Application [ Gunzip ] Not found, Can't proceed."
+ echo "Application [ Gunzip ] Not found, Can't proceed. [ $now ]" >> $errorlog
+ exit
+fi
+
+if [ -s $pfborig$alias".gz" ]; then
+ evar="ET_*"
+ # Remove Previous ET IPRep Files
+ [ -d $etdir ] && [ "$(ls -A $etdir)" ] && rm -r $etdir/$evar
+ > $tempfile; > $tempfile2
+
+ $pathgunzip -c $pfborig$alias".gz" > $pfborig$alias".raw"
+
+ # ET CSV Format (IP, Category, Score)
+ echo; echo "Processing [ $alias ]"
+ while IFS="," read a b c; do
+ # Some ET Categories are not in use (For Future Use)
+ case "$b" in
+ 1) echo $a >> $etdir/ET_Cnc;;
+ 2) echo $a >> $etdir/ET_Bot;;
+ 3) echo $a >> $etdir/ET_Spam;;
+ 4) echo $a >> $etdir/ET_Drop;;
+ 5) echo $a >> $etdir/ET_Spywarecnc;;
+ 6) echo $a >> $etdir/ET_Onlinegaming;;
+ 7) echo $a >> $etdir/ET_Drivebysrc;;
+ 8) echo $a >> $etdir/ET_Cat8;;
+ 9) echo $a >> $etdir/ET_Chatserver;;
+ 10) echo $a >> $etdir/ET_Tornode;;
+ 11) echo $a >> $etdir/ET_Cat11;;
+ 12) echo $a >> $etdir/ET_Cat12;;
+ 13) echo $a >> $etdir/ET_Compromised;;
+ 14) echo $a >> $etdir/ET_Cat14;;
+ 15) echo $a >> $etdir/ET_P2P;;
+ 16) echo $a >> $etdir/ET_Proxy;;
+ 17) echo $a >> $etdir/ET_Ipcheck;;
+ 18) echo $a >> $etdir/ET_Cat18;;
+ 19) echo $a >> $etdir/ET_Utility;;
+ 20) echo $a >> $etdir/ET_DDos;;
+ 21) echo $a >> $etdir/ET_Scanner;;
+ 22) echo $a >> $etdir/ET_Cat22;;
+ 23) echo $a >> $etdir/ET_Brute;;
+ 24) echo $a >> $etdir/ET_Fakeav;;
+ 25) echo $a >> $etdir/ET_Dyndns;;
+ 26) echo $a >> $etdir/ET_Undesireable;;
+ 27) echo $a >> $etdir/ET_Abusedtld;;
+ 28) echo $a >> $etdir/ET_Selfsignedssl;;
+ 29) echo $a >> $etdir/ET_Blackhole;;
+ 30) echo $a >> $etdir/ET_RAS;;
+ 31) echo $a >> $etdir/ET_P2Pcnc;;
+ 32) echo $a >> $etdir/ET_Sharedhosting;;
+ 33) echo $a >> $etdir/ET_Parking;;
+ 34) echo $a >> $etdir/ET_VPN;;
+ 35) echo $a >> $etdir/ET_Exesource;;
+ 36) echo $a >> $etdir/ET_Cat36;;
+ 37) echo $a >> $etdir/ET_Mobilecnc;;
+ 38) echo $a >> $etdir/ET_Mobilespyware;;
+ 39) echo $a >> $etdir/ET_Skypenode;;
+ 40) echo $a >> $etdir/ET_Bitcoin;;
+ 41) echo $a >> $etdir/ET_DDosattack;;
+ *) echo $a >> $etdir/ET_Unknown;;
+ esac
+ done <"$pfborig$alias.raw"
+ data=$(ls $etdir)
+ echo "Compiling ET IP IQRisk REP Lists based upon User Selected Categories"
+ printf "%-10s %-25s\n" " Action" "Category"
+ echo "-------------------------------------------"
+
+ for list in $data; do
+ case "$etblock" in
+ *$list*)
+ printf "%-10s %-25s\n" " Block: " "$list"
+ cat $etdir/$list >> $tempfile
+ ;;
+ esac
+ case "$etmatch" in
+ *$list*)
+ printf "%-10s %-25s\n" " Match: " "$list"
+ cat $etdir/$list >> $tempfile2
+ ;;
+ esac
+ done
+ echo "-------------------------------------------"
+
+ if [ -f $tempfile ]; then mv -f $tempfile $pfborig$alias".orig"; fi
+ if [ "$etmatch" != "x" ]; then mv -f $tempfile2 $pfbmatch/ETMatch.txt; fi
+ cicount=$(cat $etdir/$evar | grep -cv '^#\|^$'); cocount=$(grep -cv "^1\.1\.1\.1" $pfborig$alias".orig")
+ echo; echo "ET Folder count [ $cicount ] Outfile count [ $cocount ]"
+else
+ echo; echo "No ET .GZ File Found!"
+fi
+}
+
+# Process to extract IP addresses from XLSX Files
+processxlsx() {
+
+if [ ! -x $pathtar ]; then
+ echo "Application [ TAR ] Not found, Can't proceed."
+ echo "Application [ TAR ] Not found, Can't proceed. [ $now ]" >> $errorlog
+ exit
+fi
+
+if [ -s $pfborig$alias".zip" ]; then
+
+ $pathtar -xf $pfborig$alias".zip" -C $tmpxlsx
+ $pathtar -xOf $tmpxlsx*.[xX][lL][sS][xX] xl/sharedStrings.xml |
+ grep -aoEw "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" | sort | uniq > $pfborig$alias".orig"
+ rm -r $tmpxlsx*
+
+ cocount=$(grep -cv "^1\.1\.1\.1" $pfborig$alias".orig")
+ echo; echo "Download file count [ ZIP file ] Outfile count [ $cocount ]"
+else
+ echo "XLSX Download File Missing"
+ echo " [ $alias ] XLSX Download File Missing [ $now ]" >> $errorlog
+fi
+}
+
+closingprocess() {
+
+# Write "1.1.1.1" to empty Final Blocklist Files
+emptyfiles=$(find $pfbdeny -size 0)
+for i in $emptyfiles; do echo "1.1.1.1" > $i; done
+
+if [ -d "$pfborig" ] && [ "$(ls -A $pfborig)" ]; then
+ fcount=$(find $pfborig*.orig | xargs cat | grep -cv '^#\|^$')
+else
+ fcount=0
+fi
+
+if [ "$alias" == "on" ]; then
+ sort -o $masterfile $masterfile
+ sort -t . -k 1,1n -k 2,2n -k 3,3n -k 4,4n $mastercat > $tempfile; mv -f $tempfile $mastercat
+
+ echo; echo; echo "===[ FINAL Processing ]====================================="; echo
+ echo " [ Original count ] [ $fcount ]"
+ count=$(grep -c ^ $masterfile)
+ echo; echo " [ Processed Count ] [ $count ]"; echo
+
+ s1=$(grep -cv "1\.1\.1\.1" $masterfile)
+ s2=$(find $pfbdeny ! -name "*_v6.txt" -type f | xargs cat | grep -cv "^1\.1\.1\.1")
+ s3=$(sort $mastercat | uniq -d | tail -30)
+ s4=$(find $pfbdeny ! -name "*_v6.txt" -type f | xargs cat | sort | uniq -d | tail -30 | grep -v "^1\.1\.1\.1")
+
+ if [ -d "$pfbpermit" ] && [ "$(ls -A $pfbpermit)" ]; then
+ echo; echo "===[ Permit List IP Counts ]========================="; echo
+ wc -l $pfbpermit* | sort -n -r
+ fi
+ if [ -d "$pfbmatch" ] && [ "$(ls -A $pfbmatch)" ]; then
+ echo; echo "===[ Match List IP Counts ]=========================="; echo
+ wc -l $pfbmatch* | sort -n -r
+ fi
+ if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then
+ echo; echo "===[ Deny List IP Counts ]==========================="; echo
+ wc -l $pfbdeny* | sort -n -r
+ fi
+ if [ -d "$pfbnative" ] && [ "$(ls -A $pfbnative)" ]; then
+ echo; echo "===[ Native List IP Counts ] ==================================="; echo
+ wc -l $pfbnative* | sort -n -r
+ fi
+ if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then
+ emptylists=$(grep "1\.1\.1\.1" $pfbdeny* | sed -e 's/^.*[a-zA-Z]\///' -e 's/\.txt:1.1.1.1/ /')
+ if [ ! -z "$emptylists" ]; then
+ echo; echo "====================[ Empty Lists w/1.1.1.1 ]=================="; echo
+ for list in $emptylists; do
+ echo $list
+ done
+ fi
+ fi
+ if [ -d "$pfborig" ] && [ "$(ls -A $pfborig)" ]; then
+ echo; echo "====================[ Last Updated List Summary ]=============="; echo
+ ls -lahtr $pfborig* | sed -e 's/\/.*\// /' -e 's/.orig//' | awk -v OFS='\t' '{print $6" "$7,$8,$9}'
+ fi
+ echo "==============================================================="; echo
+ echo "Sanity Check (Not Including IPv6) ** These two Counts should Match! **"
+ echo "------------"
+ echo "Masterfile Count [ $s1 ]"
+ echo "Deny folder Count [ $s2 ]"; echo
+ echo "Duplication Sanity Check (Pass=No IPs reported)"
+ echo "------------------------"
+ echo "Masterfile/Deny Folder Uniq check"
+ if [ ! -z "$s3" ]; then echo $s3; fi
+ echo "Deny Folder/Masterfile Uniq check"
+ if [ ! -z "$s4" ]; then echo $s4; fi
+ echo; echo "Sync Check (Pass=No IPs reported)"
+ echo "----------"
+else
+ echo; echo "===[ FINAL Processing ]============================================="; echo
+ echo " [ Original count ] [ $fcount ]"
+ if [ -d "$pfbpermit" ] && [ "$(ls -A $pfbpermit)" ]; then
+ echo; echo "===[ Permit List IP Counts ]========================="; echo
+ wc -l $pfbpermit* | sort -n -r
+ fi
+ if [ -d "$pfbmatch" ] && [ "$(ls -A $pfbmatch)" ]; then
+ echo; echo "===[ Match List IP Counts ]=========================="; echo
+ wc -l $pfbmatch* | sort -n -r
+ fi
+ if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then
+ echo; echo "===[ Deny List IP Counts ]==========================="; echo
+ wc -l $pfbdeny* | sort -n -r
+ fi
+ if [ -d "$pfbnative" ] && [ "$(ls -A $pfbnative)" ]; then
+ echo; echo "===[ Native List IP Counts ] ==================================="; echo
+ wc -l $pfbnative* | sort -n -r
+ fi
+ if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then
+ emptylists=$(grep "1\.1\.1\.1" $pfbdeny* | sed -e 's/^.*[a-zA-Z]\///' -e 's/\.txt:1.1.1.1/ /')
+ if [ ! -z "$emptylists" ]; then
+ echo; echo "====================[ Empty Lists w/1.1.1.1 ]=================="; echo
+ for list in $emptylists; do
+ echo $list
+ done
+ fi
+ fi
+ if [ -d "$pfborig" ] && [ "$(ls -A $pfborig)" ]; then
+ echo; echo "====================[ Last Updated List Summary ]=============="; echo
+ ls -lahtr $pfborig* | sed -e 's/\/.*\// /' -e 's/.orig//' | awk -v OFS='\t' '{print $6" "$7,$8,$9}'
+ echo "==============================================================="
+ fi
+fi
+
+echo; echo "IPv4 Alias Table IP Total"; echo "-----------------------------"
+find $pfsense_alias_dir ! -name "*_v6.txt" -type f | xargs cat | grep -c ^
+
+echo; echo "IPv6 Alias Table IP Total"; echo "-----------------------------"
+find $pfsense_alias_dir -name "*_v6.txt" -type f | xargs cat | grep -c ^
+
+echo; echo "Alias Table IP Counts"; echo "-----------------------------"
+wc -l $pfsense_alias_dir*.txt | sort -n -r
+
+echo; echo "pfSense Table Stats"; echo "-------------------"
+$pathpfctl -s memory | grep "table-entries"
+pfctlcount=$($pathpfctl -vvsTables | awk '/Addresses/ {s+=$2}; END {print s}')
+echo "Table Usage Count " $pfctlcount
+}
+
+remove() {
+# Remove Lists from Masterfiles and Delete Associated Files
+echo
+for i in $cc; do
+ header=$(echo "${i%*,}")
+ if [ ! "$header" == "" ]; then
+ # Make sure that Alias Exists in Masterfile before removal.
+ masterchk=$(grep -m1 "$header[[:space:]]" $masterfile)
+ if [ ! -z "$masterchk" ]; then
+ # Grep Header with a Trailing Space character
+ grep "$header[[:space:]]" $masterfile > $tempfile
+ awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
+ cut -d' ' -f2 $masterfile > $mastercat
+ fi
+ rm -rf $pfborig$header*; rm -rf $pfbdeny$header*; rm -rf $pfbmatch$header*; rm -rf $pfbpermit$header*; rm -rf $pfbnative$header*
+ echo "The Following list has been REMOVED [ $header ]"
+ fi
+ echo
+done
+
+# Delete Masterfiles if they are empty
+emptychk=$(find $masterfile -size 0)
+if [ ! "$emptychk" == "" ]; then
+ rm -r $masterfile; rm -r $mastercat
+fi
+}
+
+
+##########
+# CALL APPROPRIATE PROCESSES using Script Argument $1
+case $1 in
+ continent)
+ continent
+ ;;
+ duplicate)
+ process255
+ duplicate
+ ;;
+ suppress)
+ suppress
+ ;;
+ p24)
+ process24
+ ;;
+ dedup)
+ deduplication
+ ;;
+ pdup)
+ pdeduplication
+ ;;
+ et)
+ processet
+ ;;
+ xlsx)
+ processxlsx
+ ;;
+ closing)
+ closingprocess
+ ;;
+ remove)
+ remove
+ ;;
+ *)
+ exit
+ ;;
+esac
+exit \ No newline at end of file