#!/bin/sh # pfBlockerNG IP Reputation Script - By BBcan177@gmail.com - 04-12-14 # Copyright (C) 2014 BBcan177@gmail.com # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License Version 2 as # published by the Free Software Foundation. You may not use, modify or # distribute this program under any other version of the GNU General # Public License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. now=$(/bin/date +%m/%d/%y' '%T) mtype=$(/usr/bin/uname -m); # Application Paths pathgrepcidr="/usr/pbi/pfblockerng-$mtype/bin/grepcidr" pathgeoip="/usr/pbi/pfblockerng-$mtype/bin/geoiplookup" pathtar=/usr/bin/tar pathgunzip=/usr/bin/gunzip pathpfctl=/sbin/pfctl # Script Arguments alias=$2 max=$3 dedup=$4 cc=$(echo $5 | sed 's/,/, /g') ccwhite=$(echo $6 | tr '[A-Z]' '[a-z]') ccblack=$(echo $7 | tr '[A-Z]' '[a-z]') etblock=$(echo $8 | sed 's/,/, /g') etmatch=$(echo $9 | sed 's/,/, /g') # File Locations pathgeoipdat=/var/db/pfblockerng/GeoIP.dat pfbsuppression=/var/db/pfblockerng/pfbsuppression.txt masterfile=/var/db/pfblockerng/masterfile mastercat=/var/db/pfblockerng/mastercat geoiplog=/var/log/pfblockerng/geoip.log errorlog=/var/log/pfblockerng/error.log # Folder Locations etdir=/var/db/pfblockerng/ET tmpxlsx=/tmp/xlsx/ pfbdeny=/var/db/pfblockerng/deny/ pfborig=/var/db/pfblockerng/original/ pfbmatch=/var/db/pfblockerng/match/ pfbpermit=/var/db/pfblockerng/permit/ pfbnative=/var/db/pfblockerng/native/ pfsense_alias_dir=/var/db/aliastables/ # Store "Match" d-dedups in matchdedup.txt file matchdedup=matchdedup.txt tempfile=/tmp/pfbtempfile tempfile2=/tmp/pfbtempfile2 dupfile=/tmp//pfbduptemp dedupfile=/tmp/pfbdeduptemp addfile=/tmp/pfBaddfile syncfile=/tmp/pfbsyncfile matchfile=/tmp/pfbmatchfile tempmatchfile=/tmp/pfbtempmatchfile if [ ! -f $masterfile ]; then touch $masterfile; fi if [ ! -f $mastercat ]; then touch $mastercat; fi if [ ! -f $tempfile ]; then touch $tempfile; fi if [ ! -f $tempfile2 ]; then touch $tempfile2; fi if [ ! -f $dupfile ]; then touch $dupfile; fi if [ ! -f $dedupfile ]; then touch $dedupfile; fi if [ ! -f $addfile ]; then touch $addfile; fi if [ ! -f $syncfile ]; then touch $syncfile; fi if [ ! -f $matchfile ]; then touch $matchfile; fi if [ ! -f $tempmatchfile ]; then touch $tempmatchfile; fi if [ ! -d $pfbmatch ]; then mkdir $pfbmatch; fi if [ ! -d $etdir ]; then mkdir $etdir; fi if [ ! -d $tmpxlsx ]; then mkdir $tmpxlsx; fi ########## # Process to condense an IP range if a "Max" amount of IP addresses are found in a /24 range per Alias Group. process24() { if [ ! -x $pathgeoip ]; then echo "Process24 - Application [ GeoIP ] Not found. Can't proceed." echo "Process24 - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog exit fi # Download MaxMind GeoIP.dat Binary on first Install. if [ ! -f $pathgeoipdat ]; then echo "Downloading [ MaxMind GeoIP.dat ] [ $now ]" >> $geoiplog /usr/local/pkg/pfblockerng/geoipupdate.sh bu fi # Exit if GeoIP.dat is not found. if [ ! -f $pathgeoipdat ]; then echo "Process24 - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed." echo "Process24 - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog exit fi count=$(grep -c ^ $pfbdeny$alias".txt") echo; echo "Original File Count [ $count ]" grep -Ev "^(#|$)" $pfbdeny$alias".txt" | sort | uniq > $tempfile > $dupfile; > $tempfile2; > $matchfile; > $tempmatchfile data="$(cut -d '.' -f 1-3 $tempfile | awk -v max="$max" '{a[$0]++}END{for(i in a){if(a[i] > max){print i}}}')" count=$(echo "$data" | grep -c ^); mcount=0; dcount=0; safe=0 if [ "$data" == "" ]; then count=0; fi matchoutfile="match"$header".txt" # Classify Repeat Offenders by Country Code if [ -f $pathgeoipdat ]; then for ip in $data; do ccheck=$($pathgeoip -f $pathgeoipdat "$ip.1" | cut -c 24-25) case "$cc" in *$ccheck*) safe=$(($safe + 1)) if [ "$ccwhite" == "match" -o "$ccblack" == "match" ]; then echo "$ip." >> $matchfile fi ;; *) echo "$ip." >> $dupfile ;; esac done else echo; echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping p24 Process"; echo echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping p24 Process [ $now ]" >> $errorlog fi # Collect Match File Details if [ -s "$matchfile" -a ! "$dedup" == "on" -a "$ccwhite" == "match" ]; then mon=$(sed -e 's/^/^/' -e 's/\./\\\./g' $matchfile) for ip in $mon; do grep $ip $tempfile >> $tempfile2 done mcount=$(grep -c ^ $tempfile2) if [ "$ccwhite" == "match" ]; then sed 's/$/0\/24/' $matchfile >> $tempmatchfile sed 's/^/\!/' $tempfile2 >> $tempmatchfile fi fi # If no Matches found remove previous Matchoutfile if exists. if [ ! -s "$tempmatchfile" -a -f $matchoutfile ]; then rm -r $matchoutfile; fi # Move Match File to the Match Folder by Individual Blocklist Name if [ -s "$tempmatchfile" ]; then mv -f $tempmatchfile $pfbmatch$matchoutfile; fi # Find Repeat Offenders in each individual Blocklist Outfile if [ -s "$dupfile" ]; then > $tempfile2 dup=$(sed -e 's/^/^/' -e 's/\./\\\./g' $dupfile) for ip in $dup; do grep $ip $tempfile >> $tempfile2 done dcount=$(grep -c ^ $tempfile2) if [ "$ccblack" == "block" ]; then awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile2 $tempfile > $pfbdeny$alias".txt" sed 's/$/0\/24/' $dupfile >> $pfbdeny$alias".txt" elif [ "$ccblack" == "match" ]; then sed 's/$/0\/24/' $dupfile >> $tempmatchfile sed 's/^/\!/' $tempfile2 >> $tempmatchfile else : fi fi if [ "$count" == "0" -a "$safe" == "0" ]; then echo; echo " Process /24 Stats [ $alias ] [ $now ] "; echo "------------------------------------------------"; fi if [ "$count" == "0" ]; then echo "Found [ 0 ] IP range(s) over the threshold of [ $max ] p24 - CC Blacklist"; fi if [ "$safe" == "0" ]; then echo "Found [ 0 ] IP range(s) over the threshold of [ $max ] p24 - CC Whitelist"; fi if [ -s "$dupfile" -o -s "$matchfile" ]; then echo echo " Process /24 Stats [ $alias ] [ $now ]" echo "--------------------------------------------------------" echo "Found [ $count ] IP range(s) over the threshold of [ $max ] on the CC Blacklist" echo "Found [ $safe ] IP range(s) over the threshold of [ $max ] on the CC Whitelist" echo echo "Found [ $dcount ] CC Blacklisted IP Address(es) are being set to [ $ccblack ]" # Skip Match Process if dedup=yes as it will create duplicates if [ "$dedup" == "on" ]; then mcount=Skipped; fi echo "Found [ $mcount ] CC Whitelisted IP Address(es) are being set to [ $ccwhite ]" if [ "$ccblack" == "block" ]; then echo; echo "Removed the following IP Ranges" cat $dupfile | tr '\n' '|'; echo else echo "Skipped, CCBlack set to [ $ccblack ]" fi sort $pfbdeny$alias".txt" | uniq > $tempfile; mv -f $tempfile $pfbdeny$alias".txt" echo "-------------------------------------------------------" cocount=$(grep -cv "^1\.1\.1\.1" $pfbdeny$alias".txt") echo "Post /24 Count [ $cocount ]"; echo fi } ########## process255() { # Remove IPs if exists over 255 IPs in a Range and replace with a single /24 Block cp $pfbdeny$alias".txt" $tempfile; > $dedupfile data255="$(cut -d '.' -f 1-3 $tempfile | awk '{a[$0]++}END{for(i in a){if(a[i] > 255){print i}}}')" if [ ! -z "$data255" ]; then for ip in $data255; do ii=$(echo "^$ip" | sed 's/\./\\\./g') grep $ii $tempfile >> $dedupfile done awk 'FNR==NR{a[$0];next}!($0 in a)' $dedupfile $tempfile > $pfbdeny$alias".txt" for ip in $data255; do echo $ip"0/24" >> $pfbdeny$alias".txt"; done fi } ########## continent() { dupcheck=yes # Check if Masterfile is Empty hcheck=$(grep -c ^ $masterfile); if [ "$hcheck" -eq "0" ]; then dupcheck=no; fi # Check if Alias exists in Masterfile lcheck=$(grep -m 1 "$alias " $masterfile ); if [ "$lcheck" == "" ]; then dupcheck=no; fi if [ "$dupcheck" == "yes" ]; then # Grep Alias with a trailing Space character grep "$alias[[:space:]]" $masterfile > $tempfile awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile cut -d' ' -f2 $masterfile > $mastercat fi grep -Ev "^(#|$)" $pfbdeny$alias".txt" | sort | uniq > $tempfile if [ ! "$hcheck" -eq "0" ]; then $pathgrepcidr -vf $mastercat $pfbdeny$alias".txt" > $tempfile; mv -f $tempfile $pfbdeny$alias".txt" fi sed -e 's/^/'$alias' /' $pfbdeny$alias".txt" >> $masterfile cut -d' ' -f2 $masterfile > $mastercat countg=$(grep -c ^ $pfborig$alias".orig") countm=$(grep -c "$alias " $masterfile); counto=$(grep -c ^ $pfbdeny$alias".txt") if [ "$countm" == "$counto" ]; then sanity="Passed"; else sanity=" ==> FAILED <== "; fi echo "----------------------------------------------------------" echo; echo " Post Duplication count [ $now ]" echo "----------------------------------------------------------" printf "%-10s %-10s %-10s %-30s\n" "Original" "Masterfile" "Outfile" "Sanity Check" echo "----------------------------------------------------------" printf "%-10s %-10s %-10s %-30s\n" "$countg" "$countm" "$counto" " [ $sanity ]" echo "----------------------------------------------------------" } ########## # Process to remove Suppressed Entries and RFC 1918 and Misc IPs on each downloaded Blocklist suppress() { if [ ! -x $pathgrepcidr ]; then echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" >> errorlog exit fi if [ -e "$pfbsuppression" ] && [ -s "$pfbsuppression" ]; then # Find '/24' Blocked IPs that are single addresses in the Suppressed IP Address List. # These '/24' Are converted to single Addresses excluding the Suppressed IPs. data="$(cat $pfbsuppression)" if [ ! -z "$data" -a ! -z "$cc" ]; then # Loop thru each Updated List to remove Suppression and RFC1918 Addresses if [ "$cc" == "suppressheader" ]; then echo; echo "===[ Suppression Stats ]========================================"; echo printf "%-20s %-10s %-10s %-10s %-10s\n" "List" "Pre" "RFC1918" "Suppress" "Masterfile" echo "----------------------------------------------------------------" exit fi for i in $cc; do counter=0 > $dupfile alias=$(echo "${i%|*}") pfbfolder=$(echo "${i#*|}") if [ ! "$alias" == "" ]; then # Count (PRE) countg=$(grep -c ^ $pfbfolder$alias".txt") grep -Ev "^(192\.168|10\.|172\.1[6789]\.|172\.2[0-9]\.|172\.3[01]\.|127\.0\.0\.1|0\.0\.0\.0|#|$)" $pfbfolder$alias".txt" | sort | uniq > $tempfile # Count (Post RFC1918) countm=$(grep -c ^ $tempfile) for ip in $data; do found=""; ddcheck=""; iptrim=$(echo $ip | cut -d '.' -f 1-3) mask=$(echo $ip | cut -d"/" -f2) found=$(grep -m1 $iptrim".0/24" $tempfile) # If a Suppression is '/32' and a Blocklist has a full '/24' Block execute the following. if [ ! "$found" == "" -a "$mask" == "32" ]; then echo " Suppression $alias: $iptrim.0/24" octet4=$(echo $ip | cut -d '.' -f 4 | sed 's/\/.*//') dcheck=$(grep $iptrim".0/24" $dupfile) if [ "$dcheck" == "" ]; then echo $iptrim".0" >> $tempfile echo $iptrim".0/24" >> $dupfile counter=$(($counter + 1)) # Add Individual IP addresses from Range excluding Suppressed IP for i in $(/usr/bin/jot 255); do if [ "$i" != "$octet4" ]; then echo $iptrim"."$i >> $tempfile counter=$(($counter + 1)) fi done fi fi done if [ -s $dupfile ]; then # Remove '/24' Suppressed Ranges awk 'FNR==NR{a[$0];next}!($0 in a)' $dupfile $tempfile > $tempfile2; mv -f $tempfile2 $tempfile fi # Remove All other Suppressions from Lists $pathgrepcidr -vf $pfbsuppression $tempfile > $pfbfolder$alias".txt" # Update Masterfiles. Don't execute if Duplication Process is Disabled if [ "$dedup" == "x" ]; then # Dont execute if Alias doesnt exist in Masterfile lcheck=$(grep -m1 "$alias " $masterfile) if [ ! "$lcheck" == "" ]; then # Replace Masterfile with changes to List. grep "$alias[[:space:]]" $masterfile > $tempfile awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile sed -e 's/^/'$alias' /' $pfbfolder$alias".txt" >> $masterfile cut -d' ' -f2 $masterfile > $mastercat fi fi countk=$(grep -c ^ $masterfile) countx=$(grep -c ^ $pfbfolder$alias".txt") counto=$(($countx - $counter)) printf "%-20s %-10s %-10s %-10s %-10s\n" "$alias" "$countg" "$countm" "$counto" "$countk" fi done fi else if [ "$cc" == "suppressheader" ]; then echo "===[ Suppression Stats ]========================================"; echo printf "%-20s %-10s %-10s %-10s %-10s\n" "List" "Pre" "RFC1918" "Suppress" "Masterfile" echo "----------------------------------------------------------------" exit fi for i in $cc; do alias=$(echo "${i%|*}") pfbfolder=$(echo "${i#*|}") if [ ! "$alias" == "" ]; then countg=$(grep -c ^ $pfbfolder$alias".txt") grep -Ev "^(192\.168|10\.|172\.1[6789]\.|172\.2[0-9]\.|172\.3[01]\.|127\.0\.0\.1|0\.0\.0\.0|#|$)" $pfbfolder$alias".txt" | sort | uniq > $tempfile; mv -f $tempfile $pfbfolder$alias".txt" countx=$(grep -c ^ $pfbfolder$alias".txt") # Update Masterfiles. Don't execute if Duplication Process is Disabled or if No Suppression Changes Found if [ "$dedup" == "x" -a "$countg" != "$countx" ]; then # Dont execute if Alias doesnt exist in Masterfile lcheck=$(grep -m1 "$alias " $masterfile) if [ ! "$lcheck" == "" ]; then # Replace Masterfile with changes to List. grep "$alias[[:space:]]" $masterfile > $tempfile awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile sed -e 's/^/'$alias' /' $pfbfolder$alias".txt" >> $masterfile cut -d' ' -f2 $masterfile > $mastercat fi fi countm=$(grep -c ^ $pfbfolder$alias".txt") counto=" - " countk=$(grep -c ^ $masterfile) printf "%-20s %-10s %-10s %-10s %-10s\n" "$alias" "$countg" "$countm" "$counto" "$countk" fi done fi } ########## # Process to remove Duplicate Entries on each downloaded Blocklist Individually duplicate() { if [ ! -x $pathgrepcidr ]; then echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" >> errorlog exit fi dupcheck=yes # Check if Masterfile is Empty hcheck=$(grep -cv "^$" $masterfile); if [ "$hcheck" -eq "0" ]; then dupcheck=no; fi # Check if Alias exists in Masterfile lcheck=$(grep -m1 "$alias " $masterfile); if [ "$lcheck" == "" ]; then dupcheck=no; fi if [ "$dupcheck" == "yes" ]; then # Grep Alias with a trailing Space character grep "$alias[[:space:]]" $masterfile > $tempfile awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile cut -d' ' -f2 $masterfile > $mastercat fi grep -Ev "^(#|$)" $pfbdeny$alias".txt" | sort | uniq > $tempfile; mv -f $tempfile $pfbdeny$alias".txt" if [ ! "$hcheck" -eq "0" ]; then $pathgrepcidr -vf $mastercat $pfbdeny$alias".txt" > $tempfile; mv -f $tempfile $pfbdeny$alias".txt" fi sed -e 's/^/'$alias' /' $pfbdeny$alias".txt" >> $masterfile cut -d' ' -f2 $masterfile > $mastercat countg=$(grep -c ^ $pfborig$alias".orig") countm=$(grep -c "$alias " $masterfile); counto=$(grep -c ^ $pfbdeny$alias".txt") if [ "$countm" == "$counto" ]; then sanity="Passed"; else sanity=" ==> FAILED <== "; fi echo "----------------------------------------------------------" printf "%-10s %-10s %-10s %-30s\n" "Original" "Masterfile" "Outfile" " [ Post Duplication count ]" echo "----------------------------------------------------------" printf "%-10s %-10s %-10s %-30s\n" "$countg" "$countm" "$counto" " [ $sanity ]" echo "----------------------------------------------------------" } ########## # De-Duplication utilizing MaxMind GeoIP Country Code Whitelisting ("dmax" variable) deduplication() { if [ ! -x $pathgeoip ]; then echo "d-duplication - Application [ GeoIP ] Not found. Can't proceed." echo "d-duplication - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog exit fi # Download MaxMind GeoIP.dat on first Install. if [ ! -f $pathgeoipdat ]; then echo "Downloading [ MaxMind GeoIP.dat ] [ $now ]" >> $geoiplog /usr/local/pkg/pfblockerng/geoipupdate.sh bu fi # Exit if GeoIP.dat is not found if [ ! -f $pathgeoipdat ]; then echo "d-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed." echo "d-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog exit fi > $tempfile; > $tempfile2; > $dupfile; > $addfile; > $dedupfile; > $matchfile; > $tempmatchfile; count=0; dcount=0; mcount=0; mmcount=0 echo; echo "Querying for Repeat Offenders" data="$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | cut -d '.' -f 1-3 $pfbdeny*.txt | awk -v max="$max" '{a[$0]++}END{for(i in a){if(a[i] > max){print i}}}' | grep -v "^1\.1\.1")" count=$(echo "$data" | grep -c ^) if [ "$data" == "" ]; then count=0; fi safe=0 # Classify Repeat Offenders by Country Code if [ -f $pathgeoipdat ]; then echo "Classifying Repeat Offenders by GeoIP" for ip in $data; do ccheck=$($pathgeoip -f $pathgeoipdat "$ip.1" | cut -c 24-25) case "$cc" in *$ccheck*) safe=$(($safe + 1)) if [ "$ccwhite" == "match" -o "$ccblack" == "match" ]; then echo "$ip." >> $matchfile fi ;; *) echo "$ip." >> $dupfile ;; esac done else echo; echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping d-dedup Process"; echo echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping d-dedup Process [ $now ]" >> $errorlog fi if [ -s "$matchfile" -a "$ccwhite" == "match" ]; then echo "Processing [ Match ] IPs" match=$(sed -e 's/^/^/' -e 's/\./\\\./g' $matchfile) for mfile in $match; do grep $mfile $pfbdeny*.txt >> $tempfile done sed 's/$/0\/24/' $matchfile >> $tempmatchfile sed -e 's/.*://' -e 's/^/\!/' $tempfile >> $tempmatchfile mv -f $tempmatchfile $pfbmatch$matchdedup mcount=$(grep -c ^ $tempfile) mmcount=$(($mcount + $mmcount)) fi # Find Repeat Offenders in each individual Blocklist Outfile if [ -s "$dupfile" ]; then echo "Processing [ Block ] IPs" dup=$(cat $dupfile) for ip in $dup; do pcount=1; ii=$(echo "^$ip" | sed 's/\./\\\./g') list=$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | xargs grep -al $ii) for blfile in $list; do header=$(echo "${blfile##*/}" | cut -d '.' -f1) grep $ii $blfile > $tempfile if [ "$ccblack" == "block" ]; then awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $blfile > $tempfile2; mv -f $tempfile2 $blfile if [ "$pcount" -eq "1" ]; then echo $ip"0/24" >> $blfile echo $header" "$ip >> $dedupfile echo $header" "$ip"0/24" >> $addfile pcount=2 else echo $header" "$ip >> $dedupfile fi else if [ "$pcount" -eq "1" ]; then matchoutfile="match"$header".txt" echo $ip"0/24" >> $pfbmatch$matchoutfile sed 's/^/\!/' $tempfile >> $pfbmatch$matchoutfile mcount=$(grep -c ^ $pfbmatch$matchoutfile) mmcount=$(($mcount + $mmcount)) pcount=2 fi fi done done # Remove Repeat Offenders in Masterfiles if [ -s "$dedupfile" ]; then echo "Removing [ Block ] IPs" > $tempfile; > $tempfile2 sed 's/\./\\\./g' $dedupfile > $tempfile2 while IFS=' ' read -r ips; do grep "$ips" $masterfile >> $tempfile; done < $tempfile2 dcount=$(grep -c ^ $tempfile) awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile cat $addfile >> $masterfile cut -d' ' -f2 $masterfile > $mastercat fi fi echo; echo "d-Duplication Process [ $now ]"; echo "------------------------------------------------" echo; echo "Found [ $count ] IP range(s) over the threshold of dmax= [ $max ]" echo "Found [ $safe ] IP range(s) classified as Whitelisted" echo; echo "Found [ $dcount ] CC Blacklisted IP Address(es) are being set to [ $ccblack ]" echo "Found [ $mmcount ] CC Whitelisted IP Address(es) are being set to [ $ccwhite ]"; echo if [ -s "$addfile" ]; then echo; echo "Removed the following IP Ranges" sed -e 's/^.* //' -e 's/0\/24//' $addfile | tr '\n' '|'; echo fi count=$(grep -c ^ $masterfile) echo " [ Post d-Deduplication count ] [ $count ]"; echo # Write "1.1.1.1" to empty Final Blocklist Files emptyfiles=$(find $pfbdeny -size 0) for i in $emptyfiles; do echo "1.1.1.1" > $i; done } ########## # Process to perform a final De-Duplication on all of the BlockLists (Excluding Country Whitelist) ("pmax" variable). pdeduplication(){ if [ ! -x $pathgeoip ]; then echo "p-duplication - Application [ GeoIP ] Not found. Can't proceed." echo "p-duplication - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog exit fi # Download MaxMind GeoIP.dat on first Install. if [ ! -f $pathgeoipdat ]; then echo "Downloading [ MaxMind GeoIP.dat ] [ $now ]" >> $geoiplog /usr/local/pkg/pfblockerng/geoipupdate.sh bu fi # Exit if GeoIP.dat is not found. if [ ! -f $pathgeoipdat ]; then echo "p-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed." echo "p-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog exit fi > $tempfile; > $tempfile2; > $dupfile; > $addfile; > $dedupfile; count=0; dcount=0 echo; echo "=====================================================================" echo; echo "Querying for Repeat Offenders" data="$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | cut -d '.' -f 1-3 $pfbdeny*.txt | awk -v max="$max" '{a[$0]++}END{for(i in a){if(a[i] > max){print i}}}' | grep -v "^1\.1\.1")" count=$(echo "$data" | grep -c ^) if [ "$data" == "" ]; then count=0; fi # Find Repeat Offenders in each individual Blocklist Outfile echo "Processing [ Block ] IPs" for ip in $data; do pcount=1; ii=$(echo "^$ip." | sed 's/\./\\\./g') list=$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | xargs grep -al $ii) for blfile in $list; do header=$(echo "${blfile##*/}" | cut -d '.' -f1) grep $ii $blfile > $tempfile awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $blfile > $tempfile2; mv -f $tempfile2 $blfile if [ "$pcount" -eq "1" ]; then echo $ip".0/24" >> $blfile echo $header" $ip." >> $dedupfile echo $header" "$ip".0/24" >> $addfile pcount=2 else echo $header" $ip." >> $dedupfile fi done done # Remove Repeat Offenders in Masterfile if [ -s "$dedupfile" ]; then echo "Removing [ Block ] IPs" > $tempfile; > $tempfile2 sed 's/\./\\\./g' $dedupfile > $tempfile2 while IFS=' ' read -r ips; do grep "$ips" $masterfile >> $tempfile; done < $tempfile2 dcount=$(grep -c ^ $tempfile) awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile cat $addfile >> $masterfile cut -d' ' -f2 $masterfile > $mastercat fi echo; echo "p-Duplication Process [ $now ]"; echo "------------------------------------------------" echo "Found [ $dcount ] IP Address(es) are being set to [ block ]" if [ -s "$addfile" ]; then echo; echo "Removed the following IP Ranges" sed -e 's/^.* //' -e 's/0\/24//' $addfile | tr '\n' '|'; echo fi count=$(grep -c ^ $masterfile) echo; echo " [ Post p-Deduplication count ] [ $count ]" # Write "1.1.1.1" to empty Final Blocklist Files emptyfiles=$(find $pfbdeny -size 0) for i in $emptyfiles; do echo "1.1.1.1" > $i; done } ########## # Process to Split ET Pro IPREP into Category Files and Compile selected Blocked categories into Outfile processet() { if [ ! -x $pathgunzip ]; then echo "Application [ Gunzip ] Not found, Can't proceed." echo "Application [ Gunzip ] Not found, Can't proceed. [ $now ]" >> $errorlog exit fi if [ -s $pfborig$alias".gz" ]; then evar="ET_*" # Remove Previous ET IPRep Files [ -d $etdir ] && [ "$(ls -A $etdir)" ] && rm -r $etdir/$evar > $tempfile; > $tempfile2 $pathgunzip -c $pfborig$alias".gz" > $pfborig$alias".raw" # ET CSV Format (IP, Category, Score) echo; echo "Processing [ $alias ]" while IFS="," read a b c; do # Some ET Categories are not in use (For Future Use) case "$b" in 1) echo $a >> $etdir/ET_Cnc;; 2) echo $a >> $etdir/ET_Bot;; 3) echo $a >> $etdir/ET_Spam;; 4) echo $a >> $etdir/ET_Drop;; 5) echo $a >> $etdir/ET_Spywarecnc;; 6) echo $a >> $etdir/ET_Onlinegaming;; 7) echo $a >> $etdir/ET_Drivebysrc;; 8) echo $a >> $etdir/ET_Cat8;; 9) echo $a >> $etdir/ET_Chatserver;; 10) echo $a >> $etdir/ET_Tornode;; 11) echo $a >> $etdir/ET_Cat11;; 12) echo $a >> $etdir/ET_Cat12;; 13) echo $a >> $etdir/ET_Compromised;; 14) echo $a >> $etdir/ET_Cat14;; 15) echo $a >> $etdir/ET_P2P;; 16) echo $a >> $etdir/ET_Proxy;; 17) echo $a >> $etdir/ET_Ipcheck;; 18) echo $a >> $etdir/ET_Cat18;; 19) echo $a >> $etdir/ET_Utility;; 20) echo $a >> $etdir/ET_DDos;; 21) echo $a >> $etdir/ET_Scanner;; 22) echo $a >> $etdir/ET_Cat22;; 23) echo $a >> $etdir/ET_Brute;; 24) echo $a >> $etdir/ET_Fakeav;; 25) echo $a >> $etdir/ET_Dyndns;; 26) echo $a >> $etdir/ET_Undesireable;; 27) echo $a >> $etdir/ET_Abusedtld;; 28) echo $a >> $etdir/ET_Selfsignedssl;; 29) echo $a >> $etdir/ET_Blackhole;; 30) echo $a >> $etdir/ET_RAS;; 31) echo $a >> $etdir/ET_P2Pcnc;; 32) echo $a >> $etdir/ET_Sharedhosting;; 33) echo $a >> $etdir/ET_Parking;; 34) echo $a >> $etdir/ET_VPN;; 35) echo $a >> $etdir/ET_Exesource;; 36) echo $a >> $etdir/ET_Cat36;; 37) echo $a >> $etdir/ET_Mobilecnc;; 38) echo $a >> $etdir/ET_Mobilespyware;; 39) echo $a >> $etdir/ET_Skypenode;; 40) echo $a >> $etdir/ET_Bitcoin;; 41) echo $a >> $etdir/ET_DDosattack;; *) echo $a >> $etdir/ET_Unknown;; esac done <"$pfborig$alias.raw" data=$(ls $etdir) echo "Compiling ET IP IQRisk REP Lists based upon User Selected Categories" printf "%-10s %-25s\n" " Action" "Category" echo "-------------------------------------------" for list in $data; do case "$etblock" in *$list*) printf "%-10s %-25s\n" " Block: " "$list" cat $etdir/$list >> $tempfile ;; esac case "$etmatch" in *$list*) printf "%-10s %-25s\n" " Match: " "$list" cat $etdir/$list >> $tempfile2 ;; esac done echo "-------------------------------------------" if [ -f $tempfile ]; then mv -f $tempfile $pfborig$alias".orig"; fi if [ "$etmatch" != "x" ]; then mv -f $tempfile2 $pfbmatch/ETMatch.txt; fi cicount=$(cat $etdir/$evar | grep -cv '^#\|^$'); cocount=$(grep -cv "^1\.1\.1\.1" $pfborig$alias".orig") echo; echo "ET Folder count [ $cicount ] Outfile count [ $cocount ]" else echo; echo "No ET .GZ File Found!" fi } # Process to extract IP addresses from XLSX Files processxlsx() { if [ ! -x $pathtar ]; then echo "Application [ TAR ] Not found, Can't proceed." echo "Application [ TAR ] Not found, Can't proceed. [ $now ]" >> $errorlog exit fi if [ -s $pfborig$alias".zip" ]; then $pathtar -xf $pfborig$alias".zip" -C $tmpxlsx $pathtar -xOf $tmpxlsx*.[xX][lL][sS][xX] xl/sharedStrings.xml | grep -aoEw "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" | sort | uniq > $pfborig$alias".orig" rm -r $tmpxlsx* cocount=$(grep -cv "^1\.1\.1\.1" $pfborig$alias".orig") echo; echo "Download file count [ ZIP file ] Outfile count [ $cocount ]" else echo "XLSX Download File Missing" echo " [ $alias ] XLSX Download File Missing [ $now ]" >> $errorlog fi } closingprocess() { # Write "1.1.1.1" to empty Final Blocklist Files emptyfiles=$(find $pfbdeny -size 0) for i in $emptyfiles; do echo "1.1.1.1" > $i; done if [ -d "$pfborig" ] && [ "$(ls -A $pfborig)" ]; then fcount=$(find $pfborig*.orig | xargs cat | grep -cv '^#\|^$') else fcount=0 fi if [ "$alias" == "on" ]; then sort -o $masterfile $masterfile sort -t . -k 1,1n -k 2,2n -k 3,3n -k 4,4n $mastercat > $tempfile; mv -f $tempfile $mastercat echo; echo; echo "===[ FINAL Processing ]====================================="; echo echo " [ Original count ] [ $fcount ]" count=$(grep -c ^ $masterfile) echo; echo " [ Processed Count ] [ $count ]"; echo s1=$(grep -cv "1\.1\.1\.1" $masterfile) s2=$(find $pfbdeny ! -name "*_v6.txt" -type f | xargs cat | grep -cv "^1\.1\.1\.1") s3=$(sort $mastercat | uniq -d | tail -30) s4=$(find $pfbdeny ! -name "*_v6.txt" -type f | xargs cat | sort | uniq -d | tail -30 | grep -v "^1\.1\.1\.1") if [ -d "$pfbpermit" ] && [ "$(ls -A $pfbpermit)" ]; then echo; echo "===[ Permit List IP Counts ]========================="; echo wc -l $pfbpermit* | sort -n -r fi if [ -d "$pfbmatch" ] && [ "$(ls -A $pfbmatch)" ]; then echo; echo "===[ Match List IP Counts ]=========================="; echo wc -l $pfbmatch* | sort -n -r fi if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then echo; echo "===[ Deny List IP Counts ]==========================="; echo wc -l $pfbdeny* | sort -n -r fi if [ -d "$pfbnative" ] && [ "$(ls -A $pfbnative)" ]; then echo; echo "===[ Native List IP Counts ] ==================================="; echo wc -l $pfbnative* | sort -n -r fi if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then emptylists=$(grep "1\.1\.1\.1" $pfbdeny* | sed -e 's/^.*[a-zA-Z]\///' -e 's/\.txt:1.1.1.1/ /') if [ ! -z "$emptylists" ]; then echo; echo "====================[ Empty Lists w/1.1.1.1 ]=================="; echo for list in $emptylists; do echo $list done fi fi if [ -d "$pfborig" ] && [ "$(ls -A $pfborig)" ]; then echo; echo "====================[ Last Updated List Summary ]=============="; echo ls -lahtr $pfborig* | sed -e 's/\/.*\// /' -e 's/.orig//' | awk -v OFS='\t' '{print $6" "$7,$8,$9}' fi echo "==============================================================="; echo echo "Sanity Check (Not Including IPv6) ** These two Counts should Match! **" echo "------------" echo "Masterfile Count [ $s1 ]" echo "Deny folder Count [ $s2 ]"; echo echo "Duplication Sanity Check (Pass=No IPs reported)" echo "------------------------" echo "Masterfile/Deny Folder Uniq check" if [ ! -z "$s3" ]; then echo $s3; fi echo "Deny Folder/Masterfile Uniq check" if [ ! -z "$s4" ]; then echo $s4; fi echo; echo "Sync Check (Pass=No IPs reported)" echo "----------" else echo; echo "===[ FINAL Processing ]============================================="; echo echo " [ Original count ] [ $fcount ]" if [ -d "$pfbpermit" ] && [ "$(ls -A $pfbpermit)" ]; then echo; echo "===[ Permit List IP Counts ]========================="; echo wc -l $pfbpermit* | sort -n -r fi if [ -d "$pfbmatch" ] && [ "$(ls -A $pfbmatch)" ]; then echo; echo "===[ Match List IP Counts ]=========================="; echo wc -l $pfbmatch* | sort -n -r fi if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then echo; echo "===[ Deny List IP Counts ]==========================="; echo wc -l $pfbdeny* | sort -n -r fi if [ -d "$pfbnative" ] && [ "$(ls -A $pfbnative)" ]; then echo; echo "===[ Native List IP Counts ] ==================================="; echo wc -l $pfbnative* | sort -n -r fi if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then emptylists=$(grep "1\.1\.1\.1" $pfbdeny* | sed -e 's/^.*[a-zA-Z]\///' -e 's/\.txt:1.1.1.1/ /') if [ ! -z "$emptylists" ]; then echo; echo "====================[ Empty Lists w/1.1.1.1 ]=================="; echo for list in $emptylists; do echo $list done fi fi if [ -d "$pfborig" ] && [ "$(ls -A $pfborig)" ]; then echo; echo "====================[ Last Updated List Summary ]=============="; echo ls -lahtr $pfborig* | sed -e 's/\/.*\// /' -e 's/.orig//' | awk -v OFS='\t' '{print $6" "$7,$8,$9}' echo "===============================================================" fi fi echo; echo "IPv4 Alias Table IP Total"; echo "-----------------------------" find $pfsense_alias_dir ! -name "*_v6.txt" -type f | xargs cat | grep -c ^ echo; echo "IPv6 Alias Table IP Total"; echo "-----------------------------" find $pfsense_alias_dir -name "*_v6.txt" -type f | xargs cat | grep -c ^ echo; echo "Alias Table IP Counts"; echo "-----------------------------" wc -l $pfsense_alias_dir*.txt | sort -n -r echo; echo "pfSense Table Stats"; echo "-------------------" $pathpfctl -s memory | grep "table-entries" pfctlcount=$($pathpfctl -vvsTables | awk '/Addresses/ {s+=$2}; END {print s}') echo "Table Usage Count " $pfctlcount } remove() { # Remove Lists from Masterfiles and Delete Associated Files echo for i in $cc; do header=$(echo "${i%*,}") if [ ! "$header" == "" ]; then # Make sure that Alias Exists in Masterfile before removal. masterchk=$(grep -m1 "$header[[:space:]]" $masterfile) if [ ! -z "$masterchk" ]; then # Grep Header with a Trailing Space character grep "$header[[:space:]]" $masterfile > $tempfile awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile cut -d' ' -f2 $masterfile > $mastercat fi rm -rf $pfborig$header*; rm -rf $pfbdeny$header*; rm -rf $pfbmatch$header*; rm -rf $pfbpermit$header*; rm -rf $pfbnative$header* echo "The Following list has been REMOVED [ $header ]" fi echo done # Delete Masterfiles if they are empty emptychk=$(find $masterfile -size 0) if [ ! "$emptychk" == "" ]; then rm -r $masterfile; rm -r $mastercat fi } ########## # CALL APPROPRIATE PROCESSES using Script Argument $1 case $1 in continent) continent ;; duplicate) process255 duplicate ;; suppress) suppress ;; p24) process24 ;; dedup) deduplication ;; pdup) pdeduplication ;; et) processet ;; xlsx) processxlsx ;; closing) closingprocess ;; remove) remove ;; *) exit ;; esac exit