diff options
Diffstat (limited to 'config/pfblockerng/pfblockerng.sh')
| -rw-r--r-- | config/pfblockerng/pfblockerng.sh | 927 |
1 files changed, 927 insertions, 0 deletions
diff --git a/config/pfblockerng/pfblockerng.sh b/config/pfblockerng/pfblockerng.sh new file mode 100644 index 00000000..c09d52e1 --- /dev/null +++ b/config/pfblockerng/pfblockerng.sh @@ -0,0 +1,927 @@ +#!/bin/sh +# pfBlockerNG IP Reputation Script - By BBcan177@gmail.com - 04-12-14 +# Copyright (C) 2014 BBcan177@gmail.com +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License Version 2 as +# published by the Free Software Foundation. You may not use, modify or +# distribute this program under any other version of the GNU General +# Public License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +now=$(/bin/date +%m/%d/%y' '%T) +mtype=$(/usr/bin/uname -m); + +# Application Paths +pathgrepcidr="/usr/pbi/pfblockerng-$mtype/bin/grepcidr" +pathgeoip="/usr/pbi/pfblockerng-$mtype/bin/geoiplookup" + +pathtar=/usr/bin/tar +pathgunzip=/usr/bin/gunzip +pathpfctl=/sbin/pfctl + +# Script Arguments +alias=$2 +max=$3 +dedup=$4 +cc=$(echo $5 | sed 's/,/, /g') +ccwhite=$(echo $6 | tr '[A-Z]' '[a-z]') +ccblack=$(echo $7 | tr '[A-Z]' '[a-z]') +etblock=$(echo $8 | sed 's/,/, /g') +etmatch=$(echo $9 | sed 's/,/, /g') + +# File Locations +pathgeoipdat=/var/db/pfblockerng/GeoIP.dat +pfbsuppression=/var/db/pfblockerng/pfbsuppression.txt +masterfile=/var/db/pfblockerng/masterfile +mastercat=/var/db/pfblockerng/mastercat +geoiplog=/var/log/pfblockerng/geoip.log +errorlog=/var/log/pfblockerng/error.log + +# Folder Locations +etdir=/var/db/pfblockerng/ET +tmpxlsx=/tmp/xlsx/ + +pfbdeny=/var/db/pfblockerng/deny/ +pfborig=/var/db/pfblockerng/original/ +pfbmatch=/var/db/pfblockerng/match/ +pfbpermit=/var/db/pfblockerng/permit/ +pfbnative=/var/db/pfblockerng/native/ +pfsense_alias_dir=/var/db/aliastables/ + +# Store "Match" d-dedups in matchdedup.txt file +matchdedup=matchdedup.txt + +tempfile=/tmp/pfbtempfile +tempfile2=/tmp/pfbtempfile2 +dupfile=/tmp//pfbduptemp +dedupfile=/tmp/pfbdeduptemp +addfile=/tmp/pfBaddfile +syncfile=/tmp/pfbsyncfile +matchfile=/tmp/pfbmatchfile +tempmatchfile=/tmp/pfbtempmatchfile + +if [ ! -f $masterfile ]; then touch $masterfile; fi +if [ ! -f $mastercat ]; then touch $mastercat; fi +if [ ! -f $tempfile ]; then touch $tempfile; fi +if [ ! -f $tempfile2 ]; then touch $tempfile2; fi +if [ ! -f $dupfile ]; then touch $dupfile; fi +if [ ! -f $dedupfile ]; then touch $dedupfile; fi +if [ ! -f $addfile ]; then touch $addfile; fi +if [ ! -f $syncfile ]; then touch $syncfile; fi +if [ ! -f $matchfile ]; then touch $matchfile; fi +if [ ! -f $tempmatchfile ]; then touch $tempmatchfile; fi +if [ ! -d $pfbmatch ]; then mkdir $pfbmatch; fi +if [ ! -d $etdir ]; then mkdir $etdir; fi +if [ ! -d $tmpxlsx ]; then mkdir $tmpxlsx; fi + +########## +# Process to condense an IP range if a "Max" amount of IP addresses are found in a /24 range per Alias Group. +process24() { + +if [ ! -x $pathgeoip ]; then + echo "Process24 - Application [ GeoIP ] Not found. Can't proceed." + echo "Process24 - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog + exit +fi + +# Download MaxMind GeoIP.dat Binary on first Install. +if [ ! -f $pathgeoipdat ]; then + echo "Downloading [ MaxMind GeoIP.dat ] [ $now ]" >> $geoiplog + /usr/local/pkg/pfblockerng/geoipupdate.sh bu +fi +# Exit if GeoIP.dat is not found. +if [ ! -f $pathgeoipdat ]; then + echo "Process24 - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed." + echo "Process24 - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog + exit +fi + +count=$(grep -c ^ $pfbdeny$alias".txt") +echo; echo "Original File Count [ $count ]" + +grep -Ev "^(#|$)" $pfbdeny$alias".txt" | sort | uniq > $tempfile +> $dupfile; > $tempfile2; > $matchfile; > $tempmatchfile +data="$(cut -d '.' -f 1-3 $tempfile | awk -v max="$max" '{a[$0]++}END{for(i in a){if(a[i] > max){print i}}}')" +count=$(echo "$data" | grep -c ^); mcount=0; dcount=0; safe=0 +if [ "$data" == "" ]; then count=0; fi +matchoutfile="match"$header".txt" +# Classify Repeat Offenders by Country Code +if [ -f $pathgeoipdat ]; then + for ip in $data; do + ccheck=$($pathgeoip -f $pathgeoipdat "$ip.1" | cut -c 24-25) + case "$cc" in + *$ccheck*) + safe=$(($safe + 1)) + if [ "$ccwhite" == "match" -o "$ccblack" == "match" ]; then + echo "$ip." >> $matchfile + fi + ;; + *) + echo "$ip." >> $dupfile + ;; + esac + done +else + echo; echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping p24 Process"; echo + echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping p24 Process [ $now ]" >> $errorlog +fi +# Collect Match File Details +if [ -s "$matchfile" -a ! "$dedup" == "on" -a "$ccwhite" == "match" ]; then + mon=$(sed -e 's/^/^/' -e 's/\./\\\./g' $matchfile) + for ip in $mon; do + grep $ip $tempfile >> $tempfile2 + done + mcount=$(grep -c ^ $tempfile2) + if [ "$ccwhite" == "match" ]; then + sed 's/$/0\/24/' $matchfile >> $tempmatchfile + sed 's/^/\!/' $tempfile2 >> $tempmatchfile + fi +fi + +# If no Matches found remove previous Matchoutfile if exists. +if [ ! -s "$tempmatchfile" -a -f $matchoutfile ]; then rm -r $matchoutfile; fi +# Move Match File to the Match Folder by Individual Blocklist Name +if [ -s "$tempmatchfile" ]; then mv -f $tempmatchfile $pfbmatch$matchoutfile; fi + +# Find Repeat Offenders in each individual Blocklist Outfile +if [ -s "$dupfile" ]; then + > $tempfile2 + dup=$(sed -e 's/^/^/' -e 's/\./\\\./g' $dupfile) + for ip in $dup; do + grep $ip $tempfile >> $tempfile2 + done + dcount=$(grep -c ^ $tempfile2) + if [ "$ccblack" == "block" ]; then + awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile2 $tempfile > $pfbdeny$alias".txt" + sed 's/$/0\/24/' $dupfile >> $pfbdeny$alias".txt" + elif [ "$ccblack" == "match" ]; then + sed 's/$/0\/24/' $dupfile >> $tempmatchfile + sed 's/^/\!/' $tempfile2 >> $tempmatchfile + else + : + fi +fi +if [ "$count" == "0" -a "$safe" == "0" ]; then echo; echo " Process /24 Stats [ $alias ] [ $now ] "; echo "------------------------------------------------"; fi +if [ "$count" == "0" ]; then echo "Found [ 0 ] IP range(s) over the threshold of [ $max ] p24 - CC Blacklist"; fi +if [ "$safe" == "0" ]; then echo "Found [ 0 ] IP range(s) over the threshold of [ $max ] p24 - CC Whitelist"; fi + +if [ -s "$dupfile" -o -s "$matchfile" ]; then +echo +echo " Process /24 Stats [ $alias ] [ $now ]" +echo "--------------------------------------------------------" +echo "Found [ $count ] IP range(s) over the threshold of [ $max ] on the CC Blacklist" +echo "Found [ $safe ] IP range(s) over the threshold of [ $max ] on the CC Whitelist" +echo +echo "Found [ $dcount ] CC Blacklisted IP Address(es) are being set to [ $ccblack ]" +# Skip Match Process if dedup=yes as it will create duplicates +if [ "$dedup" == "on" ]; then mcount=Skipped; fi +echo "Found [ $mcount ] CC Whitelisted IP Address(es) are being set to [ $ccwhite ]" +if [ "$ccblack" == "block" ]; then + echo; echo "Removed the following IP Ranges" + cat $dupfile | tr '\n' '|'; echo +else + echo "Skipped, CCBlack set to [ $ccblack ]" +fi +sort $pfbdeny$alias".txt" | uniq > $tempfile; mv -f $tempfile $pfbdeny$alias".txt" +echo "-------------------------------------------------------" +cocount=$(grep -cv "^1\.1\.1\.1" $pfbdeny$alias".txt") +echo "Post /24 Count [ $cocount ]"; echo +fi +} + + +########## +process255() { +# Remove IPs if exists over 255 IPs in a Range and replace with a single /24 Block +cp $pfbdeny$alias".txt" $tempfile; > $dedupfile + +data255="$(cut -d '.' -f 1-3 $tempfile | awk '{a[$0]++}END{for(i in a){if(a[i] > 255){print i}}}')" +if [ ! -z "$data255" ]; then + for ip in $data255; do + ii=$(echo "^$ip" | sed 's/\./\\\./g') + grep $ii $tempfile >> $dedupfile + done + awk 'FNR==NR{a[$0];next}!($0 in a)' $dedupfile $tempfile > $pfbdeny$alias".txt" + for ip in $data255; do echo $ip"0/24" >> $pfbdeny$alias".txt"; done +fi +} + + +########## +continent() { + +dupcheck=yes +# Check if Masterfile is Empty +hcheck=$(grep -c ^ $masterfile); if [ "$hcheck" -eq "0" ]; then dupcheck=no; fi +# Check if Alias exists in Masterfile +lcheck=$(grep -m 1 "$alias " $masterfile ); if [ "$lcheck" == "" ]; then dupcheck=no; fi + +if [ "$dupcheck" == "yes" ]; then + # Grep Alias with a trailing Space character + grep "$alias[[:space:]]" $masterfile > $tempfile + awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile + cut -d' ' -f2 $masterfile > $mastercat +fi + +grep -Ev "^(#|$)" $pfbdeny$alias".txt" | sort | uniq > $tempfile + +if [ ! "$hcheck" -eq "0" ]; then + $pathgrepcidr -vf $mastercat $pfbdeny$alias".txt" > $tempfile; mv -f $tempfile $pfbdeny$alias".txt" +fi + +sed -e 's/^/'$alias' /' $pfbdeny$alias".txt" >> $masterfile +cut -d' ' -f2 $masterfile > $mastercat + +countg=$(grep -c ^ $pfborig$alias".orig") +countm=$(grep -c "$alias " $masterfile); counto=$(grep -c ^ $pfbdeny$alias".txt") +if [ "$countm" == "$counto" ]; then sanity="Passed"; else sanity=" ==> FAILED <== "; fi +echo "----------------------------------------------------------" +echo; echo " Post Duplication count [ $now ]" +echo "----------------------------------------------------------" +printf "%-10s %-10s %-10s %-30s\n" "Original" "Masterfile" "Outfile" "Sanity Check" +echo "----------------------------------------------------------" +printf "%-10s %-10s %-10s %-30s\n" "$countg" "$countm" "$counto" " [ $sanity ]" +echo "----------------------------------------------------------" +} + + +########## +# Process to remove Suppressed Entries and RFC 1918 and Misc IPs on each downloaded Blocklist +suppress() { + +if [ ! -x $pathgrepcidr ]; then + echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" + echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" >> errorlog + exit +fi + +if [ -e "$pfbsuppression" ] && [ -s "$pfbsuppression" ]; then + # Find '/24' Blocked IPs that are single addresses in the Suppressed IP Address List. + # These '/24' Are converted to single Addresses excluding the Suppressed IPs. + data="$(cat $pfbsuppression)" + if [ ! -z "$data" -a ! -z "$cc" ]; then + # Loop thru each Updated List to remove Suppression and RFC1918 Addresses + if [ "$cc" == "suppressheader" ]; then + echo; echo "===[ Suppression Stats ]========================================"; echo + printf "%-20s %-10s %-10s %-10s %-10s\n" "List" "Pre" "RFC1918" "Suppress" "Masterfile" + echo "----------------------------------------------------------------" + exit + fi + + for i in $cc; do + counter=0 + > $dupfile + alias=$(echo "${i%|*}") + pfbfolder=$(echo "${i#*|}") + + if [ ! "$alias" == "" ]; then + # Count (PRE) + countg=$(grep -c ^ $pfbfolder$alias".txt") + + grep -Ev "^(192\.168|10\.|172\.1[6789]\.|172\.2[0-9]\.|172\.3[01]\.|#|$)" $pfbfolder$alias".txt" | + sort | uniq > $tempfile + # Count (Post RFC1918) + countm=$(grep -c ^ $tempfile) + + for ip in $data; do + found=""; ddcheck=""; + iptrim=$(echo $ip | cut -d '.' -f 1-3) + mask=$(echo $ip | cut -d"/" -f2) + found=$(grep -m1 $iptrim".0/24" $tempfile) + # If a Suppression is '/32' and a Blocklist has a full '/24' Block execute the following. + if [ ! "$found" == "" -a "$mask" == "32" ]; then + echo " Suppression $alias: $iptrim.0/24" + octet4=$(echo $ip | cut -d '.' -f 4 | sed 's/\/.*//') + dcheck=$(grep $iptrim".0/24" $dupfile) + if [ "$dcheck" == "" ]; then + echo $iptrim".0" >> $tempfile + echo $iptrim".0/24" >> $dupfile + counter=$(($counter + 1)) + # Add Individual IP addresses from Range excluding Suppressed IP + for i in $(/usr/bin/jot 255); do + if [ "$i" != "$octet4" ]; then + echo $iptrim"."$i >> $tempfile + counter=$(($counter + 1)) + fi + done + fi + fi + done + if [ -s $dupfile ]; then + # Remove '/24' Suppressed Ranges + awk 'FNR==NR{a[$0];next}!($0 in a)' $dupfile $tempfile > $tempfile2; mv -f $tempfile2 $tempfile + fi + # Remove All other Suppressions from Lists + $pathgrepcidr -vf $pfbsuppression $tempfile > $pfbfolder$alias".txt" + # Update Masterfiles. Don't execute if Duplication Process is Disabled + if [ "$dedup" == "x" ]; then + # Dont execute if Alias doesnt exist in Masterfile + lcheck=$(grep -m1 "$alias " $masterfile) + if [ ! "$lcheck" == "" ]; then + # Replace Masterfile with changes to List. + grep "$alias[[:space:]]" $masterfile > $tempfile + awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile + sed -e 's/^/'$alias' /' $pfbfolder$alias".txt" >> $masterfile + cut -d' ' -f2 $masterfile > $mastercat + fi + fi + countk=$(grep -c ^ $masterfile) + countx=$(grep -c ^ $pfbfolder$alias".txt") + counto=$(($countx - $counter)) + printf "%-20s %-10s %-10s %-10s %-10s\n" "$alias" "$countg" "$countm" "$counto" "$countk" + fi + done + fi +else + if [ "$cc" == "suppressheader" ]; then + echo "===[ Suppression Stats ]========================================"; echo + printf "%-20s %-10s %-10s %-10s %-10s\n" "List" "Pre" "RFC1918" "Suppress" "Masterfile" + echo "----------------------------------------------------------------" + exit + fi + for i in $cc; do + alias=$(echo "${i%|*}") + pfbfolder=$(echo "${i#*|}") + + if [ ! "$alias" == "" ]; then + countg=$(grep -c ^ $pfbfolder$alias".txt") + grep -Ev "^(192\.168|10\.|172\.1[6789]\.|172\.2[0-9]\.|172\.3[01]\.|#|$)" $pfbfolder$alias".txt" | + sort | uniq > $tempfile; mv -f $tempfile $pfbfolder$alias".txt" + countx=$(grep -c ^ $pfbfolder$alias".txt") + # Update Masterfiles. Don't execute if Duplication Process is Disabled or if No Suppression Changes Found + if [ "$dedup" == "x" -a "$countg" != "$countx" ]; then + # Dont execute if Alias doesnt exist in Masterfile + lcheck=$(grep -m1 "$alias " $masterfile) + if [ ! "$lcheck" == "" ]; then + # Replace Masterfile with changes to List. + grep "$alias[[:space:]]" $masterfile > $tempfile + awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile + sed -e 's/^/'$alias' /' $pfbfolder$alias".txt" >> $masterfile + cut -d' ' -f2 $masterfile > $mastercat + fi + fi + countm=$(grep -c ^ $pfbfolder$alias".txt") + counto=" - " + countk=$(grep -c ^ $masterfile) + printf "%-20s %-10s %-10s %-10s %-10s\n" "$alias" "$countg" "$countm" "$counto" "$countk" + fi + done +fi +} + + +########## +# Process to remove Duplicate Entries on each downloaded Blocklist Individually +duplicate() { + +if [ ! -x $pathgrepcidr ]; then + echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" + echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" >> errorlog + exit +fi + +dupcheck=yes +# Check if Masterfile is Empty +hcheck=$(grep -cv "^$" $masterfile); if [ "$hcheck" -eq "0" ]; then dupcheck=no; fi +# Check if Alias exists in Masterfile +lcheck=$(grep -m1 "$alias " $masterfile); if [ "$lcheck" == "" ]; then dupcheck=no; fi + +if [ "$dupcheck" == "yes" ]; then + # Grep Alias with a trailing Space character + grep "$alias[[:space:]]" $masterfile > $tempfile + awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile + cut -d' ' -f2 $masterfile > $mastercat +fi + +grep -Ev "^(#|$)" $pfbdeny$alias".txt" | sort | uniq > $tempfile; mv -f $tempfile $pfbdeny$alias".txt" + +if [ ! "$hcheck" -eq "0" ]; then + $pathgrepcidr -vf $mastercat $pfbdeny$alias".txt" > $tempfile; mv -f $tempfile $pfbdeny$alias".txt" +fi + +sed -e 's/^/'$alias' /' $pfbdeny$alias".txt" >> $masterfile +cut -d' ' -f2 $masterfile > $mastercat + +countg=$(grep -c ^ $pfborig$alias".orig") +countm=$(grep -c "$alias " $masterfile); counto=$(grep -c ^ $pfbdeny$alias".txt") +if [ "$countm" == "$counto" ]; then sanity="Passed"; else sanity=" ==> FAILED <== "; fi +echo "----------------------------------------------------------" +printf "%-10s %-10s %-10s %-30s\n" "Original" "Masterfile" "Outfile" " [ Post Duplication count ]" +echo "----------------------------------------------------------" +printf "%-10s %-10s %-10s %-30s\n" "$countg" "$countm" "$counto" " [ $sanity ]" +echo "----------------------------------------------------------" +} + + +########## +# De-Duplication utilizing MaxMind GeoIP Country Code Whitelisting ("dmax" variable) +deduplication() { + +if [ ! -x $pathgeoip ]; then + echo "d-duplication - Application [ GeoIP ] Not found. Can't proceed." + echo "d-duplication - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog + exit +fi + +# Download MaxMind GeoIP.dat on first Install. +if [ ! -f $pathgeoipdat ]; then + echo "Downloading [ MaxMind GeoIP.dat ] [ $now ]" >> $geoiplog + /usr/local/pkg/pfblockerng/geoipupdate.sh bu +fi + +# Exit if GeoIP.dat is not found +if [ ! -f $pathgeoipdat ]; then + echo "d-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed." + echo "d-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog + exit +fi + +> $tempfile; > $tempfile2; > $dupfile; > $addfile; > $dedupfile; > $matchfile; > $tempmatchfile; count=0; dcount=0; mcount=0; mmcount=0 +echo; echo "Querying for Repeat Offenders" +data="$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | cut -d '.' -f 1-3 $pfbdeny*.txt | + awk -v max="$max" '{a[$0]++}END{for(i in a){if(a[i] > max){print i}}}' | grep -v "^1\.1\.1")" +count=$(echo "$data" | grep -c ^) +if [ "$data" == "" ]; then count=0; fi +safe=0 +# Classify Repeat Offenders by Country Code +if [ -f $pathgeoipdat ]; then + echo "Classifying Repeat Offenders by GeoIP" + for ip in $data; do + ccheck=$($pathgeoip -f $pathgeoipdat "$ip.1" | cut -c 24-25) + case "$cc" in + *$ccheck*) + safe=$(($safe + 1)) + if [ "$ccwhite" == "match" -o "$ccblack" == "match" ]; then + echo "$ip." >> $matchfile + fi + ;; + *) + echo "$ip." >> $dupfile + ;; + esac + done +else + echo; echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping d-dedup Process"; echo + echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping d-dedup Process [ $now ]" >> $errorlog +fi +if [ -s "$matchfile" -a "$ccwhite" == "match" ]; then + echo "Processing [ Match ] IPs" + match=$(sed -e 's/^/^/' -e 's/\./\\\./g' $matchfile) + for mfile in $match; do + grep $mfile $pfbdeny*.txt >> $tempfile + done + sed 's/$/0\/24/' $matchfile >> $tempmatchfile + sed -e 's/.*://' -e 's/^/\!/' $tempfile >> $tempmatchfile + mv -f $tempmatchfile $pfbmatch$matchdedup + mcount=$(grep -c ^ $tempfile) + mmcount=$(($mcount + $mmcount)) +fi +# Find Repeat Offenders in each individual Blocklist Outfile +if [ -s "$dupfile" ]; then + echo "Processing [ Block ] IPs" + dup=$(cat $dupfile) + for ip in $dup; do + pcount=1; ii=$(echo "^$ip" | sed 's/\./\\\./g') + list=$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | xargs grep -al $ii) + for blfile in $list; do + header=$(echo "${blfile##*/}" | cut -d '.' -f1) + grep $ii $blfile > $tempfile + if [ "$ccblack" == "block" ]; then + awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $blfile > $tempfile2; mv -f $tempfile2 $blfile + if [ "$pcount" -eq "1" ]; then + echo $ip"0/24" >> $blfile + echo $header" "$ip >> $dedupfile + echo $header" "$ip"0/24" >> $addfile + pcount=2 + else + echo $header" "$ip >> $dedupfile + fi + else + if [ "$pcount" -eq "1" ]; then + matchoutfile="match"$header".txt" + echo $ip"0/24" >> $pfbmatch$matchoutfile + sed 's/^/\!/' $tempfile >> $pfbmatch$matchoutfile + mcount=$(grep -c ^ $pfbmatch$matchoutfile) + mmcount=$(($mcount + $mmcount)) + pcount=2 + fi + fi + done + done + # Remove Repeat Offenders in Masterfiles + if [ -s "$dedupfile" ]; then + echo "Removing [ Block ] IPs" + > $tempfile; > $tempfile2 + sed 's/\./\\\./g' $dedupfile > $tempfile2 + while IFS=' ' read -r ips; do grep "$ips" $masterfile >> $tempfile; done < $tempfile2 + dcount=$(grep -c ^ $tempfile) + awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile + cat $addfile >> $masterfile + cut -d' ' -f2 $masterfile > $mastercat + fi +fi + +echo; echo "d-Duplication Process [ $now ]"; echo "------------------------------------------------" +echo; echo "Found [ $count ] IP range(s) over the threshold of dmax= [ $max ]" +echo "Found [ $safe ] IP range(s) classified as Whitelisted" +echo; echo "Found [ $dcount ] CC Blacklisted IP Address(es) are being set to [ $ccblack ]" +echo "Found [ $mmcount ] CC Whitelisted IP Address(es) are being set to [ $ccwhite ]"; echo +if [ -s "$addfile" ]; then + echo; echo "Removed the following IP Ranges" + sed -e 's/^.* //' -e 's/0\/24//' $addfile | tr '\n' '|'; echo +fi +count=$(grep -c ^ $masterfile) +echo " [ Post d-Deduplication count ] [ $count ]"; echo + +# Write "1.1.1.1" to empty Final Blocklist Files +emptyfiles=$(find $pfbdeny -size 0) +for i in $emptyfiles; do echo "1.1.1.1" > $i; done +} + + +########## +# Process to perform a final De-Duplication on all of the BlockLists (Excluding Country Whitelist) ("pmax" variable). +pdeduplication(){ + +if [ ! -x $pathgeoip ]; then + echo "p-duplication - Application [ GeoIP ] Not found. Can't proceed." + echo "p-duplication - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog + exit +fi + +# Download MaxMind GeoIP.dat on first Install. +if [ ! -f $pathgeoipdat ]; then + echo "Downloading [ MaxMind GeoIP.dat ] [ $now ]" >> $geoiplog + /usr/local/pkg/pfblockerng/geoipupdate.sh bu +fi +# Exit if GeoIP.dat is not found. +if [ ! -f $pathgeoipdat ]; then + echo "p-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed." + echo "p-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog + exit +fi + +> $tempfile; > $tempfile2; > $dupfile; > $addfile; > $dedupfile; count=0; dcount=0 +echo; echo "=====================================================================" +echo; echo "Querying for Repeat Offenders" +data="$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | cut -d '.' -f 1-3 $pfbdeny*.txt | + awk -v max="$max" '{a[$0]++}END{for(i in a){if(a[i] > max){print i}}}' | grep -v "^1\.1\.1")" +count=$(echo "$data" | grep -c ^) +if [ "$data" == "" ]; then count=0; fi +# Find Repeat Offenders in each individual Blocklist Outfile +echo "Processing [ Block ] IPs" +for ip in $data; do + pcount=1; ii=$(echo "^$ip." | sed 's/\./\\\./g') + list=$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | xargs grep -al $ii) + for blfile in $list; do + header=$(echo "${blfile##*/}" | cut -d '.' -f1) + grep $ii $blfile > $tempfile + awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $blfile > $tempfile2; mv -f $tempfile2 $blfile + if [ "$pcount" -eq "1" ]; then + echo $ip".0/24" >> $blfile + echo $header" $ip." >> $dedupfile + echo $header" "$ip".0/24" >> $addfile + pcount=2 + else + echo $header" $ip." >> $dedupfile + fi + done +done +# Remove Repeat Offenders in Masterfile +if [ -s "$dedupfile" ]; then + echo "Removing [ Block ] IPs" + > $tempfile; > $tempfile2 + sed 's/\./\\\./g' $dedupfile > $tempfile2 + while IFS=' ' read -r ips; do grep "$ips" $masterfile >> $tempfile; done < $tempfile2 + dcount=$(grep -c ^ $tempfile) + awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile + cat $addfile >> $masterfile + cut -d' ' -f2 $masterfile > $mastercat +fi + +echo; echo "p-Duplication Process [ $now ]"; echo "------------------------------------------------" +echo "Found [ $dcount ] IP Address(es) are being set to [ block ]" +if [ -s "$addfile" ]; then + echo; echo "Removed the following IP Ranges" + sed -e 's/^.* //' -e 's/0\/24//' $addfile | tr '\n' '|'; echo +fi +count=$(grep -c ^ $masterfile) +echo; echo " [ Post p-Deduplication count ] [ $count ]" + +# Write "1.1.1.1" to empty Final Blocklist Files +emptyfiles=$(find $pfbdeny -size 0) +for i in $emptyfiles; do echo "1.1.1.1" > $i; done +} + + +########## +# Process to Split ET Pro IPREP into Category Files and Compile selected Blocked categories into Outfile +processet() { + +if [ ! -x $pathgunzip ]; then + echo "Application [ Gunzip ] Not found, Can't proceed." + echo "Application [ Gunzip ] Not found, Can't proceed. [ $now ]" >> $errorlog + exit +fi + +if [ -s $pfborig$alias".gz" ]; then + evar="ET_*" + # Remove Previous ET IPRep Files + [ -d $etdir ] && [ "$(ls -A $etdir)" ] && rm -r $etdir/$evar + > $tempfile; > $tempfile2 + + $pathgunzip -c $pfborig$alias".gz" > $pfborig$alias".raw" + + # ET CSV Format (IP, Category, Score) + echo; echo "Processing [ $alias ]" + while IFS="," read a b c; do + # Some ET Categories are not in use (For Future Use) + case "$b" in + 1) echo $a >> $etdir/ET_Cnc;; + 2) echo $a >> $etdir/ET_Bot;; + 3) echo $a >> $etdir/ET_Spam;; + 4) echo $a >> $etdir/ET_Drop;; + 5) echo $a >> $etdir/ET_Spywarecnc;; + 6) echo $a >> $etdir/ET_Onlinegaming;; + 7) echo $a >> $etdir/ET_Drivebysrc;; + 8) echo $a >> $etdir/ET_Cat8;; + 9) echo $a >> $etdir/ET_Chatserver;; + 10) echo $a >> $etdir/ET_Tornode;; + 11) echo $a >> $etdir/ET_Cat11;; + 12) echo $a >> $etdir/ET_Cat12;; + 13) echo $a >> $etdir/ET_Compromised;; + 14) echo $a >> $etdir/ET_Cat14;; + 15) echo $a >> $etdir/ET_P2P;; + 16) echo $a >> $etdir/ET_Proxy;; + 17) echo $a >> $etdir/ET_Ipcheck;; + 18) echo $a >> $etdir/ET_Cat18;; + 19) echo $a >> $etdir/ET_Utility;; + 20) echo $a >> $etdir/ET_DDos;; + 21) echo $a >> $etdir/ET_Scanner;; + 22) echo $a >> $etdir/ET_Cat22;; + 23) echo $a >> $etdir/ET_Brute;; + 24) echo $a >> $etdir/ET_Fakeav;; + 25) echo $a >> $etdir/ET_Dyndns;; + 26) echo $a >> $etdir/ET_Undesireable;; + 27) echo $a >> $etdir/ET_Abusedtld;; + 28) echo $a >> $etdir/ET_Selfsignedssl;; + 29) echo $a >> $etdir/ET_Blackhole;; + 30) echo $a >> $etdir/ET_RAS;; + 31) echo $a >> $etdir/ET_P2Pcnc;; + 32) echo $a >> $etdir/ET_Sharedhosting;; + 33) echo $a >> $etdir/ET_Parking;; + 34) echo $a >> $etdir/ET_VPN;; + 35) echo $a >> $etdir/ET_Exesource;; + 36) echo $a >> $etdir/ET_Cat36;; + 37) echo $a >> $etdir/ET_Mobilecnc;; + 38) echo $a >> $etdir/ET_Mobilespyware;; + 39) echo $a >> $etdir/ET_Skypenode;; + 40) echo $a >> $etdir/ET_Bitcoin;; + 41) echo $a >> $etdir/ET_DDosattack;; + *) echo $a >> $etdir/ET_Unknown;; + esac + done <"$pfborig$alias.raw" + data=$(ls $etdir) + echo "Compiling ET IP IQRisk REP Lists based upon User Selected Categories" + printf "%-10s %-25s\n" " Action" "Category" + echo "-------------------------------------------" + + for list in $data; do + case "$etblock" in + *$list*) + printf "%-10s %-25s\n" " Block: " "$list" + cat $etdir/$list >> $tempfile + ;; + esac + case "$etmatch" in + *$list*) + printf "%-10s %-25s\n" " Match: " "$list" + cat $etdir/$list >> $tempfile2 + ;; + esac + done + echo "-------------------------------------------" + + if [ -f $tempfile ]; then mv -f $tempfile $pfborig$alias".orig"; fi + if [ "$etmatch" != "x" ]; then mv -f $tempfile2 $pfbmatch/ETMatch.txt; fi + cicount=$(cat $etdir/$evar | grep -cv '^#\|^$'); cocount=$(grep -cv "^1\.1\.1\.1" $pfborig$alias".orig") + echo; echo "ET Folder count [ $cicount ] Outfile count [ $cocount ]" +else + echo; echo "No ET .GZ File Found!" +fi +} + +# Process to extract IP addresses from XLSX Files +processxlsx() { + +if [ ! -x $pathtar ]; then + echo "Application [ TAR ] Not found, Can't proceed." + echo "Application [ TAR ] Not found, Can't proceed. [ $now ]" >> $errorlog + exit +fi + +if [ -s $pfborig$alias".zip" ]; then + + $pathtar -xf $pfborig$alias".zip" -C $tmpxlsx + $pathtar -xOf $tmpxlsx*.[xX][lL][sS][xX] xl/sharedStrings.xml | + grep -aoEw "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" | sort | uniq > $pfborig$alias".orig" + rm -r $tmpxlsx* + + cocount=$(grep -cv "^1\.1\.1\.1" $pfborig$alias".orig") + echo; echo "Download file count [ ZIP file ] Outfile count [ $cocount ]" +else + echo "XLSX Download File Missing" + echo " [ $alias ] XLSX Download File Missing [ $now ]" >> $errorlog +fi +} + +closingprocess() { + +# Write "1.1.1.1" to empty Final Blocklist Files +emptyfiles=$(find $pfbdeny -size 0) +for i in $emptyfiles; do echo "1.1.1.1" > $i; done + +if [ -d "$pfborig" ] && [ "$(ls -A $pfborig)" ]; then + fcount=$(find $pfborig*.orig | xargs cat | grep -cv '^#\|^$') +else + fcount=0 +fi + +if [ "$alias" == "on" ]; then + sort -o $masterfile $masterfile + sort -t . -k 1,1n -k 2,2n -k 3,3n -k 4,4n $mastercat > $tempfile; mv -f $tempfile $mastercat + + echo; echo; echo "===[ FINAL Processing ]====================================="; echo + echo " [ Original count ] [ $fcount ]" + count=$(grep -c ^ $masterfile) + echo; echo " [ Processed Count ] [ $count ]"; echo + + s1=$(grep -cv "1\.1\.1\.1" $masterfile) + s2=$(find $pfbdeny ! -name "*_v6.txt" -type f | xargs cat | grep -cv "^1\.1\.1\.1") + s3=$(sort $mastercat | uniq -d | tail -30) + s4=$(find $pfbdeny ! -name "*_v6.txt" -type f | xargs cat | sort | uniq -d | tail -30 | grep -v "^1\.1\.1\.1") + + if [ -d "$pfbpermit" ] && [ "$(ls -A $pfbpermit)" ]; then + echo; echo "===[ Permit List IP Counts ]========================="; echo + wc -l $pfbpermit* | sort -n -r + fi + if [ -d "$pfbmatch" ] && [ "$(ls -A $pfbmatch)" ]; then + echo; echo "===[ Match List IP Counts ]=========================="; echo + wc -l $pfbmatch* | sort -n -r + fi + if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then + echo; echo "===[ Deny List IP Counts ]==========================="; echo + wc -l $pfbdeny* | sort -n -r + fi + if [ -d "$pfbnative" ] && [ "$(ls -A $pfbnative)" ]; then + echo; echo "===[ Native List IP Counts ] ==================================="; echo + wc -l $pfbnative* | sort -n -r + fi + if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then + emptylists=$(grep "1\.1\.1\.1" $pfbdeny* | sed -e 's/^.*[a-zA-Z]\///' -e 's/\.txt:1.1.1.1/ /') + if [ ! -z "$emptylists" ]; then + echo; echo "====================[ Empty Lists w/1.1.1.1 ]=================="; echo + for list in $emptylists; do + echo $list + done + fi + fi + if [ -d "$pfborig" ] && [ "$(ls -A $pfborig)" ]; then + echo; echo "====================[ Last Updated List Summary ]=============="; echo + ls -lahtr $pfborig* | sed -e 's/\/.*\// /' -e 's/.orig//' | awk -v OFS='\t' '{print $6" "$7,$8,$9}' + fi + echo "==============================================================="; echo + echo "Sanity Check (Not Including IPv6) ** These two Counts should Match! **" + echo "------------" + echo "Masterfile Count [ $s1 ]" + echo "Deny folder Count [ $s2 ]"; echo + echo "Duplication Sanity Check (Pass=No IPs reported)" + echo "------------------------" + echo "Masterfile/Deny Folder Uniq check" + if [ ! -z "$s3" ]; then echo $s3; fi + echo "Deny Folder/Masterfile Uniq check" + if [ ! -z "$s4" ]; then echo $s4; fi + echo; echo "Sync Check (Pass=No IPs reported)" + echo "----------" +else + echo; echo "===[ FINAL Processing ]============================================="; echo + echo " [ Original count ] [ $fcount ]" + if [ -d "$pfbpermit" ] && [ "$(ls -A $pfbpermit)" ]; then + echo; echo "===[ Permit List IP Counts ]========================="; echo + wc -l $pfbpermit* | sort -n -r + fi + if [ -d "$pfbmatch" ] && [ "$(ls -A $pfbmatch)" ]; then + echo; echo "===[ Match List IP Counts ]=========================="; echo + wc -l $pfbmatch* | sort -n -r + fi + if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then + echo; echo "===[ Deny List IP Counts ]==========================="; echo + wc -l $pfbdeny* | sort -n -r + fi + if [ -d "$pfbnative" ] && [ "$(ls -A $pfbnative)" ]; then + echo; echo "===[ Native List IP Counts ] ==================================="; echo + wc -l $pfbnative* | sort -n -r + fi + if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then + emptylists=$(grep "1\.1\.1\.1" $pfbdeny* | sed -e 's/^.*[a-zA-Z]\///' -e 's/\.txt:1.1.1.1/ /') + if [ ! -z "$emptylists" ]; then + echo; echo "====================[ Empty Lists w/1.1.1.1 ]=================="; echo + for list in $emptylists; do + echo $list + done + fi + fi + if [ -d "$pfborig" ] && [ "$(ls -A $pfborig)" ]; then + echo; echo "====================[ Last Updated List Summary ]=============="; echo + ls -lahtr $pfborig* | sed -e 's/\/.*\// /' -e 's/.orig//' | awk -v OFS='\t' '{print $6" "$7,$8,$9}' + echo "===============================================================" + fi +fi + +echo; echo "IPv4 Alias Table IP Total"; echo "-----------------------------" +find $pfsense_alias_dir ! -name "*_v6.txt" -type f | xargs cat | grep -c ^ + +echo; echo "IPv6 Alias Table IP Total"; echo "-----------------------------" +find $pfsense_alias_dir -name "*_v6.txt" -type f | xargs cat | grep -c ^ + +echo; echo "Alias Table IP Counts"; echo "-----------------------------" +wc -l $pfsense_alias_dir*.txt | sort -n -r + +echo; echo "pfSense Table Stats"; echo "-------------------" +$pathpfctl -s memory | grep "table-entries" +pfctlcount=$($pathpfctl -vvsTables | awk '/Addresses/ {s+=$2}; END {print s}') +echo "Table Usage Count " $pfctlcount +} + +remove() { +# Remove Lists from Masterfiles and Delete Associated Files +echo +for i in $cc; do + header=$(echo "${i%*,}") + if [ ! "$header" == "" ]; then + # Make sure that Alias Exists in Masterfile before removal. + masterchk=$(grep -m1 "$header[[:space:]]" $masterfile) + if [ ! -z "$masterchk" ]; then + # Grep Header with a Trailing Space character + grep "$header[[:space:]]" $masterfile > $tempfile + awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile + cut -d' ' -f2 $masterfile > $mastercat + fi + rm -rf $pfborig$header*; rm -rf $pfbdeny$header*; rm -rf $pfbmatch$header*; rm -rf $pfbpermit$header*; rm -rf $pfbnative$header* + echo "The Following list has been REMOVED [ $header ]" + fi + echo +done + +# Delete Masterfiles if they are empty +emptychk=$(find $masterfile -size 0) +if [ ! "$emptychk" == "" ]; then + rm -r $masterfile; rm -r $mastercat +fi +} + + +########## +# CALL APPROPRIATE PROCESSES using Script Argument $1 +case $1 in + continent) + continent + ;; + duplicate) + process255 + duplicate + ;; + suppress) + suppress + ;; + p24) + process24 + ;; + dedup) + deduplication + ;; + pdup) + pdeduplication + ;; + et) + processet + ;; + xlsx) + processxlsx + ;; + closing) + closingprocess + ;; + remove) + remove + ;; + *) + exit + ;; +esac +exit
\ No newline at end of file |