aboutsummaryrefslogtreecommitdiffstats
path: root/config/pfblockerng/pfblockerng.sh
diff options
context:
space:
mode:
Diffstat (limited to 'config/pfblockerng/pfblockerng.sh')
-rw-r--r--config/pfblockerng/pfblockerng.sh928
1 files changed, 0 insertions, 928 deletions
diff --git a/config/pfblockerng/pfblockerng.sh b/config/pfblockerng/pfblockerng.sh
deleted file mode 100644
index 335df167..00000000
--- a/config/pfblockerng/pfblockerng.sh
+++ /dev/null
@@ -1,928 +0,0 @@
-#!/bin/sh
-# pfBlockerNG IP Reputation Script - By BBcan177@gmail.com - 04-12-14
-# Copyright (C) 2014 BBcan177@gmail.com
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License Version 2 as
-# published by the Free Software Foundation. You may not use, modify or
-# distribute this program under any other version of the GNU General
-# Public License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-now=$(/bin/date +%m/%d/%y' '%T)
-mtype=$(/usr/bin/uname -m);
-pfs_version="$(cut -c 1-3 /etc/version)"
-
-# Application Paths
-pathgrepcidr="/usr/pbi/pfblockerng-$mtype/bin/grepcidr"
-pathgeoip="/usr/pbi/pfblockerng-$mtype/bin/geoiplookup"
-
-pathtar=/usr/bin/tar
-pathgunzip=/usr/bin/gunzip
-pathpfctl=/sbin/pfctl
-
-# Script Arguments
-alias=$2
-max=$3
-dedup=$4
-cc=$(echo $5 | sed 's/,/, /g')
-ccwhite=$(echo $6 | tr '[A-Z]' '[a-z]')
-ccblack=$(echo $7 | tr '[A-Z]' '[a-z]')
-etblock=$(echo $8 | sed 's/,/, /g')
-etmatch=$(echo $9 | sed 's/,/, /g')
-
-# File Locations
-pathgeoipdat=/var/db/pfblockerng/GeoIP.dat
-pfbsuppression=/var/db/pfblockerng/pfbsuppression.txt
-masterfile=/var/db/pfblockerng/masterfile
-mastercat=/var/db/pfblockerng/mastercat
-geoiplog=/var/log/pfblockerng/geoip.log
-errorlog=/var/log/pfblockerng/error.log
-
-# Folder Locations
-etdir=/var/db/pfblockerng/ET
-tmpxlsx=/tmp/xlsx/
-
-pfbdeny=/var/db/pfblockerng/deny/
-pfborig=/var/db/pfblockerng/original/
-pfbmatch=/var/db/pfblockerng/match/
-pfbpermit=/var/db/pfblockerng/permit/
-pfbnative=/var/db/pfblockerng/native/
-pfsense_alias_dir=/var/db/aliastables/
-
-# Store "Match" d-dedups in matchdedup.txt file
-matchdedup=matchdedup.txt
-
-tempfile=/tmp/pfbtempfile
-tempfile2=/tmp/pfbtempfile2
-dupfile=/tmp//pfbduptemp
-dedupfile=/tmp/pfbdeduptemp
-addfile=/tmp/pfBaddfile
-syncfile=/tmp/pfbsyncfile
-matchfile=/tmp/pfbmatchfile
-tempmatchfile=/tmp/pfbtempmatchfile
-
-if [ ! -f $masterfile ]; then touch $masterfile; fi
-if [ ! -f $mastercat ]; then touch $mastercat; fi
-if [ ! -f $tempfile ]; then touch $tempfile; fi
-if [ ! -f $tempfile2 ]; then touch $tempfile2; fi
-if [ ! -f $dupfile ]; then touch $dupfile; fi
-if [ ! -f $dedupfile ]; then touch $dedupfile; fi
-if [ ! -f $addfile ]; then touch $addfile; fi
-if [ ! -f $syncfile ]; then touch $syncfile; fi
-if [ ! -f $matchfile ]; then touch $matchfile; fi
-if [ ! -f $tempmatchfile ]; then touch $tempmatchfile; fi
-if [ ! -d $pfbmatch ]; then mkdir $pfbmatch; fi
-if [ ! -d $etdir ]; then mkdir $etdir; fi
-if [ ! -d $tmpxlsx ]; then mkdir $tmpxlsx; fi
-
-##########
-# Process to condense an IP range if a "Max" amount of IP addresses are found in a /24 range per Alias Group.
-process24() {
-
-if [ ! -x $pathgeoip ]; then
- echo "Process24 - Application [ GeoIP ] Not found. Can't proceed."
- echo "Process24 - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog
- exit
-fi
-
-# Download MaxMind GeoIP.dat Binary on first Install.
-if [ ! -f $pathgeoipdat ]; then
- echo "Downloading [ MaxMind GeoIP.dat ] [ $now ]" >> $geoiplog
- /usr/local/pkg/pfblockerng/geoipupdate.sh bu
-fi
-# Exit if GeoIP.dat is not found.
-if [ ! -f $pathgeoipdat ]; then
- echo "Process24 - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed."
- echo "Process24 - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog
- exit
-fi
-
-count=$(grep -c ^ $pfbdeny$alias".txt")
-echo; echo "Original File Count [ $count ]"
-
-grep -Ev "^(#|$)" $pfbdeny$alias".txt" | sort | uniq > $tempfile
-> $dupfile; > $tempfile2; > $matchfile; > $tempmatchfile
-data="$(cut -d '.' -f 1-3 $tempfile | awk -v max="$max" '{a[$0]++}END{for(i in a){if(a[i] > max){print i}}}')"
-count=$(echo "$data" | grep -c ^); mcount=0; dcount=0; safe=0
-if [ "$data" == "" ]; then count=0; fi
-matchoutfile="match"$header".txt"
-# Classify Repeat Offenders by Country Code
-if [ -f $pathgeoipdat ]; then
- for ip in $data; do
- ccheck=$($pathgeoip -f $pathgeoipdat "$ip.1" | cut -c 24-25)
- case "$cc" in
- *$ccheck*)
- safe=$(($safe + 1))
- if [ "$ccwhite" == "match" -o "$ccblack" == "match" ]; then
- echo "$ip." >> $matchfile
- fi
- ;;
- *)
- echo "$ip." >> $dupfile
- ;;
- esac
- done
-else
- echo; echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping p24 Process"; echo
- echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping p24 Process [ $now ]" >> $errorlog
-fi
-# Collect Match File Details
-if [ -s "$matchfile" -a ! "$dedup" == "on" -a "$ccwhite" == "match" ]; then
- mon=$(sed -e 's/^/^/' -e 's/\./\\\./g' $matchfile)
- for ip in $mon; do
- grep $ip $tempfile >> $tempfile2
- done
- mcount=$(grep -c ^ $tempfile2)
- if [ "$ccwhite" == "match" ]; then
- sed 's/$/0\/24/' $matchfile >> $tempmatchfile
- sed 's/^/\!/' $tempfile2 >> $tempmatchfile
- fi
-fi
-
-# If no Matches found remove previous Matchoutfile if exists.
-if [ ! -s "$tempmatchfile" -a -f $matchoutfile ]; then rm -r $matchoutfile; fi
-# Move Match File to the Match Folder by Individual Blocklist Name
-if [ -s "$tempmatchfile" ]; then mv -f $tempmatchfile $pfbmatch$matchoutfile; fi
-
-# Find Repeat Offenders in each individual Blocklist Outfile
-if [ -s "$dupfile" ]; then
- > $tempfile2
- dup=$(sed -e 's/^/^/' -e 's/\./\\\./g' $dupfile)
- for ip in $dup; do
- grep $ip $tempfile >> $tempfile2
- done
- dcount=$(grep -c ^ $tempfile2)
- if [ "$ccblack" == "block" ]; then
- awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile2 $tempfile > $pfbdeny$alias".txt"
- sed 's/$/0\/24/' $dupfile >> $pfbdeny$alias".txt"
- elif [ "$ccblack" == "match" ]; then
- sed 's/$/0\/24/' $dupfile >> $tempmatchfile
- sed 's/^/\!/' $tempfile2 >> $tempmatchfile
- else
- :
- fi
-fi
-if [ "$count" == "0" -a "$safe" == "0" ]; then echo; echo " Process /24 Stats [ $alias ] [ $now ] "; echo "------------------------------------------------"; fi
-if [ "$count" == "0" ]; then echo "Found [ 0 ] IP range(s) over the threshold of [ $max ] p24 - CC Blacklist"; fi
-if [ "$safe" == "0" ]; then echo "Found [ 0 ] IP range(s) over the threshold of [ $max ] p24 - CC Whitelist"; fi
-
-if [ -s "$dupfile" -o -s "$matchfile" ]; then
-echo
-echo " Process /24 Stats [ $alias ] [ $now ]"
-echo "--------------------------------------------------------"
-echo "Found [ $count ] IP range(s) over the threshold of [ $max ] on the CC Blacklist"
-echo "Found [ $safe ] IP range(s) over the threshold of [ $max ] on the CC Whitelist"
-echo
-echo "Found [ $dcount ] CC Blacklisted IP Address(es) are being set to [ $ccblack ]"
-# Skip Match Process if dedup=yes as it will create duplicates
-if [ "$dedup" == "on" ]; then mcount=Skipped; fi
-echo "Found [ $mcount ] CC Whitelisted IP Address(es) are being set to [ $ccwhite ]"
-if [ "$ccblack" == "block" ]; then
- echo; echo "Removed the following IP Ranges"
- cat $dupfile | tr '\n' '|'; echo
-else
- echo "Skipped, CCBlack set to [ $ccblack ]"
-fi
-sort $pfbdeny$alias".txt" | uniq > $tempfile; mv -f $tempfile $pfbdeny$alias".txt"
-echo "-------------------------------------------------------"
-cocount=$(grep -cv "^1\.1\.1\.1" $pfbdeny$alias".txt")
-echo "Post /24 Count [ $cocount ]"; echo
-fi
-}
-
-
-##########
-process255() {
-# Remove IPs if exists over 255 IPs in a Range and replace with a single /24 Block
-cp $pfbdeny$alias".txt" $tempfile; > $dedupfile
-
-data255="$(cut -d '.' -f 1-3 $tempfile | awk '{a[$0]++}END{for(i in a){if(a[i] > 255){print i}}}')"
-if [ ! -z "$data255" ]; then
- for ip in $data255; do
- ii=$(echo "^$ip" | sed 's/\./\\\./g')
- grep $ii $tempfile >> $dedupfile
- done
- awk 'FNR==NR{a[$0];next}!($0 in a)' $dedupfile $tempfile > $pfbdeny$alias".txt"
- for ip in $data255; do echo $ip"0/24" >> $pfbdeny$alias".txt"; done
-fi
-}
-
-
-##########
-continent() {
-
-dupcheck=yes
-# Check if Masterfile is Empty
-hcheck=$(grep -c ^ $masterfile); if [ "$hcheck" -eq "0" ]; then dupcheck=no; fi
-# Check if Alias exists in Masterfile
-lcheck=$(grep -m 1 "$alias " $masterfile ); if [ "$lcheck" == "" ]; then dupcheck=no; fi
-
-if [ "$dupcheck" == "yes" ]; then
- # Grep Alias with a trailing Space character
- grep "$alias[[:space:]]" $masterfile > $tempfile
- awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
- cut -d' ' -f2 $masterfile > $mastercat
-fi
-
-grep -Ev "^(#|$)" $pfbdeny$alias".txt" | sort | uniq > $tempfile
-
-if [ ! "$hcheck" -eq "0" ]; then
- $pathgrepcidr -vf $mastercat $pfbdeny$alias".txt" > $tempfile; mv -f $tempfile $pfbdeny$alias".txt"
-fi
-
-sed -e 's/^/'$alias' /' $pfbdeny$alias".txt" >> $masterfile
-cut -d' ' -f2 $masterfile > $mastercat
-
-countg=$(grep -c ^ $pfborig$alias".orig")
-countm=$(grep -c "$alias " $masterfile); counto=$(grep -c ^ $pfbdeny$alias".txt")
-if [ "$countm" == "$counto" ]; then sanity="Passed"; else sanity=" ==> FAILED <== "; fi
-echo "----------------------------------------------------------"
-echo; echo " Post Duplication count [ $now ]"
-echo "----------------------------------------------------------"
-printf "%-10s %-10s %-10s %-30s\n" "Original" "Masterfile" "Outfile" "Sanity Check"
-echo "----------------------------------------------------------"
-printf "%-10s %-10s %-10s %-30s\n" "$countg" "$countm" "$counto" " [ $sanity ]"
-echo "----------------------------------------------------------"
-}
-
-
-##########
-# Process to remove Suppressed Entries and RFC 1918 and Misc IPs on each downloaded Blocklist
-suppress() {
-
-if [ ! -x $pathgrepcidr ]; then
- echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]"
- echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" >> errorlog
- exit
-fi
-
-if [ -e "$pfbsuppression" ] && [ -s "$pfbsuppression" ]; then
- # Find '/24' Blocked IPs that are single addresses in the Suppressed IP Address List.
- # These '/24' Are converted to single Addresses excluding the Suppressed IPs.
- data="$(cat $pfbsuppression)"
- if [ ! -z "$data" -a ! -z "$cc" ]; then
- # Loop thru each Updated List to remove Suppression and RFC1918 Addresses
- if [ "$cc" == "suppressheader" ]; then
- echo "===[ Suppression Stats ]========================================"; echo
- printf "%-20s %-10s %-10s %-10s %-10s\n" "List" "Pre" "RFC1918" "Suppress" "Masterfile"
- echo "----------------------------------------------------------------"
- exit
- fi
-
- for i in $cc; do
- counter=0
- > $dupfile
- alias=$(echo "${i%|*}")
- pfbfolder=$(echo "${i#*|}")
-
- if [ ! "$alias" == "" ]; then
- # Count (PRE)
- countg=$(grep -c ^ $pfbfolder$alias".txt")
-
- grep -Ev "^(192\.168|10\.|172\.1[6789]\.|172\.2[0-9]\.|172\.3[01]\.|127\.0\.0\.1|0\.0\.0\.0|#|$)" $pfbfolder$alias".txt" |
- sort | uniq > $tempfile
- # Count (Post RFC1918)
- countm=$(grep -c ^ $tempfile)
-
- for ip in $data; do
- found=""; ddcheck="";
- iptrim=$(echo $ip | cut -d '.' -f 1-3)
- mask=$(echo $ip | cut -d"/" -f2)
- found=$(grep -m1 $iptrim".0/24" $tempfile)
- # If a Suppression is '/32' and a Blocklist has a full '/24' Block execute the following.
- if [ ! "$found" == "" -a "$mask" == "32" ]; then
- echo " Suppression $alias: $iptrim.0/24"
- octet4=$(echo $ip | cut -d '.' -f 4 | sed 's/\/.*//')
- dcheck=$(grep $iptrim".0/24" $dupfile)
- if [ "$dcheck" == "" ]; then
- echo $iptrim".0" >> $tempfile
- echo $iptrim".0/24" >> $dupfile
- counter=$(($counter + 1))
- # Add Individual IP addresses from Range excluding Suppressed IP
- for i in $(/usr/bin/jot 255); do
- if [ "$i" != "$octet4" ]; then
- echo $iptrim"."$i >> $tempfile
- counter=$(($counter + 1))
- fi
- done
- fi
- fi
- done
- if [ -s $dupfile ]; then
- # Remove '/24' Suppressed Ranges
- awk 'FNR==NR{a[$0];next}!($0 in a)' $dupfile $tempfile > $tempfile2; mv -f $tempfile2 $tempfile
- fi
- # Remove All other Suppressions from Lists
- $pathgrepcidr -vf $pfbsuppression $tempfile > $pfbfolder$alias".txt"
- # Update Masterfiles. Don't execute if Duplication Process is Disabled
- if [ "$dedup" == "x" ]; then
- # Dont execute if Alias doesnt exist in Masterfile
- lcheck=$(grep -m1 "$alias " $masterfile)
- if [ ! "$lcheck" == "" ]; then
- # Replace Masterfile with changes to List.
- grep "$alias[[:space:]]" $masterfile > $tempfile
- awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
- sed -e 's/^/'$alias' /' $pfbfolder$alias".txt" >> $masterfile
- cut -d' ' -f2 $masterfile > $mastercat
- fi
- fi
- countk=$(grep -c ^ $masterfile)
- countx=$(grep -c ^ $pfbfolder$alias".txt")
- counto=$(($countx - $counter))
- printf "%-20s %-10s %-10s %-10s %-10s\n" "$alias" "$countg" "$countm" "$counto" "$countk"
- fi
- done
- fi
-else
- if [ "$cc" == "suppressheader" ]; then
- echo "===[ Suppression Stats ]========================================"; echo
- printf "%-20s %-10s %-10s %-10s %-10s\n" "List" "Pre" "RFC1918" "Suppress" "Masterfile"
- echo "----------------------------------------------------------------"
- exit
- fi
- for i in $cc; do
- alias=$(echo "${i%|*}")
- pfbfolder=$(echo "${i#*|}")
-
- if [ ! "$alias" == "" ]; then
- countg=$(grep -c ^ $pfbfolder$alias".txt")
- grep -Ev "^(192\.168|10\.|172\.1[6789]\.|172\.2[0-9]\.|172\.3[01]\.|127\.0\.0\.1|0\.0\.0\.0|#|$)" $pfbfolder$alias".txt" |
- sort | uniq > $tempfile; mv -f $tempfile $pfbfolder$alias".txt"
- countx=$(grep -c ^ $pfbfolder$alias".txt")
- # Update Masterfiles. Don't execute if Duplication Process is Disabled or if No Suppression Changes Found
- if [ "$dedup" == "x" -a "$countg" != "$countx" ]; then
- # Dont execute if Alias doesnt exist in Masterfile
- lcheck=$(grep -m1 "$alias " $masterfile)
- if [ ! "$lcheck" == "" ]; then
- # Replace Masterfile with changes to List.
- grep "$alias[[:space:]]" $masterfile > $tempfile
- awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
- sed -e 's/^/'$alias' /' $pfbfolder$alias".txt" >> $masterfile
- cut -d' ' -f2 $masterfile > $mastercat
- fi
- fi
- countm=$(grep -c ^ $pfbfolder$alias".txt")
- counto=" - "
- countk=$(grep -c ^ $masterfile)
- printf "%-20s %-10s %-10s %-10s %-10s\n" "$alias" "$countg" "$countm" "$counto" "$countk"
- fi
- done
-fi
-}
-
-
-##########
-# Process to remove Duplicate Entries on each downloaded Blocklist Individually
-duplicate() {
-
-if [ ! -x $pathgrepcidr ]; then
- echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]"
- echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" >> errorlog
- exit
-fi
-
-dupcheck=yes
-# Check if Masterfile is Empty
-hcheck=$(grep -cv "^$" $masterfile); if [ "$hcheck" -eq "0" ]; then dupcheck=no; fi
-# Check if Alias exists in Masterfile
-lcheck=$(grep -m1 "$alias " $masterfile); if [ "$lcheck" == "" ]; then dupcheck=no; fi
-
-if [ "$dupcheck" == "yes" ]; then
- # Grep Alias with a trailing Space character
- grep "$alias[[:space:]]" $masterfile > $tempfile
- awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
- cut -d' ' -f2 $masterfile > $mastercat
-fi
-
-grep -Ev "^(#|$)" $pfbdeny$alias".txt" | sort | uniq > $tempfile; mv -f $tempfile $pfbdeny$alias".txt"
-
-if [ ! "$hcheck" -eq "0" ]; then
- $pathgrepcidr -vf $mastercat $pfbdeny$alias".txt" > $tempfile; mv -f $tempfile $pfbdeny$alias".txt"
-fi
-
-sed -e 's/^/'$alias' /' $pfbdeny$alias".txt" >> $masterfile
-cut -d' ' -f2 $masterfile > $mastercat
-
-countg=$(grep -c ^ $pfborig$alias".orig")
-countm=$(grep -c "$alias " $masterfile); counto=$(grep -c ^ $pfbdeny$alias".txt")
-if [ "$countm" == "$counto" ]; then sanity="Passed"; else sanity=" ==> FAILED <== "; fi
-echo "----------------------------------------------------------"
-printf "%-10s %-10s %-10s %-30s\n" "Original" "Masterfile" "Outfile" " [ Post Duplication count ]"
-echo "----------------------------------------------------------"
-printf "%-10s %-10s %-10s %-30s\n" "$countg" "$countm" "$counto" " [ $sanity ]"
-echo "----------------------------------------------------------"
-}
-
-
-##########
-# De-Duplication utilizing MaxMind GeoIP Country Code Whitelisting ("dmax" variable)
-deduplication() {
-
-if [ ! -x $pathgeoip ]; then
- echo "d-duplication - Application [ GeoIP ] Not found. Can't proceed."
- echo "d-duplication - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog
- exit
-fi
-
-# Download MaxMind GeoIP.dat on first Install.
-if [ ! -f $pathgeoipdat ]; then
- echo "Downloading [ MaxMind GeoIP.dat ] [ $now ]" >> $geoiplog
- /usr/local/pkg/pfblockerng/geoipupdate.sh bu
-fi
-
-# Exit if GeoIP.dat is not found
-if [ ! -f $pathgeoipdat ]; then
- echo "d-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed."
- echo "d-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog
- exit
-fi
-
-> $tempfile; > $tempfile2; > $dupfile; > $addfile; > $dedupfile; > $matchfile; > $tempmatchfile; count=0; dcount=0; mcount=0; mmcount=0
-echo; echo "Querying for Repeat Offenders"
-data="$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | cut -d '.' -f 1-3 $pfbdeny*.txt |
- awk -v max="$max" '{a[$0]++}END{for(i in a){if(a[i] > max){print i}}}' | grep -v "^1\.1\.1")"
-count=$(echo "$data" | grep -c ^)
-if [ "$data" == "" ]; then count=0; fi
-safe=0
-# Classify Repeat Offenders by Country Code
-if [ -f $pathgeoipdat ]; then
- echo "Classifying Repeat Offenders by GeoIP"
- for ip in $data; do
- ccheck=$($pathgeoip -f $pathgeoipdat "$ip.1" | cut -c 24-25)
- case "$cc" in
- *$ccheck*)
- safe=$(($safe + 1))
- if [ "$ccwhite" == "match" -o "$ccblack" == "match" ]; then
- echo "$ip." >> $matchfile
- fi
- ;;
- *)
- echo "$ip." >> $dupfile
- ;;
- esac
- done
-else
- echo; echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping d-dedup Process"; echo
- echo "MaxMind Binary Database Missing [ $pathgeoipdat ], skipping d-dedup Process [ $now ]" >> $errorlog
-fi
-if [ -s "$matchfile" -a "$ccwhite" == "match" ]; then
- echo "Processing [ Match ] IPs"
- match=$(sed -e 's/^/^/' -e 's/\./\\\./g' $matchfile)
- for mfile in $match; do
- grep $mfile $pfbdeny*.txt >> $tempfile
- done
- sed 's/$/0\/24/' $matchfile >> $tempmatchfile
- sed -e 's/.*://' -e 's/^/\!/' $tempfile >> $tempmatchfile
- mv -f $tempmatchfile $pfbmatch$matchdedup
- mcount=$(grep -c ^ $tempfile)
- mmcount=$(($mcount + $mmcount))
-fi
-# Find Repeat Offenders in each individual Blocklist Outfile
-if [ -s "$dupfile" ]; then
- echo "Processing [ Block ] IPs"
- dup=$(cat $dupfile)
- for ip in $dup; do
- pcount=1; ii=$(echo "^$ip" | sed 's/\./\\\./g')
- list=$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | xargs grep -al $ii)
- for blfile in $list; do
- header=$(echo "${blfile##*/}" | cut -d '.' -f1)
- grep $ii $blfile > $tempfile
- if [ "$ccblack" == "block" ]; then
- awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $blfile > $tempfile2; mv -f $tempfile2 $blfile
- if [ "$pcount" -eq "1" ]; then
- echo $ip"0/24" >> $blfile
- echo $header" "$ip >> $dedupfile
- echo $header" "$ip"0/24" >> $addfile
- pcount=2
- else
- echo $header" "$ip >> $dedupfile
- fi
- else
- if [ "$pcount" -eq "1" ]; then
- matchoutfile="match"$header".txt"
- echo $ip"0/24" >> $pfbmatch$matchoutfile
- sed 's/^/\!/' $tempfile >> $pfbmatch$matchoutfile
- mcount=$(grep -c ^ $pfbmatch$matchoutfile)
- mmcount=$(($mcount + $mmcount))
- pcount=2
- fi
- fi
- done
- done
- # Remove Repeat Offenders in Masterfiles
- if [ -s "$dedupfile" ]; then
- echo "Removing [ Block ] IPs"
- > $tempfile; > $tempfile2
- sed 's/\./\\\./g' $dedupfile > $tempfile2
- while IFS=' ' read -r ips; do grep "$ips" $masterfile >> $tempfile; done < $tempfile2
- dcount=$(grep -c ^ $tempfile)
- awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
- cat $addfile >> $masterfile
- cut -d' ' -f2 $masterfile > $mastercat
- fi
-fi
-
-echo; echo "d-Duplication Process [ $now ]"; echo "------------------------------------------------"
-echo; echo "Found [ $count ] IP range(s) over the threshold of dmax= [ $max ]"
-echo "Found [ $safe ] IP range(s) classified as Whitelisted"
-echo; echo "Found [ $dcount ] CC Blacklisted IP Address(es) are being set to [ $ccblack ]"
-echo "Found [ $mmcount ] CC Whitelisted IP Address(es) are being set to [ $ccwhite ]"; echo
-if [ -s "$addfile" ]; then
- echo; echo "Removed the following IP Ranges"
- sed -e 's/^.* //' -e 's/0\/24//' $addfile | tr '\n' '|'; echo
-fi
-count=$(grep -c ^ $masterfile)
-echo " [ Post d-Deduplication count ] [ $count ]"; echo
-
-# Write "1.1.1.1" to empty Final Blocklist Files
-emptyfiles=$(find $pfbdeny -size 0)
-for i in $emptyfiles; do echo "1.1.1.1" > $i; done
-}
-
-
-##########
-# Process to perform a final De-Duplication on all of the BlockLists (Excluding Country Whitelist) ("pmax" variable).
-pdeduplication(){
-
-if [ ! -x $pathgeoip ]; then
- echo "p-duplication - Application [ GeoIP ] Not found. Can't proceed."
- echo "p-duplication - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog
- exit
-fi
-
-# Download MaxMind GeoIP.dat on first Install.
-if [ ! -f $pathgeoipdat ]; then
- echo "Downloading [ MaxMind GeoIP.dat ] [ $now ]" >> $geoiplog
- /usr/local/pkg/pfblockerng/geoipupdate.sh bu
-fi
-# Exit if GeoIP.dat is not found.
-if [ ! -f $pathgeoipdat ]; then
- echo "p-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed."
- echo "p-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog
- exit
-fi
-
-> $tempfile; > $tempfile2; > $dupfile; > $addfile; > $dedupfile; count=0; dcount=0
-echo; echo "====================================================================="
-echo; echo "Querying for Repeat Offenders"
-data="$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | cut -d '.' -f 1-3 $pfbdeny*.txt |
- awk -v max="$max" '{a[$0]++}END{for(i in a){if(a[i] > max){print i}}}' | grep -v "^1\.1\.1")"
-count=$(echo "$data" | grep -c ^)
-if [ "$data" == "" ]; then count=0; fi
-# Find Repeat Offenders in each individual Blocklist Outfile
-echo "Processing [ Block ] IPs"
-for ip in $data; do
- pcount=1; ii=$(echo "^$ip." | sed 's/\./\\\./g')
- list=$(find $pfbdeny ! -name "pfB*.txt" ! -name "*_v6.txt" -type f | xargs grep -al $ii)
- for blfile in $list; do
- header=$(echo "${blfile##*/}" | cut -d '.' -f1)
- grep $ii $blfile > $tempfile
- awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $blfile > $tempfile2; mv -f $tempfile2 $blfile
- if [ "$pcount" -eq "1" ]; then
- echo $ip".0/24" >> $blfile
- echo $header" $ip." >> $dedupfile
- echo $header" "$ip".0/24" >> $addfile
- pcount=2
- else
- echo $header" $ip." >> $dedupfile
- fi
- done
-done
-# Remove Repeat Offenders in Masterfile
-if [ -s "$dedupfile" ]; then
- echo "Removing [ Block ] IPs"
- > $tempfile; > $tempfile2
- sed 's/\./\\\./g' $dedupfile > $tempfile2
- while IFS=' ' read -r ips; do grep "$ips" $masterfile >> $tempfile; done < $tempfile2
- dcount=$(grep -c ^ $tempfile)
- awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
- cat $addfile >> $masterfile
- cut -d' ' -f2 $masterfile > $mastercat
-fi
-
-echo; echo "p-Duplication Process [ $now ]"; echo "------------------------------------------------"
-echo "Found [ $dcount ] IP Address(es) are being set to [ block ]"
-if [ -s "$addfile" ]; then
- echo; echo "Removed the following IP Ranges"
- sed -e 's/^.* //' -e 's/0\/24//' $addfile | tr '\n' '|'; echo
-fi
-count=$(grep -c ^ $masterfile)
-echo; echo " [ Post p-Deduplication count ] [ $count ]"
-
-# Write "1.1.1.1" to empty Final Blocklist Files
-emptyfiles=$(find $pfbdeny -size 0)
-for i in $emptyfiles; do echo "1.1.1.1" > $i; done
-}
-
-
-##########
-# Process to Split ET Pro IPREP into Category Files and Compile selected Blocked categories into Outfile
-processet() {
-
-if [ ! -x $pathgunzip ]; then
- echo "Application [ Gunzip ] Not found, Can't proceed."
- echo "Application [ Gunzip ] Not found, Can't proceed. [ $now ]" >> $errorlog
- exit
-fi
-
-if [ -s $pfborig$alias".gz" ]; then
- evar="ET_*"
- # Remove Previous ET IPRep Files
- [ -d $etdir ] && [ "$(ls -A $etdir)" ] && rm -r $etdir/$evar
- > $tempfile; > $tempfile2
-
- $pathgunzip -c $pfborig$alias".gz" > $pfborig$alias".raw"
-
- # ET CSV Format (IP, Category, Score)
- echo; echo "Processing [ $alias ]"
- while IFS="," read a b c; do
- # Some ET Categories are not in use (For Future Use)
- case "$b" in
- 1) echo $a >> $etdir/ET_Cnc;;
- 2) echo $a >> $etdir/ET_Bot;;
- 3) echo $a >> $etdir/ET_Spam;;
- 4) echo $a >> $etdir/ET_Drop;;
- 5) echo $a >> $etdir/ET_Spywarecnc;;
- 6) echo $a >> $etdir/ET_Onlinegaming;;
- 7) echo $a >> $etdir/ET_Drivebysrc;;
- 8) echo $a >> $etdir/ET_Cat8;;
- 9) echo $a >> $etdir/ET_Chatserver;;
- 10) echo $a >> $etdir/ET_Tornode;;
- 11) echo $a >> $etdir/ET_Cat11;;
- 12) echo $a >> $etdir/ET_Cat12;;
- 13) echo $a >> $etdir/ET_Compromised;;
- 14) echo $a >> $etdir/ET_Cat14;;
- 15) echo $a >> $etdir/ET_P2P;;
- 16) echo $a >> $etdir/ET_Proxy;;
- 17) echo $a >> $etdir/ET_Ipcheck;;
- 18) echo $a >> $etdir/ET_Cat18;;
- 19) echo $a >> $etdir/ET_Utility;;
- 20) echo $a >> $etdir/ET_DDos;;
- 21) echo $a >> $etdir/ET_Scanner;;
- 22) echo $a >> $etdir/ET_Cat22;;
- 23) echo $a >> $etdir/ET_Brute;;
- 24) echo $a >> $etdir/ET_Fakeav;;
- 25) echo $a >> $etdir/ET_Dyndns;;
- 26) echo $a >> $etdir/ET_Undesireable;;
- 27) echo $a >> $etdir/ET_Abusedtld;;
- 28) echo $a >> $etdir/ET_Selfsignedssl;;
- 29) echo $a >> $etdir/ET_Blackhole;;
- 30) echo $a >> $etdir/ET_RAS;;
- 31) echo $a >> $etdir/ET_P2Pcnc;;
- 32) echo $a >> $etdir/ET_Sharedhosting;;
- 33) echo $a >> $etdir/ET_Parking;;
- 34) echo $a >> $etdir/ET_VPN;;
- 35) echo $a >> $etdir/ET_Exesource;;
- 36) echo $a >> $etdir/ET_Cat36;;
- 37) echo $a >> $etdir/ET_Mobilecnc;;
- 38) echo $a >> $etdir/ET_Mobilespyware;;
- 39) echo $a >> $etdir/ET_Skypenode;;
- 40) echo $a >> $etdir/ET_Bitcoin;;
- 41) echo $a >> $etdir/ET_DDosattack;;
- *) echo $a >> $etdir/ET_Unknown;;
- esac
- done <"$pfborig$alias.raw"
- data=$(ls $etdir)
- echo "Compiling ET IP IQRisk REP Lists based upon User Selected Categories"
- printf "%-10s %-25s\n" " Action" "Category"
- echo "-------------------------------------------"
-
- for list in $data; do
- case "$etblock" in
- *$list*)
- printf "%-10s %-25s\n" " Block: " "$list"
- cat $etdir/$list >> $tempfile
- ;;
- esac
- case "$etmatch" in
- *$list*)
- printf "%-10s %-25s\n" " Match: " "$list"
- cat $etdir/$list >> $tempfile2
- ;;
- esac
- done
- echo "-------------------------------------------"
-
- if [ -f $tempfile ]; then mv -f $tempfile $pfborig$alias".orig"; fi
- if [ "$etmatch" != "x" ]; then mv -f $tempfile2 $pfbmatch/ETMatch.txt; fi
- cicount=$(cat $etdir/$evar | grep -cv '^#\|^$'); cocount=$(grep -cv "^1\.1\.1\.1" $pfborig$alias".orig")
- echo; echo "ET Folder count [ $cicount ] Outfile count [ $cocount ]"
-else
- echo; echo "No ET .GZ File Found!"
-fi
-}
-
-# Process to extract IP addresses from XLSX Files
-processxlsx() {
-
-if [ ! -x $pathtar ]; then
- echo "Application [ TAR ] Not found, Can't proceed."
- echo "Application [ TAR ] Not found, Can't proceed. [ $now ]" >> $errorlog
- exit
-fi
-
-if [ -s $pfborig$alias".zip" ]; then
-
- $pathtar -xf $pfborig$alias".zip" -C $tmpxlsx
- $pathtar -xOf $tmpxlsx*.[xX][lL][sS][xX] xl/sharedStrings.xml |
- grep -aoEw "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" | sort | uniq > $pfborig$alias".orig"
- rm -r $tmpxlsx*
-
- cocount=$(grep -cv "^1\.1\.1\.1" $pfborig$alias".orig")
- echo; echo "Download file count [ ZIP file ] Outfile count [ $cocount ]"
-else
- echo "XLSX Download File Missing"
- echo " [ $alias ] XLSX Download File Missing [ $now ]" >> $errorlog
-fi
-}
-
-closingprocess() {
-
-# Write "1.1.1.1" to empty Final Blocklist Files
-emptyfiles=$(find $pfbdeny -size 0)
-for i in $emptyfiles; do echo "1.1.1.1" > $i; done
-
-if [ -d "$pfborig" ] && [ "$(ls -A $pfborig)" ]; then
- fcount=$(find $pfborig*.orig | xargs cat | grep -cv '^#\|^$')
-else
- fcount=0
-fi
-
-if [ "$alias" == "on" ]; then
- sort -o $masterfile $masterfile
- sort -t . -k 1,1n -k 2,2n -k 3,3n -k 4,4n $mastercat > $tempfile; mv -f $tempfile $mastercat
-
- echo; echo; echo "===[ FINAL Processing ]====================================="; echo
- echo " [ Original count ] [ $fcount ]"
- count=$(grep -c ^ $masterfile)
- echo; echo " [ Processed Count ] [ $count ]"; echo
-
- s1=$(grep -cv "1\.1\.1\.1" $masterfile)
- s2=$(find $pfbdeny ! -name "*_v6.txt" -type f | xargs cat | grep -cv "^1\.1\.1\.1")
- s3=$(sort $mastercat | uniq -d | tail -30)
- s4=$(find $pfbdeny ! -name "*_v6.txt" -type f | xargs cat | sort | uniq -d | tail -30 | grep -v "^1\.1\.1\.1")
-
- if [ -d "$pfbpermit" ] && [ "$(ls -A $pfbpermit)" ]; then
- echo; echo "===[ Permit List IP Counts ]========================="; echo
- wc -l $pfbpermit* | sort -n -r
- fi
- if [ -d "$pfbmatch" ] && [ "$(ls -A $pfbmatch)" ]; then
- echo; echo "===[ Match List IP Counts ]=========================="; echo
- wc -l $pfbmatch* | sort -n -r
- fi
- if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then
- echo; echo "===[ Deny List IP Counts ]==========================="; echo
- wc -l $pfbdeny* | sort -n -r
- fi
- if [ -d "$pfbnative" ] && [ "$(ls -A $pfbnative)" ]; then
- echo; echo "===[ Native List IP Counts ] ==================================="; echo
- wc -l $pfbnative* | sort -n -r
- fi
- if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then
- emptylists=$(grep "1\.1\.1\.1" $pfbdeny* | sed -e 's/^.*[a-zA-Z]\///' -e 's/\.txt:1.1.1.1/ /')
- if [ ! -z "$emptylists" ]; then
- echo; echo "====================[ Empty Lists w/1.1.1.1 ]=================="; echo
- for list in $emptylists; do
- echo $list
- done
- fi
- fi
- if [ -d "$pfborig" ] && [ "$(ls -A $pfborig)" ]; then
- echo; echo "====================[ Last Updated List Summary ]=============="; echo
- ls -lahtr $pfborig* | sed -e 's/\/.*\// /' -e 's/.orig//' | awk -v OFS='\t' '{print $6" "$7,$8,$9}'
- fi
- echo "==============================================================="; echo
- echo "Sanity Check (Not Including IPv6) ** These two Counts should Match! **"
- echo "------------"
- echo "Masterfile Count [ $s1 ]"
- echo "Deny folder Count [ $s2 ]"; echo
- echo "Duplication Sanity Check (Pass=No IPs reported)"
- echo "------------------------"
- echo "Masterfile/Deny Folder Uniq check"
- if [ ! -z "$s3" ]; then echo $s3; fi
- echo "Deny Folder/Masterfile Uniq check"
- if [ ! -z "$s4" ]; then echo $s4; fi
- echo; echo "Sync Check (Pass=No IPs reported)"
- echo "----------"
-else
- echo; echo "===[ FINAL Processing ]============================================="; echo
- echo " [ Original count ] [ $fcount ]"
- if [ -d "$pfbpermit" ] && [ "$(ls -A $pfbpermit)" ]; then
- echo; echo "===[ Permit List IP Counts ]========================="; echo
- wc -l $pfbpermit* | sort -n -r
- fi
- if [ -d "$pfbmatch" ] && [ "$(ls -A $pfbmatch)" ]; then
- echo; echo "===[ Match List IP Counts ]=========================="; echo
- wc -l $pfbmatch* | sort -n -r
- fi
- if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then
- echo; echo "===[ Deny List IP Counts ]==========================="; echo
- wc -l $pfbdeny* | sort -n -r
- fi
- if [ -d "$pfbnative" ] && [ "$(ls -A $pfbnative)" ]; then
- echo; echo "===[ Native List IP Counts ] ==================================="; echo
- wc -l $pfbnative* | sort -n -r
- fi
- if [ -d "$pfbdeny" ] && [ "$(ls -A $pfbdeny)" ]; then
- emptylists=$(grep "1\.1\.1\.1" $pfbdeny* | sed -e 's/^.*[a-zA-Z]\///' -e 's/\.txt:1.1.1.1/ /')
- if [ ! -z "$emptylists" ]; then
- echo; echo "====================[ Empty Lists w/1.1.1.1 ]=================="; echo
- for list in $emptylists; do
- echo $list
- done
- fi
- fi
- if [ -d "$pfborig" ] && [ "$(ls -A $pfborig)" ]; then
- echo; echo "====================[ Last Updated List Summary ]=============="; echo
- ls -lahtr $pfborig* | sed -e 's/\/.*\// /' -e 's/.orig//' | awk -v OFS='\t' '{print $6" "$7,$8,$9}'
- echo "==============================================================="
- fi
-fi
-
-echo; echo "IPv4 Alias Table IP Total"; echo "-----------------------------"
-find $pfsense_alias_dir ! -name "*_v6.txt" -type f | xargs cat | grep -c ^
-
-echo; echo "IPv6 Alias Table IP Total"; echo "-----------------------------"
-find $pfsense_alias_dir -name "*_v6.txt" -type f | xargs cat | grep -c ^
-
-echo; echo "Alias Table IP Counts"; echo "-----------------------------"
-wc -l $pfsense_alias_dir*.txt | sort -n -r
-
-echo; echo "pfSense Table Stats"; echo "-------------------"
-$pathpfctl -s memory | grep "table-entries"
-pfctlcount=$($pathpfctl -vvsTables | awk '/Addresses/ {s+=$2}; END {print s}')
-echo "Table Usage Count " $pfctlcount
-}
-
-remove() {
-# Remove Lists from Masterfiles and Delete Associated Files
-echo
-for i in $cc; do
- header=$(echo "${i%*,}")
- if [ ! "$header" == "" ]; then
- # Make sure that Alias Exists in Masterfile before removal.
- masterchk=$(grep -m1 "$header[[:space:]]" $masterfile)
- if [ ! -z "$masterchk" ]; then
- # Grep Header with a Trailing Space character
- grep "$header[[:space:]]" $masterfile > $tempfile
- awk 'FNR==NR{a[$0];next}!($0 in a)' $tempfile $masterfile > $tempfile2; mv -f $tempfile2 $masterfile
- cut -d' ' -f2 $masterfile > $mastercat
- fi
- rm -rf $pfborig$header*; rm -rf $pfbdeny$header*; rm -rf $pfbmatch$header*; rm -rf $pfbpermit$header*
- echo "The Following list has been REMOVED [ $header ]"
- fi
- echo
-done
-
-# Delete Masterfiles if they are empty
-emptychk=$(find $masterfile -size 0)
-if [ ! "$emptychk" == "" ]; then
- rm -r $masterfile; rm -r $mastercat
-fi
-}
-
-
-##########
-# CALL APPROPRIATE PROCESSES using Script Argument $1
-case $1 in
- continent)
- continent
- ;;
- duplicate)
- process255
- duplicate
- ;;
- suppress)
- suppress
- ;;
- p24)
- process24
- ;;
- dedup)
- deduplication
- ;;
- pdup)
- pdeduplication
- ;;
- et)
- processet
- ;;
- xlsx)
- processxlsx
- ;;
- closing)
- closingprocess
- ;;
- remove)
- remove
- ;;
- *)
- exit
- ;;
-esac
-exit \ No newline at end of file
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-14 22:54:04 +0000