From b6bbbf2f27fcb337ff79f9fb38f842f6296dde94 Mon Sep 17 00:00:00 2001 From: BBcan177 Date: Mon, 16 Feb 2015 22:01:33 -0500 Subject: pfBlockerNG - Fix Aliastable issue on Nano/Ramdisk Installations 1. On Nano/Ramdisk Installations, the /var/db/aliastables folder gets deleted on reboot. This causes a 60 second delay per alias. This PR archives the /var/db/aliastable/pfB_*.txt files to the pfBNG pbi /etc folder. On reboot the contents of this archive are restored. 2. Remove the previous Fetch Timeout code. 3. Improve the IPv6 Regex code. 4. Minor text improvements 5. Add conf_mount RW/RO to the sh script. 6. Add the aliastables() function to restore the aliastables files on reboot. --- config/pfblockerng/pfblockerng.inc | 101 ++++++++++++++++++++++++++++--------- config/pfblockerng/pfblockerng.sh | 72 +++++++++++++++++++++----- 2 files changed, 136 insertions(+), 37 deletions(-) diff --git a/config/pfblockerng/pfblockerng.inc b/config/pfblockerng/pfblockerng.inc index a1ee6abc..3db5b07b 100644 --- a/config/pfblockerng/pfblockerng.inc +++ b/config/pfblockerng/pfblockerng.inc @@ -78,6 +78,7 @@ function pfb_global() { $pfb['log'] = "{$pfb['logdir']}/pfblockerng.log"; $pfb['supptxt'] = "{$pfb['dbdir']}/pfbsuppression.txt"; $pfb['script'] = 'sh /usr/local/pkg/pfblockerng/pfblockerng.sh'; + $pfb['aliasarchive'] = "/usr/pbi/pfblockerng-" . php_uname("m") . "/etc/aliastables.tar.bz2"; # General Variables $pfb['config'] = $config['installedpackages']['pfblockerng']['config'][0]; @@ -334,6 +335,51 @@ function ip_range_to_subnet_array_temp2($ip1, $ip2) { } +// Archive Aliastables for NanoBSD and RAMDisk Installations +function pfb_aliastables($mode) { + global $g,$config,$pfb; + $earlyshellcmd = "/usr/local/pkg/pfblockerng/pfblockerng.sh aliastables"; + $msg = ""; + + // Only Execute function if Platform is NanoBSD or Ramdisks are used. + if (($g['platform'] != "pfSense") || isset($config['system']['use_mfs_tmpvar'])) { + if ($mode == "update") { + // Archive Aliastable Folder + exec ("cd {$pfb['aliasdir']}; ls -A pfB_*.txt && /usr/bin/tar -jcvf {$pfb['aliasarchive']} pfB_*.txt >/dev/null 2>&1"); + $msg = "\n\nArchiving Aliastable Folder"; + } + elseif ($mode == "conf") { + // Check conf file for earlyshellcmd + if (is_array($config['system']['earlyshellcmd'])) { + $a_earlyshellcmd = &$config['system']['earlyshellcmd']; + if (!preg_grep("/pfblockerng.sh aliastables/", $a_earlyshellcmd)) { + $a_earlyshellcmd[] = "{$earlyshellcmd}"; + $msg = "\n** Adding earlyshellcmd **\n"; + } + } + else { + $config['system']['earlyshellcmd'] = "{$earlyshellcmd}"; + $msg = "\n** Adding earlyshellcmd **\n"; + } + } + } + else { + // Remove earlyshellcmd if found. + if (is_array($config['system']['earlyshellcmd'])) { + $a_earlyshellcmd = &$config['system']['earlyshellcmd']; + if (preg_grep("/pfblockerng.sh aliastables/", $a_earlyshellcmd)) { + $a_earlyshellcmd = preg_grep("/pfblockerng.sh aliastables/", $a_earlyshellcmd, PREG_GREP_INVERT); + @unlink_if_exists("{$pfb['aliasarchive']}"); + $msg = "\n** Removing earlyshellcmd **\n"; + } + } + } + + if ($msg != "") + pfb_logger("{$msg}","1"); +} + + # Main pfBlockerNG Function function sync_package_pfblockerng($cron = "") { @@ -365,14 +411,8 @@ function sync_package_pfblockerng($cron = "") { } } - # TBC if Required ! (Fetch Timeout in 2.2) - - #apply fetch timeout to pfsense-utils.inc - $pfsense_utils = file_get_contents('/etc/inc/pfsense-utils.inc'); - $new_pfsense_utils = preg_replace("/\/usr\/bin\/fetch -q/","/usr/bin/fetch -T 5 -q",$pfsense_utils); - if ($new_pfsense_utils != $pfsense_utils) { - @file_put_contents('/etc/inc/pfsense-utils.inc',$new_pfsense_utils, LOCK_EX); - } + // Call function for NanoBSD/Ramdisk processes. + pfb_aliastables("conf"); # Collect pfSense Max Table Size Entry $pfb['table_limit'] = ($config['system']['maximumtableentries'] != "" ? $config['system']['maximumtableentries'] : "2000000"); @@ -1173,7 +1213,7 @@ function sync_package_pfblockerng($cron = "") { $pattern8 = '[A-Fa-f0-9]{1,4}:[A-Fa-f0-9]{1,4}:[A-Fa-f0-9]{1,4}::\/[0-9]{2}'; $pattern9 = '[A-Fa-f0-9]{1,4}:([A-Fa-f0-9]{1,4}::)\/[0-9]{2}'; $pattern10 = '[A-Fa-f0-9]{1,4}::\/[0-9]{2}'; - $pfb['ipv6'] = "/^($pattern1)$|^($pattern2)$|^($pattern3)$|^($pattern4)$|^($pattern5)$|^($pattern6)$|^($pattern7)$|^($pattern8)$|^($pattern9)$|^($pattern10)$/"; + $pfb['ipv6'] = "/($pattern1)|($pattern2)|($pattern3)|($pattern4)|($pattern5)|($pattern6)|($pattern7)|($pattern8)|($pattern9)|($pattern10)/"; $pfb['supp_update'] = FALSE; $list_type = array ("pfblockernglistsv4" => "_v4", "pfblockernglistsv6" => "_v6"); @@ -2223,6 +2263,9 @@ function sync_package_pfblockerng($cron = "") { #load filter file which will create the pfctl tables filter_configure(); + + // Call function for NanoBSD/Ramdisk processes. + pfb_aliastables("update"); } else { # Don't Execute on User 'Save' if (!$pfb['save']) { @@ -2253,8 +2296,11 @@ function sync_package_pfblockerng($cron = "") { $log = implode($result_pfctl); pfb_logger("{$log}","1"); } + + // Call function for NanoBSD/Ramdisk processes. + pfb_aliastables("update"); } else { - $log = "\n No Changes to Aliases, Skipping pfctl Update \n"; + $log = "\nNo Changes to Aliases, Skipping pfctl Update \n"; pfb_logger("{$log}","1"); } } @@ -2282,7 +2328,7 @@ function sync_package_pfblockerng($cron = "") { ######################################### - # Define/Apply CRON Jobs # + # Define/Apply CRON Jobs # ######################################### # Clear any existing pfBlockerNG Cron Jobs @@ -2291,13 +2337,13 @@ function sync_package_pfblockerng($cron = "") { # Replace Cron job with any User Changes to $pfb_min if ($pfb['enable'] == "on") { # Define pfBlockerNG CRON Job - $pfb_cmd = "/usr/local/bin/php /usr/local/www/pfblockerng/pfblockerng.php cron >> {$pfb['log']} 2>&1"; + $pfb_cmd = "/usr/local/bin/php /usr/local/www/pfblockerng/pfblockerng.php cron >> {$pfb['log']} 2>&1"; # $pfb['min'] ( User Defined Variable. Variable defined at start of Script ) - $pfb_hour = "*"; - $pfb_mday = "*"; - $pfb_month = "*"; - $pfb_wday = "*"; - $pfb_who = "root"; + $pfb_hour = "*"; + $pfb_mday = "*"; + $pfb_month = "*"; + $pfb_wday = "*"; + $pfb_who = "root"; install_cron_job($pfb_cmd, true, $pfb['min'], $pfb_hour, $pfb_mday, $pfb_month, $pfb_wday, $pfb_who); } @@ -2311,12 +2357,12 @@ function sync_package_pfblockerng($cron = "") { # MaxMind GeoIP Cron Hour is randomized between 0-23 Hour to minimize effect on MaxMind Website - $pfb_gmin = "0"; - $pfb_ghour = rand(0,23); - $pfb_gmday = "1,2,3,4,5,6,7"; - $pfb_gmonth = "*"; - $pfb_gwday = "2"; - $pfb_gwho = "root"; + $pfb_gmin = "0"; + $pfb_ghour = rand(0,23); + $pfb_gmday = "1,2,3,4,5,6,7"; + $pfb_gmonth = "*"; + $pfb_gwday = "2"; + $pfb_gwho = "root"; install_cron_job($pfb_gcmd, true, $pfb_gmin, $pfb_ghour, $pfb_gmday, $pfb_gmonth, $pfb_gwday, $pfb_gwho); } @@ -2413,6 +2459,15 @@ function pfblockerng_php_deinstall_command() { rmdir_recursive("{$pfb['dbdir']}"); rmdir_recursive("{$pfb['logdir']}"); + // Remove earlyshellcmd if found. + if (is_array($config['system']['earlyshellcmd'])) { + $a_earlyshellcmd = &$config['system']['earlyshellcmd']; + if (preg_grep("/pfblockerng.sh aliastables/", $a_earlyshellcmd)) { + $a_earlyshellcmd = preg_grep("/pfblockerng.sh aliastables/", $a_earlyshellcmd, PREG_GREP_INVERT); + @unlink_if_exists("{$pfb['aliasarchive']}"); + } + } + # Remove Settings from Config if (is_array($config['installedpackages']['pfblockerng'])) unset($config['installedpackages']['pfblockerng']); diff --git a/config/pfblockerng/pfblockerng.sh b/config/pfblockerng/pfblockerng.sh index cc11be6b..fd0a2f4a 100644 --- a/config/pfblockerng/pfblockerng.sh +++ b/config/pfblockerng/pfblockerng.sh @@ -35,6 +35,7 @@ etblock=$(echo $8 | sed 's/,/, /g') etmatch=$(echo $9 | sed 's/,/, /g') # File Locations +aliasarchive="/usr/pbi/pfblockerng-$mtype/etc/aliastables.tar.bz2" pathgeoipdat=/usr/pbi/pfblockerng-$mtype/share/GeoIP/GeoIP.dat pfbsuppression=/var/db/pfblockerng/pfbsuppression.txt masterfile=/var/db/pfblockerng/masterfile @@ -46,6 +47,7 @@ errorlog=/var/log/pfblockerng/error.log etdir=/var/db/pfblockerng/ET tmpxlsx=/tmp/xlsx/ +pfbdbdir=/var/db/pfblockerng/ pfbdeny=/var/db/pfblockerng/deny/ pfborig=/var/db/pfblockerng/original/ pfbmatch=/var/db/pfblockerng/match/ @@ -65,6 +67,17 @@ syncfile=/tmp/pfbsyncfile matchfile=/tmp/pfbmatchfile tempmatchfile=/tmp/pfbtempmatchfile +PLATFORM=`cat /etc/platform` +USE_MFS_TMPVAR=`/usr/bin/grep -c use_mfs_tmpvar /cf/conf/config.xml` +DISK_NAME=`/bin/df /var/db/rrd | /usr/bin/tail -1 | /usr/bin/awk '{print $1;}'` +DISK_TYPE=`/usr/bin/basename ${DISK_NAME} | /usr/bin/cut -c1-2` + +if [ "${PLATFORM}" != "pfSense" ] || [ ${USE_MFS_TMPVAR} -gt 0 ] || [ "${DISK_TYPE}" = "md" ]; then + /usr/local/bin/php /etc/rc.conf_mount_rw >/dev/null 2>&1 + if [ ! -d $pfbdbdir ]; then mkdir $pfbdbdir; fi + if [ ! -d $pfsense_alias_dir ]; then mkdir $pfsense_alias_dir; fi +fi + if [ ! -f $masterfile ]; then touch $masterfile; fi if [ ! -f $mastercat ]; then touch $mastercat; fi if [ ! -f $tempfile ]; then touch $tempfile; fi @@ -79,6 +92,16 @@ if [ ! -d $pfbmatch ]; then mkdir $pfbmatch; fi if [ ! -d $etdir ]; then mkdir $etdir; fi if [ ! -d $tmpxlsx ]; then mkdir $tmpxlsx; fi + +# Exit Function to set mount RO if required before Exiting +exitnow() { + if [ "${PLATFORM}" != "pfSense" ] || [ ${USE_MFS_TMPVAR} -gt 0 ] || [ "${DISK_TYPE}" = "md" ]; then + /usr/local/bin/php /etc/rc.conf_mount_ro >/dev/null 2>&1 + fi + exit +} + + ########## # Process to condense an IP range if a "Max" amount of IP addresses are found in a /24 range per Alias Group. process24() { @@ -86,7 +109,7 @@ process24() { if [ ! -x $pathgeoip ]; then echo "Process24 - Application [ GeoIP ] Not found. Can't proceed." echo "Process24 - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog - exit + exitnow fi # Download MaxMind GeoIP.dat Binary on first Install. @@ -98,7 +121,7 @@ fi if [ ! -f $pathgeoipdat ]; then echo "Process24 - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed." echo "Process24 - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog - exit + exitnow fi count=$(grep -c ^ $pfbdeny$alias".txt") @@ -192,6 +215,7 @@ echo "-------------------------------------------------------" cocount=$(grep -cv "^1\.1\.1\.1" $pfbdeny$alias".txt") echo "Post /24 Count [ $cocount ]"; echo fi +exitnow } @@ -247,6 +271,7 @@ printf "%-10s %-10s %-10s %-30s\n" "Original" "Masterfile" "Outfile" "Sanity Che echo "----------------------------------------------------------" printf "%-10s %-10s %-10s %-30s\n" "$countg" "$countm" "$counto" " [ $sanity ]" echo "----------------------------------------------------------" +exitnow } @@ -257,7 +282,7 @@ suppress() { if [ ! -x $pathgrepcidr ]; then echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" >> errorlog - exit + exitnow fi if [ -e "$pfbsuppression" ] && [ -s "$pfbsuppression" ]; then @@ -270,7 +295,7 @@ if [ -e "$pfbsuppression" ] && [ -s "$pfbsuppression" ]; then echo; echo "===[ Suppression Stats ]========================================"; echo printf "%-20s %-10s %-10s %-10s %-10s\n" "List" "Pre" "RFC1918" "Suppress" "Masterfile" echo "----------------------------------------------------------------" - exit + exitnow fi for i in $cc; do @@ -342,7 +367,7 @@ else echo "===[ Suppression Stats ]========================================"; echo printf "%-20s %-10s %-10s %-10s %-10s\n" "List" "Pre" "RFC1918" "Suppress" "Masterfile" echo "----------------------------------------------------------------" - exit + exitnow fi for i in $cc; do alias=$(echo "${i%|*}") @@ -372,6 +397,7 @@ else fi done fi +exitnow } @@ -382,7 +408,7 @@ duplicate() { if [ ! -x $pathgrepcidr ]; then echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" echo "Application [ Grepcidr ] Not found. Can't proceed. [ $now ]" >> errorlog - exit + exitnow fi dupcheck=yes @@ -415,6 +441,7 @@ printf "%-10s %-10s %-10s %-30s\n" "Original" "Masterfile" "Outfile" " [ Post Du echo "----------------------------------------------------------" printf "%-10s %-10s %-10s %-30s\n" "$countg" "$countm" "$counto" " [ $sanity ]" echo "----------------------------------------------------------" +exitnow } @@ -425,7 +452,7 @@ deduplication() { if [ ! -x $pathgeoip ]; then echo "d-duplication - Application [ GeoIP ] Not found. Can't proceed." echo "d-duplication - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog - exit + exitnow fi # Download MaxMind GeoIP.dat on first Install. @@ -438,7 +465,7 @@ fi if [ ! -f $pathgeoipdat ]; then echo "d-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed." echo "d-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog - exit + exitnow fi > $tempfile; > $tempfile2; > $dupfile; > $addfile; > $dedupfile; > $matchfile; > $tempmatchfile; count=0; dcount=0; mcount=0; mmcount=0 @@ -541,6 +568,7 @@ echo " [ Post d-Deduplication count ] [ $count ]"; echo # Write "1.1.1.1" to empty Final Blocklist Files emptyfiles=$(find $pfbdeny -size 0) for i in $emptyfiles; do echo "1.1.1.1" > $i; done +exitnow } @@ -551,7 +579,7 @@ pdeduplication(){ if [ ! -x $pathgeoip ]; then echo "p-duplication - Application [ GeoIP ] Not found. Can't proceed." echo "p-duplication - Application [ GeoIP ] Not found. Can't proceed. [ $now ]" >> $errorlog - exit + exitnow fi # Download MaxMind GeoIP.dat on first Install. @@ -563,7 +591,7 @@ fi if [ ! -f $pathgeoipdat ]; then echo "p-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed." echo "p-duplication - Database GeoIP [ GeoIP.Dat ] not found. Can't proceed. [ $now ]" >> $errorlog - exit + exitnow fi > $tempfile; > $tempfile2; > $dupfile; > $addfile; > $dedupfile; count=0; dcount=0 @@ -616,6 +644,7 @@ echo; echo " [ Post p-Deduplication count ] [ $count ]" # Write "1.1.1.1" to empty Final Blocklist Files emptyfiles=$(find $pfbdeny -size 0) for i in $emptyfiles; do echo "1.1.1.1" > $i; done +exitnow } @@ -626,7 +655,7 @@ processet() { if [ ! -x $pathgunzip ]; then echo "Application [ Gunzip ] Not found, Can't proceed." echo "Application [ Gunzip ] Not found, Can't proceed. [ $now ]" >> $errorlog - exit + exitnow fi if [ -s $pfborig$alias".gz" ]; then @@ -714,6 +743,7 @@ if [ -s $pfborig$alias".gz" ]; then else echo; echo "No ET .GZ File Found!" fi +exitnow } # Process to extract IP addresses from XLSX Files @@ -722,7 +752,7 @@ processxlsx() { if [ ! -x $pathtar ]; then echo "Application [ TAR ] Not found, Can't proceed." echo "Application [ TAR ] Not found, Can't proceed. [ $now ]" >> $errorlog - exit + exitnow fi if [ -s $pfborig$alias".zip" ]; then @@ -738,6 +768,7 @@ else echo "XLSX Download File Missing" echo " [ $alias ] XLSX Download File Missing [ $now ]" >> $errorlog fi +exitnow } closingprocess() { @@ -856,6 +887,7 @@ echo; echo "pfSense Table Stats"; echo "-------------------" $pathpfctl -s memory | grep "table-entries" pfctlcount=$($pathpfctl -vvsTables | awk '/Addresses/ {s+=$2}; END {print s}') echo "Table Usage Count " $pfctlcount +exitnow } remove() { @@ -883,6 +915,15 @@ emptychk=$(find $masterfile -size 0) if [ ! "$emptychk" == "" ]; then rm -r $masterfile; rm -r $mastercat fi +exitnow +} + +# Process to restore aliasables from archive on reboot ( NanoBSD and Ramdisk Installations only ) +aliastables() { + if [ "${PLATFORM}" != "pfSense" ] || [ ${USE_MFS_TMPVAR} -gt 0 ] || [ "${DISK_TYPE}" = "md" ]; then + [ -f $aliasarchive ] && cd $pfsense_alias_dir && /usr/bin/tar -jxvf $aliasarchive + fi + exitnow } @@ -920,8 +961,11 @@ case $1 in remove) remove ;; + aliastables) + aliastables + ;; *) - exit + exitnow ;; esac -exit \ No newline at end of file +exitnow \ No newline at end of file -- cgit v1.2.3