From a65a77ecf95a2f7c1092755c9978b38f3a361d89 Mon Sep 17 00:00:00 2001 From: BBcan177 Date: Tue, 3 Feb 2015 14:20:48 -0500 Subject: pfBlockerNG - Fix Cron Update process and other improvements Changes: 1) Add missing $pfbfolder variable to Cron Update function 2) Modify how the include files get loaded. 3) Reputation Tab only requires IPv4, simplify code to use IPv4 only for this function. 4) Update geoipupdate.sh to use the New PBI folder location and remove Archive folders after MaxMind Update process --- config/pfblockerng/geoipupdate.sh | 28 ++++++++++++++++------------ config/pfblockerng/pfblockerng.inc | 3 --- config/pfblockerng/pfblockerng.php | 34 ++++++++++++++++++++++------------ 3 files changed, 38 insertions(+), 27 deletions(-) diff --git a/config/pfblockerng/geoipupdate.sh b/config/pfblockerng/geoipupdate.sh index 4b8fbb63..302f81f5 100644 --- a/config/pfblockerng/geoipupdate.sh +++ b/config/pfblockerng/geoipupdate.sh @@ -23,19 +23,23 @@ # may be met by including the following in all advertising and documentation # mentioning features of or use of this database. -# Folder Locations +mtype=$(/usr/bin/uname -m); + +# Application Locations pathfetch=/usr/bin/fetch pathtar=/usr/bin/tar pathgunzip=/usr/bin/gunzip -# File Locations +# Folder Locations pathdb=/var/db/pfblockerng +pathpbi=/usr/pbi/pfblockerng-$mtype/share/GeoIP pathlog=/var/log/pfblockerng + +# File Locations errorlog=$pathlog/geoip.log -pathgeoipdatgz=$pathdb/GeoIP.dat.gz -pathgeoipdatgzv6=$pathdb/GeoIPv6.dat.gz -pathgeoipdat=$pathdb/GeoIP.dat -pathgeoipdatv6=$pathdb/GeoIPv6.dat +geoipdat=/GeoIP.dat +geoipdatv6=/GeoIPv6.dat + pathgeoipcc=$pathdb/country_continent.csv pathgeoipcsv4=$pathdb/GeoIPCountryCSV.zip pathgeoipcsvfinal4=$pathdb/GeoIPCountryWhois.csv @@ -56,12 +60,12 @@ binaryupdate() { echo " ** Downloading MaxMind GeoLite IPv4 Binary Database (For Reputation/Alerts Processes) **"; echo URL="http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz" -$pathfetch -v -o $pathgeoipdatgz -T 20 $URL +$pathfetch -v -o $pathpbi$geoipdat.gz -T 20 $URL if [ "$?" -eq "0" ]; then - $pathgunzip -f $pathgeoipdatgz + $pathgunzip -f $pathpbi$geoipdat.gz echo; echo " ( MaxMind IPv4 GeoIP.dat has been updated )"; echo echo "Current Date/Timestamp:" - /bin/ls -alh $pathgeoipdat + /bin/ls -alh $pathpbi$geoipdat echo else echo; echo " => MaxMind IPv4 GeoIP.dat Update [ FAILED ]"; echo @@ -72,12 +76,12 @@ fi echo; echo " ** Downloading MaxMind GeoLite IPv6 Binary Database (For Reputation/Alerts Processes) **"; echo URL="http://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz" -$pathfetch -v -o $pathgeoipdatgzv6 -T 20 $URL +$pathfetch -v -o $pathpbi$geoipdatv6.gz -T 20 $URL if [ "$?" -eq "0" ]; then - $pathgunzip -f $pathgeoipdatgzv6 + $pathgunzip -f $pathpbi$geoipdatv6.gz echo; echo " ( MaxMind IPv6 GeoIPv6.dat has been updated )"; echo echo "Current Date/Timestamp:" - /bin/ls -alh $pathgeoipdatv6 + /bin/ls -alh $pathpbi$geoipdatv6 echo else echo; echo " => MaxMind IPv6 GeoIPv6.dat Update [ FAILED ]"; echo diff --git a/config/pfblockerng/pfblockerng.inc b/config/pfblockerng/pfblockerng.inc index d612dbf1..66b5a313 100644 --- a/config/pfblockerng/pfblockerng.inc +++ b/config/pfblockerng/pfblockerng.inc @@ -2341,9 +2341,6 @@ function pfblockerng_php_install_command() { update_output_window(gettext("Downloading MaxMind Country Databases. This may take a minute...")); exec("/bin/sh /usr/local/pkg/pfblockerng/geoipupdate.sh all >> {$pfb['geolog']} 2>&1"); - @rename("{$pfb['dbdir']}/GeoIP.dat", "{$pfb['ccdir']}/GeoIP.dat"); - @rename("{$pfb['dbdir']}/GeoIPv6.dat", "{$pfb['ccdir']}/GeoIPv6.dat"); - update_output_window(gettext("MaxMind Country Database downloads completed...")); update_output_window(gettext("Converting MaxMind Country Databases for pfBlockerNG. This may take a few minutes...")); pfblockerng_uc_countries(); diff --git a/config/pfblockerng/pfblockerng.php b/config/pfblockerng/pfblockerng.php index 1db36334..ca8a23a9 100644 --- a/config/pfblockerng/pfblockerng.php +++ b/config/pfblockerng/pfblockerng.php @@ -44,9 +44,13 @@ require_once("functions.inc"); require_once("pkg-utils.inc"); require_once("globals.inc"); require_once("services.inc"); -require_once("/usr/local/pkg/pfblockerng/pfblockerng.inc"); -pfb_global(); +// Call Include File and Collect updated Global Settings +if (in_array($argv[1], array( 'update','dc','uc','gc','cron' ))) { + require_once("/usr/local/pkg/pfblockerng/pfblockerng.inc"); + pfb_global(); +} + // IPv6 Range to CIDR function used courtesey from: // https://github.com/stilez/pfsense-leases/blob/50cc0fa81dba5fe91bcddaea016c245d1b8479cc/etc/inc/util.inc @@ -147,7 +151,7 @@ $uname = posix_uname(); if ($uname['machine'] == "amd64") ini_set('memory_limit', '256M'); -function pfb_update_check($header_url, $list_url, $url_format) { +function pfb_update_check($header_url, $list_url, $url_format, $pfbfolder) { global $pfb; $pfb['cron_update'] = FALSE; @@ -230,6 +234,12 @@ if ($argv[1] == 'dc') { } pfblockerng_uc_countries(); pfblockerng_get_countries(); + + // Remove Original Maxmind Database Files + @unlink_if_exists("{$pfb['dbdir']}/GeoIPCountryCSV.zip"); + @unlink_if_exists("{$pfb['dbdir']}/GeoIPCountryWhois.csv"); + @unlink_if_exists("{$pfb['dbdir']}/GeoIPv6.csv"); + @unlink_if_exists("{$pfb['dbdir']}/country_continent.csv"); } if ($argv[1] == 'uc') { @@ -425,7 +435,6 @@ if ($argv[1] == 'cron') { // Function to process the downloaded Maxmind Database and format into Continent txt files. function pfblockerng_uc_countries() { global $g,$pfb; - pfb_global(); $maxmind_cont = "{$pfb['dbdir']}/country_continent.csv"; $maxmind_cc4 = "{$pfb['dbdir']}/GeoIPCountryWhois.csv"; @@ -584,7 +593,6 @@ function pfblockerng_uc_countries() { // Function to process Continent txt files and create Country ISO files and to Generate GUI XML files. function pfblockerng_get_countries() { global $g,$pfb; - pfb_global(); $files = array ( "Africa" => "{$pfb['ccdir']}/Africa_v4.txt", "Asia" => "{$pfb['ccdir']}/Asia_v4.txt", @@ -622,7 +630,9 @@ function pfblockerng_get_countries() { if (preg_match("/#/",$line)) { if ($pfb['complete']) { ${'coptions' . $type}[] = $country . '-' . $isocode . ' ('. $total .') ' . ' ' . $isocode . ''; - ${'roptions' . $type}[] = $country . '-' . $isocode . ' ('. $total .') ' . ' ' . $isocode . ''; + // Only collect IPv4 for Reputation Tab + if ($type == "4") + $roptions4[] = $country . '-' . $isocode . ' ('. $total .') ' . ' ' . $isocode . ''; // Save ISO data @file_put_contents($pfb['ccdir'] . '/' . $isocode . '_v' . $type . '.txt', $xml_data, LOCK_EX); @@ -646,7 +656,8 @@ function pfblockerng_get_countries() { if ($keycount == $lastkey) { if (preg_match("/Total Networks: 0/", $line)) { continue;} // Dont Display Countries with Null Data ${'coptions' . $type}[] = $country . '-' . $isocode . ' ('. $total .') ' . ' ' . $isocode . ''; - ${'roptions' . $type}[] = $country . '-' . $isocode . ' ('. $total .') ' . ' ' . $isocode . ''; + if ($type == "4") + $roptions4[] = $country . '-' . $isocode . ' ('. $total .') ' . ' ' . $isocode . ''; @file_put_contents($pfb['ccdir'] . '/' . $isocode . '_v' . $type . '.txt', $xml_data, LOCK_EX); unset($total, $xml_data); } @@ -962,13 +973,12 @@ EOF; // Sort Countries IPv4 Alphabetically and Build XML