From 208f74083534d47d3ac477b7ad60bea4768ff52e Mon Sep 17 00:00:00 2001 From: BBcan177 Date: Sun, 1 Feb 2015 17:16:57 -0500 Subject: pfBlockerNG - Mods to php file for MaxMind Anon. Proxy/Satellite 1) Improve/Condense the code to determine if a List needs to be Updated. 2) Condense the IPv4/V6 code for MaxMind Database Conversion into a single process and add the MaxMind Anon. Proxy and Satellite Providers capability. --- config/pfblockerng/pfblockerng.php | 524 +++++++++++++++---------------------- 1 file changed, 210 insertions(+), 314 deletions(-) (limited to 'config/pfblockerng/pfblockerng.php') diff --git a/config/pfblockerng/pfblockerng.php b/config/pfblockerng/pfblockerng.php index 20080aa4..1db36334 100644 --- a/config/pfblockerng/pfblockerng.php +++ b/config/pfblockerng/pfblockerng.php @@ -149,11 +149,12 @@ if ($uname['machine'] == "amd64") function pfb_update_check($header_url, $list_url, $url_format) { global $pfb; + $pfb['cron_update'] = FALSE; if ($url_format == "rsync" || $url_format == "html") { $log = "[ {$header_url} ]\n Skipping timestamp query\n"; pfb_logger("{$log}","1"); - return TRUE; + $pfb['cron_update'] = TRUE; } switch ($url_format) { @@ -195,14 +196,23 @@ function pfb_update_check($header_url, $list_url, $url_format) { $log = " Local timestamp: {$local_tds}\n"; pfb_logger("{$log}","1"); if ("{$remote_tds}" != "{$local_tds}") { - return TRUE; + $pfb['cron_update'] = TRUE; } else { $log = " Remote file unchanged. Download Terminated\n"; pfb_logger("{$log}","1"); - return FALSE; + $pfb['cron_update'] = FALSE; } } else { - return TRUE; + $pfb['cron_update'] = TRUE; + } + + if ($pfb['cron_update']) { + // Trigger CRON Process if Updates are Found. + $pfb['update_cron'] = TRUE; + + $log = " Updates Found\n"; + pfb_logger("{$log}","1"); + unlink_if_exists($pfbfolder . '/' . $header_url . '.txt'); } } @@ -233,6 +243,7 @@ if ($argv[1] == 'gc') { if ($argv[1] == 'cron') { $hour = date('H'); $dow = date('N'); + $pfb['update_cron'] = FALSE; # Start hour of the 'Once a day' Schedule $pfb['dailystart'] = $config['installedpackages']['pfblockerng']['config'][0]['pfb_dailystart']; @@ -344,101 +355,47 @@ if ($argv[1] == 'cron') { if (!file_exists($pfbfolder . '/' . $header_url . '.txt')) { $log = " Updates Found\n"; pfb_logger("{$log}","1"); - $updates++; + $pfb['update_cron'] = TRUE; continue; } switch ($list_cron) { case "01hour": - if (pfb_update_check($header_url, $list_url, $url_format)) { - $log = " Updates Found\n"; - pfb_logger("{$log}","1"); - unlink_if_exists($pfbfolder . '/' . $header_url . '.txt'); - $updates++; - } + pfb_update_check($header_url, $list_url, $url_format, $pfbfolder); break; case "02hours": - if (in_array($hour, $e_sch2)) { - if (pfb_update_check($header_url, $list_url, $url_format)) { - $log = " Updates Found\n"; - pfb_logger("{$log}","1"); - unlink_if_exists($pfbfolder . '/' . $header_url . '.txt'); - $updates++; - } - } + if (in_array($hour, $e_sch2)) + pfb_update_check($header_url, $list_url, $url_format, $pfbfolder); break; case "03hours": - if (in_array($hour, $e_sch3)) { - if (pfb_update_check($header_url, $list_url, $url_format)) { - $log = " Updates Found\n"; - pfb_logger("{$log}","1"); - unlink_if_exists($pfbfolder . '/' . $header_url . '.txt'); - $updates++; - } - } + if (in_array($hour, $e_sch3)) + pfb_update_check($header_url, $list_url, $url_format, $pfbfolder); break; case "04hours": - if (in_array($hour, $e_sch4)) { - if (pfb_update_check($header_url, $list_url, $url_format)) { - $log = " Updates Found\n"; - pfb_logger("{$log}","1"); - unlink_if_exists($pfbfolder . '/' . $header_url . '.txt'); - $updates++; - } - } + if (in_array($hour, $e_sch4)) + pfb_update_check($header_url, $list_url, $url_format, $pfbfolder); break; case "06hours": - if (in_array($hour, $e_sch6)) { - if (pfb_update_check($header_url, $list_url, $url_format)) { - $log = " Updates Found\n"; - pfb_logger("{$log}","1"); - unlink_if_exists($pfbfolder . '/' . $header_url . '.txt'); - $updates++; - } - } + if (in_array($hour, $e_sch6)) + pfb_update_check($header_url, $list_url, $url_format, $pfbfolder); break; case "08hours": - if (in_array($hour, $e_sch8)) { - if (pfb_update_check($header_url, $list_url, $url_format)) { - $log = " Updates Found\n"; - pfb_logger("{$log}","1"); - unlink_if_exists($pfbfolder . '/' . $header_url . '.txt'); - $updates++; - } - } + if (in_array($hour, $e_sch8)) + pfb_update_check($header_url, $list_url, $url_format, $pfbfolder); break; case "12hours": - if (in_array($hour, $e_sch12)) { - if (pfb_update_check($header_url, $list_url, $url_format)) { - $log = " Updates Found\n"; - pfb_logger("{$log}","1"); - unlink_if_exists($pfbfolder . '/' . $header_url . '.txt'); - $updates++; - } - } + if (in_array($hour, $e_sch12)) + pfb_update_check($header_url, $list_url, $url_format, $pfbfolder); break; case "EveryDay": - if ($hour == $pfb['dailystart']) { - if (pfb_update_check($header_url, $list_url, $url_format)) { - $log = " Updates Found\n"; - pfb_logger("{$log}","1"); - unlink_if_exists($pfbfolder . '/' . $header_url . '.txt'); - $updates++; - } - } + if ($hour == $pfb['dailystart']) + pfb_update_check($header_url, $list_url, $url_format, $pfbfolder); break; case "Weekly": - if ($hour == $pfb['dailystart'] && $dow == $header_dow) { - if (pfb_update_check($header_url, $list_url, $url_format)) { - $log = " Updates Found\n"; - pfb_logger("{$log}","1"); - unlink_if_exists($pfbfolder . '/' . $header_url . '.txt'); - $updates++; - } - } + if ($hour == $pfb['dailystart'] && $dow == $header_dow) + pfb_update_check($header_url, $list_url, $url_format, $pfbfolder); break; - default: { - } + default: break; } } @@ -448,7 +405,7 @@ if ($argv[1] == 'cron') { } } - if ($updates > 0) { + if ($pfb['update_cron']) { sync_package_pfblockerng("cron"); } else { $log = "\n No Updates required. \n\n"; @@ -465,14 +422,14 @@ if ($argv[1] == 'cron') { } +// Function to process the downloaded Maxmind Database and format into Continent txt files. function pfblockerng_uc_countries() { - global $g,$pfb; pfb_global(); - $maxmind_cont = "{$pfb['dbdir']}/country_continent.csv"; - $maxmind_cc4 = "{$pfb['dbdir']}/GeoIPCountryWhois.csv"; - $maxmind_cc6 = "{$pfb['dbdir']}/GeoIPv6.csv"; + $maxmind_cont = "{$pfb['dbdir']}/country_continent.csv"; + $maxmind_cc4 = "{$pfb['dbdir']}/GeoIPCountryWhois.csv"; + $maxmind_cc6 = "{$pfb['dbdir']}/GeoIPv6.csv"; # Create Folders if not Exist $folder_array = array ("{$pfb['dbdir']}","{$pfb['logdir']}","{$pfb['ccdir']}"); @@ -505,13 +462,12 @@ function pfblockerng_uc_countries() { $maxmind_ver .= "\nThese Timestamps should *match* \n"; @file_put_contents("{$pfb['logdir']}/maxmind_ver", $maxmind_ver); - // Collect ISO Codes for Each Continent - $log = "Processing Continent Data \n"; + $log = "Processing Continent Data\n"; print $log; pfb_logger("{$log}","3"); - $cont_array = array ( array($AF),array($AS),array($EU),array($NA),array($OC),array($SA)); + $cont_array = array ( array($AF),array($AS),array($EU),array($NA),array($OC),array($SA),array($AX)); if (($handle = fopen("{$maxmind_cont}",'r')) !== FALSE) { while (($cc = fgetcsv($handle)) !== FALSE) { @@ -560,244 +516,163 @@ function pfblockerng_uc_countries() { unset($cc); fclose($handle); - // Collect Country ISO Data IPv4 and Sort to Continent Array - $log = "Processing ISO IPv4 Continent/Country Data \n"; - print $log; - pfb_logger("{$log}","3"); + // Add Maxmind Anonymous Proxy and Satellite Providers to array + $cont_array[6]['continent'] = "Proxy and Satellite"; + $cont_array[6]['iso'] = "A1,A2"; + $cont_array[6]['file4'] = "{$pfb['ccdir']}/Proxy_Satellite_v4.txt"; + $cont_array[6]['file6'] = "{$pfb['ccdir']}/Proxy_Satellite_v6.txt"; - if (($handle = fopen("{$maxmind_cc4}",'r')) !== FALSE) { - while (($cc = fgetcsv($handle)) !== FALSE) { - - $ip1_key = $cc[0]; - $ip2_key = $cc[1]; - $var1_key = $cc[2]; - $var2_key = $cc[3]; - $cc_key = $cc[4]; - $country_key = $cc[5]; - $a_cidr = implode(",", ip_range_to_subnet_array_temp($cc[0],$cc[1])); - - $counter = 0; - foreach ($cont_array as $iso) { - if (preg_match("/\b$cc_key\b/", $iso['iso'])) { - $cont_array[$counter][$cc_key]['ip4'] .= $a_cidr . ","; - $cont_array[$counter][$cc_key]['country'] = $country_key; - continue; - } - $counter++; - } - } - } - unset($cc); - fclose($handle); + // Collect Country ISO data and sort to Continent arrays (IPv4 and IPv6) + foreach (array("4", "6") as $type) { + $log = "Processing ISO IPv{$type} Continent/Country Data\n"; + print $log; + pfb_logger("{$log}","3"); - // Build Continent IPv4 CIDR Files - $counter = 0; - foreach ($cont_array as $iso) { - $header = ""; - $pfb_file = ""; - $iso_key = ""; - $header .= "# Generated from MaxMind Inc. on: " . date("m/d/y G:i:s", time()) . "\n"; - $header .= "# Continent IPv4: " . $cont_array[$counter]['continent'] . "\n"; - $pfb_file = $cont_array[$counter]['file4']; - $iso_key = array_keys($iso); - foreach ($iso_key as $key) { - if (preg_match("/[A-Z]{2}/", $key)) { - $header .= "# Country: " . $iso[$key]['country'] . "\n"; - $header .= "# ISO Code: " . $key . "\n"; - $header .= "# Total Networks: " . substr_count($iso[$key]['ip4'], ",") . "\n"; - $header .= str_replace(",", "\n", $iso[$key]['ip4']); - $iso[$key]['ip4'] = ""; - } + if ($type == "4") { + $maxmind_cc = "{$pfb['dbdir']}/GeoIPCountryWhois.csv"; + } else { + $maxmind_cc = "{$pfb['dbdir']}/GeoIPv6.csv"; } - $counter++; - @file_put_contents($pfb_file, $header, LOCK_EX); - } - - - // Collect Country ISO Data IPv6 and Sort to Continent Array - $log = "Processing ISO IPv6 Continent/Country Data \n"; - print $log; - pfb_logger("{$log}","3"); - - if (($handle = fopen("{$maxmind_cc6}",'r')) !== FALSE) { - while (($cc = fgetcsv($handle)) !== FALSE) { - - $ip1_key = $cc[0]; - $ip2_key = $cc[1]; - $var1_key = $cc[2]; - $var2_key = $cc[3]; - $cc_key = $cc[4]; - $country_key = $cc[5]; - $a_cidr = implode(",", ip_range_to_subnet_array_temp($cc[0],$cc[1])); - - $counter = 0; - foreach ($cont_array as $iso) { - if (preg_match("/\b$cc_key\b/", $iso['iso'])) { - $cont_array[$counter][$cc_key]['ip6'] .= $a_cidr . ","; - continue; + $iptype = "ip{$type}"; + $filetype = "file{$type}"; + + if (($handle = fopen("{$maxmind_cc}",'r')) !== FALSE) { + while (($cc = fgetcsv($handle)) !== FALSE) { + $cc_key = $cc[4]; + $country_key = $cc[5]; + $a_cidr = implode(",", ip_range_to_subnet_array_temp($cc[0],$cc[1])); + $counter = 0; + foreach ($cont_array as $iso) { + if (preg_match("/\b$cc_key\b/", $iso['iso'])) { + $cont_array[$counter][$cc_key][$iptype] .= $a_cidr . ","; + $cont_array[$counter][$cc_key]['country'] = $country_key; + continue; + } + $counter++; } - $counter++; } } - } - unset($cc); - fclose($handle); - - // Build Continent IPv6 Files - $counter = 0; - foreach ($cont_array as $iso) { - $header = ""; - $pfb_file = ""; - $iso_key = ""; - - $header .= "# Generated from MaxMind Inc. on: " . date("m/d/y G:i:s", time()) . "\n"; - $header .= "# Continent IPv6: " . $cont_array[$counter]['continent'] . "\n"; - $pfb_file = $cont_array[$counter]['file6']; - $iso_key = array_keys($iso); - foreach ($iso_key as $key) { - if (preg_match("/[A-Z]{2}/", $key)) { - $header .= "# Country: " . $iso[$key]['country'] . "\n"; - $header .= "# ISO Code: " . $key . "\n"; - $header .= "# Total Networks: " . substr_count($iso[$key]['ip6'], ",") . "\n"; - $header .= str_replace(",", "\n", $iso[$key]['ip6']); - $iso[$key]['ip6'] = ""; + unset($cc); + fclose($handle); + + // Build Continent Files + $counter = 0; + foreach ($cont_array as $iso) { + $header = ""; + $pfb_file = ""; + $iso_key = ""; + $header .= "# Generated from MaxMind Inc. on: " . date("m/d/y G:i:s", time()) . "\n"; + $header .= "# Continent IPv{$type}: " . $cont_array[$counter]['continent'] . "\n"; + $pfb_file = $cont_array[$counter][$filetype]; + $iso_key = array_keys($iso); + foreach ($iso_key as $key) { + if (preg_match("/[A-Z]{2}|A1|A2/", $key)) { + $header .= "# Country: " . $iso[$key]['country'] . "\n"; + $header .= "# ISO Code: " . $key . "\n"; + $header .= "# Total Networks: " . substr_count($iso[$key][$iptype], ",") . "\n"; + $header .= str_replace(",", "\n", $iso[$key][$iptype]); + $iso[$key][$iptype] = ""; + } } + $counter++; + @file_put_contents($pfb_file, $header, LOCK_EX); } - $counter++; - @file_put_contents($pfb_file, $header, LOCK_EX); } - unset($cont_array); } +// Function to process Continent txt files and create Country ISO files and to Generate GUI XML files. function pfblockerng_get_countries() { - global $g,$pfb; pfb_global(); - # These arrays are used to collect the '; - $roptions4[] = $Country . '-' . $ISOCode . ' ('. $total4 .') ' . ' ' . $ISOCode . ''; - - // Save ISO IPv4 Data - @file_put_contents($pfb['ccdir'] . '/' . $ISOCode . '_v4.txt',$pfb_v4,LOCK_EX); - - // Clear Variables and Restart Continent Collection process - $total4 = 0; - $pfb_v4 = ""; - $pfb['complete'] = FALSE; + foreach ($files as $cont => $file) { + // Process the following for IPv4 and IPv6 + foreach (array("4", "6") as $type) { + $log = "IPv{$type} " . $cont . "\n"; + print $log; + pfb_logger("{$log}","3"); + + if ($type == "6") + $file = preg_replace("/v4/", "v6", $file); + $convert = explode("\n", file_get_contents($file)); + $cont_name = preg_replace("/ /", "", $cont); + $cont_name_lower = strtolower($cont_name); + $active = array("$cont" => ''); + $lastkey = count ($convert) - 1; + $pfb['complete'] = FALSE; + $keycount = 1; + $total = 0; + + foreach ($convert as $line) { + if (preg_match("/#/",$line)) { + if ($pfb['complete']) { + ${'coptions' . $type}[] = $country . '-' . $isocode . ' ('. $total .') ' . ' ' . $isocode . ''; + ${'roptions' . $type}[] = $country . '-' . $isocode . ' ('. $total .') ' . ' ' . $isocode . ''; + + // Save ISO data + @file_put_contents($pfb['ccdir'] . '/' . $isocode . '_v' . $type . '.txt', $xml_data, LOCK_EX); + + // Clear variables and restart Continent collection process + unset($total, $xml_data); + $pfb['complete'] = FALSE; + } + if (preg_match("/Total Networks: 0/", $line)) { continue;} // Don't Display Countries with Null Data + if (preg_match("/Country:\s(.*)/",$line, $matches)) { $country = $matches[1];} + if (preg_match("/ISO Code:\s(.*)/",$line, $matches)) { $isocode = $matches[1];} + } + elseif (!preg_match("/#/",$line)) { + $total++; + if (!empty($line)) + $xml_data .= $line . "\n"; + $pfb['complete'] = TRUE; } - if (preg_match("/Country:\s(.*)/",$line, $matches)) { $Country = $matches[1];} - if (preg_match("/ISO Code:\s(.*)/",$line, $matches)) { $ISOCode = $matches[1];} - } elseif (!preg_match("/#/",$line)) { - $total4++; - if (!empty($line)) - $pfb_v4 .= $line . "\n"; - $pfb['complete'] = TRUE; - } - } - unset ($ips, $convert); - - // Sort IPv4 Countries Alphabetically and Build XML '; + ${'roptions' . $type}[] = $country . '-' . $isocode . ' ('. $total .') ' . ' ' . $isocode . ''; + @file_put_contents($pfb['ccdir'] . '/' . $isocode . '_v' . $type . '.txt', $xml_data, LOCK_EX); + unset($total, $xml_data); } - if (preg_match("/Country:\s(.*)/",$line, $matches)) { $Country = $matches[1];} - if (preg_match("/ISO Code:\s(.*)/",$line, $matches)) { $ISOCode = $matches[1];} - } elseif (!preg_match("/#/",$line)) { - $total6++; - if (!empty($line)) - $pfb_v6 .= $line . "\n"; - $pfb['complete'] = TRUE; + $keycount++; } - } - - // Sort IPv6 Countries Alphabetically for Continents Tab - sort($coptions6, SORT_STRING); - $ftotal6 = count($coptions6); - $count = 1; - $options6 = ""; - - foreach ($coptions6 as $option6) { - if ($count == 1) { $options6 .= "\t" . '