0) list ($ip1bin, $ip2bin) = array($ip2bin, $ip1bin); // swap contents of ip1 <= ip2 $rangesubnets = array(); $netsize = 0; do { // at loop start, $ip1 is guaranteed strictly less than $ip2 (important for edge case trapping and preventing accidental binary wrapround) // which means the assignments $ip1 += 1 and $ip2 -= 1 will always be "binary-wrapround-safe" // step #1 if start ip (as shifted) ends in any '1's, then it must have a single cidr to itself (any cidr would include the '0' below it) if (substr($ip1bin, -1, 1) == '1') { // the start ip must be in a separate one-IP cidr range $new_subnet_ip = substr($ip1bin, $netsize, $bits - $netsize) . str_repeat('0', $netsize); $rangesubnets[$new_subnet_ip] = $bits - $netsize; $n = strrpos($ip1bin, '0'); //can't be all 1's $ip1bin = ($n == 0 ? '' : substr($ip1bin, 0, $n)) . '1' . str_repeat('0', $bits - $n - 1); // BINARY VERSION OF $ip1 += 1 } // step #2, if end ip (as shifted) ends in any zeros then that must have a cidr to itself (as cidr cant span the 1->0 gap) if (substr($ip2bin, -1, 1) == '0') { // the end ip must be in a separate one-IP cidr range $new_subnet_ip = substr($ip2bin, $netsize, $bits - $netsize) . str_repeat('0', $netsize); $rangesubnets[$new_subnet_ip] = $bits - $netsize; $n = strrpos($ip2bin, '1'); //can't be all 0's $ip2bin = ($n == 0 ? '' : substr($ip2bin, 0, $n)) . '0' . str_repeat('1', $bits - $n - 1); // BINARY VERSION OF $ip2 -= 1 // already checked for the edge case where end = start+1 and start ends in 0x1, above, so it's safe } // this is the only edge case arising from increment/decrement. // it happens if the range at start of loop is exactly 2 adjacent ips, that spanned the 1->0 gap. (we will have enumerated both by now) if (strcmp($ip2bin, $ip1bin) < 0) continue; // step #3 the start and end ip MUST now end in '0's and '1's respectively // so we have a non-trivial range AND the last N bits are no longer important for CIDR purposes. $shift = $bits - max(strrpos($ip1bin, '0'), strrpos($ip2bin, '1')); // num of low bits which are '0' in ip1 and '1' in ip2 $ip1bin = str_repeat('0', $shift) . substr($ip1bin, 0, $bits - $shift); $ip2bin = str_repeat('0', $shift) . substr($ip2bin, 0, $bits - $shift); $netsize += $shift; if ($ip1bin === $ip2bin) { // we're done. $new_subnet_ip = substr($ip1bin, $netsize, $bits - $netsize) . str_repeat('0', $netsize); $rangesubnets[$new_subnet_ip] = $bits - $netsize; continue; } // at this point there's still a remaining range, and either startip ends with '1', or endip ends with '0'. So repeat cycle. } while (strcmp($ip1bin, $ip2bin) < 0); // subnets are ordered by bit size. Re sort by IP ("naturally") and convert back to IPv4/IPv6 ksort($rangesubnets, SORT_STRING); $out = array(); foreach ($rangesubnets as $ip => $netmask) { if ($proto == 'ipv4') { $i = str_split($ip, 8); $out[] = implode('.', array( bindec($i[0]),bindec($i[1]),bindec($i[2]),bindec($i[3]))) . '/' . $netmask; } else $out[] = Net_IPv6::compress(Net_IPv6::_bin2Ip($ip)) . '/' . $netmask; } return $out; } # Set php Memory Limit $uname = posix_uname(); if ($uname['machine'] == "amd64") { ini_set('memory_limit', '256M'); } function pfb_update_check($header_url, $list_url, $url_format, $pfbfolder) { global $pfb; $pfb['cron_update'] = FALSE; if ($url_format == "rsync" || $url_format == "html") { $log = "[ {$header_url} ]\n Skipping timestamp query\n"; pfb_logger("{$log}","1"); $pfb['cron_update'] = TRUE; } switch ($url_format) { case "gz": case "gz_2": case "gz_lg": case "et": $type = '.gz'; break; case "zip": case "xlsx": $type = '.zip'; break; case "txt": $type = '.orig'; break; case "html": case "block": $type = '.raw'; break; } $log = "[ {$header_url} ]\n"; pfb_logger("{$log}","1"); $host = @parse_url($list_url); $local_file = "{$pfb['origdir']}/{$header_url}{$type}"; if (file_exists($local_file)) { // Determine if URL is Remote or Local if ($host['host'] == "127.0.0.1" || $host['host'] == $pfb['iplocal'] || empty($host['host'])) { $remote_tds = gmdate ("D, d M Y H:i:s T", filemtime($local_file)); } else { $remote_tds = @implode(preg_grep("/Last-Modified/", get_headers($list_url))); $remote_tds = preg_replace("/^Last-Modified: /","", $remote_tds); } $log = " Remote timestamp: {$remote_tds}\n"; pfb_logger("{$log}","1"); $local_tds = gmdate ("D, d M Y H:i:s T", filemtime($local_file)); $log = " Local timestamp: {$local_tds}\n"; pfb_logger("{$log}","1"); if ("{$remote_tds}" != "{$local_tds}") { $pfb['cron_update'] = TRUE; } else { $log = " Remote file unchanged. Download Terminated\n"; pfb_logger("{$log}","1"); $pfb['cron_update'] = FALSE; } } else { $pfb['cron_update'] = TRUE; } if ($pfb['cron_update']) { // Trigger CRON Process if Updates are Found. $pfb['update_cron'] = TRUE; $log = " Updates Found\n"; pfb_logger("{$log}","1"); unlink_if_exists($pfbfolder . '/' . $header_url . '.txt'); } } if ($argv[1] == 'update') { sync_package_pfblockerng("cron"); } if ($argv[1] == 'dc') { # (Options - 'bu' Binary Update for Reputation/Alerts Page, 'all' for Country update and 'bu' options. if ($pfb['cc'] == "") { exec("/bin/sh /usr/local/pkg/pfblockerng/geoipupdate.sh all >> {$pfb['geolog']} 2>&1"); } else { exec("/bin/sh /usr/local/pkg/pfblockerng/geoipupdate.sh bu >> {$pfb['geolog']} 2>&1"); } pfblockerng_uc_countries(); pfblockerng_get_countries(); // Remove Original Maxmind Database Files @unlink_if_exists("{$pfb['dbdir']}/GeoIPCountryCSV.zip"); @unlink_if_exists("{$pfb['dbdir']}/GeoIPCountryWhois.csv"); @unlink_if_exists("{$pfb['dbdir']}/GeoIPv6.csv"); @unlink_if_exists("{$pfb['dbdir']}/country_continent.csv"); } if ($argv[1] == 'uc') { pfblockerng_uc_countries(); } if ($argv[1] == 'gc') { pfblockerng_get_countries(); } if ($argv[1] == 'cron') { // Call Base Hour converter $pfb_sch = pfb_cron_base_hour(); $hour = date('G'); $dow = date('N'); $pfb['update_cron'] = FALSE; $log = " CRON PROCESS START [ NOW ]\n"; pfb_logger("{$log}","1"); $list_type = array ("pfblockernglistsv4" => "_v4", "pfblockernglistsv6" => "_v6"); foreach ($list_type as $ip_type => $vtype) { if ($config['installedpackages'][$ip_type]['config'] != "") { foreach ($config['installedpackages'][$ip_type]['config'] as $list) { if (is_array($list['row']) && $list['action'] != "Disabled") { foreach ($list['row'] as $row) { if ($row['url'] != "" && $row['state'] != "Disabled") { if ($vtype == "_v4") { $header_url = "{$row['header']}"; } else { $header_url = "{$row['header']}_v6"; } # Determine Folder Location for Alias (return array $pfbarr) pfb_determine_list_detail($list['action'], "", "", ""); $pfbfolder = $pfbarr['folder']; $list_cron = $list['cron']; $list_url = $row['url']; $header_dow = $list['dow']; $url_format = $row['format']; // Bypass update if state is defined as "Hold" and list file exists if (file_exists($pfbfolder . '/' . $header_url . '.txt') && $row['state'] == "Hold") { continue; } # Check if List file exists, if not found run Update if (!file_exists($pfbfolder . '/' . $header_url . '.txt')) { $log = " Updates Found\n"; pfb_logger("{$log}","1"); $pfb['update_cron'] = TRUE; continue; } switch ($list_cron) { case "EveryDay": if ($hour == $pfb['24hour']) { pfb_update_check($header_url, $list_url, $url_format, $pfbfolder); } break; case "Weekly": if ($hour == $pfb['24hour'] && $dow == $header_dow) { pfb_update_check($header_url, $list_url, $url_format, $pfbfolder); } break; default: if ($pfb['interval'] == "1" || in_array($hour, $pfb_sch)) { pfb_update_check($header_url, $list_url, $url_format, $pfbfolder); } break; } } } } } } } // If Continents are Defined, continue with Update Process to determine if further changes are required. $continents = array ( "Africa" => "pfB_Africa", "Antartica" => "pfB_Antartica", "Asia" => "pfB_Asia", "Europe" => "pfB_Europe", "North America" => "pfB_NAmerica", "Oceania" => "pfB_Oceania", "South America" => "pfB_SAmerica", "Top Spammers" => "pfB_Top", "Proxy and Satellite" => "pfB_PS" ); if (!$pfb['update_cron']) { foreach ($continents as $continent => $pfb_alias) { if (is_array($config['installedpackages']['pfblockerng' . strtolower(preg_replace('/ /','',$continent))]['config'])) { $continent_config = $config['installedpackages']['pfblockerng' . strtolower(preg_replace('/ /','',$continent))]['config'][0]; if ($continent_config['action'] != "Disabled" && $pfb['enable'] == "on") { $pfb['update_cron'] = TRUE; break; } } } } if ($pfb['update_cron']) { sync_package_pfblockerng("cron"); } else { sync_package_pfblockerng("noupdates"); $log = "\n No Updates required.\n CRON PROCESS ENDED\n UPDATE PROCESS ENDED\n"; pfb_logger("{$log}","1"); } # Call Log Mgmt Function // If Update GUI 'Manual view' is selected. Last output will be missed. So sleep for 5 secs. sleep(5); pfb_log_mgmt(); } // Function to process the downloaded Maxmind Database and format into Continent txt files. function pfblockerng_uc_countries() { global $g,$pfb; $maxmind_cont = "{$pfb['dbdir']}/country_continent.csv"; $maxmind_cc4 = "{$pfb['dbdir']}/GeoIPCountryWhois.csv"; $maxmind_cc6 = "{$pfb['dbdir']}/GeoIPv6.csv"; # Create Folders if not Exist $folder_array = array ("{$pfb['dbdir']}","{$pfb['logdir']}","{$pfb['ccdir']}"); foreach ($folder_array as $folder) { safe_mkdir ("{$folder}",0755); } $now = date("m/d/y G:i:s", time()); $log = "Country Code Update Start - [ NOW ]\n\n"; print "Country Code Update Start - [ $now ]\n\n"; pfb_logger("{$log}","3"); if (!file_exists($maxmind_cont) || !file_exists($maxmind_cc4) || !file_exists($maxmind_cc6)) { $log = " [ MAXMIND UPDATE FAIL, CSV Missing, using Previous Country Code Database \n"; print $log; pfb_logger("{$log}","3"); return; } # Save Date/Time Stamp to MaxMind version file $maxmind_ver = "MaxMind GeoLite Date/Time Stamps \n\n"; $remote_tds = @implode(preg_grep("/Last-Modified/", get_headers("http://geolite.maxmind.com/download/geoip/database/GeoIPCountryCSV.zip"))); $maxmind_ver .= "MaxMind_v4 \t" . $remote_tds . "\n"; $local_tds = @gmdate ("D, d M Y H:i:s T", filemtime($maxmind_cc4)); $maxmind_ver .= "Local_v4 \tLast-Modified: " . $local_tds . "\n\n"; $remote_tds = @implode(preg_grep("/Last-Modified/", get_headers("http://geolite.maxmind.com/download/geoip/database/GeoIPv6.csv.gz"))); $maxmind_ver .= "MaxMind_v6 \t" . $remote_tds . "\n"; $local_tds = @gmdate ("D, d M Y H:i:s T", filemtime($maxmind_cc6)); $maxmind_ver .= "Local_v6 \tLast-Modified: " . $local_tds . "\n"; $maxmind_ver .= "\nThese Timestamps should *match* \n"; @file_put_contents("{$pfb['logdir']}/maxmind_ver", $maxmind_ver); // Collect ISO Codes for Each Continent $log = "Processing Continent Data\n"; print $log; pfb_logger("{$log}","3"); $cont_array = array ( array($AF),array($AS),array($EU),array($NA),array($OC),array($SA),array($AX)); if (($handle = fopen("{$maxmind_cont}",'r')) !== FALSE) { while (($cc = fgetcsv($handle)) !== FALSE) { $cc_key = $cc[0]; $cont_key = $cc[1]; switch ($cont_key) { case "AF": $cont_array[0]['continent'] = "Africa"; $cont_array[0]['iso'] .= "{$cc_key},"; $cont_array[0]['file4'] = "{$pfb['ccdir']}/Africa_v4.txt"; $cont_array[0]['file6'] = "{$pfb['ccdir']}/Africa_v6.txt"; break; case "AS": $cont_array[1]['continent'] = "Asia"; $cont_array[1]['iso'] .= "{$cc_key},"; $cont_array[1]['file4'] = "{$pfb['ccdir']}/Asia_v4.txt"; $cont_array[1]['file6'] = "{$pfb['ccdir']}/Asia_v6.txt"; break; case "EU": $cont_array[2]['continent'] = "Europe"; $cont_array[2]['iso'] .= "{$cc_key},"; $cont_array[2]['file4'] = "{$pfb['ccdir']}/Europe_v4.txt"; $cont_array[2]['file6'] = "{$pfb['ccdir']}/Europe_v6.txt"; break; case "NA": $cont_array[3]['continent'] = "North America"; $cont_array[3]['iso'] .= "{$cc_key},"; $cont_array[3]['file4'] = "{$pfb['ccdir']}/North_America_v4.txt"; $cont_array[3]['file6'] = "{$pfb['ccdir']}/North_America_v6.txt"; break; case "OC": $cont_array[4]['continent'] = "Oceania"; $cont_array[4]['iso'] .= "{$cc_key},"; $cont_array[4]['file4'] = "{$pfb['ccdir']}/Oceania_v4.txt"; $cont_array[4]['file6'] = "{$pfb['ccdir']}/Oceania_v6.txt"; break; case "SA": $cont_array[5]['continent'] = "South America"; $cont_array[5]['iso'] .= "{$cc_key},"; $cont_array[5]['file4'] = "{$pfb['ccdir']}/South_America_v4.txt"; $cont_array[5]['file6'] = "{$pfb['ccdir']}/South_America_v6.txt"; break; } } } unset($cc); fclose($handle); // Add Maxmind Anonymous Proxy and Satellite Providers to array $cont_array[6]['continent'] = "Proxy and Satellite"; $cont_array[6]['iso'] = "A1,A2"; $cont_array[6]['file4'] = "{$pfb['ccdir']}/Proxy_Satellite_v4.txt"; $cont_array[6]['file6'] = "{$pfb['ccdir']}/Proxy_Satellite_v6.txt"; // Collect Country ISO data and sort to Continent arrays (IPv4 and IPv6) foreach (array("4", "6") as $type) { $log = "Processing ISO IPv{$type} Continent/Country Data\n"; print $log; pfb_logger("{$log}","3"); if ($type == "4") { $maxmind_cc = "{$pfb['dbdir']}/GeoIPCountryWhois.csv"; } else { $maxmind_cc = "{$pfb['dbdir']}/GeoIPv6.csv"; } $iptype = "ip{$type}"; $filetype = "file{$type}"; if (($handle = fopen("{$maxmind_cc}",'r')) !== FALSE) { while (($cc = fgetcsv($handle)) !== FALSE) { $cc_key = $cc[4]; $country_key = $cc[5]; $a_cidr = implode(",", ip_range_to_subnet_array_temp($cc[0],$cc[1])); $counter = 0; foreach ($cont_array as $iso) { if (preg_match("/\b$cc_key\b/", $iso['iso'])) { $cont_array[$counter][$cc_key][$iptype] .= $a_cidr . ","; $cont_array[$counter][$cc_key]['country'] = $country_key; continue; } $counter++; } } } unset($cc); fclose($handle); // Build Continent Files $counter = 0; foreach ($cont_array as $iso) { $header = ""; $pfb_file = ""; $iso_key = ""; $header .= "# Generated from MaxMind Inc. on: " . date("m/d/y G:i:s", time()) . "\n"; $header .= "# Continent IPv{$type}: " . $cont_array[$counter]['continent'] . "\n"; $pfb_file = $cont_array[$counter][$filetype]; $iso_key = array_keys($iso); foreach ($iso_key as $key) { if (preg_match("/[A-Z]{2}|A1|A2/", $key)) { $header .= "# Country: " . $iso[$key]['country'] . "\n"; $header .= "# ISO Code: " . $key . "\n"; $header .= "# Total Networks: " . substr_count($iso[$key][$iptype], ",") . "\n"; $header .= str_replace(",", "\n", $iso[$key][$iptype]); $iso[$key][$iptype] = ""; } } $counter++; @file_put_contents($pfb_file, $header, LOCK_EX); } } } // Function to process Continent txt files and create Country ISO files and to Generate GUI XML files. function pfblockerng_get_countries() { global $g,$pfb; $files = array ( "Africa" => "{$pfb['ccdir']}/Africa_v4.txt", "Asia" => "{$pfb['ccdir']}/Asia_v4.txt", "Europe" => "{$pfb['ccdir']}/Europe_v4.txt", "North America" => "{$pfb['ccdir']}/North_America_v4.txt", "Oceania" => "{$pfb['ccdir']}/Oceania_v4.txt", "South America" => "{$pfb['ccdir']}/South_America_v4.txt", "Proxy and Satellite" => "{$pfb['ccdir']}/Proxy_Satellite_v4.txt" ); // Collect Data to generate new continent XML Files. $log = "Building pfBlockerNG XML Files \n"; print $log; pfb_logger("{$log}","3"); foreach ($files as $cont => $file) { // Process the following for IPv4 and IPv6 foreach (array("4", "6") as $type) { $log = "IPv{$type} " . $cont . "\n"; print $log; pfb_logger("{$log}","3"); if ($type == "6") $file = preg_replace("/v4/", "v6", $file); $convert = explode("\n", file_get_contents($file)); $cont_name = preg_replace("/ /", "", $cont); $cont_name_lower = strtolower($cont_name); $active = array("$cont" => ''); $lastkey = count ($convert) - 1; $pfb['complete'] = FALSE; $keycount = 1; $total = 0; foreach ($convert as $line) { if (preg_match("/#/",$line)) { if ($pfb['complete']) { ${'coptions' . $type}[] = $country . '-' . $isocode . ' ('. $total .') ' . ' ' . $isocode . ''; // Only collect IPv4 for Reputation Tab if ($type == "4") $roptions4[] = $country . '-' . $isocode . ' ('. $total .') ' . ' ' . $isocode . ''; // Save ISO data @file_put_contents($pfb['ccdir'] . '/' . $isocode . '_v' . $type . '.txt', $xml_data, LOCK_EX); // Clear variables and restart Continent collection process unset($total, $xml_data); $pfb['complete'] = FALSE; } if (preg_match("/Total Networks: 0/", $line)) { continue;} // Don't Display Countries with Null Data if (preg_match("/Country:\s(.*)/",$line, $matches)) { $country = $matches[1];} if (preg_match("/ISO Code:\s(.*)/",$line, $matches)) { $isocode = $matches[1];} } elseif (!preg_match("/#/",$line)) { $total++; if (!empty($line)) $xml_data .= $line . "\n"; $pfb['complete'] = TRUE; } // Save last EOF ISO IP data if ($keycount == $lastkey) { if (preg_match("/Total Networks: 0/", $line)) { continue;} // Dont Display Countries with Null Data ${'coptions' . $type}[] = $country . '-' . $isocode . ' ('. $total .') ' . ' ' . $isocode . ''; if ($type == "4") $roptions4[] = $country . '-' . $isocode . ' ('. $total .') ' . ' ' . $isocode . ''; @file_put_contents($pfb['ccdir'] . '/' . $isocode . '_v' . $type . '.txt', $xml_data, LOCK_EX); unset($total, $xml_data); } $keycount++; } unset ($ips, $convert); // Sort IP Countries alphabetically and build XML