= $pfb['skipfeed']) { $log = " Max daily download failure attempts exceeded. Clear widget 'failed downloads' to reset.\n\n"; pfb_logger("{$log}", 1); unlink_if_exists("{$pfbfolder}/{$header}.fail"); return; } } // Attempt download, when a previous 'fail' file marker is found. if (file_exists("{$pfbfolder}/{$header}.fail")) { $log = "\t\t\tPrevious download failed.\tRe-attempt download\n"; pfb_logger("{$log}", 1); $pfb['update_cron'] = TRUE; unlink_if_exists("{$pfbfolder}/{$header}.txt"); return; } // Check if List file doesn't exist or Format is 'whois'. if (!file_exists("{$pfbfolder}/{$header}.txt") || $format == 'whois') { $log = "\t\t\t\t\t\t\tUpdate found\n"; pfb_logger("{$log}", 1); $pfb['update_cron'] = TRUE; return; } $host = @parse_url($list_url); $local_file = "{$pfborig}/{$header}.orig"; // Compare previously downloaded file timestamp with remote timestamp if (file_exists($local_file)) { if ($format == 'rsync') { $log = "\t\t\t\t( rsync )\t\tUpdate found\n"; pfb_logger("{$log}", 1); $pfb['update_cron'] = TRUE; unlink_if_exists("{$pfbfolder}/{$header}.txt"); return; } // Determine if URL is Remote or Local if (in_array($host['host'], array('127.0.0.1', $pfb['iplocal'], ''))) { clearstatcache(); $remote_tds = gmdate('D, d M Y H:i:s T', @filemtime($list_url)); } else { // Download URL headers and compare previously downloaded file with remote timestamp if (($ch = curl_init($list_url))) { curl_setopt_array($ch, $pfb['curl_defaults']); // Load curl default settings curl_setopt($ch, CURLOPT_NOBODY, true); // Exclude the body from the output curl_setopt($ch, CURLOPT_TIMEOUT, 60); // Allow downgrade of cURL settings if user configured if ($pflex == 'Flex') { curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false); curl_setopt($ch, CURLOPT_SSL_CIPHER_LIST, 'TLSv1.2, TLSv1, SSLv3'); } // Try up to 3 times to download the file before giving up for ($retries = 1; $retries <= 3; $retries++) { if (curl_exec($ch)) { $remote_stamp_raw = curl_getinfo($ch, CURLINFO_FILETIME); break; // Break on success } sleep(3); } if ($remote_stamp_raw != -1) { $remote_tds = gmdate('D, d M Y H:i:s T', $remote_stamp_raw); } } else { $remote_stamp_raw = -1; } curl_close($ch); } // If remote timestamp not found, Attempt md5 comparison if ($remote_stamp_raw == -1) { // Collect md5 checksums $remote_md5 = @md5_file($list_url); $local_md5 = @md5_file($local_file); if ($remote_md5 != $local_md5) { $log = "\t\t\t\t( md5 changed )\t\tUpdate found\n"; pfb_logger("{$log}", 1); $pfb['update_cron'] = TRUE; unlink_if_exists("{$pfbfolder}/{$header}.txt"); return; } else { $log = "\t( No remote timestamp/md5 unchanged )\t\tUpdate not required\n"; pfb_logger("{$log}", 1); return; } } else { $log = " Remote timestamp: {$remote_tds}\n"; pfb_logger("{$log}", 1); clearstatcache(); $local_tds = gmdate('D, d M Y H:i:s T', @filemtime($local_file)); $log = " Local timestamp: {$local_tds}\t"; pfb_logger("{$log}", 1); if ("{$remote_tds}" != "{$local_tds}") { $pfb['cron_update'] = TRUE; } else { $log = "Update not required\n"; pfb_logger("{$log}", 1); $pfb['cron_update'] = FALSE; } } } else { $pfb['cron_update'] = TRUE; } if ($pfb['cron_update']) { // Trigger CRON process if updates are found. $pfb['update_cron'] = TRUE; $log = "Update found\n"; pfb_logger("{$log}", 1); unlink_if_exists("{$pfbfolder}/{$header}.txt"); } return; } // Download Extras - MaxMind/Alexa feeds via cURL function pfblockerng_download_extras($timeout=600) { global $pfb; $pfberror = FALSE; pfb_logger("\nDownload Process Starting [ NOW ]\n", 3); foreach ($pfb['extras'] as $feed) { $file_dwn = "{$feed['folder']}/{$feed['file_dwn']}"; if (!pfb_download($feed['url'], $file_dwn, FALSE, "{$feed['folder']}/{$feed['file']}", '', 3, '', $timeout)) { $log = "\nFailed to Download {$feed['file']}\n"; pfb_logger("{$log}", 3); // On install, if error found when downloading MaxMind Continent lists // Return error to install process to download archive from pfSense package repo if ($feed['install']) { $pfberror = TRUE; } } } pfb_logger("Download Process Ended [ NOW ]\n\n", 3); if ($pfberror) { return FALSE; } else { return TRUE; } } // Function to update Lists/Feeds as per Cron function pfblockerng_sync_cron() { global $config, $pfb, $pfbarr; // Call base hour converter $pfb_sch = pfb_cron_base_hour(); $hour = date('G'); $dow = date('N'); $pfb['update_cron'] = FALSE; $log = " CRON PROCESS START [ NOW ]\n"; pfb_logger("{$log}", 1); $list_type = array('pfblockernglistsv4' => '_v4', 'pfblockernglistsv6' => '_v6', 'pfblockerngdnsbl' => '_v4', 'pfblockerngdnsbleasylist' => '_v4'); foreach ($list_type as $ip_type => $vtype) { if (!empty($config['installedpackages'][$ip_type]['config'])) { foreach ($config['installedpackages'][$ip_type]['config'] as $list) { if (isset($list['row']) && $list['action'] != 'Disabled' && $list['cron'] != 'Never') { foreach ($list['row'] as $row) { if (!empty($row['url']) && $row['state'] != 'Disabled') { if ($vtype == '_v4') { $header = "{$row['header']}"; } else { $header = "{$row['header']}_v6"; } // Determine folder location for alias (return array $pfbarr) pfb_determine_list_detail($list['action'], '', '', ''); $pfbfolder = $pfbarr['folder']; $pfborig = $pfbarr['orig']; // Bypass update if state is defined as 'Hold' and list file exists if ($row['state'] == 'Hold' && file_exists("{$pfbfolder}/{$header}.txt")) { continue; } // Allow cURL SSL downgrade if user configured. $pflex = FALSE; if ($row['state'] == 'Flex') { $pflex = TRUE; } switch ($list['cron']) { case 'EveryDay': if ($hour == $pfb['24hour']) { pfb_update_check($header, $row['url'], $pfbfolder, $pfborig, $pflex, $row['format']); } break; case 'Weekly': if ($hour == $pfb['24hour'] && $dow == $list['dow']) { pfb_update_check($header, $row['url'], $pfbfolder, $pfborig, $pflex, $row['format']); } break; default: if ($pfb['interval'] == '1' || in_array($hour, $pfb_sch)) { pfb_update_check($header, $row['url'], $pfbfolder, $pfborig, $pflex, $row['format']); } break; } } } } } } } // If no lists require updates, check if Continents are configured and update accordingly. if (!$pfb['update_cron']) { foreach ($pfb['continents'] as $continent => $pfb_alias) { if (isset($config['installedpackages']['pfblockerng' . strtolower(str_replace(' ', '', $continent))]['config'])) { $continent_config = $config['installedpackages']['pfblockerng' . strtolower(str_replace(' ', '', $continent))]['config'][0]; if ($continent_config['action'] != 'Disabled') { $pfb['update_cron'] = TRUE; break; } } } } if ($pfb['update_cron']) { sync_package_pfblockerng('cron'); $pfb['update_cron'] = FALSE; } else { sync_package_pfblockerng('noupdates'); $log = "\n No Updates required.\n CRON PROCESS ENDED\n UPDATE PROCESS ENDED\n"; pfb_logger("{$log}", 1); } // Call log mgmt function // If Update GUI 'Manual view' is selected. Last output will be missed. So sleep for 5 secs. sleep(5); pfb_log_mgmt(); } // Function to process the downloaded MaxMind database and format into Continent txt files. function pfblockerng_uc_countries() { global $g, $pfb; $maxmind_cont = "{$pfb['geoipshare']}/country_continent.csv"; $maxmind_cc4 = "{$pfb['geoipshare']}/GeoIPCountryWhois.csv"; $maxmind_cc6 = "{$pfb['geoipshare']}/GeoIPv6.csv"; // Create folders if not exist $folder_array = array ("{$pfb['dbdir']}", "{$pfb['logdir']}", "{$pfb['ccdir']}"); foreach ($folder_array as $folder) { safe_mkdir ("{$folder}", 0755); } $now = date('m/d/y G:i:s', time()); $log = "Country code update Start [ NOW ]\n"; if (!$g['pfblockerng_install']) { print "Country code update Start [ $now ]\n"; } pfb_logger("{$log}", 3); if (!file_exists($maxmind_cont) || !file_exists($maxmind_cc4) || !file_exists($maxmind_cc6)) { $log = " [ MAXMIND UPDATE FAIL, CSV missing, using previous Country code database \n"; if (!$g['pfblockerng_install']) { print $log; } pfb_logger("{$log}", 3); return; } // Save Date/Time stamp to MaxMind version file $local_tds4 = @gmdate('D, d M Y H:i:s T', @filemtime($maxmind_cc4)); $local_tds6 = @gmdate('D, d M Y H:i:s T', @filemtime($maxmind_cc6)); $maxmind_ver = "MaxMind GeoLite Date/Time Stamps\n"; $maxmind_ver .= "Local_v4 \tLast-Modified: {$local_tds4}\n"; $maxmind_ver .= "Local_v6 \tLast-Modified: {$local_tds6}\n"; @file_put_contents("{$pfb['logdir']}/maxmind_ver", $maxmind_ver, LOCK_EX); // Collect ISO codes for each Continent $log = " Processing Continent Data\n"; if (!$g['pfblockerng_install']) { print $log; } pfb_logger("{$log}", 3); $cont_array = array(); if (($handle = fopen("{$maxmind_cont}", 'r')) !== FALSE) { while (($cc = fgetcsv($handle)) !== FALSE) { $cc_key = $cc[0]; $cont_key = $cc[1]; switch ($cont_key) { case 'AF': $cont_array[0]['continent'] = 'Africa'; $cont_array[0]['iso'] .= "{$cc_key},"; $cont_array[0]['file4'] = "{$pfb['ccdir']}/Africa_v4.txt"; $cont_array[0]['file6'] = "{$pfb['ccdir']}/Africa_v6.txt"; break; case 'AS': $cont_array[1]['continent'] = 'Asia'; $cont_array[1]['iso'] .= "{$cc_key},"; $cont_array[1]['file4'] = "{$pfb['ccdir']}/Asia_v4.txt"; $cont_array[1]['file6'] = "{$pfb['ccdir']}/Asia_v6.txt"; break; case 'EU': $cont_array[2]['continent'] = 'Europe'; $cont_array[2]['iso'] .= "{$cc_key},"; $cont_array[2]['file4'] = "{$pfb['ccdir']}/Europe_v4.txt"; $cont_array[2]['file6'] = "{$pfb['ccdir']}/Europe_v6.txt"; break; case 'NA': $cont_array[3]['continent'] = 'North America'; $cont_array[3]['iso'] .= "{$cc_key},"; $cont_array[3]['file4'] = "{$pfb['ccdir']}/North_America_v4.txt"; $cont_array[3]['file6'] = "{$pfb['ccdir']}/North_America_v6.txt"; break; case 'OC': $cont_array[4]['continent'] = 'Oceania'; $cont_array[4]['iso'] .= "{$cc_key},"; $cont_array[4]['file4'] = "{$pfb['ccdir']}/Oceania_v4.txt"; $cont_array[4]['file6'] = "{$pfb['ccdir']}/Oceania_v6.txt"; break; case 'SA': $cont_array[5]['continent'] = 'South America'; $cont_array[5]['iso'] .= "{$cc_key},"; $cont_array[5]['file4'] = "{$pfb['ccdir']}/South_America_v4.txt"; $cont_array[5]['file6'] = "{$pfb['ccdir']}/South_America_v6.txt"; break; } } } unset($cc); fclose($handle); // Add Maxmind Anonymous Proxy and Satellite Providers to array $cont_array[6]['continent'] = 'Proxy and Satellite'; $cont_array[6]['iso'] = 'A1,A2'; $cont_array[6]['file4'] = "{$pfb['ccdir']}/Proxy_Satellite_v4.txt"; $cont_array[6]['file6'] = "{$pfb['ccdir']}/Proxy_Satellite_v6.txt"; sort($cont_array); // Collect Country ISO data and sort to Continent arrays (IPv4 and IPv6) foreach (array('4', '6') as $type) { $log = " Processing ISO IPv{$type} Continent/Country Data\n"; if (!$g['pfblockerng_install']) { print $log; } pfb_logger("{$log}", 3); if ($type == '4') { $maxmind_cc = "{$pfb['geoipshare']}/GeoIPCountryWhois.csv"; } else { $maxmind_cc = "{$pfb['geoipshare']}/GeoIPv6.csv"; } $iptype = "ip{$type}"; $filetype = "file{$type}"; if (($handle = fopen("{$maxmind_cc}", 'r')) !== FALSE) { while (($cc = fgetcsv($handle)) !== FALSE) { $cc_key = $cc[4]; $country_key = $cc[5]; $a_cidr = implode(',', ip_range_to_subnet_array_temp($cc[0], $cc[1])); foreach ($cont_array as $key => $iso) { if (strpos($iso['iso'], $cc_key) !== FALSE) { $cont_array[$key][$cc_key][$iptype] .= "{$a_cidr},"; $cont_array[$key][$cc_key]['country'] = $country_key; continue; } } } } unset($cc); fclose($handle); // Build Continent files foreach ($cont_array as $key => $iso) { $header = $pfb_file = $iso_key = ''; $header .= '# Generated from MaxMind Inc. on: ' . date('m/d/y G:i:s', time()) . "\n"; $header .= "# Continent IPv{$type}: {$cont_array[$key]['continent']}\n"; $pfb_file = $cont_array[$key][$filetype]; $iso_key = array_keys($iso); foreach ($iso_key as $ikey) { if (strlen($ikey) == 2) { $header .= "# Country: {$iso[$ikey]['country']}\n"; $header .= "# ISO Code: {$ikey}\n"; $header .= '# Total Networks: ' . substr_count($iso[$ikey][$iptype], ',') . "\n"; $header .= str_replace(',', "\n", $iso[$ikey][$iptype]); $iso[$ikey][$iptype] = ''; } } @file_put_contents($pfb_file, $header, LOCK_EX); } } } // Function to process Continent txt files and create Country ISO files and to Generate GUI XML files. function pfblockerng_get_countries() { global $g, $pfb; $files = array ( 'Africa' => "{$pfb['ccdir']}/Africa_v4.txt", 'Asia' => "{$pfb['ccdir']}/Asia_v4.txt", 'Europe' => "{$pfb['ccdir']}/Europe_v4.txt", 'North America' => "{$pfb['ccdir']}/North_America_v4.txt", 'Oceania' => "{$pfb['ccdir']}/Oceania_v4.txt", 'South America' => "{$pfb['ccdir']}/South_America_v4.txt", 'Proxy and Satellite' => "{$pfb['ccdir']}/Proxy_Satellite_v4.txt" ); // Collect data to generate new continent XML files. $log = " Building pfBlockerNG XML Files \n"; if (!$g['pfblockerng_install']) { print $log; } pfb_logger("{$log}", 3); foreach ($files as $cont => $file) { // Process the following for IPv4 and IPv6 foreach (array('4', '6') as $type) { $log = " IPv{$type} {$cont}\n"; if (!$g['pfblockerng_install']) { print $log; } pfb_logger("{$log}", 3); if ($type == '6') { $file = str_replace('v4', 'v6', $file); } $convert = explode("\n", file_get_contents($file)); $cont_name = str_replace(' ', '', $cont); $cont_name_lower = strtolower($cont_name); $active = array("$cont" => ''); $lastkey = count($convert) - 1; $pfb['complete'] = FALSE; $keycount = 1; $total = 0; $xml_data = ''; foreach ($convert as $line) { if (substr($line, 0, 1) == '#') { if ($pfb['complete']) { ${'coptions' . $type}[] = "{$country}-{$isocode} ({$total}){$isocode}"; // Only collect IPv4 for Reputation Tab if ($type == '4') { $roptions4[] = "{$country}-{$isocode} ({$total}){$isocode}"; } // Save ISO data @file_put_contents("{$pfb['ccdir']}/{$isocode}_v{$type}.txt", $xml_data, LOCK_EX); // Clear variables and restart Continent collection process unset($total, $xml_data); $pfb['complete'] = FALSE; } // Don't collect Countries with null data if (strpos($line, 'Total Networks: 0') !== FALSE) { continue; } if (strpos($line, 'Country: ') !== FALSE) { $country = str_replace('# Country: ', '', $line); } if (strpos($line, 'ISO Code: ') !== FALSE) { $isocode = str_replace('# ISO Code: ', '', $line); } } elseif (substr($line, 0, 1) != '#') { $total++; if (!empty($line)) { $xml_data .= "{$line}\n"; } $pfb['complete'] = TRUE; } // Save last EOF ISO IP data if ($keycount == $lastkey) { // Don't collect Countries with null data if (strpos($line, 'Total Networks: 0') !== FALSE) { continue; } ${'coptions' . $type}[] = "{$country}-{$isocode} ({$total}){$isocode}"; if ($type == '4') { $roptions4[] = "{$country}-{$isocode} ({$total}){$isocode}"; } @file_put_contents("{$pfb['ccdir']}/{$isocode}_v{$type}.txt", $xml_data, LOCK_EX); unset($total, $xml_data); } $keycount++; } unset ($ips, $convert); // Sort IP Countries alphabetically and build XML