= $pfb['skipfeed']) {
$log = " Max daily download failure attempts exceeded. Clear widget 'failed downloads' to reset.\n\n";
pfb_logger("{$log}", 1);
unlink_if_exists("{$pfbfolder}/{$header}.fail");
return;
}
}
// Attempt download, when a previous 'fail' file marker is found.
if (file_exists("{$pfbfolder}/{$header}.fail")) {
$log = "\t\t\tPrevious download failed.\tRe-attempt download\n";
pfb_logger("{$log}", 1);
$pfb['update_cron'] = TRUE;
unlink_if_exists("{$pfbfolder}/{$header}.txt");
return;
}
// Check if List file doesn't exist or Format is 'whois'.
if (!file_exists("{$pfbfolder}/{$header}.txt") || $format == 'whois') {
$log = "\t\t\t\t\t\t\tUpdate found\n";
pfb_logger("{$log}", 1);
$pfb['update_cron'] = TRUE;
return;
}
$host = @parse_url($list_url);
$local_file = "{$pfborig}/{$header}.orig";
// Compare previously downloaded file timestamp with remote timestamp
if (file_exists($local_file)) {
if ($format == 'rsync') {
$log = "\t\t\t\t( rsync )\t\tUpdate found\n";
pfb_logger("{$log}", 1);
$pfb['update_cron'] = TRUE;
unlink_if_exists("{$pfbfolder}/{$header}.txt");
return;
}
// Determine if URL is Remote or Local
if (in_array($host['host'], array('127.0.0.1', $pfb['iplocal'], ''))) {
clearstatcache();
$remote_tds = gmdate('D, d M Y H:i:s T', @filemtime($list_url));
}
else {
// Download URL headers and compare previously downloaded file with remote timestamp
if (($ch = curl_init($list_url))) {
curl_setopt_array($ch, $pfb['curl_defaults']); // Load curl default settings
curl_setopt($ch, CURLOPT_NOBODY, true); // Exclude the body from the output
curl_setopt($ch, CURLOPT_TIMEOUT, 60);
// Allow downgrade of cURL settings if user configured
if ($pflex == 'Flex') {
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_SSL_CIPHER_LIST, 'TLSv1.2, TLSv1, SSLv3');
}
// Try up to 3 times to download the file before giving up
for ($retries = 1; $retries <= 3; $retries++) {
if (curl_exec($ch)) {
$remote_stamp_raw = curl_getinfo($ch, CURLINFO_FILETIME);
break; // Break on success
}
sleep(3);
}
if ($remote_stamp_raw != -1) {
$remote_tds = gmdate('D, d M Y H:i:s T', $remote_stamp_raw);
}
}
else {
$remote_stamp_raw = -1;
}
curl_close($ch);
}
// If remote timestamp not found, Attempt md5 comparison
if ($remote_stamp_raw == -1) {
// Collect md5 checksums
$remote_md5 = @md5_file($list_url);
$local_md5 = @md5_file($local_file);
if ($remote_md5 != $local_md5) {
$log = "\t\t\t\t( md5 changed )\t\tUpdate found\n";
pfb_logger("{$log}", 1);
$pfb['update_cron'] = TRUE;
unlink_if_exists("{$pfbfolder}/{$header}.txt");
return;
}
else {
$log = "\t( No remote timestamp/md5 unchanged )\t\tUpdate not required\n";
pfb_logger("{$log}", 1);
return;
}
}
else {
$log = " Remote timestamp: {$remote_tds}\n";
pfb_logger("{$log}", 1);
clearstatcache();
$local_tds = gmdate('D, d M Y H:i:s T', @filemtime($local_file));
$log = " Local timestamp: {$local_tds}\t";
pfb_logger("{$log}", 1);
if ("{$remote_tds}" != "{$local_tds}") {
$pfb['cron_update'] = TRUE;
}
else {
$log = "Update not required\n";
pfb_logger("{$log}", 1);
$pfb['cron_update'] = FALSE;
}
}
} else {
$pfb['cron_update'] = TRUE;
}
if ($pfb['cron_update']) {
// Trigger CRON process if updates are found.
$pfb['update_cron'] = TRUE;
$log = "Update found\n";
pfb_logger("{$log}", 1);
unlink_if_exists("{$pfbfolder}/{$header}.txt");
}
return;
}
// Download Extras - MaxMind/Alexa feeds via cURL
function pfblockerng_download_extras($timeout=600) {
global $pfb;
$pfberror = FALSE;
pfb_logger("\nDownload Process Starting [ NOW ]\n", 3);
foreach ($pfb['extras'] as $feed) {
$file_dwn = "{$feed['folder']}/{$feed['file_dwn']}";
if (!pfb_download($feed['url'], $file_dwn, FALSE, "{$feed['folder']}/{$feed['file']}", '', 3, '', $timeout)) {
$log = "\nFailed to Download {$feed['file']}\n";
pfb_logger("{$log}", 3);
// On install, if error found when downloading MaxMind Continent lists
// Return error to install process to download archive from pfSense package repo
if ($feed['install']) {
$pfberror = TRUE;
}
}
}
pfb_logger("Download Process Ended [ NOW ]\n\n", 3);
if ($pfberror) {
return FALSE;
} else {
return TRUE;
}
}
// Function to update Lists/Feeds as per Cron
function pfblockerng_sync_cron() {
global $config, $pfb, $pfbarr;
// Call base hour converter
$pfb_sch = pfb_cron_base_hour();
$hour = date('G');
$dow = date('N');
$pfb['update_cron'] = FALSE;
$log = " CRON PROCESS START [ NOW ]\n";
pfb_logger("{$log}", 1);
$list_type = array('pfblockernglistsv4' => '_v4', 'pfblockernglistsv6' => '_v6', 'pfblockerngdnsbl' => '_v4', 'pfblockerngdnsbleasylist' => '_v4');
foreach ($list_type as $ip_type => $vtype) {
if (!empty($config['installedpackages'][$ip_type]['config'])) {
foreach ($config['installedpackages'][$ip_type]['config'] as $list) {
if (isset($list['row']) && $list['action'] != 'Disabled' && $list['cron'] != 'Never') {
foreach ($list['row'] as $row) {
if (!empty($row['url']) && $row['state'] != 'Disabled') {
if ($vtype == '_v4') {
$header = "{$row['header']}";
} else {
$header = "{$row['header']}_v6";
}
// Determine folder location for alias (return array $pfbarr)
pfb_determine_list_detail($list['action'], '', '', '');
$pfbfolder = $pfbarr['folder'];
$pfborig = $pfbarr['orig'];
// Bypass update if state is defined as 'Hold' and list file exists
if ($row['state'] == 'Hold' && file_exists("{$pfbfolder}/{$header}.txt")) {
continue;
}
// Allow cURL SSL downgrade if user configured.
$pflex = FALSE;
if ($row['state'] == 'Flex') {
$pflex = TRUE;
}
switch ($list['cron']) {
case 'EveryDay':
if ($hour == $pfb['24hour']) {
pfb_update_check($header, $row['url'], $pfbfolder, $pfborig, $pflex, $row['format']);
}
break;
case 'Weekly':
if ($hour == $pfb['24hour'] && $dow == $list['dow']) {
pfb_update_check($header, $row['url'], $pfbfolder, $pfborig, $pflex, $row['format']);
}
break;
default:
if ($pfb['interval'] == '1' || in_array($hour, $pfb_sch)) {
pfb_update_check($header, $row['url'], $pfbfolder, $pfborig, $pflex, $row['format']);
}
break;
}
}
}
}
}
}
}
// If no lists require updates, check if Continents are configured and update accordingly.
if (!$pfb['update_cron']) {
foreach ($pfb['continents'] as $continent => $pfb_alias) {
if (isset($config['installedpackages']['pfblockerng' . strtolower(str_replace(' ', '', $continent))]['config'])) {
$continent_config = $config['installedpackages']['pfblockerng' . strtolower(str_replace(' ', '', $continent))]['config'][0];
if ($continent_config['action'] != 'Disabled') {
$pfb['update_cron'] = TRUE;
break;
}
}
}
}
if ($pfb['update_cron']) {
sync_package_pfblockerng('cron');
$pfb['update_cron'] = FALSE;
} else {
sync_package_pfblockerng('noupdates');
$log = "\n No Updates required.\n CRON PROCESS ENDED\n UPDATE PROCESS ENDED\n";
pfb_logger("{$log}", 1);
}
// Call log mgmt function
// If Update GUI 'Manual view' is selected. Last output will be missed. So sleep for 5 secs.
sleep(5);
pfb_log_mgmt();
}
// Function to process the downloaded MaxMind database and format into Continent txt files.
function pfblockerng_uc_countries() {
global $g, $pfb;
$maxmind_cont = "{$pfb['geoipshare']}/country_continent.csv";
$maxmind_cc4 = "{$pfb['geoipshare']}/GeoIPCountryWhois.csv";
$maxmind_cc6 = "{$pfb['geoipshare']}/GeoIPv6.csv";
// Create folders if not exist
$folder_array = array ("{$pfb['dbdir']}", "{$pfb['logdir']}", "{$pfb['ccdir']}");
foreach ($folder_array as $folder) {
safe_mkdir ("{$folder}", 0755);
}
$now = date('m/d/y G:i:s', time());
$log = "Country code update Start [ NOW ]\n";
if (!$g['pfblockerng_install']) {
print "Country code update Start [ $now ]\n";
}
pfb_logger("{$log}", 3);
if (!file_exists($maxmind_cont) || !file_exists($maxmind_cc4) || !file_exists($maxmind_cc6)) {
$log = " [ MAXMIND UPDATE FAIL, CSV missing, using previous Country code database \n";
if (!$g['pfblockerng_install']) {
print $log;
}
pfb_logger("{$log}", 3);
return;
}
// Save Date/Time stamp to MaxMind version file
$local_tds4 = @gmdate('D, d M Y H:i:s T', @filemtime($maxmind_cc4));
$local_tds6 = @gmdate('D, d M Y H:i:s T', @filemtime($maxmind_cc6));
$maxmind_ver = "MaxMind GeoLite Date/Time Stamps\n";
$maxmind_ver .= "Local_v4 \tLast-Modified: {$local_tds4}\n";
$maxmind_ver .= "Local_v6 \tLast-Modified: {$local_tds6}\n";
@file_put_contents("{$pfb['logdir']}/maxmind_ver", $maxmind_ver, LOCK_EX);
// Collect ISO codes for each Continent
$log = " Processing Continent Data\n";
if (!$g['pfblockerng_install']) {
print $log;
}
pfb_logger("{$log}", 3);
$cont_array = array();
if (($handle = fopen("{$maxmind_cont}", 'r')) !== FALSE) {
while (($cc = fgetcsv($handle)) !== FALSE) {
$cc_key = $cc[0];
$cont_key = $cc[1];
switch ($cont_key) {
case 'AF':
$cont_array[0]['continent'] = 'Africa';
$cont_array[0]['iso'] .= "{$cc_key},";
$cont_array[0]['file4'] = "{$pfb['ccdir']}/Africa_v4.txt";
$cont_array[0]['file6'] = "{$pfb['ccdir']}/Africa_v6.txt";
break;
case 'AS':
$cont_array[1]['continent'] = 'Asia';
$cont_array[1]['iso'] .= "{$cc_key},";
$cont_array[1]['file4'] = "{$pfb['ccdir']}/Asia_v4.txt";
$cont_array[1]['file6'] = "{$pfb['ccdir']}/Asia_v6.txt";
break;
case 'EU':
$cont_array[2]['continent'] = 'Europe';
$cont_array[2]['iso'] .= "{$cc_key},";
$cont_array[2]['file4'] = "{$pfb['ccdir']}/Europe_v4.txt";
$cont_array[2]['file6'] = "{$pfb['ccdir']}/Europe_v6.txt";
break;
case 'NA':
$cont_array[3]['continent'] = 'North America';
$cont_array[3]['iso'] .= "{$cc_key},";
$cont_array[3]['file4'] = "{$pfb['ccdir']}/North_America_v4.txt";
$cont_array[3]['file6'] = "{$pfb['ccdir']}/North_America_v6.txt";
break;
case 'OC':
$cont_array[4]['continent'] = 'Oceania';
$cont_array[4]['iso'] .= "{$cc_key},";
$cont_array[4]['file4'] = "{$pfb['ccdir']}/Oceania_v4.txt";
$cont_array[4]['file6'] = "{$pfb['ccdir']}/Oceania_v6.txt";
break;
case 'SA':
$cont_array[5]['continent'] = 'South America';
$cont_array[5]['iso'] .= "{$cc_key},";
$cont_array[5]['file4'] = "{$pfb['ccdir']}/South_America_v4.txt";
$cont_array[5]['file6'] = "{$pfb['ccdir']}/South_America_v6.txt";
break;
}
}
}
unset($cc);
fclose($handle);
// Add Maxmind Anonymous Proxy and Satellite Providers to array
$cont_array[6]['continent'] = 'Proxy and Satellite';
$cont_array[6]['iso'] = 'A1,A2';
$cont_array[6]['file4'] = "{$pfb['ccdir']}/Proxy_Satellite_v4.txt";
$cont_array[6]['file6'] = "{$pfb['ccdir']}/Proxy_Satellite_v6.txt";
sort($cont_array);
// Collect Country ISO data and sort to Continent arrays (IPv4 and IPv6)
foreach (array('4', '6') as $type) {
$log = " Processing ISO IPv{$type} Continent/Country Data\n";
if (!$g['pfblockerng_install']) {
print $log;
}
pfb_logger("{$log}", 3);
if ($type == '4') {
$maxmind_cc = "{$pfb['geoipshare']}/GeoIPCountryWhois.csv";
} else {
$maxmind_cc = "{$pfb['geoipshare']}/GeoIPv6.csv";
}
$iptype = "ip{$type}";
$filetype = "file{$type}";
if (($handle = fopen("{$maxmind_cc}", 'r')) !== FALSE) {
while (($cc = fgetcsv($handle)) !== FALSE) {
$cc_key = $cc[4];
$country_key = $cc[5];
$a_cidr = implode(',', ip_range_to_subnet_array_temp($cc[0], $cc[1]));
foreach ($cont_array as $key => $iso) {
if (strpos($iso['iso'], $cc_key) !== FALSE) {
$cont_array[$key][$cc_key][$iptype] .= "{$a_cidr},";
$cont_array[$key][$cc_key]['country'] = $country_key;
continue;
}
}
}
}
unset($cc);
fclose($handle);
// Build Continent files
foreach ($cont_array as $key => $iso) {
$header = $pfb_file = $iso_key = '';
$header .= '# Generated from MaxMind Inc. on: ' . date('m/d/y G:i:s', time()) . "\n";
$header .= "# Continent IPv{$type}: {$cont_array[$key]['continent']}\n";
$pfb_file = $cont_array[$key][$filetype];
$iso_key = array_keys($iso);
foreach ($iso_key as $ikey) {
if (strlen($ikey) == 2) {
$header .= "# Country: {$iso[$ikey]['country']}\n";
$header .= "# ISO Code: {$ikey}\n";
$header .= '# Total Networks: ' . substr_count($iso[$ikey][$iptype], ',') . "\n";
$header .= str_replace(',', "\n", $iso[$ikey][$iptype]);
$iso[$ikey][$iptype] = '';
}
}
@file_put_contents($pfb_file, $header, LOCK_EX);
}
}
}
// Function to process Continent txt files and create Country ISO files and to Generate GUI XML files.
function pfblockerng_get_countries() {
global $g, $pfb;
$files = array ( 'Africa' => "{$pfb['ccdir']}/Africa_v4.txt",
'Asia' => "{$pfb['ccdir']}/Asia_v4.txt",
'Europe' => "{$pfb['ccdir']}/Europe_v4.txt",
'North America' => "{$pfb['ccdir']}/North_America_v4.txt",
'Oceania' => "{$pfb['ccdir']}/Oceania_v4.txt",
'South America' => "{$pfb['ccdir']}/South_America_v4.txt",
'Proxy and Satellite' => "{$pfb['ccdir']}/Proxy_Satellite_v4.txt"
);
// Collect data to generate new continent XML files.
$log = " Building pfBlockerNG XML Files \n";
if (!$g['pfblockerng_install']) {
print $log;
}
pfb_logger("{$log}", 3);
foreach ($files as $cont => $file) {
// Process the following for IPv4 and IPv6
foreach (array('4', '6') as $type) {
$log = " IPv{$type} {$cont}\n";
if (!$g['pfblockerng_install']) {
print $log;
}
pfb_logger("{$log}", 3);
if ($type == '6') {
$file = str_replace('v4', 'v6', $file);
}
$convert = explode("\n", file_get_contents($file));
$cont_name = str_replace(' ', '', $cont);
$cont_name_lower = strtolower($cont_name);
$active = array("$cont" => '');
$lastkey = count($convert) - 1;
$pfb['complete'] = FALSE;
$keycount = 1;
$total = 0;
$xml_data = '';
foreach ($convert as $line) {
if (substr($line, 0, 1) == '#') {
if ($pfb['complete']) {
${'coptions' . $type}[] = "{$country}-{$isocode} ({$total}){$isocode}";
// Only collect IPv4 for Reputation Tab
if ($type == '4') {
$roptions4[] = "{$country}-{$isocode} ({$total}){$isocode}";
}
// Save ISO data
@file_put_contents("{$pfb['ccdir']}/{$isocode}_v{$type}.txt", $xml_data, LOCK_EX);
// Clear variables and restart Continent collection process
unset($total, $xml_data);
$pfb['complete'] = FALSE;
}
// Don't collect Countries with null data
if (strpos($line, 'Total Networks: 0') !== FALSE) {
continue;
}
if (strpos($line, 'Country: ') !== FALSE) {
$country = str_replace('# Country: ', '', $line);
}
if (strpos($line, 'ISO Code: ') !== FALSE) {
$isocode = str_replace('# ISO Code: ', '', $line);
}
}
elseif (substr($line, 0, 1) != '#') {
$total++;
if (!empty($line)) {
$xml_data .= "{$line}\n";
}
$pfb['complete'] = TRUE;
}
// Save last EOF ISO IP data
if ($keycount == $lastkey) {
// Don't collect Countries with null data
if (strpos($line, 'Total Networks: 0') !== FALSE) {
continue;
}
${'coptions' . $type}[] = "{$country}-{$isocode} ({$total}){$isocode}";
if ($type == '4') {
$roptions4[] = "{$country}-{$isocode} ({$total}){$isocode}";
}
@file_put_contents("{$pfb['ccdir']}/{$isocode}_v{$type}.txt", $xml_data, LOCK_EX);
unset($total, $xml_data);
}
$keycount++;
}
unset ($ips, $convert);
// Sort IP Countries alphabetically and build XML