From a9e757fce0e6e8e15dad63a02472959af52f8ff4 Mon Sep 17 00:00:00 2001 From: jim-p Date: Sat, 1 May 2010 00:00:44 -0400 Subject: Add URL Table patch package for 1.2.3. --- config/urltables/urltables.inc | 18 ++ config/urltables/urltables.patch | 364 +++++++++++++++++++++++++++++++++++++++ config/urltables/urltables.xml | 65 +++++++ pkg_config.7.xml | 12 +- 4 files changed, 458 insertions(+), 1 deletion(-) create mode 100644 config/urltables/urltables.inc create mode 100644 config/urltables/urltables.patch create mode 100644 config/urltables/urltables.xml diff --git a/config/urltables/urltables.inc b/config/urltables/urltables.inc new file mode 100644 index 00000000..a2c372ce --- /dev/null +++ b/config/urltables/urltables.inc @@ -0,0 +1,18 @@ + \ No newline at end of file diff --git a/config/urltables/urltables.patch b/config/urltables/urltables.patch new file mode 100644 index 00000000..bdf16cb9 --- /dev/null +++ b/config/urltables/urltables.patch @@ -0,0 +1,364 @@ +diff --git a/etc/inc/filter.inc b/etc/inc/filter.inc +index ba71d04..17acae3 100644 +--- a/etc/inc/filter.inc ++++ b/etc/inc/filter.inc +@@ -415,7 +415,13 @@ function filter_generate_aliases() { + $extraalias = ""; + $ip = find_interface_ip($alias['address']); + $extraalias = " " . link_ip_to_carp_interface($ip); +- $aliases .= "{$alias['name']} = \"{ {$alias['address']}{$extralias} }\"\n"; ++ if ($alias['type'] != 'urltable') ++ $aliases .= "{$alias['name']} = \"{ {$alias['address']}{$extralias} }\"\n"; ++ else { ++ $urlfn = alias_expand_urltable($alias['name']); ++ if ($urlfn) ++ $aliases .= "table <{$alias['name']}> persist file \"{$urlfn}\"\n"; ++ } + } + } + +@@ -2013,7 +2019,8 @@ function generate_user_filter_rule($rule, $ngcounter) { + } else { + if ($g['debug']) + echo "{$src_table} NOT found in cache...adding\n"; +- $table_cache[$src_table] = $src_table_line; ++ if (strpos($src_table_line, 'http://') === false) ++ $table_cache[$src_table] = $src_table_line; + } + if (isset($dst_table)) + if (isset($table_cache[$dst_table])) { +@@ -2022,7 +2029,8 @@ function generate_user_filter_rule($rule, $ngcounter) { + } else { + if ($g['debug']) + echo "{$dst_table} NOT found in cache...adding\n"; +- $table_cache[$dst_table] = $dst_table_line; ++ if (strpos($dst_table_line, 'http://') === false) ++ $table_cache[$dst_table] = $dst_table_line; + } + + /* exception(s) to a user rules can go here. */ +diff --git a/etc/inc/pfsense-utils.inc b/etc/inc/pfsense-utils.inc +index c45def5..2f0f6fb 100644 +--- a/etc/inc/pfsense-utils.inc ++++ b/etc/inc/pfsense-utils.inc +@@ -3905,4 +3905,47 @@ function update_alias_names_upon_change($section, $subsection, $fielda, $fieldb, + + } + ++function process_alias_urltable($name, $url, $freq, $forceupdate=false) { ++ $aliastable_prefix = "/var/db/aliastables/"; ++ $aliastable_filename = $aliastable_prefix . $name . ".txt"; ++ ++ // Make the aliases directory if it doesn't exist ++ if (!file_exists($aliastable_prefix)) { ++ mkdir($aliastable_prefix); ++ } elseif (!is_dir($aliastable_prefix)) { ++ unlink($aliastable_prefix); ++ mkdir($aliastable_prefix); ++ } ++ ++ // If the file doesn't exist or is older than update_freq days, fetch a new copy. ++ if (!file_exists($aliastable_filename) ++ || ((time() - filemtime($aliastable_filename)) > ($freq * 86400)) ++ || $forceupdate) { ++ ++ // Try to fetch the URL supplied ++ conf_mount_rw(); ++ unlink_if_exists($aliastable_filename . ".tmp"); ++ mwexec("/usr/bin/fetch -q -o " . escapeshellarg($aliastable_filename) . ".tmp " . escapeshellarg($url)); ++ mwexec("/usr/bin/grep -v '^#' " . escapeshellarg($aliastable_filename) . ".tmp > " . escapeshellarg($aliastable_filename)); ++ unlink_if_exists($aliastable_filename . ".tmp"); ++ conf_mount_ro(); ++ if (filesize($aliastable_filename)) { ++ return true; ++ } else { ++ // If it's unfetchable or an empty file, bail ++ return false; ++ } ++ } else { ++ // File exists, and it doesn't need updated. ++ return -1; ++ } ++} ++ ++function is_valid_http_url($url) { ++ $parsed = parse_url($url); ++ if (($parsed['scheme'] == 'http') && (is_fqdn($parsed['host']) || is_ipaddr($parsed['host']))) ++ return true; ++ else ++ return false; ++} + ?> +diff --git a/etc/inc/util.inc b/etc/inc/util.inc +index 61d2e55..efeaf01 100644 +--- a/etc/inc/util.inc ++++ b/etc/inc/util.inc +@@ -487,6 +487,21 @@ function alias_expand_net($name) { + return null; + } + ++function alias_expand_urltable($name) { ++ global $aliastable; ++ $aliastable_prefix = "/var/db/aliastables/"; ++ $aliastable_filename = $aliastable_prefix . $name . ".txt"; ++ ++ if (isset($aliastable[$name]) ++ && (is_valid_http_url($aliastable[$name])) ++ && file_exists($aliastable_filename)) ++ return $aliastable_filename; ++ elseif (process_alias_urltable($name, $aliastable[$name], 0, true)) ++ return $aliastable_filename; ++ else ++ return null; ++} ++ + /* find out whether two subnets overlap */ + function check_subnets_overlap($subnet1, $bits1, $subnet2, $bits2) { + +diff --git a/etc/rc.update_urltables b/etc/rc.update_urltables +new file mode 100755 +index 0000000..506a5b0 +--- /dev/null ++++ b/etc/rc.update_urltables +@@ -0,0 +1,49 @@ ++#!/usr/local/bin/php -q ++ 0) { ++ log_error("{$argv[0]}: Starting up."); ++ ++ if ($argv[1] != "now") { ++ // Wait a little before updating. ++ $wait = mt_rand(5, 60); ++ log_error("{$argv[0]}: Sleeping for {$wait} seconds."); ++ sleep($wait); ++ } ++ ++ log_error("{$argv[0]}: Starting URL table alias updates"); ++ ++ foreach ($todo as $t) { ++ $r = process_alias_urltable($t['name'], $t['url'], $t['freq']); ++ if ($r == 1) { ++ $result = ""; ++ exec("/sbin/pfctl -t " . escapeshellarg($t['name']) . " -T replace -f /var/db/aliastables/" . escapeshellarg($t['name']) . ".txt 2>&1", $result); ++ log_error("{$argv[0]}: Updated {$t['name']} content from {$t['url']}: {$result[0]}"); ++ } elseif ($r == -1) { ++ log_error("{$argv[0]}: {$t['name']} does not need updated."); ++ } else { ++ log_error("{$argv[0]}: ERROR: could not update {$t['name']} content from {$t['url']}"); ++ } ++ } ++} ++?> +\ No newline at end of file +diff --git a/usr/local/www/firewall_aliases.php b/usr/local/www/firewall_aliases.php +index e453200..9be52a4 100755 +--- a/usr/local/www/firewall_aliases.php ++++ b/usr/local/www/firewall_aliases.php +@@ -85,7 +85,7 @@ if ($_GET['act'] == "del") { + $referenced_by = $rule['descr']; + break; + } +- if($rule['source']['address'] == $alias_name) { ++ if($rule['destination']['address'] == $alias_name) { + $is_alias_referenced = true; + $referenced_by = $rule['descr']; + break; +@@ -177,11 +177,10 @@ include("head.inc"); + + + 10) { + echo "..."; + } + ?> +diff --git a/usr/local/www/firewall_aliases_edit.php b/usr/local/www/firewall_aliases_edit.php +index 62e1272..f88b4b3 100755 +--- a/usr/local/www/firewall_aliases_edit.php ++++ b/usr/local/www/firewall_aliases_edit.php +@@ -65,10 +65,14 @@ if (isset($id) && $a_aliases[$id]) { + if ($addresssubnettest) + $pconfig['type'] = "network"; + else +- if (is_ipaddr($address[0])) ++ if (is_ipaddr($address[0])) { + $pconfig['type'] = "host"; +- else ++ } elseif (is_valid_http_url($pconfig['address'])) { ++ $pconfig['type'] = $a_aliases[$id]['type']; ++ $pconfig['address_subnet'] = $a_aliases[$id]['address_subnet']; ++ } else { + $pconfig['type'] = "port"; ++ } + + if($a_aliases[$id]['aliasurl'] <> "") { + $pconfig['type'] = "url"; +@@ -99,6 +103,9 @@ if ($_POST) { + if ($_POST['type'] == "network") { + $reqdfields[] = "address_subnet"; + $reqdfieldsn[] = "Subnet bit count"; ++ } elseif ($_POST['type'] == "urltable") { ++ $reqdfields[] = "address_subnet"; ++ $reqdfieldsn[] = "Update Frequency"; + } + + do_input_validation($_POST, $reqdfields, $reqdfieldsn, &$input_errors); +@@ -131,10 +138,18 @@ if ($_POST) { + } + + if ($_POST['type'] == "url") { +- if(stristr($_POST['address'], "http") == false) ++ if(is_valid_http_url($_POST['address']) == false) + $input_errors[] = "You must provide a valid URL to the resource."; + } + ++ if ($_POST['type'] == "urltable") { ++ if(is_valid_http_url($_POST['address']) == false) ++ $input_errors[] = "You must provide a valid URL to the resource."; ++ if (!is_numeric($_POST['address_subnet'])) { ++ $input_errors[] = "A valid update frequency must be specified."; ++ } ++ } ++ + if ($_POST['type'] == "port") + if (! is_port($_POST['address']) && ! is_portrange($_POST['address'])) + $input_errors[] = "Please specify a valid port or portrange."; +@@ -177,7 +192,25 @@ if ($_POST) { + $final_address_details .= "||"; + $isfirst = 0; + +- if($_POST['type'] == "url") { ++ if ($_POST['type'] == "urltable") { ++ $address = ""; ++ $isfirst = 0; ++ ++ /* item is a url type */ ++ if ($_POST['address']) { ++ /* fetch down and add in */ ++ $isfirst = 0; ++ $address = $_POST['address']; ++ $alias['address_subnet'] = $_POST['address_subnet']; ++ if (strtolower(substr($_POST['address'], 0, 4)) != "http") { ++ $input_errors[] = "You must provide a valid URL."; ++ $dont_update = true; ++ } elseif (! process_alias_urltable($_POST['name'], $_POST['address'], $_POST['address_subnet'], true)) { ++ $input_errors[] = "Unable to fetch usable data."; ++ $dont_update = true; ++ } ++ } ++ } elseif($_POST['type'] == "url") { + $address = ""; + $isfirst = 0; + $address_count = 2; +@@ -355,13 +388,14 @@ $hosts_str = gettext("Host(s)"); + $ip_str = gettext("IP"); + $ports_str = gettext("Port(s)"); + $port_str = gettext("Port"); ++$urltable_str = gettext("URL Table"); + $url_str = gettext("URL"); + $update_freq_str = gettext("Update Freq."); + + $networks_help = gettext("Networks can be expressed like 10.0.0.0 format. Select the CIDR (network mask) that pertains to each entry."); + $hosts_help = gettext("Enter as many hosts as you would like. Hosts should be expressed in their ip address format."); + $ports_help = gettext("Enter as many ports as you wish. Port ranges can be expressed by seperating with a colon."); +-$url_help = gettext("Enter as many urls as you wish. Also set the time that you would like the url refreshed in days. After saving {$g['product_name']} will download the URL and import the items into the alias."); ++$urltable_help = gettext("Enter a single URL and set the time that you would like the url refreshed in days. After saving {$g['product_name']} will download the URL and save the items as a persistent table, which works like the bogons list. (cron job to update is not added automatically)"); + + $jscriptstr .= << +@@ -480,6 +527,7 @@ EOD; + + + ++ + + + +@@ -525,7 +573,7 @@ EOD; + + +@@ -549,8 +597,8 @@ EOD; + + + +- +- ++ + + + diff --git a/config/urltables/urltables.xml b/config/urltables/urltables.xml new file mode 100644 index 00000000..16fe50c3 --- /dev/null +++ b/config/urltables/urltables.xml @@ -0,0 +1,65 @@ + + + + + + . + All rights reserved. + */ +/* ========================================================================== */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + */ +/* ========================================================================== */ + ]]> + + Patch to add URL Table Aliases + pfSense 1.2.3 + None + URL Table Aliases + 0.2 + URL Table Aliases + /usr/local/pkg/urltables.inc + + /usr/local/pkg/ + 077 + http://www.pfsense.com/packages/config/urltables/urltables.inc + + + /usr/local/pkg/ + 077 + http://www.pfsense.com/packages/config/urltables/urltables.patch + + + urltables_install(); + + diff --git a/pkg_config.7.xml b/pkg_config.7.xml index bec34377..644070aa 100755 --- a/pkg_config.7.xml +++ b/pkg_config.7.xml @@ -968,7 +968,7 @@ IP Range Aliases - Patch to add IP Range support to Network Aliases. WARNING! Cannot be uninstalled. + Patch to add IP Range support to Network Aliases. WARNING! Cannot be uninstalled. Probably does not play nice with URL Table patch. System http://www.pfsense.com/packages/config/iprangealiases/iprangealiases.xml 0.2 @@ -976,5 +976,15 @@ 1.2.3 1.2.3 + + URL Table Aliases + Patch to add URL Table alias support to pull a list of CIDRs/IPs from a file by URL into a persist table. WARNING! Cannot be uninstalled. Probably does not play nice with IP Range Aliases patch. This is NOT a way to add an alias for host URLs. + System + http://www.pfsense.com/packages/config/urltables/urltables.xml + 0.1 + BETA + 1.2.3 + 1.2.3 + -- cgit v1.2.3