Add URL Table patch package for 1.2.3.

4 files changed, 458 insertions, 1 deletions

diff --git a/config/urltables/urltables.inc b/config/urltables/urltables.inc
new file mode 100644
index 00000000..a2c372ce
--- /dev/null
+++ b/config/urltables/urltables.inc
@@ -0,0 +1,18 @@
+<?php
+
+function urltables_install() {
+	global $g, $config;
+	
+	// Test to make sure the patch is not already applied. 
+	$out = `patch -fslC --reverse -p1 -b .before_urltables -d / -i /usr/local/pkg/urltables.patch |& grep -ci reject`;
+	if ($out == 0) {
+		// If the patch has not already been applied, test to see if it will apply cleanly.
+		$out = `patch -fsNlC -p1 -b .before_urltables -d / -i /usr/local/pkg/urltables.patch |& grep -ci reject`;
+		if ($out == 0) {
+			// The patch should apply cleanly, let 'er rip.
+			mwexec("patch -fsNl -p1 -b .before_urltables -d / -i /usr/local/pkg/urltables.patch ");
+		}
+	}
+}
+
+?>
\ No newline at end of file
diff --git a/config/urltables/urltables.patch b/config/urltables/urltables.patch
new file mode 100644
index 00000000..bdf16cb9
--- /dev/null
+++ b/config/urltables/urltables.patch
@@ -0,0 +1,364 @@
+diff --git a/etc/inc/filter.inc b/etc/inc/filter.inc
+index ba71d04..17acae3 100644
+--- a/etc/inc/filter.inc
++++ b/etc/inc/filter.inc
+@@ -415,7 +415,13 @@ function filter_generate_aliases() {
+ 			$extraalias = "";
+ 			$ip = find_interface_ip($alias['address']);
+ 			$extraalias = " " . link_ip_to_carp_interface($ip);
+-			$aliases .= "{$alias['name']} = \"{ {$alias['address']}{$extralias} }\"\n";
++			if ($alias['type'] != 'urltable')
++				$aliases .= "{$alias['name']} = \"{ {$alias['address']}{$extralias} }\"\n";
++			else {
++				$urlfn = alias_expand_urltable($alias['name']);
++				if ($urlfn)
++					$aliases .= "table <{$alias['name']}> persist file \"{$urlfn}\"\n";
++			}
+ 		}
+ 	}
+ 
+@@ -2013,7 +2019,8 @@ function generate_user_filter_rule($rule, $ngcounter) {
+ 		} else {
+ 			if ($g['debug'])
+ 				echo "{$src_table} NOT found in cache...adding\n";
+-			$table_cache[$src_table] = $src_table_line;
++			if (strpos($src_table_line, 'http://') === false)
++				$table_cache[$src_table] = $src_table_line;
+ 		}
+ 	if (isset($dst_table))
+ 		if (isset($table_cache[$dst_table])) {
+@@ -2022,7 +2029,8 @@ function generate_user_filter_rule($rule, $ngcounter) {
+ 		} else {
+ 			if ($g['debug'])
+ 				echo "{$dst_table} NOT found in cache...adding\n";
+-			$table_cache[$dst_table] = $dst_table_line;
++			if (strpos($dst_table_line, 'http://') === false)
++				$table_cache[$dst_table] = $dst_table_line;
+ 		}
+ 
+ 	/* exception(s) to a user rules can go here. */
+diff --git a/etc/inc/pfsense-utils.inc b/etc/inc/pfsense-utils.inc
+index c45def5..2f0f6fb 100644
+--- a/etc/inc/pfsense-utils.inc
++++ b/etc/inc/pfsense-utils.inc
+@@ -3905,4 +3905,47 @@ function update_alias_names_upon_change($section, $subsection, $fielda, $fieldb,
+ 
+ }
+ 
++function process_alias_urltable($name, $url, $freq, $forceupdate=false) {
++	$aliastable_prefix = "/var/db/aliastables/";
++	$aliastable_filename = $aliastable_prefix . $name . ".txt";
++
++	// Make the aliases directory if it doesn't exist
++	if (!file_exists($aliastable_prefix)) {
++		mkdir($aliastable_prefix);
++	} elseif (!is_dir($aliastable_prefix)) {
++		unlink($aliastable_prefix);
++		mkdir($aliastable_prefix);
++	}
++
++	// If the file doesn't exist or is older than update_freq days, fetch a new copy.
++	if (!file_exists($aliastable_filename) 
++		|| ((time() - filemtime($aliastable_filename)) > ($freq * 86400))
++		|| $forceupdate) {
++
++		// Try to fetch the URL supplied
++		conf_mount_rw();
++		unlink_if_exists($aliastable_filename . ".tmp");
++		mwexec("/usr/bin/fetch -q -o " . escapeshellarg($aliastable_filename) . ".tmp " . escapeshellarg($url));
++		mwexec("/usr/bin/grep -v '^#' " . escapeshellarg($aliastable_filename) . ".tmp > " . escapeshellarg($aliastable_filename));
++		unlink_if_exists($aliastable_filename . ".tmp");
++		conf_mount_ro();
++		if (filesize($aliastable_filename)) {
++			return true;
++		} else {
++			// If it's unfetchable or an empty file, bail
++			return false;
++		}
++	} else {
++		// File exists, and it doesn't need updated.
++		return -1;
++	}
++}
++
++function is_valid_http_url($url) {
++	$parsed = parse_url($url);
++	if (($parsed['scheme'] == 'http') && (is_fqdn($parsed['host']) || is_ipaddr($parsed['host'])))
++		return true;
++	else
++		return false;
++}
+ ?>
+diff --git a/etc/inc/util.inc b/etc/inc/util.inc
+index 61d2e55..efeaf01 100644
+--- a/etc/inc/util.inc
++++ b/etc/inc/util.inc
+@@ -487,6 +487,21 @@ function alias_expand_net($name) {
+ 		return null;
+ }
+ 
++function alias_expand_urltable($name) {
++	global $aliastable;
++	$aliastable_prefix = "/var/db/aliastables/";
++	$aliastable_filename = $aliastable_prefix . $name . ".txt";
++
++	if (isset($aliastable[$name]) 
++		&& (is_valid_http_url($aliastable[$name]))
++		&& file_exists($aliastable_filename))
++		return $aliastable_filename;
++	elseif (process_alias_urltable($name, $aliastable[$name], 0, true))
++		return $aliastable_filename;
++	else
++		return null;
++}
++
+ /* find out whether two subnets overlap */
+ function check_subnets_overlap($subnet1, $bits1, $subnet2, $bits2) {
+ 
+diff --git a/etc/rc.update_urltables b/etc/rc.update_urltables
+new file mode 100755
+index 0000000..506a5b0
+--- /dev/null
++++ b/etc/rc.update_urltables
+@@ -0,0 +1,49 @@
++#!/usr/local/bin/php -q
++<?php
++require_once("config.inc");
++require_once("util.inc");
++require_once("pfsense-utils.inc");
++
++if (!is_array($config['aliases']['alias'])) {
++	// No aliases
++	exit;
++}
++
++// Gather list of urltable aliases
++$todo = array();
++foreach ($config['aliases']['alias'] as $alias) {
++	if ($alias['type'] = 'urltable') {
++		$tmp = array();
++		$tmp['name'] = $alias['name'];
++		$tmp['url']  = $alias['address'];
++		$tmp['freq'] = $alias['address_subnet'];
++		$todo[] = $tmp;
++	}
++}
++
++if (count($todo) > 0) {
++	log_error("{$argv[0]}: Starting up.");
++
++	if ($argv[1] != "now") {
++		// Wait a little before updating.
++		$wait = mt_rand(5, 60);
++		log_error("{$argv[0]}: Sleeping for {$wait} seconds.");
++		sleep($wait);
++	}
++
++	log_error("{$argv[0]}: Starting URL table alias updates");
++
++	foreach ($todo as $t) {
++		$r = process_alias_urltable($t['name'], $t['url'], $t['freq']);
++		if ($r == 1) {
++			$result = "";
++			exec("/sbin/pfctl -t " . escapeshellarg($t['name']) . " -T replace -f /var/db/aliastables/" . escapeshellarg($t['name']) . ".txt 2>&1", $result);
++			log_error("{$argv[0]}: Updated {$t['name']} content from {$t['url']}: {$result[0]}");
++		} elseif ($r == -1) {
++			log_error("{$argv[0]}: {$t['name']} does not need updated.");
++		} else {
++			log_error("{$argv[0]}: ERROR: could not update {$t['name']} content from {$t['url']}");
++		}
++	}
++}
++?>
+\ No newline at end of file
+diff --git a/usr/local/www/firewall_aliases.php b/usr/local/www/firewall_aliases.php
+index e453200..9be52a4 100755
+--- a/usr/local/www/firewall_aliases.php
++++ b/usr/local/www/firewall_aliases.php
+@@ -85,7 +85,7 @@ if ($_GET['act'] == "del") {
+ 						$referenced_by = $rule['descr'];
+ 						break;
+ 					}
+-					if($rule['source']['address'] == $alias_name) {
++					if($rule['destination']['address'] == $alias_name) {
+ 						$is_alias_referenced = true;
+ 						$referenced_by = $rule['descr'];
+ 						break;
+@@ -177,11 +177,10 @@ include("head.inc");
+   </td>
+   <td class="listr" ondblclick="document.location='firewall_aliases_edit.php?id=<?=$i;?>';">
+       <?php
+-	$addresses = implode(", ", array_slice(explode(" ", $alias['address']), 0, 10));
++	$tmpaddr = explode(" ", $alias['address']);
++	$addresses = implode(", ", array_slice($tmpaddr, 0, 10));
+ 	echo $addresses;
+-	if(count($addresses) < 10) {
+-		echo " ";
+-	} else {
++	if(count($tmpaddr) > 10) {
+ 		echo "...";
+ 	}
+     ?>
+diff --git a/usr/local/www/firewall_aliases_edit.php b/usr/local/www/firewall_aliases_edit.php
+index 62e1272..f88b4b3 100755
+--- a/usr/local/www/firewall_aliases_edit.php
++++ b/usr/local/www/firewall_aliases_edit.php
+@@ -65,10 +65,14 @@ if (isset($id) && $a_aliases[$id]) {
+ 	if ($addresssubnettest)
+ 		$pconfig['type'] = "network";
+ 	else
+-		if (is_ipaddr($address[0]))
++		if (is_ipaddr($address[0])) {
+ 			$pconfig['type'] = "host";
+-		else
++		} elseif (is_valid_http_url($pconfig['address'])) {
++			$pconfig['type'] = $a_aliases[$id]['type'];
++			$pconfig['address_subnet'] = $a_aliases[$id]['address_subnet'];
++		} else {
+ 			$pconfig['type'] = "port";
++		}
+ 
+ 	if($a_aliases[$id]['aliasurl'] <> "") {
+ 		$pconfig['type'] = "url";
+@@ -99,6 +103,9 @@ if ($_POST) {
+ 	if ($_POST['type'] == "network") {
+ 		$reqdfields[] = "address_subnet";
+ 		$reqdfieldsn[] = "Subnet bit count";
++	} elseif ($_POST['type'] == "urltable") {
++		$reqdfields[] = "address_subnet";
++		$reqdfieldsn[] = "Update Frequency";
+ 	}
+ 
+ 	do_input_validation($_POST, $reqdfields, $reqdfieldsn, &$input_errors);
+@@ -131,10 +138,18 @@ if ($_POST) {
+ 	}
+ 
+ 	if ($_POST['type'] == "url") {
+-		if(stristr($_POST['address'], "http") == false)
++		if(is_valid_http_url($_POST['address']) == false)
+ 			$input_errors[] = "You must provide a valid URL to the resource.";
+ 	}
+ 
++	if ($_POST['type'] == "urltable") {
++		if(is_valid_http_url($_POST['address']) == false)
++			$input_errors[] = "You must provide a valid URL to the resource.";
++		if (!is_numeric($_POST['address_subnet'])) {
++			$input_errors[] = "A valid update frequency must be specified.";
++		}
++	}
++
+ 	if ($_POST['type'] == "port")
+ 		if (! is_port($_POST['address']) && ! is_portrange($_POST['address']))
+ 			$input_errors[] = "Please specify a valid port or portrange.";
+@@ -177,7 +192,25 @@ if ($_POST) {
+     	$final_address_details .= "||";
+ 	$isfirst = 0;
+ 
+-	if($_POST['type'] == "url") {
++	if ($_POST['type'] == "urltable") {
++		$address = "";
++		$isfirst = 0;
++
++		/* item is a url type */
++		if ($_POST['address']) {
++			/* fetch down and add in */
++			$isfirst = 0;
++			$address = $_POST['address'];
++			$alias['address_subnet'] = $_POST['address_subnet'];
++			if (strtolower(substr($_POST['address'], 0, 4)) != "http") {
++				$input_errors[] = "You must provide a valid URL.";
++				$dont_update = true;
++			} elseif (! process_alias_urltable($_POST['name'], $_POST['address'], $_POST['address_subnet'], true)) {
++				$input_errors[] = "Unable to fetch usable data.";
++				$dont_update = true;
++			}
++		}
++	} elseif($_POST['type'] == "url") {
+ 		$address = "";
+ 		$isfirst = 0;
+ 		$address_count = 2;
+@@ -355,13 +388,14 @@ $hosts_str = gettext("Host(s)");
+ $ip_str = gettext("IP");
+ $ports_str = gettext("Port(s)");
+ $port_str = gettext("Port");
++$urltable_str = gettext("URL Table");
+ $url_str = gettext("URL");
+ $update_freq_str = gettext("Update Freq.");
+ 
+ $networks_help = gettext("Networks can be expressed like 10.0.0.0 format.  Select the CIDR (network mask) that pertains to each entry.");
+ $hosts_help = gettext("Enter as many hosts as you would like.  Hosts should be expressed in their ip address format.");
+ $ports_help = gettext("Enter as many ports as you wish.  Port ranges can be expressed by seperating with a colon.");
+-$url_help = gettext("Enter as many urls as you wish.  Also set the time that you would like the url refreshed in days.  After saving {$g['product_name']} will download the URL and import the items into the alias.");
++$urltable_help = gettext("Enter a single URL and set the time that you would like the url refreshed in days.  After saving {$g['product_name']} will download the URL and save the items as a persistent table, which works like the bogons list. (cron job to update is not added automatically)");
+ 
+ $jscriptstr .= <<<EOD
+ 
+@@ -376,6 +410,7 @@ function update_box_type() {
+ 		document.getElementById ("twocolumn").firstChild.data = "{$cidr_str}";
+ 		document.getElementById ("threecolumn").firstChild.data = "{$description_str}";
+ 		document.getElementById ("itemhelp").firstChild.data = "{$networks_help}";
++		document.getElementById ("addrowbutton").style.display = 'block';
+ 	} else if(selected == '{$hosts_str}') {
+ 		document.getElementById ("addressnetworkport").firstChild.data = "{$hosts_str}";
+ 		document.getElementById ("address_subnet").visible = false;
+@@ -384,6 +419,7 @@ function update_box_type() {
+ 		document.getElementById ("twocolumn").firstChild.data = "";
+ 		document.getElementById ("threecolumn").firstChild.data = "{$description_str}";
+ 		document.getElementById ("itemhelp").firstChild.data = "{$hosts_help}";
++		document.getElementById ("addrowbutton").style.display = 'block';
+ 	} else if(selected == '{$ports_str}') {
+ 		document.getElementById ("addressnetworkport").firstChild.data = "{$ports_str}";
+ 		document.getElementById ("address_subnet").visible = false;
+@@ -392,6 +428,7 @@ function update_box_type() {
+ 		document.getElementById ("twocolumn").firstChild.data = "";
+ 		document.getElementById ("threecolumn").firstChild.data = "{$description_str}";
+ 		document.getElementById ("itemhelp").firstChild.data = "{$ports_help}";
++		document.getElementById ("addrowbutton").style.display = 'block';
+ 	} else if(selected == '{$url_str}') {
+ 		document.getElementById ("addressnetworkport").firstChild.data = "{$url_str}";
+ 		document.getElementById ("address_subnet").visible = true;
+@@ -400,6 +437,16 @@ function update_box_type() {
+ 		document.getElementById ("twocolumn").firstChild.data = "{$update_freq_str}";
+ 		document.getElementById ("threecolumn").firstChild.data = "{$description_str}";
+ 		document.getElementById ("itemhelp").firstChild.data = "{$url_help}";
++		document.getElementById ("addrowbutton").style.display = 'block';
++	} else if(selected == '{$urltable_str}') {
++		document.getElementById ("addressnetworkport").firstChild.data = "{$url_str}";
++		document.getElementById ("address_subnet").visible = true;
++		document.getElementById ("address_subnet").disabled = false;
++		document.getElementById ("onecolumn").firstChild.data = "{$url_str}";
++		document.getElementById ("twocolumn").firstChild.data = "{$update_freq_str}";
++		document.getElementById ("threecolumn").firstChild.data = "{$description_str}";
++		document.getElementById ("itemhelp").firstChild.data = "{$urltable_help}";
++		document.getElementById ("addrowbutton").style.display = 'none';
+ 	}
+ }
+ </script>
+@@ -480,6 +527,7 @@ EOD;
+         <option value="host" <?php if ($pconfig['type'] == "host") echo "selected"; ?>>Host(s)</option>
+         <option value="network" <?php if ($pconfig['type'] == "network") echo "selected"; ?>>Network(s)</option>
+         <option value="port" <?php if ($pconfig['type'] == "port") echo "selected"; ?>>Port(s)</option>
++        <option value="urltable" <?php if ($pconfig['type'] == "urltable") echo "selected"; ?>>URL Table</option>
+       </select>
+     </td>
+   </tr>
+@@ -525,7 +573,7 @@ EOD;
+ 			        <select name="address_subnet<?php echo $tracker; ?>" class="formselect" id="address_subnet<?php echo $tracker; ?>">
+ 			          <option></option>
+ 			          <?php for ($i = 32; $i >= 1; $i--): ?>
+-			          <option value="<?=$i;?>" <?php if ($i == $address_subnet) echo "selected"; ?>><?=$i;?></option>
++			          <option value="<?=$i;?>" <?php if (($i == $address_subnet) || ($i == $pconfig['address_subnet'])) echo "selected"; ?>><?=$i;?></option>
+ 			          <?php endfor; ?>
+ 			        </select>
+ 			      </td>
+@@ -549,8 +597,8 @@ EOD;
+ 
+         </tfoot>
+ 		  </table>
+-			<a onclick="javascript:addRowTo('maintable'); typesel_change(); return false;" href="#">
+-        <img border="0" src="/themes/<?= $g['theme']; ?>/images/icons/icon_plus.gif" alt="" title="add another entry" />
++			<div id="addrowbutton"><a onclick="javascript:addRowTo('maintable'); typesel_change(); return false;" href="#">
++        <img border="0" src="/themes/<?= $g['theme']; ?>/images/icons/icon_plus.gif" alt="" title="add another entry" /></div>
+       </a>
+ 		</td>
+   </tr>
diff --git a/config/urltables/urltables.xml b/config/urltables/urltables.xml
new file mode 100644
index 00000000..16fe50c3
--- /dev/null
+++ b/config/urltables/urltables.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!DOCTYPE packagegui SYSTEM "../schema/packages.dtd">
+<?xml-stylesheet type="text/xsl" href="../xsl/package.xsl"?>
+<packagegui>
+        <copyright>
+        <![CDATA[
+/* $Id$ */
+/* ========================================================================== */
+/*
+    urltables.xml
+    part of pfSense (http://www.pfSense.com)
+    Copyright (C) 2007 to whom it may belong
+    All rights reserved.
+
+    Based on m0n0wall (http://m0n0.ch/wall)
+    Copyright (C) 2003-2006 Manuel Kasper <mk@neon1.net>.
+    All rights reserved.
+                                                                              */
+/* ========================================================================== */
+/*
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+     1. Redistributions of source code must retain the above copyright notice,
+        this list of conditions and the following disclaimer.
+
+     2. Redistributions in binary form must reproduce the above copyright
+        notice, this list of conditions and the following disclaimer in the
+        documentation and/or other materials provided with the distribution.
+
+    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+    POSSIBILITY OF SUCH DAMAGE.
+                                                                              */
+/* ========================================================================== */
+        ]]>
+        </copyright>
+    <description>Patch to add URL Table Aliases</description>
+    <requirements>pfSense 1.2.3</requirements>
+    <faq>None</faq>
+	<name>URL Table Aliases</name>
+	<version>0.2</version>
+	<title>URL Table Aliases</title>
+	<include_file>/usr/local/pkg/urltables.inc</include_file>
+	<additional_files_needed>
+		<prefix>/usr/local/pkg/</prefix>
+		<chmod>077</chmod>
+		<item>http://www.pfsense.com/packages/config/urltables/urltables.inc</item>
+	</additional_files_needed>
+	<additional_files_needed>
+		<prefix>/usr/local/pkg/</prefix>
+		<chmod>077</chmod>
+		<item>http://www.pfsense.com/packages/config/urltables/urltables.patch</item>
+	</additional_files_needed>
+	<custom_php_install_command>
+		urltables_install();
+	</custom_php_install_command>
+</packagegui>
diff --git a/pkg_config.7.xml b/pkg_config.7.xml
index bec34377..644070aa 100755
--- a/pkg_config.7.xml
+++ b/pkg_config.7.xml
@@ -968,7 +968,7 @@
     </package>
     <package>
       <name>IP Range Aliases</name>
-      <descr>Patch to add IP Range support to Network Aliases. WARNING! Cannot be uninstalled.</descr>
+      <descr>Patch to add IP Range support to Network Aliases. WARNING! Cannot be uninstalled. Probably does not play nice with URL Table patch.</descr>
       <category>System</category>
       <config_file>http://www.pfsense.com/packages/config/iprangealiases/iprangealiases.xml</config_file>
       <version>0.2</version>
@@ -976,5 +976,15 @@
       <required_version>1.2.3</required_version>
       <maximum_version>1.2.3</maximum_version>
     </package>
+    <package>
+      <name>URL Table Aliases</name>
+      <descr>Patch to add URL Table alias support to pull a list of CIDRs/IPs from a file by URL into a persist table. WARNING! Cannot be uninstalled. Probably does not play nice with IP Range Aliases patch. This is NOT a way to add an alias for host URLs.</descr>
+      <category>System</category>
+      <config_file>http://www.pfsense.com/packages/config/urltables/urltables.xml</config_file>
+      <version>0.1</version>
+      <status>BETA</status>
+      <required_version>1.2.3</required_version>
+      <maximum_version>1.2.3</maximum_version>
+    </package>
   </packages>
 </pfsensepkgs>