dansguardian - improve blacklist import and category assign

		- include dansguardian on service
		- include log, deamon and config default xml options
		- fix some typos

6 files changed, 107 insertions, 41 deletions

diff --git a/config/dansguardian/dansguardian.inc b/config/dansguardian/dansguardian.inc
index 53b45cee..69d45406 100755
--- a/config/dansguardian/dansguardian.inc
+++ b/config/dansguardian/dansguardian.inc
@@ -47,17 +47,41 @@ function dg_get_real_interface_address($iface) {
 }
 
 function sync_package_dansguardian() {
-	global $config;
-
+	global $config,$g;
+
+	#check if its booting
+	if ($g['booting']){
+		if (is_array($config['installedpackages']['dansguardian']))
+			if (!$config['installedpackages']['dansguardian']['config'][0]['enable'])
+				return;
+		if (file_exists("/var/run/dansguardian.pid"))
+			exec("/bin/ps -p `cat /var/run/dansguardian.pid` 2>&1",$return);
+			if ($return == "0")
+				return;
+	}
 	#assign xml arrays
-	if (is_array($config['installedpackages']['dansguardian']))
-		$dansguardian=$config['installedpackages']['dansguardian']['config'][0];
-	if (is_array($config['installedpackages']['dansguardianconfig']))
-		$dansguardian_config=$config['installedpackages']['dansguardianconfig']['config'][0];
+	if (!is_array($config['installedpackages']['dansguardian']))
+		$config['installedpackages']['dansguardian']['config'][0]=array('interface'=>'lo0',
+																		'daemon_options'=>'softrestart');
+	$dansguardian=$config['installedpackages']['dansguardian']['config'][0];
+	if (!is_array($config['installedpackages']['dansguardianconfig']))
+		$config['installedpackages']['dansguardianconfig']['config'][0]=array('auth_plugin'=>'',
+																		'scan_options'=>'scancleancache,createlistcachefiles,deletedownloadedtempfiles',
+																		'weightedphrasemode'=>'2',
+																		'preservecase'=>'0',
+																		'phrasefiltermode'=>'2');
+	$dansguardian_config=$config['installedpackages']['dansguardianconfig']['config'][0];
+	if (!is_array($config['installedpackages']['dansguardianlog']))
+		$config['installedpackages']['dansguardianlog']['config'][0]=array('report_level'=>'3',
+																		'report_language'=>'ukenglish',
+																		'report_options'=>'showweightedfound,usecustombannedimage,nonstandarddelimiter',
+																		'logging_options'=>'logconnectionhandlingerrors',
+																		'loglevel'=>'2',
+																		'logexceptionhits'=>'2',
+																		'logfileformat'=>'1');
+	$dansguardian_log=$config['installedpackages']['dansguardianlog']['config'][0];
 	if (is_array($config['installedpackages']['dansguardianlimits']))
 		$dansguardian_limits=$config['installedpackages']['dansguardianlimits']['config'][0];
-	if (is_array($config['installedpackages']['dansguardianlog']))
-		$dansguardian_log=$config['installedpackages']['dansguardianlog']['config'][0];
 	if (is_array($config['installedpackages']['dansguardianusers']))
 		$dansguardian_users=$config['installedpackages']['dansguardianusers']['config'][0];
 	if (is_array($config['installedpackages']['dansguardianblacklist']['config']))
@@ -330,7 +354,7 @@ function sync_package_dansguardian() {
 			$config['installedpackages']['dansguardianurlacl']['config'][$count]['bannedregexp_urllist']=base64_encode(file_get_contents($dansguardian_dir.'/lists/bannedregexpurllist.sample'));
 			$load_samples++;
 		}
-		file_put_contents($dansguardian_dir."/lists/bannedregexpurllist.".$dansguardian_url['name'],($dansguardian_url['bannedurl_enabled']?dg_text_area_decode($config['installedpackages']['dansguardianurlacl']['config'][$count]['bannedregexp_urllist']).$includes:""),LOCK_EX);
+		file_put_contents($dansguardian_dir."/lists/bannedregexpurllist.".$dansguardian_url['name'],($dansguardian_url['bannedurl_enabled']?dg_text_area_decode($config['installedpackages']['dansguardianurlacl']['config'][$count]['bannedregexp_urllist']):""),LOCK_EX);
 			
 		#greyurllist
 		if($dansguardian_url['grey_urllist'] == "" && file_exists ($dansguardian_dir.'/lists/greyurllist.sample')){
@@ -340,11 +364,11 @@ function sync_package_dansguardian() {
 		file_put_contents($dansguardian_dir."/lists/greyurllist.".$dansguardian_url['name'],($dansguardian_url['greyurl_enabled']?dg_text_area_decode($config['installedpackages']['dansguardianurlacl']['config'][$count]['grey_urllist']):""),LOCK_EX);
 			
 		#exceptionfileurllist
-		if($dansguardian_url['exception_urllist'] == "" && file_exists ($dansguardian_dir.'/lists/exceptionfileurllist.sample')){
-			$config['installedpackages']['dansguardianurlacl']['config'][$count]['exception_urllist']=base64_encode(file_get_contents($dansguardian_dir.'/lists/exceptionfileurllist.sample'));
+		if($dansguardian_url['exceptionfile_urllist'] == "" && file_exists ($dansguardian_dir.'/lists/exceptionfileurllist.sample')){
+			$config['installedpackages']['dansguardianurlacl']['config'][$count]['exceptionfile_urllist']=base64_encode(file_get_contents($dansguardian_dir.'/lists/exceptionfileurllist.sample'));
 			$load_samples++;
 		}
-		file_put_contents($dansguardian_dir."/lists/exceptionfileurllist.".$dansguardian_url['name'],($dansguardian_url['exceptionurl_enabled']?dg_text_area_decode($config['installedpackages']['dansguardianurlacl']['config'][$count]['exception_urllist']):""),LOCK_EX);
+		file_put_contents($dansguardian_dir."/lists/exceptionfileurllist.".$dansguardian_url['name'],($dansguardian_url['exceptionurl_enabled']?dg_text_area_decode($config['installedpackages']['dansguardianurlacl']['config'][$count]['exceptionfile_urllist']):""),LOCK_EX);
 		
 		#exceptionregexpurllist
 		if($dansguardian_url['exceptionregexp_urllist'] == "" && file_exists ($dansguardian_dir.'/lists/exceptionregexpurllist.sample')){
@@ -358,7 +382,8 @@ function sync_package_dansguardian() {
 			$config['installedpackages']['dansguardianurlacl']['config'][$count]['exception_urllist']=base64_encode(file_get_contents($dansguardian_dir.'/lists/exceptionurllist.sample'));
 			$load_samples++;
 		}
-		file_put_contents($dansguardian_dir."/lists/exceptionurllist.".$dansguardian_url['name'],($dansguardian_url['exceptionurl_enabled']?dg_text_area_decode($config['installedpackages']['dansguardianurlacl']['config'][$count]['exception_urllist']):""),LOCK_EX);
+		$includes=preg_replace($match,$replace,$dansguardian_url['exception_includes']);
+		file_put_contents($dansguardian_dir."/lists/exceptionurllist.".$dansguardian_url['name'],($dansguardian_url['exceptionurl_enabled']?dg_text_area_decode($config['installedpackages']['dansguardianurlacl']['config'][$count]['exception_urllist']).$includes:""),LOCK_EX);
 			
 		#urlregexplist
 		if($dansguardian_url['modify_urllist'] == "" && file_exists ($dansguardian_dir.'/lists/urlregexplist.sample')){
@@ -648,8 +673,7 @@ EOF;
 	file_put_contents("/usr/local/pkg/dansguardian_users.xml",$user_xml_header.$user_xml.$user_xml_footer,LOCK_EX);
 	
 	#check blacklist download files
-	if ($dansguardian_blacklist['cron']=="now" && $dansguardian_blacklist['blacklist']){
-		$config['installedpackages']['dansguardianblacklist']['config'][0]['cron']="never";
+	if ($dansguardian_blacklist['cron']=="force_download"){
 		log_error("Blacklist udpate process started");
 		file_notice("Dansguardian - Blacklist udpate process started","");
 		file_put_contents("/root/dansguardian_custom.script",base64_decode($dansguardian_blacklist['custom_script']),LOCK_EX);
@@ -657,6 +681,10 @@ EOF;
 			mwexec_bg("/root/dansguardian_custom.script");
 		else
 			mwexec_bg("/usr/local/bin/php /usr/local/www/dansguardian.php fetch_blacklist");
+		}
+	#update xml categories from downloaded file
+	if (preg_match("/force/",$dansguardian_blacklist['cron'])){
+		$config['installedpackages']['dansguardianblacklist']['config'][0]['cron']="never";
 		mwexec_bg("/usr/local/bin/php /usr/local/www/dansguardian.php update_lists");
 	}
 	
@@ -809,7 +837,7 @@ function dansguardian_php_install_command() {
 }
 
 function dansguardian_php_deinstall_command() {
-	mwexec("/usr/local/etc/rc.d/dansguardian.sh stop");
+	mwexec("/usr/local/etc/rc.d/dansguardian stop");
 	sleep(1);
 	conf_mount_rw();
 	unlink_if_exists("/usr/local/etc/rc.d/dansguardian");
diff --git a/config/dansguardian/dansguardian.php b/config/dansguardian/dansguardian.php
index 8998d1f8..844d9fd3 100644
--- a/config/dansguardian/dansguardian.php
+++ b/config/dansguardian/dansguardian.php
@@ -51,11 +51,9 @@ function fetch_blacklist(){
 			exec ('rm -rf /usr/local/etc/dansguardian/lists/blacklists.old');
 		rename("blacklists","blacklists.old");
 		exec('/usr/bin/tar -xvzf /usr/local/etc/dansguardian/lists/blacklist.tgz 2>&1',$output,$return);
-		if (preg_match("/x (\w+)/",$output[0],$matches)){
+		if (preg_match("/x\W+(\w+)/",$output[0],$matches)){
 			if ($matches[1] != "blacklists")
 				rename("./".$matches[1],"blacklists");
-			read_lists();
-			file_notice("Dansguardian - Blacklist applied, check site and URL access lists for categories","");
 			}
 		else
 			file_notice("Dansguardian - Could not determine Blacklist extract dir. Categories not updated","");
@@ -72,36 +70,52 @@ function read_lists(){
 	$groups= array("phraselists", "blacklists", "whitelists");
 	#assigns know list files
 	$types=array('domains','urls','banned','weighted','exception','expression');
+
 	#clean previous xml config for dansguardian lists
-	foreach ($groups as $group)
-		foreach ($types as $clean)
-			$config['installedpackages']['dansguardian'.$group.$clean]['config']=array();
+	foreach($config['installedpackages'] as $key => $values)
+		if (preg_match("/dansguardian(phrase|black|white)lists/",$key))
+			unset ($config['installedpackages'][$key]);
 	
-	#clean previous xml config for dansguardian lists
+	#find lists
 	foreach ($groups as $group)
 		if (is_dir("$dir/$group/")){
 			#read dir content and find lists
 			$lists= scandir("$dir/$group/");
 			foreach ($lists as $list)
-				if (!preg_match ("/^\./",$list)){
+				if (!preg_match ("/^\./",$list) && is_dir("$dir/$group/$list/")){
 					$category= scandir("$dir/$group/$list/"); 
 					foreach ($category as $file)
 						if (!preg_match ("/^\./",$file)){
 							#assign list to array
 							$type=split("_",$file);
-							print $type[0]." --- $list --- $file\n";
-							if (!in_array($type[0],$group_type))
-								$list_type[]=$type[0];
-							$xml_group=($list=="whitelist"?"whitelists":$group);
-							$xml_type=($type[0]=="domains.processed"?"domains":$type[0]);
-							$config['installedpackages']['dansguardian'.$xml_group.$xml_type]['config'][]=array("descr"=> "$list $file","list" => $list,"file" => "$dir/$group/$list/$file");
+							if (preg_match("/(\w+)/",$type[0],$matches));
+								$xml_type=$matches[1];
+							if ($config['installedpackages']['dansguardianblacklist']['config'][0]["liston"]=="both" && $group=="blacklists")
+								$config['installedpackages']['dansguardianwhitelists'.$xml_type]['config'][]=array("descr"=> "$list $file","list" => $list,"file" => "$dir/$group/$list/$file");
+							$config['installedpackages']['dansguardian'.$group.$xml_type]['config'][]=array("descr"=> "$list $file","list" => $list,"file" => "$dir/$group/$list/$file");
 						}
 				}
 		}
-	
-	#var_dump($config['installedpackages']['dansguardian']['config']);
-	#foreach ($types as $clean)
-	#var_dump($config['installedpackages']['dansguardianfiles'.$clean]['config']);
+	conf_mount_rw();
+	$files=array("site","url");
+	foreach ($files as $edit_xml){
+		$edit_file=file_get_contents("/usr/local/pkg/dansguardian_".$edit_xml."_acl.xml");
+		if(count($config['installedpackages']['dansguardianblacklistsdomains']['config']) > 18){
+			$edit_file=preg_replace('/size.6/','size>20',$edit_file);
+			if ($config['installedpackages']['dansguardianblacklist']['config'][0]["liston"]=="both")
+				$edit_file=preg_replace('/size.5/','size>19',$edit_file);
+			}
+		else{
+			$edit_file=preg_replace('/size.20/','size>6',$edit_file);
+			}
+		if ($config['installedpackages']['dansguardianblacklist']['config'][0]["liston"]!="both")
+			$edit_file=preg_replace('/size.19/','size>5',$edit_file);
+		file_put_contents("/usr/local/pkg/dansguardian_".$edit_xml."_acl.xml",$edit_file,LOCK_EX);
+		}
+	file_notice("Dansguardian - Blacklist applied, check site and URL access lists for categories","");
+	#foreach($config['installedpackages'] as $key => $values)
+	#	if (preg_match("/dansguardian(phrase|black|white)lists/",$key))
+	#		print "$key\n";
 	write_config();
 }
 
diff --git a/config/dansguardian/dansguardian.xml b/config/dansguardian/dansguardian.xml
index 809feffd..4be2e402 100644
--- a/config/dansguardian/dansguardian.xml
+++ b/config/dansguardian/dansguardian.xml
@@ -47,11 +47,17 @@
 	<title>Services: Dansguardian</title>
 	<include_file>/usr/local/pkg/dansguardian.inc</include_file>
 	<menu>
-		<name>dansguardian</name>
+		<name>Dansguardian</name>
 		<tooltiptext>Configure dansguardian</tooltiptext>
 		<section>Services</section>
 		<url>pkg_edit.php?xml=dansguardian.xml</url>
 	</menu>
+	<service>
+		<name>dansguardian</name>
+		<rcfile>dansguardian</rcfile>
+		<executable>dansguardian</executable>
+		<description><![CDATA[Award winning Open Source web content filter]]></description>
+	</service>
 	<additional_files_needed>
 		<item>http://www.pfsense.org/packages/config/dansguardian/dansguardian.inc</item>
 		<prefix>/usr/local/pkg/</prefix>
diff --git a/config/dansguardian/dansguardian_blacklist.xml b/config/dansguardian/dansguardian_blacklist.xml
index d518f531..cf4e3c15 100644
--- a/config/dansguardian/dansguardian_blacklist.xml
+++ b/config/dansguardian/dansguardian_blacklist.xml
@@ -95,9 +95,15 @@
 			<type>listtopic</type>
 		</field>
 		<field>
-			<fielddescr>Enable</fielddescr>
-			<fieldname>blacklist</fieldname>
-			<type>checkbox</type>
+			<fielddescr>List on</fielddescr>
+			<fieldname>liston</fieldname>
+			<description><![CDATA[Select where to list categories dowloaded form file<br>
+							List on Banned and Exception is usefull if you want to whitelist any category downloaded from url.]]></description>
+	    	<type>select</type>
+ 				<options>
+ 				<option><name>Banned only</name><value>banned</value></option>
+ 				<option><name>Banned and Exception</name><value>both</value></option>
+				</options>
 		</field>
 		<field>
 			<fielddescr>Update frequency</fielddescr>
@@ -107,10 +113,11 @@
 	    	<type>select</type>
  				<options>
  				<option><name>Never</name><value>never</value></option>
- 				<option><name>Now</name><value>now</value></option>
  				<option><name>Every day</name><value>day</value></option>
 				<option><name>Every 2 days</name><value>02days</value></option>
 				<option><name>Once a week</name><value>week</value></option>
+				<option><name>Download and update Now</name><value>force_download</value></option>
+				<option><name>Reapply downloaded list Now</name><value>force_update</value></option>
 				</options>
 		</field>
 		<field>
diff --git a/config/dansguardian/dansguardian_site_acl.xml b/config/dansguardian/dansguardian_site_acl.xml
index 69f4fa80..94242ae5 100755
--- a/config/dansguardian/dansguardian_site_acl.xml
+++ b/config/dansguardian/dansguardian_site_acl.xml
@@ -143,7 +143,7 @@
 			<source_name>descr</source_name>
 			<source_value>file</source_value>
 			<multiple/>
-			<size>20</size>
+			<size>6</size>
 		</field>
 		<field>
 			<fielddescr>Config</fielddescr>
diff --git a/config/dansguardian/dansguardian_url_acl.xml b/config/dansguardian/dansguardian_url_acl.xml
index e7fdba14..cdbe40be 100755
--- a/config/dansguardian/dansguardian_url_acl.xml
+++ b/config/dansguardian/dansguardian_url_acl.xml
@@ -143,7 +143,7 @@
 			<source_name>descr</source_name>
 			<source_value>file</source_value>
 			<multiple/>
-			<size>20</size>
+			<size>6</size>
 		</field>
 			<field>
 			<fielddescr>Config</fielddescr>
@@ -196,6 +196,17 @@
 			<type>checkbox</type>
 			<description></description>
 		</field>
+			<field>
+			<fielddescr>Include</fielddescr>
+			<fieldname>exception_includes</fieldname>
+			<description><![CDATA[Select Include files to exception url lists.]]></description>
+	    	<type>select_source</type>
+			<source><![CDATA[$config['installedpackages']['dansguardianwhitelistsurls']['config']]]></source>
+			<source_name>descr</source_name>
+			<source_value>file</source_value>
+			<multiple/>
+			<size>5</size>
+		</field>
 	<field>
 			<fielddescr>Config</fielddescr>
 			<fieldname>exception_urllist</fieldname>