From 7804046877f31f93a387f331cf5bff4bbd72a415 Mon Sep 17 00:00:00 2001 From: Marcello Coutinho Date: Wed, 14 Dec 2011 02:37:31 -0200 Subject: varnish - change varnish-dev to varnish3 with streaming support --- config/varnish3/varnish.inc | 757 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 757 insertions(+) create mode 100644 config/varnish3/varnish.inc (limited to 'config/varnish3/varnish.inc') diff --git a/config/varnish3/varnish.inc b/config/varnish3/varnish.inc new file mode 100644 index 00000000..91d09413 --- /dev/null +++ b/config/varnish3/varnish.inc @@ -0,0 +1,757 @@ + + Copyright (C) 2011 Marcello Coutinho + All rights reserved. + */ +/* ========================================================================== */ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + */ +/* ========================================================================== */ + +function varnish_settings_post_validate($post, $input_errors) { + if($post['storagesize'] && !is_numeric($post['storagesize'])) + $input_errors[] = "A valid number is required for the field 'Storage size'"; + if($post['listeningport'] && !is_numeric($post['listeningport'])) + $input_errors[] = "A valid number is required for the field 'Listening port'"; + if($post['minworkers'] && !is_numeric($post['minworkers'])) + $input_errors[] = "A valid number is required for the field 'Minimum worker threads'"; + if($post['maxworkers'] && !is_numeric($post['maxworkers'])) + $input_errors[] = "A valid number is required for the field 'Maximum worker threads'"; + if($post['timeoutworkers'] && !is_numeric($post['timeoutworkers'])) + $input_errors[] = "A valid number is required for the field 'Worker thread timeout'"; + if($post['managment']){ + $mgm= explode(":",$post['managment']); + if(!is_ipaddr($mgm[0]) || !is_numeric($mgm[1])) + $input_errors[] = "A valid ip:port is required for the field 'managment'"; + } + if($post['grace'] && ! preg_match("/^\d+(h|m|s)$/",$post['grace'])) + $input_errors[] = "A valid number with a time reference is required for the field 'Fetch grace'"; + if($post['saint'] && ! preg_match("/^\d+(h|m|s)$/",$post['saint'])) + $input_errors[] = "A valid number with a time reference is required for the field 'Saint mode'"; + +} + +function varnish_lb_directors_post_validate($post, $input_errors) { + if (preg_match("/[^a-zA-Z0-9]/", $post['directorname'])) + $input_errors[] = "The directorname name must only contain the characters a-Z or 0-9"; + if(stristr($post['directorurl'], 'http')) + $input_errors[] = "You do not need to include the http:// string in the director URL"; + if($post['grace'] && ! preg_match("/^\d+(h|m|s)$/",$post['grace'])) + $input_errors[] = "A valid number with a time reference is required for the field 'Req grace'"; +} + +function varnish_backends_post_validate($post, $input_errors) { + if (!$post['backendname'] || preg_match("/[^a-zA-Z0-9]/", $post['backendname'])) + $input_errors[] = "The backend name must only contain the characters a-Z or 0-9"; + if(!is_ipaddr($post['ipaddress'])) + $input_errors[] = "A valid IP address is required for the field 'IPAddress'"; + if($post['first_byte_timeout'] && !is_numeric($post['first_byte_timeout'])) + $input_errors[] = "A valid number is required for the field 'first byte timeout'"; + if($post['connect_timeout'] && !is_numeric($post['connect_timeout'])) + $input_errors[] = "A valid number is required for the field 'connect timeout'"; + if($post['probe_interval'] && !is_numeric($post['probe_interval'])) + $input_errors[] = "A valid number is required for the field 'probe interval'"; + if($post['probe_interval'] && !is_numeric($post['probe_interval'])) + $input_errors[] = "A valid number is required for the field 'probe interval'"; + if($post['probe_timeout'] && !is_numeric($post['probe_timeout'])) + $input_errors[] = "A valid number is required for the field 'probe timeout'"; + if($post['probe_window'] && !is_numeric($post['probe_window'])) + $input_errors[] = "A valid number is required for the field 'probe window'"; + if($post['probe_threshold'] && !is_numeric($post['probe_threshold'])) + $input_errors[] = "A valid number is required for the field 'probe threshold'"; + $x=0; + while ($post['maptype'.$x] != ""){ + if($post['grace'.$x] && ! preg_match("/^\d+(h|m|s)$/",$post['grace'.$x])){ + $input_errors[] = "A valid number with a time reference is required for the field 'grace' in map ".($x +1); + } + $x++; + +} + +} + +function varnish_install() { + create_varnish_rcd_file(); +} + +function varnish_deinstall() { + create_varnish_rcd_file(); +} + +function text_area_decode($text){ + return preg_replace('/\r\n/', "\n",base64_decode($text)); +} +function varnish_start() { + global $g, $config; + if ($config['installedpackages']['varnishsettings']['config'][0]['enablevarnish']){ + exec("chmod +x /usr/local/etc/rc.d/varnish.sh"); + mwexec("/usr/local/etc/rc.d/varnish.sh");} + else{ + exec("chmod -x /usr/local/etc/rc.d/varnish.sh"); + mwexec("/usr/bin/killall varnishd");} +} + +/* Build the URL mappings logic VCL config txt */ +function varnish_get_url_mappings_txt() { + global $g, $config, $urlmappings,$backends_in_use; + $catch_all= "unset"; + $isfirst = true; + if($config['installedpackages']['varnishlbdirectors']['config'] != "") { + foreach($config['installedpackages']['varnishlbdirectors']['config'] as $url) { + #check options + $directo_grace_time=""; + if ($url['customapping']) + $directo_grace_time.=text_area_decode($url['customapping'])."\n\t\t"; + if($url['grace']) + $directo_grace_time.=($url['grace']=="0s"?"return(pass);":"set req.grace=".$url['grace'].";"); + $fieldtype = ($url['fieldtype']?$url['fieldtype']:"=="); + $req=($url['directorurl2']?"url":"http.host"); + $director_prefix=($url['directorurl'] && $url['directorurl2']?"^http://":""); + #check url + if ( $url['directorurl'] || $url['directorurl2'] || $catch_all == "unset" ){ + if ( $url['directorurl']== "" && $url['directorurl2']== "" ){ + #director with no host or url, so director for catch all traffic not specified in config + $lasturlmappings = "\telse\t{\n\t\tset req.backend = ".$url['directorname'].";\n\t\t}\n"; + $catch_all = "set"; + $isfirst = false; + } + else{ + if(!$isfirst) + $urlmappings .= "\telse "; + $urlmappings .= "if (req.$req $fieldtype ".'"'.$url['directorurl'].$url['directorurl2'].'") {'."\n"; + #check failover + $urlbackend = "\t\t\tset req.backend = ".$url['directorname'].";"; + if ($url['failover'] && $url['failover'] != $url['directorname']){ + $tabs=($url['grace']?"\n\t\t\t":""); + $urlfailover = "\t\t\tset req.backend = ".$url['failover'].";"; + $urlmappings .= "\t\tif (req.restarts == 0) {\n".$urlbackend.$tabs.$directo_grace_time.$tabs."}"; + $urlmappings .= "\n\t\telse\t{\n".$urlfailover.$tabs.$directo_grace_time.$tabs."}\n\t\t}\n"; + $isfirst = false; + } + else{ + $tabs=($url['grace']?"\n\t\t":""); + $urlmappings .= $urlbackend.$tabs.$directo_grace_time."\n\t\t}\n"; + $isfirst = false; + } + } + } + } + } + if($config['installedpackages']['varnishbackends']['config']) + foreach($config['installedpackages']['varnishbackends']['config'] as $urlmapping) { + if($urlmapping['row']) + foreach($urlmapping['row'] as $url) { + $directo_grace_time=""; + if($url['grace']) + $directo_grace_time=($url['grace']=="0s"?"\n\t\t return(pass);":"\n\t\tset req.grace=".$url['grace'].";"); + $req=($url['maptype']?$url['maptype']:"http.host"); + $fieldtype=($url['fieldtype']?$url['fieldtype']:"=="); + if ($url['urlmapping'] != "" || $catch_all == 'unset'){ + if($url['urlmapping'] == ""){ + $catch_all = "set"; + $lasturlmappings = "\telse\t{\n\t\tset req.backend = ".$urlmapping['backendname']."BACKEND;\n\t\t}\n"; + } + else{ + if(!$isfirst) + $urlmappings .= "\telse "; + $urlmappings .= <</dev/null +killall varnishd 2>/dev/null +sleep 1 +sysctl kern.ipc.nmbclusters=65536 +sysctl kern.ipc.somaxconn=16384 +sysctl kern.maxfiles=131072 +sysctl kern.maxfilesperproc=104856 +sysctl kern.threads.max_threads_per_proc=4096 +/usr/bin/env \ +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ +/usr/local/bin/php -q -d auto_prepend_file=config.inc < +ENDOFF + +/usr/local/sbin/varnishd \ + {$listeningport} \ + -f /var/etc/default.vcl \ + {$storage_type} \ + -w {$minworkers},{$maxworkers},{$timeoutworkers} \ + {$advancedstartup} + +EOF; + + fwrite($fd, $rc_file); + fclose($fd); + exec("chmod a+rx /usr/local/etc/rc.d/varnish.sh"); +} + +function get_backend_config_txt() { + global $config, $g, $backends_in_use; + $backends=""; + if($config['installedpackages']['varnishbackends']['config'] != "") { + foreach($config['installedpackages']['varnishbackends']['config'] as $backend) { + if($backend['connect_timeout']) + $connect_timeout = $backend['connect_timeout'] . "s"; + else + $connect_timeout = "25s"; + if($backend['port']) + $connect_port = $backend['port']; + else + $connect_port = "80"; + if($backend['first_byte_timeout']) + $first_byte_timeout = $backend['first_byte_timeout'] . "s"; + else + $first_byte_timeout = "300s"; + if($backend['probe_url']) + if (preg_match("@^(http)://([a-zA-Z0-9.-]*)/(.*)$@",$backend['probe_url'],$matches)){ + $probe_url=".request =\n"; + $probe_url.="\t\t\t".'"GET /'.$matches[3].' HTTP/1.1"'."\n"; + $probe_url.="\t\t\t".'"Accept: text/*"'."\n"; + $probe_url.="\t\t\t".'"User-Agent: Varnish"'."\n"; + $probe_url.="\t\t\t".'"Host: '.$matches[2].'"'."\n"; + $probe_url.="\t\t\t".'"Connection: Close";'; + } + else{ + $probe_url = '.url = "'.$backend['probe_url'].'";'; + } + else + $probe_url ='.url = "/";'; + if($backend['probe_interval']) + $probe_interval = $backend['probe_interval'] . "s"; + else + $probe_interval = "1s"; + if($backend['probe_timeout']) + $probe_timeout = $backend['probe_timeout'] . "s"; + else + $probe_timeout = "1s"; + if($backend['probe_window']) + $probe_window = $backend['probe_window']; + else + $probe_window = "5"; + if($backend['probe_threshold']) + $probe_threshold = $backend['probe_threshold']; + else + $probe_threshold = "5"; + if (isset($probe_threshold)){ + #last parameter set ,so write conf if backend is in use + if ($backends_in_use[$backend['backendname']] != ""){ + $backends .= << + +"} + obj.status + " " + obj.response + {" + + +

Error "} + obj.status + " " + obj.response + {"

+

"} + obj.response + {"

+

Guru Meditation:

+

XID: "} + req.xid + {"

+
+

Varnish cache server

+ +138 +EOF; + +/* Grab configuration txt blocks */ +/* Please keep this sequence to determine witch backends are in use */ +$backends_in_use=array(); +$lb_config= get_lb_directors_config_txt(); +$urlmappings = varnish_get_url_mappings_txt(); +$backends = get_backend_config_txt() . $lb_config ; +#$backends .= get_lb_directors_config_txt(); + + +/* Start to build varnish default.vcl configurationf file */ +$varnish_config_file = << + +{$errorvcl}"}; + return(deliver); + +} + +{$backends} +{$vcl_hash} +sub vcl_recv { + {$vcl_recv_early} + {$vcl_recv_set_basic} + {$urlmappings} + {$vcl_recv_late} + #respect client wish to refresh the page + if (req.http.Pragma ~ "no-cache") + { + return(pass); + } + + {$vcl_recv_action_basic} + return(lookup); +} + +sub vcl_pipe { + {$vcl_pipe_early} + # If we don't set the Connection: close header, any following + # requests from the client will also be piped through and + # left untouched by varnish. We don't want that. + set req.http.connection = "close"; + # Note: no "pipe" action here - we'll fall back to the default + # pipe method so that when any changes are made there, we + # still inherit them. + {$vcl_pipe_late} +} +sub vcl_hit { + return (deliver); +} + +sub vcl_miss { + return (fetch); +} + +sub vcl_fetch { + {$vcl_fetch_stream}{$vcl_fetch_early} + {$vcl_fetch_action} + # Varnish respects the wishes of the backend application. + if (beresp.http.Pragma ~ "no-cache" || beresp.http.Cache-Control ~ "(no-cache|no-store|private)") { + return(hit_for_pass); + } + ## If the request to the backend returns a code other than 200, restart the loop + ## If the number of restarts reaches the value of the parameter max_restarts, + ## the request will be error'ed. max_restarts defaults to 4. This prevents + ## an eternal loop in the event that, e.g., the object does not exist at all. + if (beresp.status != 200 && beresp.status != 403 && beresp.status != 404 && + beresp.status != 303 && beresp.status != 302 && beresp.status != 301 && beresp.status != 401 ) { + {$vcl_saint_mode}return(restart); + } + + + {$vcl_fetch_late} + {$vcl_grace_time}return(deliver); +} + +sub vcl_deliver { + ##set resp.http.X-Served-By = server.hostname; + if (obj.hits > 0) { + set resp.http.X-Cache = "HIT"; + set resp.http.X-Cache-Hits = obj.hits; + } else { + set resp.http.X-Cache = "MISS"; + } + return(deliver); +} + +sub vcl_init { + return (ok); +} + +sub vcl_fini { + return (ok); +} + +EOF; + + $fd = fopen("/var/etc/default.vcl", "w"); + fwrite($fd, $varnish_config_file); + fclose($fd); + + varnish_sync_on_changes(); +} + +/* Uses XMLRPC to synchronize the changes to a remote node */ +function varnish_sync_on_changes() { + global $config, $g; + log_error("[varnish] varnish_xmlrpc_sync.php is starting."); + $synconchanges = $config['installedpackages']['varnishsync']['config'][0]['synconchanges']; + if(!$synconchanges) + return; + foreach ($config['installedpackages']['varnishsync']['config'] as $rs ){ + foreach($rs['row'] as $sh){ + $sync_to_ip = $sh['ipaddress']; + $password = $sh['password']; + if($password && $sync_to_ip) + varnish_do_xmlrpc_sync($sync_to_ip, $password); + } + } + log_error("[varnish] varnish_xmlrpc_sync.php is ending."); +} +/* Do the actual XMLRPC sync */ +function varnish_do_xmlrpc_sync($sync_to_ip, $password) { + global $config, $g; + + if(!$password) + return; + + if(!$sync_to_ip) + return; + + $xmlrpc_sync_neighbor = $sync_to_ip; + if($config['system']['webgui']['protocol'] != "") { + $synchronizetoip = $config['system']['webgui']['protocol']; + $synchronizetoip .= "://"; + } + $port = $config['system']['webgui']['port']; + /* if port is empty lets rely on the protocol selection */ + if($port == "") { + if($config['system']['webgui']['protocol'] == "http") + $port = "80"; + else + $port = "443"; + } + $synchronizetoip .= $sync_to_ip; + + /* xml will hold the sections to sync */ + $xml = array(); + $xml['varnishcustomvcl'] = $config['installedpackages']['varnishcustomvcl']; + $xml['varnishbackends'] = $config['installedpackages']['varnishbackends']; + $xml['varnishlbdirectors'] = $config['installedpackages']['varnishlbdirectors']; + $xml['varnishsettings'] = $config['installedpackages']['varnishsettings']; + + /* assemble xmlrpc payload */ + $params = array( + XML_RPC_encode($password), + XML_RPC_encode($xml) + ); + + /* set a few variables needed for sync code borrowed from filter.inc */ + $url = $synchronizetoip; + log_error("Beginning Varnish XMLRPC sync to {$url}:{$port}."); + $method = 'pfsense.merge_installedpackages_section_xmlrpc'; + $msg = new XML_RPC_Message($method, $params); + $cli = new XML_RPC_Client('/xmlrpc.php', $url, $port); + $cli->setCredentials('admin', $password); + if($g['debug']) + $cli->setDebug(1); + /* send our XMLRPC message and timeout after 250 seconds */ + $resp = $cli->send($msg, "250"); + if(!$resp) { + $error = "A communications error occurred while attempting varnish XMLRPC sync with {$url}:{$port}."; + log_error($error); + file_notice("sync_settings", $error, "varnish Settings Sync", ""); + } elseif($resp->faultCode()) { + $cli->setDebug(1); + $resp = $cli->send($msg, "250"); + $error = "An error code was received while attempting varnish XMLRPC sync with {$url}:{$port} - Code " . $resp->faultCode() . ": " . $resp->faultString(); + log_error($error); + file_notice("sync_settings", $error, "varnish Settings Sync", ""); + } else { + log_error("varnish XMLRPC sync successfully completed with {$url}:{$port}."); + } + + /* tell varnish to reload our settings on the destionation sync host. */ + $method = 'pfsense.exec_php'; + $execcmd = "require_once('/usr/local/pkg/varnish.inc');\n"; + $execcmd .= "sync_package_varnish();\nvarnish_start();"; + /* assemble xmlrpc payload */ + $params = array( + XML_RPC_encode($password), + XML_RPC_encode($execcmd) + ); + + log_error("varnish XMLRPC reload data {$url}:{$port}."); + $msg = new XML_RPC_Message($method, $params); + $cli = new XML_RPC_Client('/xmlrpc.php', $url, $port); + $cli->setCredentials('admin', $password); + $resp = $cli->send($msg, "250"); + if(!$resp) { + $error = "A communications error occurred while attempting varnish XMLRPC sync with {$url}:{$port} (pfsense.exec_php)."; + log_error($error); + file_notice("sync_settings", $error, "varnish Settings Sync", ""); + } elseif($resp->faultCode()) { + $cli->setDebug(1); + $resp = $cli->send($msg, "250"); + $error = "An error code was received while attempting varnish XMLRPC sync with {$url}:{$port} - Code " . $resp->faultCode() . ": " . $resp->faultString(); + log_error($error); + file_notice("sync_settings", $error, "varnish Settings Sync", ""); + } else { + log_error("varnish XMLRPC reload data success with {$url}:{$port} (pfsense.exec_php)."); + } + +} + +?> -- cgit v1.2.3