SearchVarnish Cache Plus
vmod-http allows for HTTP communication to be made to other services from VCL.
This includes communication back to Varnish in the form of a new request.
vmod-http supports both synchronous and asynchronous calls and has automatic
loop detection and prefetch URL generation capabilities.
vmod-http supports all string and integer CURLOPT options
via http.req_set_sparam() and http.req_set_iparam().
Prefetch the next video segment:
vcl 4.0;
import http;
sub vcl_recv
{
if (req.url ~ "^/vod/") {
http.init(0);
http.req_copy_headers(0);
http.req_set_method(0, "HEAD");
// Generate the next URL by incrementing the first
// number sequence found and send it back to Varnish.
// If no number sequence is found, the request is skipped.
http.req_set_url(0, http.prefetch_next_url());
http.req_send_and_finish(0);
}
}
Prefetch the next two search pages using the page query parameter:
vcl 4.0;
import http;
sub vcl_recv
{
http.init(0);
// Allow this request to loop at most twice
http.req_set_max_loops(0, 2);
http.req_copy_headers(0);
http.req_set_method(0, "HEAD");
// Generate the next URL by incrementing the first number
// sequence found after 'page=' and send it back to Varnish.
// If 'page=' and the following number sequence is not found,
// the request is skipped.
http.req_set_url(0, http.prefetch_next_url("page="));
http.req_send_and_finish(0);
}
Prefetch using a Link response header hint.
Example: Link: </images/big.jpeg>; rel=prefetch
vcl 4.0;
import http;
sub vcl_recv {
// Store Varnish's local address for later use
set req.http.X-prefetch = http.varnish_url("/");
}
sub vcl_backend_response {
if (beresp.http.Link ~ "<.+>.*(prefetch|next)") {
// Pull out the Link URL
set bereq.http.X-link = regsub(beresp.http.Link, "^.*<([^>]*)>.*$", "\1");
set bereq.http.X-prefetch = regsub(bereq.http.X-prefetch, "/$", bereq.http.X-link);
// Prefetch the Link URL back thru Varnish
http.init(0);
http.req_copy_headers(0);
http.req_set_url(0, bereq.http.X-prefetch);
http.req_send_and_finish(0);
}
}
Send a request to another service and include the response cookie.
vcl 4.0;
import cookieplus;
import http;
sub vcl_recv
{
// We need to initialize the HTTP object here, as it is going to be used
// in vcl_deliver with `http.req_sent(0)`, which requires the object to be
// initialized before usage.
http.init(0);
// Send the request if we do not have a session
// We will read the response later
if (!cookieplus.get("session")) {
http.req_copy_headers(0);
http.req_set_url(0, "https://example.com/api/authorize");
http.req_send(0);
}
}
sub vcl_deliver
{
// Get the session response cookies
if (http.req_sent(0)) {
http.resp_wait(0);
if (http.resp_get_status(0) == 200) {
http.resp_copy_headers(0, "Set-Cookie");
} else {
return(synth(401, "UNAUTHORIZED"));
}
}
}
Send a read only copy of all requests to another service.
vcl 4.0;
import http;
import std;
sub vcl_recv
{
std.cache_req_body(100KB);
http.init(0);
http.req_copy_headers(0);
http.req_set_header(0, "X-Varnish-fork", http.version());
http.req_set_max_loops(0, -1); // Remove loop counter header
http.req_set_method(0, req.method);
http.req_copy_body(0);
http.req_set_url(0, "http://service_host" + req.url);
http.req_send_and_finish(0);
}
Send a POST request with a custom body using CURLOPT_POSTFIELDS.
The CURLOPT_ prefix is optional on all curl options.
vcl 4.0;
import http;
sub vcl_deliver
{
http.init(0);
http.req_set_method(0, "POST");
http.req_set_url(0, "https://some.host/data/service");
http.req_set_sparam(0, "POSTFIELDS", "postdata is here");
http.req_send_and_finish(0);
}
vcl 4.0;
import goto;
import http;
import json;
sub vcl_init {
new api = goto.dns_director("api.example.com");
}
sub vcl_recv {
if (req.http.Host == "www.example.com") {
// Pull JSON thru Varnish
http.init(0);
http.req_copy_headers(0);
http.req_set_url(0, http.varnish_url("/auth/"));
http.req_set_header(0, "Host", "api.example.com");
http.req_send(0);
http.resp_wait(0);
// Invalid response
if (http.resp_get_status(0) != 200) {
return (synth(401));
}
// Parse the response
json.parse(http.resp_get_body(0));
// Invalid authorization
if (!json.is_object() || !json.get("auth") || json.get("error")) {
return (synth(401));
}
// Pull data out of the JSON response
set req.http.X-auth = json.get("auth");
unset req.http.Authorization;
} else if (req.http.Host == "api.example.com") {
set req.backend_hint = api.backend();
return (hash);
}
}
VOID init(INT name, ENUM log_level)
Description
Initialize an http call, by name.
Return value
None
name
The name of the http instance.
By default, name values from 0 to 10 are supported.
The name must be used on all calls which reference this instance.
Referencing an uninitialized name in a http call will result in a Varnish assertion error.
log_level
The level of logging to perform for this request.
Options are NONE, LOW, MEDIUM, HIGH, DEBUG. Defaults to HIGH.
VOID finish(INT name)
Description
Cleanup and finish an http call. All resources associated with the http call
will be released, including the name. Referencing the name again will result
in a Varnish assertion error. name can be safely reused in another init() call.
Calling finish() on an http request is optional. All http calls will be safely finished
when both the parent request and the actual http call are completed.
Return value
None
name
The name of the http instance.
STRING prefetch_next_url(STRING prefix = "", STRING url_prefix = "", STRING url = "", INT count = 1, ENUM {DECIMAL, HEX, HEX_UPPER} base = DECIMAL)
Description
Generate the next prefetch URL. This works by finding a numeric sequence in the
current URL and incrementing it by count. If no numeric sequence is found, an empty URL
is returned which will cause the http request to be skipped. Cannot be used in a backend
context (sub vcl_backend_*).
Return value
The next prefetch URL with the numeric sequence incremented by count.
prefix
Optional. If prefix is used, the numeric sequence used is searched for after prefix.
prefix is a regular expression. If prefix is not found, an empty URL is
returned. This value is static and cannot change between calls.
url_prefix
Optional. url_prefix is the URL prefix. It is appended before req.url to
compose the full URL needed for the http request. It must start with http:// or
https://, followed by a hostname or IP, and an optional port. It must not contain a
trailing slash character, /. If not defined, server.ip and std.port(server.ip)
are used with http://. If the port detected is 443, https:// is used.
url
Optional. url is the URL string to use. If not defined, req.url is used.
count
Optional. count is the amount to add to the numeric sequence found when generating the
next prefetch URL. Defaults to 1.
base
Optional. base is the format that the url is in. Either DECIMAL, HEX, or
HEX_UPPER. DECIMAL indicates that the url string will be in decimal, base 10.
HEX indicates that the URL string will be in hexadecimal, base 16. It’s alpha
characters of the string are lower case. HEX_UPPER is the same as HEX but the
alpha characters are upper case. If not defined, base will be set to DECIMAL.
STRING varnish_url(STRING url)
Description
Generate a Varnish callback URL. This is done by appending the URL to
server.ip and std.port(server.ip) with http:// as the protocol.
If the port detected is 443, https:// is used. Cannot be used in a backend
context (sub vcl_backend_*).
Return value
The Varnish callback URL.
url
The URL to use.
STRING backend_url(BACKEND be, STRING url)
Description
Generate a URL from a backend or director. This is done by appending the URL to the resolved
backend’s IP address and port. If the SSL flag is set, https:// is used as the protocol.
Otherwise, http:// is used as the protocol. This can only be used in sub vcl_backend_fetch.
Return value
The generated URL.
url
The URL to use.
VOID req_set_url(INT name, STRING url)
Description
Set the URL for a request.
Return value
None
name
The name of the http instance.
url
The URL for a request. If url is empty, the request is not performed.
url must start with http:// or https:// and contain the host or IP destination
and the trailing URL.
VOID req_set_method(INT name, STRING method)
Description
Set the request method. Ex: GET, HEAD, POST.
Return value
None
name
The name of the http instance.
method
The method for this request. Defaults to GET.
VOID req_set_header(INT name, STRING header, STRING value)
Description
Set a request header.
Return value
None
name
The name of the http instance.
header
The header name.
value
The header value.
VOID req_copy_headers(INT name, STRING list)
Description
Copy all Varnish request headers into this request.
Return value
None
name
The name of the http instance.
list
Optional. A comma seperated list of request headers to copy. If blank, all request headers are copied over. Defaults to copying all headers.
VOID req_unset_header(INT name, STRING header)
Description
Unset a request header.
Return value
None
name
The name of the http instance.
header
The header name to unset.
VOID req_copy_body(INT name)
Description
Copy the request body to this request. std.cache_req_body() must be called before using this function.
Return value
None
name
The name of the http instance.
VOID req_set_sparam(INT name, STRING param, STRING value)
Description
Set a string parameter for this request.
Please see CURLOPT options for supported parameters.
The CURLOPT_ prefix is optional.
Return value
None
name
The name of the http instance.
param
The parameter name.
value
The parameter value.
VOID req_set_iparam(INT name, STRING param, INT value)
Description
Set an integer parameter for this request.
Please see CURLOPT options for supported parameters.
The CURLOPT_ prefix is optional.
Return value
None
name
The name of the http instance.
param
The parameter name.
value
The parameter value.
VOID req_set_max_loops(INT name, INT loops)
Description
Set the maximum number of times this request can loop when doing a request
back to Varnish. This is done via a special request header, VMOD-http-loops,
which keeps track of the number of times the request passes thru this VMOD.
This has a default value of 1, meaning this VMOD will only make a single
call to Varnish. Increasing this value in conjunction with prefetch_next_url()
allows multiple URLs to be prefetched from a single client call.
Return value
None
name
The name of the http instance.
loops
The maximum number of loops this request can have through Varnish.
Defaults to 1. A negative value removes the loop counting header
and potentially allows unlimited loops.
INT req_get_loops(INT name)
Description
Get the current loop iteration for this request. The first request starts at
0 and the first loop is 1.
Return value
The current loop iteration.
name
The name of the http instance.
BOOL req_is_loop(INT name)
Description
Return true if the request has looped at least once.
Return value
true if the request has looped more than once, otherwise false.
name
The name of the http instance.
VOID req_send(INT name)
Description
Send the request.
Return value
None
name
The name of the http instance.
VOID req_send_and_finish(INT name)
Description
Send the request and finish the name. Any overhead associated with response
parsing will be skipped. The request will be safely processed in the background
and finished when completed.
name cannot be used after calling this function.
Return value
None
name
The name of the http instance.
BOOL req_sent(INT name)
Description
Has the request been sent? If true, it is safe to call resp_wait(). Useful if
a request has been conditionally sent.
Return value
true if the request was sent, otherwise false.
name
The name of the http instance.
BOOL resp_is_ready(INT name)
Description
Is the response ready? Errors are considered ready responses. Can be called at anytime, before or after the request has been sent.
Return value
true if the response is ready, otherwise false.
name
The name of the http instance.
VOID resp_wait(INT name)
Description
Wait for a response. When this function returns, the response is ready. Errors are considered ready responses. Can only be called after a request has been sent.
Return value
None
name
The name of the http instance.
INT resp_get_status(INT name)
Get the status code for a response. If no response status was received, due to error, this function will return 0. Can only be called after a response is ready.
Return value
The response status code.
name
The name of the http instance.
INT resp_get_errorcode(INT name)
Description
Get the internal response error code. If no error has happened, this function
will return 0. Otherwise a non zero error code will be returned. Can only be called
after a response is ready.
Return value
The internal response error code. 0 if no error has occurred.
name
The name of the http instance.
STRING resp_get_reason(INT name)
Description
Get the response reason. Can only be called after a response is ready.
This is the string after the response status, ex: 200 OK, the reason is OK.
Return value
The response reason.
name
The name of the http instance.
STRING resp_get_header(INT name, STRING header, STRING default)
Description
Get a response header. If not found, default is returned.
Can only be called after a response is ready.
Return value
The header value if found, otherwise default.
name
The name of the http instance.
header
The response header name.
default
The default return value if header is not found.
VOID resp_copy_headers(INT name, STRING header)
Description
Copy all matching response headers into the Varnish response. Can only be called after a response is ready.
Return value
None
name
The name of the http instance.
header
The response header name. If blank, all response headers are copied over.
STRING resp_get_body(INT name)
Description
Get the response body. Can only be called after a response is ready.
Return value
The response body.
name
The name of the http instance.
BLOB resp_get_body_blob(INT name)
Description
Get the response body as a blob. Can only be called after a response is ready.
Return value
The response body as a blob.
name
The name of the http instance.
INT resp_get_body_len(INT name)
Description
Get the response body length. Can only be called after a response is ready.
Return value
The response body length.
name
The name of the http instance.
STRING version()
Description
Get the vmod-http version.
Return value
The version.
VOID debug_print(INT name)
Description
Print the request and response to varnishlog.
Return value
None
name
The name of the http instance.
VOID set_log_level(INT name, ENUM log_level)
Description
Change the logging level of a request.
Return value
None
name
The name of the http instance.
log_level
The level of logging to perform for this request.
Options are NONE, LOW, MEDIUM, HIGH, DEBUG.
VOID debug_print_stats()
Description
Print the internal vmod-http stats.
Return value
None