traefik routes cacheable content to varnish.

Web & varnish IP addresses generated to group vars from pulumi state

ansible/group_vars/haproxy.yml

@@ -1,12 +1,21 @@
-haproxy_traefik_ip: 
-  - "10.24.1.1"
-  - "10.25.0.4"
 haproxy_traefik_port: 80
+haproxy_varnish_port: 80
 haproxy_cookie_value: "{{ inventory_hostname | default('server-1') }}"
 haproxy_dynamic_cookie_key: "mysecretphrase"
 haproxy_stats_auth: "admin:strongpassword"
 haproxy_certs_dir: "/etc/haproxy/certs"
-
 certbot_cloudflare_secrets_dir: "/root/.secrets/certbot"
 certbot_cloudflare_ini_path: "/root/.secrets/certbot/cloudflare.ini"
-certbot_cloudflare_api_token: "REPLACE_WITH_REAL_TOKEN"
+haproxy_varnish_ip:
+  - 10.24.2.1
+  - 10.24.2.2
+  - 10.25.2.1
+  - 10.25.2.2
+haproxy_traefik_ip:
+  - 10.24.3.6
+  - 10.24.3.3
+  - 10.25.3.4
+certbot_cloudflare_domains:
+  - k9e.no
+  - planetposen.no
+  - whoami.schleppe.cloud

ansible/roles/haproxy/templates/haproxy.cfg.j2

@@ -35,13 +35,37 @@ defaults
     errorfile 503 /etc/haproxy/errors/503.http
     errorfile 504 /etc/haproxy/errors/504.http
 
-# Front door: public HTTP
-frontend fe_http
+# Front door: main frontend dedicated to end users
+frontend ft_web
     bind :80
 
-    http-request set-header X-Forwarded-Proto https
-    option forwardfor
+    # Cache routing acl definitions
+    acl static_content path_end .jpg .jpeg .gif .png .css .js .htm .html
+    acl pseudo_static path_end .php ! path_beg /dynamic/
+    acl image_php path_beg /images.php
+    acl varnish_available nbsrv(bk_varnish_uri) ge 1
 
+    # Caches health detection + routing decision
+    use_backend bk_varnish_uri if varnish_available static_content
+    use_backend bk_varnish_uri if varnish_available pseudo_static
+    use_backend bk_varnish_url_param if varnish_available image_php
+
+    # Read debug query parameter
+    http-request set-var(txn.debug) urlp(debug)
+    # Define what "debug enabled" means
+    acl debug_enabled var(txn.debug) -m str -i 1 true yes on
+    # Debug headers
+    http-request set-var(txn.http_ver) req.ver
+    http-response add-header X-HA-HTTP-Version	%[var(txn.http_ver)] if debug_enabled
+    http-response add-header X-HA-TLS-Version	%[ssl_fc_protocol] if debug_enabled
+    http-response add-header X-HA-Frontend	%[fe_name] if debug_enabled
+    http-response add-header X-HA-Backend  	%[be_name] if debug_enabled
+    http-response add-header X-HA-Server   	%[srv_name] if debug_enabled
+    http-response add-header X-HA-Server	%[hostname] if debug_enabled
+    http-response add-header X-Debug-Client-IP	%[src] if debug_enabled
+    http-response add-header Cache-Control	no-store if debug_enabled
+
+    # dynamic content or all caches are unavailable
     default_backend be_traefik_http
 
 # Front door: public HTTPS
@@ -58,47 +82,45 @@ frontend fe_https
     # acl is_h2 ssl_fc_alpn -i h2
     # http-response set-header Alt-Svc "h3=\":443\"; ma=900" if is_h2
 
-    # =========================================================
-    # Debug response headers (enabled via ?debug=1)
+    # Cache routing acl definitions
+    acl static_content path_end .jpg .jpeg .gif .png .css .js .htm .html
+    acl pseudo_static path_end .php ! path_beg /dynamic/
+    acl image_php path_beg /images.php
+    acl varnish_available nbsrv(bk_varnish_uri) ge 1
+
+    # Caches health detection + routing decision
+    use_backend bk_varnish_uri if varnish_available static_content
+    use_backend bk_varnish_uri if varnish_available pseudo_static
+    use_backend bk_varnish_url_param if varnish_available image_php
 
     # Read debug query parameter
     http-request set-var(txn.debug) urlp(debug)
-
     # Define what "debug enabled" means
     acl debug_enabled var(txn.debug) -m str -i 1 true yes on
-
+    # Debug headers
     http-request set-var(txn.http_ver) req.ver
-    http-response add-header X-Debug-HTTP-Version %[var(txn.http_ver)] if debug_enabled
-    http-response add-header X-Debug-Served-By haproxy-https if debug_enabled
-    http-response add-header X-Debug-Frontend %[fe_name]    if debug_enabled
-    http-response add-header X-Debug-Backend  %[be_name]    if debug_enabled
-    http-response add-header X-Debug-Server   %[srv_name]   if debug_enabled
-
-    # Client & network
-    http-response add-header X-Debug-Client-IP   %[src] if debug_enabled
-    # http-response add-header X-Debug-Client-Port %[sp]  if debug_enabled
-    # http-response add-header X-Debug-XFF %[req.hdr(X-Forwarded-For)] if debug_enabled
-
-    # TLS / HTTPS details
-    http-response add-header X-Debug-TLS %[ssl_fc] if debug_enabled
-    http-response add-header X-Debug-TLS-Version %[ssl_fc_protocol] if debug_enabled
-    http-response add-header X-Debug-TLS-Cipher  %[ssl_fc_cipher]   if debug_enabled
-
-    # Request identity & correlation
-    http-response add-header X-Debug-Request-ID %[unique-id] if debug_enabled
-    http-response add-header X-Debug-Method     %[method]   if debug_enabled
-
-    # Safety: prevent caching of debug responses
-    http-response add-header Cache-Control no-store if debug_enabled
+    http-response add-header X-HA-HTTP-Version	%[var(txn.http_ver)] if debug_enabled
+    http-response add-header X-HA-TLS-Version	%[ssl_fc_protocol] if debug_enabled
+    http-response add-header X-HA-Frontend	%[fe_name] if debug_enabled
+    http-response add-header X-HA-Backend  	%[be_name] if debug_enabled
+    http-response add-header X-HA-Server   	%[srv_name] if debug_enabled
+    http-response add-header X-HA-Server	%[hostname] if debug_enabled
+    http-response add-header X-Debug-Client-IP	%[src] if debug_enabled
+    http-response add-header Cache-Control	no-store if debug_enabled
 
+    # dynamic content or all caches are unavailable
     default_backend be_traefik_http
 
-
 # Backend: Traefik VM
 backend be_traefik_http
     mode http
     balance roundrobin
-    cookie LB_SERVER insert indirect nocache dynamic
+    # app servers must say if everything is fine on their side
+    # and they can process requests
+    option httpchk
+    option httpchk GET /appcheck
+    http-check expect rstring [oO][kK]
+    cookie LB_SERVER insert indirect nocache
     dynamic-cookie-key {{ haproxy_dynamic_cookie_key }}
 
     # Health check: Traefik should respond with 404 for unknown host; that's still "alive".
@@ -109,6 +131,39 @@ backend be_traefik_http
         server traefik{{ loop.index }} {{ ip }}:{{ haproxy_traefik_port }} check cookie {{ haproxy_cookie_value }}
     {% endfor %}
 
+# VARNISH
+# static backend with balance based on the uri, including the query string
+# to avoid caching an object on several caches
+backend bk_varnish_uri
+    balance uri # in latest HAProxy version, one can add 'whole' keyword
+
+    # Varnish must tell it's ready to accept traffic
+    option httpchk HEAD /varnishcheck
+    http-check expect status 200
+
+    # client IP information
+    option forwardfor
+
+    # avoid request redistribution when the number of caches changes (crash or start up)
+    hash-type consistent
+    {% for ip in haproxy_varnish_ip %}
+        server varnish{{ loop.index }} {{ ip }}:{{ haproxy_varnish_port }} check
+    {% endfor %}
+
+# cache backend with balance based on the value of the URL parameter called "id"
+# to avoid caching an object on several caches
+backend bk_varnish_url_param
+    balance url_param id
+
+    # client IP information
+    option forwardfor
+
+    # avoid request redistribution when the number of caches changes (crash or start up)
+    hash-type consistent
+    {% for ip in haproxy_varnish_ip %}
+        server varnish{{ loop.index }} {{ ip }}:{{ haproxy_varnish_port }} track bk_varnish_uri/varnish{{ loop.index }}
+    {% endfor %}
+
 # Frontend: HAProxy prometheus exporter metrics
 frontend fe_metrics
     bind :8405

ansible/scripts/generate-inventory.sh

@@ -0,0 +1,43 @@
+#!/usr/local/bin/bash
+#
+# Usage: ./scripts/generate-inventory.sh | pbcopy
+
+cd ../hetzner-pulumi
+pulumi stack output --json | jq -r '
+  # extract dc (nbg / va) positionally from hostname
+  def dc:
+    (.name | capture("-(?<dc>nbg|hel|ash|va)[0-9]*-").dc);
+
+  def region:
+    if dc == "nbg" then "eu" else "us" end;
+
+  def pad($n):
+    tostring as $s
+    | ($n - ($s|length)) as $k
+    | if $k > 0 then ($s + (" " * $k)) else $s end;
+
+  .inventory.vms
+  | map({
+      region: region,
+      role:   (.name | split("-")[0]),
+      idx:    (.name | capture("-(?<n>[0-9]+)$").n),
+      ip:     .publicIpv4,
+      dc:     dc
+    })
+  | group_by(.region)
+  | .[]
+  | .[0].region as $r
+  | "[\($r)]",
+    (
+      sort_by(.role, (.idx | tonumber))
+      | .[]
+      | (
+          ("\(.role)-\(.dc)-\(.idx)" | pad(15)) +
+          ("ansible_host=\(.ip)"     | pad(30)) +
+          ("ansible_port=22"         | pad(18)) +
+          "ansible_user=root"
+        )
+    ),
+    ""
+'
+

ansible/scripts/update-config_certbot-domains.sh

@@ -0,0 +1,14 @@
+#!/usr/local/bin/bash
+#
+# Usage: ./scripts/update-config_certbot-domains.sh | pbcopy
+
+CERTBOT_EXPORT_KEY=certbot_cloudflare_domains
+
+EXPORT_VARIABLES="$(pwd)/group_vars/haproxy.yml"
+yq -i 'del(.certbot_cloudflare_domains)' $EXPORT_VARIABLES
+
+cd ../hetzner-pulumi
+pulumi stack output --json | jq -r --arg key $CERTBOT_EXPORT_KEY '
+  ($key + ":\n") +
+  (.inventory.domains | map("  - " + .) | join("\n"))
+' >> $EXPORT_VARIABLES

ansible/scripts/update-config_varnish-ips.sh

@@ -0,0 +1,20 @@
+#!/usr/local/bin/bash
+#
+# Usage: ./scripts/update-config_varnishserver-ips.sh
+
+IP_EXPORT_KEY=haproxy_varnish_ip
+ANSIBLE_DIR="$(pwd)"
+PULIMI_DIR="$(pwd)/../hetzner-pulumi"
+
+EXPORT_VARIABLES="$(pwd)/group_vars/haproxy.yml"
+yq -i 'del(.haproxy_varnish_ip)' $EXPORT_VARIABLES
+
+cd $PULIMI_DIR
+pulumi stack output --json | jq -r --arg key $IP_EXPORT_KEY '
+  def varnish_private_ips:
+    .inventory.vms
+    | map(select(.name | startswith("varnish")) | .privateIp);
+
+  ($key + ":\n") +
+  (varnish_private_ips | map("  - " + .) | join("\n"))
+' >> $EXPORT_VARIABLES

ansible/scripts/update-config_webserver-ips.sh

@@ -0,0 +1,35 @@
+#!/usr/local/bin/bash
+#
+# Usage: ./scripts/update-config_webserver-ips.sh
+
+IP_EXPORT_KEY=haproxy_traefik_ip
+ANSIBLE_DIR="$(pwd)"
+PULIMI_DIR="$(pwd)/../hetzner-pulumi"
+
+EXPORT_VARIABLES="$(pwd)/group_vars/haproxy.yml"
+yq -i 'del(.haproxy_traefik_ip)' $EXPORT_VARIABLES
+
+cd ../hetzner-pulumi
+pulumi stack output --json | jq -r --arg key $IP_EXPORT_KEY '
+  def web_private_ips:
+    .inventory.vms
+    | map(select(.name | startswith("web")) | .privateIp);
+
+  ($key + ":\n") +
+  (web_private_ips | map("  - " + .) | join("\n"))
+' >> $EXPORT_VARIABLES
+
+cd $ANSIBLE_DIR
+EXPORT_VARIABLES="$(pwd)/group_vars/varnish.yml"
+yq -i 'del(.haproxy_traefik_ip)' $EXPORT_VARIABLES
+
+cd $PULIMI_DIR
+pulumi stack output --json | jq -r --arg key $IP_EXPORT_KEY '
+  def varnish_private_ips:
+    .inventory.vms
+    | map(select(.name | startswith("web")) | .privateIp);
+
+  ($key + ":\n") +
+  (varnish_private_ips | map("  - " + .) | join("\n"))
+' >> $EXPORT_VARIABLES
+