diff --git a/ansible/group_vars/haproxy.yml b/ansible/group_vars/haproxy.yml index 6d53a1e..5ed7723 100644 --- a/ansible/group_vars/haproxy.yml +++ b/ansible/group_vars/haproxy.yml @@ -1,12 +1,21 @@ -haproxy_traefik_ip: - - "10.24.1.1" - - "10.25.0.4" haproxy_traefik_port: 80 +haproxy_varnish_port: 80 haproxy_cookie_value: "{{ inventory_hostname | default('server-1') }}" haproxy_dynamic_cookie_key: "mysecretphrase" haproxy_stats_auth: "admin:strongpassword" haproxy_certs_dir: "/etc/haproxy/certs" - certbot_cloudflare_secrets_dir: "/root/.secrets/certbot" certbot_cloudflare_ini_path: "/root/.secrets/certbot/cloudflare.ini" -certbot_cloudflare_api_token: "REPLACE_WITH_REAL_TOKEN" +haproxy_varnish_ip: + - 10.24.2.1 + - 10.24.2.2 + - 10.25.2.1 + - 10.25.2.2 +haproxy_traefik_ip: + - 10.24.3.6 + - 10.24.3.3 + - 10.25.3.4 +certbot_cloudflare_domains: + - k9e.no + - planetposen.no + - whoami.schleppe.cloud diff --git a/ansible/roles/haproxy/templates/haproxy.cfg.j2 b/ansible/roles/haproxy/templates/haproxy.cfg.j2 index cdb4143..f09a70c 100644 --- a/ansible/roles/haproxy/templates/haproxy.cfg.j2 +++ b/ansible/roles/haproxy/templates/haproxy.cfg.j2 @@ -35,13 +35,37 @@ defaults errorfile 503 /etc/haproxy/errors/503.http errorfile 504 /etc/haproxy/errors/504.http -# Front door: public HTTP -frontend fe_http +# Front door: main frontend dedicated to end users +frontend ft_web bind :80 - http-request set-header X-Forwarded-Proto https - option forwardfor + # Cache routing acl definitions + acl static_content path_end .jpg .jpeg .gif .png .css .js .htm .html + acl pseudo_static path_end .php ! path_beg /dynamic/ + acl image_php path_beg /images.php + acl varnish_available nbsrv(bk_varnish_uri) ge 1 + # Caches health detection + routing decision + use_backend bk_varnish_uri if varnish_available static_content + use_backend bk_varnish_uri if varnish_available pseudo_static + use_backend bk_varnish_url_param if varnish_available image_php + + # Read debug query parameter + http-request set-var(txn.debug) urlp(debug) + # Define what "debug enabled" means + acl debug_enabled var(txn.debug) -m str -i 1 true yes on + # Debug headers + http-request set-var(txn.http_ver) req.ver + http-response add-header X-HA-HTTP-Version %[var(txn.http_ver)] if debug_enabled + http-response add-header X-HA-TLS-Version %[ssl_fc_protocol] if debug_enabled + http-response add-header X-HA-Frontend %[fe_name] if debug_enabled + http-response add-header X-HA-Backend %[be_name] if debug_enabled + http-response add-header X-HA-Server %[srv_name] if debug_enabled + http-response add-header X-HA-Server %[hostname] if debug_enabled + http-response add-header X-Debug-Client-IP %[src] if debug_enabled + http-response add-header Cache-Control no-store if debug_enabled + + # dynamic content or all caches are unavailable default_backend be_traefik_http # Front door: public HTTPS @@ -58,47 +82,45 @@ frontend fe_https # acl is_h2 ssl_fc_alpn -i h2 # http-response set-header Alt-Svc "h3=\":443\"; ma=900" if is_h2 - # ========================================================= - # Debug response headers (enabled via ?debug=1) + # Cache routing acl definitions + acl static_content path_end .jpg .jpeg .gif .png .css .js .htm .html + acl pseudo_static path_end .php ! path_beg /dynamic/ + acl image_php path_beg /images.php + acl varnish_available nbsrv(bk_varnish_uri) ge 1 + + # Caches health detection + routing decision + use_backend bk_varnish_uri if varnish_available static_content + use_backend bk_varnish_uri if varnish_available pseudo_static + use_backend bk_varnish_url_param if varnish_available image_php # Read debug query parameter http-request set-var(txn.debug) urlp(debug) - # Define what "debug enabled" means acl debug_enabled var(txn.debug) -m str -i 1 true yes on - + # Debug headers http-request set-var(txn.http_ver) req.ver - http-response add-header X-Debug-HTTP-Version %[var(txn.http_ver)] if debug_enabled - http-response add-header X-Debug-Served-By haproxy-https if debug_enabled - http-response add-header X-Debug-Frontend %[fe_name] if debug_enabled - http-response add-header X-Debug-Backend %[be_name] if debug_enabled - http-response add-header X-Debug-Server %[srv_name] if debug_enabled - - # Client & network - http-response add-header X-Debug-Client-IP %[src] if debug_enabled - # http-response add-header X-Debug-Client-Port %[sp] if debug_enabled - # http-response add-header X-Debug-XFF %[req.hdr(X-Forwarded-For)] if debug_enabled - - # TLS / HTTPS details - http-response add-header X-Debug-TLS %[ssl_fc] if debug_enabled - http-response add-header X-Debug-TLS-Version %[ssl_fc_protocol] if debug_enabled - http-response add-header X-Debug-TLS-Cipher %[ssl_fc_cipher] if debug_enabled - - # Request identity & correlation - http-response add-header X-Debug-Request-ID %[unique-id] if debug_enabled - http-response add-header X-Debug-Method %[method] if debug_enabled - - # Safety: prevent caching of debug responses - http-response add-header Cache-Control no-store if debug_enabled + http-response add-header X-HA-HTTP-Version %[var(txn.http_ver)] if debug_enabled + http-response add-header X-HA-TLS-Version %[ssl_fc_protocol] if debug_enabled + http-response add-header X-HA-Frontend %[fe_name] if debug_enabled + http-response add-header X-HA-Backend %[be_name] if debug_enabled + http-response add-header X-HA-Server %[srv_name] if debug_enabled + http-response add-header X-HA-Server %[hostname] if debug_enabled + http-response add-header X-Debug-Client-IP %[src] if debug_enabled + http-response add-header Cache-Control no-store if debug_enabled + # dynamic content or all caches are unavailable default_backend be_traefik_http - # Backend: Traefik VM backend be_traefik_http mode http balance roundrobin - cookie LB_SERVER insert indirect nocache dynamic + # app servers must say if everything is fine on their side + # and they can process requests + option httpchk + option httpchk GET /appcheck + http-check expect rstring [oO][kK] + cookie LB_SERVER insert indirect nocache dynamic-cookie-key {{ haproxy_dynamic_cookie_key }} # Health check: Traefik should respond with 404 for unknown host; that's still "alive". @@ -109,6 +131,39 @@ backend be_traefik_http server traefik{{ loop.index }} {{ ip }}:{{ haproxy_traefik_port }} check cookie {{ haproxy_cookie_value }} {% endfor %} +# VARNISH +# static backend with balance based on the uri, including the query string +# to avoid caching an object on several caches +backend bk_varnish_uri + balance uri # in latest HAProxy version, one can add 'whole' keyword + + # Varnish must tell it's ready to accept traffic + option httpchk HEAD /varnishcheck + http-check expect status 200 + + # client IP information + option forwardfor + + # avoid request redistribution when the number of caches changes (crash or start up) + hash-type consistent + {% for ip in haproxy_varnish_ip %} + server varnish{{ loop.index }} {{ ip }}:{{ haproxy_varnish_port }} check + {% endfor %} + +# cache backend with balance based on the value of the URL parameter called "id" +# to avoid caching an object on several caches +backend bk_varnish_url_param + balance url_param id + + # client IP information + option forwardfor + + # avoid request redistribution when the number of caches changes (crash or start up) + hash-type consistent + {% for ip in haproxy_varnish_ip %} + server varnish{{ loop.index }} {{ ip }}:{{ haproxy_varnish_port }} track bk_varnish_uri/varnish{{ loop.index }} + {% endfor %} + # Frontend: HAProxy prometheus exporter metrics frontend fe_metrics bind :8405 diff --git a/ansible/scripts/generate-inventory.sh b/ansible/scripts/generate-inventory.sh new file mode 100755 index 0000000..994a066 --- /dev/null +++ b/ansible/scripts/generate-inventory.sh @@ -0,0 +1,43 @@ +#!/usr/local/bin/bash +# +# Usage: ./scripts/generate-inventory.sh | pbcopy + +cd ../hetzner-pulumi +pulumi stack output --json | jq -r ' + # extract dc (nbg / va) positionally from hostname + def dc: + (.name | capture("-(?nbg|hel|ash|va)[0-9]*-").dc); + + def region: + if dc == "nbg" then "eu" else "us" end; + + def pad($n): + tostring as $s + | ($n - ($s|length)) as $k + | if $k > 0 then ($s + (" " * $k)) else $s end; + + .inventory.vms + | map({ + region: region, + role: (.name | split("-")[0]), + idx: (.name | capture("-(?[0-9]+)$").n), + ip: .publicIpv4, + dc: dc + }) + | group_by(.region) + | .[] + | .[0].region as $r + | "[\($r)]", + ( + sort_by(.role, (.idx | tonumber)) + | .[] + | ( + ("\(.role)-\(.dc)-\(.idx)" | pad(15)) + + ("ansible_host=\(.ip)" | pad(30)) + + ("ansible_port=22" | pad(18)) + + "ansible_user=root" + ) + ), + "" +' + diff --git a/ansible/scripts/update-config_certbot-domains.sh b/ansible/scripts/update-config_certbot-domains.sh new file mode 100644 index 0000000..a95708d --- /dev/null +++ b/ansible/scripts/update-config_certbot-domains.sh @@ -0,0 +1,14 @@ +#!/usr/local/bin/bash +# +# Usage: ./scripts/update-config_certbot-domains.sh | pbcopy + +CERTBOT_EXPORT_KEY=certbot_cloudflare_domains + +EXPORT_VARIABLES="$(pwd)/group_vars/haproxy.yml" +yq -i 'del(.certbot_cloudflare_domains)' $EXPORT_VARIABLES + +cd ../hetzner-pulumi +pulumi stack output --json | jq -r --arg key $CERTBOT_EXPORT_KEY ' + ($key + ":\n") + + (.inventory.domains | map(" - " + .) | join("\n")) +' >> $EXPORT_VARIABLES diff --git a/ansible/scripts/update-config_varnish-ips.sh b/ansible/scripts/update-config_varnish-ips.sh new file mode 100644 index 0000000..e62d37a --- /dev/null +++ b/ansible/scripts/update-config_varnish-ips.sh @@ -0,0 +1,20 @@ +#!/usr/local/bin/bash +# +# Usage: ./scripts/update-config_varnishserver-ips.sh + +IP_EXPORT_KEY=haproxy_varnish_ip +ANSIBLE_DIR="$(pwd)" +PULIMI_DIR="$(pwd)/../hetzner-pulumi" + +EXPORT_VARIABLES="$(pwd)/group_vars/haproxy.yml" +yq -i 'del(.haproxy_varnish_ip)' $EXPORT_VARIABLES + +cd $PULIMI_DIR +pulumi stack output --json | jq -r --arg key $IP_EXPORT_KEY ' + def varnish_private_ips: + .inventory.vms + | map(select(.name | startswith("varnish")) | .privateIp); + + ($key + ":\n") + + (varnish_private_ips | map(" - " + .) | join("\n")) +' >> $EXPORT_VARIABLES diff --git a/ansible/scripts/update-config_webserver-ips.sh b/ansible/scripts/update-config_webserver-ips.sh new file mode 100644 index 0000000..862b162 --- /dev/null +++ b/ansible/scripts/update-config_webserver-ips.sh @@ -0,0 +1,35 @@ +#!/usr/local/bin/bash +# +# Usage: ./scripts/update-config_webserver-ips.sh + +IP_EXPORT_KEY=haproxy_traefik_ip +ANSIBLE_DIR="$(pwd)" +PULIMI_DIR="$(pwd)/../hetzner-pulumi" + +EXPORT_VARIABLES="$(pwd)/group_vars/haproxy.yml" +yq -i 'del(.haproxy_traefik_ip)' $EXPORT_VARIABLES + +cd ../hetzner-pulumi +pulumi stack output --json | jq -r --arg key $IP_EXPORT_KEY ' + def web_private_ips: + .inventory.vms + | map(select(.name | startswith("web")) | .privateIp); + + ($key + ":\n") + + (web_private_ips | map(" - " + .) | join("\n")) +' >> $EXPORT_VARIABLES + +cd $ANSIBLE_DIR +EXPORT_VARIABLES="$(pwd)/group_vars/varnish.yml" +yq -i 'del(.haproxy_traefik_ip)' $EXPORT_VARIABLES + +cd $PULIMI_DIR +pulumi stack output --json | jq -r --arg key $IP_EXPORT_KEY ' + def varnish_private_ips: + .inventory.vms + | map(select(.name | startswith("web")) | .privateIp); + + ($key + ":\n") + + (varnish_private_ips | map(" - " + .) | join("\n")) +' >> $EXPORT_VARIABLES +