add redis handlers

plays/mysql.yml

@@ -0,0 +1,17 @@
+---
+- name: Install & configure mysql server
+  hosts: all
+  vars:
+    custom_firewall_ports:
+      - 111
+      - 2049
+      - 3306
+      - 6379
+    apt_packages:
+      - nfs-kernel-server
+    install_redis_exporter: true
+    install_mysqld_exporter: true
+    description: 'database & filestorage node for hosting app.retailor.io'
+  roles:
+    - role: roles/firewall
+    - role: roles/mysql

plays/redis.yml

@@ -0,0 +1,12 @@
+---
+- name: Install & configure redis
+  hosts: all
+  vars:
+    custom_firewall_ports:
+      - 6379
+    install_redis_exporter: true
+    description: 'cache node for retailor redis cache'
+  roles:
+    - role: roles/firewall
+    - role: roles/redis
+

plays/traefik.yml

@@ -4,5 +4,4 @@
   roles:
     - role: roles/firewall
     - role: roles/docker
-    - role: roles/env
     - role: roles/traefik

roles/mysql/defaults/main.yml

@@ -0,0 +1,36 @@
+---
+# ---------------------------------------------------------------------------
+mysql_root_password: "{{ lookup('env', 'MYSQL_ROOT_PASSWORD') | default('', true) }}"
+mysql_replication_user: "repl"
+mysql_replication_password: "{{ lookup('env', 'MYSQL_REPL_PASSWORD') | default('', true) }}"
+mysql_app_user: "{{ lookup('env', 'MYSQL_APP_USER') | default('appuser', true) }}"
+mysql_app_password: "{{ lookup('env', 'MYSQL_APP_PASSWORD') | default('', true) }}"
+mysql_app_database: "{{ lookup('env', 'MYSQL_APP_DATABASE') | default('appdb', true) }}"
+
+mysql_version: "{{ lookup('env', 'MYSQL_VERSION') | default('8.0', true) }}"
+mysql_server_id: "{{ lookup('env', 'HOSTNAME') | default('mysql', true) }}"
+
+# --- Sizing ------------------------------------------------------------
+# Set per-environment (override in host_vars if primary/replicas differ in size).
+# Rule of thumb: innodb_buffer_pool_size ~= 70-75% of RAM on a dedicated DB host.
+mysql_total_ram_mb: "{{ lookup('env', 'MYSQL_TOTAL_RAM_MB') | default(8192, true) | int }}"
+mysql_innodb_buffer_pool_size: "{{ (mysql_total_ram_mb | int * 0.70) | round | int }}M"
+
+mysql_max_connections: 200
+
+# --- Networking / security ----------------------------------------------
+mysql_bind_address: "0.0.0.0"     # bound to all interfaces; ufw restricts who can reach it
+mysql_port: 3306
+mysql_allowed_cidrs:              # who is allowed to hit port 3306 via firewall
+  - "{{ lookup('env', 'MYSQL_ALLOWED_CIDR') | default('10.0.1.0/24', true) }}"
+
+# --- Replication ----------------------------------------------------------
+mysql_gtid_mode: "ON"
+mysql_binlog_format: "ROW"
+mysql_binlog_expire_logs_seconds: 604800   # 7 days
+
+# --- Backups ---------------------------------------------------------------
+mysql_backup_enabled: "{{ lookup('env', 'MYSQL_BACKUP_ENABLED') | default('true', true) | bool }}"
+mysql_backup_dir: /var/backups/mysql
+mysql_backup_retention_days: 7
+mysql_backup_hour: 2    # 2am local, staggered per-host below if needed

roles/mysql/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: restart mysql
+  systemd:
+    name: mysql
+    state: restarted
+

roles/mysql/tasks/main.yml

@@ -0,0 +1,155 @@
+---
+- name: Install prerequisite packages
+  apt:
+    name:
+      - gnupg2
+      - lsb-release
+      - python3-pymysql   # required by Ansible's community.mysql modules
+      - ufw
+    state: present
+    update_cache: true
+
+- name: Install MySQL server
+  apt:
+    name:
+      - mysql-server-8.0
+      - mysql-client-8.0
+    state: present
+  notify: restart mysql
+
+- name: Ensure MySQL data/log directories exist with correct ownership
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: mysql
+    group: mysql
+    mode: '0750'
+  loop:
+    - /var/log/mysql
+    - "{{ mysql_backup_dir }}"
+
+- name: Deploy tuned my.cnf
+  template:
+    src: my.cnf.j2
+    dest: /etc/mysql/mysql.conf.d/99-production.cnf
+    owner: root
+    group: root
+    mode: '0644'
+    # validate: 'mysqld --defaults-file=%s --validate-config'
+  notify: restart mysql
+
+- name: Start and enable MySQL
+  systemd:
+    name: mysql
+    state: started
+    enabled: true
+
+- name: Set MySQL root password (idempotent — only runs if not already set)
+  community.mysql.mysql_user:
+    name: root
+    password: "{{ mysql_root_password }}"
+    host: localhost
+    login_unix_socket: /var/run/mysqld/mysqld.sock
+    state: present
+  no_log: true
+
+- name: Create .my.cnf for root so future plays authenticate automatically
+  template:
+    src: root_my_cnf.j2
+    dest: /root/.my.cnf
+    owner: root
+    group: root
+    mode: '0600'
+  no_log: true
+
+# --- mysql_secure_installation equivalent, done idempotently ---------------
+- name: Remove anonymous MySQL users
+  community.mysql.mysql_user:
+    name: ''
+    host_all: true
+    state: absent
+    login_unix_socket: /var/run/mysqld/mysqld.sock
+
+- name: Remove test database
+  community.mysql.mysql_db:
+    name: test
+    state: absent
+    login_unix_socket: /var/run/mysqld/mysqld.sock
+
+- name: Disallow remote root login
+  community.mysql.mysql_user:
+    name: root
+    host: "{{ item }}"
+    state: absent
+    login_unix_socket: /var/run/mysqld/mysqld.sock
+  loop:
+    - "{{ ansible_hostname }}"
+    - "%"
+  ignore_errors: true   # fine if they don't exist
+
+# --- Firewall: only allow MySQL port from known CIDRs ----------------------
+- name: Allow MySQL port from trusted CIDRs only
+  community.general.ufw:
+    rule: allow
+    port: "{{ mysql_port | string }}"
+    proto: tcp
+    from_ip: "{{ item }}"
+  loop: "{{ mysql_allowed_cidrs }}"
+
+- name: Reload ufw
+  community.general.ufw:
+    state: enabled
+
+# --- Application DB + user --------------------------------------------------
+- name: Create retailor database
+  community.mysql.mysql_db:
+    name: "{{ mysql_app_database }}"
+    state: present
+    login_unix_socket: /var/run/mysqld/mysqld.sock
+
+- name: Create forge user
+  community.mysql.mysql_user:
+    name: "{{ mysql_app_user }}"
+    password: "{{ mysql_app_password }}"
+    priv: "{{ mysql_app_database }}.*:ALL"
+    host: '%'
+    state: present
+    login_unix_socket: /var/run/mysqld/mysqld.sock
+  no_log: true
+
+- name: Create replication user
+  community.mysql.mysql_user:
+    name: "{{ mysql_replication_user }}"
+    password: "{{ mysql_replication_password }}"
+    priv: "*.*:REPLICATION SLAVE"
+    host: '%'
+    state: present
+    login_unix_socket: /var/run/mysqld/mysqld.sock
+  no_log: true
+
+- name: Deploy logrotate config
+  template:
+    src: logrotate_mysql.j2
+    dest: /etc/logrotate.d/mysql-server
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Deploy backup script
+  template:
+    src: mysql_backup.sh.j2
+    dest: /usr/local/bin/mysql_backup.sh
+    owner: root
+    group: root
+    mode: '0750'
+  when: mysql_backup_enabled
+
+- name: Schedule nightly backup cron
+  cron:
+    name: "mysql nightly backup"
+    hour: "{{ mysql_backup_hour }}"
+    minute: "0"
+    job: "/usr/local/bin/mysql_backup.sh >> /var/log/mysql/backup.log 2>&1"
+    user: root
+  when: mysql_backup_enabled
+

roles/mysql/templates/logrotate_mysql.j2

@@ -0,0 +1,14 @@
+/var/log/mysql/*.log {
+    daily
+    rotate 14
+    missingok
+    compress
+    delaycompress
+    notifempty
+    create 640 mysql adm
+    sharedscripts
+    postrotate
+        /usr/bin/mysqladmin flush-logs
+    endscript
+}
+

roles/mysql/templates/my.cnf.j2

@@ -0,0 +1,35 @@
+# Production MySQL config — generated by Ansible for retailor.io. Edit by hand might cause drift!
+
+[mysqld]
+bind-address            = {{ mysql_bind_address }}
+port                     = {{ mysql_port }}
+max_connections          = {{ mysql_max_connections }}
+server_id                = {{ mysql_server_id }}
+
+# --- InnoDB ------------------------------------------------------------
+innodb_buffer_pool_size  = {{ mysql_innodb_buffer_pool_size }}
+innodb_log_file_size     = 512M
+innodb_flush_log_at_trx_commit = 1        # full ACID durability; set to 2 only if you accept losing ~1s of txns on crash
+innodb_flush_method      = O_DIRECT
+innodb_file_per_table    = 1
+
+# --- Replication (GTID-based) -------------------------------------------
+gtid_mode                = {{ mysql_gtid_mode }}
+enforce_gtid_consistency = ON
+log_bin                  = /var/log/mysql/mysql-bin.log
+binlog_format             = {{ mysql_binlog_format }}
+binlog_expire_logs_seconds = {{ mysql_binlog_expire_logs_seconds }}
+sync_binlog               = 1
+read_only                 = {{ 'ON' if 'mysql_replica' in group_names else 'OFF' }}
+relay_log                 = /var/log/mysql/mysql-relay-bin
+
+# --- Logging -------------------------------------------------------------
+slow_query_log            = 1
+slow_query_log_file       = /var/log/mysql/mysql-slow.log
+long_query_time            = 2
+log_error                 = /var/log/mysql/error.log
+
+# --- Security --------------------------------------------------------------
+local_infile               = 0
+skip_name_resolve          = 1
+

roles/mysql/templates/root_my_cnf.j2

@@ -0,0 +1,4 @@
+[client]
+user = root
+password = "{{ mysql_root_password }}"
+

roles/redis/defaults/main.yml

@@ -0,0 +1,41 @@
+---
+# --- Role / topology ---
+redis_role: "{{ lookup('env', 'REDIS_ROLE') | default('primary', true) }}"               # primary | replica
+redis_primary_host: "{{ lookup('env', 'REDIS_PRIMARY_HOST') | default('', true) }}"
+
+# --- Network ---
+redis_bind_addresses: "127.0.0.1 10.0.0.0/8"
+redis_bind_addresses: "0.0.0.0"
+redis_port: 6379
+redis_trusted_cidr: "{{ lookup('env', 'REDIS_ALLOWED_CIDR') | default('10.0.1.0/24', true) }}"
+
+# --- Auth (override via vault; do NOT commit plaintext) ---
+redis_password: "{{ lookup('env', 'REDIS_PASSWORD') | default('', false) }}"
+
+# --- Memory ---
+redis_maxmemory: "{{ lookup('env', 'REDIS_TOTAL_RAM_GB') | default('2', true) | int }}gb"
+redis_maxmemory_policy: "allkeys-lru"  # use "noeviction" if Redis is a source of truth
+
+# --- Persistence ---
+redis_appendonly: "yes"
+redis_appendfsync: "everysec"          # everysec | always | no
+
+# --- Dangerous command handling ---
+redis_disable_flushall: true
+redis_disable_flushdb: true
+redis_disable_debug: true
+redis_disable_keys: true
+redis_rename_config: true
+redis_rename_shutdown: true
+
+# --- Package ---
+redis_package_state: present
+
+# --- Limits ---
+redis_nofile_limit: 65536
+
+# TODO(kevinmidboe): just reference the certs acme.sh generates
+redis_tls_enabled: "{{ lookup('env', 'REDIS_TLS_ENABLED') | default('', false) }}"
+redis_tls_cert_path: ""
+redis_tls_key_path: ""
+redis_tls_ca_path: ""

roles/redis/handlers/main.yml

@@ -0,0 +1,20 @@
+---
+- name: Reload sysctl
+  ansible.builtin.command: sysctl -p /etc/sysctl.d/99-redis.conf
+  changed_when: false
+ 
+- name: Reload systemd
+  ansible.builtin.systemd:
+    daemon_reload: true
+ 
+- name: Enable and start disable-thp
+  ansible.builtin.systemd:
+    name: disable-thp.service
+    enabled: true
+    state: started
+ 
+- name: Restart redis
+  ansible.builtin.systemd:
+    name: redis-server
+    state: restarted
+ 

roles/redis/tasks/main.yml

@@ -0,0 +1,155 @@
+---
+- name: Update apt cache
+  ansible.builtin.apt:
+    update_cache: true
+    cache_valid_time: 3600
+  when: ansible_os_family == "Debian"
+
+- name: Install redis-server and firewall tooling
+  ansible.builtin.apt:
+    name:
+      - redis-server
+      - ufw
+    state: "{{ redis_package_state }}"
+
+- name: Ensure redis data/log/run directories exist with correct ownership
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: redis
+    group: redis
+    mode: "0750"
+  loop:
+    - /var/lib/redis
+    - /var/log/redis
+    - /run/redis
+
+# --- Kernel tuning ---
+- name: Apply sysctl tuning for Redis
+  ansible.builtin.copy:
+    dest: /etc/sysctl.d/99-redis.conf
+    content: |
+      vm.overcommit_memory = 1
+      vm.swappiness = 1
+      net.core.somaxconn = 511
+    mode: "0644"
+  notify: Reload sysctl
+
+- name: Disable Transparent Huge Pages via systemd unit
+  ansible.builtin.copy:
+    dest: /etc/systemd/system/disable-thp.service
+    content: |
+      [Unit]
+      Description=Disable Transparent Huge Pages (THP)
+      Before=redis-server.service
+
+      [Service]
+      Type=oneshot
+      ExecStart=/bin/sh -c 'echo never > /sys/kernel/mm/transparent_hugepage/enabled'
+      ExecStart=/bin/sh -c 'echo never > /sys/kernel/mm/transparent_hugepage/defrag'
+      RemainAfterExit=yes
+
+      [Install]
+      WantedBy=multi-user.target
+    mode: "0644"
+  notify:
+    - Reload systemd
+    - Enable and start disable-thp
+
+- name: Raise nofile limits for redis user
+  ansible.builtin.copy:
+    dest: /etc/security/limits.d/redis.conf
+    content: |
+      redis soft nofile {{ redis_nofile_limit }}
+      redis hard nofile {{ redis_nofile_limit }}
+    mode: "0644"
+
+- name: Add systemd override for redis-server (limits + restart policy)
+  ansible.builtin.file:
+    path: /etc/systemd/system/redis-server.service.d
+    state: directory
+    mode: "0755"
+
+- name: Deploy systemd override.conf
+  ansible.builtin.copy:
+    dest: /etc/systemd/system/redis-server.service.d/override.conf
+    content: |
+      [Service]
+      LimitNOFILE={{ redis_nofile_limit }}
+      Restart=always
+      RestartSec=5
+    mode: "0644"
+  notify:
+    - Reload systemd
+    - Restart redis
+
+# --- Main config ---
+- name: Deploy redis.conf from template
+  ansible.builtin.template:
+    src: redis.conf.j2
+    dest: /etc/redis/redis.conf
+    owner: redis
+    group: redis
+    mode: "0640"
+  notify: Restart redis
+
+- name: Deploy logrotate config
+  ansible.builtin.copy:
+    dest: /etc/logrotate.d/redis-server
+    content: |
+      /var/log/redis/redis-server.log {
+        weekly
+        rotate 8
+        compress
+        delaycompress
+        missingok
+        notifempty
+        create 640 redis redis
+      }
+    mode: "0644"
+
+# --- Firewall ---
+- name: Allow SSH through ufw
+  community.general.ufw:
+    rule: allow
+    name: OpenSSH
+
+- name: Allow Redis port from trusted CIDR only
+  community.general.ufw:
+    rule: allow
+    port: "{{ redis_port | string }}"
+    proto: tcp
+    src: "{{ redis_trusted_cidr }}"
+
+- name: Set default ufw policies
+  community.general.ufw:
+    state: enabled
+    policy: deny
+    direction: incoming
+
+- name: Allow outgoing traffic
+  community.general.ufw:
+    policy: allow
+    direction: outgoing
+
+# --- Service ---
+- name: Ensure redis-server is enabled and started
+  ansible.builtin.systemd:
+    name: redis-server
+    enabled: true
+    state: started
+    daemon_reload: true
+
+- name: Verify Redis responds to PING
+  ansible.builtin.command: >
+    redis-cli -a {{ redis_password }} --no-auth-warning
+    -h 127.0.0.1 -p {{ redis_port }} ping
+  register: redis_ping_result
+  changed_when: false
+  no_log: true
+
+- name: Assert Redis is healthy
+  ansible.builtin.assert:
+    that:
+      - "'PONG' in redis_ping_result.stdout"
+    fail_msg: "Redis did not respond with PONG after configuration."

roles/traefik/defaults/main.yml

@@ -4,12 +4,15 @@ traefik_web_port: 80
 traefik_websecure_port: 443
 traefik_dashboard_port: 8080
 traefik_metrics_port: 8100
+
+traefik_image: "traefik:v3.1"
 traefik_dashboard_insecure: true   # ufw should only allow from tailscale or vnet
 traefik_acme_staging: "{{ lookup('env', 'TRAEFIK_ACME_STAGING') | default('false', true) }}"
 
-domain: "{{ lookup('env', 'DOMAIN') | default('', true) }}"
+traefik_acme_email: "{{ lookup('env', 'TRAEFIK_ACME_EMAIL') | default('', true) }}"
+traefik_domain: "{{ lookup('env', 'DOMAIN') | default('', true) }}"
 # Single source of truth for static routing. Expected to be overridden
 # from group_vars, -e, or an env-var lookup at the play level — see
 # playbook.yml for the TRAEFIK_ROUTES example.
-traefik_web_private_ips: "{{ lookup('env', 'WEB_IPS') | default('[]', true) | from_json }}"
+traefik_web_private_ips: "{{ lookup('env', 'WEB_IPS') | from_json }}"
 traefik_analytics_private_ip: "{{ lookup('env', 'ANALYTICS_IP') | default('', true) }}"

roles/traefik/tasks/install.yml

@@ -5,7 +5,7 @@
       traefik_web_private_ips is empty. Set the WEB_IPS env var
       (JSON list) before running, or override the variable directly, e.g.
       -e traefik_web_private_ips='["10.0.1.11","10.0.1.12"]'
-  when: env_vars.WEB_IPS | length == 0
+  when: traefik_web_private_ips | length == 0
 
 - name: Ensure Traefik config directories exist
   ansible.builtin.file:

roles/traefik/templates/dynamic-analytics.yml.j2

@@ -3,22 +3,22 @@
 http:
   routers:
     analytics:
-      rule: "Host(`analytics.{{ env_vars.DOMAIN }}`)"
+      rule: "Host(`analytics.{{ traefik_domain }}`)"
       entryPoints:
         - websecure
       service: analytics-svc
       tls:
         certResolver: letsencrypt
         domains:
-          - main: "analytics.{{ env_vars.DOMAIN }}"
+          - main: "analytics.{{ traefik_domain }}"
             sans:
-              - "blog.{{ env_vars.DOMAIN }}"
+              - "blog-1.{{ traefik_domain }}"
 
   services:
     analytics-svc:
       loadBalancer:
         servers:
-          - url: "http://{{ env_vars.ANALYTICS_IP }}:3000"
+          - url: "http://{{ traefik_analytics_private_ip }}:3000"
         healthCheck:
           path: /api/heartbeat
           interval: "10s"

roles/traefik/templates/dynamic-hosts.yml.j2

@@ -3,22 +3,22 @@
 http:
   routers:
     retailor-web:
-      rule: "Host(`app.{{ env_vars.DOMAIN }}`)"
+      rule: "Host(`app.{{ traefik_domain }}`)"
       service: retailor-web-svc
       entryPoints:
         - websecure
       tls:
         certResolver: letsencrypt
         domains:
-          - main: "app.{{ env_vars.DOMAIN }}"
+          - main: "app.{{ traefik_domain }}"
             sans:
-              - "blog.{{ env_vars.DOMAIN }}"
+              - "blog-1.{{ traefik_domain }}"
 
   services:
     retailor-web-svc:
       loadBalancer:
         servers:
-{% for web_ip in env_vars.WEB_IPS | from_json %}
+{% for web_ip in traefik_web_private_ips %}
           - url: "http://{{ web_ip }}:80"
 {% endfor %}
         healthCheck: