configure dns resolvers for lego

plays/elasticsearch.yml

@@ -1,12 +1,6 @@
 ---
 - name: Install ELK stack using Docker
   hosts: all
-  vars:
-    elk_version: "9.4.4"
   roles:
-    # TODO check for docker install before purging it. This nukes containers
-    # we might not want destroyed.
-    # Leave a file, or read the same input requirements into a apt check
-    # which skips purge if satisfied.
     - role: roles/docker
     - role: roles/elasticsearch

roles/elasticsearch/README.md

@@ -1,23 +1,247 @@
-# elasticsearch
+# elk_stack
 
-Play configures ELK stack using docker & is available without HTTPS. Configure container variables in `tasks/SERVICE.yml` files, environment variables for the services & Java can also be configured here.
+Deploys Elasticsearch, Logstash and Kibana as a single Docker Compose stack
+on a Debian host that already has Docker (and the `docker compose` plugin)
+installed and running. All three services join a shared bridge network
+(`elk-network` by default) so they can reach each other by container/service
+name — no hardcoded IPs, no `--link`.
 
-The following are manual steps required during setup.
+## Requirements
 
-## elastic
+- Debian host with Docker Engine + Docker Compose plugin already installed
+- Ansible collection: `community.docker` (`ansible-galaxy collection install community.docker`)
+- Python `docker` + `requests` modules on the target (the role installs these via pip)
+- Enough RAM for the JVM heaps you configure (Elasticsearch + Logstash default to ~1.5G combined; leave headroom for OS/Docker)
 
-After creating elasticsearch container SSH into the running host and generate a new password for user `elastic` using command:
+## What it does
 
-```bash
-docker exec -it elasticsearch /usr/share/elasticsearch/bin/elasticsearch-reset-password -u elastic
+1. Verifies Docker and Docker Compose are usable.
+2. Creates `{{ elk_base_dir }}` (default `/opt/elk`) with a `config/` tree.
+3. Templates:
+   - `config/elasticsearch/elasticsearch.yml`
+   - `config/kibana/kibana.yml`
+   - `config/logstash/logstash.yml`, `pipelines.yml`, and three pipeline
+     configs: `pipeline/main.conf` (generic beats/tcp/udp), `pipeline/pipeline-nginx.conf`
+     (nginx access logs), `pipeline/pipeline-laravel.conf` (Laravel logs)
+   - `docker-compose.yml` — one file, three services, one shared network, a named volume for ES data
+4. Brings the stack up with `docker compose up`.
+5. Waits for Elasticsearch to respond, then (if security is enabled) sets the
+   `kibana_system` and `logstash_system` built-in user passwords via the
+   Elasticsearch security API, and recreates Kibana/Logstash so they pick up
+   working credentials.
+6. Waits for Kibana's `/api/status` to come back healthy.
+
+## Networking
+
+All three containers run on the `elk_network_name` bridge network
+(default `elk-network`, subnet `172.28.0.0/24`). Inside that network, all
+inter-service traffic uses `http://` or `https://` consistently based on
+`elk_tls_enabled` (default stack, TLS on):
+
+- Elasticsearch is reachable at `https://elasticsearch:9200`
+- Kibana is configured with `elasticsearch.hosts: ["https://elasticsearch:9200"]`
+- Logstash's pipelines all point at `https://elasticsearch:9200`
+
+With `elk_tls_enabled: false`, all three of the above use `http://` instead.
+
+Host ports (`elk_elasticsearch_http_port`, `elk_kibana_http_port`,
+`elk_logstash_beats_port`, `elk_logstash_nginx_beats_port`,
+`elk_logstash_laravel_beats_port`, etc.) are only for reaching the stack
+*from outside* — the containers themselves never use `localhost` to talk to
+each other.
+
+## Security
+
+`elk_security_enabled: true` by default (recommended). This turns on
+Elastic's built-in security (basic auth, no TLS between containers since
+traffic stays on the internal Docker network). Change these before running
+in anything but a scratch environment — ideally via Ansible Vault:
+
+```yaml
+elk_elastic_password: "..."
+elk_kibana_system_password: "..."
+elk_logstash_system_password: "..."
+elk_kibana_encryption_key: "..."   # 32+ random chars, e.g. `openssl rand -hex 32`
 ```
 
-## kibana
+Set `elk_security_enabled: false` for a quick, unauthenticated local/dev
+stack (no passwords) — do not do this on anything reachable from untrusted
+networks.
 
-Create a password for `kibana_system` user:
+**Note:** `elk_tls_enabled` and `elk_security_enabled` are independent
+switches you can set separately, but Elasticsearch only applies TLS
+(`xpack.security.http.ssl.*`) when X-Pack security itself is on. So if
+`elk_tls_enabled: true` and `elk_security_enabled: false`, this role
+force-enables security anyway purely to make TLS possible — `elastic`,
+`kibana_system`, and `logstash_system` all get real passwords from
+`elk_elastic_password`/`elk_kibana_system_password`/
+`elk_logstash_system_password` in that mode too, exactly as if
+`elk_security_enabled: true` had been set, so change those from their
+placeholder defaults the same way you would for a normal secured stack.
+Setting both `elk_tls_enabled: false` and `elk_security_enabled: false` is
+the only way to get a fully open, unauthenticated, plain-HTTP stack.
 
-```bash
-export ELASTIC_PASSWORD=
-export KIBANA_PASSWORD=
-curl -s -X POST -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" http://elasticsearch:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}";
+## Example playbook
+
+```yaml
+- hosts: elk_servers
+  become: true
+  roles:
+    - role: elk_stack
+      vars:
+        elk_version: "8.15.3"
+        elk_elasticsearch_heap_size: "2g"
+        elk_logstash_heap_size: "1g"
+        elk_elastic_password: "{{ vault_elk_elastic_password }}"
+        elk_kibana_system_password: "{{ vault_elk_kibana_password }}"
+        elk_logstash_system_password: "{{ vault_elk_logstash_password }}"
+        elk_kibana_encryption_key: "{{ vault_elk_kibana_enc_key }}"
 ```
+
+## Customizing the Logstash pipeline
+
+Three pipelines run side by side inside the same Logstash process, each on
+its own beats port (they can't share a port):
+
+| Pipeline | Template | Beats port var | Purpose |
+|---|---|---|---|
+| `main` | `main.conf.j2` | `elk_logstash_beats_port` (5044) | Generic beats input, plus raw JSON over TCP/UDP (5000) |
+| `nginx_pipeline` | `pipeline-nginx.conf.j2` | `elk_logstash_nginx_beats_port` (5045) | nginx access log parsing (grok + geoip) |
+| `laravel_pipeline` | `pipeline-laravel.conf.j2` | `elk_logstash_laravel_beats_port` (5046) | Laravel log ingestion |
+
+All three ship to Elasticsearch using the same `elk_tls_enabled`/
+`elk_security_enabled` logic as the rest of the stack (TLS scheme + the
+`logstash_system` credentials, when applicable). To add your own
+filters/inputs/outputs, either edit these templates directly, or drop an
+additional `*.conf.j2` file into `templates/` and extend `pipelines.yml.j2`
+(with a new `pipeline.id`/`path.config` entry and a beats port that doesn't
+collide with the ones above) and `tasks/main.yml` (a matching `template:`
+task) to deploy it.
+
+## Idempotency notes
+
+- Re-running the role only recreates containers whose config actually
+  changed (via handlers), plus a compose reconciliation pass.
+- Setting the `kibana_system`/`logstash_system` passwords is safe to repeat;
+  Elasticsearch just resets them to the same value each time and the role
+  only recreates those two containers when the API call reports a change.
+
+## TLS (public certs for a private-IP node)
+
+`elk_tls_enabled: true` (default) gets Elasticsearch, Kibana, and Logstash a
+**real, publicly-trusted certificate** for a DNS name that happens to point
+at a private IP (e.g. `logs-eu.domain.com -> 10.0.99.5`), using the ACME
+**DNS-01** challenge against Cloudflare. This works even though the host
+itself is unreachable from the internet, because DNS-01 only requires the
+ability to create a `_acme-challenge` TXT record — no inbound HTTP/HTTPS
+access to the node is needed at all.
+
+### Why DNS-01, and why one shared cert
+
+- Public CAs don't issue certificates for bare IP SANs (especially not
+  RFC1918/private addresses) — the workaround is a cert for a **DNS name**
+  that resolves to the private IP, with clients connecting via that name.
+- Since all three services live on the same node behind the same DNS name,
+  they share **one cert** rather than one per service. Elastic explicitly
+  supports (and this is a common pattern) using [a single shared HTTP CA
+  across Elasticsearch, Kibana, and other stack components](https://www.elastic.co/docs/deploy-manage/deploy/self-managed/tutorial-self-managed-secure)
+  — separate CAs/certs per component exist to isolate trust domains, which
+  isn't a goal here since everything is already inside the same trust
+  boundary (one box, one operator, one Docker network).
+- Because the cert comes from a **public** CA (Let's Encrypt, via lego's
+  default ACME server), Kibana and Logstash don't need a custom CA
+  certificate distributed to them to trust Elasticsearch's cert — their
+  default trust stores already trust Let's Encrypt's root. This removes an
+  entire category of "distribute the CA cert everywhere" complexity that
+  self-signed/internal-CA setups require.
+
+### What this role does
+
+1. Expects `elk_cloudflare_token_env_file` (default
+   `/etc/lego/cloudflare-token.env`) to **already exist** on the box,
+   containing `CF_DNS_API_TOKEN=...`, root-owned, mode `0600`. This role
+   only *reads* that file — it never receives, logs, or stores the token
+   itself. See "IaC responsibilities" below for where this file comes from.
+2. Installs the [`lego`](https://github.com/go-acme/lego) ACME client (a
+   single static Go binary — no Python/snap dependencies, easy to pin an
+   exact version).
+3. On first run, issues a certificate for `elk_tls_domain` via `lego ... run`,
+   landing at `{{ elk_lego_certs_dir }}/{{ elk_tls_domain }}.{crt,key,issuer.crt}`.
+4. Mounts that directory read-only into all three containers and wires
+   `xpack.security.http.ssl.*` (Elasticsearch), `server.ssl.*` (Kibana), and
+   `ssl_enabled` (Logstash's elasticsearch output) to point at it.
+5. Installs a `lego-renew.service` + `lego-renew.timer` systemd pair that
+   runs daily, checks expiry, and **only** re-issues (and only then restarts
+   the three containers, via lego's `--run-hook`) when the cert is within
+   `elk_tls_renew_days` of expiring. No-op checks never restart anything.
+
+### IaC responsibilities (Pulumi, not this role)
+
+This role deliberately does **not** manage Cloudflare DNS records, zones, or
+API tokens — that's Pulumi's job, since it already holds the Cloudflare
+credentials and provisions the box. The pseudocode below sketches what
+Pulumi needs to set up per node:
+
+```python
+# Pulumi pseudocode - see conversation history / infra repo for the real version
+apex_zone = cloudflare.get_zone(name="domain.com")
+
+# A record already exists in this setup: logs-eu.domain.com -> 10.0.99.x
+# (proxied=False - a private IP can't sit behind Cloudflare's proxy/edge)
+
+# Scoped API token: Zone.DNS.Edit, restricted to the apex zone's ID (or a
+# delegated child zone, e.g. logs-eu.domain.com as its own zone, for harder
+# isolation). This is the tightest scope Cloudflare's token model supports -
+# there's no way to restrict a token to a single record name/pattern.
+acme_dns_token = cloudflare.ApiToken(
+    "logs-eu-acme-dns-token",
+    policies=[{
+        "permission_groups": [dns_edit_permission_group],
+        "resources": {f"com.cloudflare.api.account.zone.{apex_zone.id}": "*"},
+    }],
+    # optional: condition.request_ip.in_ to pin the token to the node's
+    # egress IP, and/or expires_on for periodic rotation via Pulumi re-runs
+)
+
+# Hand the token to the VM via cloud-init, written root-only - Pulumi never
+# passes this to Ansible as a variable.
+cloud_init_write_files = [{
+    "path": "/etc/lego/cloudflare-token.env",
+    "owner": "root:root",
+    "permissions": "0600",
+    "content": f"CF_DNS_API_TOKEN={acme_dns_token.value}\n",
+}]
+```
+
+For stronger isolation than a single shared zone token, NS-delegate
+`logs-eu.domain.com` (and `logs-us.domain.com`, etc.) to their own
+Cloudflare zones, and scope each region's token to only its own zone ID —
+that way a compromised `logs-eu` node's token can't touch `logs-us` or the
+apex domain's records at all.
+
+### Key variables
+
+| Variable | Purpose |
+|---|---|
+| `elk_tls_enabled` | Master switch; `false` reverts to plain HTTP |
+| `elk_tls_domain` | The DNS name (e.g. `logs-eu.domain.com`) the cert covers and clients must connect via |
+| `elk_tls_acme_email` | Contact address registered with the ACME account |
+| `elk_tls_acme_server` | ACME endpoint — point at Let's Encrypt **staging** while testing to avoid production rate limits |
+| `elk_cloudflare_token_env_file` | Path to the pre-existing, cloud-init-provisioned token file |
+| `elk_tls_renew_days` | Renew when fewer than this many days remain before expiry |
+| `elk_lego_version` | Pinned lego release to install |
+
+### Operational notes
+
+- **Testing**: set `elk_tls_acme_server` to
+  `https://acme-staging-v02.api.letsencrypt.org/directory` first — staging
+  certs aren't publicly trusted but validate the whole DNS-01/Cloudflare
+  flow without touching Let's Encrypt's production rate limits.
+- **Downtime on renewal**: renewal restarts all three containers via
+  `docker compose restart`, which is brief (~10-20s) but not zero-downtime.
+  Acceptable for a single-node stack; a zero-downtime rolling reload would
+  require a multi-node Elasticsearch cluster, which is out of scope here.
+- **Clients must use the DNS name, not the raw IP** — connecting to
+  `https://10.0.99.5:9200` directly will fail certificate hostname
+  verification even though the cert is otherwise valid, since the cert's
+  SAN is `logs-eu.domain.com`, not the IP.

roles/elasticsearch/defaults/main.yml

@@ -0,0 +1,135 @@
+---
+# ---------------------------------------------------------------------------
+# Environment variable overrides
+# ---------------------------------------------------------------------------
+# Every setting below can also be supplied as an env var on the CONTROL node
+# (i.e. wherever `ansible-playbook` runs - a CI job, your laptop, etc.), not
+# the remote target host. `lookup('env', ...)` reads from the control node's
+# process environment at template-render time, which is exactly what you
+# want for injecting CI/CD secrets (e.g. GitHub Actions `env:`/`secrets:`,
+# a `.env` file sourced before the playbook runs) without touching inventory
+# or vault files.
+#
+# Precedence, highest first: `-e` / extra vars on the CLI > host/group_vars
+# in inventory > playbook `vars:` > these defaults (env var, if set, else
+# the literal fallback shown). So an env var set here is easy to override
+# per-host or per-run if needed - it's just the last thing consulted.
+#
+# Booleans: env vars are always strings, so `elk_tls_enabled`/
+# `elk_security_enabled` run through `| bool` to turn "true"/"false"/"1"/"0"/
+# "yes"/"no" (case-insensitive) into real Ansible booleans - a raw string
+# "false" would otherwise be truthy in Jinja `{% if %}` conditionals.
+# ---------------------------------------------------------------------------
+
+# ---------------------------------------------------------------------------
+# General
+# ---------------------------------------------------------------------------
+elk_version: "{{ lookup('env', 'ELK_VERSION') | default('8.15.3', true) }}"
+
+elk_base_dir: /opt/elk
+elk_compose_project_name: elk
+
+elk_network_name: elk-network
+elk_network_subnet: "172.28.0.0/24"
+
+# Set true only after you've configured elk_elastic_password etc. for real use.
+# When false, security is disabled for a quick local/dev stack.
+elk_security_enabled: "{{ lookup('env', 'ELK_SECURITY_ENABLED') | default('true', true) | bool }}"
+
+# Change these in production (e.g. via vault, or ELK_ELASTIC_PASSWORD etc.
+# from CI secrets). These fallbacks are intentionally obvious placeholders
+# so nobody ships them by accident.
+elk_elastic_password: "{{ lookup('env', 'ELK_ELASTIC_PASSWORD') | default('ChangeMe_Elastic_123!', true) }}"
+elk_kibana_system_password: "{{ lookup('env', 'ELK_KIBANA_SYSTEM_PASSWORD') | default('ChangeMe_Kibana_123!', true) }}"
+elk_logstash_system_password: "{{ lookup('env', 'ELK_LOGSTASH_SYSTEM_PASSWORD') | default('ChangeMe_Logstash_123!', true) }}"
+
+# A pre-shared 32+ char key used to encrypt the Kibana saved-objects store.
+elk_kibana_encryption_key: "{{ lookup('env', 'ELK_KIBANA_ENCRYPTION_KEY') | default('ChangeMe_32Char_Kibana_EncKey!!!', true) }}"
+
+# ---------------------------------------------------------------------------
+# Elasticsearch
+# ---------------------------------------------------------------------------
+elk_elasticsearch_container_name: elasticsearch
+elk_elasticsearch_http_port: 9200
+elk_elasticsearch_transport_port: 9300
+elk_elasticsearch_heap_size: "1g"
+elk_elasticsearch_cluster_name: "elk-retailor"
+elk_elasticsearch_node_name: "{{ lookup('env', 'ELK_ELASTIC_NODE_NAME') | default('es01', true) }}"
+elk_elasticsearch_data_volume: elk_esdata
+
+# ---------------------------------------------------------------------------
+# Kibana
+# ---------------------------------------------------------------------------
+elk_kibana_container_name: kibana
+elk_kibana_http_port: 5601
+
+# ---------------------------------------------------------------------------
+# Logstash
+# ---------------------------------------------------------------------------
+elk_logstash_container_name: logstash
+elk_logstash_heap_size: "512m"
+elk_logstash_tcp_port: 5000
+elk_logstash_udp_port: 5000
+elk_logstash_http_port: 9600
+
+# Each pipeline that has its own beats input needs its own port - they all
+# run inside the same Logstash process, so two pipelines can't bind the same
+# port. main.conf (generic beats/tcp/udp) and the nginx/laravel example
+# pipelines are each wired to a distinct one below.
+elk_logstash_beats_port: 5044
+elk_logstash_nginx_beats_port: 5045
+elk_logstash_laravel_beats_port: 5046
+
+# ---------------------------------------------------------------------------
+# Docker image repository (override for an internal mirror/proxy)
+# ---------------------------------------------------------------------------
+elk_docker_image_repo: "docker.elastic.co"
+
+# ---------------------------------------------------------------------------
+# TLS via lego (ACME DNS-01 against Cloudflare)
+# ---------------------------------------------------------------------------
+# Master switch. When false, the stack runs plain HTTP as before (nothing
+# below this point applies).
+elk_tls_enabled: "{{ lookup('env', 'ELK_TLS_ENABLED') | default('true', true) | bool }}"
+
+# The public DNS name this node is reachable at, e.g. "logs.domain.com".
+# An A record for this name -> the node's private IP is assumed to already
+# exist (managed by IaC, not this role). This name is what ES/Kibana/Logstash
+# put in their certs and what clients must connect to (not the raw IP) for
+# TLS verification to succeed.
+elk_tls_domain: "{{ lookup('env', 'ELK_TLS_DOMAIN') | default('logs.domain.com', true) }}"
+
+# Contact email lego registers with the ACME account (Let's Encrypt notifies
+# this address about certificate/account issues, not renewal reminders,
+# since renewal is automated).
+elk_tls_acme_email: "{{ lookup('env', 'ELK_TLS_ACME_EMAIL') | default('webmaster@domain.com', true) }}"
+
+# Let's Encrypt production endpoint by default. Point this at the staging
+# endpoint while testing to avoid production rate limits:
+# https://acme-staging-v02.api.letsencrypt.org/directory
+elk_tls_acme_server: "{{ lookup('env', 'ELK_TLS_ACME_SERVER') | default('https://acme-v02.api.letsencrypt.org/directory', true) }}"
+
+# lego binary version/arch and install location.
+elk_lego_version: "4.20.4"
+elk_lego_arch: "amd64"   # amd64 | arm64
+elk_lego_install_dir: /usr/local/bin
+elk_lego_path: /etc/lego              # lego's --path (accounts + certificates live under here)
+elk_lego_certs_dir: "{{ elk_lego_path }}/certificates"
+
+# Path to the Cloudflare token env file. This is expected to already exist,
+# dropped by cloud-init at provisioning time (see IaC notes) as
+# root:root 0600, containing a line like:
+#   CF_DNS_API_TOKEN=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+# This role only READS this file - it never receives or stores the token
+# itself, so the Cloudflare credential never touches Ansible inventory,
+# vault, logs, or the control node.
+elk_cloudflare_token_env_file: "{{ lookup('env', 'ELK_CLOUDFLARE_TOKEN_ENV_FILE') | default('/etc/lego/cloudflare-token.env', true) }}"
+
+# Threshold used on renewal checks - lego renews if the current cert has
+# fewer than this many days left before expiry.
+elk_tls_renew_days: 30
+
+# Renewal check cadence via systemd timer.
+elk_tls_renew_on_calendar: "22:00:00"
+elk_tls_renew_randomized_delay: "30m"
+

roles/elasticsearch/handlers/main.yml

@@ -0,0 +1,38 @@
+---
+- name: restart elk stack
+  community.docker.docker_compose_v2:
+    project_src: "{{ elk_base_dir }}"
+    project_name: "{{ elk_compose_project_name }}"
+    state: present
+    recreate: always
+  listen: "restart elk stack"
+
+- name: recreate elasticsearch
+  community.docker.docker_compose_v2:
+    project_src: "{{ elk_base_dir }}"
+    project_name: "{{ elk_compose_project_name }}"
+    services:
+      - elasticsearch
+    state: present
+    recreate: always
+  listen: "recreate elasticsearch"
+
+- name: recreate logstash
+  community.docker.docker_compose_v2:
+    project_src: "{{ elk_base_dir }}"
+    project_name: "{{ elk_compose_project_name }}"
+    services:
+      - logstash
+    state: present
+    recreate: always
+  listen: "recreate logstash"
+
+- name: recreate kibana
+  community.docker.docker_compose_v2:
+    project_src: "{{ elk_base_dir }}"
+    project_name: "{{ elk_compose_project_name }}"
+    services:
+      - kibana
+    state: present
+    recreate: always
+  listen: "recreate kibana"

roles/elasticsearch/meta/main.yml

@@ -1,16 +1,16 @@
 ---
 galaxy_info:
-  author: Your Name
-  description: Ansible role to deploy prometheus & grafana using Docker
-  license: MIT
-  min_ansible_version: "2.9"
+  role_name: elk_stack
+  author: your-name
+  description: Deploys Elasticsearch, Logstash and Kibana via Docker Compose on a shared network
+  min_ansible_version: "2.14"
   platforms:
     - name: Debian
       versions:
-        - all
-  galaxy_tags:
-    - monitoring
-    - cadvisor
-  dependencies:
-    - docker
+        - bullseye
+        - bookworm
 
+dependencies: []
+
+collections:
+  - community.docker

roles/elasticsearch/tasks/elasticsearch.yml

@@ -1,41 +0,0 @@
----
-- name: Pull Elasticsearch Docker image
-  docker_image:
-    name: docker.elastic.co/elasticsearch/elasticsearch-wolfi:{{ elk_version }}
-    source: pull
-
-- name: Create Elasticsearch configuration file directory on host
-  file:
-    path: /etc/elasticsearch
-    state: directory
-    mode: '0755'
-
-    # - name: Create Elasticsearch configuration file
-    #   template:
-    #     src: elasticsearch.yml.j2
-    #     dest: /etc/elasticsearch/elasticsearch.yml
-
-- name: Start Elasticsearch container
-  docker_container:
-    name: elasticsearch
-    image: docker.elastic.co/elasticsearch/elasticsearch-wolfi:{{ elk_version }}
-    state: started
-    restart: yes
-    restart_policy: unless-stopped
-    published_ports:
-      - "9200:9200"
-    volumes:
-      - /etc/elasticsearch/esdata:/usr/share/elasticsearch/data
-    networks:
-      - name: elk_network
-        ipv4_address: 172.19.0.2
-    env:
-      node.name: elasticsearch
-      cluster.name: retailor-elk
-      discovery.type: single-node
-      bootstrap.memory_lock: "true"
-      # limits elasticsearch to 2 GB of RAM
-      ES_JAVA_OPTS: "-Xms1g -Xmx2g"
-      # disables SSL & xpack security
-      xpack.security.http.ssl.enabled: "false"
-

roles/elasticsearch/tasks/elk-network.yml

@@ -1,11 +0,0 @@
----
-- name: Create a Docker network for Elasticsearch
-  docker_network:
-    name: elk_network
-    state: present
-    driver: bridge
-    ipam_config:
-      - subnet: "172.19.0.0/16"
-        gateway: "172.19.0.1"
-        iprange: "172.19.0.0/24"
-

roles/elasticsearch/tasks/kibana.yml

@@ -1,37 +0,0 @@
----
-- name: Create a Docker network for Kibana
-  docker_network:
-    name: elk_network
-    state: present
-
-- name: Create kibana directory on host
-  file:
-    path: /etc/kibana
-    state: directory
-    mode: "0755"
-
-- name: Pull Kibana Docker image
-  docker_image:
-    name: docker.elastic.co/kibana/kibana:{{ elk_version }}
-    source: pull
-
-# TODO rember to move CA cert from elastic to Kibana
-# docker cp elasticsearch:/usr/share/elasticsearch/config/certs/http_ca.crt .
-# docker cp http_ca.crt kibana:/usr/share/kibana/config/certs/ca/http_ca.crt
-- name: Start Kibana container
-  docker_container:
-    name: kibana
-    image: docker.elastic.co/kibana/kibana:{{ elk_version }}
-    state: started
-    restart: yes
-    restart_policy: unless-stopped
-    published_ports:
-      - "5601:5601"
-    env:
-      ELASTICSEARCH_HOSTS: "{{ env_vars.ELASTIC_HOSTS }}" 
-      ELASTICSEARCH_USERNAME: kibana_system
-      ELASTICSEARCH_PASSWORD: "{{ env_vars.KIBANA_PASSWORD }}"
-      TELEMETRY_ENABLED: "false"
-    networks:
-      - name: elk_network
-        ipv4_address: 172.19.0.3

roles/elasticsearch/tasks/logstash.yml

@@ -1,65 +0,0 @@
----
-- name: Create a Docker network for Logstash
-  docker_network:
-    name: elk_network
-    state: present
-
-- name: Create logstash directory on host
-  file:
-    path: /etc/logstash
-    state: directory
-    mode: "0755"
-
-- name: Copy logstash config
-  copy:
-    src: templates/pipelines.yml.j2
-    dest: /etc/logstash/pipelines.yml
-
-- name: Create logstash directory on host
-  file:
-    path: /etc/logstash/pipeline
-    state: directory
-    mode: "0755"
-
-- name: Copy logstash input configs
-  copy:
-    src: "{{ item }}"
-    dest: /etc/logstash/pipeline/{{ item | basename | regex_replace('\.j2$', '') }}
-  loop: "{{ query('fileglob', 'templates/logstash-conf.d/*.j2') }}"
-
-- name: Pull Logstash Docker image
-  docker_image:
-    name: docker.elastic.co/logstash/logstash:{{ elk_version }}
-    source: pull
-
-# TODO rember to move CA cert from elastic to Logstash
-# docker cp elasticsearch:/usr/share/elasticsearch/config/certs/http_ca.crt .
-# docker cp http_ca.crt logstash:/usr/share/logstash/config/certs/ca/http_ca.crt
-- name: Start Logstash container
-  docker_container:
-    name: logstash
-    image: docker.elastic.co/logstash/logstash:{{ elk_version }}
-    state: started
-    restart: yes
-    restart_policy: unless-stopped
-    command:
-      - /bin/bash
-      - -c
-      - |
-        echo "Waiting for Elasticsearch availability";
-        until curl -s {{ env_vars.ELASTIC_HOSTS }} | grep -q "missing authentication credentials"; do sleep 1; done;
-        echo "Starting logstash";
-        /usr/share/logstash/bin/logstash
-    published_ports:
-      - "5044-5049:5044-5049"
-    volumes:
-      - /etc/logstash/pipelines.yml:/usr/share/logstash/config/pipelines.yml
-      - /etc/logstash/pipeline:/usr/share/logstash/pipeline
-    env:
-      xpack.monitoring.enabled: "false"
-      ELASTIC_USER: elastic
-      ELASTIC_PASSWORD: "{{ env_vars.ELASTIC_PASSWORD }}"
-      ELASTIC_HOSTS: "{{ env_vars.ELASTIC_HOSTS }}"
-    networks:
-      - name: elk_network
-        ipv4_address: 172.19.0.4

roles/elasticsearch/tasks/main.yml

@@ -1,8 +1,213 @@
 ---
-# ensure we have variables from .env files
-- include_tasks: ../roles/env/tasks/main.yml
+# ---------------------------------------------------------------------------
+# TLS (elk_tls_enabled) and X-Pack security (elk_security_enabled) are
+# documented as independent switches, but Elasticsearch only honors
+# xpack.security.http.ssl.* when xpack.security.enabled is also true. So
+# whenever TLS is on, security (and therefore basic auth / ELASTIC_PASSWORD)
+# is force-enabled too - elk_effective_security_enabled is the single fact
+# every template and task below should use instead of elk_security_enabled
+# directly, so this coupling only has to be expressed once.
+# ---------------------------------------------------------------------------
+- name: Derive effective security state (TLS implies security)
+  ansible.builtin.set_fact:
+    elk_effective_security_enabled: "{{ elk_security_enabled or elk_tls_enabled }}"
 
-- include_tasks: elk-network.yml
-- include_tasks: elasticsearch.yml
-- include_tasks: kibana.yml
-- include_tasks: logstash.yml
+- name: Assert Docker is available
+  ansible.builtin.command: docker info
+  register: elk_docker_info
+  changed_when: false
+  failed_when: elk_docker_info.rc != 0
+
+- name: Ensure Docker Compose plugin is installed
+  ansible.builtin.command: docker compose version
+  register: elk_compose_check
+  changed_when: false
+  failed_when: elk_compose_check.rc != 0
+
+- name: Ensure we have pip installed
+  apt:
+    pkg: "python3-pip"
+    state: present
+
+- name: Ensure python docker SDK is present for community.docker modules
+  apt:
+    pkg:
+      - python3-docker
+      - python3-requests
+    state: present
+
+- name: Create base ELK directory structure
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  loop:
+    - "{{ elk_base_dir }}"
+    - "{{ elk_base_dir }}/config"
+    - "{{ elk_base_dir }}/config/elasticsearch"
+    - "{{ elk_base_dir }}/config/kibana"
+    - "{{ elk_base_dir }}/config/logstash"
+    - "{{ elk_base_dir }}/config/logstash/pipeline"
+
+# ---------------------------------------------------------------------------
+# TLS (must happen before compose/config templating below, since those
+# templates reference the cert file paths lego produces)
+# ---------------------------------------------------------------------------
+- name: Issue/renew TLS certificate and set up renewal automation
+  ansible.builtin.include_tasks: tls.yml
+  when: elk_tls_enabled
+
+- name: Set connection facts used by health checks below
+  ansible.builtin.set_fact:
+    elk_scheme: "{{ 'https' if elk_tls_enabled else 'http' }}"
+    elk_validate_certs: "{{ not elk_tls_enabled }}"
+    elk_elasticsearch_client_host: "{{ elk_tls_domain if elk_tls_enabled else elk_elasticsearch_container_name }}"
+
+# ---------------------------------------------------------------------------
+# Configs
+# ---------------------------------------------------------------------------
+- name: Template elasticsearch.yml
+  ansible.builtin.template:
+    src: elasticsearch.yml.j2
+    dest: "{{ elk_base_dir }}/config/elasticsearch/elasticsearch.yml"
+    mode: "0644"
+  notify: "recreate elasticsearch"
+
+- name: Template kibana.yml
+  ansible.builtin.template:
+    src: kibana.yml.j2
+    dest: "{{ elk_base_dir }}/config/kibana/kibana.yml"
+    mode: "0644"
+  notify: "recreate kibana"
+
+- name: Template logstash.yml
+  ansible.builtin.template:
+    src: logstash.yml.j2
+    dest: "{{ elk_base_dir }}/config/logstash/logstash.yml"
+    mode: "0644"
+  notify: "recreate logstash"
+
+- name: Template logstash pipelines.yml
+  ansible.builtin.template:
+    src: pipelines.yml.j2
+    dest: "{{ elk_base_dir }}/config/logstash/pipelines.yml"
+    mode: "0644"
+  notify: "recreate logstash"
+
+- name: Template logstash main pipeline
+  ansible.builtin.template:
+    src: main.conf.j2
+    dest: "{{ elk_base_dir }}/config/logstash/pipeline/main.conf"
+    mode: "0644"
+  notify: "recreate logstash"
+
+- name: Template logstash nginx pipeline
+  ansible.builtin.template:
+    src: pipeline-nginx.conf.j2
+    dest: "{{ elk_base_dir }}/config/logstash/pipeline/pipeline-nginx.conf"
+    mode: "0644"
+  notify: "recreate logstash"
+
+- name: Template logstash laravel pipeline
+  ansible.builtin.template:
+    src: pipeline-laravel.conf.j2
+    dest: "{{ elk_base_dir }}/config/logstash/pipeline/pipeline-laravel.conf"
+    mode: "0644"
+  notify: "recreate logstash"
+
+- name: Template docker-compose.yml
+  ansible.builtin.template:
+    src: docker-compose.yml.j2
+    dest: "{{ elk_base_dir }}/docker-compose.yml"
+    mode: "0644"
+  notify: "restart elk stack"
+
+# ---------------------------------------------------------------------------
+# Bring up the stack
+# ---------------------------------------------------------------------------
+- name: Flush handlers so config changes apply before we continue
+  ansible.builtin.meta: flush_handlers
+
+- name: Start ELK stack via docker compose
+  community.docker.docker_compose_v2:
+    project_src: "{{ elk_base_dir }}"
+    project_name: "{{ elk_compose_project_name }}"
+    state: present
+  register: elk_compose_result
+
+- name: Wait for Elasticsearch to answer on its HTTP port
+  ansible.builtin.uri:
+    url: "{{ elk_scheme }}://localhost:{{ elk_elasticsearch_http_port }}"
+    validate_certs: "{{ elk_validate_certs }}"
+    status_code:
+      - 200
+      - 401  # 401 is expected once security is enabled but before we authenticate
+  register: elk_es_wait
+  until: elk_es_wait.status in [200, 401]
+  retries: 30
+  delay: 10
+
+# ---------------------------------------------------------------------------
+# Security bootstrap: set built-in user passwords (idempotent)
+# ---------------------------------------------------------------------------
+- name: Set kibana_system password
+  ansible.builtin.uri:
+    url: "{{ elk_scheme }}://localhost:{{ elk_elasticsearch_http_port }}/_security/user/kibana_system/_password"
+    validate_certs: "{{ elk_validate_certs }}"
+    method: POST
+    user: elastic
+    password: "{{ elk_elastic_password }}"
+    force_basic_auth: true
+    body_format: json
+    body:
+      password: "{{ elk_kibana_system_password }}"
+    status_code: 200
+  when: elk_effective_security_enabled
+  register: elk_set_kibana_pw
+  retries: 5
+  delay: 10
+  until: elk_set_kibana_pw is succeeded
+
+- name: Set logstash_system password
+  ansible.builtin.uri:
+    url: "{{ elk_scheme }}://localhost:{{ elk_elasticsearch_http_port }}/_security/user/logstash_system/_password"
+    validate_certs: "{{ elk_validate_certs }}"
+    method: POST
+    user: elastic
+    password: "{{ elk_elastic_password }}"
+    force_basic_auth: true
+    body_format: json
+    body:
+      password: "{{ elk_logstash_system_password }}"
+    status_code: 200
+  when: elk_effective_security_enabled
+  register: elk_set_logstash_pw
+  retries: 5
+  delay: 10
+  until: elk_set_logstash_pw is succeeded
+
+- name: Recreate kibana and logstash if passwords were just set for the first time
+  community.docker.docker_compose_v2:
+    project_src: "{{ elk_base_dir }}"
+    project_name: "{{ elk_compose_project_name }}"
+    services:
+      - kibana
+      - logstash
+    state: present
+    recreate: always
+  when: >
+    elk_effective_security_enabled and
+    (elk_set_kibana_pw is changed or elk_set_logstash_pw is changed)
+
+- name: Wait for Kibana to answer on its HTTP port
+  ansible.builtin.uri:
+    url: "{{ elk_scheme }}://localhost:{{ elk_kibana_http_port }}/api/status"
+    validate_certs: "{{ elk_validate_certs }}"
+    status_code:
+      - 200
+  register: elk_kibana_wait
+  until: elk_kibana_wait.status == 200
+  retries: 30
+  delay: 10

roles/elasticsearch/tasks/tls.yml

@@ -0,0 +1,177 @@
+---
+# ---------------------------------------------------------------------------
+# TLS certificate lifecycle via lego (ACME DNS-01 against Cloudflare).
+#
+# This file only READS the Cloudflare token from elk_cloudflare_token_env_file.
+# It never receives the token as an Ansible variable - the token is expected
+# to already exist on disk (root:root, 0600), placed there by cloud-init at
+# provisioning time. See the role README for the IaC side of this.
+# ---------------------------------------------------------------------------
+
+- name: Assert the Cloudflare token file exists
+  ansible.builtin.stat:
+    path: "{{ elk_cloudflare_token_env_file }}"
+  register: elk_cf_token_stat
+
+- name: Fail early with a clear message if the token file is missing
+  ansible.builtin.fail:
+    msg: >
+      {{ elk_cloudflare_token_env_file }} does not exist. This role expects
+      cloud-init (or equivalent provisioning) to have already written a
+      scoped Cloudflare API token here before Ansible runs. See the IaC
+      notes in the role README.
+  when: not elk_cf_token_stat.stat.exists
+
+- name: Verify the token file is not world/group readable
+  ansible.builtin.assert:
+    that:
+      - elk_cf_token_stat.stat.mode == "0600"
+    fail_msg: >
+      {{ elk_cloudflare_token_env_file }} should be mode 0600
+      (currently {{ elk_cf_token_stat.stat.mode }}) since it holds a
+      Cloudflare API credential.
+    success_msg: "Cloudflare token file permissions look correct."
+
+- name: Create lego directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0700"
+  loop:
+    - "{{ elk_lego_path }}"
+    - "{{ elk_lego_path }}/accounts"
+
+- name: Create lego certificates directory (traversable by container users)
+  ansible.builtin.file:
+    path: "{{ elk_lego_certs_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Check installed lego version (if any)
+  ansible.builtin.command: "{{ elk_lego_install_dir }}/lego --version"
+  register: elk_lego_installed_version
+  changed_when: false
+  failed_when: false
+
+- name: Download and install lego binary
+  when: >
+    elk_lego_installed_version.rc != 0 or
+    elk_lego_version not in (elk_lego_installed_version.stdout | default(''))
+  block:
+    - name: Download lego release tarball
+      ansible.builtin.get_url:
+        url: "https://github.com/go-acme/lego/releases/download/v{{ elk_lego_version }}/lego_v{{ elk_lego_version }}_linux_{{ elk_lego_arch }}.tar.gz"
+        dest: "/tmp/lego_{{ elk_lego_version }}.tar.gz"
+        mode: "0644"
+
+    - name: Extract lego binary
+      ansible.builtin.unarchive:
+        src: "/tmp/lego_{{ elk_lego_version }}.tar.gz"
+        dest: /tmp
+        remote_src: true
+        include:
+          - lego
+
+    - name: Install lego binary
+      ansible.builtin.copy:
+        src: /tmp/lego
+        dest: "{{ elk_lego_install_dir }}/lego"
+        owner: root
+        group: root
+        mode: "0755"
+        remote_src: true
+
+    - name: Clean up downloaded artifacts
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: absent
+      loop:
+        - "/tmp/lego_{{ elk_lego_version }}.tar.gz"
+        - /tmp/lego
+
+- name: Check whether a certificate already exists for this domain
+  ansible.builtin.stat:
+    path: "{{ elk_lego_certs_dir }}/{{ elk_tls_domain }}.crt"
+  register: elk_cert_stat
+
+# First issuance uses `lego ... run`, which is what actually requests a new
+# certificate; `renew` (used by the systemd timer below) refuses to run
+# against a domain with no existing certificate.
+- name: Issue the initial certificate (first run only)
+  ansible.builtin.shell: |
+    set -o pipefail
+    set -a
+    source {{ elk_cloudflare_token_env_file }}
+    set +a
+    {{ elk_lego_install_dir }}/lego \
+      --accept-tos \
+      --email "{{ elk_tls_acme_email }}" \
+      --server "{{ elk_tls_acme_server }}" \
+      --domains "{{ elk_tls_domain }}" \
+      --dns cloudflare \
+      --dns.resolvers "1.1.1.1:53,8.8.8.8:53" \
+      --path "{{ elk_lego_path }}" \
+      run
+  args:
+    executable: /bin/bash
+  when: not elk_cert_stat.stat.exists
+  register: elk_lego_first_issue
+  notify: "restart elk stack"
+
+- name: Ensure issued cert/key are readable by the containers (root-owned, group-readable)
+  ansible.builtin.file:
+    path: "{{ item }}"
+    owner: root
+    group: root
+    mode: "0644"
+  loop:
+    - "{{ elk_lego_certs_dir }}/{{ elk_tls_domain }}.crt"
+    - "{{ elk_lego_certs_dir }}/{{ elk_tls_domain }}.issuer.crt"
+  when: elk_cert_stat.stat.exists or elk_lego_first_issue is not skipped
+
+- name: Private key stays root-only-readable but group-readable for the container runtime
+  ansible.builtin.file:
+    path: "{{ elk_lego_certs_dir }}/{{ elk_tls_domain }}.key"
+    owner: root
+    group: root
+    mode: "0640"
+  when: elk_cert_stat.stat.exists or elk_lego_first_issue is not skipped
+
+# ---------------------------------------------------------------------------
+# Ongoing renewal: systemd oneshot service + daily timer. lego's `renew`
+# subcommand only actually re-issues (and only then fires --run-hook) when
+# the certificate is within elk_tls_renew_days of expiring, so this is safe
+# to run daily without causing unnecessary container restarts.
+# ---------------------------------------------------------------------------
+- name: Template lego renewal systemd service
+  ansible.builtin.template:
+    src: lego-renew.service.j2
+    dest: /etc/systemd/system/lego-renew.service
+    owner: root
+    group: root
+    mode: "0644"
+  register: elk_lego_service_unit
+
+- name: Template lego renewal systemd timer
+  ansible.builtin.template:
+    src: lego-renew.timer.j2
+    dest: /etc/systemd/system/lego-renew.timer
+    owner: root
+    group: root
+    mode: "0644"
+  register: elk_lego_timer_unit
+
+- name: Reload systemd if units changed
+  ansible.builtin.systemd:
+    daemon_reload: true
+  when: elk_lego_service_unit is changed or elk_lego_timer_unit is changed
+
+- name: Enable and start the lego renewal timer
+  ansible.builtin.systemd:
+    name: lego-renew.timer
+    enabled: true
+    state: started

roles/elasticsearch/templates/docker-compose.yml.j2

@@ -0,0 +1,147 @@
+# ansible_managed
+version: "3.8"
+
+networks:
+  {{ elk_network_name }}:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: {{ elk_network_subnet }}
+
+volumes:
+  {{ elk_elasticsearch_data_volume }}:
+    driver: local
+
+services:
+  elasticsearch:
+    image: "{{ elk_docker_image_repo }}/elasticsearch/elasticsearch:{{ elk_version }}"
+    container_name: "{{ elk_elasticsearch_container_name }}"
+    restart: unless-stopped
+    environment:
+      - node.name={{ elk_elasticsearch_node_name }}
+      - cluster.name={{ elk_elasticsearch_cluster_name }}
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms{{ elk_elasticsearch_heap_size }} -Xmx{{ elk_elasticsearch_heap_size }}"
+      # xpack.security.enabled must be true whenever TLS is on - Elasticsearch
+      # only honors xpack.security.http.ssl.* when security itself is
+      # enabled. elk_effective_security_enabled captures that (it's true if
+      # either elk_security_enabled or elk_tls_enabled is true). Whenever
+      # security ends up on for either reason, ELASTIC_PASSWORD must also be
+      # set - otherwise the image generates a random, unknown elastic
+      # password and nothing else in this stack could authenticate.
+      - xpack.security.enabled={{ elk_effective_security_enabled | lower }}
+{% if elk_effective_security_enabled %}
+      - ELASTIC_PASSWORD={{ elk_elastic_password }}
+{% endif %}
+{% if elk_tls_enabled %}
+      - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.certificate=/usr/share/elasticsearch/config/certs/{{ elk_tls_domain }}.crt
+      - xpack.security.http.ssl.key=/usr/share/elasticsearch/config/certs/{{ elk_tls_domain }}.key
+      - xpack.security.http.ssl.certificate_authorities=/usr/share/elasticsearch/config/certs/{{ elk_tls_domain }}.issuer.crt
+      # Single node - transport SSL stays off. Turn this on (and add a
+      # matching transport cert) if this ever becomes a multi-node cluster.
+      - xpack.security.transport.ssl.enabled=false
+{% else %}
+      - xpack.security.http.ssl.enabled=false
+      - xpack.security.transport.ssl.enabled=false
+{% endif %}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - {{ elk_elasticsearch_data_volume }}:/usr/share/elasticsearch/data
+      - {{ elk_base_dir }}/config/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
+{% if elk_tls_enabled %}
+      - {{ elk_lego_certs_dir }}:/usr/share/elasticsearch/config/certs:ro
+{% endif %}
+    ports:
+      - "{{ elk_elasticsearch_http_port }}:9200"
+      - "{{ elk_elasticsearch_transport_port }}:9300"
+    networks:
+      {{ elk_network_name }}:
+{% if elk_tls_enabled %}
+        aliases:
+          - "{{ elk_tls_domain }}"
+{% endif %}
+    healthcheck:
+      test:
+        - CMD-SHELL
+{% set es_scheme = 'https' if elk_tls_enabled else 'http' %}
+{% set curl_tls_flag = '-k ' if elk_tls_enabled else '' %}
+{# xpack.security.enabled follows elk_effective_security_enabled (see
+   above), so basic auth is required here under that same condition. #}
+{% if elk_effective_security_enabled %}
+        - "curl -s {{ curl_tls_flag }}-u elastic:{{ elk_elastic_password }} {{ es_scheme }}://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"\\|\"status\":\"yellow\"'"
+{% else %}
+        - "curl -s {{ curl_tls_flag }}{{ es_scheme }}://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"\\|\"status\":\"yellow\"'"
+{% endif %}
+      interval: 10s
+      timeout: 10s
+      retries: 20
+      start_period: 30s
+
+  logstash:
+    image: "{{ elk_docker_image_repo }}/logstash/logstash:{{ elk_version }}"
+    container_name: "{{ elk_logstash_container_name }}"
+    restart: unless-stopped
+    environment:
+      - "LS_JAVA_OPTS=-Xms{{ elk_logstash_heap_size }} -Xmx{{ elk_logstash_heap_size }}"
+      - xpack.monitoring.enabled=false
+{% if elk_effective_security_enabled %}
+      - ELASTIC_USER=logstash_system
+      - ELASTIC_PASSWORD={{ elk_logstash_system_password }}
+{% endif %}
+    volumes:
+      - {{ elk_base_dir }}/config/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
+      - {{ elk_base_dir }}/config/logstash/pipelines.yml:/usr/share/logstash/config/pipelines.yml:ro
+      - {{ elk_base_dir }}/config/logstash/pipeline:/usr/share/logstash/pipeline:ro
+{% if elk_tls_enabled %}
+      - {{ elk_lego_certs_dir }}:/usr/share/logstash/config/certs:ro
+{% endif %}
+    ports:
+      - "{{ elk_logstash_beats_port }}:5044"
+      - "{{ elk_logstash_nginx_beats_port }}:5045"
+      - "{{ elk_logstash_laravel_beats_port }}:5046"
+      - "{{ elk_logstash_tcp_port }}:5000/tcp"
+      - "{{ elk_logstash_udp_port }}:5000/udp"
+      - "{{ elk_logstash_http_port }}:9600"
+    networks:
+      - {{ elk_network_name }}
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+
+  kibana:
+    image: "{{ elk_docker_image_repo }}/kibana/kibana:{{ elk_version }}"
+    container_name: "{{ elk_kibana_container_name }}"
+    restart: unless-stopped
+    environment:
+      - SERVER_NAME={{ elk_kibana_container_name }}
+      - ELASTICSEARCH_HOSTS={{ 'https' if elk_tls_enabled else 'http' }}://{{ elk_elasticsearch_client_host }}:9200
+{% if elk_effective_security_enabled %}
+      - ELASTICSEARCH_USERNAME=kibana_system
+      - ELASTICSEARCH_PASSWORD={{ elk_kibana_system_password }}
+      - XPACK_SECURITY_ENCRYPTIONKEY={{ elk_kibana_encryption_key }}
+{% endif %}
+{% if elk_tls_enabled %}
+      - SERVER_SSL_ENABLED=true
+      - SERVER_SSL_CERTIFICATE=/usr/share/kibana/config/certs/{{ elk_tls_domain }}.crt
+      - SERVER_SSL_KEY=/usr/share/kibana/config/certs/{{ elk_tls_domain }}.key
+      # Public CA (Let's Encrypt) is already trusted by the container's
+      # system trust store, so no custom CA needs to be supplied here for
+      # the Elasticsearch connection.
+{% endif %}
+    volumes:
+      - {{ elk_base_dir }}/config/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml:ro
+{% if elk_tls_enabled %}
+      - {{ elk_lego_certs_dir }}:/usr/share/kibana/config/certs:ro
+{% endif %}
+    ports:
+      - "{{ elk_kibana_http_port }}:5601"
+    networks:
+      - {{ elk_network_name }}
+    depends_on:
+      elasticsearch:
+        condition: service_healthy

roles/elasticsearch/templates/elasticsearch.yml.j2

@@ -0,0 +1,29 @@
+# ansible_managed
+cluster.name: "{{ elk_elasticsearch_cluster_name }}"
+node.name: "{{ elk_elasticsearch_node_name }}"
+
+network.host: 0.0.0.0
+discovery.type: single-node
+
+# TLS (elk_tls_enabled) is a transport-layer concern independent of X-Pack
+# security (elk_security_enabled) - HOWEVER Elasticsearch only honors
+# xpack.security.http.ssl.* when xpack.security.enabled is true, so security
+# is force-enabled (elk_effective_security_enabled, set in tasks/main.yml)
+# whenever TLS is on, even if elk_security_enabled itself is false. The
+# ELASTIC_PASSWORD env var is set under the same condition (see
+# docker-compose.yml.j2), so basic auth is always usable whenever this is true.
+xpack.security.enabled: {{ elk_effective_security_enabled | lower }}
+{% if elk_tls_enabled %}
+xpack.security.http.ssl.enabled: true
+xpack.security.http.ssl.certificate: certs/{{ elk_tls_domain }}.crt
+xpack.security.http.ssl.key: certs/{{ elk_tls_domain }}.key
+xpack.security.http.ssl.certificate_authorities: ["certs/{{ elk_tls_domain }}.issuer.crt"]
+xpack.security.transport.ssl.enabled: false
+{% else %}
+xpack.security.http.ssl.enabled: false
+xpack.security.transport.ssl.enabled: false
+{% endif %}
+
+# Reachable at {{ 'https' if elk_tls_enabled else 'http' }}://{{ elk_elasticsearch_client_host }}:9200 from other
+# containers on the {{ elk_network_name }} docker network, and from outside
+# the host at {{ 'https' if elk_tls_enabled else 'http' }}://{{ elk_tls_domain if elk_tls_enabled else elk_elasticsearch_client_host }}:{{ elk_elasticsearch_http_port }}

roles/elasticsearch/templates/filebeat.yml.j2

@@ -1,272 +0,0 @@
-###################### Filebeat Configuration Example #########################
-
-# This file is an example configuration file highlighting only the most common
-# options. The filebeat.reference.yml file from the same directory contains all the
-# supported options with more comments. You can use it as a reference.
-#
-# You can find the full configuration reference here:
-# https://www.elastic.co/guide/en/beats/filebeat/index.html
-
-# For more available modules and options, please see the filebeat.reference.yml sample
-# configuration file.
-
-# ============================== Filebeat inputs ===============================
-
-filebeat.inputs:
-
-# Each - is an input. Most options can be set at the input level, so
-# you can use different inputs for various configurations.
-# Below are the input specific configurations.
-
-- type: log
-
-  # Change to true to enable this input configuration.
-  enabled: false
-
-  # Paths that should be crawled and fetched. Glob based paths.
-  paths:
-    - /var/log/ngnix/*.log
-    #- c:\programdata\elasticsearch\logs\*
-
-  # Exclude lines. A list of regular expressions to match. It drops the lines that are
-  # matching any regular expression from the list.
-  #exclude_lines: ['^DBG']
-
-  # Include lines. A list of regular expressions to match. It exports the lines that are
-  # matching any regular expression from the list.
-  #include_lines: ['^ERR', '^WARN']
-
-  # Exclude files. A list of regular expressions to match. Filebeat drops the files that
-  # are matching any regular expression from the list. By default, no files are dropped.
-  #exclude_files: ['.gz$']
-
-  # Optional additional fields. These fields can be freely picked
-  # to add additional information to the crawled log files for filtering
-  #fields:
-  #  level: debug
-  #  review: 1
-
-  ### Multiline options
-
-  # Multiline can be used for log messages spanning multiple lines. This is common
-  # for Java Stack Traces or C-Line Continuation
-
-  # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [
-  #multiline.pattern: ^\[
-
-  # Defines if the pattern set under pattern should be negated or not. Default is false.
-  #multiline.negate: false
-
-  # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern
-  # that was (not) matched before or after or as long as a pattern is not matched based on negate.
-  # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
-  #multiline.match: after
-
-# filestream is an input for collecting log messages from files. It is going to replace log input in the future.
-- type: filestream
-
-  # Change to true to enable this input configuration.
-  enabled: true
-
-  # Paths that should be crawled and fetched. Glob based paths.
-  paths:
-    - /var/log/nginx/*.log
-    #- c:\programdata\elasticsearch\logs\*
-
-  exclude_files: ['\.gz$']
-
-  # Exclude lines. A list of regular expressions to match. It drops the lines that are
-  # matching any regular expression from the list.
-  #exclude_lines: ['^DBG']
-
-  # Include lines. A list of regular expressions to match. It exports the lines that are
-  # matching any regular expression from the list.
-  #include_lines: ['^ERR', '^WARN']
-
-  # Exclude files. A list of regular expressions to match. Filebeat drops the files that
-  # are matching any regular expression from the list. By default, no files are dropped.
-  #prospector.scanner.exclude_files: ['.gz$']
-
-  # Optional additional fields. These fields can be freely picked
-  # to add additional information to the crawled log files for filtering
-  #fields:
-  #  level: debug
-  #  review: 1
-
-# ============================== Filebeat modules ==============================
-
-filebeat.config.modules:
-  # Glob pattern for configuration loading
-  path: ${path.config}/modules.d/*.yml
-
-  # Set to true to enable config reloading
-  reload.enabled: false
-
-  # Period on which files under path should be checked for changes
-  #reload.period: 10s
-
-# ======================= Elasticsearch template setting =======================
-
-setup.template.settings:
-  index.number_of_shards: 1
-  #index.codec: best_compression
-  #_source.enabled: false
-
-
-# ================================== General ===================================
-
-# The name of the shipper that publishes the network data. It can be used to group
-# all the transactions sent by a single shipper in the web interface.
-#name:
-
-# The tags of the shipper are included in their own field with each
-# transaction published.
-#tags: ["service-X", "web-tier"]
-
-# Optional fields that you can specify to add additional information to the
-# output.
-#fields:
-#  env: staging
-
-# ================================= Dashboards =================================
-# These settings control loading the sample dashboards to the Kibana index. Loading
-# the dashboards is disabled by default and can be enabled either by setting the
-# options here or by using the `setup` command.
-#setup.dashboards.enabled: false
-
-# The URL from where to download the dashboards archive. By default this URL
-# has a value which is computed based on the Beat name and version. For released
-# versions, this URL points to the dashboard archive on the artifacts.elastic.co
-# website.
-#setup.dashboards.url:
-
-# =================================== Kibana ===================================
-
-# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
-# This requires a Kibana endpoint configuration.
-setup.kibana:
-
-  # Kibana Host
-  # Scheme and port can be left out and will be set to the default (http and 5601)
-  # In case you specify and additional path, the scheme is required: http://localhost:5601/path
-  # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
-  #host: "localhost:5601"
-
-  # Kibana Space ID
-  # ID of the Kibana Space into which the dashboards should be loaded. By default,
-  # the Default Space will be used.
-  #space.id:
-
-# =============================== Elastic Cloud ================================
-
-# These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/).
-
-# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
-# `setup.kibana.host` options.
-# You can find the `cloud.id` in the Elastic Cloud web UI.
-#cloud.id:
-
-# The cloud.auth setting overwrites the `output.elasticsearch.username` and
-# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
-#cloud.auth:
-
-# ================================== Outputs ===================================
-
-# Configure what output to use when sending the data collected by the beat.
-
-# ---------------------------- Elasticsearch Output ----------------------------
-# output.elasticsearch:
-  # Array of hosts to connect to.
-  # hosts: ["elastic.schleppe:9200"]
-
-  # Protocol - either `http` (default) or `https`.
-  # protocol: "https"
-
-  # Authentication credentials - either API key or username/password.
-  #api_key: "id:api_key"
-  #username: "elastic"
-  #password: "changeme"
-
-# ------------------------------ Logstash Output -------------------------------
-output.logstash:
-  # The Logstash hosts
-  hosts: ["elasticsearch:5400"]
-
-  # Optional SSL. By default is off.
-  # List of root certificates for HTTPS server verifications
-  # ssl.certificate_authorities: ["/etc/elk-certs/elk-ssl.crt"]
-
-  # Certificate for SSL client authentication
-  # ssl.certificate: "/etc/elk-certs/elk-ssl.crt"
-
-  # Client Certificate Key
-  # ssl.key: "/etc/elk-certs/elk-ssl.key"
-
-# ================================= Processors =================================
-processors:
-  - add_host_metadata:
-      when.not.contains.tags: forwarded
-  - add_cloud_metadata: ~
-  - add_docker_metadata: ~
-  - add_kubernetes_metadata: ~
-
-# ================================== Logging ===================================
-
-# Sets log level. The default log level is info.
-# Available log levels are: error, warning, info, debug
-# logging.level: debug
-
-# At debug level, you can selectively enable logging only for some components.
-# To enable all selectors use ["*"]. Examples of other selectors are "beat",
-# "publisher", "service".
-#logging.selectors: ["*"]
-
-# ============================= X-Pack Monitoring ==============================
-# Filebeat can export internal metrics to a central Elasticsearch monitoring
-# cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The
-# reporting is disabled by default.
-
-# Set to true to enable the monitoring reporter.
-#monitoring.enabled: false
-
-# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
-# Filebeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
-# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
-#monitoring.cluster_uuid:
-
-# Uncomment to send the metrics to Elasticsearch. Most settings from the
-# Elasticsearch output are accepted here as well.
-# Note that the settings should point to your Elasticsearch *monitoring* cluster.
-# Any setting that is not set is automatically inherited from the Elasticsearch
-# output configuration, so if you have the Elasticsearch output configured such
-# that it is pointing to your Elasticsearch monitoring cluster, you can simply
-# uncomment the following line.
-#monitoring.elasticsearch:
-
-# ============================== Instrumentation ===============================
-
-# Instrumentation support for the filebeat.
-#instrumentation:
-    # Set to true to enable instrumentation of filebeat.
-    #enabled: false
-
-    # Environment in which filebeat is running on (eg: staging, production, etc.)
-    #environment: ""
-
-    # APM Server hosts to report instrumentation results to.
-    #hosts:
-    #  - http://localhost:8200
-
-    # API Key for the APM Server(s).
-    # If api_key is set then secret_token will be ignored.
-    #api_key:
-
-    # Secret token for the APM Server(s).
-    #secret_token:
-
-
-# ================================= Migration ==================================
-
-# This allows to enable 6.7 migration aliases
-#migration.6_to_7.enabled: true
-

roles/elasticsearch/templates/kibana.yml.j2

@@ -0,0 +1,28 @@
+# ansible_managed
+server.name: "{{ elk_kibana_container_name }}"
+server.host: "0.0.0.0"
+server.port: 5601
+
+# Talk to Elasticsearch over the shared docker network using its service name.
+elasticsearch.hosts: ["{{ 'https' if elk_tls_enabled else 'http' }}://{{ elk_elasticsearch_client_host }}:9200"]
+
+{% if elk_effective_security_enabled %}
+elasticsearch.username: "kibana_system"
+elasticsearch.password: "{{ elk_kibana_system_password }}"
+xpack.security.encryptionKey: "{{ elk_kibana_encryption_key }}"
+{% endif %}
+
+{% if elk_tls_enabled %}
+server.ssl.enabled: true
+server.ssl.certificate: /usr/share/kibana/config/certs/{{ elk_tls_domain }}.crt
+server.ssl.key: /usr/share/kibana/config/certs/{{ elk_tls_domain }}.key
+
+server.publicBaseUrl: "https://{{ elk_tls_domain }}:{{ elk_kibana_http_port }}"
+
+# No elasticsearch.ssl.certificateAuthorities entry needed here: the cert
+# chain is issued by a public CA (Let's Encrypt) which the container's
+# default trust store already trusts.
+{% endif %}
+
+monitoring.ui.container.elasticsearch.enabled: true
+i18n.locale: "en"

roles/elasticsearch/templates/lego-renew.service.j2

@@ -0,0 +1,29 @@
+# ansible_managed
+[Unit]
+Description=Renew ELK TLS certificate via lego (Cloudflare DNS-01)
+After=network-online.target docker.service
+Wants=network-online.target
+Requires=docker.service
+
+[Service]
+Type=oneshot
+EnvironmentFile={{ elk_cloudflare_token_env_file }}
+ExecStart={{ elk_lego_install_dir }}/lego \
+  --accept-tos \
+  --email "{{ elk_tls_acme_email }}" \
+  --server "{{ elk_tls_acme_server }}" \
+  --domains "{{ elk_tls_domain }}" \
+  --dns cloudflare \
+  --dns.resolvers "1.1.1.1:53,8.8.8.8:53" \
+  --path "{{ elk_lego_path }}" \
+  --run-hook "/usr/bin/docker compose -f {{ elk_base_dir }}/docker-compose.yml -p {{ elk_compose_project_name }} restart elasticsearch kibana logstash" \
+  renew --days {{ elk_tls_renew_days }}
+User=root
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=lego-renew
+
+# Light hardening. NoNewPrivileges/ProtectSystem kept loose enough that
+# lego can write certs under elk_lego_path and reach the docker socket
+# via the docker CLI.
+PrivateTmp=yes

roles/elasticsearch/templates/lego-renew.timer.j2

@@ -0,0 +1,11 @@
+# ansible_managed
+[Unit]
+Description=Daily timer for ELK TLS certificate renewal check (lego)
+
+[Timer]
+OnCalendar=*-*-* {{ elk_tls_renew_on_calendar }}
+RandomizedDelaySec={{ elk_tls_renew_randomized_delay }}
+Persistent=true
+
+[Install]
+WantedBy=timers.target

roles/elasticsearch/templates/logstash-conf.d/laravel_pipeline.conf.j2

@@ -1,17 +0,0 @@
-input {
-  beats {
-    port => 5045
-  }
-}
-
-filter {
-}
-
-output {
-  elasticsearch {
-    index => "laravel-logs-%{+YYYY.MM}"
-    hosts => "${ELASTIC_HOSTS}"
-    user => "elastic"
-    password => "${ELASTIC_PASSWORD}"
-  }
-}

roles/elasticsearch/templates/logstash-conf.d/main_pipeline.conf.j2.disabled

@@ -1,24 +0,0 @@
-input {
-  beats {
-    port => 5044
-  }
-}
-
-filter {
-  if [pipeline_id] == "nginx" {
-    mutate { add_field => { "route" => "nginx_pipeline" } }
-  } else if [pipeline_id] == "laravel" {
-    mutate { add_field => { "route" => "laravel_pipeline" } }
-  }
-}
-
-output {
-  if [pipeline_id] == "nginx" {
-    pipeline { send_to => "nginx_pipeline" }
-  } else if [pipeline_id] == "laravel" {
-    pipeline { send_to => "laravel_pipeline" }
-  } else {
-    # Handle unknown cases
-    stdout { codec => rubydebug }
-  }
-}

roles/elasticsearch/templates/logstash.yml.j2

@@ -0,0 +1,6 @@
+# ansible_managed
+http.host: "0.0.0.0"
+http.port: 9600
+
+path.config: /usr/share/logstash/pipeline
+xpack.monitoring.enabled: false

roles/elasticsearch/templates/main.conf.j2

@@ -0,0 +1,38 @@
+# ansible_managed
+input {
+  beats {
+    port => 5044
+  }
+  tcp {
+    port => 5000
+    codec => json_lines
+  }
+  udp {
+    port => 5000
+    codec => json_lines
+  }
+}
+
+filter {
+  # Add filters here as needed (grok, mutate, date, etc.)
+}
+
+output {
+  elasticsearch {
+    # Reach Elasticsearch over the shared docker network using its service name.
+    hosts => ["{{ 'https' if elk_tls_enabled else 'http' }}://{{ elk_elasticsearch_client_host }}:9200"]
+{% if elk_effective_security_enabled %}
+    user => "logstash_system"
+    password => "{{ elk_logstash_system_password }}"
+{% endif %}
+{% if elk_tls_enabled %}
+    ssl_enabled => true
+    # Public CA (Let's Encrypt) - trusted by Logstash's default JVM trust
+    # store already, no custom CA needs to be supplied here.
+{% endif %}
+    index => "logstash-%{+YYYY.MM.dd}"
+  }
+
+  # Uncomment for local debugging of pipeline output:
+  # stdout { codec => rubydebug }
+}

roles/elasticsearch/templates/pipeline-laravel.conf.j2

@@ -0,0 +1,26 @@
+# ansible_managed
+input {
+  beats {
+    port => 5046
+  }
+}
+
+filter {
+}
+
+output {
+  elasticsearch {
+    # Reach Elasticsearch over the shared docker network using its service name.
+    hosts => ["{{ 'https' if elk_tls_enabled else 'http' }}://{{ elk_elasticsearch_client_host }}:9200"]
+{% if elk_effective_security_enabled %}
+    user => "logstash_system"
+    password => "{{ elk_logstash_system_password }}"
+{% endif %}
+{% if elk_tls_enabled %}
+    ssl_enabled => true
+    # Public CA (Let's Encrypt) - trusted by Logstash's default JVM trust
+    # store already, no custom CA needs to be supplied here.
+{% endif %}
+    index => "laravel-logs-%{+YYYY.MM}"
+  }
+}

roles/elasticsearch/templates/pipeline-nginx.conf.j2

@@ -1,6 +1,7 @@
+# ansible_managed
 input {
   beats {
-    port => 5044
+    port => 5045
   }
 }
 
@@ -36,10 +37,17 @@ filter {
 
 output {
   elasticsearch {
+    # Reach Elasticsearch over the shared docker network using its service name.
+    hosts => ["{{ 'https' if elk_tls_enabled else 'http' }}://{{ elk_elasticsearch_client_host }}:9200"]
+{% if elk_effective_security_enabled %}
+    user => "logstash_system"
+    password => "{{ elk_logstash_system_password }}"
+{% endif %}
+{% if elk_tls_enabled %}
+    ssl_enabled => true
+    # Public CA (Let's Encrypt) - trusted by Logstash's default JVM trust
+    # store already, no custom CA needs to be supplied here.
+{% endif %}
     index => "weblogs-%{+YYYY.MM}"
-    hosts => "${ELASTIC_HOSTS}"
-    user => "elastic"
-    password => "${ELASTIC_PASSWORD}"
-    document_type => "nginx_logs"
   }
 }

roles/elasticsearch/templates/pipelines.yml.j2

@@ -1,5 +1,9 @@
+# ansible_managed
+- pipeline.id: main
+  path.config: "/usr/share/logstash/pipeline/main.conf"
+
 - pipeline.id: nginx_pipeline
-  path.config: "/usr/share/logstash/pipeline/nginx_pipeline.conf"
+  path.config: "/usr/share/logstash/pipeline/pipeline-nginx.conf"
 
 - pipeline.id: laravel_pipeline
-  path.config: "/usr/share/logstash/pipeline/laravel_pipeline.conf"
+  path.config: "/usr/share/logstash/pipeline/pipeline-laravel.conf"

roles/oh-my-zsh/templates/zshrc.j2

@@ -1,4 +1,4 @@
-{{ ansible_managed | comment }}
+# ansible_managed
 
 {% set oh_my_zsh = user.oh_my_zsh|default({}) %}
 

roles/umami/templates/docker-compose.yml.j2

@@ -1,4 +1,4 @@
-{{ ansible_managed | comment }}
+# ansible_managed
 # Based on: https://github.com/umami-software/umami/blob/master/docker-compose.yml
 services:
   umami: