From 86d8986630e04ba318a7de00f9e56d06adf95244 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Fri, 7 Aug 2026 16:23:00 +0200 Subject: [PATCH] update elasticsearch w/ ssl & xpack support --- plays/elasticsearch.yml | 6 - roles/elasticsearch/README.md | 250 +++++++++++++++- roles/elasticsearch/defaults/main.yml | 135 +++++++++ roles/elasticsearch/handlers/main.yml | 38 +++ roles/elasticsearch/meta/main.yml | 20 +- roles/elasticsearch/tasks/elasticsearch.yml | 41 --- roles/elasticsearch/tasks/elk-network.yml | 11 - roles/elasticsearch/tasks/kibana.yml | 37 --- roles/elasticsearch/tasks/logstash.yml | 65 ----- roles/elasticsearch/tasks/main.yml | 217 +++++++++++++- roles/elasticsearch/tasks/tls.yml | 176 ++++++++++++ .../templates/docker-compose.yml.j2 | 147 ++++++++++ .../templates/elasticsearch.yml.j2 | 29 ++ roles/elasticsearch/templates/filebeat.yml.j2 | 272 ------------------ roles/elasticsearch/templates/kibana.yml.j2 | 28 ++ .../templates/lego-renew.service.j2 | 28 ++ .../templates/lego-renew.timer.j2 | 11 + .../logstash-conf.d/laravel_pipeline.conf.j2 | 17 -- .../main_pipeline.conf.j2.disabled | 24 -- roles/elasticsearch/templates/logstash.yml.j2 | 6 + roles/elasticsearch/templates/main.conf.j2 | 38 +++ .../templates/pipeline-laravel.conf.j2 | 26 ++ ...ipeline.conf.j2 => pipeline-nginx.conf.j2} | 18 +- .../elasticsearch/templates/pipelines.yml.j2 | 8 +- 24 files changed, 1139 insertions(+), 509 deletions(-) create mode 100644 roles/elasticsearch/defaults/main.yml create mode 100644 roles/elasticsearch/handlers/main.yml delete mode 100644 roles/elasticsearch/tasks/elasticsearch.yml delete mode 100644 roles/elasticsearch/tasks/elk-network.yml delete mode 100644 roles/elasticsearch/tasks/kibana.yml delete mode 100644 roles/elasticsearch/tasks/logstash.yml create mode 100644 roles/elasticsearch/tasks/tls.yml create mode 100644 roles/elasticsearch/templates/docker-compose.yml.j2 create mode 100644 roles/elasticsearch/templates/elasticsearch.yml.j2 delete mode 100644 roles/elasticsearch/templates/filebeat.yml.j2 create mode 100644 roles/elasticsearch/templates/kibana.yml.j2 create mode 100644 roles/elasticsearch/templates/lego-renew.service.j2 create mode 100644 roles/elasticsearch/templates/lego-renew.timer.j2 delete mode 100644 roles/elasticsearch/templates/logstash-conf.d/laravel_pipeline.conf.j2 delete mode 100644 roles/elasticsearch/templates/logstash-conf.d/main_pipeline.conf.j2.disabled create mode 100644 roles/elasticsearch/templates/logstash.yml.j2 create mode 100644 roles/elasticsearch/templates/main.conf.j2 create mode 100644 roles/elasticsearch/templates/pipeline-laravel.conf.j2 rename roles/elasticsearch/templates/{logstash-conf.d/nginx_pipeline.conf.j2 => pipeline-nginx.conf.j2} (61%) diff --git a/plays/elasticsearch.yml b/plays/elasticsearch.yml index b04fb68..af4c0ec 100644 --- a/plays/elasticsearch.yml +++ b/plays/elasticsearch.yml @@ -1,12 +1,6 @@ --- - name: Install ELK stack using Docker hosts: all - vars: - elk_version: "9.4.4" roles: - # TODO check for docker install before purging it. This nukes containers - # we might not want destroyed. - # Leave a file, or read the same input requirements into a apt check - # which skips purge if satisfied. - role: roles/docker - role: roles/elasticsearch diff --git a/roles/elasticsearch/README.md b/roles/elasticsearch/README.md index ec787fe..e3eb641 100644 --- a/roles/elasticsearch/README.md +++ b/roles/elasticsearch/README.md @@ -1,23 +1,247 @@ -# elasticsearch +# elk_stack -Play configures ELK stack using docker & is available without HTTPS. Configure container variables in `tasks/SERVICE.yml` files, environment variables for the services & Java can also be configured here. +Deploys Elasticsearch, Logstash and Kibana as a single Docker Compose stack +on a Debian host that already has Docker (and the `docker compose` plugin) +installed and running. All three services join a shared bridge network +(`elk-network` by default) so they can reach each other by container/service +name — no hardcoded IPs, no `--link`. -The following are manual steps required during setup. +## Requirements -## elastic +- Debian host with Docker Engine + Docker Compose plugin already installed +- Ansible collection: `community.docker` (`ansible-galaxy collection install community.docker`) +- Python `docker` + `requests` modules on the target (the role installs these via pip) +- Enough RAM for the JVM heaps you configure (Elasticsearch + Logstash default to ~1.5G combined; leave headroom for OS/Docker) -After creating elasticsearch container SSH into the running host and generate a new password for user `elastic` using command: +## What it does -```bash -docker exec -it elasticsearch /usr/share/elasticsearch/bin/elasticsearch-reset-password -u elastic +1. Verifies Docker and Docker Compose are usable. +2. Creates `{{ elk_base_dir }}` (default `/opt/elk`) with a `config/` tree. +3. Templates: + - `config/elasticsearch/elasticsearch.yml` + - `config/kibana/kibana.yml` + - `config/logstash/logstash.yml`, `pipelines.yml`, and three pipeline + configs: `pipeline/main.conf` (generic beats/tcp/udp), `pipeline/pipeline-nginx.conf` + (nginx access logs), `pipeline/pipeline-laravel.conf` (Laravel logs) + - `docker-compose.yml` — one file, three services, one shared network, a named volume for ES data +4. Brings the stack up with `docker compose up`. +5. Waits for Elasticsearch to respond, then (if security is enabled) sets the + `kibana_system` and `logstash_system` built-in user passwords via the + Elasticsearch security API, and recreates Kibana/Logstash so they pick up + working credentials. +6. Waits for Kibana's `/api/status` to come back healthy. + +## Networking + +All three containers run on the `elk_network_name` bridge network +(default `elk-network`, subnet `172.28.0.0/24`). Inside that network, all +inter-service traffic uses `http://` or `https://` consistently based on +`elk_tls_enabled` (default stack, TLS on): + +- Elasticsearch is reachable at `https://elasticsearch:9200` +- Kibana is configured with `elasticsearch.hosts: ["https://elasticsearch:9200"]` +- Logstash's pipelines all point at `https://elasticsearch:9200` + +With `elk_tls_enabled: false`, all three of the above use `http://` instead. + +Host ports (`elk_elasticsearch_http_port`, `elk_kibana_http_port`, +`elk_logstash_beats_port`, `elk_logstash_nginx_beats_port`, +`elk_logstash_laravel_beats_port`, etc.) are only for reaching the stack +*from outside* — the containers themselves never use `localhost` to talk to +each other. + +## Security + +`elk_security_enabled: true` by default (recommended). This turns on +Elastic's built-in security (basic auth, no TLS between containers since +traffic stays on the internal Docker network). Change these before running +in anything but a scratch environment — ideally via Ansible Vault: + +```yaml +elk_elastic_password: "..." +elk_kibana_system_password: "..." +elk_logstash_system_password: "..." +elk_kibana_encryption_key: "..." # 32+ random chars, e.g. `openssl rand -hex 32` ``` -## kibana +Set `elk_security_enabled: false` for a quick, unauthenticated local/dev +stack (no passwords) — do not do this on anything reachable from untrusted +networks. -Create a password for `kibana_system` user: +**Note:** `elk_tls_enabled` and `elk_security_enabled` are independent +switches you can set separately, but Elasticsearch only applies TLS +(`xpack.security.http.ssl.*`) when X-Pack security itself is on. So if +`elk_tls_enabled: true` and `elk_security_enabled: false`, this role +force-enables security anyway purely to make TLS possible — `elastic`, +`kibana_system`, and `logstash_system` all get real passwords from +`elk_elastic_password`/`elk_kibana_system_password`/ +`elk_logstash_system_password` in that mode too, exactly as if +`elk_security_enabled: true` had been set, so change those from their +placeholder defaults the same way you would for a normal secured stack. +Setting both `elk_tls_enabled: false` and `elk_security_enabled: false` is +the only way to get a fully open, unauthenticated, plain-HTTP stack. -```bash -export ELASTIC_PASSWORD= -export KIBANA_PASSWORD= -curl -s -X POST -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" http://elasticsearch:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; +## Example playbook + +```yaml +- hosts: elk_servers + become: true + roles: + - role: elk_stack + vars: + elk_version: "8.15.3" + elk_elasticsearch_heap_size: "2g" + elk_logstash_heap_size: "1g" + elk_elastic_password: "{{ vault_elk_elastic_password }}" + elk_kibana_system_password: "{{ vault_elk_kibana_password }}" + elk_logstash_system_password: "{{ vault_elk_logstash_password }}" + elk_kibana_encryption_key: "{{ vault_elk_kibana_enc_key }}" ``` + +## Customizing the Logstash pipeline + +Three pipelines run side by side inside the same Logstash process, each on +its own beats port (they can't share a port): + +| Pipeline | Template | Beats port var | Purpose | +|---|---|---|---| +| `main` | `main.conf.j2` | `elk_logstash_beats_port` (5044) | Generic beats input, plus raw JSON over TCP/UDP (5000) | +| `nginx_pipeline` | `pipeline-nginx.conf.j2` | `elk_logstash_nginx_beats_port` (5045) | nginx access log parsing (grok + geoip) | +| `laravel_pipeline` | `pipeline-laravel.conf.j2` | `elk_logstash_laravel_beats_port` (5046) | Laravel log ingestion | + +All three ship to Elasticsearch using the same `elk_tls_enabled`/ +`elk_security_enabled` logic as the rest of the stack (TLS scheme + the +`logstash_system` credentials, when applicable). To add your own +filters/inputs/outputs, either edit these templates directly, or drop an +additional `*.conf.j2` file into `templates/` and extend `pipelines.yml.j2` +(with a new `pipeline.id`/`path.config` entry and a beats port that doesn't +collide with the ones above) and `tasks/main.yml` (a matching `template:` +task) to deploy it. + +## Idempotency notes + +- Re-running the role only recreates containers whose config actually + changed (via handlers), plus a compose reconciliation pass. +- Setting the `kibana_system`/`logstash_system` passwords is safe to repeat; + Elasticsearch just resets them to the same value each time and the role + only recreates those two containers when the API call reports a change. + +## TLS (public certs for a private-IP node) + +`elk_tls_enabled: true` (default) gets Elasticsearch, Kibana, and Logstash a +**real, publicly-trusted certificate** for a DNS name that happens to point +at a private IP (e.g. `logs-eu.domain.com -> 10.0.99.5`), using the ACME +**DNS-01** challenge against Cloudflare. This works even though the host +itself is unreachable from the internet, because DNS-01 only requires the +ability to create a `_acme-challenge` TXT record — no inbound HTTP/HTTPS +access to the node is needed at all. + +### Why DNS-01, and why one shared cert + +- Public CAs don't issue certificates for bare IP SANs (especially not + RFC1918/private addresses) — the workaround is a cert for a **DNS name** + that resolves to the private IP, with clients connecting via that name. +- Since all three services live on the same node behind the same DNS name, + they share **one cert** rather than one per service. Elastic explicitly + supports (and this is a common pattern) using [a single shared HTTP CA + across Elasticsearch, Kibana, and other stack components](https://www.elastic.co/docs/deploy-manage/deploy/self-managed/tutorial-self-managed-secure) + — separate CAs/certs per component exist to isolate trust domains, which + isn't a goal here since everything is already inside the same trust + boundary (one box, one operator, one Docker network). +- Because the cert comes from a **public** CA (Let's Encrypt, via lego's + default ACME server), Kibana and Logstash don't need a custom CA + certificate distributed to them to trust Elasticsearch's cert — their + default trust stores already trust Let's Encrypt's root. This removes an + entire category of "distribute the CA cert everywhere" complexity that + self-signed/internal-CA setups require. + +### What this role does + +1. Expects `elk_cloudflare_token_env_file` (default + `/etc/lego/cloudflare-token.env`) to **already exist** on the box, + containing `CF_DNS_API_TOKEN=...`, root-owned, mode `0600`. This role + only *reads* that file — it never receives, logs, or stores the token + itself. See "IaC responsibilities" below for where this file comes from. +2. Installs the [`lego`](https://github.com/go-acme/lego) ACME client (a + single static Go binary — no Python/snap dependencies, easy to pin an + exact version). +3. On first run, issues a certificate for `elk_tls_domain` via `lego ... run`, + landing at `{{ elk_lego_certs_dir }}/{{ elk_tls_domain }}.{crt,key,issuer.crt}`. +4. Mounts that directory read-only into all three containers and wires + `xpack.security.http.ssl.*` (Elasticsearch), `server.ssl.*` (Kibana), and + `ssl_enabled` (Logstash's elasticsearch output) to point at it. +5. Installs a `lego-renew.service` + `lego-renew.timer` systemd pair that + runs daily, checks expiry, and **only** re-issues (and only then restarts + the three containers, via lego's `--run-hook`) when the cert is within + `elk_tls_renew_days` of expiring. No-op checks never restart anything. + +### IaC responsibilities (Pulumi, not this role) + +This role deliberately does **not** manage Cloudflare DNS records, zones, or +API tokens — that's Pulumi's job, since it already holds the Cloudflare +credentials and provisions the box. The pseudocode below sketches what +Pulumi needs to set up per node: + +```python +# Pulumi pseudocode - see conversation history / infra repo for the real version +apex_zone = cloudflare.get_zone(name="domain.com") + +# A record already exists in this setup: logs-eu.domain.com -> 10.0.99.x +# (proxied=False - a private IP can't sit behind Cloudflare's proxy/edge) + +# Scoped API token: Zone.DNS.Edit, restricted to the apex zone's ID (or a +# delegated child zone, e.g. logs-eu.domain.com as its own zone, for harder +# isolation). This is the tightest scope Cloudflare's token model supports - +# there's no way to restrict a token to a single record name/pattern. +acme_dns_token = cloudflare.ApiToken( + "logs-eu-acme-dns-token", + policies=[{ + "permission_groups": [dns_edit_permission_group], + "resources": {f"com.cloudflare.api.account.zone.{apex_zone.id}": "*"}, + }], + # optional: condition.request_ip.in_ to pin the token to the node's + # egress IP, and/or expires_on for periodic rotation via Pulumi re-runs +) + +# Hand the token to the VM via cloud-init, written root-only - Pulumi never +# passes this to Ansible as a variable. +cloud_init_write_files = [{ + "path": "/etc/lego/cloudflare-token.env", + "owner": "root:root", + "permissions": "0600", + "content": f"CF_DNS_API_TOKEN={acme_dns_token.value}\n", +}] +``` + +For stronger isolation than a single shared zone token, NS-delegate +`logs-eu.domain.com` (and `logs-us.domain.com`, etc.) to their own +Cloudflare zones, and scope each region's token to only its own zone ID — +that way a compromised `logs-eu` node's token can't touch `logs-us` or the +apex domain's records at all. + +### Key variables + +| Variable | Purpose | +|---|---| +| `elk_tls_enabled` | Master switch; `false` reverts to plain HTTP | +| `elk_tls_domain` | The DNS name (e.g. `logs-eu.domain.com`) the cert covers and clients must connect via | +| `elk_tls_acme_email` | Contact address registered with the ACME account | +| `elk_tls_acme_server` | ACME endpoint — point at Let's Encrypt **staging** while testing to avoid production rate limits | +| `elk_cloudflare_token_env_file` | Path to the pre-existing, cloud-init-provisioned token file | +| `elk_tls_renew_days` | Renew when fewer than this many days remain before expiry | +| `elk_lego_version` | Pinned lego release to install | + +### Operational notes + +- **Testing**: set `elk_tls_acme_server` to + `https://acme-staging-v02.api.letsencrypt.org/directory` first — staging + certs aren't publicly trusted but validate the whole DNS-01/Cloudflare + flow without touching Let's Encrypt's production rate limits. +- **Downtime on renewal**: renewal restarts all three containers via + `docker compose restart`, which is brief (~10-20s) but not zero-downtime. + Acceptable for a single-node stack; a zero-downtime rolling reload would + require a multi-node Elasticsearch cluster, which is out of scope here. +- **Clients must use the DNS name, not the raw IP** — connecting to + `https://10.0.99.5:9200` directly will fail certificate hostname + verification even though the cert is otherwise valid, since the cert's + SAN is `logs-eu.domain.com`, not the IP. diff --git a/roles/elasticsearch/defaults/main.yml b/roles/elasticsearch/defaults/main.yml new file mode 100644 index 0000000..7c1343d --- /dev/null +++ b/roles/elasticsearch/defaults/main.yml @@ -0,0 +1,135 @@ +--- +# --------------------------------------------------------------------------- +# Environment variable overrides +# --------------------------------------------------------------------------- +# Every setting below can also be supplied as an env var on the CONTROL node +# (i.e. wherever `ansible-playbook` runs - a CI job, your laptop, etc.), not +# the remote target host. `lookup('env', ...)` reads from the control node's +# process environment at template-render time, which is exactly what you +# want for injecting CI/CD secrets (e.g. GitHub Actions `env:`/`secrets:`, +# a `.env` file sourced before the playbook runs) without touching inventory +# or vault files. +# +# Precedence, highest first: `-e` / extra vars on the CLI > host/group_vars +# in inventory > playbook `vars:` > these defaults (env var, if set, else +# the literal fallback shown). So an env var set here is easy to override +# per-host or per-run if needed - it's just the last thing consulted. +# +# Booleans: env vars are always strings, so `elk_tls_enabled`/ +# `elk_security_enabled` run through `| bool` to turn "true"/"false"/"1"/"0"/ +# "yes"/"no" (case-insensitive) into real Ansible booleans - a raw string +# "false" would otherwise be truthy in Jinja `{% if %}` conditionals. +# --------------------------------------------------------------------------- + +# --------------------------------------------------------------------------- +# General +# --------------------------------------------------------------------------- +elk_version: "{{ lookup('env', 'ELK_VERSION') | default('8.15.3', true) }}" + +elk_base_dir: /opt/elk +elk_compose_project_name: elk + +elk_network_name: elk-network +elk_network_subnet: "172.28.0.0/24" + +# Set true only after you've configured elk_elastic_password etc. for real use. +# When false, security is disabled for a quick local/dev stack. +elk_security_enabled: "{{ lookup('env', 'ELK_SECURITY_ENABLED') | default('true', true) | bool }}" + +# Change these in production (e.g. via vault, or ELK_ELASTIC_PASSWORD etc. +# from CI secrets). These fallbacks are intentionally obvious placeholders +# so nobody ships them by accident. +elk_elastic_password: "{{ lookup('env', 'ELK_ELASTIC_PASSWORD') | default('ChangeMe_Elastic_123!', true) }}" +elk_kibana_system_password: "{{ lookup('env', 'ELK_KIBANA_SYSTEM_PASSWORD') | default('ChangeMe_Kibana_123!', true) }}" +elk_logstash_system_password: "{{ lookup('env', 'ELK_LOGSTASH_SYSTEM_PASSWORD') | default('ChangeMe_Logstash_123!', true) }}" + +# A pre-shared 32+ char key used to encrypt the Kibana saved-objects store. +elk_kibana_encryption_key: "{{ lookup('env', 'ELK_KIBANA_ENCRYPTION_KEY') | default('ChangeMe_32Char_Kibana_EncKey!!!', true) }}" + +# --------------------------------------------------------------------------- +# Elasticsearch +# --------------------------------------------------------------------------- +elk_elasticsearch_container_name: elasticsearch +elk_elasticsearch_http_port: 9200 +elk_elasticsearch_transport_port: 9300 +elk_elasticsearch_heap_size: "1g" +elk_elasticsearch_cluster_name: "elk-retailor" +elk_elasticsearch_node_name: "{{ lookup('env', 'ELK_ELASTIC_NODE_NAME') | default('es01', true) }}" +elk_elasticsearch_data_volume: elk_esdata + +# --------------------------------------------------------------------------- +# Kibana +# --------------------------------------------------------------------------- +elk_kibana_container_name: kibana +elk_kibana_http_port: 5601 + +# --------------------------------------------------------------------------- +# Logstash +# --------------------------------------------------------------------------- +elk_logstash_container_name: logstash +elk_logstash_heap_size: "512m" +elk_logstash_tcp_port: 5000 +elk_logstash_udp_port: 5000 +elk_logstash_http_port: 9600 + +# Each pipeline that has its own beats input needs its own port - they all +# run inside the same Logstash process, so two pipelines can't bind the same +# port. main.conf (generic beats/tcp/udp) and the nginx/laravel example +# pipelines are each wired to a distinct one below. +elk_logstash_beats_port: 5044 +elk_logstash_nginx_beats_port: 5045 +elk_logstash_laravel_beats_port: 5046 + +# --------------------------------------------------------------------------- +# Docker image repository (override for an internal mirror/proxy) +# --------------------------------------------------------------------------- +elk_docker_image_repo: "docker.elastic.co" + +# --------------------------------------------------------------------------- +# TLS via lego (ACME DNS-01 against Cloudflare) +# --------------------------------------------------------------------------- +# Master switch. When false, the stack runs plain HTTP as before (nothing +# below this point applies). +elk_tls_enabled: "{{ lookup('env', 'ELK_TLS_ENABLED') | default('true', true) | bool }}" + +# The public DNS name this node is reachable at, e.g. "logs.domain.com". +# An A record for this name -> the node's private IP is assumed to already +# exist (managed by IaC, not this role). This name is what ES/Kibana/Logstash +# put in their certs and what clients must connect to (not the raw IP) for +# TLS verification to succeed. +elk_tls_domain: "{{ lookup('env', 'ELK_TLS_DOMAIN') | default('logs.domain.com', true) }}" + +# Contact email lego registers with the ACME account (Let's Encrypt notifies +# this address about certificate/account issues, not renewal reminders, +# since renewal is automated). +elk_tls_acme_email: "{{ lookup('env', 'ELK_TLS_ACME_EMAIL') | default('webmaster@domain.com', true) }}" + +# Let's Encrypt production endpoint by default. Point this at the staging +# endpoint while testing to avoid production rate limits: +# https://acme-staging-v02.api.letsencrypt.org/directory +elk_tls_acme_server: "{{ lookup('env', 'ELK_TLS_ACME_SERVER') | default('https://acme-v02.api.letsencrypt.org/directory', true) }}" + +# lego binary version/arch and install location. +elk_lego_version: "4.20.4" +elk_lego_arch: "amd64" # amd64 | arm64 +elk_lego_install_dir: /usr/local/bin +elk_lego_path: /etc/lego # lego's --path (accounts + certificates live under here) +elk_lego_certs_dir: "{{ elk_lego_path }}/certificates" + +# Path to the Cloudflare token env file. This is expected to already exist, +# dropped by cloud-init at provisioning time (see IaC notes) as +# root:root 0600, containing a line like: +# CF_DNS_API_TOKEN=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +# This role only READS this file - it never receives or stores the token +# itself, so the Cloudflare credential never touches Ansible inventory, +# vault, logs, or the control node. +elk_cloudflare_token_env_file: "{{ lookup('env', 'ELK_CLOUDFLARE_TOKEN_ENV_FILE') | default('/etc/lego/cloudflare-token.env', true) }}" + +# Threshold used on renewal checks - lego renews if the current cert has +# fewer than this many days left before expiry. +elk_tls_renew_days: 30 + +# Renewal check cadence via systemd timer. +elk_tls_renew_on_calendar: "22:00:00" +elk_tls_renew_randomized_delay: "30m" + diff --git a/roles/elasticsearch/handlers/main.yml b/roles/elasticsearch/handlers/main.yml new file mode 100644 index 0000000..4ae4593 --- /dev/null +++ b/roles/elasticsearch/handlers/main.yml @@ -0,0 +1,38 @@ +--- +- name: restart elk stack + community.docker.docker_compose_v2: + project_src: "{{ elk_base_dir }}" + project_name: "{{ elk_compose_project_name }}" + state: present + recreate: always + listen: "restart elk stack" + +- name: recreate elasticsearch + community.docker.docker_compose_v2: + project_src: "{{ elk_base_dir }}" + project_name: "{{ elk_compose_project_name }}" + services: + - elasticsearch + state: present + recreate: always + listen: "recreate elasticsearch" + +- name: recreate logstash + community.docker.docker_compose_v2: + project_src: "{{ elk_base_dir }}" + project_name: "{{ elk_compose_project_name }}" + services: + - logstash + state: present + recreate: always + listen: "recreate logstash" + +- name: recreate kibana + community.docker.docker_compose_v2: + project_src: "{{ elk_base_dir }}" + project_name: "{{ elk_compose_project_name }}" + services: + - kibana + state: present + recreate: always + listen: "recreate kibana" diff --git a/roles/elasticsearch/meta/main.yml b/roles/elasticsearch/meta/main.yml index f46d0e3..7ba7553 100644 --- a/roles/elasticsearch/meta/main.yml +++ b/roles/elasticsearch/meta/main.yml @@ -1,16 +1,16 @@ --- galaxy_info: - author: Your Name - description: Ansible role to deploy prometheus & grafana using Docker - license: MIT - min_ansible_version: "2.9" + role_name: elk_stack + author: your-name + description: Deploys Elasticsearch, Logstash and Kibana via Docker Compose on a shared network + min_ansible_version: "2.14" platforms: - name: Debian versions: - - all - galaxy_tags: - - monitoring - - cadvisor - dependencies: - - docker + - bullseye + - bookworm +dependencies: [] + +collections: + - community.docker diff --git a/roles/elasticsearch/tasks/elasticsearch.yml b/roles/elasticsearch/tasks/elasticsearch.yml deleted file mode 100644 index 92120c9..0000000 --- a/roles/elasticsearch/tasks/elasticsearch.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -- name: Pull Elasticsearch Docker image - docker_image: - name: docker.elastic.co/elasticsearch/elasticsearch-wolfi:{{ elk_version }} - source: pull - -- name: Create Elasticsearch configuration file directory on host - file: - path: /etc/elasticsearch - state: directory - mode: '0755' - - # - name: Create Elasticsearch configuration file - # template: - # src: elasticsearch.yml.j2 - # dest: /etc/elasticsearch/elasticsearch.yml - -- name: Start Elasticsearch container - docker_container: - name: elasticsearch - image: docker.elastic.co/elasticsearch/elasticsearch-wolfi:{{ elk_version }} - state: started - restart: yes - restart_policy: unless-stopped - published_ports: - - "9200:9200" - volumes: - - /etc/elasticsearch/esdata:/usr/share/elasticsearch/data - networks: - - name: elk_network - ipv4_address: 172.19.0.2 - env: - node.name: elasticsearch - cluster.name: retailor-elk - discovery.type: single-node - bootstrap.memory_lock: "true" - # limits elasticsearch to 2 GB of RAM - ES_JAVA_OPTS: "-Xms1g -Xmx2g" - # disables SSL & xpack security - xpack.security.http.ssl.enabled: "false" - diff --git a/roles/elasticsearch/tasks/elk-network.yml b/roles/elasticsearch/tasks/elk-network.yml deleted file mode 100644 index d9c581b..0000000 --- a/roles/elasticsearch/tasks/elk-network.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: Create a Docker network for Elasticsearch - docker_network: - name: elk_network - state: present - driver: bridge - ipam_config: - - subnet: "172.19.0.0/16" - gateway: "172.19.0.1" - iprange: "172.19.0.0/24" - diff --git a/roles/elasticsearch/tasks/kibana.yml b/roles/elasticsearch/tasks/kibana.yml deleted file mode 100644 index 3d9d08f..0000000 --- a/roles/elasticsearch/tasks/kibana.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- name: Create a Docker network for Kibana - docker_network: - name: elk_network - state: present - -- name: Create kibana directory on host - file: - path: /etc/kibana - state: directory - mode: "0755" - -- name: Pull Kibana Docker image - docker_image: - name: docker.elastic.co/kibana/kibana:{{ elk_version }} - source: pull - -# TODO rember to move CA cert from elastic to Kibana -# docker cp elasticsearch:/usr/share/elasticsearch/config/certs/http_ca.crt . -# docker cp http_ca.crt kibana:/usr/share/kibana/config/certs/ca/http_ca.crt -- name: Start Kibana container - docker_container: - name: kibana - image: docker.elastic.co/kibana/kibana:{{ elk_version }} - state: started - restart: yes - restart_policy: unless-stopped - published_ports: - - "5601:5601" - env: - ELASTICSEARCH_HOSTS: "{{ env_vars.ELASTIC_HOSTS }}" - ELASTICSEARCH_USERNAME: kibana_system - ELASTICSEARCH_PASSWORD: "{{ env_vars.KIBANA_PASSWORD }}" - TELEMETRY_ENABLED: "false" - networks: - - name: elk_network - ipv4_address: 172.19.0.3 diff --git a/roles/elasticsearch/tasks/logstash.yml b/roles/elasticsearch/tasks/logstash.yml deleted file mode 100644 index 87b4935..0000000 --- a/roles/elasticsearch/tasks/logstash.yml +++ /dev/null @@ -1,65 +0,0 @@ ---- -- name: Create a Docker network for Logstash - docker_network: - name: elk_network - state: present - -- name: Create logstash directory on host - file: - path: /etc/logstash - state: directory - mode: "0755" - -- name: Copy logstash config - copy: - src: templates/pipelines.yml.j2 - dest: /etc/logstash/pipelines.yml - -- name: Create logstash directory on host - file: - path: /etc/logstash/pipeline - state: directory - mode: "0755" - -- name: Copy logstash input configs - copy: - src: "{{ item }}" - dest: /etc/logstash/pipeline/{{ item | basename | regex_replace('\.j2$', '') }} - loop: "{{ query('fileglob', 'templates/logstash-conf.d/*.j2') }}" - -- name: Pull Logstash Docker image - docker_image: - name: docker.elastic.co/logstash/logstash:{{ elk_version }} - source: pull - -# TODO rember to move CA cert from elastic to Logstash -# docker cp elasticsearch:/usr/share/elasticsearch/config/certs/http_ca.crt . -# docker cp http_ca.crt logstash:/usr/share/logstash/config/certs/ca/http_ca.crt -- name: Start Logstash container - docker_container: - name: logstash - image: docker.elastic.co/logstash/logstash:{{ elk_version }} - state: started - restart: yes - restart_policy: unless-stopped - command: - - /bin/bash - - -c - - | - echo "Waiting for Elasticsearch availability"; - until curl -s {{ env_vars.ELASTIC_HOSTS }} | grep -q "missing authentication credentials"; do sleep 1; done; - echo "Starting logstash"; - /usr/share/logstash/bin/logstash - published_ports: - - "5044-5049:5044-5049" - volumes: - - /etc/logstash/pipelines.yml:/usr/share/logstash/config/pipelines.yml - - /etc/logstash/pipeline:/usr/share/logstash/pipeline - env: - xpack.monitoring.enabled: "false" - ELASTIC_USER: elastic - ELASTIC_PASSWORD: "{{ env_vars.ELASTIC_PASSWORD }}" - ELASTIC_HOSTS: "{{ env_vars.ELASTIC_HOSTS }}" - networks: - - name: elk_network - ipv4_address: 172.19.0.4 diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 48bd97c..76e9b22 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -1,8 +1,213 @@ --- -# ensure we have variables from .env files -- include_tasks: ../roles/env/tasks/main.yml +# --------------------------------------------------------------------------- +# TLS (elk_tls_enabled) and X-Pack security (elk_security_enabled) are +# documented as independent switches, but Elasticsearch only honors +# xpack.security.http.ssl.* when xpack.security.enabled is also true. So +# whenever TLS is on, security (and therefore basic auth / ELASTIC_PASSWORD) +# is force-enabled too - elk_effective_security_enabled is the single fact +# every template and task below should use instead of elk_security_enabled +# directly, so this coupling only has to be expressed once. +# --------------------------------------------------------------------------- +- name: Derive effective security state (TLS implies security) + ansible.builtin.set_fact: + elk_effective_security_enabled: "{{ elk_security_enabled or elk_tls_enabled }}" -- include_tasks: elk-network.yml -- include_tasks: elasticsearch.yml -- include_tasks: kibana.yml -- include_tasks: logstash.yml +- name: Assert Docker is available + ansible.builtin.command: docker info + register: elk_docker_info + changed_when: false + failed_when: elk_docker_info.rc != 0 + +- name: Ensure Docker Compose plugin is installed + ansible.builtin.command: docker compose version + register: elk_compose_check + changed_when: false + failed_when: elk_compose_check.rc != 0 + +- name: Ensure we have pip installed + apt: + pkg: "python3-pip" + state: present + +- name: Ensure python docker SDK is present for community.docker modules + apt: + pkg: + - python3-docker + - python3-requests + state: present + +- name: Create base ELK directory structure + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0755" + loop: + - "{{ elk_base_dir }}" + - "{{ elk_base_dir }}/config" + - "{{ elk_base_dir }}/config/elasticsearch" + - "{{ elk_base_dir }}/config/kibana" + - "{{ elk_base_dir }}/config/logstash" + - "{{ elk_base_dir }}/config/logstash/pipeline" + +# --------------------------------------------------------------------------- +# TLS (must happen before compose/config templating below, since those +# templates reference the cert file paths lego produces) +# --------------------------------------------------------------------------- +- name: Issue/renew TLS certificate and set up renewal automation + ansible.builtin.include_tasks: tls.yml + when: elk_tls_enabled + +- name: Set connection facts used by health checks below + ansible.builtin.set_fact: + elk_scheme: "{{ 'https' if elk_tls_enabled else 'http' }}" + elk_validate_certs: "{{ not elk_tls_enabled }}" + elk_elasticsearch_client_host: "{{ elk_tls_domain if elk_tls_enabled else elk_elasticsearch_container_name }}" + +# --------------------------------------------------------------------------- +# Configs +# --------------------------------------------------------------------------- +- name: Template elasticsearch.yml + ansible.builtin.template: + src: elasticsearch.yml.j2 + dest: "{{ elk_base_dir }}/config/elasticsearch/elasticsearch.yml" + mode: "0644" + notify: "recreate elasticsearch" + +- name: Template kibana.yml + ansible.builtin.template: + src: kibana.yml.j2 + dest: "{{ elk_base_dir }}/config/kibana/kibana.yml" + mode: "0644" + notify: "recreate kibana" + +- name: Template logstash.yml + ansible.builtin.template: + src: logstash.yml.j2 + dest: "{{ elk_base_dir }}/config/logstash/logstash.yml" + mode: "0644" + notify: "recreate logstash" + +- name: Template logstash pipelines.yml + ansible.builtin.template: + src: pipelines.yml.j2 + dest: "{{ elk_base_dir }}/config/logstash/pipelines.yml" + mode: "0644" + notify: "recreate logstash" + +- name: Template logstash main pipeline + ansible.builtin.template: + src: main.conf.j2 + dest: "{{ elk_base_dir }}/config/logstash/pipeline/main.conf" + mode: "0644" + notify: "recreate logstash" + +- name: Template logstash nginx pipeline + ansible.builtin.template: + src: pipeline-nginx.conf.j2 + dest: "{{ elk_base_dir }}/config/logstash/pipeline/pipeline-nginx.conf" + mode: "0644" + notify: "recreate logstash" + +- name: Template logstash laravel pipeline + ansible.builtin.template: + src: pipeline-laravel.conf.j2 + dest: "{{ elk_base_dir }}/config/logstash/pipeline/pipeline-laravel.conf" + mode: "0644" + notify: "recreate logstash" + +- name: Template docker-compose.yml + ansible.builtin.template: + src: docker-compose.yml.j2 + dest: "{{ elk_base_dir }}/docker-compose.yml" + mode: "0644" + notify: "restart elk stack" + +# --------------------------------------------------------------------------- +# Bring up the stack +# --------------------------------------------------------------------------- +- name: Flush handlers so config changes apply before we continue + ansible.builtin.meta: flush_handlers + +- name: Start ELK stack via docker compose + community.docker.docker_compose_v2: + project_src: "{{ elk_base_dir }}" + project_name: "{{ elk_compose_project_name }}" + state: present + register: elk_compose_result + +- name: Wait for Elasticsearch to answer on its HTTP port + ansible.builtin.uri: + url: "{{ elk_scheme }}://localhost:{{ elk_elasticsearch_http_port }}" + validate_certs: "{{ elk_validate_certs }}" + status_code: + - 200 + - 401 # 401 is expected once security is enabled but before we authenticate + register: elk_es_wait + until: elk_es_wait.status in [200, 401] + retries: 30 + delay: 10 + +# --------------------------------------------------------------------------- +# Security bootstrap: set built-in user passwords (idempotent) +# --------------------------------------------------------------------------- +- name: Set kibana_system password + ansible.builtin.uri: + url: "{{ elk_scheme }}://localhost:{{ elk_elasticsearch_http_port }}/_security/user/kibana_system/_password" + validate_certs: "{{ elk_validate_certs }}" + method: POST + user: elastic + password: "{{ elk_elastic_password }}" + force_basic_auth: true + body_format: json + body: + password: "{{ elk_kibana_system_password }}" + status_code: 200 + when: elk_effective_security_enabled + register: elk_set_kibana_pw + retries: 5 + delay: 10 + until: elk_set_kibana_pw is succeeded + +- name: Set logstash_system password + ansible.builtin.uri: + url: "{{ elk_scheme }}://localhost:{{ elk_elasticsearch_http_port }}/_security/user/logstash_system/_password" + validate_certs: "{{ elk_validate_certs }}" + method: POST + user: elastic + password: "{{ elk_elastic_password }}" + force_basic_auth: true + body_format: json + body: + password: "{{ elk_logstash_system_password }}" + status_code: 200 + when: elk_effective_security_enabled + register: elk_set_logstash_pw + retries: 5 + delay: 10 + until: elk_set_logstash_pw is succeeded + +- name: Recreate kibana and logstash if passwords were just set for the first time + community.docker.docker_compose_v2: + project_src: "{{ elk_base_dir }}" + project_name: "{{ elk_compose_project_name }}" + services: + - kibana + - logstash + state: present + recreate: always + when: > + elk_effective_security_enabled and + (elk_set_kibana_pw is changed or elk_set_logstash_pw is changed) + +- name: Wait for Kibana to answer on its HTTP port + ansible.builtin.uri: + url: "{{ elk_scheme }}://localhost:{{ elk_kibana_http_port }}/api/status" + validate_certs: "{{ elk_validate_certs }}" + status_code: + - 200 + register: elk_kibana_wait + until: elk_kibana_wait.status == 200 + retries: 30 + delay: 10 diff --git a/roles/elasticsearch/tasks/tls.yml b/roles/elasticsearch/tasks/tls.yml new file mode 100644 index 0000000..5412ff8 --- /dev/null +++ b/roles/elasticsearch/tasks/tls.yml @@ -0,0 +1,176 @@ +--- +# --------------------------------------------------------------------------- +# TLS certificate lifecycle via lego (ACME DNS-01 against Cloudflare). +# +# This file only READS the Cloudflare token from elk_cloudflare_token_env_file. +# It never receives the token as an Ansible variable - the token is expected +# to already exist on disk (root:root, 0600), placed there by cloud-init at +# provisioning time. See the role README for the IaC side of this. +# --------------------------------------------------------------------------- + +- name: Assert the Cloudflare token file exists + ansible.builtin.stat: + path: "{{ elk_cloudflare_token_env_file }}" + register: elk_cf_token_stat + +- name: Fail early with a clear message if the token file is missing + ansible.builtin.fail: + msg: > + {{ elk_cloudflare_token_env_file }} does not exist. This role expects + cloud-init (or equivalent provisioning) to have already written a + scoped Cloudflare API token here before Ansible runs. See the IaC + notes in the role README. + when: not elk_cf_token_stat.stat.exists + +- name: Verify the token file is not world/group readable + ansible.builtin.assert: + that: + - elk_cf_token_stat.stat.mode == "0600" + fail_msg: > + {{ elk_cloudflare_token_env_file }} should be mode 0600 + (currently {{ elk_cf_token_stat.stat.mode }}) since it holds a + Cloudflare API credential. + success_msg: "Cloudflare token file permissions look correct." + +- name: Create lego directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0700" + loop: + - "{{ elk_lego_path }}" + - "{{ elk_lego_path }}/accounts" + +- name: Create lego certificates directory (traversable by container users) + ansible.builtin.file: + path: "{{ elk_lego_certs_dir }}" + state: directory + owner: root + group: root + mode: "0755" + +- name: Check installed lego version (if any) + ansible.builtin.command: "{{ elk_lego_install_dir }}/lego --version" + register: elk_lego_installed_version + changed_when: false + failed_when: false + +- name: Download and install lego binary + when: > + elk_lego_installed_version.rc != 0 or + elk_lego_version not in (elk_lego_installed_version.stdout | default('')) + block: + - name: Download lego release tarball + ansible.builtin.get_url: + url: "https://github.com/go-acme/lego/releases/download/v{{ elk_lego_version }}/lego_v{{ elk_lego_version }}_linux_{{ elk_lego_arch }}.tar.gz" + dest: "/tmp/lego_{{ elk_lego_version }}.tar.gz" + mode: "0644" + + - name: Extract lego binary + ansible.builtin.unarchive: + src: "/tmp/lego_{{ elk_lego_version }}.tar.gz" + dest: /tmp + remote_src: true + include: + - lego + + - name: Install lego binary + ansible.builtin.copy: + src: /tmp/lego + dest: "{{ elk_lego_install_dir }}/lego" + owner: root + group: root + mode: "0755" + remote_src: true + + - name: Clean up downloaded artifacts + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - "/tmp/lego_{{ elk_lego_version }}.tar.gz" + - /tmp/lego + +- name: Check whether a certificate already exists for this domain + ansible.builtin.stat: + path: "{{ elk_lego_certs_dir }}/{{ elk_tls_domain }}.crt" + register: elk_cert_stat + +# First issuance uses `lego ... run`, which is what actually requests a new +# certificate; `renew` (used by the systemd timer below) refuses to run +# against a domain with no existing certificate. +- name: Issue the initial certificate (first run only) + ansible.builtin.shell: | + set -o pipefail + set -a + source {{ elk_cloudflare_token_env_file }} + set +a + {{ elk_lego_install_dir }}/lego \ + --accept-tos \ + --email "{{ elk_tls_acme_email }}" \ + --server "{{ elk_tls_acme_server }}" \ + --domains "{{ elk_tls_domain }}" \ + --dns cloudflare \ + --path "{{ elk_lego_path }}" \ + run + args: + executable: /bin/bash + when: not elk_cert_stat.stat.exists + register: elk_lego_first_issue + notify: "restart elk stack" + +- name: Ensure issued cert/key are readable by the containers (root-owned, group-readable) + ansible.builtin.file: + path: "{{ item }}" + owner: root + group: root + mode: "0644" + loop: + - "{{ elk_lego_certs_dir }}/{{ elk_tls_domain }}.crt" + - "{{ elk_lego_certs_dir }}/{{ elk_tls_domain }}.issuer.crt" + when: elk_cert_stat.stat.exists or elk_lego_first_issue is not skipped + +- name: Private key stays root-only-readable but group-readable for the container runtime + ansible.builtin.file: + path: "{{ elk_lego_certs_dir }}/{{ elk_tls_domain }}.key" + owner: root + group: root + mode: "0640" + when: elk_cert_stat.stat.exists or elk_lego_first_issue is not skipped + +# --------------------------------------------------------------------------- +# Ongoing renewal: systemd oneshot service + daily timer. lego's `renew` +# subcommand only actually re-issues (and only then fires --run-hook) when +# the certificate is within elk_tls_renew_days of expiring, so this is safe +# to run daily without causing unnecessary container restarts. +# --------------------------------------------------------------------------- +- name: Template lego renewal systemd service + ansible.builtin.template: + src: lego-renew.service.j2 + dest: /etc/systemd/system/lego-renew.service + owner: root + group: root + mode: "0644" + register: elk_lego_service_unit + +- name: Template lego renewal systemd timer + ansible.builtin.template: + src: lego-renew.timer.j2 + dest: /etc/systemd/system/lego-renew.timer + owner: root + group: root + mode: "0644" + register: elk_lego_timer_unit + +- name: Reload systemd if units changed + ansible.builtin.systemd: + daemon_reload: true + when: elk_lego_service_unit is changed or elk_lego_timer_unit is changed + +- name: Enable and start the lego renewal timer + ansible.builtin.systemd: + name: lego-renew.timer + enabled: true + state: started diff --git a/roles/elasticsearch/templates/docker-compose.yml.j2 b/roles/elasticsearch/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..9d72e5f --- /dev/null +++ b/roles/elasticsearch/templates/docker-compose.yml.j2 @@ -0,0 +1,147 @@ +# ansible_managed +version: "3.8" + +networks: + {{ elk_network_name }}: + driver: bridge + ipam: + config: + - subnet: {{ elk_network_subnet }} + +volumes: + {{ elk_elasticsearch_data_volume }}: + driver: local + +services: + elasticsearch: + image: "{{ elk_docker_image_repo }}/elasticsearch/elasticsearch:{{ elk_version }}" + container_name: "{{ elk_elasticsearch_container_name }}" + restart: unless-stopped + environment: + - node.name={{ elk_elasticsearch_node_name }} + - cluster.name={{ elk_elasticsearch_cluster_name }} + - discovery.type=single-node + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms{{ elk_elasticsearch_heap_size }} -Xmx{{ elk_elasticsearch_heap_size }}" + # xpack.security.enabled must be true whenever TLS is on - Elasticsearch + # only honors xpack.security.http.ssl.* when security itself is + # enabled. elk_effective_security_enabled captures that (it's true if + # either elk_security_enabled or elk_tls_enabled is true). Whenever + # security ends up on for either reason, ELASTIC_PASSWORD must also be + # set - otherwise the image generates a random, unknown elastic + # password and nothing else in this stack could authenticate. + - xpack.security.enabled={{ elk_effective_security_enabled | lower }} +{% if elk_effective_security_enabled %} + - ELASTIC_PASSWORD={{ elk_elastic_password }} +{% endif %} +{% if elk_tls_enabled %} + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.certificate=/usr/share/elasticsearch/config/certs/{{ elk_tls_domain }}.crt + - xpack.security.http.ssl.key=/usr/share/elasticsearch/config/certs/{{ elk_tls_domain }}.key + - xpack.security.http.ssl.certificate_authorities=/usr/share/elasticsearch/config/certs/{{ elk_tls_domain }}.issuer.crt + # Single node - transport SSL stays off. Turn this on (and add a + # matching transport cert) if this ever becomes a multi-node cluster. + - xpack.security.transport.ssl.enabled=false +{% else %} + - xpack.security.http.ssl.enabled=false + - xpack.security.transport.ssl.enabled=false +{% endif %} + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - {{ elk_elasticsearch_data_volume }}:/usr/share/elasticsearch/data + - {{ elk_base_dir }}/config/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro +{% if elk_tls_enabled %} + - {{ elk_lego_certs_dir }}:/usr/share/elasticsearch/config/certs:ro +{% endif %} + ports: + - "{{ elk_elasticsearch_http_port }}:9200" + - "{{ elk_elasticsearch_transport_port }}:9300" + networks: + {{ elk_network_name }}: +{% if elk_tls_enabled %} + aliases: + - "{{ elk_tls_domain }}" +{% endif %} + healthcheck: + test: + - CMD-SHELL +{% set es_scheme = 'https' if elk_tls_enabled else 'http' %} +{% set curl_tls_flag = '-k ' if elk_tls_enabled else '' %} +{# xpack.security.enabled follows elk_effective_security_enabled (see + above), so basic auth is required here under that same condition. #} +{% if elk_effective_security_enabled %} + - "curl -s {{ curl_tls_flag }}-u elastic:{{ elk_elastic_password }} {{ es_scheme }}://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"\\|\"status\":\"yellow\"'" +{% else %} + - "curl -s {{ curl_tls_flag }}{{ es_scheme }}://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"\\|\"status\":\"yellow\"'" +{% endif %} + interval: 10s + timeout: 10s + retries: 20 + start_period: 30s + + logstash: + image: "{{ elk_docker_image_repo }}/logstash/logstash:{{ elk_version }}" + container_name: "{{ elk_logstash_container_name }}" + restart: unless-stopped + environment: + - "LS_JAVA_OPTS=-Xms{{ elk_logstash_heap_size }} -Xmx{{ elk_logstash_heap_size }}" + - xpack.monitoring.enabled=false +{% if elk_effective_security_enabled %} + - ELASTIC_USER=logstash_system + - ELASTIC_PASSWORD={{ elk_logstash_system_password }} +{% endif %} + volumes: + - {{ elk_base_dir }}/config/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml:ro + - {{ elk_base_dir }}/config/logstash/pipelines.yml:/usr/share/logstash/config/pipelines.yml:ro + - {{ elk_base_dir }}/config/logstash/pipeline:/usr/share/logstash/pipeline:ro +{% if elk_tls_enabled %} + - {{ elk_lego_certs_dir }}:/usr/share/logstash/config/certs:ro +{% endif %} + ports: + - "{{ elk_logstash_beats_port }}:5044" + - "{{ elk_logstash_nginx_beats_port }}:5045" + - "{{ elk_logstash_laravel_beats_port }}:5046" + - "{{ elk_logstash_tcp_port }}:5000/tcp" + - "{{ elk_logstash_udp_port }}:5000/udp" + - "{{ elk_logstash_http_port }}:9600" + networks: + - {{ elk_network_name }} + depends_on: + elasticsearch: + condition: service_healthy + + kibana: + image: "{{ elk_docker_image_repo }}/kibana/kibana:{{ elk_version }}" + container_name: "{{ elk_kibana_container_name }}" + restart: unless-stopped + environment: + - SERVER_NAME={{ elk_kibana_container_name }} + - ELASTICSEARCH_HOSTS={{ 'https' if elk_tls_enabled else 'http' }}://{{ elk_elasticsearch_client_host }}:9200 +{% if elk_effective_security_enabled %} + - ELASTICSEARCH_USERNAME=kibana_system + - ELASTICSEARCH_PASSWORD={{ elk_kibana_system_password }} + - XPACK_SECURITY_ENCRYPTIONKEY={{ elk_kibana_encryption_key }} +{% endif %} +{% if elk_tls_enabled %} + - SERVER_SSL_ENABLED=true + - SERVER_SSL_CERTIFICATE=/usr/share/kibana/config/certs/{{ elk_tls_domain }}.crt + - SERVER_SSL_KEY=/usr/share/kibana/config/certs/{{ elk_tls_domain }}.key + # Public CA (Let's Encrypt) is already trusted by the container's + # system trust store, so no custom CA needs to be supplied here for + # the Elasticsearch connection. +{% endif %} + volumes: + - {{ elk_base_dir }}/config/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml:ro +{% if elk_tls_enabled %} + - {{ elk_lego_certs_dir }}:/usr/share/kibana/config/certs:ro +{% endif %} + ports: + - "{{ elk_kibana_http_port }}:5601" + networks: + - {{ elk_network_name }} + depends_on: + elasticsearch: + condition: service_healthy diff --git a/roles/elasticsearch/templates/elasticsearch.yml.j2 b/roles/elasticsearch/templates/elasticsearch.yml.j2 new file mode 100644 index 0000000..83f3b2a --- /dev/null +++ b/roles/elasticsearch/templates/elasticsearch.yml.j2 @@ -0,0 +1,29 @@ +# ansible_managed +cluster.name: "{{ elk_elasticsearch_cluster_name }}" +node.name: "{{ elk_elasticsearch_node_name }}" + +network.host: 0.0.0.0 +discovery.type: single-node + +# TLS (elk_tls_enabled) is a transport-layer concern independent of X-Pack +# security (elk_security_enabled) - HOWEVER Elasticsearch only honors +# xpack.security.http.ssl.* when xpack.security.enabled is true, so security +# is force-enabled (elk_effective_security_enabled, set in tasks/main.yml) +# whenever TLS is on, even if elk_security_enabled itself is false. The +# ELASTIC_PASSWORD env var is set under the same condition (see +# docker-compose.yml.j2), so basic auth is always usable whenever this is true. +xpack.security.enabled: {{ elk_effective_security_enabled | lower }} +{% if elk_tls_enabled %} +xpack.security.http.ssl.enabled: true +xpack.security.http.ssl.certificate: certs/{{ elk_tls_domain }}.crt +xpack.security.http.ssl.key: certs/{{ elk_tls_domain }}.key +xpack.security.http.ssl.certificate_authorities: ["certs/{{ elk_tls_domain }}.issuer.crt"] +xpack.security.transport.ssl.enabled: false +{% else %} +xpack.security.http.ssl.enabled: false +xpack.security.transport.ssl.enabled: false +{% endif %} + +# Reachable at {{ 'https' if elk_tls_enabled else 'http' }}://{{ elk_elasticsearch_client_host }}:9200 from other +# containers on the {{ elk_network_name }} docker network, and from outside +# the host at {{ 'https' if elk_tls_enabled else 'http' }}://{{ elk_tls_domain if elk_tls_enabled else elk_elasticsearch_client_host }}:{{ elk_elasticsearch_http_port }} diff --git a/roles/elasticsearch/templates/filebeat.yml.j2 b/roles/elasticsearch/templates/filebeat.yml.j2 deleted file mode 100644 index ceefb07..0000000 --- a/roles/elasticsearch/templates/filebeat.yml.j2 +++ /dev/null @@ -1,272 +0,0 @@ -###################### Filebeat Configuration Example ######################### - -# This file is an example configuration file highlighting only the most common -# options. The filebeat.reference.yml file from the same directory contains all the -# supported options with more comments. You can use it as a reference. -# -# You can find the full configuration reference here: -# https://www.elastic.co/guide/en/beats/filebeat/index.html - -# For more available modules and options, please see the filebeat.reference.yml sample -# configuration file. - -# ============================== Filebeat inputs =============================== - -filebeat.inputs: - -# Each - is an input. Most options can be set at the input level, so -# you can use different inputs for various configurations. -# Below are the input specific configurations. - -- type: log - - # Change to true to enable this input configuration. - enabled: false - - # Paths that should be crawled and fetched. Glob based paths. - paths: - - /var/log/ngnix/*.log - #- c:\programdata\elasticsearch\logs\* - - # Exclude lines. A list of regular expressions to match. It drops the lines that are - # matching any regular expression from the list. - #exclude_lines: ['^DBG'] - - # Include lines. A list of regular expressions to match. It exports the lines that are - # matching any regular expression from the list. - #include_lines: ['^ERR', '^WARN'] - - # Exclude files. A list of regular expressions to match. Filebeat drops the files that - # are matching any regular expression from the list. By default, no files are dropped. - #exclude_files: ['.gz$'] - - # Optional additional fields. These fields can be freely picked - # to add additional information to the crawled log files for filtering - #fields: - # level: debug - # review: 1 - - ### Multiline options - - # Multiline can be used for log messages spanning multiple lines. This is common - # for Java Stack Traces or C-Line Continuation - - # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [ - #multiline.pattern: ^\[ - - # Defines if the pattern set under pattern should be negated or not. Default is false. - #multiline.negate: false - - # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern - # that was (not) matched before or after or as long as a pattern is not matched based on negate. - # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash - #multiline.match: after - -# filestream is an input for collecting log messages from files. It is going to replace log input in the future. -- type: filestream - - # Change to true to enable this input configuration. - enabled: true - - # Paths that should be crawled and fetched. Glob based paths. - paths: - - /var/log/nginx/*.log - #- c:\programdata\elasticsearch\logs\* - - exclude_files: ['\.gz$'] - - # Exclude lines. A list of regular expressions to match. It drops the lines that are - # matching any regular expression from the list. - #exclude_lines: ['^DBG'] - - # Include lines. A list of regular expressions to match. It exports the lines that are - # matching any regular expression from the list. - #include_lines: ['^ERR', '^WARN'] - - # Exclude files. A list of regular expressions to match. Filebeat drops the files that - # are matching any regular expression from the list. By default, no files are dropped. - #prospector.scanner.exclude_files: ['.gz$'] - - # Optional additional fields. These fields can be freely picked - # to add additional information to the crawled log files for filtering - #fields: - # level: debug - # review: 1 - -# ============================== Filebeat modules ============================== - -filebeat.config.modules: - # Glob pattern for configuration loading - path: ${path.config}/modules.d/*.yml - - # Set to true to enable config reloading - reload.enabled: false - - # Period on which files under path should be checked for changes - #reload.period: 10s - -# ======================= Elasticsearch template setting ======================= - -setup.template.settings: - index.number_of_shards: 1 - #index.codec: best_compression - #_source.enabled: false - - -# ================================== General =================================== - -# The name of the shipper that publishes the network data. It can be used to group -# all the transactions sent by a single shipper in the web interface. -#name: - -# The tags of the shipper are included in their own field with each -# transaction published. -#tags: ["service-X", "web-tier"] - -# Optional fields that you can specify to add additional information to the -# output. -#fields: -# env: staging - -# ================================= Dashboards ================================= -# These settings control loading the sample dashboards to the Kibana index. Loading -# the dashboards is disabled by default and can be enabled either by setting the -# options here or by using the `setup` command. -#setup.dashboards.enabled: false - -# The URL from where to download the dashboards archive. By default this URL -# has a value which is computed based on the Beat name and version. For released -# versions, this URL points to the dashboard archive on the artifacts.elastic.co -# website. -#setup.dashboards.url: - -# =================================== Kibana =================================== - -# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. -# This requires a Kibana endpoint configuration. -setup.kibana: - - # Kibana Host - # Scheme and port can be left out and will be set to the default (http and 5601) - # In case you specify and additional path, the scheme is required: http://localhost:5601/path - # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 - #host: "localhost:5601" - - # Kibana Space ID - # ID of the Kibana Space into which the dashboards should be loaded. By default, - # the Default Space will be used. - #space.id: - -# =============================== Elastic Cloud ================================ - -# These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/). - -# The cloud.id setting overwrites the `output.elasticsearch.hosts` and -# `setup.kibana.host` options. -# You can find the `cloud.id` in the Elastic Cloud web UI. -#cloud.id: - -# The cloud.auth setting overwrites the `output.elasticsearch.username` and -# `output.elasticsearch.password` settings. The format is `:`. -#cloud.auth: - -# ================================== Outputs =================================== - -# Configure what output to use when sending the data collected by the beat. - -# ---------------------------- Elasticsearch Output ---------------------------- -# output.elasticsearch: - # Array of hosts to connect to. - # hosts: ["elastic.schleppe:9200"] - - # Protocol - either `http` (default) or `https`. - # protocol: "https" - - # Authentication credentials - either API key or username/password. - #api_key: "id:api_key" - #username: "elastic" - #password: "changeme" - -# ------------------------------ Logstash Output ------------------------------- -output.logstash: - # The Logstash hosts - hosts: ["elasticsearch:5400"] - - # Optional SSL. By default is off. - # List of root certificates for HTTPS server verifications - # ssl.certificate_authorities: ["/etc/elk-certs/elk-ssl.crt"] - - # Certificate for SSL client authentication - # ssl.certificate: "/etc/elk-certs/elk-ssl.crt" - - # Client Certificate Key - # ssl.key: "/etc/elk-certs/elk-ssl.key" - -# ================================= Processors ================================= -processors: - - add_host_metadata: - when.not.contains.tags: forwarded - - add_cloud_metadata: ~ - - add_docker_metadata: ~ - - add_kubernetes_metadata: ~ - -# ================================== Logging =================================== - -# Sets log level. The default log level is info. -# Available log levels are: error, warning, info, debug -# logging.level: debug - -# At debug level, you can selectively enable logging only for some components. -# To enable all selectors use ["*"]. Examples of other selectors are "beat", -# "publisher", "service". -#logging.selectors: ["*"] - -# ============================= X-Pack Monitoring ============================== -# Filebeat can export internal metrics to a central Elasticsearch monitoring -# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The -# reporting is disabled by default. - -# Set to true to enable the monitoring reporter. -#monitoring.enabled: false - -# Sets the UUID of the Elasticsearch cluster under which monitoring data for this -# Filebeat instance will appear in the Stack Monitoring UI. If output.elasticsearch -# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch. -#monitoring.cluster_uuid: - -# Uncomment to send the metrics to Elasticsearch. Most settings from the -# Elasticsearch output are accepted here as well. -# Note that the settings should point to your Elasticsearch *monitoring* cluster. -# Any setting that is not set is automatically inherited from the Elasticsearch -# output configuration, so if you have the Elasticsearch output configured such -# that it is pointing to your Elasticsearch monitoring cluster, you can simply -# uncomment the following line. -#monitoring.elasticsearch: - -# ============================== Instrumentation =============================== - -# Instrumentation support for the filebeat. -#instrumentation: - # Set to true to enable instrumentation of filebeat. - #enabled: false - - # Environment in which filebeat is running on (eg: staging, production, etc.) - #environment: "" - - # APM Server hosts to report instrumentation results to. - #hosts: - # - http://localhost:8200 - - # API Key for the APM Server(s). - # If api_key is set then secret_token will be ignored. - #api_key: - - # Secret token for the APM Server(s). - #secret_token: - - -# ================================= Migration ================================== - -# This allows to enable 6.7 migration aliases -#migration.6_to_7.enabled: true - diff --git a/roles/elasticsearch/templates/kibana.yml.j2 b/roles/elasticsearch/templates/kibana.yml.j2 new file mode 100644 index 0000000..2a111ff --- /dev/null +++ b/roles/elasticsearch/templates/kibana.yml.j2 @@ -0,0 +1,28 @@ +# ansible_managed +server.name: "{{ elk_kibana_container_name }}" +server.host: "0.0.0.0" +server.port: 5601 + +# Talk to Elasticsearch over the shared docker network using its service name. +elasticsearch.hosts: ["{{ 'https' if elk_tls_enabled else 'http' }}://{{ elk_elasticsearch_client_host }}:9200"] + +{% if elk_effective_security_enabled %} +elasticsearch.username: "kibana_system" +elasticsearch.password: "{{ elk_kibana_system_password }}" +xpack.security.encryptionKey: "{{ elk_kibana_encryption_key }}" +{% endif %} + +{% if elk_tls_enabled %} +server.ssl.enabled: true +server.ssl.certificate: /usr/share/kibana/config/certs/{{ elk_tls_domain }}.crt +server.ssl.key: /usr/share/kibana/config/certs/{{ elk_tls_domain }}.key + +server.publicBaseUrl: "https://{{ elk_tls_domain }}:{{ elk_kibana_http_port }}" + +# No elasticsearch.ssl.certificateAuthorities entry needed here: the cert +# chain is issued by a public CA (Let's Encrypt) which the container's +# default trust store already trusts. +{% endif %} + +monitoring.ui.container.elasticsearch.enabled: true +i18n.locale: "en" diff --git a/roles/elasticsearch/templates/lego-renew.service.j2 b/roles/elasticsearch/templates/lego-renew.service.j2 new file mode 100644 index 0000000..3d9d2e5 --- /dev/null +++ b/roles/elasticsearch/templates/lego-renew.service.j2 @@ -0,0 +1,28 @@ +# ansible_managed +[Unit] +Description=Renew ELK TLS certificate via lego (Cloudflare DNS-01) +After=network-online.target docker.service +Wants=network-online.target +Requires=docker.service + +[Service] +Type=oneshot +EnvironmentFile={{ elk_cloudflare_token_env_file }} +ExecStart={{ elk_lego_install_dir }}/lego \ + --accept-tos \ + --email "{{ elk_tls_acme_email }}" \ + --server "{{ elk_tls_acme_server }}" \ + --domains "{{ elk_tls_domain }}" \ + --dns cloudflare \ + --path "{{ elk_lego_path }}" \ + --run-hook "/usr/bin/docker compose -f {{ elk_base_dir }}/docker-compose.yml -p {{ elk_compose_project_name }} restart elasticsearch kibana logstash" \ + renew --days {{ elk_tls_renew_days }} +User=root +StandardOutput=journal +StandardError=journal +SyslogIdentifier=lego-renew + +# Light hardening. NoNewPrivileges/ProtectSystem kept loose enough that +# lego can write certs under elk_lego_path and reach the docker socket +# via the docker CLI. +PrivateTmp=yes diff --git a/roles/elasticsearch/templates/lego-renew.timer.j2 b/roles/elasticsearch/templates/lego-renew.timer.j2 new file mode 100644 index 0000000..e81eb65 --- /dev/null +++ b/roles/elasticsearch/templates/lego-renew.timer.j2 @@ -0,0 +1,11 @@ +# ansible_managed +[Unit] +Description=Daily timer for ELK TLS certificate renewal check (lego) + +[Timer] +OnCalendar=*-*-* {{ elk_tls_renew_on_calendar }} +RandomizedDelaySec={{ elk_tls_renew_randomized_delay }} +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/roles/elasticsearch/templates/logstash-conf.d/laravel_pipeline.conf.j2 b/roles/elasticsearch/templates/logstash-conf.d/laravel_pipeline.conf.j2 deleted file mode 100644 index 3b53762..0000000 --- a/roles/elasticsearch/templates/logstash-conf.d/laravel_pipeline.conf.j2 +++ /dev/null @@ -1,17 +0,0 @@ -input { - beats { - port => 5045 - } -} - -filter { -} - -output { - elasticsearch { - index => "laravel-logs-%{+YYYY.MM}" - hosts => "${ELASTIC_HOSTS}" - user => "elastic" - password => "${ELASTIC_PASSWORD}" - } -} diff --git a/roles/elasticsearch/templates/logstash-conf.d/main_pipeline.conf.j2.disabled b/roles/elasticsearch/templates/logstash-conf.d/main_pipeline.conf.j2.disabled deleted file mode 100644 index ea5db9a..0000000 --- a/roles/elasticsearch/templates/logstash-conf.d/main_pipeline.conf.j2.disabled +++ /dev/null @@ -1,24 +0,0 @@ -input { - beats { - port => 5044 - } -} - -filter { - if [pipeline_id] == "nginx" { - mutate { add_field => { "route" => "nginx_pipeline" } } - } else if [pipeline_id] == "laravel" { - mutate { add_field => { "route" => "laravel_pipeline" } } - } -} - -output { - if [pipeline_id] == "nginx" { - pipeline { send_to => "nginx_pipeline" } - } else if [pipeline_id] == "laravel" { - pipeline { send_to => "laravel_pipeline" } - } else { - # Handle unknown cases - stdout { codec => rubydebug } - } -} diff --git a/roles/elasticsearch/templates/logstash.yml.j2 b/roles/elasticsearch/templates/logstash.yml.j2 new file mode 100644 index 0000000..1b3a9fe --- /dev/null +++ b/roles/elasticsearch/templates/logstash.yml.j2 @@ -0,0 +1,6 @@ +# ansible_managed +http.host: "0.0.0.0" +http.port: 9600 + +path.config: /usr/share/logstash/pipeline +xpack.monitoring.enabled: false diff --git a/roles/elasticsearch/templates/main.conf.j2 b/roles/elasticsearch/templates/main.conf.j2 new file mode 100644 index 0000000..4d54bd8 --- /dev/null +++ b/roles/elasticsearch/templates/main.conf.j2 @@ -0,0 +1,38 @@ +# ansible_managed +input { + beats { + port => 5044 + } + tcp { + port => 5000 + codec => json_lines + } + udp { + port => 5000 + codec => json_lines + } +} + +filter { + # Add filters here as needed (grok, mutate, date, etc.) +} + +output { + elasticsearch { + # Reach Elasticsearch over the shared docker network using its service name. + hosts => ["{{ 'https' if elk_tls_enabled else 'http' }}://{{ elk_elasticsearch_client_host }}:9200"] +{% if elk_effective_security_enabled %} + user => "logstash_system" + password => "{{ elk_logstash_system_password }}" +{% endif %} +{% if elk_tls_enabled %} + ssl_enabled => true + # Public CA (Let's Encrypt) - trusted by Logstash's default JVM trust + # store already, no custom CA needs to be supplied here. +{% endif %} + index => "logstash-%{+YYYY.MM.dd}" + } + + # Uncomment for local debugging of pipeline output: + # stdout { codec => rubydebug } +} diff --git a/roles/elasticsearch/templates/pipeline-laravel.conf.j2 b/roles/elasticsearch/templates/pipeline-laravel.conf.j2 new file mode 100644 index 0000000..a990f11 --- /dev/null +++ b/roles/elasticsearch/templates/pipeline-laravel.conf.j2 @@ -0,0 +1,26 @@ +# ansible_managed +input { + beats { + port => 5046 + } +} + +filter { +} + +output { + elasticsearch { + # Reach Elasticsearch over the shared docker network using its service name. + hosts => ["{{ 'https' if elk_tls_enabled else 'http' }}://{{ elk_elasticsearch_client_host }}:9200"] +{% if elk_effective_security_enabled %} + user => "logstash_system" + password => "{{ elk_logstash_system_password }}" +{% endif %} +{% if elk_tls_enabled %} + ssl_enabled => true + # Public CA (Let's Encrypt) - trusted by Logstash's default JVM trust + # store already, no custom CA needs to be supplied here. +{% endif %} + index => "laravel-logs-%{+YYYY.MM}" + } +} diff --git a/roles/elasticsearch/templates/logstash-conf.d/nginx_pipeline.conf.j2 b/roles/elasticsearch/templates/pipeline-nginx.conf.j2 similarity index 61% rename from roles/elasticsearch/templates/logstash-conf.d/nginx_pipeline.conf.j2 rename to roles/elasticsearch/templates/pipeline-nginx.conf.j2 index 2908e52..c78d453 100644 --- a/roles/elasticsearch/templates/logstash-conf.d/nginx_pipeline.conf.j2 +++ b/roles/elasticsearch/templates/pipeline-nginx.conf.j2 @@ -1,6 +1,7 @@ +# ansible_managed input { beats { - port => 5044 + port => 5045 } } @@ -36,10 +37,17 @@ filter { output { elasticsearch { + # Reach Elasticsearch over the shared docker network using its service name. + hosts => ["{{ 'https' if elk_tls_enabled else 'http' }}://{{ elk_elasticsearch_client_host }}:9200"] +{% if elk_effective_security_enabled %} + user => "logstash_system" + password => "{{ elk_logstash_system_password }}" +{% endif %} +{% if elk_tls_enabled %} + ssl_enabled => true + # Public CA (Let's Encrypt) - trusted by Logstash's default JVM trust + # store already, no custom CA needs to be supplied here. +{% endif %} index => "weblogs-%{+YYYY.MM}" - hosts => "${ELASTIC_HOSTS}" - user => "elastic" - password => "${ELASTIC_PASSWORD}" - document_type => "nginx_logs" } } diff --git a/roles/elasticsearch/templates/pipelines.yml.j2 b/roles/elasticsearch/templates/pipelines.yml.j2 index 7a911a7..a97a132 100644 --- a/roles/elasticsearch/templates/pipelines.yml.j2 +++ b/roles/elasticsearch/templates/pipelines.yml.j2 @@ -1,5 +1,9 @@ +# ansible_managed +- pipeline.id: main + path.config: "/usr/share/logstash/pipeline/main.conf" + - pipeline.id: nginx_pipeline - path.config: "/usr/share/logstash/pipeline/nginx_pipeline.conf" + path.config: "/usr/share/logstash/pipeline/pipeline-nginx.conf" - pipeline.id: laravel_pipeline - path.config: "/usr/share/logstash/pipeline/laravel_pipeline.conf" + path.config: "/usr/share/logstash/pipeline/pipeline-laravel.conf"