update elasticsearch w/ ssl & xpack support

This commit is contained in:
2026-08-07 16:23:00 +02:00
parent 871b8d1f83
commit 86d8986630
24 changed files with 1139 additions and 509 deletions
@@ -1,41 +0,0 @@
---
- name: Pull Elasticsearch Docker image
docker_image:
name: docker.elastic.co/elasticsearch/elasticsearch-wolfi:{{ elk_version }}
source: pull
- name: Create Elasticsearch configuration file directory on host
file:
path: /etc/elasticsearch
state: directory
mode: '0755'
# - name: Create Elasticsearch configuration file
# template:
# src: elasticsearch.yml.j2
# dest: /etc/elasticsearch/elasticsearch.yml
- name: Start Elasticsearch container
docker_container:
name: elasticsearch
image: docker.elastic.co/elasticsearch/elasticsearch-wolfi:{{ elk_version }}
state: started
restart: yes
restart_policy: unless-stopped
published_ports:
- "9200:9200"
volumes:
- /etc/elasticsearch/esdata:/usr/share/elasticsearch/data
networks:
- name: elk_network
ipv4_address: 172.19.0.2
env:
node.name: elasticsearch
cluster.name: retailor-elk
discovery.type: single-node
bootstrap.memory_lock: "true"
# limits elasticsearch to 2 GB of RAM
ES_JAVA_OPTS: "-Xms1g -Xmx2g"
# disables SSL & xpack security
xpack.security.http.ssl.enabled: "false"
-11
View File
@@ -1,11 +0,0 @@
---
- name: Create a Docker network for Elasticsearch
docker_network:
name: elk_network
state: present
driver: bridge
ipam_config:
- subnet: "172.19.0.0/16"
gateway: "172.19.0.1"
iprange: "172.19.0.0/24"
-37
View File
@@ -1,37 +0,0 @@
---
- name: Create a Docker network for Kibana
docker_network:
name: elk_network
state: present
- name: Create kibana directory on host
file:
path: /etc/kibana
state: directory
mode: "0755"
- name: Pull Kibana Docker image
docker_image:
name: docker.elastic.co/kibana/kibana:{{ elk_version }}
source: pull
# TODO rember to move CA cert from elastic to Kibana
# docker cp elasticsearch:/usr/share/elasticsearch/config/certs/http_ca.crt .
# docker cp http_ca.crt kibana:/usr/share/kibana/config/certs/ca/http_ca.crt
- name: Start Kibana container
docker_container:
name: kibana
image: docker.elastic.co/kibana/kibana:{{ elk_version }}
state: started
restart: yes
restart_policy: unless-stopped
published_ports:
- "5601:5601"
env:
ELASTICSEARCH_HOSTS: "{{ env_vars.ELASTIC_HOSTS }}"
ELASTICSEARCH_USERNAME: kibana_system
ELASTICSEARCH_PASSWORD: "{{ env_vars.KIBANA_PASSWORD }}"
TELEMETRY_ENABLED: "false"
networks:
- name: elk_network
ipv4_address: 172.19.0.3
-65
View File
@@ -1,65 +0,0 @@
---
- name: Create a Docker network for Logstash
docker_network:
name: elk_network
state: present
- name: Create logstash directory on host
file:
path: /etc/logstash
state: directory
mode: "0755"
- name: Copy logstash config
copy:
src: templates/pipelines.yml.j2
dest: /etc/logstash/pipelines.yml
- name: Create logstash directory on host
file:
path: /etc/logstash/pipeline
state: directory
mode: "0755"
- name: Copy logstash input configs
copy:
src: "{{ item }}"
dest: /etc/logstash/pipeline/{{ item | basename | regex_replace('\.j2$', '') }}
loop: "{{ query('fileglob', 'templates/logstash-conf.d/*.j2') }}"
- name: Pull Logstash Docker image
docker_image:
name: docker.elastic.co/logstash/logstash:{{ elk_version }}
source: pull
# TODO rember to move CA cert from elastic to Logstash
# docker cp elasticsearch:/usr/share/elasticsearch/config/certs/http_ca.crt .
# docker cp http_ca.crt logstash:/usr/share/logstash/config/certs/ca/http_ca.crt
- name: Start Logstash container
docker_container:
name: logstash
image: docker.elastic.co/logstash/logstash:{{ elk_version }}
state: started
restart: yes
restart_policy: unless-stopped
command:
- /bin/bash
- -c
- |
echo "Waiting for Elasticsearch availability";
until curl -s {{ env_vars.ELASTIC_HOSTS }} | grep -q "missing authentication credentials"; do sleep 1; done;
echo "Starting logstash";
/usr/share/logstash/bin/logstash
published_ports:
- "5044-5049:5044-5049"
volumes:
- /etc/logstash/pipelines.yml:/usr/share/logstash/config/pipelines.yml
- /etc/logstash/pipeline:/usr/share/logstash/pipeline
env:
xpack.monitoring.enabled: "false"
ELASTIC_USER: elastic
ELASTIC_PASSWORD: "{{ env_vars.ELASTIC_PASSWORD }}"
ELASTIC_HOSTS: "{{ env_vars.ELASTIC_HOSTS }}"
networks:
- name: elk_network
ipv4_address: 172.19.0.4
+211 -6
View File
@@ -1,8 +1,213 @@
---
# ensure we have variables from .env files
- include_tasks: ../roles/env/tasks/main.yml
# ---------------------------------------------------------------------------
# TLS (elk_tls_enabled) and X-Pack security (elk_security_enabled) are
# documented as independent switches, but Elasticsearch only honors
# xpack.security.http.ssl.* when xpack.security.enabled is also true. So
# whenever TLS is on, security (and therefore basic auth / ELASTIC_PASSWORD)
# is force-enabled too - elk_effective_security_enabled is the single fact
# every template and task below should use instead of elk_security_enabled
# directly, so this coupling only has to be expressed once.
# ---------------------------------------------------------------------------
- name: Derive effective security state (TLS implies security)
ansible.builtin.set_fact:
elk_effective_security_enabled: "{{ elk_security_enabled or elk_tls_enabled }}"
- include_tasks: elk-network.yml
- include_tasks: elasticsearch.yml
- include_tasks: kibana.yml
- include_tasks: logstash.yml
- name: Assert Docker is available
ansible.builtin.command: docker info
register: elk_docker_info
changed_when: false
failed_when: elk_docker_info.rc != 0
- name: Ensure Docker Compose plugin is installed
ansible.builtin.command: docker compose version
register: elk_compose_check
changed_when: false
failed_when: elk_compose_check.rc != 0
- name: Ensure we have pip installed
apt:
pkg: "python3-pip"
state: present
- name: Ensure python docker SDK is present for community.docker modules
apt:
pkg:
- python3-docker
- python3-requests
state: present
- name: Create base ELK directory structure
ansible.builtin.file:
path: "{{ item }}"
state: directory
owner: root
group: root
mode: "0755"
loop:
- "{{ elk_base_dir }}"
- "{{ elk_base_dir }}/config"
- "{{ elk_base_dir }}/config/elasticsearch"
- "{{ elk_base_dir }}/config/kibana"
- "{{ elk_base_dir }}/config/logstash"
- "{{ elk_base_dir }}/config/logstash/pipeline"
# ---------------------------------------------------------------------------
# TLS (must happen before compose/config templating below, since those
# templates reference the cert file paths lego produces)
# ---------------------------------------------------------------------------
- name: Issue/renew TLS certificate and set up renewal automation
ansible.builtin.include_tasks: tls.yml
when: elk_tls_enabled
- name: Set connection facts used by health checks below
ansible.builtin.set_fact:
elk_scheme: "{{ 'https' if elk_tls_enabled else 'http' }}"
elk_validate_certs: "{{ not elk_tls_enabled }}"
elk_elasticsearch_client_host: "{{ elk_tls_domain if elk_tls_enabled else elk_elasticsearch_container_name }}"
# ---------------------------------------------------------------------------
# Configs
# ---------------------------------------------------------------------------
- name: Template elasticsearch.yml
ansible.builtin.template:
src: elasticsearch.yml.j2
dest: "{{ elk_base_dir }}/config/elasticsearch/elasticsearch.yml"
mode: "0644"
notify: "recreate elasticsearch"
- name: Template kibana.yml
ansible.builtin.template:
src: kibana.yml.j2
dest: "{{ elk_base_dir }}/config/kibana/kibana.yml"
mode: "0644"
notify: "recreate kibana"
- name: Template logstash.yml
ansible.builtin.template:
src: logstash.yml.j2
dest: "{{ elk_base_dir }}/config/logstash/logstash.yml"
mode: "0644"
notify: "recreate logstash"
- name: Template logstash pipelines.yml
ansible.builtin.template:
src: pipelines.yml.j2
dest: "{{ elk_base_dir }}/config/logstash/pipelines.yml"
mode: "0644"
notify: "recreate logstash"
- name: Template logstash main pipeline
ansible.builtin.template:
src: main.conf.j2
dest: "{{ elk_base_dir }}/config/logstash/pipeline/main.conf"
mode: "0644"
notify: "recreate logstash"
- name: Template logstash nginx pipeline
ansible.builtin.template:
src: pipeline-nginx.conf.j2
dest: "{{ elk_base_dir }}/config/logstash/pipeline/pipeline-nginx.conf"
mode: "0644"
notify: "recreate logstash"
- name: Template logstash laravel pipeline
ansible.builtin.template:
src: pipeline-laravel.conf.j2
dest: "{{ elk_base_dir }}/config/logstash/pipeline/pipeline-laravel.conf"
mode: "0644"
notify: "recreate logstash"
- name: Template docker-compose.yml
ansible.builtin.template:
src: docker-compose.yml.j2
dest: "{{ elk_base_dir }}/docker-compose.yml"
mode: "0644"
notify: "restart elk stack"
# ---------------------------------------------------------------------------
# Bring up the stack
# ---------------------------------------------------------------------------
- name: Flush handlers so config changes apply before we continue
ansible.builtin.meta: flush_handlers
- name: Start ELK stack via docker compose
community.docker.docker_compose_v2:
project_src: "{{ elk_base_dir }}"
project_name: "{{ elk_compose_project_name }}"
state: present
register: elk_compose_result
- name: Wait for Elasticsearch to answer on its HTTP port
ansible.builtin.uri:
url: "{{ elk_scheme }}://localhost:{{ elk_elasticsearch_http_port }}"
validate_certs: "{{ elk_validate_certs }}"
status_code:
- 200
- 401 # 401 is expected once security is enabled but before we authenticate
register: elk_es_wait
until: elk_es_wait.status in [200, 401]
retries: 30
delay: 10
# ---------------------------------------------------------------------------
# Security bootstrap: set built-in user passwords (idempotent)
# ---------------------------------------------------------------------------
- name: Set kibana_system password
ansible.builtin.uri:
url: "{{ elk_scheme }}://localhost:{{ elk_elasticsearch_http_port }}/_security/user/kibana_system/_password"
validate_certs: "{{ elk_validate_certs }}"
method: POST
user: elastic
password: "{{ elk_elastic_password }}"
force_basic_auth: true
body_format: json
body:
password: "{{ elk_kibana_system_password }}"
status_code: 200
when: elk_effective_security_enabled
register: elk_set_kibana_pw
retries: 5
delay: 10
until: elk_set_kibana_pw is succeeded
- name: Set logstash_system password
ansible.builtin.uri:
url: "{{ elk_scheme }}://localhost:{{ elk_elasticsearch_http_port }}/_security/user/logstash_system/_password"
validate_certs: "{{ elk_validate_certs }}"
method: POST
user: elastic
password: "{{ elk_elastic_password }}"
force_basic_auth: true
body_format: json
body:
password: "{{ elk_logstash_system_password }}"
status_code: 200
when: elk_effective_security_enabled
register: elk_set_logstash_pw
retries: 5
delay: 10
until: elk_set_logstash_pw is succeeded
- name: Recreate kibana and logstash if passwords were just set for the first time
community.docker.docker_compose_v2:
project_src: "{{ elk_base_dir }}"
project_name: "{{ elk_compose_project_name }}"
services:
- kibana
- logstash
state: present
recreate: always
when: >
elk_effective_security_enabled and
(elk_set_kibana_pw is changed or elk_set_logstash_pw is changed)
- name: Wait for Kibana to answer on its HTTP port
ansible.builtin.uri:
url: "{{ elk_scheme }}://localhost:{{ elk_kibana_http_port }}/api/status"
validate_certs: "{{ elk_validate_certs }}"
status_code:
- 200
register: elk_kibana_wait
until: elk_kibana_wait.status == 200
retries: 30
delay: 10
+176
View File
@@ -0,0 +1,176 @@
---
# ---------------------------------------------------------------------------
# TLS certificate lifecycle via lego (ACME DNS-01 against Cloudflare).
#
# This file only READS the Cloudflare token from elk_cloudflare_token_env_file.
# It never receives the token as an Ansible variable - the token is expected
# to already exist on disk (root:root, 0600), placed there by cloud-init at
# provisioning time. See the role README for the IaC side of this.
# ---------------------------------------------------------------------------
- name: Assert the Cloudflare token file exists
ansible.builtin.stat:
path: "{{ elk_cloudflare_token_env_file }}"
register: elk_cf_token_stat
- name: Fail early with a clear message if the token file is missing
ansible.builtin.fail:
msg: >
{{ elk_cloudflare_token_env_file }} does not exist. This role expects
cloud-init (or equivalent provisioning) to have already written a
scoped Cloudflare API token here before Ansible runs. See the IaC
notes in the role README.
when: not elk_cf_token_stat.stat.exists
- name: Verify the token file is not world/group readable
ansible.builtin.assert:
that:
- elk_cf_token_stat.stat.mode == "0600"
fail_msg: >
{{ elk_cloudflare_token_env_file }} should be mode 0600
(currently {{ elk_cf_token_stat.stat.mode }}) since it holds a
Cloudflare API credential.
success_msg: "Cloudflare token file permissions look correct."
- name: Create lego directories
ansible.builtin.file:
path: "{{ item }}"
state: directory
owner: root
group: root
mode: "0700"
loop:
- "{{ elk_lego_path }}"
- "{{ elk_lego_path }}/accounts"
- name: Create lego certificates directory (traversable by container users)
ansible.builtin.file:
path: "{{ elk_lego_certs_dir }}"
state: directory
owner: root
group: root
mode: "0755"
- name: Check installed lego version (if any)
ansible.builtin.command: "{{ elk_lego_install_dir }}/lego --version"
register: elk_lego_installed_version
changed_when: false
failed_when: false
- name: Download and install lego binary
when: >
elk_lego_installed_version.rc != 0 or
elk_lego_version not in (elk_lego_installed_version.stdout | default(''))
block:
- name: Download lego release tarball
ansible.builtin.get_url:
url: "https://github.com/go-acme/lego/releases/download/v{{ elk_lego_version }}/lego_v{{ elk_lego_version }}_linux_{{ elk_lego_arch }}.tar.gz"
dest: "/tmp/lego_{{ elk_lego_version }}.tar.gz"
mode: "0644"
- name: Extract lego binary
ansible.builtin.unarchive:
src: "/tmp/lego_{{ elk_lego_version }}.tar.gz"
dest: /tmp
remote_src: true
include:
- lego
- name: Install lego binary
ansible.builtin.copy:
src: /tmp/lego
dest: "{{ elk_lego_install_dir }}/lego"
owner: root
group: root
mode: "0755"
remote_src: true
- name: Clean up downloaded artifacts
ansible.builtin.file:
path: "{{ item }}"
state: absent
loop:
- "/tmp/lego_{{ elk_lego_version }}.tar.gz"
- /tmp/lego
- name: Check whether a certificate already exists for this domain
ansible.builtin.stat:
path: "{{ elk_lego_certs_dir }}/{{ elk_tls_domain }}.crt"
register: elk_cert_stat
# First issuance uses `lego ... run`, which is what actually requests a new
# certificate; `renew` (used by the systemd timer below) refuses to run
# against a domain with no existing certificate.
- name: Issue the initial certificate (first run only)
ansible.builtin.shell: |
set -o pipefail
set -a
source {{ elk_cloudflare_token_env_file }}
set +a
{{ elk_lego_install_dir }}/lego \
--accept-tos \
--email "{{ elk_tls_acme_email }}" \
--server "{{ elk_tls_acme_server }}" \
--domains "{{ elk_tls_domain }}" \
--dns cloudflare \
--path "{{ elk_lego_path }}" \
run
args:
executable: /bin/bash
when: not elk_cert_stat.stat.exists
register: elk_lego_first_issue
notify: "restart elk stack"
- name: Ensure issued cert/key are readable by the containers (root-owned, group-readable)
ansible.builtin.file:
path: "{{ item }}"
owner: root
group: root
mode: "0644"
loop:
- "{{ elk_lego_certs_dir }}/{{ elk_tls_domain }}.crt"
- "{{ elk_lego_certs_dir }}/{{ elk_tls_domain }}.issuer.crt"
when: elk_cert_stat.stat.exists or elk_lego_first_issue is not skipped
- name: Private key stays root-only-readable but group-readable for the container runtime
ansible.builtin.file:
path: "{{ elk_lego_certs_dir }}/{{ elk_tls_domain }}.key"
owner: root
group: root
mode: "0640"
when: elk_cert_stat.stat.exists or elk_lego_first_issue is not skipped
# ---------------------------------------------------------------------------
# Ongoing renewal: systemd oneshot service + daily timer. lego's `renew`
# subcommand only actually re-issues (and only then fires --run-hook) when
# the certificate is within elk_tls_renew_days of expiring, so this is safe
# to run daily without causing unnecessary container restarts.
# ---------------------------------------------------------------------------
- name: Template lego renewal systemd service
ansible.builtin.template:
src: lego-renew.service.j2
dest: /etc/systemd/system/lego-renew.service
owner: root
group: root
mode: "0644"
register: elk_lego_service_unit
- name: Template lego renewal systemd timer
ansible.builtin.template:
src: lego-renew.timer.j2
dest: /etc/systemd/system/lego-renew.timer
owner: root
group: root
mode: "0644"
register: elk_lego_timer_unit
- name: Reload systemd if units changed
ansible.builtin.systemd:
daemon_reload: true
when: elk_lego_service_unit is changed or elk_lego_timer_unit is changed
- name: Enable and start the lego renewal timer
ansible.builtin.systemd:
name: lego-renew.timer
enabled: true
state: started