From c404d08b896c497ac8a6e381f84511d776f85f5d Mon Sep 17 00:00:00 2001 From: irl Date: Mon, 25 May 2026 15:08:46 +0100 Subject: [PATCH 1/2] feat(node_exporter): use tailnet only --- roles/node_exporter/tasks/main.yml | 5 +++++ roles/node_exporter/templates/etc/node-exporter-web.yml | 2 ++ 2 files changed, 7 insertions(+) diff --git a/roles/node_exporter/tasks/main.yml b/roles/node_exporter/tasks/main.yml index 2278dc4..a8e7218 100644 --- a/roles/node_exporter/tasks/main.yml +++ b/roles/node_exporter/tasks/main.yml @@ -1,4 +1,9 @@ --- +- name: Node Exporter | AUDIT | Get Tailscale IP + ansible.builtin.shell: tailscale ip -4 2>/dev/null + register: node_exporter_tailscale_ipv4 + changed_when: false + - name: Node Exporter | PATCH | Install node-exporter become: true ansible.builtin.dnf: diff --git a/roles/node_exporter/templates/etc/node-exporter-web.yml b/roles/node_exporter/templates/etc/node-exporter-web.yml index 786c1ce..5c86870 100644 --- a/roles/node_exporter/templates/etc/node-exporter-web.yml +++ b/roles/node_exporter/templates/etc/node-exporter-web.yml @@ -1,4 +1,6 @@ --- +listen_address: {{ node_exporter_tailscale_ipv4 }}:9090 + tls_server_config: cert_file: /etc/ssl/node-exporter.crt key_file: /etc/ssl/node-exporter.key From 0d92344f16b103ea31b70dd4124b3276c61cd57e Mon Sep 17 00:00:00 2001 From: irl Date: Mon, 25 May 2026 17:23:34 +0100 Subject: [PATCH 2/2] feat(podman_prometheus): add hostname labels to targets --- playbooks/services.yml | 37 +++++++------- roles/baseline/tasks/tailscale.yml | 12 +++++ roles/node_exporter/tasks/main.yml | 48 ++----------------- roles/podman_prometheus/handlers/main.yml | 8 ++++ roles/podman_prometheus/tasks/main.yml | 4 ++ .../systemd/prometheus-ts.container | 16 +++++++ .../templates/home/podman/prometheus.yml | 28 ++++------- 7 files changed, 74 insertions(+), 79 deletions(-) create mode 100644 roles/podman_prometheus/templates/home/podman/config/containers/systemd/prometheus-ts.container diff --git a/playbooks/services.yml b/playbooks/services.yml index d973017..117a537 100644 --- a/playbooks/services.yml +++ b/playbooks/services.yml @@ -75,24 +75,6 @@ - role: sr2c.core.podman_headscale tags: headscale -- name: Deploy and update the Prometheus server - hosts: - - prometheus - roles: - - role: sr2c.core.baseline - vars: - baseline_epel_packages_allowed: - - node-exporter - tags: bootstrap - - role: freeipa.ansible_freeipa.ipaclient - become: true - state: present - tags: bootstrap - - role: sr2c.core.node_exporter - tags: prometheus - - role: sr2c.core.podman_prometheus - tags: prometheus - - name: Baseline for generic servers (manual or externally managed application deployment) hosts: - generic @@ -131,3 +113,22 @@ tags: prometheus - role: sr2c.core.radius tags: radius + +- name: Deploy and update the Prometheus server + hosts: + - prometheus + roles: + - role: sr2c.core.baseline + vars: + baseline_epel_packages_allowed: + - node-exporter + tags: bootstrap + - role: freeipa.ansible_freeipa.ipaclient + become: true + state: present + tags: bootstrap + - role: sr2c.core.node_exporter + tags: prometheus + - role: sr2c.core.podman_prometheus + tags: prometheus + diff --git a/roles/baseline/tasks/tailscale.yml b/roles/baseline/tasks/tailscale.yml index f0e011e..342a232 100644 --- a/roles/baseline/tasks/tailscale.yml +++ b/roles/baseline/tasks/tailscale.yml @@ -38,3 +38,15 @@ when: tailscale_status.rc != 0 or "Logged out" in tailscale_status.stdout no_log: yes # Hide auth key from logs become: true + +- name: Tailscale | PATCH | Add Tailscale interface to internal zone + ansible.posix.firewalld: + zone: internal + interface: "{{ item }}" + permanent: yes + immediate: yes + state: enabled + with_items: + - tailscale0 + become: true + diff --git a/roles/node_exporter/tasks/main.yml b/roles/node_exporter/tasks/main.yml index a8e7218..07a2df0 100644 --- a/roles/node_exporter/tasks/main.yml +++ b/roles/node_exporter/tasks/main.yml @@ -1,6 +1,7 @@ --- -- name: Node Exporter | AUDIT | Get Tailscale IP - ansible.builtin.shell: tailscale ip -4 2>/dev/null +- name: Node Exporter | AUDIT | Get Tailscale IP address + become: true + ansible.builtin.shell: tailscale ip -4 register: node_exporter_tailscale_ipv4 changed_when: false @@ -10,52 +11,12 @@ name: node-exporter state: present -- name: Node Exporter | PATCH | Generate private TLS key - community.crypto.openssl_privatekey: - path: /etc/ssl/node-exporter.key - size: 4096 - owner: prometheus - group: root - mode: '0440' - become: true - -- name: Node Exporter | PATCH | Create certificate signing request - community.crypto.openssl_csr: - path: /etc/ssl/node-exporter.csr - privatekey_path: /etc/ssl/node-exporter.key - common_name: "{{ inventory_hostname }}" - subject_alt_name: "DNS:{{ inventory_hostname }}" - owner: root - group: root - mode: '0400' - become: true - -- name: Generate self-signed certificate - community.crypto.x509_certificate: - provider: selfsigned - path: /etc/ssl/node-exporter.crt - privatekey_path: /etc/ssl/node-exporter.key - csr_path: /etc/ssl/node-exporter.csr - owner: prometheus - group: root - mode: '0440' - become: true - -- name: Node Exporter | PATCH | Install node-exporter web configuration - become: true - ansible.builtin.template: - src: etc/node-exporter-web.yml - dest: /etc/node-exporter-web.yml - owner: root - group: root - mode: "0444" - - name: Node Exporter | PATCH | Set command line arguments become: true ansible.builtin.lineinfile: path: /etc/default/prometheus-node-exporter regexp: "^ARGS" - line: "ARGS='--web.config.file=\"/etc/node-exporter-web.yml\"{% if node_exporter_textfile_directory is defined %} --collector.textfile.directory {{ node_exporter_textfile_directory }}{% endif %}'" + line: "ARGS='--web.listen-address={{ node_exporter_tailscale_ipv4.stdout }}:9100{% if node_exporter_textfile_directory is defined %} --collector.textfile.directory {{ node_exporter_textfile_directory }}{% endif %}'" notify: Restart Node Exporter - name: Node Exporter | PATCH | Ensure node-exporter is enabled and running @@ -83,6 +44,7 @@ become: true ansible.posix.firewalld: service: node-exporter + zone: internal permanent: true state: enabled immediate: true diff --git a/roles/podman_prometheus/handlers/main.yml b/roles/podman_prometheus/handlers/main.yml index 91c6ade..9012a6e 100644 --- a/roles/podman_prometheus/handlers/main.yml +++ b/roles/podman_prometheus/handlers/main.yml @@ -23,6 +23,14 @@ become: true become_user: "{{ podman_prometheus_podman_rootless_user }}" +- name: Restart Prometheus-TS + ansible.builtin.systemd_service: + name: prometheus-ts + scope: user + state: restarted + become: true + become_user: "{{ podman_prometheus_podman_rootless_user }}" + - name: Restart nginx ansible.builtin.systemd_service: name: nginx diff --git a/roles/podman_prometheus/tasks/main.yml b/roles/podman_prometheus/tasks/main.yml index f9f6554..1496aea 100644 --- a/roles/podman_prometheus/tasks/main.yml +++ b/roles/podman_prometheus/tasks/main.yml @@ -111,10 +111,13 @@ - alertmanager.container - grafana.container - prometheus.container + - prometheus-ts.container become: true notify: + - Restart Alertmanager - Restart Grafana - Restart Prometheus + - Restart Prometheus-TS - name: Podman Prometheus | PATCH | Install network quadlets ansible.builtin.template: @@ -178,6 +181,7 @@ - grafana - nginx - prometheus + - prometheus-ts become: true become_user: "{{ podman_prometheus_podman_rootless_user }}" diff --git a/roles/podman_prometheus/templates/home/podman/config/containers/systemd/prometheus-ts.container b/roles/podman_prometheus/templates/home/podman/config/containers/systemd/prometheus-ts.container new file mode 100644 index 0000000..d41ee86 --- /dev/null +++ b/roles/podman_prometheus/templates/home/podman/config/containers/systemd/prometheus-ts.container @@ -0,0 +1,16 @@ +[Container] +ContainerName=prometheus-ts +Image=docker.io/tailscale/tailscale:latest +HostName=prometheus +Environment=TS_AUTH_KEY={{ podman_prometheus_ts_auth_key }} +Environment=TS_STATE_DIR=/var/lib/tailscale +Environment=TS_USERSPACE=true +Environment=TS_EXTRA_ARGS="--login-server https://hs.sr2.uk/" +Network=monitor.network + +[Service] +Restart=on-failure + +[Install] +WantedBy=default.target + diff --git a/roles/podman_prometheus/templates/home/podman/prometheus.yml b/roles/podman_prometheus/templates/home/podman/prometheus.yml index 4870f78..9522b54 100644 --- a/roles/podman_prometheus/templates/home/podman/prometheus.yml +++ b/roles/podman_prometheus/templates/home/podman/prometheus.yml @@ -13,26 +13,18 @@ scrape_configs: - targets: ['alertmanager:9093'] - job_name: 'node' scrape_interval: 5s - scheme: https - basic_auth: - username: metrics - password: "{{ node_exporter_password }}" - tls_config: - insecure_skip_verify: true + scheme: http static_configs: + - targets: ['{{ node_exporter_tailscale_ipv4.stdout }}:9100'] + labels: + instance: "{{ inventory_hostname }}" + hostname: "{{ inventory_hostname }}" +{% for host in (groups['ipaservers'] + groups['keycloak'] + groups['radius'] + groups['generic']) %} - targets: - - 'host.containers.internal:9100' -{% for host in groups['ipaservers'] %} - - '{{ host }}:9100' -{% endfor %} -{% for host in groups['keycloak'] %} - - '{{ host }}:9100' -{% endfor %} -{% for host in groups['radius'] %} - - '{{ host }}:9100' -{% endfor %} -{% for host in groups['generic'] %} - - '{{ host }}:9100' + - "{{ hostvars[host]['node_exporter_tailscale_ipv4'].stdout }}:9100" + labels: + instance: "{{ host }}" + hostname: "{{ host }}" {% endfor %} file_sd_configs: - files: