feat(podman_prometheus): add hostname labels to targets

playbooks/services.yml

@@ -75,24 +75,6 @@
     - role: sr2c.core.podman_headscale
       tags: headscale
 
-- name: Deploy and update the Prometheus server
-  hosts:
-    - prometheus
-  roles:
-    - role: sr2c.core.baseline
-      vars:
-        baseline_epel_packages_allowed:
-          - node-exporter
-      tags: bootstrap
-    - role: freeipa.ansible_freeipa.ipaclient
-      become: true
-      state: present
-      tags: bootstrap
-    - role: sr2c.core.node_exporter
-      tags: prometheus
-    - role: sr2c.core.podman_prometheus
-      tags: prometheus
-
 - name: Baseline for generic servers (manual or externally managed application deployment)
   hosts:
     - generic
@@ -131,3 +113,22 @@
       tags: prometheus
     - role: sr2c.core.radius
       tags: radius
+
+- name: Deploy and update the Prometheus server
+  hosts:
+    - prometheus
+  roles:
+    - role: sr2c.core.baseline
+      vars:
+        baseline_epel_packages_allowed:
+          - node-exporter
+      tags: bootstrap
+    - role: freeipa.ansible_freeipa.ipaclient
+      become: true
+      state: present
+      tags: bootstrap
+    - role: sr2c.core.node_exporter
+      tags: prometheus
+    - role: sr2c.core.podman_prometheus
+      tags: prometheus
+

roles/baseline/tasks/tailscale.yml

@@ -38,3 +38,15 @@
   when: tailscale_status.rc != 0 or "Logged out" in tailscale_status.stdout
   no_log: yes  # Hide auth key from logs
   become: true
+
+- name: Tailscale | PATCH | Add Tailscale interface to internal zone
+  ansible.posix.firewalld:
+    zone: internal
+    interface: "{{ item }}"
+    permanent: yes
+    immediate: yes
+    state: enabled
+  with_items:
+    - tailscale0
+  become: true
+

roles/node_exporter/tasks/main.yml

@@ -1,56 +1,22 @@
 ---
+- name: Node Exporter | AUDIT | Get Tailscale IP address
+  become: true
+  ansible.builtin.shell: tailscale ip -4
+  register: node_exporter_tailscale_ipv4
+  changed_when: false
+
 - name: Node Exporter | PATCH | Install node-exporter
   become: true
   ansible.builtin.dnf:
     name: node-exporter
     state: present
 
-- name: Node Exporter | PATCH | Generate private TLS key
-  community.crypto.openssl_privatekey:
-    path: /etc/ssl/node-exporter.key
-    size: 4096
-    owner: prometheus
-    group: root
-    mode: '0440'
-  become: true
-
-- name: Node Exporter | PATCH | Create certificate signing request
-  community.crypto.openssl_csr:
-    path: /etc/ssl/node-exporter.csr
-    privatekey_path: /etc/ssl/node-exporter.key
-    common_name: "{{ inventory_hostname }}"
-    subject_alt_name: "DNS:{{ inventory_hostname }}"
-    owner: root
-    group: root
-    mode: '0400'
-  become: true
-
-- name: Generate self-signed certificate
-  community.crypto.x509_certificate:
-    provider: selfsigned
-    path: /etc/ssl/node-exporter.crt
-    privatekey_path: /etc/ssl/node-exporter.key
-    csr_path: /etc/ssl/node-exporter.csr
-    owner: prometheus
-    group: root
-    mode: '0440'
-  become: true
-
-- name: Node Exporter | PATCH | Install node-exporter web configuration
-  become: true
-  ansible.builtin.template:
-    src: etc/node-exporter-web.yml
-    dest: /etc/node-exporter-web.yml
-    owner: root
-    group: root
-    mode: "0444"
-
 - name: Node Exporter | PATCH | Set command line arguments
   become: true
   ansible.builtin.lineinfile:
     path: /etc/default/prometheus-node-exporter
     regexp: "^ARGS"
-    line: "ARGS='--web.config.file=\"/etc/node-exporter-web.yml\"{% if node_exporter_textfile_directory is defined %} --collector.textfile.directory {{ node_exporter_textfile_directory }}{% endif %}'"
+    line: "ARGS='--web.listen-address={{ node_exporter_tailscale_ipv4.stdout }}:9100{% if node_exporter_textfile_directory is defined %} --collector.textfile.directory {{ node_exporter_textfile_directory }}{% endif %}'"
   notify: Restart Node Exporter
 
 - name: Node Exporter | PATCH | Ensure node-exporter is enabled and running
@@ -78,6 +44,7 @@
   become: true
   ansible.posix.firewalld:
     service: node-exporter
+    zone: internal
     permanent: true
     state: enabled
     immediate: true

roles/node_exporter/templates/etc/node-exporter-web.yml

@@ -1,4 +1,6 @@
 ---
+listen_address: {{ node_exporter_tailscale_ipv4 }}:9090
+
 tls_server_config:
   cert_file: /etc/ssl/node-exporter.crt
   key_file: /etc/ssl/node-exporter.key

roles/podman_prometheus/handlers/main.yml

@@ -23,6 +23,14 @@
   become: true
   become_user: "{{ podman_prometheus_podman_rootless_user }}"
 
+- name: Restart Prometheus-TS
+  ansible.builtin.systemd_service:
+    name: prometheus-ts
+    scope: user
+    state: restarted
+  become: true
+  become_user: "{{ podman_prometheus_podman_rootless_user }}"
+
 - name: Restart nginx
   ansible.builtin.systemd_service:
     name: nginx

roles/podman_prometheus/tasks/main.yml

@@ -111,10 +111,13 @@
     - alertmanager.container
     - grafana.container
     - prometheus.container
+    - prometheus-ts.container
   become: true
   notify:
+    - Restart Alertmanager
     - Restart Grafana
     - Restart Prometheus
+    - Restart Prometheus-TS
 
 - name: Podman Prometheus | PATCH | Install network quadlets
   ansible.builtin.template:
@@ -178,6 +181,7 @@
     - grafana
     - nginx
     - prometheus
+    - prometheus-ts
   become: true
   become_user: "{{ podman_prometheus_podman_rootless_user }}"
 

roles/podman_prometheus/templates/home/podman/config/containers/systemd/prometheus-ts.container

@@ -0,0 +1,16 @@
+[Container]
+ContainerName=prometheus-ts
+Image=docker.io/tailscale/tailscale:latest
+HostName=prometheus
+Environment=TS_AUTH_KEY={{ podman_prometheus_ts_auth_key }}
+Environment=TS_STATE_DIR=/var/lib/tailscale
+Environment=TS_USERSPACE=true
+Environment=TS_EXTRA_ARGS="--login-server https://hs.sr2.uk/"
+Network=monitor.network
+
+[Service]
+Restart=on-failure
+
+[Install]
+WantedBy=default.target
+

roles/podman_prometheus/templates/home/podman/prometheus.yml

@@ -13,26 +13,18 @@ scrape_configs:
       - targets: ['alertmanager:9093']
   - job_name: 'node'
     scrape_interval: 5s
-    scheme: https
-    basic_auth:
-      username: metrics
-      password: "{{ node_exporter_password }}"
-    tls_config:
-      insecure_skip_verify: true
+    scheme: http
     static_configs:
+      - targets: ['{{ node_exporter_tailscale_ipv4.stdout }}:9100']
+        labels:
+          instance: "{{ inventory_hostname }}"
+          hostname: "{{ inventory_hostname }}"
+{% for host in (groups['ipaservers'] + groups['keycloak'] + groups['radius'] + groups['generic']) %}
       - targets:
-        - 'host.containers.internal:9100'
-{% for host in groups['ipaservers'] %}
-        - '{{ host }}:9100'
-{% endfor %}
-{% for host in groups['keycloak'] %}
-        - '{{ host }}:9100'
-{% endfor %}
-{% for host in groups['radius'] %}
-        - '{{ host }}:9100'
-{% endfor %}
-{% for host in groups['generic'] %}
-        - '{{ host }}:9100'
+        - "{{ hostvars[host]['node_exporter_tailscale_ipv4'].stdout }}:9100"
+        labels:
+          instance: "{{ host }}"
+          hostname: "{{ host }}"
 {% endfor %}
     file_sd_configs:
       - files: