playbooks/services.yml

@@ -75,6 +75,24 @@
     - role: sr2c.core.podman_headscale
       tags: headscale
 
+- name: Deploy and update the Prometheus server
+  hosts:
+    - prometheus
+  roles:
+    - role: sr2c.core.baseline
+      vars:
+        baseline_epel_packages_allowed:
+          - node-exporter
+      tags: bootstrap
+    - role: freeipa.ansible_freeipa.ipaclient
+      become: true
+      state: present
+      tags: bootstrap
+    - role: sr2c.core.node_exporter
+      tags: prometheus
+    - role: sr2c.core.podman_prometheus
+      tags: prometheus
+
 - name: Baseline for generic servers (manual or externally managed application deployment)
   hosts:
     - generic
@@ -113,22 +131,3 @@
       tags: prometheus
     - role: sr2c.core.radius
       tags: radius
-
-- name: Deploy and update the Prometheus server
-  hosts:
-    - prometheus
-  roles:
-    - role: sr2c.core.baseline
-      vars:
-        baseline_epel_packages_allowed:
-          - node-exporter
-      tags: bootstrap
-    - role: freeipa.ansible_freeipa.ipaclient
-      become: true
-      state: present
-      tags: bootstrap
-    - role: sr2c.core.node_exporter
-      tags: prometheus
-    - role: sr2c.core.podman_prometheus
-      tags: prometheus
-

roles/baseline/tasks/tailscale.yml

@@ -38,15 +38,3 @@
   when: tailscale_status.rc != 0 or "Logged out" in tailscale_status.stdout
   no_log: yes  # Hide auth key from logs
   become: true
-
-- name: Tailscale | PATCH | Add Tailscale interface to internal zone
-  ansible.posix.firewalld:
-    zone: internal
-    interface: "{{ item }}"
-    permanent: yes
-    immediate: yes
-    state: enabled
-  with_items:
-    - tailscale0
-  become: true
-

roles/node_exporter/tasks/main.yml

@@ -1,22 +1,56 @@
 ---
-- name: Node Exporter | AUDIT | Get Tailscale IP address
-  become: true
-  ansible.builtin.shell: tailscale ip -4
-  register: node_exporter_tailscale_ipv4
-  changed_when: false
-
 - name: Node Exporter | PATCH | Install node-exporter
   become: true
   ansible.builtin.dnf:
     name: node-exporter
     state: present
 
+- name: Node Exporter | PATCH | Generate private TLS key
+  community.crypto.openssl_privatekey:
+    path: /etc/ssl/node-exporter.key
+    size: 4096
+    owner: prometheus
+    group: root
+    mode: '0440'
+  become: true
+
+- name: Node Exporter | PATCH | Create certificate signing request
+  community.crypto.openssl_csr:
+    path: /etc/ssl/node-exporter.csr
+    privatekey_path: /etc/ssl/node-exporter.key
+    common_name: "{{ inventory_hostname }}"
+    subject_alt_name: "DNS:{{ inventory_hostname }}"
+    owner: root
+    group: root
+    mode: '0400'
+  become: true
+
+- name: Generate self-signed certificate
+  community.crypto.x509_certificate:
+    provider: selfsigned
+    path: /etc/ssl/node-exporter.crt
+    privatekey_path: /etc/ssl/node-exporter.key
+    csr_path: /etc/ssl/node-exporter.csr
+    owner: prometheus
+    group: root
+    mode: '0440'
+  become: true
+
+- name: Node Exporter | PATCH | Install node-exporter web configuration
+  become: true
+  ansible.builtin.template:
+    src: etc/node-exporter-web.yml
+    dest: /etc/node-exporter-web.yml
+    owner: root
+    group: root
+    mode: "0444"
+
 - name: Node Exporter | PATCH | Set command line arguments
   become: true
   ansible.builtin.lineinfile:
     path: /etc/default/prometheus-node-exporter
     regexp: "^ARGS"
-    line: "ARGS='--web.listen-address={{ node_exporter_tailscale_ipv4.stdout }}:9100{% if node_exporter_textfile_directory is defined %} --collector.textfile.directory {{ node_exporter_textfile_directory }}{% endif %}'"
+    line: "ARGS='--web.config.file=\"/etc/node-exporter-web.yml\"{% if node_exporter_textfile_directory is defined %} --collector.textfile.directory {{ node_exporter_textfile_directory }}{% endif %}'"
   notify: Restart Node Exporter
 
 - name: Node Exporter | PATCH | Ensure node-exporter is enabled and running
@@ -44,7 +78,6 @@
   become: true
   ansible.posix.firewalld:
     service: node-exporter
-    zone: internal
     permanent: true
     state: enabled
     immediate: true

roles/node_exporter/templates/etc/node-exporter-web.yml

@@ -1,6 +1,4 @@
 ---
-listen_address: {{ node_exporter_tailscale_ipv4 }}:9090
-
 tls_server_config:
   cert_file: /etc/ssl/node-exporter.crt
   key_file: /etc/ssl/node-exporter.key

roles/podman_prometheus/handlers/main.yml

@@ -23,14 +23,6 @@
   become: true
   become_user: "{{ podman_prometheus_podman_rootless_user }}"
 
-- name: Restart Prometheus-TS
-  ansible.builtin.systemd_service:
-    name: prometheus-ts
-    scope: user
-    state: restarted
-  become: true
-  become_user: "{{ podman_prometheus_podman_rootless_user }}"
-
 - name: Restart nginx
   ansible.builtin.systemd_service:
     name: nginx

roles/podman_prometheus/tasks/main.yml

@@ -111,13 +111,10 @@
     - alertmanager.container
     - grafana.container
     - prometheus.container
-    - prometheus-ts.container
   become: true
   notify:
-    - Restart Alertmanager
     - Restart Grafana
     - Restart Prometheus
-    - Restart Prometheus-TS
 
 - name: Podman Prometheus | PATCH | Install network quadlets
   ansible.builtin.template:
@@ -181,7 +178,6 @@
     - grafana
     - nginx
     - prometheus
-    - prometheus-ts
   become: true
   become_user: "{{ podman_prometheus_podman_rootless_user }}"
 

roles/podman_prometheus/templates/home/podman/config/containers/systemd/prometheus-ts.container

@@ -1,16 +0,0 @@
-[Container]
-ContainerName=prometheus-ts
-Image=docker.io/tailscale/tailscale:latest
-HostName=prometheus
-Environment=TS_AUTH_KEY={{ podman_prometheus_ts_auth_key }}
-Environment=TS_STATE_DIR=/var/lib/tailscale
-Environment=TS_USERSPACE=true
-Environment=TS_EXTRA_ARGS="--login-server https://hs.sr2.uk/"
-Network=monitor.network
-
-[Service]
-Restart=on-failure
-
-[Install]
-WantedBy=default.target
-

roles/podman_prometheus/templates/home/podman/prometheus.yml

@@ -13,18 +13,26 @@ scrape_configs:
       - targets: ['alertmanager:9093']
   - job_name: 'node'
     scrape_interval: 5s
-    scheme: http
+    scheme: https
+    basic_auth:
+      username: metrics
+      password: "{{ node_exporter_password }}"
+    tls_config:
+      insecure_skip_verify: true
     static_configs:
-      - targets: ['{{ node_exporter_tailscale_ipv4.stdout }}:9100']
-        labels:
-          instance: "{{ inventory_hostname }}"
-          hostname: "{{ inventory_hostname }}"
-{% for host in (groups['ipaservers'] + groups['keycloak'] + groups['radius'] + groups['generic']) %}
       - targets:
-        - "{{ hostvars[host]['node_exporter_tailscale_ipv4'].stdout }}:9100"
+        - 'host.containers.internal:9100'
-        labels:
+{% for host in groups['ipaservers'] %}
-          instance: "{{ host }}"
+        - '{{ host }}:9100'
-          hostname: "{{ host }}"
+{% endfor %}
+{% for host in groups['keycloak'] %}
+        - '{{ host }}:9100'
+{% endfor %}
+{% for host in groups['radius'] %}
+        - '{{ host }}:9100'
+{% endfor %}
+{% for host in groups['generic'] %}
+        - '{{ host }}:9100'
 {% endfor %}
     file_sd_configs:
       - files: