feat(podman_prometheus): initial role for running prometheus in podman

roles/podman_prometheus/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+podman_prometheus_podman_rootless_user: prometheus

roles/podman_prometheus/handlers/main.yml

@@ -0,0 +1,8 @@
+---
+- name: Restart Prometheus
+  ansible.builtin.systemd_service:
+    name: prometheus
+    scope: user
+    state: restarted
+  become: true
+  become_user: "{{ podman_prometheus_podman_rootless_user }}"

roles/podman_prometheus/tasks/main.yml

@@ -0,0 +1,148 @@
+---
+- name: Podman Prometheus | PATCH | Install data plate
+  ansible.builtin.template:
+    src: etc/motd.d/10-data-plate.txt
+    dest: /etc/motd.d/10-data-plate.txt
+    owner: root
+    group: root
+    mode: "0444"
+  become: true
+
+- name: Podman Prometheus | PATCH | Install podman and verify rootless podman user
+  ansible.builtin.include_role:
+    role: sr2c.core.podman_host
+  vars:
+    podman_host_minimum_unpriv_port: 80
+    podman_host_rootless_users: ["{{ podman_prometheus_podman_rootless_user }}"]
+
+- name: Podman Prometheus | AUDIT | Get subuid range for user
+  ansible.builtin.command:
+    cmd: "getsubids {{ podman_prometheus_podman_rootless_user }}"
+  register: _podman_prometheus_user_subuid
+  changed_when: false
+
+- name: Podman Prometheus | AUDIT | Get subgid range for user
+  ansible.builtin.command:
+    cmd: "getsubids -g {{ podman_prometheus_podman_rootless_user }}"
+  register: _podman_prometheus_user_subgid
+  changed_when: false
+
+- name: Podman Prometheus | AUDIT | Parse outputs of getsubids and store results
+  ansible.builtin.set_fact:
+    _podman_prometheus_user_subuid_start: "{{ (_podman_prometheus_user_subuid.stdout_lines[0].split()[2] | int) }}"
+    _podman_prometheus_user_subgid_start: "{{ (_podman_prometheus_user_subgid.stdout_lines[0].split()[2] | int) }}"
+
+# Prometheus runs with UID/GID 65534 inside the container
+- name: Podman Prometheus | PATCH | Create data directory for Prometheus
+  ansible.builtin.file:
+    path: "/home/{{ podman_prometheus_podman_rootless_user }}/prometheus-data"
+    owner: "{{ _podman_prometheus_user_subuid_start + 65533 }}"
+    group: "{{ _podman_prometheus_user_subgid_start + 65533 }}"
+    mode: "0700"
+    state: "directory"
+  become: true
+
+# Prometheus runs with UID/GID 65534 inside the container
+- name: Podman Prometheus | PATCH | Create service discovery directory for Prometheus
+  ansible.builtin.file:
+    path: "/home/{{ podman_prometheus_podman_rootless_user }}/file-configs"
+    owner: "{{ _podman_prometheus_user_subuid_start + 65533 }}"
+    group: "{{ _podman_prometheus_user_subgid_start + 65533 }}"
+    mode: "0700"
+    state: "directory"
+  become: true
+
+# Prometheus runs with UID/GID 65534 inside the container
+- name: Podman Prometheus | PATCH | Install Prometheus configuration
+  ansible.builtin.template:
+    src: home/podman/prometheus.yml
+    dest: "/home/{{ podman_prometheus_podman_rootless_user }}/prometheus.yml"
+    mode: "0400"
+    owner: "{{ _podman_prometheus_user_subuid_start + 65533 }}"
+    group: "{{ _podman_prometheus_user_subgid_start + 65533 }}"
+  become: true
+  notify:
+    - Restart Prometheus
+
+- name: Podman Prometheus | PATCH | Install container quadlets
+  ansible.builtin.template:
+    src: "home/podman/config/containers/systemd/{{ item }}"
+    dest: "/home/{{ podman_prometheus_podman_rootless_user }}/.config/containers/systemd/{{ item }}"
+    owner: "{{ podman_prometheus_podman_rootless_user }}"
+    mode: "0400"
+  with_items:
+    - prometheus.container
+  become: true
+  notify:
+    - Restart Prometheus
+
+- name: Podman Prometheus | PATCH | Install network quadlets
+  ansible.builtin.template:
+    src: "home/podman/config/containers/systemd/{{ item }}"
+    dest: "/home/{{ podman_prometheus_podman_rootless_user }}/.config/containers/systemd/{{ item }}"
+    owner: "{{ podman_prometheus_podman_rootless_user }}"
+    mode: "0400"
+  with_items:
+    - frontend.network
+  become: true
+  notify:
+    - Restart Prometheus
+    - Restart nginx
+
+- name: Podman Prometheus | AUDIT | Verify quadlets are correctly defined
+  ansible.builtin.command: /usr/libexec/podman/quadlet -dryrun -user
+  register: podman_prometheus_quadlet_result
+  ignore_errors: true
+  changed_when: false
+  become: true
+  become_user: "{{ podman_prometheus_podman_rootless_user }}"
+
+- name: Podman Prometheus | AUDIT | Assert that the quadlet verification succeeded
+  ansible.builtin.assert:
+    that:
+      - podman_prometheus_quadlet_result.rc == 0
+    fail_msg: "'/usr/libexec/podman/quadlet -dryrun -user' failed! Output withheld to prevent leaking secrets."
+
+- name: Podman Prometheus | PATCH | Set up nginx and Let's Encrypt certificate
+  ansible.builtin.include_role:
+    name: sr2c.core.podman_nginx
+  vars:
+    podman_nginx_frontend_network: frontend
+    podman_nginx_podman_rootless_user: "{{ podman_prometheus_podman_rootless_user }}"
+    podman_nginx_primary_hostname: "{{ inventory_hostname }}"
+
+- name: Podman Prometheus | PATCH | Install production nginx configuration file
+  ansible.builtin.template:
+    src: home/podman/nginx.conf
+    dest: "/home/{{ podman_prometheus_podman_rootless_user }}/nginx/nginx.conf"
+    owner: "{{ podman_prometheus_podman_rootless_user }}"
+    group: "{{ podman_prometheus_podman_rootless_user }}"
+    mode: "0644"
+  become: true
+  notify:
+    - Restart nginx
+
+- name: Podman Prometheus | PATCH | Make sure Prometheus and Nginx are running now and started on boot
+  ansible.builtin.systemd_service:
+    name: "{{ item }}.service"
+    enabled: true
+    state: started
+    masked: false
+    daemon_reload: true
+    scope: user
+  with_items:
+    - nginx
+    - prometheus
+  become: true
+  become_user: "{{ podman_prometheus_podman_rootless_user }}"
+
+- name: Podman Prometheus | PATCH | Set up ClouDNS monitoring
+  sr2c.core.cloudns_monitor:
+    name: "Prometheus - {{ inventory_hostname[:19] }}"
+    host: "{{ inventory_hostname }}"
+    ip: "{{ inventory_hostname }}"
+    http_status_code: "200"
+    emails: "{{ cloudns_monitoring_emails }}"
+    auth_id: "{{ cloudns_auth_id }}"
+    auth_password: "{{ cloudns_auth_password }}"
+  delegate_to: localhost

roles/podman_prometheus/templates/etc/motd.d/10-data-plate.txt

@@ -0,0 +1,14 @@
+    =========================================================
+    A Prometheus instance is hosted on this server.
+    Podman user: {{ podman_prometheus_podman_rootless_user }}
+    =========================================================
+    # Become the podman user
+    sudo -iu {{ podman_prometheus_podman_rootless_user }}
+    # Check the Prometheus status
+    systemctl --user status prometheus.service
+    # Restart Prometheus
+    systemctl --user restart prometheus.service
+    # Follow the logs for Prometheus
+    journalctl --user -fu prometheus.service
+    =========================================================
+

roles/podman_prometheus/templates/home/podman/config/containers/systemd/frontend.network

@@ -0,0 +1,2 @@
+[Network]
+NetworkName=frontend

roles/podman_prometheus/templates/home/podman/config/containers/systemd/prometheus.container

@@ -0,0 +1,13 @@
+[Container]
+ContainerName=prometheus
+Image=quay.io/prometheus/prometheus:v3.8.1
+Network=frontend.network
+Volume=/home/{{ podman_prometheus_podman_rootless_user }}/prometheus-data:/prometheus:rw,Z
+Volume=/home/{{ podman_prometheus_podman_rootless_user }}/file-configs:/file-configs:ro,Z
+Volume=/home/{{ podman_prometheus_podman_rootless_user }}/prometheus.yml:/etc/prometheus/prometheus.yml:ro,Z
+
+[Service]
+Restart=on-failure
+
+[Install]
+WantedBy=default.target

roles/podman_prometheus/templates/home/podman/nginx.conf

@@ -0,0 +1,52 @@
+# {{ ansible_managed }}
+
+resolver 10.89.0.1 ipv6=off valid=10s;
+
+# Mitigate httpoxy attack
+proxy_set_header Proxy "";
+
+server {
+    listen 80;
+    listen [::]:80;
+
+    server_name {{ inventory_hostname }};
+    server_tokens off;
+
+    location /.well-known/acme-challenge/ {
+        root /var/www/certbot;
+    }
+
+    location / {
+        return 301 https://{{ inventory_hostname }}$request_uri;
+    }
+}
+
+upstream prometheus {
+    zone prometheus_upstream 64k;
+	server prometheus:9090 resolve;
+}
+
+server {
+	server_name {{ inventory_hostname }};
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+    server_tokens off;
+
+    ssl_certificate /etc/letsencrypt/live/{{ inventory_hostname }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ inventory_hostname }}/privkey.pem;
+
+	add_header Strict-Transport-Security "max-age=31536000" always;
+	add_header Referrer-Policy origin always;  # make sure outgoing links don't show the URL to the Prometheus instance
+    add_header X-Content-Type-Options "nosniff" always;
+    add_header X-XSS-Protection "1; mode=block" always;
+
+  location / {
+    proxy_set_header Host $http_host;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    proxy_set_header X-Forwarded-Proto $scheme;
+
+    proxy_read_timeout 180;
+    proxy_pass http://prometheus;
+  }
+}

roles/podman_prometheus/templates/home/podman/prometheus.yml

@@ -0,0 +1,33 @@
+global:
+  scrape_interval: 15s
+  external_labels:
+    monitor: '{{ inventory_hostname }}'
+
+scrape_configs:
+  - job_name: 'prometheus'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['localhost:9090']
+  - job_name: 'node'
+    scrape_interval: 5s
+    scheme: https
+    basic_auth:
+      username: metrics
+      password: "{{ node_exporter_password }}"
+    tls_config:
+      insecure_skip_verify: true
+    static_configs:
+      - targets:
+        - 'host.containers.internal:9100'
+{% for host in groups['ipaservers'] %}
+        - '{{ host }}:9100'
+{% endfor %}
+{% for host in groups['keycloak'] %}
+        - '{{ host }}:9100'
+{% endfor %}
+{% for host in groups['generic'] %}
+        - '{{ host }}:9100'
+{% endfor %}
+    file_sd_configs:
+      - files:
+          - "/file-configs/*.yml"