dump

nixos/hosts/monitoring/prometheus.nix

@@ -1,51 +1,137 @@
-{ config, pkgs, modulesPath, lib, ... }:
+{
+  config,
+  pkgs,
+  modulesPath,
+  lib,
+  ...
+}: let
+  monitor_hostname = "monitor.lab";
+  traefik_hostname = "traefik.lab";
+  sandbox_hostname = "sandbox.lab";
+  forgejo_hostname = "forgejo.lab";
+  runner01_hostname = "forgejo-runner-01.lab";
+  dnsmasq_hostname = "dns.lab";
+  media_hostname = "media.lab";
+  mail_hostname = "mail.lab";
+  keycloak_hostname = "keycloak.lab";
 
-let
-  monitor_ip = "monitor.lab";
-  traefik_ip = "traefik.lab";
-  sandbox_ip = "sandbox.lab";
-  forgejo_ip = "forgejo.lab";
-  runner01_ip = "forgejo-runner-01.lab";
-  dnsmasq_ip = "dns.lab";
+  monitored_hosts = [
+    monitor_hostname
+    traefik_hostname
+    sandbox_hostname
+    forgejo_hostname
+    runner01_hostname
+    dnsmasq_hostname
+    media_hostname
+    mail_hostname
+    keycloak_hostname
+  ];
+
+  generateTargets = port:
+    map (host: "${host}:${toString port}") monitored_hosts;
+
+  instance_relabel_config = [
+    {
+      source_labels = ["__address__"];
+      regex = "([^:]+):\\d+"; # Captures everything before the last colon
+      target_label = "instance";
+      replacement = "$1";
+    }
+  ];
+
+  node_exporter_port = 9100;
+  node_exporter_job = {
+    job_name = "node";
+    static_configs = [{targets = generateTargets node_exporter_port;}];
+    relabel_configs = instance_relabel_config;
+  };
+
+  promtail_port = 9080;
+  promtail_job = {
+    job_name = "promtail";
+    static_configs = [{targets = generateTargets promtail_port;}];
+    relabel_configs = instance_relabel_config;
+  };
 
-  prometheus_exporter_port = 9100;
-  postgres_exporter_port = 9187;
   prometheus_port = 9090;
   alertmanager_port = 9093;
   grafana_port = 3000;
-  promtail_port = 9080;
-  traefik_monitor_port = 8082;
-  forgejo_monitor_port = 3000;
-  dnsmasq_exporter_port = 9153;
-
-  exporters = {
-    node = [
-        "${monitor_ip}:${toString prometheus_exporter_port}"
-        "${traefik_ip}:${toString prometheus_exporter_port}"
-        "${sandbox_ip}:${toString prometheus_exporter_port}"
-        "${forgejo_ip}:${toString prometheus_exporter_port}"
-        "${runner01_ip}:${toString prometheus_exporter_port}"
+  monitoring_infra_job = {
+    job_name = "monitoring_infra";
+    static_configs = [
+      {
+        targets = [
+          "${monitor_hostname}:${toString prometheus_port}"
+          "${monitor_hostname}:${toString alertmanager_port}"
+          "${monitor_hostname}:${toString grafana_port}"
+        ];
+      }
     ];
-    promtail = [
-        "${monitor_ip}:${toString promtail_port}"
-        "${traefik_ip}:${toString promtail_port}"
-        "${sandbox_ip}:${toString promtail_port}"
-        "${forgejo_ip}:${toString promtail_port}"
-        "${runner01_ip}:${toString promtail_port}"
-    ];
-    grafana = [ "${monitor_ip}:${toString grafana_port}" ];
-    prometheus = [ "${monitor_ip}:${toString prometheus_port}" ];
-    alertmanager = [ "${monitor_ip}:${toString alertmanager_port}" ];
-    traefik = [ "${traefik_ip}:${toString traefik_monitor_port}" ];
-    gitea = [ "${forgejo_ip}:${toString forgejo_monitor_port}" ];
-    postgres = [ "${forgejo_ip}:${toString postgres_exporter_port}" ];
-
-    dnsmasq = [ "${dnsmasq_ip}:${toString dnsmasq_exporter_port}" ];
+    relabel_configs = instance_relabel_config;
   };
 
+  traefik_monitor_port = 8082;
+  traefik_job = {
+    job_name = "traefik";
+    static_configs = [{targets = ["${traefik_hostname}:${toString traefik_monitor_port}"];}];
+    relabel_configs = instance_relabel_config;
+  };
 
+  forgejo_monitor_port = 3000;
+  forgejo_job = {
+    job_name = "forgejo";
+    static_configs = [{targets = ["${forgejo_hostname}:${toString forgejo_monitor_port}"];}];
+    relabel_configs = instance_relabel_config;
+  };
+
+  postgres_exporter_port = 9187;
+  postgres_job = {
+    job_name = "postgres";
+    static_configs = [{targets = ["${forgejo_hostname}:${toString postgres_exporter_port}"];}];
+    relabel_configs = instance_relabel_config;
+  };
+
+  dnsmasq_exporter_port = 9153;
+  dnsmasq_job = {
+    job_name = "dnsmasq";
+    static_configs = [{targets = ["${dnsmasq_hostname}:${toString dnsmasq_exporter_port}"];}];
+    relabel_configs = instance_relabel_config;
+  };
+
+  # --- Media Stack Scrape Job ---
+  media_stack_job = {
+    job_name = "media_stack";
+    static_configs = [
+      {
+        targets = [
+          "${media_hostname}:9707" # sonarr
+          "${media_hostname}:9708" # readarr
+          "${media_hostname}:9709" # radarr
+          "${media_hostname}:9710" # prowlarr
+          "${media_hostname}:9711" # lidarr
+          "${media_hostname}:9712" # bazarr
+        ];
+      }
+    ];
+    relabel_configs = instance_relabel_config;
+  };
+
+  jellyfin_port = 8096;
+  jellyfin_exporter_port = 9594;
+  jellyfin_job = {
+    job_name = "jellyfin";
+    static_configs = [
+      {
+        targets = [
+          "${media_hostname}:${toString jellyfin_port}"
+          "${monitor_hostname}:${toString jellyfin_exporter_port}"
+        ];
+      }
+    ];
+    relabel_configs = instance_relabel_config;
+  };
 in {
-  networking.firewall.allowedTCPPorts = [ 9090 ];
+  networking.firewall.allowedTCPPorts = [9090];
 
   services.prometheus = {
     enable = true;
@@ -61,10 +147,17 @@ in {
       "--web.enable-admin-api"
     ];
 
-    scrapeConfigs = lib.mapAttrsToList (job_name: targets: {
-      inherit job_name;
-      static_configs = [ { inherit targets; } ];
-    }) exporters;
+    scrapeConfigs = [
+      node_exporter_job
+      promtail_job
+      monitoring_infra_job
+      traefik_job
+      forgejo_job
+      postgres_job
+      dnsmasq_job
+      media_stack_job
+      jellyfin_job
+    ];
 
     # 🔔 Alerts provisioning
     ruleFiles = [