colmena initial implementation for sandbox and monitor

machines/monitor/prometheus.nix

@@ -0,0 +1,185 @@
+{
+  config,
+  pkgs,
+  modulesPath,
+  lib,
+  ...
+}: let
+  monitor_hostname = "monitor.lab";
+  traefik_hostname = "traefik.lab";
+  sandbox_hostname = "sandbox.lab";
+  forgejo_hostname = "forgejo.lab";
+  runner01_hostname = "forgejo-runner-01.lab";
+  dnsmasq_hostname = "dns.lab";
+  media_hostname = "media.lab";
+  mail_hostname = "mail.lab";
+  keycloak_hostname = "keycloak.lab";
+
+  monitored_hosts = [
+    monitor_hostname
+    traefik_hostname
+    sandbox_hostname
+    forgejo_hostname
+    runner01_hostname
+    dnsmasq_hostname
+    media_hostname
+    mail_hostname
+    keycloak_hostname
+  ];
+
+  # integrate colmena names and targetHost to generate nodeexporters
+  generateTargets = port:
+    map (host: "${host}:${toString port}") monitored_hosts;
+
+  instance_relabel_config = [
+    {
+      source_labels = ["__address__"];
+      regex = "([^:]+):\\d+"; # Captures everything before the last colon
+      target_label = "instance";
+      replacement = "$1";
+    }
+  ];
+
+  node_exporter_port = 9100;
+  node_exporter_job = {
+    job_name = "node";
+    static_configs = [{targets = generateTargets node_exporter_port;}];
+    relabel_configs = instance_relabel_config;
+  };
+
+  promtail_port = 9080;
+  promtail_job = {
+    job_name = "promtail";
+    static_configs = [{targets = generateTargets promtail_port;}];
+    relabel_configs = instance_relabel_config;
+  };
+
+  prometheus_target = "${monitor_hostname}:9090";
+  prometheus_job = {
+    job_name = "prometheus";
+    static_configs = [{targets = [prometheus_target];}];
+    relabel_configs = instance_relabel_config;
+  };
+
+  alertmanager_target = "${monitor_hostname}:9093";
+  alertmanager_job = {
+    job_name = "alertmanager";
+    static_configs = [{targets = [alertmanager_target];}];
+    relabel_configs = instance_relabel_config;
+  };
+
+  grafana_target = "${monitor_hostname}:3000";
+  grafana_job = {
+    job_name = "grafana";
+    static_configs = [{targets = [grafana_target];}];
+    relabel_configs = instance_relabel_config;
+  };
+
+  traefik_monitor_port = 8082;
+  traefik_job = {
+    job_name = "traefik";
+    static_configs = [{targets = ["${traefik_hostname}:${toString traefik_monitor_port}"];}];
+    relabel_configs = instance_relabel_config;
+  };
+
+  forgejo_monitor_port = 3000;
+  forgejo_job = {
+    job_name = "forgejo";
+    static_configs = [{targets = ["${forgejo_hostname}:${toString forgejo_monitor_port}"];}];
+    relabel_configs = instance_relabel_config;
+  };
+
+  postgres_exporter_port = 9187;
+  postgres_job = {
+    job_name = "postgres";
+    static_configs = [{targets = ["${forgejo_hostname}:${toString postgres_exporter_port}"];}];
+    relabel_configs = instance_relabel_config;
+  };
+
+  dnsmasq_exporter_port = 9153;
+  dnsmasq_job = {
+    job_name = "dnsmasq";
+    static_configs = [{targets = ["${dnsmasq_hostname}:${toString dnsmasq_exporter_port}"];}];
+    relabel_configs = instance_relabel_config;
+  };
+
+  # --- Media Stack Scrape Job ---
+  media_stack_job = {
+    job_name = "media_stack";
+    static_configs = [
+      {
+        targets = [
+          "${media_hostname}:9707" # sonarr
+          "${media_hostname}:9708" # readarr
+          "${media_hostname}:9709" # radarr
+          "${media_hostname}:9710" # prowlarr
+          "${media_hostname}:9711" # lidarr
+          "${media_hostname}:9712" # bazarr
+        ];
+      }
+    ];
+    relabel_configs = instance_relabel_config;
+  };
+
+  jellyfin_port = 8096;
+  jellyfin_exporter_port = 9594;
+  jellyfin_job = {
+    job_name = "jellyfin";
+    static_configs = [
+      {
+        targets = [
+          "${media_hostname}:${toString jellyfin_port}"
+          "${monitor_hostname}:${toString jellyfin_exporter_port}"
+        ];
+      }
+    ];
+    relabel_configs = instance_relabel_config;
+  };
+in {
+  networking.firewall.allowedTCPPorts = [9090];
+
+  services.prometheus = {
+    enable = true;
+    retentionTime = "7d";
+    globalConfig = {
+      scrape_timeout = "10s";
+      scrape_interval = "30s";
+      # A short evaluation_interval will check alerting rules very often.
+      # It can be costly if you run Prometheus with 100+ alerts.
+      evaluation_interval = "20s";
+    };
+    extraFlags = [
+      "--web.enable-admin-api"
+    ];
+
+    scrapeConfigs = [
+      node_exporter_job
+      promtail_job
+      prometheus_job
+      alertmanager_job
+      grafana_job
+      traefik_job
+      forgejo_job
+      postgres_job
+      dnsmasq_job
+      media_stack_job
+      jellyfin_job
+    ];
+
+    alertmanagers = [
+      {
+        scheme = "http";
+        static_configs = [{targets = [alertmanager_target];}];
+      }
+    ];
+
+    ruleFiles = [
+      (pkgs.writeText "prometheus-alerts.yml" (builtins.readFile ./provisioning/alerts/prometheus-alerts.yml))
+      (pkgs.writeText "loki-alerts.yml" (builtins.readFile ./provisioning/alerts/loki-alerts.yml))
+      (pkgs.writeText "promtail-alerts.yml" (builtins.readFile ./provisioning/alerts/promtail-alerts.yml))
+      (pkgs.writeText "postgres-alerts.yml" (builtins.readFile ./provisioning/alerts/postgres-alerts.yml))
+      (pkgs.writeText "traefik-alerts.yml" (builtins.readFile ./provisioning/alerts/traefik-alerts.yml))
+      (pkgs.writeText "node-exporter-alerts.yml" (builtins.readFile ./provisioning/alerts/node-exporter-alerts.yml))
+    ];
+  };
+}