From a39a314e4c3801fc5e34ee7e39ce49224391c4f7 Mon Sep 17 00:00:00 2001
From: Andrea Ciceri <andrea.ciceri@autistici.org>
Date: Thu, 25 Jul 2024 16:44:28 +0200
Subject: [PATCH] Prometheus pushgateway

---
 modules/prometheus-exporters/default.nix | 76 ++++++++++++++++--------
 modules/prometheus/default.nix           | 62 +++++++++++++++++++
 2 files changed, 112 insertions(+), 26 deletions(-)

diff --git a/modules/prometheus-exporters/default.nix b/modules/prometheus-exporters/default.nix
index ceaab99..a56d1cb 100644
--- a/modules/prometheus-exporters/default.nix
+++ b/modules/prometheus-exporters/default.nix
@@ -1,32 +1,56 @@
 {
   config,
   pkgs,
+  lib,
   ...
-}: {
-  services.prometheus.exporters.node = {
-    enable = true;
-    enabledCollectors = [
-      "cpu"
-      "conntrack"
-      "diskstats"
-      "entropy"
-      "filefd"
-      "filesystem"
-      "loadavg"
-      "mdadm"
-      "meminfo"
-      "netdev"
-      "netstat"
-      "stat"
-      "time"
-      "vmstat"
-      "systemd"
-      "logind"
-      "interrupts"
-      "ksmd"
-      "textfile"
-      "pressure"
-    ];
-    extraFlags = ["--collector.ethtool" "--collector.softirqs" "--collector.tcpstat" "--collector.wifi"];
+}: let
+  hostname = config.networking.hostName;
+  mkFor = hosts: lib.mkIf (builtins.elem hostname hosts);
+in {
+  services.prometheus.exporters = {
+    node = mkFor ["sisko" "picard"] {
+      enable = true;
+      enabledCollectors = [
+        "cpu"
+        "conntrack"
+        "diskstats"
+        "entropy"
+        "filefd"
+        "filesystem"
+        "loadavg"
+        "mdadm"
+        "meminfo"
+        "netdev"
+        "netstat"
+        "stat"
+        "time"
+        "vmstat"
+        "systemd"
+        "logind"
+        "interrupts"
+        "ksmd"
+        "textfile"
+        "pressure"
+      ];
+      extraFlags = ["--collector.ethtool" "--collector.softirqs" "--collector.tcpstat" "--collector.wifi"];
+    };
+    wireguard = mkFor ["sisko" "picard"] {
+      enable = true;
+    };
+    zfs = mkFor ["picard"] {
+      enable = true;
+    };
+    # restic = mkFor ["sisko"] {
+    #   enable = true;
+    # };
+    postgres = mkFor ["sisko"] {
+      enable = true;
+    };
+    nginx = mkFor ["sisko"] {
+      enable = true;
+    };
+    smartctl = mkFor ["sisko"] {
+      enable = true;
+    };
   };
 }
diff --git a/modules/prometheus/default.nix b/modules/prometheus/default.nix
index e77b7d6..c35b04e 100644
--- a/modules/prometheus/default.nix
+++ b/modules/prometheus/default.nix
@@ -3,6 +3,12 @@
 in {
   services.prometheus = {
     enable = true;
+    pushgateway = {
+      enable = true;
+      web = {
+        listen-address = "sisko.fleet:9094";
+      };
+    };
     checkConfig = false; # Otherwise it will fail because it cannot access bearer_token_file
     webExternalUrl = "https://status.aciceri.dev";
     globalConfig.scrape_interval = "10s";
@@ -17,6 +23,14 @@ in {
           }
         ];
       }
+      {
+        job_name = "pushgateway";
+        static_configs = [
+          {
+            targets = [cfg.pushgateway.web.listen-address];
+          }
+        ];
+      }
       {
         job_name = "node";
         static_configs = [
@@ -25,6 +39,54 @@ in {
           }
         ];
       }
+      {
+        job_name = "wireguard";
+        static_configs = [
+          {
+            targets = builtins.map (host: "${host}.fleet:9586") ["picard"];
+          }
+        ];
+      }
+      {
+        job_name = "zfs";
+        static_configs = [
+          {
+            targets = builtins.map (host: "${host}.fleet:9134") ["picard"];
+          }
+        ];
+      }
+      {
+        job_name = "restic";
+        static_configs = [
+          {
+            targets = builtins.map (host: "${host}.fleet:9753") ["sisko"];
+          }
+        ];
+      }
+      {
+        job_name = "postgres";
+        static_configs = [
+          {
+            targets = builtins.map (host: "${host}.fleet:9187") ["sisko"];
+          }
+        ];
+      }
+      {
+        job_name = "nginx";
+        static_configs = [
+          {
+            targets = builtins.map (host: "${host}.fleet:9117") ["sisko"];
+          }
+        ];
+      }
+      {
+        job_name = "smartctl";
+        static_configs = [
+          {
+            targets = builtins.map (host: "${host}.fleet:9633") ["sisko"];
+          }
+        ];
+      }
     ];
   };
   environment.persistence."/persist".directories = [