Prometheus pushgateway

modules/prometheus-exporters/default.nix

@@ -1,9 +1,14 @@
 {
   config,
   pkgs,
+  lib,
   ...
-}: {
+}: let
-  services.prometheus.exporters.node = {
+  hostname = config.networking.hostName;
+  mkFor = hosts: lib.mkIf (builtins.elem hostname hosts);
+in {
+  services.prometheus.exporters = {
+    node = mkFor ["sisko" "picard"] {
       enable = true;
       enabledCollectors = [
         "cpu"
@@ -29,4 +34,23 @@
       ];
       extraFlags = ["--collector.ethtool" "--collector.softirqs" "--collector.tcpstat" "--collector.wifi"];
     };
+    wireguard = mkFor ["sisko" "picard"] {
+      enable = true;
+    };
+    zfs = mkFor ["picard"] {
+      enable = true;
+    };
+    # restic = mkFor ["sisko"] {
+    #   enable = true;
+    # };
+    postgres = mkFor ["sisko"] {
+      enable = true;
+    };
+    nginx = mkFor ["sisko"] {
+      enable = true;
+    };
+    smartctl = mkFor ["sisko"] {
+      enable = true;
+    };
+  };
 }

modules/prometheus/default.nix

@@ -3,6 +3,12 @@
 in {
   services.prometheus = {
     enable = true;
+    pushgateway = {
+      enable = true;
+      web = {
+        listen-address = "sisko.fleet:9094";
+      };
+    };
     checkConfig = false; # Otherwise it will fail because it cannot access bearer_token_file
     webExternalUrl = "https://status.aciceri.dev";
     globalConfig.scrape_interval = "10s";
@@ -17,6 +23,14 @@ in {
           }
         ];
       }
+      {
+        job_name = "pushgateway";
+        static_configs = [
+          {
+            targets = [cfg.pushgateway.web.listen-address];
+          }
+        ];
+      }
       {
         job_name = "node";
         static_configs = [
@@ -25,6 +39,54 @@ in {
           }
         ];
       }
+      {
+        job_name = "wireguard";
+        static_configs = [
+          {
+            targets = builtins.map (host: "${host}.fleet:9586") ["picard"];
+          }
+        ];
+      }
+      {
+        job_name = "zfs";
+        static_configs = [
+          {
+            targets = builtins.map (host: "${host}.fleet:9134") ["picard"];
+          }
+        ];
+      }
+      {
+        job_name = "restic";
+        static_configs = [
+          {
+            targets = builtins.map (host: "${host}.fleet:9753") ["sisko"];
+          }
+        ];
+      }
+      {
+        job_name = "postgres";
+        static_configs = [
+          {
+            targets = builtins.map (host: "${host}.fleet:9187") ["sisko"];
+          }
+        ];
+      }
+      {
+        job_name = "nginx";
+        static_configs = [
+          {
+            targets = builtins.map (host: "${host}.fleet:9117") ["sisko"];
+          }
+        ];
+      }
+      {
+        job_name = "smartctl";
+        static_configs = [
+          {
+            targets = builtins.map (host: "${host}.fleet:9633") ["sisko"];
+          }
+        ];
+      }
     ];
   };
   environment.persistence."/persist".directories = [