diff --git a/hosts/default.nix b/hosts/default.nix index 8adb65e..27ff71f 100644 --- a/hosts/default.nix +++ b/hosts/default.nix @@ -172,6 +172,8 @@ "hercules-ci-secrets-json".owner = "hercules-ci-agent"; "cachix-personal-token".owner = "ccr"; "home-planimetry".owner = "hass"; + "home-assistant-token".owner = "prometheus"; + "grafana-password".owner = "grafana"; "cloudflare-dyndns-api-token" = {}; "restic-hetzner-password" = {}; # "minio-credentials".owner = "minio"; diff --git a/hosts/picard/default.nix b/hosts/picard/default.nix index 3dc217d..97be64f 100644 --- a/hosts/picard/default.nix +++ b/hosts/picard/default.nix @@ -41,6 +41,7 @@ "mount-rock5b" "adb" "guix" + "prometheus-exporters" ] ++ [ ./disko.nix diff --git a/hosts/sisko/default.nix b/hosts/sisko/default.nix index 6ec4fd2..8aba34a 100644 --- a/hosts/sisko/default.nix +++ b/hosts/sisko/default.nix @@ -29,13 +29,16 @@ "forgejo" # # "jellyfin" # "immich" + "prometheus" + "grafana" + "prometheus-exporters" + "loki" + "promtail" ] ++ [ ./disko.nix ]; - # FIXME why is this needed? - nixpkgs.config.permittedInsecurePackages = ["openssl-1.1.1w"]; # boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_8; # boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_testing; boot.kernelPackages = let @@ -94,6 +97,7 @@ "/var/log" "/var/lib/containers" "/var/lib/postgresql" + "/home/${config.ccr.username}/.ssh" ]; files = [ "/etc/machine-id" diff --git a/modules/cloudflare-dyndns/default.nix b/modules/cloudflare-dyndns/default.nix index 0e6be46..10fe6dc 100644 --- a/modules/cloudflare-dyndns/default.nix +++ b/modules/cloudflare-dyndns/default.nix @@ -16,6 +16,7 @@ "syncv3.matrix.aciceri.dev" "jellyfin.aciceri.dev" "photos.aciceri.dev" + "status.aciceri.dev" ]; apiTokenFile = config.age.secrets.cloudflare-dyndns-api-token.path; }; diff --git a/modules/grafana/default.nix b/modules/grafana/default.nix new file mode 100644 index 0000000..98624b8 --- /dev/null +++ b/modules/grafana/default.nix @@ -0,0 +1,30 @@ +{config, ...}: let + cfg = config.services.grafana; +in { + services.grafana = { + enable = true; + settings = { + server = { + domain = "status.aciceri.dev"; + http_addr = "127.0.0.1"; + http_port = 2342; + root_url = "https://${config.services.grafana.settings.server.domain}:443/"; + }; + security = { + admin_user = "andrea"; + admin_password = "$__file{${config.age.secrets.grafana-password.path}}"; + }; + }; + }; + environment.persistence."/persist".directories = [ + cfg.dataDir + ]; + + services.nginx.virtualHosts = { + "status.aciceri.dev" = { + enableACME = true; + forceSSL = true; + locations."/".proxyPass = "http://127.0.0.1:${builtins.toString cfg.settings.server.http_port}"; + }; + }; +} diff --git a/modules/home-assistant/default.nix b/modules/home-assistant/default.nix index b21f37c..e40f0ed 100644 --- a/modules/home-assistant/default.nix +++ b/modules/home-assistant/default.nix @@ -94,6 +94,7 @@ in { "media_player" "wyoming" "wake_on_lan" + "prometheus" ]; extraPackages = python3Packages: with python3Packages; [ @@ -148,6 +149,9 @@ in { ]; shell_command.turn_off_picard = ''${pkgs.openssh}/bin/ssh -i /var/lib/hass/.ssh/id_ed25519 -o StrictHostKeyChecking=no hass@picard.fleet "exec sudo \$(readlink \$(which systemctl)) poweroff"''; # shell_command.turn_off_picard = ''whoami''; + prometheus = { + namespace = "hass"; + }; }; }; diff --git a/modules/loki/default.nix b/modules/loki/default.nix new file mode 100644 index 0000000..79d9738 --- /dev/null +++ b/modules/loki/default.nix @@ -0,0 +1,67 @@ +{config, ...}: let + cfg = config.services.loki; +in { + services.loki = { + enable = true; + configuration = { + # Basic stuff + auth_enabled = false; + server = { + http_listen_port = 3100; + log_level = "warn"; + }; + common = { + path_prefix = config.services.loki.dataDir; + storage.filesystem = { + chunks_directory = "${cfg.dataDir}/chunks"; + rules_directory = "${cfg.dataDir}/rules"; + }; + replication_factor = 1; + ring.kvstore.store = "inmemory"; + ring.instance_addr = "127.0.0.1"; + }; + + ingester.chunk_encoding = "snappy"; + + limits_config = { + retention_period = "120h"; + ingestion_burst_size_mb = 16; + reject_old_samples = true; + reject_old_samples_max_age = "12h"; + }; + + table_manager = { + retention_deletes_enabled = true; + retention_period = "120h"; + }; + + compactor = { + retention_enabled = true; + compaction_interval = "10m"; + working_directory = "${cfg.dataDir}/compactor"; + delete_request_cancel_period = "10m"; # don't wait 24h before processing the delete_request + retention_delete_delay = "2h"; + retention_delete_worker_count = 150; + delete_request_store = "filesystem"; + }; + + schema_config.configs = [ + { + from = "2020-11-08"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index.prefix = "index_"; + index.period = "24h"; + } + ]; + + query_range.cache_results = true; + limits_config.split_queries_by_interval = "24h"; + }; + }; + + environment.persistence."/persist".directories = [ + cfg.dataDir + ]; +} diff --git a/modules/prometheus-exporters/default.nix b/modules/prometheus-exporters/default.nix new file mode 100644 index 0000000..ceaab99 --- /dev/null +++ b/modules/prometheus-exporters/default.nix @@ -0,0 +1,32 @@ +{ + config, + pkgs, + ... +}: { + services.prometheus.exporters.node = { + enable = true; + enabledCollectors = [ + "cpu" + "conntrack" + "diskstats" + "entropy" + "filefd" + "filesystem" + "loadavg" + "mdadm" + "meminfo" + "netdev" + "netstat" + "stat" + "time" + "vmstat" + "systemd" + "logind" + "interrupts" + "ksmd" + "textfile" + "pressure" + ]; + extraFlags = ["--collector.ethtool" "--collector.softirqs" "--collector.tcpstat" "--collector.wifi"]; + }; +} diff --git a/modules/prometheus/default.nix b/modules/prometheus/default.nix new file mode 100644 index 0000000..e77b7d6 --- /dev/null +++ b/modules/prometheus/default.nix @@ -0,0 +1,33 @@ +{config, ...}: let + cfg = config.services.prometheus; +in { + services.prometheus = { + enable = true; + checkConfig = false; # Otherwise it will fail because it cannot access bearer_token_file + webExternalUrl = "https://status.aciceri.dev"; + globalConfig.scrape_interval = "10s"; + scrapeConfigs = [ + { + job_name = "hass"; + metrics_path = "/api/prometheus"; + bearer_token_file = config.age.secrets.home-assistant-token.path; + static_configs = [ + { + targets = ["sisko.fleet:${builtins.toString config.services.home-assistant.config.http.server_port}"]; + } + ]; + } + { + job_name = "node"; + static_configs = [ + { + targets = builtins.map (host: "${host}.fleet:9100") ["sisko" "picard"]; + } + ]; + } + ]; + }; + environment.persistence."/persist".directories = [ + "/var/lib/${cfg.stateDir}" + ]; +} diff --git a/modules/promtail/default.nix b/modules/promtail/default.nix new file mode 100644 index 0000000..74f2eec --- /dev/null +++ b/modules/promtail/default.nix @@ -0,0 +1,54 @@ +{ + pkgs, + lib, + config, + ... +}: let + conf = { + server = { + http_listen_port = 28183; + grpc_listen_port = 0; + }; + clients = [ + { + url = "http://sisko.fleet:${builtins.toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push"; + } + ]; + positions = { + filename = "/tmp/positions.yaml"; + }; + scrape_configs = [ + { + job_name = "journal"; + journal = { + max_age = "12h"; + labels = { + job = "systemd-journal"; + host = config.networking.hostName; + }; + }; + relabel_configs = [ + { + source_labels = ["__journal__systemd_unit"]; + target_label = "unit"; + } + ]; + } + ]; + }; + configFile = pkgs.writeTextFile { + name = "promtail.yaml"; + text = lib.generators.toYAML {} conf; + }; +in { + systemd.services.promtail = { + description = "Promtail service for Loki"; + wantedBy = ["multi-user.target"]; + + serviceConfig = { + ExecStart = '' + ${pkgs.grafana-loki}/bin/promtail --config.file ${configFile} + ''; + }; + }; +} diff --git a/modules/promtail/protmail.yaml b/modules/promtail/protmail.yaml new file mode 100644 index 0000000..e69de29 diff --git a/secrets/grafana-password.age b/secrets/grafana-password.age new file mode 100644 index 0000000..b042c07 --- /dev/null +++ b/secrets/grafana-password.age @@ -0,0 +1,29 @@ +age-encryption.org/v1 +-> ssh-rsa /AagBw +TevAEnbonePC8z7uK+0ywO2fOZ2wqerIuF3jS3dGSNJLG3xn+92lHiBbVxc0uza7 +LcetZrcEkqoAqo1CGWnUlvmxm/MYF+bGKsM7wgXoi54XIBXTzcSFiK8Il+htNQsH +l11yN4OfCLlG8YT7Yr/1oZHd/k1sSaDW//3tpM4yftZhjZub/Q5m9LAUYZ3fWbPO +lS7AcDwgYPx0wERlfms9t+n9z62qev3leSuQLvTYwktfgAB7XFEIBI2DHZYIZ7IM +gUVXkBHZbMCrxS08b5IP2R7ajiV4RUz8c36TbZY4WWXId3eUkiV1HERzSDmzxtCH +LqZ+GCQRA33g5kXP/5Lxwknr9J4bdk6sBrFl28nawaUyKz7yCBZBZ7hwrvxXJkjS +zOZ+DLSHKqJXYWD1Juq5QtPhsimTr1FhA9ibm6OtjmtMg+TaJpVwXIR0wWrKXSfs +u+kKUWQMFRB07iA+ho86BRvIkhOt8EsrrwXtAjKXCUVm0D9HPfj6R6tzAMT/5Obb +6SmRmlhiJlNB2eqocaTuZEtiHPVyFNFWlYUqaw4wncogVS+Tc8hEuSuZB6kX9nCt +B473PEsqAl5cjkJOFy6VFIdce0S3gflcRceKd7v6Yzvu+UXZbwCOQC75jidsDoJj +6aBfGDhNtTt5zyvuiktlHXpamOKFd1LJPqMfjzM7ka0 +-> ssh-rsa QHr3/A +J5pKmu/0IK971olnG7abbezq3wC/izp9i1+TRwwRdMIWdJEXIeNDkWbZa8IzOFwM +1Dz+g4PiIJf1Rmfij+8fXodoTIkh0oF3Nr3bx+3IkGmUfV56SRlLXZQBJsT1qQQ6 +ejPhJwvtcOZbbRNECkLeaRErAS6rfNWXc0dn6q9WzBOG8pIXUgFjvP3ak7LpCBz9 +yuJvFF1uExydOf4xM9B17ar72qEdMksWSiCsld8wpAPyxkMeaJpDSXc50sYFt2P9 +tqT8LGS/cHwhlKMzywbYRClp6tXmV5lit+MEucodOQLgFVk/tOySOPeMfjgud++F +yWcYGVZxuine00trnSwuUHTJVvRWJ7xUUdX1Gi/ZPb6SeJaIcWB/VA0e3OvST5J3 +0Zpma/ZqD61ivL6yaj49xiz+sT74tHN8+duPcbiaHMb3DkmjNmptI5x1fJx/k7Dp +WC4Sy4s86tSbrebRrJU3oDGEaXPeJS40QDRhhNTvhOPOOR6K724IWAAFABiHVWOd +o4lXaL3wUaLrgbKIX8f7gwKsGjziMW43HicpnaLOH30OeU4mDtRPRVEboJ5kbsFj +uup5YILnAuhtVjrt+vL0hVbQf0Ll2678jneuu9p816nYj1W58kG6jyGd5n+8jPPi +4g/COiYemaD/qnOvNc51E4vKieV99reaAwZtFV2HwQM +-> ssh-ed25519 +vdRnA yYxNN5g4AFwd+KobTSE6f+ROdXqwBRl9dmq6ZpilmA8 +oEKqsUekQMziv74yx5Y9M5MYy85LoFMFYM5rwUgdcSI +--- sKSqHbosmsAkMN3OUyVWcKMsqAmKkn9fcAi5kdNqDWY + 9WVrn&ю㢹_|Q[CWYذ \ No newline at end of file diff --git a/secrets/home-assistant-token.age b/secrets/home-assistant-token.age new file mode 100644 index 0000000..00542aa --- /dev/null +++ b/secrets/home-assistant-token.age @@ -0,0 +1,29 @@ +age-encryption.org/v1 +-> ssh-rsa /AagBw +Q2vgJ5hIP92ny8yk2gih/ZbmUpz+CdGMeQnX2ekORZAkWSuCQvGnBM9gAvRyhoa2 +2pmPOjP72HaL8fapcdZlcKHZ7IiQuT+3VWekQmIrrMSlSIwVnvasnOBInxA9f91k +hPR4Np6szQJjMLY7jsgwC0FrrUbgkZIVJp+y+zW+Vk09jdVb84vnBX0QFu/T01Qe +B0GfAmyZnWnxaKkNyNZb3afd+dpWZiZ79jnA4qOkjfXhhmiW2Ng23XdkHfjS8ekN +UptlWH8zbBwhzcFCGDkLEMs6/IewozqvzyS4eqmDfwj8saO78ue2R5pE2skNwq/q +EOneH592KqEIWNFgQPS6P4gjmyWtMnJsWm5SvEcg6mDaE1e6acJ7kb/kYI5zyK6y +PF7lzqsoge/+ptvbQxTwlvhhJIGob3Wqf2/soep+o6C4Vh87Wt2zhHmKR7Mt4Nrl +IIs94GU9SDTz1zDOjaBW/msPDagskVpQnu7z3p9iTnBKdxs4WEpugbPZTUVioChA +c3T8PxLy4pzhm88Rz1KNK9h++uTCPIooGOHVQj+WQpUf+ifWEWiwYEdeTbeutjgF +jz3Ntl9YOzm/w1JCzOVdOge7dvfx0J6qAFbf4OkcLhf8bv2ljSL429NH1lp3DCHY +qwN4wlKpOMGThw3pS2SVd6GqINwehrbS4OVobE/kVH8 +-> ssh-rsa QHr3/A +OjdbkVy7w5q3kxFm/4uMbjuIMyVuLSqSWQDvfE1T3vEwkIRJ1w8DOIwahO1qP6CL +oZOH9A5xnS+UcJOSdTqiN4hHC03uuxx1unh/CHPm6zsSksCDHkLvVVhHMaVINvQz +YtV/bek9AWdgT6zMke0pv+zzuCqFGLT7Og1k+aCHtECkF3mB6Etm2P7knggJ2BX3 +L9YzsxSO4jj0PBFGX6nHR71dfq2bctik+mKW8LjS1cQ6plOdEsNHUR7s0bwoslrA +KdD6WOHoEOlxfTLiJmNB0A5pZb+iXJOP+ygrpC6WfJHU8nEWZzglNaVqrv14pieH +uez3nji2ZRsfkeXZI4vQJJ8EQ4LJtNhqki9+AcDYxX8pPUQs3e77ytcMYUMkhZmf +p8rC8eXPP69vS+Ia1xyL2nGqPmggZh7iT1VKOC8kXcHX0UB9WZXcJqPeDtQUO6l9 +cbGFSF9GkWfuVya7tl7rnRQVQs0Ko6XjOAFiOF8WA0YFNACV/2wVawH4rJ6Km48z +Gv+65Zk3yCXP231saE/Ztv3W43XLiJVDuw2RlUFXpJarGqAAZBhSC4qtDAgzHYLU +CxPrRtoIzOMv4iTzQmjJQFpArOBXU0yWZkaVwn57w6jEzk9NyqTZ02Oxb7DwK528 +F/fQOyw1b3GYJY7igv8+KbB+Bup9QQuvHxuxpRaqnek +-> ssh-ed25519 +vdRnA 2bZe+2cbP6T+Aa1g4lWhnOLkJdT7YqfCxTLKZ6wOKhw +i0+UOKIioQz01GfATEmNZVdGeIM2QRIiaUyRdqTgXCM +--- +1qA7qnzAo1u6/yQytQoq7MsZ7owcIa5uAqdg8UQ3tY +q|Ti'ګ^%;^4ǁ‚}+%ϡ ~آE>р0%gEMg.FPWǭ \$Y;2^;LiXM@%\Vࢼs$sXԬ̢|尹\Jܾ3!NC)ƿrQƟAQO0 ã( \ No newline at end of file diff --git a/secrets/secrets.nix b/secrets/secrets.nix index 0e092b5..8f3c4ef 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -17,10 +17,12 @@ in "aws-credentials.age".publicKeys = [ccr-ssh ccr-gpg picard sisko]; "nextcloud-admin-pass.age".publicKeys = [ccr-ssh ccr-gpg sisko]; "home-planimetry.age".publicKeys = [ccr-ssh ccr-gpg sisko]; + "home-assistant-token.age".publicKeys = [ccr-ssh ccr-gpg sisko]; "chatgpt-token.age".publicKeys = [ccr-ssh ccr-gpg kirk mothership picard deltaflyer]; "cloudflare-dyndns-api-token.age".publicKeys = [ccr-ssh ccr-gpg sisko]; "restic-hetzner-password.age".publicKeys = [ccr-ssh ccr-gpg picard sisko kirk]; "hass-ssh-key.age".publicKeys = [ccr-ssh ccr-gpg sisko]; + "grafana-password.age".publicKeys = [ccr-ssh ccr-gpg sisko]; "matrix-registration-shared-secret.age".publicKeys = [ccr-ssh ccr-gpg sisko]; "matrix-sliding-sync-secret.age".publicKeys = [ccr-ssh ccr-gpg sisko]; "forgejo-runners-token.age".publicKeys = [ccr-ssh ccr-gpg picard];