hydra, cgit and vm-sala on mothership

hosts/mothership/default.nix

@@ -13,6 +13,11 @@
     "mosh"
     "ccr"
     "nix"
+    "vm-sala"
+    "hydra"
+    "nix-serve"
+    "cgit"
+    "docker"
   ];
 
   ccr = {

modules/cgit/config.nix

@@ -0,0 +1,57 @@
+{
+  lib,
+  pkgs,
+  ...
+}: let
+  repos-path = "/var/lib/cgit-repos";
+  cgit-setup-repos =
+    pkgs.writers.writePython3 "cgit-setup-repos" {
+      libraries = with pkgs.python3Packages; [PyGithub];
+    } ''
+      from github import Github
+      from pathlib import Path
+
+      c = Path("${repos-path}")
+      c.unlink(missing_ok=True)
+
+      with open(c, "w") as f:
+          for repo in Github().get_user("aciceri").get_repos():
+              f.writelines([
+                  f"repo.url={repo.name}\n"
+                  f"repo.path=/home/ccr/projects/aciceri/{repo.name}/.git\n"
+                  f"repo.desc={repo.description}\n"
+              ])
+    '';
+in {
+  services.nginx.virtualHosts."git.aciceri.dev" = {
+    cgit = {
+      enable = true;
+      css = "/custom.css";
+      # scan-path = "/home/ccr/projects/aciceri";
+      virtual-root = "/";
+      cache-size = 1000;
+      include = [
+        (builtins.toString (pkgs.writeText "cgit-extra" ''
+          source-filter=${pkgs.cgit-pink}/lib/cgit/filters/syntax-highlighting.py
+          about-filter=${pkgs.cgit-pink}/lib/cgit/filters/about-formatting.sh
+        ''))
+        repos-path
+      ];
+    };
+    forceSSL = true;
+    enableACME = true;
+    # locations."/" = {
+    #   proxyPass = "http://127.0.0.1:${builtins.toString config.services.hydra.port}";
+    # };
+  };
+
+  systemd.services.cgit-setup-repos = {
+    description = "Update GitHub personal repos for cgit";
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+    };
+    wantedBy = ["multi-user.target"];
+    script = builtins.toString cgit-setup-repos;
+  };
+}

modules/cgit/default.nix

@@ -24,6 +24,11 @@ with lib; let
       pkgs.writeText name (lib.generators.toKeyValue {listsAsDuplicateKeys = true;} values);
   };
 in {
+  imports = [
+    ../nginx-base
+    ./config.nix
+  ];
+
   options.services.nginx.virtualHosts = mkOption {
     type = types.attrsOf (types.submodule ({config, ...}: let
       cfg = config.cgit;

modules/hydra/config.nix

@@ -0,0 +1,8 @@
+{
+  services.my-hydra.repos = {
+    emacs = {};
+    nixfleet = {};
+    trotten = {};
+    blog = {};
+  };
+}

modules/hydra/default.nix

@@ -0,0 +1,206 @@
+{
+  lib,
+  config,
+  pkgs,
+  ...
+}: let
+  cfg = config.services.my-hydra;
+  toSpec = {
+    name,
+    owner,
+    ...
+  }: let
+    spec = {
+      enabled = 1;
+      hidden = false;
+      description = "Declarative specification jobset automatically generated";
+      checkinterval = 120;
+      schedulingshares = 10000;
+      enableemail = false;
+      emailoverride = "";
+      keepnr = 1;
+      nixexprinput = "src";
+      nixexprpath = "jobsets.nix";
+      inputs = {
+        src = {
+          type = "path";
+          value = pkgs.writeTextFile {
+            name = "src";
+            text = builtins.readFile ./jobsets.nix;
+            destination = "/jobsets.nix";
+          };
+          emailresponsible = false;
+        };
+        repoInfoPath = {
+          type = "path";
+          value = pkgs.writeTextFile {
+            name = "repo";
+            text = builtins.toJSON {
+              inherit name owner;
+            };
+          };
+          emailresponsible = false;
+        };
+        prs = {
+          type = "githubpulls";
+          value = "${owner} ${name}";
+          emailresponsible = false;
+        };
+      };
+    };
+    drv = pkgs.writeTextFile {
+      name = "hydra-jobset-specification-${name}";
+      text = builtins.toJSON spec;
+      destination = "/spec.json";
+    };
+  in "${drv}";
+in {
+  imports = [
+    ./config.nix
+    ../nginx-base
+  ];
+
+  options.services.my-hydra = {
+    domain = lib.mkOption {
+      type = lib.types.str;
+      default = "hydra.aciceri.dev";
+    };
+    repos = lib.mkOption {
+      type = lib.types.attrsOf (lib.types.submodule ({
+        name,
+        config,
+        ...
+      }: {
+        options = {
+          name = lib.mkOption {
+            type = lib.types.str;
+            default = name;
+          };
+          owner = lib.mkOption {
+            type = lib.types.str;
+            default = "aciceri";
+          };
+          description = lib.mkOption {
+            type = lib.types.str;
+            default = config.homepage;
+          };
+          homepage = lib.mkOption {
+            type = lib.types.str;
+            default = "https://github.com/${config.owner}/${config.name}";
+          };
+          reportStatus = lib.mkOption {
+            type = lib.types.bool;
+            default = true;
+          };
+        };
+      }));
+      default = {};
+    };
+  };
+
+  config = {
+    # TODO manage `hydra` user ssh key declaratively
+    nix.extraOptions = ''
+      allowed-uris = https://github.com/ git://git.savannah.gnu.org/
+    '';
+
+    services.hydra = {
+      enable = true;
+      hydraURL = "https://${cfg.domain}";
+      notificationSender = "hydra@mothership.fleet";
+      buildMachinesFiles = [];
+      useSubstitutes = true;
+      extraConfig =
+        ''
+          <github_authorization>
+          include ${config.age.secrets.hydra-github-token.path}
+          </github_authorization>
+        ''
+        + (lib.concatMapStrings (repo:
+          lib.optionalString repo.reportStatus
+          ''
+            <githubstatus>
+              jobs = ${repo.name}.*
+              excludeBuildFromContext = 1
+              useShortContext = 1
+            </githubstatus>
+          '') (builtins.attrValues cfg.repos));
+    };
+
+    systemd.services.hydra-setup = {
+      description = "Hydra CI setup";
+      serviceConfig.Type = "oneshot";
+      serviceConfig.RemainAfterExit = true;
+      wantedBy = ["multi-user.target"];
+      requires = ["hydra-init.service"];
+      after = ["hydra-init.service"];
+      environment = builtins.removeAttrs (config.systemd.services.hydra-init.environment) ["PATH"];
+      script =
+        ''
+          PATH=$PATH:${lib.makeBinPath (with pkgs; [yq-go curl config.services.hydra.package])}
+          PASSWORD="$(cat ${config.age.secrets.hydra-admin-password.path})"
+          if [ ! -e ~hydra/.setup-is-complete ]; then
+            hydra-create-user admin \
+              --full-name "Andrea Ciceri" \
+              --email-address "andrea.ciceri@autistici.org" \
+              --password "$PASSWORD" \
+              --role admin
+            touch ~hydra/.setup-is-complete
+          fi
+
+          curl --head -X GET --retry 5 --retry-connrefused --retry-delay 1 http://localhost:3000
+
+          CURRENT_REPOS=$(curl -s -H "Accept: application/json" http://localhost:3000 | yq ".[].name")
+          DECLARED_REPOS="${lib.concatStringsSep " " (builtins.attrNames cfg.repos)}"
+
+          curl -H "Accept: application/json" \
+            -H 'Origin: http://localhost:3000' \
+            -H 'Content-Type: application/json' \
+            -d "{\"username\": \"admin\", \"password\": \"$PASSWORD\"}" \
+            --request "POST" localhost:3000/login \
+            --cookie-jar cookie
+
+          for repo in $CURRENT_REPOS; do
+            echo $repo
+            [[ ! "$DECLARED_REPOS" =~ (\ |^)$repo(\ |$) ]] && \
+              curl -H "Accept: application/json" \
+                --request "DELETE" \
+                --cookie cookie \
+                http://localhost:3000/project/$repo
+          done
+        ''
+        + lib.concatMapStrings (repo: ''
+          curl -H "Accept: application/json" \
+            -H 'Content-Type: application/json' \
+            --request "PUT" \
+            localhost:3000/project/${repo.name} \
+            --cookie cookie \
+            -d '{
+              "name": "${repo.name}",
+              "displayname": "${repo.name}",
+              "description": "${repo.description}",
+              "homepage": "${repo.homepage}",
+              "owner": "admin",
+              "enabled": true,
+              "visible": true,
+              "declarative": {
+                "file": "spec.json",
+                "type": "path",
+                "value": "${toSpec repo}"
+              }
+            }'
+        '') (builtins.attrValues cfg.repos)
+        + ''
+          rm cookie
+        '';
+    };
+
+    services.nginx.virtualHosts."${cfg.domain}" = {
+      forceSSL = true;
+      enableACME = true;
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:${builtins.toString config.services.hydra.port}";
+      };
+    };
+  };
+}

modules/hydra/jobsets.nix

@@ -0,0 +1,73 @@
+{
+  repoInfoPath,
+  prs,
+  ...
+}: let
+  minutes = 60;
+  hours = 60 * minutes;
+  days = 24 * hours;
+  filterAttrs = pred: set:
+    builtins.listToAttrs (builtins.concatMap (name: let
+      v = set.${name};
+    in
+      if pred name v
+      then [
+        {
+          inherit name;
+          value = v;
+        }
+      ]
+      else []) (builtins.attrNames set));
+  mapAttrs' = f: set:
+    builtins.listToAttrs (map (attr: f attr set.${attr}) (builtins.attrNames set));
+
+  mkJobset = {
+    enabled ? 1,
+    hidden ? false,
+    type ? 1,
+    description ? "",
+    checkinterval ? 5 * minutes,
+    schedulingshares ? 100,
+    enableemail ? false,
+    emailoverride ? "",
+    keepnr ? 1,
+    flake,
+  } @ args: {inherit enabled hidden type description checkinterval schedulingshares enableemail emailoverride keepnr flake;};
+
+  mkSpec = contents: let
+    escape = builtins.replaceStrings [''"''] [''\"''];
+    contentsJson = builtins.toJSON contents;
+  in
+    builtins.derivation {
+      name = "spec.json";
+      system = "x86_64-linux";
+      preferLocalBuild = true;
+      allowSubstitutes = false;
+      builder = "/bin/sh";
+      args = [
+        (builtins.toFile "builder.sh" ''
+          echo "${escape contentsJson}" > $out
+        '')
+      ];
+    };
+
+  repo = builtins.fromJSON (builtins.readFile repoInfoPath);
+
+  pullRequests = builtins.fromJSON (builtins.readFile prs);
+  pullRequestsToBuild = filterAttrs (n: pr: pr.head.repo != null && pr.head.repo.owner.login == repo.owner && pr.head.repo.name == repo.name) pullRequests;
+in {
+  jobsets = mkSpec ({
+      master = mkJobset {
+        description = "${repo.name}'s master branch";
+        flake = "git+ssh://git@github.com/${repo.owner}/${repo.name}?ref=master";
+      };
+    }
+    // (mapAttrs' (n: pr: {
+        name = "pullRequest_${n}";
+        value = mkJobset {
+          description = pr.title;
+          flake = "git+ssh://git@github.com/${repo.owner}/${repo.name}?ref=${pr.head.ref}";
+        };
+      })
+      pullRequests));
+}

modules/nginx-base/default.nix

@@ -0,0 +1,19 @@
+{
+  security.acme = {
+    acceptTerms = true;
+    defaults.email = "andrea.ciceri@autistici.org";
+  };
+
+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+  ];
+
+  services.nginx = {
+    enable = true;
+    recommendedGzipSettings = true;
+    recommendedOptimisation = true;
+    recommendedProxySettings = true;
+    recommendedTlsSettings = true;
+  };
+}

modules/nix-serve/default.nix

@@ -0,0 +1,29 @@
+{
+  config,
+  lib,
+  ...
+}: let
+  cfg = config.services.my-nix-serve;
+in {
+  imports = [../nginx-base];
+  options.services.my-nix-serve = {
+    domain = lib.mkOption {
+      type = lib.types.str;
+      default = "cache.aciceri.dev";
+    };
+  };
+  config = {
+    services.nix-serve = {
+      enable = true;
+      secretKeyFile = config.age.secrets.cache-private-key.path;
+      # Public key: cache.aciceri.dev:4e9sFjWPUOjGwTJE98PXinJJZLwPz0m5nKsAe63MY3E=
+    };
+    services.nginx.virtualHosts."${cfg.domain}" = {
+      forceSSL = true;
+      enableACME = true;
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:${builtins.toString config.services.nix-serve.port}";
+      };
+    };
+  };
+}

modules/vm-sala/default.nix

@@ -0,0 +1,87 @@
+{
+  pkgs,
+  lib,
+  fleetFlake,
+  ...
+}: {
+  security.polkit.enable = true;
+  virtualisation.libvirtd.enable = true;
+
+  networking.firewall.allowedTCPPorts = [
+    2222
+  ];
+
+  imports = [../nginx-base];
+
+  services.nginx.virtualHosts."git.slavni.aciceri.dev" = {
+    forceSSL = true;
+    enableACME = true;
+    locations."/" = {
+      proxyPass = "http://127.0.0.1:13000";
+    };
+  };
+
+  systemd.services.vm-sala = let
+    initial-config = fleetFlake.inputs.nixos-generators.nixosGenerate {
+      system = "x86_64-linux";
+      modules = [
+        fleetFlake.inputs.nixos-vscode-server.nixosModule
+        ({
+          modulesPath,
+          lib,
+          config,
+          ...
+        }: {
+          services.vscode-server = {
+            enable = true;
+            enableFHS = true;
+          };
+          system.build.qcow = lib.mkForce (import "${toString modulesPath}/../lib/make-disk-image.nix" {
+            inherit lib config pkgs;
+            diskSize = 50 * 1024;
+            format = "qcow2";
+            partitionTableType = "hybrid";
+          });
+          services.openssh.enable = true;
+          environment.systemPackages = with pkgs; [
+            vim
+            git
+            htop
+          ];
+          users.users.root = {
+            password = "password";
+            openssh.authorizedKeys.keys = [
+              (import "${fleetFlake.outPath}/lib").keys.users.ccr-ssh
+              "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC7qikwR0a4LDoMQIVvtX+gyJ41OsAOWe8RcXc4ksIBP9x1nCcSrItlgC2soADB77QGIgyeyLGmnTCtMj5/s8NdREAycPeXLii1WRakbT7oZ/hTEmvAgObpadeYJn3LhaUDNtmsnAqqh2pRpCpSsAdhfIt+YyV4VgRYSfaa12Ozp/H6NI9bJMNttmG8TmY9V4zyskV9bE+up9y8Yuck2bZV/GjQe6UWgxsiC3XPSrFFGuxyaFMRGsc8h86xVwTAmwaHESEFhRvHD4EtdPNss0jqmSI6m4AoSZQ2wq7eiH8ZiYzERF0FnEFf4UsyOTM7j78bfogNLfKrdcEIPLrNNFFc3Iarfe9CJn3DdSnwwPnhFU1MBBXSbGOp1IyN3+gpjHwLMPzozlDAVqOwx6XpnpF78VpeknFBHCbkcKC/R0MXzqf900wH3i2HvfB7v9e9EUFzCQ0vUC+1Og+BFw3F5VSo0QtZyLc4BJ/akBs5mEE6TnuWQa/GhlY8Lz7wbcV1AaBOAQdx+NTbL/+Q31SJ1XsXtGtXCrwMY9noUTyVfpGVXo7Mn4HSslmeQ9SKfYKjyetkBR/1f8a47O3rCggjBy1AlfLjgbERnXy+0Ma4T8lnPZAKt3s9Ya1JupZ7SO7D5j7WfPKP+60c372/RrX1wXsxEeLvBJ0jd8GnSCXDOuvHTQ=="
+            ];
+          };
+        })
+      ];
+      format = "qcow";
+    };
+    image = "${initial-config}/nixos.qcow2";
+    start-vm = pkgs.writeShellApplication {
+      name = "start-vm";
+      runtimeInputs = with pkgs; [qemu];
+      text = ''
+        [ ! -f /var/lib/vm-sala/nixos.qcow2 ] && \
+          install ${image} /var/lib/vm-sala
+
+        qemu-system-x86_64 \
+          -enable-kvm \
+          -cpu host \
+          -smp 2 \
+          -m 4096 \
+          -nic user,model=virtio-net-pci,hostfwd=tcp::2222-:22,hostfwd=tcp::13000-:3000 \
+          -nographic \
+          -drive file=/var/lib/vm-sala/nixos.qcow2
+      '';
+    };
+  in {
+    wantedBy = ["multi-user.target"];
+    after = ["network.target"];
+    serviceConfig = {
+      ExecStart = "${start-vm}/bin/start-vm";
+    };
+  };
+}