checks/default.nix

@@ -6,6 +6,8 @@ let
 in
 {
   x86_64-linux = lib.mergeAttrsList [
+    flake-inputs.self.nixosConfigurations.hetzner-1.config.serviceTests
+
     {
       nix = checkLib.mkLint {
         name = "nix-lints";

configuration/default.nix

@@ -1,5 +1,5 @@
 {
-  config,
+  lib,
   modulesPath,
   flake-inputs,
   ...
@@ -9,30 +9,13 @@
     flake-inputs.disko.nixosModules.disko
     flake-inputs.sops-nix.nixosModules.sops
     flake-inputs.tlaternet-webserver.nixosModules.default
-
     "${modulesPath}/profiles/minimal.nix"
-    (import ../modules)
 
-    ./services/backups.nix
-    ./services/battery-manager.nix
-    ./services/conduit
-    ./services/crowdsec.nix
-    ./services/foundryvtt.nix
-    ./services/gitea.nix
-    ./services/immich.nix
-    ./services/metrics
-    ./services/minecraft.nix
-    ./services/nextcloud.nix
-    ./services/webserver.nix
-    ./services/wireguard.nix
-    # ./services/starbound.nix -- Not currently used
-    ./services/postgres.nix
+    ../modules
     ./nginx
-    ./sops.nix
+    ./services
   ];
 
-  nixpkgs.overlays = [ (_: prev: { local = import ../pkgs { pkgs = prev; }; }) ];
-
   nix = {
     extraOptions = ''
       experimental-features = nix-command flakes
@@ -42,49 +25,9 @@
     settings.trusted-users = [ "@wheel" ];
   };
 
-  # Optimization for minecraft servers, see:
-  # https://bugs.mojang.com/browse/MC-183518
-  boot.kernelParams = [
-    "highres=off"
-    "nohz=off"
-  ];
-
   networking = {
     usePredictableInterfaceNames = false;
     useDHCP = false;
-
-    firewall = {
-      allowedTCPPorts = [
-        # http
-        80
-        443
-        # ssh
-        2222
-        # matrix
-        8448
-        # starbound
-        21025
-
-        config.services.coturn.listening-port
-        config.services.coturn.tls-listening-port
-        config.services.coturn.alt-listening-port
-        config.services.coturn.alt-tls-listening-port
-      ];
-
-      allowedUDPPorts = [
-        config.services.coturn.listening-port
-        config.services.coturn.tls-listening-port
-        config.services.coturn.alt-listening-port
-        config.services.coturn.alt-tls-listening-port
-      ];
-
-      allowedUDPPortRanges = [
-        {
-          from = config.services.coturn.min-port;
-          to = config.services.coturn.max-port;
-        }
-      ];
-    };
   };
 
   systemd.network.enable = true;
@@ -124,9 +67,10 @@
       services.sudo.rssh = true;
     };
   };
+  sops.defaultSopsFile = ../keys/production.yaml;
 
   # Remove some unneeded packages
-  environment.defaultPackages = [ ];
+  environment.defaultPackages = lib.mkForce [ ];
 
   system.stateVersion = "20.09";
 }

configuration/nginx/logging.nix

@@ -1,4 +1,10 @@
-{ config, lib, ... }:
+{
+  flake-inputs,
+  pkgs,
+  config,
+  lib,
+  ...
+}:
 let
   hostNames = lib.attrNames config.services.nginx.virtualHosts;
   logPath = name: "/var/log/nginx/${name}/access.log";
@@ -80,5 +86,55 @@ in
         };
       };
     };
+
+    serviceTests =
+      let
+        testHostConfig =
+          { config, ... }:
+          {
+            imports = [
+              ./.
+              ../../modules/serviceTests/mocks.nix
+            ];
+
+            networking.firewall.allowedTCPPorts = [ 80 ];
+
+            services.nginx = {
+              domain = "testHost";
+              virtualHosts."${config.services.nginx.domain}".locations."/".return = "200 ok";
+            };
+          };
+      in
+      {
+        nginxMetricsWork = pkgs.testers.runNixOSTest {
+          name = "nginx-metrics-work";
+          node.specialArgs = { inherit flake-inputs; };
+
+          nodes = {
+            testHost = testHostConfig;
+
+            client =
+              { pkgs, ... }:
+              {
+                environment.systemPackages = [ pkgs.curl ];
+              };
+          };
+
+          testScript = ''
+            import time
+
+            start_all()
+
+            testHost.wait_for_unit("nginx.service")
+            client.succeed("curl --max-time 10 http://testHost")
+
+            # Wait a bit for the prometheus exporter to scrape our logs
+            time.sleep(5)
+
+            res = testHost.succeed("curl localhost:${builtins.toString config.services.prometheus.exporters.nginxlog.port}/metrics")
+            assert 'nginxlog_http_response_count_total{method="GET",status="200",vhost="testHost"} 1' in res, res
+          '';
+        };
+      };
   };
 }

configuration/nginx/ssl.nix

@@ -1,4 +1,5 @@
 {
+  flake-inputs,
   pkgs,
   config,
   lib,
@@ -64,5 +65,73 @@
         in
         ''${pkgs.runtimeShell} -c '${confirm}' '';
     };
+
+    sops.secrets = {
+      "porkbun/api-key".owner = "acme";
+      "porkbun/secret-api-key".owner = "acme";
+    };
+
+    serviceTests =
+      let
+        testHostConfig =
+          { config, ... }:
+          {
+            imports = [
+              ./.
+              ../../modules/serviceTests/mocks.nix
+            ];
+
+            networking.firewall.allowedTCPPorts = [ 443 ];
+
+            security.acme.certs."tlater.net".extraDomainNames = [ config.services.nginx.domain ];
+
+            services.nginx = {
+              domain = "testHost";
+
+              virtualHosts."${config.services.nginx.domain}" = {
+                useACMEHost = "tlater.net";
+                onlySSL = true;
+                enableHSTS = true;
+                locations."/".return = "200 ok";
+              };
+            };
+          };
+      in
+      {
+        testNginxSSL = pkgs.testers.runNixOSTest {
+          name = "test-nginx-ssl";
+
+          node.specialArgs = { inherit flake-inputs; };
+          nodes = {
+            testHost = testHostConfig;
+
+            client =
+              { pkgs, ... }:
+              {
+                environment.systemPackages = [ pkgs.curl ];
+              };
+          };
+
+          testScript = ''
+            start_all()
+
+            testHost.wait_for_unit("nginx.service")
+            testHost.copy_from_vm("/var/lib/acme/tlater.net/", "certs")
+            client.copy_from_host(f"{testHost.out_dir}/certs", "/certs")
+
+            res = client.succeed(" ".join([
+                "curl",
+                "--show-error",
+                "--silent",
+                "--dump-header -",
+                "--cacert /certs/tlater.net/fullchain.pem",
+                "https://testHost",
+                "-o /dev/null"
+            ]))
+
+            assert "strict-transport-security: max-age=15552000; includeSubDomains" in res
+          '';
+        };
+      };
   };
 }

configuration/services/backups.nix

@@ -265,5 +265,18 @@ in
       };
       groups.backup = { };
     };
+
+    sops.secrets = {
+      "restic/storagebox-backups" = {
+        owner = "root";
+        group = "backup";
+        mode = "0440";
+      };
+      "restic/storagebox-ssh-key" = {
+        owner = "backup";
+        group = "backup";
+        mode = "0040";
+      };
+    };
   };
 }

configuration/services/battery-manager.nix

@@ -13,4 +13,9 @@
       log_level = "DEBUG";
     };
   };
+
+  sops.secrets = {
+    "battery-manager/email" = { };
+    "battery-manager/password" = { };
+  };
 }

configuration/services/conduit/default.nix

@@ -17,6 +17,36 @@ in
     ./matrix-hookshot.nix
   ];
 
+  networking.firewall = {
+    allowedTCPPorts = [
+      # These are for "normal" clients
+      80
+      443
+
+      # Federation happens on 8448
+      8448
+
+      config.services.coturn.listening-port
+      config.services.coturn.tls-listening-port
+      config.services.coturn.alt-listening-port
+      config.services.coturn.alt-tls-listening-port
+    ];
+
+    allowedUDPPorts = [
+      config.services.coturn.listening-port
+      config.services.coturn.tls-listening-port
+      config.services.coturn.alt-listening-port
+      config.services.coturn.alt-tls-listening-port
+    ];
+
+    allowedUDPPortRanges = [
+      {
+        from = config.services.coturn.min-port;
+        to = config.services.coturn.max-port;
+      }
+    ];
+  };
+
   services = {
     matrix-conduit = {
       enable = true;
@@ -179,4 +209,11 @@ in
   systemd.services.coturn.serviceConfig.SupplementaryGroups = [
     config.security.acme.certs."tlater.net".group
   ];
+
+  sops.secrets = {
+    "turn/env" = { };
+    "turn/secret" = {
+      owner = "turnserver";
+    };
+  };
 }

configuration/services/conduit/heisenbridge.nix

@@ -75,4 +75,10 @@ in
         # AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
       };
     };
+
+  sops.secrets = {
+    # Accessed via systemd cred through /run/secrets/heisebridge
+    "heisenbridge/as-token" = { };
+    "heisenbridge/hs-token" = { };
+  };
 }

configuration/services/conduit/matrix-hookshot.nix

@@ -163,4 +163,10 @@ in
       metrics.enabled = true;
     };
   };
+
+  sops.secrets = {
+    # Accessed via systemd cred through /run/secrets/matrix-hookshot
+    "matrix-hookshot/as-token" = { };
+    "matrix-hookshot/hs-token" = { };
+  };
 }

configuration/services/default.nix

@@ -0,0 +1,18 @@
+{
+  imports = [
+    ./backups.nix
+    ./battery-manager.nix
+    ./conduit
+    ./crowdsec.nix
+    ./foundryvtt.nix
+    ./gitea.nix
+    ./immich.nix
+    ./metrics
+    ./minecraft.nix
+    ./nextcloud.nix
+    ./postgres.nix
+    # ./starbound.nix -- Not currently used
+    ./webserver.nix
+    ./wireguard.nix
+  ];
+}

configuration/services/foundryvtt.nix

@@ -11,6 +11,11 @@ in
 {
   imports = [ flake-inputs.foundryvtt.nixosModules.foundryvtt ];
 
+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+  ];
+
   services = {
     foundryvtt = {
       enable = true;

configuration/services/gitea.nix

@@ -8,6 +8,11 @@ let
   domain = "gitea.${config.services.nginx.domain}";
 in
 {
+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+  ];
+
   services = {
     forgejo = {
       enable = true;

configuration/services/immich.nix

@@ -8,6 +8,11 @@ let
   hostName = "immich.${config.services.nginx.domain}";
 in
 {
+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+  ];
+
   services = {
     immich = {
       enable = true;

configuration/services/metrics/grafana.nix

@@ -3,6 +3,11 @@ let
   domain = "metrics.${config.services.nginx.domain}";
 in
 {
+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+  ];
+
   services.grafana = {
     enable = true;
     settings = {
@@ -67,4 +72,15 @@ in
       };
     };
   };
+
+  sops.secrets = {
+    "grafana/adminPassword" = {
+      owner = "grafana";
+      group = "grafana";
+    };
+    "grafana/secretKey" = {
+      owner = "grafana";
+      group = "grafana";
+    };
+  };
 }

configuration/services/metrics/victoriametrics.nix

@@ -4,7 +4,7 @@ let
   blackbox_port = config.services.prometheus.exporters.blackbox.port;
 in
 {
-  config.services.victoriametrics = {
+  services.victoriametrics = {
     enable = true;
     extraOptions = [ "-storage.minFreeDiskSpaceBytes=5GB" ];
 
@@ -96,4 +96,10 @@ in
       victorialogs.targets = [ config.services.victorialogs.bindAddress ];
     };
   };
+
+  sops.secrets."forgejo/metrics-token" = {
+    owner = "forgejo";
+    group = "metrics";
+    mode = "0440";
+  };
 }

configuration/services/nextcloud.nix

@@ -9,6 +9,11 @@ let
   hostName = "nextcloud.${config.services.nginx.domain}";
 in
 {
+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+  ];
+
   services = {
     nextcloud = {
       inherit hostName;
@@ -100,4 +105,9 @@ in
 
   # Ensure that this service doesn't start before postgres is ready
   systemd.services.nextcloud-setup.after = [ "postgresql.service" ];
+
+  sops.secrets."nextcloud/tlater" = {
+    owner = "nextcloud";
+    group = "nextcloud";
+  };
 }

configuration/services/starbound.nix

@@ -1,8 +1,15 @@
-{ pkgs, lib, ... }:
+{
+  flake-inputs,
+  pkgs,
+  lib,
+  ...
+}:
 let
   inherit (lib) concatStringsSep;
 in
 {
+  networking.firewall.allowedTCPPorts = [ 21025 ];
+
   # Sadly, steam-run requires some X libs
   environment.noXlibs = false;
 
@@ -11,7 +18,9 @@ in
     after = [ "network.target" ];
 
     serviceConfig = {
-      ExecStart = "${pkgs.local.starbound}/bin/launch-starbound ${./configs/starbound.json}";
+      ExecStart = "${
+        flake-inputs.self.packages.${pkgs.system}.starbound
+      }/bin/launch-starbound ${./configs/starbound.json}";
 
       Type = "simple";
 
@@ -114,4 +123,7 @@ in
     paths = [ "/var/lib/private/starbound/storage/universe/" ];
     pauseServices = [ "starbound.service" ];
   };
+
+  # Accessed via systemd cred through /run/secrets/steam
+  sops.secrets."steam/tlater" = { };
 }

configuration/services/webserver.nix

@@ -3,6 +3,11 @@ let
   inherit (config.services.nginx) domain;
 in
 {
+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+  ];
+
   services.tlaternet-webserver = {
     enable = true;
     listen = {

configuration/services/wireguard.nix

@@ -62,4 +62,10 @@
       };
     };
   };
+
+  sops.secrets."wireguard/server-key" = {
+    owner = "root";
+    group = "systemd-network";
+    mode = "0440";
+  };
 }

configuration/sops.nix

@@ -1,89 +0,0 @@
-{
-  sops = {
-    defaultSopsFile = ../keys/production.yaml;
-
-    secrets = {
-      "battery-manager/email" = { };
-
-      "battery-manager/password" = { };
-
-      # Gitea
-      "forgejo/metrics-token" = {
-        owner = "forgejo";
-        group = "metrics";
-        mode = "0440";
-      };
-
-      # Grafana
-      "grafana/adminPassword" = {
-        owner = "grafana";
-        group = "grafana";
-      };
-      "grafana/secretKey" = {
-        owner = "grafana";
-        group = "grafana";
-      };
-
-      # Heisenbridge
-      "heisenbridge/as-token" = { };
-      "heisenbridge/hs-token" = { };
-
-      # Matrix-hookshot
-      "matrix-hookshot/as-token" = { };
-      "matrix-hookshot/hs-token" = { };
-
-      # Nextcloud
-      "nextcloud/tlater" = {
-        owner = "nextcloud";
-        group = "nextcloud";
-      };
-
-      # Porkbub/ACME
-      "porkbun/api-key" = {
-        owner = "acme";
-      };
-      "porkbun/secret-api-key" = {
-        owner = "acme";
-      };
-
-      # Restic
-      "restic/local-backups" = {
-        owner = "root";
-        group = "backup";
-        mode = "0440";
-      };
-      "restic/storagebox-backups" = {
-        owner = "root";
-        group = "backup";
-        mode = "0440";
-      };
-      "restic/storagebox-ssh-key" = {
-        owner = "backup";
-        group = "backup";
-        mode = "0040";
-      };
-
-      # Steam
-      "steam/tlater" = { };
-
-      # Turn
-      "turn/env" = { };
-      "turn/secret" = {
-        owner = "turnserver";
-      };
-      "turn/ssl-key" = {
-        owner = "turnserver";
-      };
-      "turn/ssl-cert" = {
-        owner = "turnserver";
-      };
-
-      # Wireguard
-      "wireguard/server-key" = {
-        owner = "root";
-        group = "systemd-network";
-        mode = "0440";
-      };
-    };
-  };
-}

flake.lock

@@ -136,11 +136,11 @@
         "pyproject-nix": "pyproject-nix"
       },
       "locked": {
-        "lastModified": 1754978539,
-        "narHash": "sha256-nrDovydywSKRbWim9Ynmgj8SBm8LK3DI2WuhIqzOHYI=",
+        "lastModified": 1763413832,
+        "narHash": "sha256-dkqBwDXiv8MPoFyIvOuC4bVubAP+TlVZUkVMB78TTSg=",
         "owner": "nix-community",
         "repo": "dream2nix",
-        "rev": "fbec3263cb4895ac86ee9506cdc4e6919a1a2214",
+        "rev": "5658fba3a0b6b7d5cb0460b949651f64f644a743",
         "type": "github"
       },
       "original": {
@@ -356,11 +356,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1762868777,
-        "narHash": "sha256-QqS72GvguP56oKDNUckWUPNJHjsdeuXh5RyoKz0wJ+E=",
+        "lastModified": 1763319842,
+        "narHash": "sha256-YG19IyrTdnVn0l3DvcUYm85u3PaqBt6tI6VvolcuHnA=",
         "owner": "cachix",
         "repo": "pre-commit-hooks.nix",
-        "rev": "c5c3147730384576196fb5da048a6e45dee10d56",
+        "rev": "7275fa67fbbb75891c16d9dee7d88e58aea2d761",
         "type": "github"
       },
       "original": {

keys/staging.yaml

@@ -22,18 +22,14 @@ matrix-hookshot:
 wireguard:
     server-key: ENC[AES256_GCM,data:FvY897XdKoa/mckE8JQLCkklsnYD6Wz1wpsu5t3uhEnW3iarnDQxF9msuYU=,iv:jqGXfekM+Vs+J9b5nlZ5Skd1ZKHajoUo2Dc4tMYPm1w=,tag:EehikjI/FCU8wqtpvJRamQ==,type:str]
 restic:
-    local-backups: ENC[AES256_GCM,data:3QjEv03t7wE=,iv:y/6Lv4eUbZZfGPwUONykz8VNL62cAJuWaJy9yk3aAmk=,tag:wMlGsepuG9JjwtUKGWSibw==,type:str]
     storagebox-backups: ENC[AES256_GCM,data:NEHk57B3YtI=,iv:0/qnqMVK0662sgfDQoLxcW7L09SKF8E5liCnjaQ2+2k=,tag:RU0BPwGgvI9bgOPr8VItmA==,type:str]
     storagebox-ssh-key: ENC[AES256_GCM,data:65+kbJPO90y+rRh3Q5cqLDtQa3VFfbaDPPo1nJLqxgAB7Wm3J7K4qUYAKPcYnkWV4/xFz63R2uCNaq5xv+vuZA==,iv:O7AeE/ujp5p1P7nff7PpghQfN2tQUYBSWL+EHRbE5yA=,tag:Pu/+bEAQuqwmD1Rc//t0cA==,type:str]
 turn:
     env: ENC[AES256_GCM,data:xjIz/AY109lyiL5N01p5T3HcYco/rM5CJSRTtg==,iv:16bW6OpyOK/QL0QPGQp/Baa9xyT8E3ZsYkwqmjuofk0=,tag:J5re3uKxIykw3YunvQWBgg==,type:str]
     secret: ENC[AES256_GCM,data:eQ7dAocoZtg=,iv:fgzjTPv30WqTKlLy+yMn5MsKQgjhPnwlGFFwYEg3gWs=,tag:1ze33U1NBkgMX/9SiaBNQg==,type:str]
-    ssl-key: ENC[AES256_GCM,data:RYfwHjBvwFXgXxXIEuWUzaycTdrCvmPivsNvvUIwDRynS5G2Dl6RCVp1w9zuLvoNun5ncUPGGuLMmVqN2wkJlw==,iv:UKI3bVTY7iTDNvp5UqrZ3QlQkMZ5p2bjgODEc6DCBfQ=,tag:sz7VTyRWyZxAsP4nE48DnA==,type:str]
-    #ENC[AES256_GCM,data:bxhKzU5Tzezl749CDu8e8kxa7ahGuZFaPa9K3kxuD+4sg5Hi3apgDlC0n8oK0DeiK4Ks7+9Cyw==,iv:T/zVJUpNAv1rR0a9+6SDTG08ws2A1hFBs5Ia3TpT0uk=,tag:uGXb1VryM+lIJ8r0I5durA==,type:comment]
-    ssl-cert: ENC[AES256_GCM,data:xHUr14CjKslgbGh/n5jYSOuCw9JRxS6YXE4fxS+aJzFcNeSeGNqoipPeuJupZGBnQP/FCqohiHY=,iv:/OEsVqRshGL9NIvntMC42EPZSNL0u6EfhtUBqgV7qog=,tag:4pxtNjuvy/ibm6nDtKdSkw==,type:str]
 sops:
-    lastmodified: "2025-02-07T17:43:24Z"
-    mac: ENC[AES256_GCM,data:akmD/bfgeTyFzW1quvM16cdj0fC6+CbJ8WyX9173H11yKGxvE1USQYcErpl1SHOx9Jk8LVb7f+MsUm2fjQF1MEq6xaWI74jem12lZ9CGXFaTL7e87JvfbK7pV+aKpxSBBNFyJgbYm30ibdUwxwKmNVfPb1e0HT9qwenvoV7RobM=,iv:mKqOW0ULXL711uczUbRf9NPo6uPTQoS/IbR46S+JID4=,tag:vE6NYzYLbQHDImov1XGTcg==,type:str]
+    lastmodified: "2025-11-19T16:42:43Z"
+    mac: ENC[AES256_GCM,data:4YivckDS+jBX3Bkon0bTAm3SXya4v2ieZyqeBXjBUYZeCmelIng7bn2dP7791O6RK6RvSXAGhiykWgGRW/boG3QM8VLxDMSRTKovJo5k6oxtFJC8OLDJoh1EC5BQLznJDKl4So6FgYPEtdQ6rx+Q6Ah7JSMtQilxRoe/hYapT90=,iv:9BGtS585gVbvH6l96/YYZiY1DrwB565vPaNNtFC9vbk=,tag:HsZuDMqPFHTMPxQsD36LNQ==,type:str]
     pgp:
         - created_at: "2025-10-03T21:38:26Z"
           enc: |-
@@ -67,4 +63,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: 2f5caa73e7ceea4fcc8d2881fde587e6737d2dbc
     unencrypted_suffix: _unencrypted
-    version: 3.9.2
+    version: 3.11.0

modules/default.nix

@@ -1 +1,6 @@
-{ imports = [ ./crowdsec ]; }
+{
+  imports = [
+    ./crowdsec
+    ./serviceTests/stub.nix
+  ];
+}

modules/serviceTests/mocks.nix

@@ -0,0 +1,27 @@
+/**
+  Module containing mock definitions for service test runners.
+*/
+{ flake-inputs, lib, ... }:
+{
+  imports = [
+    flake-inputs.sops-nix.nixosModules.sops
+    ../.
+    ../../configuration/services/backups.nix
+  ];
+
+  sops.defaultSopsFile = ../../keys/staging.yaml;
+  environment.etc."staging.key" = {
+    mode = "0400";
+    source = ../../keys/hosts/staging.key;
+  };
+  services.openssh = {
+    enable = true;
+    hostKeys = lib.mkForce [
+      {
+        type = "rsa";
+        bits = 4096;
+        path = "/etc/staging.key";
+      }
+    ];
+  };
+}

modules/serviceTests/stub.nix

@@ -0,0 +1,20 @@
+/**
+  Module to make writing service-specific tests easy.
+*/
+{ lib, ... }:
+let
+  inherit (lib) mkOption types;
+in
+{
+  options = {
+    serviceTests = mkOption {
+      type = types.attrsOf types.package;
+
+      description = ''
+        NixOS tests to run.
+      '';
+
+      default = { };
+    };
+  };
+}