feat(metrics): Add victorialogs

Flake lock file updates:

• Updated input 'disko':
    'github:nix-community/disko/fa5746ecea1772cf59b3f34c5816ab3531478142?narHash=sha256-xFnU%2BuUl48Icas2wPQ%2BZzlL2O3n8f6J2LrzNK9f2nng%3D' (2025-02-15)
  → 'github:nix-community/disko/15dbf8cebd8e2655a883b74547108e089f051bf0?narHash=sha256-lSOXdgW/1zi/SSu7xp71v%2B55D5Egz8ACv0STkj7fhbs%3D' (2025-02-18)
• Updated input 'foundryvtt':
    'github:reckenrode/nix-foundryvtt/0a72a4bf64224c6584fd1b9e9f0012dd09af979a?narHash=sha256-vM9C1gFiQGa3nTYqmTBI8MoiUfprkQdepUBbxV7ECMQ%3D' (2025-01-17)
  → 'github:reckenrode/nix-foundryvtt/a7fa493ba2c623cf90e83756b62285b3b58f18d2?narHash=sha256-u3m%2BawbdL%2B0BKk8IWidsWMr%2BR0ian3GZMUlH7623kd8%3D' (2025-02-16)
• Updated input 'nixpkgs':
    'github:nixos/nixpkgs/30d4471a8a2a13b716530d3aad60b9846ea5ff83?narHash=sha256-jGiez5BtGGJUB/LXzRa%2B4AQurMO9acc1B69kBfgQhJc%3D' (2025-02-15)
  → 'github:nixos/nixpkgs/11415c7ae8539d6292f2928317ee7a8410b28bb9?narHash=sha256-SSYxFhqCOb3aiPb6MmN68yEzBIltfom8IgRz7phHscM%3D' (2025-02-21)
• Updated input 'nixpkgs-unstable':
    'github:nixos/nixpkgs/31ff66eb77d02e9ac34b7256a02edb1c43fb9998?narHash=sha256-3bnOIZz8KXtzcaXGuH9Eriv0HiQyr1EIfcye%2BVHLQZE%3D' (2025-02-15)
  → 'github:nixos/nixpkgs/8465e233b0668cf162c608a92e62e8d78c1ba7e4?narHash=sha256-wzBbGGZ6i1VVBA/cDJaLfuuGYCUriD7fwsLgJJHRVRk%3D' (2025-02-22)

configuration/nginx.nix

@@ -53,7 +53,7 @@
         "*.tlater.com"
       ];
       dnsProvider = "porkbun";
-      group = "nginx";
+      group = "ssl-cert";
       credentialFiles = {
         PORKBUN_API_KEY_FILE = config.sops.secrets."porkbun/api-key".path;
         PORKBUN_SECRET_API_KEY_FILE = config.sops.secrets."porkbun/secret-api-key".path;
@@ -61,6 +61,12 @@
     };
   };
 
+  users.groups.ssl-cert = { };
+
+  systemd.services.nginx.serviceConfig.SupplementaryGroups = [
+    config.security.acme.certs."tlater.net".group
+  ];
+
   services.backups.acme = {
     user = "acme";
     paths = lib.mapAttrsToList (

configuration/services/battery-manager.nix

@@ -4,9 +4,13 @@
 
   services.batteryManager = {
     enable = true;
-    battery = "3ca39300-c523-4315-b9a3-d030f85a9373";
 
     emailFile = "${config.sops.secrets."battery-manager/email".path}";
     passwordFile = "${config.sops.secrets."battery-manager/password".path}";
+
+    settings = {
+      battery_id = "3ca39300-c523-4315-b9a3-d030f85a9373";
+      log_level = "DEBUG";
+    };
   };
 }

configuration/services/conduit/default.nix

@@ -50,6 +50,10 @@ in
   # See also https://gitlab.com/famedly/conduit/-/issues/314
   systemd.services.conduit.serviceConfig.EnvironmentFile = config.sops.secrets."turn/env".path;
 
+  systemd.services.coturn.serviceConfig.SupplementaryGroups = [
+    config.security.acme.certs."tlater.net".group
+  ];
+
   services.coturn = {
     enable = true;
     no-cli = true;
@@ -59,11 +63,8 @@ in
     relay-ips = [ "116.202.158.55" ];
 
     # SSL config
-    #
+    pkey = "${config.security.acme.certs."tlater.net".directory}/key.pem";
-    # TODO(tlater): Switch to letsencrypt once google fix:
+    cert = "${config.security.acme.certs."tlater.net".directory}/fullchain.pem";
-    #  https://github.com/vector-im/element-android/issues/1533
-    pkey = config.sops.secrets."turn/ssl-key".path;
-    cert = config.sops.secrets."turn/ssl-cert".path;
 
     # Based on suggestions from
     # https://github.com/matrix-org/synapse/blob/develop/docs/turn-howto.md

configuration/services/metrics/default.nix

@@ -5,5 +5,6 @@
     ./exporters.nix
     ./grafana.nix
     ./victoriametrics.nix
+    ./victorialogs.nix
   ];
 }

configuration/services/metrics/exporters.nix

@@ -20,6 +20,16 @@ in
               timeout = "5s";
               http.preferred_ip_protocol = "ip4";
             };
+
+            turn_server = {
+              prober = "tcp";
+              timeout = "5s";
+              tcp = {
+                preferred_ip_protocol = "ip4";
+                source_ip_address = "116.202.158.55";
+                tls = true;
+              };
+            };
           };
         };
       };

configuration/services/metrics/grafana.nix

@@ -1,4 +1,9 @@
-{ config, ... }:
+{
+  pkgs,
+  config,
+  flake-inputs,
+  ...
+}:
 let
   domain = "metrics.${config.services.nginx.domain}";
 in
@@ -28,6 +33,11 @@ in
       };
     };
 
+    declarativePlugins = [
+      pkgs.grafanaPlugins.victoriametrics-metrics-datasource
+      flake-inputs.nixpkgs-unstable.legacyPackages.${pkgs.system}.grafanaPlugins.victoriametrics-logs-datasource
+    ];
+
     provision = {
       enable = true;
 
@@ -35,7 +45,16 @@ in
         {
           name = "Victoriametrics - tlater.net";
           url = "http://localhost:8428";
-          type = "prometheus";
+          type = "victoriametrics-metrics-datasource";
+          access = "proxy";
+          isDefault = true;
+        }
+
+        {
+          name = "Victorialogs - tlater.net";
+          url = "http://${config.services.victorialogs.bindAddress}";
+          type = "victoriametrics-logs-datasource";
+          access = "proxy";
         }
       ];
     };

configuration/services/metrics/victorialogs.nix

@@ -0,0 +1,110 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  cfg = config.services.victorialogs;
+  pkg = pkgs.victoriametrics;
+  dirname = "victorialogs";
+in
+{
+  options.services.victorialogs =
+    let
+      inherit (lib.types) str;
+    in
+    {
+      listenAddress = lib.mkOption {
+        default = ":9428";
+        type = str;
+      };
+
+      bindAddress = lib.mkOption {
+        readOnly = true;
+        type = str;
+        description = ''
+          Final address on which victorialogs listens.
+        '';
+      };
+    };
+
+  config = {
+    services.victorialogs.bindAddress =
+      (lib.optionalString (lib.hasPrefix ":" cfg.listenAddress) "127.0.0.1") + cfg.listenAddress;
+
+    services.journald.upload = {
+      enable = true;
+      settings.Upload = {
+        URL = "http://${cfg.bindAddress}/insert/journald";
+        NetworkTimeoutSec = "20s";
+      };
+    };
+
+    systemd.services."systemd-journal-upload".after = [ "victorialogs.service" ];
+
+    systemd.services.victorialogs = {
+      description = "VictoriaLogs log database";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      startLimitBurst = 5;
+
+      serviceConfig = {
+        ExecStart = lib.escapeShellArgs [
+          "${pkg}/bin/victoria-logs"
+          "-storageDataPath=/var/lib/${dirname}"
+          "-httpListenAddr=${cfg.listenAddress}"
+        ];
+
+        DynamicUser = true;
+        RestartSec = 1;
+        Restart = "on-failure";
+        RuntimeDirectory = dirname;
+        RuntimeDirectoryMode = "0700";
+        StateDirectory = dirname;
+        StateDirectoryMode = "0700";
+
+        LimitNOFILE = 1048576;
+
+        # Hardening
+        DeviceAllow = [ "/dev/null rw" ];
+        DevicePolicy = "strict";
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        PrivateTmp = true;
+        PrivateUsers = true;
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "invisible";
+        ProtectSystem = "full";
+        RemoveIPC = true;
+        RestrictAddressFamilies = [
+          "AF_INET"
+          "AF_INET6"
+          "AF_UNIX"
+        ];
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = [
+          "@system-service"
+          "~@privileged"
+        ];
+      };
+
+      postStart = lib.mkBefore ''
+        until ${lib.getBin pkgs.curl}/bin/curl -s -o /dev/null http://${cfg.bindAddress}/ping; do
+          sleep 1;
+        done
+      '';
+    };
+  };
+}

configuration/services/metrics/victoriametrics.nix

@@ -40,6 +40,30 @@ in
         };
       };
 
+      blackbox_turn = {
+        targets = [ "turn.tlater.net:${toString config.services.coturn.tls-listening-port}" ];
+
+        extraSettings = {
+          metrics_path = "/probe";
+          params.module = [ "turn_server" ];
+
+          relabel_configs = [
+            {
+              source_labels = [ "__address__" ];
+              target_label = "__param_target";
+            }
+            {
+              source_labels = [ "__param_target" ];
+              target_label = "instance";
+            }
+            {
+              target_label = "__address__";
+              replacement = "${blackbox_host}:${toString blackbox_port}";
+            }
+          ];
+        };
+      };
+
       blackbox_exporter.targets = [ "${blackbox_host}:${toString blackbox_port}" ];
 
       coturn.targets = [ "127.0.0.1:9641" ];

configuration/sops.nix

@@ -1,18 +1,11 @@
-{ config, lib, ... }:
 {
   sops = {
     defaultSopsFile = ../keys/production.yaml;
 
     secrets = {
-      "battery-manager/email" = lib.mkIf config.services.batteryManager.enable {
+      "battery-manager/email" = { };
-        owner = "battery-manager";
-        group = "battery-manager";
-      };
 
-      "battery-manager/password" = lib.mkIf config.services.batteryManager.enable {
+      "battery-manager/password" = { };
-        owner = "battery-manager";
-        group = "battery-manager";
-      };
 
       # Gitea
       "forgejo/metrics-token" = {

flake.nix

@@ -26,8 +26,6 @@
       url = "git+ssh://git@github.com/sonnenshift/battery-manager";
       inputs.nixpkgs.follows = "nixpkgs";
     };
-
-    nixpkgs-crowdsec.url = "github:tlater/nixpkgs/tlater/fix-crowdsec";
   };
 
   outputs =
@@ -106,7 +104,6 @@
         in
         {
           default = vm.config.system.build.vm;
-          crowdsec = pkgs.callPackage "${inputs.nixpkgs-crowdsec}/pkgs/by-name/cr/crowdsec/package.nix" { };
           crowdsec-hub = localPkgs.crowdsec.hub;
           crowdsec-firewall-bouncer = localPkgs.crowdsec.firewall-bouncer;
         };

modules/crowdsec/default.nix

@@ -1,5 +1,4 @@
 {
-  flake-inputs,
   pkgs,
   lib,
   config,
@@ -9,8 +8,6 @@ let
   cfg = config.security.crowdsec;
   settingsFormat = pkgs.formats.yaml { };
 
-  crowdsec = flake-inputs.self.packages.${pkgs.system}.crowdsec;
-
   hub = pkgs.fetchFromGitHub {
     owner = "crowdsecurity";
     repo = "hub";
@@ -19,14 +16,14 @@ let
   };
 
   cscli = pkgs.writeShellScriptBin "cscli" ''
-    export PATH="$PATH:${crowdsec}/bin/"
+    export PATH="$PATH:${cfg.package}/bin/"
 
     sudo=exec
     if [ "$USER" != "crowdsec" ]; then
         sudo='exec /run/wrappers/bin/sudo -u crowdsec'
     fi
 
-    $sudo ${crowdsec}/bin/cscli "$@"
+    $sudo ${cfg.package}/bin/cscli "$@"
   '';
 
   acquisitions = ''
@@ -53,7 +50,7 @@ in
 
       package = lib.mkOption {
         type = package;
-        default = crowdsec;
+        default = pkgs.crowdsec;
       };
 
       stateDirectory = lib.mkOption {

pkgs/crowdsec/_sources/generated.json

@@ -21,7 +21,7 @@
     },
     "crowdsec-hub": {
         "cargoLocks": null,
-        "date": "2025-01-30",
+        "date": "2025-02-22",
         "extract": null,
         "name": "crowdsec-hub",
         "passthru": null,
@@ -33,10 +33,10 @@
             "name": null,
             "owner": "crowdsecurity",
             "repo": "hub",
-            "rev": "8f102f5ac79af59d3024ca2771b65ec87411ac02",
+            "rev": "f9883cd6c7d1913c13e4a3a69d9a0b887a7d57df",
-            "sha256": "sha256-8K1HkBg0++Au1dr2KMrl9b2ruqXdo+vqWngOCwL11Mo=",
+            "sha256": "sha256-45pUln7Qj5luY9I9BE2qhzjH7kv4IbYvNoEX3/4AVVg=",
             "type": "github"
         },
-        "version": "8f102f5ac79af59d3024ca2771b65ec87411ac02"
+        "version": "f9883cd6c7d1913c13e4a3a69d9a0b887a7d57df"
     }
 }

pkgs/crowdsec/_sources/generated.nix

@@ -14,14 +14,14 @@
   };
   crowdsec-hub = {
     pname = "crowdsec-hub";
-    version = "8f102f5ac79af59d3024ca2771b65ec87411ac02";
+    version = "f9883cd6c7d1913c13e4a3a69d9a0b887a7d57df";
     src = fetchFromGitHub {
       owner = "crowdsecurity";
       repo = "hub";
-      rev = "8f102f5ac79af59d3024ca2771b65ec87411ac02";
+      rev = "f9883cd6c7d1913c13e4a3a69d9a0b887a7d57df";
       fetchSubmodules = false;
-      sha256 = "sha256-8K1HkBg0++Au1dr2KMrl9b2ruqXdo+vqWngOCwL11Mo=";
+      sha256 = "sha256-45pUln7Qj5luY9I9BE2qhzjH7kv4IbYvNoEX3/4AVVg=";
     };
-    date = "2025-01-30";
+    date = "2025-02-22";
   };
 }