feat(metrics): Add blackbox exporter

configuration/services/metrics/exporters.nix

@@ -10,6 +10,20 @@ in
 {
   services.prometheus = {
     exporters = {
+      blackbox = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        configFile = yaml.generate "blackbox.yaml" {
+          modules = {
+            http_2xx = {
+              prober = "http";
+              timeout = "5s";
+              http.preferred_ip_protocol = "ip4";
+            };
+          };
+        };
+      };
+
       # Periodically check domain registration status
       domain = {
         enable = true;
@@ -71,7 +85,6 @@ in
     # TODO(tlater):
     #   - wireguard (?)
     #   - postgres (?)
-    #   - blackbox (?) (curl to see if http and similar is up)
     #   - ssl_exporter (?)
   };
 }

configuration/services/metrics/options.nix

@@ -47,7 +47,7 @@ in
               };
 
               extraSettings = mkOption {
-                type = types.anything;
+                inherit (pkgs.formats.yaml { }) type;
                 description = ''
                   Other settings to set for this scrape config.
                 '';
@@ -217,6 +217,7 @@ in
             name: exporter:
             # A bunch of deprecated exporters that need to be ignored
             !(builtins.elem name [
+              "blackbox"
               "minio"
               "tor"
               "unifi-poller"

configuration/services/metrics/victoriametrics.nix

@@ -1,4 +1,8 @@
-{ config, ... }:
+{ config, lib, ... }:
+let
+  blackbox_host = config.services.prometheus.exporters.blackbox.listenAddress;
+  blackbox_port = config.services.prometheus.exporters.blackbox.port;
+in
 {
   config.services.victoriametrics = {
     enable = true;
@@ -9,6 +13,35 @@
         targets = [ "127.0.0.1:${toString config.services.forgejo.settings.server.HTTP_PORT}" ];
         extraSettings.authorization.credentials_file = config.sops.secrets."forgejo/metrics-token".path;
       };
+
+      blackbox = {
+        static_configs = lib.singleton {
+          targets = lib.mapAttrsToList (vHost: _: "https://${vHost}") config.services.nginx.virtualHosts;
+        };
+
+        extraSettings = {
+          metrics_path = "/probe";
+          params.module = [ "http_2xx" ];
+
+          relabel_configs = [
+            {
+              source_labels = [ "__address__" ];
+              target_label = "__param_target";
+            }
+            {
+              source_labels = [ "__param_target" ];
+              target_label = "instance";
+            }
+            {
+              target_label = "__address__";
+              replacement = "${blackbox_host}:${toString blackbox_port}";
+            }
+          ];
+        };
+      };
+
+      blackbox_exporter.targets = [ "${blackbox_host}:${toString blackbox_port}" ];
+
       coturn.targets = [ "127.0.0.1:9641" ];
 
       crowdsec.targets =