WIP: Add metrics

configuration/services/metrics/default.nix

@@ -0,0 +1,9 @@
+{
+  imports = [
+    ./options.nix
+
+    ./exporters.nix
+    ./grafana.nix
+    ./victoriametrics.nix
+  ];
+}

configuration/services/metrics/exporters.nix

@@ -0,0 +1,100 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}: let
+  yaml = pkgs.formats.yaml {};
+in {
+  services.prometheus = {
+    exporters = {
+      # Periodically check domain registration status
+      domain = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        extraFlags = let
+          conf.domains = [
+            "tlater.net"
+            "tlater.com"
+          ];
+        in [
+          "--config=${yaml.generate "domains.yml" conf}"
+        ];
+      };
+
+      # System statistics
+      node = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+      };
+      systemd = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        extraFlags = [
+          # Disabled by default because only supported from systemd 235+
+          "--systemd.collector.enable-restart-count"
+          "--systemd.collector.enable-ip-accounting"
+        ];
+      };
+
+      # Various nginx metrics
+      nginx = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+      };
+
+      nginxlog = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        group = "nginx";
+
+        settings.namespaces =
+          lib.mapAttrsToList (name: virtualHost: {
+            inherit name;
+            metrics_override.prefix = "nginxlog";
+            namespace_label = "vhost";
+
+            format = lib.concatStringsSep " " [
+              "$remote_addr - $remote_user [$time_local]"
+              ''"$request" $status $body_bytes_sent''
+              ''"$http_referer" "$http_user_agent"''
+              ''rt=$request_time uct="$upstream_connect_time"''
+              ''uht="$upstream_header_time" urt="$upstream_response_time"''
+            ];
+
+            source.files = [
+              "/var/log/nginx/${name}/access.log"
+            ];
+          })
+          config.services.nginx.virtualHosts;
+      };
+    };
+
+    extraExporters = {
+      fail2ban = let
+        cfg = config.services.prometheus.extraExporters.fail2ban;
+      in {
+        port = 9191;
+        serviceOpts = {
+          after = ["fail2ban.service"];
+          requires = ["fail2ban.service"];
+          serviceConfig = {
+            Group = "fail2ban";
+            RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"];
+            ExecStart = lib.concatStringsSep " " [
+              "${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter"
+              "--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock"
+              "--web.listen-address='${cfg.listenAddress}:${toString cfg.port}'"
+              "--collector.f2b.exit-on-socket-connection-error=true"
+            ];
+          };
+        };
+      };
+    };
+
+    # TODO(tlater):
+    #   - wireguard (?)
+    #   - postgres (?)
+    #   - blackbox (?) (curl to see if http and similar is up)
+  };
+}

configuration/services/metrics/grafana.nix

@@ -0,0 +1,48 @@
+{config, ...}: let
+  domain = "metrics.${config.services.nginx.domain}";
+in {
+  services.grafana = {
+    enable = true;
+    settings = {
+      server.http_port = 3001; # Default overlaps with gitea
+
+      security = {
+        admin_user = "tlater";
+        admin_password = "$__file{${config.sops.secrets."grafana/adminPassword".path}}";
+        secret_key = "$__file{${config.sops.secrets."grafana/secretKey".path}}";
+        cookie_secure = true;
+        cookie_samesite = "strict";
+        content_security_policy = true;
+      };
+
+      database = {
+        user = "grafana";
+        name = "grafana";
+        type = "postgres";
+        host = "/run/postgresql";
+      };
+    };
+
+    provision = {
+      enable = true;
+
+      datasources.settings.datasources = [
+        {
+          name = "Victoriametrics - tlater.net";
+          url = "http://localhost:8428";
+          type = "prometheus";
+        }
+      ];
+    };
+  };
+
+  services.nginx.virtualHosts."${domain}" = {
+    forceSSL = true;
+    enableACME = true;
+    extraConfig = ''
+      add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+      access_log /var/log/nginx/${domain}/access.log upstream_time;
+    '';
+    locations."/".proxyPass = "http://localhost:${toString config.services.grafana.settings.server.http_port}";
+  };
+}

configuration/services/metrics/options.nix

@@ -0,0 +1,90 @@
+{
+  config,
+  lib,
+  ...
+}: let
+  inherit (lib) types mkOption mkDefault;
+in {
+  options.services.prometheus = {
+    extraExporters = mkOption {
+      type = types.attrsOf (types.submodule {
+        options = {
+          port = mkOption {
+            type = types.int;
+            description = "The port on which this exporter listens.";
+          };
+          listenAddress = mkOption {
+            type = types.str;
+            default = "127.0.0.1";
+            description = "Address to listen on.";
+          };
+          serviceOpts = mkOption {
+            type = types.attrs;
+            description = "An attrset to be merged with the exporter's systemd service.";
+          };
+        };
+      });
+    };
+
+    allExporters = mkOption {
+      internal = true;
+      description = "The full list of scraping-relevant settings of all exporters, extra or built-in.";
+      type = types.attrsOf (types.submodule {
+        port = mkOption {
+          type = types.int;
+        };
+        listenAddress = mkOption {
+          type = types.str;
+        };
+        extraSettings = mkOption {
+          type = types.anything;
+          default = {};
+        };
+      });
+    };
+  };
+
+  config = {
+    systemd.services = lib.mapAttrs' (name: exporter:
+      lib.nameValuePair "prometheus-${name}-exporter" (lib.mkMerge [
+        {
+          # Shamelessly copied from upstream because the upstream
+          # module is an intractable mess
+          wantedBy = ["multi-user.target"];
+          after = ["network.target"];
+          serviceConfig.Restart = mkDefault "always";
+          serviceConfig.PrivateTmp = mkDefault true;
+          serviceConfig.WorkingDirectory = mkDefault /tmp;
+          serviceConfig.DynamicUser = mkDefault true;
+          # Hardening
+          serviceConfig.CapabilityBoundingSet = mkDefault [""];
+          serviceConfig.DeviceAllow = [""];
+          serviceConfig.LockPersonality = true;
+          serviceConfig.MemoryDenyWriteExecute = true;
+          serviceConfig.NoNewPrivileges = true;
+          serviceConfig.PrivateDevices = mkDefault true;
+          serviceConfig.ProtectClock = mkDefault true;
+          serviceConfig.ProtectControlGroups = true;
+          serviceConfig.ProtectHome = true;
+          serviceConfig.ProtectHostname = true;
+          serviceConfig.ProtectKernelLogs = true;
+          serviceConfig.ProtectKernelModules = true;
+          serviceConfig.ProtectKernelTunables = true;
+          serviceConfig.ProtectSystem = mkDefault "strict";
+          serviceConfig.RemoveIPC = true;
+          serviceConfig.RestrictAddressFamilies = ["AF_INET" "AF_INET6"];
+          serviceConfig.RestrictNamespaces = true;
+          serviceConfig.RestrictRealtime = true;
+          serviceConfig.RestrictSUIDSGID = true;
+          serviceConfig.SystemCallArchitectures = "native";
+          serviceConfig.UMask = "0077";
+        }
+        exporter.serviceOpts
+      ]))
+    config.services.prometheus.extraExporters;
+
+    services.prometheus.allExporters = lib.mapAttrs (name: exporter: {
+      inherit (exporter) listenAddress port;
+    }) (config.services.prometheus.exporters ++ config.services.prometheus.extraExporters);
+  };
+}

configuration/services/metrics/victoriametrics.nix

@@ -0,0 +1,31 @@
+{config, ...}: {
+  services.victoriametrics = let
+    scrapeConfigFromExporters = conf: conf // {inherit (config.services.prometheus.exporters.${conf.name}) listenAddress port;};
+    scrapeConfigFromLocalExporters = conf: conf // {inherit (config.services.prometheus.local-exporters.${conf.name}) listenAddress port;};
+  in {
+    enable = true;
+    vmagent-scraping.static_configs =
+      [
+        {
+          name = "gitea";
+          listenAddress = "127.0.0.1";
+          port = 3000;
+        }
+        {
+          name = "coturn";
+          listenAddress = "127.0.0.1";
+          port = 9641;
+        }
+      ]
+      ++ (map scrapeConfigFromLocalExporters [
+        {name = "prometheus-fail2ban-exporter";}
+      ])
+      ++ (map scrapeConfigFromExporters [
+        {name = "domain";}
+        {name = "node";}
+        {name = "nginx";}
+        {name = "nginxlog";}
+        {name = "systemd";}
+      ]);
+  };
+}