fail2ban: Add metrics

configuration/services/metrics/default.nix

@@ -0,0 +1,144 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}: let
+  domain = "metrics.${config.services.nginx.domain}";
+  yaml = pkgs.formats.yaml {};
+in {
+  imports = [
+    ./exporters.nix
+  ];
+
+  services.victoriametrics.enable = true;
+
+  services.grafana = {
+    enable = true;
+    settings = {
+      server.http_port = 3001; # Default overlaps with gitea
+
+      security = {
+        admin_user = "tlater";
+        admin_password = "$__file{${config.sops.secrets."grafana/adminPassword".path}}";
+        secret_key = "$__file{${config.sops.secrets."grafana/secretKey".path}}";
+        cookie_secure = true;
+        cookie_samesite = "strict";
+        content_security_policy = true;
+      };
+
+      database = {
+        user = "grafana";
+        name = "grafana";
+        type = "postgres";
+        host = "/run/postgresql";
+      };
+    };
+
+    provision = {
+      enable = true;
+
+      datasources.settings.datasources = [
+        {
+          name = "Victoriametrics - tlater.net";
+          url = "http://localhost:8428";
+          type = "prometheus";
+        }
+      ];
+    };
+  };
+
+  services.prometheus.exporters = {
+    node = {
+      enable = true;
+      enabledCollectors = ["systemd"];
+      listenAddress = "127.0.0.1";
+    };
+
+    nginx = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+    };
+
+    nginxlog = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      group = "nginx";
+
+      settings.namespaces =
+        lib.mapAttrsToList (name: virtualHost: {
+          inherit name;
+          metrics_override.prefix = "nginxlog";
+          namespace_label = "vhost";
+
+          format = lib.concatStringsSep " " [
+            "$remote_addr - $remote_user [$time_local]"
+            ''"$request" $status $body_bytes_sent''
+            ''"$http_referer" "$http_user_agent"''
+            ''rt=$request_time uct="$upstream_connect_time"''
+            ''uht="$upstream_header_time" urt="$upstream_response_time"''
+          ];
+
+          source.files = [
+            "/var/log/nginx/${name}/access.log"
+          ];
+        })
+        config.services.nginx.virtualHosts;
+    };
+  };
+
+  services.prometheus.local-exporters = {
+    prometheus-fail2ban-exporter = rec {
+      enable = true;
+      after = ["fail2ban.service"];
+
+      port = 9191;
+      listenAddress = "127.0.0.1";
+
+      serviceConfig = {
+        Group = "fail2ban";
+
+        RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"];
+
+        ExecStart = lib.concatStringsSep " " [
+          "${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter"
+          "--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock"
+          "--web.listen-address='${listenAddress}:${toString port}'"
+        ];
+      };
+    };
+  };
+
+  systemd.services.export-to-victoriametrics = let
+    promscrape = yaml.generate "prometheus.yml" {
+      scrape_configs = [
+        {
+          job_name = "tlater.net";
+          static_configs = [
+            {
+              targets =
+                lib.mapAttrsToList (name: exporter: "${exporter.listenAddress}:${toString exporter.port}")
+                (lib.filterAttrs (name: exporter: (builtins.isAttrs exporter) && exporter.enable)
+                  (config.services.prometheus.exporters // config.services.prometheus.local-exporters));
+            }
+          ];
+        }
+      ];
+    };
+  in {
+    enable = true;
+    path = [pkgs.victoriametrics];
+    wantedBy = ["multi-user.target"];
+    script = "vmagent -promscrape.config=${promscrape} -remoteWrite.url=http://localhost:8428/api/v1/write";
+  };
+
+  services.nginx.virtualHosts."${domain}" = {
+    forceSSL = true;
+    enableACME = true;
+    extraConfig = ''
+      add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+      access_log /var/log/nginx/${domain}/access.log upstream_time;
+    '';
+    locations."/".proxyPass = "http://localhost:3001";
+  };
+}

configuration/services/metrics/exporters.nix

@@ -0,0 +1,45 @@
+{
+  config,
+  lib,
+  ...
+}: {
+  options.services.prometheus.local-exporters = lib.mkOption {
+    type = lib.types.anything;
+  };
+
+  config.systemd.services = lib.mapAttrs (_: exporter:
+    lib.mkMerge [
+      {
+        wantedBy = ["multi-user.target"];
+        after = ["network.target"];
+
+        serviceConfig = {
+          Restart = "always";
+          PrivateTmp = true;
+          WorkingDirectory = "/tmp";
+          DynamicUser = true;
+          LockPersonality = true;
+          MemoryDenyWriteExecute = true;
+          NonNewPrivileges = true;
+          PrivateDevices = true;
+          ProtectClock = true;
+          ProtectControlGroups = true;
+          ProtectHome = true;
+          ProtectHostname = true;
+          ProtectKernelLogs = true;
+          ProtectKernelModules = true;
+          ProtectKernelTunables = true;
+          ProtectSystem = "strict";
+          RemoveIPC = true;
+          RestrictAddressFamilies = lib.mkDefault ["AF_INET" "AF_INET6"];
+          RestrictNamespaces = true;
+          RestrictRealtime = true;
+          RestrictSUIDSGID = true;
+          SystemCallArchitectures = "native";
+          UMask = "0077";
+        };
+      }
+      (removeAttrs exporter ["port" "listenAddress"])
+    ])
+  config.services.prometheus.local-exporters;
+}