fail2ban: Add metrics

configuration/default.nix

@@ -16,9 +16,10 @@
 
     ./services/backups.nix
     ./services/conduit.nix
+    ./services/fail2ban.nix
     ./services/foundryvtt.nix
     ./services/gitea.nix
-    ./services/metrics.nix
+    ./services/metrics
     ./services/nextcloud.nix
     ./services/webserver.nix
     ./services/wireguard.nix
@@ -137,6 +138,8 @@
     recommendedProxySettings = true;
     clientMaxBodySize = "10G";
     domain = "tlater.net";
+
+    statusPage = true; # For metrics, should be accessible only from localhost
   };
 
   security.acme = {
@@ -144,27 +147,6 @@
     acceptTerms = true;
   };
 
-  services.fail2ban = {
-    enable = true;
-    extraPackages = [pkgs.ipset];
-    banaction = "iptables-ipset-proto6-allports";
-    bantime-increment.enable = true;
-
-    jails = {
-      nginx-botsearch = ''
-        enabled = true
-        logpath = /var/log/nginx/access.log
-      '';
-    };
-
-    ignoreIP = [
-      "127.0.0.0/8"
-      "10.0.0.0/8"
-      "172.16.0.0/12"
-      "192.168.0.0/16"
-    ];
-  };
-
   # Remove some unneeded packages
   environment.defaultPackages = [];
 

configuration/services/fail2ban.nix

@@ -0,0 +1,42 @@
+{pkgs, ...}: {
+  services.fail2ban = {
+    enable = true;
+    extraPackages = [pkgs.ipset];
+    banaction = "iptables-ipset-proto6-allports";
+    bantime-increment.enable = true;
+
+    jails = {
+      nginx-botsearch = ''
+        enabled = true
+        logpath = /var/log/nginx/access.log
+      '';
+    };
+
+    ignoreIP = [
+      "127.0.0.0/8"
+      "10.0.0.0/8"
+      "172.16.0.0/12"
+      "192.168.0.0/16"
+    ];
+  };
+
+  # Allow metrics services to connect to the socket as well
+  users.groups.fail2ban = {};
+  systemd.services.fail2ban.serviceConfig = {
+    ExecStartPost =
+      "+"
+      + (pkgs.writeShellScript "fail2ban-post-start" ''
+        while ! [ -S /var/run/fail2ban/fail2ban.sock ]; do
+            sleep 1
+        done
+
+        while ! ${pkgs.netcat}/bin/nc -zU /var/run/fail2ban/fail2ban.sock; do
+            sleep 1
+        done
+
+        ${pkgs.coreutils}/bin/chown root:fail2ban /var/run/fail2ban /var/run/fail2ban/fail2ban.sock
+        ${pkgs.coreutils}/bin/chmod 660 /var/run/fail2ban/fail2ban.sock
+        ${pkgs.coreutils}/bin/chmod 710 /var/run/fail2ban
+      '');
+  };
+}

configuration/services/metrics/default.nix

@@ -7,6 +7,10 @@
   domain = "metrics.${config.services.nginx.domain}";
   yaml = pkgs.formats.yaml {};
 in {
+  imports = [
+    ./exporters.nix
+  ];
+
   services.victoriametrics.enable = true;
 
   services.grafana = {
@@ -50,6 +54,38 @@ in {
       enabledCollectors = ["systemd"];
       listenAddress = "127.0.0.1";
     };
+
+    nginx = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+    };
+
+    nginxlog = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+    };
+  };
+
+  services.prometheus.local-exporters = {
+    prometheus-fail2ban-exporter = rec {
+      enable = true;
+      after = ["fail2ban.service"];
+
+      port = 9191;
+      listenAddress = "127.0.0.1";
+
+      serviceConfig = {
+        Group = "fail2ban";
+
+        RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"];
+
+        ExecStart = lib.concatStringsSep " " [
+          "${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter"
+          "--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock"
+          "--web.listen-address='${listenAddress}:${toString port}'"
+        ];
+      };
+    };
   };
 
   systemd.services.export-to-victoriametrics = let
@@ -62,7 +98,7 @@ in {
               targets =
                 lib.mapAttrsToList (name: exporter: "${exporter.listenAddress}:${toString exporter.port}")
                 (lib.filterAttrs (name: exporter: (builtins.isAttrs exporter) && exporter.enable)
-                  config.services.prometheus.exporters);
+                  (config.services.prometheus.exporters // config.services.prometheus.local-exporters));
             }
           ];
         }

configuration/services/metrics/exporters.nix

@@ -0,0 +1,45 @@
+{
+  config,
+  lib,
+  ...
+}: {
+  options.services.prometheus.local-exporters = lib.mkOption {
+    type = lib.types.anything;
+  };
+
+  config.systemd.services = lib.mapAttrs (_: exporter:
+    lib.mkMerge [
+      {
+        wantedBy = ["multi-user.target"];
+        after = ["network.target"];
+
+        serviceConfig = {
+          Restart = "always";
+          PrivateTmp = true;
+          WorkingDirectory = "/tmp";
+          DynamicUser = true;
+          LockPersonality = true;
+          MemoryDenyWriteExecute = true;
+          NonNewPrivileges = true;
+          PrivateDevices = true;
+          ProtectClock = true;
+          ProtectControlGroups = true;
+          ProtectHome = true;
+          ProtectHostname = true;
+          ProtectKernelLogs = true;
+          ProtectKernelModules = true;
+          ProtectKernelTunables = true;
+          ProtectSystem = "strict";
+          RemoveIPC = true;
+          RestrictAddressFamilies = lib.mkDefault ["AF_INET" "AF_INET6"];
+          RestrictNamespaces = true;
+          RestrictRealtime = true;
+          RestrictSUIDSGID = true;
+          SystemCallArchitectures = "native";
+          UMask = "0077";
+        };
+      }
+      (removeAttrs exporter ["port" "listenAddress"])
+    ])
+  config.services.prometheus.local-exporters;
+}

flake.nix

@@ -78,7 +78,7 @@
     # Utility scripts #
     ###################
     packages.${system} = let
-      inherit (nixpkgs.legacyPackages.${system}) writeShellScript;
+      inherit (nixpkgs.legacyPackages.${system}) writeShellScript writeShellScriptBin;
       vm = nixpkgs.lib.nixosSystem {
         inherit system;
         specialArgs.flake-inputs = inputs;
@@ -106,6 +106,14 @@
           "${vm.config.system.build.vm}/bin/run-tlaternet-vm"
         '';
 
+      update-pkgs = let
+        nvfetcher-bin = "${nvfetcher.packages.${system}.default}/bin/nvfetcher";
+      in
+        writeShellScriptBin "update-pkgs" ''
+          cd "$(git rev-parse --show-toplevel)/pkgs"
+          ${nvfetcher-bin} -o _sources_pkgs -c nvfetcher.toml
+        '';
+
       update-nextcloud-apps = let
         nvfetcher-bin = "${nvfetcher.packages.${system}.default}/bin/nvfetcher";
       in

pkgs/_sources_pkgs/generated.json

@@ -0,0 +1,21 @@
+{
+    "prometheus-fail2ban-exporter": {
+        "cargoLocks": null,
+        "date": null,
+        "extract": null,
+        "name": "prometheus-fail2ban-exporter",
+        "passthru": null,
+        "pinned": false,
+        "src": {
+            "deepClone": false,
+            "fetchSubmodules": false,
+            "leaveDotGit": false,
+            "name": null,
+            "rev": "v0.10.0",
+            "sha256": "sha256-8nIW1XaHCBqQCoLkV1ZYE3NTbVZ6c+UOqYD08XQiv+4=",
+            "type": "git",
+            "url": "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter"
+        },
+        "version": "v0.10.0"
+    }
+}

pkgs/_sources_pkgs/generated.nix

@@ -0,0 +1,16 @@
+# This file was generated by nvfetcher, please do not modify it manually.
+{ fetchgit, fetchurl, fetchFromGitHub, dockerTools }:
+{
+  prometheus-fail2ban-exporter = {
+    pname = "prometheus-fail2ban-exporter";
+    version = "v0.10.0";
+    src = fetchgit {
+      url = "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter";
+      rev = "v0.10.0";
+      fetchSubmodules = false;
+      deepClone = false;
+      leaveDotGit = false;
+      sha256 = "sha256-8nIW1XaHCBqQCoLkV1ZYE3NTbVZ6c+UOqYD08XQiv+4=";
+    };
+  };
+}

pkgs/default.nix

@@ -7,6 +7,9 @@
 in
   {
     starbound = callPackage ./starbound {};
+    prometheus-fail2ban-exporter = callPackage ./prometheus/fail2ban-exporter.nix {
+      sources = pkgs.callPackage ./_sources_pkgs/generated.nix {};
+    };
   }
   // (
     # Add nextcloud apps

pkgs/nvfetcher.toml

@@ -0,0 +1,3 @@
+[prometheus-fail2ban-exporter]
+src.manual = "v0.10.0" # No gitlab support in nvfetcher
+fetch.git = "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter"

pkgs/prometheus/fail2ban-exporter.nix

@@ -0,0 +1,8 @@
+{
+  buildGoModule,
+  sources,
+}:
+buildGoModule {
+  inherit (sources.prometheus-fail2ban-exporter) pname src version;
+  vendorHash = "sha256-qU6opwhhvzbQOhfGVyiVgKhfCSB0Z4eSRAJnv6ht2I0=";
+}