metrics: Add metrics with victoriametrics + grafana

configuration/default.nix

@@ -16,8 +16,10 @@
 
     ./services/backups.nix
     ./services/conduit.nix
+    ./services/fail2ban.nix
     ./services/foundryvtt.nix
     ./services/gitea.nix
+    ./services/metrics
     ./services/nextcloud.nix
     ./services/webserver.nix
     ./services/wireguard.nix
@@ -136,34 +138,45 @@
     recommendedProxySettings = true;
     clientMaxBodySize = "10G";
     domain = "tlater.net";
+
+    statusPage = true; # For metrics, should be accessible only from localhost
+
+    commonHttpConfig = ''
+      log_format upstream_time '$remote_addr - $remote_user [$time_local] '
+                         '"$request" $status $body_bytes_sent '
+                         '"$http_referer" "$http_user_agent" '
+                         'rt=$request_time uct="$upstream_connect_time" '
+                         'uht="$upstream_header_time" urt="$upstream_response_time"';
+    '';
   };
 
+  services.logrotate = {
+    enable = true;
+
+    settings = lib.mapAttrs' (virtualHost: _:
+      lib.nameValuePair "/var/log/nginx/${virtualHost}/access.log" {
+        frequency = "daily";
+        rotate = 2;
+        compress = true;
+        delaycompress = true;
+        su = "${config.services.nginx.user} ${config.services.nginx.group}";
+        postrotate = "[ ! -f /var/run/nginx/nginx.pid ] || kill -USR1 `cat /var/run/nginx/nginx.pid`";
+      })
+    config.services.nginx.virtualHosts;
+  };
+  systemd.tmpfiles.rules =
+    lib.mapAttrsToList (
+      virtualHost: _:
+      #
+      "d /var/log/nginx/${virtualHost} 0750 ${config.services.nginx.user} ${config.services.nginx.group}"
+    )
+    config.services.nginx.virtualHosts;
+
   security.acme = {
     defaults.email = "tm@tlater.net";
     acceptTerms = true;
   };
 
-  services.fail2ban = {
-    enable = true;
-    extraPackages = [pkgs.ipset];
-    banaction = "iptables-ipset-proto6-allports";
-    bantime-increment.enable = true;
-
-    jails = {
-      nginx-botsearch = ''
-        enabled = true
-        logpath = /var/log/nginx/access.log
-      '';
-    };
-
-    ignoreIP = [
-      "127.0.0.0/8"
-      "10.0.0.0/8"
-      "172.16.0.0/12"
-      "192.168.0.0/16"
-    ];
-  };
-
   # Remove some unneeded packages
   environment.defaultPackages = [];
 

configuration/services/conduit.nix

@@ -173,6 +173,9 @@ in {
       # Various other security settings
       no-tlsv1
       no-tlsv1_1
+
+      # Monitoring
+      prometheus
     '';
   };
 
@@ -205,6 +208,7 @@ in {
     addSSL = true;
     extraConfig = ''
       merge_slashes off;
+      access_log /var/log/nginx/${domain}/access.log upstream_time;
     '';
 
     locations = {

configuration/services/fail2ban.nix

@@ -0,0 +1,42 @@
+{pkgs, ...}: {
+  services.fail2ban = {
+    enable = true;
+    extraPackages = [pkgs.ipset];
+    banaction = "iptables-ipset-proto6-allports";
+    bantime-increment.enable = true;
+
+    jails = {
+      nginx-botsearch = ''
+        enabled = true
+        logpath = /var/log/nginx/access.log
+      '';
+    };
+
+    ignoreIP = [
+      "127.0.0.0/8"
+      "10.0.0.0/8"
+      "172.16.0.0/12"
+      "192.168.0.0/16"
+    ];
+  };
+
+  # Allow metrics services to connect to the socket as well
+  users.groups.fail2ban = {};
+  systemd.services.fail2ban.serviceConfig = {
+    ExecStartPost =
+      "+"
+      + (pkgs.writeShellScript "fail2ban-post-start" ''
+        while ! [ -S /var/run/fail2ban/fail2ban.sock ]; do
+            sleep 1
+        done
+
+        while ! ${pkgs.netcat}/bin/nc -zU /var/run/fail2ban/fail2ban.sock; do
+            sleep 1
+        done
+
+        ${pkgs.coreutils}/bin/chown root:fail2ban /var/run/fail2ban /var/run/fail2ban/fail2ban.sock
+        ${pkgs.coreutils}/bin/chmod 660 /var/run/fail2ban/fail2ban.sock
+        ${pkgs.coreutils}/bin/chmod 710 /var/run/fail2ban
+      '');
+  };
+}

configuration/services/foundryvtt.nix

@@ -25,6 +25,7 @@ in {
     enableACME = true;
     extraConfig = ''
       add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+      access_log /var/log/nginx/${domain}/access.log upstream_time;
     '';
 
     locations."/" = {

configuration/services/gitea.nix

@@ -1,6 +1,7 @@
 {
   pkgs,
   config,
+  lib,
   ...
 }: let
   domain = "gitea.${config.services.nginx.domain}";
@@ -19,11 +20,23 @@ in {
         SSH_PORT = 2222;
       };
 
+      metrics = {
+        ENABLED = true;
+        TOKEN = "#metricstoken#";
+      };
       service.DISABLE_REGISTRATION = true;
       session.COOKIE_SECURE = true;
     };
   };
 
+  systemd.services.gitea.serviceConfig.ExecStartPre = let
+    replaceSecretBin = "${pkgs.replace-secret}/bin/replace-secret";
+    secretPath = config.sops.secrets."gitea/metrics-token".path;
+    runConfig = "${config.services.gitea.customDir}/conf/app.ini";
+  in [
+    "+${replaceSecretBin} '#metricstoken#' '${secretPath}' '${runConfig}'"
+  ];
+
   # Set up SSL
   services.nginx.virtualHosts."${domain}" = let
     httpAddress = config.services.gitea.settings.server.HTTP_ADDR;
@@ -33,9 +46,18 @@ in {
     enableACME = true;
     extraConfig = ''
       add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+      access_log /var/log/nginx/${domain}/access.log upstream_time;
     '';
 
     locations."/".proxyPass = "http://${httpAddress}:${toString httpPort}";
+    locations."/metrics" = {
+      extraConfig = ''
+        access_log off;
+        allow 127.0.0.1;
+        ${lib.optionalString config.networking.enableIPv6 "allow ::1;"}
+        deny all;
+      '';
+    };
   };
 
   # Block repeated failed login attempts

configuration/services/metrics/default.nix

@@ -0,0 +1,9 @@
+{
+  imports = [
+    ./options.nix
+
+    ./exporters.nix
+    ./grafana.nix
+    ./victoriametrics.nix
+  ];
+}

configuration/services/metrics/exporters.nix

@@ -0,0 +1,101 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}: let
+  yaml = pkgs.formats.yaml {};
+in {
+  services.prometheus = {
+    exporters = {
+      # Periodically check domain registration status
+      domain = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        extraFlags = let
+          conf.domains = [
+            "tlater.net"
+            "tlater.com"
+          ];
+        in [
+          "--config=${yaml.generate "domains.yml" conf}"
+        ];
+      };
+
+      # System statistics
+      node = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+      };
+      systemd = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        extraFlags = [
+          # Disabled by default because only supported from systemd 235+
+          "--systemd.collector.enable-restart-count"
+          "--systemd.collector.enable-ip-accounting"
+        ];
+      };
+
+      # Various nginx metrics
+      nginx = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+      };
+
+      nginxlog = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        group = "nginx";
+
+        settings.namespaces =
+          lib.mapAttrsToList (name: virtualHost: {
+            inherit name;
+            metrics_override.prefix = "nginxlog";
+            namespace_label = "vhost";
+
+            format = lib.concatStringsSep " " [
+              "$remote_addr - $remote_user [$time_local]"
+              ''"$request" $status $body_bytes_sent''
+              ''"$http_referer" "$http_user_agent"''
+              ''rt=$request_time uct="$upstream_connect_time"''
+              ''uht="$upstream_header_time" urt="$upstream_response_time"''
+            ];
+
+            source.files = [
+              "/var/log/nginx/${name}/access.log"
+            ];
+          })
+          config.services.nginx.virtualHosts;
+      };
+    };
+
+    extraExporters = {
+      fail2ban = let
+        cfg = config.services.prometheus.extraExporters.fail2ban;
+      in {
+        port = 9191;
+        serviceOpts = {
+          after = ["fail2ban.service"];
+          requires = ["fail2ban.service"];
+          serviceConfig = {
+            Group = "fail2ban";
+            RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"];
+            ExecStart = lib.concatStringsSep " " [
+              "${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter"
+              "--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock"
+              "--web.listen-address='${cfg.listenAddress}:${toString cfg.port}'"
+              "--collector.f2b.exit-on-socket-connection-error=true"
+            ];
+          };
+        };
+      };
+    };
+
+    # TODO(tlater):
+    #   - wireguard (?)
+    #   - postgres (?)
+    #   - blackbox (?) (curl to see if http and similar is up)
+    #   - ssl_exporter (?)
+  };
+}

configuration/services/metrics/grafana.nix

@@ -0,0 +1,48 @@
+{config, ...}: let
+  domain = "metrics.${config.services.nginx.domain}";
+in {
+  services.grafana = {
+    enable = true;
+    settings = {
+      server.http_port = 3001; # Default overlaps with gitea
+
+      security = {
+        admin_user = "tlater";
+        admin_password = "$__file{${config.sops.secrets."grafana/adminPassword".path}}";
+        secret_key = "$__file{${config.sops.secrets."grafana/secretKey".path}}";
+        cookie_secure = true;
+        cookie_samesite = "strict";
+        content_security_policy = true;
+      };
+
+      database = {
+        user = "grafana";
+        name = "grafana";
+        type = "postgres";
+        host = "/run/postgresql";
+      };
+    };
+
+    provision = {
+      enable = true;
+
+      datasources.settings.datasources = [
+        {
+          name = "Victoriametrics - tlater.net";
+          url = "http://localhost:8428";
+          type = "prometheus";
+        }
+      ];
+    };
+  };
+
+  services.nginx.virtualHosts."${domain}" = {
+    forceSSL = true;
+    enableACME = true;
+    extraConfig = ''
+      add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+      access_log /var/log/nginx/${domain}/access.log upstream_time;
+    '';
+    locations."/".proxyPass = "http://localhost:${toString config.services.grafana.settings.server.http_port}";
+  };
+}

configuration/services/metrics/options.nix

@@ -0,0 +1,204 @@
+{
+  pkgs,
+  config,
+  lib,
+  ...
+}: let
+  inherit (lib) types mkOption mkDefault;
+  yaml = pkgs.formats.yaml {};
+in {
+  options = {
+    services.prometheus = {
+      extraExporters = mkOption {
+        type = types.attrsOf (types.submodule {
+          options = {
+            port = mkOption {
+              type = types.int;
+              description = "The port on which this exporter listens.";
+            };
+            listenAddress = mkOption {
+              type = types.str;
+              default = "127.0.0.1";
+              description = "Address to listen on.";
+            };
+            serviceOpts = mkOption {
+              type = types.attrs;
+              description = "An attrset to be merged with the exporter's systemd service.";
+            };
+          };
+        });
+      };
+    };
+
+    services.victoriametrics.scrapeConfigs = mkOption {
+      type = types.attrsOf (types.submodule ({
+        name,
+        self,
+        ...
+      }: {
+        options = {
+          job_name = mkOption {
+            type = types.str;
+            default = name;
+          };
+
+          extraSettings = mkOption {
+            type = types.anything;
+            description = ''
+              Other settings to set for this scrape config.
+            '';
+            default = {};
+          };
+
+          targets = mkOption {
+            type = types.listOf types.str;
+            description = lib.mdDoc ''
+              Addresses scrape targets for this config listen on.
+
+              Shortcut for `static_configs = lib.singleton {targets = [<targets>];}`
+            '';
+            default = [];
+          };
+
+          static_configs = mkOption {
+            default = [];
+            type = types.listOf (types.submodule {
+              options = {
+                targets = mkOption {
+                  type = types.listOf types.str;
+                  description = lib.mdDoc ''
+                    The addresses scrape targets for this config listen on.
+
+                    Must in `listenAddress:port` format.
+                  '';
+                };
+                labels = mkOption {
+                  type = types.attrsOf types.str;
+                  description = lib.mdDoc ''
+                    Labels to apply to all targets defined for this static config.
+                  '';
+                  default = {};
+                };
+              };
+            });
+          };
+        };
+      }));
+    };
+  };
+
+  config = {
+    systemd.services = lib.mkMerge [
+      (lib.mapAttrs' (name: exporter:
+        lib.nameValuePair "prometheus-${name}-exporter" (lib.mkMerge [
+          {
+            # Shamelessly copied from upstream because the upstream
+            # module is an intractable mess
+            wantedBy = ["multi-user.target"];
+            after = ["network.target"];
+            serviceConfig.Restart = mkDefault "always";
+            serviceConfig.PrivateTmp = mkDefault true;
+            serviceConfig.WorkingDirectory = mkDefault /tmp;
+            serviceConfig.DynamicUser = mkDefault true;
+            # Hardening
+            serviceConfig.CapabilityBoundingSet = mkDefault [""];
+            serviceConfig.DeviceAllow = [""];
+            serviceConfig.LockPersonality = true;
+            serviceConfig.MemoryDenyWriteExecute = true;
+            serviceConfig.NoNewPrivileges = true;
+            serviceConfig.PrivateDevices = mkDefault true;
+            serviceConfig.ProtectClock = mkDefault true;
+            serviceConfig.ProtectControlGroups = true;
+            serviceConfig.ProtectHome = true;
+            serviceConfig.ProtectHostname = true;
+            serviceConfig.ProtectKernelLogs = true;
+            serviceConfig.ProtectKernelModules = true;
+            serviceConfig.ProtectKernelTunables = true;
+            serviceConfig.ProtectSystem = mkDefault "strict";
+            serviceConfig.RemoveIPC = true;
+            serviceConfig.RestrictAddressFamilies = ["AF_INET" "AF_INET6"];
+            serviceConfig.RestrictNamespaces = true;
+            serviceConfig.RestrictRealtime = true;
+            serviceConfig.RestrictSUIDSGID = true;
+            serviceConfig.SystemCallArchitectures = "native";
+            serviceConfig.UMask = "0077";
+          }
+          exporter.serviceOpts
+        ]))
+      config.services.prometheus.extraExporters)
+
+      {
+        vmagent-scrape-exporters = let
+          listenAddress = config.services.victoriametrics.listenAddress;
+          vmAddr = (lib.optionalString (lib.hasPrefix ":" listenAddress) "127.0.0.1") + listenAddress;
+          promscrape = yaml.generate "prometheus.yml" {
+            scrape_configs = lib.mapAttrsToList (_: scrape:
+              lib.recursiveUpdate {
+                inherit (scrape) job_name;
+                static_configs =
+                  scrape.static_configs
+                  ++ lib.optional (scrape.targets != []) {targets = scrape.targets;};
+              }
+              scrape.extraSettings)
+            config.services.victoriametrics.scrapeConfigs;
+          };
+        in {
+          enable = true;
+          path = [pkgs.victoriametrics];
+          wantedBy = ["multi-user.target"];
+          after = ["network.target" "victoriametrics.service"];
+          serviceConfig = {
+            ExecStart = [
+              (lib.concatStringsSep " " [
+                "${pkgs.victoriametrics}/bin/vmagent"
+                "-promscrape.config=${promscrape}"
+                "-remoteWrite.url=http://${vmAddr}/api/v1/write"
+                "-remoteWrite.tmpDataPath=%t/vmagent"
+              ])
+            ];
+            SupplementaryGroups = "metrics";
+
+            DynamicUser = true;
+            RuntimeDirectory = "vmagent";
+            CapabilityBoundingSet = [""];
+            DeviceAllow = [""];
+            LockPersonality = true;
+            MemoryDenyWriteExecute = true;
+            NoNewPrivileges = true;
+            PrivateDevices = true;
+            ProtectClock = true;
+            ProtectControlGroups = true;
+            ProtectHome = true;
+            ProtectHostname = true;
+            ProtectKernelLogs = true;
+            ProtectKernelModules = true;
+            ProtectKernelTunables = true;
+            ProtectSystem = "strict";
+            RemoveIPC = true;
+            RestrictAddressFamilies = ["AF_INET" "AF_INET6"];
+            RestrictNamespaces = true;
+            RestrictRealtime = true;
+            RestrictSUIDSGID = true;
+            SystemCallArchitectures = "native";
+            UMask = "0077";
+          };
+        };
+      }
+    ];
+
+    users.groups.metrics = {};
+
+    services.victoriametrics.scrapeConfigs = let
+      allExporters =
+        lib.mapAttrs (name: exporter: {
+          inherit (exporter) listenAddress port;
+        }) ((lib.filterAttrs (_: exporter: builtins.isAttrs exporter && exporter.enable)
+          config.services.prometheus.exporters)
+        // config.services.prometheus.extraExporters);
+    in
+      lib.mapAttrs (_: exporter: {
+        targets = ["${exporter.listenAddress}:${toString exporter.port}"];
+      })
+      allExporters;
+  };
+}

configuration/services/metrics/victoriametrics.nix

@@ -0,0 +1,13 @@
+{config, ...}: {
+  config.services.victoriametrics = {
+    enable = true;
+
+    scrapeConfigs = {
+      gitea = {
+        targets = ["127.0.0.1:${toString config.services.gitea.settings.server.HTTP_PORT}"];
+        extraSettings.authorization.credentials_file = config.sops.secrets."gitea/metrics-token".path;
+      };
+      coturn.targets = ["127.0.0.1:9641"];
+    };
+  };
+}

configuration/services/nextcloud.nix

@@ -50,6 +50,9 @@ in {
   services.nginx.virtualHosts."${hostName}" = {
     forceSSL = true;
     enableACME = true;
+    extraConfig = ''
+      access_log /var/log/nginx/${hostName}/access.log upstream_time;
+    '';
   };
 
   # Block repeated failed login attempts

configuration/services/postgres.nix

@@ -16,6 +16,12 @@
     # that operation needs to be performed manually on the system as
     # well.
     ensureUsers = [
+      {
+        name = "grafana";
+        ensurePermissions = {
+          "DATABASE grafana" = "ALL PRIVILEGES";
+        };
+      }
       {
         name = "nextcloud";
         ensurePermissions = {
@@ -25,6 +31,7 @@
     ];
 
     ensureDatabases = [
+      "grafana"
       "nextcloud"
     ];
   };

configuration/services/webserver.nix

@@ -19,6 +19,7 @@ in {
     enableACME = true;
     extraConfig = ''
       add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+      access_log /var/log/nginx/${domain}/access.log upstream_time;
     '';
 
     locations."/".proxyPass = "http://${addr}:${toString port}";

configuration/sops.nix

@@ -3,28 +3,44 @@
     defaultSopsFile = ../keys/production.yaml;
 
     secrets = {
+      # Gitea
+      "gitea/metrics-token" = {
+        owner = "gitea";
+        group = "metrics";
+        mode = "0440";
+      };
+
+      # Grafana
+      "grafana/adminPassword" = {
+        owner = "grafana";
+        group = "grafana";
+      };
+      "grafana/secretKey" = {
+        owner = "grafana";
+        group = "grafana";
+      };
+
+      # Heisenbridge
+      "heisenbridge/as-token" = {};
+      "heisenbridge/hs-token" = {};
+
+      # Nextcloud
       "nextcloud/tlater" = {
         owner = "nextcloud";
         group = "nextcloud";
       };
 
-      "steam/tlater" = {};
-
-      "heisenbridge/as-token" = {};
-      "heisenbridge/hs-token" = {};
-
-      "wireguard/server-key" = {
-        owner = "root";
-        group = "systemd-network";
-        mode = "0440";
-      };
-
+      # Restic
       "restic/local-backups" = {
         owner = "root";
         group = "backup";
         mode = "0440";
       };
 
+      # Steam
+      "steam/tlater" = {};
+
+      # Turn
       "turn/env" = {};
       "turn/secret" = {
         owner = "turnserver";
@@ -35,6 +51,13 @@
       "turn/ssl-cert" = {
         owner = "turnserver";
       };
+
+      # Wireguard
+      "wireguard/server-key" = {
+        owner = "root";
+        group = "systemd-network";
+        mode = "0440";
+      };
     };
   };
 }

flake.nix

@@ -78,7 +78,7 @@
     # Utility scripts #
     ###################
     packages.${system} = let
-      inherit (nixpkgs.legacyPackages.${system}) writeShellScript;
+      inherit (nixpkgs.legacyPackages.${system}) writeShellScript writeShellScriptBin;
       vm = nixpkgs.lib.nixosSystem {
         inherit system;
         specialArgs.flake-inputs = inputs;
@@ -106,6 +106,14 @@
           "${vm.config.system.build.vm}/bin/run-tlaternet-vm"
         '';
 
+      update-pkgs = let
+        nvfetcher-bin = "${nvfetcher.packages.${system}.default}/bin/nvfetcher";
+      in
+        writeShellScriptBin "update-pkgs" ''
+          cd "$(git rev-parse --show-toplevel)/pkgs"
+          ${nvfetcher-bin} -o _sources_pkgs -c nvfetcher.toml
+        '';
+
       update-nextcloud-apps = let
         nvfetcher-bin = "${nvfetcher.packages.${system}.default}/bin/nvfetcher";
       in

keys/production.yaml

@@ -1,3 +1,8 @@
+gitea:
+    metrics-token: ENC[AES256_GCM,data:/7/zvVl2ZOBoekrJR32vl/QQcG5XqTmltgpHEMUpbXVeqwnq29idzE2Qyjau96ZHObmSI73/ZtW95uXF6LH9Qw==,iv:iWZECCZSh1CN7wMBqstXR5QWtriR7QLKVqhekGnpXl0=,tag:HEr9km8VYmruBzf0I/5HuA==,type:str]
+grafana:
+    adminPassword: ENC[AES256_GCM,data:/qw//J7cOkIGa58bG4GgdzndvKof32AmQeWB00IX8WhA22PDCOc4VdUEoB3wVJJqI/ucoHFInYyhg2rFYoYBesBjAt0QS3+O+8WblIunUuYeqlBuYJJK1TLhy6ql6+aqvfiW/rJLm4LpgA7CboyDD2OYHcAbvGSD2GWwFcHTR/Y=,iv:KK6p8GKzc9SBDZZFkEwCdIjSxriPGNMDNcr97tfbwTI=,tag:gLRNSGdJWFD+V9K5TfJvXw==,type:str]
+    secretKey: ENC[AES256_GCM,data:OUXWOE6I3a26SrFEOczWNIwyR3Rx62fbsRBBcfh0xyEbxOIPhexH6lIqlVG9Ltwra9+rAldNM4/0BydtxIDj7A==,iv:fiNO/or5yZnhpDPMANDnEC5dtXmbKBZsV+BPmvCN/HI=,tag:Q0M0OtLWdWAJgQmUlL//fg==,type:str]
 nextcloud:
     tlater: ENC[AES256_GCM,data:zNsPm4uFaIRe3LjcwmayRg==,iv:5wam6bP5zP708jC9UrLV0s8qspl3Pm4fPzbMFYBUyPQ=,tag:apnJUMeJwMn9q0NhO4ptmA==,type:str]
 steam:
@@ -21,8 +26,8 @@ sops:
     azure_kv: []
     hc_vault: []
     age: []
-    lastmodified: "2023-09-23T18:55:44Z"
-    mac: ENC[AES256_GCM,data:psqgXozY9L7nduZ11GF+mbIrZ4RUySqBixkWL5z0cYeoLA3URb/dr028LCmNgQS9l8aJVsjVkyLBJIU/8wmiUNqRy/VI5iqV5mu+sxXhUVwFL0dAAWP1lOKwwT5uGK89/ioqkphgzuWD37vGe2vYddKkJF0M+zlz12fqkMjaisU=,iv:UyRoJbfuGU3K/Mp5DQ1kY0Z+nKSSo46BGNAcxt+vAvc=,tag:HkP6+qxQ8J/xAYJXYoG/6g==,type:str]
+    lastmodified: "2023-10-12T18:40:26Z"
+    mac: ENC[AES256_GCM,data:F+yQ20jCtLRKeQDFVKoqrYCgtwGkXxrK6aQO0MFZTIMJAnbTVPM2ZJGQ1RxXb+Zs4T+44EEc2xN4LjeANvgpE6MfOz2VTw+sEEjcYwUyB6RcXHia9XlFLa8lh7/Wx/9DxlSFjjSrxmDkNB6r+n5UF81cdRXF2E9ibdH346ST98A=,iv:xVxFN1IDKrLskaGqnWvOWx1zUII0jRSjQxEsaTf2GNw=,tag:lnp1AvgMOXXlg1vFjHEWUQ==,type:str]
     pgp:
         - created_at: "2022-10-12T00:46:51Z"
           enc: |

keys/staging.yaml

@@ -1,3 +1,8 @@
+gitea:
+    metrics-token: ENC[AES256_GCM,data:J4QdfI1wKyM=,iv:8fqCbftyhj90eIVFxjEp9RXKC1y1IaLnV1r2MOdY15M=,tag:8W/juv1OZh4hJco02qXO6g==,type:str]
+grafana:
+    adminPassword: ENC[AES256_GCM,data:dYfaxUpQpzA=,iv:j5wSem8C5+V4c5qRzXQJhsU7/FOtpvrnaEyFBmW6zJ4=,tag:oc8n3TkEbjF2gjuOobZuLA==,type:str]
+    secretKey: ENC[AES256_GCM,data:Atruvh2MsNY=,iv:y2MaCUCEzGIydHp6G0DJHfk289S1is0twKm2oUYwDhM=,tag:nAWeg+YqaYqk6k22oBkAhQ==,type:str]
 nextcloud:
     tlater: ENC[AES256_GCM,data:91kDcO4hpng=,iv:ayuILRmRru4ZxTCur9H2xHuLjkDzwPdS/4lEog/tesU=,tag:qYhJxnNDcCwUM7xe7Tlcjw==,type:str]
 steam:
@@ -21,8 +26,8 @@ sops:
     azure_kv: []
     hc_vault: []
     age: []
-    lastmodified: "2023-09-22T21:07:02Z"
-    mac: ENC[AES256_GCM,data:gItC41S8MInLmikdH1okhPs+FVf8sCF/iQeJ5reigBunHkOngoc6nOFANyAcNZETszzhgTLXXtmVNEjW46v6K7D6nmoi/zwpedUxwzMwDC5I28VTMDHVMAThYSGtdo6kig8i2pi8rzEQd1DStxMv3TWML5y6DDTlFsd3lfudaHA=,iv:zXebvIVPR76GwUhpactwRgF/eEmx2OBkT18E8lkwzRA=,tag:6HyISACbFCGlpIIgkFeA/A==,type:str]
+    lastmodified: "2023-10-07T02:17:50Z"
+    mac: ENC[AES256_GCM,data:vZDq33YIn0Nf1FQ2+ySezox6igiw6zNFCu3l3kaIsBKo1797pohmAxj2Lcc+OmlBjj98khaBIlbQuA5ULM+uPN5ILaz3NuXD5PZtsV+rL2PsLNMW9FBSmJ0m0YQrt0nZ0tpzifn12XghcSK2IXv+FnxlfrAJCxDvr5tRm90uUwU=,iv:ct8CzIWjaoJ1UjZcdFSr8lZ626vA0RvM883V6H5plWc=,tag:waJNtp/UbRDOfyzNElrung==,type:str]
     pgp:
         - created_at: "2022-10-12T16:48:23Z"
           enc: |

pkgs/_sources_pkgs/generated.json

@@ -0,0 +1,21 @@
+{
+    "prometheus-fail2ban-exporter": {
+        "cargoLocks": null,
+        "date": null,
+        "extract": null,
+        "name": "prometheus-fail2ban-exporter",
+        "passthru": null,
+        "pinned": false,
+        "src": {
+            "deepClone": false,
+            "fetchSubmodules": false,
+            "leaveDotGit": false,
+            "name": null,
+            "rev": "v0.10.0",
+            "sha256": "sha256-8nIW1XaHCBqQCoLkV1ZYE3NTbVZ6c+UOqYD08XQiv+4=",
+            "type": "git",
+            "url": "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter"
+        },
+        "version": "v0.10.0"
+    }
+}

pkgs/_sources_pkgs/generated.nix

@@ -0,0 +1,16 @@
+# This file was generated by nvfetcher, please do not modify it manually.
+{ fetchgit, fetchurl, fetchFromGitHub, dockerTools }:
+{
+  prometheus-fail2ban-exporter = {
+    pname = "prometheus-fail2ban-exporter";
+    version = "v0.10.0";
+    src = fetchgit {
+      url = "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter";
+      rev = "v0.10.0";
+      fetchSubmodules = false;
+      deepClone = false;
+      leaveDotGit = false;
+      sha256 = "sha256-8nIW1XaHCBqQCoLkV1ZYE3NTbVZ6c+UOqYD08XQiv+4=";
+    };
+  };
+}

pkgs/default.nix

@@ -7,6 +7,9 @@
 in
   {
     starbound = callPackage ./starbound {};
+    prometheus-fail2ban-exporter = callPackage ./prometheus/fail2ban-exporter.nix {
+      sources = pkgs.callPackage ./_sources_pkgs/generated.nix {};
+    };
   }
   // (
     # Add nextcloud apps

pkgs/nvfetcher.toml

@@ -0,0 +1,3 @@
+[prometheus-fail2ban-exporter]
+src.manual = "v0.10.0" # No gitlab support in nvfetcher
+fetch.git = "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter"

pkgs/prometheus/fail2ban-exporter.nix

@@ -0,0 +1,8 @@
+{
+  buildGoModule,
+  sources,
+}:
+buildGoModule {
+  inherit (sources.prometheus-fail2ban-exporter) pname src version;
+  vendorHash = "sha256-qU6opwhhvzbQOhfGVyiVgKhfCSB0Z4eSRAJnv6ht2I0=";
+}