fail2ban: Add metrics

configuration/default.nix

@@ -14,9 +14,12 @@
     "${modulesPath}/profiles/minimal.nix"
     (import ../modules)
 
+    ./services/backups.nix
     ./services/conduit.nix
+    ./services/fail2ban.nix
     ./services/foundryvtt.nix
     ./services/gitea.nix
+    ./services/metrics
     ./services/nextcloud.nix
     ./services/webserver.nix
     ./services/wireguard.nix
@@ -135,34 +138,45 @@
     recommendedProxySettings = true;
     clientMaxBodySize = "10G";
     domain = "tlater.net";
+
+    statusPage = true; # For metrics, should be accessible only from localhost
+
+    commonHttpConfig = ''
+      log_format upstream_time '$remote_addr - $remote_user [$time_local] '
+                         '"$request" $status $body_bytes_sent '
+                         '"$http_referer" "$http_user_agent" '
+                         'rt=$request_time uct="$upstream_connect_time" '
+                         'uht="$upstream_header_time" urt="$upstream_response_time"';
+    '';
   };
 
+  services.logrotate = {
+    enable = true;
+
+    settings = lib.mapAttrs' (virtualHost: _:
+      lib.nameValuePair "/var/log/nginx/${virtualHost}/access.log" {
+        frequency = "daily";
+        rotate = 2;
+        compress = true;
+        delaycompress = true;
+        su = "${config.services.nginx.user} ${config.services.nginx.group}";
+        postrotate = "[ ! -f /var/run/nginx/nginx.pid ] || kill -USR1 `cat /var/run/nginx/nginx.pid`";
+      })
+    config.services.nginx.virtualHosts;
+  };
+  systemd.tmpfiles.rules =
+    lib.mapAttrsToList (
+      virtualHost: _:
+      #
+      "d /var/log/nginx/${virtualHost} 0750 ${config.services.nginx.user} ${config.services.nginx.group}"
+    )
+    config.services.nginx.virtualHosts;
+
   security.acme = {
     defaults.email = "tm@tlater.net";
     acceptTerms = true;
   };
 
-  services.fail2ban = {
-    enable = true;
-    extraPackages = [pkgs.ipset];
-    banaction = "iptables-ipset-proto6-allports";
-    bantime-increment.enable = true;
-
-    jails = {
-      nginx-botsearch = ''
-        enabled = true
-        logpath = /var/log/nginx/access.log
-      '';
-    };
-
-    ignoreIP = [
-      "127.0.0.0/8"
-      "10.0.0.0/8"
-      "172.16.0.0/12"
-      "192.168.0.0/16"
-    ];
-  };
-
   # Remove some unneeded packages
   environment.defaultPackages = [];
 

configuration/services/backups.nix

@@ -0,0 +1,228 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}: let
+  inherit (lib) types optional singleton;
+  mkShutdownScript = service:
+    pkgs.writeShellScript "backup-${service}-shutdown" ''
+      if systemctl is-active --quiet '${service}'; then
+        touch '/tmp/${service}-was-active'
+        systemctl stop '${service}'
+      fi
+    '';
+  mkRestartScript = service:
+    pkgs.writeShellScript "backup-${service}-restart" ''
+      if [ -f '/tmp/${service}-was-active' ]; then
+        rm '/tmp/${service}-was-active'
+        systemctl start '${service}'
+      fi
+    '';
+  writeScript = name: packages: text:
+    lib.getExe (pkgs.writeShellApplication {
+      inherit name text;
+      runtimeInputs = packages;
+    });
+in {
+  options = {
+    services.backups = lib.mkOption {
+      description = lib.mdDoc ''
+        Configure restic backups with a specific tag.
+      '';
+      type = types.attrsOf (types.submodule ({
+        config,
+        name,
+        ...
+      }: {
+        options = {
+          user = lib.mkOption {
+            type = types.str;
+            description = ''
+              The user as which to run the backup.
+            '';
+          };
+          paths = lib.mkOption {
+            type = types.listOf types.str;
+            description = ''
+              The paths to back up.
+            '';
+          };
+          tag = lib.mkOption {
+            type = types.str;
+            description = ''
+              The restic tag to mark the backup with.
+            '';
+            default = name;
+          };
+          preparation = {
+            packages = lib.mkOption {
+              type = types.listOf types.package;
+              default = [];
+              description = ''
+                The list of packages to make available in the
+                preparation script.
+              '';
+            };
+            text = lib.mkOption {
+              type = types.nullOr types.str;
+              default = null;
+              description = ''
+                The preparation script to run before the backup.
+
+                This should include things like database dumps and
+                enabling maintenance modes. If a service needs to be
+                shut down for backups, use `pauseServices` instead.
+              '';
+            };
+          };
+          cleanup = {
+            packages = lib.mkOption {
+              type = types.listOf types.package;
+              default = [];
+              description = ''
+                The list of packages to make available in the
+                cleanup script.
+              '';
+            };
+            text = lib.mkOption {
+              type = types.nullOr types.str;
+              default = null;
+              description = ''
+                The cleanup script to run after the backup.
+
+                This should do things like cleaning up database dumps
+                and disabling maintenance modes.
+              '';
+            };
+          };
+          pauseServices = lib.mkOption {
+            type = types.listOf types.str;
+            default = [];
+            description = ''
+              The systemd services that need to be shut down before
+              the backup can run. Services will be restarted after the
+              backup is complete.
+
+              This is intended to be used for services that do not
+              support hot backups.
+            '';
+          };
+        };
+      }));
+    };
+  };
+
+  config = lib.mkIf (config.services.backups != {}) {
+    systemd.services =
+      {
+        restic-prune = {
+          # Doesn't hurt to finish the ongoing prune
+          restartIfChanged = false;
+
+          environment = {
+            RESTIC_PASSWORD_FILE = config.sops.secrets."restic/local-backups".path;
+            RESTIC_REPOSITORY = "/var/lib/backups/";
+          };
+
+          path = with pkgs; [
+            restic
+          ];
+
+          script = ''
+            # TODO(tlater): In an append-only setup, we should be
+            # careful with this; an attacker could delete backups by
+            # simply appending ad infinitum:
+            # https://restic.readthedocs.io/en/stable/060_forget.html#security-considerations-in-append-only-mode
+            restic forget --keep-last 3 --prune
+            restic check
+          '';
+
+          serviceConfig = {
+            DynamicUser = true;
+            Group = "backup";
+
+            CacheDirectory = "restic-prune";
+            CacheDirectoryMode = "0700";
+            ReadWritePaths = "/var/lib/backups/";
+
+            # Ensure we don't leave behind any files with the
+            # temporary UID of this service.
+            ExecStopPost = "+${pkgs.coreutils}/bin/chown -R root:backup /var/lib/backups/";
+          };
+        };
+      }
+      // lib.mapAttrs' (name: backup:
+        lib.nameValuePair "backup-${name}" {
+          # Don't want to restart mid-backup
+          restartIfChanged = false;
+
+          environment = {
+            RESTIC_CACHE_DIR = "%C/backup-${name}";
+            RESTIC_PASSWORD_FILE = config.sops.secrets."restic/local-backups".path;
+            # TODO(tlater): If I ever add more than one repo, service
+            # shutdown/restarting will potentially break if multiple
+            # backups for the same service overlap. A more clever
+            # sentinel file with reference counts would probably solve
+            # this.
+            RESTIC_REPOSITORY = "/var/lib/backups/";
+          };
+
+          serviceConfig = {
+            User = backup.user;
+            Group = "backup";
+            RuntimeDirectory = "backup-${name}";
+            CacheDirectory = "backup-${name}";
+            CacheDirectoryMode = "0700";
+            PrivateTmp = true;
+
+            ExecStart = [
+              (lib.concatStringsSep " " (["${pkgs.restic}/bin/restic" "backup" "--tag" name] ++ backup.paths))
+            ];
+
+            ExecStartPre =
+              map (service: "+${mkShutdownScript service}") backup.pauseServices
+              ++ singleton (writeScript "backup-${name}-repo-init" [pkgs.restic pkgs.coreutils] ''
+                restic snapshots || (restic init && chmod -R g+rwx "$RESTIC_REPOSITORY"/*)
+              '')
+              ++ optional (backup.preparation.text != null)
+              (writeScript "backup-${name}-prepare" backup.preparation.packages backup.preparation.text);
+
+            # TODO(tlater): Add repo pruning/checking
+            ExecStopPost =
+              map (service: "+${mkRestartScript service}") backup.pauseServices
+              ++ optional (backup.cleanup.text != null)
+              (writeScript "backup-${name}-cleanup" backup.cleanup.packages backup.cleanup.text);
+          };
+        })
+      config.services.backups;
+
+    systemd.timers =
+      {
+        restic-prune = {
+          wantedBy = ["timers.target"];
+          timerConfig.OnCalendar = "Thursday 03:00:00 UTC";
+          # Don't make this persistent, in case the server was offline
+          # for a while. This job cannot run at the same time as any
+          # of the backup jobs.
+        };
+      }
+      // lib.mapAttrs' (name: backup:
+        lib.nameValuePair "backup-${name}" {
+          wantedBy = ["timers.target"];
+          timerConfig = {
+            OnCalendar = "Wednesday 02:30:00 UTC";
+            RandomizedDelaySec = "1h";
+            FixedRandomDelay = true;
+            Persistent = true;
+          };
+        })
+      config.services.backups;
+
+    users.groups.backup = {};
+
+    systemd.tmpfiles.rules = [
+      "d /var/lib/backups/ 0770 root backup"
+    ];
+  };
+}

configuration/services/conduit.nix

@@ -205,6 +205,7 @@ in {
     addSSL = true;
     extraConfig = ''
       merge_slashes off;
+      access_log /var/log/nginx/${domain}/access.log upstream_time;
     '';
 
     locations = {
@@ -231,4 +232,14 @@ in {
       };
     };
   };
+
+  services.backups.conduit = {
+    user = "root";
+    paths = [
+      "/var/lib/private/matrix-conduit/"
+    ];
+    # Other services store their data in conduit, so no other services
+    # need to be shut down currently.
+    pauseServices = ["conduit.service"];
+  };
 }

configuration/services/fail2ban.nix

@@ -0,0 +1,42 @@
+{pkgs, ...}: {
+  services.fail2ban = {
+    enable = true;
+    extraPackages = [pkgs.ipset];
+    banaction = "iptables-ipset-proto6-allports";
+    bantime-increment.enable = true;
+
+    jails = {
+      nginx-botsearch = ''
+        enabled = true
+        logpath = /var/log/nginx/access.log
+      '';
+    };
+
+    ignoreIP = [
+      "127.0.0.0/8"
+      "10.0.0.0/8"
+      "172.16.0.0/12"
+      "192.168.0.0/16"
+    ];
+  };
+
+  # Allow metrics services to connect to the socket as well
+  users.groups.fail2ban = {};
+  systemd.services.fail2ban.serviceConfig = {
+    ExecStartPost =
+      "+"
+      + (pkgs.writeShellScript "fail2ban-post-start" ''
+        while ! [ -S /var/run/fail2ban/fail2ban.sock ]; do
+            sleep 1
+        done
+
+        while ! ${pkgs.netcat}/bin/nc -zU /var/run/fail2ban/fail2ban.sock; do
+            sleep 1
+        done
+
+        ${pkgs.coreutils}/bin/chown root:fail2ban /var/run/fail2ban /var/run/fail2ban/fail2ban.sock
+        ${pkgs.coreutils}/bin/chmod 660 /var/run/fail2ban/fail2ban.sock
+        ${pkgs.coreutils}/bin/chmod 710 /var/run/fail2ban
+      '');
+  };
+}

configuration/services/foundryvtt.nix

@@ -25,6 +25,7 @@ in {
     enableACME = true;
     extraConfig = ''
       add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+      access_log /var/log/nginx/${domain}/access.log upstream_time;
     '';
 
     locations."/" = {

configuration/services/gitea.nix

@@ -1,4 +1,8 @@
-{config, ...}: let
+{
+  pkgs,
+  config,
+  ...
+}: let
   domain = "gitea.${config.services.nginx.domain}";
 in {
   services.gitea = {
@@ -29,6 +33,7 @@ in {
     enableACME = true;
     extraConfig = ''
       add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+      access_log /var/log/nginx/${domain}/access.log upstream_time;
     '';
 
     locations."/".proxyPass = "http://${httpAddress}:${toString httpPort}";
@@ -52,4 +57,24 @@ in {
       enabled = true
     '';
   };
+
+  services.backups.gitea = {
+    user = "gitea";
+    paths = [
+      "/var/lib/gitea/gitea-db.sql"
+      "/var/lib/gitea/repositories/"
+      "/var/lib/gitea/data/"
+      "/var/lib/gitea/custom/"
+      # Conf is backed up via nix
+    ];
+    preparation = {
+      packages = [config.services.postgresql.package];
+      text = "pg_dump ${config.services.gitea.database.name} --file=/var/lib/gitea/gitea-db.sql";
+    };
+    cleanup = {
+      packages = [pkgs.coreutils];
+      text = "rm /var/lib/gitea/gitea-db.sql";
+    };
+    pauseServices = ["gitea.service"];
+  };
 }

configuration/services/metrics/default.nix

@@ -0,0 +1,144 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}: let
+  domain = "metrics.${config.services.nginx.domain}";
+  yaml = pkgs.formats.yaml {};
+in {
+  imports = [
+    ./exporters.nix
+  ];
+
+  services.victoriametrics.enable = true;
+
+  services.grafana = {
+    enable = true;
+    settings = {
+      server.http_port = 3001; # Default overlaps with gitea
+
+      security = {
+        admin_user = "tlater";
+        admin_password = "$__file{${config.sops.secrets."grafana/adminPassword".path}}";
+        secret_key = "$__file{${config.sops.secrets."grafana/secretKey".path}}";
+        cookie_secure = true;
+        cookie_samesite = "strict";
+        content_security_policy = true;
+      };
+
+      database = {
+        user = "grafana";
+        name = "grafana";
+        type = "postgres";
+        host = "/run/postgresql";
+      };
+    };
+
+    provision = {
+      enable = true;
+
+      datasources.settings.datasources = [
+        {
+          name = "Victoriametrics - tlater.net";
+          url = "http://localhost:8428";
+          type = "prometheus";
+        }
+      ];
+    };
+  };
+
+  services.prometheus.exporters = {
+    node = {
+      enable = true;
+      enabledCollectors = ["systemd"];
+      listenAddress = "127.0.0.1";
+    };
+
+    nginx = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+    };
+
+    nginxlog = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      group = "nginx";
+
+      settings.namespaces =
+        lib.mapAttrsToList (name: virtualHost: {
+          inherit name;
+          metrics_override.prefix = "nginxlog";
+          namespace_label = "vhost";
+
+          format = lib.concatStringsSep " " [
+            "$remote_addr - $remote_user [$time_local]"
+            ''"$request" $status $body_bytes_sent''
+            ''"$http_referer" "$http_user_agent"''
+            ''rt=$request_time uct="$upstream_connect_time"''
+            ''uht="$upstream_header_time" urt="$upstream_response_time"''
+          ];
+
+          source.files = [
+            "/var/log/nginx/${name}/access.log"
+          ];
+        })
+        config.services.nginx.virtualHosts;
+    };
+  };
+
+  services.prometheus.local-exporters = {
+    prometheus-fail2ban-exporter = rec {
+      enable = true;
+      after = ["fail2ban.service"];
+
+      port = 9191;
+      listenAddress = "127.0.0.1";
+
+      serviceConfig = {
+        Group = "fail2ban";
+
+        RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"];
+
+        ExecStart = lib.concatStringsSep " " [
+          "${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter"
+          "--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock"
+          "--web.listen-address='${listenAddress}:${toString port}'"
+        ];
+      };
+    };
+  };
+
+  systemd.services.export-to-victoriametrics = let
+    promscrape = yaml.generate "prometheus.yml" {
+      scrape_configs = [
+        {
+          job_name = "tlater.net";
+          static_configs = [
+            {
+              targets =
+                lib.mapAttrsToList (name: exporter: "${exporter.listenAddress}:${toString exporter.port}")
+                (lib.filterAttrs (name: exporter: (builtins.isAttrs exporter) && exporter.enable)
+                  (config.services.prometheus.exporters // config.services.prometheus.local-exporters));
+            }
+          ];
+        }
+      ];
+    };
+  in {
+    enable = true;
+    path = [pkgs.victoriametrics];
+    wantedBy = ["multi-user.target"];
+    script = "vmagent -promscrape.config=${promscrape} -remoteWrite.url=http://localhost:8428/api/v1/write";
+  };
+
+  services.nginx.virtualHosts."${domain}" = {
+    forceSSL = true;
+    enableACME = true;
+    extraConfig = ''
+      add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+      access_log /var/log/nginx/${domain}/access.log upstream_time;
+    '';
+    locations."/".proxyPass = "http://localhost:3001";
+  };
+}

configuration/services/metrics/exporters.nix

@@ -0,0 +1,45 @@
+{
+  config,
+  lib,
+  ...
+}: {
+  options.services.prometheus.local-exporters = lib.mkOption {
+    type = lib.types.anything;
+  };
+
+  config.systemd.services = lib.mapAttrs (_: exporter:
+    lib.mkMerge [
+      {
+        wantedBy = ["multi-user.target"];
+        after = ["network.target"];
+
+        serviceConfig = {
+          Restart = "always";
+          PrivateTmp = true;
+          WorkingDirectory = "/tmp";
+          DynamicUser = true;
+          LockPersonality = true;
+          MemoryDenyWriteExecute = true;
+          NonNewPrivileges = true;
+          PrivateDevices = true;
+          ProtectClock = true;
+          ProtectControlGroups = true;
+          ProtectHome = true;
+          ProtectHostname = true;
+          ProtectKernelLogs = true;
+          ProtectKernelModules = true;
+          ProtectKernelTunables = true;
+          ProtectSystem = "strict";
+          RemoveIPC = true;
+          RestrictAddressFamilies = lib.mkDefault ["AF_INET" "AF_INET6"];
+          RestrictNamespaces = true;
+          RestrictRealtime = true;
+          RestrictSUIDSGID = true;
+          SystemCallArchitectures = "native";
+          UMask = "0077";
+        };
+      }
+      (removeAttrs exporter ["port" "listenAddress"])
+    ])
+  config.services.prometheus.local-exporters;
+}

configuration/services/nextcloud.nix

@@ -50,6 +50,9 @@ in {
   services.nginx.virtualHosts."${hostName}" = {
     forceSSL = true;
     enableACME = true;
+    extraConfig = ''
+      access_log /var/log/nginx/${hostName}/access.log upstream_time;
+    '';
   };
 
   # Block repeated failed login attempts
@@ -74,4 +77,33 @@ in {
       bantime = 86400
     '';
   };
+
+  services.backups.nextcloud = {
+    user = "nextcloud";
+    paths = [
+      "/var/lib/nextcloud/nextcloud-db.sql"
+      "/var/lib/nextcloud/data/"
+      "/var/lib/nextcloud/config/config.php"
+    ];
+    preparation = {
+      packages = [
+        config.services.postgresql.package
+        config.services.nextcloud.occ
+      ];
+      text = ''
+        nextcloud-occ maintenance:mode --on
+        pg_dump ${config.services.nextcloud.config.dbname} --file=/var/lib/nextcloud/nextcloud-db.sql
+      '';
+    };
+    cleanup = {
+      packages = [
+        pkgs.coreutils
+        config.services.nextcloud.occ
+      ];
+      text = ''
+        rm /var/lib/nextcloud/nextcloud-db.sql
+        nextcloud-occ maintenance:mode --off
+      '';
+    };
+  };
 }

configuration/services/postgres.nix

@@ -16,6 +16,12 @@
     # that operation needs to be performed manually on the system as
     # well.
     ensureUsers = [
+      {
+        name = "grafana";
+        ensurePermissions = {
+          "DATABASE grafana" = "ALL PRIVILEGES";
+        };
+      }
       {
         name = "nextcloud";
         ensurePermissions = {
@@ -25,6 +31,7 @@
     ];
 
     ensureDatabases = [
+      "grafana"
       "nextcloud"
     ];
   };

configuration/services/starbound.nix

@@ -110,4 +110,12 @@ in {
       # ProtectHome = "read-only"; # See further up
     };
   };
+
+  services.backups.starbound = {
+    user = "root";
+    paths = [
+      "/var/lib/private/starbound/storage/universe/"
+    ];
+    pauseServices = ["starbound.service"];
+  };
 }

configuration/services/webserver.nix

@@ -19,6 +19,7 @@ in {
     enableACME = true;
     extraConfig = ''
       add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+      access_log /var/log/nginx/${domain}/access.log upstream_time;
     '';
 
     locations."/".proxyPass = "http://${addr}:${toString port}";

configuration/sops.nix

@@ -3,22 +3,37 @@
     defaultSopsFile = ../keys/production.yaml;
 
     secrets = {
+      # Grafana
+      "grafana/adminPassword" = {
+        owner = "grafana";
+        group = "grafana";
+      };
+      "grafana/secretKey" = {
+        owner = "grafana";
+        group = "grafana";
+      };
+
+      # Heisenbridge
+      "heisenbridge/as-token" = {};
+      "heisenbridge/hs-token" = {};
+
+      # Nextcloud
       "nextcloud/tlater" = {
         owner = "nextcloud";
         group = "nextcloud";
       };
 
-      "steam/tlater" = {};
+      # Restic
-
+      "restic/local-backups" = {
-      "heisenbridge/as-token" = {};
-      "heisenbridge/hs-token" = {};
-
-      "wireguard/server-key" = {
         owner = "root";
-        group = "systemd-network";
+        group = "backup";
         mode = "0440";
       };
 
+      # Steam
+      "steam/tlater" = {};
+
+      # Turn
       "turn/env" = {};
       "turn/secret" = {
         owner = "turnserver";
@@ -29,6 +44,13 @@
       "turn/ssl-cert" = {
         owner = "turnserver";
       };
+
+      # Wireguard
+      "wireguard/server-key" = {
+        owner = "root";
+        group = "systemd-network";
+        mode = "0440";
+      };
     };
   };
 }

flake.nix

@@ -78,7 +78,7 @@
     # Utility scripts #
     ###################
     packages.${system} = let
-      inherit (nixpkgs.legacyPackages.${system}) writeShellScript;
+      inherit (nixpkgs.legacyPackages.${system}) writeShellScript writeShellScriptBin;
       vm = nixpkgs.lib.nixosSystem {
         inherit system;
         specialArgs.flake-inputs = inputs;
@@ -106,6 +106,14 @@
           "${vm.config.system.build.vm}/bin/run-tlaternet-vm"
         '';
 
+      update-pkgs = let
+        nvfetcher-bin = "${nvfetcher.packages.${system}.default}/bin/nvfetcher";
+      in
+        writeShellScriptBin "update-pkgs" ''
+          cd "$(git rev-parse --show-toplevel)/pkgs"
+          ${nvfetcher-bin} -o _sources_pkgs -c nvfetcher.toml
+        '';
+
       update-nextcloud-apps = let
         nvfetcher-bin = "${nvfetcher.packages.${system}.default}/bin/nvfetcher";
       in

keys/production.yaml

@@ -1,3 +1,5 @@
+metrics:
+    tlater: ENC[AES256_GCM,data:4nB0H45nlongb0x1OOKzNXIk96PovZ7OwENovnBZUwMl9ncfYwTHT30OlLsPA75w1govH0jyBRkn1Pe/qHzY1Zt53B8=,iv:AfZ4So6HnjOXzqiHM3WpOsQZJs2CEckuxGfyDxc4TNA=,tag:fIXOKJSVDLpLbiLd2zAu9w==,type:str]
 nextcloud:
     tlater: ENC[AES256_GCM,data:zNsPm4uFaIRe3LjcwmayRg==,iv:5wam6bP5zP708jC9UrLV0s8qspl3Pm4fPzbMFYBUyPQ=,tag:apnJUMeJwMn9q0NhO4ptmA==,type:str]
 steam:
@@ -7,6 +9,8 @@ heisenbridge:
     hs-token: ENC[AES256_GCM,data:u52WpkQFd/J7JFoE/rfNluebyZQLOokvkVdL7+AEAvrhJhrkJli1ztkD79lbC+6tGUH4tT3T+nX9wvGKnrRUQg==,iv:as+9fVuvMg2IoE2WIKD9mHi+znhNcWRh5Zq+yr0xcDQ=,tag:mZ7fh7U0MfgI8hyq/28Bcg==,type:str]
 wireguard:
     server-key: ENC[AES256_GCM,data:mXb7ZznJHf5CgV8rI4uzPBATMRbmd7LimgtCkQM9kAjbIaGwUBqJZBN3fXs=,iv:3Po1Orinzov9rnEm9cLzgJY1PeD+5Jl9115MriABHh8=,tag:E/2CjDO1JCvJzxCnqKcNyw==,type:str]
+restic:
+    local-backups: ENC[AES256_GCM,data:NLNVlR9G9bLSZOkMoPvkbBbAZlKkmiUbdWHOFDnaefuy9wNLH53ctOIyS0rSsQLaJCSBTpgPSWIIXUSuzoK/eA==,iv:DzuujmyJJP4GiE5z7KOOGUEzUgOwmtf/7UYhwkyLe9g=,tag:cElFhpVC7S6HYlB6UyN7PQ==,type:str]
 turn:
     env: ENC[AES256_GCM,data:kt5nhVo9pb/ZbPUEcqSYXxN9YMgQKnFb5VRfFFS/qoIaJ73uD2fuJKqcxAyVRrdLqnSAWSQBgTgunBzdP7xqLAK2qt8DYAQWHkIe9uxFbSXZpdmw,iv:9lq6SFwTFN4GGm6gPiJpUMasMdnHVF6XLGYrsyG3kjU=,tag:428Qf9DOiiHt/Wjb188b8g==,type:str]
     secret: ENC[AES256_GCM,data:si7ee6Xfhdgdyzbp6aQpF7pz3TmTBb7iQ82lRPVXNDg9JfHI+lbmgAsSnRLX5qMCA6P9R045sSMosqidL8QwRg==,iv:SrhpZKK8D45yxCEfDb9P3TwtA14+qEI+wcRqcN/a6pw=,tag:PiwV+mOL9xHJgJft6sc61g==,type:str]
@@ -19,8 +23,8 @@ sops:
     azure_kv: []
     hc_vault: []
     age: []
-    lastmodified: "2023-04-23T17:34:53Z"
+    lastmodified: "2023-09-25T00:42:25Z"
-    mac: ENC[AES256_GCM,data:UaGB4uwmYGVbKud5KrvdKeYTnYrs8nnQsT590KIS/b/9JhpQo5JXFtHsm1AteEBg9ygmY6tYKDcK4AXwz/uR/m3CW5If03dBNG8F9Uy3dPL5KaebC/EsNVIaRavWTbSZgqhnBgYeM+HkeQPskSWuwviSNU0D7d1n98Q89Y0kQfA=,iv:kEsRh8hb1amd2qozyxwYHCHdX80c2mO5Mm7npKX3DKc=,tag:p5GPd0OZvowghT92pxxXeA==,type:str]
+    mac: ENC[AES256_GCM,data:28o/elUKslgn5auYfr34N9fE7B6EoZ6njL6yT0emjfoTjsCADJOLcHfUDNWb3AMP3Z5e/w8WsxI7MpwuwUXRvZ6u9Kui1IBcQu/V6GEzpBVw7JkLHZvsUFHOj/uEBcPvON7pKfXtG3vdH8FF1cxeenFm1Z0cX4C0WrNaxumGknA=,iv:GYK0/JZtCkbVorus+9HQbtxAnIXviiNkoC9dMqTHflM=,tag:R3N5hf/UV2nqyOI50Imr6g==,type:str]
     pgp:
         - created_at: "2022-10-12T00:46:51Z"
           enc: |

keys/staging.yaml

@@ -1,3 +1,6 @@
+grafana:
+    adminPassword: ENC[AES256_GCM,data:dYfaxUpQpzA=,iv:j5wSem8C5+V4c5qRzXQJhsU7/FOtpvrnaEyFBmW6zJ4=,tag:oc8n3TkEbjF2gjuOobZuLA==,type:str]
+    secretKey: ENC[AES256_GCM,data:Atruvh2MsNY=,iv:y2MaCUCEzGIydHp6G0DJHfk289S1is0twKm2oUYwDhM=,tag:nAWeg+YqaYqk6k22oBkAhQ==,type:str]
 nextcloud:
     tlater: ENC[AES256_GCM,data:91kDcO4hpng=,iv:ayuILRmRru4ZxTCur9H2xHuLjkDzwPdS/4lEog/tesU=,tag:qYhJxnNDcCwUM7xe7Tlcjw==,type:str]
 steam:
@@ -7,6 +10,8 @@ heisenbridge:
     hs-token: ENC[AES256_GCM,data:VBwvwomv0Xg=,iv:q6INtJ+rg+QiXj8uBdBzQYQZUBBXp+9odxDHwvu8Jxc=,tag:XKhm8nxygAkKaiVPJ2Fcdg==,type:str]
 wireguard:
     server-key: ENC[AES256_GCM,data:FvY897XdKoa/mckE8JQLCkklsnYD6Wz1wpsu5t3uhEnW3iarnDQxF9msuYU=,iv:jqGXfekM+Vs+J9b5nlZ5Skd1ZKHajoUo2Dc4tMYPm1w=,tag:EehikjI/FCU8wqtpvJRamQ==,type:str]
+restic:
+    local-backups: ENC[AES256_GCM,data:3QjEv03t7wE=,iv:y/6Lv4eUbZZfGPwUONykz8VNL62cAJuWaJy9yk3aAmk=,tag:wMlGsepuG9JjwtUKGWSibw==,type:str]
 turn:
     env: ENC[AES256_GCM,data:xjIz/AY109lyiL5N01p5T3HcYco/rM5CJSRTtg==,iv:16bW6OpyOK/QL0QPGQp/Baa9xyT8E3ZsYkwqmjuofk0=,tag:J5re3uKxIykw3YunvQWBgg==,type:str]
     secret: ENC[AES256_GCM,data:eQ7dAocoZtg=,iv:fgzjTPv30WqTKlLy+yMn5MsKQgjhPnwlGFFwYEg3gWs=,tag:1ze33U1NBkgMX/9SiaBNQg==,type:str]
@@ -19,8 +24,8 @@ sops:
     azure_kv: []
     hc_vault: []
     age: []
-    lastmodified: "2023-04-23T17:35:16Z"
+    lastmodified: "2023-10-01T23:00:06Z"
-    mac: ENC[AES256_GCM,data:4cW8k6o3jET8k+yJGyApjOyuSUQb+d+4wX/RTNnpbt+867sExQrZUrOMif/u8S4WmcKVSJgvrzuxK9hpDPYhJ1d/5YuHH1Dyj7QDRdhbZYHhkpPus0ZVTEpSknZzx2eWH1ch/fyJJknlrBlfb/tz50Dv+w9mhkL7qteaIq+Vmsc=,iv:YMfAuGwu1kAM0wGkq3kzVMnC72yo7ZT04BuEwoLRPIA=,tag:6I1VRzteRaLuxN+sfLA5Mw==,type:str]
+    mac: ENC[AES256_GCM,data:oEJ3Nwlx5YTVLvWa12On1O+LakU42rsAD1wD52MTlzuwgyRZ/g49pL6pQiL6S0uE7wC0EOqOvg2pCtDxxHe3WNjEpcxnWWftdEjw2laLnBuOqduQmVW+Sn23SzoRkl7PwOH1jTQHzRyciyYkJT1/vCNnbNdKg1eqnbpxPysg6/A=,iv:dC8eNEXhzC8Nx1rfXQdDKtlO01QhyW9ncNFEK/yakrg=,tag:vQ4AW/mqnA9Vs5NNzFsYWQ==,type:str]
     pgp:
         - created_at: "2022-10-12T16:48:23Z"
           enc: |

pkgs/_sources_pkgs/generated.json

@@ -0,0 +1,21 @@
+{
+    "prometheus-fail2ban-exporter": {
+        "cargoLocks": null,
+        "date": null,
+        "extract": null,
+        "name": "prometheus-fail2ban-exporter",
+        "passthru": null,
+        "pinned": false,
+        "src": {
+            "deepClone": false,
+            "fetchSubmodules": false,
+            "leaveDotGit": false,
+            "name": null,
+            "rev": "v0.10.0",
+            "sha256": "sha256-8nIW1XaHCBqQCoLkV1ZYE3NTbVZ6c+UOqYD08XQiv+4=",
+            "type": "git",
+            "url": "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter"
+        },
+        "version": "v0.10.0"
+    }
+}

pkgs/_sources_pkgs/generated.nix

@@ -0,0 +1,16 @@
+# This file was generated by nvfetcher, please do not modify it manually.
+{ fetchgit, fetchurl, fetchFromGitHub, dockerTools }:
+{
+  prometheus-fail2ban-exporter = {
+    pname = "prometheus-fail2ban-exporter";
+    version = "v0.10.0";
+    src = fetchgit {
+      url = "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter";
+      rev = "v0.10.0";
+      fetchSubmodules = false;
+      deepClone = false;
+      leaveDotGit = false;
+      sha256 = "sha256-8nIW1XaHCBqQCoLkV1ZYE3NTbVZ6c+UOqYD08XQiv+4=";
+    };
+  };
+}

pkgs/default.nix

@@ -7,6 +7,9 @@
 in
   {
     starbound = callPackage ./starbound {};
+    prometheus-fail2ban-exporter = callPackage ./prometheus/fail2ban-exporter.nix {
+      sources = pkgs.callPackage ./_sources_pkgs/generated.nix {};
+    };
   }
   // (
     # Add nextcloud apps

pkgs/nvfetcher.toml

@@ -0,0 +1,3 @@
+[prometheus-fail2ban-exporter]
+src.manual = "v0.10.0" # No gitlab support in nvfetcher
+fetch.git = "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter"

pkgs/prometheus/fail2ban-exporter.nix

@@ -0,0 +1,8 @@
+{
+  buildGoModule,
+  sources,
+}:
+buildGoModule {
+  inherit (sources.prometheus-fail2ban-exporter) pname src version;
+  vendorHash = "sha256-qU6opwhhvzbQOhfGVyiVgKhfCSB0Z4eSRAJnv6ht2I0=";
+}