gitea: Add monitoring

configuration/services/gitea.nix

@@ -1,6 +1,7 @@
 {
   pkgs,
   config,
+  lib,
   ...
 }: let
   domain = "gitea.${config.services.nginx.domain}";
@@ -19,11 +20,23 @@ in {
         SSH_PORT = 2222;
       };
 
+      metrics = {
+        ENABLED = true;
+        TOKEN = "#metricstoken#";
+      };
       service.DISABLE_REGISTRATION = true;
       session.COOKIE_SECURE = true;
     };
   };
 
+  systemd.services.gitea.serviceConfig.ExecStartPre = let
+    replaceSecretBin = "${pkgs.replace-secret}/bin/replace-secret";
+    secretPath = config.sops.secrets."gitea/metrics-token".path;
+    runConfig = "${config.services.gitea.customDir}/conf/app.ini";
+  in [
+    "+${replaceSecretBin} '#metricstoken#' '${secretPath}' '${runConfig}'"
+  ];
+
   # Set up SSL
   services.nginx.virtualHosts."${domain}" = let
     httpAddress = config.services.gitea.settings.server.HTTP_ADDR;
@@ -37,6 +50,14 @@ in {
     '';
 
     locations."/".proxyPass = "http://${httpAddress}:${toString httpPort}";
+    locations."/metrics" = {
+      extraConfig = ''
+        access_log off;
+        allow 127.0.0.1;
+        ${lib.optionalString config.networking.enableIPv6 "allow ::1;"}
+        deny all;
+      '';
+    };
   };
 
   # Block repeated failed login attempts

configuration/services/metrics/default.nix

@@ -138,12 +138,29 @@ in {
           job_name = "tlater.net";
           static_configs = [
             {
-              targets =
+              targets = let
-                lib.mapAttrsToList (name: exporter: "${exporter.listenAddress}:${toString exporter.port}")
+                exporters = config.services.prometheus.exporters;
-                (lib.filterAttrs (name: exporter: (builtins.isAttrs exporter) && exporter.enable)
+                localExporters = config.services.prometheus.local-exporters;
-                  (config.services.prometheus.exporters // config.services.prometheus.local-exporters))
+              in
-                ++ [
+                map (exporter: "${exporter.listenAddress}:${toString exporter.port}") [
-                  "127.0.0.1:9641" # coturn
+                  exporters.domain
+                  exporters.node
+                  exporters.nginx
+                  exporters.nginxlog
+                  exporters.systemd
+
+                  localExporters.prometheus-fail2ban-exporter
+
+                  {
+                    # coturn
+                    listenAddress = "127.0.0.1";
+                    port = "9641";
+                  }
+                  {
+                    # gitea
+                    listenAddress = "127.0.0.1";
+                    port = "3000";
+                  }
                 ];
             }
           ];

configuration/sops.nix

@@ -3,6 +3,12 @@
     defaultSopsFile = ../keys/production.yaml;
 
     secrets = {
+      # Gitea
+      "gitea/metrics-token" = {
+        owner = "gitea";
+        group = "gitea";
+      };
+
       # Grafana
       "grafana/adminPassword" = {
         owner = "grafana";

keys/staging.yaml

@@ -1,3 +1,5 @@
+gitea:
+    metrics-token: ENC[AES256_GCM,data:J4QdfI1wKyM=,iv:8fqCbftyhj90eIVFxjEp9RXKC1y1IaLnV1r2MOdY15M=,tag:8W/juv1OZh4hJco02qXO6g==,type:str]
 grafana:
     adminPassword: ENC[AES256_GCM,data:dYfaxUpQpzA=,iv:j5wSem8C5+V4c5qRzXQJhsU7/FOtpvrnaEyFBmW6zJ4=,tag:oc8n3TkEbjF2gjuOobZuLA==,type:str]
     secretKey: ENC[AES256_GCM,data:Atruvh2MsNY=,iv:y2MaCUCEzGIydHp6G0DJHfk289S1is0twKm2oUYwDhM=,tag:nAWeg+YqaYqk6k22oBkAhQ==,type:str]
@@ -24,8 +26,8 @@ sops:
     azure_kv: []
     hc_vault: []
     age: []
-    lastmodified: "2023-10-01T23:00:06Z"
+    lastmodified: "2023-10-07T02:17:50Z"
-    mac: ENC[AES256_GCM,data:oEJ3Nwlx5YTVLvWa12On1O+LakU42rsAD1wD52MTlzuwgyRZ/g49pL6pQiL6S0uE7wC0EOqOvg2pCtDxxHe3WNjEpcxnWWftdEjw2laLnBuOqduQmVW+Sn23SzoRkl7PwOH1jTQHzRyciyYkJT1/vCNnbNdKg1eqnbpxPysg6/A=,iv:dC8eNEXhzC8Nx1rfXQdDKtlO01QhyW9ncNFEK/yakrg=,tag:vQ4AW/mqnA9Vs5NNzFsYWQ==,type:str]
+    mac: ENC[AES256_GCM,data:vZDq33YIn0Nf1FQ2+ySezox6igiw6zNFCu3l3kaIsBKo1797pohmAxj2Lcc+OmlBjj98khaBIlbQuA5ULM+uPN5ILaz3NuXD5PZtsV+rL2PsLNMW9FBSmJ0m0YQrt0nZ0tpzifn12XghcSK2IXv+FnxlfrAJCxDvr5tRm90uUwU=,iv:ct8CzIWjaoJ1UjZcdFSr8lZ626vA0RvM883V6H5plWc=,tag:waJNtp/UbRDOfyzNElrung==,type:str]
     pgp:
         - created_at: "2022-10-12T16:48:23Z"
           enc: |