From eb539f6ee70d8f2dde0503dfa58e2e1f5c8d9087 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tristan=20Dani=C3=ABl=20Maat?= <tm@tlater.net>
Date: Sat, 7 Oct 2023 03:58:33 +0200
Subject: [PATCH 1/4] metrics: Add domain monitoring

---
 configuration/services/metrics/default.nix | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/configuration/services/metrics/default.nix b/configuration/services/metrics/default.nix
index 0c02556..4086ef2 100644
--- a/configuration/services/metrics/default.nix
+++ b/configuration/services/metrics/default.nix
@@ -49,6 +49,19 @@ in {
   };
 
   services.prometheus.exporters = {
+    domain = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      extraFlags = let
+        conf.domains = [
+          "tlater.net"
+          "tlater.com"
+        ];
+      in [
+        "--config=${yaml.generate "domains.yml" conf}"
+      ];
+    };
+
     node = {
       enable = true;
       enabledCollectors = ["systemd"];

From 214c59b7b34a0cce592e87eec74e18ccb47c3868 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tristan=20Dani=C3=ABl=20Maat?= <tm@tlater.net>
Date: Sat, 7 Oct 2023 03:58:59 +0200
Subject: [PATCH 2/4] metrics: Add systemd monitoring

---
 configuration/services/metrics/default.nix | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/configuration/services/metrics/default.nix b/configuration/services/metrics/default.nix
index 4086ef2..53a8a1d 100644
--- a/configuration/services/metrics/default.nix
+++ b/configuration/services/metrics/default.nix
@@ -64,7 +64,6 @@ in {
 
     node = {
       enable = true;
-      enabledCollectors = ["systemd"];
       listenAddress = "127.0.0.1";
     };
 
@@ -98,6 +97,16 @@ in {
         })
         config.services.nginx.virtualHosts;
     };
+
+    systemd = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      extraFlags = [
+        # Disabled by default because only supported from systemd 235+
+        "--systemd.collector.enable-restart-count"
+        "--systemd.collector.enable-ip-accounting"
+      ];
+    };
   };
 
   services.prometheus.local-exporters = {

From c373911a1bfacd0576d1739ec9f9800d98a18587 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tristan=20Dani=C3=ABl=20Maat?= <tm@tlater.net>
Date: Sat, 7 Oct 2023 03:59:23 +0200
Subject: [PATCH 3/4] conduit: Add coturn monitoring

---
 configuration/services/conduit.nix         | 3 +++
 configuration/services/metrics/default.nix | 5 ++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/configuration/services/conduit.nix b/configuration/services/conduit.nix
index dcd0103..8257592 100644
--- a/configuration/services/conduit.nix
+++ b/configuration/services/conduit.nix
@@ -173,6 +173,9 @@ in {
       # Various other security settings
       no-tlsv1
       no-tlsv1_1
+
+      # Monitoring
+      prometheus
     '';
   };
 
diff --git a/configuration/services/metrics/default.nix b/configuration/services/metrics/default.nix
index 53a8a1d..3347467 100644
--- a/configuration/services/metrics/default.nix
+++ b/configuration/services/metrics/default.nix
@@ -141,7 +141,10 @@ in {
               targets =
                 lib.mapAttrsToList (name: exporter: "${exporter.listenAddress}:${toString exporter.port}")
                 (lib.filterAttrs (name: exporter: (builtins.isAttrs exporter) && exporter.enable)
-                  (config.services.prometheus.exporters // config.services.prometheus.local-exporters));
+                  (config.services.prometheus.exporters // config.services.prometheus.local-exporters))
+                ++ [
+                  "127.0.0.1:9641" # coturn
+                ];
             }
           ];
         }

From 6f8d95781c7e963e0b3c319277f15164264c095b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tristan=20Dani=C3=ABl=20Maat?= <tm@tlater.net>
Date: Sat, 7 Oct 2023 04:15:52 +0200
Subject: [PATCH 4/4] gitea: Add monitoring

---
 configuration/services/gitea.nix           | 21 ++++++++++++++++
 configuration/services/metrics/default.nix | 29 +++++++++++++++++-----
 2 files changed, 44 insertions(+), 6 deletions(-)

diff --git a/configuration/services/gitea.nix b/configuration/services/gitea.nix
index 6d6dafd..f167230 100644
--- a/configuration/services/gitea.nix
+++ b/configuration/services/gitea.nix
@@ -1,6 +1,7 @@
 {
   pkgs,
   config,
+  lib,
   ...
 }: let
   domain = "gitea.${config.services.nginx.domain}";
@@ -19,11 +20,23 @@ in {
         SSH_PORT = 2222;
       };
 
+      metrics = {
+        ENABLED = true;
+        TOKEN = "#metricstoken#";
+      };
       service.DISABLE_REGISTRATION = true;
       session.COOKIE_SECURE = true;
     };
   };
 
+  systemd.services.gitea.serviceConfig.ExecStartPre = let
+    replaceSecretBin = "${pkgs.replace-secret}/bin/replace-secret";
+    secretPath = config.sops.secrets."gitea/metrics-token".path;
+    runConfig = "${config.services.gitea.customDir}/conf/app.ini";
+  in [
+    "${replaceSecretBin} '#metricstoken#' '${secretPath}' '${runConfig}'"
+  ];
+
   # Set up SSL
   services.nginx.virtualHosts."${domain}" = let
     httpAddress = config.services.gitea.settings.server.HTTP_ADDR;
@@ -37,6 +50,14 @@ in {
     '';
 
     locations."/".proxyPass = "http://${httpAddress}:${toString httpPort}";
+    locations."/metrics" = {
+      extraConfig = ''
+        access_log off;
+        allow 127.0.0.1;
+        ${lib.optionalString config.networking.enableIPv6 "allow ::1;"}
+        deny all;
+      '';
+    };
   };
 
   # Block repeated failed login attempts
diff --git a/configuration/services/metrics/default.nix b/configuration/services/metrics/default.nix
index 3347467..4b163d3 100644
--- a/configuration/services/metrics/default.nix
+++ b/configuration/services/metrics/default.nix
@@ -138,12 +138,29 @@ in {
           job_name = "tlater.net";
           static_configs = [
             {
-              targets =
-                lib.mapAttrsToList (name: exporter: "${exporter.listenAddress}:${toString exporter.port}")
-                (lib.filterAttrs (name: exporter: (builtins.isAttrs exporter) && exporter.enable)
-                  (config.services.prometheus.exporters // config.services.prometheus.local-exporters))
-                ++ [
-                  "127.0.0.1:9641" # coturn
+              targets = let
+                exporters = config.services.prometheus.exporters;
+                localExporters = config.services.prometheus.local-exporters;
+              in
+                map (exporter: "${exporter.listenAddress}:${toString exporter.port}") [
+                  exporters.domain
+                  exporters.node
+                  exporters.nginx
+                  exporters.nginxlog
+                  exporters.systemd
+
+                  localExporters.prometheus-fail2ban-exporter
+
+                  {
+                    # coturn
+                    listenAddress = "127.0.0.1";
+                    port = "9641";
+                  }
+                  {
+                    # gitea
+                    listenAddress = "127.0.0.1";
+                    port = "3000";
+                  }
                 ];
             }
           ];