From afcc959bf9f2d696a4e6b48e36ec66eb580a4536 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tristan=20Dani=C3=ABl=20Maat?= <tm@tlater.net>
Date: Mon, 25 Sep 2023 03:32:04 +0200
Subject: [PATCH] Add metrics

---
 configuration/default.nix           |  1 +
 configuration/services/metrics.nix  | 86 +++++++++++++++++++++++++++++
 configuration/services/postgres.nix |  7 +++
 configuration/sops.nix              | 10 ++++
 keys/production.yaml                |  6 +-
 keys/staging.yaml                   |  7 ++-
 6 files changed, 113 insertions(+), 4 deletions(-)
 create mode 100644 configuration/services/metrics.nix

diff --git a/configuration/default.nix b/configuration/default.nix
index 5d491af..610851a 100644
--- a/configuration/default.nix
+++ b/configuration/default.nix
@@ -18,6 +18,7 @@
     ./services/conduit.nix
     ./services/foundryvtt.nix
     ./services/gitea.nix
+    ./services/metrics.nix
     ./services/nextcloud.nix
     ./services/webserver.nix
     ./services/wireguard.nix
diff --git a/configuration/services/metrics.nix b/configuration/services/metrics.nix
new file mode 100644
index 0000000..12a48e0
--- /dev/null
+++ b/configuration/services/metrics.nix
@@ -0,0 +1,86 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}: let
+  domain = "metrics.${config.services.nginx.domain}";
+  yaml = pkgs.formats.yaml {};
+in {
+  services.victoriametrics.enable = true;
+
+  services.grafana = {
+    enable = true;
+    settings = {
+      server.http_port = 3001; # Default overlaps with gitea
+
+      security = {
+        admin_user = "tlater";
+        admin_password = "$__file{${config.sops.secrets."grafana/adminPassword".path}}";
+        secret_key = "$__file{${config.sops.secrets."grafana/secretKey".path}}";
+        cookie_secure = true;
+        cookie_samesite = "strict";
+        content_security_policy = true;
+      };
+
+      database = {
+        user = "grafana";
+        name = "grafana";
+        type = "postgres";
+        host = "/run/postgresql";
+      };
+    };
+
+    provision = {
+      enable = true;
+
+      datasources.settings.datasources = [
+        {
+          name = "Victoriametrics - tlater.net";
+          url = "http://localhost:8428";
+          type = "prometheus";
+        }
+      ];
+    };
+  };
+
+  services.prometheus.exporters = {
+    node = {
+      enable = true;
+      enabledCollectors = ["systemd"];
+      listenAddress = "127.0.0.1";
+    };
+  };
+
+  systemd.services.export-to-victoriametrics = let
+    promscrape = yaml.generate "prometheus.yml" {
+      scrape_configs = [
+        {
+          job_name = "tlater.net";
+          static_configs = [
+            {
+              targets =
+                lib.mapAttrsToList (name: exporter: "${exporter.listenAddress}:${toString exporter.port}")
+                (lib.filterAttrs (name: exporter: (builtins.isAttrs exporter) && exporter.enable)
+                  config.services.prometheus.exporters);
+            }
+          ];
+        }
+      ];
+    };
+  in {
+    enable = true;
+    path = [pkgs.victoriametrics];
+    wantedBy = ["multi-user.target"];
+    script = "vmagent -promscrape.config=${promscrape} -remoteWrite.url=http://localhost:8428/api/v1/write";
+  };
+
+  services.nginx.virtualHosts."${domain}" = {
+    forceSSL = true;
+    enableACME = true;
+    extraConfig = ''
+      add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+    '';
+    locations."/".proxyPass = "http://localhost:3001";
+  };
+}
diff --git a/configuration/services/postgres.nix b/configuration/services/postgres.nix
index 6c584bb..923007d 100644
--- a/configuration/services/postgres.nix
+++ b/configuration/services/postgres.nix
@@ -16,6 +16,12 @@
     # that operation needs to be performed manually on the system as
     # well.
     ensureUsers = [
+      {
+        name = "grafana";
+        ensurePermissions = {
+          "DATABASE grafana" = "ALL PRIVILEGES";
+        };
+      }
       {
         name = "nextcloud";
         ensurePermissions = {
@@ -25,6 +31,7 @@
     ];
 
     ensureDatabases = [
+      "grafana"
       "nextcloud"
     ];
   };
diff --git a/configuration/sops.nix b/configuration/sops.nix
index bf98433..6eae9fc 100644
--- a/configuration/sops.nix
+++ b/configuration/sops.nix
@@ -3,6 +3,16 @@
     defaultSopsFile = ../keys/production.yaml;
 
     secrets = {
+      # Grafana
+      "grafana/adminPassword" = {
+        owner = "grafana";
+        group = "grafana";
+      };
+      "grafana/secretKey" = {
+        owner = "grafana";
+        group = "grafana";
+      };
+
       # Heisenbridge
       "heisenbridge/as-token" = {};
       "heisenbridge/hs-token" = {};
diff --git a/keys/production.yaml b/keys/production.yaml
index f8d259d..87ef3c4 100644
--- a/keys/production.yaml
+++ b/keys/production.yaml
@@ -1,3 +1,5 @@
+metrics:
+    tlater: ENC[AES256_GCM,data:4nB0H45nlongb0x1OOKzNXIk96PovZ7OwENovnBZUwMl9ncfYwTHT30OlLsPA75w1govH0jyBRkn1Pe/qHzY1Zt53B8=,iv:AfZ4So6HnjOXzqiHM3WpOsQZJs2CEckuxGfyDxc4TNA=,tag:fIXOKJSVDLpLbiLd2zAu9w==,type:str]
 nextcloud:
     tlater: ENC[AES256_GCM,data:zNsPm4uFaIRe3LjcwmayRg==,iv:5wam6bP5zP708jC9UrLV0s8qspl3Pm4fPzbMFYBUyPQ=,tag:apnJUMeJwMn9q0NhO4ptmA==,type:str]
 steam:
@@ -21,8 +23,8 @@ sops:
     azure_kv: []
     hc_vault: []
     age: []
-    lastmodified: "2023-09-23T18:55:44Z"
-    mac: ENC[AES256_GCM,data:psqgXozY9L7nduZ11GF+mbIrZ4RUySqBixkWL5z0cYeoLA3URb/dr028LCmNgQS9l8aJVsjVkyLBJIU/8wmiUNqRy/VI5iqV5mu+sxXhUVwFL0dAAWP1lOKwwT5uGK89/ioqkphgzuWD37vGe2vYddKkJF0M+zlz12fqkMjaisU=,iv:UyRoJbfuGU3K/Mp5DQ1kY0Z+nKSSo46BGNAcxt+vAvc=,tag:HkP6+qxQ8J/xAYJXYoG/6g==,type:str]
+    lastmodified: "2023-09-25T00:42:25Z"
+    mac: ENC[AES256_GCM,data:28o/elUKslgn5auYfr34N9fE7B6EoZ6njL6yT0emjfoTjsCADJOLcHfUDNWb3AMP3Z5e/w8WsxI7MpwuwUXRvZ6u9Kui1IBcQu/V6GEzpBVw7JkLHZvsUFHOj/uEBcPvON7pKfXtG3vdH8FF1cxeenFm1Z0cX4C0WrNaxumGknA=,iv:GYK0/JZtCkbVorus+9HQbtxAnIXviiNkoC9dMqTHflM=,tag:R3N5hf/UV2nqyOI50Imr6g==,type:str]
     pgp:
         - created_at: "2022-10-12T00:46:51Z"
           enc: |
diff --git a/keys/staging.yaml b/keys/staging.yaml
index a6b0849..86ce700 100644
--- a/keys/staging.yaml
+++ b/keys/staging.yaml
@@ -1,3 +1,6 @@
+grafana:
+    adminPassword: ENC[AES256_GCM,data:dYfaxUpQpzA=,iv:j5wSem8C5+V4c5qRzXQJhsU7/FOtpvrnaEyFBmW6zJ4=,tag:oc8n3TkEbjF2gjuOobZuLA==,type:str]
+    secretKey: ENC[AES256_GCM,data:Atruvh2MsNY=,iv:y2MaCUCEzGIydHp6G0DJHfk289S1is0twKm2oUYwDhM=,tag:nAWeg+YqaYqk6k22oBkAhQ==,type:str]
 nextcloud:
     tlater: ENC[AES256_GCM,data:91kDcO4hpng=,iv:ayuILRmRru4ZxTCur9H2xHuLjkDzwPdS/4lEog/tesU=,tag:qYhJxnNDcCwUM7xe7Tlcjw==,type:str]
 steam:
@@ -21,8 +24,8 @@ sops:
     azure_kv: []
     hc_vault: []
     age: []
-    lastmodified: "2023-09-22T21:07:02Z"
-    mac: ENC[AES256_GCM,data:gItC41S8MInLmikdH1okhPs+FVf8sCF/iQeJ5reigBunHkOngoc6nOFANyAcNZETszzhgTLXXtmVNEjW46v6K7D6nmoi/zwpedUxwzMwDC5I28VTMDHVMAThYSGtdo6kig8i2pi8rzEQd1DStxMv3TWML5y6DDTlFsd3lfudaHA=,iv:zXebvIVPR76GwUhpactwRgF/eEmx2OBkT18E8lkwzRA=,tag:6HyISACbFCGlpIIgkFeA/A==,type:str]
+    lastmodified: "2023-10-01T23:00:06Z"
+    mac: ENC[AES256_GCM,data:oEJ3Nwlx5YTVLvWa12On1O+LakU42rsAD1wD52MTlzuwgyRZ/g49pL6pQiL6S0uE7wC0EOqOvg2pCtDxxHe3WNjEpcxnWWftdEjw2laLnBuOqduQmVW+Sn23SzoRkl7PwOH1jTQHzRyciyYkJT1/vCNnbNdKg1eqnbpxPysg6/A=,iv:dC8eNEXhzC8Nx1rfXQdDKtlO01QhyW9ncNFEK/yakrg=,tag:vQ4AW/mqnA9Vs5NNzFsYWQ==,type:str]
     pgp:
         - created_at: "2022-10-12T16:48:23Z"
           enc: |