gitea: Add monitoring

configuration/services/foundryvtt.nix

@@ -8,11 +8,11 @@
 in {
   imports = [flake-inputs.foundryvtt.nixosModules.foundryvtt];
 
-  # services.foundryvtt = {
+  services.foundryvtt = {
-  #   enable = true;
+    enable = true;
-  #   hostName = domain;
+    hostName = domain;
-  #   minifyStaticFiles = true;
+    minifyStaticFiles = true;
-  # };
+  };
 
   # Want to start it manually when I need it, not have it constantly
   # running

configuration/services/metrics/default.nix

@@ -1,9 +1,186 @@
 {
+  config,
+  pkgs,
+  lib,
+  ...
+}: let
+  domain = "metrics.${config.services.nginx.domain}";
+  yaml = pkgs.formats.yaml {};
+in {
   imports = [
-    ./options.nix
-
     ./exporters.nix
-    ./grafana.nix
+  ];
-    ./victoriametrics.nix
+
+  services.victoriametrics.enable = true;
+
+  services.grafana = {
+    enable = true;
+    settings = {
+      server.http_port = 3001; # Default overlaps with gitea
+
+      security = {
+        admin_user = "tlater";
+        admin_password = "$__file{${config.sops.secrets."grafana/adminPassword".path}}";
+        secret_key = "$__file{${config.sops.secrets."grafana/secretKey".path}}";
+        cookie_secure = true;
+        cookie_samesite = "strict";
+        content_security_policy = true;
+      };
+
+      database = {
+        user = "grafana";
+        name = "grafana";
+        type = "postgres";
+        host = "/run/postgresql";
+      };
+    };
+
+    provision = {
+      enable = true;
+
+      datasources.settings.datasources = [
+        {
+          name = "Victoriametrics - tlater.net";
+          url = "http://localhost:8428";
+          type = "prometheus";
+        }
+      ];
+    };
+  };
+
+  services.prometheus.exporters = {
+    domain = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      extraFlags = let
+        conf.domains = [
+          "tlater.net"
+          "tlater.com"
+        ];
+      in [
+        "--config=${yaml.generate "domains.yml" conf}"
+      ];
+    };
+
+    node = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+    };
+
+    nginx = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+    };
+
+    nginxlog = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      group = "nginx";
+
+      settings.namespaces =
+        lib.mapAttrsToList (name: virtualHost: {
+          inherit name;
+          metrics_override.prefix = "nginxlog";
+          namespace_label = "vhost";
+
+          format = lib.concatStringsSep " " [
+            "$remote_addr - $remote_user [$time_local]"
+            ''"$request" $status $body_bytes_sent''
+            ''"$http_referer" "$http_user_agent"''
+            ''rt=$request_time uct="$upstream_connect_time"''
+            ''uht="$upstream_header_time" urt="$upstream_response_time"''
+          ];
+
+          source.files = [
+            "/var/log/nginx/${name}/access.log"
+          ];
+        })
+        config.services.nginx.virtualHosts;
+    };
+
+    systemd = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      extraFlags = [
+        # Disabled by default because only supported from systemd 235+
+        "--systemd.collector.enable-restart-count"
+        "--systemd.collector.enable-ip-accounting"
+      ];
+    };
+  };
+
+  services.prometheus.local-exporters = {
+    prometheus-fail2ban-exporter = rec {
+      enable = true;
+      after = ["fail2ban.service"];
+
+      port = 9191;
+      listenAddress = "127.0.0.1";
+
+      serviceConfig = {
+        Group = "fail2ban";
+
+        RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"];
+
+        ExecStart = lib.concatStringsSep " " [
+          "${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter"
+          "--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock"
+          "--web.listen-address='${listenAddress}:${toString port}'"
+        ];
+      };
+    };
+  };
+
+  systemd.services.export-to-victoriametrics = let
+    promscrape = yaml.generate "prometheus.yml" {
+      scrape_configs = [
+        {
+          job_name = "tlater.net";
+          static_configs = [
+            {
+              targets = let
+                exporters = config.services.prometheus.exporters;
+                localExporters = config.services.prometheus.local-exporters;
+              in
+                map (exporter: "${exporter.listenAddress}:${toString exporter.port}") [
+                  exporters.domain
+                  exporters.node
+                  exporters.nginx
+                  exporters.nginxlog
+                  exporters.systemd
+
+                  localExporters.prometheus-fail2ban-exporter
+
+                  {
+                    # coturn
+                    listenAddress = "127.0.0.1";
+                    port = "9641";
+                  }
+                  {
+                    # gitea
+                    listenAddress = "127.0.0.1";
+                    port = "3000";
+                  }
                 ];
             }
+          ];
+        }
+      ];
+    };
+  in {
+    enable = true;
+    path = [pkgs.victoriametrics];
+    wantedBy = ["multi-user.target"];
+    script = "vmagent -promscrape.config=${promscrape} -remoteWrite.url=http://localhost:8428/api/v1/write";
+  };
+
+  services.nginx.virtualHosts."${domain}" = {
+    forceSSL = true;
+    enableACME = true;
+    extraConfig = ''
+      add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+      access_log /var/log/nginx/${domain}/access.log upstream_time;
+    '';
+    locations."/".proxyPass = "http://localhost:3001";
+  };
+}

configuration/services/metrics/exporters.nix

@@ -1,100 +1,45 @@
 {
   config,
-  pkgs,
   lib,
   ...
-}: let
+}: {
-  yaml = pkgs.formats.yaml {};
+  options.services.prometheus.local-exporters = lib.mkOption {
-in {
+    type = lib.types.anything;
-  services.prometheus = {
-    exporters = {
-      # Periodically check domain registration status
-      domain = {
-        enable = true;
-        listenAddress = "127.0.0.1";
-        extraFlags = let
-          conf.domains = [
-            "tlater.net"
-            "tlater.com"
-          ];
-        in [
-          "--config=${yaml.generate "domains.yml" conf}"
-        ];
   };
 
-      # System statistics
+  config.systemd.services = lib.mapAttrs (_: exporter:
-      node = {
+    lib.mkMerge [
-        enable = true;
+      {
-        listenAddress = "127.0.0.1";
+        wantedBy = ["multi-user.target"];
-      };
+        after = ["network.target"];
-      systemd = {
-        enable = true;
-        listenAddress = "127.0.0.1";
-        extraFlags = [
-          # Disabled by default because only supported from systemd 235+
-          "--systemd.collector.enable-restart-count"
-          "--systemd.collector.enable-ip-accounting"
-        ];
-      };
 
-      # Various nginx metrics
-      nginx = {
-        enable = true;
-        listenAddress = "127.0.0.1";
-      };
-
-      nginxlog = {
-        enable = true;
-        listenAddress = "127.0.0.1";
-        group = "nginx";
-
-        settings.namespaces =
-          lib.mapAttrsToList (name: virtualHost: {
-            inherit name;
-            metrics_override.prefix = "nginxlog";
-            namespace_label = "vhost";
-
-            format = lib.concatStringsSep " " [
-              "$remote_addr - $remote_user [$time_local]"
-              ''"$request" $status $body_bytes_sent''
-              ''"$http_referer" "$http_user_agent"''
-              ''rt=$request_time uct="$upstream_connect_time"''
-              ''uht="$upstream_header_time" urt="$upstream_response_time"''
-            ];
-
-            source.files = [
-              "/var/log/nginx/${name}/access.log"
-            ];
-          })
-          config.services.nginx.virtualHosts;
-      };
-    };
-
-    extraExporters = {
-      fail2ban = let
-        cfg = config.services.prometheus.extraExporters.fail2ban;
-      in {
-        port = 9191;
-        serviceOpts = {
-          after = ["fail2ban.service"];
-          requires = ["fail2ban.service"];
         serviceConfig = {
-            Group = "fail2ban";
+          Restart = "always";
-            RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"];
+          PrivateTmp = true;
-            ExecStart = lib.concatStringsSep " " [
+          WorkingDirectory = "/tmp";
-              "${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter"
+          DynamicUser = true;
-              "--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock"
+          LockPersonality = true;
-              "--web.listen-address='${cfg.listenAddress}:${toString cfg.port}'"
+          MemoryDenyWriteExecute = true;
-              "--collector.f2b.exit-on-socket-connection-error=true"
+          NonNewPrivileges = true;
-            ];
+          PrivateDevices = true;
-          };
+          ProtectClock = true;
-        };
+          ProtectControlGroups = true;
-      };
+          ProtectHome = true;
-    };
+          ProtectHostname = true;
-
+          ProtectKernelLogs = true;
-    # TODO(tlater):
+          ProtectKernelModules = true;
-    #   - wireguard (?)
+          ProtectKernelTunables = true;
-    #   - postgres (?)
+          ProtectSystem = "strict";
-    #   - blackbox (?) (curl to see if http and similar is up)
+          RemoveIPC = true;
+          RestrictAddressFamilies = lib.mkDefault ["AF_INET" "AF_INET6"];
+          RestrictNamespaces = true;
+          RestrictRealtime = true;
+          RestrictSUIDSGID = true;
+          SystemCallArchitectures = "native";
+          UMask = "0077";
         };
       }
+      (removeAttrs exporter ["port" "listenAddress"])
+    ])
+  config.services.prometheus.local-exporters;
+}

configuration/services/metrics/grafana.nix

@@ -1,48 +0,0 @@
-{config, ...}: let
-  domain = "metrics.${config.services.nginx.domain}";
-in {
-  services.grafana = {
-    enable = true;
-    settings = {
-      server.http_port = 3001; # Default overlaps with gitea
-
-      security = {
-        admin_user = "tlater";
-        admin_password = "$__file{${config.sops.secrets."grafana/adminPassword".path}}";
-        secret_key = "$__file{${config.sops.secrets."grafana/secretKey".path}}";
-        cookie_secure = true;
-        cookie_samesite = "strict";
-        content_security_policy = true;
-      };
-
-      database = {
-        user = "grafana";
-        name = "grafana";
-        type = "postgres";
-        host = "/run/postgresql";
-      };
-    };
-
-    provision = {
-      enable = true;
-
-      datasources.settings.datasources = [
-        {
-          name = "Victoriametrics - tlater.net";
-          url = "http://localhost:8428";
-          type = "prometheus";
-        }
-      ];
-    };
-  };
-
-  services.nginx.virtualHosts."${domain}" = {
-    forceSSL = true;
-    enableACME = true;
-    extraConfig = ''
-      add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
-      access_log /var/log/nginx/${domain}/access.log upstream_time;
-    '';
-    locations."/".proxyPass = "http://localhost:${toString config.services.grafana.settings.server.http_port}";
-  };
-}

configuration/services/metrics/options.nix

@@ -1,90 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}: let
-  inherit (lib) types mkOption mkDefault;
-in {
-  options.services.prometheus = {
-    extraExporters = mkOption {
-      type = types.attrsOf (types.submodule {
-        options = {
-          port = mkOption {
-            type = types.int;
-            description = "The port on which this exporter listens.";
-          };
-          listenAddress = mkOption {
-            type = types.str;
-            default = "127.0.0.1";
-            description = "Address to listen on.";
-          };
-          serviceOpts = mkOption {
-            type = types.attrs;
-            description = "An attrset to be merged with the exporter's systemd service.";
-          };
-        };
-      });
-    };
-
-    allExporters = mkOption {
-      internal = true;
-      description = "The full list of scraping-relevant settings of all exporters, extra or built-in.";
-      type = types.attrsOf (types.submodule {
-        port = mkOption {
-          type = types.int;
-        };
-        listenAddress = mkOption {
-          type = types.str;
-        };
-        extraSettings = mkOption {
-          type = types.anything;
-          default = {};
-        };
-      });
-    };
-  };
-
-  config = {
-    systemd.services = lib.mapAttrs' (name: exporter:
-      lib.nameValuePair "prometheus-${name}-exporter" (lib.mkMerge [
-        {
-          # Shamelessly copied from upstream because the upstream
-          # module is an intractable mess
-          wantedBy = ["multi-user.target"];
-          after = ["network.target"];
-          serviceConfig.Restart = mkDefault "always";
-          serviceConfig.PrivateTmp = mkDefault true;
-          serviceConfig.WorkingDirectory = mkDefault /tmp;
-          serviceConfig.DynamicUser = mkDefault true;
-          # Hardening
-          serviceConfig.CapabilityBoundingSet = mkDefault [""];
-          serviceConfig.DeviceAllow = [""];
-          serviceConfig.LockPersonality = true;
-          serviceConfig.MemoryDenyWriteExecute = true;
-          serviceConfig.NoNewPrivileges = true;
-          serviceConfig.PrivateDevices = mkDefault true;
-          serviceConfig.ProtectClock = mkDefault true;
-          serviceConfig.ProtectControlGroups = true;
-          serviceConfig.ProtectHome = true;
-          serviceConfig.ProtectHostname = true;
-          serviceConfig.ProtectKernelLogs = true;
-          serviceConfig.ProtectKernelModules = true;
-          serviceConfig.ProtectKernelTunables = true;
-          serviceConfig.ProtectSystem = mkDefault "strict";
-          serviceConfig.RemoveIPC = true;
-          serviceConfig.RestrictAddressFamilies = ["AF_INET" "AF_INET6"];
-          serviceConfig.RestrictNamespaces = true;
-          serviceConfig.RestrictRealtime = true;
-          serviceConfig.RestrictSUIDSGID = true;
-          serviceConfig.SystemCallArchitectures = "native";
-          serviceConfig.UMask = "0077";
-        }
-        exporter.serviceOpts
-      ]))
-    config.services.prometheus.extraExporters;
-
-    services.prometheus.allExporters = lib.mapAttrs (name: exporter: {
-      inherit (exporter) listenAddress port;
-    }) (config.services.prometheus.exporters ++ config.services.prometheus.extraExporters);
-  };
-}

configuration/services/metrics/victoriametrics.nix

@@ -1,31 +0,0 @@
-{config, ...}: {
-  services.victoriametrics = let
-    scrapeConfigFromExporters = conf: conf // {inherit (config.services.prometheus.exporters.${conf.name}) listenAddress port;};
-    scrapeConfigFromLocalExporters = conf: conf // {inherit (config.services.prometheus.local-exporters.${conf.name}) listenAddress port;};
-  in {
-    enable = true;
-    vmagent-scraping.static_configs =
-      [
-        {
-          name = "gitea";
-          listenAddress = "127.0.0.1";
-          port = 3000;
-        }
-        {
-          name = "coturn";
-          listenAddress = "127.0.0.1";
-          port = 9641;
-        }
-      ]
-      ++ (map scrapeConfigFromLocalExporters [
-        {name = "prometheus-fail2ban-exporter";}
-      ])
-      ++ (map scrapeConfigFromExporters [
-        {name = "domain";}
-        {name = "node";}
-        {name = "nginx";}
-        {name = "nginxlog";}
-        {name = "systemd";}
-      ]);
-  };
-}