metrics: Add metrics with victoriametrics + grafana

2023-10-07 22:14:43 +02:00 · 2023-10-07 22:14:43 +02:00 · 55a4aaf48b
commit 55a4aaf48b
parent 78a9eac9bb
22 changed files with 580 additions and 26 deletions
--- a/configuration/services/metrics/exporters.nix
+++ b/configuration/services/metrics/exporters.nix
@ -0,0 +1,101 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}: let
+  yaml = pkgs.formats.yaml {};
+in {
+  services.prometheus = {
+    exporters = {
+      # Periodically check domain registration status
+      domain = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        extraFlags = let
+          conf.domains = [
+            "tlater.net"
+            "tlater.com"
+          ];
+        in [
+          "--config=${yaml.generate "domains.yml" conf}"
+        ];
+      };
+
+      # System statistics
+      node = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+      };
+      systemd = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        extraFlags = [
+          # Disabled by default because only supported from systemd 235+
+          "--systemd.collector.enable-restart-count"
+          "--systemd.collector.enable-ip-accounting"
+        ];
+      };
+
+      # Various nginx metrics
+      nginx = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+      };
+
+      nginxlog = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        group = "nginx";
+
+        settings.namespaces =
+          lib.mapAttrsToList (name: virtualHost: {
+            inherit name;
+            metrics_override.prefix = "nginxlog";
+            namespace_label = "vhost";
+
+            format = lib.concatStringsSep " " [
+              "$remote_addr - $remote_user [$time_local]"
+              ''"$request" $status $body_bytes_sent''
+              ''"$http_referer" "$http_user_agent"''
+              ''rt=$request_time uct="$upstream_connect_time"''
+              ''uht="$upstream_header_time" urt="$upstream_response_time"''
+            ];
+
+            source.files = [
+              "/var/log/nginx/${name}/access.log"
+            ];
+          })
+          config.services.nginx.virtualHosts;
+      };
+    };
+
+    extraExporters = {
+      fail2ban = let
+        cfg = config.services.prometheus.extraExporters.fail2ban;
+      in {
+        port = 9191;
+        serviceOpts = {
+          after = ["fail2ban.service"];
+          requires = ["fail2ban.service"];
+          serviceConfig = {
+            Group = "fail2ban";
+            RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"];
+            ExecStart = lib.concatStringsSep " " [
+              "${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter"
+              "--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock"
+              "--web.listen-address='${cfg.listenAddress}:${toString cfg.port}'"
+              "--collector.f2b.exit-on-socket-connection-error=true"
+            ];
+          };
+        };
+      };
+    };
+
+    # TODO(tlater):
+    #   - wireguard (?)
+    #   - postgres (?)
+    #   - blackbox (?) (curl to see if http and similar is up)
+    #   - ssl_exporter (?)
+  };
+}