tlaternet-server/configuration/services/metrics/default.nix

{
  config,
  pkgs,
  lib,
  ...
}: let
  domain = "metrics.${config.services.nginx.domain}";
  yaml = pkgs.formats.yaml {};
in {
  imports = [
    ./exporters.nix
  ];

  services.victoriametrics.enable = true;

  services.grafana = {
    enable = true;
    settings = {
      server.http_port = 3001; # Default overlaps with gitea

      security = {
        admin_user = "tlater";
        admin_password = "$__file{${config.sops.secrets."grafana/adminPassword".path}}";
        secret_key = "$__file{${config.sops.secrets."grafana/secretKey".path}}";
        cookie_secure = true;
        cookie_samesite = "strict";
        content_security_policy = true;
      };

      database = {
        user = "grafana";
        name = "grafana";
        type = "postgres";
        host = "/run/postgresql";
      };
    };

    provision = {
      enable = true;

      datasources.settings.datasources = [
        {
          name = "Victoriametrics - tlater.net";
          url = "http://localhost:8428";
          type = "prometheus";
        }
      ];
    };
  };

  services.prometheus.exporters = {
    node = {
      enable = true;
      enabledCollectors = ["systemd"];
      listenAddress = "127.0.0.1";
    };

    nginx = {
      enable = true;
      listenAddress = "127.0.0.1";
    };

    nginxlog = {
      enable = true;
      listenAddress = "127.0.0.1";
      group = "nginx";

      settings.namespaces =
        lib.mapAttrsToList (name: virtualHost: {
          inherit name;
          metrics_override.prefix = "nginxlog";
          namespace_label = "vhost";

          format = lib.concatStringsSep " " [
            "$remote_addr - $remote_user [$time_local]"
            ''"$request" $status $body_bytes_sent''
            ''"$http_referer" "$http_user_agent"''
            ''rt=$request_time uct="$upstream_connect_time"''
            ''uht="$upstream_header_time" urt="$upstream_response_time"''
          ];

          source.files = [
            "/var/log/nginx/${name}/access.log"
          ];
        })
        config.services.nginx.virtualHosts;
    };
  };

  services.prometheus.local-exporters = {
    prometheus-fail2ban-exporter = rec {
      enable = true;
      after = ["fail2ban.service"];

      port = 9191;
      listenAddress = "127.0.0.1";

      serviceConfig = {
        Group = "fail2ban";

        RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"];

        ExecStart = lib.concatStringsSep " " [
          "${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter"
          "--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock"
          "--web.listen-address='${listenAddress}:${toString port}'"
        ];
      };
    };
  };

  systemd.services.export-to-victoriametrics = let
    promscrape = yaml.generate "prometheus.yml" {
      scrape_configs = [
        {
          job_name = "tlater.net";
          static_configs = [
            {
              targets =
                lib.mapAttrsToList (name: exporter: "${exporter.listenAddress}:${toString exporter.port}")
                (lib.filterAttrs (name: exporter: (builtins.isAttrs exporter) && exporter.enable)
                  (config.services.prometheus.exporters // config.services.prometheus.local-exporters));
            }
          ];
        }
      ];
    };
  in {
    enable = true;
    path = [pkgs.victoriametrics];
    wantedBy = ["multi-user.target"];
    script = "vmagent -promscrape.config=${promscrape} -remoteWrite.url=http://localhost:8428/api/v1/write";
  };

  services.nginx.virtualHosts."${domain}" = {
    forceSSL = true;
    enableACME = true;
    extraConfig = ''
      add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
      access_log /var/log/nginx/${domain}/access.log upstream_time;
    '';
    locations."/".proxyPass = "http://localhost:3001";
  };
}