diff --git a/configuration/services/foundryvtt.nix b/configuration/services/foundryvtt.nix index a4978fd..d573480 100644 --- a/configuration/services/foundryvtt.nix +++ b/configuration/services/foundryvtt.nix @@ -8,11 +8,11 @@ in { imports = [flake-inputs.foundryvtt.nixosModules.foundryvtt]; - # services.foundryvtt = { - # enable = true; - # hostName = domain; - # minifyStaticFiles = true; - # }; + services.foundryvtt = { + enable = true; + hostName = domain; + minifyStaticFiles = true; + }; # Want to start it manually when I need it, not have it constantly # running diff --git a/configuration/services/metrics/default.nix b/configuration/services/metrics/default.nix index 84e126a..4b163d3 100644 --- a/configuration/services/metrics/default.nix +++ b/configuration/services/metrics/default.nix @@ -1,9 +1,186 @@ { + config, + pkgs, + lib, + ... +}: let + domain = "metrics.${config.services.nginx.domain}"; + yaml = pkgs.formats.yaml {}; +in { imports = [ - ./options.nix - ./exporters.nix - ./grafana.nix - ./victoriametrics.nix ]; + + services.victoriametrics.enable = true; + + services.grafana = { + enable = true; + settings = { + server.http_port = 3001; # Default overlaps with gitea + + security = { + admin_user = "tlater"; + admin_password = "$__file{${config.sops.secrets."grafana/adminPassword".path}}"; + secret_key = "$__file{${config.sops.secrets."grafana/secretKey".path}}"; + cookie_secure = true; + cookie_samesite = "strict"; + content_security_policy = true; + }; + + database = { + user = "grafana"; + name = "grafana"; + type = "postgres"; + host = "/run/postgresql"; + }; + }; + + provision = { + enable = true; + + datasources.settings.datasources = [ + { + name = "Victoriametrics - tlater.net"; + url = "http://localhost:8428"; + type = "prometheus"; + } + ]; + }; + }; + + services.prometheus.exporters = { + domain = { + enable = true; + listenAddress = "127.0.0.1"; + extraFlags = let + conf.domains = [ + "tlater.net" + "tlater.com" + ]; + in [ + "--config=${yaml.generate "domains.yml" conf}" + ]; + }; + + node = { + enable = true; + listenAddress = "127.0.0.1"; + }; + + nginx = { + enable = true; + listenAddress = "127.0.0.1"; + }; + + nginxlog = { + enable = true; + listenAddress = "127.0.0.1"; + group = "nginx"; + + settings.namespaces = + lib.mapAttrsToList (name: virtualHost: { + inherit name; + metrics_override.prefix = "nginxlog"; + namespace_label = "vhost"; + + format = lib.concatStringsSep " " [ + "$remote_addr - $remote_user [$time_local]" + ''"$request" $status $body_bytes_sent'' + ''"$http_referer" "$http_user_agent"'' + ''rt=$request_time uct="$upstream_connect_time"'' + ''uht="$upstream_header_time" urt="$upstream_response_time"'' + ]; + + source.files = [ + "/var/log/nginx/${name}/access.log" + ]; + }) + config.services.nginx.virtualHosts; + }; + + systemd = { + enable = true; + listenAddress = "127.0.0.1"; + extraFlags = [ + # Disabled by default because only supported from systemd 235+ + "--systemd.collector.enable-restart-count" + "--systemd.collector.enable-ip-accounting" + ]; + }; + }; + + services.prometheus.local-exporters = { + prometheus-fail2ban-exporter = rec { + enable = true; + after = ["fail2ban.service"]; + + port = 9191; + listenAddress = "127.0.0.1"; + + serviceConfig = { + Group = "fail2ban"; + + RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"]; + + ExecStart = lib.concatStringsSep " " [ + "${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter" + "--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock" + "--web.listen-address='${listenAddress}:${toString port}'" + ]; + }; + }; + }; + + systemd.services.export-to-victoriametrics = let + promscrape = yaml.generate "prometheus.yml" { + scrape_configs = [ + { + job_name = "tlater.net"; + static_configs = [ + { + targets = let + exporters = config.services.prometheus.exporters; + localExporters = config.services.prometheus.local-exporters; + in + map (exporter: "${exporter.listenAddress}:${toString exporter.port}") [ + exporters.domain + exporters.node + exporters.nginx + exporters.nginxlog + exporters.systemd + + localExporters.prometheus-fail2ban-exporter + + { + # coturn + listenAddress = "127.0.0.1"; + port = "9641"; + } + { + # gitea + listenAddress = "127.0.0.1"; + port = "3000"; + } + ]; + } + ]; + } + ]; + }; + in { + enable = true; + path = [pkgs.victoriametrics]; + wantedBy = ["multi-user.target"]; + script = "vmagent -promscrape.config=${promscrape} -remoteWrite.url=http://localhost:8428/api/v1/write"; + }; + + services.nginx.virtualHosts."${domain}" = { + forceSSL = true; + enableACME = true; + extraConfig = '' + add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always; + access_log /var/log/nginx/${domain}/access.log upstream_time; + ''; + locations."/".proxyPass = "http://localhost:3001"; + }; } diff --git a/configuration/services/metrics/exporters.nix b/configuration/services/metrics/exporters.nix index b4af2a0..fc56316 100644 --- a/configuration/services/metrics/exporters.nix +++ b/configuration/services/metrics/exporters.nix @@ -1,100 +1,45 @@ { config, - pkgs, lib, ... -}: let - yaml = pkgs.formats.yaml {}; -in { - services.prometheus = { - exporters = { - # Periodically check domain registration status - domain = { - enable = true; - listenAddress = "127.0.0.1"; - extraFlags = let - conf.domains = [ - "tlater.net" - "tlater.com" - ]; - in [ - "--config=${yaml.generate "domains.yml" conf}" - ]; - }; - - # System statistics - node = { - enable = true; - listenAddress = "127.0.0.1"; - }; - systemd = { - enable = true; - listenAddress = "127.0.0.1"; - extraFlags = [ - # Disabled by default because only supported from systemd 235+ - "--systemd.collector.enable-restart-count" - "--systemd.collector.enable-ip-accounting" - ]; - }; - - # Various nginx metrics - nginx = { - enable = true; - listenAddress = "127.0.0.1"; - }; - - nginxlog = { - enable = true; - listenAddress = "127.0.0.1"; - group = "nginx"; - - settings.namespaces = - lib.mapAttrsToList (name: virtualHost: { - inherit name; - metrics_override.prefix = "nginxlog"; - namespace_label = "vhost"; - - format = lib.concatStringsSep " " [ - "$remote_addr - $remote_user [$time_local]" - ''"$request" $status $body_bytes_sent'' - ''"$http_referer" "$http_user_agent"'' - ''rt=$request_time uct="$upstream_connect_time"'' - ''uht="$upstream_header_time" urt="$upstream_response_time"'' - ]; - - source.files = [ - "/var/log/nginx/${name}/access.log" - ]; - }) - config.services.nginx.virtualHosts; - }; - }; - - extraExporters = { - fail2ban = let - cfg = config.services.prometheus.extraExporters.fail2ban; - in { - port = 9191; - serviceOpts = { - after = ["fail2ban.service"]; - requires = ["fail2ban.service"]; - serviceConfig = { - Group = "fail2ban"; - RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"]; - ExecStart = lib.concatStringsSep " " [ - "${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter" - "--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock" - "--web.listen-address='${cfg.listenAddress}:${toString cfg.port}'" - "--collector.f2b.exit-on-socket-connection-error=true" - ]; - }; - }; - }; - }; - - # TODO(tlater): - # - wireguard (?) - # - postgres (?) - # - blackbox (?) (curl to see if http and similar is up) +}: { + options.services.prometheus.local-exporters = lib.mkOption { + type = lib.types.anything; }; + + config.systemd.services = lib.mapAttrs (_: exporter: + lib.mkMerge [ + { + wantedBy = ["multi-user.target"]; + after = ["network.target"]; + + serviceConfig = { + Restart = "always"; + PrivateTmp = true; + WorkingDirectory = "/tmp"; + DynamicUser = true; + LockPersonality = true; + MemoryDenyWriteExecute = true; + NonNewPrivileges = true; + PrivateDevices = true; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectSystem = "strict"; + RemoveIPC = true; + RestrictAddressFamilies = lib.mkDefault ["AF_INET" "AF_INET6"]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + SystemCallArchitectures = "native"; + UMask = "0077"; + }; + } + (removeAttrs exporter ["port" "listenAddress"]) + ]) + config.services.prometheus.local-exporters; } diff --git a/configuration/services/metrics/grafana.nix b/configuration/services/metrics/grafana.nix deleted file mode 100644 index 8538dc7..0000000 --- a/configuration/services/metrics/grafana.nix +++ /dev/null @@ -1,48 +0,0 @@ -{config, ...}: let - domain = "metrics.${config.services.nginx.domain}"; -in { - services.grafana = { - enable = true; - settings = { - server.http_port = 3001; # Default overlaps with gitea - - security = { - admin_user = "tlater"; - admin_password = "$__file{${config.sops.secrets."grafana/adminPassword".path}}"; - secret_key = "$__file{${config.sops.secrets."grafana/secretKey".path}}"; - cookie_secure = true; - cookie_samesite = "strict"; - content_security_policy = true; - }; - - database = { - user = "grafana"; - name = "grafana"; - type = "postgres"; - host = "/run/postgresql"; - }; - }; - - provision = { - enable = true; - - datasources.settings.datasources = [ - { - name = "Victoriametrics - tlater.net"; - url = "http://localhost:8428"; - type = "prometheus"; - } - ]; - }; - }; - - services.nginx.virtualHosts."${domain}" = { - forceSSL = true; - enableACME = true; - extraConfig = '' - add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always; - access_log /var/log/nginx/${domain}/access.log upstream_time; - ''; - locations."/".proxyPass = "http://localhost:${toString config.services.grafana.settings.server.http_port}"; - }; -} diff --git a/configuration/services/metrics/options.nix b/configuration/services/metrics/options.nix deleted file mode 100644 index 65bcf83..0000000 --- a/configuration/services/metrics/options.nix +++ /dev/null @@ -1,90 +0,0 @@ -{ - config, - lib, - ... -}: let - inherit (lib) types mkOption mkDefault; -in { - options.services.prometheus = { - extraExporters = mkOption { - type = types.attrsOf (types.submodule { - options = { - port = mkOption { - type = types.int; - description = "The port on which this exporter listens."; - }; - listenAddress = mkOption { - type = types.str; - default = "127.0.0.1"; - description = "Address to listen on."; - }; - serviceOpts = mkOption { - type = types.attrs; - description = "An attrset to be merged with the exporter's systemd service."; - }; - }; - }); - }; - - allExporters = mkOption { - internal = true; - description = "The full list of scraping-relevant settings of all exporters, extra or built-in."; - type = types.attrsOf (types.submodule { - port = mkOption { - type = types.int; - }; - listenAddress = mkOption { - type = types.str; - }; - extraSettings = mkOption { - type = types.anything; - default = {}; - }; - }); - }; - }; - - config = { - systemd.services = lib.mapAttrs' (name: exporter: - lib.nameValuePair "prometheus-${name}-exporter" (lib.mkMerge [ - { - # Shamelessly copied from upstream because the upstream - # module is an intractable mess - wantedBy = ["multi-user.target"]; - after = ["network.target"]; - serviceConfig.Restart = mkDefault "always"; - serviceConfig.PrivateTmp = mkDefault true; - serviceConfig.WorkingDirectory = mkDefault /tmp; - serviceConfig.DynamicUser = mkDefault true; - # Hardening - serviceConfig.CapabilityBoundingSet = mkDefault [""]; - serviceConfig.DeviceAllow = [""]; - serviceConfig.LockPersonality = true; - serviceConfig.MemoryDenyWriteExecute = true; - serviceConfig.NoNewPrivileges = true; - serviceConfig.PrivateDevices = mkDefault true; - serviceConfig.ProtectClock = mkDefault true; - serviceConfig.ProtectControlGroups = true; - serviceConfig.ProtectHome = true; - serviceConfig.ProtectHostname = true; - serviceConfig.ProtectKernelLogs = true; - serviceConfig.ProtectKernelModules = true; - serviceConfig.ProtectKernelTunables = true; - serviceConfig.ProtectSystem = mkDefault "strict"; - serviceConfig.RemoveIPC = true; - serviceConfig.RestrictAddressFamilies = ["AF_INET" "AF_INET6"]; - serviceConfig.RestrictNamespaces = true; - serviceConfig.RestrictRealtime = true; - serviceConfig.RestrictSUIDSGID = true; - serviceConfig.SystemCallArchitectures = "native"; - serviceConfig.UMask = "0077"; - } - exporter.serviceOpts - ])) - config.services.prometheus.extraExporters; - - services.prometheus.allExporters = lib.mapAttrs (name: exporter: { - inherit (exporter) listenAddress port; - }) (config.services.prometheus.exporters ++ config.services.prometheus.extraExporters); - }; -} diff --git a/configuration/services/metrics/victoriametrics.nix b/configuration/services/metrics/victoriametrics.nix deleted file mode 100644 index a9fb3a2..0000000 --- a/configuration/services/metrics/victoriametrics.nix +++ /dev/null @@ -1,31 +0,0 @@ -{config, ...}: { - services.victoriametrics = let - scrapeConfigFromExporters = conf: conf // {inherit (config.services.prometheus.exporters.${conf.name}) listenAddress port;}; - scrapeConfigFromLocalExporters = conf: conf // {inherit (config.services.prometheus.local-exporters.${conf.name}) listenAddress port;}; - in { - enable = true; - vmagent-scraping.static_configs = - [ - { - name = "gitea"; - listenAddress = "127.0.0.1"; - port = 3000; - } - { - name = "coturn"; - listenAddress = "127.0.0.1"; - port = 9641; - } - ] - ++ (map scrapeConfigFromLocalExporters [ - {name = "prometheus-fail2ban-exporter";} - ]) - ++ (map scrapeConfigFromExporters [ - {name = "domain";} - {name = "node";} - {name = "nginx";} - {name = "nginxlog";} - {name = "systemd";} - ]); - }; -}