diff --git a/configuration/services/foundryvtt.nix b/configuration/services/foundryvtt.nix
index d573480..a4978fd 100644
--- a/configuration/services/foundryvtt.nix
+++ b/configuration/services/foundryvtt.nix
@@ -8,11 +8,11 @@
in {
imports = [flake-inputs.foundryvtt.nixosModules.foundryvtt];
- services.foundryvtt = {
- enable = true;
- hostName = domain;
- minifyStaticFiles = true;
- };
+ # services.foundryvtt = {
+ # enable = true;
+ # hostName = domain;
+ # minifyStaticFiles = true;
+ # };
# Want to start it manually when I need it, not have it constantly
# running
diff --git a/configuration/services/metrics/default.nix b/configuration/services/metrics/default.nix
index 4b163d3..84e126a 100644
--- a/configuration/services/metrics/default.nix
+++ b/configuration/services/metrics/default.nix
@@ -1,186 +1,9 @@
{
- config,
- pkgs,
- lib,
- ...
-}: let
- domain = "metrics.${config.services.nginx.domain}";
- yaml = pkgs.formats.yaml {};
-in {
imports = [
+ ./options.nix
+
./exporters.nix
+ ./grafana.nix
+ ./victoriametrics.nix
];
-
- services.victoriametrics.enable = true;
-
- services.grafana = {
- enable = true;
- settings = {
- server.http_port = 3001; # Default overlaps with gitea
-
- security = {
- admin_user = "tlater";
- admin_password = "$__file{${config.sops.secrets."grafana/adminPassword".path}}";
- secret_key = "$__file{${config.sops.secrets."grafana/secretKey".path}}";
- cookie_secure = true;
- cookie_samesite = "strict";
- content_security_policy = true;
- };
-
- database = {
- user = "grafana";
- name = "grafana";
- type = "postgres";
- host = "/run/postgresql";
- };
- };
-
- provision = {
- enable = true;
-
- datasources.settings.datasources = [
- {
- name = "Victoriametrics - tlater.net";
- url = "http://localhost:8428";
- type = "prometheus";
- }
- ];
- };
- };
-
- services.prometheus.exporters = {
- domain = {
- enable = true;
- listenAddress = "127.0.0.1";
- extraFlags = let
- conf.domains = [
- "tlater.net"
- "tlater.com"
- ];
- in [
- "--config=${yaml.generate "domains.yml" conf}"
- ];
- };
-
- node = {
- enable = true;
- listenAddress = "127.0.0.1";
- };
-
- nginx = {
- enable = true;
- listenAddress = "127.0.0.1";
- };
-
- nginxlog = {
- enable = true;
- listenAddress = "127.0.0.1";
- group = "nginx";
-
- settings.namespaces =
- lib.mapAttrsToList (name: virtualHost: {
- inherit name;
- metrics_override.prefix = "nginxlog";
- namespace_label = "vhost";
-
- format = lib.concatStringsSep " " [
- "$remote_addr - $remote_user [$time_local]"
- ''"$request" $status $body_bytes_sent''
- ''"$http_referer" "$http_user_agent"''
- ''rt=$request_time uct="$upstream_connect_time"''
- ''uht="$upstream_header_time" urt="$upstream_response_time"''
- ];
-
- source.files = [
- "/var/log/nginx/${name}/access.log"
- ];
- })
- config.services.nginx.virtualHosts;
- };
-
- systemd = {
- enable = true;
- listenAddress = "127.0.0.1";
- extraFlags = [
- # Disabled by default because only supported from systemd 235+
- "--systemd.collector.enable-restart-count"
- "--systemd.collector.enable-ip-accounting"
- ];
- };
- };
-
- services.prometheus.local-exporters = {
- prometheus-fail2ban-exporter = rec {
- enable = true;
- after = ["fail2ban.service"];
-
- port = 9191;
- listenAddress = "127.0.0.1";
-
- serviceConfig = {
- Group = "fail2ban";
-
- RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"];
-
- ExecStart = lib.concatStringsSep " " [
- "${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter"
- "--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock"
- "--web.listen-address='${listenAddress}:${toString port}'"
- ];
- };
- };
- };
-
- systemd.services.export-to-victoriametrics = let
- promscrape = yaml.generate "prometheus.yml" {
- scrape_configs = [
- {
- job_name = "tlater.net";
- static_configs = [
- {
- targets = let
- exporters = config.services.prometheus.exporters;
- localExporters = config.services.prometheus.local-exporters;
- in
- map (exporter: "${exporter.listenAddress}:${toString exporter.port}") [
- exporters.domain
- exporters.node
- exporters.nginx
- exporters.nginxlog
- exporters.systemd
-
- localExporters.prometheus-fail2ban-exporter
-
- {
- # coturn
- listenAddress = "127.0.0.1";
- port = "9641";
- }
- {
- # gitea
- listenAddress = "127.0.0.1";
- port = "3000";
- }
- ];
- }
- ];
- }
- ];
- };
- in {
- enable = true;
- path = [pkgs.victoriametrics];
- wantedBy = ["multi-user.target"];
- script = "vmagent -promscrape.config=${promscrape} -remoteWrite.url=http://localhost:8428/api/v1/write";
- };
-
- services.nginx.virtualHosts."${domain}" = {
- forceSSL = true;
- enableACME = true;
- extraConfig = ''
- add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
- access_log /var/log/nginx/${domain}/access.log upstream_time;
- '';
- locations."/".proxyPass = "http://localhost:3001";
- };
}
diff --git a/configuration/services/metrics/exporters.nix b/configuration/services/metrics/exporters.nix
index fc56316..b4af2a0 100644
--- a/configuration/services/metrics/exporters.nix
+++ b/configuration/services/metrics/exporters.nix
@@ -1,45 +1,100 @@
{
config,
+ pkgs,
lib,
...
-}: {
- options.services.prometheus.local-exporters = lib.mkOption {
- type = lib.types.anything;
- };
+}: let
+ yaml = pkgs.formats.yaml {};
+in {
+ services.prometheus = {
+ exporters = {
+ # Periodically check domain registration status
+ domain = {
+ enable = true;
+ listenAddress = "127.0.0.1";
+ extraFlags = let
+ conf.domains = [
+ "tlater.net"
+ "tlater.com"
+ ];
+ in [
+ "--config=${yaml.generate "domains.yml" conf}"
+ ];
+ };
- config.systemd.services = lib.mapAttrs (_: exporter:
- lib.mkMerge [
- {
- wantedBy = ["multi-user.target"];
- after = ["network.target"];
+ # System statistics
+ node = {
+ enable = true;
+ listenAddress = "127.0.0.1";
+ };
+ systemd = {
+ enable = true;
+ listenAddress = "127.0.0.1";
+ extraFlags = [
+ # Disabled by default because only supported from systemd 235+
+ "--systemd.collector.enable-restart-count"
+ "--systemd.collector.enable-ip-accounting"
+ ];
+ };
- serviceConfig = {
- Restart = "always";
- PrivateTmp = true;
- WorkingDirectory = "/tmp";
- DynamicUser = true;
- LockPersonality = true;
- MemoryDenyWriteExecute = true;
- NonNewPrivileges = true;
- PrivateDevices = true;
- ProtectClock = true;
- ProtectControlGroups = true;
- ProtectHome = true;
- ProtectHostname = true;
- ProtectKernelLogs = true;
- ProtectKernelModules = true;
- ProtectKernelTunables = true;
- ProtectSystem = "strict";
- RemoveIPC = true;
- RestrictAddressFamilies = lib.mkDefault ["AF_INET" "AF_INET6"];
- RestrictNamespaces = true;
- RestrictRealtime = true;
- RestrictSUIDSGID = true;
- SystemCallArchitectures = "native";
- UMask = "0077";
+ # Various nginx metrics
+ nginx = {
+ enable = true;
+ listenAddress = "127.0.0.1";
+ };
+
+ nginxlog = {
+ enable = true;
+ listenAddress = "127.0.0.1";
+ group = "nginx";
+
+ settings.namespaces =
+ lib.mapAttrsToList (name: virtualHost: {
+ inherit name;
+ metrics_override.prefix = "nginxlog";
+ namespace_label = "vhost";
+
+ format = lib.concatStringsSep " " [
+ "$remote_addr - $remote_user [$time_local]"
+ ''"$request" $status $body_bytes_sent''
+ ''"$http_referer" "$http_user_agent"''
+ ''rt=$request_time uct="$upstream_connect_time"''
+ ''uht="$upstream_header_time" urt="$upstream_response_time"''
+ ];
+
+ source.files = [
+ "/var/log/nginx/${name}/access.log"
+ ];
+ })
+ config.services.nginx.virtualHosts;
+ };
+ };
+
+ extraExporters = {
+ fail2ban = let
+ cfg = config.services.prometheus.extraExporters.fail2ban;
+ in {
+ port = 9191;
+ serviceOpts = {
+ after = ["fail2ban.service"];
+ requires = ["fail2ban.service"];
+ serviceConfig = {
+ Group = "fail2ban";
+ RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"];
+ ExecStart = lib.concatStringsSep " " [
+ "${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter"
+ "--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock"
+ "--web.listen-address='${cfg.listenAddress}:${toString cfg.port}'"
+ "--collector.f2b.exit-on-socket-connection-error=true"
+ ];
+ };
};
- }
- (removeAttrs exporter ["port" "listenAddress"])
- ])
- config.services.prometheus.local-exporters;
+ };
+ };
+
+ # TODO(tlater):
+ # - wireguard (?)
+ # - postgres (?)
+ # - blackbox (?) (curl to see if http and similar is up)
+ };
}
diff --git a/configuration/services/metrics/grafana.nix b/configuration/services/metrics/grafana.nix
new file mode 100644
index 0000000..8538dc7
--- /dev/null
+++ b/configuration/services/metrics/grafana.nix
@@ -0,0 +1,48 @@
+{config, ...}: let
+ domain = "metrics.${config.services.nginx.domain}";
+in {
+ services.grafana = {
+ enable = true;
+ settings = {
+ server.http_port = 3001; # Default overlaps with gitea
+
+ security = {
+ admin_user = "tlater";
+ admin_password = "$__file{${config.sops.secrets."grafana/adminPassword".path}}";
+ secret_key = "$__file{${config.sops.secrets."grafana/secretKey".path}}";
+ cookie_secure = true;
+ cookie_samesite = "strict";
+ content_security_policy = true;
+ };
+
+ database = {
+ user = "grafana";
+ name = "grafana";
+ type = "postgres";
+ host = "/run/postgresql";
+ };
+ };
+
+ provision = {
+ enable = true;
+
+ datasources.settings.datasources = [
+ {
+ name = "Victoriametrics - tlater.net";
+ url = "http://localhost:8428";
+ type = "prometheus";
+ }
+ ];
+ };
+ };
+
+ services.nginx.virtualHosts."${domain}" = {
+ forceSSL = true;
+ enableACME = true;
+ extraConfig = ''
+ add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+ access_log /var/log/nginx/${domain}/access.log upstream_time;
+ '';
+ locations."/".proxyPass = "http://localhost:${toString config.services.grafana.settings.server.http_port}";
+ };
+}
diff --git a/configuration/services/metrics/options.nix b/configuration/services/metrics/options.nix
new file mode 100644
index 0000000..65bcf83
--- /dev/null
+++ b/configuration/services/metrics/options.nix
@@ -0,0 +1,90 @@
+{
+ config,
+ lib,
+ ...
+}: let
+ inherit (lib) types mkOption mkDefault;
+in {
+ options.services.prometheus = {
+ extraExporters = mkOption {
+ type = types.attrsOf (types.submodule {
+ options = {
+ port = mkOption {
+ type = types.int;
+ description = "The port on which this exporter listens.";
+ };
+ listenAddress = mkOption {
+ type = types.str;
+ default = "127.0.0.1";
+ description = "Address to listen on.";
+ };
+ serviceOpts = mkOption {
+ type = types.attrs;
+ description = "An attrset to be merged with the exporter's systemd service.";
+ };
+ };
+ });
+ };
+
+ allExporters = mkOption {
+ internal = true;
+ description = "The full list of scraping-relevant settings of all exporters, extra or built-in.";
+ type = types.attrsOf (types.submodule {
+ port = mkOption {
+ type = types.int;
+ };
+ listenAddress = mkOption {
+ type = types.str;
+ };
+ extraSettings = mkOption {
+ type = types.anything;
+ default = {};
+ };
+ });
+ };
+ };
+
+ config = {
+ systemd.services = lib.mapAttrs' (name: exporter:
+ lib.nameValuePair "prometheus-${name}-exporter" (lib.mkMerge [
+ {
+ # Shamelessly copied from upstream because the upstream
+ # module is an intractable mess
+ wantedBy = ["multi-user.target"];
+ after = ["network.target"];
+ serviceConfig.Restart = mkDefault "always";
+ serviceConfig.PrivateTmp = mkDefault true;
+ serviceConfig.WorkingDirectory = mkDefault /tmp;
+ serviceConfig.DynamicUser = mkDefault true;
+ # Hardening
+ serviceConfig.CapabilityBoundingSet = mkDefault [""];
+ serviceConfig.DeviceAllow = [""];
+ serviceConfig.LockPersonality = true;
+ serviceConfig.MemoryDenyWriteExecute = true;
+ serviceConfig.NoNewPrivileges = true;
+ serviceConfig.PrivateDevices = mkDefault true;
+ serviceConfig.ProtectClock = mkDefault true;
+ serviceConfig.ProtectControlGroups = true;
+ serviceConfig.ProtectHome = true;
+ serviceConfig.ProtectHostname = true;
+ serviceConfig.ProtectKernelLogs = true;
+ serviceConfig.ProtectKernelModules = true;
+ serviceConfig.ProtectKernelTunables = true;
+ serviceConfig.ProtectSystem = mkDefault "strict";
+ serviceConfig.RemoveIPC = true;
+ serviceConfig.RestrictAddressFamilies = ["AF_INET" "AF_INET6"];
+ serviceConfig.RestrictNamespaces = true;
+ serviceConfig.RestrictRealtime = true;
+ serviceConfig.RestrictSUIDSGID = true;
+ serviceConfig.SystemCallArchitectures = "native";
+ serviceConfig.UMask = "0077";
+ }
+ exporter.serviceOpts
+ ]))
+ config.services.prometheus.extraExporters;
+
+ services.prometheus.allExporters = lib.mapAttrs (name: exporter: {
+ inherit (exporter) listenAddress port;
+ }) (config.services.prometheus.exporters ++ config.services.prometheus.extraExporters);
+ };
+}
diff --git a/configuration/services/metrics/victoriametrics.nix b/configuration/services/metrics/victoriametrics.nix
new file mode 100644
index 0000000..a9fb3a2
--- /dev/null
+++ b/configuration/services/metrics/victoriametrics.nix
@@ -0,0 +1,31 @@
+{config, ...}: {
+ services.victoriametrics = let
+ scrapeConfigFromExporters = conf: conf // {inherit (config.services.prometheus.exporters.${conf.name}) listenAddress port;};
+ scrapeConfigFromLocalExporters = conf: conf // {inherit (config.services.prometheus.local-exporters.${conf.name}) listenAddress port;};
+ in {
+ enable = true;
+ vmagent-scraping.static_configs =
+ [
+ {
+ name = "gitea";
+ listenAddress = "127.0.0.1";
+ port = 3000;
+ }
+ {
+ name = "coturn";
+ listenAddress = "127.0.0.1";
+ port = 9641;
+ }
+ ]
+ ++ (map scrapeConfigFromLocalExporters [
+ {name = "prometheus-fail2ban-exporter";}
+ ])
+ ++ (map scrapeConfigFromExporters [
+ {name = "domain";}
+ {name = "node";}
+ {name = "nginx";}
+ {name = "nginxlog";}
+ {name = "systemd";}
+ ]);
+ };
+}