From 612b63b4b38aa7aa62cf460ed23dacf0775be406 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tristan=20Dani=C3=ABl=20Maat?= <tm@tlater.net>
Date: Mon, 25 Sep 2023 03:32:04 +0200
Subject: [PATCH 1/9] Add metrics
---
configuration/default.nix | 1 +
configuration/services/metrics.nix | 86 +++++++++++++++++++++++++++++
configuration/services/postgres.nix | 7 +++
configuration/sops.nix | 10 ++++
keys/production.yaml | 6 +-
keys/staging.yaml | 7 ++-
6 files changed, 113 insertions(+), 4 deletions(-)
create mode 100644 configuration/services/metrics.nix
diff --git a/configuration/default.nix b/configuration/default.nix
index 5d491af..610851a 100644
--- a/configuration/default.nix
+++ b/configuration/default.nix
@@ -18,6 +18,7 @@
./services/conduit.nix
./services/foundryvtt.nix
./services/gitea.nix
+ ./services/metrics.nix
./services/nextcloud.nix
./services/webserver.nix
./services/wireguard.nix
diff --git a/configuration/services/metrics.nix b/configuration/services/metrics.nix
new file mode 100644
index 0000000..12a48e0
--- /dev/null
+++ b/configuration/services/metrics.nix
@@ -0,0 +1,86 @@
+{
+ config,
+ pkgs,
+ lib,
+ ...
+}: let
+ domain = "metrics.${config.services.nginx.domain}";
+ yaml = pkgs.formats.yaml {};
+in {
+ services.victoriametrics.enable = true;
+
+ services.grafana = {
+ enable = true;
+ settings = {
+ server.http_port = 3001; # Default overlaps with gitea
+
+ security = {
+ admin_user = "tlater";
+ admin_password = "$__file{${config.sops.secrets."grafana/adminPassword".path}}";
+ secret_key = "$__file{${config.sops.secrets."grafana/secretKey".path}}";
+ cookie_secure = true;
+ cookie_samesite = "strict";
+ content_security_policy = true;
+ };
+
+ database = {
+ user = "grafana";
+ name = "grafana";
+ type = "postgres";
+ host = "/run/postgresql";
+ };
+ };
+
+ provision = {
+ enable = true;
+
+ datasources.settings.datasources = [
+ {
+ name = "Victoriametrics - tlater.net";
+ url = "http://localhost:8428";
+ type = "prometheus";
+ }
+ ];
+ };
+ };
+
+ services.prometheus.exporters = {
+ node = {
+ enable = true;
+ enabledCollectors = ["systemd"];
+ listenAddress = "127.0.0.1";
+ };
+ };
+
+ systemd.services.export-to-victoriametrics = let
+ promscrape = yaml.generate "prometheus.yml" {
+ scrape_configs = [
+ {
+ job_name = "tlater.net";
+ static_configs = [
+ {
+ targets =
+ lib.mapAttrsToList (name: exporter: "${exporter.listenAddress}:${toString exporter.port}")
+ (lib.filterAttrs (name: exporter: (builtins.isAttrs exporter) && exporter.enable)
+ config.services.prometheus.exporters);
+ }
+ ];
+ }
+ ];
+ };
+ in {
+ enable = true;
+ path = [pkgs.victoriametrics];
+ wantedBy = ["multi-user.target"];
+ script = "vmagent -promscrape.config=${promscrape} -remoteWrite.url=http://localhost:8428/api/v1/write";
+ };
+
+ services.nginx.virtualHosts."${domain}" = {
+ forceSSL = true;
+ enableACME = true;
+ extraConfig = ''
+ add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+ '';
+ locations."/".proxyPass = "http://localhost:3001";
+ };
+}
diff --git a/configuration/services/postgres.nix b/configuration/services/postgres.nix
index 6c584bb..923007d 100644
--- a/configuration/services/postgres.nix
+++ b/configuration/services/postgres.nix
@@ -16,6 +16,12 @@
# that operation needs to be performed manually on the system as
# well.
ensureUsers = [
+ {
+ name = "grafana";
+ ensurePermissions = {
+ "DATABASE grafana" = "ALL PRIVILEGES";
+ };
+ }
{
name = "nextcloud";
ensurePermissions = {
@@ -25,6 +31,7 @@
];
ensureDatabases = [
+ "grafana"
"nextcloud"
];
};
diff --git a/configuration/sops.nix b/configuration/sops.nix
index bf98433..6eae9fc 100644
--- a/configuration/sops.nix
+++ b/configuration/sops.nix
@@ -3,6 +3,16 @@
defaultSopsFile = ../keys/production.yaml;
secrets = {
+ # Grafana
+ "grafana/adminPassword" = {
+ owner = "grafana";
+ group = "grafana";
+ };
+ "grafana/secretKey" = {
+ owner = "grafana";
+ group = "grafana";
+ };
+
# Heisenbridge
"heisenbridge/as-token" = {};
"heisenbridge/hs-token" = {};
diff --git a/keys/production.yaml b/keys/production.yaml
index f8d259d..87ef3c4 100644
--- a/keys/production.yaml
+++ b/keys/production.yaml
@@ -1,3 +1,5 @@
+metrics:
+ tlater: ENC[AES256_GCM,data:4nB0H45nlongb0x1OOKzNXIk96PovZ7OwENovnBZUwMl9ncfYwTHT30OlLsPA75w1govH0jyBRkn1Pe/qHzY1Zt53B8=,iv:AfZ4So6HnjOXzqiHM3WpOsQZJs2CEckuxGfyDxc4TNA=,tag:fIXOKJSVDLpLbiLd2zAu9w==,type:str]
nextcloud:
tlater: ENC[AES256_GCM,data:zNsPm4uFaIRe3LjcwmayRg==,iv:5wam6bP5zP708jC9UrLV0s8qspl3Pm4fPzbMFYBUyPQ=,tag:apnJUMeJwMn9q0NhO4ptmA==,type:str]
steam:
@@ -21,8 +23,8 @@ sops:
azure_kv: []
hc_vault: []
age: []
- lastmodified: "2023-09-23T18:55:44Z"
- mac: ENC[AES256_GCM,data:psqgXozY9L7nduZ11GF+mbIrZ4RUySqBixkWL5z0cYeoLA3URb/dr028LCmNgQS9l8aJVsjVkyLBJIU/8wmiUNqRy/VI5iqV5mu+sxXhUVwFL0dAAWP1lOKwwT5uGK89/ioqkphgzuWD37vGe2vYddKkJF0M+zlz12fqkMjaisU=,iv:UyRoJbfuGU3K/Mp5DQ1kY0Z+nKSSo46BGNAcxt+vAvc=,tag:HkP6+qxQ8J/xAYJXYoG/6g==,type:str]
+ lastmodified: "2023-09-25T00:42:25Z"
+ mac: ENC[AES256_GCM,data:28o/elUKslgn5auYfr34N9fE7B6EoZ6njL6yT0emjfoTjsCADJOLcHfUDNWb3AMP3Z5e/w8WsxI7MpwuwUXRvZ6u9Kui1IBcQu/V6GEzpBVw7JkLHZvsUFHOj/uEBcPvON7pKfXtG3vdH8FF1cxeenFm1Z0cX4C0WrNaxumGknA=,iv:GYK0/JZtCkbVorus+9HQbtxAnIXviiNkoC9dMqTHflM=,tag:R3N5hf/UV2nqyOI50Imr6g==,type:str]
pgp:
- created_at: "2022-10-12T00:46:51Z"
enc: |
diff --git a/keys/staging.yaml b/keys/staging.yaml
index a6b0849..86ce700 100644
--- a/keys/staging.yaml
+++ b/keys/staging.yaml
@@ -1,3 +1,6 @@
+grafana:
+ adminPassword: ENC[AES256_GCM,data:dYfaxUpQpzA=,iv:j5wSem8C5+V4c5qRzXQJhsU7/FOtpvrnaEyFBmW6zJ4=,tag:oc8n3TkEbjF2gjuOobZuLA==,type:str]
+ secretKey: ENC[AES256_GCM,data:Atruvh2MsNY=,iv:y2MaCUCEzGIydHp6G0DJHfk289S1is0twKm2oUYwDhM=,tag:nAWeg+YqaYqk6k22oBkAhQ==,type:str]
nextcloud:
tlater: ENC[AES256_GCM,data:91kDcO4hpng=,iv:ayuILRmRru4ZxTCur9H2xHuLjkDzwPdS/4lEog/tesU=,tag:qYhJxnNDcCwUM7xe7Tlcjw==,type:str]
steam:
@@ -21,8 +24,8 @@ sops:
azure_kv: []
hc_vault: []
age: []
- lastmodified: "2023-09-22T21:07:02Z"
- mac: ENC[AES256_GCM,data:gItC41S8MInLmikdH1okhPs+FVf8sCF/iQeJ5reigBunHkOngoc6nOFANyAcNZETszzhgTLXXtmVNEjW46v6K7D6nmoi/zwpedUxwzMwDC5I28VTMDHVMAThYSGtdo6kig8i2pi8rzEQd1DStxMv3TWML5y6DDTlFsd3lfudaHA=,iv:zXebvIVPR76GwUhpactwRgF/eEmx2OBkT18E8lkwzRA=,tag:6HyISACbFCGlpIIgkFeA/A==,type:str]
+ lastmodified: "2023-10-01T23:00:06Z"
+ mac: ENC[AES256_GCM,data:oEJ3Nwlx5YTVLvWa12On1O+LakU42rsAD1wD52MTlzuwgyRZ/g49pL6pQiL6S0uE7wC0EOqOvg2pCtDxxHe3WNjEpcxnWWftdEjw2laLnBuOqduQmVW+Sn23SzoRkl7PwOH1jTQHzRyciyYkJT1/vCNnbNdKg1eqnbpxPysg6/A=,iv:dC8eNEXhzC8Nx1rfXQdDKtlO01QhyW9ncNFEK/yakrg=,tag:vQ4AW/mqnA9Vs5NNzFsYWQ==,type:str]
pgp:
- created_at: "2022-10-12T16:48:23Z"
enc: |
From cb4527d52560f168b3248c976347ca947257d920 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tristan=20Dani=C3=ABl=20Maat?= <tm@tlater.net>
Date: Mon, 2 Oct 2023 22:53:56 +0200
Subject: [PATCH 2/9] nginx: Add metrics
---
configuration/default.nix | 32 +++++++++++++++++++++++++++
configuration/services/conduit.nix | 1 +
configuration/services/foundryvtt.nix | 1 +
configuration/services/gitea.nix | 1 +
configuration/services/metrics.nix | 32 +++++++++++++++++++++++++++
configuration/services/nextcloud.nix | 3 +++
configuration/services/webserver.nix | 1 +
7 files changed, 71 insertions(+)
diff --git a/configuration/default.nix b/configuration/default.nix
index 610851a..14775b7 100644
--- a/configuration/default.nix
+++ b/configuration/default.nix
@@ -137,8 +137,40 @@
recommendedProxySettings = true;
clientMaxBodySize = "10G";
domain = "tlater.net";
+
+ statusPage = true; # For metrics, should be accessible only from localhost
+
+ commonHttpConfig = ''
+ log_format upstream_time '$remote_addr - $remote_user [$time_local] '
+ '"$request" $status $body_bytes_sent '
+ '"$http_referer" "$http_user_agent" '
+ 'rt=$request_time uct="$upstream_connect_time" '
+ 'uht="$upstream_header_time" urt="$upstream_response_time"';
+ '';
};
+ services.logrotate = {
+ enable = true;
+
+ settings = lib.mapAttrs' (virtualHost: _:
+ lib.nameValuePair "/var/log/nginx/${virtualHost}/access.log" {
+ frequency = "daily";
+ rotate = 2;
+ compress = true;
+ delaycompress = true;
+ su = "${config.services.nginx.user} ${config.services.nginx.group}";
+ postrotate = "[ ! -f /var/run/nginx/nginx.pid ] || kill -USR1 `cat /var/run/nginx/nginx.pid`";
+ })
+ config.services.nginx.virtualHosts;
+ };
+ systemd.tmpfiles.rules =
+ lib.mapAttrsToList (
+ virtualHost: _:
+ #
+ "d /var/log/nginx/${virtualHost} 0750 ${config.services.nginx.user} ${config.services.nginx.group}"
+ )
+ config.services.nginx.virtualHosts;
+
security.acme = {
defaults.email = "tm@tlater.net";
acceptTerms = true;
diff --git a/configuration/services/conduit.nix b/configuration/services/conduit.nix
index 3f8fd40..dcd0103 100644
--- a/configuration/services/conduit.nix
+++ b/configuration/services/conduit.nix
@@ -205,6 +205,7 @@ in {
addSSL = true;
extraConfig = ''
merge_slashes off;
+ access_log /var/log/nginx/${domain}/access.log upstream_time;
'';
locations = {
diff --git a/configuration/services/foundryvtt.nix b/configuration/services/foundryvtt.nix
index 7bb2286..d573480 100644
--- a/configuration/services/foundryvtt.nix
+++ b/configuration/services/foundryvtt.nix
@@ -25,6 +25,7 @@ in {
enableACME = true;
extraConfig = ''
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+ access_log /var/log/nginx/${domain}/access.log upstream_time;
'';
locations."/" = {
diff --git a/configuration/services/gitea.nix b/configuration/services/gitea.nix
index 27353f6..6d6dafd 100644
--- a/configuration/services/gitea.nix
+++ b/configuration/services/gitea.nix
@@ -33,6 +33,7 @@ in {
enableACME = true;
extraConfig = ''
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+ access_log /var/log/nginx/${domain}/access.log upstream_time;
'';
locations."/".proxyPass = "http://${httpAddress}:${toString httpPort}";
diff --git a/configuration/services/metrics.nix b/configuration/services/metrics.nix
index 12a48e0..ad71a98 100644
--- a/configuration/services/metrics.nix
+++ b/configuration/services/metrics.nix
@@ -50,6 +50,37 @@ in {
enabledCollectors = ["systemd"];
listenAddress = "127.0.0.1";
};
+
+ nginx = {
+ enable = true;
+ listenAddress = "127.0.0.1";
+ };
+
+ nginxlog = {
+ enable = true;
+ listenAddress = "127.0.0.1";
+ group = "nginx";
+
+ settings.namespaces =
+ lib.mapAttrsToList (name: virtualHost: {
+ inherit name;
+ metrics_override.prefix = "nginxlog";
+ namespace_label = "vhost";
+
+ format = lib.concatStringsSep " " [
+ "$remote_addr - $remote_user [$time_local]"
+ ''"$request" $status $body_bytes_sent''
+ ''"$http_referer" "$http_user_agent"''
+ ''rt=$request_time uct="$upstream_connect_time"''
+ ''uht="$upstream_header_time" urt="$upstream_response_time"''
+ ];
+
+ source.files = [
+ "/var/log/nginx/${name}/access.log"
+ ];
+ })
+ config.services.nginx.virtualHosts;
+ };
};
systemd.services.export-to-victoriametrics = let
@@ -80,6 +111,7 @@ in {
enableACME = true;
extraConfig = ''
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+ access_log /var/log/nginx/${domain}/access.log upstream_time;
'';
locations."/".proxyPass = "http://localhost:3001";
};
diff --git a/configuration/services/nextcloud.nix b/configuration/services/nextcloud.nix
index fbca607..81f38a3 100644
--- a/configuration/services/nextcloud.nix
+++ b/configuration/services/nextcloud.nix
@@ -50,6 +50,9 @@ in {
services.nginx.virtualHosts."${hostName}" = {
forceSSL = true;
enableACME = true;
+ extraConfig = ''
+ access_log /var/log/nginx/${hostName}/access.log upstream_time;
+ '';
};
# Block repeated failed login attempts
diff --git a/configuration/services/webserver.nix b/configuration/services/webserver.nix
index 4a8bee4..085b1f7 100644
--- a/configuration/services/webserver.nix
+++ b/configuration/services/webserver.nix
@@ -19,6 +19,7 @@ in {
enableACME = true;
extraConfig = ''
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+ access_log /var/log/nginx/${domain}/access.log upstream_time;
'';
locations."/".proxyPass = "http://${addr}:${toString port}";
From 3de03a32aca4f691e9a42336be8e4af0c7cfa523 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tristan=20Dani=C3=ABl=20Maat?= <tm@tlater.net>
Date: Mon, 2 Oct 2023 22:54:21 +0200
Subject: [PATCH 3/9] fail2ban: Add metrics
---
configuration/default.nix | 24 +---------
configuration/services/fail2ban.nix | 42 +++++++++++++++++
.../{metrics.nix => metrics/default.nix} | 28 +++++++++++-
configuration/services/metrics/exporters.nix | 45 +++++++++++++++++++
flake.nix | 10 ++++-
pkgs/_sources_pkgs/generated.json | 21 +++++++++
pkgs/_sources_pkgs/generated.nix | 16 +++++++
pkgs/default.nix | 3 ++
pkgs/nvfetcher.toml | 3 ++
pkgs/prometheus/fail2ban-exporter.nix | 8 ++++
10 files changed, 176 insertions(+), 24 deletions(-)
create mode 100644 configuration/services/fail2ban.nix
rename configuration/services/{metrics.nix => metrics/default.nix} (80%)
create mode 100644 configuration/services/metrics/exporters.nix
create mode 100644 pkgs/_sources_pkgs/generated.json
create mode 100644 pkgs/_sources_pkgs/generated.nix
create mode 100644 pkgs/nvfetcher.toml
create mode 100644 pkgs/prometheus/fail2ban-exporter.nix
diff --git a/configuration/default.nix b/configuration/default.nix
index 14775b7..81e7241 100644
--- a/configuration/default.nix
+++ b/configuration/default.nix
@@ -16,9 +16,10 @@
./services/backups.nix
./services/conduit.nix
+ ./services/fail2ban.nix
./services/foundryvtt.nix
./services/gitea.nix
- ./services/metrics.nix
+ ./services/metrics
./services/nextcloud.nix
./services/webserver.nix
./services/wireguard.nix
@@ -176,27 +177,6 @@
acceptTerms = true;
};
- services.fail2ban = {
- enable = true;
- extraPackages = [pkgs.ipset];
- banaction = "iptables-ipset-proto6-allports";
- bantime-increment.enable = true;
-
- jails = {
- nginx-botsearch = ''
- enabled = true
- logpath = /var/log/nginx/access.log
- '';
- };
-
- ignoreIP = [
- "127.0.0.0/8"
- "10.0.0.0/8"
- "172.16.0.0/12"
- "192.168.0.0/16"
- ];
- };
-
# Remove some unneeded packages
environment.defaultPackages = [];
diff --git a/configuration/services/fail2ban.nix b/configuration/services/fail2ban.nix
new file mode 100644
index 0000000..ace3219
--- /dev/null
+++ b/configuration/services/fail2ban.nix
@@ -0,0 +1,42 @@
+{pkgs, ...}: {
+ services.fail2ban = {
+ enable = true;
+ extraPackages = [pkgs.ipset];
+ banaction = "iptables-ipset-proto6-allports";
+ bantime-increment.enable = true;
+
+ jails = {
+ nginx-botsearch = ''
+ enabled = true
+ logpath = /var/log/nginx/access.log
+ '';
+ };
+
+ ignoreIP = [
+ "127.0.0.0/8"
+ "10.0.0.0/8"
+ "172.16.0.0/12"
+ "192.168.0.0/16"
+ ];
+ };
+
+ # Allow metrics services to connect to the socket as well
+ users.groups.fail2ban = {};
+ systemd.services.fail2ban.serviceConfig = {
+ ExecStartPost =
+ "+"
+ + (pkgs.writeShellScript "fail2ban-post-start" ''
+ while ! [ -S /var/run/fail2ban/fail2ban.sock ]; do
+ sleep 1
+ done
+
+ while ! ${pkgs.netcat}/bin/nc -zU /var/run/fail2ban/fail2ban.sock; do
+ sleep 1
+ done
+
+ ${pkgs.coreutils}/bin/chown root:fail2ban /var/run/fail2ban /var/run/fail2ban/fail2ban.sock
+ ${pkgs.coreutils}/bin/chmod 660 /var/run/fail2ban/fail2ban.sock
+ ${pkgs.coreutils}/bin/chmod 710 /var/run/fail2ban
+ '');
+ };
+}
diff --git a/configuration/services/metrics.nix b/configuration/services/metrics/default.nix
similarity index 80%
rename from configuration/services/metrics.nix
rename to configuration/services/metrics/default.nix
index ad71a98..0c02556 100644
--- a/configuration/services/metrics.nix
+++ b/configuration/services/metrics/default.nix
@@ -7,6 +7,10 @@
domain = "metrics.${config.services.nginx.domain}";
yaml = pkgs.formats.yaml {};
in {
+ imports = [
+ ./exporters.nix
+ ];
+
services.victoriametrics.enable = true;
services.grafana = {
@@ -83,6 +87,28 @@ in {
};
};
+ services.prometheus.local-exporters = {
+ prometheus-fail2ban-exporter = rec {
+ enable = true;
+ after = ["fail2ban.service"];
+
+ port = 9191;
+ listenAddress = "127.0.0.1";
+
+ serviceConfig = {
+ Group = "fail2ban";
+
+ RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"];
+
+ ExecStart = lib.concatStringsSep " " [
+ "${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter"
+ "--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock"
+ "--web.listen-address='${listenAddress}:${toString port}'"
+ ];
+ };
+ };
+ };
+
systemd.services.export-to-victoriametrics = let
promscrape = yaml.generate "prometheus.yml" {
scrape_configs = [
@@ -93,7 +119,7 @@ in {
targets =
lib.mapAttrsToList (name: exporter: "${exporter.listenAddress}:${toString exporter.port}")
(lib.filterAttrs (name: exporter: (builtins.isAttrs exporter) && exporter.enable)
- config.services.prometheus.exporters);
+ (config.services.prometheus.exporters // config.services.prometheus.local-exporters));
}
];
}
diff --git a/configuration/services/metrics/exporters.nix b/configuration/services/metrics/exporters.nix
new file mode 100644
index 0000000..fc56316
--- /dev/null
+++ b/configuration/services/metrics/exporters.nix
@@ -0,0 +1,45 @@
+{
+ config,
+ lib,
+ ...
+}: {
+ options.services.prometheus.local-exporters = lib.mkOption {
+ type = lib.types.anything;
+ };
+
+ config.systemd.services = lib.mapAttrs (_: exporter:
+ lib.mkMerge [
+ {
+ wantedBy = ["multi-user.target"];
+ after = ["network.target"];
+
+ serviceConfig = {
+ Restart = "always";
+ PrivateTmp = true;
+ WorkingDirectory = "/tmp";
+ DynamicUser = true;
+ LockPersonality = true;
+ MemoryDenyWriteExecute = true;
+ NonNewPrivileges = true;
+ PrivateDevices = true;
+ ProtectClock = true;
+ ProtectControlGroups = true;
+ ProtectHome = true;
+ ProtectHostname = true;
+ ProtectKernelLogs = true;
+ ProtectKernelModules = true;
+ ProtectKernelTunables = true;
+ ProtectSystem = "strict";
+ RemoveIPC = true;
+ RestrictAddressFamilies = lib.mkDefault ["AF_INET" "AF_INET6"];
+ RestrictNamespaces = true;
+ RestrictRealtime = true;
+ RestrictSUIDSGID = true;
+ SystemCallArchitectures = "native";
+ UMask = "0077";
+ };
+ }
+ (removeAttrs exporter ["port" "listenAddress"])
+ ])
+ config.services.prometheus.local-exporters;
+}
diff --git a/flake.nix b/flake.nix
index b6db610..d8ff1a8 100644
--- a/flake.nix
+++ b/flake.nix
@@ -78,7 +78,7 @@
# Utility scripts #
###################
packages.${system} = let
- inherit (nixpkgs.legacyPackages.${system}) writeShellScript;
+ inherit (nixpkgs.legacyPackages.${system}) writeShellScript writeShellScriptBin;
vm = nixpkgs.lib.nixosSystem {
inherit system;
specialArgs.flake-inputs = inputs;
@@ -106,6 +106,14 @@
"${vm.config.system.build.vm}/bin/run-tlaternet-vm"
'';
+ update-pkgs = let
+ nvfetcher-bin = "${nvfetcher.packages.${system}.default}/bin/nvfetcher";
+ in
+ writeShellScriptBin "update-pkgs" ''
+ cd "$(git rev-parse --show-toplevel)/pkgs"
+ ${nvfetcher-bin} -o _sources_pkgs -c nvfetcher.toml
+ '';
+
update-nextcloud-apps = let
nvfetcher-bin = "${nvfetcher.packages.${system}.default}/bin/nvfetcher";
in
diff --git a/pkgs/_sources_pkgs/generated.json b/pkgs/_sources_pkgs/generated.json
new file mode 100644
index 0000000..b3faf9a
--- /dev/null
+++ b/pkgs/_sources_pkgs/generated.json
@@ -0,0 +1,21 @@
+{
+ "prometheus-fail2ban-exporter": {
+ "cargoLocks": null,
+ "date": null,
+ "extract": null,
+ "name": "prometheus-fail2ban-exporter",
+ "passthru": null,
+ "pinned": false,
+ "src": {
+ "deepClone": false,
+ "fetchSubmodules": false,
+ "leaveDotGit": false,
+ "name": null,
+ "rev": "v0.10.0",
+ "sha256": "sha256-8nIW1XaHCBqQCoLkV1ZYE3NTbVZ6c+UOqYD08XQiv+4=",
+ "type": "git",
+ "url": "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter"
+ },
+ "version": "v0.10.0"
+ }
+}
\ No newline at end of file
diff --git a/pkgs/_sources_pkgs/generated.nix b/pkgs/_sources_pkgs/generated.nix
new file mode 100644
index 0000000..bb015b4
--- /dev/null
+++ b/pkgs/_sources_pkgs/generated.nix
@@ -0,0 +1,16 @@
+# This file was generated by nvfetcher, please do not modify it manually.
+{ fetchgit, fetchurl, fetchFromGitHub, dockerTools }:
+{
+ prometheus-fail2ban-exporter = {
+ pname = "prometheus-fail2ban-exporter";
+ version = "v0.10.0";
+ src = fetchgit {
+ url = "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter";
+ rev = "v0.10.0";
+ fetchSubmodules = false;
+ deepClone = false;
+ leaveDotGit = false;
+ sha256 = "sha256-8nIW1XaHCBqQCoLkV1ZYE3NTbVZ6c+UOqYD08XQiv+4=";
+ };
+ };
+}
diff --git a/pkgs/default.nix b/pkgs/default.nix
index 3818a26..3130ae0 100644
--- a/pkgs/default.nix
+++ b/pkgs/default.nix
@@ -7,6 +7,9 @@
in
{
starbound = callPackage ./starbound {};
+ prometheus-fail2ban-exporter = callPackage ./prometheus/fail2ban-exporter.nix {
+ sources = pkgs.callPackage ./_sources_pkgs/generated.nix {};
+ };
}
// (
# Add nextcloud apps
diff --git a/pkgs/nvfetcher.toml b/pkgs/nvfetcher.toml
new file mode 100644
index 0000000..8c53200
--- /dev/null
+++ b/pkgs/nvfetcher.toml
@@ -0,0 +1,3 @@
+[prometheus-fail2ban-exporter]
+src.manual = "v0.10.0" # No gitlab support in nvfetcher
+fetch.git = "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter"
diff --git a/pkgs/prometheus/fail2ban-exporter.nix b/pkgs/prometheus/fail2ban-exporter.nix
new file mode 100644
index 0000000..50b4973
--- /dev/null
+++ b/pkgs/prometheus/fail2ban-exporter.nix
@@ -0,0 +1,8 @@
+{
+ buildGoModule,
+ sources,
+}:
+buildGoModule {
+ inherit (sources.prometheus-fail2ban-exporter) pname src version;
+ vendorHash = "sha256-qU6opwhhvzbQOhfGVyiVgKhfCSB0Z4eSRAJnv6ht2I0=";
+}
From eb539f6ee70d8f2dde0503dfa58e2e1f5c8d9087 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tristan=20Dani=C3=ABl=20Maat?= <tm@tlater.net>
Date: Sat, 7 Oct 2023 03:58:33 +0200
Subject: [PATCH 4/9] metrics: Add domain monitoring
---
configuration/services/metrics/default.nix | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/configuration/services/metrics/default.nix b/configuration/services/metrics/default.nix
index 0c02556..4086ef2 100644
--- a/configuration/services/metrics/default.nix
+++ b/configuration/services/metrics/default.nix
@@ -49,6 +49,19 @@ in {
};
services.prometheus.exporters = {
+ domain = {
+ enable = true;
+ listenAddress = "127.0.0.1";
+ extraFlags = let
+ conf.domains = [
+ "tlater.net"
+ "tlater.com"
+ ];
+ in [
+ "--config=${yaml.generate "domains.yml" conf}"
+ ];
+ };
+
node = {
enable = true;
enabledCollectors = ["systemd"];
From 214c59b7b34a0cce592e87eec74e18ccb47c3868 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tristan=20Dani=C3=ABl=20Maat?= <tm@tlater.net>
Date: Sat, 7 Oct 2023 03:58:59 +0200
Subject: [PATCH 5/9] metrics: Add systemd monitoring
---
configuration/services/metrics/default.nix | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/configuration/services/metrics/default.nix b/configuration/services/metrics/default.nix
index 4086ef2..53a8a1d 100644
--- a/configuration/services/metrics/default.nix
+++ b/configuration/services/metrics/default.nix
@@ -64,7 +64,6 @@ in {
node = {
enable = true;
- enabledCollectors = ["systemd"];
listenAddress = "127.0.0.1";
};
@@ -98,6 +97,16 @@ in {
})
config.services.nginx.virtualHosts;
};
+
+ systemd = {
+ enable = true;
+ listenAddress = "127.0.0.1";
+ extraFlags = [
+ # Disabled by default because only supported from systemd 235+
+ "--systemd.collector.enable-restart-count"
+ "--systemd.collector.enable-ip-accounting"
+ ];
+ };
};
services.prometheus.local-exporters = {
From c373911a1bfacd0576d1739ec9f9800d98a18587 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tristan=20Dani=C3=ABl=20Maat?= <tm@tlater.net>
Date: Sat, 7 Oct 2023 03:59:23 +0200
Subject: [PATCH 6/9] conduit: Add coturn monitoring
---
configuration/services/conduit.nix | 3 +++
configuration/services/metrics/default.nix | 5 ++++-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/configuration/services/conduit.nix b/configuration/services/conduit.nix
index dcd0103..8257592 100644
--- a/configuration/services/conduit.nix
+++ b/configuration/services/conduit.nix
@@ -173,6 +173,9 @@ in {
# Various other security settings
no-tlsv1
no-tlsv1_1
+
+ # Monitoring
+ prometheus
'';
};
diff --git a/configuration/services/metrics/default.nix b/configuration/services/metrics/default.nix
index 53a8a1d..3347467 100644
--- a/configuration/services/metrics/default.nix
+++ b/configuration/services/metrics/default.nix
@@ -141,7 +141,10 @@ in {
targets =
lib.mapAttrsToList (name: exporter: "${exporter.listenAddress}:${toString exporter.port}")
(lib.filterAttrs (name: exporter: (builtins.isAttrs exporter) && exporter.enable)
- (config.services.prometheus.exporters // config.services.prometheus.local-exporters));
+ (config.services.prometheus.exporters // config.services.prometheus.local-exporters))
+ ++ [
+ "127.0.0.1:9641" # coturn
+ ];
}
];
}
From 345159601e139bf4553c82f8e3b0fbe6f2ca46e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tristan=20Dani=C3=ABl=20Maat?= <tm@tlater.net>
Date: Sat, 7 Oct 2023 04:15:52 +0200
Subject: [PATCH 7/9] gitea: Add monitoring
---
configuration/services/gitea.nix | 21 ++++++++++++++++
configuration/services/metrics/default.nix | 29 +++++++++++++++++-----
configuration/sops.nix | 6 +++++
keys/staging.yaml | 6 +++--
4 files changed, 54 insertions(+), 8 deletions(-)
diff --git a/configuration/services/gitea.nix b/configuration/services/gitea.nix
index 6d6dafd..013842e 100644
--- a/configuration/services/gitea.nix
+++ b/configuration/services/gitea.nix
@@ -1,6 +1,7 @@
{
pkgs,
config,
+ lib,
...
}: let
domain = "gitea.${config.services.nginx.domain}";
@@ -19,11 +20,23 @@ in {
SSH_PORT = 2222;
};
+ metrics = {
+ ENABLED = true;
+ TOKEN = "#metricstoken#";
+ };
service.DISABLE_REGISTRATION = true;
session.COOKIE_SECURE = true;
};
};
+ systemd.services.gitea.serviceConfig.ExecStartPre = let
+ replaceSecretBin = "${pkgs.replace-secret}/bin/replace-secret";
+ secretPath = config.sops.secrets."gitea/metrics-token".path;
+ runConfig = "${config.services.gitea.customDir}/conf/app.ini";
+ in [
+ "+${replaceSecretBin} '#metricstoken#' '${secretPath}' '${runConfig}'"
+ ];
+
# Set up SSL
services.nginx.virtualHosts."${domain}" = let
httpAddress = config.services.gitea.settings.server.HTTP_ADDR;
@@ -37,6 +50,14 @@ in {
'';
locations."/".proxyPass = "http://${httpAddress}:${toString httpPort}";
+ locations."/metrics" = {
+ extraConfig = ''
+ access_log off;
+ allow 127.0.0.1;
+ ${lib.optionalString config.networking.enableIPv6 "allow ::1;"}
+ deny all;
+ '';
+ };
};
# Block repeated failed login attempts
diff --git a/configuration/services/metrics/default.nix b/configuration/services/metrics/default.nix
index 3347467..4b163d3 100644
--- a/configuration/services/metrics/default.nix
+++ b/configuration/services/metrics/default.nix
@@ -138,12 +138,29 @@ in {
job_name = "tlater.net";
static_configs = [
{
- targets =
- lib.mapAttrsToList (name: exporter: "${exporter.listenAddress}:${toString exporter.port}")
- (lib.filterAttrs (name: exporter: (builtins.isAttrs exporter) && exporter.enable)
- (config.services.prometheus.exporters // config.services.prometheus.local-exporters))
- ++ [
- "127.0.0.1:9641" # coturn
+ targets = let
+ exporters = config.services.prometheus.exporters;
+ localExporters = config.services.prometheus.local-exporters;
+ in
+ map (exporter: "${exporter.listenAddress}:${toString exporter.port}") [
+ exporters.domain
+ exporters.node
+ exporters.nginx
+ exporters.nginxlog
+ exporters.systemd
+
+ localExporters.prometheus-fail2ban-exporter
+
+ {
+ # coturn
+ listenAddress = "127.0.0.1";
+ port = "9641";
+ }
+ {
+ # gitea
+ listenAddress = "127.0.0.1";
+ port = "3000";
+ }
];
}
];
diff --git a/configuration/sops.nix b/configuration/sops.nix
index 6eae9fc..190dd95 100644
--- a/configuration/sops.nix
+++ b/configuration/sops.nix
@@ -3,6 +3,12 @@
defaultSopsFile = ../keys/production.yaml;
secrets = {
+ # Gitea
+ "gitea/metrics-token" = {
+ owner = "gitea";
+ group = "gitea";
+ };
+
# Grafana
"grafana/adminPassword" = {
owner = "grafana";
diff --git a/keys/staging.yaml b/keys/staging.yaml
index 86ce700..73f0f94 100644
--- a/keys/staging.yaml
+++ b/keys/staging.yaml
@@ -1,3 +1,5 @@
+gitea:
+ metrics-token: ENC[AES256_GCM,data:J4QdfI1wKyM=,iv:8fqCbftyhj90eIVFxjEp9RXKC1y1IaLnV1r2MOdY15M=,tag:8W/juv1OZh4hJco02qXO6g==,type:str]
grafana:
adminPassword: ENC[AES256_GCM,data:dYfaxUpQpzA=,iv:j5wSem8C5+V4c5qRzXQJhsU7/FOtpvrnaEyFBmW6zJ4=,tag:oc8n3TkEbjF2gjuOobZuLA==,type:str]
secretKey: ENC[AES256_GCM,data:Atruvh2MsNY=,iv:y2MaCUCEzGIydHp6G0DJHfk289S1is0twKm2oUYwDhM=,tag:nAWeg+YqaYqk6k22oBkAhQ==,type:str]
@@ -24,8 +26,8 @@ sops:
azure_kv: []
hc_vault: []
age: []
- lastmodified: "2023-10-01T23:00:06Z"
- mac: ENC[AES256_GCM,data:oEJ3Nwlx5YTVLvWa12On1O+LakU42rsAD1wD52MTlzuwgyRZ/g49pL6pQiL6S0uE7wC0EOqOvg2pCtDxxHe3WNjEpcxnWWftdEjw2laLnBuOqduQmVW+Sn23SzoRkl7PwOH1jTQHzRyciyYkJT1/vCNnbNdKg1eqnbpxPysg6/A=,iv:dC8eNEXhzC8Nx1rfXQdDKtlO01QhyW9ncNFEK/yakrg=,tag:vQ4AW/mqnA9Vs5NNzFsYWQ==,type:str]
+ lastmodified: "2023-10-07T02:17:50Z"
+ mac: ENC[AES256_GCM,data:vZDq33YIn0Nf1FQ2+ySezox6igiw6zNFCu3l3kaIsBKo1797pohmAxj2Lcc+OmlBjj98khaBIlbQuA5ULM+uPN5ILaz3NuXD5PZtsV+rL2PsLNMW9FBSmJ0m0YQrt0nZ0tpzifn12XghcSK2IXv+FnxlfrAJCxDvr5tRm90uUwU=,iv:ct8CzIWjaoJ1UjZcdFSr8lZ626vA0RvM883V6H5plWc=,tag:waJNtp/UbRDOfyzNElrung==,type:str]
pgp:
- created_at: "2022-10-12T16:48:23Z"
enc: |
From 8dc5e133637df038c53af8774225a75dddc853fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tristan=20Dani=C3=ABl=20Maat?= <tm@tlater.net>
Date: Sat, 7 Oct 2023 22:11:39 +0200
Subject: [PATCH 8/9] DONTPUSH: Disable foundryvtt for now
---
configuration/services/foundryvtt.nix | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/configuration/services/foundryvtt.nix b/configuration/services/foundryvtt.nix
index 7bb2286..607defa 100644
--- a/configuration/services/foundryvtt.nix
+++ b/configuration/services/foundryvtt.nix
@@ -8,11 +8,11 @@
in {
imports = [flake-inputs.foundryvtt.nixosModules.foundryvtt];
- services.foundryvtt = {
- enable = true;
- hostName = domain;
- minifyStaticFiles = true;
- };
+ # services.foundryvtt = {
+ # enable = true;
+ # hostName = domain;
+ # minifyStaticFiles = true;
+ # };
# Want to start it manually when I need it, not have it constantly
# running
From 8dfdaa899f158317dbcdf56b4f7dbb3162451eaf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tristan=20Dani=C3=ABl=20Maat?= <tm@tlater.net>
Date: Sat, 7 Oct 2023 22:14:43 +0200
Subject: [PATCH 9/9] WIP: Add metrics
---
configuration/default.nix | 55 ++++++----
configuration/services/conduit.nix | 4 +
configuration/services/fail2ban.nix | 42 ++++++++
configuration/services/foundryvtt.nix | 1 +
configuration/services/gitea.nix | 22 ++++
configuration/services/metrics/default.nix | 9 ++
configuration/services/metrics/exporters.nix | 100 ++++++++++++++++++
configuration/services/metrics/grafana.nix | 48 +++++++++
configuration/services/metrics/options.nix | 90 ++++++++++++++++
.../services/metrics/victoriametrics.nix | 31 ++++++
configuration/services/nextcloud.nix | 3 +
configuration/services/postgres.nix | 7 ++
configuration/services/webserver.nix | 1 +
configuration/sops.nix | 16 +++
flake.nix | 10 +-
keys/production.yaml | 6 +-
keys/staging.yaml | 9 +-
pkgs/_sources_pkgs/generated.json | 21 ++++
pkgs/_sources_pkgs/generated.nix | 16 +++
pkgs/default.nix | 3 +
pkgs/nvfetcher.toml | 3 +
pkgs/prometheus/fail2ban-exporter.nix | 8 ++
22 files changed, 479 insertions(+), 26 deletions(-)
create mode 100644 configuration/services/fail2ban.nix
create mode 100644 configuration/services/metrics/default.nix
create mode 100644 configuration/services/metrics/exporters.nix
create mode 100644 configuration/services/metrics/grafana.nix
create mode 100644 configuration/services/metrics/options.nix
create mode 100644 configuration/services/metrics/victoriametrics.nix
create mode 100644 pkgs/_sources_pkgs/generated.json
create mode 100644 pkgs/_sources_pkgs/generated.nix
create mode 100644 pkgs/nvfetcher.toml
create mode 100644 pkgs/prometheus/fail2ban-exporter.nix
diff --git a/configuration/default.nix b/configuration/default.nix
index 5d491af..81e7241 100644
--- a/configuration/default.nix
+++ b/configuration/default.nix
@@ -16,8 +16,10 @@
./services/backups.nix
./services/conduit.nix
+ ./services/fail2ban.nix
./services/foundryvtt.nix
./services/gitea.nix
+ ./services/metrics
./services/nextcloud.nix
./services/webserver.nix
./services/wireguard.nix
@@ -136,34 +138,45 @@
recommendedProxySettings = true;
clientMaxBodySize = "10G";
domain = "tlater.net";
+
+ statusPage = true; # For metrics, should be accessible only from localhost
+
+ commonHttpConfig = ''
+ log_format upstream_time '$remote_addr - $remote_user [$time_local] '
+ '"$request" $status $body_bytes_sent '
+ '"$http_referer" "$http_user_agent" '
+ 'rt=$request_time uct="$upstream_connect_time" '
+ 'uht="$upstream_header_time" urt="$upstream_response_time"';
+ '';
};
+ services.logrotate = {
+ enable = true;
+
+ settings = lib.mapAttrs' (virtualHost: _:
+ lib.nameValuePair "/var/log/nginx/${virtualHost}/access.log" {
+ frequency = "daily";
+ rotate = 2;
+ compress = true;
+ delaycompress = true;
+ su = "${config.services.nginx.user} ${config.services.nginx.group}";
+ postrotate = "[ ! -f /var/run/nginx/nginx.pid ] || kill -USR1 `cat /var/run/nginx/nginx.pid`";
+ })
+ config.services.nginx.virtualHosts;
+ };
+ systemd.tmpfiles.rules =
+ lib.mapAttrsToList (
+ virtualHost: _:
+ #
+ "d /var/log/nginx/${virtualHost} 0750 ${config.services.nginx.user} ${config.services.nginx.group}"
+ )
+ config.services.nginx.virtualHosts;
+
security.acme = {
defaults.email = "tm@tlater.net";
acceptTerms = true;
};
- services.fail2ban = {
- enable = true;
- extraPackages = [pkgs.ipset];
- banaction = "iptables-ipset-proto6-allports";
- bantime-increment.enable = true;
-
- jails = {
- nginx-botsearch = ''
- enabled = true
- logpath = /var/log/nginx/access.log
- '';
- };
-
- ignoreIP = [
- "127.0.0.0/8"
- "10.0.0.0/8"
- "172.16.0.0/12"
- "192.168.0.0/16"
- ];
- };
-
# Remove some unneeded packages
environment.defaultPackages = [];
diff --git a/configuration/services/conduit.nix b/configuration/services/conduit.nix
index 3f8fd40..8257592 100644
--- a/configuration/services/conduit.nix
+++ b/configuration/services/conduit.nix
@@ -173,6 +173,9 @@ in {
# Various other security settings
no-tlsv1
no-tlsv1_1
+
+ # Monitoring
+ prometheus
'';
};
@@ -205,6 +208,7 @@ in {
addSSL = true;
extraConfig = ''
merge_slashes off;
+ access_log /var/log/nginx/${domain}/access.log upstream_time;
'';
locations = {
diff --git a/configuration/services/fail2ban.nix b/configuration/services/fail2ban.nix
new file mode 100644
index 0000000..ace3219
--- /dev/null
+++ b/configuration/services/fail2ban.nix
@@ -0,0 +1,42 @@
+{pkgs, ...}: {
+ services.fail2ban = {
+ enable = true;
+ extraPackages = [pkgs.ipset];
+ banaction = "iptables-ipset-proto6-allports";
+ bantime-increment.enable = true;
+
+ jails = {
+ nginx-botsearch = ''
+ enabled = true
+ logpath = /var/log/nginx/access.log
+ '';
+ };
+
+ ignoreIP = [
+ "127.0.0.0/8"
+ "10.0.0.0/8"
+ "172.16.0.0/12"
+ "192.168.0.0/16"
+ ];
+ };
+
+ # Allow metrics services to connect to the socket as well
+ users.groups.fail2ban = {};
+ systemd.services.fail2ban.serviceConfig = {
+ ExecStartPost =
+ "+"
+ + (pkgs.writeShellScript "fail2ban-post-start" ''
+ while ! [ -S /var/run/fail2ban/fail2ban.sock ]; do
+ sleep 1
+ done
+
+ while ! ${pkgs.netcat}/bin/nc -zU /var/run/fail2ban/fail2ban.sock; do
+ sleep 1
+ done
+
+ ${pkgs.coreutils}/bin/chown root:fail2ban /var/run/fail2ban /var/run/fail2ban/fail2ban.sock
+ ${pkgs.coreutils}/bin/chmod 660 /var/run/fail2ban/fail2ban.sock
+ ${pkgs.coreutils}/bin/chmod 710 /var/run/fail2ban
+ '');
+ };
+}
diff --git a/configuration/services/foundryvtt.nix b/configuration/services/foundryvtt.nix
index 607defa..a4978fd 100644
--- a/configuration/services/foundryvtt.nix
+++ b/configuration/services/foundryvtt.nix
@@ -25,6 +25,7 @@ in {
enableACME = true;
extraConfig = ''
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+ access_log /var/log/nginx/${domain}/access.log upstream_time;
'';
locations."/" = {
diff --git a/configuration/services/gitea.nix b/configuration/services/gitea.nix
index 27353f6..013842e 100644
--- a/configuration/services/gitea.nix
+++ b/configuration/services/gitea.nix
@@ -1,6 +1,7 @@
{
pkgs,
config,
+ lib,
...
}: let
domain = "gitea.${config.services.nginx.domain}";
@@ -19,11 +20,23 @@ in {
SSH_PORT = 2222;
};
+ metrics = {
+ ENABLED = true;
+ TOKEN = "#metricstoken#";
+ };
service.DISABLE_REGISTRATION = true;
session.COOKIE_SECURE = true;
};
};
+ systemd.services.gitea.serviceConfig.ExecStartPre = let
+ replaceSecretBin = "${pkgs.replace-secret}/bin/replace-secret";
+ secretPath = config.sops.secrets."gitea/metrics-token".path;
+ runConfig = "${config.services.gitea.customDir}/conf/app.ini";
+ in [
+ "+${replaceSecretBin} '#metricstoken#' '${secretPath}' '${runConfig}'"
+ ];
+
# Set up SSL
services.nginx.virtualHosts."${domain}" = let
httpAddress = config.services.gitea.settings.server.HTTP_ADDR;
@@ -33,9 +46,18 @@ in {
enableACME = true;
extraConfig = ''
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+ access_log /var/log/nginx/${domain}/access.log upstream_time;
'';
locations."/".proxyPass = "http://${httpAddress}:${toString httpPort}";
+ locations."/metrics" = {
+ extraConfig = ''
+ access_log off;
+ allow 127.0.0.1;
+ ${lib.optionalString config.networking.enableIPv6 "allow ::1;"}
+ deny all;
+ '';
+ };
};
# Block repeated failed login attempts
diff --git a/configuration/services/metrics/default.nix b/configuration/services/metrics/default.nix
new file mode 100644
index 0000000..84e126a
--- /dev/null
+++ b/configuration/services/metrics/default.nix
@@ -0,0 +1,9 @@
+{
+ imports = [
+ ./options.nix
+
+ ./exporters.nix
+ ./grafana.nix
+ ./victoriametrics.nix
+ ];
+}
diff --git a/configuration/services/metrics/exporters.nix b/configuration/services/metrics/exporters.nix
new file mode 100644
index 0000000..b4af2a0
--- /dev/null
+++ b/configuration/services/metrics/exporters.nix
@@ -0,0 +1,100 @@
+{
+ config,
+ pkgs,
+ lib,
+ ...
+}: let
+ yaml = pkgs.formats.yaml {};
+in {
+ services.prometheus = {
+ exporters = {
+ # Periodically check domain registration status
+ domain = {
+ enable = true;
+ listenAddress = "127.0.0.1";
+ extraFlags = let
+ conf.domains = [
+ "tlater.net"
+ "tlater.com"
+ ];
+ in [
+ "--config=${yaml.generate "domains.yml" conf}"
+ ];
+ };
+
+ # System statistics
+ node = {
+ enable = true;
+ listenAddress = "127.0.0.1";
+ };
+ systemd = {
+ enable = true;
+ listenAddress = "127.0.0.1";
+ extraFlags = [
+ # Disabled by default because only supported from systemd 235+
+ "--systemd.collector.enable-restart-count"
+ "--systemd.collector.enable-ip-accounting"
+ ];
+ };
+
+ # Various nginx metrics
+ nginx = {
+ enable = true;
+ listenAddress = "127.0.0.1";
+ };
+
+ nginxlog = {
+ enable = true;
+ listenAddress = "127.0.0.1";
+ group = "nginx";
+
+ settings.namespaces =
+ lib.mapAttrsToList (name: virtualHost: {
+ inherit name;
+ metrics_override.prefix = "nginxlog";
+ namespace_label = "vhost";
+
+ format = lib.concatStringsSep " " [
+ "$remote_addr - $remote_user [$time_local]"
+ ''"$request" $status $body_bytes_sent''
+ ''"$http_referer" "$http_user_agent"''
+ ''rt=$request_time uct="$upstream_connect_time"''
+ ''uht="$upstream_header_time" urt="$upstream_response_time"''
+ ];
+
+ source.files = [
+ "/var/log/nginx/${name}/access.log"
+ ];
+ })
+ config.services.nginx.virtualHosts;
+ };
+ };
+
+ extraExporters = {
+ fail2ban = let
+ cfg = config.services.prometheus.extraExporters.fail2ban;
+ in {
+ port = 9191;
+ serviceOpts = {
+ after = ["fail2ban.service"];
+ requires = ["fail2ban.service"];
+ serviceConfig = {
+ Group = "fail2ban";
+ RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"];
+ ExecStart = lib.concatStringsSep " " [
+ "${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter"
+ "--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock"
+ "--web.listen-address='${cfg.listenAddress}:${toString cfg.port}'"
+ "--collector.f2b.exit-on-socket-connection-error=true"
+ ];
+ };
+ };
+ };
+ };
+
+ # TODO(tlater):
+ # - wireguard (?)
+ # - postgres (?)
+ # - blackbox (?) (curl to see if http and similar is up)
+ };
+}
diff --git a/configuration/services/metrics/grafana.nix b/configuration/services/metrics/grafana.nix
new file mode 100644
index 0000000..8538dc7
--- /dev/null
+++ b/configuration/services/metrics/grafana.nix
@@ -0,0 +1,48 @@
+{config, ...}: let
+ domain = "metrics.${config.services.nginx.domain}";
+in {
+ services.grafana = {
+ enable = true;
+ settings = {
+ server.http_port = 3001; # Default overlaps with gitea
+
+ security = {
+ admin_user = "tlater";
+ admin_password = "$__file{${config.sops.secrets."grafana/adminPassword".path}}";
+ secret_key = "$__file{${config.sops.secrets."grafana/secretKey".path}}";
+ cookie_secure = true;
+ cookie_samesite = "strict";
+ content_security_policy = true;
+ };
+
+ database = {
+ user = "grafana";
+ name = "grafana";
+ type = "postgres";
+ host = "/run/postgresql";
+ };
+ };
+
+ provision = {
+ enable = true;
+
+ datasources.settings.datasources = [
+ {
+ name = "Victoriametrics - tlater.net";
+ url = "http://localhost:8428";
+ type = "prometheus";
+ }
+ ];
+ };
+ };
+
+ services.nginx.virtualHosts."${domain}" = {
+ forceSSL = true;
+ enableACME = true;
+ extraConfig = ''
+ add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+ access_log /var/log/nginx/${domain}/access.log upstream_time;
+ '';
+ locations."/".proxyPass = "http://localhost:${toString config.services.grafana.settings.server.http_port}";
+ };
+}
diff --git a/configuration/services/metrics/options.nix b/configuration/services/metrics/options.nix
new file mode 100644
index 0000000..65bcf83
--- /dev/null
+++ b/configuration/services/metrics/options.nix
@@ -0,0 +1,90 @@
+{
+ config,
+ lib,
+ ...
+}: let
+ inherit (lib) types mkOption mkDefault;
+in {
+ options.services.prometheus = {
+ extraExporters = mkOption {
+ type = types.attrsOf (types.submodule {
+ options = {
+ port = mkOption {
+ type = types.int;
+ description = "The port on which this exporter listens.";
+ };
+ listenAddress = mkOption {
+ type = types.str;
+ default = "127.0.0.1";
+ description = "Address to listen on.";
+ };
+ serviceOpts = mkOption {
+ type = types.attrs;
+ description = "An attrset to be merged with the exporter's systemd service.";
+ };
+ };
+ });
+ };
+
+ allExporters = mkOption {
+ internal = true;
+ description = "The full list of scraping-relevant settings of all exporters, extra or built-in.";
+ type = types.attrsOf (types.submodule {
+ port = mkOption {
+ type = types.int;
+ };
+ listenAddress = mkOption {
+ type = types.str;
+ };
+ extraSettings = mkOption {
+ type = types.anything;
+ default = {};
+ };
+ });
+ };
+ };
+
+ config = {
+ systemd.services = lib.mapAttrs' (name: exporter:
+ lib.nameValuePair "prometheus-${name}-exporter" (lib.mkMerge [
+ {
+ # Shamelessly copied from upstream because the upstream
+ # module is an intractable mess
+ wantedBy = ["multi-user.target"];
+ after = ["network.target"];
+ serviceConfig.Restart = mkDefault "always";
+ serviceConfig.PrivateTmp = mkDefault true;
+ serviceConfig.WorkingDirectory = mkDefault /tmp;
+ serviceConfig.DynamicUser = mkDefault true;
+ # Hardening
+ serviceConfig.CapabilityBoundingSet = mkDefault [""];
+ serviceConfig.DeviceAllow = [""];
+ serviceConfig.LockPersonality = true;
+ serviceConfig.MemoryDenyWriteExecute = true;
+ serviceConfig.NoNewPrivileges = true;
+ serviceConfig.PrivateDevices = mkDefault true;
+ serviceConfig.ProtectClock = mkDefault true;
+ serviceConfig.ProtectControlGroups = true;
+ serviceConfig.ProtectHome = true;
+ serviceConfig.ProtectHostname = true;
+ serviceConfig.ProtectKernelLogs = true;
+ serviceConfig.ProtectKernelModules = true;
+ serviceConfig.ProtectKernelTunables = true;
+ serviceConfig.ProtectSystem = mkDefault "strict";
+ serviceConfig.RemoveIPC = true;
+ serviceConfig.RestrictAddressFamilies = ["AF_INET" "AF_INET6"];
+ serviceConfig.RestrictNamespaces = true;
+ serviceConfig.RestrictRealtime = true;
+ serviceConfig.RestrictSUIDSGID = true;
+ serviceConfig.SystemCallArchitectures = "native";
+ serviceConfig.UMask = "0077";
+ }
+ exporter.serviceOpts
+ ]))
+ config.services.prometheus.extraExporters;
+
+ services.prometheus.allExporters = lib.mapAttrs (name: exporter: {
+ inherit (exporter) listenAddress port;
+ }) (config.services.prometheus.exporters ++ config.services.prometheus.extraExporters);
+ };
+}
diff --git a/configuration/services/metrics/victoriametrics.nix b/configuration/services/metrics/victoriametrics.nix
new file mode 100644
index 0000000..a9fb3a2
--- /dev/null
+++ b/configuration/services/metrics/victoriametrics.nix
@@ -0,0 +1,31 @@
+{config, ...}: {
+ services.victoriametrics = let
+ scrapeConfigFromExporters = conf: conf // {inherit (config.services.prometheus.exporters.${conf.name}) listenAddress port;};
+ scrapeConfigFromLocalExporters = conf: conf // {inherit (config.services.prometheus.local-exporters.${conf.name}) listenAddress port;};
+ in {
+ enable = true;
+ vmagent-scraping.static_configs =
+ [
+ {
+ name = "gitea";
+ listenAddress = "127.0.0.1";
+ port = 3000;
+ }
+ {
+ name = "coturn";
+ listenAddress = "127.0.0.1";
+ port = 9641;
+ }
+ ]
+ ++ (map scrapeConfigFromLocalExporters [
+ {name = "prometheus-fail2ban-exporter";}
+ ])
+ ++ (map scrapeConfigFromExporters [
+ {name = "domain";}
+ {name = "node";}
+ {name = "nginx";}
+ {name = "nginxlog";}
+ {name = "systemd";}
+ ]);
+ };
+}
diff --git a/configuration/services/nextcloud.nix b/configuration/services/nextcloud.nix
index fbca607..81f38a3 100644
--- a/configuration/services/nextcloud.nix
+++ b/configuration/services/nextcloud.nix
@@ -50,6 +50,9 @@ in {
services.nginx.virtualHosts."${hostName}" = {
forceSSL = true;
enableACME = true;
+ extraConfig = ''
+ access_log /var/log/nginx/${hostName}/access.log upstream_time;
+ '';
};
# Block repeated failed login attempts
diff --git a/configuration/services/postgres.nix b/configuration/services/postgres.nix
index 6c584bb..923007d 100644
--- a/configuration/services/postgres.nix
+++ b/configuration/services/postgres.nix
@@ -16,6 +16,12 @@
# that operation needs to be performed manually on the system as
# well.
ensureUsers = [
+ {
+ name = "grafana";
+ ensurePermissions = {
+ "DATABASE grafana" = "ALL PRIVILEGES";
+ };
+ }
{
name = "nextcloud";
ensurePermissions = {
@@ -25,6 +31,7 @@
];
ensureDatabases = [
+ "grafana"
"nextcloud"
];
};
diff --git a/configuration/services/webserver.nix b/configuration/services/webserver.nix
index 4a8bee4..085b1f7 100644
--- a/configuration/services/webserver.nix
+++ b/configuration/services/webserver.nix
@@ -19,6 +19,7 @@ in {
enableACME = true;
extraConfig = ''
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+ access_log /var/log/nginx/${domain}/access.log upstream_time;
'';
locations."/".proxyPass = "http://${addr}:${toString port}";
diff --git a/configuration/sops.nix b/configuration/sops.nix
index bf98433..190dd95 100644
--- a/configuration/sops.nix
+++ b/configuration/sops.nix
@@ -3,6 +3,22 @@
defaultSopsFile = ../keys/production.yaml;
secrets = {
+ # Gitea
+ "gitea/metrics-token" = {
+ owner = "gitea";
+ group = "gitea";
+ };
+
+ # Grafana
+ "grafana/adminPassword" = {
+ owner = "grafana";
+ group = "grafana";
+ };
+ "grafana/secretKey" = {
+ owner = "grafana";
+ group = "grafana";
+ };
+
# Heisenbridge
"heisenbridge/as-token" = {};
"heisenbridge/hs-token" = {};
diff --git a/flake.nix b/flake.nix
index b6db610..d8ff1a8 100644
--- a/flake.nix
+++ b/flake.nix
@@ -78,7 +78,7 @@
# Utility scripts #
###################
packages.${system} = let
- inherit (nixpkgs.legacyPackages.${system}) writeShellScript;
+ inherit (nixpkgs.legacyPackages.${system}) writeShellScript writeShellScriptBin;
vm = nixpkgs.lib.nixosSystem {
inherit system;
specialArgs.flake-inputs = inputs;
@@ -106,6 +106,14 @@
"${vm.config.system.build.vm}/bin/run-tlaternet-vm"
'';
+ update-pkgs = let
+ nvfetcher-bin = "${nvfetcher.packages.${system}.default}/bin/nvfetcher";
+ in
+ writeShellScriptBin "update-pkgs" ''
+ cd "$(git rev-parse --show-toplevel)/pkgs"
+ ${nvfetcher-bin} -o _sources_pkgs -c nvfetcher.toml
+ '';
+
update-nextcloud-apps = let
nvfetcher-bin = "${nvfetcher.packages.${system}.default}/bin/nvfetcher";
in
diff --git a/keys/production.yaml b/keys/production.yaml
index f8d259d..87ef3c4 100644
--- a/keys/production.yaml
+++ b/keys/production.yaml
@@ -1,3 +1,5 @@
+metrics:
+ tlater: ENC[AES256_GCM,data:4nB0H45nlongb0x1OOKzNXIk96PovZ7OwENovnBZUwMl9ncfYwTHT30OlLsPA75w1govH0jyBRkn1Pe/qHzY1Zt53B8=,iv:AfZ4So6HnjOXzqiHM3WpOsQZJs2CEckuxGfyDxc4TNA=,tag:fIXOKJSVDLpLbiLd2zAu9w==,type:str]
nextcloud:
tlater: ENC[AES256_GCM,data:zNsPm4uFaIRe3LjcwmayRg==,iv:5wam6bP5zP708jC9UrLV0s8qspl3Pm4fPzbMFYBUyPQ=,tag:apnJUMeJwMn9q0NhO4ptmA==,type:str]
steam:
@@ -21,8 +23,8 @@ sops:
azure_kv: []
hc_vault: []
age: []
- lastmodified: "2023-09-23T18:55:44Z"
- mac: ENC[AES256_GCM,data:psqgXozY9L7nduZ11GF+mbIrZ4RUySqBixkWL5z0cYeoLA3URb/dr028LCmNgQS9l8aJVsjVkyLBJIU/8wmiUNqRy/VI5iqV5mu+sxXhUVwFL0dAAWP1lOKwwT5uGK89/ioqkphgzuWD37vGe2vYddKkJF0M+zlz12fqkMjaisU=,iv:UyRoJbfuGU3K/Mp5DQ1kY0Z+nKSSo46BGNAcxt+vAvc=,tag:HkP6+qxQ8J/xAYJXYoG/6g==,type:str]
+ lastmodified: "2023-09-25T00:42:25Z"
+ mac: ENC[AES256_GCM,data:28o/elUKslgn5auYfr34N9fE7B6EoZ6njL6yT0emjfoTjsCADJOLcHfUDNWb3AMP3Z5e/w8WsxI7MpwuwUXRvZ6u9Kui1IBcQu/V6GEzpBVw7JkLHZvsUFHOj/uEBcPvON7pKfXtG3vdH8FF1cxeenFm1Z0cX4C0WrNaxumGknA=,iv:GYK0/JZtCkbVorus+9HQbtxAnIXviiNkoC9dMqTHflM=,tag:R3N5hf/UV2nqyOI50Imr6g==,type:str]
pgp:
- created_at: "2022-10-12T00:46:51Z"
enc: |
diff --git a/keys/staging.yaml b/keys/staging.yaml
index a6b0849..73f0f94 100644
--- a/keys/staging.yaml
+++ b/keys/staging.yaml
@@ -1,3 +1,8 @@
+gitea:
+ metrics-token: ENC[AES256_GCM,data:J4QdfI1wKyM=,iv:8fqCbftyhj90eIVFxjEp9RXKC1y1IaLnV1r2MOdY15M=,tag:8W/juv1OZh4hJco02qXO6g==,type:str]
+grafana:
+ adminPassword: ENC[AES256_GCM,data:dYfaxUpQpzA=,iv:j5wSem8C5+V4c5qRzXQJhsU7/FOtpvrnaEyFBmW6zJ4=,tag:oc8n3TkEbjF2gjuOobZuLA==,type:str]
+ secretKey: ENC[AES256_GCM,data:Atruvh2MsNY=,iv:y2MaCUCEzGIydHp6G0DJHfk289S1is0twKm2oUYwDhM=,tag:nAWeg+YqaYqk6k22oBkAhQ==,type:str]
nextcloud:
tlater: ENC[AES256_GCM,data:91kDcO4hpng=,iv:ayuILRmRru4ZxTCur9H2xHuLjkDzwPdS/4lEog/tesU=,tag:qYhJxnNDcCwUM7xe7Tlcjw==,type:str]
steam:
@@ -21,8 +26,8 @@ sops:
azure_kv: []
hc_vault: []
age: []
- lastmodified: "2023-09-22T21:07:02Z"
- mac: ENC[AES256_GCM,data:gItC41S8MInLmikdH1okhPs+FVf8sCF/iQeJ5reigBunHkOngoc6nOFANyAcNZETszzhgTLXXtmVNEjW46v6K7D6nmoi/zwpedUxwzMwDC5I28VTMDHVMAThYSGtdo6kig8i2pi8rzEQd1DStxMv3TWML5y6DDTlFsd3lfudaHA=,iv:zXebvIVPR76GwUhpactwRgF/eEmx2OBkT18E8lkwzRA=,tag:6HyISACbFCGlpIIgkFeA/A==,type:str]
+ lastmodified: "2023-10-07T02:17:50Z"
+ mac: ENC[AES256_GCM,data:vZDq33YIn0Nf1FQ2+ySezox6igiw6zNFCu3l3kaIsBKo1797pohmAxj2Lcc+OmlBjj98khaBIlbQuA5ULM+uPN5ILaz3NuXD5PZtsV+rL2PsLNMW9FBSmJ0m0YQrt0nZ0tpzifn12XghcSK2IXv+FnxlfrAJCxDvr5tRm90uUwU=,iv:ct8CzIWjaoJ1UjZcdFSr8lZ626vA0RvM883V6H5plWc=,tag:waJNtp/UbRDOfyzNElrung==,type:str]
pgp:
- created_at: "2022-10-12T16:48:23Z"
enc: |
diff --git a/pkgs/_sources_pkgs/generated.json b/pkgs/_sources_pkgs/generated.json
new file mode 100644
index 0000000..b3faf9a
--- /dev/null
+++ b/pkgs/_sources_pkgs/generated.json
@@ -0,0 +1,21 @@
+{
+ "prometheus-fail2ban-exporter": {
+ "cargoLocks": null,
+ "date": null,
+ "extract": null,
+ "name": "prometheus-fail2ban-exporter",
+ "passthru": null,
+ "pinned": false,
+ "src": {
+ "deepClone": false,
+ "fetchSubmodules": false,
+ "leaveDotGit": false,
+ "name": null,
+ "rev": "v0.10.0",
+ "sha256": "sha256-8nIW1XaHCBqQCoLkV1ZYE3NTbVZ6c+UOqYD08XQiv+4=",
+ "type": "git",
+ "url": "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter"
+ },
+ "version": "v0.10.0"
+ }
+}
\ No newline at end of file
diff --git a/pkgs/_sources_pkgs/generated.nix b/pkgs/_sources_pkgs/generated.nix
new file mode 100644
index 0000000..bb015b4
--- /dev/null
+++ b/pkgs/_sources_pkgs/generated.nix
@@ -0,0 +1,16 @@
+# This file was generated by nvfetcher, please do not modify it manually.
+{ fetchgit, fetchurl, fetchFromGitHub, dockerTools }:
+{
+ prometheus-fail2ban-exporter = {
+ pname = "prometheus-fail2ban-exporter";
+ version = "v0.10.0";
+ src = fetchgit {
+ url = "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter";
+ rev = "v0.10.0";
+ fetchSubmodules = false;
+ deepClone = false;
+ leaveDotGit = false;
+ sha256 = "sha256-8nIW1XaHCBqQCoLkV1ZYE3NTbVZ6c+UOqYD08XQiv+4=";
+ };
+ };
+}
diff --git a/pkgs/default.nix b/pkgs/default.nix
index 3818a26..3130ae0 100644
--- a/pkgs/default.nix
+++ b/pkgs/default.nix
@@ -7,6 +7,9 @@
in
{
starbound = callPackage ./starbound {};
+ prometheus-fail2ban-exporter = callPackage ./prometheus/fail2ban-exporter.nix {
+ sources = pkgs.callPackage ./_sources_pkgs/generated.nix {};
+ };
}
// (
# Add nextcloud apps
diff --git a/pkgs/nvfetcher.toml b/pkgs/nvfetcher.toml
new file mode 100644
index 0000000..8c53200
--- /dev/null
+++ b/pkgs/nvfetcher.toml
@@ -0,0 +1,3 @@
+[prometheus-fail2ban-exporter]
+src.manual = "v0.10.0" # No gitlab support in nvfetcher
+fetch.git = "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter"
diff --git a/pkgs/prometheus/fail2ban-exporter.nix b/pkgs/prometheus/fail2ban-exporter.nix
new file mode 100644
index 0000000..50b4973
--- /dev/null
+++ b/pkgs/prometheus/fail2ban-exporter.nix
@@ -0,0 +1,8 @@
+{
+ buildGoModule,
+ sources,
+}:
+buildGoModule {
+ inherit (sources.prometheus-fail2ban-exporter) pname src version;
+ vendorHash = "sha256-qU6opwhhvzbQOhfGVyiVgKhfCSB0Z4eSRAJnv6ht2I0=";
+}