tlaternet/tlaternet-server
1
0
Fork 0
Code Issues 10 Pull requests Activity

WIP: Add metrics

Browse source
This commit is contained in:
Tristan Daniël Maat 2023-10-07 22:14:43 +02:00
parent 8dc5e13363
commit 8eb01bb23d
Signed by: tlater
GPG key ID: 49670FD774E43268
22 changed files with 576 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

55
configuration/default.nix
View file

@ -16,8 +16,10 @@
./services/backups.nix
./services/conduit.nix
./services/fail2ban.nix
./services/foundryvtt.nix
./services/gitea.nix
./services/metrics
./services/nextcloud.nix
./services/webserver.nix
./services/wireguard.nix
@ -136,34 +138,45 @@
recommendedProxySettings = true;
clientMaxBodySize = "10G";
domain = "tlater.net";
statusPage = true; # For metrics, should be accessible only from localhost
commonHttpConfig = ''
log_format upstream_time '$remote_addr - $remote_user [$time_local] '
'"$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent" '
'rt=$request_time uct="$upstream_connect_time" '
'uht="$upstream_header_time" urt="$upstream_response_time"';
'';
};
services.logrotate = {
enable = true;
settings = lib.mapAttrs' (virtualHost: _:
lib.nameValuePair "/var/log/nginx/${virtualHost}/access.log" {
frequency = "daily";
rotate = 2;
compress = true;
delaycompress = true;
su = "${config.services.nginx.user} ${config.services.nginx.group}";
postrotate = "[ ! -f /var/run/nginx/nginx.pid ] || kill -USR1 `cat /var/run/nginx/nginx.pid`";
})
config.services.nginx.virtualHosts;
};
systemd.tmpfiles.rules =
lib.mapAttrsToList (
virtualHost: _:
#
"d /var/log/nginx/${virtualHost} 0750 ${config.services.nginx.user} ${config.services.nginx.group}"
)
config.services.nginx.virtualHosts;
security.acme = {
defaults.email = "tm@tlater.net";
acceptTerms = true;
};
services.fail2ban = {
enable = true;
extraPackages = [pkgs.ipset];
banaction = "iptables-ipset-proto6-allports";
bantime-increment.enable = true;
jails = {
nginx-botsearch = ''
enabled = true
logpath = /var/log/nginx/access.log
'';
};
ignoreIP = [
"127.0.0.0/8"
"10.0.0.0/8"
"172.16.0.0/12"
"192.168.0.0/16"
];
};
# Remove some unneeded packages
environment.defaultPackages = [];

4
configuration/services/conduit.nix
View file

@ -173,6 +173,9 @@ in {
# Various other security settings
no-tlsv1
no-tlsv1_1
# Monitoring
prometheus
'';
};
@ -205,6 +208,7 @@ in {
addSSL = true;
extraConfig = ''
merge_slashes off;
access_log /var/log/nginx/${domain}/access.log upstream_time;
'';
locations = {

42
configuration/services/fail2ban.nix Normal file
View file

@ -0,0 +1,42 @@
{pkgs, ...}: {
services.fail2ban = {
enable = true;
extraPackages = [pkgs.ipset];
banaction = "iptables-ipset-proto6-allports";
bantime-increment.enable = true;
jails = {
nginx-botsearch = ''
enabled = true
logpath = /var/log/nginx/access.log
'';
};
ignoreIP = [
"127.0.0.0/8"
"10.0.0.0/8"
"172.16.0.0/12"
"192.168.0.0/16"
];
};
# Allow metrics services to connect to the socket as well
users.groups.fail2ban = {};
systemd.services.fail2ban.serviceConfig = {
ExecStartPost =
"+"
+ (pkgs.writeShellScript "fail2ban-post-start" ''
while ! [ -S /var/run/fail2ban/fail2ban.sock ]; do
sleep 1
done
while ! ${pkgs.netcat}/bin/nc -zU /var/run/fail2ban/fail2ban.sock; do
sleep 1
done
${pkgs.coreutils}/bin/chown root:fail2ban /var/run/fail2ban /var/run/fail2ban/fail2ban.sock
${pkgs.coreutils}/bin/chmod 660 /var/run/fail2ban/fail2ban.sock
${pkgs.coreutils}/bin/chmod 710 /var/run/fail2ban
'');
};
}

1
configuration/services/foundryvtt.nix
View file

@ -25,6 +25,7 @@ in {
enableACME = true;
extraConfig = ''
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
access_log /var/log/nginx/${domain}/access.log upstream_time;
'';
locations."/" = {

22
configuration/services/gitea.nix
View file

@ -1,6 +1,7 @@
{
pkgs,
config,
lib,
...
}: let
domain = "gitea.${config.services.nginx.domain}";
@ -19,11 +20,23 @@ in {
SSH_PORT = 2222;
};
metrics = {
ENABLED = true;
TOKEN = "#metricstoken#";
};
service.DISABLE_REGISTRATION = true;
session.COOKIE_SECURE = true;
};
};
systemd.services.gitea.serviceConfig.ExecStartPre = let
replaceSecretBin = "${pkgs.replace-secret}/bin/replace-secret";
secretPath = config.sops.secrets."gitea/metrics-token".path;
runConfig = "${config.services.gitea.customDir}/conf/app.ini";
in [
"+${replaceSecretBin} '#metricstoken#' '${secretPath}' '${runConfig}'"
];
# Set up SSL
services.nginx.virtualHosts."${domain}" = let
httpAddress = config.services.gitea.settings.server.HTTP_ADDR;
@ -33,9 +46,18 @@ in {
enableACME = true;
extraConfig = ''
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
access_log /var/log/nginx/${domain}/access.log upstream_time;
'';
locations."/".proxyPass = "http://${httpAddress}:${toString httpPort}";
locations."/metrics" = {
extraConfig = ''
access_log off;
allow 127.0.0.1;
${lib.optionalString config.networking.enableIPv6 "allow ::1;"}
deny all;
'';
};
};
# Block repeated failed login attempts

9
configuration/services/metrics/default.nix Normal file
View file

@ -0,0 +1,9 @@
{
imports = [
./options.nix
./exporters.nix
./grafana.nix
./victoriametrics.nix
];
}

100
configuration/services/metrics/exporters.nix Normal file
View file

@ -0,0 +1,100 @@
{
config,
pkgs,
lib,
...
}: let
yaml = pkgs.formats.yaml {};
in {
services.prometheus = {
exporters = {
# Periodically check domain registration status
domain = {
enable = true;
listenAddress = "127.0.0.1";
extraFlags = let
conf.domains = [
"tlater.net"
"tlater.com"
];
in [
"--config=${yaml.generate "domains.yml" conf}"
];
};
# System statistics
node = {
enable = true;
listenAddress = "127.0.0.1";
};
systemd = {
enable = true;
listenAddress = "127.0.0.1";
extraFlags = [
# Disabled by default because only supported from systemd 235+
"--systemd.collector.enable-restart-count"
"--systemd.collector.enable-ip-accounting"
];
};
# Various nginx metrics
nginx = {
enable = true;
listenAddress = "127.0.0.1";
};
nginxlog = {
enable = true;
listenAddress = "127.0.0.1";
group = "nginx";
settings.namespaces =
lib.mapAttrsToList (name: virtualHost: {
inherit name;
metrics_override.prefix = "nginxlog";
namespace_label = "vhost";
format = lib.concatStringsSep " " [
"$remote_addr - $remote_user [$time_local]"
''"$request" $status $body_bytes_sent''
''"$http_referer" "$http_user_agent"''
''rt=$request_time uct="$upstream_connect_time"''
''uht="$upstream_header_time" urt="$upstream_response_time"''
];
source.files = [
"/var/log/nginx/${name}/access.log"
];
})
config.services.nginx.virtualHosts;
};
};
extraExporters = {
fail2ban = let
cfg = config.services.prometheus.extraExporters.fail2ban;
in {
port = 9191;
serviceOpts = {
after = ["fail2ban.service"];
requires = ["fail2ban.service"];
serviceConfig = {
Group = "fail2ban";
RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"];
ExecStart = lib.concatStringsSep " " [
"${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter"
"--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock"
"--web.listen-address='${cfg.listenAddress}:${toString cfg.port}'"
"--collector.f2b.exit-on-socket-connection-error=true"
];
};
};
};
};
# TODO(tlater):
# - wireguard (?)
# - postgres (?)
# - blackbox (?) (curl to see if http and similar is up)
};
}

48
configuration/services/metrics/grafana.nix Normal file
View file

@ -0,0 +1,48 @@
{config, ...}: let
domain = "metrics.${config.services.nginx.domain}";
in {
services.grafana = {
enable = true;
settings = {
server.http_port = 3001; # Default overlaps with gitea
security = {
admin_user = "tlater";
admin_password = "$__file{${config.sops.secrets."grafana/adminPassword".path}}";
secret_key = "$__file{${config.sops.secrets."grafana/secretKey".path}}";
cookie_secure = true;
cookie_samesite = "strict";
content_security_policy = true;
};
database = {
user = "grafana";
name = "grafana";
type = "postgres";
host = "/run/postgresql";
};
};
provision = {
enable = true;
datasources.settings.datasources = [
{
name = "Victoriametrics - tlater.net";
url = "http://localhost:8428";
type = "prometheus";
}
];
};
};
services.nginx.virtualHosts."${domain}" = {
forceSSL = true;
enableACME = true;
extraConfig = ''
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
access_log /var/log/nginx/${domain}/access.log upstream_time;
'';
locations."/".proxyPass = "http://localhost:${toString config.services.grafana.settings.server.http_port}";
};
}

204
configuration/services/metrics/options.nix Normal file
View file

@ -0,0 +1,204 @@
{
pkgs,
config,
lib,
...
}: let
inherit (lib) types mkOption mkDefault;
yaml = pkgs.formats.yaml {};
in {
options = {
services.prometheus = {
extraExporters = mkOption {
type = types.attrsOf (types.submodule {
options = {
port = mkOption {
type = types.int;
description = "The port on which this exporter listens.";
};
listenAddress = mkOption {
type = types.str;
default = "127.0.0.1";
description = "Address to listen on.";
};
serviceOpts = mkOption {
type = types.attrs;
description = "An attrset to be merged with the exporter's systemd service.";
};
};
});
};
};
services.victoriametrics.scrapeConfigs = mkOption {
type = types.attrsOf (types.submodule ({
name,
self,
...
}: {
options = {
job_name = mkOption {
type = types.str;
default = name;
};
extraSettings = mkOption {
type = types.anything;
description = ''
Other settings to set for this scrape config.
'';
default = {};
};
targets = mkOption {
type = types.listOf types.str;
description = lib.mdDoc ''
Addresses scrape targets for this config listen on.
Shortcut for `static_configs = lib.singleton {targets = [<targets>];}`
'';
default = [];
};
static_configs = mkOption {
default = [];
type = types.listOf (types.submodule {
options = {
targets = mkOption {
type = types.listOf types.str;
description = lib.mdDoc ''
The addresses scrape targets for this config listen on.
Must in `listenAddress:port` format.
'';
};
labels = mkOption {
type = types.attrsOf types.str;
description = lib.mdDoc ''
Labels to apply to all targets defined for this static config.
'';
default = {};
};
};
});
};
};
}));
};
};
config = {
systemd.services = lib.mkMerge [
(lib.mapAttrs' (name: exporter:
lib.nameValuePair "prometheus-${name}-exporter" (lib.mkMerge [
{
# Shamelessly copied from upstream because the upstream
# module is an intractable mess
wantedBy = ["multi-user.target"];
after = ["network.target"];
serviceConfig.Restart = mkDefault "always";
serviceConfig.PrivateTmp = mkDefault true;
serviceConfig.WorkingDirectory = mkDefault /tmp;
serviceConfig.DynamicUser = mkDefault true;
# Hardening
serviceConfig.CapabilityBoundingSet = mkDefault [""];
serviceConfig.DeviceAllow = [""];
serviceConfig.LockPersonality = true;
serviceConfig.MemoryDenyWriteExecute = true;
serviceConfig.NoNewPrivileges = true;
serviceConfig.PrivateDevices = mkDefault true;
serviceConfig.ProtectClock = mkDefault true;
serviceConfig.ProtectControlGroups = true;
serviceConfig.ProtectHome = true;
serviceConfig.ProtectHostname = true;
serviceConfig.ProtectKernelLogs = true;
serviceConfig.ProtectKernelModules = true;
serviceConfig.ProtectKernelTunables = true;
serviceConfig.ProtectSystem = mkDefault "strict";
serviceConfig.RemoveIPC = true;
serviceConfig.RestrictAddressFamilies = ["AF_INET" "AF_INET6"];
serviceConfig.RestrictNamespaces = true;
serviceConfig.RestrictRealtime = true;
serviceConfig.RestrictSUIDSGID = true;
serviceConfig.SystemCallArchitectures = "native";
serviceConfig.UMask = "0077";
}
exporter.serviceOpts
]))
config.services.prometheus.extraExporters)
{
vmagent-scrape-exporters = let
listenAddress = config.services.victoriametrics.listenAddress;
vmAddr = (lib.optionalString (lib.hasPrefix ":" listenAddress) "127.0.0.1") + listenAddress;
promscrape = yaml.generate "prometheus.yml" {
scrape_configs = lib.mapAttrsToList (_: scrape:
lib.recursiveUpdate {
inherit (scrape) job_name;
static_configs =
scrape.static_configs
++ lib.optional (scrape.targets != []) {targets = scrape.targets;};
}
scrape.extraSettings)
config.services.victoriametrics.scrapeConfigs;
};
in {
enable = true;
path = [pkgs.victoriametrics];
wantedBy = ["multi-user.target"];
after = ["network.target" "victoriametrics.service"];
serviceConfig = {
ExecStart = [
(lib.concatStringsSep " " [
"${pkgs.victoriametrics}/bin/vmagent"
"-promscrape.config=${promscrape}"
"-remoteWrite.url=http://${vmAddr}/api/v1/write"
"-remoteWrite.tmpDataPath=%t/vmagent"
])
];
SupplementaryGroups = "metrics";
DynamicUser = true;
RuntimeDirectory = "vmagent";
CapabilityBoundingSet = [""];
DeviceAllow = [""];
LockPersonality = true;
MemoryDenyWriteExecute = true;
NoNewPrivileges = true;
PrivateDevices = true;
ProtectClock = true;
ProtectControlGroups = true;
ProtectHome = true;
ProtectHostname = true;
ProtectKernelLogs = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectSystem = "strict";
RemoveIPC = true;
RestrictAddressFamilies = ["AF_INET" "AF_INET6"];
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
SystemCallArchitectures = "native";
UMask = "0077";
};
};
}
];
users.groups.metrics = {};
services.victoriametrics.scrapeConfigs = let
allExporters =
lib.mapAttrs (name: exporter: {
inherit (exporter) listenAddress port;
}) ((lib.filterAttrs (_: exporter: builtins.isAttrs exporter && exporter.enable)
config.services.prometheus.exporters)
// config.services.prometheus.extraExporters);
in
lib.mapAttrs (_: exporter: {
targets = ["${exporter.listenAddress}:${toString exporter.port}"];
})
allExporters;
};
}

13
configuration/services/metrics/victoriametrics.nix Normal file
View file

@ -0,0 +1,13 @@
{config, ...}: {
config.services.victoriametrics = {
enable = true;
scrapeConfigs = {
gitea = {
targets = ["127.0.0.1:${toString config.services.gitea.settings.server.HTTP_PORT}"];
extraSettings.authorization.credentials_file = config.sops.secrets."gitea/metrics-token".path;
};
coturn.targets = ["127.0.0.1:9641"];
};
};
}

3
configuration/services/nextcloud.nix
View file

@ -50,6 +50,9 @@ in {
services.nginx.virtualHosts."${hostName}" = {
forceSSL = true;
enableACME = true;
extraConfig = ''
access_log /var/log/nginx/${hostName}/access.log upstream_time;
'';
};
# Block repeated failed login attempts

7
configuration/services/postgres.nix
View file

@ -16,6 +16,12 @@
# that operation needs to be performed manually on the system as
# well.
ensureUsers = [
{
name = "grafana";
ensurePermissions = {
"DATABASE grafana" = "ALL PRIVILEGES";
};
}
{
name = "nextcloud";
ensurePermissions = {
@ -25,6 +31,7 @@
];
ensureDatabases = [
"grafana"
"nextcloud"
];
};

1
configuration/services/webserver.nix
View file

@ -19,6 +19,7 @@ in {
enableACME = true;
extraConfig = ''
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
access_log /var/log/nginx/${domain}/access.log upstream_time;
'';
locations."/".proxyPass = "http://${addr}:${toString port}";

17
configuration/sops.nix
View file

@ -3,6 +3,23 @@
defaultSopsFile = ../keys/production.yaml;
secrets = {
# Gitea
"gitea/metrics-token" = {
owner = "gitea";
group = "metrics";
mode = "0440";
};
# Grafana
"grafana/adminPassword" = {
owner = "grafana";
group = "grafana";
};
"grafana/secretKey" = {
owner = "grafana";
group = "grafana";
};
# Heisenbridge
"heisenbridge/as-token" = {};
"heisenbridge/hs-token" = {};