From 7b9a77d087af07f6a55f636933ee9e785f481c07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20Dani=C3=ABl=20Maat?= Date: Thu, 20 Jan 2022 22:47:36 +0000 Subject: [PATCH] WIP: Add monitoring services --- configuration/default.nix | 23 ++- configuration/services/monitoring.nix | 192 ++++++++++++++++++++++++++ 2 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 configuration/services/monitoring.nix diff --git a/configuration/default.nix b/configuration/default.nix index cae99d1..a53c286 100644 --- a/configuration/default.nix +++ b/configuration/default.nix @@ -1,9 +1,12 @@ { config, pkgs, lib, ... }: -{ +let inherit (lib.attrsets) mapAttrs; + +in { imports = [ ./services/gitea.nix ./services/minecraft.nix + ./services/monitoring.nix ./services/nextcloud.nix ./services/webserver.nix ./ids.nix @@ -67,6 +70,13 @@ recommendedProxySettings = true; clientMaxBodySize = "10G"; domain = "tlater.net"; + commonHttpConfig = '' + log_format custom '$remote_addr - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_referrer" "$http_user_agent" ' + '$upstream_response_time $request_length $request_time'; + access_log /var/log/nginx/access.log custom; + ''; virtualHosts = let host = port: extra: @@ -83,9 +93,20 @@ "${domain}" = host 3002 { serverAliases = [ "www.${domain}" ]; }; "gitea.${domain}" = host 3000 { }; "nextcloud.${domain}" = host 3001 { }; + "grafana.${domain}" = host 3003 { }; }; }; + # Allow nginxlog group users to read the nginx log + users.groups.nginxlog.gid = null; + systemd.services.nginx.serviceConfig = { + SupplementaryGroups = [ "nginxlog" ]; + LogsDirectoryMode = lib.mkOverride 99 "0751"; + ExecStartPost = [ + "+${pkgs.coreutils}/bin/chown nginx:nginxlog \${LOGS_DIRECTORY}/access.log \${LOGS_DIRECTORY}/error.log" + ]; + }; + security.acme = { email = "tm@tlater.net"; acceptTerms = true; diff --git a/configuration/services/monitoring.nix b/configuration/services/monitoring.nix new file mode 100644 index 0000000..b32a211 --- /dev/null +++ b/configuration/services/monitoring.nix @@ -0,0 +1,192 @@ +{ config, lib, pkgs, ... }: + +let + inherit (builtins) attrNames concatStringsSep; + inherit (lib) stringAfter; + inherit (lib.attrsets) filterAttrs mapAttrsToList; + inherit (pkgs) openssl writeText; + + domain = "grafana.${config.services.nginx.domain}"; + keydir = "/run/tempsecrets.d"; + certdir = "/run/tempcerts.d"; + nonTlsExporters = + filterAttrs (_: exporter: exporter.enable && exporter.extraFlags == [ ]) + config.services.prometheus.exporters; + tlsExporters = + filterAttrs (_: exporter: exporter.enable && exporter.extraFlags != [ ]) + config.services.prometheus.exporters; +in { + services.grafana = { + inherit domain; + + enable = true; + port = 3003; + + security = { + adminUser = "tlater"; + adminPasswordFile = "/run/secrets/grafana-admin-pass"; + }; + + extraOptions = { + # All services grafana is allowed to source from + SECURITY_DATA_SOURCE_PROXY_WHITELIST = "localhost:4000"; + + # We want this to always go through the nixos config + SECURITY_DISABLE_INITIAL_ADMIN_CREATION = "true"; + + # Our nginx host only forwards this through https, so we can use + # cookie_secure + SECURITY_COOKIE_SECURE = "true"; + + # These security settings aren't set by default yet, but + # probably will be in the future + SECURITY_COOKIE_SAMESITE = "true"; + SECURITY_X_XSS_PROTECTION = "true"; + }; + + provision = { + enable = true; + datasources = [{ + name = "Prometheus"; + type = "prometheus"; + url = "https://localhost:4000"; + jsonData = { + tlsAuth = true; + tlsAuthWithCACert = true; + }; + # Currently, Grafana doesn't support specifying key/cert from + # a file, which makes this very tricky to automate. + # + # We'd need to set jsonSecureData, which would be + # world-readable, and completely break authentication. + # + # See this discussion: + # https://github.com/grafana/grafana/discussions/44296 + # + # For now, hand-add key/cert every time the server restarts, + # if this becomes more permanent, maybe write a script that + # updates the key via API? + editable = true; + }]; + }; + }; + + services.prometheus = let + # See https://github.com/prometheus/exporter-toolkit/blob/master/docs/web-configuration.md#web-configuration + makeTlsConfig = client: server: + writeText "web.yml" '' + tls_server_config: + key_file: ${keydir}/${server}.pem + cert_file: ${certdir}/${server}.pem + + client_auth_type: RequireAndVerifyClientCert + client_ca_file: ${certdir}/${client}.pem + ''; + in { + enable = true; + port = 4000; + extraFlags = + [ "--web.config.file=${makeTlsConfig "grafana" "prometheus"}" ]; + + # From the documentation: + # + # > When credentials are stored in external files (password_file, + # > bearer_token_file, etc), they will not be visible to promtool + # > and it will report errors, despite a correct configuration. + checkConfig = false; + + exporters = { + node = { + enable = true; + enabledCollectors = [ "systemd" ]; + port = 4001; + extraFlags = + [ "--web.config=${makeTlsConfig "prometheus" "node-exporter"}" ]; + }; + + nginxlog = { + enable = true; + group = "nginxlog"; + port = 4002; + # Note: No way to enable TLS/auth here + settings.namespaces = [{ + name = "nginx"; + format = concatStringsSep " " [ + "$remote_addr - $remote_user [$time_local]" + ''"$request" $status $body_bytes_sent'' + ''"$http_referrer" "$http_user_agent"'' + "$upstream_response_time $request_length $request_time" + ]; + source.files = [ "/var/log/nginx/access.log" ]; + }]; + }; + + nextcloud = { + enable = true; + port = 4003; + url = "https://127.0.0.1:3001"; + }; + }; + + scrapeConfigs = (mapAttrsToList (name: exporter: { + job_name = name; + scheme = "https"; + tls_config = { + ca_file = "${certdir}/${name}-exporter.pem"; + cert_file = "${certdir}/prometheus.pem"; + key_file = "${keydir}/prometheus.pem"; + server_name = "localhost"; + }; + static_configs = + [{ targets = [ "127.0.0.1:${toString exporter.port}" ]; }]; + }) tlsExporters) ++ mapAttrsToList (name: exporter: { + job_name = name; + scheme = "http"; + static_configs = + [{ targets = [ "127.0.0.1:${toString exporter.port}" ]; }]; + }) nonTlsExporters; + }; + + system.activationScripts = { + # This will seem a bit strange, and it probably *is*; The + # keys/certs here are only used for the various prometheus/grafana + # services to authenticate against each other. + # + # Since they aren't used to actually encrypt anything but + # communication that happens once, it's not necessary to keep the + # keys around. They're only used internally, and frequently + # switching them doesn't cause any issues. In fact, a single-use + # key protocol would probably be more secure. + # + # Sadly, neither of these services support anything more usable + # than https, so we need to generate keys. We opt to regenerate + # them at each system activation. + # + # CN=localhost is not really a risk here - this only matters if an + # attacker can spoof a service on the correct port somehow, in + # which case they either have root or full access to that server's + # user anyway. Since we use TLS auth, no secrets would be leaked, + # so in the worst case this exploit would enable an attacker to + # DoS that specific data source... Which they could do by taking + # over the service already anyway. + setupMonitoringAuth = let + opensslBin = "${openssl}/bin/openssl"; + services = [ "grafana" "prometheus" ] + ++ (map (name: "${name}-exporter") (attrNames tlsExporters)); + in stringAfter ([ "specialfs" "users" "groups" ]) ('' + [ -e /run/current-system ] || echo setting up monitoring secrets... + specialMount ramfs '${keydir}' nodev,nosuid,mode=0751 ramfs + specialMount ramfs '${certdir}' nodev,nosuid,mode=0751 ramfs + '' + concatStringsSep "\n" (map (service: '' + ${opensslBin} req -batch -x509 -newkey ed25519 -nodes \ + -subj '/CN=localhost' \ + -addext "subjectAltName = DNS:localhost" \ + -keyout '${keydir}/${service}.pem' \ + -out '${certdir}/${service}.pem' + + chown ${service}:${service} '${keydir}/${service}.pem' + chmod u=r '${keydir}/${service}.pem' + chmod =r '${certdir}/${service}.pem' + '') services)); + }; +}