WIP: Add monitoring services

.sops.yaml

@@ -0,0 +1,7 @@
+keys:
+  - &tlater 535B61015823443941C744DD12264F6BBDFABA89
+
+creation_rules:
+  - key_groups:
+      - pgp:
+          - *tlater

configuration/default.nix

@@ -1,9 +1,12 @@
 { config, pkgs, lib, ... }:
 
-{
+let inherit (lib.attrsets) mapAttrs;
+
+in {
   imports = [
     ./services/gitea.nix
     ./services/minecraft.nix
+    ./services/monitoring.nix
     ./services/nextcloud.nix
     ./services/webserver.nix
     ./ids.nix
@@ -34,6 +37,16 @@
 
   time.timeZone = "Europe/London";
 
+  sops = {
+    gnupg = {
+      home = "/var/lib/sops";
+      sshKeyPaths = [ ];
+    };
+
+    defaultSopsFile = "/etc/sops/secrets.yaml";
+    validateSopsFiles = false;
+  };
+
   users.users.tlater = {
     isNormalUser = true;
     extraGroups = [ "wheel" ];
@@ -57,6 +70,13 @@
     recommendedProxySettings = true;
     clientMaxBodySize = "10G";
     domain = "tlater.net";
+    commonHttpConfig = ''
+      log_format custom '$remote_addr - $remote_user [$time_local] '
+                        '"$request" $status $body_bytes_sent '
+                        '"$http_referrer" "$http_user_agent" '
+                        '$upstream_response_time $request_length $request_time';
+      access_log /var/log/nginx/access.log custom;
+    '';
 
     virtualHosts = let
       host = port: extra:
@@ -73,9 +93,20 @@
       "${domain}" = host 3002 { serverAliases = [ "www.${domain}" ]; };
       "gitea.${domain}" = host 3000 { };
       "nextcloud.${domain}" = host 3001 { };
+      "grafana.${domain}" = host 3003 { };
     };
   };
 
+  # Allow nginxlog group users to read the nginx log
+  users.groups.nginxlog.gid = null;
+  systemd.services.nginx.serviceConfig = {
+    SupplementaryGroups = [ "nginxlog" ];
+    LogsDirectoryMode = lib.mkOverride 99 "0751";
+    ExecStartPost = [
+      "+${pkgs.coreutils}/bin/chown nginx:nginxlog \${LOGS_DIRECTORY}/access.log \${LOGS_DIRECTORY}/error.log"
+    ];
+  };
+
   security.acme = {
     email = "tm@tlater.net";
     acceptTerms = true;

configuration/services/monitoring.nix

@@ -0,0 +1,186 @@
+{ config, lib, pkgs, ... }:
+
+let
+  inherit (builtins) attrNames concatStringsSep;
+  inherit (lib) stringAfter;
+  inherit (lib.attrsets) filterAttrs mapAttrsToList;
+  inherit (pkgs) openssl writeText;
+
+  domain = "grafana.${config.services.nginx.domain}";
+  keydir = "/run/tempsecrets.d";
+  certdir = "/run/tempcerts.d";
+  nonTlsExporters =
+    filterAttrs (_: exporter: exporter.enable && exporter.extraFlags == [ ])
+    config.services.prometheus.exporters;
+  tlsExporters =
+    filterAttrs (_: exporter: exporter.enable && exporter.extraFlags != [ ])
+    config.services.prometheus.exporters;
+in {
+  services.grafana = {
+    inherit domain;
+
+    enable = true;
+    port = 3003;
+
+    security = {
+      adminUser = "tlater";
+      adminPasswordFile = "/run/secrets/grafana-admin-pass";
+    };
+
+    extraOptions = {
+      # All services grafana is allowed to source from
+      SECURITY_DATA_SOURCE_PROXY_WHITELIST = "localhost:4000";
+
+      # We want this to always go through the nixos config
+      SECURITY_DISABLE_INITIAL_ADMIN_CREATION = "true";
+
+      # Our nginx host only forwards this through https, so we can use
+      # cookie_secure
+      SECURITY_COOKIE_SECURE = "true";
+
+      # These security settings aren't set by default yet, but
+      # probably will be in the future
+      SECURITY_COOKIE_SAMESITE = "true";
+      SECURITY_X_XSS_PROTECTION = "true";
+    };
+
+    provision = {
+      enable = true;
+      datasources = [{
+        name = "Prometheus";
+        type = "prometheus";
+        url = "https://localhost:4000";
+        jsonData = {
+          tlsAuth = true;
+          tlsAuthWithCACert = true;
+        };
+        # Currently, Grafana doesn't support specifying key/cert from
+        # a file, which makes this very tricky to automate.
+        #
+        # We'd need to set jsonSecureData, which would be
+        # world-readable, and completely break authentication.
+        #
+        # See this discussion:
+        # https://github.com/grafana/grafana/discussions/44296
+        #
+        # For now, hand-add key/cert every time the server restarts,
+        # if this becomes more permanent, maybe write a script that
+        # updates the key via API?
+        editable = true;
+      }];
+    };
+  };
+
+  services.prometheus = let
+    # See https://github.com/prometheus/exporter-toolkit/blob/master/docs/web-configuration.md#web-configuration
+    makeTlsConfig = client: server:
+      writeText "web.yml" ''
+        tls_server_config:
+          key_file: ${keydir}/${server}.pem
+          cert_file: ${certdir}/${server}.pem
+
+          client_auth_type: RequireAndVerifyClientCert
+          client_ca_file: ${certdir}/${client}.pem
+      '';
+  in {
+    enable = true;
+    port = 4000;
+    extraFlags =
+      [ "--web.config.file=${makeTlsConfig "grafana" "prometheus"}" ];
+
+    # From the documentation:
+    #
+    # > When credentials are stored in external files (password_file,
+    # > bearer_token_file, etc), they will not be visible to promtool
+    # > and it will report errors, despite a correct configuration.
+    checkConfig = false;
+
+    exporters = {
+      node = {
+        enable = true;
+        enabledCollectors = [ "systemd" ];
+        port = 4001;
+        extraFlags =
+          [ "--web.config=${makeTlsConfig "prometheus" "node-exporter"}" ];
+      };
+
+      nginxlog = {
+        enable = true;
+        group = "nginxlog";
+        port = 4002;
+        # Note: No way to enable TLS/auth here
+        settings.namespaces = [{
+          name = "nginx";
+          format = concatStringsSep " " [
+            "$remote_addr - $remote_user [$time_local]"
+            ''"$request" $status $body_bytes_sent''
+            ''"$http_referrer" "$http_user_agent"''
+            "$upstream_response_time $request_length $request_time"
+          ];
+          source.files = [ "/var/log/nginx/access.log" ];
+        }];
+      };
+    };
+
+    scrapeConfigs = (mapAttrsToList (name: exporter: {
+      job_name = name;
+      scheme = "https";
+      tls_config = {
+        ca_file = "${certdir}/${name}-exporter.pem";
+        cert_file = "${certdir}/prometheus.pem";
+        key_file = "${keydir}/prometheus.pem";
+        server_name = "localhost";
+      };
+      static_configs =
+        [{ targets = [ "127.0.0.1:${toString exporter.port}" ]; }];
+    }) tlsExporters) ++ mapAttrsToList (name: exporter: {
+      job_name = name;
+      scheme = "http";
+      static_configs =
+        [{ targets = [ "127.0.0.1:${toString exporter.port}" ]; }];
+    }) nonTlsExporters;
+  };
+
+  system.activationScripts = {
+    # This will seem a bit strange, and it probably *is*; The
+    # keys/certs here are only used for the various prometheus/grafana
+    # services to authenticate against each other.
+    #
+    # Since they aren't used to actually encrypt anything but
+    # communication that happens once, it's not necessary to keep the
+    # keys around. They're only used internally, and frequently
+    # switching them doesn't cause any issues. In fact, a single-use
+    # key protocol would probably be more secure.
+    #
+    # Sadly, neither of these services support anything more usable
+    # than https, so we need to generate keys. We opt to regenerate
+    # them at each system activation.
+    #
+    # CN=localhost is not really a risk here - this only matters if an
+    # attacker can spoof a service on the correct port somehow, in
+    # which case they either have root or full access to that server's
+    # user anyway. Since we use TLS auth, no secrets would be leaked,
+    # so in the worst case this exploit would enable an attacker to
+    # DoS that specific data source... Which they could do by taking
+    # over the service already anyway.
+    setupMonitoringAuth = let
+      opensslBin = "${openssl}/bin/openssl";
+      services = [ "grafana" "prometheus" ]
+        ++ (map (name: "${name}-exporter") (attrNames tlsExporters));
+    in stringAfter ([ "specialfs" "users" "groups" ]) (''
+      [ -e /run/current-system ] || echo setting up monitoring secrets...
+      specialMount ramfs '${keydir}' nodev,nosuid,mode=0751 ramfs
+      specialMount ramfs '${certdir}' nodev,nosuid,mode=0751 ramfs
+    '' + concatStringsSep "\n" (map (service: ''
+      ${opensslBin} req -batch -x509 -newkey ed25519 -nodes \
+          -subj '/CN=localhost' \
+          -addext "subjectAltName = DNS:localhost" \
+          -keyout '${keydir}/${service}.pem' \
+          -out '${certdir}/${service}.pem'
+
+      chown ${service}:${service} '${keydir}/${service}.pem'
+      chmod u=r '${keydir}/${service}.pem'
+      chmod =r '${certdir}/${service}.pem'
+    '') services));
+  };
+}

flake.lock

@@ -73,6 +73,7 @@
         "flake-utils": "flake-utils",
         "nixos-hardware": "nixos-hardware",
         "nixpkgs": "nixpkgs",
+        "sops-nix": "sops-nix",
         "tlaternet-templates": "tlaternet-templates",
         "tlaternet-webserver": "tlaternet-webserver"
       }
@@ -102,6 +103,26 @@
         "type": "github"
       }
     },
+    "sops-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1641374494,
+        "narHash": "sha256-a56G6Um43+0+n+yNYhRCh/mSvDdRVzQHSKcFaDEB9/8=",
+        "owner": "Mic92",
+        "repo": "sops-nix",
+        "rev": "7edb4b080023ef12f39262a3aa7aab31015a7a0e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "Mic92",
+        "repo": "sops-nix",
+        "type": "github"
+      }
+    },
     "tlaternet-templates": {
       "inputs": {
         "flake-utils": [

flake.nix

@@ -5,6 +5,10 @@
     nixpkgs.url = "github:nixos/nixpkgs/nixos-21.11";
     nixos-hardware.url = "github:nixos/nixos-hardware/master";
     flake-utils.url = "github:numtide/flake-utils";
+    sops-nix = {
+      url = "github:Mic92/sops-nix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
     tlaternet-webserver = {
       url = "git+https://gitea.tlater.net/tlaternet/tlaternet.git";
       inputs = {
@@ -21,8 +25,8 @@
     };
   };
 
-  outputs = { self, nixpkgs, nixos-hardware, flake-utils, tlaternet-webserver
+  outputs = { self, nixpkgs, nixos-hardware, flake-utils, sops-nix
-    , tlaternet-templates, ... }@inputs:
+    , tlaternet-webserver, tlaternet-templates, ... }@inputs:
     let
       overlays = [
         (final: prev: {
@@ -35,6 +39,7 @@
             local-lib = self.lib.${prev.system};
           };
         })
+        sops-nix.overlay
       ];
 
     in {
@@ -44,6 +49,8 @@
           inherit system;
 
           modules = [
+            sops-nix.nixosModules.sops
+
             ({ modulesPath, ... }: {
               imports = [ (modulesPath + "/profiles/headless.nix") ];
               nixpkgs.overlays = overlays;
@@ -61,6 +68,8 @@
           inherit system;
 
           modules = [
+            sops-nix.nixosModule
+
             ({ modulesPath, ... }: {
               imports = [ (modulesPath + "/profiles/headless.nix") ];
               nixpkgs.overlays = overlays;
@@ -78,6 +87,13 @@
               # can easily test locally with the VM.
               services.nginx.domain = lib.mkOverride 99 "localhost";
 
+              # Use a default password for the grafana instance for
+              # easy testing.
+              services.grafana.security = {
+                adminPassword = "insecure";
+                adminPasswordFile = lib.mkOverride 99 null;
+              };
+
               # # Set up VM settings to match real VPS
               # virtualisation.memorySize = 3941;
               # virtualisation.cores = 2;
@@ -94,19 +110,25 @@
               nixfmt
               git-lfs
 
+              sops-init-gpg-key
+
               # For the minecraft mod update script
               (python3.withPackages (pypkgs:
                 with pypkgs; [
                   dateutil
                   requests
 
-                  ipython
+                  # ipython
-                  python-language-server
+                  # python-language-server
-                  pyls-black
+                  # pyls-black
-                  pyls-isort
+                  # pyls-isort
-                  pyls-mypy
+                  # pyls-mypy
                 ]))
             ];
+
+            # nativeBuildInputs = [ sops-import-keys-hook ]; Breaks the shellHook somehow
+            sopsPGPKeyDirs = [ "./keys/hosts/" "./keys/users/" ];
+
             shellHook = ''
               export QEMU_OPTS="-m 3941 -smp 2 -curses"
               export QEMU_NET_OPTS="hostfwd=::3022-:2222,hostfwd=::3080-:80,hostfwd=::3443-:443,hostfwd=::3021-:2221,hostfwd=::25565-:25565"

pkgs/minecraft/forge-server.nix

@@ -9,7 +9,7 @@ let
     url = "${mirror}/${version}/forge-${version}-installer.jar";
     curlOpts = "--globoff";
     # Forge doesn't seem to like newer shas
-    sha1 = "e97821e5431bdcaa46e12048769922e2cdb5e2e1";
+    sha1 = "sha1-oHNpyrgHluRrAXWZJg9j+OInAwA=";
   };
 
   unpackCmd = "mkdir -p src; cp $curSrc src/forge-${version}-installer.jar";