WIP: nginx: Add metrics

configuration/default.nix

@@ -137,8 +137,22 @@
     recommendedProxySettings = true;
     clientMaxBodySize = "10G";
     domain = "tlater.net";
+
+    statusPage = true; # For metrics, should be accessible only from localhost
+
+    commonHttpConfig = ''
+      log_format upstream_time '$remote_addr - $remote_user [$time_local] '
+                         '"$request" $status $body_bytes_sent '
+                         '"$http_referer" "$http_user_agent" '
+                         'rt=$request_time uct="$upstream_connect_time" '
+                         'uht="$upstream_header_time" urt="$upstream_response_time"';
+    '';
   };
 
+  systemd.tmpfiles.rules =
+    lib.mapAttrsToList (virtualHost: config: "d /var/log/nginx/${virtualHost} 0750 nginx nginx")
+    config.services.nginx.virtualHosts;
+
   security.acme = {
     defaults.email = "tm@tlater.net";
     acceptTerms = true;

configuration/services/conduit.nix

@@ -205,6 +205,7 @@ in {
     addSSL = true;
     extraConfig = ''
       merge_slashes off;
+      access_log /var/log/nginx/${domain}/access.log upstream_time;
     '';
 
     locations = {

configuration/services/foundryvtt.nix

@@ -25,6 +25,7 @@ in {
     enableACME = true;
     extraConfig = ''
       add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+      access_log /var/log/nginx/${domain}/access.log upstream_time;
     '';
 
     locations."/" = {

configuration/services/gitea.nix

@@ -33,6 +33,7 @@ in {
     enableACME = true;
     extraConfig = ''
       add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+      access_log /var/log/nginx/${domain}/access.log upstream_time;
     '';
 
     locations."/".proxyPass = "http://${httpAddress}:${toString httpPort}";

configuration/services/metrics.nix

@@ -50,6 +50,37 @@ in {
       enabledCollectors = ["systemd"];
       listenAddress = "127.0.0.1";
     };
+
+    nginx = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+    };
+
+    nginxlog = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      group = "nginx";
+
+      settings.namespaces =
+        lib.mapAttrsToList (name: virtualHost: {
+          inherit name;
+          metrics_override.prefix = "nginxlog";
+          namespace_label = "vhost";
+
+          format = lib.concatStringsSep " " [
+            "$remote_addr - $remote_user [$time_local]"
+            ''"$request" $status $body_bytes_sent''
+            ''"$http_referer" "$http_user_agent"''
+            ''rt=$request_time uct="$upstream_connect_time"''
+            ''uht="$upstream_header_time" urt="$upstream_response_time"''
+          ];
+
+          source.files = [
+            "/var/log/nginx/${name}/access.log"
+          ];
+        })
+        config.services.nginx.virtualHosts;
+    };
   };
 
   systemd.services.export-to-victoriametrics = let
@@ -80,6 +111,7 @@ in {
     enableACME = true;
     extraConfig = ''
       add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+      access_log /var/log/nginx/${domain}/access.log upstream_time;
     '';
     locations."/".proxyPass = "http://localhost:3001";
   };

configuration/services/nextcloud.nix

@@ -50,6 +50,9 @@ in {
   services.nginx.virtualHosts."${hostName}" = {
     forceSSL = true;
     enableACME = true;
+    extraConfig = ''
+      access_log /var/log/nginx/${hostName}/access.log upstream_time;
+    '';
   };
 
   # Block repeated failed login attempts

configuration/services/webserver.nix

@@ -19,6 +19,7 @@ in {
     enableACME = true;
     extraConfig = ''
       add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+      access_log /var/log/nginx/${domain}/access.log upstream_time;
     '';
 
     locations."/".proxyPass = "http://${addr}:${toString port}";