Compare commits
	
		
			No commits in common. "55a4aaf48b2333104f3adb2d6f6bc3afe5bd1ed7" and "87dd9daa4f6392c36504be8005422dc6bf0100fc" have entirely different histories.
		
	
	
		
			55a4aaf48b
			...
			87dd9daa4f
		
	
		
					 22 changed files with 37 additions and 597 deletions
				
			
		|  | @ -16,10 +16,8 @@ | ||||||
| 
 | 
 | ||||||
|     ./services/backups.nix |     ./services/backups.nix | ||||||
|     ./services/conduit.nix |     ./services/conduit.nix | ||||||
|     ./services/fail2ban.nix |  | ||||||
|     ./services/foundryvtt.nix |     ./services/foundryvtt.nix | ||||||
|     ./services/gitea.nix |     ./services/gitea.nix | ||||||
|     ./services/metrics |  | ||||||
|     ./services/nextcloud.nix |     ./services/nextcloud.nix | ||||||
|     ./services/webserver.nix |     ./services/webserver.nix | ||||||
|     ./services/wireguard.nix |     ./services/wireguard.nix | ||||||
|  | @ -138,45 +136,34 @@ | ||||||
|     recommendedProxySettings = true; |     recommendedProxySettings = true; | ||||||
|     clientMaxBodySize = "10G"; |     clientMaxBodySize = "10G"; | ||||||
|     domain = "tlater.net"; |     domain = "tlater.net"; | ||||||
| 
 |  | ||||||
|     statusPage = true; # For metrics, should be accessible only from localhost |  | ||||||
| 
 |  | ||||||
|     commonHttpConfig = '' |  | ||||||
|       log_format upstream_time '$remote_addr - $remote_user [$time_local] ' |  | ||||||
|                          '"$request" $status $body_bytes_sent ' |  | ||||||
|                          '"$http_referer" "$http_user_agent" ' |  | ||||||
|                          'rt=$request_time uct="$upstream_connect_time" ' |  | ||||||
|                          'uht="$upstream_header_time" urt="$upstream_response_time"'; |  | ||||||
|     ''; |  | ||||||
|   }; |   }; | ||||||
| 
 | 
 | ||||||
|   services.logrotate = { |  | ||||||
|     enable = true; |  | ||||||
| 
 |  | ||||||
|     settings = lib.mapAttrs' (virtualHost: _: |  | ||||||
|       lib.nameValuePair "/var/log/nginx/${virtualHost}/access.log" { |  | ||||||
|         frequency = "daily"; |  | ||||||
|         rotate = 2; |  | ||||||
|         compress = true; |  | ||||||
|         delaycompress = true; |  | ||||||
|         su = "${config.services.nginx.user} ${config.services.nginx.group}"; |  | ||||||
|         postrotate = "[ ! -f /var/run/nginx/nginx.pid ] || kill -USR1 `cat /var/run/nginx/nginx.pid`"; |  | ||||||
|       }) |  | ||||||
|     config.services.nginx.virtualHosts; |  | ||||||
|   }; |  | ||||||
|   systemd.tmpfiles.rules = |  | ||||||
|     lib.mapAttrsToList ( |  | ||||||
|       virtualHost: _: |  | ||||||
|       # |  | ||||||
|       "d /var/log/nginx/${virtualHost} 0750 ${config.services.nginx.user} ${config.services.nginx.group}" |  | ||||||
|     ) |  | ||||||
|     config.services.nginx.virtualHosts; |  | ||||||
| 
 |  | ||||||
|   security.acme = { |   security.acme = { | ||||||
|     defaults.email = "tm@tlater.net"; |     defaults.email = "tm@tlater.net"; | ||||||
|     acceptTerms = true; |     acceptTerms = true; | ||||||
|   }; |   }; | ||||||
| 
 | 
 | ||||||
|  |   services.fail2ban = { | ||||||
|  |     enable = true; | ||||||
|  |     extraPackages = [pkgs.ipset]; | ||||||
|  |     banaction = "iptables-ipset-proto6-allports"; | ||||||
|  |     bantime-increment.enable = true; | ||||||
|  | 
 | ||||||
|  |     jails = { | ||||||
|  |       nginx-botsearch = '' | ||||||
|  |         enabled = true | ||||||
|  |         logpath = /var/log/nginx/access.log | ||||||
|  |       ''; | ||||||
|  |     }; | ||||||
|  | 
 | ||||||
|  |     ignoreIP = [ | ||||||
|  |       "127.0.0.0/8" | ||||||
|  |       "10.0.0.0/8" | ||||||
|  |       "172.16.0.0/12" | ||||||
|  |       "192.168.0.0/16" | ||||||
|  |     ]; | ||||||
|  |   }; | ||||||
|  | 
 | ||||||
|   # Remove some unneeded packages |   # Remove some unneeded packages | ||||||
|   environment.defaultPackages = []; |   environment.defaultPackages = []; | ||||||
| 
 | 
 | ||||||
|  |  | ||||||
|  | @ -173,9 +173,6 @@ in { | ||||||
|       # Various other security settings |       # Various other security settings | ||||||
|       no-tlsv1 |       no-tlsv1 | ||||||
|       no-tlsv1_1 |       no-tlsv1_1 | ||||||
| 
 |  | ||||||
|       # Monitoring |  | ||||||
|       prometheus |  | ||||||
|     ''; |     ''; | ||||||
|   }; |   }; | ||||||
| 
 | 
 | ||||||
|  | @ -208,7 +205,6 @@ in { | ||||||
|     addSSL = true; |     addSSL = true; | ||||||
|     extraConfig = '' |     extraConfig = '' | ||||||
|       merge_slashes off; |       merge_slashes off; | ||||||
|       access_log /var/log/nginx/${domain}/access.log upstream_time; |  | ||||||
|     ''; |     ''; | ||||||
| 
 | 
 | ||||||
|     locations = { |     locations = { | ||||||
|  |  | ||||||
|  | @ -1,42 +0,0 @@ | ||||||
| {pkgs, ...}: { |  | ||||||
|   services.fail2ban = { |  | ||||||
|     enable = true; |  | ||||||
|     extraPackages = [pkgs.ipset]; |  | ||||||
|     banaction = "iptables-ipset-proto6-allports"; |  | ||||||
|     bantime-increment.enable = true; |  | ||||||
| 
 |  | ||||||
|     jails = { |  | ||||||
|       nginx-botsearch = '' |  | ||||||
|         enabled = true |  | ||||||
|         logpath = /var/log/nginx/access.log |  | ||||||
|       ''; |  | ||||||
|     }; |  | ||||||
| 
 |  | ||||||
|     ignoreIP = [ |  | ||||||
|       "127.0.0.0/8" |  | ||||||
|       "10.0.0.0/8" |  | ||||||
|       "172.16.0.0/12" |  | ||||||
|       "192.168.0.0/16" |  | ||||||
|     ]; |  | ||||||
|   }; |  | ||||||
| 
 |  | ||||||
|   # Allow metrics services to connect to the socket as well |  | ||||||
|   users.groups.fail2ban = {}; |  | ||||||
|   systemd.services.fail2ban.serviceConfig = { |  | ||||||
|     ExecStartPost = |  | ||||||
|       "+" |  | ||||||
|       + (pkgs.writeShellScript "fail2ban-post-start" '' |  | ||||||
|         while ! [ -S /var/run/fail2ban/fail2ban.sock ]; do |  | ||||||
|             sleep 1 |  | ||||||
|         done |  | ||||||
| 
 |  | ||||||
|         while ! ${pkgs.netcat}/bin/nc -zU /var/run/fail2ban/fail2ban.sock; do |  | ||||||
|             sleep 1 |  | ||||||
|         done |  | ||||||
| 
 |  | ||||||
|         ${pkgs.coreutils}/bin/chown root:fail2ban /var/run/fail2ban /var/run/fail2ban/fail2ban.sock |  | ||||||
|         ${pkgs.coreutils}/bin/chmod 660 /var/run/fail2ban/fail2ban.sock |  | ||||||
|         ${pkgs.coreutils}/bin/chmod 710 /var/run/fail2ban |  | ||||||
|       ''); |  | ||||||
|   }; |  | ||||||
| } |  | ||||||
|  | @ -25,7 +25,6 @@ in { | ||||||
|     enableACME = true; |     enableACME = true; | ||||||
|     extraConfig = '' |     extraConfig = '' | ||||||
|       add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always; |       add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always; | ||||||
|       access_log /var/log/nginx/${domain}/access.log upstream_time; |  | ||||||
|     ''; |     ''; | ||||||
| 
 | 
 | ||||||
|     locations."/" = { |     locations."/" = { | ||||||
|  |  | ||||||
|  | @ -1,7 +1,6 @@ | ||||||
| { | { | ||||||
|   pkgs, |   pkgs, | ||||||
|   config, |   config, | ||||||
|   lib, |  | ||||||
|   ... |   ... | ||||||
| }: let | }: let | ||||||
|   domain = "gitea.${config.services.nginx.domain}"; |   domain = "gitea.${config.services.nginx.domain}"; | ||||||
|  | @ -20,23 +19,11 @@ in { | ||||||
|         SSH_PORT = 2222; |         SSH_PORT = 2222; | ||||||
|       }; |       }; | ||||||
| 
 | 
 | ||||||
|       metrics = { |  | ||||||
|         ENABLED = true; |  | ||||||
|         TOKEN = "#metricstoken#"; |  | ||||||
|       }; |  | ||||||
|       service.DISABLE_REGISTRATION = true; |       service.DISABLE_REGISTRATION = true; | ||||||
|       session.COOKIE_SECURE = true; |       session.COOKIE_SECURE = true; | ||||||
|     }; |     }; | ||||||
|   }; |   }; | ||||||
| 
 | 
 | ||||||
|   systemd.services.gitea.serviceConfig.ExecStartPre = let |  | ||||||
|     replaceSecretBin = "${pkgs.replace-secret}/bin/replace-secret"; |  | ||||||
|     secretPath = config.sops.secrets."gitea/metrics-token".path; |  | ||||||
|     runConfig = "${config.services.gitea.customDir}/conf/app.ini"; |  | ||||||
|   in [ |  | ||||||
|     "+${replaceSecretBin} '#metricstoken#' '${secretPath}' '${runConfig}'" |  | ||||||
|   ]; |  | ||||||
| 
 |  | ||||||
|   # Set up SSL |   # Set up SSL | ||||||
|   services.nginx.virtualHosts."${domain}" = let |   services.nginx.virtualHosts."${domain}" = let | ||||||
|     httpAddress = config.services.gitea.settings.server.HTTP_ADDR; |     httpAddress = config.services.gitea.settings.server.HTTP_ADDR; | ||||||
|  | @ -46,18 +33,9 @@ in { | ||||||
|     enableACME = true; |     enableACME = true; | ||||||
|     extraConfig = '' |     extraConfig = '' | ||||||
|       add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always; |       add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always; | ||||||
|       access_log /var/log/nginx/${domain}/access.log upstream_time; |  | ||||||
|     ''; |     ''; | ||||||
| 
 | 
 | ||||||
|     locations."/".proxyPass = "http://${httpAddress}:${toString httpPort}"; |     locations."/".proxyPass = "http://${httpAddress}:${toString httpPort}"; | ||||||
|     locations."/metrics" = { |  | ||||||
|       extraConfig = '' |  | ||||||
|         access_log off; |  | ||||||
|         allow 127.0.0.1; |  | ||||||
|         ${lib.optionalString config.networking.enableIPv6 "allow ::1;"} |  | ||||||
|         deny all; |  | ||||||
|       ''; |  | ||||||
|     }; |  | ||||||
|   }; |   }; | ||||||
| 
 | 
 | ||||||
|   # Block repeated failed login attempts |   # Block repeated failed login attempts | ||||||
|  |  | ||||||
|  | @ -1,9 +0,0 @@ | ||||||
| { |  | ||||||
|   imports = [ |  | ||||||
|     ./options.nix |  | ||||||
| 
 |  | ||||||
|     ./exporters.nix |  | ||||||
|     ./grafana.nix |  | ||||||
|     ./victoriametrics.nix |  | ||||||
|   ]; |  | ||||||
| } |  | ||||||
|  | @ -1,101 +0,0 @@ | ||||||
| { |  | ||||||
|   config, |  | ||||||
|   pkgs, |  | ||||||
|   lib, |  | ||||||
|   ... |  | ||||||
| }: let |  | ||||||
|   yaml = pkgs.formats.yaml {}; |  | ||||||
| in { |  | ||||||
|   services.prometheus = { |  | ||||||
|     exporters = { |  | ||||||
|       # Periodically check domain registration status |  | ||||||
|       domain = { |  | ||||||
|         enable = true; |  | ||||||
|         listenAddress = "127.0.0.1"; |  | ||||||
|         extraFlags = let |  | ||||||
|           conf.domains = [ |  | ||||||
|             "tlater.net" |  | ||||||
|             "tlater.com" |  | ||||||
|           ]; |  | ||||||
|         in [ |  | ||||||
|           "--config=${yaml.generate "domains.yml" conf}" |  | ||||||
|         ]; |  | ||||||
|       }; |  | ||||||
| 
 |  | ||||||
|       # System statistics |  | ||||||
|       node = { |  | ||||||
|         enable = true; |  | ||||||
|         listenAddress = "127.0.0.1"; |  | ||||||
|       }; |  | ||||||
|       systemd = { |  | ||||||
|         enable = true; |  | ||||||
|         listenAddress = "127.0.0.1"; |  | ||||||
|         extraFlags = [ |  | ||||||
|           # Disabled by default because only supported from systemd 235+ |  | ||||||
|           "--systemd.collector.enable-restart-count" |  | ||||||
|           "--systemd.collector.enable-ip-accounting" |  | ||||||
|         ]; |  | ||||||
|       }; |  | ||||||
| 
 |  | ||||||
|       # Various nginx metrics |  | ||||||
|       nginx = { |  | ||||||
|         enable = true; |  | ||||||
|         listenAddress = "127.0.0.1"; |  | ||||||
|       }; |  | ||||||
| 
 |  | ||||||
|       nginxlog = { |  | ||||||
|         enable = true; |  | ||||||
|         listenAddress = "127.0.0.1"; |  | ||||||
|         group = "nginx"; |  | ||||||
| 
 |  | ||||||
|         settings.namespaces = |  | ||||||
|           lib.mapAttrsToList (name: virtualHost: { |  | ||||||
|             inherit name; |  | ||||||
|             metrics_override.prefix = "nginxlog"; |  | ||||||
|             namespace_label = "vhost"; |  | ||||||
| 
 |  | ||||||
|             format = lib.concatStringsSep " " [ |  | ||||||
|               "$remote_addr - $remote_user [$time_local]" |  | ||||||
|               ''"$request" $status $body_bytes_sent'' |  | ||||||
|               ''"$http_referer" "$http_user_agent"'' |  | ||||||
|               ''rt=$request_time uct="$upstream_connect_time"'' |  | ||||||
|               ''uht="$upstream_header_time" urt="$upstream_response_time"'' |  | ||||||
|             ]; |  | ||||||
| 
 |  | ||||||
|             source.files = [ |  | ||||||
|               "/var/log/nginx/${name}/access.log" |  | ||||||
|             ]; |  | ||||||
|           }) |  | ||||||
|           config.services.nginx.virtualHosts; |  | ||||||
|       }; |  | ||||||
|     }; |  | ||||||
| 
 |  | ||||||
|     extraExporters = { |  | ||||||
|       fail2ban = let |  | ||||||
|         cfg = config.services.prometheus.extraExporters.fail2ban; |  | ||||||
|       in { |  | ||||||
|         port = 9191; |  | ||||||
|         serviceOpts = { |  | ||||||
|           after = ["fail2ban.service"]; |  | ||||||
|           requires = ["fail2ban.service"]; |  | ||||||
|           serviceConfig = { |  | ||||||
|             Group = "fail2ban"; |  | ||||||
|             RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"]; |  | ||||||
|             ExecStart = lib.concatStringsSep " " [ |  | ||||||
|               "${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter" |  | ||||||
|               "--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock" |  | ||||||
|               "--web.listen-address='${cfg.listenAddress}:${toString cfg.port}'" |  | ||||||
|               "--collector.f2b.exit-on-socket-connection-error=true" |  | ||||||
|             ]; |  | ||||||
|           }; |  | ||||||
|         }; |  | ||||||
|       }; |  | ||||||
|     }; |  | ||||||
| 
 |  | ||||||
|     # TODO(tlater): |  | ||||||
|     #   - wireguard (?) |  | ||||||
|     #   - postgres (?) |  | ||||||
|     #   - blackbox (?) (curl to see if http and similar is up) |  | ||||||
|     #   - ssl_exporter (?) |  | ||||||
|   }; |  | ||||||
| } |  | ||||||
|  | @ -1,48 +0,0 @@ | ||||||
| {config, ...}: let |  | ||||||
|   domain = "metrics.${config.services.nginx.domain}"; |  | ||||||
| in { |  | ||||||
|   services.grafana = { |  | ||||||
|     enable = true; |  | ||||||
|     settings = { |  | ||||||
|       server.http_port = 3001; # Default overlaps with gitea |  | ||||||
| 
 |  | ||||||
|       security = { |  | ||||||
|         admin_user = "tlater"; |  | ||||||
|         admin_password = "$__file{${config.sops.secrets."grafana/adminPassword".path}}"; |  | ||||||
|         secret_key = "$__file{${config.sops.secrets."grafana/secretKey".path}}"; |  | ||||||
|         cookie_secure = true; |  | ||||||
|         cookie_samesite = "strict"; |  | ||||||
|         content_security_policy = true; |  | ||||||
|       }; |  | ||||||
| 
 |  | ||||||
|       database = { |  | ||||||
|         user = "grafana"; |  | ||||||
|         name = "grafana"; |  | ||||||
|         type = "postgres"; |  | ||||||
|         host = "/run/postgresql"; |  | ||||||
|       }; |  | ||||||
|     }; |  | ||||||
| 
 |  | ||||||
|     provision = { |  | ||||||
|       enable = true; |  | ||||||
| 
 |  | ||||||
|       datasources.settings.datasources = [ |  | ||||||
|         { |  | ||||||
|           name = "Victoriametrics - tlater.net"; |  | ||||||
|           url = "http://localhost:8428"; |  | ||||||
|           type = "prometheus"; |  | ||||||
|         } |  | ||||||
|       ]; |  | ||||||
|     }; |  | ||||||
|   }; |  | ||||||
| 
 |  | ||||||
|   services.nginx.virtualHosts."${domain}" = { |  | ||||||
|     forceSSL = true; |  | ||||||
|     enableACME = true; |  | ||||||
|     extraConfig = '' |  | ||||||
|       add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always; |  | ||||||
|       access_log /var/log/nginx/${domain}/access.log upstream_time; |  | ||||||
|     ''; |  | ||||||
|     locations."/".proxyPass = "http://localhost:${toString config.services.grafana.settings.server.http_port}"; |  | ||||||
|   }; |  | ||||||
| } |  | ||||||
|  | @ -1,204 +0,0 @@ | ||||||
| { |  | ||||||
|   pkgs, |  | ||||||
|   config, |  | ||||||
|   lib, |  | ||||||
|   ... |  | ||||||
| }: let |  | ||||||
|   inherit (lib) types mkOption mkDefault; |  | ||||||
|   yaml = pkgs.formats.yaml {}; |  | ||||||
| in { |  | ||||||
|   options = { |  | ||||||
|     services.prometheus = { |  | ||||||
|       extraExporters = mkOption { |  | ||||||
|         type = types.attrsOf (types.submodule { |  | ||||||
|           options = { |  | ||||||
|             port = mkOption { |  | ||||||
|               type = types.int; |  | ||||||
|               description = "The port on which this exporter listens."; |  | ||||||
|             }; |  | ||||||
|             listenAddress = mkOption { |  | ||||||
|               type = types.str; |  | ||||||
|               default = "127.0.0.1"; |  | ||||||
|               description = "Address to listen on."; |  | ||||||
|             }; |  | ||||||
|             serviceOpts = mkOption { |  | ||||||
|               type = types.attrs; |  | ||||||
|               description = "An attrset to be merged with the exporter's systemd service."; |  | ||||||
|             }; |  | ||||||
|           }; |  | ||||||
|         }); |  | ||||||
|       }; |  | ||||||
|     }; |  | ||||||
| 
 |  | ||||||
|     services.victoriametrics.scrapeConfigs = mkOption { |  | ||||||
|       type = types.attrsOf (types.submodule ({ |  | ||||||
|         name, |  | ||||||
|         self, |  | ||||||
|         ... |  | ||||||
|       }: { |  | ||||||
|         options = { |  | ||||||
|           job_name = mkOption { |  | ||||||
|             type = types.str; |  | ||||||
|             default = name; |  | ||||||
|           }; |  | ||||||
| 
 |  | ||||||
|           extraSettings = mkOption { |  | ||||||
|             type = types.anything; |  | ||||||
|             description = '' |  | ||||||
|               Other settings to set for this scrape config. |  | ||||||
|             ''; |  | ||||||
|             default = {}; |  | ||||||
|           }; |  | ||||||
| 
 |  | ||||||
|           targets = mkOption { |  | ||||||
|             type = types.listOf types.str; |  | ||||||
|             description = lib.mdDoc '' |  | ||||||
|               Addresses scrape targets for this config listen on. |  | ||||||
| 
 |  | ||||||
|               Shortcut for `static_configs = lib.singleton {targets = [<targets>];}` |  | ||||||
|             ''; |  | ||||||
|             default = []; |  | ||||||
|           }; |  | ||||||
| 
 |  | ||||||
|           static_configs = mkOption { |  | ||||||
|             default = []; |  | ||||||
|             type = types.listOf (types.submodule { |  | ||||||
|               options = { |  | ||||||
|                 targets = mkOption { |  | ||||||
|                   type = types.listOf types.str; |  | ||||||
|                   description = lib.mdDoc '' |  | ||||||
|                     The addresses scrape targets for this config listen on. |  | ||||||
| 
 |  | ||||||
|                     Must in `listenAddress:port` format. |  | ||||||
|                   ''; |  | ||||||
|                 }; |  | ||||||
|                 labels = mkOption { |  | ||||||
|                   type = types.attrsOf types.str; |  | ||||||
|                   description = lib.mdDoc '' |  | ||||||
|                     Labels to apply to all targets defined for this static config. |  | ||||||
|                   ''; |  | ||||||
|                   default = {}; |  | ||||||
|                 }; |  | ||||||
|               }; |  | ||||||
|             }); |  | ||||||
|           }; |  | ||||||
|         }; |  | ||||||
|       })); |  | ||||||
|     }; |  | ||||||
|   }; |  | ||||||
| 
 |  | ||||||
|   config = { |  | ||||||
|     systemd.services = lib.mkMerge [ |  | ||||||
|       (lib.mapAttrs' (name: exporter: |  | ||||||
|         lib.nameValuePair "prometheus-${name}-exporter" (lib.mkMerge [ |  | ||||||
|           { |  | ||||||
|             # Shamelessly copied from upstream because the upstream |  | ||||||
|             # module is an intractable mess |  | ||||||
|             wantedBy = ["multi-user.target"]; |  | ||||||
|             after = ["network.target"]; |  | ||||||
|             serviceConfig.Restart = mkDefault "always"; |  | ||||||
|             serviceConfig.PrivateTmp = mkDefault true; |  | ||||||
|             serviceConfig.WorkingDirectory = mkDefault /tmp; |  | ||||||
|             serviceConfig.DynamicUser = mkDefault true; |  | ||||||
|             # Hardening |  | ||||||
|             serviceConfig.CapabilityBoundingSet = mkDefault [""]; |  | ||||||
|             serviceConfig.DeviceAllow = [""]; |  | ||||||
|             serviceConfig.LockPersonality = true; |  | ||||||
|             serviceConfig.MemoryDenyWriteExecute = true; |  | ||||||
|             serviceConfig.NoNewPrivileges = true; |  | ||||||
|             serviceConfig.PrivateDevices = mkDefault true; |  | ||||||
|             serviceConfig.ProtectClock = mkDefault true; |  | ||||||
|             serviceConfig.ProtectControlGroups = true; |  | ||||||
|             serviceConfig.ProtectHome = true; |  | ||||||
|             serviceConfig.ProtectHostname = true; |  | ||||||
|             serviceConfig.ProtectKernelLogs = true; |  | ||||||
|             serviceConfig.ProtectKernelModules = true; |  | ||||||
|             serviceConfig.ProtectKernelTunables = true; |  | ||||||
|             serviceConfig.ProtectSystem = mkDefault "strict"; |  | ||||||
|             serviceConfig.RemoveIPC = true; |  | ||||||
|             serviceConfig.RestrictAddressFamilies = ["AF_INET" "AF_INET6"]; |  | ||||||
|             serviceConfig.RestrictNamespaces = true; |  | ||||||
|             serviceConfig.RestrictRealtime = true; |  | ||||||
|             serviceConfig.RestrictSUIDSGID = true; |  | ||||||
|             serviceConfig.SystemCallArchitectures = "native"; |  | ||||||
|             serviceConfig.UMask = "0077"; |  | ||||||
|           } |  | ||||||
|           exporter.serviceOpts |  | ||||||
|         ])) |  | ||||||
|       config.services.prometheus.extraExporters) |  | ||||||
| 
 |  | ||||||
|       { |  | ||||||
|         vmagent-scrape-exporters = let |  | ||||||
|           listenAddress = config.services.victoriametrics.listenAddress; |  | ||||||
|           vmAddr = (lib.optionalString (lib.hasPrefix ":" listenAddress) "127.0.0.1") + listenAddress; |  | ||||||
|           promscrape = yaml.generate "prometheus.yml" { |  | ||||||
|             scrape_configs = lib.mapAttrsToList (_: scrape: |  | ||||||
|               lib.recursiveUpdate { |  | ||||||
|                 inherit (scrape) job_name; |  | ||||||
|                 static_configs = |  | ||||||
|                   scrape.static_configs |  | ||||||
|                   ++ lib.optional (scrape.targets != []) {targets = scrape.targets;}; |  | ||||||
|               } |  | ||||||
|               scrape.extraSettings) |  | ||||||
|             config.services.victoriametrics.scrapeConfigs; |  | ||||||
|           }; |  | ||||||
|         in { |  | ||||||
|           enable = true; |  | ||||||
|           path = [pkgs.victoriametrics]; |  | ||||||
|           wantedBy = ["multi-user.target"]; |  | ||||||
|           after = ["network.target" "victoriametrics.service"]; |  | ||||||
|           serviceConfig = { |  | ||||||
|             ExecStart = [ |  | ||||||
|               (lib.concatStringsSep " " [ |  | ||||||
|                 "${pkgs.victoriametrics}/bin/vmagent" |  | ||||||
|                 "-promscrape.config=${promscrape}" |  | ||||||
|                 "-remoteWrite.url=http://${vmAddr}/api/v1/write" |  | ||||||
|                 "-remoteWrite.tmpDataPath=%t/vmagent" |  | ||||||
|               ]) |  | ||||||
|             ]; |  | ||||||
|             SupplementaryGroups = "metrics"; |  | ||||||
| 
 |  | ||||||
|             DynamicUser = true; |  | ||||||
|             RuntimeDirectory = "vmagent"; |  | ||||||
|             CapabilityBoundingSet = [""]; |  | ||||||
|             DeviceAllow = [""]; |  | ||||||
|             LockPersonality = true; |  | ||||||
|             MemoryDenyWriteExecute = true; |  | ||||||
|             NoNewPrivileges = true; |  | ||||||
|             PrivateDevices = true; |  | ||||||
|             ProtectClock = true; |  | ||||||
|             ProtectControlGroups = true; |  | ||||||
|             ProtectHome = true; |  | ||||||
|             ProtectHostname = true; |  | ||||||
|             ProtectKernelLogs = true; |  | ||||||
|             ProtectKernelModules = true; |  | ||||||
|             ProtectKernelTunables = true; |  | ||||||
|             ProtectSystem = "strict"; |  | ||||||
|             RemoveIPC = true; |  | ||||||
|             RestrictAddressFamilies = ["AF_INET" "AF_INET6"]; |  | ||||||
|             RestrictNamespaces = true; |  | ||||||
|             RestrictRealtime = true; |  | ||||||
|             RestrictSUIDSGID = true; |  | ||||||
|             SystemCallArchitectures = "native"; |  | ||||||
|             UMask = "0077"; |  | ||||||
|           }; |  | ||||||
|         }; |  | ||||||
|       } |  | ||||||
|     ]; |  | ||||||
| 
 |  | ||||||
|     users.groups.metrics = {}; |  | ||||||
| 
 |  | ||||||
|     services.victoriametrics.scrapeConfigs = let |  | ||||||
|       allExporters = |  | ||||||
|         lib.mapAttrs (name: exporter: { |  | ||||||
|           inherit (exporter) listenAddress port; |  | ||||||
|         }) ((lib.filterAttrs (_: exporter: builtins.isAttrs exporter && exporter.enable) |  | ||||||
|           config.services.prometheus.exporters) |  | ||||||
|         // config.services.prometheus.extraExporters); |  | ||||||
|     in |  | ||||||
|       lib.mapAttrs (_: exporter: { |  | ||||||
|         targets = ["${exporter.listenAddress}:${toString exporter.port}"]; |  | ||||||
|       }) |  | ||||||
|       allExporters; |  | ||||||
|   }; |  | ||||||
| } |  | ||||||
|  | @ -1,13 +0,0 @@ | ||||||
| {config, ...}: { |  | ||||||
|   config.services.victoriametrics = { |  | ||||||
|     enable = true; |  | ||||||
| 
 |  | ||||||
|     scrapeConfigs = { |  | ||||||
|       gitea = { |  | ||||||
|         targets = ["127.0.0.1:${toString config.services.gitea.settings.server.HTTP_PORT}"]; |  | ||||||
|         extraSettings.authorization.credentials_file = config.sops.secrets."gitea/metrics-token".path; |  | ||||||
|       }; |  | ||||||
|       coturn.targets = ["127.0.0.1:9641"]; |  | ||||||
|     }; |  | ||||||
|   }; |  | ||||||
| } |  | ||||||
|  | @ -50,9 +50,6 @@ in { | ||||||
|   services.nginx.virtualHosts."${hostName}" = { |   services.nginx.virtualHosts."${hostName}" = { | ||||||
|     forceSSL = true; |     forceSSL = true; | ||||||
|     enableACME = true; |     enableACME = true; | ||||||
|     extraConfig = '' |  | ||||||
|       access_log /var/log/nginx/${hostName}/access.log upstream_time; |  | ||||||
|     ''; |  | ||||||
|   }; |   }; | ||||||
| 
 | 
 | ||||||
|   # Block repeated failed login attempts |   # Block repeated failed login attempts | ||||||
|  |  | ||||||
|  | @ -16,12 +16,6 @@ | ||||||
|     # that operation needs to be performed manually on the system as |     # that operation needs to be performed manually on the system as | ||||||
|     # well. |     # well. | ||||||
|     ensureUsers = [ |     ensureUsers = [ | ||||||
|       { |  | ||||||
|         name = "grafana"; |  | ||||||
|         ensurePermissions = { |  | ||||||
|           "DATABASE grafana" = "ALL PRIVILEGES"; |  | ||||||
|         }; |  | ||||||
|       } |  | ||||||
|       { |       { | ||||||
|         name = "nextcloud"; |         name = "nextcloud"; | ||||||
|         ensurePermissions = { |         ensurePermissions = { | ||||||
|  | @ -31,7 +25,6 @@ | ||||||
|     ]; |     ]; | ||||||
| 
 | 
 | ||||||
|     ensureDatabases = [ |     ensureDatabases = [ | ||||||
|       "grafana" |  | ||||||
|       "nextcloud" |       "nextcloud" | ||||||
|     ]; |     ]; | ||||||
|   }; |   }; | ||||||
|  |  | ||||||
|  | @ -19,7 +19,6 @@ in { | ||||||
|     enableACME = true; |     enableACME = true; | ||||||
|     extraConfig = '' |     extraConfig = '' | ||||||
|       add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always; |       add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always; | ||||||
|       access_log /var/log/nginx/${domain}/access.log upstream_time; |  | ||||||
|     ''; |     ''; | ||||||
| 
 | 
 | ||||||
|     locations."/".proxyPass = "http://${addr}:${toString port}"; |     locations."/".proxyPass = "http://${addr}:${toString port}"; | ||||||
|  |  | ||||||
|  | @ -3,44 +3,28 @@ | ||||||
|     defaultSopsFile = ../keys/production.yaml; |     defaultSopsFile = ../keys/production.yaml; | ||||||
| 
 | 
 | ||||||
|     secrets = { |     secrets = { | ||||||
|       # Gitea |  | ||||||
|       "gitea/metrics-token" = { |  | ||||||
|         owner = "gitea"; |  | ||||||
|         group = "metrics"; |  | ||||||
|         mode = "0440"; |  | ||||||
|       }; |  | ||||||
| 
 |  | ||||||
|       # Grafana |  | ||||||
|       "grafana/adminPassword" = { |  | ||||||
|         owner = "grafana"; |  | ||||||
|         group = "grafana"; |  | ||||||
|       }; |  | ||||||
|       "grafana/secretKey" = { |  | ||||||
|         owner = "grafana"; |  | ||||||
|         group = "grafana"; |  | ||||||
|       }; |  | ||||||
| 
 |  | ||||||
|       # Heisenbridge |  | ||||||
|       "heisenbridge/as-token" = {}; |  | ||||||
|       "heisenbridge/hs-token" = {}; |  | ||||||
| 
 |  | ||||||
|       # Nextcloud |  | ||||||
|       "nextcloud/tlater" = { |       "nextcloud/tlater" = { | ||||||
|         owner = "nextcloud"; |         owner = "nextcloud"; | ||||||
|         group = "nextcloud"; |         group = "nextcloud"; | ||||||
|       }; |       }; | ||||||
| 
 | 
 | ||||||
|       # Restic |       "steam/tlater" = {}; | ||||||
|  | 
 | ||||||
|  |       "heisenbridge/as-token" = {}; | ||||||
|  |       "heisenbridge/hs-token" = {}; | ||||||
|  | 
 | ||||||
|  |       "wireguard/server-key" = { | ||||||
|  |         owner = "root"; | ||||||
|  |         group = "systemd-network"; | ||||||
|  |         mode = "0440"; | ||||||
|  |       }; | ||||||
|  | 
 | ||||||
|       "restic/local-backups" = { |       "restic/local-backups" = { | ||||||
|         owner = "root"; |         owner = "root"; | ||||||
|         group = "backup"; |         group = "backup"; | ||||||
|         mode = "0440"; |         mode = "0440"; | ||||||
|       }; |       }; | ||||||
| 
 | 
 | ||||||
|       # Steam |  | ||||||
|       "steam/tlater" = {}; |  | ||||||
| 
 |  | ||||||
|       # Turn |  | ||||||
|       "turn/env" = {}; |       "turn/env" = {}; | ||||||
|       "turn/secret" = { |       "turn/secret" = { | ||||||
|         owner = "turnserver"; |         owner = "turnserver"; | ||||||
|  | @ -51,13 +35,6 @@ | ||||||
|       "turn/ssl-cert" = { |       "turn/ssl-cert" = { | ||||||
|         owner = "turnserver"; |         owner = "turnserver"; | ||||||
|       }; |       }; | ||||||
| 
 |  | ||||||
|       # Wireguard |  | ||||||
|       "wireguard/server-key" = { |  | ||||||
|         owner = "root"; |  | ||||||
|         group = "systemd-network"; |  | ||||||
|         mode = "0440"; |  | ||||||
|       }; |  | ||||||
|     }; |     }; | ||||||
|   }; |   }; | ||||||
| } | } | ||||||
|  |  | ||||||
							
								
								
									
										10
									
								
								flake.nix
									
										
									
									
									
								
							
							
						
						
									
										10
									
								
								flake.nix
									
										
									
									
									
								
							|  | @ -78,7 +78,7 @@ | ||||||
|     # Utility scripts # |     # Utility scripts # | ||||||
|     ################### |     ################### | ||||||
|     packages.${system} = let |     packages.${system} = let | ||||||
|       inherit (nixpkgs.legacyPackages.${system}) writeShellScript writeShellScriptBin; |       inherit (nixpkgs.legacyPackages.${system}) writeShellScript; | ||||||
|       vm = nixpkgs.lib.nixosSystem { |       vm = nixpkgs.lib.nixosSystem { | ||||||
|         inherit system; |         inherit system; | ||||||
|         specialArgs.flake-inputs = inputs; |         specialArgs.flake-inputs = inputs; | ||||||
|  | @ -106,14 +106,6 @@ | ||||||
|           "${vm.config.system.build.vm}/bin/run-tlaternet-vm" |           "${vm.config.system.build.vm}/bin/run-tlaternet-vm" | ||||||
|         ''; |         ''; | ||||||
| 
 | 
 | ||||||
|       update-pkgs = let |  | ||||||
|         nvfetcher-bin = "${nvfetcher.packages.${system}.default}/bin/nvfetcher"; |  | ||||||
|       in |  | ||||||
|         writeShellScriptBin "update-pkgs" '' |  | ||||||
|           cd "$(git rev-parse --show-toplevel)/pkgs" |  | ||||||
|           ${nvfetcher-bin} -o _sources_pkgs -c nvfetcher.toml |  | ||||||
|         ''; |  | ||||||
| 
 |  | ||||||
|       update-nextcloud-apps = let |       update-nextcloud-apps = let | ||||||
|         nvfetcher-bin = "${nvfetcher.packages.${system}.default}/bin/nvfetcher"; |         nvfetcher-bin = "${nvfetcher.packages.${system}.default}/bin/nvfetcher"; | ||||||
|       in |       in | ||||||
|  |  | ||||||
|  | @ -1,8 +1,3 @@ | ||||||
| gitea: |  | ||||||
|     metrics-token: ENC[AES256_GCM,data:/7/zvVl2ZOBoekrJR32vl/QQcG5XqTmltgpHEMUpbXVeqwnq29idzE2Qyjau96ZHObmSI73/ZtW95uXF6LH9Qw==,iv:iWZECCZSh1CN7wMBqstXR5QWtriR7QLKVqhekGnpXl0=,tag:HEr9km8VYmruBzf0I/5HuA==,type:str] |  | ||||||
| grafana: |  | ||||||
|     adminPassword: ENC[AES256_GCM,data:/qw//J7cOkIGa58bG4GgdzndvKof32AmQeWB00IX8WhA22PDCOc4VdUEoB3wVJJqI/ucoHFInYyhg2rFYoYBesBjAt0QS3+O+8WblIunUuYeqlBuYJJK1TLhy6ql6+aqvfiW/rJLm4LpgA7CboyDD2OYHcAbvGSD2GWwFcHTR/Y=,iv:KK6p8GKzc9SBDZZFkEwCdIjSxriPGNMDNcr97tfbwTI=,tag:gLRNSGdJWFD+V9K5TfJvXw==,type:str] |  | ||||||
|     secretKey: ENC[AES256_GCM,data:OUXWOE6I3a26SrFEOczWNIwyR3Rx62fbsRBBcfh0xyEbxOIPhexH6lIqlVG9Ltwra9+rAldNM4/0BydtxIDj7A==,iv:fiNO/or5yZnhpDPMANDnEC5dtXmbKBZsV+BPmvCN/HI=,tag:Q0M0OtLWdWAJgQmUlL//fg==,type:str] |  | ||||||
| nextcloud: | nextcloud: | ||||||
|     tlater: ENC[AES256_GCM,data:zNsPm4uFaIRe3LjcwmayRg==,iv:5wam6bP5zP708jC9UrLV0s8qspl3Pm4fPzbMFYBUyPQ=,tag:apnJUMeJwMn9q0NhO4ptmA==,type:str] |     tlater: ENC[AES256_GCM,data:zNsPm4uFaIRe3LjcwmayRg==,iv:5wam6bP5zP708jC9UrLV0s8qspl3Pm4fPzbMFYBUyPQ=,tag:apnJUMeJwMn9q0NhO4ptmA==,type:str] | ||||||
| steam: | steam: | ||||||
|  | @ -26,8 +21,8 @@ sops: | ||||||
|     azure_kv: [] |     azure_kv: [] | ||||||
|     hc_vault: [] |     hc_vault: [] | ||||||
|     age: [] |     age: [] | ||||||
|     lastmodified: "2023-10-12T18:40:26Z" |     lastmodified: "2023-09-23T18:55:44Z" | ||||||
|     mac: ENC[AES256_GCM,data:F+yQ20jCtLRKeQDFVKoqrYCgtwGkXxrK6aQO0MFZTIMJAnbTVPM2ZJGQ1RxXb+Zs4T+44EEc2xN4LjeANvgpE6MfOz2VTw+sEEjcYwUyB6RcXHia9XlFLa8lh7/Wx/9DxlSFjjSrxmDkNB6r+n5UF81cdRXF2E9ibdH346ST98A=,iv:xVxFN1IDKrLskaGqnWvOWx1zUII0jRSjQxEsaTf2GNw=,tag:lnp1AvgMOXXlg1vFjHEWUQ==,type:str] |     mac: ENC[AES256_GCM,data:psqgXozY9L7nduZ11GF+mbIrZ4RUySqBixkWL5z0cYeoLA3URb/dr028LCmNgQS9l8aJVsjVkyLBJIU/8wmiUNqRy/VI5iqV5mu+sxXhUVwFL0dAAWP1lOKwwT5uGK89/ioqkphgzuWD37vGe2vYddKkJF0M+zlz12fqkMjaisU=,iv:UyRoJbfuGU3K/Mp5DQ1kY0Z+nKSSo46BGNAcxt+vAvc=,tag:HkP6+qxQ8J/xAYJXYoG/6g==,type:str] | ||||||
|     pgp: |     pgp: | ||||||
|         - created_at: "2022-10-12T00:46:51Z" |         - created_at: "2022-10-12T00:46:51Z" | ||||||
|           enc: | |           enc: | | ||||||
|  |  | ||||||
|  | @ -1,8 +1,3 @@ | ||||||
| gitea: |  | ||||||
|     metrics-token: ENC[AES256_GCM,data:J4QdfI1wKyM=,iv:8fqCbftyhj90eIVFxjEp9RXKC1y1IaLnV1r2MOdY15M=,tag:8W/juv1OZh4hJco02qXO6g==,type:str] |  | ||||||
| grafana: |  | ||||||
|     adminPassword: ENC[AES256_GCM,data:dYfaxUpQpzA=,iv:j5wSem8C5+V4c5qRzXQJhsU7/FOtpvrnaEyFBmW6zJ4=,tag:oc8n3TkEbjF2gjuOobZuLA==,type:str] |  | ||||||
|     secretKey: ENC[AES256_GCM,data:Atruvh2MsNY=,iv:y2MaCUCEzGIydHp6G0DJHfk289S1is0twKm2oUYwDhM=,tag:nAWeg+YqaYqk6k22oBkAhQ==,type:str] |  | ||||||
| nextcloud: | nextcloud: | ||||||
|     tlater: ENC[AES256_GCM,data:91kDcO4hpng=,iv:ayuILRmRru4ZxTCur9H2xHuLjkDzwPdS/4lEog/tesU=,tag:qYhJxnNDcCwUM7xe7Tlcjw==,type:str] |     tlater: ENC[AES256_GCM,data:91kDcO4hpng=,iv:ayuILRmRru4ZxTCur9H2xHuLjkDzwPdS/4lEog/tesU=,tag:qYhJxnNDcCwUM7xe7Tlcjw==,type:str] | ||||||
| steam: | steam: | ||||||
|  | @ -26,8 +21,8 @@ sops: | ||||||
|     azure_kv: [] |     azure_kv: [] | ||||||
|     hc_vault: [] |     hc_vault: [] | ||||||
|     age: [] |     age: [] | ||||||
|     lastmodified: "2023-10-07T02:17:50Z" |     lastmodified: "2023-09-22T21:07:02Z" | ||||||
|     mac: ENC[AES256_GCM,data:vZDq33YIn0Nf1FQ2+ySezox6igiw6zNFCu3l3kaIsBKo1797pohmAxj2Lcc+OmlBjj98khaBIlbQuA5ULM+uPN5ILaz3NuXD5PZtsV+rL2PsLNMW9FBSmJ0m0YQrt0nZ0tpzifn12XghcSK2IXv+FnxlfrAJCxDvr5tRm90uUwU=,iv:ct8CzIWjaoJ1UjZcdFSr8lZ626vA0RvM883V6H5plWc=,tag:waJNtp/UbRDOfyzNElrung==,type:str] |     mac: ENC[AES256_GCM,data:gItC41S8MInLmikdH1okhPs+FVf8sCF/iQeJ5reigBunHkOngoc6nOFANyAcNZETszzhgTLXXtmVNEjW46v6K7D6nmoi/zwpedUxwzMwDC5I28VTMDHVMAThYSGtdo6kig8i2pi8rzEQd1DStxMv3TWML5y6DDTlFsd3lfudaHA=,iv:zXebvIVPR76GwUhpactwRgF/eEmx2OBkT18E8lkwzRA=,tag:6HyISACbFCGlpIIgkFeA/A==,type:str] | ||||||
|     pgp: |     pgp: | ||||||
|         - created_at: "2022-10-12T16:48:23Z" |         - created_at: "2022-10-12T16:48:23Z" | ||||||
|           enc: | |           enc: | | ||||||
|  |  | ||||||
|  | @ -1,21 +0,0 @@ | ||||||
| { |  | ||||||
|     "prometheus-fail2ban-exporter": { |  | ||||||
|         "cargoLocks": null, |  | ||||||
|         "date": null, |  | ||||||
|         "extract": null, |  | ||||||
|         "name": "prometheus-fail2ban-exporter", |  | ||||||
|         "passthru": null, |  | ||||||
|         "pinned": false, |  | ||||||
|         "src": { |  | ||||||
|             "deepClone": false, |  | ||||||
|             "fetchSubmodules": false, |  | ||||||
|             "leaveDotGit": false, |  | ||||||
|             "name": null, |  | ||||||
|             "rev": "v0.10.0", |  | ||||||
|             "sha256": "sha256-8nIW1XaHCBqQCoLkV1ZYE3NTbVZ6c+UOqYD08XQiv+4=", |  | ||||||
|             "type": "git", |  | ||||||
|             "url": "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter" |  | ||||||
|         }, |  | ||||||
|         "version": "v0.10.0" |  | ||||||
|     } |  | ||||||
| } |  | ||||||
|  | @ -1,16 +0,0 @@ | ||||||
| # This file was generated by nvfetcher, please do not modify it manually. |  | ||||||
| { fetchgit, fetchurl, fetchFromGitHub, dockerTools }: |  | ||||||
| { |  | ||||||
|   prometheus-fail2ban-exporter = { |  | ||||||
|     pname = "prometheus-fail2ban-exporter"; |  | ||||||
|     version = "v0.10.0"; |  | ||||||
|     src = fetchgit { |  | ||||||
|       url = "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter"; |  | ||||||
|       rev = "v0.10.0"; |  | ||||||
|       fetchSubmodules = false; |  | ||||||
|       deepClone = false; |  | ||||||
|       leaveDotGit = false; |  | ||||||
|       sha256 = "sha256-8nIW1XaHCBqQCoLkV1ZYE3NTbVZ6c+UOqYD08XQiv+4="; |  | ||||||
|     }; |  | ||||||
|   }; |  | ||||||
| } |  | ||||||
|  | @ -7,9 +7,6 @@ | ||||||
| in | in | ||||||
|   { |   { | ||||||
|     starbound = callPackage ./starbound {}; |     starbound = callPackage ./starbound {}; | ||||||
|     prometheus-fail2ban-exporter = callPackage ./prometheus/fail2ban-exporter.nix { |  | ||||||
|       sources = pkgs.callPackage ./_sources_pkgs/generated.nix {}; |  | ||||||
|     }; |  | ||||||
|   } |   } | ||||||
|   // ( |   // ( | ||||||
|     # Add nextcloud apps |     # Add nextcloud apps | ||||||
|  |  | ||||||
|  | @ -1,3 +0,0 @@ | ||||||
| [prometheus-fail2ban-exporter] |  | ||||||
| src.manual = "v0.10.0" # No gitlab support in nvfetcher |  | ||||||
| fetch.git = "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter" |  | ||||||
|  | @ -1,8 +0,0 @@ | ||||||
| { |  | ||||||
|   buildGoModule, |  | ||||||
|   sources, |  | ||||||
| }: |  | ||||||
| buildGoModule { |  | ||||||
|   inherit (sources.prometheus-fail2ban-exporter) pname src version; |  | ||||||
|   vendorHash = "sha256-qU6opwhhvzbQOhfGVyiVgKhfCSB0Z4eSRAJnv6ht2I0="; |  | ||||||
| } |  | ||||||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue