metrics: Add metrics with victoriametrics + grafana
This commit is contained in:
parent
78a9eac9bb
commit
55a4aaf48b
|
@ -16,8 +16,10 @@
|
||||||
|
|
||||||
./services/backups.nix
|
./services/backups.nix
|
||||||
./services/conduit.nix
|
./services/conduit.nix
|
||||||
|
./services/fail2ban.nix
|
||||||
./services/foundryvtt.nix
|
./services/foundryvtt.nix
|
||||||
./services/gitea.nix
|
./services/gitea.nix
|
||||||
|
./services/metrics
|
||||||
./services/nextcloud.nix
|
./services/nextcloud.nix
|
||||||
./services/webserver.nix
|
./services/webserver.nix
|
||||||
./services/wireguard.nix
|
./services/wireguard.nix
|
||||||
|
@ -136,34 +138,45 @@
|
||||||
recommendedProxySettings = true;
|
recommendedProxySettings = true;
|
||||||
clientMaxBodySize = "10G";
|
clientMaxBodySize = "10G";
|
||||||
domain = "tlater.net";
|
domain = "tlater.net";
|
||||||
|
|
||||||
|
statusPage = true; # For metrics, should be accessible only from localhost
|
||||||
|
|
||||||
|
commonHttpConfig = ''
|
||||||
|
log_format upstream_time '$remote_addr - $remote_user [$time_local] '
|
||||||
|
'"$request" $status $body_bytes_sent '
|
||||||
|
'"$http_referer" "$http_user_agent" '
|
||||||
|
'rt=$request_time uct="$upstream_connect_time" '
|
||||||
|
'uht="$upstream_header_time" urt="$upstream_response_time"';
|
||||||
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
services.logrotate = {
|
||||||
|
enable = true;
|
||||||
|
|
||||||
|
settings = lib.mapAttrs' (virtualHost: _:
|
||||||
|
lib.nameValuePair "/var/log/nginx/${virtualHost}/access.log" {
|
||||||
|
frequency = "daily";
|
||||||
|
rotate = 2;
|
||||||
|
compress = true;
|
||||||
|
delaycompress = true;
|
||||||
|
su = "${config.services.nginx.user} ${config.services.nginx.group}";
|
||||||
|
postrotate = "[ ! -f /var/run/nginx/nginx.pid ] || kill -USR1 `cat /var/run/nginx/nginx.pid`";
|
||||||
|
})
|
||||||
|
config.services.nginx.virtualHosts;
|
||||||
|
};
|
||||||
|
systemd.tmpfiles.rules =
|
||||||
|
lib.mapAttrsToList (
|
||||||
|
virtualHost: _:
|
||||||
|
#
|
||||||
|
"d /var/log/nginx/${virtualHost} 0750 ${config.services.nginx.user} ${config.services.nginx.group}"
|
||||||
|
)
|
||||||
|
config.services.nginx.virtualHosts;
|
||||||
|
|
||||||
security.acme = {
|
security.acme = {
|
||||||
defaults.email = "tm@tlater.net";
|
defaults.email = "tm@tlater.net";
|
||||||
acceptTerms = true;
|
acceptTerms = true;
|
||||||
};
|
};
|
||||||
|
|
||||||
services.fail2ban = {
|
|
||||||
enable = true;
|
|
||||||
extraPackages = [pkgs.ipset];
|
|
||||||
banaction = "iptables-ipset-proto6-allports";
|
|
||||||
bantime-increment.enable = true;
|
|
||||||
|
|
||||||
jails = {
|
|
||||||
nginx-botsearch = ''
|
|
||||||
enabled = true
|
|
||||||
logpath = /var/log/nginx/access.log
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
ignoreIP = [
|
|
||||||
"127.0.0.0/8"
|
|
||||||
"10.0.0.0/8"
|
|
||||||
"172.16.0.0/12"
|
|
||||||
"192.168.0.0/16"
|
|
||||||
];
|
|
||||||
};
|
|
||||||
|
|
||||||
# Remove some unneeded packages
|
# Remove some unneeded packages
|
||||||
environment.defaultPackages = [];
|
environment.defaultPackages = [];
|
||||||
|
|
||||||
|
|
|
@ -173,6 +173,9 @@ in {
|
||||||
# Various other security settings
|
# Various other security settings
|
||||||
no-tlsv1
|
no-tlsv1
|
||||||
no-tlsv1_1
|
no-tlsv1_1
|
||||||
|
|
||||||
|
# Monitoring
|
||||||
|
prometheus
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -205,6 +208,7 @@ in {
|
||||||
addSSL = true;
|
addSSL = true;
|
||||||
extraConfig = ''
|
extraConfig = ''
|
||||||
merge_slashes off;
|
merge_slashes off;
|
||||||
|
access_log /var/log/nginx/${domain}/access.log upstream_time;
|
||||||
'';
|
'';
|
||||||
|
|
||||||
locations = {
|
locations = {
|
||||||
|
|
42
configuration/services/fail2ban.nix
Normal file
42
configuration/services/fail2ban.nix
Normal file
|
@ -0,0 +1,42 @@
|
||||||
|
{pkgs, ...}: {
|
||||||
|
services.fail2ban = {
|
||||||
|
enable = true;
|
||||||
|
extraPackages = [pkgs.ipset];
|
||||||
|
banaction = "iptables-ipset-proto6-allports";
|
||||||
|
bantime-increment.enable = true;
|
||||||
|
|
||||||
|
jails = {
|
||||||
|
nginx-botsearch = ''
|
||||||
|
enabled = true
|
||||||
|
logpath = /var/log/nginx/access.log
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
ignoreIP = [
|
||||||
|
"127.0.0.0/8"
|
||||||
|
"10.0.0.0/8"
|
||||||
|
"172.16.0.0/12"
|
||||||
|
"192.168.0.0/16"
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
# Allow metrics services to connect to the socket as well
|
||||||
|
users.groups.fail2ban = {};
|
||||||
|
systemd.services.fail2ban.serviceConfig = {
|
||||||
|
ExecStartPost =
|
||||||
|
"+"
|
||||||
|
+ (pkgs.writeShellScript "fail2ban-post-start" ''
|
||||||
|
while ! [ -S /var/run/fail2ban/fail2ban.sock ]; do
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
while ! ${pkgs.netcat}/bin/nc -zU /var/run/fail2ban/fail2ban.sock; do
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
${pkgs.coreutils}/bin/chown root:fail2ban /var/run/fail2ban /var/run/fail2ban/fail2ban.sock
|
||||||
|
${pkgs.coreutils}/bin/chmod 660 /var/run/fail2ban/fail2ban.sock
|
||||||
|
${pkgs.coreutils}/bin/chmod 710 /var/run/fail2ban
|
||||||
|
'');
|
||||||
|
};
|
||||||
|
}
|
|
@ -25,6 +25,7 @@ in {
|
||||||
enableACME = true;
|
enableACME = true;
|
||||||
extraConfig = ''
|
extraConfig = ''
|
||||||
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
|
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
|
||||||
|
access_log /var/log/nginx/${domain}/access.log upstream_time;
|
||||||
'';
|
'';
|
||||||
|
|
||||||
locations."/" = {
|
locations."/" = {
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
{
|
{
|
||||||
pkgs,
|
pkgs,
|
||||||
config,
|
config,
|
||||||
|
lib,
|
||||||
...
|
...
|
||||||
}: let
|
}: let
|
||||||
domain = "gitea.${config.services.nginx.domain}";
|
domain = "gitea.${config.services.nginx.domain}";
|
||||||
|
@ -19,11 +20,23 @@ in {
|
||||||
SSH_PORT = 2222;
|
SSH_PORT = 2222;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
metrics = {
|
||||||
|
ENABLED = true;
|
||||||
|
TOKEN = "#metricstoken#";
|
||||||
|
};
|
||||||
service.DISABLE_REGISTRATION = true;
|
service.DISABLE_REGISTRATION = true;
|
||||||
session.COOKIE_SECURE = true;
|
session.COOKIE_SECURE = true;
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
systemd.services.gitea.serviceConfig.ExecStartPre = let
|
||||||
|
replaceSecretBin = "${pkgs.replace-secret}/bin/replace-secret";
|
||||||
|
secretPath = config.sops.secrets."gitea/metrics-token".path;
|
||||||
|
runConfig = "${config.services.gitea.customDir}/conf/app.ini";
|
||||||
|
in [
|
||||||
|
"+${replaceSecretBin} '#metricstoken#' '${secretPath}' '${runConfig}'"
|
||||||
|
];
|
||||||
|
|
||||||
# Set up SSL
|
# Set up SSL
|
||||||
services.nginx.virtualHosts."${domain}" = let
|
services.nginx.virtualHosts."${domain}" = let
|
||||||
httpAddress = config.services.gitea.settings.server.HTTP_ADDR;
|
httpAddress = config.services.gitea.settings.server.HTTP_ADDR;
|
||||||
|
@ -33,9 +46,18 @@ in {
|
||||||
enableACME = true;
|
enableACME = true;
|
||||||
extraConfig = ''
|
extraConfig = ''
|
||||||
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
|
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
|
||||||
|
access_log /var/log/nginx/${domain}/access.log upstream_time;
|
||||||
'';
|
'';
|
||||||
|
|
||||||
locations."/".proxyPass = "http://${httpAddress}:${toString httpPort}";
|
locations."/".proxyPass = "http://${httpAddress}:${toString httpPort}";
|
||||||
|
locations."/metrics" = {
|
||||||
|
extraConfig = ''
|
||||||
|
access_log off;
|
||||||
|
allow 127.0.0.1;
|
||||||
|
${lib.optionalString config.networking.enableIPv6 "allow ::1;"}
|
||||||
|
deny all;
|
||||||
|
'';
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
# Block repeated failed login attempts
|
# Block repeated failed login attempts
|
||||||
|
|
9
configuration/services/metrics/default.nix
Normal file
9
configuration/services/metrics/default.nix
Normal file
|
@ -0,0 +1,9 @@
|
||||||
|
{
|
||||||
|
imports = [
|
||||||
|
./options.nix
|
||||||
|
|
||||||
|
./exporters.nix
|
||||||
|
./grafana.nix
|
||||||
|
./victoriametrics.nix
|
||||||
|
];
|
||||||
|
}
|
101
configuration/services/metrics/exporters.nix
Normal file
101
configuration/services/metrics/exporters.nix
Normal file
|
@ -0,0 +1,101 @@
|
||||||
|
{
|
||||||
|
config,
|
||||||
|
pkgs,
|
||||||
|
lib,
|
||||||
|
...
|
||||||
|
}: let
|
||||||
|
yaml = pkgs.formats.yaml {};
|
||||||
|
in {
|
||||||
|
services.prometheus = {
|
||||||
|
exporters = {
|
||||||
|
# Periodically check domain registration status
|
||||||
|
domain = {
|
||||||
|
enable = true;
|
||||||
|
listenAddress = "127.0.0.1";
|
||||||
|
extraFlags = let
|
||||||
|
conf.domains = [
|
||||||
|
"tlater.net"
|
||||||
|
"tlater.com"
|
||||||
|
];
|
||||||
|
in [
|
||||||
|
"--config=${yaml.generate "domains.yml" conf}"
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
# System statistics
|
||||||
|
node = {
|
||||||
|
enable = true;
|
||||||
|
listenAddress = "127.0.0.1";
|
||||||
|
};
|
||||||
|
systemd = {
|
||||||
|
enable = true;
|
||||||
|
listenAddress = "127.0.0.1";
|
||||||
|
extraFlags = [
|
||||||
|
# Disabled by default because only supported from systemd 235+
|
||||||
|
"--systemd.collector.enable-restart-count"
|
||||||
|
"--systemd.collector.enable-ip-accounting"
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
# Various nginx metrics
|
||||||
|
nginx = {
|
||||||
|
enable = true;
|
||||||
|
listenAddress = "127.0.0.1";
|
||||||
|
};
|
||||||
|
|
||||||
|
nginxlog = {
|
||||||
|
enable = true;
|
||||||
|
listenAddress = "127.0.0.1";
|
||||||
|
group = "nginx";
|
||||||
|
|
||||||
|
settings.namespaces =
|
||||||
|
lib.mapAttrsToList (name: virtualHost: {
|
||||||
|
inherit name;
|
||||||
|
metrics_override.prefix = "nginxlog";
|
||||||
|
namespace_label = "vhost";
|
||||||
|
|
||||||
|
format = lib.concatStringsSep " " [
|
||||||
|
"$remote_addr - $remote_user [$time_local]"
|
||||||
|
''"$request" $status $body_bytes_sent''
|
||||||
|
''"$http_referer" "$http_user_agent"''
|
||||||
|
''rt=$request_time uct="$upstream_connect_time"''
|
||||||
|
''uht="$upstream_header_time" urt="$upstream_response_time"''
|
||||||
|
];
|
||||||
|
|
||||||
|
source.files = [
|
||||||
|
"/var/log/nginx/${name}/access.log"
|
||||||
|
];
|
||||||
|
})
|
||||||
|
config.services.nginx.virtualHosts;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
extraExporters = {
|
||||||
|
fail2ban = let
|
||||||
|
cfg = config.services.prometheus.extraExporters.fail2ban;
|
||||||
|
in {
|
||||||
|
port = 9191;
|
||||||
|
serviceOpts = {
|
||||||
|
after = ["fail2ban.service"];
|
||||||
|
requires = ["fail2ban.service"];
|
||||||
|
serviceConfig = {
|
||||||
|
Group = "fail2ban";
|
||||||
|
RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"];
|
||||||
|
ExecStart = lib.concatStringsSep " " [
|
||||||
|
"${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter"
|
||||||
|
"--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock"
|
||||||
|
"--web.listen-address='${cfg.listenAddress}:${toString cfg.port}'"
|
||||||
|
"--collector.f2b.exit-on-socket-connection-error=true"
|
||||||
|
];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
# TODO(tlater):
|
||||||
|
# - wireguard (?)
|
||||||
|
# - postgres (?)
|
||||||
|
# - blackbox (?) (curl to see if http and similar is up)
|
||||||
|
# - ssl_exporter (?)
|
||||||
|
};
|
||||||
|
}
|
48
configuration/services/metrics/grafana.nix
Normal file
48
configuration/services/metrics/grafana.nix
Normal file
|
@ -0,0 +1,48 @@
|
||||||
|
{config, ...}: let
|
||||||
|
domain = "metrics.${config.services.nginx.domain}";
|
||||||
|
in {
|
||||||
|
services.grafana = {
|
||||||
|
enable = true;
|
||||||
|
settings = {
|
||||||
|
server.http_port = 3001; # Default overlaps with gitea
|
||||||
|
|
||||||
|
security = {
|
||||||
|
admin_user = "tlater";
|
||||||
|
admin_password = "$__file{${config.sops.secrets."grafana/adminPassword".path}}";
|
||||||
|
secret_key = "$__file{${config.sops.secrets."grafana/secretKey".path}}";
|
||||||
|
cookie_secure = true;
|
||||||
|
cookie_samesite = "strict";
|
||||||
|
content_security_policy = true;
|
||||||
|
};
|
||||||
|
|
||||||
|
database = {
|
||||||
|
user = "grafana";
|
||||||
|
name = "grafana";
|
||||||
|
type = "postgres";
|
||||||
|
host = "/run/postgresql";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
provision = {
|
||||||
|
enable = true;
|
||||||
|
|
||||||
|
datasources.settings.datasources = [
|
||||||
|
{
|
||||||
|
name = "Victoriametrics - tlater.net";
|
||||||
|
url = "http://localhost:8428";
|
||||||
|
type = "prometheus";
|
||||||
|
}
|
||||||
|
];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
services.nginx.virtualHosts."${domain}" = {
|
||||||
|
forceSSL = true;
|
||||||
|
enableACME = true;
|
||||||
|
extraConfig = ''
|
||||||
|
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
|
||||||
|
access_log /var/log/nginx/${domain}/access.log upstream_time;
|
||||||
|
'';
|
||||||
|
locations."/".proxyPass = "http://localhost:${toString config.services.grafana.settings.server.http_port}";
|
||||||
|
};
|
||||||
|
}
|
204
configuration/services/metrics/options.nix
Normal file
204
configuration/services/metrics/options.nix
Normal file
|
@ -0,0 +1,204 @@
|
||||||
|
{
|
||||||
|
pkgs,
|
||||||
|
config,
|
||||||
|
lib,
|
||||||
|
...
|
||||||
|
}: let
|
||||||
|
inherit (lib) types mkOption mkDefault;
|
||||||
|
yaml = pkgs.formats.yaml {};
|
||||||
|
in {
|
||||||
|
options = {
|
||||||
|
services.prometheus = {
|
||||||
|
extraExporters = mkOption {
|
||||||
|
type = types.attrsOf (types.submodule {
|
||||||
|
options = {
|
||||||
|
port = mkOption {
|
||||||
|
type = types.int;
|
||||||
|
description = "The port on which this exporter listens.";
|
||||||
|
};
|
||||||
|
listenAddress = mkOption {
|
||||||
|
type = types.str;
|
||||||
|
default = "127.0.0.1";
|
||||||
|
description = "Address to listen on.";
|
||||||
|
};
|
||||||
|
serviceOpts = mkOption {
|
||||||
|
type = types.attrs;
|
||||||
|
description = "An attrset to be merged with the exporter's systemd service.";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
});
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
services.victoriametrics.scrapeConfigs = mkOption {
|
||||||
|
type = types.attrsOf (types.submodule ({
|
||||||
|
name,
|
||||||
|
self,
|
||||||
|
...
|
||||||
|
}: {
|
||||||
|
options = {
|
||||||
|
job_name = mkOption {
|
||||||
|
type = types.str;
|
||||||
|
default = name;
|
||||||
|
};
|
||||||
|
|
||||||
|
extraSettings = mkOption {
|
||||||
|
type = types.anything;
|
||||||
|
description = ''
|
||||||
|
Other settings to set for this scrape config.
|
||||||
|
'';
|
||||||
|
default = {};
|
||||||
|
};
|
||||||
|
|
||||||
|
targets = mkOption {
|
||||||
|
type = types.listOf types.str;
|
||||||
|
description = lib.mdDoc ''
|
||||||
|
Addresses scrape targets for this config listen on.
|
||||||
|
|
||||||
|
Shortcut for `static_configs = lib.singleton {targets = [<targets>];}`
|
||||||
|
'';
|
||||||
|
default = [];
|
||||||
|
};
|
||||||
|
|
||||||
|
static_configs = mkOption {
|
||||||
|
default = [];
|
||||||
|
type = types.listOf (types.submodule {
|
||||||
|
options = {
|
||||||
|
targets = mkOption {
|
||||||
|
type = types.listOf types.str;
|
||||||
|
description = lib.mdDoc ''
|
||||||
|
The addresses scrape targets for this config listen on.
|
||||||
|
|
||||||
|
Must in `listenAddress:port` format.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
labels = mkOption {
|
||||||
|
type = types.attrsOf types.str;
|
||||||
|
description = lib.mdDoc ''
|
||||||
|
Labels to apply to all targets defined for this static config.
|
||||||
|
'';
|
||||||
|
default = {};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
});
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}));
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
config = {
|
||||||
|
systemd.services = lib.mkMerge [
|
||||||
|
(lib.mapAttrs' (name: exporter:
|
||||||
|
lib.nameValuePair "prometheus-${name}-exporter" (lib.mkMerge [
|
||||||
|
{
|
||||||
|
# Shamelessly copied from upstream because the upstream
|
||||||
|
# module is an intractable mess
|
||||||
|
wantedBy = ["multi-user.target"];
|
||||||
|
after = ["network.target"];
|
||||||
|
serviceConfig.Restart = mkDefault "always";
|
||||||
|
serviceConfig.PrivateTmp = mkDefault true;
|
||||||
|
serviceConfig.WorkingDirectory = mkDefault /tmp;
|
||||||
|
serviceConfig.DynamicUser = mkDefault true;
|
||||||
|
# Hardening
|
||||||
|
serviceConfig.CapabilityBoundingSet = mkDefault [""];
|
||||||
|
serviceConfig.DeviceAllow = [""];
|
||||||
|
serviceConfig.LockPersonality = true;
|
||||||
|
serviceConfig.MemoryDenyWriteExecute = true;
|
||||||
|
serviceConfig.NoNewPrivileges = true;
|
||||||
|
serviceConfig.PrivateDevices = mkDefault true;
|
||||||
|
serviceConfig.ProtectClock = mkDefault true;
|
||||||
|
serviceConfig.ProtectControlGroups = true;
|
||||||
|
serviceConfig.ProtectHome = true;
|
||||||
|
serviceConfig.ProtectHostname = true;
|
||||||
|
serviceConfig.ProtectKernelLogs = true;
|
||||||
|
serviceConfig.ProtectKernelModules = true;
|
||||||
|
serviceConfig.ProtectKernelTunables = true;
|
||||||
|
serviceConfig.ProtectSystem = mkDefault "strict";
|
||||||
|
serviceConfig.RemoveIPC = true;
|
||||||
|
serviceConfig.RestrictAddressFamilies = ["AF_INET" "AF_INET6"];
|
||||||
|
serviceConfig.RestrictNamespaces = true;
|
||||||
|
serviceConfig.RestrictRealtime = true;
|
||||||
|
serviceConfig.RestrictSUIDSGID = true;
|
||||||
|
serviceConfig.SystemCallArchitectures = "native";
|
||||||
|
serviceConfig.UMask = "0077";
|
||||||
|
}
|
||||||
|
exporter.serviceOpts
|
||||||
|
]))
|
||||||
|
config.services.prometheus.extraExporters)
|
||||||
|
|
||||||
|
{
|
||||||
|
vmagent-scrape-exporters = let
|
||||||
|
listenAddress = config.services.victoriametrics.listenAddress;
|
||||||
|
vmAddr = (lib.optionalString (lib.hasPrefix ":" listenAddress) "127.0.0.1") + listenAddress;
|
||||||
|
promscrape = yaml.generate "prometheus.yml" {
|
||||||
|
scrape_configs = lib.mapAttrsToList (_: scrape:
|
||||||
|
lib.recursiveUpdate {
|
||||||
|
inherit (scrape) job_name;
|
||||||
|
static_configs =
|
||||||
|
scrape.static_configs
|
||||||
|
++ lib.optional (scrape.targets != []) {targets = scrape.targets;};
|
||||||
|
}
|
||||||
|
scrape.extraSettings)
|
||||||
|
config.services.victoriametrics.scrapeConfigs;
|
||||||
|
};
|
||||||
|
in {
|
||||||
|
enable = true;
|
||||||
|
path = [pkgs.victoriametrics];
|
||||||
|
wantedBy = ["multi-user.target"];
|
||||||
|
after = ["network.target" "victoriametrics.service"];
|
||||||
|
serviceConfig = {
|
||||||
|
ExecStart = [
|
||||||
|
(lib.concatStringsSep " " [
|
||||||
|
"${pkgs.victoriametrics}/bin/vmagent"
|
||||||
|
"-promscrape.config=${promscrape}"
|
||||||
|
"-remoteWrite.url=http://${vmAddr}/api/v1/write"
|
||||||
|
"-remoteWrite.tmpDataPath=%t/vmagent"
|
||||||
|
])
|
||||||
|
];
|
||||||
|
SupplementaryGroups = "metrics";
|
||||||
|
|
||||||
|
DynamicUser = true;
|
||||||
|
RuntimeDirectory = "vmagent";
|
||||||
|
CapabilityBoundingSet = [""];
|
||||||
|
DeviceAllow = [""];
|
||||||
|
LockPersonality = true;
|
||||||
|
MemoryDenyWriteExecute = true;
|
||||||
|
NoNewPrivileges = true;
|
||||||
|
PrivateDevices = true;
|
||||||
|
ProtectClock = true;
|
||||||
|
ProtectControlGroups = true;
|
||||||
|
ProtectHome = true;
|
||||||
|
ProtectHostname = true;
|
||||||
|
ProtectKernelLogs = true;
|
||||||
|
ProtectKernelModules = true;
|
||||||
|
ProtectKernelTunables = true;
|
||||||
|
ProtectSystem = "strict";
|
||||||
|
RemoveIPC = true;
|
||||||
|
RestrictAddressFamilies = ["AF_INET" "AF_INET6"];
|
||||||
|
RestrictNamespaces = true;
|
||||||
|
RestrictRealtime = true;
|
||||||
|
RestrictSUIDSGID = true;
|
||||||
|
SystemCallArchitectures = "native";
|
||||||
|
UMask = "0077";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
users.groups.metrics = {};
|
||||||
|
|
||||||
|
services.victoriametrics.scrapeConfigs = let
|
||||||
|
allExporters =
|
||||||
|
lib.mapAttrs (name: exporter: {
|
||||||
|
inherit (exporter) listenAddress port;
|
||||||
|
}) ((lib.filterAttrs (_: exporter: builtins.isAttrs exporter && exporter.enable)
|
||||||
|
config.services.prometheus.exporters)
|
||||||
|
// config.services.prometheus.extraExporters);
|
||||||
|
in
|
||||||
|
lib.mapAttrs (_: exporter: {
|
||||||
|
targets = ["${exporter.listenAddress}:${toString exporter.port}"];
|
||||||
|
})
|
||||||
|
allExporters;
|
||||||
|
};
|
||||||
|
}
|
13
configuration/services/metrics/victoriametrics.nix
Normal file
13
configuration/services/metrics/victoriametrics.nix
Normal file
|
@ -0,0 +1,13 @@
|
||||||
|
{config, ...}: {
|
||||||
|
config.services.victoriametrics = {
|
||||||
|
enable = true;
|
||||||
|
|
||||||
|
scrapeConfigs = {
|
||||||
|
gitea = {
|
||||||
|
targets = ["127.0.0.1:${toString config.services.gitea.settings.server.HTTP_PORT}"];
|
||||||
|
extraSettings.authorization.credentials_file = config.sops.secrets."gitea/metrics-token".path;
|
||||||
|
};
|
||||||
|
coturn.targets = ["127.0.0.1:9641"];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
|
@ -50,6 +50,9 @@ in {
|
||||||
services.nginx.virtualHosts."${hostName}" = {
|
services.nginx.virtualHosts."${hostName}" = {
|
||||||
forceSSL = true;
|
forceSSL = true;
|
||||||
enableACME = true;
|
enableACME = true;
|
||||||
|
extraConfig = ''
|
||||||
|
access_log /var/log/nginx/${hostName}/access.log upstream_time;
|
||||||
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
# Block repeated failed login attempts
|
# Block repeated failed login attempts
|
||||||
|
|
|
@ -16,6 +16,12 @@
|
||||||
# that operation needs to be performed manually on the system as
|
# that operation needs to be performed manually on the system as
|
||||||
# well.
|
# well.
|
||||||
ensureUsers = [
|
ensureUsers = [
|
||||||
|
{
|
||||||
|
name = "grafana";
|
||||||
|
ensurePermissions = {
|
||||||
|
"DATABASE grafana" = "ALL PRIVILEGES";
|
||||||
|
};
|
||||||
|
}
|
||||||
{
|
{
|
||||||
name = "nextcloud";
|
name = "nextcloud";
|
||||||
ensurePermissions = {
|
ensurePermissions = {
|
||||||
|
@ -25,6 +31,7 @@
|
||||||
];
|
];
|
||||||
|
|
||||||
ensureDatabases = [
|
ensureDatabases = [
|
||||||
|
"grafana"
|
||||||
"nextcloud"
|
"nextcloud"
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
|
|
@ -19,6 +19,7 @@ in {
|
||||||
enableACME = true;
|
enableACME = true;
|
||||||
extraConfig = ''
|
extraConfig = ''
|
||||||
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
|
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
|
||||||
|
access_log /var/log/nginx/${domain}/access.log upstream_time;
|
||||||
'';
|
'';
|
||||||
|
|
||||||
locations."/".proxyPass = "http://${addr}:${toString port}";
|
locations."/".proxyPass = "http://${addr}:${toString port}";
|
||||||
|
|
|
@ -3,6 +3,23 @@
|
||||||
defaultSopsFile = ../keys/production.yaml;
|
defaultSopsFile = ../keys/production.yaml;
|
||||||
|
|
||||||
secrets = {
|
secrets = {
|
||||||
|
# Gitea
|
||||||
|
"gitea/metrics-token" = {
|
||||||
|
owner = "gitea";
|
||||||
|
group = "metrics";
|
||||||
|
mode = "0440";
|
||||||
|
};
|
||||||
|
|
||||||
|
# Grafana
|
||||||
|
"grafana/adminPassword" = {
|
||||||
|
owner = "grafana";
|
||||||
|
group = "grafana";
|
||||||
|
};
|
||||||
|
"grafana/secretKey" = {
|
||||||
|
owner = "grafana";
|
||||||
|
group = "grafana";
|
||||||
|
};
|
||||||
|
|
||||||
# Heisenbridge
|
# Heisenbridge
|
||||||
"heisenbridge/as-token" = {};
|
"heisenbridge/as-token" = {};
|
||||||
"heisenbridge/hs-token" = {};
|
"heisenbridge/hs-token" = {};
|
||||||
|
|
10
flake.nix
10
flake.nix
|
@ -78,7 +78,7 @@
|
||||||
# Utility scripts #
|
# Utility scripts #
|
||||||
###################
|
###################
|
||||||
packages.${system} = let
|
packages.${system} = let
|
||||||
inherit (nixpkgs.legacyPackages.${system}) writeShellScript;
|
inherit (nixpkgs.legacyPackages.${system}) writeShellScript writeShellScriptBin;
|
||||||
vm = nixpkgs.lib.nixosSystem {
|
vm = nixpkgs.lib.nixosSystem {
|
||||||
inherit system;
|
inherit system;
|
||||||
specialArgs.flake-inputs = inputs;
|
specialArgs.flake-inputs = inputs;
|
||||||
|
@ -106,6 +106,14 @@
|
||||||
"${vm.config.system.build.vm}/bin/run-tlaternet-vm"
|
"${vm.config.system.build.vm}/bin/run-tlaternet-vm"
|
||||||
'';
|
'';
|
||||||
|
|
||||||
|
update-pkgs = let
|
||||||
|
nvfetcher-bin = "${nvfetcher.packages.${system}.default}/bin/nvfetcher";
|
||||||
|
in
|
||||||
|
writeShellScriptBin "update-pkgs" ''
|
||||||
|
cd "$(git rev-parse --show-toplevel)/pkgs"
|
||||||
|
${nvfetcher-bin} -o _sources_pkgs -c nvfetcher.toml
|
||||||
|
'';
|
||||||
|
|
||||||
update-nextcloud-apps = let
|
update-nextcloud-apps = let
|
||||||
nvfetcher-bin = "${nvfetcher.packages.${system}.default}/bin/nvfetcher";
|
nvfetcher-bin = "${nvfetcher.packages.${system}.default}/bin/nvfetcher";
|
||||||
in
|
in
|
||||||
|
|
|
@ -1,3 +1,8 @@
|
||||||
|
gitea:
|
||||||
|
metrics-token: ENC[AES256_GCM,data:/7/zvVl2ZOBoekrJR32vl/QQcG5XqTmltgpHEMUpbXVeqwnq29idzE2Qyjau96ZHObmSI73/ZtW95uXF6LH9Qw==,iv:iWZECCZSh1CN7wMBqstXR5QWtriR7QLKVqhekGnpXl0=,tag:HEr9km8VYmruBzf0I/5HuA==,type:str]
|
||||||
|
grafana:
|
||||||
|
adminPassword: ENC[AES256_GCM,data:/qw//J7cOkIGa58bG4GgdzndvKof32AmQeWB00IX8WhA22PDCOc4VdUEoB3wVJJqI/ucoHFInYyhg2rFYoYBesBjAt0QS3+O+8WblIunUuYeqlBuYJJK1TLhy6ql6+aqvfiW/rJLm4LpgA7CboyDD2OYHcAbvGSD2GWwFcHTR/Y=,iv:KK6p8GKzc9SBDZZFkEwCdIjSxriPGNMDNcr97tfbwTI=,tag:gLRNSGdJWFD+V9K5TfJvXw==,type:str]
|
||||||
|
secretKey: ENC[AES256_GCM,data:OUXWOE6I3a26SrFEOczWNIwyR3Rx62fbsRBBcfh0xyEbxOIPhexH6lIqlVG9Ltwra9+rAldNM4/0BydtxIDj7A==,iv:fiNO/or5yZnhpDPMANDnEC5dtXmbKBZsV+BPmvCN/HI=,tag:Q0M0OtLWdWAJgQmUlL//fg==,type:str]
|
||||||
nextcloud:
|
nextcloud:
|
||||||
tlater: ENC[AES256_GCM,data:zNsPm4uFaIRe3LjcwmayRg==,iv:5wam6bP5zP708jC9UrLV0s8qspl3Pm4fPzbMFYBUyPQ=,tag:apnJUMeJwMn9q0NhO4ptmA==,type:str]
|
tlater: ENC[AES256_GCM,data:zNsPm4uFaIRe3LjcwmayRg==,iv:5wam6bP5zP708jC9UrLV0s8qspl3Pm4fPzbMFYBUyPQ=,tag:apnJUMeJwMn9q0NhO4ptmA==,type:str]
|
||||||
steam:
|
steam:
|
||||||
|
@ -21,8 +26,8 @@ sops:
|
||||||
azure_kv: []
|
azure_kv: []
|
||||||
hc_vault: []
|
hc_vault: []
|
||||||
age: []
|
age: []
|
||||||
lastmodified: "2023-09-23T18:55:44Z"
|
lastmodified: "2023-10-12T18:40:26Z"
|
||||||
mac: ENC[AES256_GCM,data:psqgXozY9L7nduZ11GF+mbIrZ4RUySqBixkWL5z0cYeoLA3URb/dr028LCmNgQS9l8aJVsjVkyLBJIU/8wmiUNqRy/VI5iqV5mu+sxXhUVwFL0dAAWP1lOKwwT5uGK89/ioqkphgzuWD37vGe2vYddKkJF0M+zlz12fqkMjaisU=,iv:UyRoJbfuGU3K/Mp5DQ1kY0Z+nKSSo46BGNAcxt+vAvc=,tag:HkP6+qxQ8J/xAYJXYoG/6g==,type:str]
|
mac: ENC[AES256_GCM,data:F+yQ20jCtLRKeQDFVKoqrYCgtwGkXxrK6aQO0MFZTIMJAnbTVPM2ZJGQ1RxXb+Zs4T+44EEc2xN4LjeANvgpE6MfOz2VTw+sEEjcYwUyB6RcXHia9XlFLa8lh7/Wx/9DxlSFjjSrxmDkNB6r+n5UF81cdRXF2E9ibdH346ST98A=,iv:xVxFN1IDKrLskaGqnWvOWx1zUII0jRSjQxEsaTf2GNw=,tag:lnp1AvgMOXXlg1vFjHEWUQ==,type:str]
|
||||||
pgp:
|
pgp:
|
||||||
- created_at: "2022-10-12T00:46:51Z"
|
- created_at: "2022-10-12T00:46:51Z"
|
||||||
enc: |
|
enc: |
|
||||||
|
|
|
@ -1,3 +1,8 @@
|
||||||
|
gitea:
|
||||||
|
metrics-token: ENC[AES256_GCM,data:J4QdfI1wKyM=,iv:8fqCbftyhj90eIVFxjEp9RXKC1y1IaLnV1r2MOdY15M=,tag:8W/juv1OZh4hJco02qXO6g==,type:str]
|
||||||
|
grafana:
|
||||||
|
adminPassword: ENC[AES256_GCM,data:dYfaxUpQpzA=,iv:j5wSem8C5+V4c5qRzXQJhsU7/FOtpvrnaEyFBmW6zJ4=,tag:oc8n3TkEbjF2gjuOobZuLA==,type:str]
|
||||||
|
secretKey: ENC[AES256_GCM,data:Atruvh2MsNY=,iv:y2MaCUCEzGIydHp6G0DJHfk289S1is0twKm2oUYwDhM=,tag:nAWeg+YqaYqk6k22oBkAhQ==,type:str]
|
||||||
nextcloud:
|
nextcloud:
|
||||||
tlater: ENC[AES256_GCM,data:91kDcO4hpng=,iv:ayuILRmRru4ZxTCur9H2xHuLjkDzwPdS/4lEog/tesU=,tag:qYhJxnNDcCwUM7xe7Tlcjw==,type:str]
|
tlater: ENC[AES256_GCM,data:91kDcO4hpng=,iv:ayuILRmRru4ZxTCur9H2xHuLjkDzwPdS/4lEog/tesU=,tag:qYhJxnNDcCwUM7xe7Tlcjw==,type:str]
|
||||||
steam:
|
steam:
|
||||||
|
@ -21,8 +26,8 @@ sops:
|
||||||
azure_kv: []
|
azure_kv: []
|
||||||
hc_vault: []
|
hc_vault: []
|
||||||
age: []
|
age: []
|
||||||
lastmodified: "2023-09-22T21:07:02Z"
|
lastmodified: "2023-10-07T02:17:50Z"
|
||||||
mac: ENC[AES256_GCM,data:gItC41S8MInLmikdH1okhPs+FVf8sCF/iQeJ5reigBunHkOngoc6nOFANyAcNZETszzhgTLXXtmVNEjW46v6K7D6nmoi/zwpedUxwzMwDC5I28VTMDHVMAThYSGtdo6kig8i2pi8rzEQd1DStxMv3TWML5y6DDTlFsd3lfudaHA=,iv:zXebvIVPR76GwUhpactwRgF/eEmx2OBkT18E8lkwzRA=,tag:6HyISACbFCGlpIIgkFeA/A==,type:str]
|
mac: ENC[AES256_GCM,data:vZDq33YIn0Nf1FQ2+ySezox6igiw6zNFCu3l3kaIsBKo1797pohmAxj2Lcc+OmlBjj98khaBIlbQuA5ULM+uPN5ILaz3NuXD5PZtsV+rL2PsLNMW9FBSmJ0m0YQrt0nZ0tpzifn12XghcSK2IXv+FnxlfrAJCxDvr5tRm90uUwU=,iv:ct8CzIWjaoJ1UjZcdFSr8lZ626vA0RvM883V6H5plWc=,tag:waJNtp/UbRDOfyzNElrung==,type:str]
|
||||||
pgp:
|
pgp:
|
||||||
- created_at: "2022-10-12T16:48:23Z"
|
- created_at: "2022-10-12T16:48:23Z"
|
||||||
enc: |
|
enc: |
|
||||||
|
|
21
pkgs/_sources_pkgs/generated.json
Normal file
21
pkgs/_sources_pkgs/generated.json
Normal file
|
@ -0,0 +1,21 @@
|
||||||
|
{
|
||||||
|
"prometheus-fail2ban-exporter": {
|
||||||
|
"cargoLocks": null,
|
||||||
|
"date": null,
|
||||||
|
"extract": null,
|
||||||
|
"name": "prometheus-fail2ban-exporter",
|
||||||
|
"passthru": null,
|
||||||
|
"pinned": false,
|
||||||
|
"src": {
|
||||||
|
"deepClone": false,
|
||||||
|
"fetchSubmodules": false,
|
||||||
|
"leaveDotGit": false,
|
||||||
|
"name": null,
|
||||||
|
"rev": "v0.10.0",
|
||||||
|
"sha256": "sha256-8nIW1XaHCBqQCoLkV1ZYE3NTbVZ6c+UOqYD08XQiv+4=",
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter"
|
||||||
|
},
|
||||||
|
"version": "v0.10.0"
|
||||||
|
}
|
||||||
|
}
|
16
pkgs/_sources_pkgs/generated.nix
Normal file
16
pkgs/_sources_pkgs/generated.nix
Normal file
|
@ -0,0 +1,16 @@
|
||||||
|
# This file was generated by nvfetcher, please do not modify it manually.
|
||||||
|
{ fetchgit, fetchurl, fetchFromGitHub, dockerTools }:
|
||||||
|
{
|
||||||
|
prometheus-fail2ban-exporter = {
|
||||||
|
pname = "prometheus-fail2ban-exporter";
|
||||||
|
version = "v0.10.0";
|
||||||
|
src = fetchgit {
|
||||||
|
url = "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter";
|
||||||
|
rev = "v0.10.0";
|
||||||
|
fetchSubmodules = false;
|
||||||
|
deepClone = false;
|
||||||
|
leaveDotGit = false;
|
||||||
|
sha256 = "sha256-8nIW1XaHCBqQCoLkV1ZYE3NTbVZ6c+UOqYD08XQiv+4=";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
|
@ -7,6 +7,9 @@
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
starbound = callPackage ./starbound {};
|
starbound = callPackage ./starbound {};
|
||||||
|
prometheus-fail2ban-exporter = callPackage ./prometheus/fail2ban-exporter.nix {
|
||||||
|
sources = pkgs.callPackage ./_sources_pkgs/generated.nix {};
|
||||||
|
};
|
||||||
}
|
}
|
||||||
// (
|
// (
|
||||||
# Add nextcloud apps
|
# Add nextcloud apps
|
||||||
|
|
3
pkgs/nvfetcher.toml
Normal file
3
pkgs/nvfetcher.toml
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
[prometheus-fail2ban-exporter]
|
||||||
|
src.manual = "v0.10.0" # No gitlab support in nvfetcher
|
||||||
|
fetch.git = "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter"
|
8
pkgs/prometheus/fail2ban-exporter.nix
Normal file
8
pkgs/prometheus/fail2ban-exporter.nix
Normal file
|
@ -0,0 +1,8 @@
|
||||||
|
{
|
||||||
|
buildGoModule,
|
||||||
|
sources,
|
||||||
|
}:
|
||||||
|
buildGoModule {
|
||||||
|
inherit (sources.prometheus-fail2ban-exporter) pname src version;
|
||||||
|
vendorHash = "sha256-qU6opwhhvzbQOhfGVyiVgKhfCSB0Z4eSRAJnv6ht2I0=";
|
||||||
|
}
|
Loading…
Reference in a new issue