WIP: Add metrics
This commit is contained in:
parent
8dc5e13363
commit
8eb01bb23d
GPG key ID:
49670FD774E43268
22 changed files with 576 additions and 26 deletions
9
configuration/services/metrics/default.nix
Normal file
9
configuration/services/metrics/default.nix
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
imports = [
|
||||
./options.nix
|
||||
|
||||
./exporters.nix
|
||||
./grafana.nix
|
||||
./victoriametrics.nix
|
||||
];
|
||||
}
|
||||
100
configuration/services/metrics/exporters.nix
Normal file
100
configuration/services/metrics/exporters.nix
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
{
|
||||
config,
|
||||
pkgs,
|
||||
lib,
|
||||
...
|
||||
}: let
|
||||
yaml = pkgs.formats.yaml {};
|
||||
in {
|
||||
services.prometheus = {
|
||||
exporters = {
|
||||
# Periodically check domain registration status
|
||||
domain = {
|
||||
enable = true;
|
||||
listenAddress = "127.0.0.1";
|
||||
extraFlags = let
|
||||
conf.domains = [
|
||||
"tlater.net"
|
||||
"tlater.com"
|
||||
];
|
||||
in [
|
||||
"--config=${yaml.generate "domains.yml" conf}"
|
||||
];
|
||||
};
|
||||
|
||||
# System statistics
|
||||
node = {
|
||||
enable = true;
|
||||
listenAddress = "127.0.0.1";
|
||||
};
|
||||
systemd = {
|
||||
enable = true;
|
||||
listenAddress = "127.0.0.1";
|
||||
extraFlags = [
|
||||
# Disabled by default because only supported from systemd 235+
|
||||
"--systemd.collector.enable-restart-count"
|
||||
"--systemd.collector.enable-ip-accounting"
|
||||
];
|
||||
};
|
||||
|
||||
# Various nginx metrics
|
||||
nginx = {
|
||||
enable = true;
|
||||
listenAddress = "127.0.0.1";
|
||||
};
|
||||
|
||||
nginxlog = {
|
||||
enable = true;
|
||||
listenAddress = "127.0.0.1";
|
||||
group = "nginx";
|
||||
|
||||
settings.namespaces =
|
||||
lib.mapAttrsToList (name: virtualHost: {
|
||||
inherit name;
|
||||
metrics_override.prefix = "nginxlog";
|
||||
namespace_label = "vhost";
|
||||
|
||||
format = lib.concatStringsSep " " [
|
||||
"$remote_addr - $remote_user [$time_local]"
|
||||
''"$request" $status $body_bytes_sent''
|
||||
''"$http_referer" "$http_user_agent"''
|
||||
''rt=$request_time uct="$upstream_connect_time"''
|
||||
''uht="$upstream_header_time" urt="$upstream_response_time"''
|
||||
];
|
||||
|
||||
source.files = [
|
||||
"/var/log/nginx/${name}/access.log"
|
||||
];
|
||||
})
|
||||
config.services.nginx.virtualHosts;
|
||||
};
|
||||
};
|
||||
|
||||
extraExporters = {
|
||||
fail2ban = let
|
||||
cfg = config.services.prometheus.extraExporters.fail2ban;
|
||||
in {
|
||||
port = 9191;
|
||||
serviceOpts = {
|
||||
after = ["fail2ban.service"];
|
||||
requires = ["fail2ban.service"];
|
||||
serviceConfig = {
|
||||
Group = "fail2ban";
|
||||
RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"];
|
||||
ExecStart = lib.concatStringsSep " " [
|
||||
"${pkgs.local.prometheus-fail2ban-exporter}/bin/fail2ban-prometheus-exporter"
|
||||
"--collector.f2b.socket=/var/run/fail2ban/fail2ban.sock"
|
||||
"--web.listen-address='${cfg.listenAddress}:${toString cfg.port}'"
|
||||
"--collector.f2b.exit-on-socket-connection-error=true"
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
# TODO(tlater):
|
||||
# - wireguard (?)
|
||||
# - postgres (?)
|
||||
# - blackbox (?) (curl to see if http and similar is up)
|
||||
};
|
||||
}
|
||||
48
configuration/services/metrics/grafana.nix
Normal file
48
configuration/services/metrics/grafana.nix
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
{config, ...}: let
|
||||
domain = "metrics.${config.services.nginx.domain}";
|
||||
in {
|
||||
services.grafana = {
|
||||
enable = true;
|
||||
settings = {
|
||||
server.http_port = 3001; # Default overlaps with gitea
|
||||
|
||||
security = {
|
||||
admin_user = "tlater";
|
||||
admin_password = "$__file{${config.sops.secrets."grafana/adminPassword".path}}";
|
||||
secret_key = "$__file{${config.sops.secrets."grafana/secretKey".path}}";
|
||||
cookie_secure = true;
|
||||
cookie_samesite = "strict";
|
||||
content_security_policy = true;
|
||||
};
|
||||
|
||||
database = {
|
||||
user = "grafana";
|
||||
name = "grafana";
|
||||
type = "postgres";
|
||||
host = "/run/postgresql";
|
||||
};
|
||||
};
|
||||
|
||||
provision = {
|
||||
enable = true;
|
||||
|
||||
datasources.settings.datasources = [
|
||||
{
|
||||
name = "Victoriametrics - tlater.net";
|
||||
url = "http://localhost:8428";
|
||||
type = "prometheus";
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
services.nginx.virtualHosts."${domain}" = {
|
||||
forceSSL = true;
|
||||
enableACME = true;
|
||||
extraConfig = ''
|
||||
add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
|
||||
access_log /var/log/nginx/${domain}/access.log upstream_time;
|
||||
'';
|
||||
locations."/".proxyPass = "http://localhost:${toString config.services.grafana.settings.server.http_port}";
|
||||
};
|
||||
}
|
||||
204
configuration/services/metrics/options.nix
Normal file
204
configuration/services/metrics/options.nix
Normal file
|
|
@ -0,0 +1,204 @@
|
|||
{
|
||||
pkgs,
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}: let
|
||||
inherit (lib) types mkOption mkDefault;
|
||||
yaml = pkgs.formats.yaml {};
|
||||
in {
|
||||
options = {
|
||||
services.prometheus = {
|
||||
extraExporters = mkOption {
|
||||
type = types.attrsOf (types.submodule {
|
||||
options = {
|
||||
port = mkOption {
|
||||
type = types.int;
|
||||
description = "The port on which this exporter listens.";
|
||||
};
|
||||
listenAddress = mkOption {
|
||||
type = types.str;
|
||||
default = "127.0.0.1";
|
||||
description = "Address to listen on.";
|
||||
};
|
||||
serviceOpts = mkOption {
|
||||
type = types.attrs;
|
||||
description = "An attrset to be merged with the exporter's systemd service.";
|
||||
};
|
||||
};
|
||||
});
|
||||
};
|
||||
};
|
||||
|
||||
services.victoriametrics.scrapeConfigs = mkOption {
|
||||
type = types.attrsOf (types.submodule ({
|
||||
name,
|
||||
self,
|
||||
...
|
||||
}: {
|
||||
options = {
|
||||
job_name = mkOption {
|
||||
type = types.str;
|
||||
default = name;
|
||||
};
|
||||
|
||||
extraSettings = mkOption {
|
||||
type = types.anything;
|
||||
description = ''
|
||||
Other settings to set for this scrape config.
|
||||
'';
|
||||
default = {};
|
||||
};
|
||||
|
||||
targets = mkOption {
|
||||
type = types.listOf types.str;
|
||||
description = lib.mdDoc ''
|
||||
Addresses scrape targets for this config listen on.
|
||||
|
||||
Shortcut for `static_configs = lib.singleton {targets = [<targets>];}`
|
||||
'';
|
||||
default = [];
|
||||
};
|
||||
|
||||
static_configs = mkOption {
|
||||
default = [];
|
||||
type = types.listOf (types.submodule {
|
||||
options = {
|
||||
targets = mkOption {
|
||||
type = types.listOf types.str;
|
||||
description = lib.mdDoc ''
|
||||
The addresses scrape targets for this config listen on.
|
||||
|
||||
Must in `listenAddress:port` format.
|
||||
'';
|
||||
};
|
||||
labels = mkOption {
|
||||
type = types.attrsOf types.str;
|
||||
description = lib.mdDoc ''
|
||||
Labels to apply to all targets defined for this static config.
|
||||
'';
|
||||
default = {};
|
||||
};
|
||||
};
|
||||
});
|
||||
};
|
||||
};
|
||||
}));
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
systemd.services = lib.mkMerge [
|
||||
(lib.mapAttrs' (name: exporter:
|
||||
lib.nameValuePair "prometheus-${name}-exporter" (lib.mkMerge [
|
||||
{
|
||||
# Shamelessly copied from upstream because the upstream
|
||||
# module is an intractable mess
|
||||
wantedBy = ["multi-user.target"];
|
||||
after = ["network.target"];
|
||||
serviceConfig.Restart = mkDefault "always";
|
||||
serviceConfig.PrivateTmp = mkDefault true;
|
||||
serviceConfig.WorkingDirectory = mkDefault /tmp;
|
||||
serviceConfig.DynamicUser = mkDefault true;
|
||||
# Hardening
|
||||
serviceConfig.CapabilityBoundingSet = mkDefault [""];
|
||||
serviceConfig.DeviceAllow = [""];
|
||||
serviceConfig.LockPersonality = true;
|
||||
serviceConfig.MemoryDenyWriteExecute = true;
|
||||
serviceConfig.NoNewPrivileges = true;
|
||||
serviceConfig.PrivateDevices = mkDefault true;
|
||||
serviceConfig.ProtectClock = mkDefault true;
|
||||
serviceConfig.ProtectControlGroups = true;
|
||||
serviceConfig.ProtectHome = true;
|
||||
serviceConfig.ProtectHostname = true;
|
||||
serviceConfig.ProtectKernelLogs = true;
|
||||
serviceConfig.ProtectKernelModules = true;
|
||||
serviceConfig.ProtectKernelTunables = true;
|
||||
serviceConfig.ProtectSystem = mkDefault "strict";
|
||||
serviceConfig.RemoveIPC = true;
|
||||
serviceConfig.RestrictAddressFamilies = ["AF_INET" "AF_INET6"];
|
||||
serviceConfig.RestrictNamespaces = true;
|
||||
serviceConfig.RestrictRealtime = true;
|
||||
serviceConfig.RestrictSUIDSGID = true;
|
||||
serviceConfig.SystemCallArchitectures = "native";
|
||||
serviceConfig.UMask = "0077";
|
||||
}
|
||||
exporter.serviceOpts
|
||||
]))
|
||||
config.services.prometheus.extraExporters)
|
||||
|
||||
{
|
||||
vmagent-scrape-exporters = let
|
||||
listenAddress = config.services.victoriametrics.listenAddress;
|
||||
vmAddr = (lib.optionalString (lib.hasPrefix ":" listenAddress) "127.0.0.1") + listenAddress;
|
||||
promscrape = yaml.generate "prometheus.yml" {
|
||||
scrape_configs = lib.mapAttrsToList (_: scrape:
|
||||
lib.recursiveUpdate {
|
||||
inherit (scrape) job_name;
|
||||
static_configs =
|
||||
scrape.static_configs
|
||||
++ lib.optional (scrape.targets != []) {targets = scrape.targets;};
|
||||
}
|
||||
scrape.extraSettings)
|
||||
config.services.victoriametrics.scrapeConfigs;
|
||||
};
|
||||
in {
|
||||
enable = true;
|
||||
path = [pkgs.victoriametrics];
|
||||
wantedBy = ["multi-user.target"];
|
||||
after = ["network.target" "victoriametrics.service"];
|
||||
serviceConfig = {
|
||||
ExecStart = [
|
||||
(lib.concatStringsSep " " [
|
||||
"${pkgs.victoriametrics}/bin/vmagent"
|
||||
"-promscrape.config=${promscrape}"
|
||||
"-remoteWrite.url=http://${vmAddr}/api/v1/write"
|
||||
"-remoteWrite.tmpDataPath=%t/vmagent"
|
||||
])
|
||||
];
|
||||
SupplementaryGroups = "metrics";
|
||||
|
||||
DynamicUser = true;
|
||||
RuntimeDirectory = "vmagent";
|
||||
CapabilityBoundingSet = [""];
|
||||
DeviceAllow = [""];
|
||||
LockPersonality = true;
|
||||
MemoryDenyWriteExecute = true;
|
||||
NoNewPrivileges = true;
|
||||
PrivateDevices = true;
|
||||
ProtectClock = true;
|
||||
ProtectControlGroups = true;
|
||||
ProtectHome = true;
|
||||
ProtectHostname = true;
|
||||
ProtectKernelLogs = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectSystem = "strict";
|
||||
RemoveIPC = true;
|
||||
RestrictAddressFamilies = ["AF_INET" "AF_INET6"];
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
RestrictSUIDSGID = true;
|
||||
SystemCallArchitectures = "native";
|
||||
UMask = "0077";
|
||||
};
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
users.groups.metrics = {};
|
||||
|
||||
services.victoriametrics.scrapeConfigs = let
|
||||
allExporters =
|
||||
lib.mapAttrs (name: exporter: {
|
||||
inherit (exporter) listenAddress port;
|
||||
}) ((lib.filterAttrs (_: exporter: builtins.isAttrs exporter && exporter.enable)
|
||||
config.services.prometheus.exporters)
|
||||
// config.services.prometheus.extraExporters);
|
||||
in
|
||||
lib.mapAttrs (_: exporter: {
|
||||
targets = ["${exporter.listenAddress}:${toString exporter.port}"];
|
||||
})
|
||||
allExporters;
|
||||
};
|
||||
}
|
||||
13
configuration/services/metrics/victoriametrics.nix
Normal file
13
configuration/services/metrics/victoriametrics.nix
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
{config, ...}: {
|
||||
config.services.victoriametrics = {
|
||||
enable = true;
|
||||
|
||||
scrapeConfigs = {
|
||||
gitea = {
|
||||
targets = ["127.0.0.1:${toString config.services.gitea.settings.server.HTTP_PORT}"];
|
||||
extraSettings.authorization.credentials_file = config.sops.secrets."gitea/metrics-token".path;
|
||||
};
|
||||
coturn.targets = ["127.0.0.1:9641"];
|
||||
};
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue