tlaternet/tlaternet-server
1
0
Fork 0
Code Issues10 Pull requests Releases Activity

metrics: Add metrics with victoriametrics + grafana

Browse source
This commit is contained in:
Tristan Daniël Maat 2023-10-07 22:14:43 +02:00
parent 78a9eac9bb
commit 55a4aaf48b
Signed by: tlater
GPG key ID: 49670FD774E43268
22 changed files with 580 additions and 26 deletions
configuration/services/metrics
options.nix

204
configuration/services/metrics/options.nix Normal file
View file

@ -0,0 +1,204 @@
{
pkgs,
config,
lib,
...
}: let
inherit (lib) types mkOption mkDefault;
yaml = pkgs.formats.yaml {};
in {
options = {
services.prometheus = {
extraExporters = mkOption {
type = types.attrsOf (types.submodule {
options = {
port = mkOption {
type = types.int;
description = "The port on which this exporter listens.";
};
listenAddress = mkOption {
type = types.str;
default = "127.0.0.1";
description = "Address to listen on.";
};
serviceOpts = mkOption {
type = types.attrs;
description = "An attrset to be merged with the exporter's systemd service.";
};
};
});
};
};
services.victoriametrics.scrapeConfigs = mkOption {
type = types.attrsOf (types.submodule ({
name,
self,
...
}: {
options = {
job_name = mkOption {
type = types.str;
default = name;
};
extraSettings = mkOption {
type = types.anything;
description = ''
Other settings to set for this scrape config.
'';
default = {};
};
targets = mkOption {
type = types.listOf types.str;
description = lib.mdDoc ''
Addresses scrape targets for this config listen on.
Shortcut for `static_configs = lib.singleton {targets = [<targets>];}`
'';
default = [];
};
static_configs = mkOption {
default = [];
type = types.listOf (types.submodule {
options = {
targets = mkOption {
type = types.listOf types.str;
description = lib.mdDoc ''
The addresses scrape targets for this config listen on.
Must in `listenAddress:port` format.
'';
};
labels = mkOption {
type = types.attrsOf types.str;
description = lib.mdDoc ''
Labels to apply to all targets defined for this static config.
'';
default = {};
};
};
});
};
};
}));
};
};
config = {
systemd.services = lib.mkMerge [
(lib.mapAttrs' (name: exporter:
lib.nameValuePair "prometheus-${name}-exporter" (lib.mkMerge [
{
# Shamelessly copied from upstream because the upstream
# module is an intractable mess
wantedBy = ["multi-user.target"];
after = ["network.target"];
serviceConfig.Restart = mkDefault "always";
serviceConfig.PrivateTmp = mkDefault true;
serviceConfig.WorkingDirectory = mkDefault /tmp;
serviceConfig.DynamicUser = mkDefault true;
# Hardening
serviceConfig.CapabilityBoundingSet = mkDefault [""];
serviceConfig.DeviceAllow = [""];
serviceConfig.LockPersonality = true;
serviceConfig.MemoryDenyWriteExecute = true;
serviceConfig.NoNewPrivileges = true;
serviceConfig.PrivateDevices = mkDefault true;
serviceConfig.ProtectClock = mkDefault true;
serviceConfig.ProtectControlGroups = true;
serviceConfig.ProtectHome = true;
serviceConfig.ProtectHostname = true;
serviceConfig.ProtectKernelLogs = true;
serviceConfig.ProtectKernelModules = true;
serviceConfig.ProtectKernelTunables = true;
serviceConfig.ProtectSystem = mkDefault "strict";
serviceConfig.RemoveIPC = true;
serviceConfig.RestrictAddressFamilies = ["AF_INET" "AF_INET6"];
serviceConfig.RestrictNamespaces = true;
serviceConfig.RestrictRealtime = true;
serviceConfig.RestrictSUIDSGID = true;
serviceConfig.SystemCallArchitectures = "native";
serviceConfig.UMask = "0077";
}
exporter.serviceOpts
]))
config.services.prometheus.extraExporters)
{
vmagent-scrape-exporters = let
listenAddress = config.services.victoriametrics.listenAddress;
vmAddr = (lib.optionalString (lib.hasPrefix ":" listenAddress) "127.0.0.1") + listenAddress;
promscrape = yaml.generate "prometheus.yml" {
scrape_configs = lib.mapAttrsToList (_: scrape:
lib.recursiveUpdate {
inherit (scrape) job_name;
static_configs =
scrape.static_configs
++ lib.optional (scrape.targets != []) {targets = scrape.targets;};
}
scrape.extraSettings)
config.services.victoriametrics.scrapeConfigs;
};
in {
enable = true;
path = [pkgs.victoriametrics];
wantedBy = ["multi-user.target"];
after = ["network.target" "victoriametrics.service"];
serviceConfig = {
ExecStart = [
(lib.concatStringsSep " " [
"${pkgs.victoriametrics}/bin/vmagent"
"-promscrape.config=${promscrape}"
"-remoteWrite.url=http://${vmAddr}/api/v1/write"
"-remoteWrite.tmpDataPath=%t/vmagent"
])
];
SupplementaryGroups = "metrics";
DynamicUser = true;
RuntimeDirectory = "vmagent";
CapabilityBoundingSet = [""];
DeviceAllow = [""];
LockPersonality = true;
MemoryDenyWriteExecute = true;
NoNewPrivileges = true;
PrivateDevices = true;
ProtectClock = true;
ProtectControlGroups = true;
ProtectHome = true;
ProtectHostname = true;
ProtectKernelLogs = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectSystem = "strict";
RemoveIPC = true;
RestrictAddressFamilies = ["AF_INET" "AF_INET6"];
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
SystemCallArchitectures = "native";
UMask = "0077";
};
};
}
];
users.groups.metrics = {};
services.victoriametrics.scrapeConfigs = let
allExporters =
lib.mapAttrs (name: exporter: {
inherit (exporter) listenAddress port;
}) ((lib.filterAttrs (_: exporter: builtins.isAttrs exporter && exporter.enable)
config.services.prometheus.exporters)
// config.services.prometheus.extraExporters);
in
lib.mapAttrs (_: exporter: {
targets = ["${exporter.listenAddress}:${toString exporter.port}"];
})
allExporters;
};
}