tlaternet/tlaternet-server
1
0
Fork 0
Code Issues 10 Pull requests Activity

WIP: Add atomic backups with restic

Browse source
This commit is contained in:
Tristan Daniël Maat 2023-09-22 06:20:36 +02:00
parent ab5e088016
commit 729b442158
Signed by: tlater
GPG key ID: 49670FD774E43268
8 changed files with 263 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

1
configuration/default.nix
View file

@ -14,6 +14,7 @@
"${modulesPath}/profiles/minimal.nix"
(import ../modules)
./services/backups.nix
./services/conduit.nix
./services/foundryvtt.nix
./services/gitea.nix

180
configuration/services/backups.nix Normal file
View file

@ -0,0 +1,180 @@
{
config,
pkgs,
lib,
...
}: let
inherit (lib) types optional singleton;
mkShutdownScript = service:
pkgs.writeShellScript "backup-${service}-shutdown" ''
if systemctl is-active --quiet '${service}'; then
touch '/tmp/${service}-was-active'
systemctl stop '${service}'
fi
'';
mkRestartScript = service:
pkgs.writeShellScript "backup-${service}-restart" ''
if [ -f '/tmp/${service}-was-active' ]; then
rm '/tmp/${service}-was-active'
systemctl start '${service}'
fi
'';
writeScript = name: packages: text:
lib.getExe (pkgs.writeShellApplication {
inherit name text;
runtimeInputs = packages;
});
in {
options = {
services.backups = lib.mkOption {
description = lib.mdDoc ''
Configure restic backups with a specific tag.
'';
type = types.attrsOf (types.submodule ({
config,
name,
...
}: {
options = {
user = lib.mkOption {
type = types.str;
description = ''
The user as which to run the backup.
'';
};
paths = lib.mkOption {
type = types.listOf types.str;
description = ''
The paths to back up.
'';
};
tag = lib.mkOption {
type = types.str;
description = ''
The restic tag to mark the backup with.
'';
default = name;
};
preparation = {
packages = lib.mkOption {
type = types.listOf types.package;
default = [];
description = ''
The list of packages to make available in the
preparation script.
'';
};
text = lib.mkOption {
type = types.nullOr types.str;
default = null;
description = ''
The preparation script to run before the backup.
This should include things like database dumps and
enabling maintenance modes. If a service needs to be
shut down for backups, use `pauseServices` instead.
'';
};
};
cleanup = {
packages = lib.mkOption {
type = types.listOf types.package;
default = [];
description = ''
The list of packages to make available in the
cleanup script.
'';
};
text = lib.mkOption {
type = types.nullOr types.str;
default = null;
description = ''
The cleanup script to run after the backup.
This should do things like cleaning up database dumps
and disabling maintenance modes.
'';
};
};
pauseServices = lib.mkOption {
type = types.listOf types.str;
default = [];
description = ''
The systemd services that need to be shut down before
the backup can run. Services will be restarted after the
backup is complete.
This is intended to be used for services that do not
support hot backups.
'';
};
};
}));
};
};
config = lib.mkIf (config.services.backups != {}) {
systemd.services = lib.mapAttrs' (name: backup:
lib.nameValuePair "backup-${name}" {
# Don't want to restart mid-backup
restartIfChanged = false;
environment = {
RESTIC_CACHE_DIR = "%C/backup-${name}";
RESTIC_PASSWORD_FILE = config.sops.secrets."restic/local-backups".path;
# TODO(tlater): If I ever add more than one repo, service
# shutdown/restarting will potentially break if multiple
# backups for the same service overlap. A more clever
# sentinel file with reference counts would probably solve
# this.
RESTIC_REPOSITORY = "/var/lib/backups/";
};
serviceConfig = {
User = backup.user;
Group = "backup";
RuntimeDirectory = "backup-${name}";
CacheDirectory = "backup-${name}";
CacheDirectoryMode = "0700";
PrivateTmp = true;
ExecStart = [
(lib.concatStringsSep " " (["${pkgs.restic}/bin/restic" "backup" "--tag" name] ++ backup.paths))
];
ExecStartPre =
map (service: "+${mkShutdownScript service}") backup.pauseServices
++ singleton (writeScript "backup-${name}-repo-init" [pkgs.restic pkgs.coreutils] ''
restic snapshots || (restic init && chmod -R g+rwx "$RESTIC_REPOSITORY"/*)
'')
++ optional (backup.preparation.text != null)
(writeScript "backup-${name}-prepare" backup.preparation.packages backup.preparation.text);
# TODO(tlater): Add repo pruning/checking
ExecStopPost =
map (service: "+${mkRestartScript service}") backup.pauseServices
++ optional (backup.cleanup.text != null)
(writeScript "backup-${name}-cleanup" backup.cleanup.packages backup.cleanup.text);
};
})
config.services.backups;
systemd.timers = lib.mapAttrs' (name: backup:
lib.nameValuePair "backup-${name}" {
wantedBy = ["timers.target"];
timerConfig = {
OnCalendar = "*-*-* 02:30:00 UTC";
RandomizedDelaySec = "1h";
FixedRandomDelay = true;
Persistent = true;
};
})
config.services.backups;
users.groups.backup = {};
systemd.tmpfiles.rules = [
"d /var/lib/backups/ 0770 root backup"
];
};
}

10
configuration/services/conduit.nix
View file

@ -231,4 +231,14 @@ in {
};
};
};
services.backups.conduit = {
user = "root";
paths = [
"/var/lib/matrix-conduit/"
];
# Other services store their data in conduit, so no other services
# need to be shut down currently.
pauseServices = ["conduit.service"];
};
}

26
configuration/services/gitea.nix
View file

@ -1,4 +1,8 @@
{config, ...}: let
{
pkgs,
config,
...
}: let
domain = "gitea.${config.services.nginx.domain}";
in {
services.gitea = {
@ -52,4 +56,24 @@ in {
enabled = true
'';
};
services.backups.gitea = {
user = "gitea";
paths = [
"/var/lib/gitea/gitea-db.sql"
"/var/lib/gitea/repositories/"
"/var/lib/gitea/data/"
"/var/lib/gitea/custom/"
# Conf is backed up via nix
];
preparation = {
packages = [config.services.postgresql.package];
text = "pg_dump ${config.services.gitea.database.name} --file=/var/lib/gitea/gitea-db.sql";
};
cleanup = {
packages = [pkgs.coreutils];
text = "rm /var/lib/gitea/gitea-db.sql";
};
pauseServices = ["gitea.service"];
};
}

29
configuration/services/nextcloud.nix
View file

@ -74,4 +74,33 @@ in {
bantime = 86400
'';
};
services.backups.nextcloud = {
user = "nextcloud";
paths = [
"/var/lib/nextcloud/nextcloud-db.sql"
"/var/lib/nextcloud/data/"
"/var/lib/nextcloud/config/config.php"
];
preparation = {
packages = [
config.services.postgresql.package
config.services.nextcloud.occ
];
text = ''
nextcloud-occ maintenance:mode --on
pg_dump ${config.services.nextcloud.config.dbname} --file=/var/lib/nextcloud/nextcloud-db.sql
'';
};
cleanup = {
packages = [
pkgs.coreutils
config.services.nextcloud.occ
];
text = ''
rm /var/lib/nextcloud/nextcloud-db.sql
nextcloud-occ maintenance:mode --off
'';
};
};
}

8
configuration/services/starbound.nix
View file

@ -110,4 +110,12 @@ in {
# ProtectHome = "read-only"; # See further up
};
};
services.backups.starbound = {
user = "root";
paths = [
"/var/lib/starbound/storage/universe/"
];
pauseServices = ["starbound.service"];
};
}

6
configuration/sops.nix
View file

@ -19,6 +19,12 @@
mode = "0440";
};
"restic/local-backups" = {
owner = "root";
group = "backup";
mode = "0440";
};
"turn/env" = {};
"turn/secret" = {
owner = "turnserver";

6
keys/staging.yaml
View file

@ -7,6 +7,8 @@ heisenbridge:
hs-token: ENC[AES256_GCM,data:VBwvwomv0Xg=,iv:q6INtJ+rg+QiXj8uBdBzQYQZUBBXp+9odxDHwvu8Jxc=,tag:XKhm8nxygAkKaiVPJ2Fcdg==,type:str]
wireguard:
server-key: ENC[AES256_GCM,data:FvY897XdKoa/mckE8JQLCkklsnYD6Wz1wpsu5t3uhEnW3iarnDQxF9msuYU=,iv:jqGXfekM+Vs+J9b5nlZ5Skd1ZKHajoUo2Dc4tMYPm1w=,tag:EehikjI/FCU8wqtpvJRamQ==,type:str]
restic:
local-backups: ENC[AES256_GCM,data:3QjEv03t7wE=,iv:y/6Lv4eUbZZfGPwUONykz8VNL62cAJuWaJy9yk3aAmk=,tag:wMlGsepuG9JjwtUKGWSibw==,type:str]
turn:
env: ENC[AES256_GCM,data:xjIz/AY109lyiL5N01p5T3HcYco/rM5CJSRTtg==,iv:16bW6OpyOK/QL0QPGQp/Baa9xyT8E3ZsYkwqmjuofk0=,tag:J5re3uKxIykw3YunvQWBgg==,type:str]
secret: ENC[AES256_GCM,data:eQ7dAocoZtg=,iv:fgzjTPv30WqTKlLy+yMn5MsKQgjhPnwlGFFwYEg3gWs=,tag:1ze33U1NBkgMX/9SiaBNQg==,type:str]
@ -19,8 +21,8 @@ sops:
azure_kv: []
hc_vault: []
age: []
lastmodified: "2023-04-23T17:35:16Z"
mac: ENC[AES256_GCM,data:4cW8k6o3jET8k+yJGyApjOyuSUQb+d+4wX/RTNnpbt+867sExQrZUrOMif/u8S4WmcKVSJgvrzuxK9hpDPYhJ1d/5YuHH1Dyj7QDRdhbZYHhkpPus0ZVTEpSknZzx2eWH1ch/fyJJknlrBlfb/tz50Dv+w9mhkL7qteaIq+Vmsc=,iv:YMfAuGwu1kAM0wGkq3kzVMnC72yo7ZT04BuEwoLRPIA=,tag:6I1VRzteRaLuxN+sfLA5Mw==,type:str]
lastmodified: "2023-09-22T21:07:02Z"
mac: ENC[AES256_GCM,data:gItC41S8MInLmikdH1okhPs+FVf8sCF/iQeJ5reigBunHkOngoc6nOFANyAcNZETszzhgTLXXtmVNEjW46v6K7D6nmoi/zwpedUxwzMwDC5I28VTMDHVMAThYSGtdo6kig8i2pi8rzEQd1DStxMv3TWML5y6DDTlFsd3lfudaHA=,iv:zXebvIVPR76GwUhpactwRgF/eEmx2OBkT18E8lkwzRA=,tag:6HyISACbFCGlpIIgkFeA/A==,type:str]
pgp:
- created_at: "2022-10-12T16:48:23Z"
enc: |