tlaternet-server/configuration/services/backups.nix

{
  config,
  pkgs,
  lib,
  ...
}: let
  inherit (lib) types optional singleton;
  mkShutdownScript = service:
    pkgs.writeShellScript "backup-${service}-shutdown" ''
      if systemctl is-active --quiet '${service}'; then
        touch '/tmp/${service}-was-active'
        systemctl stop '${service}'
      fi
    '';
  mkRestartScript = service:
    pkgs.writeShellScript "backup-${service}-restart" ''
      if [ -f '/tmp/${service}-was-active' ]; then
        rm '/tmp/${service}-was-active'
        systemctl start '${service}'
      fi
    '';
  writeScript = name: packages: text:
    lib.getExe (pkgs.writeShellApplication {
      inherit name text;
      runtimeInputs = packages;
    });
in {
  options = {
    services.backups = lib.mkOption {
      description = lib.mdDoc ''
        Configure restic backups with a specific tag.
      '';
      type = types.attrsOf (types.submodule ({
        config,
        name,
        ...
      }: {
        options = {
          user = lib.mkOption {
            type = types.str;
            description = ''
              The user as which to run the backup.
            '';
          };
          paths = lib.mkOption {
            type = types.listOf types.str;
            description = ''
              The paths to back up.
            '';
          };
          tag = lib.mkOption {
            type = types.str;
            description = ''
              The restic tag to mark the backup with.
            '';
            default = name;
          };
          preparation = {
            packages = lib.mkOption {
              type = types.listOf types.package;
              default = [];
              description = ''
                The list of packages to make available in the
                preparation script.
              '';
            };
            text = lib.mkOption {
              type = types.nullOr types.str;
              default = null;
              description = ''
                The preparation script to run before the backup.

                This should include things like database dumps and
                enabling maintenance modes. If a service needs to be
                shut down for backups, use `pauseServices` instead.
              '';
            };
          };
          cleanup = {
            packages = lib.mkOption {
              type = types.listOf types.package;
              default = [];
              description = ''
                The list of packages to make available in the
                cleanup script.
              '';
            };
            text = lib.mkOption {
              type = types.nullOr types.str;
              default = null;
              description = ''
                The cleanup script to run after the backup.

                This should do things like cleaning up database dumps
                and disabling maintenance modes.
              '';
            };
          };
          pauseServices = lib.mkOption {
            type = types.listOf types.str;
            default = [];
            description = ''
              The systemd services that need to be shut down before
              the backup can run. Services will be restarted after the
              backup is complete.

              This is intended to be used for services that do not
              support hot backups.
            '';
          };
        };
      }));
    };
  };

  config = lib.mkIf (config.services.backups != {}) {
    systemd.services = lib.mapAttrs' (name: backup:
      lib.nameValuePair "backup-${name}" {
        # Don't want to restart mid-backup
        restartIfChanged = false;

        environment = {
          RESTIC_CACHE_DIR = "%C/backup-${name}";
          RESTIC_PASSWORD_FILE = config.sops.secrets."restic/local-backups".path;
          # TODO(tlater): If I ever add more than one repo, service
          # shutdown/restarting will potentially break if multiple
          # backups for the same service overlap. A more clever
          # sentinel file with reference counts would probably solve
          # this.
          RESTIC_REPOSITORY = "/var/lib/backups/";
        };

        serviceConfig = {
          User = backup.user;
          Group = "backup";
          RuntimeDirectory = "backup-${name}";
          CacheDirectory = "backup-${name}";
          CacheDirectoryMode = "0700";
          PrivateTmp = true;

          ExecStart = [
            (lib.concatStringsSep " " (["${pkgs.restic}/bin/restic" "backup" "--tag" name] ++ backup.paths))
          ];

          ExecStartPre =
            map (service: "+${mkShutdownScript service}") backup.pauseServices
            ++ singleton (writeScript "backup-${name}-repo-init" [pkgs.restic pkgs.coreutils] ''
              restic snapshots || (restic init && chmod -R g+rwx "$RESTIC_REPOSITORY"/*)
            '')
            ++ optional (backup.preparation.text != null)
            (writeScript "backup-${name}-prepare" backup.preparation.packages backup.preparation.text);

          # TODO(tlater): Add repo pruning/checking
          ExecStopPost =
            map (service: "+${mkRestartScript service}") backup.pauseServices
            ++ optional (backup.cleanup.text != null)
            (writeScript "backup-${name}-cleanup" backup.cleanup.packages backup.cleanup.text);
        };
      })
    config.services.backups;

    systemd.timers = lib.mapAttrs' (name: backup:
      lib.nameValuePair "backup-${name}" {
        wantedBy = ["timers.target"];
        timerConfig = {
          OnCalendar = "*-*-* 02:30:00 UTC";
          RandomizedDelaySec = "1h";
          FixedRandomDelay = true;
          Persistent = true;
        };
      })
    config.services.backups;

    users.groups.backup = {};

    systemd.tmpfiles.rules = [
      "d /var/lib/backups/ 0770 root backup"
    ];
  };
}