backups: Switch to hetzner storage box

configuration/services/backups.nix

@@ -24,6 +24,24 @@
       inherit name text;
       runtimeInputs = packages;
     });
+
+  # *NOT* a TOML file, for some reason quotes are interpreted
+  # *literally
+  rcloneConfig = pkgs.writeText "rclone.conf" ''
+    [storagebox]
+    type = sftp
+    user = u395933
+    host = u395933.your-storagebox.de
+    port = 23
+    key_file = ${config.sops.secrets."restic/storagebox-ssh-key".path}
+    shell_type = unix
+  '';
+
+  resticEnv = {
+    RESTIC_PASSWORD_FILE = config.sops.secrets."restic/storagebox-backups".path;
+    RESTIC_REPOSITORY = "rclone:storagebox:backups";
+    RCLONE_CONFIG = rcloneConfig;
+  };
 in {
   options = {
     services.backups = lib.mkOption {
@@ -120,13 +138,11 @@ in {
           # Doesn't hurt to finish the ongoing prune
           restartIfChanged = false;
 
-          environment = {
-            RESTIC_PASSWORD_FILE = config.sops.secrets."restic/local-backups".path;
-            RESTIC_REPOSITORY = "/var/lib/backups/";
-            RESTIC_CACHE_DIR = "%C/restic-prune";
-          };
+          environment = resticEnv;
 
           path = with pkgs; [
+            openssh
+            rclone
             restic
           ];
 
@@ -145,11 +161,6 @@ in {
 
             CacheDirectory = "restic-prune";
             CacheDirectoryMode = "0700";
-            ReadWritePaths = "/var/lib/backups/";
-
-            # Ensure we don't leave behind any files with the
-            # temporary UID of this service.
-            ExecStopPost = "+${pkgs.coreutils}/bin/chown -R root:backup /var/lib/backups/";
           };
         };
       }
@@ -158,17 +169,24 @@ in {
           # Don't want to restart mid-backup
           restartIfChanged = false;
 
-          environment = {
+          environment =
+            resticEnv
+            // {
               RESTIC_CACHE_DIR = "%C/backup-${name}";
-            RESTIC_PASSWORD_FILE = config.sops.secrets."restic/local-backups".path;
+            };
+
+          path = with pkgs; [
+            coreutils
+            openssh
+            rclone
+            restic
+          ];
+
           # TODO(tlater): If I ever add more than one repo, service
           # shutdown/restarting will potentially break if multiple
           # backups for the same service overlap. A more clever
           # sentinel file with reference counts would probably solve
           # this.
-            RESTIC_REPOSITORY = "/var/lib/backups/";
-          };
-
           serviceConfig = {
             User = backup.user;
             Group = "backup";
@@ -183,8 +201,8 @@ in {
 
             ExecStartPre =
               map (service: "+${mkShutdownScript service}") backup.pauseServices
-              ++ singleton (writeScript "backup-${name}-repo-init" [pkgs.restic pkgs.coreutils] ''
-                restic snapshots || (restic init && chmod -R g+rwx "$RESTIC_REPOSITORY"/*)
+              ++ singleton (writeScript "backup-${name}-repo-init" [] ''
+                restic snapshots || restic init
               '')
               ++ optional (backup.preparation.text != null)
               (writeScript "backup-${name}-prepare" backup.preparation.packages backup.preparation.text);
@@ -220,10 +238,14 @@ in {
         })
       config.services.backups;
 
-    users.groups.backup = {};
-
-    systemd.tmpfiles.rules = [
-      "d /var/lib/backups/ 0770 root backup"
-    ];
+    users = {
+      # This user is only used to own the ssh key, because apparently
+      # the ssh client checks file permissions and is stuck in 1980.
+      users.backup = {
+        group = "backup";
+        isSystemUser = true;
+      };
+      groups.backup = {};
+    };
   };
 }

configuration/sops.nix

@@ -36,6 +36,16 @@
         group = "backup";
         mode = "0440";
       };
+      "restic/storagebox-backups" = {
+        owner = "root";
+        group = "backup";
+        mode = "0440";
+      };
+      "restic/storagebox-ssh-key" = {
+        owner = "backup";
+        group = "backup";
+        mode = "0040";
+      };
 
       # Steam
       "steam/tlater" = {};

flake.lock

@@ -354,11 +354,11 @@
         "nixpkgs-stable": "nixpkgs-stable"
       },
       "locked": {
-        "lastModified": 1710039806,
-        "narHash": "sha256-vC2fo/phnetp6ub/nRv6mgAi5LbhJ6ujGQWrRD2VgNs=",
+        "lastModified": 1710433464,
+        "narHash": "sha256-IXlPoWgIRovZ32mYvqqdBgOQln71LouE/HBhbKc1wcw=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "f8d5c8baa83fe620a28c0db633be9db3e34474b4",
+        "rev": "6c32d3b9c7593f4b466ec5404e59fc09a803a090",
         "type": "github"
       },
       "original": {

keys/production.yaml

@@ -14,6 +14,8 @@ wireguard:
     server-key: ENC[AES256_GCM,data:mXb7ZznJHf5CgV8rI4uzPBATMRbmd7LimgtCkQM9kAjbIaGwUBqJZBN3fXs=,iv:3Po1Orinzov9rnEm9cLzgJY1PeD+5Jl9115MriABHh8=,tag:E/2CjDO1JCvJzxCnqKcNyw==,type:str]
 restic:
     local-backups: ENC[AES256_GCM,data:NLNVlR9G9bLSZOkMoPvkbBbAZlKkmiUbdWHOFDnaefuy9wNLH53ctOIyS0rSsQLaJCSBTpgPSWIIXUSuzoK/eA==,iv:DzuujmyJJP4GiE5z7KOOGUEzUgOwmtf/7UYhwkyLe9g=,tag:cElFhpVC7S6HYlB6UyN7PQ==,type:str]
+    storagebox-backups: ENC[AES256_GCM,data:UyT8jCkKlfYJXjWLI9MbYfeVhY5d89N3aj1Olj54/aBOP3gwcrx6gU56Pwa1xKZ3lR13AVs/b4wF9sbvP7Kqqg==,iv:0HM+DgH4iCiWpjRvAYCFQGEy4xIBQwAM+PkkzOsizw0=,tag:jbrqo1In2O4jVM5e7fjOzg==,type:str]
+    storagebox-ssh-key: ENC[AES256_GCM,data:7aYlKX7I8Bsur3nm4nV9eSW3lmIxBCeCUMbPX3qgcotPbyPYaUqD3MOCnFRepajYkFXAgMX4jknqLfoO9xYc4bavDFjOY8Ww/KmLay7ces6tDnkK6tTRxcNRPUBqEzaFiPNsZc2UwHnmHOF0rKvQusvhCOplYao3xxz5McTHC7IEriUApSNudWCg3qGbyAmxkGEw7tRfh6IiUXEOFeaXDZd78dWZlWIIeospmA1hcVhkLGrjMmoikt/YANHUpWPbd+B9E6x+s2eIzFdvztRjarBluWPZuX981b+hcOm1/+HY/tJ/jzgyVbX1rjmdgZ9jZqdKO/vkOkijHWXlwpQ0QJ2s8p5MURPGRsC7W5jfmGbVKrubxfQC2mSJRJgBaj1wX3yI4GbfCXNdbpseAMy7t8OmN/iMN57lGnD3uX8CWWD327PIWp3SgwMDIZtJRlMBu3vMUrBdNnrrlYoLgf821tX7JWW6L5g1EK/bcBZqZZ/6rE+Q9fLiJHTsj2lyTzQZRLKsn0YePlcIMOWHO2CG/aWrfycdSKKjaKGG,iv:OVnEIMFB4h/EQ8zV3XOpVXLDrV5t4roNYDFQz99m4sQ=,tag:mKWF12uD1TLla/MoJs2zNg==,type:str]
 turn:
     env: ENC[AES256_GCM,data:kt5nhVo9pb/ZbPUEcqSYXxN9YMgQKnFb5VRfFFS/qoIaJ73uD2fuJKqcxAyVRrdLqnSAWSQBgTgunBzdP7xqLAK2qt8DYAQWHkIe9uxFbSXZpdmw,iv:9lq6SFwTFN4GGm6gPiJpUMasMdnHVF6XLGYrsyG3kjU=,tag:428Qf9DOiiHt/Wjb188b8g==,type:str]
     secret: ENC[AES256_GCM,data:si7ee6Xfhdgdyzbp6aQpF7pz3TmTBb7iQ82lRPVXNDg9JfHI+lbmgAsSnRLX5qMCA6P9R045sSMosqidL8QwRg==,iv:SrhpZKK8D45yxCEfDb9P3TwtA14+qEI+wcRqcN/a6pw=,tag:PiwV+mOL9xHJgJft6sc61g==,type:str]
@@ -26,67 +28,62 @@ sops:
     azure_kv: []
     hc_vault: []
     age: []
-    lastmodified: "2023-12-28T00:07:08Z"
-    mac: ENC[AES256_GCM,data:P2bNJLjzn69Kg2bJHXmofER7J8wbEj9C4jq9ePWewXBOt45GEiqgnqIaISwZkyzQmm9cxZd95Lr780ICwoKDFdtSCCcC7CdYxYEfyyhnvU3W2qzEghvkypL8JbiEtPSlQ9xOlCk7p41A9eRrV+JziIVSv5UEUs4NubrG9Mkwv3k=,iv:Yq2gANTTgx6cFxkdustUZ1MPszxGSkao/bS1KHAkzJc=,tag:kqJibocgRQXkxTJze6O5MA==,type:str]
+    lastmodified: "2024-03-18T04:05:08Z"
+    mac: ENC[AES256_GCM,data:/v/XRBizRUn8UR1HOIaKrY11/5I8RmsctmxXg0wcCXO983YgcGYTEqfXOSRJzkh/MOjB0c8jpdfE7tyqYgB2iYjm+HmRFVsGNB484e+3ukAMsxz4CTz26RWC/LEemQeBOa5RB3rrPXXo1LrJCBN9Y+T/PVgO2xshUSxyhRFQmMA=,iv:km1m8k/vrCVT8ugvqAZNORLu0NSW66B58btH5k4e//Y=,tag:RZXviobM03IQXN1FwDUlVA==,type:str]
     pgp:
-        - created_at: "2024-03-02T21:16:50Z"
+        - created_at: "2024-03-18T04:02:00Z"
           enc: |-
             -----BEGIN PGP MESSAGE-----
 
-            hQIMAzWu0p84AOApARAAi+GxJ9z+cMaMgENnDC0Kq6ZJZ/rkXnUIjVxpdXLVhnCc
-            E2S8NoXJI5jcqsYI08wVQm7OWzsNK6GuJET1i3YdHVDOiwYK+WNGeMA6JdIuJzXV
-            EDcuarLusygqIV1UcZCwTl362zuLi5kPs/fGsn7BJeI8Q7CtMEP1cmCk0LlHotjz
-            Pl53bUos1WUqSv0EQw9Cz1dhL6LGlUtoIJaPbB9OO/+chzQCFUJGbCO5KJ/+3fFq
-            2DhQZw1GvgNf9/66f39tgY+jeQq5OyuoFSpuzyjxCeK+eX6Jkxs4zOVlcJoztSVc
-            FEiPIO4YfcgDXToLJWSWA2uGJ+KCvqDXDWyPATQupytAItw05oFyfZOPuh45Wj46
-            6Dm9QYKZMsFj6xfgNl6VEK0KK34zi0EcBKm4wmfF8hw4o5T2U542iPzgKv53jbC2
-            F1dn7GI8ZkSGDPlw7UWSIRLmRYilZhbR+2RJX23nXoarP9oxigCpqhIGBGizdBEx
-            PpUYQjiPUuytk/B3DP+0q01lVvdqcxchA3s88iZwc5GSwBfEMVJ2MJOFkiwIkttO
-            9PkmtXAaFAt7jjRCzhH05/S7g9xt/1zid/lHCGKcfaZJqX6YIu9+mXeERsZ7OdMs
-            uur8T7r14DC4ffPOYQR6BIfNZ3vPUyEP2/fSncAtyDFKO2Cc6ry3JvxBCdPGErjS
-            XgFwk6xHtOsIU3ozokW3aupo5eSNBEPpfIK28P0ivouIZsU64sVJFjc7zPpZnaF+
-            bEnAXMK8FrHvYZz3v4+LSaYZyoKWYly0wCWrSOZTEphTJHFrW/KsJ2hmVTpjS58=
-            =qqF7
+            hQEMA7x7stsXx45CAQf7BjF+HR3WKdMyAV6R1M0+lqDz6hBHKyGH7YBB/QZBqRbK
+            3hdABIwWUsqpHjleEOp/Gj0VhZqwagqHxK4Fp5G0r3QBupbAO8u/+DNI8wll0Nva
+            dlOh0Jqp4E17TkERMQL02rrQ1ZmpOYmPkCd2//xkmWAQ1LatHWeRVSRxQBuMtPQi
+            btrefcQNjQCvS9/60dp8oTu8nxlFA4iHCBQKNIKVGqQH7jkdIfMPdUILjCkCiyCc
+            h+OxlHZZnpU6U9A+hjMBinvCzebSkZh48VX/T33Kr+4b0CBr1gR9MSXKG9f2MPQP
+            PMl6rPvqSqG6ddN9QDI+0HEHYaRvxPIV8uDS36tVxNJeAQHB5/6Lt7hJdYWgwf5E
+            TLgbZ0IxB17++6K++GlaG8WHO65l1jzmkPlN+ZGcwnhibDxnZjP6kqGqDFcZP4ge
+            cnV0KnhYcC59IooQYrWKzAJex9rnwPo7MGKV6XwZOQ==
+            =Hy9T
             -----END PGP MESSAGE-----
           fp: 535B61015823443941C744DD12264F6BBDFABA89
-        - created_at: "2024-03-02T21:16:50Z"
+        - created_at: "2024-03-18T04:02:00Z"
           enc: |-
             -----BEGIN PGP MESSAGE-----
 
-            hQIMA9ahl2ynTH87ARAApU/UkNVGbtqxwQ83Zl3f7Zp/PTIeLtcvmuOUjSnPYrYi
-            60H1ZPVJUhAv+gcTwRBZ+aN39mUI43qBgCjNu7Z7Bmevf+TXCvK1CwsxuxVbG1tl
-            sL8FtVH0p8KETq+v8aylTzaV339BmEgnLOBLCE9oP+PhLEERqIT1sz5CeaI71z4F
-            wETPCfJKEouCQpT0P6hSN1f/9h43PZDQQW5MLY2m1o8t+pFHfowADIlsAmZziXBf
-            t/IezzM7oo/QKITpLI8NND9nZfvG7leubG3L2TIL0xIgQeLBs4a+jfFSpt8DR0ii
-            YGf1RgrtpnlkA4B75KHTfEq1LMEn0wOJj89Z38x5MZEw3suUc8W+1PcKoKIgt4Dw
-            RN4K+CS/4Ud8pNLoO+zZ4moRlM9ltWpCJ9kSHNeMShxtsIEPxkhh3CqWU+Ta/4er
-            1W2bkII2ieS4mLlJM6qqLYAb8VJpaKi3BQmB66KtDS4n4HEXvOO+nurmz9luKZZt
-            1e3t8ABBowOu+LOVxUbx9DKFObBJ1CDDPQHxRDmGxeSz3ZccHlXsC83QSHCtcm8G
-            uFtUZLOCaR0iB7DbEUX43p40xFZ5ieqY9XDC3uGJfzoEZRfaX05I3MX267EZBKSp
-            H6kyYPnTBqI0UhIsDtd6AWd9huqOZ/TrWubTeDf07s6VDusMYrtE+WaVczaYUkPS
-            WAHYUCmSFUN5z3Emds26kMUQvWTKMvx8TgaEf9LwOfjo4LXhvNKjU5yi+hqZqlO8
-            AOvcgnksjHUhonEl7GLaOvPPiyoB6F6ZuOFlzOeL1OB3QxJiEoRFbF8=
-            =574h
+            hQIMA9ahl2ynTH87AQ/+LNXxC3acjs2+c38gHZRW6Am4XFx1t/4tfxIgaaK/Boq8
+            PGU5CFNOMDGv8u/cwyDbfNM7GuL5g7vrLmBXzSV5ErZqc6bJ0+ZCNPTRIxP1Vxph
+            tWiDIyTwuqUzxWpOlSzii2Sqhlp8CyiWzBe95eIr96XzDCCtfzyCZ0BYyKgpHHxB
+            BltH0/+0JZFiR19zvf6M99AHwM8OddRQkXav+mRIJQpA87ovVZcAv5skYGJgNCqN
+            55fbskuYmqEnloQCZVJ2+2ZXK5Qn/uq4fLJCiIdZm4YsctJnV5spzZIL+dcOty65
+            Plk77BWzLaU5UOKCBAJWrK8oZSTGOrp4VZqb62DuqMRejG0JXmneIVk7p79yn5eA
+            ANVMGRF4b8RP9YUhzE8HACFzQebKpUU8XKv9+qsmO9Le5jUhU3UQeCSSzT/T5Dr8
+            kLDNtmW+mliQnxFlKcVWq2JIG+HaQD1KLOAl0JBNCOSLif2ofaHahuZ15agbYeis
+            hyrBY92EhzqYXHk/Kzv4ff4r+WUs9NN7R4Gg+wfWvMcTtVfbi4Ht+pjjTtCZwK1C
+            M8JebQn0NZSpVi3e7Xaz1fQ5Tqrg8PHZtkYGoIHLRPJQwLn9PHYtGzC3rFAk+Fqq
+            5WWHELxfcsZ6DakAGSXPK/80QhEZkpGmKizTwrEde+7fpEPxjdzUqlmH3rv7mFzS
+            WAGSiBIMjLR6ofb65vpghbwh6gXkpCtgUyINRhx/D+Kj5Z4lGD1u1I05DT1xD6VJ
+            FAbnH7oZ3PJecoAXgRT05FndFA1xfPMCkugmec8ML/sEZt+c3kbrXaA=
+            =MqS3
             -----END PGP MESSAGE-----
           fp: 8a3737d48f1035fe6c3a0a8fd6a1976ca74c7f3b
-        - created_at: "2024-03-02T21:16:50Z"
+        - created_at: "2024-03-18T04:02:00Z"
           enc: |-
             -----BEGIN PGP MESSAGE-----
 
-            hQIMA0f3HaPlrXn5AQ/8Cvb9YhG/wYRhu+B3iSTCOq1xiPbCOPs9BcCg85G2yI+5
-            LA8G18XVDpaxlT4lyOE3p4XEbJkP+ceLhNbldiQns9HCDQXanRonZndLjwSdEuRj
-            /A/ql2Q27Xhad34Bu0n+hoNfQ2qKjjx6q5lbbKLIIGOvEBF35oImnWF+Vc2cYpzp
-            J0PT/gkKkGnBCihiUrmpISU+7grFMFT59UnWSthCpACG8ocjzF0PBdzPOj2QSDiv
-            eDiPEdd72KcGXVfRodrdAbApFXJx2goaxYobAFCyC7G3UHJTliCOEG/5PNSb8lSl
-            Xv8NJnYI7bs8bRMFTvpEIsogrVeXy0yDl+qogQWPKYwpStn6yqOMIvs2C476nY/f
-            llRLfjJLTEmPuq+JYhWfZ4o0tOZNECmq4DiAg30ePqThZNXJLNyk9sfkjuDz+zbh
-            rYnJ1Xb1UM7ZKyjGcxSU9eAba0MBJpVZa/ZDrb4GjysPq+rsEb8LO6WPPbYfLbr/
-            kfiK7e4Rv4AgUdd7NjRwBHJSjIFCul8I2hF4v/vp+da11CktPXC0sJNsYXWBR1I+
-            FeKxc+WkLTfuS6evb8Y+UuyQkTDI3mb13QfXaX1V8I63LivdCE7zsTOlnOWPT1k7
-            cqhQ2VpNxBtt7gNG7MAYHn9KAwGbyQ/Ma6Qx//ftjmf47b8qnZuJe8HEg0Nh5uDS
-            WAGdbEL/ZXTT4ZxNm/QHVctZVzCAqDUMIkMK4vCCR+Bs8FvLFUo6YoVEnajqTSj8
-            pkEyS0RuM68KTpivAjDhqlY4vJsMmiRBjx/q5rSwi29vOuhK9ttSj38=
-            =KQd3
+            hQIMA0f3HaPlrXn5AQ/+N5NK5UJdtw5e7O9T4hfIhtMXci/og1cJiI64daSyNeDH
+            jq+CPJ8e73yiTPwu6wHHqfuEhlEuI6sJY0ZJVFU+h4SIBtG21veGEz7GzlYgBCJm
+            xvJHXjtM8AprqnFVO7Fj9QA/ik5QBP6ZpkOY7j8/qf1G7alOIne/MYRALXDvvIMH
+            HTWE+Y2N57yZK55Pokmdw10hawbrn/N1nt2Y7sa1+5TlRNtuA/+zLkXtEjRr5U3N
+            DId+hqCKgXDqKLBMkh4mZUTGOGsk8eeKAWwyPp9+8A5/0rfy+xOJYEjjHICXQMSE
+            zfe6qvj/fRJKGzT5lEzD+ZKHlR0zHEwGRfHqrVUTdPcPdKj3DZILjsoe5ba4VlAp
+            sS0CAYTg3YuWMT4iHuOQlY5IoQxHHrn7k8ox5iZULFecg58f6r6iJL3AepDYWAey
+            gtQXYBeaeCm5Ddwmd6TBVz8Q4bCVYIrHbVeAhSDkxfrWLc5UORggvLEWiXilGDJi
+            DzAv0MVHE2Wa3eOJLq05K2/LBqRBD1XYM3dcS6JSdFxWWMzvLdUOB4dAuPt9gpl9
+            liaA13Blw/ev+U4ADxptrl+QuYRbWz3z6rniYpluSrTbVCKFRoHXSGFPy5u8/N6O
+            QyjfoovIBxXKnbUq2kMoFa/qFpc1pDUn0sjQNsUBdtorAu3Up4icyoih7qwx2J3S
+            WAGB1jHWMfcsBJqPwjRYkqBf6MuwHAZWdd+zvj/fKfft9jtxLcCGOIM6QdfiWbl0
+            Wq4gHdH7OhSy+ZgRnaBRt/GAkzkHvfG68HfulviHZ1h2mrQN1y3mxpg=
+            =RCYB
             -----END PGP MESSAGE-----
           fp: 0af7641adb8aa843136cf6d047f71da3e5ad79f9
     unencrypted_suffix: _unencrypted