"""The backup-server role of the PBS driver: datastores, groups, jobs, remotes and the actions on them (#52).""" from __future__ import annotations import pytest from proxmoxer.core import ResourceException from napalm_proxmox import ProxmoxBackupDriver from tests.pbs_fake import FakePbsApi, forbidden # --------------------------------------------------------------------------- # # What the recorded server did not have yet, in the shapes of the PBS API schema # --------------------------------------------------------------------------- # T0 = 1793487600 # 2026-11-01 GROUPS_ROOT = [ { "backup-type": "vm", "backup-id": "101", "backup-count": 3, "last-backup": T0 + 2 * 86400, "owner": "backup@pbs", "files": ["drive-scsi0.img.fidx", "index.json.blob"], "comment": "web-01", }, { "backup-type": "ct", "backup-id": "200", "backup-count": 1, "last-backup": T0, "owner": "backup@pbs", "files": ["root.pxar.didx"], }, ] SNAPSHOTS_ROOT = [ { "backup-type": "vm", "backup-id": "101", "backup-time": T0, "size": 1000, "protected": False, "files": [], "verification": {"state": "ok", "upid": "UPID:pbs:1"}, }, { "backup-type": "vm", "backup-id": "101", "backup-time": T0 + 86400, "size": 1100, "protected": False, "files": [], "verification": {"state": "failed", "upid": "UPID:pbs:2"}, }, { "backup-type": "vm", "backup-id": "101", "backup-time": T0 + 2 * 86400, "size": 1200, "protected": True, "files": [], "comment": "before upgrade", }, { "backup-type": "ct", "backup-id": "200", "backup-time": T0, "protected": False, "files": [], }, ] GROUPS_PROD = [ {"backup-type": "host", "backup-id": "db-01", "backup-count": 0, "last-backup": T0, "files": []} ] GC = [ { "store": "backup", "schedule": "daily", "last-run-state": "OK", "last-run-endtime": T0 + 3600, "next-run": T0 + 86400, "upid": "UPID:pbs:gc", "disk-bytes": 0, "disk-chunks": 0, "index-data-bytes": 0, "index-file-count": 0, "pending-bytes": 0, "pending-chunks": 0, "removed-bad": 0, "removed-bytes": 0, "removed-chunks": 0, "still-bad": 0, } ] VERIFY = [ { "id": "v-weekly", "store": "backup", "schedule": "sat 03:00", "last-run-state": "WARNINGS: 2", "last-run-endtime": T0, "next-run": T0 + 7 * 86400, } ] PRUNE = [{"id": "p-daily", "store": "backup", "ns": "prod", "schedule": "daily", "disable": True}] SYNC = [ { "id": "offsite", "store": "backup", "remote": "pbs-offsite", "remote-store": "mirror", "sync-direction": "push", "schedule": "hourly", "last-run-state": "connection refused", "last-run-endtime": T0, "next-run": T0 + 3600, "comment": "second site", } ] TAPE = [ { "id": "t-monthly", "store": "backup", "pool": "monthly", "drive": "lto8", "schedule": "monthly", "last-run-upid": "UPID:pbs:tape", } ] REMOTES = [ { "name": "pbs-offsite", "host": "pbs2.example.com", "port": 8007, "auth-id": "sync@pbs", "fingerprint": "bb:" * 31 + "bb", "password": "never-reported", } ] ALL_RIGHTS = {"/": {"Datastore.Audit": True, "Sys.Audit": True, "Remote.Audit": True}} def _driver(**responses) -> tuple[ProxmoxBackupDriver, FakePbsApi]: drv = ProxmoxBackupDriver("pbs1", "root", "secret") api = FakePbsApi(**responses) drv._api = api drv._node_name = "localhost" return drv, api def _with_groups(**more): return _driver( **{ "admin/datastore/backup/namespace": [{"ns": ""}, {"ns": "prod"}], "admin/datastore/backup/groups": GROUPS_ROOT, "admin/datastore/backup/groups?ns=prod": GROUPS_PROD, "admin/datastore/backup/snapshots": SNAPSHOTS_ROOT, "admin/datastore/backup/snapshots?ns=prod": [], **more, } ) # --------------------------------------------------------------------------- # # Rights: a list the token may not see comes back empty, not refused # --------------------------------------------------------------------------- # class TestRights: def test_a_token_without_datastore_audit_is_refused_not_read_as_empty(self): """A privilege-separated token without an ACL entry gets [] for the datastore list; that is no answer, not a server without datastores.""" drv, _ = _driver(**{"access/permissions": {}, "admin/datastore": []}) with pytest.raises(PermissionError, match="Datastore.Audit"): drv.get_backup_datastores() def test_remotes_need_remote_audit(self): drv, _ = _driver(**{"config/remote": []}) # recorded rights lack Remote.Audit with pytest.raises(PermissionError, match="Remote.Audit"): drv.get_backup_remotes() def test_a_right_on_a_parent_path_counts(self): drv, _ = _driver(**{"access/permissions": ALL_RIGHTS, "config/remote": REMOTES}) assert [r["name"] for r in drv.get_backup_remotes()] == ["pbs-offsite"] def test_a_refused_path_raises(self): drv, _ = _driver(**{"status/datastore-usage": forbidden()}) with pytest.raises(Exception, match="403"): drv.get_backup_datastores() # --------------------------------------------------------------------------- # # Datastores (recorded) # --------------------------------------------------------------------------- # class TestDatastores: def test_recorded_removable_datastore(self): drv, _ = _driver() assert drv.get_backup_datastores() == [ { "name": "backup", # a removable datastore's path is relative to its mount point "path": None, "backend": "removable", "total": 999715438592, "used": 34838138880, "available": 964877299712, "estimated_full": None, "maintenance": None, "comment": None, } ] def test_s3_backend_estimate_and_maintenance(self): drv, _ = _driver( **{ "config/datastore": [ { "name": "cold", "path": "/var/cache/pbs/cold", "backend": "type=s3,client=minio,bucket=pbs", "maintenance-mode": "type=read-only,message=migrating", "comment": "object store", } ], "status/datastore-usage": [ { "store": "cold", "backend-type": "s3", "mount-status": "nonremovable", "total": 100, "used": 40, "avail": 60, "estimated-full-date": 1800000000, } ], } ) (store,) = drv.get_backup_datastores() assert (store["backend"], store["path"], store["estimated_full"]) == ( "s3", "/var/cache/pbs/cold", 1800000000, ) assert (store["maintenance"], store["comment"]) == ("read-only", "object store") def test_an_unmounted_removable_datastore_is_offline(self): drv, _ = _driver( **{ "status/datastore-usage": [ {"store": "backup", "backend-type": "filesystem", "mount-status": "notmounted"} ] } ) (store,) = drv.get_backup_datastores() assert (store["maintenance"], store["total"], store["used"]) == ("unmounted", None, None) # --------------------------------------------------------------------------- # # Groups and snapshots # --------------------------------------------------------------------------- # class TestGroups: def test_recorded_empty_datastore_has_no_groups(self): drv, _ = _driver( **{"admin/datastore/backup/groups": [], "admin/datastore/backup/snapshots": []} ) assert drv.get_backup_groups() == [] def test_each_group_with_its_newest_snapshot(self): drv, _ = _with_groups() groups = {(g["namespace"], g["type"], g["id"]): g for g in drv.get_backup_groups()} assert groups[("", "vm", "101")] == { "datastore": "backup", "namespace": "", "type": "vm", "id": "101", "owner": "backup@pbs", "count": 3, "newest": { "time": T0 + 2 * 86400, "size": 1200, "verify": None, # the newest was never verified "protected": True, "comment": "before upgrade", }, "oldest_time": T0, "comment": "web-01", } assert groups[("", "ct", "200")]["newest"]["size"] is None def test_namespaces_are_walked(self): drv, _ = _with_groups() prod = [g for g in drv.get_backup_groups() if g["namespace"] == "prod"] assert [(g["type"], g["id"], g["count"], g["newest"]) for g in prod] == [ ("host", "db-01", 0, None) ] def test_one_datastore_only(self): drv, api = _with_groups() drv.get_backup_groups("backup") assert all("/other/" not in path for _, path, _ in api.calls) def test_an_unmounted_removable_datastore_is_left_out_of_the_listing(self): """Recorded: the USB disk behind the datastore was unplugged. Its content cannot be read (400), which must not fail the listing of every datastore; get_backup_datastores reports it as "unmounted".""" not_mounted = ResourceException(400, "Bad Request", "datastore 'backup' is not mounted") drv, _ = _driver( **{ "admin/datastore": [ {"backend-type": "filesystem", "mount-status": "notmounted", "store": "backup"} ], "admin/datastore/backup/namespace": not_mounted, } ) assert drv.get_backup_groups() == [] with pytest.raises(ValueError, match="not mounted"): drv.get_backup_groups("backup") def test_snapshots_of_one_group_newest_first(self): drv, api = _with_groups( **{ "admin/datastore/backup/snapshots?backup-id=101&backup-type=vm": [ s for s in SNAPSHOTS_ROOT if s["backup-id"] == "101" ] } ) snaps = drv.get_backup_snapshots("backup", "", "vm", "101") assert [(s["time"], s["verify"]) for s in snaps] == [ (T0 + 2 * 86400, None), (T0 + 86400, "failed"), (T0, "ok"), ] def test_snapshots_in_a_namespace_name_it(self): drv, api = _with_groups( **{"admin/datastore/backup/snapshots?backup-id=db-01&backup-type=host&ns=prod": []} ) assert drv.get_backup_snapshots("backup", "prod", "host", "db-01") == [] # --------------------------------------------------------------------------- # # Jobs # --------------------------------------------------------------------------- # class TestJobs: def test_recorded_server_has_only_its_unscheduled_gc(self): drv, _ = _driver() assert drv.get_backup_jobs() == [ { "id": "backup", "kind": "gc", "datastore": "backup", "namespace": None, "schedule": None, "enabled": False, "last_run": None, "last_state": None, "last_error": None, "next_run": None, "remote": None, "direction": None, "comment": None, } ] def test_every_kind_and_how_its_last_run_ended(self): drv, _ = _driver( **{ "admin/gc": GC, "admin/verify": VERIFY, "admin/prune": PRUNE, "admin/sync": SYNC, "tape/backup": TAPE, } ) jobs = {(j["kind"], j["id"]): j for j in drv.get_backup_jobs()} assert set(jobs) == { ("gc", "backup"), ("verify", "v-weekly"), ("prune", "p-daily"), ("sync", "offsite"), ("tape", "t-monthly"), } gc = jobs[("gc", "backup")] assert (gc["enabled"], gc["last_state"], gc["last_run"], gc["next_run"]) == ( True, "ok", T0 + 3600, T0 + 86400, ) verify = jobs[("verify", "v-weekly")] assert (verify["last_state"], verify["last_error"]) == ("warning", "WARNINGS: 2") prune = jobs[("prune", "p-daily")] assert (prune["enabled"], prune["namespace"], prune["last_state"]) == (False, "prod", None) sync = jobs[("sync", "offsite")] assert (sync["last_state"], sync["last_error"]) == ("error", "connection refused") assert (sync["remote"], sync["direction"], sync["comment"]) == ( "pbs-offsite", "push", "second site", ) tape = jobs[("tape", "t-monthly")] assert (tape["last_state"], tape["last_run"]) == ("running", None) def test_a_sync_without_a_remote_is_local(self): local = [{**SYNC[0], "remote": None, "sync-direction": None, "id": "local"}] drv, _ = _driver( **{"admin/sync": local, "admin/verify": [], "admin/prune": [], "tape/backup": []} ) sync = next(j for j in drv.get_backup_jobs() if j["kind"] == "sync") assert (sync["remote"], sync["direction"]) == (None, "pull") def test_a_failed_listing_fails_the_whole_read(self): drv, _ = _driver(**{"admin/sync": forbidden()}) with pytest.raises(Exception, match="403"): drv.get_backup_jobs() # --------------------------------------------------------------------------- # # Remotes # --------------------------------------------------------------------------- # def test_remotes_never_carry_their_secret(): drv, _ = _driver(**{"access/permissions": ALL_RIGHTS, "config/remote": REMOTES}) assert drv.get_backup_remotes() == [ { "name": "pbs-offsite", "host": "pbs2.example.com", "port": 8007, "fingerprint": "bb:" * 31 + "bb", "auth_id": "sync@pbs", "comment": None, } ] # --------------------------------------------------------------------------- # # Actions and tasks # --------------------------------------------------------------------------- # SNAPSHOT = { "datastore": "backup", "namespace": "", "group_type": "vm", "group_id": "101", "time": T0, } class TestActions: def test_garbage_collection(self): drv, api = _driver(**{"POST admin/datastore/backup/gc": "UPID:pbs:gc-now"}) assert drv.start_garbage_collection("backup") == {"task": "UPID:pbs:gc-now"} @pytest.mark.parametrize( ("kind", "method", "path"), [ ("verify", "POST", "admin/verify/j1/run"), ("prune", "POST", "admin/prune/j1/run"), ("sync", "POST", "admin/sync/j1/run"), ("tape", "POST", "tape/backup/j1"), ("gc", "POST", "admin/datastore/j1/gc"), ], ) def test_run_a_job_now(self, kind, method, path): drv, api = _driver(**{f"{method} {path}": "UPID:pbs:run"}) assert drv.run_backup_job(kind, "j1") == {"task": "UPID:pbs:run"} assert api.calls[-1][:2] == (method, path) def test_an_unknown_kind_is_refused(self): drv, _ = _driver() with pytest.raises(ValueError, match="kind"): drv.run_backup_job("restore", "j1") # type: ignore[arg-type] def test_a_run_without_a_task_id_is_an_error(self): drv, _ = _driver(**{"POST admin/sync/j1/run": None}) with pytest.raises(RuntimeError, match="task"): drv.run_backup_job("sync", "j1") def test_verify_one_snapshot(self): drv, api = _driver(**{"POST admin/datastore/backup/verify": "UPID:pbs:verify"}) assert drv.verify_backup_snapshot(SNAPSHOT) == {"task": "UPID:pbs:verify"} assert api.calls[-1][2] == {"backup-type": "vm", "backup-id": "101", "backup-time": T0} def test_namespace_is_passed_where_given(self): drv, api = _driver(**{"POST admin/datastore/backup/verify": "UPID:pbs:verify"}) drv.verify_backup_snapshot({**SNAPSHOT, "namespace": "prod"}) assert api.calls[-1][2]["ns"] == "prod" def test_protect_and_unprotect(self): drv, api = _driver(**{"PUT admin/datastore/backup/protected": None}) drv.set_backup_snapshot_protected(SNAPSHOT, True) assert api.calls[-1] == ( "PUT", "admin/datastore/backup/protected", {"backup-type": "vm", "backup-id": "101", "backup-time": T0, "protected": 1}, ) drv.set_backup_snapshot_protected(SNAPSHOT, False) assert api.calls[-1][2]["protected"] == 0 def test_delete(self): drv, api = _driver(**{"DELETE admin/datastore/backup/snapshots": None}) drv.delete_backup_snapshot(SNAPSHOT) assert api.calls[-1] == ( "DELETE", "admin/datastore/backup/snapshots", {"backup-type": "vm", "backup-id": "101", "backup-time": T0}, ) class TestTasks: UPID = "UPID:pbs:000003E8:0000A000:00000002:6AE8E898:garbage_collection:backup:root@pam!netork:" def _task(self, status, log=()): return _driver( **{ f"nodes/localhost/tasks/{self.UPID}/status": status, f"nodes/localhost/tasks/{self.UPID}/log?limit=0": [ {"n": i + 1, "t": t} for i, t in enumerate(log) ], } )[0] def test_running(self): drv = self._task({"upid": self.UPID, "status": "running"}, ["starting garbage collection"]) assert drv.get_backup_task(self.UPID) == { "task": self.UPID, "running": True, "state": "running", "error": None, "log": ["starting garbage collection"], } @pytest.mark.parametrize( ("exitstatus", "state", "error"), [ ("OK", "ok", None), ("WARNINGS: 1", "warning", "WARNINGS: 1"), ("unable to acquire lock", "error", "unable to acquire lock"), ], ) def test_finished(self, exitstatus, state, error): drv = self._task({"upid": self.UPID, "status": "stopped", "exitstatus": exitstatus}) task = drv.get_backup_task(self.UPID) assert (task["running"], task["state"], task["error"]) == (False, state, error)