driver / lint (pull_request) Successful in 14s
driver / standard (pull_request) Successful in 16s
driver / types (pull_request) Successful in 1m5s
driver / test (3.13) (pull_request) Successful in 1m4s
driver / test (3.12) (pull_request) Successful in 1m10s
driver / lowest (pull_request) Successful in 31s
driver / build (pull_request) Successful in 30s
driver / security (pull_request) Successful in 1m0s
CI / driver (pull_request) Successful in 2m20s
driver / publish (pull_request) Skipped
A second driver in this package, ProxmoxBackupDriver. Its API and web UI are on 8007, not on PVE's 8006, and its node is always localhost; open() proves the login with GET /version, so a bad token fails there. Roles: BackupServerDriver (napalm-device-types 4.25.0) first, then StorageDriver for the disks and ZFS pools the server manages. It reads datastores, the backup groups of every namespace with their newest snapshot, a group's snapshots, the GC, verify, prune, sync and tape jobs with how their last run ended, and the remotes without their secrets; it starts garbage collection and jobs, verifies, protects and deletes snapshots, and follows the tasks. As a host it reports facts, interfaces, CPU and memory, disks, pools, its pbs-subscription and the same packages, updates, host readers and sshd hardening as a PVE node. A privilege-separated token without an ACL entry gets an empty list from PBS, not a refusal, so every read checks /access/permissions and raises PermissionError. A removable datastore that is not mounted is reported as unmounted and left out of the group listing. The fixture is what a PBS 4.2 answered, sanitised; groups, snapshots and jobs it did not have yet follow the API schema.
554 lines
19 KiB
Python
554 lines
19 KiB
Python
"""The backup-server role of the PBS driver: datastores, groups, jobs, remotes and
|
|
the actions on them (#52)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from proxmoxer.core import ResourceException
|
|
|
|
from napalm_proxmox import ProxmoxBackupDriver
|
|
from tests.pbs_fake import FakePbsApi, forbidden
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# What the recorded server did not have yet, in the shapes of the PBS API schema
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
T0 = 1793487600 # 2026-11-01
|
|
|
|
GROUPS_ROOT = [
|
|
{
|
|
"backup-type": "vm",
|
|
"backup-id": "101",
|
|
"backup-count": 3,
|
|
"last-backup": T0 + 2 * 86400,
|
|
"owner": "backup@pbs",
|
|
"files": ["drive-scsi0.img.fidx", "index.json.blob"],
|
|
"comment": "web-01",
|
|
},
|
|
{
|
|
"backup-type": "ct",
|
|
"backup-id": "200",
|
|
"backup-count": 1,
|
|
"last-backup": T0,
|
|
"owner": "backup@pbs",
|
|
"files": ["root.pxar.didx"],
|
|
},
|
|
]
|
|
SNAPSHOTS_ROOT = [
|
|
{
|
|
"backup-type": "vm",
|
|
"backup-id": "101",
|
|
"backup-time": T0,
|
|
"size": 1000,
|
|
"protected": False,
|
|
"files": [],
|
|
"verification": {"state": "ok", "upid": "UPID:pbs:1"},
|
|
},
|
|
{
|
|
"backup-type": "vm",
|
|
"backup-id": "101",
|
|
"backup-time": T0 + 86400,
|
|
"size": 1100,
|
|
"protected": False,
|
|
"files": [],
|
|
"verification": {"state": "failed", "upid": "UPID:pbs:2"},
|
|
},
|
|
{
|
|
"backup-type": "vm",
|
|
"backup-id": "101",
|
|
"backup-time": T0 + 2 * 86400,
|
|
"size": 1200,
|
|
"protected": True,
|
|
"files": [],
|
|
"comment": "before upgrade",
|
|
},
|
|
{
|
|
"backup-type": "ct",
|
|
"backup-id": "200",
|
|
"backup-time": T0,
|
|
"protected": False,
|
|
"files": [],
|
|
},
|
|
]
|
|
GROUPS_PROD = [
|
|
{"backup-type": "host", "backup-id": "db-01", "backup-count": 0, "last-backup": T0, "files": []}
|
|
]
|
|
|
|
GC = [
|
|
{
|
|
"store": "backup",
|
|
"schedule": "daily",
|
|
"last-run-state": "OK",
|
|
"last-run-endtime": T0 + 3600,
|
|
"next-run": T0 + 86400,
|
|
"upid": "UPID:pbs:gc",
|
|
"disk-bytes": 0,
|
|
"disk-chunks": 0,
|
|
"index-data-bytes": 0,
|
|
"index-file-count": 0,
|
|
"pending-bytes": 0,
|
|
"pending-chunks": 0,
|
|
"removed-bad": 0,
|
|
"removed-bytes": 0,
|
|
"removed-chunks": 0,
|
|
"still-bad": 0,
|
|
}
|
|
]
|
|
VERIFY = [
|
|
{
|
|
"id": "v-weekly",
|
|
"store": "backup",
|
|
"schedule": "sat 03:00",
|
|
"last-run-state": "WARNINGS: 2",
|
|
"last-run-endtime": T0,
|
|
"next-run": T0 + 7 * 86400,
|
|
}
|
|
]
|
|
PRUNE = [{"id": "p-daily", "store": "backup", "ns": "prod", "schedule": "daily", "disable": True}]
|
|
SYNC = [
|
|
{
|
|
"id": "offsite",
|
|
"store": "backup",
|
|
"remote": "pbs-offsite",
|
|
"remote-store": "mirror",
|
|
"sync-direction": "push",
|
|
"schedule": "hourly",
|
|
"last-run-state": "connection refused",
|
|
"last-run-endtime": T0,
|
|
"next-run": T0 + 3600,
|
|
"comment": "second site",
|
|
}
|
|
]
|
|
TAPE = [
|
|
{
|
|
"id": "t-monthly",
|
|
"store": "backup",
|
|
"pool": "monthly",
|
|
"drive": "lto8",
|
|
"schedule": "monthly",
|
|
"last-run-upid": "UPID:pbs:tape",
|
|
}
|
|
]
|
|
REMOTES = [
|
|
{
|
|
"name": "pbs-offsite",
|
|
"host": "pbs2.example.com",
|
|
"port": 8007,
|
|
"auth-id": "sync@pbs",
|
|
"fingerprint": "bb:" * 31 + "bb",
|
|
"password": "never-reported",
|
|
}
|
|
]
|
|
ALL_RIGHTS = {"/": {"Datastore.Audit": True, "Sys.Audit": True, "Remote.Audit": True}}
|
|
|
|
|
|
def _driver(**responses) -> tuple[ProxmoxBackupDriver, FakePbsApi]:
|
|
drv = ProxmoxBackupDriver("pbs1", "root", "secret")
|
|
api = FakePbsApi(**responses)
|
|
drv._api = api
|
|
drv._node_name = "localhost"
|
|
return drv, api
|
|
|
|
|
|
def _with_groups(**more):
|
|
return _driver(
|
|
**{
|
|
"admin/datastore/backup/namespace": [{"ns": ""}, {"ns": "prod"}],
|
|
"admin/datastore/backup/groups": GROUPS_ROOT,
|
|
"admin/datastore/backup/groups?ns=prod": GROUPS_PROD,
|
|
"admin/datastore/backup/snapshots": SNAPSHOTS_ROOT,
|
|
"admin/datastore/backup/snapshots?ns=prod": [],
|
|
**more,
|
|
}
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Rights: a list the token may not see comes back empty, not refused
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
class TestRights:
|
|
def test_a_token_without_datastore_audit_is_refused_not_read_as_empty(self):
|
|
"""A privilege-separated token without an ACL entry gets [] for the
|
|
datastore list; that is no answer, not a server without datastores."""
|
|
drv, _ = _driver(**{"access/permissions": {}, "admin/datastore": []})
|
|
with pytest.raises(PermissionError, match="Datastore.Audit"):
|
|
drv.get_backup_datastores()
|
|
|
|
def test_remotes_need_remote_audit(self):
|
|
drv, _ = _driver(**{"config/remote": []}) # recorded rights lack Remote.Audit
|
|
with pytest.raises(PermissionError, match="Remote.Audit"):
|
|
drv.get_backup_remotes()
|
|
|
|
def test_a_right_on_a_parent_path_counts(self):
|
|
drv, _ = _driver(**{"access/permissions": ALL_RIGHTS, "config/remote": REMOTES})
|
|
assert [r["name"] for r in drv.get_backup_remotes()] == ["pbs-offsite"]
|
|
|
|
def test_a_refused_path_raises(self):
|
|
drv, _ = _driver(**{"status/datastore-usage": forbidden()})
|
|
with pytest.raises(Exception, match="403"):
|
|
drv.get_backup_datastores()
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Datastores (recorded)
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
class TestDatastores:
|
|
def test_recorded_removable_datastore(self):
|
|
drv, _ = _driver()
|
|
assert drv.get_backup_datastores() == [
|
|
{
|
|
"name": "backup",
|
|
# a removable datastore's path is relative to its mount point
|
|
"path": None,
|
|
"backend": "removable",
|
|
"total": 999715438592,
|
|
"used": 34838138880,
|
|
"available": 964877299712,
|
|
"estimated_full": None,
|
|
"maintenance": None,
|
|
"comment": None,
|
|
}
|
|
]
|
|
|
|
def test_s3_backend_estimate_and_maintenance(self):
|
|
drv, _ = _driver(
|
|
**{
|
|
"config/datastore": [
|
|
{
|
|
"name": "cold",
|
|
"path": "/var/cache/pbs/cold",
|
|
"backend": "type=s3,client=minio,bucket=pbs",
|
|
"maintenance-mode": "type=read-only,message=migrating",
|
|
"comment": "object store",
|
|
}
|
|
],
|
|
"status/datastore-usage": [
|
|
{
|
|
"store": "cold",
|
|
"backend-type": "s3",
|
|
"mount-status": "nonremovable",
|
|
"total": 100,
|
|
"used": 40,
|
|
"avail": 60,
|
|
"estimated-full-date": 1800000000,
|
|
}
|
|
],
|
|
}
|
|
)
|
|
(store,) = drv.get_backup_datastores()
|
|
assert (store["backend"], store["path"], store["estimated_full"]) == (
|
|
"s3",
|
|
"/var/cache/pbs/cold",
|
|
1800000000,
|
|
)
|
|
assert (store["maintenance"], store["comment"]) == ("read-only", "object store")
|
|
|
|
def test_an_unmounted_removable_datastore_is_offline(self):
|
|
drv, _ = _driver(
|
|
**{
|
|
"status/datastore-usage": [
|
|
{"store": "backup", "backend-type": "filesystem", "mount-status": "notmounted"}
|
|
]
|
|
}
|
|
)
|
|
(store,) = drv.get_backup_datastores()
|
|
assert (store["maintenance"], store["total"], store["used"]) == ("unmounted", None, None)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Groups and snapshots
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
class TestGroups:
|
|
def test_recorded_empty_datastore_has_no_groups(self):
|
|
drv, _ = _driver(
|
|
**{"admin/datastore/backup/groups": [], "admin/datastore/backup/snapshots": []}
|
|
)
|
|
assert drv.get_backup_groups() == []
|
|
|
|
def test_each_group_with_its_newest_snapshot(self):
|
|
drv, _ = _with_groups()
|
|
groups = {(g["namespace"], g["type"], g["id"]): g for g in drv.get_backup_groups()}
|
|
assert groups[("", "vm", "101")] == {
|
|
"datastore": "backup",
|
|
"namespace": "",
|
|
"type": "vm",
|
|
"id": "101",
|
|
"owner": "backup@pbs",
|
|
"count": 3,
|
|
"newest": {
|
|
"time": T0 + 2 * 86400,
|
|
"size": 1200,
|
|
"verify": None, # the newest was never verified
|
|
"protected": True,
|
|
"comment": "before upgrade",
|
|
},
|
|
"oldest_time": T0,
|
|
"comment": "web-01",
|
|
}
|
|
assert groups[("", "ct", "200")]["newest"]["size"] is None
|
|
|
|
def test_namespaces_are_walked(self):
|
|
drv, _ = _with_groups()
|
|
prod = [g for g in drv.get_backup_groups() if g["namespace"] == "prod"]
|
|
assert [(g["type"], g["id"], g["count"], g["newest"]) for g in prod] == [
|
|
("host", "db-01", 0, None)
|
|
]
|
|
|
|
def test_one_datastore_only(self):
|
|
drv, api = _with_groups()
|
|
drv.get_backup_groups("backup")
|
|
assert all("/other/" not in path for _, path, _ in api.calls)
|
|
|
|
def test_an_unmounted_removable_datastore_is_left_out_of_the_listing(self):
|
|
"""Recorded: the USB disk behind the datastore was unplugged. Its content
|
|
cannot be read (400), which must not fail the listing of every datastore;
|
|
get_backup_datastores reports it as "unmounted"."""
|
|
not_mounted = ResourceException(400, "Bad Request", "datastore 'backup' is not mounted")
|
|
drv, _ = _driver(
|
|
**{
|
|
"admin/datastore": [
|
|
{"backend-type": "filesystem", "mount-status": "notmounted", "store": "backup"}
|
|
],
|
|
"admin/datastore/backup/namespace": not_mounted,
|
|
}
|
|
)
|
|
assert drv.get_backup_groups() == []
|
|
with pytest.raises(ValueError, match="not mounted"):
|
|
drv.get_backup_groups("backup")
|
|
|
|
def test_snapshots_of_one_group_newest_first(self):
|
|
drv, api = _with_groups(
|
|
**{
|
|
"admin/datastore/backup/snapshots?backup-id=101&backup-type=vm": [
|
|
s for s in SNAPSHOTS_ROOT if s["backup-id"] == "101"
|
|
]
|
|
}
|
|
)
|
|
snaps = drv.get_backup_snapshots("backup", "", "vm", "101")
|
|
assert [(s["time"], s["verify"]) for s in snaps] == [
|
|
(T0 + 2 * 86400, None),
|
|
(T0 + 86400, "failed"),
|
|
(T0, "ok"),
|
|
]
|
|
|
|
def test_snapshots_in_a_namespace_name_it(self):
|
|
drv, api = _with_groups(
|
|
**{"admin/datastore/backup/snapshots?backup-id=db-01&backup-type=host&ns=prod": []}
|
|
)
|
|
assert drv.get_backup_snapshots("backup", "prod", "host", "db-01") == []
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Jobs
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
class TestJobs:
|
|
def test_recorded_server_has_only_its_unscheduled_gc(self):
|
|
drv, _ = _driver()
|
|
assert drv.get_backup_jobs() == [
|
|
{
|
|
"id": "backup",
|
|
"kind": "gc",
|
|
"datastore": "backup",
|
|
"namespace": None,
|
|
"schedule": None,
|
|
"enabled": False,
|
|
"last_run": None,
|
|
"last_state": None,
|
|
"last_error": None,
|
|
"next_run": None,
|
|
"remote": None,
|
|
"direction": None,
|
|
"comment": None,
|
|
}
|
|
]
|
|
|
|
def test_every_kind_and_how_its_last_run_ended(self):
|
|
drv, _ = _driver(
|
|
**{
|
|
"admin/gc": GC,
|
|
"admin/verify": VERIFY,
|
|
"admin/prune": PRUNE,
|
|
"admin/sync": SYNC,
|
|
"tape/backup": TAPE,
|
|
}
|
|
)
|
|
jobs = {(j["kind"], j["id"]): j for j in drv.get_backup_jobs()}
|
|
assert set(jobs) == {
|
|
("gc", "backup"),
|
|
("verify", "v-weekly"),
|
|
("prune", "p-daily"),
|
|
("sync", "offsite"),
|
|
("tape", "t-monthly"),
|
|
}
|
|
gc = jobs[("gc", "backup")]
|
|
assert (gc["enabled"], gc["last_state"], gc["last_run"], gc["next_run"]) == (
|
|
True,
|
|
"ok",
|
|
T0 + 3600,
|
|
T0 + 86400,
|
|
)
|
|
verify = jobs[("verify", "v-weekly")]
|
|
assert (verify["last_state"], verify["last_error"]) == ("warning", "WARNINGS: 2")
|
|
prune = jobs[("prune", "p-daily")]
|
|
assert (prune["enabled"], prune["namespace"], prune["last_state"]) == (False, "prod", None)
|
|
sync = jobs[("sync", "offsite")]
|
|
assert (sync["last_state"], sync["last_error"]) == ("error", "connection refused")
|
|
assert (sync["remote"], sync["direction"], sync["comment"]) == (
|
|
"pbs-offsite",
|
|
"push",
|
|
"second site",
|
|
)
|
|
tape = jobs[("tape", "t-monthly")]
|
|
assert (tape["last_state"], tape["last_run"]) == ("running", None)
|
|
|
|
def test_a_sync_without_a_remote_is_local(self):
|
|
local = [{**SYNC[0], "remote": None, "sync-direction": None, "id": "local"}]
|
|
drv, _ = _driver(
|
|
**{"admin/sync": local, "admin/verify": [], "admin/prune": [], "tape/backup": []}
|
|
)
|
|
sync = next(j for j in drv.get_backup_jobs() if j["kind"] == "sync")
|
|
assert (sync["remote"], sync["direction"]) == (None, "pull")
|
|
|
|
def test_a_failed_listing_fails_the_whole_read(self):
|
|
drv, _ = _driver(**{"admin/sync": forbidden()})
|
|
with pytest.raises(Exception, match="403"):
|
|
drv.get_backup_jobs()
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Remotes
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_remotes_never_carry_their_secret():
|
|
drv, _ = _driver(**{"access/permissions": ALL_RIGHTS, "config/remote": REMOTES})
|
|
assert drv.get_backup_remotes() == [
|
|
{
|
|
"name": "pbs-offsite",
|
|
"host": "pbs2.example.com",
|
|
"port": 8007,
|
|
"fingerprint": "bb:" * 31 + "bb",
|
|
"auth_id": "sync@pbs",
|
|
"comment": None,
|
|
}
|
|
]
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Actions and tasks
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
SNAPSHOT = {
|
|
"datastore": "backup",
|
|
"namespace": "",
|
|
"group_type": "vm",
|
|
"group_id": "101",
|
|
"time": T0,
|
|
}
|
|
|
|
|
|
class TestActions:
|
|
def test_garbage_collection(self):
|
|
drv, api = _driver(**{"POST admin/datastore/backup/gc": "UPID:pbs:gc-now"})
|
|
assert drv.start_garbage_collection("backup") == {"task": "UPID:pbs:gc-now"}
|
|
|
|
@pytest.mark.parametrize(
|
|
("kind", "method", "path"),
|
|
[
|
|
("verify", "POST", "admin/verify/j1/run"),
|
|
("prune", "POST", "admin/prune/j1/run"),
|
|
("sync", "POST", "admin/sync/j1/run"),
|
|
("tape", "POST", "tape/backup/j1"),
|
|
("gc", "POST", "admin/datastore/j1/gc"),
|
|
],
|
|
)
|
|
def test_run_a_job_now(self, kind, method, path):
|
|
drv, api = _driver(**{f"{method} {path}": "UPID:pbs:run"})
|
|
assert drv.run_backup_job(kind, "j1") == {"task": "UPID:pbs:run"}
|
|
assert api.calls[-1][:2] == (method, path)
|
|
|
|
def test_an_unknown_kind_is_refused(self):
|
|
drv, _ = _driver()
|
|
with pytest.raises(ValueError, match="kind"):
|
|
drv.run_backup_job("restore", "j1") # type: ignore[arg-type]
|
|
|
|
def test_a_run_without_a_task_id_is_an_error(self):
|
|
drv, _ = _driver(**{"POST admin/sync/j1/run": None})
|
|
with pytest.raises(RuntimeError, match="task"):
|
|
drv.run_backup_job("sync", "j1")
|
|
|
|
def test_verify_one_snapshot(self):
|
|
drv, api = _driver(**{"POST admin/datastore/backup/verify": "UPID:pbs:verify"})
|
|
assert drv.verify_backup_snapshot(SNAPSHOT) == {"task": "UPID:pbs:verify"}
|
|
assert api.calls[-1][2] == {"backup-type": "vm", "backup-id": "101", "backup-time": T0}
|
|
|
|
def test_namespace_is_passed_where_given(self):
|
|
drv, api = _driver(**{"POST admin/datastore/backup/verify": "UPID:pbs:verify"})
|
|
drv.verify_backup_snapshot({**SNAPSHOT, "namespace": "prod"})
|
|
assert api.calls[-1][2]["ns"] == "prod"
|
|
|
|
def test_protect_and_unprotect(self):
|
|
drv, api = _driver(**{"PUT admin/datastore/backup/protected": None})
|
|
drv.set_backup_snapshot_protected(SNAPSHOT, True)
|
|
assert api.calls[-1] == (
|
|
"PUT",
|
|
"admin/datastore/backup/protected",
|
|
{"backup-type": "vm", "backup-id": "101", "backup-time": T0, "protected": 1},
|
|
)
|
|
drv.set_backup_snapshot_protected(SNAPSHOT, False)
|
|
assert api.calls[-1][2]["protected"] == 0
|
|
|
|
def test_delete(self):
|
|
drv, api = _driver(**{"DELETE admin/datastore/backup/snapshots": None})
|
|
drv.delete_backup_snapshot(SNAPSHOT)
|
|
assert api.calls[-1] == (
|
|
"DELETE",
|
|
"admin/datastore/backup/snapshots",
|
|
{"backup-type": "vm", "backup-id": "101", "backup-time": T0},
|
|
)
|
|
|
|
|
|
class TestTasks:
|
|
UPID = "UPID:pbs:000003E8:0000A000:00000002:6AE8E898:garbage_collection:backup:root@pam!netork:"
|
|
|
|
def _task(self, status, log=()):
|
|
return _driver(
|
|
**{
|
|
f"nodes/localhost/tasks/{self.UPID}/status": status,
|
|
f"nodes/localhost/tasks/{self.UPID}/log?limit=0": [
|
|
{"n": i + 1, "t": t} for i, t in enumerate(log)
|
|
],
|
|
}
|
|
)[0]
|
|
|
|
def test_running(self):
|
|
drv = self._task({"upid": self.UPID, "status": "running"}, ["starting garbage collection"])
|
|
assert drv.get_backup_task(self.UPID) == {
|
|
"task": self.UPID,
|
|
"running": True,
|
|
"state": "running",
|
|
"error": None,
|
|
"log": ["starting garbage collection"],
|
|
}
|
|
|
|
@pytest.mark.parametrize(
|
|
("exitstatus", "state", "error"),
|
|
[
|
|
("OK", "ok", None),
|
|
("WARNINGS: 1", "warning", "WARNINGS: 1"),
|
|
("unable to acquire lock", "error", "unable to acquire lock"),
|
|
],
|
|
)
|
|
def test_finished(self, exitstatus, state, error):
|
|
drv = self._task({"upid": self.UPID, "status": "stopped", "exitstatus": exitstatus})
|
|
task = drv.get_backup_task(self.UPID)
|
|
assert (task["running"], task["state"], task["error"]) == (False, state, error)
|