fix(vm_provision_mixin): stale same-named cloud image cache causes checksum mismatch

_download_cloud_image() cached downloaded images under just the URL's
basename (e.g. ubuntu-26.04-server-cloudimg-amd64.img). Ubuntu's per-build
download URLs change daily under that same stable basename
(.../release-20260713/... vs .../release-20260714/...), so a previous
day's cached file satisfied the "already cached" check and got checksum-
verified against the *new* day's expected hash from NetOrk's daily catalog
sync — failing outright and aborting the whole provisioning job, even
though a plain retry would have re-downloaded and succeeded (the bad file
was already being deleted on mismatch, just never re-fetched).

Found live during a NetOrk deploy: "Checksum mismatch for
https://cloud-images.ubuntu.com/.../release-20260713/
ubuntu-26.04-server-cloudimg-amd64.img: expected 0826c500..., got
3ee4f67f...".

Fix: key the cache path on a hash of the full URL (not just the
basename), and retry the download once after a checksum-mismatch cleanup
before raising.
This commit is contained in:
Christian Manivong
2026-07-14 16:23:13 +02:00
parent 3181ade728
commit 38f0c0a656
2 changed files with 112 additions and 23 deletions
+46 -20
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import base64
import hashlib
import logging
import time
import yaml
@@ -74,38 +75,63 @@ class ProxmoxVMProvisionMixin:
"""
Download image_url to the node's image cache dir if not already present.
Returns the local path on the hypervisor node. Verifies image_checksum
(format "<algo>:<hex>", e.g. "sha256:abc123...") if given, re-downloading
is left to the caller's next attempt if verification fails.
Returns the local path on the hypervisor node. The cache filename is
prefixed with a hash of the *full* URL, not just its basename —
Ubuntu (and others) publish per-build URLs that change daily under a
stable basename (e.g. .../release-20260713/ubuntu-26.04-server-
cloudimg-amd64.img), so keying the cache on the basename alone let a
stale previous-day build satisfy the "already cached" check and fail
checksum verification against today's expected hash.
Verifies image_checksum (format "<algo>:<hex>", e.g. "sha256:abc123...")
if given. On mismatch, removes the bad file and retries the download
once (covers a corrupted/partial transfer or a stale same-keyed file)
before raising.
"""
filename = image_url.rstrip("/").rsplit("/", 1)[-1]
local_path = f"{_IMAGE_CACHE_DIR}/{filename}"
url_hash = hashlib.sha256(image_url.encode()).hexdigest()[:12]
local_path = f"{_IMAGE_CACHE_DIR}/{url_hash}-{filename}"
exists = self._run_node_command(
f"mkdir -p {_IMAGE_CACHE_DIR} && test -f {local_path} && echo EXISTS || echo MISSING",
timeout=30,
)
if "EXISTS" not in exists:
_logger.info(f"Downloading cloud image {image_url} -> {local_path}")
self._run_node_command(
f"wget -q -O {local_path}.tmp '{image_url}' && mv {local_path}.tmp {local_path}",
timeout=timeout,
algo, _, expected = (image_checksum or "").partition(":")
algo = (algo or "sha256").lower()
max_attempts = 2
for attempt in range(1, max_attempts + 1):
exists = self._run_node_command(
f"mkdir -p {_IMAGE_CACHE_DIR} && test -f {local_path} "
f"&& echo EXISTS || echo MISSING",
timeout=30,
)
if "EXISTS" not in exists:
_logger.info(f"Downloading cloud image {image_url} -> {local_path}")
self._run_node_command(
f"wget -q -O {local_path}.tmp '{image_url}' "
f"&& mv {local_path}.tmp {local_path}",
timeout=timeout,
)
if not image_checksum:
return local_path
if image_checksum:
algo, _, expected = image_checksum.partition(":")
algo = (algo or "sha256").lower()
actual = self._run_node_command(
f"{algo}sum {local_path} | awk '{{print $1}}'", timeout=60
)
if actual.lower() != expected.lower():
# Remove the bad file so a retry re-downloads instead of reusing it.
self._run_node_command(f"rm -f {local_path}", timeout=30)
if actual.lower() == expected.lower():
return local_path
# Remove the bad file so the next attempt re-downloads instead of
# reusing it.
self._run_node_command(f"rm -f {local_path}", timeout=30)
if attempt == max_attempts:
raise RuntimeError(
f"Checksum mismatch for {image_url}: expected {expected}, got {actual}"
)
_logger.warning(
f"Checksum mismatch for {image_url} on attempt {attempt}/{max_attempts} "
"— retrying download"
)
return local_path
raise AssertionError("unreachable") # loop always returns or raises above
def _find_default_image_storage(self) -> str:
"""Find a storage suitable for VM root disks (content includes 'images').