fix(vm_provision_mixin): stale same-named cloud image cache causes checksum mismatch
_download_cloud_image() cached downloaded images under just the URL's basename (e.g. ubuntu-26.04-server-cloudimg-amd64.img). Ubuntu's per-build download URLs change daily under that same stable basename (.../release-20260713/... vs .../release-20260714/...), so a previous day's cached file satisfied the "already cached" check and got checksum- verified against the *new* day's expected hash from NetOrk's daily catalog sync — failing outright and aborting the whole provisioning job, even though a plain retry would have re-downloaded and succeeded (the bad file was already being deleted on mismatch, just never re-fetched). Found live during a NetOrk deploy: "Checksum mismatch for https://cloud-images.ubuntu.com/.../release-20260713/ ubuntu-26.04-server-cloudimg-amd64.img: expected 0826c500..., got 3ee4f67f...". Fix: key the cache path on a hash of the full URL (not just the basename), and retry the download once after a checksum-mismatch cleanup before raising.
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import logging
|
||||
import time
|
||||
import yaml
|
||||
@@ -74,38 +75,63 @@ class ProxmoxVMProvisionMixin:
|
||||
"""
|
||||
Download image_url to the node's image cache dir if not already present.
|
||||
|
||||
Returns the local path on the hypervisor node. Verifies image_checksum
|
||||
(format "<algo>:<hex>", e.g. "sha256:abc123...") if given, re-downloading
|
||||
is left to the caller's next attempt if verification fails.
|
||||
Returns the local path on the hypervisor node. The cache filename is
|
||||
prefixed with a hash of the *full* URL, not just its basename —
|
||||
Ubuntu (and others) publish per-build URLs that change daily under a
|
||||
stable basename (e.g. .../release-20260713/ubuntu-26.04-server-
|
||||
cloudimg-amd64.img), so keying the cache on the basename alone let a
|
||||
stale previous-day build satisfy the "already cached" check and fail
|
||||
checksum verification against today's expected hash.
|
||||
|
||||
Verifies image_checksum (format "<algo>:<hex>", e.g. "sha256:abc123...")
|
||||
if given. On mismatch, removes the bad file and retries the download
|
||||
once (covers a corrupted/partial transfer or a stale same-keyed file)
|
||||
before raising.
|
||||
"""
|
||||
filename = image_url.rstrip("/").rsplit("/", 1)[-1]
|
||||
local_path = f"{_IMAGE_CACHE_DIR}/{filename}"
|
||||
url_hash = hashlib.sha256(image_url.encode()).hexdigest()[:12]
|
||||
local_path = f"{_IMAGE_CACHE_DIR}/{url_hash}-{filename}"
|
||||
|
||||
exists = self._run_node_command(
|
||||
f"mkdir -p {_IMAGE_CACHE_DIR} && test -f {local_path} && echo EXISTS || echo MISSING",
|
||||
timeout=30,
|
||||
)
|
||||
if "EXISTS" not in exists:
|
||||
_logger.info(f"Downloading cloud image {image_url} -> {local_path}")
|
||||
self._run_node_command(
|
||||
f"wget -q -O {local_path}.tmp '{image_url}' && mv {local_path}.tmp {local_path}",
|
||||
timeout=timeout,
|
||||
algo, _, expected = (image_checksum or "").partition(":")
|
||||
algo = (algo or "sha256").lower()
|
||||
|
||||
max_attempts = 2
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
exists = self._run_node_command(
|
||||
f"mkdir -p {_IMAGE_CACHE_DIR} && test -f {local_path} "
|
||||
f"&& echo EXISTS || echo MISSING",
|
||||
timeout=30,
|
||||
)
|
||||
if "EXISTS" not in exists:
|
||||
_logger.info(f"Downloading cloud image {image_url} -> {local_path}")
|
||||
self._run_node_command(
|
||||
f"wget -q -O {local_path}.tmp '{image_url}' "
|
||||
f"&& mv {local_path}.tmp {local_path}",
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
if not image_checksum:
|
||||
return local_path
|
||||
|
||||
if image_checksum:
|
||||
algo, _, expected = image_checksum.partition(":")
|
||||
algo = (algo or "sha256").lower()
|
||||
actual = self._run_node_command(
|
||||
f"{algo}sum {local_path} | awk '{{print $1}}'", timeout=60
|
||||
)
|
||||
if actual.lower() != expected.lower():
|
||||
# Remove the bad file so a retry re-downloads instead of reusing it.
|
||||
self._run_node_command(f"rm -f {local_path}", timeout=30)
|
||||
if actual.lower() == expected.lower():
|
||||
return local_path
|
||||
|
||||
# Remove the bad file so the next attempt re-downloads instead of
|
||||
# reusing it.
|
||||
self._run_node_command(f"rm -f {local_path}", timeout=30)
|
||||
if attempt == max_attempts:
|
||||
raise RuntimeError(
|
||||
f"Checksum mismatch for {image_url}: expected {expected}, got {actual}"
|
||||
)
|
||||
_logger.warning(
|
||||
f"Checksum mismatch for {image_url} on attempt {attempt}/{max_attempts} "
|
||||
"— retrying download"
|
||||
)
|
||||
|
||||
return local_path
|
||||
raise AssertionError("unreachable") # loop always returns or raises above
|
||||
|
||||
def _find_default_image_storage(self) -> str:
|
||||
"""Find a storage suitable for VM root disks (content includes 'images').
|
||||
|
||||
Reference in New Issue
Block a user