feat: get_clusters() and create_vm_from_cloud_init(node=) #10

Merged
christianmanivong merged 3 commits from feat/cluster-inventory into main 2026-11-01 19:11:34 +00:00
15 changed files with 814 additions and 29 deletions

No files matched your search

+2
View File
@@ -11,5 +11,7 @@ jobs:
uses: NAPALM/driver-standard/.gitea/workflows/driver.yml@main
with:
standard-ref: main
# Until napalm-device-types with the cluster shapes (ClusterDict) is on PyPI.
pre-install: "git+https://git.netork.io/NAPALM/napalm-device-types.git"
secrets:
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+15
View File
@@ -7,7 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
- `get_clusters()` on both drivers. The vCenter reports each cluster and each host
added to a datacenter directly (as a cluster of one with no name), with the id
`<instanceUuid>/<MoRef>`; an ESXi host reports itself, `local`, with its system
UUID as the id. Each cluster lists its hosts (state, maintenance mode, CPU and
memory; a disconnected host keeps only its name and state), the accessible
datastores and the port groups its hosts see, each with the hosts that see it
(NetOrk/netork#484).
- `create_vm_from_cloud_init(node=…)`: the host to create the VM on. A host that does
not exist, is not connected, is in maintenance mode or does not see the datastore
and port groups is refused with ValueError, before the image is downloaded
(NetOrk/netork#1822).
### Changed
- Provisioning chooses the host before it downloads the image, so a request no host
can take fails at once.
- Python 3.12 is the floor (`requires-python = ">=3.12"`): netOrk runs every driver
on 3.12, so 3.10 and 3.11 are no longer tested (NAPALM/driver-standard#6).
- The floor on napalm is 5.1: older releases import `pkg_resources`, which a Python
+3 -1
View File
@@ -31,6 +31,7 @@ hardware**: see [Harvesting fixtures](#harvesting-fixtures).
| `get_virtual_networks` | ✅ | ✅ (+ dvPortgroups) | port groups |
| `get_hypervisor_hosts` | ✅ | ✅ | connection state, maintenance mode, config issues, managing vCenter |
| `get_licenses` | ✅ | ✅ | license manager: name and edition key, never the key |
| `get_clusters` | ✅ (the host, a cluster of one) | ✅ (each cluster and standalone host) | compute resources, hosts, datastores, port groups |
| `reboot_host` | ✅ (maintenance mode only) | — | `RebootHost_Task` |
| `create_vm_from_cloud_init`, `destroy_vm`, `get_vm_status` | ✅ | ✅ | see [Provisioning](#provisioning) |
| `get_network_targets`, `get_image_storages` | ✅ | ✅ | port groups, datastores |
@@ -119,7 +120,8 @@ offers for Proxmox (qcow2/raw):
is cached by URL in `image_cache_dir`; the download is not kept.
2. A host is chosen that is connected, not in maintenance mode, and sees the
datastore and every requested port group (the named datastore, or the one
with the most free space).
with the most free space). `node` names the host to use; one that does not
exist or does not qualify is refused with ValueError before the download.
3. The VM is created from a minimal OVF descriptor (PVSCSI disk, VMXNET3
NICs) and the disk streamed in over NFC.
4. Requested MACs are pinned, the disk grown to `disk_resize_gb`, and a
+20 -1
View File
@@ -6,7 +6,7 @@ from pathlib import Path
from typing import Any, ClassVar
from napalm.base.exceptions import ConnectionException
from napalm_device_types import HypervisorDriver, LicenseMixin
from napalm_device_types import ClusterDict, HypervisorDriver, LicenseMixin
from napalm_device_types.models import (
HypervisorHostDict,
LicenseDict,
@@ -18,6 +18,7 @@ from napalm_device_types.models import (
from napalm_vmware import _session, paths
from napalm_vmware._inventory import Inventory
from napalm_vmware.parse.clusters import ClusterSources, host_cluster, manager_clusters
from napalm_vmware.parse.environment import environment
from napalm_vmware.parse.hosts import hypervisor_hosts
from napalm_vmware.parse.licenses import license_list
@@ -160,5 +161,23 @@ class VmwareBaseDriver(LicenseMixin, HypervisorDriver):
def get_hypervisor_hosts(self) -> list[HypervisorHostDict]:
return hypervisor_hosts(self._hosts(), standalone=self.STANDALONE)
def get_clusters(self) -> list[ClusterDict]:
collect = self._inventory.collect
src = ClusterSources(
hosts=self._hosts(),
datastores=collect("Datastore", paths.DATASTORE),
networks=collect("Network", paths.NETWORK),
dv_portgroups=collect("DistributedVirtualPortgroup", paths.DV_PORTGROUP),
)
if self.STANDALONE:
return [host_cluster(src)]
clusters = collect("ClusterComputeResource", paths.CLUSTER)
return manager_clusters(
collect("ComputeResource", paths.CLUSTER),
{c["_moref"] for c in clusters},
src,
instance_uuid=self._inventory.about().get("instanceUuid"),
)
def get_licenses(self) -> list[LicenseDict]:
return license_list(self._inventory.licenses())
+225
View File
@@ -0,0 +1,225 @@
"""``get_clusters``: clusters, their hosts, and the datastores and port groups they see.
A vCenter runs two kinds of compute resource: a ``ClusterComputeResource``,
and a plain ``ComputeResource`` for a host added to a datacenter directly.
Each is one cluster here; the plain one has no name, since the contract keeps
names for clusters an operator made. An ESXi host is a cluster of one.
A cluster's datastores and port groups are the ones its hosts list, each with
the hosts that list it. vSphere has no cluster quorum and no cpuinfo flags, so
``quorate`` and ``cpu_flags`` are always None.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
from napalm_device_types import (
ClusterDict,
ClusterNetworkDict,
ClusterNodeDict,
ClusterStorageDict,
)
from napalm_vmware.parse.networks import dv_vlan, is_uplink
Row = dict[str, Any]
_MB = 1024 * 1024
_STATUS = {"connected": "online", "disconnected": "offline", "notResponding": "offline"}
#: What a datastore can hold for netOrk: VM disks and the cloud-init seed ISO.
_CONTENT = ["images", "iso"]
#: A standard port group's "VLAN" 4095 trunks every VLAN to the guest.
_TRUNK = 4095
@dataclass
class ClusterSources:
"""The inventory rows every cluster draws from."""
hosts: list[Row]
datastores: list[Row]
networks: list[Row] # ``Network`` objects, distributed port groups included
dv_portgroups: list[Row]
def manager_clusters(
compute_resources: list[Row],
cluster_refs: set[str],
src: ClusterSources,
*,
instance_uuid: str | None,
) -> list[ClusterDict]:
"""vCenter: one cluster per compute resource; ``cluster_refs`` are the real clusters.
The id is ``<instanceUuid>/<MoRef>``: a MoRef alone repeats across vCenters.
"""
hosts = {h["_moref"]: h for h in src.hosts}
clusters = []
for cr in compute_resources:
members = [hosts[ref] for ref in cr.get("host") or [] if ref in hosts]
name = cr.get("name") if cr["_moref"] in cluster_refs else None
ident = f"{instance_uuid}/{cr['_moref']}" if instance_uuid else cr["_moref"]
clusters.append(_cluster(ident, name, members, src, local=False))
return sorted(clusters, key=lambda c: (c["name"] is None, c["name"] or _first_node(c)))
def host_cluster(src: ClusterSources) -> ClusterDict:
"""ESXi: the host itself, a cluster of one with no name."""
if not src.hosts:
raise RuntimeError("The inventory shows no host")
host = src.hosts[0]
ident = (host.get("hardware.systemInfo") or {}).get("uuid")
return _cluster(ident, None, [host], src, local=True)
def _first_node(cluster: ClusterDict) -> str:
return cluster["nodes"][0]["name"] if cluster["nodes"] else ""
def _cluster(
ident: str | None, name: str | None, hosts: list[Row], src: ClusterSources, *, local: bool
) -> ClusterDict:
hosts = sorted(hosts, key=_host_name)
return {
"id": ident,
"name": name,
"quorate": None,
"nodes": [_node(h, local) for h in hosts],
"storages": _storages(hosts, src.datastores),
"networks": _networks(hosts, src),
}
def _host_name(host: Row) -> str:
return str(host.get("name") or host["_moref"])
# -- nodes ---------------------------------------------------------------------
def _node(host: Row, local: bool) -> ClusterNodeDict:
status = _STATUS.get(host.get("runtime.connectionState") or "", "unknown")
hardware: Row = {}
stats: Row = {}
# A host vCenter cannot reach reports what it last knew, if anything.
if status != "offline":
hardware = host.get("summary.hardware") or {}
stats = host.get("summary.quickStats") or {}
memory_used = stats.get("overallMemoryUsage")
return {
"name": _host_name(host),
"status": status,
"local": local,
"maintenance": host.get("runtime.inMaintenanceMode"),
"cpu_threads": hardware.get("numCpuThreads"),
"cpu_cores": hardware.get("numCpuCores"),
"cpu_model": hardware.get("cpuModel"),
"cpu_flags": None,
"cpu_usage": _cpu_usage(hardware, stats),
"memory_total": hardware.get("memorySize"),
"memory_used": memory_used * _MB if memory_used is not None else None,
}
def _cpu_usage(hardware: Row, stats: Row) -> float | None:
"""Used MHz over the host's total MHz (``cpuMhz`` per core)."""
used = stats.get("overallCpuUsage")
total = (hardware.get("cpuMhz") or 0) * (hardware.get("numCpuCores") or 0)
if used is None or not total:
return None
return min(1.0, float(used) / total)
# -- storages ------------------------------------------------------------------
def _mounts(hosts: list[Row], key: str) -> dict[str, list[str]]:
"""MoRef -> names of the hosts whose ``key`` list holds it, in host order."""
seen: dict[str, list[str]] = {}
for host in hosts:
for ref in host.get(key) or []:
seen.setdefault(ref, []).append(_host_name(host))
return seen
def _storages(hosts: list[Row], datastores: list[Row]) -> list[ClusterStorageDict]:
mounts = _mounts(hosts, "datastore")
result: list[ClusterStorageDict] = []
for ds in datastores:
summary = ds.get("summary") or {}
if ds["_moref"] not in mounts or not summary.get("accessible"):
continue
total = summary.get("capacity")
free = summary.get("freeSpace")
result.append(
{
"name": ds.get("name") or summary.get("name") or ds["_moref"],
"type": str(summary.get("type", "")).lower(),
"shared": bool(summary.get("multipleHostAccess")),
"content": list(_CONTENT),
"nodes": mounts[ds["_moref"]],
"total": total,
"used": total - free if total is not None and free is not None else None,
}
)
return sorted(result, key=lambda s: s["name"])
# -- networks ------------------------------------------------------------------
def _networks(hosts: list[Row], src: ClusterSources) -> list[ClusterNetworkDict]:
return [
{
"name": name,
"kind": "portgroup",
# The port group fixes the VLAN; a NIC cannot add a tag of its own.
"vlan_aware": False,
"fixed_vlan_tag": _fixed_tag(vlans),
"nodes": nodes,
}
for name, (nodes, vlans) in sorted(_seen_networks(hosts, src).items())
]
def _seen_networks(
hosts: list[Row], src: ClusterSources
) -> dict[str, tuple[list[str], set[int | None]]]:
"""Port group name -> (the hosts that have it, the VLAN it carries on each)."""
names = {n["_moref"]: n.get("name") or n["_moref"] for n in src.networks}
dv = {pg["_moref"]: pg for pg in src.dv_portgroups}
seen: dict[str, tuple[list[str], set[int | None]]] = {}
for host in hosts:
for ref in _vm_networks(host, dv):
name = names.get(ref, ref)
nodes, vlans = seen.setdefault(name, ([], set()))
if _host_name(host) not in nodes:
nodes.append(_host_name(host))
vlans.add(_vlan(ref, name, host, dv))
return seen
def _vm_networks(host: Row, dv: dict[str, Row]) -> list[str]:
"""The host's networks a VM can use: a vDS's uplink port group carries none."""
return [ref for ref in host.get("network") or [] if not (ref in dv and is_uplink(dv[ref]))]
def _vlan(ref: str, name: str, host: Row, dv: dict[str, Row]) -> int | None:
"""The VLAN ``ref`` carries on ``host``; None where it cannot be read."""
if ref in dv:
return dv_vlan(dv[ref].get("config.defaultPortConfig"))
for pg in host.get("config.network.portgroup") or []:
spec = pg.get("spec") or {}
if spec.get("name") == name:
return int(spec.get("vlanId", 0))
return None
def _fixed_tag(vlans: set[int | None]) -> int | None:
"""One VLAN on every host, and a single tag at that; else None."""
if len(vlans) != 1:
return None
(vlan,) = vlans
return vlan if vlan not in (None, 0, _TRUNK) else None
+2 -2
View File
@@ -76,7 +76,7 @@ def _network(name: str, kind: str, bridge: str, vlan: int) -> dict[str, Any]:
}
def _is_uplink(pg: dict[str, Any]) -> bool:
def is_uplink(pg: dict[str, Any]) -> bool:
if "config.uplink" in pg:
return bool(pg["config.uplink"])
return bool(_UPLINK_NAME.search(pg.get("name", "")))
@@ -97,7 +97,7 @@ def virtual_networks(
)
switch_names = {s["_moref"]: s.get("name", s["_moref"]) for s in dv_switches}
for pg in dv_portgroups:
if _is_uplink(pg):
if is_uplink(pg):
continue
dvs = pg.get("config.distributedVirtualSwitch", "")
vlan = dv_vlan(pg.get("config.defaultPortConfig"))
+7 -1
View File
@@ -78,6 +78,11 @@ COMPUTE_RESOURCE = ("parent", "resourcePool")
FOLDER = ("parent",)
DATACENTER = ("name", "vmFolder")
# get_clusters: every compute resource and its hosts. Read once as
# ComputeResource (clusters included, a subtype) and once as
# ClusterComputeResource to tell the two apart.
CLUSTER = ("name", "host")
#: Managed object type name -> paths, in the order harvest.py dumps them.
ALL = {
"HostSystem": HOST,
@@ -86,7 +91,8 @@ ALL = {
"DistributedVirtualPortgroup": DV_PORTGROUP,
"DistributedVirtualSwitch": DV_SWITCH,
"Network": NETWORK,
"ComputeResource": COMPUTE_RESOURCE,
"ComputeResource": tuple(dict.fromkeys(COMPUTE_RESOURCE + CLUSTER)),
"ClusterComputeResource": CLUSTER,
"Folder": FOLDER,
"Datacenter": DATACENTER,
}
+55 -15
View File
@@ -2,9 +2,10 @@
A host qualifies when it is connected, not in maintenance mode, and sees both
a usable datastore and every requested network. Among those, the pair with
the most free space wins -- or the named datastore, when one is given. The
resource pool comes from the host's compute resource and the VM folder from
the datacenter above it; a standalone ESXi host has the same shape
the most free space wins -- or the named datastore, when one is given; a
named host narrows the choice to that host and is refused if it does not
qualify. The resource pool comes from the host's compute resource and the VM
folder from the datacenter above it; a standalone ESXi host has the same shape
(``ha-compute-res`` under ``ha-datacenter``), so one path serves both.
"""
@@ -16,6 +17,10 @@ from typing import Any
Row = dict[str, Any]
class NodeRefused(ValueError):
"""The host named for a new VM does not exist or cannot take it."""
@dataclass
class Inventory:
hosts: list[Row]
@@ -71,15 +76,35 @@ def _datacenter(inv: Inventory, compute_resource: str) -> Row:
return datacenters[node]
def choose_placement(inv: Inventory, storage: str | None, networks: list[str]) -> Placement:
usable = _usable_datastores(inv, storage)
names = {n["_moref"]: n.get("name", "") for n in inv.networks}
def _refusal(host: Row) -> str | None:
"""Why ``host`` cannot take a new VM, or None if it can."""
if host.get("runtime.connectionState") != "connected":
return "is not connected"
if host.get("runtime.inMaintenanceMode"):
return "is in maintenance mode"
return None
def _candidates(inv: Inventory, host_name: str | None) -> list[Row]:
"""The hosts placement may choose from: every eligible one, or the named one."""
hosts = sorted(inv.hosts, key=lambda h: h.get("name", ""))
if host_name is None:
return [h for h in hosts if _refusal(h) is None]
named = [h for h in hosts if h.get("name") == host_name]
if not named:
raise NodeRefused(f"There is no host named {host_name!r}")
reason = _refusal(named[0])
if reason:
raise NodeRefused(f"Host {host_name} {reason}")
return named[:1]
def _best(
hosts: list[Row], usable: dict[str, Row], names: dict[str, str], networks: list[str]
) -> tuple[Row, Row, list[str]] | None:
"""The host/datastore pair with the most free space, and the host's networks."""
best: tuple[int, Row, Row, list[str]] | None = None
for host in sorted(inv.hosts, key=lambda h: h.get("name", "")):
if host.get("runtime.connectionState") != "connected" or host.get(
"runtime.inMaintenanceMode"
):
continue
for host in hosts:
host_nets = _host_networks(host, names, networks)
if host_nets is None:
continue
@@ -88,11 +113,26 @@ def choose_placement(inv: Inventory, storage: str | None, networks: list[str]) -
free = int((ds or {}).get("summary", {}).get("freeSpace", 0))
if ds is not None and (best is None or free > best[0]):
best = (free, host, ds, host_nets)
return best[1:] if best else None
def choose_placement(
inv: Inventory, storage: str | None, networks: list[str], *, host_name: str | None = None
) -> Placement:
"""Where a VM on ``storage`` with ``networks`` goes; ``host_name`` pins the host.
Raises NodeRefused (a ValueError) when the named host does not exist or
cannot take the VM, and ValueError when no host can.
"""
usable = _usable_datastores(inv, storage)
names = {n["_moref"]: n.get("name", "") for n in inv.networks}
best = _best(_candidates(inv, host_name), usable, names, networks)
wanted = f"datastore {storage or '(any)'} and networks {networks}"
if best is None and host_name is not None:
raise NodeRefused(f"Host {host_name} does not see {wanted}")
if best is None:
raise ValueError(
f"There is no host that sees datastore {storage or '(any)'} and networks {networks}"
)
_, host, ds, host_nets = best
raise ValueError(f"There is no host that sees {wanted}")
host, ds, host_nets = best
compute = next(c for c in inv.compute_resources if c["_moref"] == host.get("parent"))
dc = _datacenter(inv, compute["_moref"])
return Placement(
+21 -4
View File
@@ -35,7 +35,7 @@ from napalm_vmware.provision import transfer
from napalm_vmware.provision.image import ImageCache
from napalm_vmware.provision.ovf import ovf_descriptor
from napalm_vmware.provision.placement import Inventory as PlacementInventory
from napalm_vmware.provision.placement import Placement, choose_placement
from napalm_vmware.provision.placement import NodeRefused, Placement, choose_placement
from napalm_vmware.provision.seed import meta_data, nocloud_iso, user_data
logger = logging.getLogger(__name__)
@@ -153,6 +153,7 @@ class VmwareProvisioningMixin:
storage: str | None = None,
cpu_type: str | None = None,
guest_os: str = "linux",
node: str | None = None,
download_timeout: int = 300,
timeout: int = 180,
) -> VMProvisionResultDict:
@@ -163,13 +164,11 @@ class VmwareProvisioningMixin:
# one, and there is no get_vm_cpu_types() that could offer a model.
raise ValueError(f"VMware cannot give a single VM a CPU type ({cpu_type!r})")
self._check_nics(nics)
place = self._place(storage, nics, node)
try:
vmdk, image_size = ImageCache(self._image_cache_dir).vmdk(
image_url, image_checksum, download_timeout
)
place = choose_placement(
self._placement_inventory(), storage, [n["bridge"] for n in nics]
)
except ValueError as exc:
raise RuntimeError(str(exc)) from exc
descriptor = ovf_descriptor(
@@ -196,6 +195,24 @@ class VmwareProvisioningMixin:
)
return {"vmid": props["config.instanceUuid"], "name": name, "node": place.host_name}
def _place(self, storage: str | None, nics: list[NICConfigDict], node: str | None) -> Placement:
"""Where the VM goes, decided before the image is fetched.
A ``node`` that does not exist or cannot take the VM stays a
ValueError, as the contract has it; no host at all is a RuntimeError.
"""
try:
return choose_placement(
self._placement_inventory(),
storage,
[n["bridge"] for n in nics],
host_name=node,
)
except NodeRefused:
raise
except ValueError as exc:
raise RuntimeError(str(exc)) from exc
def _run_task(self, call: Any, *args: Any, **kwargs: Any) -> None:
wait_for_task(self._inventory, invoke(call, *args, **kwargs), _TASK_TIMEOUT)
+2 -3
View File
@@ -5,13 +5,12 @@
"napalm_vmware/base.py::VmwareBaseDriver._find_vm": 12,
"napalm_vmware/parse/facts.py::host_serial": 11,
"napalm_vmware/parse/interfaces.py::host_interfaces": 11,
"napalm_vmware/parse/vms.py::_vm": 13,
"napalm_vmware/provision/placement.py::choose_placement": 16
"napalm_vmware/parse/vms.py::_vm": 13
},
"function_lines": {
"napalm_vmware/provision/ovf.py::ovf_descriptor": 80,
"napalm_vmware/provisioning.py::VmwareProvisioningMixin._import_ovf": 56,
"napalm_vmware/provisioning.py::VmwareProvisioningMixin.create_vm_from_cloud_init": 57
"napalm_vmware/provisioning.py::VmwareProvisioningMixin.create_vm_from_cloud_init": 56
},
"mypy_relaxed": [
"napalm_vmware._inventory",
+74
View File
@@ -74,6 +74,7 @@ class TestIdentity:
"get_virtual_networks",
"get_hypervisor_hosts",
"get_licenses",
"get_clusters",
],
)
@pytest.mark.parametrize("cls", DRIVERS)
@@ -248,6 +249,79 @@ class TestVcenterGetters:
assert vcenter.get_virtual_networks()["DC0_DVPG0"]["type"] == "dvportgroup"
def _vcsim_topology(data, compute, clusters, networks):
"""What the harvested fixtures predate: compute resources, Network objects,
and each host's datastores and networks -- as vcsim reports them."""
objects = data["objects"]
objects["ComputeResource"] = compute
objects["ClusterComputeResource"] = [c for c in compute if c["_moref"] in clusters]
objects["Network"] = networks
for host in objects["HostSystem"]:
host["datastore"] = [ds["_moref"] for ds in objects["Datastore"]]
host["network"] = [n["_moref"] for n in networks]
class TestClusters:
@pytest.fixture
def vcenter(self, vcenter_data):
_vcsim_topology(
vcenter_data,
[
{"_moref": "domain-s24", "name": "DC0_H0", "host": ["host-21"]},
{
"_moref": "domain-c28",
"name": "DC0_C0",
"host": ["host-37", "host-47", "host-57"],
},
],
{"domain-c28"},
[
{"_moref": "network-6", "name": "VM Network"},
{"_moref": "dvportgroup-10", "name": "DVS0-DVUplinks-8"},
{"_moref": "dvportgroup-12", "name": "DC0_DVPG0"},
],
)
return _driver(VmwareVcenterDriver, vcenter_data, hostname="vc01.example.lan")
@pytest.fixture
def esxi(self, esxi_data):
_vcsim_topology(
esxi_data,
[{"_moref": "ha-compute-res", "name": "localhost.localdomain", "host": ["ha-host"]}],
set(),
[{"_moref": "HaNetwork-VM Network", "name": "VM Network"}],
)
return _driver(VmwareEsxiDriver, esxi_data)
def test_vcenter_reports_its_cluster_and_its_standalone_host(self, vcenter, vcenter_data):
uuid = vcenter_data["about"]["instanceUuid"]
clusters = vcenter.get_clusters()
assert [(c["name"], c["id"]) for c in clusters] == [
("DC0_C0", f"{uuid}/domain-c28"),
(None, f"{uuid}/domain-s24"),
]
assert [n["name"] for n in clusters[0]["nodes"]] == ["DC0_C0_H0", "DC0_C0_H1", "DC0_C0_H2"]
assert [n["name"] for n in clusters[1]["nodes"]] == ["DC0_H0"]
assert {n["name"] for n in clusters[0]["networks"]} == {"VM Network", "DC0_DVPG0"}
assert clusters[0]["storages"][0]["nodes"] == ["DC0_C0_H0", "DC0_C0_H1", "DC0_C0_H2"]
def test_esxi_is_a_cluster_of_one_with_itself_local(self, esxi, esxi_host):
(cluster,) = esxi.get_clusters()
assert (cluster["id"], cluster["name"]) == (esxi_host["hardware.systemInfo"]["uuid"], None)
assert [(n["name"], n["local"], n["status"]) for n in cluster["nodes"]] == [
(esxi_host["name"], True, "online")
]
assert [s["name"] for s in cluster["storages"]] == ["LocalDS_0"]
assert [n["name"] for n in cluster["networks"]] == ["VM Network"]
@pytest.mark.parametrize("name", ["vcenter", "esxi"])
def test_a_failed_read_raises_instead_of_reporting_nothing(self, request, name):
driver = request.getfixturevalue(name)
driver._inventory.collect = MagicMock(side_effect=RuntimeError("session gone"))
with pytest.raises(RuntimeError, match="session gone"):
driver.get_clusters()
class TestFindVm:
def test_by_name_vmid_or_moref(self, esxi, esxi_data):
vm0 = esxi_data["objects"]["VirtualMachine"][0]
+280
View File
@@ -0,0 +1,280 @@
"""``get_clusters``: clusters, their hosts, datastores and port groups, from plain rows."""
from __future__ import annotations
import pytest
from napalm_vmware.parse.clusters import ClusterSources, host_cluster, manager_clusters
MB = 1024 * 1024
UUID = "35bb7c82-8526-5aa7-a886-cd7f8e6be786"
def _host(moref, name, *, state="connected", maint=False, datastores=(), networks=(), pgs=()):
return {
"_moref": moref,
"name": name,
"runtime.connectionState": state,
"runtime.inMaintenanceMode": maint,
"summary.hardware": {
"cpuMhz": 2000,
"cpuModel": "Xeon",
"memorySize": 64 * 1024 * MB,
"numCpuCores": 8,
"numCpuThreads": 16,
},
"summary.quickStats": {"overallCpuUsage": 4000, "overallMemoryUsage": 1024},
"hardware.systemInfo": {"uuid": f"uuid-{moref}"},
"datastore": list(datastores),
"network": list(networks),
"config.network.portgroup": [
{"spec": {"name": pg, "vlanId": vlan, "vswitchName": "vSwitch0"}} for pg, vlan in pgs
],
}
def _ds(moref, name, *, shared=False, accessible=True, capacity=1000, free=400, kind="VMFS"):
return {
"_moref": moref,
"name": name,
"summary": {
"accessible": accessible,
"capacity": capacity,
"freeSpace": free,
"multipleHostAccess": shared,
"type": kind,
},
}
@pytest.fixture
def src():
return ClusterSources(
hosts=[
_host(
"host-1",
"esx01",
datastores=["ds-san", "ds-local1", "ds-gone"],
networks=["network-1", "network-10", "dvportgroup-12", "dvportgroup-10"],
pgs=[("VM Network", 0), ("Storage", 30)],
),
_host(
"host-2",
"esx02",
state="disconnected",
datastores=["ds-san"],
networks=["network-1", "dvportgroup-12"],
pgs=[("VM Network", 0)],
),
_host(
"host-3",
"esx03",
maint=True,
datastores=["ds-san"],
networks=["network-1", "dvportgroup-12"],
pgs=[("VM Network", 0)],
),
_host("host-9", "lonely", datastores=["ds-local9"], networks=["network-1"]),
],
datastores=[
_ds("ds-san", "san", shared=True, capacity=10_000, free=2_500),
_ds("ds-local1", "local1", kind="vmfs"),
_ds("ds-gone", "gone", accessible=False),
_ds("ds-local9", "local9", kind="NFS"),
],
networks=[
{"_moref": "network-1", "name": "VM Network"},
{"_moref": "network-10", "name": "Storage"},
{"_moref": "dvportgroup-10", "name": "DVS0-DVUplinks-8"},
{"_moref": "dvportgroup-12", "name": "DC0_DVPG0"},
],
dv_portgroups=[
{
"_moref": "dvportgroup-10",
"name": "DVS0-DVUplinks-8",
"config.defaultPortConfig": {"vlan": {"_type": "TrunkVlanSpec"}},
},
{
"_moref": "dvportgroup-12",
"name": "DC0_DVPG0",
"config.defaultPortConfig": {
"vlan": {"_type": "VmwareDistributedVirtualSwitchVlanIdSpec", "vlanId": 42}
},
},
],
)
COMPUTE = [
{"_moref": "domain-c7", "name": "Prod", "host": ["host-1", "host-2", "host-3"]},
{"_moref": "domain-s9", "name": "lonely", "host": ["host-9"]},
]
@pytest.fixture
def clusters(src):
return manager_clusters(COMPUTE, {"domain-c7"}, src, instance_uuid=UUID)
def _standalone(src, index):
"""``src`` as an ESXi host sees it: one host, ``src.hosts[index]``."""
src.hosts = [src.hosts[index]]
return host_cluster(src)
def _by_name(items):
return {i["name"]: i for i in items}
class TestManager:
def test_one_entry_per_cluster_and_per_standalone_host(self, clusters):
assert [(c["name"], c["id"]) for c in clusters] == [
("Prod", f"{UUID}/domain-c7"),
(None, f"{UUID}/domain-s9"),
]
assert all(c["quorate"] is None for c in clusters)
def test_the_moref_alone_without_an_instance_uuid(self, src):
(prod, _) = manager_clusters(COMPUTE, {"domain-c7"}, src, instance_uuid=None)
assert prod["id"] == "domain-c7"
def test_a_cluster_lists_only_its_own_hosts(self, clusters):
prod, lonely = clusters
assert [n["name"] for n in prod["nodes"]] == ["esx01", "esx02", "esx03"]
assert [n["name"] for n in lonely["nodes"]] == ["lonely"]
assert all(n["local"] is False for c in clusters for n in c["nodes"])
def test_an_empty_cluster_is_still_a_cluster(self, src):
(empty,) = manager_clusters(
[{"_moref": "domain-c8", "name": "Empty"}], {"domain-c8"}, src, instance_uuid=UUID
)
assert (empty["name"], empty["nodes"], empty["storages"], empty["networks"]) == (
"Empty",
[],
[],
[],
)
class TestNodes:
def test_a_connected_host(self, clusters):
node = _by_name(clusters[0]["nodes"])["esx01"]
assert node == {
"name": "esx01",
"status": "online",
"local": False,
"maintenance": False,
"cpu_threads": 16,
"cpu_cores": 8,
"cpu_model": "Xeon",
"cpu_flags": None,
"cpu_usage": 0.25,
"memory_total": 64 * 1024 * MB,
"memory_used": 1024 * MB,
}
def test_a_disconnected_host_keeps_its_name_and_state_but_no_capacity(self, clusters):
node = _by_name(clusters[0]["nodes"])["esx02"]
assert (node["status"], node["maintenance"]) == ("offline", False)
capacity = ("cpu_threads", "cpu_cores", "cpu_model", "cpu_usage")
assert all(node[k] is None for k in (*capacity, "memory_total", "memory_used"))
def test_a_host_in_maintenance(self, clusters):
node = _by_name(clusters[0]["nodes"])["esx03"]
assert (node["status"], node["maintenance"], node["cpu_cores"]) == ("online", True, 8)
@pytest.mark.parametrize(
("state", "status"),
[("notResponding", "offline"), ("disconnected", "offline"), (None, "unknown")],
)
def test_status_from_the_connection_state(self, src, state, status):
src.hosts[3]["runtime.connectionState"] = state
node = _standalone(src, 3)["nodes"][0]
assert node["status"] == status
@pytest.mark.parametrize(
"change",
[
lambda h: h["summary.hardware"].update(cpuMhz=0),
lambda h: h["summary.hardware"].pop("numCpuCores"),
lambda h: h.pop("summary.quickStats"),
],
)
def test_cpu_usage_is_none_when_it_cannot_be_computed(self, src, change):
change(src.hosts[0])
node = _standalone(src, 0)["nodes"][0]
assert node["cpu_usage"] is None
def test_cpu_usage_never_exceeds_one(self, src):
src.hosts[0]["summary.quickStats"]["overallCpuUsage"] = 99_999
assert _standalone(src, 0)["nodes"][0]["cpu_usage"] == 1.0
class TestStorages:
def test_shared_and_local_datastores_with_the_hosts_that_mount_them(self, clusters):
storages = _by_name(clusters[0]["storages"])
assert storages["san"] == {
"name": "san",
"type": "vmfs",
"shared": True,
"content": ["images", "iso"],
"nodes": ["esx01", "esx02", "esx03"],
"total": 10_000,
"used": 7_500,
}
assert (storages["local1"]["shared"], storages["local1"]["nodes"]) == (False, ["esx01"])
def test_an_inaccessible_datastore_is_left_out(self, clusters):
assert "gone" not in _by_name(clusters[0]["storages"])
def test_only_the_datastores_of_the_clusters_hosts(self, clusters):
assert sorted(_by_name(clusters[0]["storages"])) == ["local1", "san"]
assert [s["name"] for s in clusters[1]["storages"]] == ["local9"]
assert clusters[1]["storages"][0]["type"] == "nfs"
class TestNetworks:
def test_standard_and_distributed_port_groups(self, clusters):
networks = _by_name(clusters[0]["networks"])
assert networks["VM Network"] == {
"name": "VM Network",
"kind": "portgroup",
"vlan_aware": False,
"fixed_vlan_tag": None,
"nodes": ["esx01", "esx02", "esx03"],
}
assert networks["DC0_DVPG0"]["fixed_vlan_tag"] == 42
def test_a_port_group_on_one_host_only(self, clusters):
storage = _by_name(clusters[0]["networks"])["Storage"]
assert (storage["nodes"], storage["fixed_vlan_tag"]) == (["esx01"], 30)
def test_the_uplink_port_group_is_no_network_for_a_vm(self, clusters):
assert "DVS0-DVUplinks-8" not in _by_name(clusters[0]["networks"])
def test_a_vlan_that_differs_between_hosts_is_not_reported(self, src, clusters):
src.hosts[1]["config.network.portgroup"][0]["spec"]["vlanId"] = 7
prod = manager_clusters(COMPUTE, {"domain-c7"}, src, instance_uuid=UUID)[0]
assert _by_name(prod["networks"])["VM Network"]["fixed_vlan_tag"] is None
def test_a_trunk_port_group_has_no_fixed_tag(self, src):
src.hosts[0]["config.network.portgroup"][1]["spec"]["vlanId"] = 4095
networks = _by_name(_standalone(src, 0)["networks"])
assert networks["Storage"]["fixed_vlan_tag"] is None
class TestStandaloneHost:
def test_the_host_is_a_cluster_of_one(self, src):
cluster = _standalone(src, 3)
assert (cluster["id"], cluster["name"], cluster["quorate"]) == ("uuid-host-9", None, None)
assert [(n["name"], n["local"]) for n in cluster["nodes"]] == [("lonely", True)]
assert [s["name"] for s in cluster["storages"]] == ["local9"]
assert [n["name"] for n in cluster["networks"]] == ["VM Network"]
def test_no_id_without_a_system_uuid(self, src):
del src.hosts[3]["hardware.systemInfo"]
assert _standalone(src, 3)["id"] is None
def test_no_host_is_a_failed_read(self, src):
with pytest.raises(RuntimeError, match="no host"):
host_cluster(ClusterSources([], [], [], []))
+41 -1
View File
@@ -5,7 +5,7 @@ from __future__ import annotations
import pytest
from napalm_vmware.provision.placement import Inventory as Inv
from napalm_vmware.provision.placement import choose_placement
from napalm_vmware.provision.placement import NodeRefused, choose_placement
def _host(moref, name, *, datastores, networks, parent="domain-c1", state="connected", maint=False):
@@ -112,3 +112,43 @@ def test_datacenter_is_found_through_nested_folders(inv):
inv.compute_resources[0]["parent"] = "group-h2"
inv.folders.append({"_moref": "group-h2", "parent": "group-h1"})
assert choose_placement(inv, storage=None, networks=["VM Network"]).datacenter_name == "DC0"
class TestNamedHost:
"""``create_vm_from_cloud_init(node=...)``: only that host is considered."""
def test_the_named_host_is_used_even_with_less_free_space_elsewhere(self, inv):
inv.hosts[0]["datastore"] = ["ds-1"]
p = choose_placement(inv, storage=None, networks=["VM Network"], host_name="esx01")
assert (p.host, p.datastore_name, p.networks) == ("host-1", "local1", ["net-1"])
def test_the_best_datastore_of_the_named_host(self, inv):
p = choose_placement(inv, storage=None, networks=["VM Network"], host_name="esx02")
assert (p.host, p.datastore_name) == ("host-2", "san")
def test_an_unknown_host(self, inv):
with pytest.raises(NodeRefused, match="no host named 'esx99'"):
choose_placement(inv, storage=None, networks=["VM Network"], host_name="esx99")
@pytest.mark.parametrize(
("change", "message"),
[
(lambda h: h.update({"runtime.inMaintenanceMode": True}), "maintenance"),
(lambda h: h.update({"runtime.connectionState": "disconnected"}), "not connected"),
],
)
def test_a_host_that_cannot_take_a_vm(self, inv, change, message):
change(inv.hosts[1])
with pytest.raises(NodeRefused, match=f"esx02.*{message}"):
choose_placement(inv, storage=None, networks=["VM Network"], host_name="esx02")
def test_a_host_without_the_networks(self, inv):
with pytest.raises(NodeRefused, match="esx02 does not see"):
choose_placement(inv, storage=None, networks=["DC0_DVPG0"], host_name="esx02")
def test_a_host_without_the_datastore(self, inv):
with pytest.raises(NodeRefused, match="esx02 does not see datastore local1"):
choose_placement(inv, storage="local1", networks=["VM Network"], host_name="esx02")
def test_a_refused_host_is_a_value_error(self):
assert issubclass(NodeRefused, ValueError)
+26 -1
View File
@@ -11,7 +11,7 @@ import yaml
from napalm_vmware import VmwareEsxiDriver
from napalm_vmware import provisioning as prov
from napalm_vmware.provision.placement import Placement
from napalm_vmware.provision.placement import NodeRefused, Placement
from tests.conftest import FakeInventory
PLACE = Placement(
@@ -170,6 +170,31 @@ class TestCreate:
_create(driver)
staged.import_ovf.assert_not_called()
def test_an_image_refusal_is_a_runtime_error(self, driver, staged):
staged.cache.return_value.vmdk.side_effect = ValueError("checksum mismatch")
with pytest.raises(RuntimeError, match="checksum mismatch"):
_create(driver)
staged.import_ovf.assert_not_called()
def test_no_node_leaves_the_host_to_placement(self, driver, staged):
with patch.object(prov, "choose_placement", return_value=PLACE) as choose:
_create(driver)
assert choose.call_args.kwargs["host_name"] is None
def test_the_node_is_the_only_host_placement_considers(self, driver, staged):
with patch.object(prov, "choose_placement", return_value=PLACE) as choose:
assert _create(driver, node="esx01")["node"] == "esx01"
assert choose.call_args.kwargs["host_name"] == "esx01"
def test_a_refused_node_is_a_value_error_before_anything_happens(self, driver, staged):
refusal = NodeRefused("There is no host named 'esx99'")
with patch.object(prov, "choose_placement", side_effect=refusal):
with pytest.raises(ValueError, match="esx99") as raised:
_create(driver, node="esx99")
assert not isinstance(raised.value, RuntimeError)
staged.cache.return_value.vmdk.assert_not_called()
staged.import_ovf.assert_not_called()
MAC = "00:0c:29:39:31:62"
+41
View File
@@ -150,3 +150,44 @@ def test_provision_and_destroy(which, image_url, tmp_path, request):
finally:
driver.destroy_vm(result["vmid"])
assert not any(vm["vmid"] == result["vmid"] for vm in driver.get_vms())
def test_vcenter_clusters(vcenter):
clusters = vcenter.get_clusters()
named = [c for c in clusters if c["name"]]
assert [c["name"] for c in named] == ["DC0_C0"]
assert len(named[0]["nodes"]) == 3
assert any(c["name"] is None and len(c["nodes"]) == 1 for c in clusters)
assert all(c["id"] and "/" in c["id"] for c in clusters)
assert "DC0_DVPG0" in {n["name"] for n in named[0]["networks"]}
assert named[0]["storages"]
def test_esxi_cluster(esxi):
(cluster,) = esxi.get_clusters()
(node,) = cluster["nodes"]
assert (cluster["name"], node["local"], node["status"]) == (None, True, "online")
assert cluster["id"]
assert node["cpu_cores"] and node["memory_total"]
assert [n["name"] for n in cluster["networks"]] == ["VM Network"]
def test_provision_on_a_named_node(vcenter, image_url, tmp_path):
vcenter._image_cache_dir = tmp_path / "cache"
(cluster,) = [c for c in vcenter.get_clusters() if c["name"]]
node = cluster["nodes"][-1]["name"]
kwargs = dict(
image_url=image_url,
cpu=1,
memory=512,
nics=[{"bridge": "VM Network", "dhcp": True}],
cloud_init_config={},
)
with pytest.raises(ValueError, match="no host named"):
vcenter.create_vm_from_cloud_init("netork-nowhere", node="nope", **kwargs)
result = vcenter.create_vm_from_cloud_init("netork-pinned", node=node, **kwargs)
try:
assert result["node"] == node
assert next(vm for vm in vcenter.get_vms() if vm["vmid"] == result["vmid"])["node"] == node
finally:
vcenter.destroy_vm(result["vmid"])