christianmanivong
eb3a296aff
napalm-hpe-aruba-procurve ships napalm_procurve; guessed from its name, the ratchet measured nothing and wrote an empty baseline without a word.
323 lines
10 KiB
Python
323 lines
10 KiB
Python
"""The quality ratchet: measured from the AST, held against a baseline (STANDARDS.md §6).
|
|
|
|
Ported from netOrk's scripts/quality_ratchet.py (#402), Python only, plus the
|
|
two settings a driver ratchets in pyproject.toml: modules exempt from strict
|
|
mypy, and the coverage floor.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ast
|
|
import json
|
|
import pathlib
|
|
import textwrap
|
|
|
|
import pytest
|
|
|
|
from napalm_standard import ratchet as qr
|
|
|
|
|
|
def _functions(source: str) -> dict:
|
|
return qr.python_functions_in(textwrap.dedent(source), "mod.py")
|
|
|
|
|
|
class TestMeasuring:
|
|
def test_function_length_and_qualified_names(self) -> None:
|
|
funcs = _functions(
|
|
"""
|
|
def top():
|
|
return 1
|
|
|
|
class C:
|
|
def method(self):
|
|
def inner():
|
|
return 2
|
|
return inner()
|
|
"""
|
|
)
|
|
|
|
assert funcs["mod.py::top"]["lines"] == 2
|
|
assert funcs["mod.py::C.method"]["lines"] == 4
|
|
assert funcs["mod.py::C.method.inner"]["lines"] == 2
|
|
|
|
def test_complexity_counts_decision_points(self) -> None:
|
|
funcs = _functions(
|
|
"""
|
|
def f(a, b, xs):
|
|
if a and b:
|
|
pass
|
|
elif a:
|
|
pass
|
|
for x in xs:
|
|
while x:
|
|
x -= 1
|
|
try:
|
|
pass
|
|
except ValueError:
|
|
pass
|
|
return [y for y in xs if y] if a else None
|
|
"""
|
|
)
|
|
|
|
# 1 + if + and + elif + for + while + except + comprehension + its if + ternary
|
|
assert funcs["mod.py::f"]["complexity"] == 10
|
|
|
|
def test_match_cases_count(self) -> None:
|
|
funcs = _functions(
|
|
"""
|
|
def f(x):
|
|
match x:
|
|
case 1:
|
|
return 1
|
|
case _:
|
|
return 0
|
|
"""
|
|
)
|
|
|
|
assert funcs["mod.py::f"]["complexity"] == 3
|
|
|
|
def test_a_nested_function_counts_for_itself_only(self) -> None:
|
|
funcs = _functions(
|
|
"""
|
|
def outer():
|
|
def inner(x):
|
|
if x:
|
|
return 1
|
|
return 0
|
|
return inner
|
|
"""
|
|
)
|
|
|
|
assert funcs["mod.py::outer"]["complexity"] == 1
|
|
assert funcs["mod.py::outer.inner"]["complexity"] == 2
|
|
|
|
@pytest.mark.parametrize(
|
|
("source", "typed"),
|
|
[
|
|
("def f(a: int) -> int:\n return a\n", True),
|
|
("def f(a) -> int:\n return a\n", False),
|
|
("def f(a: int):\n return a\n", False),
|
|
("def f(*a: int, **k: int) -> None:\n pass\n", True),
|
|
("def f(*a) -> None:\n pass\n", False),
|
|
("class C:\n def f(self, a: int) -> None:\n pass\n", True),
|
|
],
|
|
)
|
|
def test_typed(self, source: str, typed: bool) -> None:
|
|
[entry] = _functions(source).values()
|
|
assert entry["typed"] is typed
|
|
|
|
def test_private_names_and_what_they_contain_are_not_public(self) -> None:
|
|
funcs = _functions(
|
|
"""
|
|
def _helper():
|
|
def inner():
|
|
pass
|
|
|
|
class _Hidden:
|
|
def method(self):
|
|
pass
|
|
|
|
class Shown:
|
|
def _private(self):
|
|
pass
|
|
def public(self):
|
|
pass
|
|
"""
|
|
)
|
|
|
|
public = {k for k, v in funcs.items() if v["public"]}
|
|
assert public == {"mod.py::Shown.public"}
|
|
|
|
def test_the_walker_yields_each_function_once_with_its_qualified_name(self) -> None:
|
|
tree = ast.parse(
|
|
"def top():\n pass\n\nclass C:\n def m(self):\n def inner():\n"
|
|
" pass\n"
|
|
)
|
|
walked = [
|
|
(name, node.name, in_class)
|
|
for name, node, in_class, _ in qr.iter_python_functions(tree)
|
|
]
|
|
assert walked == [("top", "top", False), ("C.m", "m", True), ("C.m.inner", "inner", False)]
|
|
|
|
def test_a_function_inside_a_block_is_still_measured(self) -> None:
|
|
funcs = _functions(
|
|
"""
|
|
def outer(flag):
|
|
with lock():
|
|
def _stream():
|
|
if flag:
|
|
return 1
|
|
return 0
|
|
return _stream
|
|
|
|
class C:
|
|
if True:
|
|
def m(self):
|
|
pass
|
|
"""
|
|
)
|
|
|
|
assert funcs["mod.py::outer._stream"]["complexity"] == 2
|
|
assert "mod.py::C.m" in funcs
|
|
|
|
|
|
# ── pyproject settings ───────────────────────────────────────────────────
|
|
|
|
|
|
def test_relaxed_mypy_modules_are_the_overrides_that_loosen_strict() -> None:
|
|
pyproject = {
|
|
"tool": {
|
|
"mypy": {
|
|
"strict": True,
|
|
"overrides": [
|
|
{"module": "napalm.*", "ignore_missing_imports": True},
|
|
{"module": ["pkg.old", "pkg.older"], "ignore_errors": True},
|
|
{"module": "pkg.legacy", "disallow_untyped_defs": False},
|
|
{"module": "pkg.loose", "strict": False},
|
|
],
|
|
}
|
|
}
|
|
}
|
|
|
|
assert qr.relaxed_mypy_modules(pyproject) == [
|
|
"pkg.legacy",
|
|
"pkg.loose",
|
|
"pkg.old",
|
|
"pkg.older",
|
|
]
|
|
|
|
|
|
def test_coverage_floor_is_read_from_pyproject() -> None:
|
|
assert qr.coverage_floor({"tool": {"coverage": {"report": {"fail_under": 63.5}}}}) == 63.5
|
|
assert qr.coverage_floor({}) == 0
|
|
|
|
|
|
# ── the ratchet against a repo ───────────────────────────────────────────
|
|
|
|
LONG = "def long(x: int) -> int:\n" + " x += 1\n" * 60 + " return x\n"
|
|
|
|
|
|
def _repo(tmp_path: pathlib.Path, source: str, pyproject: str = "") -> pathlib.Path:
|
|
(tmp_path / "pkg").mkdir()
|
|
(tmp_path / "pkg" / "mod.py").write_text(source)
|
|
(tmp_path / "pyproject.toml").write_text(
|
|
'[project]\nname = "pkg"\n\n' + (pyproject or "[tool.coverage.report]\nfail_under = 50\n")
|
|
)
|
|
return tmp_path
|
|
|
|
|
|
def test_measure_reports_only_what_is_over_the_limit(tmp_path: pathlib.Path) -> None:
|
|
repo = _repo(
|
|
tmp_path, LONG + "\ndef ok(a: int) -> int:\n return a\n\ndef raw(a):\n pass\n"
|
|
)
|
|
|
|
measured = qr.measure(repo, "pkg")
|
|
|
|
assert measured["function_lines"] == {"pkg/mod.py::long": 62}
|
|
assert measured["function_complexity"] == {}
|
|
assert measured["python_file_lines"] == {}
|
|
assert measured["untyped_public"] == ["pkg/mod.py::raw"]
|
|
assert measured["coverage_floor"] == 50
|
|
|
|
|
|
def test_a_file_over_the_hard_maximum_is_measured(tmp_path: pathlib.Path) -> None:
|
|
repo = _repo(tmp_path, "x = 1\n" * 1001)
|
|
|
|
assert qr.measure(repo, "pkg")["python_file_lines"] == {"pkg/mod.py": 1001}
|
|
|
|
|
|
def test_without_a_baseline_everything_over_the_limit_is_new(tmp_path: pathlib.Path) -> None:
|
|
repo = _repo(tmp_path, LONG)
|
|
|
|
problems = qr.problems(qr.measure(repo, "pkg"), {})
|
|
|
|
assert problems == ["function_lines: pkg/mod.py::long = 62 (new)"]
|
|
|
|
|
|
def test_a_recorded_excess_may_stay_but_not_grow(tmp_path: pathlib.Path) -> None:
|
|
repo = _repo(tmp_path, LONG)
|
|
measured = qr.measure(repo, "pkg")
|
|
|
|
assert qr.problems(measured, {"function_lines": {"pkg/mod.py::long": 62}}) == []
|
|
assert qr.problems(measured, {"function_lines": {"pkg/mod.py::long": 61}}) == [
|
|
"function_lines: pkg/mod.py::long = 62 (recorded 61)"
|
|
]
|
|
|
|
|
|
def test_a_new_untyped_public_function_is_a_problem(tmp_path: pathlib.Path) -> None:
|
|
repo = _repo(tmp_path, "def raw(a):\n pass\n")
|
|
|
|
assert qr.problems(qr.measure(repo, "pkg"), {"untyped_public": []}) == [
|
|
"untyped_public: pkg/mod.py::raw (new)"
|
|
]
|
|
|
|
|
|
def test_a_new_relaxed_mypy_module_is_a_problem(tmp_path: pathlib.Path) -> None:
|
|
pyproject = (
|
|
"[tool.mypy]\nstrict = true\n\n[[tool.mypy.overrides]]\n"
|
|
'module = "pkg.mod"\nignore_errors = true\n'
|
|
)
|
|
repo = _repo(tmp_path, "", pyproject)
|
|
|
|
assert qr.problems(qr.measure(repo, "pkg"), {}) == ["mypy_relaxed: pkg.mod (new)"]
|
|
assert qr.problems(qr.measure(repo, "pkg"), {"mypy_relaxed": ["pkg.mod"]}) == []
|
|
|
|
|
|
def test_a_lowered_coverage_floor_is_a_problem(tmp_path: pathlib.Path) -> None:
|
|
repo = _repo(tmp_path, "")
|
|
|
|
assert qr.problems(qr.measure(repo, "pkg"), {"coverage_floor": 60}) == [
|
|
"coverage_floor: 50 (recorded 60)"
|
|
]
|
|
|
|
|
|
def test_progress_shows_up_as_stale_entries(tmp_path: pathlib.Path) -> None:
|
|
repo = _repo(tmp_path, "def ok(a: int) -> int:\n return a\n")
|
|
baseline = {
|
|
"function_lines": {"pkg/mod.py::gone": 70},
|
|
"untyped_public": ["pkg/mod.py::fixed"],
|
|
"mypy_relaxed": ["pkg.mod"],
|
|
"coverage_floor": 40,
|
|
}
|
|
|
|
assert qr.stale_entries(qr.measure(repo, "pkg"), baseline) == [
|
|
"function_lines: pkg/mod.py::gone",
|
|
"untyped_public: pkg/mod.py::fixed",
|
|
"mypy_relaxed: pkg.mod",
|
|
"coverage_floor: 40 -> 50",
|
|
]
|
|
|
|
|
|
def test_write_then_check_is_clean(tmp_path: pathlib.Path) -> None:
|
|
repo = _repo(tmp_path, LONG)
|
|
|
|
assert qr.run(repo, "pkg", write=True)[0] == 0
|
|
baseline = json.loads((repo / qr.BASELINE).read_text())
|
|
|
|
assert baseline["function_lines"] == {"pkg/mod.py::long": 62}
|
|
assert qr.run(repo, "pkg") == (0, "within the baseline")
|
|
|
|
|
|
def test_check_fails_on_a_problem_and_on_a_stale_entry(tmp_path: pathlib.Path) -> None:
|
|
repo = _repo(tmp_path, LONG)
|
|
(repo / "tests").mkdir()
|
|
(repo / qr.BASELINE).write_text(json.dumps({"function_lines": {"pkg/mod.py::old": 99}}))
|
|
|
|
code, text = qr.run(repo, "pkg")
|
|
|
|
assert code == 1
|
|
assert "pkg/mod.py::long = 62 (new)" in text
|
|
assert "stale: function_lines: pkg/mod.py::old" in text
|
|
|
|
|
|
def test_a_package_that_does_not_exist_is_an_error_not_an_empty_baseline(
|
|
tmp_path: pathlib.Path,
|
|
) -> None:
|
|
"""procurve's import package is napalm_procurve: guessed from the name, the
|
|
ratchet measured nothing and wrote an empty baseline without a word."""
|
|
repo = _repo(tmp_path, LONG)
|
|
|
|
with pytest.raises(FileNotFoundError, match="napalm_missing"):
|
|
qr.run(repo, "napalm_missing", write=True)
|
|
assert not (repo / qr.BASELINE).exists()
|