4fb7e0bc29
AGENTS.md has always said that a skill shipping scripts/ puts its tests in
tests/<name>/, but nothing checked it: 54 of the 100 such skills had no suite
at all, including docx, pptx, xlsx, pdf and scanpy. All 100 now do.
tests/_meta is the guard. It runs the shared structural contract across every
skill in a single process -- safe because it parses scripts with ast and never
imports them -- and fails when a skill ships scripts/ without a suite or
without a [skills.<name>] entry in skill-requirements.toml. It needs no
scientific packages and finishes in seconds, so skill-tests.yml blocks every
pull request on it, plus the packages=[] suites.
tests/_contract holds what the per-skill suites were each reimplementing:
frontmatter conformance, the 500-line limit, no tests or bytecode under
skills/, local links resolving, scripts parsing, no eval/exec/os.system, no
standard-library shadowing, no hardcoded local paths, valid shell scripts. Also
the --help contract, which skips when a skill's packages are absent and runs
for real under --isolated, and shared behaviour for the files docx/pptx/xlsx
and five schematic-shipping skills carry byte-identical copies of, with drift
detection so they cannot diverge silently.
67 new suites, 3325 test functions. The existing suites were retrofitted: 18
no longer pin an exact skill version, so a version bump no longer breaks a
test; 11 duplicated structural methods removed; 19 wired to the --help
contract; and 5 that failed collection without their packages now skip
cleanly. run_all.py in the bare project environment goes from 6 failures to 0.
Writing the tests surfaced 19 defects in the skills, fixed here with version
bumps. The ones that changed scientific output:
- openpiv reported vorticity 0 for a rotating flow, from a sign error in
openpiv's y-up coordinate relabelling; solid-body rotation now gives 2w
exactly, on grids of either orientation
- deepchem returned solubility predictions in z-scored space while labelling
them log(mol/L), because it transformed a y-less dataset instead of
untransforming the output
- neuropixels-analysis had the Allen and IBL ISI thresholds swapped,
contradicting its own references/QUALITY_METRICS.md and inverting the two
standards' relative strictness
- scanpy's summarize() raised TypeError on every AnnData under anndata 0.13,
which reports an unnamed None key on .layers; scanpy convert was broken
- experimental-design's Latin hypercube was never reproducible: pyDOE3 draws
from its own default_rng and ignores numpy's global seed
- primekg shipped a hardcoded path naming a person, which is why
no_personal_paths is now a contract rule
The remainder is upstream API drift, each verified against the installed
package: retired symbols in bioservices 1.16, gget helpers that returned lists
where a string was written, ArviZ 1.x kwargs in pymc, a positional-only
factory in pymoo, ReduceLROnPlateau(verbose=) in torch 2.13, a removed scvelo
parameter, and PyPDF2 in scientific-slides.
Two manifest environments could not build and are pinned: gget, where an
unpinned scanpy walked back to 1.9.8 and pulled llvmlite 0.36 which does not
compile on 3.13, and pymatgen, pinned to the snapshot its own _common.py
enforces rather than loosening that check. deepchem gains torch, without which
no model class exists.
python tests/run_all.py --isolated: 101 passed, 0 failed.
109 lines
4.0 KiB
Python
109 lines
4.0 KiB
Python
"""Tests for the latex-posters helpers.
|
|
|
|
The two schematic scripts are byte-identical to the copies in
|
|
`scientific-schematics` and `literature-review`, so their behaviour comes from
|
|
the shared contract. What is specific here is `review_poster.sh`, the shell
|
|
helper that renders and inspects a compiled poster.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
import skill_contract
|
|
|
|
SKILL_ROOT = Path(__file__).resolve().parents[2] / "skills" / "latex-posters"
|
|
SCRIPTS = SKILL_ROOT / "scripts"
|
|
sys.path.insert(0, str(SCRIPTS))
|
|
|
|
SchematicTests = skill_contract.schematic.schematic_test_case(SKILL_ROOT)
|
|
CliHelpTests = skill_contract.cli.help_test_case(SKILL_ROOT)
|
|
|
|
REVIEW_SCRIPT = SCRIPTS / "review_poster.sh"
|
|
|
|
|
|
class ReviewScriptTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.text = REVIEW_SCRIPT.read_text(encoding="utf-8")
|
|
|
|
def test_it_is_executable_shell_with_a_shebang(self) -> None:
|
|
self.assertTrue(REVIEW_SCRIPT.is_file())
|
|
self.assertTrue(REVIEW_SCRIPT.stat().st_mode & 0o111, "not executable")
|
|
self.assertTrue(self.text.startswith("#!"), "no shebang")
|
|
|
|
def test_it_parses_as_bash(self) -> None:
|
|
syntax = subprocess.run(
|
|
["bash", "-n", str(REVIEW_SCRIPT)],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30,
|
|
)
|
|
self.assertEqual(syntax.returncode, 0, syntax.stderr)
|
|
|
|
def _run(self, *args: str) -> subprocess.CompletedProcess:
|
|
return subprocess.run(
|
|
["bash", str(REVIEW_SCRIPT), *args],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=60,
|
|
cwd=SCRIPTS,
|
|
)
|
|
|
|
def test_no_argument_exits_non_zero_with_usage(self) -> None:
|
|
result = self._run()
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertIn("Usage:", result.stdout + result.stderr)
|
|
|
|
def test_a_missing_file_exits_non_zero_and_names_it(self) -> None:
|
|
result = self._run("no-such-poster.pdf")
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertIn("no-such-poster.pdf", result.stdout + result.stderr)
|
|
|
|
def test_a_missing_poppler_tool_degrades_instead_of_aborting(self) -> None:
|
|
# Deliberately no `set -e`: this is a report, and one unavailable
|
|
# inspector must not truncate the remaining checks or the manual
|
|
# checklist. Running with an empty PATH-ish environment proves it.
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
poster = Path(directory) / "poster.pdf"
|
|
poster.write_bytes(b"%PDF-1.4\n%%EOF\n")
|
|
result = subprocess.run(
|
|
["bash", str(REVIEW_SCRIPT), str(poster)],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=60,
|
|
cwd=SCRIPTS,
|
|
)
|
|
self.assertEqual(result.returncode, 0)
|
|
output = result.stdout
|
|
# Every numbered section still runs, right through to the summary.
|
|
for section in ("[1]", "[2]", "[3]", "[4]", "[5]", "[6]", "[7]"):
|
|
with self.subTest(section=section):
|
|
self.assertIn(section, output)
|
|
self.assertIn("Quality Check Complete", output)
|
|
self.assertNotIn("Traceback", result.stderr)
|
|
|
|
def test_every_python_script_it_invokes_is_shipped(self) -> None:
|
|
for token in self.text.split():
|
|
if token.endswith(".py"):
|
|
with self.subTest(script=token):
|
|
self.assertTrue(
|
|
(SCRIPTS / Path(token).name).is_file(),
|
|
f"{token} is referenced but not shipped",
|
|
)
|
|
|
|
|
|
class AssetTests(unittest.TestCase):
|
|
def test_every_documented_asset_is_shipped(self) -> None:
|
|
# The skill's value is its templates; a SKILL.md that names one it does
|
|
# not ship sends the agent looking for a file that is not there.
|
|
problems = skill_contract.structure.link_problems(SKILL_ROOT)
|
|
self.assertEqual(problems, [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|