From 9a866cc7db43f00165b536f1024467e0d480f921 Mon Sep 17 00:00:00 2001 From: luto Date: Wed, 14 Feb 2024 00:47:01 +0100 Subject: [PATCH] add parametrized tests, close #3 --- docs/syntax.md | 20 +++++ src/shellinspector/__main__.py | 38 +++++++--- src/shellinspector/parser.py | 90 ++++++++++++++++++++++- tests/e2e/600_examplestest.ispec | 2 + tests/e2e/600_examplestest.ispec.examples | 3 + tests/test_parser.py | 18 +++++ 6 files changed, 158 insertions(+), 13 deletions(-) create mode 100644 tests/e2e/600_examplestest.ispec create mode 100644 tests/e2e/600_examplestest.ispec.examples diff --git a/docs/syntax.md b/docs/syntax.md index 3e5ef0f..893a721 100644 --- a/docs/syntax.md +++ b/docs/syntax.md @@ -210,6 +210,26 @@ $ echo $DNS_SERVER DNS_SERVER=1.1.1.1 ``` +## Parametrized tests + +To run many similar tests, create a file called `test.ispec.examples` next to +`test.ispec`: + +`test.ispec`: + +``` +$~ {PY_EXE} --version +{PY_VERSION} +``` + +`test.ispec.examples` + +``` +PY_EXE PY_VERSION +python3.10 3.10 +python3.11 3.11 +``` + ## Examples ``` diff --git a/src/shellinspector/__main__.py b/src/shellinspector/__main__.py index f1f7e84..14b8ac1 100644 --- a/src/shellinspector/__main__.py +++ b/src/shellinspector/__main__.py @@ -39,8 +39,8 @@ def get_ssh_config(target_host): } -def handle_spec_file(runner, path): - LOGGER.info("handling %s", path) +def parse_spec_file(path): + LOGGER.debug("parsing %s", path) spec_file = Path(path).resolve() @@ -51,6 +51,9 @@ def handle_spec_file(runner, path): lines = spec_file.read_text().splitlines() specfile = parse(spec_file, lines) + for i, command in enumerate(specfile.commands): + LOGGER.debug("command[%s]: %s", i, command.short) + if specfile.errors: for error in specfile.errors: LOGGER.error( @@ -63,13 +66,10 @@ def handle_spec_file(runner, path): return False - for i, command in enumerate(specfile.commands): - LOGGER.debug("command[%s]: %s", i, command.short) - - return runner.run(specfile) + return specfile -def run(target_host, spec_files, identity, verbose): +def run(target_host, spec_file_paths, identity, verbose): ssh_config = get_ssh_config(target_host) ssh_config["ssh_key"] = identity @@ -85,8 +85,28 @@ def run(target_host, spec_files, identity, verbose): runner.add_reporter(ConsoleReporter()) success = True + spec_files = [] + + for spec_file_path in spec_file_paths: + spec_file = parse_spec_file(spec_file_path) + + if spec_file.examples: + for example in spec_file.examples: + spec_files.append(spec_file.as_example(example)) + else: + spec_files.append(spec_file) + for spec_file in spec_files: - success = success & handle_spec_file(runner, spec_file) + if len(spec_files) > 1: + if spec_file.applied_example: + example_str = ",".join( + f"{k}={v}" for k, v in spec_file.applied_example.items() + ) + print(f"{spec_file.path} (w/ {example_str})") + else: + print(f"{spec_file.path}") + + success = success & runner.run(spec_file) return 0 if success else 1 @@ -112,7 +132,7 @@ def parse_args(argv=None): default=False, ) parser.add_argument( - "spec_files", + "spec_file_paths", nargs="+", ) diff --git a/src/shellinspector/parser.py b/src/shellinspector/parser.py index 53b0bd7..f7d7288 100644 --- a/src/shellinspector/parser.py +++ b/src/shellinspector/parser.py @@ -1,5 +1,6 @@ import re from dataclasses import dataclass +from dataclasses import replace from enum import Enum from pathlib import Path @@ -55,12 +56,38 @@ class Specfile: commands: list[Command] errors: list[Error] environment: dict[str, str] + examples: list[dict[str, str]] + applied_example: dict - def __init__(self, path, commands=None, errors=None, environment=None): + def __init__( + self, path, commands=None, errors=None, environment=None, examples=None + ): self.path = Path(path) self.commands = commands or [] self.errors = errors or [] self.environment = environment or {} + self.examples = examples or [] + self.applied_example = None + + def copy(self): + return Specfile( + self.path, + [replace(c) for c in self.commands], + [replace(e) for e in self.errors], + self.environment.copy(), + [e.copy() for e in self.examples], + ) + + def as_example(self, example): + copy = self.copy() + copy.applied_example = example + + for cmd in copy.commands: + cmd.command = cmd.command.format(**example) + cmd.line = cmd.line.format(**example) + cmd.expected = cmd.expected.format(**example) + + return copy # parse a line like @@ -89,13 +116,13 @@ def parse_env(path, lines: list[str]): for line_no, line in enumerate(lines, 1): try: + if line.startswith("#") or not line.strip(): + continue + k, _, v = line.partition("=") k = k.strip() v = v.strip() - if k.startswith("#"): - continue - if not v: errors.append( Error( @@ -121,8 +148,54 @@ def parse_env(path, lines: list[str]): return environment, errors +def parse_examples(path, lines: list[str]): + errors = [] + keys = None + examples = [] + + if len(lines) <= 1: + return examples + + for line_no, line in enumerate(lines, 1): + if line.startswith("#") or not line.strip(): + continue + + if keys is None: + keys = line.split() + continue + + try: + values = line.split() + + if len(values) != len(keys): + errors.append( + Error( + path, + line_no, + line, + f"Number of values ({len(values)}) does not match" + f"number of keys in header ({len(keys)})", + ) + ) + continue + + examples.append(dict(zip(keys, values))) + except Exception as ex: + errors.append( + Error( + path, + line_no, + line, + str(ex), + ) + ) + + return examples, errors + + def parse(path: str, lines: list[str]) -> Specfile: specfile = Specfile(path) + env_path = specfile.path.with_suffix(".ispec.env") if env_path.exists(): @@ -130,6 +203,15 @@ def parse(path: str, lines: list[str]) -> Specfile: specfile.environment.update(environment) specfile.errors.extend(errors) + examples_path = specfile.path.with_suffix(".ispec.examples") + + if examples_path.exists(): + examples, errors = parse_examples( + examples_path, examples_path.read_text().splitlines() + ) + specfile.examples = examples + specfile.errors.extend(errors) + for line_no, line in enumerate(lines, 1): # comment if line.startswith("#"): diff --git a/tests/e2e/600_examplestest.ispec b/tests/e2e/600_examplestest.ispec new file mode 100644 index 0000000..f961bde --- /dev/null +++ b/tests/e2e/600_examplestest.ispec @@ -0,0 +1,2 @@ +[@local]$ echo Python {PY_COMMAND} --version | grep -Po '[0-9.]+' +{PY_VERSION} diff --git a/tests/e2e/600_examplestest.ispec.examples b/tests/e2e/600_examplestest.ispec.examples new file mode 100644 index 0000000..a073634 --- /dev/null +++ b/tests/e2e/600_examplestest.ispec.examples @@ -0,0 +1,3 @@ +PY_COMMAND PY_VERSION +python3.10 3.10 +python3.11 3.11 diff --git a/tests/test_parser.py b/tests/test_parser.py index 495b0d8..94ffe24 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -315,6 +315,24 @@ def test_environment(): } +def test_examples(): + path = Path(__file__).parent / "e2e/600_examplestest.ispec" + specfile = parse(path, []) + + assert len(specfile.errors) == 0 + + assert specfile.examples == [ + { + "PY_COMMAND": "python3.10", + "PY_VERSION": "3.10", + }, + { + "PY_COMMAND": "python3.11", + "PY_VERSION": "3.11", + }, + ] + + def test_include_missing_file(): path = Path(__file__).parent / "virtual.ispec" specfile = parse(