4"""Pytest integration for ESI cosimulation tests.
6Provides the ``@cosim_test`` decorator which automates the full lifecycle of a
7cosimulation test: running a PyCDE hardware script, compiling the design with
8a simulator (e.g. Verilator), launching the simulator, injecting connection
9parameters into the test function, and tearing everything down afterwards.
11Decorated functions run in an isolated child process (via ``fork``) so that
12simulator state never leaks between tests. When applied to a class, the
13hardware compilation is performed once and shared across all ``test_*`` methods.
17 from esiaccel.cosim.pytest import cosim_test
19 @cosim_test("path/to/hw_script.py")
20 def test_my_design(conn: AcceleratorConnection):
21 # conn is already connected to the running simulator
25from __future__
import annotations
28from dataclasses
import dataclass, field, replace
33import multiprocessing.connection
35from pathlib
import Path
43from typing
import Any, Callable, Dict, Optional, Pattern, Sequence, Union
49from .simulator
import (available_simulators, get_simulator,
50 is_simulator_available, Simulator, SourceFiles)
52LogMatcher = Union[str, Pattern[str], Callable[[str, str], bool]]
53SourceGeneratorFunc = Callable[[
"CosimPytestConfig", Path], Path]
54SourceGeneratorArg = Union[str, Path, SourceGeneratorFunc]
56_logger = logging.getLogger(
"esiaccel.cosim.pytest")
57_DEFAULT_FAILURE_PATTERN = re.compile(
r"\berror\b", re.IGNORECASE)
58_DEFAULT_WARN_PATTERN = re.compile(
r"\bwarn(ing)?\b", re.IGNORECASE)
61_DEFAULT_TIMEOUT_S: float = 120.0
65 """Read a boolean environment variable.
68 var_name: Name of the environment variable to read.
69 default: Default value if the variable is not set.
72 The boolean value of the environment variable, or the default value.
74 value = os.environ.get(var_name)
77 return value.lower()
in (
"true",
"1",
"yes",
"on")
81 """Read a path environment variable.
84 var_name: Name of the environment variable to read.
87 The path value of the environment variable, or None if not set.
89 value = os.environ.get(var_name)
90 return Path(value)
if value
else None
94 """Get a unique identifier for this pytest invocation.
96 For normal runs, uses the current process PID (the pytest process itself).
97 For xdist workers, uses the parent PID (the main pytest controller) so that
98 all workers share the same top-level run directory.
100 if os.environ.get(
"PYTEST_XDIST_WORKER"):
102 return f
"pytest-{os.getppid()}"
103 return f
"pytest-{os.getpid()}"
107 """Get the xdist worker ID if running under pytest-xdist.
109 Returns the worker ID (e.g., "gw0", "gw1") or None if not using xdist.
111 return os.environ.get(
"PYTEST_XDIST_WORKER")
115 """Build a test directory path including pytest run ID and xdist worker (if present).
117 For class-based tests with xdist: "pytest-{pid}/gw{n}/ClassName/test_method".
118 For class-based tests without xdist: "pytest-{pid}/ClassName/test_method".
119 For function tests with xdist: "pytest-{pid}/gw{n}/test_function".
120 For function tests without xdist: "pytest-{pid}/test_function".
124 if config.pytest_run_id:
125 parts.append(config.pytest_run_id)
127 if config.xdist_worker_id:
128 parts.append(config.xdist_worker_id)
130 if config.class_name:
131 parts.append(config.class_name)
134 parts.append(config.test_name)
137 return "unknown-test"
139 return "/".join(parts)
142@dataclass(frozen=True)
144 """Immutable configuration for a single cosim test or test class.
147 source_generator: Path to the PyCDE hardware generation script, or a
148 callable ``(config, tmp_dir) -> sources_dir``.
149 args: Arguments passed to the script; ``{tmp_dir}`` is interpolated.
150 simulator: Simulator backend name (e.g. ``"verilator"``).
151 top: Top-level module name for the simulator.
152 debug: If True, enable verbose simulator output.
153 timeout_s: Maximum wall-clock seconds before the test is killed.
154 failure_matcher: Pattern applied to simulator output to detect errors.
155 warning_matcher: Pattern applied to simulator output to detect warnings.
156 tmp_dir_root: Root directory for temporary directories. If None, uses system temp.
157 delete_tmp_dir: If True, delete temporary directories after test completion.
158 class_name: Name of the test class (for class-based tests). None for function tests.
159 test_name: Name of the test function or method.
160 pytest_run_id: Unique identifier for the pytest run (e.g., "pytest-12345").
161 xdist_worker_id: ID of the xdist worker if running under pytest-xdist (e.g., "gw0").
162 save_waveform: If True, dump waveform file. Format depends on backend. Requires debug=True.
165 source_generator: SourceGeneratorArg
166 args: Sequence[str] = (
"{tmp_dir}",)
167 simulator: str =
"verilator"
168 top: str =
"ESI_Cosim_Top"
170 timeout_s: float = _DEFAULT_TIMEOUT_S
171 failure_matcher: Optional[LogMatcher] = _DEFAULT_FAILURE_PATTERN
172 warning_matcher: Optional[LogMatcher] = _DEFAULT_WARN_PATTERN
173 tmp_dir_root: Optional[Path] =
None
174 delete_tmp_dir: bool =
True
175 class_name: Optional[str] =
None
176 test_name: Optional[str] =
None
177 pytest_run_id: Optional[str] =
None
178 xdist_worker_id: Optional[str] =
None
179 save_waveform: bool =
False
184 """Cached compilation artifacts shared across methods of a test class."""
192 """Outcome of a child-process test execution, passed back via queue."""
196 failure_lines: Sequence[str] = field(default_factory=list)
197 warning_lines: Sequence[str] = field(default_factory=list)
198 stdout_lines: Sequence[str] = field(default_factory=list)
199 stderr_lines: Sequence[str] = field(default_factory=list)
202@contextlib.contextmanager
204 """Context manager that temporarily changes the working directory."""
214 """Return True if *line* matches the given matcher.
216 The matcher may be a plain string (regex search), a compiled regex,
217 or a callable ``(line, stream) -> bool``.
219 if isinstance(matcher, str):
220 return bool(re.search(matcher, line, re.IGNORECASE))
221 elif isinstance(matcher, re.Pattern):
222 return bool(matcher.search(line))
224 return matcher(line, stream)
228 stdout_lines: Sequence[str],
229 stderr_lines: Sequence[str],
230 config: CosimPytestConfig,
231) -> tuple[list[str], list[str]]:
232 """Scan simulator output for failures and warnings.
235 A ``(failures, warnings)`` tuple of tagged log lines.
237 failures: list[str] = []
238 warnings: list[str] = []
240 for stream, lines
in ((
"stdout", stdout_lines), (
"stderr", stderr_lines)):
242 tagged = f
"[{stream}] {line}"
243 if config.failure_matcher
and _line_matches(config.failure_matcher, line,
245 failures.append(tagged)
246 if config.warning_matcher
and _line_matches(config.warning_matcher, line,
248 warnings.append(tagged)
250 return failures, warnings
254 """Interpolate ``{tmp_dir}`` placeholders in script arguments."""
255 return [arg.format(tmp_dir=tmp_dir)
for arg
in args]
259 """Generate hardware sources via a normalized source-generator callable.
261 If ``config.source_generator`` is a string/path, it is wrapped into a callable
262 that runs the PyCDE script. If it is already callable, it is used directly.
264 source_spec = config.source_generator
265 if isinstance(source_spec, (str, Path)):
266 source_generator: SourceGeneratorFunc = (
268 source_spec, inner_config, inner_tmp_dir))
270 source_generator = source_spec
271 return source_generator(config, tmp_dir)
275 run_dir: Path) -> Simulator:
276 """Instantiate a ``Simulator`` from the generated source files."""
278 hw_dir = sources_dir /
"hw"
279 sources.add_dir(hw_dir
if hw_dir.exists()
else sources_dir)
281 return get_simulator(config.simulator, sources, run_dir, config.debug,
282 config.save_waveform)
286 tmp_dir: Path) -> Path:
287 """Execute the PyCDE hardware script and run codegen if a manifest exists.
290 The directory containing the generated sources (same as *tmp_dir*).
292 script = Path(script_path).
resolve()
295 subprocess.run([sys.executable, str(script), *script_args],
300 manifest_path = tmp_dir /
"esi_system_manifest.json"
301 if manifest_path.exists():
302 generated_dir = tmp_dir /
"generated"
303 generated_dir.mkdir(parents=
True, exist_ok=
True)
307 sys.executable,
"-m",
"esiaccel.codegen",
"--file",
308 str(manifest_path),
"--output-dir",
314 except subprocess.CalledProcessError
as e:
316 _logger.warning(
"codegen failed (non-fatal): %s", e)
322 "host",
"hostname",
"port",
"sources_dir",
"conn",
"accelerator"
324_INJECTED_ANNOTATIONS = frozenset({Accelerator, AcceleratorConnection})
328 """Return True if *name*/*annotation* will be supplied by the decorator."""
329 return name
in _INJECTED_NAMES
or annotation
in _INJECTED_ANNOTATIONS
333 target: Callable[..., Any],
334 kwargs: Dict[str, Any],
337 sources_dir: Optional[Path] =
None,
339 """Build the keyword arguments to inject into the test function.
341 Inspects the target's signature and automatically supplies ``host``,
342 ``port``, ``sources_dir``, ``AcceleratorConnection``, or ``Accelerator``
343 parameters that the test declares but the caller did not provide.
345 sig = inspect.signature(target)
346 updated = dict(kwargs)
348 for name, param
in sig.parameters.items():
351 if name
in (
"host",
"hostname"):
355 elif name ==
"sources_dir" and sources_dir
is not None:
356 updated[name] = sources_dir
357 elif param.annotation
is AcceleratorConnection
or name ==
"conn":
359 elif param.annotation
is Accelerator
or name ==
"accelerator":
361 updated[name] = conn.build_accelerator()
367 """Return a signature with injected parameters removed.
369 Pytest uses function signatures to determine fixture requirements. This
370 hides the parameters that the decorator injects (``host``, ``port``,
371 ``conn``, etc.) so pytest does not try to resolve them as fixtures.
372 Uses :func:`_is_injected_param` as the single source of truth.
374 sig = inspect.signature(target)
376 p
for p
in sig.parameters.values()
379 return sig.replace(parameters=kept)
383 """Copy pre-compiled simulator artifacts into the per-test run directory.
385 Copies the *entire* compile directory so that all backends (Verilator,
386 Questa, etc.) find their artefacts regardless of internal layout.
388 if compile_dir
is None:
390 run_dir.mkdir(parents=
True, exist_ok=
True)
391 for item
in compile_dir.iterdir():
392 dst = run_dir / item.name
394 shutil.copytree(item, dst, dirs_exist_ok=
True)
396 shutil.copy2(item, dst)
400 """Run the hw script and compile the simulator once for a whole test class.
402 The resulting ``_ClassCompileCache`` is reused by each test method to avoid
403 redundant compilations.
407 if config.tmp_dir_root
is not None:
410 compile_root = config.tmp_dir_root / test_dir /
"__compile__"
411 compile_root.mkdir(parents=
True, exist_ok=
True)
413 compile_root = Path(tempfile.mkdtemp(prefix=
"esi-pytest-class-compile-",))
416 compile_dir = compile_root /
"compile"
421 raise RuntimeError(f
"Simulator compile failed with exit code {rc}")
424 if config.delete_tmp_dir
and not config.debug:
425 shutil.rmtree(compile_root, ignore_errors=
True)
430 result_pipe: Optional[multiprocessing.connection.Connection],
431 target: Callable[..., Any],
432 config: CosimPytestConfig,
434 kwargs: Dict[str, Any],
435 class_cache: Optional[_ClassCompileCache],
437 """Entry point for the forked child process.
439 Compiles (or reuses cached compilation), starts the simulator, injects
440 connection parameters, calls the test function, scans logs for failures,
441 and sends a ``_ChildResult`` through *result_pipe*. When *result_pipe* is
442 ``None``, returns the result directly for callers that already have process
443 isolation (Windows pytest-xdist workers).
447 compile_stdout: list[str] = []
448 compile_stderr: list[str] = []
449 stdout_lines: list[str] = []
450 stderr_lines: list[str] = []
452 def on_stdout(line: str):
453 stdout_lines.append(line)
455 def on_stderr(line: str):
456 stderr_lines.append(line)
465 if config.tmp_dir_root
is not None:
468 run_root = config.tmp_dir_root / test_dir
469 run_root.mkdir(parents=
True, exist_ok=
True)
471 run_root = Path(tempfile.mkdtemp(prefix=
"esi-pytest-run-",))
472 if class_cache
is None:
476 sim._run_stdout_cb = on_stdout
477 sim._run_stderr_cb = on_stderr
478 sim._compile_stdout_cb = compile_stdout.append
479 sim._compile_stderr_cb = compile_stderr.append
483 raise RuntimeError(f
"Simulator compile failed with exit code {rc}")
485 sources_dir = class_cache.sources_dir
488 sim._run_stdout_cb = on_stdout
489 sim._run_stderr_cb = on_stderr
493 sim_proc = sim.run_proc()
498 sources_dir=sources_dir)
499 target(*args, **injected_kwargs)
501 failure_lines, warning_lines =
_scan_logs(stdout_lines, stderr_lines,
504 raise AssertionError(
"Detected simulator failures:\n" +
505 "\n".join(failure_lines))
508 warning_lines=warning_lines,
509 failure_lines=failure_lines,
510 stdout_lines=compile_stdout + stdout_lines,
511 stderr_lines=compile_stderr + stderr_lines)
514 traceback=traceback.format_exc(),
515 warning_lines=
_scan_logs(stdout_lines, stderr_lines,
517 stdout_lines=compile_stdout + stdout_lines,
518 stderr_lines=compile_stderr + stderr_lines)
520 if sim_proc
is not None and sim_proc.proc.poll()
is None:
521 sim_proc.force_stop()
524 should_delete = config.delete_tmp_dir
and not config.debug
525 if run_root
is not None and should_delete:
526 shutil.rmtree(run_root, ignore_errors=
True)
528 if result_pipe
is not None:
530 result_pipe.send(result)
541 target: Callable[..., Any],
542 config: CosimPytestConfig,
544 kwargs: Dict[str, Any],
545 class_cache: Optional[_ClassCompileCache] =
None,
547 """Fork a child process to run *target* and wait for its result.
549 Handles timeouts, collects warnings, and re-raises any failure from
550 the child as an ``AssertionError`` in the parent.
553 ctx: Any = multiprocessing.get_context(
"fork")
558 import pytest
as _pytest
559 if os.environ.get(
"PYTEST_XDIST_WORKER")
is None:
560 _pytest.skip(
"Windows cosim tests require pytest-xdist isolation; run "
561 "with `pytest -n 1` or more")
564 inline_result =
_run_child(
None, target, config, args, kwargs, class_cache)
565 for line
in inline_result.stdout_lines:
566 _logger.debug(
"sim stdout: %s", line)
567 for line
in inline_result.stderr_lines:
568 _logger.debug(
"sim stderr: %s", line)
569 for warning
in inline_result.warning_lines:
570 _logger.warning(
"cosim warning: %s", warning)
571 if not inline_result.success:
572 parts = [inline_result.traceback]
573 if inline_result.stdout_lines:
574 parts.append(
"\n=== Simulator stdout ===")
575 parts.extend(inline_result.stdout_lines[-200:])
576 if inline_result.stderr_lines:
577 parts.append(
"\n=== Simulator stderr ===")
578 parts.extend(inline_result.stderr_lines[-200:])
579 raise AssertionError(
"\n".join(parts))
581 reader, writer = ctx.Pipe(duplex=
False)
582 process = ctx.Process(
584 args=(writer, target, config, args, kwargs, class_cache),
591 result: Optional[_ChildResult] =
None
592 if reader.poll(timeout=config.timeout_s):
593 result = reader.recv()
595 process.join(timeout=10)
596 if process.is_alive():
598 process.join(timeout=5)
601 if config.timeout_s
is not None:
602 raise AssertionError(
603 f
"Cosim test timed out after {config.timeout_s} seconds")
605 f
"Cosim child exited without returning a result (exit code: {process.exitcode})"
609 for line
in result.stdout_lines:
610 _logger.debug(
"sim stdout: %s", line)
611 for line
in result.stderr_lines:
612 _logger.debug(
"sim stderr: %s", line)
613 for warning
in result.warning_lines:
614 _logger.warning(
"cosim warning: %s", warning)
616 if not result.success:
617 parts = [result.traceback]
618 if result.stdout_lines:
619 parts.append(
"\n=== Simulator stdout ===")
620 parts.extend(result.stdout_lines[-200:])
621 if result.stderr_lines:
622 parts.append(
"\n=== Simulator stderr ===")
623 parts.extend(result.stderr_lines[-200:])
624 raise AssertionError(
"\n".join(parts))
628 target: Callable[..., Any],
629 config: CosimPytestConfig,
630 class_cache_getter: Optional[Callable[[], _ClassCompileCache]] =
None,
631) -> Callable[..., Any]:
632 """Wrap a single test function so it runs inside ``_run_isolated``."""
634 test_config = replace(config, test_name=target.__name__)
636 @functools.wraps(target)
637 def _wrapper(*args, **kwargs):
638 if not is_simulator_available(test_config.simulator):
639 available = available_simulators()
641 f
"Simulator '{test_config.simulator}' not available; available simulators: "
642 f
"{', '.join(available) if available else 'none'}")
643 cache = class_cache_getter()
if class_cache_getter
is not None else None
644 _run_isolated(target, test_config, args, kwargs, class_cache=cache)
651 """Wrap every ``test_*`` method of a class with cosim isolation.
653 Compilation is performed once (lazily, on first method invocation) and
654 the resulting artifacts are shared across all methods via a thread-safe
658 class_config = replace(config, class_name=target_cls.__name__)
659 lock = threading.Lock()
660 cache_holder: dict[str, _ClassCompileCache] = {}
662 def _get_cache() -> _ClassCompileCache:
664 if "cache" not in cache_holder:
666 return cache_holder[
"cache"]
668 for name, member
in list(vars(target_cls).items()):
669 if name.startswith(
"test")
and callable(member):
675 class_cache_getter=_get_cache),
681 source_generator: SourceGeneratorArg,
682 args: Sequence[str] = (
"{tmp_dir}",),
683 simulator: str =
"verilator",
684 top: str =
"ESI_Cosim_Top",
685 debug: Optional[bool] =
None,
686 timeout_s: float = _DEFAULT_TIMEOUT_S,
687 failure_matcher: Optional[LogMatcher] = _DEFAULT_FAILURE_PATTERN,
688 warning_matcher: Optional[LogMatcher] = _DEFAULT_WARN_PATTERN,
689 tmp_dir_root: Optional[Path] =
None,
690 delete_tmp_dir: Optional[bool] =
None,
691 save_waveform: Optional[bool] =
None,
693 """Decorator that turns a function or class into a cosimulation test.
695 The decorated target is executed in a forked child process with a freshly
696 compiled and running simulator. Connection parameters (``host``, ``port``,
697 ``acc``, etc.) are injected automatically based on the function signature.
699 When applied to a class, the hardware script is run and compiled once;
700 each ``test_*`` method gets its own simulator process but skips
704 source_generator: Path to the PyCDE script that generates the hardware, or
705 a callable ``(config, tmp_dir) -> sources_dir`` that generates
707 args: Arguments forwarded to the script; ``{tmp_dir}`` is interpolated
708 with the temporary build directory.
709 simulator: Simulator backend (default ``"verilator"``).
710 top: Top-level module name.
711 debug: Enable verbose simulator output. Defaults to the value of the
712 ``ESIACCEL_PYTEST_DEBUG`` environment variable if set, otherwise False.
713 timeout_s: Wall-clock timeout in seconds (default 120).
714 failure_matcher: Pattern to detect errors in simulator output.
715 warning_matcher: Pattern to detect warnings in simulator output.
716 tmp_dir_root: Root directory for temporary test files. Defaults to the value
717 of the ``ESIACCEL_PYTEST_TMP_DIR`` environment variable if set, otherwise
718 uses the system temporary directory. Run directories are organized in a
719 hierarchy to avoid collisions during parallel execution:
721 - Normal run: ``pytest-{pid}/ClassName/test_method/`` or
722 ``pytest-{pid}/test_function/``
723 - xdist parallel: ``pytest-{pid}/gw0/ClassName/test_method/`` (gw0, gw1, etc.
724 for different workers)
725 delete_tmp_dir: Whether to delete temporary directories after test
726 completion. When tmp_dir_root is set (custom debugging directory),
727 directories are kept by default; set ``ESIACCEL_PYTEST_DELETE_TMP_DIR=true``
728 to delete them. When using the system temp directory, defaults to True
729 (delete to avoid clutter). Always False when debug mode is enabled.
730 save_waveform: Whether to save waveform dumps (format depends on the
731 simulator backend, e.g. FST for Verilator, VCD for Questa). Requires
732 debug mode to be enabled. Defaults to the value of the
733 ``ESIACCEL_PYTEST_SAVE_WAVEFORM`` environment variable if set, otherwise
739 if tmp_dir_root
is None:
741 if delete_tmp_dir
is None:
745 default_delete = tmp_dir_root
is None
746 delete_tmp_dir =
_get_env_bool(
"ESIACCEL_PYTEST_DELETE_TMP_DIR",
748 if save_waveform
is None:
749 save_waveform =
_get_env_bool(
"ESIACCEL_PYTEST_SAVE_WAVEFORM",
False)
752 if save_waveform
and not debug:
754 "save_waveform requires debug mode to be enabled; disabling waveform dumping"
756 save_waveform =
False
763 source_generator=source_generator,
769 failure_matcher=failure_matcher,
770 warning_matcher=warning_matcher,
771 tmp_dir_root=tmp_dir_root,
772 delete_tmp_dir=delete_tmp_dir,
773 pytest_run_id=pytest_run_id,
774 xdist_worker_id=xdist_worker_id,
775 save_waveform=save_waveform,
778 def _decorator(target):
779 if inspect.isclass(target):
783 raise TypeError(
"@cosim_test can decorate functions or classes")
static mlir::Operation * resolve(Context &context, mlir::SymbolRefAttr sym)
"AcceleratorConnection" connect(str platform, str connection_str)
Optional[Path] _get_env_path(str var_name)
_copy_compiled_artifacts(Optional[Path] compile_dir, Path run_dir)
type _decorate_class(type target_cls, CosimPytestConfig config)
_run_isolated(Callable[..., Any] target, CosimPytestConfig config, Sequence[Any] args, Dict[str, Any] kwargs, Optional[_ClassCompileCache] class_cache=None)
tuple[list[str], list[str]] _scan_logs(Sequence[str] stdout_lines, Sequence[str] stderr_lines, CosimPytestConfig config)
Dict[str, Any] _resolve_injected_params(Callable[..., Any] target, Dict[str, Any] kwargs, str host, int port, Optional[Path] sources_dir=None)
Path _run_hw_script(Union[str, Path] script_path, CosimPytestConfig config, Path tmp_dir)
Path _generate_sources(CosimPytestConfig config, Path tmp_dir)
bool _line_matches(LogMatcher matcher, str line, str stream)
str _get_test_dir_name(CosimPytestConfig config)
Simulator _create_simulator(CosimPytestConfig config, Path sources_dir, Path run_dir)
list[str] _render_args(Sequence[str] args, Path tmp_dir)
Optional[str] _get_xdist_worker_id()
inspect.Signature _visible_signature(Callable[..., Any] target)
bool _get_env_bool(str var_name, bool default)
bool _is_injected_param(str name, Any annotation)
_ChildResult _run_child(Optional[multiprocessing.connection.Connection] result_pipe, Callable[..., Any] target, CosimPytestConfig config, Sequence[Any] args, Dict[str, Any] kwargs, Optional[_ClassCompileCache] class_cache)
_ClassCompileCache _compile_once_for_class(CosimPytestConfig config)
Callable[..., Any] _decorate_function(Callable[..., Any] target, CosimPytestConfig config, Optional[Callable[[], _ClassCompileCache]] class_cache_getter=None)