4"""Pytest integration for ESI cosimulation tests.
6Provides the ``@cosim_test`` decorator which automates the full lifecycle of a
7cosimulation test: running a PyCDE hardware script, compiling the design with
8a simulator (e.g. Verilator), launching the simulator, injecting connection
9parameters into the test function, and tearing everything down afterwards.
11Decorated functions run in an isolated child process (via ``fork``) so that
12simulator state never leaks between tests. When applied to a class, the
13hardware compilation is performed once and shared across all ``test_*`` methods.
17 from esiaccel.cosim.pytest import cosim_test
19 @cosim_test("path/to/hw_script.py")
20 def test_my_design(conn: AcceleratorConnection):
21 # conn is already connected to the running simulator
25from __future__
import annotations
28from dataclasses
import dataclass, field, replace
33import multiprocessing.connection
35from pathlib
import Path
43from typing
import Any, Callable, Dict, Optional, Pattern, Sequence, Union
48from .simulator
import get_simulator, Simulator, SourceFiles
50LogMatcher = Union[str, Pattern[str], Callable[[str, str], bool]]
51SourceGeneratorFunc = Callable[[
"CosimPytestConfig", Path], Path]
52SourceGeneratorArg = Union[str, Path, SourceGeneratorFunc]
54_logger = logging.getLogger(
"esiaccel.cosim.pytest")
55_DEFAULT_FAILURE_PATTERN = re.compile(
r"\berror\b", re.IGNORECASE)
56_DEFAULT_WARN_PATTERN = re.compile(
r"\bwarn(ing)?\b", re.IGNORECASE)
59_DEFAULT_TIMEOUT_S: float = 120.0
63 """Read a boolean environment variable.
66 var_name: Name of the environment variable to read.
67 default: Default value if the variable is not set.
70 The boolean value of the environment variable, or the default value.
72 value = os.environ.get(var_name)
75 return value.lower()
in (
"true",
"1",
"yes",
"on")
79 """Read a path environment variable.
82 var_name: Name of the environment variable to read.
85 The path value of the environment variable, or None if not set.
87 value = os.environ.get(var_name)
88 return Path(value)
if value
else None
92 """Get a unique identifier for this pytest invocation.
94 For normal runs, uses the current process PID (the pytest process itself).
95 For xdist workers, uses the parent PID (the main pytest controller) so that
96 all workers share the same top-level run directory.
98 if os.environ.get(
"PYTEST_XDIST_WORKER"):
100 return f
"pytest-{os.getppid()}"
101 return f
"pytest-{os.getpid()}"
105 """Get the xdist worker ID if running under pytest-xdist.
107 Returns the worker ID (e.g., "gw0", "gw1") or None if not using xdist.
109 return os.environ.get(
"PYTEST_XDIST_WORKER")
113 """Build a test directory path including pytest run ID and xdist worker (if present).
115 For class-based tests with xdist: "pytest-{pid}/gw{n}/ClassName/test_method".
116 For class-based tests without xdist: "pytest-{pid}/ClassName/test_method".
117 For function tests with xdist: "pytest-{pid}/gw{n}/test_function".
118 For function tests without xdist: "pytest-{pid}/test_function".
122 if config.pytest_run_id:
123 parts.append(config.pytest_run_id)
125 if config.xdist_worker_id:
126 parts.append(config.xdist_worker_id)
128 if config.class_name:
129 parts.append(config.class_name)
132 parts.append(config.test_name)
135 return "unknown-test"
137 return "/".join(parts)
140@dataclass(frozen=True)
142 """Immutable configuration for a single cosim test or test class.
145 source_generator: Path to the PyCDE hardware generation script, or a
146 callable ``(config, tmp_dir) -> sources_dir``.
147 args: Arguments passed to the script; ``{tmp_dir}`` is interpolated.
148 simulator: Simulator backend name (e.g. ``"verilator"``).
149 top: Top-level module name for the simulator.
150 debug: If True, enable verbose simulator output.
151 timeout_s: Maximum wall-clock seconds before the test is killed.
152 failure_matcher: Pattern applied to simulator output to detect errors.
153 warning_matcher: Pattern applied to simulator output to detect warnings.
154 tmp_dir_root: Root directory for temporary directories. If None, uses system temp.
155 delete_tmp_dir: If True, delete temporary directories after test completion.
156 class_name: Name of the test class (for class-based tests). None for function tests.
157 test_name: Name of the test function or method.
158 pytest_run_id: Unique identifier for the pytest run (e.g., "pytest-12345").
159 xdist_worker_id: ID of the xdist worker if running under pytest-xdist (e.g., "gw0").
160 save_waveform: If True, dump waveform file. Format depends on backend. Requires debug=True.
163 source_generator: SourceGeneratorArg
164 args: Sequence[str] = (
"{tmp_dir}",)
165 simulator: str =
"verilator"
166 top: str =
"ESI_Cosim_Top"
168 timeout_s: float = _DEFAULT_TIMEOUT_S
169 failure_matcher: Optional[LogMatcher] = _DEFAULT_FAILURE_PATTERN
170 warning_matcher: Optional[LogMatcher] = _DEFAULT_WARN_PATTERN
171 tmp_dir_root: Optional[Path] =
None
172 delete_tmp_dir: bool =
True
173 class_name: Optional[str] =
None
174 test_name: Optional[str] =
None
175 pytest_run_id: Optional[str] =
None
176 xdist_worker_id: Optional[str] =
None
177 save_waveform: bool =
False
182 """Cached compilation artifacts shared across methods of a test class."""
190 """Outcome of a child-process test execution, passed back via queue."""
194 failure_lines: Sequence[str] = field(default_factory=list)
195 warning_lines: Sequence[str] = field(default_factory=list)
196 stdout_lines: Sequence[str] = field(default_factory=list)
197 stderr_lines: Sequence[str] = field(default_factory=list)
200@contextlib.contextmanager
202 """Context manager that temporarily changes the working directory."""
212 """Return True if *line* matches the given matcher.
214 The matcher may be a plain string (regex search), a compiled regex,
215 or a callable ``(line, stream) -> bool``.
217 if isinstance(matcher, str):
218 return bool(re.search(matcher, line, re.IGNORECASE))
219 elif isinstance(matcher, re.Pattern):
220 return bool(matcher.search(line))
222 return matcher(line, stream)
226 stdout_lines: Sequence[str],
227 stderr_lines: Sequence[str],
228 config: CosimPytestConfig,
229) -> tuple[list[str], list[str]]:
230 """Scan simulator output for failures and warnings.
233 A ``(failures, warnings)`` tuple of tagged log lines.
235 failures: list[str] = []
236 warnings: list[str] = []
238 for stream, lines
in ((
"stdout", stdout_lines), (
"stderr", stderr_lines)):
240 tagged = f
"[{stream}] {line}"
241 if config.failure_matcher
and _line_matches(config.failure_matcher, line,
243 failures.append(tagged)
244 if config.warning_matcher
and _line_matches(config.warning_matcher, line,
246 warnings.append(tagged)
248 return failures, warnings
252 """Interpolate ``{tmp_dir}`` placeholders in script arguments."""
253 return [arg.format(tmp_dir=tmp_dir)
for arg
in args]
257 """Generate hardware sources via a normalized source-generator callable.
259 If ``config.source_generator`` is a string/path, it is wrapped into a callable
260 that runs the PyCDE script. If it is already callable, it is used directly.
262 source_spec = config.source_generator
263 if isinstance(source_spec, (str, Path)):
264 source_generator: SourceGeneratorFunc = (
266 source_spec, inner_config, inner_tmp_dir))
268 source_generator = source_spec
269 return source_generator(config, tmp_dir)
273 run_dir: Path) -> Simulator:
274 """Instantiate a ``Simulator`` from the generated source files."""
276 hw_dir = sources_dir /
"hw"
277 sources.add_dir(hw_dir
if hw_dir.exists()
else sources_dir)
279 return get_simulator(config.simulator, sources, run_dir, config.debug,
280 config.save_waveform)
284 tmp_dir: Path) -> Path:
285 """Execute the PyCDE hardware script and run codegen if a manifest exists.
288 The directory containing the generated sources (same as *tmp_dir*).
290 script = Path(script_path).
resolve()
293 subprocess.run([sys.executable, str(script), *script_args],
298 manifest_path = tmp_dir /
"esi_system_manifest.json"
299 if manifest_path.exists():
300 generated_dir = tmp_dir /
"generated"
301 generated_dir.mkdir(parents=
True, exist_ok=
True)
305 sys.executable,
"-m",
"esiaccel.codegen",
"--file",
306 str(manifest_path),
"--output-dir",
312 except subprocess.CalledProcessError
as e:
314 _logger.warning(
"codegen failed (non-fatal): %s", e)
320 "host",
"hostname",
"port",
"sources_dir",
"conn",
"accelerator"
322_INJECTED_ANNOTATIONS = frozenset({Accelerator, AcceleratorConnection})
326 """Return True if *name*/*annotation* will be supplied by the decorator."""
327 return name
in _INJECTED_NAMES
or annotation
in _INJECTED_ANNOTATIONS
331 target: Callable[..., Any],
332 kwargs: Dict[str, Any],
335 sources_dir: Optional[Path] =
None,
337 """Build the keyword arguments to inject into the test function.
339 Inspects the target's signature and automatically supplies ``host``,
340 ``port``, ``sources_dir``, ``AcceleratorConnection``, or ``Accelerator``
341 parameters that the test declares but the caller did not provide.
343 sig = inspect.signature(target)
344 updated = dict(kwargs)
346 for name, param
in sig.parameters.items():
349 if name
in (
"host",
"hostname"):
353 elif name ==
"sources_dir" and sources_dir
is not None:
354 updated[name] = sources_dir
355 elif param.annotation
is AcceleratorConnection
or name ==
"conn":
357 elif param.annotation
is Accelerator
or name ==
"accelerator":
359 updated[name] = conn.build_accelerator()
365 """Return a signature with injected parameters removed.
367 Pytest uses function signatures to determine fixture requirements. This
368 hides the parameters that the decorator injects (``host``, ``port``,
369 ``conn``, etc.) so pytest does not try to resolve them as fixtures.
370 Uses :func:`_is_injected_param` as the single source of truth.
372 sig = inspect.signature(target)
374 p
for p
in sig.parameters.values()
377 return sig.replace(parameters=kept)
381 """Copy pre-compiled simulator artifacts into the per-test run directory.
383 Copies the *entire* compile directory so that all backends (Verilator,
384 Questa, etc.) find their artefacts regardless of internal layout.
386 if compile_dir
is None:
388 run_dir.mkdir(parents=
True, exist_ok=
True)
389 for item
in compile_dir.iterdir():
390 dst = run_dir / item.name
392 shutil.copytree(item, dst, dirs_exist_ok=
True)
394 shutil.copy2(item, dst)
398 """Run the hw script and compile the simulator once for a whole test class.
400 The resulting ``_ClassCompileCache`` is reused by each test method to avoid
401 redundant compilations.
405 if config.tmp_dir_root
is not None:
408 compile_root = config.tmp_dir_root / test_dir /
"__compile__"
409 compile_root.mkdir(parents=
True, exist_ok=
True)
411 compile_root = Path(tempfile.mkdtemp(prefix=
"esi-pytest-class-compile-",))
414 compile_dir = compile_root /
"compile"
419 raise RuntimeError(f
"Simulator compile failed with exit code {rc}")
422 if config.delete_tmp_dir
and not config.debug:
423 shutil.rmtree(compile_root, ignore_errors=
True)
428 result_pipe: multiprocessing.connection.Connection,
429 target: Callable[..., Any],
430 config: CosimPytestConfig,
432 kwargs: Dict[str, Any],
433 class_cache: Optional[_ClassCompileCache],
435 """Entry point for the forked child process.
437 Compiles (or reuses cached compilation), starts the simulator, injects
438 connection parameters, calls the test function, scans logs for failures,
439 and sends a ``_ChildResult`` through *result_pipe*.
443 compile_stdout: list[str] = []
444 compile_stderr: list[str] = []
445 stdout_lines: list[str] = []
446 stderr_lines: list[str] = []
448 def on_stdout(line: str):
449 stdout_lines.append(line)
451 def on_stderr(line: str):
452 stderr_lines.append(line)
461 if config.tmp_dir_root
is not None:
464 run_root = config.tmp_dir_root / test_dir
465 run_root.mkdir(parents=
True, exist_ok=
True)
467 run_root = Path(tempfile.mkdtemp(prefix=
"esi-pytest-run-",))
468 if class_cache
is None:
472 sim._run_stdout_cb = on_stdout
473 sim._run_stderr_cb = on_stderr
474 sim._compile_stdout_cb = compile_stdout.append
475 sim._compile_stderr_cb = compile_stderr.append
479 raise RuntimeError(f
"Simulator compile failed with exit code {rc}")
481 sources_dir = class_cache.sources_dir
484 sim._run_stdout_cb = on_stdout
485 sim._run_stderr_cb = on_stderr
489 sim_proc = sim.run_proc()
494 sources_dir=sources_dir)
495 target(*args, **injected_kwargs)
497 failure_lines, warning_lines =
_scan_logs(stdout_lines, stderr_lines,
500 raise AssertionError(
"Detected simulator failures:\n" +
501 "\n".join(failure_lines))
505 all_stdout = compile_stdout + stdout_lines
506 all_stderr = compile_stderr + stderr_lines
509 warning_lines=warning_lines,
510 failure_lines=failure_lines,
511 stdout_lines=all_stdout,
512 stderr_lines=all_stderr))
514 all_stdout = compile_stdout + stdout_lines
515 all_stderr = compile_stderr + stderr_lines
518 traceback=traceback.format_exc(),
519 warning_lines=
_scan_logs(stdout_lines, stderr_lines,
521 stdout_lines=all_stdout,
522 stderr_lines=all_stderr))
525 if sim_proc
is not None and sim_proc.proc.poll()
is None:
526 sim_proc.force_stop()
529 should_delete = config.delete_tmp_dir
and not config.debug
530 if run_root
is not None and should_delete:
531 shutil.rmtree(run_root, ignore_errors=
True)
539 target: Callable[..., Any],
540 config: CosimPytestConfig,
542 kwargs: Dict[str, Any],
543 class_cache: Optional[_ClassCompileCache] =
None,
545 """Fork a child process to run *target* and wait for its result.
547 Handles timeouts, collects warnings, and re-raises any failure from
548 the child as an ``AssertionError`` in the parent.
551 ctx = multiprocessing.get_context(
"fork")
553 import pytest
as _pytest
554 _pytest.skip(
"fork start method unavailable on this platform")
555 reader, writer = ctx.Pipe(duplex=
False)
556 process = ctx.Process(
558 args=(writer, target, config, args, kwargs, class_cache),
565 result: Optional[_ChildResult] =
None
566 if reader.poll(timeout=config.timeout_s):
567 result = reader.recv()
569 process.join(timeout=10)
570 if process.is_alive():
572 process.join(timeout=5)
575 if config.timeout_s
is not None:
576 raise AssertionError(
577 f
"Cosim test timed out after {config.timeout_s} seconds")
579 f
"Cosim child exited without returning a result (exit code: {process.exitcode})"
583 for line
in result.stdout_lines:
584 _logger.debug(
"sim stdout: %s", line)
585 for line
in result.stderr_lines:
586 _logger.debug(
"sim stderr: %s", line)
587 for warning
in result.warning_lines:
588 _logger.warning(
"cosim warning: %s", warning)
590 if not result.success:
591 parts = [result.traceback]
592 if result.stdout_lines:
593 parts.append(
"\n=== Simulator stdout ===")
594 parts.extend(result.stdout_lines[-200:])
595 if result.stderr_lines:
596 parts.append(
"\n=== Simulator stderr ===")
597 parts.extend(result.stderr_lines[-200:])
598 raise AssertionError(
"\n".join(parts))
602 target: Callable[..., Any],
603 config: CosimPytestConfig,
604 class_cache_getter: Optional[Callable[[], _ClassCompileCache]] =
None,
605) -> Callable[..., Any]:
606 """Wrap a single test function so it runs inside ``_run_isolated``."""
608 test_config = replace(config, test_name=target.__name__)
610 @functools.wraps(target)
611 def _wrapper(*args, **kwargs):
612 cache = class_cache_getter()
if class_cache_getter
is not None else None
613 _run_isolated(target, test_config, args, kwargs, class_cache=cache)
620 """Wrap every ``test_*`` method of a class with cosim isolation.
622 Compilation is performed once (lazily, on first method invocation) and
623 the resulting artifacts are shared across all methods via a thread-safe
627 class_config = replace(config, class_name=target_cls.__name__)
628 lock = threading.Lock()
629 cache_holder: dict[str, _ClassCompileCache] = {}
631 def _get_cache() -> _ClassCompileCache:
633 if "cache" not in cache_holder:
635 return cache_holder[
"cache"]
637 for name, member
in list(vars(target_cls).items()):
638 if name.startswith(
"test")
and callable(member):
644 class_cache_getter=_get_cache),
650 source_generator: SourceGeneratorArg,
651 args: Sequence[str] = (
"{tmp_dir}",),
652 simulator: str =
"verilator",
653 top: str =
"ESI_Cosim_Top",
654 debug: Optional[bool] =
None,
655 timeout_s: float = _DEFAULT_TIMEOUT_S,
656 failure_matcher: Optional[LogMatcher] = _DEFAULT_FAILURE_PATTERN,
657 warning_matcher: Optional[LogMatcher] = _DEFAULT_WARN_PATTERN,
658 tmp_dir_root: Optional[Path] =
None,
659 delete_tmp_dir: Optional[bool] =
None,
660 save_waveform: Optional[bool] =
None,
662 """Decorator that turns a function or class into a cosimulation test.
664 The decorated target is executed in a forked child process with a freshly
665 compiled and running simulator. Connection parameters (``host``, ``port``,
666 ``acc``, etc.) are injected automatically based on the function signature.
668 When applied to a class, the hardware script is run and compiled once;
669 each ``test_*`` method gets its own simulator process but skips
673 source_generator: Path to the PyCDE script that generates the hardware, or
674 a callable ``(config, tmp_dir) -> sources_dir`` that generates
676 args: Arguments forwarded to the script; ``{tmp_dir}`` is interpolated
677 with the temporary build directory.
678 simulator: Simulator backend (default ``"verilator"``).
679 top: Top-level module name.
680 debug: Enable verbose simulator output. Defaults to the value of the
681 ``ESIACCEL_PYTEST_DEBUG`` environment variable if set, otherwise False.
682 timeout_s: Wall-clock timeout in seconds (default 120).
683 failure_matcher: Pattern to detect errors in simulator output.
684 warning_matcher: Pattern to detect warnings in simulator output.
685 tmp_dir_root: Root directory for temporary test files. Defaults to the value
686 of the ``ESIACCEL_PYTEST_TMP_DIR`` environment variable if set, otherwise
687 uses the system temporary directory. Run directories are organized in a
688 hierarchy to avoid collisions during parallel execution:
690 - Normal run: ``pytest-{pid}/ClassName/test_method/`` or
691 ``pytest-{pid}/test_function/``
692 - xdist parallel: ``pytest-{pid}/gw0/ClassName/test_method/`` (gw0, gw1, etc.
693 for different workers)
694 delete_tmp_dir: Whether to delete temporary directories after test
695 completion. When tmp_dir_root is set (custom debugging directory),
696 directories are kept by default; set ``ESIACCEL_PYTEST_DELETE_TMP_DIR=true``
697 to delete them. When using the system temp directory, defaults to True
698 (delete to avoid clutter). Always False when debug mode is enabled.
699 save_waveform: Whether to save waveform dumps (format depends on the
700 simulator backend, e.g. FST for Verilator, VCD for Questa). Requires
701 debug mode to be enabled. Defaults to the value of the
702 ``ESIACCEL_PYTEST_SAVE_WAVEFORM`` environment variable if set, otherwise
708 if tmp_dir_root
is None:
710 if delete_tmp_dir
is None:
714 default_delete = tmp_dir_root
is None
715 delete_tmp_dir =
_get_env_bool(
"ESIACCEL_PYTEST_DELETE_TMP_DIR",
717 if save_waveform
is None:
718 save_waveform =
_get_env_bool(
"ESIACCEL_PYTEST_SAVE_WAVEFORM",
False)
721 if save_waveform
and not debug:
723 "save_waveform requires debug mode to be enabled; disabling waveform dumping"
725 save_waveform =
False
732 source_generator=source_generator,
738 failure_matcher=failure_matcher,
739 warning_matcher=warning_matcher,
740 tmp_dir_root=tmp_dir_root,
741 delete_tmp_dir=delete_tmp_dir,
742 pytest_run_id=pytest_run_id,
743 xdist_worker_id=xdist_worker_id,
744 save_waveform=save_waveform,
747 def _decorator(target):
748 if inspect.isclass(target):
752 raise TypeError(
"@cosim_test can decorate functions or classes")
static mlir::Operation * resolve(Context &context, mlir::SymbolRefAttr sym)
"AcceleratorConnection" connect(str platform, str connection_str)
Optional[Path] _get_env_path(str var_name)
_copy_compiled_artifacts(Optional[Path] compile_dir, Path run_dir)
type _decorate_class(type target_cls, CosimPytestConfig config)
_run_isolated(Callable[..., Any] target, CosimPytestConfig config, Sequence[Any] args, Dict[str, Any] kwargs, Optional[_ClassCompileCache] class_cache=None)
_run_child(multiprocessing.connection.Connection result_pipe, Callable[..., Any] target, CosimPytestConfig config, Sequence[Any] args, Dict[str, Any] kwargs, Optional[_ClassCompileCache] class_cache)
tuple[list[str], list[str]] _scan_logs(Sequence[str] stdout_lines, Sequence[str] stderr_lines, CosimPytestConfig config)
Dict[str, Any] _resolve_injected_params(Callable[..., Any] target, Dict[str, Any] kwargs, str host, int port, Optional[Path] sources_dir=None)
Path _run_hw_script(Union[str, Path] script_path, CosimPytestConfig config, Path tmp_dir)
Path _generate_sources(CosimPytestConfig config, Path tmp_dir)
bool _line_matches(LogMatcher matcher, str line, str stream)
str _get_test_dir_name(CosimPytestConfig config)
Simulator _create_simulator(CosimPytestConfig config, Path sources_dir, Path run_dir)
list[str] _render_args(Sequence[str] args, Path tmp_dir)
Optional[str] _get_xdist_worker_id()
inspect.Signature _visible_signature(Callable[..., Any] target)
bool _get_env_bool(str var_name, bool default)
bool _is_injected_param(str name, Any annotation)
_ClassCompileCache _compile_once_for_class(CosimPytestConfig config)
Callable[..., Any] _decorate_function(Callable[..., Any] target, CosimPytestConfig config, Optional[Callable[[], _ClassCompileCache]] class_cache_getter=None)