CIRCT  19.0.0git
esi-cosim.py
Go to the documentation of this file.
1 #!/usr/bin/env python3
2 
3 # ===- esi-cosim.py - ESI cosimulation launch utility --------*- python -*-===//
4 #
5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
6 # See https://llvm.org/LICENSE.txt for license information.
7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
8 #
9 # ===----------------------------------------------------------------------===//
10 #
11 # Utility script to start a simulation and launch a command to interact with it
12 # via ESI cosimulation.
13 #
14 # ===----------------------------------------------------------------------===//
15 
16 import argparse
17 import os
18 from pathlib import Path
19 import re
20 import signal
21 import socket
22 import subprocess
23 import sys
24 import textwrap
25 import time
26 from typing import Dict, List
27 
28 _thisdir = Path(__file__).parent
29 CosimCollateralDir = _thisdir.parent / "cosim"
30 
31 
32 def is_port_open(port) -> bool:
33  """Check if a TCP port is open locally."""
34  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
35  result = sock.connect_ex(('127.0.0.1', port))
36  sock.close()
37  return True if result == 0 else False
38 
39 
41 
42  def __init__(self, top: str) -> None:
43  # User source files.
44  self.user: List[Path] = []
45  # DPI shared objects.
46  self.dpi_so: List[str] = ["EsiCosimDpiServer"]
47  # DPI SV files.
48  self.dpi_sv: List[Path] = [
49  CosimCollateralDir / "Cosim_DpiPkg.sv",
50  CosimCollateralDir / "Cosim_Endpoint.sv",
51  CosimCollateralDir / "Cosim_Manifest.sv",
52  CosimCollateralDir / "Cosim_MMIO.sv",
53  ]
54  # Name of the top module.
55  self.toptop = top
56 
57  def add_dir(self, dir: Path):
58  """Add all the RTL files in a directory to the source list."""
59  for file in sorted(dir.iterdir()):
60  if file.is_file() and (file.suffix == ".sv" or file.suffix == ".v"):
61  self.user.append(file)
62  elif file.is_dir():
63  self.add_diradd_dir(file)
64 
65  def dpi_so_paths(self) -> List[Path]:
66  """Return a list of all the DPI shared object files."""
67 
68  def find_so(name: str) -> Path:
69  for path in Simulator.get_env().get("LD_LIBRARY_PATH", "").split(":"):
70  if os.name == "nt":
71  so = Path(path) / f"{name}.dll"
72  else:
73  so = Path(path) / f"lib{name}.so"
74  if so.exists():
75  return so
76  raise FileNotFoundError(f"Could not find {name}.so in LD_LIBRARY_PATH")
77 
78  return [find_so(name) for name in self.dpi_so]
79 
80  @property
81  def rtl_sources(self) -> List[Path]:
82  """Return a list of all the RTL source files."""
83  return self.dpi_sv + self.user
84 
85 
86 class Simulator:
87 
88  # Some RTL simulators don't use stderr for error messages. Everything goes to
89  # stdout. Boo! They should feel bad about this. Also, they can specify that
90  # broken behavior by overriding this.
91  UsesStderr = True
92 
93  def __init__(self, sources: SourceFiles, run_dir: Path, debug: bool):
94  self.sourcessources = sources
95  self.run_dirrun_dir = run_dir
96  self.debugdebug = debug
97 
98  @staticmethod
99  def get_env() -> Dict[str, str]:
100  """Get the environment variables to locate shared objects."""
101 
102  env = os.environ.copy()
103  env["LIBRARY_PATH"] = env.get("LIBRARY_PATH", "") + ":" + str(
104  _thisdir.parent / "lib")
105  env["LD_LIBRARY_PATH"] = env.get("LD_LIBRARY_PATH", "") + ":" + str(
106  _thisdir.parent / "lib")
107  return env
108 
109  def compile_commands(self) -> List[List[str]]:
110  """Compile the sources. Returns the exit code of the simulation compiler."""
111  assert False, "Must be implemented by subclass"
112 
113  def compile(self) -> int:
114  cmds = self.compile_commandscompile_commands()
115  self.run_dirrun_dir.mkdir(parents=True, exist_ok=True)
116  with (self.run_dirrun_dir / "compile_stdout.log").open("w") as stdout, (
117  self.run_dirrun_dir / "compile_stderr.log").open("w") as stderr:
118  for cmd in cmds:
119  cp = subprocess.run(cmd,
120  env=Simulator.get_env(),
121  capture_output=True,
122  text=True)
123  stdout.write(cp.stdout)
124  stderr.write(cp.stderr)
125  if cp.returncode != 0:
126  print("====== Compilation failure:")
127  if self.UsesStderrUsesStderr:
128  print(cp.stderr)
129  else:
130  print(cp.stdout)
131  return cp.returncode
132  return 0
133 
134  def run_command(self, gui: bool) -> List[str]:
135  """Return the command to run the simulation."""
136  assert False, "Must be implemented by subclass"
137 
138  def run(self, inner_command: str, gui: bool = False) -> int:
139  """Start the simulation then run the command specified. Kill the simulation
140  when the command exits."""
141 
142  # 'simProc' is accessed in the finally block. Declare it here to avoid
143  # syntax errors in that block.
144  simProc = None
145  try:
146  # Open log files
147  self.run_dirrun_dir.mkdir(parents=True, exist_ok=True)
148  simStdout = open(self.run_dirrun_dir / "sim_stdout.log", "w")
149  simStderr = open(self.run_dirrun_dir / "sim_stderr.log", "w")
150 
151  # Erase the config file if it exists. We don't want to read
152  # an old config.
153  portFileName = self.run_dirrun_dir / "cosim.cfg"
154  if os.path.exists(portFileName):
155  os.remove(portFileName)
156 
157  # Run the simulation.
158  simEnv = Simulator.get_env()
159  if self.debugdebug:
160  simEnv["COSIM_DEBUG_FILE"] = "cosim_debug.log"
161  simProc = subprocess.Popen(self.run_commandrun_command(gui),
162  stdout=simStdout,
163  stderr=simStderr,
164  env=simEnv,
165  cwd=self.run_dirrun_dir,
166  preexec_fn=os.setsid)
167  simStderr.close()
168  simStdout.close()
169 
170  # Get the port which the simulation RPC selected.
171  checkCount = 0
172  while (not os.path.exists(portFileName)) and \
173  simProc.poll() is None:
174  time.sleep(0.1)
175  checkCount += 1
176  if checkCount > 200 and not gui:
177  raise Exception(f"Cosim never wrote cfg file: {portFileName}")
178  port = -1
179  while port < 0:
180  portFile = open(portFileName, "r")
181  for line in portFile.readlines():
182  m = re.match("port: (\\d+)", line)
183  if m is not None:
184  port = int(m.group(1))
185  portFile.close()
186 
187  # Wait for the simulation to start accepting RPC connections.
188  checkCount = 0
189  while not is_port_open(port):
190  checkCount += 1
191  if checkCount > 200:
192  raise Exception(f"Cosim RPC port ({port}) never opened")
193  if simProc.poll() is not None:
194  raise Exception("Simulation exited early")
195  time.sleep(0.05)
196 
197  # Run the inner command, passing the connection info via environment vars.
198  testEnv = os.environ.copy()
199  testEnv["ESI_COSIM_PORT"] = str(port)
200  testEnv["ESI_COSIM_HOST"] = "localhost"
201  return subprocess.run(inner_command, cwd=os.getcwd(),
202  env=testEnv).returncode
203  finally:
204  # Make sure to stop the simulation no matter what.
205  if simProc:
206  os.killpg(os.getpgid(simProc.pid), signal.SIGINT)
207  # Allow the simulation time to flush its outputs.
208  try:
209  simProc.wait(timeout=1.0)
210  except subprocess.TimeoutExpired:
211  # If the simulation doesn't exit of its own free will, kill it.
212  simProc.kill()
213 
214 
216  """Run and compile funcs for Verilator."""
217 
218  DefaultDriver = CosimCollateralDir / "driver.cpp"
219 
220  def __init__(self, sources: SourceFiles, run_dir: Path, debug: bool):
221  super().__init__(sources, run_dir, debug)
222 
223  self.verilatorverilator = "verilator"
224  if "VERILATOR_PATH" in os.environ:
225  self.verilatorverilator = os.environ["VERILATOR_PATH"]
226 
227  def compile_commands(self) -> List[List[str]]:
228  cmd: List[str] = [
229  self.verilatorverilator,
230  "--cc",
231  "--top-module",
232  self.sourcessources.top,
233  "-DSIMULATION",
234  "-sv",
235  "--build",
236  "--exe",
237  "--assert",
238  str(Verilator.DefaultDriver),
239  ]
240  cflags = [
241  "-DTOP_MODULE=" + self.sourcessources.top,
242  ]
243  if self.debugdebug:
244  cmd += ["--trace", "--trace-params", "--trace-structs"]
245  cflags.append("-DTRACE")
246  if len(cflags) > 0:
247  cmd += ["-CFLAGS", " ".join(cflags)]
248  if len(self.sourcessources.dpi_so) > 0:
249  cmd += ["-LDFLAGS", " ".join(["-l" + so for so in self.sourcessources.dpi_so])]
250  cmd += [str(p) for p in self.sourcessources.rtl_sources]
251  return [cmd]
252 
253  def run_command(self, gui: bool):
254  if gui:
255  raise RuntimeError("Verilator does not support GUI mode.")
256  exe = Path.cwd() / "obj_dir" / ("V" + self.sourcessources.top)
257  return [str(exe)]
258 
259 
261  """Run and compile funcs for Questasim."""
262 
263  DefaultDriver = CosimCollateralDir / "driver.sv"
264 
265  # Questa doesn't use stderr for error messages. Everything goes to stdout.
266  UsesStderr = False
267 
268  def internal_compile_commands(self) -> List[str]:
269  cmds = [
270  "onerror { quit -f -code 1 }",
271  ]
272  sources = self.sourcessources.rtl_sources
273  sources.append(Questa.DefaultDriver)
274  for src in sources:
275  cmds.append(f"vlog -incr +acc -sv +define+TOP_MODULE={self.sources.top}"
276  f" +define+SIMULATION {str(src)}")
277  cmds.append(f"vopt -incr driver -o driver_opt +acc")
278  return cmds
279 
280  def compile_commands(self) -> List[List[str]]:
281  with open("compile.do", "w") as f:
282  for cmd in self.internal_compile_commandsinternal_compile_commands():
283  f.write(cmd)
284  f.write("\n")
285  f.write("quit\n")
286  return [
287  ["vsim", "-batch", "-do", "compile.do"],
288  ]
289 
290  def run_command(self, gui: bool) -> List[str]:
291  vsim = "vsim"
292  # Note: vsim exit codes say nothing about the test run's pass/fail even
293  # if $fatal is encountered in the simulation.
294  if gui:
295  cmd = [
296  vsim,
297  "driver_opt",
298  ]
299  else:
300  cmd = [
301  vsim,
302  "driver_opt",
303  "-batch",
304  "-do",
305  "run -all",
306  ]
307  for lib in self.sourcessources.dpi_so_paths():
308  svLib = os.path.splitext(lib)[0]
309  cmd.append("-sv_lib")
310  cmd.append(svLib)
311  if len(self.sourcessources.dpi_so) > 0:
312  cmd.append("-cpppath")
313  cmd.append("/usr/bin/clang++")
314  return cmd
315 
316  def run(self, inner_command: str, gui: bool = False) -> int:
317  """Override the Simulator.run() to add a soft link in the run directory (to
318  the work directory) before running vsim the usual way."""
319 
320  # Create a soft link to the work directory.
321  workDir = self.run_dirrun_dir / "work"
322  if not workDir.exists():
323  os.symlink(Path(os.getcwd()) / "work", workDir)
324 
325  # Run the simulation.
326  return super().run(inner_command, gui)
327 
328 
329 def __main__(args):
330  argparser = argparse.ArgumentParser(
331  description="Wrap a 'inner_cmd' in an ESI cosimulation environment.",
332  formatter_class=argparse.RawDescriptionHelpFormatter,
333  epilog=textwrap.dedent("""
334  Notes:
335  - For Verilator, libEsiCosimDpiServer.so must be in the dynamic
336  library runtime search path (LD_LIBRARY_PATH) and link time path
337  (LIBRARY_PATH). If it is installed to a standard location (e.g.
338  /usr/lib), this should be handled automatically.
339  - This script needs to sit in the same directory as the ESI support
340  SystemVerilog (e.g. Cosim_DpiPkg.sv, Cosim_MMIO.sv, etc.). It can,
341  however, be soft linked to a different location.
342  - The simulator executable(s) must be in your PATH.
343  """))
344 
345  argparser.add_argument(
346  "--sim",
347  type=str,
348  default="verilator",
349  help="Name of the RTL simulator to use or path to an executable.")
350  argparser.add_argument("--rundir",
351  default="run",
352  help="Directory in which simulation should be run.")
353  argparser.add_argument(
354  "--top",
355  default="ESI_Cosim_Top",
356  help="Name of the 'top' module to use in the simulation.")
357  argparser.add_argument("--no-compile",
358  action="store_true",
359  help="Do not run the compile.")
360  argparser.add_argument("--debug",
361  action="store_true",
362  help="Enable debug output.")
363  argparser.add_argument("--gui",
364  action="store_true",
365  help="Run the simulator in GUI mode (if supported).")
366  argparser.add_argument("--source",
367  help="Directories containing the source files.",
368  default="hw")
369 
370  argparser.add_argument("inner_cmd",
371  nargs=argparse.REMAINDER,
372  help="Command to run in the simulation environment.")
373 
374  if len(args) <= 1:
375  argparser.print_help()
376  return
377  args = argparser.parse_args(args[1:])
378 
379  sources = SourceFiles(args.top)
380  sources.add_dir(Path(args.source))
381 
382  if args.sim == "verilator":
383  sim = Verilator(sources, Path(args.rundir), args.debug)
384  elif args.sim == "questa":
385  sim = Questa(sources, Path(args.rundir), args.debug)
386  else:
387  print("Unknown simulator: " + args.sim)
388  print("Supported simulators: ")
389  print(" - verilator")
390  print(" - questa")
391  return 1
392 
393  if not args.no_compile:
394  rc = sim.compile()
395  if rc != 0:
396  return rc
397  return sim.run(args.inner_cmd[1:], gui=args.gui)
398 
399 
400 if __name__ == '__main__':
401  sys.exit(__main__(sys.argv))
static StringAttr append(StringAttr base, const Twine &suffix)
Return a attribute with the specified suffix appended.
List[List[str]] compile_commands(self)
Definition: esi-cosim.py:280
int run(self, str inner_command, bool gui=False)
Definition: esi-cosim.py:316
List[str] internal_compile_commands(self)
Definition: esi-cosim.py:268
List[str] run_command(self, bool gui)
Definition: esi-cosim.py:290
Dict[str, str] get_env()
Definition: esi-cosim.py:99
int compile(self)
Definition: esi-cosim.py:113
List[str] run_command(self, bool gui)
Definition: esi-cosim.py:134
int run(self, str inner_command, bool gui=False)
Definition: esi-cosim.py:138
def __init__(self, SourceFiles sources, Path run_dir, bool debug)
Definition: esi-cosim.py:93
List[List[str]] compile_commands(self)
Definition: esi-cosim.py:109
List[Path] rtl_sources(self)
Definition: esi-cosim.py:81
List[Path] dpi_so_paths(self)
Definition: esi-cosim.py:65
None __init__(self, str top)
Definition: esi-cosim.py:42
def add_dir(self, Path dir)
Definition: esi-cosim.py:57
def __init__(self, SourceFiles sources, Path run_dir, bool debug)
Definition: esi-cosim.py:220
def run_command(self, bool gui)
Definition: esi-cosim.py:253
List[List[str]] compile_commands(self)
Definition: esi-cosim.py:227
Direction get(bool isOutput)
Returns an output direction if isOutput is true, otherwise returns an input direction.
Definition: CalyxOps.cpp:54
bool is_port_open(port)
Definition: esi-cosim.py:32
def __main__(args)
Definition: esi-cosim.py:329