CIRCT  20.0.0git
esi-cosim.py
Go to the documentation of this file.
1 #!/usr/bin/env python3
2 
3 # ===- esi-cosim.py - ESI cosimulation launch utility --------*- python -*-===//
4 #
5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
6 # See https://llvm.org/LICENSE.txt for license information.
7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
8 #
9 # ===----------------------------------------------------------------------===//
10 #
11 # Utility script to start a simulation and launch a command to interact with it
12 # via ESI cosimulation.
13 #
14 # ===----------------------------------------------------------------------===//
15 
16 import argparse
17 import os
18 from pathlib import Path
19 import re
20 import signal
21 import socket
22 import subprocess
23 import sys
24 import textwrap
25 import time
26 from typing import Dict, List
27 
28 _thisdir = Path(__file__).parent
29 CosimCollateralDir = _thisdir.parent / "cosim"
30 
31 
32 def is_port_open(port) -> bool:
33  """Check if a TCP port is open locally."""
34  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
35  result = sock.connect_ex(('127.0.0.1', port))
36  sock.close()
37  return True if result == 0 else False
38 
39 
41 
42  def __init__(self, top: str) -> None:
43  # User source files.
44  self.user: List[Path] = []
45  # DPI shared objects.
46  self.dpi_so: List[str] = ["EsiCosimDpiServer"]
47  # DPI SV files.
48  self.dpi_sv: List[Path] = [
49  CosimCollateralDir / "Cosim_DpiPkg.sv",
50  CosimCollateralDir / "Cosim_Endpoint.sv",
51  CosimCollateralDir / "Cosim_Manifest.sv",
52  ]
53  # Name of the top module.
54  self.toptop = top
55 
56  def add_dir(self, dir: Path):
57  """Add all the RTL files in a directory to the source list."""
58  for file in sorted(dir.iterdir()):
59  if file.is_file() and (file.suffix == ".sv" or file.suffix == ".v"):
60  self.user.append(file)
61  elif file.is_dir():
62  self.add_diradd_dir(file)
63 
64  def dpi_so_paths(self) -> List[Path]:
65  """Return a list of all the DPI shared object files."""
66 
67  def find_so(name: str) -> Path:
68  for path in Simulator.get_env().get("LD_LIBRARY_PATH", "").split(":"):
69  if os.name == "nt":
70  so = Path(path) / f"{name}.dll"
71  else:
72  so = Path(path) / f"lib{name}.so"
73  if so.exists():
74  return so
75  raise FileNotFoundError(f"Could not find {name}.so in LD_LIBRARY_PATH")
76 
77  return [find_so(name) for name in self.dpi_so]
78 
79  @property
80  def rtl_sources(self) -> List[Path]:
81  """Return a list of all the RTL source files."""
82  return self.dpi_sv + self.user
83 
84 
85 class Simulator:
86 
87  # Some RTL simulators don't use stderr for error messages. Everything goes to
88  # stdout. Boo! They should feel bad about this. Also, they can specify that
89  # broken behavior by overriding this.
90  UsesStderr = True
91 
92  def __init__(self, sources: SourceFiles, run_dir: Path, debug: bool):
93  self.sourcessources = sources
94  self.run_dirrun_dir = run_dir
95  self.debugdebug = debug
96 
97  @staticmethod
98  def get_env() -> Dict[str, str]:
99  """Get the environment variables to locate shared objects."""
100 
101  env = os.environ.copy()
102  env["LIBRARY_PATH"] = env.get("LIBRARY_PATH", "") + ":" + str(
103  _thisdir.parent / "lib")
104  env["LD_LIBRARY_PATH"] = env.get("LD_LIBRARY_PATH", "") + ":" + str(
105  _thisdir.parent / "lib")
106  return env
107 
108  def compile_commands(self) -> List[List[str]]:
109  """Compile the sources. Returns the exit code of the simulation compiler."""
110  assert False, "Must be implemented by subclass"
111 
112  def compile(self) -> int:
113  cmds = self.compile_commandscompile_commands()
114  self.run_dirrun_dir.mkdir(parents=True, exist_ok=True)
115  with (self.run_dirrun_dir / "compile_stdout.log").open("w") as stdout, (
116  self.run_dirrun_dir / "compile_stderr.log").open("w") as stderr:
117  for cmd in cmds:
118  cp = subprocess.run(cmd,
119  env=Simulator.get_env(),
120  capture_output=True,
121  text=True)
122  stdout.write(cp.stdout)
123  stderr.write(cp.stderr)
124  if cp.returncode != 0:
125  print("====== Compilation failure:")
126  if self.UsesStderrUsesStderr:
127  print(cp.stderr)
128  else:
129  print(cp.stdout)
130  return cp.returncode
131  return 0
132 
133  def run_command(self, gui: bool) -> List[str]:
134  """Return the command to run the simulation."""
135  assert False, "Must be implemented by subclass"
136 
137  def run(self, inner_command: str, gui: bool = False) -> int:
138  """Start the simulation then run the command specified. Kill the simulation
139  when the command exits."""
140 
141  # 'simProc' is accessed in the finally block. Declare it here to avoid
142  # syntax errors in that block.
143  simProc = None
144  try:
145  # Open log files
146  self.run_dirrun_dir.mkdir(parents=True, exist_ok=True)
147  simStdout = open(self.run_dirrun_dir / "sim_stdout.log", "w")
148  simStderr = open(self.run_dirrun_dir / "sim_stderr.log", "w")
149 
150  # Erase the config file if it exists. We don't want to read
151  # an old config.
152  portFileName = self.run_dirrun_dir / "cosim.cfg"
153  if os.path.exists(portFileName):
154  os.remove(portFileName)
155 
156  # Run the simulation.
157  simEnv = Simulator.get_env()
158  if self.debugdebug:
159  simEnv["COSIM_DEBUG_FILE"] = "cosim_debug.log"
160  simProc = subprocess.Popen(self.run_commandrun_command(gui),
161  stdout=simStdout,
162  stderr=simStderr,
163  env=simEnv,
164  cwd=self.run_dirrun_dir,
165  preexec_fn=os.setsid)
166  simStderr.close()
167  simStdout.close()
168 
169  # Get the port which the simulation RPC selected.
170  checkCount = 0
171  while (not os.path.exists(portFileName)) and \
172  simProc.poll() is None:
173  time.sleep(0.1)
174  checkCount += 1
175  if checkCount > 200 and not gui:
176  raise Exception(f"Cosim never wrote cfg file: {portFileName}")
177  port = -1
178  while port < 0:
179  portFile = open(portFileName, "r")
180  for line in portFile.readlines():
181  m = re.match("port: (\\d+)", line)
182  if m is not None:
183  port = int(m.group(1))
184  portFile.close()
185 
186  # Wait for the simulation to start accepting RPC connections.
187  checkCount = 0
188  while not is_port_open(port):
189  checkCount += 1
190  if checkCount > 200:
191  raise Exception(f"Cosim RPC port ({port}) never opened")
192  if simProc.poll() is not None:
193  raise Exception("Simulation exited early")
194  time.sleep(0.05)
195 
196  # Run the inner command, passing the connection info via environment vars.
197  testEnv = os.environ.copy()
198  testEnv["ESI_COSIM_PORT"] = str(port)
199  testEnv["ESI_COSIM_HOST"] = "localhost"
200  return subprocess.run(inner_command, cwd=os.getcwd(),
201  env=testEnv).returncode
202  finally:
203  # Make sure to stop the simulation no matter what.
204  if simProc:
205  os.killpg(os.getpgid(simProc.pid), signal.SIGINT)
206  # Allow the simulation time to flush its outputs.
207  try:
208  simProc.wait(timeout=1.0)
209  except subprocess.TimeoutExpired:
210  # If the simulation doesn't exit of its own free will, kill it.
211  simProc.kill()
212 
213 
215  """Run and compile funcs for Verilator."""
216 
217  DefaultDriver = CosimCollateralDir / "driver.cpp"
218 
219  def __init__(self, sources: SourceFiles, run_dir: Path, debug: bool):
220  super().__init__(sources, run_dir, debug)
221 
222  self.verilatorverilator = "verilator"
223  if "VERILATOR_PATH" in os.environ:
224  self.verilatorverilator = os.environ["VERILATOR_PATH"]
225 
226  def compile_commands(self) -> List[List[str]]:
227  cmd: List[str] = [
228  self.verilatorverilator,
229  "--cc",
230  "--top-module",
231  self.sourcessources.top,
232  "-DSIMULATION",
233  "-sv",
234  "--build",
235  "--exe",
236  "--assert",
237  str(Verilator.DefaultDriver),
238  ]
239  cflags = [
240  "-DTOP_MODULE=" + self.sourcessources.top,
241  ]
242  if self.debugdebug:
243  cmd += ["--trace", "--trace-params", "--trace-structs"]
244  cflags.append("-DTRACE")
245  if len(cflags) > 0:
246  cmd += ["-CFLAGS", " ".join(cflags)]
247  if len(self.sourcessources.dpi_so) > 0:
248  cmd += ["-LDFLAGS", " ".join(["-l" + so for so in self.sourcessources.dpi_so])]
249  cmd += [str(p) for p in self.sourcessources.rtl_sources]
250  return [cmd]
251 
252  def run_command(self, gui: bool):
253  if gui:
254  raise RuntimeError("Verilator does not support GUI mode.")
255  exe = Path.cwd() / "obj_dir" / ("V" + self.sourcessources.top)
256  return [str(exe)]
257 
258 
260  """Run and compile funcs for Questasim."""
261 
262  DefaultDriver = CosimCollateralDir / "driver.sv"
263 
264  # Questa doesn't use stderr for error messages. Everything goes to stdout.
265  UsesStderr = False
266 
267  def internal_compile_commands(self) -> List[str]:
268  cmds = [
269  "onerror { quit -f -code 1 }",
270  ]
271  sources = self.sourcessources.rtl_sources
272  sources.append(Questa.DefaultDriver)
273  for src in sources:
274  cmds.append(f"vlog -incr +acc -sv +define+TOP_MODULE={self.sources.top}"
275  f" +define+SIMULATION {str(src)}")
276  cmds.append(f"vopt -incr driver -o driver_opt +acc")
277  return cmds
278 
279  def compile_commands(self) -> List[List[str]]:
280  with open("compile.do", "w") as f:
281  for cmd in self.internal_compile_commandsinternal_compile_commands():
282  f.write(cmd)
283  f.write("\n")
284  f.write("quit\n")
285  return [
286  ["vsim", "-batch", "-do", "compile.do"],
287  ]
288 
289  def run_command(self, gui: bool) -> List[str]:
290  vsim = "vsim"
291  # Note: vsim exit codes say nothing about the test run's pass/fail even
292  # if $fatal is encountered in the simulation.
293  if gui:
294  cmd = [
295  vsim,
296  "driver_opt",
297  ]
298  else:
299  cmd = [
300  vsim,
301  "driver_opt",
302  "-batch",
303  "-do",
304  "run -all",
305  ]
306  for lib in self.sourcessources.dpi_so_paths():
307  svLib = os.path.splitext(lib)[0]
308  cmd.append("-sv_lib")
309  cmd.append(svLib)
310  return cmd
311 
312  def run(self, inner_command: str, gui: bool = False) -> int:
313  """Override the Simulator.run() to add a soft link in the run directory (to
314  the work directory) before running vsim the usual way."""
315 
316  # Create a soft link to the work directory.
317  workDir = self.run_dirrun_dir / "work"
318  if not workDir.exists():
319  os.symlink(Path(os.getcwd()) / "work", workDir)
320 
321  # Run the simulation.
322  return super().run(inner_command, gui)
323 
324 
325 def __main__(args):
326  argparser = argparse.ArgumentParser(
327  description="Wrap a 'inner_cmd' in an ESI cosimulation environment.",
328  formatter_class=argparse.RawDescriptionHelpFormatter,
329  epilog=textwrap.dedent("""
330  Notes:
331  - For Verilator, libEsiCosimDpiServer.so must be in the dynamic
332  library runtime search path (LD_LIBRARY_PATH) and link time path
333  (LIBRARY_PATH). If it is installed to a standard location (e.g.
334  /usr/lib), this should be handled automatically.
335  - This script needs to sit in the same directory as the ESI support
336  SystemVerilog (e.g. Cosim_DpiPkg.sv, Cosim_MMIO.sv, etc.). It can,
337  however, be soft linked to a different location.
338  - The simulator executable(s) must be in your PATH.
339  """))
340 
341  argparser.add_argument(
342  "--sim",
343  type=str,
344  default="verilator",
345  help="Name of the RTL simulator to use or path to an executable.")
346  argparser.add_argument("--rundir",
347  default="run",
348  help="Directory in which simulation should be run.")
349  argparser.add_argument(
350  "--top",
351  default="ESI_Cosim_Top",
352  help="Name of the 'top' module to use in the simulation.")
353  argparser.add_argument("--no-compile",
354  action="store_true",
355  help="Do not run the compile.")
356  argparser.add_argument("--debug",
357  action="store_true",
358  help="Enable debug output.")
359  argparser.add_argument("--gui",
360  action="store_true",
361  help="Run the simulator in GUI mode (if supported).")
362  argparser.add_argument("--source",
363  help="Directories containing the source files.",
364  default="hw")
365 
366  argparser.add_argument("inner_cmd",
367  nargs=argparse.REMAINDER,
368  help="Command to run in the simulation environment.")
369 
370  if len(args) <= 1:
371  argparser.print_help()
372  return
373  args = argparser.parse_args(args[1:])
374 
375  sources = SourceFiles(args.top)
376  sources.add_dir(Path(args.source))
377 
378  if args.sim == "verilator":
379  sim = Verilator(sources, Path(args.rundir), args.debug)
380  elif args.sim == "questa":
381  sim = Questa(sources, Path(args.rundir), args.debug)
382  else:
383  print("Unknown simulator: " + args.sim)
384  print("Supported simulators: ")
385  print(" - verilator")
386  print(" - questa")
387  return 1
388 
389  if not args.no_compile:
390  rc = sim.compile()
391  if rc != 0:
392  return rc
393  return sim.run(args.inner_cmd[1:], gui=args.gui)
394 
395 
396 if __name__ == '__main__':
397  sys.exit(__main__(sys.argv))
static StringAttr append(StringAttr base, const Twine &suffix)
Return a attribute with the specified suffix appended.
List[List[str]] compile_commands(self)
Definition: esi-cosim.py:279
int run(self, str inner_command, bool gui=False)
Definition: esi-cosim.py:312
List[str] internal_compile_commands(self)
Definition: esi-cosim.py:267
List[str] run_command(self, bool gui)
Definition: esi-cosim.py:289
Dict[str, str] get_env()
Definition: esi-cosim.py:98
int compile(self)
Definition: esi-cosim.py:112
List[str] run_command(self, bool gui)
Definition: esi-cosim.py:133
int run(self, str inner_command, bool gui=False)
Definition: esi-cosim.py:137
def __init__(self, SourceFiles sources, Path run_dir, bool debug)
Definition: esi-cosim.py:92
List[List[str]] compile_commands(self)
Definition: esi-cosim.py:108
List[Path] rtl_sources(self)
Definition: esi-cosim.py:80
List[Path] dpi_so_paths(self)
Definition: esi-cosim.py:64
None __init__(self, str top)
Definition: esi-cosim.py:42
def add_dir(self, Path dir)
Definition: esi-cosim.py:56
def __init__(self, SourceFiles sources, Path run_dir, bool debug)
Definition: esi-cosim.py:219
def run_command(self, bool gui)
Definition: esi-cosim.py:252
List[List[str]] compile_commands(self)
Definition: esi-cosim.py:226
Direction get(bool isOutput)
Returns an output direction if isOutput is true, otherwise returns an input direction.
Definition: CalyxOps.cpp:55
bool is_port_open(port)
Definition: esi-cosim.py:32
def __main__(args)
Definition: esi-cosim.py:325