CIRCT  18.0.0git
HandshakeToHW.cpp
Go to the documentation of this file.
1 //===- HandshakeToHW.cpp - Translate Handshake into HW ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 
7 //===----------------------------------------------------------------------===//
8 //
9 // This is the main Handshake to HW Conversion Pass Implementation.
10 //
11 //===----------------------------------------------------------------------===//
12 
14 #include "../PassDetail.h"
17 #include "circt/Dialect/HW/HWOps.h"
25 #include "mlir/Dialect/Arith/IR/Arith.h"
26 #include "mlir/Dialect/MemRef/IR/MemRef.h"
27 #include "mlir/IR/ImplicitLocOpBuilder.h"
28 #include "mlir/Pass/PassManager.h"
29 #include "mlir/Transforms/DialectConversion.h"
30 #include "llvm/ADT/TypeSwitch.h"
31 #include "llvm/Support/MathExtras.h"
32 #include <optional>
33 
34 using namespace mlir;
35 using namespace circt;
36 using namespace circt::handshake;
37 using namespace circt::hw;
38 
39 using NameUniquer = std::function<std::string(Operation *)>;
40 
41 namespace {
42 
43 static Type tupleToStruct(TypeRange types) {
44  return toValidType(mlir::TupleType::get(types[0].getContext(), types));
45 }
46 
47 // Shared state used by various functions; captured in a struct to reduce the
48 // number of arguments that we have to pass around.
49 struct HandshakeLoweringState {
50  ModuleOp parentModule;
51  NameUniquer nameUniquer;
52 };
53 
54 // A type converter is needed to perform the in-flight materialization of "raw"
55 // (non-ESI channel) types to their ESI channel correspondents. This comes into
56 // effect when backedges exist in the input IR.
57 class ESITypeConverter : public TypeConverter {
58 public:
59  ESITypeConverter() {
60  addConversion([](Type type) -> Type { return esiWrapper(type); });
61 
62  addTargetMaterialization(
63  [&](mlir::OpBuilder &builder, mlir::Type resultType,
64  mlir::ValueRange inputs,
65  mlir::Location loc) -> std::optional<mlir::Value> {
66  if (inputs.size() != 1)
67  return std::nullopt;
68  return inputs[0];
69  });
70 
71  addSourceMaterialization(
72  [&](mlir::OpBuilder &builder, mlir::Type resultType,
73  mlir::ValueRange inputs,
74  mlir::Location loc) -> std::optional<mlir::Value> {
75  if (inputs.size() != 1)
76  return std::nullopt;
77  return inputs[0];
78  });
79  }
80 };
81 
82 } // namespace
83 
84 /// Returns a submodule name resulting from an operation, without discriminating
85 /// type information.
86 static std::string getBareSubModuleName(Operation *oldOp) {
87  // The dialect name is separated from the operation name by '.', which is not
88  // valid in SystemVerilog module names. In case this name is used in
89  // SystemVerilog output, replace '.' with '_'.
90  std::string subModuleName = oldOp->getName().getStringRef().str();
91  std::replace(subModuleName.begin(), subModuleName.end(), '.', '_');
92  return subModuleName;
93 }
94 
95 static std::string getCallName(Operation *op) {
96  auto callOp = dyn_cast<handshake::InstanceOp>(op);
97  return callOp ? callOp.getModule().str() : getBareSubModuleName(op);
98 }
99 
100 /// Extracts the type of the data-carrying type of opType. If opType is an ESI
101 /// channel, getHandshakeBundleDataType extracts the data-carrying type, else,
102 /// assume that opType itself is the data-carrying type.
103 static Type getOperandDataType(Value op) {
104  auto opType = op.getType();
105  if (auto channelType = opType.dyn_cast<esi::ChannelType>())
106  return channelType.getInner();
107  return opType;
108 }
109 
110 /// Filters NoneType's from the input.
111 static SmallVector<Type> filterNoneTypes(ArrayRef<Type> input) {
112  SmallVector<Type> filterRes;
113  llvm::copy_if(input, std::back_inserter(filterRes),
114  [](Type type) { return !type.isa<NoneType>(); });
115  return filterRes;
116 }
117 
118 /// Returns a set of types which may uniquely identify the provided op. Return
119 /// value is <inputTypes, outputTypes>.
120 using DiscriminatingTypes = std::pair<SmallVector<Type>, SmallVector<Type>>;
122  return TypeSwitch<Operation *, DiscriminatingTypes>(op)
123  .Case<MemoryOp, ExternalMemoryOp>([&](auto memOp) {
124  return DiscriminatingTypes{{},
125  {memOp.getMemRefType().getElementType()}};
126  })
127  .Default([&](auto) {
128  // By default, all in- and output types which is not a control type
129  // (NoneType) are discriminating types.
130  std::vector<Type> inTypes, outTypes;
131  llvm::transform(op->getOperands(), std::back_inserter(inTypes),
133  llvm::transform(op->getResults(), std::back_inserter(outTypes),
135  return DiscriminatingTypes{filterNoneTypes(inTypes),
136  filterNoneTypes(outTypes)};
137  });
138 }
139 
140 /// Get type name. Currently we only support integer or index types.
141 /// The emitted type aligns with the getFIRRTLType() method. Thus all integers
142 /// other than signed integers will be emitted as unsigned.
143 // NOLINTNEXTLINE(misc-no-recursion)
144 static std::string getTypeName(Location loc, Type type) {
145  std::string typeName;
146  // Builtin types
147  if (type.isIntOrIndex()) {
148  if (auto indexType = type.dyn_cast<IndexType>())
149  typeName += "_ui" + std::to_string(indexType.kInternalStorageBitWidth);
150  else if (type.isSignedInteger())
151  typeName += "_si" + std::to_string(type.getIntOrFloatBitWidth());
152  else
153  typeName += "_ui" + std::to_string(type.getIntOrFloatBitWidth());
154  } else if (auto tupleType = type.dyn_cast<TupleType>()) {
155  typeName += "_tuple";
156  for (auto elementType : tupleType.getTypes())
157  typeName += getTypeName(loc, elementType);
158  } else if (auto structType = type.dyn_cast<hw::StructType>()) {
159  typeName += "_struct";
160  for (auto element : structType.getElements())
161  typeName += "_" + element.name.str() + getTypeName(loc, element.type);
162  } else
163  emitError(loc) << "unsupported data type '" << type << "'";
164 
165  return typeName;
166 }
167 
168 /// Construct a name for creating HW sub-module.
169 static std::string getSubModuleName(Operation *oldOp) {
170  if (auto instanceOp = dyn_cast<handshake::InstanceOp>(oldOp); instanceOp)
171  return instanceOp.getModule().str();
172 
173  std::string subModuleName = getBareSubModuleName(oldOp);
174 
175  // Add value of the constant operation.
176  if (auto constOp = dyn_cast<handshake::ConstantOp>(oldOp)) {
177  if (auto intAttr = constOp.getValue().dyn_cast<IntegerAttr>()) {
178  auto intType = intAttr.getType();
179 
180  if (intType.isSignedInteger())
181  subModuleName += "_c" + std::to_string(intAttr.getSInt());
182  else if (intType.isUnsignedInteger())
183  subModuleName += "_c" + std::to_string(intAttr.getUInt());
184  else
185  subModuleName += "_c" + std::to_string((uint64_t)intAttr.getInt());
186  } else
187  oldOp->emitError("unsupported constant type");
188  }
189 
190  // Add discriminating in- and output types.
191  auto [inTypes, outTypes] = getHandshakeDiscriminatingTypes(oldOp);
192  if (!inTypes.empty())
193  subModuleName += "_in";
194  for (auto inType : inTypes)
195  subModuleName += getTypeName(oldOp->getLoc(), inType);
196 
197  if (!outTypes.empty())
198  subModuleName += "_out";
199  for (auto outType : outTypes)
200  subModuleName += getTypeName(oldOp->getLoc(), outType);
201 
202  // Add memory ID.
203  if (auto memOp = dyn_cast<handshake::MemoryOp>(oldOp))
204  subModuleName += "_id" + std::to_string(memOp.getId());
205 
206  // Add compare kind.
207  if (auto comOp = dyn_cast<mlir::arith::CmpIOp>(oldOp))
208  subModuleName += "_" + stringifyEnum(comOp.getPredicate()).str();
209 
210  // Add buffer information.
211  if (auto bufferOp = dyn_cast<handshake::BufferOp>(oldOp)) {
212  subModuleName += "_" + std::to_string(bufferOp.getNumSlots()) + "slots";
213  if (bufferOp.isSequential())
214  subModuleName += "_seq";
215  else
216  subModuleName += "_fifo";
217 
218  if (auto initValues = bufferOp.getInitValues()) {
219  subModuleName += "_init";
220  for (const Attribute e : *initValues) {
221  assert(e.isa<IntegerAttr>());
222  subModuleName +=
223  "_" + std::to_string(e.dyn_cast<IntegerAttr>().getInt());
224  }
225  }
226  }
227 
228  // Add control information.
229  if (auto ctrlInterface = dyn_cast<handshake::ControlInterface>(oldOp);
230  ctrlInterface && ctrlInterface.isControl()) {
231  // Add some additional discriminating info for non-typed operations.
232  subModuleName += "_" + std::to_string(oldOp->getNumOperands()) + "ins_" +
233  std::to_string(oldOp->getNumResults()) + "outs";
234  subModuleName += "_ctrl";
235  } else {
236  assert(
237  (!inTypes.empty() || !outTypes.empty()) &&
238  "Insufficient discriminating type info generated for the operation!");
239  }
240 
241  return subModuleName;
242 }
243 
244 //===----------------------------------------------------------------------===//
245 // HW Sub-module Related Functions
246 //===----------------------------------------------------------------------===//
247 
248 /// Check whether a submodule with the same name has been created elsewhere in
249 /// the top level module. Return the matched module operation if true, otherwise
250 /// return nullptr.
251 static HWModuleLike checkSubModuleOp(mlir::ModuleOp parentModule,
252  StringRef modName) {
253  if (auto mod = parentModule.lookupSymbol<HWModuleOp>(modName))
254  return mod;
255  if (auto mod = parentModule.lookupSymbol<HWModuleExternOp>(modName))
256  return mod;
257  return {};
258 }
259 
260 static HWModuleLike checkSubModuleOp(mlir::ModuleOp parentModule,
261  Operation *oldOp) {
262  HWModuleLike targetModule;
263  if (auto instanceOp = dyn_cast<handshake::InstanceOp>(oldOp))
264  targetModule = checkSubModuleOp(parentModule, instanceOp.getModule());
265  else
266  targetModule = checkSubModuleOp(parentModule, getSubModuleName(oldOp));
267 
268  if (isa<handshake::InstanceOp>(oldOp))
269  assert(targetModule &&
270  "handshake.instance target modules should always have been lowered "
271  "before the modules that reference them!");
272  return targetModule;
273 }
274 
275 /// Returns a vector of PortInfo's which defines the HW interface of the
276 /// to-be-converted op.
277 static ModulePortInfo getPortInfoForOp(Operation *op) {
278  return getPortInfoForOpTypes(op, op->getOperandTypes(), op->getResultTypes());
279 }
280 
281 static llvm::SmallVector<hw::detail::FieldInfo>
282 portToFieldInfo(llvm::ArrayRef<hw::PortInfo> portInfo) {
283  llvm::SmallVector<hw::detail::FieldInfo> fieldInfo;
284  for (auto port : portInfo)
285  fieldInfo.push_back({port.name, port.type});
286 
287  return fieldInfo;
288 }
289 
290 // Convert any handshake.extmemory operations and the top-level I/O
291 // associated with these.
292 static LogicalResult convertExtMemoryOps(HWModuleOp mod) {
293  auto ports = mod.getPortList();
294  auto *ctx = mod.getContext();
295 
296  // Gather memref ports to be converted.
297  llvm::DenseMap<unsigned, Value> memrefPorts;
298  for (auto [i, arg] : llvm::enumerate(mod.getBodyBlock()->getArguments())) {
299  auto channel = arg.getType().dyn_cast<esi::ChannelType>();
300  if (channel && channel.getInner().isa<MemRefType>())
301  memrefPorts[i] = arg;
302  }
303 
304  if (memrefPorts.empty())
305  return success(); // nothing to do.
306 
307  OpBuilder b(mod);
308 
309  auto getMemoryIOInfo = [&](Location loc, Twine portName, unsigned argIdx,
310  ArrayRef<hw::PortInfo> info,
311  hw::ModulePort::Direction direction) {
312  auto type = hw::StructType::get(ctx, portToFieldInfo(info));
313  auto portInfo =
314  hw::PortInfo{{b.getStringAttr(portName), type, direction}, argIdx};
315  return portInfo;
316  };
317 
318  for (auto [i, arg] : memrefPorts) {
319  // Insert ports into the module
320  auto memName = mod.getArgName(i);
321 
322  // Get the attached extmemory external module.
323  auto extmemInstance = cast<hw::InstanceOp>(*arg.getUsers().begin());
324  auto extmemMod =
325  cast<hw::HWModuleExternOp>(extmemInstance.getReferencedModuleSlow());
326  auto portInfo = extmemMod.getPortList();
327 
328  // The extmemory external module's interface is a direct wrapping of the
329  // original handshake.extmemory operation in- and output types. Remove the
330  // first input argument (the !esi.channel<memref> op) since that is what
331  // we're replacing with a materialized interface.
332  portInfo.eraseInput(0);
333 
334  // Add memory input - this is the output of the extmemory op.
335  SmallVector<PortInfo> outputs(portInfo.getOutputs());
336  auto inPortInfo =
337  getMemoryIOInfo(arg.getLoc(), memName.strref() + "_in", i, outputs,
339  mod.insertPorts({{i, inPortInfo}}, {});
340  auto newInPort = mod.getArgumentForInput(i);
341  // Replace the extmemory submodule outputs with the newly created inputs.
342  b.setInsertionPointToStart(mod.getBodyBlock());
343  auto newInPortExploded = b.create<hw::StructExplodeOp>(
344  arg.getLoc(), extmemMod.getOutputTypes(), newInPort);
345  extmemInstance.replaceAllUsesWith(newInPortExploded.getResults());
346 
347  // Add memory output - this is the inputs of the extmemory op (without the
348  // first argument);
349  unsigned outArgI = mod.getNumOutputPorts();
350  SmallVector<PortInfo> inputs(portInfo.getInputs());
351  auto outPortInfo =
352  getMemoryIOInfo(arg.getLoc(), memName.strref() + "_out", outArgI,
354 
355  auto memOutputArgs = extmemInstance.getOperands().drop_front();
356  b.setInsertionPoint(mod.getBodyBlock()->getTerminator());
357  auto memOutputStruct = b.create<hw::StructCreateOp>(
358  arg.getLoc(), outPortInfo.type, memOutputArgs);
359  mod.appendOutputs({{outPortInfo.name, memOutputStruct}});
360 
361  // Erase the extmemory submodule instace since the i/o has now been
362  // plumbed.
363  extmemMod.erase();
364  extmemInstance.erase();
365 
366  // Erase the original memref argument of the top-level i/o now that it's use
367  // has been removed.
368  mod.modifyPorts(/*insertInputs*/ {}, /*insertOutputs*/ {},
369  /*eraseInputs*/ {i + 1}, /*eraseOutputs*/ {});
370  }
371 
372  return success();
373 }
374 
375 namespace {
376 
377 // Input handshakes contain a resolved valid and (optional )data signal, and
378 // a to-be-assigned ready signal.
379 struct InputHandshake {
380  Value valid;
381  std::shared_ptr<Backedge> ready;
382  Value data;
383 };
384 
385 // Output handshakes contain a resolved ready, and to-be-assigned valid and
386 // (optional) data signals.
387 struct OutputHandshake {
388  std::shared_ptr<Backedge> valid;
389  Value ready;
390  std::shared_ptr<Backedge> data;
391 };
392 
393 /// A helper struct that acts like a wire. Can be used to interact with the
394 /// RTLBuilder when multiple built components should be connected.
395 struct HandshakeWire {
396  HandshakeWire(BackedgeBuilder &bb, Type dataType) {
397  MLIRContext *ctx = dataType.getContext();
398  auto i1Type = IntegerType::get(ctx, 1);
399  valid = std::make_shared<Backedge>(bb.get(i1Type));
400  ready = std::make_shared<Backedge>(bb.get(i1Type));
401  data = std::make_shared<Backedge>(bb.get(dataType));
402  }
403 
404  // Functions that allow to treat a wire like an input or output port.
405  // **Careful**: Such a port will not be updated when backedges are resolved.
406  InputHandshake getAsInput() { return {*valid, ready, *data}; }
407  OutputHandshake getAsOutput() { return {valid, *ready, data}; }
408 
409  std::shared_ptr<Backedge> valid;
410  std::shared_ptr<Backedge> ready;
411  std::shared_ptr<Backedge> data;
412 };
413 
414 template <typename T, typename TInner>
415 llvm::SmallVector<T> extractValues(llvm::SmallVector<TInner> &container,
416  llvm::function_ref<T(TInner &)> extractor) {
417  llvm::SmallVector<T> result;
418  llvm::transform(container, std::back_inserter(result), extractor);
419  return result;
420 }
421 struct UnwrappedIO {
422  llvm::SmallVector<InputHandshake> inputs;
423  llvm::SmallVector<OutputHandshake> outputs;
424 
425  llvm::SmallVector<Value> getInputValids() {
426  return extractValues<Value, InputHandshake>(
427  inputs, [](auto &hs) { return hs.valid; });
428  }
429  llvm::SmallVector<std::shared_ptr<Backedge>> getInputReadys() {
430  return extractValues<std::shared_ptr<Backedge>, InputHandshake>(
431  inputs, [](auto &hs) { return hs.ready; });
432  }
433  llvm::SmallVector<Value> getInputDatas() {
434  return extractValues<Value, InputHandshake>(
435  inputs, [](auto &hs) { return hs.data; });
436  }
437  llvm::SmallVector<std::shared_ptr<Backedge>> getOutputValids() {
438  return extractValues<std::shared_ptr<Backedge>, OutputHandshake>(
439  outputs, [](auto &hs) { return hs.valid; });
440  }
441  llvm::SmallVector<Value> getOutputReadys() {
442  return extractValues<Value, OutputHandshake>(
443  outputs, [](auto &hs) { return hs.ready; });
444  }
445  llvm::SmallVector<std::shared_ptr<Backedge>> getOutputDatas() {
446  return extractValues<std::shared_ptr<Backedge>, OutputHandshake>(
447  outputs, [](auto &hs) { return hs.data; });
448  }
449 };
450 
451 // A class containing a bunch of syntactic sugar to reduce builder function
452 // verbosity.
453 // @todo: should be moved to support.
454 struct RTLBuilder {
455  RTLBuilder(hw::ModulePortInfo info, OpBuilder &builder, Location loc,
456  Value clk = Value(), Value rst = Value())
457  : info(std::move(info)), b(builder), loc(loc), clk(clk), rst(rst) {}
458 
459  Value constant(const APInt &apv, std::optional<StringRef> name = {}) {
460  // Cannot use zero-width APInt's in DenseMap's, see
461  // https://github.com/llvm/llvm-project/issues/58013
462  bool isZeroWidth = apv.getBitWidth() == 0;
463  if (!isZeroWidth) {
464  auto it = constants.find(apv);
465  if (it != constants.end())
466  return it->second;
467  }
468 
469  auto cval = b.create<hw::ConstantOp>(loc, apv);
470  if (!isZeroWidth)
471  constants[apv] = cval;
472  return cval;
473  }
474 
475  Value constant(unsigned width, int64_t value,
476  std::optional<StringRef> name = {}) {
477  return constant(APInt(width, value));
478  }
479  std::pair<Value, Value> wrap(Value data, Value valid,
480  std::optional<StringRef> name = {}) {
481  auto wrapOp = b.create<esi::WrapValidReadyOp>(loc, data, valid);
482  return {wrapOp.getResult(0), wrapOp.getResult(1)};
483  }
484  std::pair<Value, Value> unwrap(Value channel, Value ready,
485  std::optional<StringRef> name = {}) {
486  auto unwrapOp = b.create<esi::UnwrapValidReadyOp>(loc, channel, ready);
487  return {unwrapOp.getResult(0), unwrapOp.getResult(1)};
488  }
489 
490  // Various syntactic sugar functions.
491  Value reg(StringRef name, Value in, Value rstValue, Value clk = Value(),
492  Value rst = Value()) {
493  Value resolvedClk = clk ? clk : this->clk;
494  Value resolvedRst = rst ? rst : this->rst;
495  assert(resolvedClk &&
496  "No global clock provided to this RTLBuilder - a clock "
497  "signal must be provided to the reg(...) function.");
498  assert(resolvedRst &&
499  "No global reset provided to this RTLBuilder - a reset "
500  "signal must be provided to the reg(...) function.");
501 
502  return b.create<seq::CompRegOp>(loc, in.getType(), in, resolvedClk, name,
503  resolvedRst, rstValue, hw::InnerSymAttr());
504  }
505 
506  Value cmp(Value lhs, Value rhs, comb::ICmpPredicate predicate,
507  std::optional<StringRef> name = {}) {
508  return b.create<comb::ICmpOp>(loc, predicate, lhs, rhs);
509  }
510 
511  Value buildNamedOp(llvm::function_ref<Value()> f,
512  std::optional<StringRef> name) {
513  Value v = f();
514  StringAttr nameAttr;
515  Operation *op = v.getDefiningOp();
516  if (name.has_value()) {
517  op->setAttr("sv.namehint", b.getStringAttr(*name));
518  nameAttr = b.getStringAttr(*name);
519  }
520  return v;
521  }
522 
523  // Bitwise 'and'.
524  Value bAnd(ValueRange values, std::optional<StringRef> name = {}) {
525  return buildNamedOp(
526  [&]() { return b.create<comb::AndOp>(loc, values, false); }, name);
527  }
528 
529  Value bOr(ValueRange values, std::optional<StringRef> name = {}) {
530  return buildNamedOp(
531  [&]() { return b.create<comb::OrOp>(loc, values, false); }, name);
532  }
533 
534  // Bitwise 'not'.
535  Value bNot(Value value, std::optional<StringRef> name = {}) {
536  auto allOnes = constant(value.getType().getIntOrFloatBitWidth(), -1);
537  std::string inferedName;
538  if (!name) {
539  // Try to create a name from the input value.
540  if (auto valueName =
541  value.getDefiningOp()->getAttrOfType<StringAttr>("sv.namehint")) {
542  inferedName = ("not_" + valueName.getValue()).str();
543  name = inferedName;
544  }
545  }
546 
547  return buildNamedOp(
548  [&]() { return b.create<comb::XorOp>(loc, value, allOnes); }, name);
549 
550  return b.createOrFold<comb::XorOp>(loc, value, allOnes, false);
551  }
552 
553  Value shl(Value value, Value shift, std::optional<StringRef> name = {}) {
554  return buildNamedOp(
555  [&]() { return b.create<comb::ShlOp>(loc, value, shift); }, name);
556  }
557 
558  Value concat(ValueRange values, std::optional<StringRef> name = {}) {
559  return buildNamedOp([&]() { return b.create<comb::ConcatOp>(loc, values); },
560  name);
561  }
562 
563  // Packs a list of values into a hw.struct.
564  Value pack(ValueRange values, Type structType = Type(),
565  std::optional<StringRef> name = {}) {
566  if (!structType)
567  structType = tupleToStruct(values.getTypes());
568  return buildNamedOp(
569  [&]() { return b.create<hw::StructCreateOp>(loc, structType, values); },
570  name);
571  }
572 
573  // Unpacks a hw.struct into a list of values.
574  ValueRange unpack(Value value) {
575  auto structType = value.getType().cast<hw::StructType>();
576  llvm::SmallVector<Type> innerTypes;
577  structType.getInnerTypes(innerTypes);
578  return b.create<hw::StructExplodeOp>(loc, innerTypes, value).getResults();
579  }
580 
581  llvm::SmallVector<Value> toBits(Value v, std::optional<StringRef> name = {}) {
582  llvm::SmallVector<Value> bits;
583  for (unsigned i = 0, e = v.getType().getIntOrFloatBitWidth(); i != e; ++i)
584  bits.push_back(b.create<comb::ExtractOp>(loc, v, i, /*bitWidth=*/1));
585  return bits;
586  }
587 
588  // OR-reduction of the bits in 'v'.
589  Value rOr(Value v, std::optional<StringRef> name = {}) {
590  return buildNamedOp([&]() { return bOr(toBits(v)); }, name);
591  }
592 
593  // Extract bits v[hi:lo] (inclusive).
594  Value extract(Value v, unsigned lo, unsigned hi,
595  std::optional<StringRef> name = {}) {
596  unsigned width = hi - lo + 1;
597  return buildNamedOp(
598  [&]() { return b.create<comb::ExtractOp>(loc, v, lo, width); }, name);
599  }
600 
601  // Truncates 'value' to its lower 'width' bits.
602  Value truncate(Value value, unsigned width,
603  std::optional<StringRef> name = {}) {
604  return extract(value, 0, width - 1, name);
605  }
606 
607  Value zext(Value value, unsigned outWidth,
608  std::optional<StringRef> name = {}) {
609  unsigned inWidth = value.getType().getIntOrFloatBitWidth();
610  assert(inWidth <= outWidth && "zext: input width must be <- output width.");
611  if (inWidth == outWidth)
612  return value;
613  auto c0 = constant(outWidth - inWidth, 0);
614  return concat({c0, value}, name);
615  }
616 
617  Value sext(Value value, unsigned outWidth,
618  std::optional<StringRef> name = {}) {
619  return comb::createOrFoldSExt(loc, value, b.getIntegerType(outWidth), b);
620  }
621 
622  // Extracts a single bit v[bit].
623  Value bit(Value v, unsigned index, std::optional<StringRef> name = {}) {
624  return extract(v, index, index, name);
625  }
626 
627  // Creates a hw.array of the given values.
628  Value arrayCreate(ValueRange values, std::optional<StringRef> name = {}) {
629  return buildNamedOp(
630  [&]() { return b.create<hw::ArrayCreateOp>(loc, values); }, name);
631  }
632 
633  // Extract the 'index'th value from the input array.
634  Value arrayGet(Value array, Value index, std::optional<StringRef> name = {}) {
635  return buildNamedOp(
636  [&]() { return b.create<hw::ArrayGetOp>(loc, array, index); }, name);
637  }
638 
639  // Muxes a range of values.
640  // The select signal is expected to be a decimal value which selects starting
641  // from the lowest index of value.
642  Value mux(Value index, ValueRange values,
643  std::optional<StringRef> name = {}) {
644  if (values.size() == 2)
645  return b.create<comb::MuxOp>(loc, index, values[1], values[0]);
646 
647  return arrayGet(arrayCreate(values), index, name);
648  }
649 
650  // Muxes a range of values. The select signal is expected to be a 1-hot
651  // encoded value.
652  Value ohMux(Value index, ValueRange inputs) {
653  // Confirm the select input can be a one-hot encoding for the inputs.
654  unsigned numInputs = inputs.size();
655  assert(numInputs == index.getType().getIntOrFloatBitWidth() &&
656  "one-hot select can't mux inputs");
657 
658  // Start the mux tree with zero value.
659  // Todo: clean up when handshake supports i0.
660  auto dataType = inputs[0].getType();
661  unsigned width =
662  dataType.isa<NoneType>() ? 0 : dataType.getIntOrFloatBitWidth();
663  Value muxValue = constant(width, 0);
664 
665  // Iteratively chain together muxes from the high bit to the low bit.
666  for (size_t i = numInputs - 1; i != 0; --i) {
667  Value input = inputs[i];
668  Value selectBit = bit(index, i);
669  muxValue = mux(selectBit, {muxValue, input});
670  }
671 
672  return muxValue;
673  }
674 
675  hw::ModulePortInfo info;
676  OpBuilder &b;
677  Location loc;
678  Value clk, rst;
679  DenseMap<APInt, Value> constants;
680 };
681 
682 /// Creates a Value that has an assigned zero value. For structs, this
683 /// corresponds to assigning zero to each element recursively.
684 static Value createZeroDataConst(RTLBuilder &s, Location loc, Type type) {
685  return TypeSwitch<Type, Value>(type)
686  .Case<NoneType>([&](NoneType) { return s.constant(0, 0); })
687  .Case<IntType, IntegerType>([&](auto type) {
688  return s.constant(type.getIntOrFloatBitWidth(), 0);
689  })
690  .Case<hw::StructType>([&](auto structType) {
691  SmallVector<Value> zeroValues;
692  for (auto field : structType.getElements())
693  zeroValues.push_back(createZeroDataConst(s, loc, field.type));
694  return s.b.create<hw::StructCreateOp>(loc, structType, zeroValues);
695  })
696  .Default([&](Type) -> Value {
697  emitError(loc) << "unsupported type for zero value: " << type;
698  assert(false);
699  return {};
700  });
701 }
702 
703 static void
704 addSequentialIOOperandsIfNeeded(Operation *op,
705  llvm::SmallVectorImpl<Value> &operands) {
706  if (op->hasTrait<mlir::OpTrait::HasClock>()) {
707  // Parent should at this point be a hw.module and have clock and reset
708  // ports.
709  auto parent = cast<hw::HWModuleOp>(op->getParentOp());
710  operands.push_back(
711  parent.getArgumentForInput(parent.getNumInputPorts() - 2));
712  operands.push_back(
713  parent.getArgumentForInput(parent.getNumInputPorts() - 1));
714  }
715 }
716 
717 template <typename T>
718 class HandshakeConversionPattern : public OpConversionPattern<T> {
719 public:
720  HandshakeConversionPattern(ESITypeConverter &typeConverter,
721  MLIRContext *context, OpBuilder &submoduleBuilder,
722  HandshakeLoweringState &ls)
723  : OpConversionPattern<T>::OpConversionPattern(typeConverter, context),
724  submoduleBuilder(submoduleBuilder), ls(ls) {}
725 
726  using OpAdaptor = typename T::Adaptor;
727 
728  LogicalResult
729  matchAndRewrite(T op, OpAdaptor adaptor,
730  ConversionPatternRewriter &rewriter) const override {
731 
732  // Check if a submodule has already been created for the op. If so,
733  // instantiate the submodule. Else, run the pattern-defined module
734  // builder.
735  hw::HWModuleLike implModule = checkSubModuleOp(ls.parentModule, op);
736  if (!implModule) {
737  auto portInfo = ModulePortInfo(getPortInfoForOp(op));
738 
739  submoduleBuilder.setInsertionPoint(op->getParentOp());
740  implModule = submoduleBuilder.create<hw::HWModuleOp>(
741  op.getLoc(), submoduleBuilder.getStringAttr(getSubModuleName(op)),
742  portInfo, [&](OpBuilder &b, hw::HWModulePortAccessor &ports) {
743  // if 'op' has clock trait, extract these and provide them to the
744  // RTL builder.
745  Value clk, rst;
746  if (op->template hasTrait<mlir::OpTrait::HasClock>()) {
747  clk = ports.getInput("clock");
748  rst = ports.getInput("reset");
749  }
750 
751  BackedgeBuilder bb(b, op.getLoc());
752  RTLBuilder s(ports.getPortList(), b, op.getLoc(), clk, rst);
753  this->buildModule(op, bb, s, ports);
754  });
755  }
756 
757  // Instantiate the submodule.
758  llvm::SmallVector<Value> operands = adaptor.getOperands();
759  addSequentialIOOperandsIfNeeded(op, operands);
760  rewriter.replaceOpWithNewOp<hw::InstanceOp>(
761  op, implModule, rewriter.getStringAttr(ls.nameUniquer(op)), operands);
762  return success();
763  }
764 
765  virtual void buildModule(T op, BackedgeBuilder &bb, RTLBuilder &builder,
766  hw::HWModulePortAccessor &ports) const = 0;
767 
768  // Syntactic sugar functions.
769  // Unwraps an ESI-interfaced module into its constituent handshake signals.
770  // Backedges are created for the to-be-resolved signals, and output ports
771  // are assigned to their wrapped counterparts.
772  UnwrappedIO unwrapIO(RTLBuilder &s, BackedgeBuilder &bb,
773  hw::HWModulePortAccessor &ports) const {
774  UnwrappedIO unwrapped;
775  for (auto port : ports.getInputs()) {
776  if (!isa<esi::ChannelType>(port.getType()))
777  continue;
778  InputHandshake hs;
779  auto ready = std::make_shared<Backedge>(bb.get(s.b.getI1Type()));
780  auto [data, valid] = s.unwrap(port, *ready);
781  hs.data = data;
782  hs.valid = valid;
783  hs.ready = ready;
784  unwrapped.inputs.push_back(hs);
785  }
786  for (auto &outputInfo : ports.getPortList().getOutputs()) {
787  esi::ChannelType channelType =
788  dyn_cast<esi::ChannelType>(outputInfo.type);
789  if (!channelType)
790  continue;
791  OutputHandshake hs;
792  Type innerType = channelType.getInner();
793  auto data = std::make_shared<Backedge>(bb.get(innerType));
794  auto valid = std::make_shared<Backedge>(bb.get(s.b.getI1Type()));
795  auto [dataCh, ready] = s.wrap(*data, *valid);
796  hs.data = data;
797  hs.valid = valid;
798  hs.ready = ready;
799  ports.setOutput(outputInfo.name, dataCh);
800  unwrapped.outputs.push_back(hs);
801  }
802  return unwrapped;
803  }
804 
805  void setAllReadyWithCond(RTLBuilder &s, ArrayRef<InputHandshake> inputs,
806  OutputHandshake &output, Value cond) const {
807  auto validAndReady = s.bAnd({output.ready, cond});
808  for (auto &input : inputs)
809  input.ready->setValue(validAndReady);
810  }
811 
812  void buildJoinLogic(RTLBuilder &s, ArrayRef<InputHandshake> inputs,
813  OutputHandshake &output) const {
814  llvm::SmallVector<Value> valids;
815  for (auto &input : inputs)
816  valids.push_back(input.valid);
817  Value allValid = s.bAnd(valids);
818  output.valid->setValue(allValid);
819  setAllReadyWithCond(s, inputs, output, allValid);
820  }
821 
822  // Builds mux logic for the given inputs and outputs.
823  // Note: it is assumed that the caller has removed the 'select' signal from
824  // the 'unwrapped' inputs and provide it as a separate argument.
825  void buildMuxLogic(RTLBuilder &s, UnwrappedIO &unwrapped,
826  InputHandshake &select) const {
827  // ============================= Control logic =============================
828  size_t numInputs = unwrapped.inputs.size();
829  size_t selectWidth = llvm::Log2_64_Ceil(numInputs);
830  Value truncatedSelect =
831  select.data.getType().getIntOrFloatBitWidth() > selectWidth
832  ? s.truncate(select.data, selectWidth)
833  : select.data;
834 
835  // Decimal-to-1-hot decoder. 'shl' operands must be identical in size.
836  auto selectZext = s.zext(truncatedSelect, numInputs);
837  auto select1h = s.shl(s.constant(numInputs, 1), selectZext);
838  auto &res = unwrapped.outputs[0];
839 
840  // Mux input valid signals.
841  auto selectedInputValid =
842  s.mux(truncatedSelect, unwrapped.getInputValids());
843  // Result is valid when the selected input and the select input is valid.
844  auto selAndInputValid = s.bAnd({selectedInputValid, select.valid});
845  res.valid->setValue(selAndInputValid);
846  auto resValidAndReady = s.bAnd({selAndInputValid, res.ready});
847 
848  // Select is ready when result is valid and ready (result transacting).
849  select.ready->setValue(resValidAndReady);
850 
851  // Assign each input ready signal if it is currently selected.
852  for (auto [inIdx, in] : llvm::enumerate(unwrapped.inputs)) {
853  // Extract the selection bit for this input.
854  auto isSelected = s.bit(select1h, inIdx);
855 
856  // '&' that with the result valid and ready, and assign to the input
857  // ready signal.
858  auto activeAndResultValidAndReady =
859  s.bAnd({isSelected, resValidAndReady});
860  in.ready->setValue(activeAndResultValidAndReady);
861  }
862 
863  // ============================== Data logic ===============================
864  res.data->setValue(s.mux(truncatedSelect, unwrapped.getInputDatas()));
865  }
866 
867  // Builds fork logic between the single input and multiple outputs' control
868  // networks. Caller is expected to handle data separately.
869  void buildForkLogic(RTLBuilder &s, BackedgeBuilder &bb, InputHandshake &input,
870  ArrayRef<OutputHandshake> outputs) const {
871  auto c0I1 = s.constant(1, 0);
872  llvm::SmallVector<Value> doneWires;
873  for (auto [i, output] : llvm::enumerate(outputs)) {
874  auto doneBE = bb.get(s.b.getI1Type());
875  auto emitted = s.bAnd({doneBE, s.bNot(*input.ready)});
876  auto emittedReg = s.reg("emitted_" + std::to_string(i), emitted, c0I1);
877  auto outValid = s.bAnd({s.bNot(emittedReg), input.valid});
878  output.valid->setValue(outValid);
879  auto validReady = s.bAnd({output.ready, outValid});
880  auto done = s.bOr({validReady, emittedReg}, "done" + std::to_string(i));
881  doneBE.setValue(done);
882  doneWires.push_back(done);
883  }
884  input.ready->setValue(s.bAnd(doneWires, "allDone"));
885  }
886 
887  // Builds a unit-rate actor around an inner operation. 'unitBuilder' is a
888  // function which takes the set of unwrapped data inputs, and returns a
889  // value which should be assigned to the output data value.
890  void buildUnitRateJoinLogic(
891  RTLBuilder &s, UnwrappedIO &unwrappedIO,
892  llvm::function_ref<Value(ValueRange)> unitBuilder) const {
893  assert(unwrappedIO.outputs.size() == 1 &&
894  "Expected exactly one output for unit-rate join actor");
895  // Control logic.
896  this->buildJoinLogic(s, unwrappedIO.inputs, unwrappedIO.outputs[0]);
897 
898  // Data logic.
899  auto unitRes = unitBuilder(unwrappedIO.getInputDatas());
900  unwrappedIO.outputs[0].data->setValue(unitRes);
901  }
902 
903  void buildUnitRateForkLogic(
904  RTLBuilder &s, BackedgeBuilder &bb, UnwrappedIO &unwrappedIO,
905  llvm::function_ref<llvm::SmallVector<Value>(Value)> unitBuilder) const {
906  assert(unwrappedIO.inputs.size() == 1 &&
907  "Expected exactly one input for unit-rate fork actor");
908  // Control logic.
909  this->buildForkLogic(s, bb, unwrappedIO.inputs[0], unwrappedIO.outputs);
910 
911  // Data logic.
912  auto unitResults = unitBuilder(unwrappedIO.inputs[0].data);
913  assert(unitResults.size() == unwrappedIO.outputs.size() &&
914  "Expected unit builder to return one result per output");
915  for (auto [res, outport] : llvm::zip(unitResults, unwrappedIO.outputs))
916  outport.data->setValue(res);
917  }
918 
919  void buildExtendLogic(RTLBuilder &s, UnwrappedIO &unwrappedIO,
920  bool signExtend) const {
921  size_t outWidth =
922  toValidType(static_cast<Value>(*unwrappedIO.outputs[0].data).getType())
923  .getIntOrFloatBitWidth();
924  buildUnitRateJoinLogic(s, unwrappedIO, [&](ValueRange inputs) {
925  if (signExtend)
926  return s.sext(inputs[0], outWidth);
927  return s.zext(inputs[0], outWidth);
928  });
929  }
930 
931  void buildTruncateLogic(RTLBuilder &s, UnwrappedIO &unwrappedIO,
932  unsigned targetWidth) const {
933  size_t outWidth =
934  toValidType(static_cast<Value>(*unwrappedIO.outputs[0].data).getType())
935  .getIntOrFloatBitWidth();
936  buildUnitRateJoinLogic(s, unwrappedIO, [&](ValueRange inputs) {
937  return s.truncate(inputs[0], outWidth);
938  });
939  }
940 
941  /// Return the number of bits needed to index the given number of values.
942  static size_t getNumIndexBits(uint64_t numValues) {
943  return numValues > 1 ? llvm::Log2_64_Ceil(numValues) : 1;
944  }
945 
946  Value buildPriorityArbiter(RTLBuilder &s, ArrayRef<Value> inputs,
947  Value defaultValue,
948  DenseMap<size_t, Value> &indexMapping) const {
949  auto numInputs = inputs.size();
950  auto priorityArb = defaultValue;
951 
952  for (size_t i = numInputs; i > 0; --i) {
953  size_t inputIndex = i - 1;
954  size_t oneHotIndex = size_t{1} << inputIndex;
955  auto constIndex = s.constant(numInputs, oneHotIndex);
956  indexMapping[inputIndex] = constIndex;
957  priorityArb = s.mux(inputs[inputIndex], {priorityArb, constIndex});
958  }
959  return priorityArb;
960  }
961 
962 private:
963  OpBuilder &submoduleBuilder;
964  HandshakeLoweringState &ls;
965 };
966 
967 class ForkConversionPattern : public HandshakeConversionPattern<ForkOp> {
968 public:
969  using HandshakeConversionPattern<ForkOp>::HandshakeConversionPattern;
970  void buildModule(ForkOp op, BackedgeBuilder &bb, RTLBuilder &s,
971  hw::HWModulePortAccessor &ports) const override {
972  auto unwrapped = unwrapIO(s, bb, ports);
973  buildUnitRateForkLogic(s, bb, unwrapped, [&](Value input) {
974  return llvm::SmallVector<Value>(unwrapped.outputs.size(), input);
975  });
976  }
977 };
978 
979 class JoinConversionPattern : public HandshakeConversionPattern<JoinOp> {
980 public:
981  using HandshakeConversionPattern<JoinOp>::HandshakeConversionPattern;
982  void buildModule(JoinOp op, BackedgeBuilder &bb, RTLBuilder &s,
983  hw::HWModulePortAccessor &ports) const override {
984  auto unwrappedIO = unwrapIO(s, bb, ports);
985  buildJoinLogic(s, unwrappedIO.inputs, unwrappedIO.outputs[0]);
986  unwrappedIO.outputs[0].data->setValue(s.constant(0, 0));
987  };
988 };
989 
990 class SyncConversionPattern : public HandshakeConversionPattern<SyncOp> {
991 public:
992  using HandshakeConversionPattern<SyncOp>::HandshakeConversionPattern;
993  void buildModule(SyncOp op, BackedgeBuilder &bb, RTLBuilder &s,
994  hw::HWModulePortAccessor &ports) const override {
995  auto unwrappedIO = unwrapIO(s, bb, ports);
996 
997  // A helper wire that will be used to connect the two built logics
998  HandshakeWire wire(bb, s.b.getNoneType());
999 
1000  OutputHandshake output = wire.getAsOutput();
1001  buildJoinLogic(s, unwrappedIO.inputs, output);
1002 
1003  InputHandshake input = wire.getAsInput();
1004 
1005  // The state-keeping fork logic is required here, as the circuit isn't
1006  // allowed to wait for all the consumers to be ready. Connecting the ready
1007  // signals of the outputs to their corresponding valid signals leads to
1008  // combinatorial cycles. The paper which introduced compositional dataflow
1009  // circuits explicitly mentions this limitation:
1010  // http://arcade.cs.columbia.edu/df-memocode17.pdf
1011  buildForkLogic(s, bb, input, unwrappedIO.outputs);
1012 
1013  // Directly connect the data wires, only the control signals need to be
1014  // combined.
1015  for (auto &&[in, out] : llvm::zip(unwrappedIO.inputs, unwrappedIO.outputs))
1016  out.data->setValue(in.data);
1017  };
1018 };
1019 
1020 class MuxConversionPattern : public HandshakeConversionPattern<MuxOp> {
1021 public:
1022  using HandshakeConversionPattern<MuxOp>::HandshakeConversionPattern;
1023  void buildModule(MuxOp op, BackedgeBuilder &bb, RTLBuilder &s,
1024  hw::HWModulePortAccessor &ports) const override {
1025  auto unwrappedIO = unwrapIO(s, bb, ports);
1026 
1027  // Extract select signal from the unwrapped IO.
1028  auto select = unwrappedIO.inputs[0];
1029  unwrappedIO.inputs.erase(unwrappedIO.inputs.begin());
1030  buildMuxLogic(s, unwrappedIO, select);
1031  };
1032 };
1033 
1034 class InstanceConversionPattern
1035  : public HandshakeConversionPattern<handshake::InstanceOp> {
1036 public:
1037  using HandshakeConversionPattern<
1038  handshake::InstanceOp>::HandshakeConversionPattern;
1039  void buildModule(handshake::InstanceOp op, BackedgeBuilder &bb, RTLBuilder &s,
1040  hw::HWModulePortAccessor &ports) const override {
1041  assert(false &&
1042  "If we indeed perform conversion in post-order, this "
1043  "should never be called. The base HandshakeConversionPattern logic "
1044  "will instantiate the external module.");
1045  }
1046 };
1047 
1048 class ReturnConversionPattern
1049  : public OpConversionPattern<handshake::ReturnOp> {
1050 public:
1051  using OpConversionPattern::OpConversionPattern;
1052  LogicalResult
1053  matchAndRewrite(ReturnOp op, OpAdaptor adaptor,
1054  ConversionPatternRewriter &rewriter) const override {
1055  // Locate existing output op, Append operands to output op, and move to
1056  // the end of the block.
1057  auto parent = cast<hw::HWModuleOp>(op->getParentOp());
1058  auto outputOp = *parent.getBodyBlock()->getOps<hw::OutputOp>().begin();
1059  outputOp->setOperands(adaptor.getOperands());
1060  outputOp->moveAfter(&parent.getBodyBlock()->back());
1061  rewriter.eraseOp(op);
1062  return success();
1063  }
1064 };
1065 
1066 // Converts an arbitrary operation into a unit rate actor. A unit rate actor
1067 // will transact once all inputs are valid and its output is ready.
1068 template <typename TIn, typename TOut = TIn>
1069 class UnitRateConversionPattern : public HandshakeConversionPattern<TIn> {
1070 public:
1071  using HandshakeConversionPattern<TIn>::HandshakeConversionPattern;
1072  void buildModule(TIn op, BackedgeBuilder &bb, RTLBuilder &s,
1073  hw::HWModulePortAccessor &ports) const override {
1074  auto unwrappedIO = this->unwrapIO(s, bb, ports);
1075  this->buildUnitRateJoinLogic(s, unwrappedIO, [&](ValueRange inputs) {
1076  // Create TOut - it is assumed that TOut trivially
1077  // constructs from the input data signals of TIn.
1078  // To disambiguate ambiguous builders with default arguments (e.g.,
1079  // twoState UnitAttr), specify attribute array explicitly.
1080  return s.b.create<TOut>(op.getLoc(), inputs,
1081  /* attributes */ ArrayRef<NamedAttribute>{});
1082  });
1083  };
1084 };
1085 
1086 class PackConversionPattern : public HandshakeConversionPattern<PackOp> {
1087 public:
1088  using HandshakeConversionPattern<PackOp>::HandshakeConversionPattern;
1089  void buildModule(PackOp op, BackedgeBuilder &bb, RTLBuilder &s,
1090  hw::HWModulePortAccessor &ports) const override {
1091  auto unwrappedIO = unwrapIO(s, bb, ports);
1092  buildUnitRateJoinLogic(s, unwrappedIO,
1093  [&](ValueRange inputs) { return s.pack(inputs); });
1094  };
1095 };
1096 
1097 class StructCreateConversionPattern
1098  : public HandshakeConversionPattern<hw::StructCreateOp> {
1099 public:
1100  using HandshakeConversionPattern<
1101  hw::StructCreateOp>::HandshakeConversionPattern;
1102  void buildModule(hw::StructCreateOp op, BackedgeBuilder &bb, RTLBuilder &s,
1103  hw::HWModulePortAccessor &ports) const override {
1104  auto unwrappedIO = unwrapIO(s, bb, ports);
1105  auto structType = op.getResult().getType();
1106  buildUnitRateJoinLogic(s, unwrappedIO, [&](ValueRange inputs) {
1107  return s.pack(inputs, structType);
1108  });
1109  };
1110 };
1111 
1112 class UnpackConversionPattern : public HandshakeConversionPattern<UnpackOp> {
1113 public:
1114  using HandshakeConversionPattern<UnpackOp>::HandshakeConversionPattern;
1115  void buildModule(UnpackOp op, BackedgeBuilder &bb, RTLBuilder &s,
1116  hw::HWModulePortAccessor &ports) const override {
1117  auto unwrappedIO = unwrapIO(s, bb, ports);
1118  buildUnitRateForkLogic(s, bb, unwrappedIO,
1119  [&](Value input) { return s.unpack(input); });
1120  };
1121 };
1122 
1123 class ConditionalBranchConversionPattern
1124  : public HandshakeConversionPattern<ConditionalBranchOp> {
1125 public:
1126  using HandshakeConversionPattern<
1127  ConditionalBranchOp>::HandshakeConversionPattern;
1128  void buildModule(ConditionalBranchOp op, BackedgeBuilder &bb, RTLBuilder &s,
1129  hw::HWModulePortAccessor &ports) const override {
1130  auto unwrappedIO = unwrapIO(s, bb, ports);
1131  auto cond = unwrappedIO.inputs[0];
1132  auto arg = unwrappedIO.inputs[1];
1133  auto trueRes = unwrappedIO.outputs[0];
1134  auto falseRes = unwrappedIO.outputs[1];
1135 
1136  auto condArgValid = s.bAnd({cond.valid, arg.valid});
1137 
1138  // Connect valid signal of both results.
1139  trueRes.valid->setValue(s.bAnd({cond.data, condArgValid}));
1140  falseRes.valid->setValue(s.bAnd({s.bNot(cond.data), condArgValid}));
1141 
1142  // Connecte data signals of both results.
1143  trueRes.data->setValue(arg.data);
1144  falseRes.data->setValue(arg.data);
1145 
1146  // Connect ready signal of input and condition.
1147  auto selectedResultReady =
1148  s.mux(cond.data, {falseRes.ready, trueRes.ready});
1149  auto condArgReady = s.bAnd({selectedResultReady, condArgValid});
1150  arg.ready->setValue(condArgReady);
1151  cond.ready->setValue(condArgReady);
1152  };
1153 };
1154 
1155 template <typename TIn, bool signExtend>
1156 class ExtendConversionPattern : public HandshakeConversionPattern<TIn> {
1157 public:
1158  using HandshakeConversionPattern<TIn>::HandshakeConversionPattern;
1159  void buildModule(TIn op, BackedgeBuilder &bb, RTLBuilder &s,
1160  hw::HWModulePortAccessor &ports) const override {
1161  auto unwrappedIO = this->unwrapIO(s, bb, ports);
1162  this->buildExtendLogic(s, unwrappedIO, /*signExtend=*/signExtend);
1163  };
1164 };
1165 
1166 class ComparisonConversionPattern
1167  : public HandshakeConversionPattern<arith::CmpIOp> {
1168 public:
1169  using HandshakeConversionPattern<arith::CmpIOp>::HandshakeConversionPattern;
1170  void buildModule(arith::CmpIOp op, BackedgeBuilder &bb, RTLBuilder &s,
1171  hw::HWModulePortAccessor &ports) const override {
1172  auto unwrappedIO = this->unwrapIO(s, bb, ports);
1173  auto buildCompareLogic = [&](comb::ICmpPredicate predicate) {
1174  return buildUnitRateJoinLogic(s, unwrappedIO, [&](ValueRange inputs) {
1175  return s.b.create<comb::ICmpOp>(op.getLoc(), predicate, inputs[0],
1176  inputs[1]);
1177  });
1178  };
1179 
1180  switch (op.getPredicate()) {
1181  case arith::CmpIPredicate::eq:
1182  return buildCompareLogic(comb::ICmpPredicate::eq);
1183  case arith::CmpIPredicate::ne:
1184  return buildCompareLogic(comb::ICmpPredicate::ne);
1185  case arith::CmpIPredicate::slt:
1186  return buildCompareLogic(comb::ICmpPredicate::slt);
1187  case arith::CmpIPredicate::ult:
1188  return buildCompareLogic(comb::ICmpPredicate::ult);
1189  case arith::CmpIPredicate::sle:
1190  return buildCompareLogic(comb::ICmpPredicate::sle);
1191  case arith::CmpIPredicate::ule:
1192  return buildCompareLogic(comb::ICmpPredicate::ule);
1193  case arith::CmpIPredicate::sgt:
1194  return buildCompareLogic(comb::ICmpPredicate::sgt);
1195  case arith::CmpIPredicate::ugt:
1196  return buildCompareLogic(comb::ICmpPredicate::ugt);
1197  case arith::CmpIPredicate::sge:
1198  return buildCompareLogic(comb::ICmpPredicate::sge);
1199  case arith::CmpIPredicate::uge:
1200  return buildCompareLogic(comb::ICmpPredicate::uge);
1201  }
1202  assert(false && "invalid CmpIOp");
1203  };
1204 };
1205 
1206 class TruncateConversionPattern
1207  : public HandshakeConversionPattern<arith::TruncIOp> {
1208 public:
1209  using HandshakeConversionPattern<arith::TruncIOp>::HandshakeConversionPattern;
1210  void buildModule(arith::TruncIOp op, BackedgeBuilder &bb, RTLBuilder &s,
1211  hw::HWModulePortAccessor &ports) const override {
1212  auto unwrappedIO = this->unwrapIO(s, bb, ports);
1213  unsigned targetBits =
1214  toValidType(op.getResult().getType()).getIntOrFloatBitWidth();
1215  buildTruncateLogic(s, unwrappedIO, targetBits);
1216  };
1217 };
1218 
1219 class ControlMergeConversionPattern
1220  : public HandshakeConversionPattern<ControlMergeOp> {
1221 public:
1222  using HandshakeConversionPattern<ControlMergeOp>::HandshakeConversionPattern;
1223  void buildModule(ControlMergeOp op, BackedgeBuilder &bb, RTLBuilder &s,
1224  hw::HWModulePortAccessor &ports) const override {
1225  auto unwrappedIO = this->unwrapIO(s, bb, ports);
1226  auto resData = unwrappedIO.outputs[0];
1227  auto resIndex = unwrappedIO.outputs[1];
1228 
1229  // Define some common types and values that will be used.
1230  unsigned numInputs = unwrappedIO.inputs.size();
1231  auto indexType = s.b.getIntegerType(numInputs);
1232  Value noWinner = s.constant(numInputs, 0);
1233  Value c0I1 = s.constant(1, 0);
1234 
1235  // Declare register for storing arbitration winner.
1236  auto won = bb.get(indexType);
1237  Value wonReg = s.reg("won_reg", won, noWinner);
1238 
1239  // Declare wire for arbitration winner.
1240  auto win = bb.get(indexType);
1241 
1242  // Declare wire for whether the circuit just fired and emitted both
1243  // outputs.
1244  auto fired = bb.get(s.b.getI1Type());
1245 
1246  // Declare registers for storing if each output has been emitted.
1247  auto resultEmitted = bb.get(s.b.getI1Type());
1248  Value resultEmittedReg = s.reg("result_emitted_reg", resultEmitted, c0I1);
1249  auto indexEmitted = bb.get(s.b.getI1Type());
1250  Value indexEmittedReg = s.reg("index_emitted_reg", indexEmitted, c0I1);
1251 
1252  // Declare wires for if each output is done.
1253  auto resultDone = bb.get(s.b.getI1Type());
1254  auto indexDone = bb.get(s.b.getI1Type());
1255 
1256  // Create predicates to assert if the win wire or won register hold a
1257  // valid index.
1258  auto hasWinnerCondition = s.rOr({win});
1259  auto hadWinnerCondition = s.rOr({wonReg});
1260 
1261  // Create an arbiter based on a simple priority-encoding scheme to assign
1262  // an index to the win wire. If the won register is set, just use that. In
1263  // the case that won is not set and no input is valid, set a sentinel
1264  // value to indicate no winner was chosen. The constant values are
1265  // remembered in a map so they can be re-used later to assign the arg
1266  // ready outputs.
1267  DenseMap<size_t, Value> argIndexValues;
1268  Value priorityArb = buildPriorityArbiter(s, unwrappedIO.getInputValids(),
1269  noWinner, argIndexValues);
1270  priorityArb = s.mux(hadWinnerCondition, {priorityArb, wonReg});
1271  win.setValue(priorityArb);
1272 
1273  // Create the logic to assign the result and index outputs. The result
1274  // valid output will always be assigned, and if isControl is not set, the
1275  // result data output will also be assigned. The index valid and data
1276  // outputs will always be assigned. The win wire from the arbiter is used
1277  // to index into a tree of muxes to select the chosen input's signal(s),
1278  // and is fed directly to the index output. Both the result and index
1279  // valid outputs are gated on the win wire being set to something other
1280  // than the sentinel value.
1281  auto resultNotEmitted = s.bNot(resultEmittedReg);
1282  auto resultValid = s.bAnd({hasWinnerCondition, resultNotEmitted});
1283  resData.valid->setValue(resultValid);
1284  resData.data->setValue(s.ohMux(win, unwrappedIO.getInputDatas()));
1285 
1286  auto indexNotEmitted = s.bNot(indexEmittedReg);
1287  auto indexValid = s.bAnd({hasWinnerCondition, indexNotEmitted});
1288  resIndex.valid->setValue(indexValid);
1289 
1290  // Use the one-hot win wire to select the index to output in the index
1291  // data.
1292  SmallVector<Value, 8> indexOutputs;
1293  for (size_t i = 0; i < numInputs; ++i)
1294  indexOutputs.push_back(s.constant(64, i));
1295 
1296  auto indexOutput = s.ohMux(win, indexOutputs);
1297  resIndex.data->setValue(indexOutput);
1298 
1299  // Create the logic to set the won register. If the fired wire is
1300  // asserted, we have finished this round and can and reset the register to
1301  // the sentinel value that indicates there is no winner. Otherwise, we
1302  // need to hold the value of the win register until we can fire.
1303  won.setValue(s.mux(fired, {win, noWinner}));
1304 
1305  // Create the logic to set the done wires for the result and index. For
1306  // both outputs, the done wire is asserted when the output is valid and
1307  // ready, or the emitted register for that output is set.
1308  auto resultValidAndReady = s.bAnd({resultValid, resData.ready});
1309  resultDone.setValue(s.bOr({resultValidAndReady, resultEmittedReg}));
1310 
1311  auto indexValidAndReady = s.bAnd({indexValid, resIndex.ready});
1312  indexDone.setValue(s.bOr({indexValidAndReady, indexEmittedReg}));
1313 
1314  // Create the logic to set the fired wire. It is asserted when both result
1315  // and index are done.
1316  fired.setValue(s.bAnd({resultDone, indexDone}));
1317 
1318  // Create the logic to assign the emitted registers. If the fired wire is
1319  // asserted, we have finished this round and can reset the registers to 0.
1320  // Otherwise, we need to hold the values of the done registers until we
1321  // can fire.
1322  resultEmitted.setValue(s.mux(fired, {resultDone, c0I1}));
1323  indexEmitted.setValue(s.mux(fired, {indexDone, c0I1}));
1324 
1325  // Create the logic to assign the arg ready outputs. The logic is
1326  // identical for each arg. If the fired wire is asserted, and the win wire
1327  // holds an arg's index, that arg is ready.
1328  auto winnerOrDefault = s.mux(fired, {noWinner, win});
1329  for (auto [i, ir] : llvm::enumerate(unwrappedIO.getInputReadys())) {
1330  auto &indexValue = argIndexValues[i];
1331  ir->setValue(s.cmp(winnerOrDefault, indexValue, comb::ICmpPredicate::eq));
1332  }
1333  };
1334 };
1335 
1336 class MergeConversionPattern : public HandshakeConversionPattern<MergeOp> {
1337 public:
1338  using HandshakeConversionPattern<MergeOp>::HandshakeConversionPattern;
1339  void buildModule(MergeOp op, BackedgeBuilder &bb, RTLBuilder &s,
1340  hw::HWModulePortAccessor &ports) const override {
1341  auto unwrappedIO = this->unwrapIO(s, bb, ports);
1342  auto resData = unwrappedIO.outputs[0];
1343 
1344  // Define some common types and values that will be used.
1345  unsigned numInputs = unwrappedIO.inputs.size();
1346  auto indexType = s.b.getIntegerType(numInputs);
1347  Value noWinner = s.constant(numInputs, 0);
1348 
1349  // Declare wire for arbitration winner.
1350  auto win = bb.get(indexType);
1351 
1352  // Create predicates to assert if the win wire holds a valid index.
1353  auto hasWinnerCondition = s.rOr(win);
1354 
1355  // Create an arbiter based on a simple priority-encoding scheme to assign an
1356  // index to the win wire. In the case that no input is valid, set a sentinel
1357  // value to indicate no winner was chosen. The constant values are
1358  // remembered in a map so they can be re-used later to assign the arg ready
1359  // outputs.
1360  DenseMap<size_t, Value> argIndexValues;
1361  Value priorityArb = buildPriorityArbiter(s, unwrappedIO.getInputValids(),
1362  noWinner, argIndexValues);
1363  win.setValue(priorityArb);
1364 
1365  // Create the logic to assign the result outputs. The result valid and data
1366  // outputs will always be assigned. The win wire from the arbiter is used to
1367  // index into a tree of muxes to select the chosen input's signal(s). The
1368  // result outputs are gated on the win wire being non-zero.
1369 
1370  resData.valid->setValue(hasWinnerCondition);
1371  resData.data->setValue(s.ohMux(win, unwrappedIO.getInputDatas()));
1372 
1373  // Create the logic to set the done wires for the result. The done wire is
1374  // asserted when the output is valid and ready, or the emitted register is
1375  // set.
1376  auto resultValidAndReady = s.bAnd({hasWinnerCondition, resData.ready});
1377 
1378  // Create the logic to assign the arg ready outputs. The logic is
1379  // identical for each arg. If the fired wire is asserted, and the win wire
1380  // holds an arg's index, that arg is ready.
1381  auto winnerOrDefault = s.mux(resultValidAndReady, {noWinner, win});
1382  for (auto [i, ir] : llvm::enumerate(unwrappedIO.getInputReadys())) {
1383  auto &indexValue = argIndexValues[i];
1384  ir->setValue(s.cmp(winnerOrDefault, indexValue, comb::ICmpPredicate::eq));
1385  }
1386  };
1387 };
1388 
1389 class LoadConversionPattern
1390  : public HandshakeConversionPattern<handshake::LoadOp> {
1391 public:
1392  using HandshakeConversionPattern<
1393  handshake::LoadOp>::HandshakeConversionPattern;
1394  void buildModule(handshake::LoadOp op, BackedgeBuilder &bb, RTLBuilder &s,
1395  hw::HWModulePortAccessor &ports) const override {
1396  auto unwrappedIO = this->unwrapIO(s, bb, ports);
1397  auto addrFromUser = unwrappedIO.inputs[0];
1398  auto dataFromMem = unwrappedIO.inputs[1];
1399  auto controlIn = unwrappedIO.inputs[2];
1400  auto dataToUser = unwrappedIO.outputs[0];
1401  auto addrToMem = unwrappedIO.outputs[1];
1402 
1403  addrToMem.data->setValue(addrFromUser.data);
1404  dataToUser.data->setValue(dataFromMem.data);
1405 
1406  // The valid/ready logic between user address/control to memoryAddr is
1407  // join logic.
1408  buildJoinLogic(s, {addrFromUser, controlIn}, addrToMem);
1409 
1410  // The valid/ready logic between memoryData and outputData is a direct
1411  // connection.
1412  dataToUser.valid->setValue(dataFromMem.valid);
1413  dataFromMem.ready->setValue(dataToUser.ready);
1414  };
1415 };
1416 
1417 class StoreConversionPattern
1418  : public HandshakeConversionPattern<handshake::StoreOp> {
1419 public:
1420  using HandshakeConversionPattern<
1421  handshake::StoreOp>::HandshakeConversionPattern;
1422  void buildModule(handshake::StoreOp op, BackedgeBuilder &bb, RTLBuilder &s,
1423  hw::HWModulePortAccessor &ports) const override {
1424  auto unwrappedIO = this->unwrapIO(s, bb, ports);
1425  auto addrFromUser = unwrappedIO.inputs[0];
1426  auto dataFromUser = unwrappedIO.inputs[1];
1427  auto controlIn = unwrappedIO.inputs[2];
1428  auto dataToMem = unwrappedIO.outputs[0];
1429  auto addrToMem = unwrappedIO.outputs[1];
1430 
1431  // Create a gate that will be asserted when all outputs are ready.
1432  auto outputsReady = s.bAnd({dataToMem.ready, addrToMem.ready});
1433 
1434  // Build the standard join logic from the inputs to the inputsValid and
1435  // outputsReady signals.
1436  HandshakeWire joinWire(bb, s.b.getNoneType());
1437  joinWire.ready->setValue(outputsReady);
1438  OutputHandshake joinOutput = joinWire.getAsOutput();
1439  buildJoinLogic(s, {dataFromUser, addrFromUser, controlIn}, joinOutput);
1440 
1441  // Output address and data signals are connected directly.
1442  addrToMem.data->setValue(addrFromUser.data);
1443  dataToMem.data->setValue(dataFromUser.data);
1444 
1445  // Output valid signals are connected from the inputsValid wire.
1446  addrToMem.valid->setValue(*joinWire.valid);
1447  dataToMem.valid->setValue(*joinWire.valid);
1448  };
1449 };
1450 
1451 class MemoryConversionPattern
1452  : public HandshakeConversionPattern<handshake::MemoryOp> {
1453 public:
1454  using HandshakeConversionPattern<
1455  handshake::MemoryOp>::HandshakeConversionPattern;
1456  void buildModule(handshake::MemoryOp op, BackedgeBuilder &bb, RTLBuilder &s,
1457  hw::HWModulePortAccessor &ports) const override {
1458  auto loc = op.getLoc();
1459 
1460  // Gather up the load and store ports.
1461  auto unwrappedIO = this->unwrapIO(s, bb, ports);
1462  struct LoadPort {
1463  InputHandshake &addr;
1464  OutputHandshake &data;
1465  OutputHandshake &done;
1466  };
1467  struct StorePort {
1468  InputHandshake &addr;
1469  InputHandshake &data;
1470  OutputHandshake &done;
1471  };
1472  SmallVector<LoadPort, 4> loadPorts;
1473  SmallVector<StorePort, 4> storePorts;
1474 
1475  unsigned stCount = op.getStCount();
1476  unsigned ldCount = op.getLdCount();
1477  for (unsigned i = 0, e = ldCount; i != e; ++i) {
1478  LoadPort port = {unwrappedIO.inputs[stCount * 2 + i],
1479  unwrappedIO.outputs[i],
1480  unwrappedIO.outputs[ldCount + stCount + i]};
1481  loadPorts.push_back(port);
1482  }
1483 
1484  for (unsigned i = 0, e = stCount; i != e; ++i) {
1485  StorePort port = {unwrappedIO.inputs[i * 2 + 1],
1486  unwrappedIO.inputs[i * 2],
1487  unwrappedIO.outputs[ldCount + i]};
1488  storePorts.push_back(port);
1489  }
1490 
1491  // used to drive the data wire of the control-only channels.
1492  auto c0I0 = s.constant(0, 0);
1493 
1494  auto cl2dim = llvm::Log2_64_Ceil(op.getMemRefType().getShape()[0]);
1495  auto hlmem = s.b.create<seq::HLMemOp>(
1496  loc, s.clk, s.rst, "_handshake_memory_" + std::to_string(op.getId()),
1497  op.getMemRefType().getShape(), op.getMemRefType().getElementType());
1498 
1499  // Create load ports...
1500  for (auto &ld : loadPorts) {
1501  llvm::SmallVector<Value> addresses = {s.truncate(ld.addr.data, cl2dim)};
1502  auto readData = s.b.create<seq::ReadPortOp>(loc, hlmem.getHandle(),
1503  addresses, ld.addr.valid,
1504  /*latency=*/0);
1505  ld.data.data->setValue(readData);
1506  ld.done.data->setValue(c0I0);
1507  // Create control fork for the load address valid and ready signals.
1508  buildForkLogic(s, bb, ld.addr, {ld.data, ld.done});
1509  }
1510 
1511  // Create store ports...
1512  for (auto &st : storePorts) {
1513  // Create a register to buffer the valid path by 1 cycle, to match the
1514  // write latency of 1.
1515  auto writeValidBufferMuxBE = bb.get(s.b.getI1Type());
1516  auto writeValidBuffer =
1517  s.reg("writeValidBuffer", writeValidBufferMuxBE, s.constant(1, 0));
1518  st.done.valid->setValue(writeValidBuffer);
1519  st.done.data->setValue(c0I0);
1520 
1521  // Create the logic for when both the buffered write valid signal and the
1522  // store complete ready signal are asserted.
1523  auto storeCompleted =
1524  s.bAnd({st.done.ready, writeValidBuffer}, "storeCompleted");
1525 
1526  // Create a signal for when the write valid buffer is empty or the output
1527  // is ready.
1528  auto notWriteValidBuffer = s.bNot(writeValidBuffer);
1529  auto emptyOrComplete =
1530  s.bOr({notWriteValidBuffer, storeCompleted}, "emptyOrComplete");
1531 
1532  // Connect the gate to both the store address ready and store data ready
1533  st.addr.ready->setValue(emptyOrComplete);
1534  st.data.ready->setValue(emptyOrComplete);
1535 
1536  // Create a wire for when both the store address and data are valid.
1537  auto writeValid = s.bAnd({st.addr.valid, st.data.valid}, "writeValid");
1538 
1539  // Create a mux that drives the buffer input. If the emptyOrComplete
1540  // signal is asserted, the mux selects the writeValid signal. Otherwise,
1541  // it selects the buffer output, keeping the output registered until the
1542  // emptyOrComplete signal is asserted.
1543  writeValidBufferMuxBE.setValue(
1544  s.mux(emptyOrComplete, {writeValidBuffer, writeValid}));
1545 
1546  // Instantiate the write port operation - truncate address width to memory
1547  // width.
1548  llvm::SmallVector<Value> addresses = {s.truncate(st.addr.data, cl2dim)};
1549  s.b.create<seq::WritePortOp>(loc, hlmem.getHandle(), addresses,
1550  st.data.data, writeValid,
1551  /*latency=*/1);
1552  }
1553  }
1554 }; // namespace
1555 
1556 class SinkConversionPattern : public HandshakeConversionPattern<SinkOp> {
1557 public:
1558  using HandshakeConversionPattern<SinkOp>::HandshakeConversionPattern;
1559  void buildModule(SinkOp op, BackedgeBuilder &bb, RTLBuilder &s,
1560  hw::HWModulePortAccessor &ports) const override {
1561  auto unwrappedIO = this->unwrapIO(s, bb, ports);
1562  // A sink is always ready to accept a new value.
1563  unwrappedIO.inputs[0].ready->setValue(s.constant(1, 1));
1564  };
1565 };
1566 
1567 class SourceConversionPattern : public HandshakeConversionPattern<SourceOp> {
1568 public:
1569  using HandshakeConversionPattern<SourceOp>::HandshakeConversionPattern;
1570  void buildModule(SourceOp op, BackedgeBuilder &bb, RTLBuilder &s,
1571  hw::HWModulePortAccessor &ports) const override {
1572  auto unwrappedIO = this->unwrapIO(s, bb, ports);
1573  // A source always provides a new (i0-typed) value.
1574  unwrappedIO.outputs[0].valid->setValue(s.constant(1, 1));
1575  unwrappedIO.outputs[0].data->setValue(s.constant(0, 0));
1576  };
1577 };
1578 
1579 class ConstantConversionPattern
1580  : public HandshakeConversionPattern<handshake::ConstantOp> {
1581 public:
1582  using HandshakeConversionPattern<
1583  handshake::ConstantOp>::HandshakeConversionPattern;
1584  void buildModule(handshake::ConstantOp op, BackedgeBuilder &bb, RTLBuilder &s,
1585  hw::HWModulePortAccessor &ports) const override {
1586  auto unwrappedIO = this->unwrapIO(s, bb, ports);
1587  unwrappedIO.outputs[0].valid->setValue(unwrappedIO.inputs[0].valid);
1588  unwrappedIO.inputs[0].ready->setValue(unwrappedIO.outputs[0].ready);
1589  auto constantValue = op->getAttrOfType<IntegerAttr>("value").getValue();
1590  unwrappedIO.outputs[0].data->setValue(s.constant(constantValue));
1591  };
1592 };
1593 
1594 class BufferConversionPattern : public HandshakeConversionPattern<BufferOp> {
1595 public:
1596  using HandshakeConversionPattern<BufferOp>::HandshakeConversionPattern;
1597  void buildModule(BufferOp op, BackedgeBuilder &bb, RTLBuilder &s,
1598  hw::HWModulePortAccessor &ports) const override {
1599  auto unwrappedIO = this->unwrapIO(s, bb, ports);
1600  auto input = unwrappedIO.inputs[0];
1601  auto output = unwrappedIO.outputs[0];
1602  InputHandshake lastStage;
1603  SmallVector<int64_t> initValues;
1604 
1605  // For now, always build seq buffers.
1606  if (op.getInitValues())
1607  initValues = op.getInitValueArray();
1608 
1609  lastStage =
1610  buildSeqBufferLogic(s, bb, toValidType(op.getDataType()),
1611  op.getNumSlots(), input, output, initValues);
1612 
1613  // Connect the last stage to the output handshake.
1614  output.data->setValue(lastStage.data);
1615  output.valid->setValue(lastStage.valid);
1616  lastStage.ready->setValue(output.ready);
1617  };
1618 
1619  struct SeqBufferStage {
1620  SeqBufferStage(Type dataType, InputHandshake &preStage, BackedgeBuilder &bb,
1621  RTLBuilder &s, size_t index,
1622  std::optional<int64_t> initValue)
1623  : dataType(dataType), preStage(preStage), s(s), bb(bb), index(index) {
1624 
1625  // Todo: Change when i0 support is added.
1626  c0s = createZeroDataConst(s, s.loc, dataType);
1627  currentStage.ready = std::make_shared<Backedge>(bb.get(s.b.getI1Type()));
1628 
1629  auto hasInitValue = s.constant(1, initValue.has_value());
1630  auto validBE = bb.get(s.b.getI1Type());
1631  auto validReg = s.reg(getRegName("valid"), validBE, hasInitValue);
1632  auto readyBE = bb.get(s.b.getI1Type());
1633 
1634  Value initValueCs = c0s;
1635  if (initValue.has_value())
1636  initValueCs = s.constant(dataType.getIntOrFloatBitWidth(), *initValue);
1637 
1638  // This could/should be revised but needs a larger rethinking to avoid
1639  // introducing new bugs.
1640  Value dataReg =
1641  buildDataBufferLogic(validReg, initValueCs, validBE, readyBE);
1642  buildControlBufferLogic(validReg, readyBE, dataReg);
1643  }
1644 
1645  StringAttr getRegName(StringRef name) {
1646  return s.b.getStringAttr(name + std::to_string(index) + "_reg");
1647  }
1648 
1649  void buildControlBufferLogic(Value validReg, Backedge &readyBE,
1650  Value dataReg) {
1651  auto c0I1 = s.constant(1, 0);
1652  auto readyRegWire = bb.get(s.b.getI1Type());
1653  auto readyReg = s.reg(getRegName("ready"), readyRegWire, c0I1);
1654 
1655  // Create the logic to drive the current stage valid and potentially
1656  // data.
1657  currentStage.valid = s.mux(readyReg, {validReg, readyReg},
1658  "controlValid" + std::to_string(index));
1659 
1660  // Create the logic to drive the current stage ready.
1661  auto notReadyReg = s.bNot(readyReg);
1662  readyBE.setValue(notReadyReg);
1663 
1664  auto succNotReady = s.bNot(*currentStage.ready);
1665  auto neitherReady = s.bAnd({succNotReady, notReadyReg});
1666  auto ctrlNotReady = s.mux(neitherReady, {readyReg, validReg});
1667  auto bothReady = s.bAnd({*currentStage.ready, readyReg});
1668 
1669  // Create a mux for emptying the register when both are ready.
1670  auto resetSignal = s.mux(bothReady, {ctrlNotReady, c0I1});
1671  readyRegWire.setValue(resetSignal);
1672 
1673  // Add same logic for the data path if necessary.
1674  auto ctrlDataRegBE = bb.get(dataType);
1675  auto ctrlDataReg = s.reg(getRegName("ctrl_data"), ctrlDataRegBE, c0s);
1676  auto dataResult = s.mux(readyReg, {dataReg, ctrlDataReg});
1677  currentStage.data = dataResult;
1678 
1679  auto dataNotReadyMux = s.mux(neitherReady, {ctrlDataReg, dataReg});
1680  auto dataResetSignal = s.mux(bothReady, {dataNotReadyMux, c0s});
1681  ctrlDataRegBE.setValue(dataResetSignal);
1682  }
1683 
1684  Value buildDataBufferLogic(Value validReg, Value initValue,
1685  Backedge &validBE, Backedge &readyBE) {
1686  // Create a signal for when the valid register is empty or the successor
1687  // is ready to accept new token.
1688  auto notValidReg = s.bNot(validReg);
1689  auto emptyOrReady = s.bOr({notValidReg, readyBE});
1690  preStage.ready->setValue(emptyOrReady);
1691 
1692  // Create a mux that drives the register input. If the emptyOrReady
1693  // signal is asserted, the mux selects the predValid signal. Otherwise,
1694  // it selects the register output, keeping the output registered
1695  // unchanged.
1696  auto validRegMux = s.mux(emptyOrReady, {validReg, preStage.valid});
1697 
1698  // Now we can drive the valid register.
1699  validBE.setValue(validRegMux);
1700 
1701  // Create a mux that drives the date register.
1702  auto dataRegBE = bb.get(dataType);
1703  auto dataReg =
1704  s.reg(getRegName("data"),
1705  s.mux(emptyOrReady, {dataRegBE, preStage.data}), initValue);
1706  dataRegBE.setValue(dataReg);
1707  return dataReg;
1708  }
1709 
1710  InputHandshake getOutput() { return currentStage; }
1711 
1712  Type dataType;
1713  InputHandshake &preStage;
1714  InputHandshake currentStage;
1715  RTLBuilder &s;
1716  BackedgeBuilder &bb;
1717  size_t index;
1718 
1719  // A zero-valued constant of equal type as the data type of this buffer.
1720  Value c0s;
1721  };
1722 
1723  InputHandshake buildSeqBufferLogic(RTLBuilder &s, BackedgeBuilder &bb,
1724  Type dataType, unsigned size,
1725  InputHandshake &input,
1726  OutputHandshake &output,
1727  llvm::ArrayRef<int64_t> initValues) const {
1728  // Prime the buffer building logic with an initial stage, which just
1729  // wraps the input handshake.
1730  InputHandshake currentStage = input;
1731 
1732  for (unsigned i = 0; i < size; ++i) {
1733  bool isInitialized = i < initValues.size();
1734  auto initValue =
1735  isInitialized ? std::optional<int64_t>(initValues[i]) : std::nullopt;
1736  currentStage = SeqBufferStage(dataType, currentStage, bb, s, i, initValue)
1737  .getOutput();
1738  }
1739 
1740  return currentStage;
1741  };
1742 };
1743 
1744 class IndexCastConversionPattern
1745  : public HandshakeConversionPattern<arith::IndexCastOp> {
1746 public:
1747  using HandshakeConversionPattern<
1748  arith::IndexCastOp>::HandshakeConversionPattern;
1749  void buildModule(arith::IndexCastOp op, BackedgeBuilder &bb, RTLBuilder &s,
1750  hw::HWModulePortAccessor &ports) const override {
1751  auto unwrappedIO = this->unwrapIO(s, bb, ports);
1752  unsigned sourceBits =
1753  toValidType(op.getIn().getType()).getIntOrFloatBitWidth();
1754  unsigned targetBits =
1755  toValidType(op.getResult().getType()).getIntOrFloatBitWidth();
1756  if (targetBits < sourceBits)
1757  buildTruncateLogic(s, unwrappedIO, targetBits);
1758  else
1759  buildExtendLogic(s, unwrappedIO, /*signExtend=*/true);
1760  };
1761 };
1762 
1763 template <typename T>
1764 class ExtModuleConversionPattern : public OpConversionPattern<T> {
1765 public:
1766  ExtModuleConversionPattern(ESITypeConverter &typeConverter,
1767  MLIRContext *context, OpBuilder &submoduleBuilder,
1768  HandshakeLoweringState &ls)
1769  : OpConversionPattern<T>::OpConversionPattern(typeConverter, context),
1770  submoduleBuilder(submoduleBuilder), ls(ls) {}
1771  using OpAdaptor = typename T::Adaptor;
1772 
1773  LogicalResult
1774  matchAndRewrite(T op, OpAdaptor adaptor,
1775  ConversionPatternRewriter &rewriter) const override {
1776 
1777  hw::HWModuleLike implModule = checkSubModuleOp(ls.parentModule, op);
1778  if (!implModule) {
1779  auto portInfo = ModulePortInfo(getPortInfoForOp(op));
1780  implModule = submoduleBuilder.create<hw::HWModuleExternOp>(
1781  op.getLoc(), submoduleBuilder.getStringAttr(getSubModuleName(op)),
1782  portInfo);
1783  }
1784 
1785  llvm::SmallVector<Value> operands = adaptor.getOperands();
1786  addSequentialIOOperandsIfNeeded(op, operands);
1787  rewriter.replaceOpWithNewOp<hw::InstanceOp>(
1788  op, implModule, rewriter.getStringAttr(ls.nameUniquer(op)), operands);
1789  return success();
1790  }
1791 
1792 private:
1793  OpBuilder &submoduleBuilder;
1794  HandshakeLoweringState &ls;
1795 };
1796 
1797 class FuncOpConversionPattern : public OpConversionPattern<handshake::FuncOp> {
1798 public:
1799  using OpConversionPattern::OpConversionPattern;
1800 
1801  LogicalResult
1802  matchAndRewrite(handshake::FuncOp op, OpAdaptor operands,
1803  ConversionPatternRewriter &rewriter) const override {
1804  ModulePortInfo ports =
1805  getPortInfoForOpTypes(op, op.getArgumentTypes(), op.getResultTypes());
1806 
1807  HWModuleLike hwModule;
1808  if (op.isExternal()) {
1809  hwModule = rewriter.create<hw::HWModuleExternOp>(
1810  op.getLoc(), rewriter.getStringAttr(op.getName()), ports);
1811  } else {
1812  auto hwModuleOp = rewriter.create<hw::HWModuleOp>(
1813  op.getLoc(), rewriter.getStringAttr(op.getName()), ports);
1814  auto args = hwModuleOp.getBodyBlock()->getArguments().drop_back(2);
1815  rewriter.inlineBlockBefore(&op.getBody().front(),
1816  hwModuleOp.getBodyBlock()->getTerminator(),
1817  args);
1818  hwModule = hwModuleOp;
1819  }
1820 
1821  // Was any predeclaration associated with this func? If so, replace uses
1822  // with the newly created module and erase the predeclaration.
1823  if (auto predecl =
1824  op->getAttrOfType<FlatSymbolRefAttr>(kPredeclarationAttr)) {
1825  auto *parentOp = op->getParentOp();
1826  auto *predeclModule =
1827  SymbolTable::lookupSymbolIn(parentOp, predecl.getValue());
1828  if (predeclModule) {
1829  if (failed(SymbolTable::replaceAllSymbolUses(
1830  predeclModule, hwModule.getModuleNameAttr(), parentOp)))
1831  return failure();
1832  rewriter.eraseOp(predeclModule);
1833  }
1834  }
1835 
1836  rewriter.eraseOp(op);
1837  return success();
1838  }
1839 };
1840 
1841 } // namespace
1842 
1843 //===----------------------------------------------------------------------===//
1844 // HW Top-module Related Functions
1845 //===----------------------------------------------------------------------===//
1846 
1847 static LogicalResult convertFuncOp(ESITypeConverter &typeConverter,
1848  ConversionTarget &target,
1849  handshake::FuncOp op,
1850  OpBuilder &moduleBuilder) {
1851 
1852  std::map<std::string, unsigned> instanceNameCntr;
1853  NameUniquer instanceUniquer = [&](Operation *op) {
1854  std::string instName = getCallName(op);
1855  if (auto idAttr = op->getAttrOfType<IntegerAttr>("handshake_id"); idAttr) {
1856  // We use a special naming convention for operations which have a
1857  // 'handshake_id' attribute.
1858  instName += "_id" + std::to_string(idAttr.getValue().getZExtValue());
1859  } else {
1860  // Fallback to just prefixing with an integer.
1861  instName += std::to_string(instanceNameCntr[instName]++);
1862  }
1863  return instName;
1864  };
1865 
1866  auto ls = HandshakeLoweringState{op->getParentOfType<mlir::ModuleOp>(),
1867  instanceUniquer};
1868  RewritePatternSet patterns(op.getContext());
1869  patterns.insert<FuncOpConversionPattern, ReturnConversionPattern>(
1870  op.getContext());
1871  patterns.insert<JoinConversionPattern, ForkConversionPattern,
1872  SyncConversionPattern>(typeConverter, op.getContext(),
1873  moduleBuilder, ls);
1874 
1875  patterns.insert<
1876  // Comb operations.
1877  UnitRateConversionPattern<arith::AddIOp, comb::AddOp>,
1878  UnitRateConversionPattern<arith::SubIOp, comb::SubOp>,
1879  UnitRateConversionPattern<arith::MulIOp, comb::MulOp>,
1880  UnitRateConversionPattern<arith::DivUIOp, comb::DivSOp>,
1881  UnitRateConversionPattern<arith::DivSIOp, comb::DivUOp>,
1882  UnitRateConversionPattern<arith::RemUIOp, comb::ModUOp>,
1883  UnitRateConversionPattern<arith::RemSIOp, comb::ModSOp>,
1884  UnitRateConversionPattern<arith::AndIOp, comb::AndOp>,
1885  UnitRateConversionPattern<arith::OrIOp, comb::OrOp>,
1886  UnitRateConversionPattern<arith::XOrIOp, comb::XorOp>,
1887  UnitRateConversionPattern<arith::ShLIOp, comb::OrOp>,
1888  UnitRateConversionPattern<arith::ShRUIOp, comb::ShrUOp>,
1889  UnitRateConversionPattern<arith::ShRSIOp, comb::ShrSOp>,
1890  UnitRateConversionPattern<arith::SelectOp, comb::MuxOp>,
1891  // HW operations.
1892  StructCreateConversionPattern,
1893  // Handshake operations.
1894  ConditionalBranchConversionPattern, MuxConversionPattern,
1895  PackConversionPattern, UnpackConversionPattern,
1896  ComparisonConversionPattern, BufferConversionPattern,
1897  SourceConversionPattern, SinkConversionPattern, ConstantConversionPattern,
1898  MergeConversionPattern, ControlMergeConversionPattern,
1899  LoadConversionPattern, StoreConversionPattern, MemoryConversionPattern,
1900  InstanceConversionPattern,
1901  // Arith operations.
1902  ExtendConversionPattern<arith::ExtUIOp, /*signExtend=*/false>,
1903  ExtendConversionPattern<arith::ExtSIOp, /*signExtend=*/true>,
1904  TruncateConversionPattern, IndexCastConversionPattern>(
1905  typeConverter, op.getContext(), moduleBuilder, ls);
1906 
1907  if (failed(applyPartialConversion(op, target, std::move(patterns))))
1908  return op->emitOpError() << "error during conversion";
1909  return success();
1910 }
1911 
1912 namespace {
1913 class HandshakeToHWPass : public HandshakeToHWBase<HandshakeToHWPass> {
1914 public:
1915  void runOnOperation() override {
1916  mlir::ModuleOp mod = getOperation();
1917 
1918  // Lowering to HW requires that every value is used exactly once. Check
1919  // whether this precondition is met, and if not, exit.
1920  for (auto f : mod.getOps<handshake::FuncOp>()) {
1921  if (failed(verifyAllValuesHasOneUse(f))) {
1922  f.emitOpError() << "HandshakeToHW: failed to verify that all values "
1923  "are used exactly once. Remember to run the "
1924  "fork/sink materialization pass before HW lowering.";
1925  signalPassFailure();
1926  return;
1927  }
1928  }
1929 
1930  // Resolve the instance graph to get a top-level module.
1931  std::string topLevel;
1933  SmallVector<std::string> sortedFuncs;
1934  if (resolveInstanceGraph(mod, uses, topLevel, sortedFuncs).failed()) {
1935  signalPassFailure();
1936  return;
1937  }
1938 
1939  ESITypeConverter typeConverter;
1940  ConversionTarget target(getContext());
1941  // All top-level logic of a handshake module will be the interconnectivity
1942  // between instantiated modules.
1943  target.addLegalOp<hw::HWModuleOp, hw::HWModuleExternOp, hw::OutputOp,
1944  hw::InstanceOp>();
1945  target
1946  .addIllegalDialect<handshake::HandshakeDialect, arith::ArithDialect>();
1947 
1948  // Convert the handshake.func operations in post-order wrt. the instance
1949  // graph. This ensures that any referenced submodules (through
1950  // handshake.instance) has already been lowered, and their HW module
1951  // equivalents are available.
1952  OpBuilder submoduleBuilder(mod.getContext());
1953  submoduleBuilder.setInsertionPointToStart(mod.getBody());
1954  for (auto &funcName : llvm::reverse(sortedFuncs)) {
1955  auto funcOp = mod.lookupSymbol<handshake::FuncOp>(funcName);
1956  assert(funcOp && "handshake.func not found in module!");
1957  if (failed(
1958  convertFuncOp(typeConverter, target, funcOp, submoduleBuilder))) {
1959  signalPassFailure();
1960  return;
1961  }
1962  }
1963 
1964  // Second stage: Convert any handshake.extmemory operations and the
1965  // top-level I/O associated with these.
1966  for (auto hwModule : mod.getOps<hw::HWModuleOp>())
1967  if (failed(convertExtMemoryOps(hwModule)))
1968  return signalPassFailure();
1969  }
1970 };
1971 } // end anonymous namespace
1972 
1973 std::unique_ptr<mlir::Pass> circt::createHandshakeToHWPass() {
1974  return std::make_unique<HandshakeToHWPass>();
1975 }
assert(baseType &&"element must be base type")
return wrap(CMemoryType::get(unwrap(ctx), baseType, numElements))
MlirType elementType
Definition: CHIRRTL.cpp:25
static std::string valueName(Operation *scopeOp, Value v)
Convenience function for getting the SSA name of v under the scope of operation scopeOp.
Definition: CalyxOps.cpp:119
static SmallVector< T > concat(const SmallVectorImpl< T > &a, const SmallVectorImpl< T > &b)
Returns a new vector containing the concatenation of vectors a and b.
Definition: CalyxOps.cpp:538
static Type tupleToStruct(TupleType tuple)
Definition: DCToHW.cpp:43
std::function< std::string(Operation *)> NameUniquer
Definition: DCToHW.cpp:40
static void buildModule(OpBuilder &builder, OperationState &result, StringAttr name, ArrayRef< PortInfo > ports, ArrayAttr annotations, bool withAnnotations=true)
Definition: FIRRTLOps.cpp:782
int32_t width
Definition: FIRRTL.cpp:27
@ Input
Definition: HW.h:32
@ Output
Definition: HW.h:32
static std::string getCallName(Operation *op)
static SmallVector< Type > filterNoneTypes(ArrayRef< Type > input)
Filters NoneType's from the input.
static Type getOperandDataType(Value op)
Extracts the type of the data-carrying type of opType.
static DiscriminatingTypes getHandshakeDiscriminatingTypes(Operation *op)
static ModulePortInfo getPortInfoForOp(Operation *op)
Returns a vector of PortInfo's which defines the HW interface of the to-be-converted op.
static std::string getBareSubModuleName(Operation *oldOp)
Returns a submodule name resulting from an operation, without discriminating type information.
static std::string getSubModuleName(Operation *oldOp)
Construct a name for creating HW sub-module.
static HWModuleLike checkSubModuleOp(mlir::ModuleOp parentModule, StringRef modName)
Check whether a submodule with the same name has been created elsewhere in the top level module.
std::pair< SmallVector< Type >, SmallVector< Type > > DiscriminatingTypes
Returns a set of types which may uniquely identify the provided op.
static LogicalResult convertFuncOp(ESITypeConverter &typeConverter, ConversionTarget &target, handshake::FuncOp op, OpBuilder &moduleBuilder)
static llvm::SmallVector< hw::detail::FieldInfo > portToFieldInfo(llvm::ArrayRef< hw::PortInfo > portInfo)
static std::string getTypeName(Location loc, Type type)
Get type name.
static LogicalResult convertExtMemoryOps(HWModuleOp mod)
static EvaluatorValuePtr unwrap(OMEvaluatorValue c)
Definition: OM.cpp:78
llvm::SmallVector< StringAttr > inputs
llvm::SmallVector< StringAttr > outputs
Builder builder
static size_t bits(::capnp::schema::Type::Reader type)
Return the number of bits used by a Capnp type.
Definition: Schema.cpp:121
static int64_t size(hw::ArrayType mType, capnp::schema::Field::Reader cField)
Returns the expected size of an array (capnp list) in 64-bit words.
Definition: Schema.cpp:193
Instantiate one of these and use it to build typed backedges.
Backedge get(mlir::Type resultType, mlir::LocationAttr optionalLoc={})
Create a typed backedge.
Backedge is a wrapper class around a Value.
void setValue(mlir::Value)
Direction get(bool isOutput)
Returns an output direction if isOutput is true, otherwise returns an input direction.
Definition: CalyxOps.cpp:53
Value createOrFoldSExt(Location loc, Value value, Type destTy, OpBuilder &builder)
Create a sign extension operation from a value of integer type to an equal or larger integer type.
Definition: CombOps.cpp:25
mlir::Type innerType(mlir::Type type)
Definition: ESITypes.cpp:177
hw::ModulePortInfo getPortInfoForOpTypes(mlir::Operation *op, TypeRange inputs, TypeRange outputs)
std::map< std::string, std::set< std::string > > InstanceGraph
Iterates over the handshake::FuncOp's in the program to build an instance graph.
LogicalResult resolveInstanceGraph(ModuleOp moduleOp, InstanceGraph &instanceGraph, std::string &topLevel, SmallVectorImpl< std::string > &sortedFuncs)
Iterates over the handshake::FuncOp's in the program to build an instance graph.
Definition: PassHelpers.cpp:38
static constexpr const char * kPredeclarationAttr
Definition: HandshakeToHW.h:37
LogicalResult verifyAllValuesHasOneUse(handshake::FuncOp op)
Type toValidType(Type t)
esi::ChannelType esiWrapper(mlir::Type t)
This file defines an intermediate representation for circuits acting as an abstraction for constraint...
std::unique_ptr< mlir::Pass > createHandshakeToHWPass()
def reg(value, clock, reset=None, reset_value=None, name=None, sym_name=None)
Definition: seq.py:16
This holds a decoded list of input/inout and output ports for a module or instance.