25 #include "mlir/Dialect/Arith/IR/Arith.h"
26 #include "mlir/Dialect/MemRef/IR/MemRef.h"
27 #include "mlir/IR/ImplicitLocOpBuilder.h"
28 #include "mlir/Pass/Pass.h"
29 #include "mlir/Pass/PassManager.h"
30 #include "mlir/Transforms/DialectConversion.h"
31 #include "llvm/ADT/TypeSwitch.h"
32 #include "llvm/Support/MathExtras.h"
36 #define GEN_PASS_DEF_HANDSHAKETOHW
37 #include "circt/Conversion/Passes.h.inc"
41 using namespace circt;
55 struct HandshakeLoweringState {
56 ModuleOp parentModule;
63 class ESITypeConverter :
public TypeConverter {
66 addConversion([](Type type) -> Type {
return esiWrapper(type); });
68 addTargetMaterialization(
69 [&](mlir::OpBuilder &builder, mlir::Type resultType,
70 mlir::ValueRange inputs,
71 mlir::Location loc) -> std::optional<mlir::Value> {
72 if (inputs.size() != 1)
77 addSourceMaterialization(
78 [&](mlir::OpBuilder &builder, mlir::Type resultType,
79 mlir::ValueRange inputs,
80 mlir::Location loc) -> std::optional<mlir::Value> {
81 if (inputs.size() != 1)
96 std::string subModuleName = oldOp->getName().getStringRef().str();
97 std::replace(subModuleName.begin(), subModuleName.end(),
'.',
'_');
102 auto callOp = dyn_cast<handshake::InstanceOp>(op);
110 auto opType = op.getType();
111 if (
auto channelType = dyn_cast<esi::ChannelType>(opType))
112 return channelType.getInner();
118 SmallVector<Type> filterRes;
119 llvm::copy_if(input, std::back_inserter(filterRes),
120 [](Type type) {
return !isa<NoneType>(type); });
128 return TypeSwitch<Operation *, DiscriminatingTypes>(op)
129 .Case<MemoryOp, ExternalMemoryOp>([&](
auto memOp) {
131 {memOp.getMemRefType().getElementType()}};
136 std::vector<Type> inTypes, outTypes;
137 llvm::transform(op->getOperands(), std::back_inserter(inTypes),
139 llvm::transform(op->getResults(), std::back_inserter(outTypes),
151 std::string typeName;
153 if (type.isIntOrIndex()) {
154 if (
auto indexType = dyn_cast<IndexType>(type))
155 typeName +=
"_ui" + std::to_string(indexType.kInternalStorageBitWidth);
156 else if (type.isSignedInteger())
157 typeName +=
"_si" + std::to_string(type.getIntOrFloatBitWidth());
159 typeName +=
"_ui" + std::to_string(type.getIntOrFloatBitWidth());
160 }
else if (
auto tupleType = dyn_cast<TupleType>(type)) {
161 typeName +=
"_tuple";
164 }
else if (
auto structType = dyn_cast<hw::StructType>(type)) {
165 typeName +=
"_struct";
166 for (
auto element : structType.getElements())
167 typeName +=
"_" + element.name.str() +
getTypeName(loc, element.type);
169 emitError(loc) <<
"unsupported data type '" << type <<
"'";
176 if (
auto instanceOp = dyn_cast<handshake::InstanceOp>(oldOp); instanceOp)
177 return instanceOp.getModule().str();
182 if (
auto constOp = dyn_cast<handshake::ConstantOp>(oldOp)) {
183 if (
auto intAttr = dyn_cast<IntegerAttr>(constOp.getValue())) {
184 auto intType = intAttr.getType();
186 if (intType.isSignedInteger())
187 subModuleName +=
"_c" + std::to_string(intAttr.getSInt());
188 else if (intType.isUnsignedInteger())
189 subModuleName +=
"_c" + std::to_string(intAttr.getUInt());
191 subModuleName +=
"_c" + std::to_string((uint64_t)intAttr.getInt());
193 oldOp->emitError(
"unsupported constant type");
198 if (!inTypes.empty())
199 subModuleName +=
"_in";
200 for (
auto inType : inTypes)
201 subModuleName +=
getTypeName(oldOp->getLoc(), inType);
203 if (!outTypes.empty())
204 subModuleName +=
"_out";
205 for (
auto outType : outTypes)
206 subModuleName +=
getTypeName(oldOp->getLoc(), outType);
209 if (
auto memOp = dyn_cast<handshake::MemoryOp>(oldOp))
210 subModuleName +=
"_id" + std::to_string(memOp.getId());
213 if (
auto comOp = dyn_cast<mlir::arith::CmpIOp>(oldOp))
214 subModuleName +=
"_" + stringifyEnum(comOp.getPredicate()).str();
217 if (
auto bufferOp = dyn_cast<handshake::BufferOp>(oldOp)) {
218 subModuleName +=
"_" + std::to_string(bufferOp.getNumSlots()) +
"slots";
219 if (bufferOp.isSequential())
220 subModuleName +=
"_seq";
222 subModuleName +=
"_fifo";
224 if (
auto initValues = bufferOp.getInitValues()) {
225 subModuleName +=
"_init";
226 for (
const Attribute e : *initValues) {
227 assert(isa<IntegerAttr>(e));
229 "_" + std::to_string(dyn_cast<IntegerAttr>(e).
getInt());
235 if (
auto ctrlInterface = dyn_cast<handshake::ControlInterface>(oldOp);
236 ctrlInterface && ctrlInterface.isControl()) {
238 subModuleName +=
"_" + std::to_string(oldOp->getNumOperands()) +
"ins_" +
239 std::to_string(oldOp->getNumResults()) +
"outs";
240 subModuleName +=
"_ctrl";
243 (!inTypes.empty() || !outTypes.empty()) &&
244 "Insufficient discriminating type info generated for the operation!");
247 return subModuleName;
259 if (
auto mod = parentModule.lookupSymbol<HWModuleOp>(modName))
261 if (
auto mod = parentModule.lookupSymbol<HWModuleExternOp>(modName))
268 HWModuleLike targetModule;
269 if (
auto instanceOp = dyn_cast<handshake::InstanceOp>(oldOp))
274 if (isa<handshake::InstanceOp>(oldOp))
276 "handshake.instance target modules should always have been lowered "
277 "before the modules that reference them!");
287 static llvm::SmallVector<hw::detail::FieldInfo>
289 llvm::SmallVector<hw::detail::FieldInfo> fieldInfo;
290 for (
auto port : portInfo)
291 fieldInfo.push_back({port.name, port.type});
299 auto *ctx = mod.getContext();
302 llvm::DenseMap<unsigned, Value> memrefPorts;
303 for (
auto [i, arg] : llvm::enumerate(mod.getBodyBlock()->getArguments())) {
304 auto channel = dyn_cast<esi::ChannelType>(arg.getType());
305 if (channel && isa<MemRefType>(channel.getInner()))
306 memrefPorts[i] = arg;
309 if (memrefPorts.empty())
314 auto getMemoryIOInfo = [&](Location loc, Twine portName,
unsigned argIdx,
315 ArrayRef<hw::PortInfo> info,
319 hw::PortInfo{{b.getStringAttr(portName), type, direction}, argIdx};
323 for (
auto [i, arg] : memrefPorts) {
325 auto memName = mod.getArgName(i);
328 auto extmemInstance = cast<hw::InstanceOp>(*arg.getUsers().begin());
330 cast<hw::HWModuleExternOp>(SymbolTable::lookupNearestSymbolFrom(
331 extmemInstance, extmemInstance.getModuleNameAttr()));
342 SmallVector<PortInfo> outputs(portInfo.
getOutputs());
344 getMemoryIOInfo(arg.getLoc(), memName.strref() +
"_in", i, outputs,
346 mod.insertPorts({{i, inPortInfo}}, {});
347 auto newInPort = mod.getArgumentForInput(i);
349 b.setInsertionPointToStart(mod.getBodyBlock());
350 auto newInPortExploded = b.create<hw::StructExplodeOp>(
351 arg.getLoc(), extmemMod.getOutputTypes(), newInPort);
352 extmemInstance.replaceAllUsesWith(newInPortExploded.getResults());
356 unsigned outArgI = mod.getNumOutputPorts();
357 SmallVector<PortInfo> inputs(portInfo.
getInputs());
359 getMemoryIOInfo(arg.getLoc(), memName.strref() +
"_out", outArgI,
362 auto memOutputArgs = extmemInstance.getOperands().drop_front();
363 b.setInsertionPoint(mod.getBodyBlock()->getTerminator());
365 arg.getLoc(), outPortInfo.type, memOutputArgs);
366 mod.appendOutputs({{outPortInfo.name, memOutputStruct}});
371 extmemInstance.erase();
375 mod.modifyPorts( {}, {},
386 struct InputHandshake {
388 std::shared_ptr<Backedge> ready;
394 struct OutputHandshake {
395 std::shared_ptr<Backedge> valid;
397 std::shared_ptr<Backedge>
data;
402 struct HandshakeWire {
404 MLIRContext *ctx = dataType.getContext();
406 valid = std::make_shared<Backedge>(bb.
get(i1Type));
407 ready = std::make_shared<Backedge>(bb.
get(i1Type));
408 data = std::make_shared<Backedge>(bb.
get(dataType));
413 InputHandshake getAsInput() {
return {*valid, ready, *
data}; }
414 OutputHandshake getAsOutput() {
return {valid, *ready,
data}; }
416 std::shared_ptr<Backedge> valid;
417 std::shared_ptr<Backedge> ready;
418 std::shared_ptr<Backedge>
data;
421 template <
typename T,
typename TInner>
422 llvm::SmallVector<T> extractValues(llvm::SmallVector<TInner> &container,
423 llvm::function_ref<T(TInner &)> extractor) {
424 llvm::SmallVector<T> result;
425 llvm::transform(container, std::back_inserter(result), extractor);
429 llvm::SmallVector<InputHandshake> inputs;
430 llvm::SmallVector<OutputHandshake> outputs;
432 llvm::SmallVector<Value> getInputValids() {
433 return extractValues<Value, InputHandshake>(
434 inputs, [](
auto &hs) {
return hs.valid; });
436 llvm::SmallVector<std::shared_ptr<Backedge>> getInputReadys() {
437 return extractValues<std::shared_ptr<Backedge>, InputHandshake>(
438 inputs, [](
auto &hs) {
return hs.ready; });
440 llvm::SmallVector<Value> getInputDatas() {
441 return extractValues<Value, InputHandshake>(
442 inputs, [](
auto &hs) {
return hs.data; });
444 llvm::SmallVector<std::shared_ptr<Backedge>> getOutputValids() {
445 return extractValues<std::shared_ptr<Backedge>, OutputHandshake>(
446 outputs, [](
auto &hs) {
return hs.valid; });
448 llvm::SmallVector<Value> getOutputReadys() {
449 return extractValues<Value, OutputHandshake>(
450 outputs, [](
auto &hs) {
return hs.ready; });
452 llvm::SmallVector<std::shared_ptr<Backedge>> getOutputDatas() {
453 return extractValues<std::shared_ptr<Backedge>, OutputHandshake>(
454 outputs, [](
auto &hs) {
return hs.data; });
462 RTLBuilder(hw::ModulePortInfo info, OpBuilder &builder, Location loc,
463 Value clk = Value(), Value rst = Value())
464 : info(std::move(info)), b(builder), loc(loc),
clk(
clk), rst(rst) {}
466 Value constant(
const APInt &apv, std::optional<StringRef> name = {}) {
469 bool isZeroWidth = apv.getBitWidth() == 0;
471 auto it = constants.find(apv);
472 if (it != constants.end())
478 constants[apv] = cval;
482 Value constant(
unsigned width, int64_t value,
483 std::optional<StringRef> name = {}) {
484 return constant(APInt(
width, value));
486 std::pair<Value, Value>
wrap(Value data, Value valid,
487 std::optional<StringRef> name = {}) {
488 auto wrapOp = b.create<esi::WrapValidReadyOp>(loc,
data, valid);
489 return {wrapOp.getResult(0), wrapOp.getResult(1)};
491 std::pair<Value, Value>
unwrap(Value channel, Value ready,
492 std::optional<StringRef> name = {}) {
493 auto unwrapOp = b.create<esi::UnwrapValidReadyOp>(loc, channel, ready);
494 return {unwrapOp.getResult(0), unwrapOp.getResult(1)};
498 Value
reg(StringRef name, Value in, Value rstValue, Value clk = Value(),
499 Value rst = Value()) {
500 Value resolvedClk =
clk ?
clk : this->
clk;
501 Value resolvedRst = rst ? rst : this->rst;
503 "No global clock provided to this RTLBuilder - a clock "
504 "signal must be provided to the reg(...) function.");
506 "No global reset provided to this RTLBuilder - a reset "
507 "signal must be provided to the reg(...) function.");
509 return b.create<
seq::CompRegOp>(loc, in, resolvedClk, resolvedRst, rstValue,
513 Value cmp(Value lhs, Value rhs, comb::ICmpPredicate predicate,
514 std::optional<StringRef> name = {}) {
515 return b.
create<comb::ICmpOp>(loc, predicate, lhs, rhs);
518 Value buildNamedOp(llvm::function_ref<Value()> f,
519 std::optional<StringRef> name) {
522 Operation *op = v.getDefiningOp();
523 if (name.has_value()) {
524 op->setAttr(
"sv.namehint", b.getStringAttr(*name));
525 nameAttr = b.getStringAttr(*name);
531 Value bAnd(ValueRange values, std::optional<StringRef> name = {}) {
533 [&]() {
return b.create<
comb::AndOp>(loc, values,
false); }, name);
536 Value bOr(ValueRange values, std::optional<StringRef> name = {}) {
538 [&]() {
return b.create<
comb::OrOp>(loc, values,
false); }, name);
542 Value bNot(Value value, std::optional<StringRef> name = {}) {
543 auto allOnes = constant(value.getType().getIntOrFloatBitWidth(), -1);
544 std::string inferedName;
548 value.getDefiningOp()->getAttrOfType<StringAttr>(
"sv.namehint")) {
549 inferedName = (
"not_" +
valueName.getValue()).str();
555 [&]() {
return b.create<
comb::XorOp>(loc, value, allOnes); }, name);
557 return b.createOrFold<
comb::XorOp>(loc, value, allOnes,
false);
560 Value shl(Value value, Value shift, std::optional<StringRef> name = {}) {
562 [&]() {
return b.create<
comb::ShlOp>(loc, value, shift); }, name);
565 Value
concat(ValueRange values, std::optional<StringRef> name = {}) {
566 return buildNamedOp([&]() {
return b.create<
comb::ConcatOp>(loc, values); },
571 Value pack(ValueRange values, Type structType = Type(),
572 std::optional<StringRef> name = {}) {
581 ValueRange unpack(Value value) {
582 auto structType = cast<hw::StructType>(value.getType());
583 llvm::SmallVector<Type> innerTypes;
584 structType.getInnerTypes(innerTypes);
585 return b.create<hw::StructExplodeOp>(loc, innerTypes, value).getResults();
588 llvm::SmallVector<Value> toBits(Value v, std::optional<StringRef> name = {}) {
589 llvm::SmallVector<Value> bits;
590 for (
unsigned i = 0, e = v.getType().getIntOrFloatBitWidth(); i != e; ++i)
596 Value rOr(Value v, std::optional<StringRef> name = {}) {
597 return buildNamedOp([&]() {
return bOr(toBits(v)); }, name);
601 Value extract(Value v,
unsigned lo,
unsigned hi,
602 std::optional<StringRef> name = {}) {
603 unsigned width = hi - lo + 1;
609 Value truncate(Value value,
unsigned width,
610 std::optional<StringRef> name = {}) {
611 return extract(value, 0,
width - 1, name);
614 Value zext(Value value,
unsigned outWidth,
615 std::optional<StringRef> name = {}) {
616 unsigned inWidth = value.getType().getIntOrFloatBitWidth();
617 assert(inWidth <= outWidth &&
"zext: input width must be <- output width.");
618 if (inWidth == outWidth)
620 auto c0 = constant(outWidth - inWidth, 0);
621 return concat({c0, value}, name);
624 Value sext(Value value,
unsigned outWidth,
625 std::optional<StringRef> name = {}) {
630 Value bit(Value v,
unsigned index, std::optional<StringRef> name = {}) {
631 return extract(v, index, index, name);
635 Value arrayCreate(ValueRange values, std::optional<StringRef> name = {}) {
641 Value arrayGet(Value array, Value index, std::optional<StringRef> name = {}) {
643 [&]() {
return b.create<
hw::ArrayGetOp>(loc, array, index); }, name);
649 Value mux(Value index, ValueRange values,
650 std::optional<StringRef> name = {}) {
651 if (values.size() == 2)
652 return b.create<
comb::MuxOp>(loc, index, values[1], values[0]);
654 return arrayGet(arrayCreate(values), index, name);
659 Value ohMux(Value index, ValueRange inputs) {
661 unsigned numInputs = inputs.size();
662 assert(numInputs == index.getType().getIntOrFloatBitWidth() &&
663 "one-hot select can't mux inputs");
667 auto dataType = inputs[0].getType();
669 isa<NoneType>(dataType) ? 0 : dataType.getIntOrFloatBitWidth();
670 Value muxValue = constant(
width, 0);
673 for (
size_t i = numInputs - 1; i != 0; --i) {
674 Value input = inputs[i];
675 Value selectBit = bit(index, i);
676 muxValue = mux(selectBit, {muxValue, input});
682 hw::ModulePortInfo info;
686 DenseMap<APInt, Value> constants;
691 static Value createZeroDataConst(RTLBuilder &s, Location loc, Type type) {
692 return TypeSwitch<Type, Value>(type)
693 .Case<NoneType>([&](NoneType) {
return s.constant(0, 0); })
694 .Case<IntType, IntegerType>([&](
auto type) {
695 return s.constant(type.getIntOrFloatBitWidth(), 0);
697 .Case<hw::StructType>([&](
auto structType) {
698 SmallVector<Value> zeroValues;
699 for (
auto field : structType.getElements())
700 zeroValues.push_back(createZeroDataConst(s, loc, field.type));
703 .Default([&](Type) -> Value {
704 emitError(loc) <<
"unsupported type for zero value: " << type;
711 addSequentialIOOperandsIfNeeded(Operation *op,
712 llvm::SmallVectorImpl<Value> &operands) {
716 auto parent = cast<hw::HWModuleOp>(op->getParentOp());
718 parent.getArgumentForInput(parent.getNumInputPorts() - 2));
720 parent.getArgumentForInput(parent.getNumInputPorts() - 1));
724 template <
typename T>
727 HandshakeConversionPattern(ESITypeConverter &typeConverter,
728 MLIRContext *context, OpBuilder &submoduleBuilder,
729 HandshakeLoweringState &ls)
731 submoduleBuilder(submoduleBuilder), ls(ls) {}
733 using OpAdaptor =
typename T::Adaptor;
736 matchAndRewrite(T op, OpAdaptor adaptor,
737 ConversionPatternRewriter &rewriter)
const override {
746 submoduleBuilder.setInsertionPoint(op->getParentOp());
749 portInfo, [&](OpBuilder &b, hw::HWModulePortAccessor &ports) {
753 if (op->template hasTrait<mlir::OpTrait::HasClock>()) {
754 clk = ports.getInput(
"clock");
755 rst = ports.getInput(
"reset");
759 RTLBuilder s(ports.getPortList(), b, op.getLoc(), clk, rst);
765 llvm::SmallVector<Value> operands = adaptor.getOperands();
766 addSequentialIOOperandsIfNeeded(op, operands);
767 rewriter.replaceOpWithNewOp<hw::InstanceOp>(
768 op, implModule, rewriter.getStringAttr(ls.nameUniquer(op)), operands);
773 hw::HWModulePortAccessor &ports)
const = 0;
780 hw::HWModulePortAccessor &ports)
const {
781 UnwrappedIO unwrapped;
782 for (
auto port : ports.getInputs()) {
783 if (!isa<esi::ChannelType>(port.getType()))
786 auto ready = std::make_shared<Backedge>(bb.
get(s.b.getI1Type()));
787 auto [
data, valid] = s.unwrap(port, *ready);
791 unwrapped.inputs.push_back(hs);
793 for (
auto &outputInfo : ports.getPortList().getOutputs()) {
795 dyn_cast<esi::ChannelType>(outputInfo.type);
801 auto valid = std::make_shared<Backedge>(bb.
get(s.b.getI1Type()));
802 auto [dataCh, ready] = s.wrap(*data, *valid);
806 ports.setOutput(outputInfo.name, dataCh);
807 unwrapped.outputs.push_back(hs);
812 void setAllReadyWithCond(RTLBuilder &s, ArrayRef<InputHandshake> inputs,
813 OutputHandshake &output, Value cond)
const {
814 auto validAndReady = s.bAnd({output.ready, cond});
815 for (
auto &input : inputs)
816 input.ready->setValue(validAndReady);
819 void buildJoinLogic(RTLBuilder &s, ArrayRef<InputHandshake> inputs,
820 OutputHandshake &output)
const {
821 llvm::SmallVector<Value> valids;
822 for (
auto &input : inputs)
823 valids.push_back(input.valid);
824 Value allValid = s.bAnd(valids);
825 output.valid->setValue(allValid);
826 setAllReadyWithCond(s, inputs, output, allValid);
832 void buildMuxLogic(RTLBuilder &s, UnwrappedIO &unwrapped,
833 InputHandshake &select)
const {
835 size_t numInputs = unwrapped.inputs.size();
836 size_t selectWidth = llvm::Log2_64_Ceil(numInputs);
837 Value truncatedSelect =
838 select.data.getType().getIntOrFloatBitWidth() > selectWidth
839 ? s.truncate(select.data, selectWidth)
843 auto selectZext = s.zext(truncatedSelect, numInputs);
844 auto select1h = s.shl(s.constant(numInputs, 1), selectZext);
845 auto &res = unwrapped.outputs[0];
848 auto selectedInputValid =
849 s.mux(truncatedSelect, unwrapped.getInputValids());
851 auto selAndInputValid = s.bAnd({selectedInputValid, select.valid});
852 res.valid->setValue(selAndInputValid);
853 auto resValidAndReady = s.bAnd({selAndInputValid, res.ready});
856 select.ready->setValue(resValidAndReady);
859 for (
auto [inIdx, in] : llvm::enumerate(unwrapped.inputs)) {
861 auto isSelected = s.bit(select1h, inIdx);
865 auto activeAndResultValidAndReady =
866 s.bAnd({isSelected, resValidAndReady});
867 in.ready->setValue(activeAndResultValidAndReady);
871 res.data->setValue(s.mux(truncatedSelect, unwrapped.getInputDatas()));
876 void buildForkLogic(RTLBuilder &s,
BackedgeBuilder &bb, InputHandshake &input,
877 ArrayRef<OutputHandshake> outputs)
const {
878 auto c0I1 = s.constant(1, 0);
879 llvm::SmallVector<Value> doneWires;
880 for (
auto [i, output] : llvm::enumerate(outputs)) {
881 auto doneBE = bb.
get(s.b.getI1Type());
882 auto emitted = s.bAnd({doneBE, s.bNot(*input.ready)});
883 auto emittedReg = s.reg(
"emitted_" + std::to_string(i), emitted, c0I1);
884 auto outValid = s.bAnd({s.bNot(emittedReg), input.valid});
886 auto validReady = s.bAnd({output.ready, outValid});
887 auto done = s.bOr({validReady, emittedReg},
"done" + std::to_string(i));
888 doneBE.setValue(done);
889 doneWires.push_back(done);
891 input.ready->setValue(s.bAnd(doneWires,
"allDone"));
897 void buildUnitRateJoinLogic(
898 RTLBuilder &s, UnwrappedIO &unwrappedIO,
899 llvm::function_ref<Value(ValueRange)> unitBuilder)
const {
900 assert(unwrappedIO.outputs.size() == 1 &&
901 "Expected exactly one output for unit-rate join actor");
903 this->buildJoinLogic(s, unwrappedIO.inputs, unwrappedIO.outputs[0]);
906 auto unitRes = unitBuilder(unwrappedIO.getInputDatas());
907 unwrappedIO.outputs[0].data->setValue(unitRes);
910 void buildUnitRateForkLogic(
912 llvm::function_ref<llvm::SmallVector<Value>(Value)> unitBuilder)
const {
913 assert(unwrappedIO.inputs.size() == 1 &&
914 "Expected exactly one input for unit-rate fork actor");
916 this->buildForkLogic(s, bb, unwrappedIO.inputs[0], unwrappedIO.outputs);
919 auto unitResults = unitBuilder(unwrappedIO.inputs[0].data);
920 assert(unitResults.size() == unwrappedIO.outputs.size() &&
921 "Expected unit builder to return one result per output");
922 for (
auto [res, outport] : llvm::zip(unitResults, unwrappedIO.outputs))
923 outport.data->setValue(res);
926 void buildExtendLogic(RTLBuilder &s, UnwrappedIO &unwrappedIO,
927 bool signExtend)
const {
929 toValidType(
static_cast<Value
>(*unwrappedIO.outputs[0].data).getType())
930 .getIntOrFloatBitWidth();
931 buildUnitRateJoinLogic(s, unwrappedIO, [&](ValueRange inputs) {
933 return s.sext(inputs[0], outWidth);
934 return s.zext(inputs[0], outWidth);
938 void buildTruncateLogic(RTLBuilder &s, UnwrappedIO &unwrappedIO,
939 unsigned targetWidth)
const {
941 toValidType(
static_cast<Value
>(*unwrappedIO.outputs[0].data).getType())
942 .getIntOrFloatBitWidth();
943 buildUnitRateJoinLogic(s, unwrappedIO, [&](ValueRange inputs) {
944 return s.truncate(inputs[0], outWidth);
949 static size_t getNumIndexBits(uint64_t numValues) {
950 return numValues > 1 ? llvm::Log2_64_Ceil(numValues) : 1;
953 Value buildPriorityArbiter(RTLBuilder &s, ArrayRef<Value> inputs,
955 DenseMap<size_t, Value> &indexMapping)
const {
956 auto numInputs = inputs.size();
957 auto priorityArb = defaultValue;
959 for (
size_t i = numInputs; i > 0; --i) {
960 size_t inputIndex = i - 1;
961 size_t oneHotIndex =
size_t{1} << inputIndex;
962 auto constIndex = s.constant(numInputs, oneHotIndex);
963 indexMapping[inputIndex] = constIndex;
964 priorityArb = s.mux(inputs[inputIndex], {priorityArb, constIndex});
970 OpBuilder &submoduleBuilder;
971 HandshakeLoweringState &ls;
974 class ForkConversionPattern :
public HandshakeConversionPattern<ForkOp> {
976 using HandshakeConversionPattern<ForkOp>::HandshakeConversionPattern;
978 hw::HWModulePortAccessor &ports)
const override {
979 auto unwrapped = unwrapIO(s, bb, ports);
980 buildUnitRateForkLogic(s, bb, unwrapped, [&](Value input) {
981 return llvm::SmallVector<Value>(unwrapped.outputs.size(), input);
986 class JoinConversionPattern :
public HandshakeConversionPattern<JoinOp> {
988 using HandshakeConversionPattern<JoinOp>::HandshakeConversionPattern;
990 hw::HWModulePortAccessor &ports)
const override {
991 auto unwrappedIO = unwrapIO(s, bb, ports);
992 buildJoinLogic(s, unwrappedIO.inputs, unwrappedIO.outputs[0]);
993 unwrappedIO.outputs[0].data->setValue(s.constant(0, 0));
997 class SyncConversionPattern :
public HandshakeConversionPattern<SyncOp> {
999 using HandshakeConversionPattern<SyncOp>::HandshakeConversionPattern;
1001 hw::HWModulePortAccessor &ports)
const override {
1002 auto unwrappedIO = unwrapIO(s, bb, ports);
1005 HandshakeWire wire(bb, s.b.getNoneType());
1007 OutputHandshake output = wire.getAsOutput();
1008 buildJoinLogic(s, unwrappedIO.inputs, output);
1010 InputHandshake input = wire.getAsInput();
1018 buildForkLogic(s, bb, input, unwrappedIO.outputs);
1022 for (
auto &&[in, out] : llvm::zip(unwrappedIO.inputs, unwrappedIO.outputs))
1023 out.data->setValue(in.data);
1027 class MuxConversionPattern :
public HandshakeConversionPattern<MuxOp> {
1029 using HandshakeConversionPattern<MuxOp>::HandshakeConversionPattern;
1031 hw::HWModulePortAccessor &ports)
const override {
1032 auto unwrappedIO = unwrapIO(s, bb, ports);
1035 auto select = unwrappedIO.inputs[0];
1036 unwrappedIO.inputs.erase(unwrappedIO.inputs.begin());
1037 buildMuxLogic(s, unwrappedIO, select);
1041 class InstanceConversionPattern
1042 :
public HandshakeConversionPattern<handshake::InstanceOp> {
1044 using HandshakeConversionPattern<
1045 handshake::InstanceOp>::HandshakeConversionPattern;
1047 hw::HWModulePortAccessor &ports)
const override {
1049 "If we indeed perform conversion in post-order, this "
1050 "should never be called. The base HandshakeConversionPattern logic "
1051 "will instantiate the external module.");
1055 class ReturnConversionPattern
1058 using OpConversionPattern::OpConversionPattern;
1060 matchAndRewrite(ReturnOp op, OpAdaptor adaptor,
1061 ConversionPatternRewriter &rewriter)
const override {
1064 auto parent = cast<hw::HWModuleOp>(op->getParentOp());
1065 auto outputOp = *parent.getBodyBlock()->getOps<hw::OutputOp>().begin();
1066 outputOp->setOperands(adaptor.getOperands());
1067 outputOp->moveAfter(&parent.getBodyBlock()->back());
1068 rewriter.eraseOp(op);
1075 template <
typename TIn,
typename TOut = TIn>
1076 class UnitRateConversionPattern :
public HandshakeConversionPattern<TIn> {
1078 using HandshakeConversionPattern<TIn>::HandshakeConversionPattern;
1080 hw::HWModulePortAccessor &ports)
const override {
1081 auto unwrappedIO = this->unwrapIO(s, bb, ports);
1082 this->buildUnitRateJoinLogic(s, unwrappedIO, [&](ValueRange inputs) {
1087 return s.b.create<TOut>(op.getLoc(), inputs,
1088 ArrayRef<NamedAttribute>{});
1093 class PackConversionPattern :
public HandshakeConversionPattern<PackOp> {
1095 using HandshakeConversionPattern<PackOp>::HandshakeConversionPattern;
1097 hw::HWModulePortAccessor &ports)
const override {
1098 auto unwrappedIO = unwrapIO(s, bb, ports);
1099 buildUnitRateJoinLogic(s, unwrappedIO,
1100 [&](ValueRange inputs) {
return s.pack(inputs); });
1104 class StructCreateConversionPattern
1105 :
public HandshakeConversionPattern<hw::StructCreateOp> {
1107 using HandshakeConversionPattern<
1110 hw::HWModulePortAccessor &ports)
const override {
1111 auto unwrappedIO = unwrapIO(s, bb, ports);
1112 auto structType = op.getResult().getType();
1113 buildUnitRateJoinLogic(s, unwrappedIO, [&](ValueRange inputs) {
1114 return s.pack(inputs, structType);
1119 class UnpackConversionPattern :
public HandshakeConversionPattern<UnpackOp> {
1121 using HandshakeConversionPattern<UnpackOp>::HandshakeConversionPattern;
1123 hw::HWModulePortAccessor &ports)
const override {
1124 auto unwrappedIO = unwrapIO(s, bb, ports);
1125 buildUnitRateForkLogic(s, bb, unwrappedIO,
1126 [&](Value input) {
return s.unpack(input); });
1130 class ConditionalBranchConversionPattern
1131 :
public HandshakeConversionPattern<ConditionalBranchOp> {
1133 using HandshakeConversionPattern<
1134 ConditionalBranchOp>::HandshakeConversionPattern;
1136 hw::HWModulePortAccessor &ports)
const override {
1137 auto unwrappedIO = unwrapIO(s, bb, ports);
1138 auto cond = unwrappedIO.inputs[0];
1139 auto arg = unwrappedIO.inputs[1];
1140 auto trueRes = unwrappedIO.outputs[0];
1141 auto falseRes = unwrappedIO.outputs[1];
1143 auto condArgValid = s.bAnd({cond.valid, arg.valid});
1146 trueRes.valid->setValue(s.bAnd({cond.data, condArgValid}));
1147 falseRes.valid->setValue(s.bAnd({s.bNot(cond.data), condArgValid}));
1150 trueRes.data->setValue(arg.data);
1151 falseRes.data->setValue(arg.data);
1154 auto selectedResultReady =
1155 s.mux(cond.data, {falseRes.ready, trueRes.ready});
1156 auto condArgReady = s.bAnd({selectedResultReady, condArgValid});
1157 arg.ready->setValue(condArgReady);
1158 cond.ready->setValue(condArgReady);
1162 template <
typename TIn,
bool signExtend>
1163 class ExtendConversionPattern :
public HandshakeConversionPattern<TIn> {
1165 using HandshakeConversionPattern<TIn>::HandshakeConversionPattern;
1167 hw::HWModulePortAccessor &ports)
const override {
1168 auto unwrappedIO = this->unwrapIO(s, bb, ports);
1169 this->buildExtendLogic(s, unwrappedIO, signExtend);
1173 class ComparisonConversionPattern
1174 :
public HandshakeConversionPattern<arith::CmpIOp> {
1176 using HandshakeConversionPattern<arith::CmpIOp>::HandshakeConversionPattern;
1178 hw::HWModulePortAccessor &ports)
const override {
1179 auto unwrappedIO = this->unwrapIO(s, bb, ports);
1180 auto buildCompareLogic = [&](comb::ICmpPredicate predicate) {
1181 return buildUnitRateJoinLogic(s, unwrappedIO, [&](ValueRange inputs) {
1182 return s.b.create<comb::ICmpOp>(op.getLoc(), predicate, inputs[0],
1187 switch (op.getPredicate()) {
1188 case arith::CmpIPredicate::eq:
1189 return buildCompareLogic(comb::ICmpPredicate::eq);
1190 case arith::CmpIPredicate::ne:
1191 return buildCompareLogic(comb::ICmpPredicate::ne);
1192 case arith::CmpIPredicate::slt:
1193 return buildCompareLogic(comb::ICmpPredicate::slt);
1194 case arith::CmpIPredicate::ult:
1195 return buildCompareLogic(comb::ICmpPredicate::ult);
1196 case arith::CmpIPredicate::sle:
1197 return buildCompareLogic(comb::ICmpPredicate::sle);
1198 case arith::CmpIPredicate::ule:
1199 return buildCompareLogic(comb::ICmpPredicate::ule);
1200 case arith::CmpIPredicate::sgt:
1201 return buildCompareLogic(comb::ICmpPredicate::sgt);
1202 case arith::CmpIPredicate::ugt:
1203 return buildCompareLogic(comb::ICmpPredicate::ugt);
1204 case arith::CmpIPredicate::sge:
1205 return buildCompareLogic(comb::ICmpPredicate::sge);
1206 case arith::CmpIPredicate::uge:
1207 return buildCompareLogic(comb::ICmpPredicate::uge);
1209 assert(
false &&
"invalid CmpIOp");
1213 class TruncateConversionPattern
1214 :
public HandshakeConversionPattern<arith::TruncIOp> {
1216 using HandshakeConversionPattern<arith::TruncIOp>::HandshakeConversionPattern;
1218 hw::HWModulePortAccessor &ports)
const override {
1219 auto unwrappedIO = this->unwrapIO(s, bb, ports);
1220 unsigned targetBits =
1221 toValidType(op.getResult().getType()).getIntOrFloatBitWidth();
1222 buildTruncateLogic(s, unwrappedIO, targetBits);
1226 class ControlMergeConversionPattern
1227 :
public HandshakeConversionPattern<ControlMergeOp> {
1229 using HandshakeConversionPattern<ControlMergeOp>::HandshakeConversionPattern;
1231 hw::HWModulePortAccessor &ports)
const override {
1232 auto unwrappedIO = this->unwrapIO(s, bb, ports);
1233 auto resData = unwrappedIO.outputs[0];
1234 auto resIndex = unwrappedIO.outputs[1];
1237 unsigned numInputs = unwrappedIO.inputs.size();
1238 auto indexType = s.b.getIntegerType(numInputs);
1239 Value noWinner = s.constant(numInputs, 0);
1240 Value c0I1 = s.constant(1, 0);
1243 auto won = bb.
get(indexType);
1244 Value wonReg = s.reg(
"won_reg", won, noWinner);
1247 auto win = bb.
get(indexType);
1251 auto fired = bb.
get(s.b.getI1Type());
1254 auto resultEmitted = bb.
get(s.b.getI1Type());
1255 Value resultEmittedReg = s.reg(
"result_emitted_reg", resultEmitted, c0I1);
1256 auto indexEmitted = bb.
get(s.b.getI1Type());
1257 Value indexEmittedReg = s.reg(
"index_emitted_reg", indexEmitted, c0I1);
1260 auto resultDone = bb.
get(s.b.getI1Type());
1261 auto indexDone = bb.
get(s.b.getI1Type());
1265 auto hasWinnerCondition = s.rOr({win});
1266 auto hadWinnerCondition = s.rOr({wonReg});
1274 DenseMap<size_t, Value> argIndexValues;
1275 Value priorityArb = buildPriorityArbiter(s, unwrappedIO.getInputValids(),
1276 noWinner, argIndexValues);
1277 priorityArb = s.mux(hadWinnerCondition, {priorityArb, wonReg});
1278 win.setValue(priorityArb);
1288 auto resultNotEmitted = s.bNot(resultEmittedReg);
1289 auto resultValid = s.bAnd({hasWinnerCondition, resultNotEmitted});
1290 resData.valid->setValue(resultValid);
1291 resData.data->setValue(s.ohMux(win, unwrappedIO.getInputDatas()));
1293 auto indexNotEmitted = s.bNot(indexEmittedReg);
1294 auto indexValid = s.bAnd({hasWinnerCondition, indexNotEmitted});
1295 resIndex.valid->setValue(indexValid);
1299 SmallVector<Value, 8> indexOutputs;
1300 for (
size_t i = 0; i < numInputs; ++i)
1301 indexOutputs.push_back(s.constant(64, i));
1303 auto indexOutput = s.ohMux(win, indexOutputs);
1304 resIndex.data->setValue(indexOutput);
1310 won.setValue(s.mux(fired, {win, noWinner}));
1315 auto resultValidAndReady = s.bAnd({resultValid, resData.ready});
1316 resultDone.setValue(s.bOr({resultValidAndReady, resultEmittedReg}));
1318 auto indexValidAndReady = s.bAnd({indexValid, resIndex.ready});
1319 indexDone.setValue(s.bOr({indexValidAndReady, indexEmittedReg}));
1323 fired.setValue(s.bAnd({resultDone, indexDone}));
1329 resultEmitted.setValue(s.mux(fired, {resultDone, c0I1}));
1330 indexEmitted.setValue(s.mux(fired, {indexDone, c0I1}));
1335 auto winnerOrDefault = s.mux(fired, {noWinner, win});
1336 for (
auto [i, ir] : llvm::enumerate(unwrappedIO.getInputReadys())) {
1337 auto &indexValue = argIndexValues[i];
1338 ir->setValue(s.cmp(winnerOrDefault, indexValue, comb::ICmpPredicate::eq));
1343 class MergeConversionPattern :
public HandshakeConversionPattern<MergeOp> {
1345 using HandshakeConversionPattern<MergeOp>::HandshakeConversionPattern;
1347 hw::HWModulePortAccessor &ports)
const override {
1348 auto unwrappedIO = this->unwrapIO(s, bb, ports);
1349 auto resData = unwrappedIO.outputs[0];
1352 unsigned numInputs = unwrappedIO.inputs.size();
1353 auto indexType = s.b.getIntegerType(numInputs);
1354 Value noWinner = s.constant(numInputs, 0);
1357 auto win = bb.
get(indexType);
1360 auto hasWinnerCondition = s.rOr(win);
1367 DenseMap<size_t, Value> argIndexValues;
1368 Value priorityArb = buildPriorityArbiter(s, unwrappedIO.getInputValids(),
1369 noWinner, argIndexValues);
1370 win.setValue(priorityArb);
1377 resData.valid->setValue(hasWinnerCondition);
1378 resData.data->setValue(s.ohMux(win, unwrappedIO.getInputDatas()));
1383 auto resultValidAndReady = s.bAnd({hasWinnerCondition, resData.ready});
1388 auto winnerOrDefault = s.mux(resultValidAndReady, {noWinner, win});
1389 for (
auto [i, ir] : llvm::enumerate(unwrappedIO.getInputReadys())) {
1390 auto &indexValue = argIndexValues[i];
1391 ir->setValue(s.cmp(winnerOrDefault, indexValue, comb::ICmpPredicate::eq));
1396 class LoadConversionPattern
1397 :
public HandshakeConversionPattern<handshake::LoadOp> {
1399 using HandshakeConversionPattern<
1400 handshake::LoadOp>::HandshakeConversionPattern;
1402 hw::HWModulePortAccessor &ports)
const override {
1403 auto unwrappedIO = this->unwrapIO(s, bb, ports);
1404 auto addrFromUser = unwrappedIO.inputs[0];
1405 auto dataFromMem = unwrappedIO.inputs[1];
1406 auto controlIn = unwrappedIO.inputs[2];
1407 auto dataToUser = unwrappedIO.outputs[0];
1408 auto addrToMem = unwrappedIO.outputs[1];
1410 addrToMem.data->setValue(addrFromUser.data);
1411 dataToUser.data->setValue(dataFromMem.data);
1415 buildJoinLogic(s, {addrFromUser, controlIn}, addrToMem);
1419 dataToUser.valid->setValue(dataFromMem.valid);
1420 dataFromMem.ready->setValue(dataToUser.ready);
1424 class StoreConversionPattern
1425 :
public HandshakeConversionPattern<handshake::StoreOp> {
1427 using HandshakeConversionPattern<
1428 handshake::StoreOp>::HandshakeConversionPattern;
1430 hw::HWModulePortAccessor &ports)
const override {
1431 auto unwrappedIO = this->unwrapIO(s, bb, ports);
1432 auto addrFromUser = unwrappedIO.inputs[0];
1433 auto dataFromUser = unwrappedIO.inputs[1];
1434 auto controlIn = unwrappedIO.inputs[2];
1435 auto dataToMem = unwrappedIO.outputs[0];
1436 auto addrToMem = unwrappedIO.outputs[1];
1439 auto outputsReady = s.bAnd({dataToMem.ready, addrToMem.ready});
1443 HandshakeWire joinWire(bb, s.b.getNoneType());
1444 joinWire.ready->setValue(outputsReady);
1445 OutputHandshake joinOutput = joinWire.getAsOutput();
1446 buildJoinLogic(s, {dataFromUser, addrFromUser, controlIn}, joinOutput);
1449 addrToMem.data->setValue(addrFromUser.data);
1450 dataToMem.data->setValue(dataFromUser.data);
1453 addrToMem.valid->setValue(*joinWire.valid);
1454 dataToMem.valid->setValue(*joinWire.valid);
1458 class MemoryConversionPattern
1459 :
public HandshakeConversionPattern<handshake::MemoryOp> {
1461 using HandshakeConversionPattern<
1462 handshake::MemoryOp>::HandshakeConversionPattern;
1464 hw::HWModulePortAccessor &ports)
const override {
1465 auto loc = op.getLoc();
1468 auto unwrappedIO = this->unwrapIO(s, bb, ports);
1470 InputHandshake &
addr;
1471 OutputHandshake &
data;
1472 OutputHandshake &done;
1475 InputHandshake &
addr;
1476 InputHandshake &
data;
1477 OutputHandshake &done;
1479 SmallVector<LoadPort, 4> loadPorts;
1480 SmallVector<StorePort, 4> storePorts;
1482 unsigned stCount = op.getStCount();
1483 unsigned ldCount = op.getLdCount();
1484 for (
unsigned i = 0, e = ldCount; i != e; ++i) {
1485 LoadPort port = {unwrappedIO.inputs[stCount * 2 + i],
1486 unwrappedIO.outputs[i],
1487 unwrappedIO.outputs[ldCount + stCount + i]};
1488 loadPorts.push_back(port);
1491 for (
unsigned i = 0, e = stCount; i != e; ++i) {
1492 StorePort port = {unwrappedIO.inputs[i * 2 + 1],
1493 unwrappedIO.inputs[i * 2],
1494 unwrappedIO.outputs[ldCount + i]};
1495 storePorts.push_back(port);
1499 auto c0I0 = s.constant(0, 0);
1501 auto cl2dim = llvm::Log2_64_Ceil(op.getMemRefType().getShape()[0]);
1502 auto hlmem = s.b.create<seq::HLMemOp>(
1503 loc, s.clk, s.rst,
"_handshake_memory_" + std::to_string(op.getId()),
1504 op.getMemRefType().getShape(), op.getMemRefType().getElementType());
1507 for (
auto &ld : loadPorts) {
1508 llvm::SmallVector<Value> addresses = {s.truncate(ld.addr.data, cl2dim)};
1509 auto readData = s.b.create<seq::ReadPortOp>(loc, hlmem.getHandle(),
1510 addresses, ld.addr.valid,
1512 ld.data.data->setValue(readData);
1513 ld.done.data->setValue(c0I0);
1515 buildForkLogic(s, bb, ld.addr, {ld.data, ld.done});
1519 for (
auto &st : storePorts) {
1522 auto writeValidBufferMuxBE = bb.
get(s.b.getI1Type());
1523 auto writeValidBuffer =
1524 s.reg(
"writeValidBuffer", writeValidBufferMuxBE, s.constant(1, 0));
1525 st.done.valid->
setValue(writeValidBuffer);
1526 st.done.data->setValue(c0I0);
1530 auto storeCompleted =
1531 s.bAnd({st.done.ready, writeValidBuffer},
"storeCompleted");
1535 auto notWriteValidBuffer = s.bNot(writeValidBuffer);
1536 auto emptyOrComplete =
1537 s.bOr({notWriteValidBuffer, storeCompleted},
"emptyOrComplete");
1540 st.addr.ready->setValue(emptyOrComplete);
1541 st.data.ready->setValue(emptyOrComplete);
1544 auto writeValid = s.bAnd({st.addr.valid, st.data.valid},
"writeValid");
1550 writeValidBufferMuxBE.setValue(
1551 s.mux(emptyOrComplete, {writeValidBuffer, writeValid}));
1555 llvm::SmallVector<Value> addresses = {s.truncate(st.addr.data, cl2dim)};
1556 s.b.create<seq::WritePortOp>(loc, hlmem.getHandle(), addresses,
1557 st.data.data, writeValid,
1563 class SinkConversionPattern :
public HandshakeConversionPattern<SinkOp> {
1565 using HandshakeConversionPattern<SinkOp>::HandshakeConversionPattern;
1567 hw::HWModulePortAccessor &ports)
const override {
1568 auto unwrappedIO = this->unwrapIO(s, bb, ports);
1570 unwrappedIO.inputs[0].ready->setValue(s.constant(1, 1));
1574 class SourceConversionPattern :
public HandshakeConversionPattern<SourceOp> {
1576 using HandshakeConversionPattern<SourceOp>::HandshakeConversionPattern;
1578 hw::HWModulePortAccessor &ports)
const override {
1579 auto unwrappedIO = this->unwrapIO(s, bb, ports);
1581 unwrappedIO.outputs[0].valid->setValue(s.constant(1, 1));
1582 unwrappedIO.outputs[0].data->setValue(s.constant(0, 0));
1586 class ConstantConversionPattern
1587 :
public HandshakeConversionPattern<handshake::ConstantOp> {
1589 using HandshakeConversionPattern<
1590 handshake::ConstantOp>::HandshakeConversionPattern;
1592 hw::HWModulePortAccessor &ports)
const override {
1593 auto unwrappedIO = this->unwrapIO(s, bb, ports);
1594 unwrappedIO.outputs[0].valid->setValue(unwrappedIO.inputs[0].valid);
1595 unwrappedIO.inputs[0].ready->setValue(unwrappedIO.outputs[0].ready);
1596 auto constantValue = op->getAttrOfType<IntegerAttr>(
"value").getValue();
1597 unwrappedIO.outputs[0].data->setValue(s.constant(constantValue));
1601 class BufferConversionPattern :
public HandshakeConversionPattern<BufferOp> {
1603 using HandshakeConversionPattern<BufferOp>::HandshakeConversionPattern;
1605 hw::HWModulePortAccessor &ports)
const override {
1606 auto unwrappedIO = this->unwrapIO(s, bb, ports);
1607 auto input = unwrappedIO.inputs[0];
1608 auto output = unwrappedIO.outputs[0];
1609 InputHandshake lastStage;
1610 SmallVector<int64_t> initValues;
1613 if (op.getInitValues())
1614 initValues = op.getInitValueArray();
1617 buildSeqBufferLogic(s, bb,
toValidType(op.getDataType()),
1618 op.getNumSlots(), input, output, initValues);
1621 output.data->setValue(lastStage.data);
1622 output.valid->setValue(lastStage.valid);
1623 lastStage.ready->setValue(output.ready);
1626 struct SeqBufferStage {
1627 SeqBufferStage(Type dataType, InputHandshake &preStage,
BackedgeBuilder &bb,
1628 RTLBuilder &s,
size_t index,
1629 std::optional<int64_t> initValue)
1630 : dataType(dataType), preStage(preStage), s(s), bb(bb), index(index) {
1633 c0s = createZeroDataConst(s, s.loc, dataType);
1634 currentStage.ready = std::make_shared<Backedge>(bb.
get(s.b.getI1Type()));
1636 auto hasInitValue = s.constant(1, initValue.has_value());
1637 auto validBE = bb.
get(s.b.getI1Type());
1638 auto validReg = s.reg(getRegName(
"valid"), validBE, hasInitValue);
1639 auto readyBE = bb.
get(s.b.getI1Type());
1641 Value initValueCs = c0s;
1642 if (initValue.has_value())
1643 initValueCs = s.constant(dataType.getIntOrFloatBitWidth(), *initValue);
1648 buildDataBufferLogic(validReg, initValueCs, validBE, readyBE);
1649 buildControlBufferLogic(validReg, readyBE, dataReg);
1652 StringAttr getRegName(StringRef name) {
1653 return s.b.getStringAttr(name + std::to_string(index) +
"_reg");
1656 void buildControlBufferLogic(Value validReg,
Backedge &readyBE,
1658 auto c0I1 = s.constant(1, 0);
1659 auto readyRegWire = bb.
get(s.b.getI1Type());
1660 auto readyReg = s.reg(getRegName(
"ready"), readyRegWire, c0I1);
1664 currentStage.valid = s.mux(readyReg, {validReg, readyReg},
1665 "controlValid" + std::to_string(index));
1668 auto notReadyReg = s.bNot(readyReg);
1671 auto succNotReady = s.bNot(*currentStage.ready);
1672 auto neitherReady = s.bAnd({succNotReady, notReadyReg});
1673 auto ctrlNotReady = s.mux(neitherReady, {readyReg, validReg});
1674 auto bothReady = s.bAnd({*currentStage.ready, readyReg});
1677 auto resetSignal = s.mux(bothReady, {ctrlNotReady, c0I1});
1678 readyRegWire.setValue(resetSignal);
1681 auto ctrlDataRegBE = bb.
get(dataType);
1682 auto ctrlDataReg = s.reg(getRegName(
"ctrl_data"), ctrlDataRegBE, c0s);
1683 auto dataResult = s.mux(readyReg, {dataReg, ctrlDataReg});
1684 currentStage.data = dataResult;
1686 auto dataNotReadyMux = s.mux(neitherReady, {ctrlDataReg, dataReg});
1687 auto dataResetSignal = s.mux(bothReady, {dataNotReadyMux, c0s});
1688 ctrlDataRegBE.
setValue(dataResetSignal);
1691 Value buildDataBufferLogic(Value validReg, Value initValue,
1695 auto notValidReg = s.bNot(validReg);
1696 auto emptyOrReady = s.bOr({notValidReg, readyBE});
1697 preStage.ready->setValue(emptyOrReady);
1703 auto validRegMux = s.mux(emptyOrReady, {validReg, preStage.valid});
1709 auto dataRegBE = bb.
get(dataType);
1711 s.reg(getRegName(
"data"),
1712 s.mux(emptyOrReady, {dataRegBE, preStage.data}), initValue);
1717 InputHandshake getOutput() {
return currentStage; }
1720 InputHandshake &preStage;
1721 InputHandshake currentStage;
1730 InputHandshake buildSeqBufferLogic(RTLBuilder &s,
BackedgeBuilder &bb,
1731 Type dataType,
unsigned size,
1732 InputHandshake &input,
1733 OutputHandshake &output,
1734 llvm::ArrayRef<int64_t> initValues)
const {
1737 InputHandshake currentStage = input;
1739 for (
unsigned i = 0; i < size; ++i) {
1740 bool isInitialized = i < initValues.size();
1742 isInitialized ? std::optional<int64_t>(initValues[i]) : std::nullopt;
1743 currentStage = SeqBufferStage(dataType, currentStage, bb, s, i, initValue)
1747 return currentStage;
1751 class IndexCastConversionPattern
1752 :
public HandshakeConversionPattern<arith::IndexCastOp> {
1754 using HandshakeConversionPattern<
1755 arith::IndexCastOp>::HandshakeConversionPattern;
1757 hw::HWModulePortAccessor &ports)
const override {
1758 auto unwrappedIO = this->unwrapIO(s, bb, ports);
1759 unsigned sourceBits =
1760 toValidType(op.getIn().getType()).getIntOrFloatBitWidth();
1761 unsigned targetBits =
1762 toValidType(op.getResult().getType()).getIntOrFloatBitWidth();
1763 if (targetBits < sourceBits)
1764 buildTruncateLogic(s, unwrappedIO, targetBits);
1766 buildExtendLogic(s, unwrappedIO,
true);
1770 template <
typename T>
1773 ExtModuleConversionPattern(ESITypeConverter &typeConverter,
1774 MLIRContext *context, OpBuilder &submoduleBuilder,
1775 HandshakeLoweringState &ls)
1777 submoduleBuilder(submoduleBuilder), ls(ls) {}
1778 using OpAdaptor =
typename T::Adaptor;
1781 matchAndRewrite(T op, OpAdaptor adaptor,
1782 ConversionPatternRewriter &rewriter)
const override {
1792 llvm::SmallVector<Value> operands = adaptor.getOperands();
1793 addSequentialIOOperandsIfNeeded(op, operands);
1794 rewriter.replaceOpWithNewOp<hw::InstanceOp>(
1795 op, implModule, rewriter.getStringAttr(ls.nameUniquer(op)), operands);
1800 OpBuilder &submoduleBuilder;
1801 HandshakeLoweringState &ls;
1806 using OpConversionPattern::OpConversionPattern;
1809 matchAndRewrite(handshake::FuncOp op, OpAdaptor operands,
1810 ConversionPatternRewriter &rewriter)
const override {
1814 HWModuleLike hwModule;
1815 if (op.isExternal()) {
1817 op.getLoc(), rewriter.getStringAttr(op.getName()), ports);
1820 op.getLoc(), rewriter.getStringAttr(op.getName()), ports);
1821 auto args = hwModuleOp.getBodyBlock()->getArguments().drop_back(2);
1822 rewriter.inlineBlockBefore(&op.getBody().front(),
1823 hwModuleOp.getBodyBlock()->getTerminator(),
1825 hwModule = hwModuleOp;
1832 auto *parentOp = op->getParentOp();
1833 auto *predeclModule =
1834 SymbolTable::lookupSymbolIn(parentOp, predecl.getValue());
1835 if (predeclModule) {
1836 if (failed(SymbolTable::replaceAllSymbolUses(
1837 predeclModule, hwModule.getModuleNameAttr(), parentOp)))
1839 rewriter.eraseOp(predeclModule);
1843 rewriter.eraseOp(op);
1855 ConversionTarget &target,
1856 handshake::FuncOp op,
1857 OpBuilder &moduleBuilder) {
1859 std::map<std::string, unsigned> instanceNameCntr;
1860 NameUniquer instanceUniquer = [&](Operation *op) {
1862 if (
auto idAttr = op->getAttrOfType<IntegerAttr>(
"handshake_id"); idAttr) {
1865 instName +=
"_id" + std::to_string(idAttr.getValue().getZExtValue());
1868 instName += std::to_string(instanceNameCntr[instName]++);
1873 auto ls = HandshakeLoweringState{op->getParentOfType<mlir::ModuleOp>(),
1875 RewritePatternSet
patterns(op.getContext());
1876 patterns.insert<FuncOpConversionPattern, ReturnConversionPattern>(
1878 patterns.insert<JoinConversionPattern, ForkConversionPattern,
1879 SyncConversionPattern>(typeConverter, op.getContext(),
1884 UnitRateConversionPattern<arith::AddIOp, comb::AddOp>,
1885 UnitRateConversionPattern<arith::SubIOp, comb::SubOp>,
1886 UnitRateConversionPattern<arith::MulIOp, comb::MulOp>,
1887 UnitRateConversionPattern<arith::DivUIOp, comb::DivSOp>,
1888 UnitRateConversionPattern<arith::DivSIOp, comb::DivUOp>,
1889 UnitRateConversionPattern<arith::RemUIOp, comb::ModUOp>,
1890 UnitRateConversionPattern<arith::RemSIOp, comb::ModSOp>,
1891 UnitRateConversionPattern<arith::AndIOp, comb::AndOp>,
1892 UnitRateConversionPattern<arith::OrIOp, comb::OrOp>,
1893 UnitRateConversionPattern<arith::XOrIOp, comb::XorOp>,
1894 UnitRateConversionPattern<arith::ShLIOp, comb::ShlOp>,
1895 UnitRateConversionPattern<arith::ShRUIOp, comb::ShrUOp>,
1896 UnitRateConversionPattern<arith::ShRSIOp, comb::ShrSOp>,
1897 UnitRateConversionPattern<arith::SelectOp, comb::MuxOp>,
1899 StructCreateConversionPattern,
1901 ConditionalBranchConversionPattern, MuxConversionPattern,
1902 PackConversionPattern, UnpackConversionPattern,
1903 ComparisonConversionPattern, BufferConversionPattern,
1904 SourceConversionPattern, SinkConversionPattern, ConstantConversionPattern,
1905 MergeConversionPattern, ControlMergeConversionPattern,
1906 LoadConversionPattern, StoreConversionPattern, MemoryConversionPattern,
1907 InstanceConversionPattern,
1909 ExtendConversionPattern<arith::ExtUIOp,
false>,
1910 ExtendConversionPattern<arith::ExtSIOp,
true>,
1911 TruncateConversionPattern, IndexCastConversionPattern>(
1912 typeConverter, op.getContext(), moduleBuilder, ls);
1914 if (failed(applyPartialConversion(op, target, std::move(
patterns))))
1915 return op->emitOpError() <<
"error during conversion";
1920 class HandshakeToHWPass
1921 :
public circt::impl::HandshakeToHWBase<HandshakeToHWPass> {
1923 void runOnOperation()
override {
1924 mlir::ModuleOp mod = getOperation();
1928 for (
auto f : mod.getOps<handshake::FuncOp>()) {
1930 f.emitOpError() <<
"HandshakeToHW: failed to verify that all values "
1931 "are used exactly once. Remember to run the "
1932 "fork/sink materialization pass before HW lowering.";
1933 signalPassFailure();
1939 std::string topLevel;
1941 SmallVector<std::string> sortedFuncs;
1943 signalPassFailure();
1947 ESITypeConverter typeConverter;
1948 ConversionTarget target(getContext());
1954 .addIllegalDialect<handshake::HandshakeDialect, arith::ArithDialect>();
1960 OpBuilder submoduleBuilder(mod.getContext());
1961 submoduleBuilder.setInsertionPointToStart(mod.getBody());
1962 for (
auto &funcName : llvm::reverse(sortedFuncs)) {
1963 auto funcOp = mod.lookupSymbol<handshake::FuncOp>(funcName);
1964 assert(funcOp &&
"handshake.func not found in module!");
1966 convertFuncOp(typeConverter, target, funcOp, submoduleBuilder))) {
1967 signalPassFailure();
1976 return signalPassFailure();
1982 return std::make_unique<HandshakeToHWPass>();
assert(baseType &&"element must be base type")
return wrap(CMemoryType::get(unwrap(ctx), baseType, numElements))
static std::string valueName(Operation *scopeOp, Value v)
Convenience function for getting the SSA name of v under the scope of operation scopeOp.
static SmallVector< T > concat(const SmallVectorImpl< T > &a, const SmallVectorImpl< T > &b)
Returns a new vector containing the concatenation of vectors a and b.
static Type tupleToStruct(TupleType tuple)
std::function< std::string(Operation *)> NameUniquer
static void buildModule(OpBuilder &builder, OperationState &result, StringAttr name, ArrayRef< PortInfo > ports, ArrayAttr annotations, ArrayAttr layers)
static std::string getCallName(Operation *op)
static SmallVector< Type > filterNoneTypes(ArrayRef< Type > input)
Filters NoneType's from the input.
static Type getOperandDataType(Value op)
Extracts the type of the data-carrying type of opType.
static DiscriminatingTypes getHandshakeDiscriminatingTypes(Operation *op)
static ModulePortInfo getPortInfoForOp(Operation *op)
Returns a vector of PortInfo's which defines the HW interface of the to-be-converted op.
static std::string getBareSubModuleName(Operation *oldOp)
Returns a submodule name resulting from an operation, without discriminating type information.
static std::string getSubModuleName(Operation *oldOp)
Construct a name for creating HW sub-module.
static HWModuleLike checkSubModuleOp(mlir::ModuleOp parentModule, StringRef modName)
Check whether a submodule with the same name has been created elsewhere in the top level module.
std::pair< SmallVector< Type >, SmallVector< Type > > DiscriminatingTypes
Returns a set of types which may uniquely identify the provided op.
static LogicalResult convertFuncOp(ESITypeConverter &typeConverter, ConversionTarget &target, handshake::FuncOp op, OpBuilder &moduleBuilder)
static llvm::SmallVector< hw::detail::FieldInfo > portToFieldInfo(llvm::ArrayRef< hw::PortInfo > portInfo)
static std::string getTypeName(Location loc, Type type)
Get type name.
static LogicalResult convertExtMemoryOps(HWModuleOp mod)
static EvaluatorValuePtr unwrap(OMEvaluatorValue c)
static std::optional< APInt > getInt(Value value)
Helper to convert a value to a constant integer if it is one.
Instantiate one of these and use it to build typed backedges.
Backedge get(mlir::Type resultType, mlir::LocationAttr optionalLoc={})
Create a typed backedge.
Backedge is a wrapper class around a Value.
void setValue(mlir::Value)
Channels are the basic communication primitives.
const Type * getInner() const
def create(cls, result_type, reset=None, reset_value=None, name=None, sym_name=None, **kwargs)
Direction get(bool isOutput)
Returns an output direction if isOutput is true, otherwise returns an input direction.
Value createOrFoldSExt(Location loc, Value value, Type destTy, OpBuilder &builder)
Create a sign extension operation from a value of integer type to an equal or larger integer type.
mlir::Type innerType(mlir::Type type)
hw::ModulePortInfo getPortInfoForOpTypes(mlir::Operation *op, TypeRange inputs, TypeRange outputs)
Returns the hw::ModulePortInfo that corresponds to the given handshake operation and its in- and outp...
std::map< std::string, std::set< std::string > > InstanceGraph
Iterates over the handshake::FuncOp's in the program to build an instance graph.
LogicalResult resolveInstanceGraph(ModuleOp moduleOp, InstanceGraph &instanceGraph, std::string &topLevel, SmallVectorImpl< std::string > &sortedFuncs)
Iterates over the handshake::FuncOp's in the program to build an instance graph.
static constexpr const char * kPredeclarationAttr
Attribute name for the name of a predeclaration of the to-be-lowered hw.module from a handshake funct...
esi::ChannelType esiWrapper(Type t)
Wraps a type into an ESI ChannelType type.
LogicalResult verifyAllValuesHasOneUse(handshake::FuncOp op)
Checks all block arguments and values within op to ensure that all values have exactly one use.
Type toValidType(Type t)
Converts 't' into a valid HW type.
The InstanceGraph op interface, see InstanceGraphInterface.td for more details.
std::unique_ptr< mlir::Pass > createHandshakeToHWPass()
def reg(value, clock, reset=None, reset_value=None, name=None, sym_name=None)
This holds a decoded list of input/inout and output ports for a module or instance.
void eraseInput(size_t idx)
PortDirectionRange getInputs()
PortDirectionRange getOutputs()