21 #include "mlir/IR/ImplicitLocOpBuilder.h"
22 #include "mlir/Pass/Pass.h"
23 #include "llvm/ADT/BitVector.h"
24 #include "llvm/ADT/DenseMap.h"
25 #include "llvm/ADT/EquivalenceClasses.h"
26 #include "llvm/ADT/PostOrderIterator.h"
27 #include "llvm/Support/Debug.h"
29 #define DEBUG_TYPE "firrtl-lower-xmr"
33 #define GEN_PASS_DEF_LOWERXMR
34 #include "circt/Dialect/FIRRTL/Passes.h.inc"
38 using namespace circt;
39 using namespace firrtl;
40 using hw::InnerRefAttr;
62 using NextNodeOnPath = std::optional<size_t>;
63 using SymOrIndexOp = PointerUnion<Attribute, Operation *>;
67 [[maybe_unused]] llvm::raw_ostream &
operator<<(llvm::raw_ostream &os,
68 const XMRNode &node) {
70 if (
auto attr = dyn_cast<Attribute>(node.info))
71 os <<
"path=" << attr;
73 auto subOp = cast<RefSubOp>(cast<Operation *>(node.info));
74 os <<
"index=" << subOp.getIndex() <<
" (-> " << subOp.getType() <<
")";
76 os <<
", next=" << node.next <<
")";
87 circuitNamespace = &ns;
89 llvm::EquivalenceClasses<Value, ValueComparator> eq;
90 dataFlowClasses = &eq;
93 SmallVector<RefResolveOp> resolveOps;
94 SmallVector<RefSubOp> indexingOps;
95 SmallVector<Operation *> forceAndReleaseOps;
98 auto transferFunc = [&](Operation *op) -> LogicalResult {
99 return TypeSwitch<Operation *, LogicalResult>(op)
100 .Case<RefSendOp>([&](RefSendOp send) {
103 Value xmrDef = send.getBase();
104 if (isZeroWidth(send.getType().getType())) {
105 markForRemoval(send);
109 if (
auto verbExpr = xmrDef.getDefiningOp<VerbatimExprOp>())
110 if (verbExpr.getSymbolsAttr().empty() && verbExpr->hasOneUse()) {
115 auto inRef = InnerRefAttr();
116 auto ind = addReachingSendsEntry(send.getResult(), inRef);
117 xmrPathSuffix[ind] = verbExpr.getText();
118 markForRemoval(verbExpr);
119 markForRemoval(send);
126 ImplicitLocOpBuilder b(xmrDef.getLoc(), &getContext());
127 b.setInsertionPointAfterValue(xmrDef);
128 SmallString<32> opName;
129 auto nameKind = NameKindEnum::DroppableName;
135 opName = name +
"_probe";
136 nameKind = NameKindEnum::InterestingName;
137 }
else if (
auto *xmrDefOp = xmrDef.getDefiningOp()) {
140 if (
auto name = xmrDefOp->getAttrOfType<StringAttr>(
"name")) {
141 (Twine(name.strref()) +
"_probe").
toVector(opName);
142 nameKind = NameKindEnum::InterestingName;
145 xmrDef = b.create<NodeOp>(xmrDef, opName, nameKind).getResult();
149 addReachingSendsEntry(send.getResult(),
getInnerRefTo(xmrDef));
150 markForRemoval(send);
153 .Case<RWProbeOp>([&](RWProbeOp rwprobe) {
154 if (!isZeroWidth(rwprobe.getType().getType()))
155 addReachingSendsEntry(rwprobe.getResult(), rwprobe.getTarget());
156 markForRemoval(rwprobe);
159 .Case<MemOp>([&](MemOp mem) {
165 for (
const auto &res : llvm::enumerate(mem.getResults()))
166 if (isa<RefType>(mem.getResult(res.index()).getType())) {
168 auto ind = addReachingSendsEntry(res.value(), inRef);
169 xmrPathSuffix[ind] =
"Memory";
172 refPortsToRemoveMap[mem].resize(1);
177 [&](
auto inst) {
return handleInstanceOp(inst, instanceGraph); })
178 .Case<FConnectLike>([&](FConnectLike
connect) {
180 if (!isa<RefType>(
connect.getSrc().getType()))
184 type_cast<RefType>(
connect.getSrc().getType()).getType()))
201 .Case<RefSubOp>([&](RefSubOp op) -> LogicalResult {
203 if (isZeroWidth(op.getType().getType()))
207 indexingOps.push_back(op);
210 .Case<RefResolveOp>([&](RefResolveOp resolve) {
221 markForRemoval(resolve);
222 if (!isZeroWidth(resolve.getType()))
223 dataFlowClasses->unionSets(resolve.getRef(), resolve.getResult());
224 resolveOps.push_back(resolve);
227 .Case<RefCastOp>([&](RefCastOp op) {
229 if (!isZeroWidth(op.getType().getType()))
230 dataFlowClasses->unionSets(op.getInput(), op.getResult());
233 .Case<Forceable>([&](Forceable op) {
235 if (type_isa<RefType>(op.getDataRaw().getType())) {
241 if (!op.isForceable() || op.getDataRef().use_empty() ||
242 isZeroWidth(op.getDataType()))
248 .Case<RefForceOp, RefForceInitialOp, RefReleaseOp,
249 RefReleaseInitialOp>([&](
auto op) {
250 forceAndReleaseOps.push_back(op);
253 .Default([&](
auto) {
return success(); });
256 SmallVector<FModuleOp> publicModules;
259 for (
auto node : llvm::post_order(&instanceGraph)) {
260 auto module = dyn_cast<FModuleOp>(*node->getModule());
263 LLVM_DEBUG(llvm::dbgs()
264 <<
"Traversing module:" << module.getModuleNameAttr() <<
"\n");
266 if (module.isPublic())
267 publicModules.push_back(module);
269 auto result = module.walk([&](Operation *op) {
270 if (transferFunc(op).failed())
271 return WalkResult::interrupt();
272 return WalkResult::advance();
275 if (result.wasInterrupted())
276 return signalPassFailure();
287 while (!indexingOps.empty()) {
289 decltype(indexingOps) worklist;
290 worklist.swap(indexingOps);
292 for (
auto op : worklist) {
294 getRemoteRefSend(op.getInput(),
false);
297 indexingOps.push_back(op);
299 addReachingSendsEntry(op.getResult(), op.getOperation(),
303 if (worklist.size() == indexingOps.size()) {
304 auto op = worklist.front();
305 getRemoteRefSend(op.getInput());
307 "indexing through probe of unknown origin (input probe?)")
308 .attachNote(op.getInput().getLoc())
309 .append(
"indexing through this reference");
310 return signalPassFailure();
315 size_t numPorts = module.getNumPorts();
316 for (
size_t portNum = 0; portNum < numPorts; ++portNum)
317 if (isa<RefType>(module.getPortType(portNum))) {
318 setPortToRemove(module, portNum, numPorts);
323 for (
auto I = dataFlowClasses->begin(), E = dataFlowClasses->end();
328 llvm::interleave(llvm::make_range(dataFlowClasses->member_begin(I),
329 dataFlowClasses->member_end()),
331 llvm::dbgs() <<
"\n dataflow at leader::" << I->getData() <<
"\n =>";
332 auto iter = dataflowAt.find(I->getData());
333 if (iter != dataflowAt.end()) {
334 for (auto init = refSendPathList[iter->getSecond()]; init.next;
335 init = refSendPathList[*init.next])
336 llvm::dbgs() <<
"\n " << init;
338 llvm::dbgs() <<
"\n Done\n";
341 for (
auto refResolve : resolveOps)
342 if (handleRefResolve(refResolve).failed())
343 return signalPassFailure();
344 for (
auto *op : forceAndReleaseOps)
345 if (failed(handleForceReleaseOp(op)))
346 return signalPassFailure();
347 for (
auto module : publicModules) {
348 if (failed(handlePublicModuleRefPorts(module)))
349 return signalPassFailure();
354 moduleNamespaces.clear();
355 visitedModules.clear();
357 refSendPathList.clear();
358 dataFlowClasses =
nullptr;
359 refPortsToRemoveMap.clear();
361 xmrPathSuffix.clear();
362 circuitNamespace =
nullptr;
364 pathInsertPoint = {};
369 auto modName = mod.getModuleName();
370 if (
auto ext = dyn_cast<FExtModuleOp>(*mod)) {
372 if (
auto defname = ext.getDefname(); defname && !defname->empty())
375 (Twine(
"ref_") + modName).
toVector(prefix);
381 const Twine &prefix,
bool backTick =
false) {
382 return StringAttr::get(&getContext(), Twine(backTick ?
"`" :
"") + prefix +
383 "_" + mod.getPortName(portIndex));
387 ImplicitLocOpBuilder builder,
388 mlir::FlatSymbolRefAttr &ref,
389 SmallString<128> &stringLeaf) {
390 assert(stringLeaf.empty());
392 auto remoteOpPath = getRemoteRefSend(refVal);
395 SmallVector<Attribute> refSendPath;
396 SmallVector<RefSubOp> indexing;
398 while (remoteOpPath) {
399 lastIndex = *remoteOpPath;
400 auto entr = refSendPathList[*remoteOpPath];
402 TypeSwitch<XMRNode::SymOrIndexOp>(entr.info)
403 .Case<Attribute>([&](
auto attr) {
407 refSendPath.push_back(attr);
410 [&](
auto *op) { indexing.push_back(cast<RefSubOp>(op)); });
411 remoteOpPath = entr.next;
413 auto iter = xmrPathSuffix.find(lastIndex);
417 if (iter != xmrPathSuffix.end()) {
418 if (!refSendPath.empty())
419 stringLeaf.append(
".");
420 stringLeaf.append(iter->getSecond());
423 assert(!(refSendPath.empty() && stringLeaf.empty()) &&
424 "nothing to index through");
437 for (
auto subOp : llvm::reverse(indexing)) {
438 TypeSwitch<FIRRTLBaseType>(subOp.getInput().getType().getType())
439 .Case<FVectorType, OpenVectorType>([&](
auto vecType) {
440 (Twine(
"[") + Twine(subOp.getIndex()) +
"]").
toVector(stringLeaf);
442 .Case<BundleType, OpenBundleType>([&](
auto bundleType) {
443 auto fieldName = bundleType.getElementName(subOp.getIndex());
444 stringLeaf.append({
".", fieldName});
448 if (!refSendPath.empty())
451 getOrCreatePath(builder.getArrayAttr(refSendPath), builder)
458 ImplicitLocOpBuilder &builder,
459 FlatSymbolRefAttr &ref, StringAttr &xmrAttr) {
460 auto remoteOpPath = getRemoteRefSend(refVal);
464 SmallString<128> xmrString;
465 if (failed(resolveReferencePath(refVal, builder, ref, xmrString)))
468 xmrString.empty() ? StringAttr{} : builder.getStringAttr(xmrString);
475 return TypeSwitch<Operation *, LogicalResult>(op)
476 .Case<RefForceOp, RefForceInitialOp, RefReleaseOp, RefReleaseInitialOp>(
479 auto destType = op.getDest().getType();
480 if (isZeroWidth(destType.getType())) {
485 ImplicitLocOpBuilder builder(op.getLoc(), op);
486 FlatSymbolRefAttr ref;
488 if (failed(resolveReference(op.getDest(), builder, ref, str)))
491 Value xmr = builder.create<XMRRefOp>(destType, ref, str);
492 op.getDestMutable().assign(xmr);
495 .Default([](
auto *op) {
496 return op->emitError(
"unexpected operation kind");
503 if (resWidth.has_value() && *resWidth == 0) {
505 ImplicitLocOpBuilder builder(resolve.getLoc(), resolve);
507 auto zeroC = builder.createOrFold<BitCastOp>(
508 resolve.getType(), builder.create<ConstantOp>(
510 resolve.getResult().replaceAllUsesWith(zeroC);
514 FlatSymbolRefAttr ref;
516 ImplicitLocOpBuilder builder(resolve.getLoc(), resolve);
517 if (failed(resolveReference(resolve.getRef(), builder, ref, str)))
520 Value result = builder.create<XMRDerefOp>(resolve.getType(), ref, str);
521 resolve.getResult().replaceAllUsesWith(result);
526 if (refPortsToRemoveMap[op].size() < numPorts)
527 refPortsToRemoveMap[op].resize(numPorts);
528 refPortsToRemoveMap[op].set(index);
534 Operation *mod = inst.getReferencedModule(instanceGraph);
535 if (
auto extRefMod = dyn_cast<FExtModuleOp>(mod)) {
539 auto internalPaths = extRefMod.getInternalPaths();
540 auto numPorts = inst.getNumResults();
541 SmallString<128> circuitRefPrefix;
544 auto getPath = [&](
size_t portNo) {
548 cast<InternalPathAttr>(internalPaths->getValue()[portNo])
554 if (circuitRefPrefix.empty())
555 getRefABIPrefix(extRefMod, circuitRefPrefix);
557 return getRefABIMacroForPort(extRefMod, portNo, circuitRefPrefix,
true);
560 for (
const auto &res : llvm::enumerate(inst.getResults())) {
561 if (!isa<RefType>(inst.getResult(res.index()).getType()))
565 auto ind = addReachingSendsEntry(res.value(), inRef);
567 xmrPathSuffix[ind] = getPath(res.index());
569 setPortToRemove(inst, res.index(), numPorts);
570 setPortToRemove(extRefMod, res.index(), numPorts);
574 auto refMod = dyn_cast<FModuleOp>(mod);
575 bool multiplyInstantiated = !visitedModules.insert(refMod).second;
576 for (
size_t portNum = 0, numPorts = inst.getNumResults();
577 portNum < numPorts; ++portNum) {
578 auto instanceResult = inst.getResult(portNum);
579 if (!isa<RefType>(instanceResult.getType()))
582 return inst.emitOpError(
"cannot lower ext modules with RefType ports");
584 setPortToRemove(inst, portNum, numPorts);
586 if (instanceResult.use_empty() ||
587 isZeroWidth(type_cast<RefType>(instanceResult.getType()).getType()))
589 auto refModuleArg = refMod.getArgument(portNum);
594 auto remoteOpPath = getRemoteRefSend(refModuleArg);
606 if (multiplyInstantiated)
607 return refMod.emitOpError(
608 "multiply instantiated module with input RefType port '")
609 << refMod.getPortName(portNum) <<
"'";
610 dataFlowClasses->unionSets(
611 dataFlowClasses->getOrInsertLeaderValue(refModuleArg),
612 dataFlowClasses->getOrInsertLeaderValue(instanceResult));
619 auto *body = getOperation().getBodyBlock();
622 SmallString<128> circuitRefPrefix;
623 SmallVector<std::tuple<StringAttr, StringAttr, ArrayAttr>> ports;
625 ImplicitLocOpBuilder::atBlockBegin(module.getLoc(), body);
626 for (
size_t portIndex = 0, numPorts = module.getNumPorts();
627 portIndex != numPorts; ++portIndex) {
628 auto refType = type_dyn_cast<RefType>(module.getPortType(portIndex));
629 if (!refType || isZeroWidth(refType.getType()) ||
633 cast<mlir::TypedValue<RefType>>(module.getArgument(portIndex));
634 mlir::FlatSymbolRefAttr ref;
635 SmallString<128> stringLeaf;
636 if (failed(resolveReferencePath(portValue, declBuilder, ref, stringLeaf)))
639 SmallString<128> formatString;
641 formatString +=
"{{0}}";
642 formatString += stringLeaf;
646 if (circuitRefPrefix.empty())
647 getRefABIPrefix(module, circuitRefPrefix);
649 getRefABIMacroForPort(module, portIndex, circuitRefPrefix);
650 declBuilder.create<sv::MacroDeclOp>(macroName, ArrayAttr(), StringAttr());
651 ports.emplace_back(macroName, declBuilder.getStringAttr(formatString),
652 ref ? declBuilder.getArrayAttr({ref}) : ArrayAttr{});
661 auto fileBuilder = ImplicitLocOpBuilder(module.getLoc(), module);
662 fileBuilder.create<emit::FileOp>(circuitRefPrefix +
".sv", [&] {
663 for (
auto [macroName, formatString, symbols] : ports) {
665 formatString, symbols);
674 return moduleNamespaces.try_emplace(module, module).first->second;
678 if (
auto arg = dyn_cast<BlockArgument>(val))
680 cast<FModuleLike>(arg.getParentBlock()->getParentOp()),
682 [&](FModuleLike mod) -> hw::InnerSymbolNamespace & {
683 return getModuleNamespace(mod);
690 [&](FModuleLike mod) -> hw::InnerSymbolNamespace & {
691 return getModuleNamespace(mod);
698 bool errorIfNotFound =
true) {
699 auto iter = dataflowAt.find(dataFlowClasses->getOrInsertLeaderValue(val));
700 if (iter != dataflowAt.end())
701 return iter->getSecond();
702 if (!errorIfNotFound)
706 if (BlockArgument arg = dyn_cast<BlockArgument>(val))
707 arg.getOwner()->getParentOp()->emitError(
708 "reference dataflow cannot be traced back to the remote read op "
710 << dyn_cast<FModuleOp>(arg.getOwner()->getParentOp())
711 .getPortName(arg.getArgNumber())
714 val.getDefiningOp()->emitOpError(
715 "reference dataflow cannot be traced back to the remote read op");
722 std::optional<size_t> continueFrom = std::nullopt) {
723 auto leader = dataFlowClasses->getOrInsertLeaderValue(atRefVal);
724 auto indx = refSendPathList.size();
725 dataflowAt[leader] = indx;
726 refSendPathList.push_back({info, continueFrom});
734 for (Operation *op : llvm::reverse(opsToRemove))
736 for (
auto iter : refPortsToRemoveMap)
737 if (
auto mod = dyn_cast<FModuleOp>(iter.getFirst()))
738 mod.erasePorts(iter.getSecond());
739 else if (
auto mod = dyn_cast<FExtModuleOp>(iter.getFirst()))
740 mod.erasePorts(iter.getSecond());
741 else if (
auto inst = dyn_cast<InstanceOp>(iter.getFirst())) {
742 ImplicitLocOpBuilder b(inst.getLoc(), inst);
743 inst.erasePorts(b, iter.getSecond());
745 }
else if (
auto mem = dyn_cast<MemOp>(iter.getFirst())) {
747 ImplicitLocOpBuilder builder(mem.getLoc(), mem);
748 SmallVector<Attribute, 4> resultNames;
749 SmallVector<Type, 4> resultTypes;
750 SmallVector<Attribute, 4> portAnnotations;
751 SmallVector<Value, 4> oldResults;
752 for (
const auto &res : llvm::enumerate(mem.getResults())) {
753 if (isa<RefType>(mem.getResult(res.index()).getType()))
755 resultNames.push_back(mem.getPortName(res.index()));
756 resultTypes.push_back(res.value().getType());
757 portAnnotations.push_back(mem.getPortAnnotation(res.index()));
758 oldResults.push_back(res.value());
760 auto newMem = builder.create<MemOp>(
761 resultTypes, mem.getReadLatency(), mem.getWriteLatency(),
762 mem.getDepth(), RUWAttr::Undefined,
763 builder.getArrayAttr(resultNames), mem.getNameAttr(),
764 mem.getNameKind(), mem.getAnnotations(),
765 builder.getArrayAttr(portAnnotations), mem.getInnerSymAttr(),
766 mem.getInitAttr(), mem.getPrefixAttr());
767 for (
const auto &res : llvm::enumerate(oldResults))
768 res.value().replaceAllUsesWith(newMem.getResult(res.index()));
772 refPortsToRemoveMap.clear();
774 refSendPathList.clear();
782 ImplicitLocOpBuilder &builder) {
783 assert(pathArray && !pathArray.empty());
785 auto pathIter = pathCache.find(pathArray);
786 if (pathIter != pathCache.end())
787 return pathIter->second;
790 OpBuilder::InsertionGuard guard(builder);
794 if (pathInsertPoint.isSet())
795 builder.restoreInsertionPoint(pathInsertPoint);
797 builder.setInsertionPointToStart(getOperation().
getBodyBlock());
800 hw::HierPathOp path =
803 builder.create<hw::HierPathOp>(
804 circuitNamespace->newName(
"xmrPath"), pathArray)})
806 path.setVisibility(SymbolTable::Visibility::Private);
810 pathInsertPoint = builder.saveInsertionPoint();
840 return lhs.getImpl() < rhs.getImpl();
861 OpBuilder::InsertPoint pathInsertPoint = {};
865 return std::make_unique<LowerXMRPass>();
assert(baseType &&"element must be base type")
static std::vector< mlir::Value > toVector(mlir::ValueRange range)
static Block * getBodyBlock(FModuleLike mod)
hw::InnerSymbolNamespace & getModuleNamespace(FModuleLike module)
Get the cached namespace for a module.
LogicalResult resolveReference(mlir::TypedValue< RefType > refVal, ImplicitLocOpBuilder &builder, FlatSymbolRefAttr &ref, StringAttr &xmrAttr)
DenseMap< Operation *, hw::InnerSymbolNamespace > moduleNamespaces
Cached module namespaces.
DenseMap< size_t, SmallString< 128 > > xmrPathSuffix
Record the internal path to an external module or a memory.
InnerRefAttr getInnerRefTo(Value val)
size_t addReachingSendsEntry(Value atRefVal, XMRNode::SymOrIndexOp info, std::optional< size_t > continueFrom=std::nullopt)
hw::HierPathOp getOrCreatePath(ArrayAttr pathArray, ImplicitLocOpBuilder &builder)
Return a HierPathOp for the provided pathArray.
LogicalResult resolveReferencePath(mlir::TypedValue< RefType > refVal, ImplicitLocOpBuilder builder, mlir::FlatSymbolRefAttr &ref, SmallString< 128 > &stringLeaf)
DenseMap< Value, size_t > dataflowAt
Map of a reference value to an entry into refSendPathList.
void setPortToRemove(Operation *op, size_t index, size_t numPorts)
llvm::EquivalenceClasses< Value, ValueComparator > * dataFlowClasses
void markForRemoval(Operation *op)
LogicalResult handlePublicModuleRefPorts(FModuleOp module)
void getRefABIPrefix(FModuleLike mod, SmallVectorImpl< char > &prefix)
Generate the ABI ref_<module> prefix string into prefix.
void runOnOperation() override
DenseMap< Attribute, hw::HierPathOp > pathCache
A cache of already created HierPathOps.
LogicalResult handleRefResolve(RefResolveOp resolve)
DenseMap< Operation *, llvm::BitVector > refPortsToRemoveMap
SmallVector< XMRNode > refSendPathList
refSendPathList is used to construct a path to the RefSendOp.
LogicalResult handleInstanceOp(InstanceOp inst, InstanceGraph &instanceGraph)
LogicalResult handleForceReleaseOp(Operation *op)
DenseSet< Operation * > visitedModules
InnerRefAttr getInnerRefTo(Operation *op)
StringAttr getRefABIMacroForPort(FModuleLike mod, size_t portIndex, const Twine &prefix, bool backTick=false)
Get full macro name as StringAttr for the specified ref port.
CircuitNamespace * circuitNamespace
bool isZeroWidth(FIRRTLBaseType t)
std::optional< size_t > getRemoteRefSend(Value val, bool errorIfNotFound=true)
SmallVector< Operation * > opsToRemove
RefResolve, RefSend, and Connects involving them that will be removed.
int32_t getBitWidthOrSentinel()
If this is an IntType, AnalogType, or sugar type for a single bit (Clock, Reset, etc) then return the...
This graph tracks modules and where they are instantiated.
def connect(destination, source)
Direction get(bool isOutput)
Returns an output direction if isOutput is true, otherwise returns an input direction.
FieldRef getFieldRefFromValue(Value value, bool lookThroughCasts=false)
Get the FieldRef from a value.
hw::InnerRefAttr getInnerRefTo(const hw::InnerSymTarget &target, GetNamespaceCallback getNamespace)
Obtain an inner reference to the target (operation or port), adding an inner symbol as necessary.
std::pair< std::string, bool > getFieldName(const FieldRef &fieldRef, bool nameSafe=false)
Get a string identifier representing the FieldRef.
llvm::raw_ostream & operator<<(llvm::raw_ostream &os, const InstanceInfo::LatticeValue &value)
std::unique_ptr< mlir::Pass > createLowerXMRPass()
std::optional< int64_t > getBitWidth(FIRRTLBaseType type, bool ignoreFlip=false)
IntegerAttr getIntZerosAttr(Type type)
Utility for generating a constant zero attribute.
The InstanceGraph op interface, see InstanceGraphInterface.td for more details.
llvm::EquivalenceClasses wants comparable elements.
bool operator()(const Value &lhs, const Value &rhs) const
The namespace of a CircuitOp, generally inhabited by modules.