22#include "mlir/IR/ImplicitLocOpBuilder.h"
23#include "mlir/Pass/Pass.h"
24#include "llvm/ADT/BitVector.h"
25#include "llvm/ADT/DenseMap.h"
26#include "llvm/ADT/EquivalenceClasses.h"
27#include "llvm/ADT/PostOrderIterator.h"
28#include "llvm/Support/Debug.h"
30#define DEBUG_TYPE "firrtl-lower-xmr"
34#define GEN_PASS_DEF_LOWERXMR
35#include "circt/Dialect/FIRRTL/Passes.h.inc"
40using namespace firrtl;
41using hw::InnerRefAttr;
63 using NextNodeOnPath = std::optional<size_t>;
64 using SymOrIndexOp = PointerUnion<Attribute, Operation *>;
68[[maybe_unused]] llvm::raw_ostream &
operator<<(llvm::raw_ostream &os,
69 const XMRNode &node) {
71 if (
auto attr = dyn_cast<Attribute>(node.info))
72 os <<
"path=" << attr;
74 auto subOp = cast<RefSubOp>(cast<Operation *>(node.info));
75 os <<
"index=" << subOp.getIndex() <<
" (-> " << subOp.getType() <<
")";
77 os <<
", next=" << node.next <<
")";
86 ModuleState(FModuleOp &moduleOp) : body(moduleOp.
getBodyBlock()) {}
92 Value getOrCreateXMRRefOp(Type type, FlatSymbolRefAttr symbol,
93 StringAttr suffix, ImplicitLocOpBuilder &builder) {
95 auto it = xmrRefCache.find({type, symbol, suffix});
96 if (it != xmrRefCache.end())
97 return it->getSecond();
100 OpBuilder::InsertionGuard guard(builder);
101 if (xmrRefPoint.isSet())
102 builder.restoreInsertionPoint(xmrRefPoint);
104 builder.setInsertionPointToStart(body);
106 Value xmr = builder.create<XMRRefOp>(type, symbol, suffix);
107 xmrRefCache.insert({{type, symbol, suffix}, xmr});
109 xmrRefPoint = builder.saveInsertionPoint();
119 DenseMap<std::tuple<Type, SymbolRefAttr, StringAttr>, Value> xmrRefCache;
122 OpBuilder::InsertPoint xmrRefPoint;
135 &ns, OpBuilder::InsertPoint(getOperation().
getBodyBlock(),
139 llvm::EquivalenceClasses<Value> eq;
143 SmallVector<RefResolveOp> resolveOps;
144 SmallVector<RefSubOp> indexingOps;
145 SmallVector<Operation *> forceAndReleaseOps;
148 auto transferFunc = [&](Operation *op) -> LogicalResult {
149 return TypeSwitch<Operation *, LogicalResult>(op)
150 .Case<RefSendOp>([&](RefSendOp send) {
153 Value xmrDef = send.getBase();
159 if (
auto verbExpr = xmrDef.getDefiningOp<VerbatimExprOp>())
160 if (verbExpr.getSymbolsAttr().empty() && verbExpr->hasOneUse()) {
165 auto inRef = InnerRefAttr();
176 ImplicitLocOpBuilder b(xmrDef.getLoc(), &getContext());
177 b.setInsertionPointAfterValue(xmrDef);
178 SmallString<32> opName;
179 auto nameKind = NameKindEnum::DroppableName;
185 opName = name +
"_probe";
186 nameKind = NameKindEnum::InterestingName;
187 }
else if (
auto *xmrDefOp = xmrDef.getDefiningOp()) {
190 if (
auto name = xmrDefOp->getAttrOfType<StringAttr>(
"name")) {
191 (Twine(name.strref()) +
"_probe").
toVector(opName);
192 nameKind = NameKindEnum::InterestingName;
195 xmrDef = b.create<NodeOp>(xmrDef, opName, nameKind).getResult();
203 .Case<RWProbeOp>([&](RWProbeOp rwprobe) {
209 .Case<MemOp>([&](MemOp mem) {
215 for (
const auto &res : llvm::enumerate(mem.getResults()))
216 if (isa<RefType>(mem.getResult(res.index()).getType())) {
228 .Case<FConnectLike>([&](FConnectLike connect) {
230 if (!isa<RefType>(connect.getSrc().getType()))
234 type_cast<RefType>(connect.getSrc().getType()).getType()))
251 .Case<RefSubOp>([&](RefSubOp op) -> LogicalResult {
257 indexingOps.push_back(op);
260 .Case<RefResolveOp>([&](RefResolveOp resolve) {
274 resolveOps.push_back(resolve);
277 .Case<RefCastOp>([&](RefCastOp op) {
283 .Case<Forceable>([&](Forceable op) {
285 if (type_isa<RefType>(op.getDataRaw().getType())) {
291 if (!op.isForceable() || op.getDataRef().use_empty() ||
298 .Case<RefForceOp, RefForceInitialOp, RefReleaseOp,
299 RefReleaseInitialOp>([&](
auto op) {
300 forceAndReleaseOps.push_back(op);
303 .Default([&](
auto) {
return success(); });
306 SmallVector<FModuleOp> publicModules;
310 DenseSet<InstanceGraphNode *> visited;
311 for (
auto *root : instanceGraph) {
312 for (
auto *node : llvm::post_order_ext(root, visited)) {
313 auto module = dyn_cast<FModuleOp>(*node->getModule());
316 LLVM_DEBUG(llvm::dbgs() <<
"Traversing module:"
317 << module.getModuleNameAttr() <<
"\n");
321 if (module.isPublic())
322 publicModules.push_back(module);
324 auto result =
module.walk([&](Operation *op) {
325 if (transferFunc(op).failed())
326 return WalkResult::interrupt();
327 return WalkResult::advance();
330 if (result.wasInterrupted())
331 return signalPassFailure();
339 while (!indexingOps.empty()) {
341 decltype(indexingOps) worklist;
342 worklist.swap(indexingOps);
344 for (
auto op : worklist) {
349 indexingOps.push_back(op);
355 if (worklist.size() == indexingOps.size()) {
356 auto op = worklist.front();
359 "indexing through probe of unknown origin (input probe?)")
360 .attachNote(op.getInput().getLoc())
361 .append(
"indexing through this reference");
362 return signalPassFailure();
367 size_t numPorts =
module.getNumPorts();
368 for (
size_t portNum = 0; portNum < numPorts; ++portNum)
369 if (isa<RefType>(module.getPortType(portNum))) {
382 llvm::dbgs() <<
"\n dataflow at leader::" << I->getData() <<
"\n =>";
387 llvm::dbgs() <<
"\n " << init;
389 llvm::dbgs() <<
"\n Done\n";
392 for (
auto refResolve : resolveOps)
394 return signalPassFailure();
395 for (
auto *op : forceAndReleaseOps)
397 return signalPassFailure();
398 for (
auto module : publicModules) {
400 return signalPassFailure();
419 auto modName = mod.getModuleName();
420 if (
auto ext = dyn_cast<FExtModuleOp>(*mod)) {
422 if (
auto defname = ext.getDefname(); defname && !defname->empty())
425 (Twine(
"ref_") + modName).
toVector(prefix);
431 const Twine &prefix,
bool backTick =
false) {
432 return StringAttr::get(&getContext(), Twine(backTick ?
"`" :
"") + prefix +
433 "_" + mod.getPortName(portIndex));
437 ImplicitLocOpBuilder builder,
438 mlir::FlatSymbolRefAttr &ref,
439 SmallString<128> &stringLeaf) {
440 assert(stringLeaf.empty());
442 auto remoteOpPath = getRemoteRefSend(refVal);
445 SmallVector<Attribute> refSendPath;
446 SmallVector<RefSubOp> indexing;
448 while (remoteOpPath) {
449 lastIndex = *remoteOpPath;
450 auto entr = refSendPathList[*remoteOpPath];
452 TypeSwitch<XMRNode::SymOrIndexOp>(entr.info)
453 .Case<Attribute>([&](
auto attr) {
457 refSendPath.push_back(attr);
460 [&](
auto *op) { indexing.push_back(cast<RefSubOp>(op)); });
461 remoteOpPath = entr.next;
463 auto iter = xmrPathSuffix.find(lastIndex);
467 if (iter != xmrPathSuffix.end()) {
468 if (!refSendPath.empty())
469 stringLeaf.append(
".");
470 stringLeaf.append(iter->getSecond());
473 assert(!(refSendPath.empty() && stringLeaf.empty()) &&
474 "nothing to index through");
487 for (
auto subOp : llvm::reverse(indexing)) {
488 TypeSwitch<FIRRTLBaseType>(subOp.getInput().getType().getType())
489 .Case<FVectorType, OpenVectorType>([&](
auto vecType) {
490 (Twine(
"[") + Twine(subOp.getIndex()) +
"]").
toVector(stringLeaf);
492 .Case<BundleType, OpenBundleType>([&](
auto bundleType) {
493 auto fieldName = bundleType.getElementName(subOp.getIndex());
494 stringLeaf.append({
".", fieldName});
498 if (!refSendPath.empty())
500 ref = FlatSymbolRefAttr::get(
502 ->getOrCreatePath(builder.getArrayAttr(refSendPath),
510 ImplicitLocOpBuilder &builder,
511 FlatSymbolRefAttr &ref, StringAttr &xmrAttr) {
512 auto remoteOpPath = getRemoteRefSend(refVal);
516 SmallString<128> xmrString;
517 if (failed(resolveReferencePath(refVal, builder, ref, xmrString)))
520 xmrString.empty() ? StringAttr{} : builder.getStringAttr(xmrString);
527 return TypeSwitch<Operation *, LogicalResult>(op)
528 .Case<RefForceOp, RefForceInitialOp, RefReleaseOp, RefReleaseInitialOp>(
531 auto destType = op.getDest().getType();
532 if (isZeroWidth(destType.getType())) {
537 ImplicitLocOpBuilder builder(op.getLoc(), op);
538 FlatSymbolRefAttr ref;
540 if (failed(resolveReference(op.getDest(), builder, ref, str)))
544 moduleStates.find(op->template getParentOfType<FModuleOp>())
546 .getOrCreateXMRRefOp(destType, ref, str, builder);
547 op.getDestMutable().assign(xmr);
550 .Default([](
auto *op) {
551 return op->emitError(
"unexpected operation kind");
558 if (resWidth.has_value() && *resWidth == 0) {
560 ImplicitLocOpBuilder builder(resolve.getLoc(), resolve);
561 auto zeroUintType = UIntType::get(builder.getContext(), 0);
562 auto zeroC = builder.createOrFold<BitCastOp>(
563 resolve.getType(), builder.create<ConstantOp>(
565 resolve.getResult().replaceAllUsesWith(zeroC);
569 FlatSymbolRefAttr ref;
571 ImplicitLocOpBuilder builder(resolve.getLoc(), resolve);
572 if (failed(resolveReference(resolve.getRef(), builder, ref, str)))
575 Value result = builder.create<XMRDerefOp>(resolve.getType(), ref, str);
576 resolve.getResult().replaceAllUsesWith(result);
581 if (refPortsToRemoveMap[op].size() < numPorts)
582 refPortsToRemoveMap[op].resize(numPorts);
583 refPortsToRemoveMap[op].set(index);
589 Operation *mod = inst.getReferencedModule(instanceGraph);
590 if (
auto extRefMod = dyn_cast<FExtModuleOp>(mod)) {
594 auto internalPaths = extRefMod.getInternalPaths();
595 auto numPorts = inst.getNumResults();
596 SmallString<128> circuitRefPrefix;
599 auto getPath = [&](
size_t portNo) {
603 cast<InternalPathAttr>(internalPaths->getValue()[portNo])
609 if (circuitRefPrefix.empty())
610 getRefABIPrefix(extRefMod, circuitRefPrefix);
612 return getRefABIMacroForPort(extRefMod, portNo, circuitRefPrefix,
true);
615 for (
const auto &res : llvm::enumerate(inst.getResults())) {
616 if (!isa<RefType>(inst.getResult(res.index()).getType()))
620 auto ind = addReachingSendsEntry(res.value(), inRef);
622 xmrPathSuffix[ind] = getPath(res.index());
624 setPortToRemove(inst, res.index(), numPorts);
625 setPortToRemove(extRefMod, res.index(), numPorts);
629 auto refMod = dyn_cast<FModuleOp>(mod);
630 bool multiplyInstantiated = !visitedModules.insert(refMod).second;
631 for (
size_t portNum = 0, numPorts = inst.getNumResults();
632 portNum < numPorts; ++portNum) {
633 auto instanceResult = inst.getResult(portNum);
634 if (!isa<RefType>(instanceResult.getType()))
637 return inst.emitOpError(
"cannot lower ext modules with RefType ports");
639 setPortToRemove(inst, portNum, numPorts);
641 if (instanceResult.use_empty() ||
642 isZeroWidth(type_cast<RefType>(instanceResult.getType()).getType()))
644 auto refModuleArg = refMod.getArgument(portNum);
645 if (inst.getPortDirection(portNum) == Direction::Out) {
649 auto remoteOpPath = getRemoteRefSend(refModuleArg);
661 if (multiplyInstantiated)
662 return refMod.emitOpError(
663 "multiply instantiated module with input RefType port '")
664 << refMod.getPortName(portNum) <<
"'";
665 dataFlowClasses->unionSets(
666 dataFlowClasses->getOrInsertLeaderValue(refModuleArg),
667 dataFlowClasses->getOrInsertLeaderValue(instanceResult));
674 auto *body = getOperation().getBodyBlock();
677 SmallString<128> circuitRefPrefix;
678 SmallVector<std::tuple<StringAttr, StringAttr, ArrayAttr>> ports;
680 ImplicitLocOpBuilder::atBlockBegin(module.getLoc(), body);
681 for (
size_t portIndex = 0, numPorts = module.getNumPorts();
682 portIndex != numPorts; ++portIndex) {
683 auto refType = type_dyn_cast<RefType>(module.getPortType(portIndex));
684 if (!refType || isZeroWidth(refType.getType()) ||
685 module.getPortDirection(portIndex) != Direction::Out)
688 cast<mlir::TypedValue<RefType>>(
module.getArgument(portIndex));
689 mlir::FlatSymbolRefAttr ref;
690 SmallString<128> stringLeaf;
691 if (failed(resolveReferencePath(portValue, declBuilder, ref, stringLeaf)))
694 SmallString<128> formatString;
696 formatString +=
"{{0}}";
697 formatString += stringLeaf;
701 if (circuitRefPrefix.empty())
702 getRefABIPrefix(module, circuitRefPrefix);
704 getRefABIMacroForPort(module, portIndex, circuitRefPrefix);
705 declBuilder.create<sv::MacroDeclOp>(macroName, ArrayAttr(), StringAttr());
706 ports.emplace_back(macroName, declBuilder.getStringAttr(formatString),
707 ref ? declBuilder.getArrayAttr({ref}) : ArrayAttr{});
716 auto fileBuilder = ImplicitLocOpBuilder(module.getLoc(), module);
717 fileBuilder.create<emit::FileOp>(circuitRefPrefix +
".sv", [&] {
718 for (
auto [macroName, formatString, symbols] : ports) {
719 fileBuilder.create<sv::MacroDefOp>(FlatSymbolRefAttr::get(macroName),
720 formatString, symbols);
729 return moduleNamespaces.try_emplace(module, module).first->second;
733 if (
auto arg = dyn_cast<BlockArgument>(val))
734 return ::getInnerRefTo(
735 cast<FModuleLike>(arg.getParentBlock()->getParentOp()),
738 return getModuleNamespace(mod);
744 return ::getInnerRefTo(op,
746 return getModuleNamespace(mod);
753 bool errorIfNotFound =
true) {
754 auto iter = dataflowAt.find(dataFlowClasses->getOrInsertLeaderValue(val));
755 if (iter != dataflowAt.end())
756 return iter->getSecond();
757 if (!errorIfNotFound)
761 if (BlockArgument arg = dyn_cast<BlockArgument>(val))
762 arg.getOwner()->getParentOp()->emitError(
763 "reference dataflow cannot be traced back to the remote read op "
765 << dyn_cast<FModuleOp>(arg.getOwner()->getParentOp())
766 .getPortName(arg.getArgNumber())
769 val.getDefiningOp()->emitOpError(
770 "reference dataflow cannot be traced back to the remote read op");
777 std::optional<size_t> continueFrom = std::nullopt) {
778 auto leader = dataFlowClasses->getOrInsertLeaderValue(atRefVal);
779 auto indx = refSendPathList.size();
780 dataflowAt[leader] = indx;
781 refSendPathList.push_back({info, continueFrom});
789 for (Operation *op : llvm::reverse(opsToRemove))
791 for (
auto iter : refPortsToRemoveMap)
792 if (
auto mod = dyn_cast<FModuleOp>(iter.getFirst()))
793 mod.erasePorts(iter.getSecond());
794 else if (
auto mod = dyn_cast<FExtModuleOp>(iter.getFirst()))
795 mod.erasePorts(iter.getSecond());
796 else if (
auto inst = dyn_cast<InstanceOp>(iter.getFirst())) {
797 ImplicitLocOpBuilder b(inst.getLoc(), inst);
798 inst.erasePorts(b, iter.getSecond());
800 }
else if (
auto mem = dyn_cast<MemOp>(iter.getFirst())) {
802 ImplicitLocOpBuilder builder(mem.getLoc(), mem);
803 SmallVector<Attribute, 4> resultNames;
804 SmallVector<Type, 4> resultTypes;
805 SmallVector<Attribute, 4> portAnnotations;
806 SmallVector<Value, 4> oldResults;
807 for (
const auto &res : llvm::enumerate(mem.getResults())) {
808 if (isa<RefType>(mem.getResult(res.index()).getType()))
810 resultNames.push_back(mem.getPortName(res.index()));
811 resultTypes.push_back(res.value().getType());
812 portAnnotations.push_back(mem.getPortAnnotation(res.index()));
813 oldResults.push_back(res.value());
815 auto newMem = builder.create<MemOp>(
816 resultTypes, mem.getReadLatency(), mem.getWriteLatency(),
817 mem.getDepth(), RUWAttr::Undefined,
818 builder.getArrayAttr(resultNames), mem.getNameAttr(),
819 mem.getNameKind(), mem.getAnnotations(),
820 builder.getArrayAttr(portAnnotations), mem.getInnerSymAttr(),
821 mem.getInitAttr(), mem.getPrefixAttr());
822 for (
const auto &res : llvm::enumerate(oldResults))
823 res.value().replaceAllUsesWith(newMem.getResult(res.index()));
827 refPortsToRemoveMap.clear();
829 refSendPathList.clear();
830 moduleStates.clear();
876 return std::make_unique<LowerXMRPass>();
assert(baseType &&"element must be base type")
static std::vector< mlir::Value > toVector(mlir::ValueRange range)
static Block * getBodyBlock(FModuleLike mod)
LogicalResult resolveReference(mlir::TypedValue< RefType > refVal, ImplicitLocOpBuilder &builder, FlatSymbolRefAttr &ref, StringAttr &xmrAttr)
DenseMap< Operation *, hw::InnerSymbolNamespace > moduleNamespaces
Cached module namespaces.
llvm::EquivalenceClasses< Value > * dataFlowClasses
DenseMap< size_t, SmallString< 128 > > xmrPathSuffix
Record the internal path to an external module or a memory.
InnerRefAttr getInnerRefTo(Value val)
size_t addReachingSendsEntry(Value atRefVal, XMRNode::SymOrIndexOp info, std::optional< size_t > continueFrom=std::nullopt)
DenseMap< FModuleOp, ModuleState > moduleStates
Per-module helpers for creating operations within modules.
LogicalResult resolveReferencePath(mlir::TypedValue< RefType > refVal, ImplicitLocOpBuilder builder, mlir::FlatSymbolRefAttr &ref, SmallString< 128 > &stringLeaf)
DenseMap< Value, size_t > dataflowAt
Map of a reference value to an entry into refSendPathList.
void setPortToRemove(Operation *op, size_t index, size_t numPorts)
hw::InnerSymbolNamespace & getModuleNamespace(FModuleLike module)
Get the cached namespace for a module.
void markForRemoval(Operation *op)
hw::HierPathCache * hierPathCache
Utility to create HerPathOps at a predefined location in the circuit.
LogicalResult handlePublicModuleRefPorts(FModuleOp module)
void getRefABIPrefix(FModuleLike mod, SmallVectorImpl< char > &prefix)
Generate the ABI ref_<module> prefix string into prefix.
void runOnOperation() override
LogicalResult handleRefResolve(RefResolveOp resolve)
DenseMap< Operation *, llvm::BitVector > refPortsToRemoveMap
SmallVector< XMRNode > refSendPathList
refSendPathList is used to construct a path to the RefSendOp.
LogicalResult handleInstanceOp(InstanceOp inst, InstanceGraph &instanceGraph)
LogicalResult handleForceReleaseOp(Operation *op)
std::optional< size_t > getRemoteRefSend(Value val, bool errorIfNotFound=true)
DenseSet< Operation * > visitedModules
InnerRefAttr getInnerRefTo(Operation *op)
StringAttr getRefABIMacroForPort(FModuleLike mod, size_t portIndex, const Twine &prefix, bool backTick=false)
Get full macro name as StringAttr for the specified ref port.
CircuitNamespace * circuitNamespace
bool isZeroWidth(FIRRTLBaseType t)
SmallVector< Operation * > opsToRemove
RefResolve, RefSend, and Connects involving them that will be removed.
int32_t getBitWidthOrSentinel()
If this is an IntType, AnalogType, or sugar type for a single bit (Clock, Reset, etc) then return the...
This graph tracks modules and where they are instantiated.
FieldRef getFieldRefFromValue(Value value, bool lookThroughCasts=false)
Get the FieldRef from a value.
hw::InnerRefAttr getInnerRefTo(const hw::InnerSymTarget &target, GetNamespaceCallback getNamespace)
Obtain an inner reference to the target (operation or port), adding an inner symbol as necessary.
llvm::raw_ostream & operator<<(llvm::raw_ostream &os, const InstanceInfo::LatticeValue &value)
std::pair< std::string, bool > getFieldName(const FieldRef &fieldRef, bool nameSafe=false)
Get a string identifier representing the FieldRef.
std::unique_ptr< mlir::Pass > createLowerXMRPass()
std::optional< int64_t > getBitWidth(FIRRTLBaseType type, bool ignoreFlip=false)
IntegerAttr getIntZerosAttr(Type type)
Utility for generating a constant zero attribute.
The InstanceGraph op interface, see InstanceGraphInterface.td for more details.
The namespace of a CircuitOp, generally inhabited by modules.