21 #include "mlir/IR/BuiltinOps.h"
22 #include "mlir/IR/ImplicitLocOpBuilder.h"
23 #include "llvm/ADT/BitVector.h"
24 #include "llvm/ADT/DenseMap.h"
25 #include "llvm/ADT/EquivalenceClasses.h"
26 #include "llvm/ADT/PostOrderIterator.h"
27 #include "llvm/Support/Debug.h"
29 #define DEBUG_TYPE "firrtl-lower-xmr"
31 using namespace circt;
32 using namespace firrtl;
33 using hw::InnerRefAttr;
55 using NextNodeOnPath = std::optional<size_t>;
56 using SymOrIndexOp = PointerUnion<Attribute, Operation *>;
60 [[maybe_unused]] llvm::raw_ostream &
operator<<(llvm::raw_ostream &os,
61 const XMRNode &node) {
63 if (
auto attr = dyn_cast<Attribute>(node.info))
64 os <<
"path=" << attr;
66 auto subOp = cast<RefSubOp>(cast<Operation *>(node.info));
67 os <<
"index=" << subOp.getIndex() <<
" (-> " << subOp.getType() <<
")";
69 os <<
", next=" << node.next <<
")";
80 circuitNamespace = &ns;
82 llvm::EquivalenceClasses<Value, ValueComparator> eq;
83 dataFlowClasses = &eq;
86 SmallVector<RefResolveOp> resolveOps;
87 SmallVector<RefSubOp> indexingOps;
88 SmallVector<Operation *> forceAndReleaseOps;
91 auto transferFunc = [&](Operation &op) -> LogicalResult {
92 return TypeSwitch<Operation *, LogicalResult>(&op)
93 .Case<RefSendOp>([&](RefSendOp send) {
96 Value xmrDef = send.getBase();
97 if (isZeroWidth(send.getType().getType())) {
102 if (
auto verbExpr = xmrDef.getDefiningOp<VerbatimExprOp>())
103 if (verbExpr.getSymbolsAttr().empty() && verbExpr->hasOneUse()) {
108 auto inRef = InnerRefAttr();
109 auto ind = addReachingSendsEntry(send.getResult(), inRef);
110 xmrPathSuffix[ind] = verbExpr.getText();
111 markForRemoval(verbExpr);
112 markForRemoval(send);
119 ImplicitLocOpBuilder b(xmrDef.getLoc(), &getContext());
120 b.setInsertionPointAfterValue(xmrDef);
121 SmallString<32> opName;
122 auto nameKind = NameKindEnum::DroppableName;
128 opName = name +
"_probe";
129 nameKind = NameKindEnum::InterestingName;
130 }
else if (
auto *xmrDefOp = xmrDef.getDefiningOp()) {
133 if (
auto name = xmrDefOp->getAttrOfType<StringAttr>(
"name")) {
134 (Twine(name.strref()) +
"_probe").
toVector(opName);
135 nameKind = NameKindEnum::InterestingName;
138 xmrDef = b.create<NodeOp>(xmrDef, opName, nameKind).getResult();
142 addReachingSendsEntry(send.getResult(),
getInnerRefTo(xmrDef));
143 markForRemoval(send);
146 .Case<RWProbeOp>([&](RWProbeOp rwprobe) {
147 if (!isZeroWidth(rwprobe.getType().getType()))
148 addReachingSendsEntry(rwprobe.getResult(), rwprobe.getTarget());
149 markForRemoval(rwprobe);
152 .Case<MemOp>([&](MemOp mem) {
158 for (
const auto &res : llvm::enumerate(mem.getResults()))
159 if (isa<RefType>(mem.getResult(res.index()).getType())) {
161 auto ind = addReachingSendsEntry(res.value(), inRef);
162 xmrPathSuffix[ind] =
"Memory";
165 refPortsToRemoveMap[mem].resize(1);
170 [&](
auto inst) {
return handleInstanceOp(inst, instanceGraph); })
171 .Case<FConnectLike>([&](FConnectLike
connect) {
173 if (!isa<RefType>(
connect.getSrc().getType()))
177 type_cast<RefType>(
connect.getSrc().getType()).getType()))
194 .Case<RefSubOp>([&](RefSubOp op) -> LogicalResult {
196 if (isZeroWidth(op.getType().getType()))
200 indexingOps.push_back(op);
203 .Case<RefResolveOp>([&](RefResolveOp resolve) {
214 markForRemoval(resolve);
215 if (!isZeroWidth(resolve.getType()))
216 dataFlowClasses->unionSets(resolve.getRef(), resolve.getResult());
217 resolveOps.push_back(resolve);
220 .Case<RefCastOp>([&](RefCastOp op) {
222 if (!isZeroWidth(op.getType().getType()))
223 dataFlowClasses->unionSets(op.getInput(), op.getResult());
226 .Case<Forceable>([&](Forceable op) {
228 if (type_isa<RefType>(op.getDataRaw().getType())) {
234 if (!op.isForceable() || op.getDataRef().use_empty() ||
235 isZeroWidth(op.getDataType()))
241 .Case<RefForceOp, RefForceInitialOp, RefReleaseOp,
242 RefReleaseInitialOp>([&](
auto op) {
243 forceAndReleaseOps.push_back(op);
246 .Default([&](
auto) {
return success(); });
249 SmallVector<FModuleOp> publicModules;
252 for (
auto node : llvm::post_order(&instanceGraph)) {
253 auto module = dyn_cast<FModuleOp>(*node->getModule());
256 LLVM_DEBUG(llvm::dbgs()
257 <<
"Traversing module:" << module.getModuleNameAttr() <<
"\n");
259 if (module.isPublic())
260 publicModules.push_back(module);
262 for (Operation &op : module.getBodyBlock()->getOperations())
263 if (transferFunc(op).failed())
264 return signalPassFailure();
275 while (!indexingOps.empty()) {
277 decltype(indexingOps) worklist;
278 worklist.swap(indexingOps);
280 for (
auto op : worklist) {
282 getRemoteRefSend(op.getInput(),
false);
285 indexingOps.push_back(op);
287 addReachingSendsEntry(op.getResult(), op.getOperation(),
291 if (worklist.size() == indexingOps.size()) {
292 auto op = worklist.front();
293 getRemoteRefSend(op.getInput());
295 "indexing through probe of unknown origin (input probe?)")
296 .attachNote(op.getInput().getLoc())
297 .append(
"indexing through this reference");
298 return signalPassFailure();
303 size_t numPorts = module.getNumPorts();
304 for (
size_t portNum = 0; portNum < numPorts; ++portNum)
305 if (isa<RefType>(module.getPortType(portNum))) {
306 setPortToRemove(module, portNum, numPorts);
311 for (
auto I = dataFlowClasses->begin(), E = dataFlowClasses->end();
316 llvm::interleave(llvm::make_range(dataFlowClasses->member_begin(I),
317 dataFlowClasses->member_end()),
319 llvm::dbgs() <<
"\n dataflow at leader::" << I->getData() <<
"\n =>";
320 auto iter = dataflowAt.find(I->getData());
321 if (iter != dataflowAt.end()) {
322 for (auto init = refSendPathList[iter->getSecond()]; init.next;
323 init = refSendPathList[*init.next])
324 llvm::dbgs() <<
"\n " << init;
326 llvm::dbgs() <<
"\n Done\n";
329 for (
auto refResolve : resolveOps)
330 if (handleRefResolve(refResolve).failed())
331 return signalPassFailure();
332 for (
auto *op : forceAndReleaseOps)
333 if (failed(handleForceReleaseOp(op)))
334 return signalPassFailure();
335 for (
auto module : publicModules) {
336 if (failed(handlePublicModuleRefPorts(module)))
337 return signalPassFailure();
342 moduleNamespaces.clear();
343 visitedModules.clear();
345 refSendPathList.clear();
346 dataFlowClasses =
nullptr;
347 refPortsToRemoveMap.clear();
349 xmrPathSuffix.clear();
350 circuitNamespace =
nullptr;
352 pathInsertPoint = {};
357 auto modName = mod.getModuleName();
358 if (
auto ext = dyn_cast<FExtModuleOp>(*mod)) {
360 if (
auto defname = ext.getDefname(); defname && !defname->empty())
363 (Twine(
"ref_") + modName).
toVector(prefix);
369 const Twine &prefix,
bool backTick =
false) {
370 return StringAttr::get(&getContext(), Twine(backTick ?
"`" :
"") + prefix +
371 "_" + mod.getPortName(portIndex));
376 mlir::FlatSymbolRefAttr &ref,
377 SmallString<128> &stringLeaf) {
378 assert(stringLeaf.empty());
380 auto remoteOpPath = getRemoteRefSend(refVal);
383 SmallVector<Attribute> refSendPath;
384 SmallVector<RefSubOp> indexing;
386 while (remoteOpPath) {
387 lastIndex = *remoteOpPath;
388 auto entr = refSendPathList[*remoteOpPath];
389 TypeSwitch<XMRNode::SymOrIndexOp>(entr.info)
390 .Case<Attribute>([&](
auto attr) {
394 refSendPath.push_back(attr);
397 [&](
auto *op) { indexing.push_back(cast<RefSubOp>(op)); });
398 remoteOpPath = entr.next;
400 auto iter = xmrPathSuffix.find(lastIndex);
404 if (iter != xmrPathSuffix.end()) {
405 if (!refSendPath.empty())
406 stringLeaf.append(
".");
407 stringLeaf.append(iter->getSecond());
410 assert(!(refSendPath.empty() && stringLeaf.empty()) &&
411 "nothing to index through");
424 for (
auto subOp : llvm::reverse(indexing)) {
425 TypeSwitch<FIRRTLBaseType>(subOp.getInput().getType().getType())
426 .Case<FVectorType, OpenVectorType>([&](
auto vecType) {
427 (Twine(
"[") + Twine(subOp.getIndex()) +
"]").
toVector(stringLeaf);
429 .Case<BundleType, OpenBundleType>([&](
auto bundleType) {
430 auto fieldName = bundleType.getElementName(subOp.getIndex());
431 stringLeaf.append({
".", fieldName});
435 if (!refSendPath.empty())
446 FlatSymbolRefAttr &ref, StringAttr &xmrAttr) {
447 auto remoteOpPath = getRemoteRefSend(refVal);
451 SmallString<128> xmrString;
452 if (failed(resolveReferencePath(refVal,
builder, ref, xmrString)))
455 xmrString.empty() ? StringAttr{} :
builder.getStringAttr(xmrString);
462 return TypeSwitch<Operation *, LogicalResult>(op)
463 .Case<RefForceOp, RefForceInitialOp, RefReleaseOp, RefReleaseInitialOp>(
466 auto destType = op.getDest().getType();
467 if (isZeroWidth(destType.getType())) {
472 ImplicitLocOpBuilder
builder(op.getLoc(), op);
473 FlatSymbolRefAttr ref;
475 if (failed(resolveReference(op.getDest(),
builder, ref, str)))
478 Value xmr =
builder.create<XMRRefOp>(destType, ref, str);
479 op.getDestMutable().assign(xmr);
482 .Default([](
auto *op) {
483 return op->emitError(
"unexpected operation kind");
490 if (resWidth.has_value() && *resWidth == 0) {
492 ImplicitLocOpBuilder
builder(resolve.getLoc(), resolve);
494 auto zeroC =
builder.createOrFold<BitCastOp>(
495 resolve.getType(),
builder.create<ConstantOp>(
497 resolve.getResult().replaceAllUsesWith(zeroC);
501 FlatSymbolRefAttr ref;
503 ImplicitLocOpBuilder
builder(resolve.getLoc(), resolve);
504 if (failed(resolveReference(resolve.getRef(),
builder, ref, str)))
507 Value result =
builder.create<XMRDerefOp>(resolve.getType(), ref, str);
508 resolve.getResult().replaceAllUsesWith(result);
513 if (refPortsToRemoveMap[op].size() < numPorts)
514 refPortsToRemoveMap[op].resize(numPorts);
515 refPortsToRemoveMap[op].set(index);
521 Operation *mod = inst.getReferencedModule(instanceGraph);
522 if (
auto extRefMod = dyn_cast<FExtModuleOp>(mod)) {
526 auto internalPaths = extRefMod.getInternalPaths();
527 auto numPorts = inst.getNumResults();
528 SmallString<128> circuitRefPrefix;
531 auto getPath = [&](
size_t portNo) {
535 cast<InternalPathAttr>(internalPaths->getValue()[portNo])
541 if (circuitRefPrefix.empty())
542 getRefABIPrefix(extRefMod, circuitRefPrefix);
544 return getRefABIMacroForPort(extRefMod, portNo, circuitRefPrefix,
true);
547 for (
const auto &res : llvm::enumerate(inst.getResults())) {
548 if (!isa<RefType>(inst.getResult(res.index()).getType()))
552 auto ind = addReachingSendsEntry(res.value(), inRef);
554 xmrPathSuffix[ind] = getPath(res.index());
556 setPortToRemove(inst, res.index(), numPorts);
557 setPortToRemove(extRefMod, res.index(), numPorts);
561 auto refMod = dyn_cast<FModuleOp>(mod);
562 bool multiplyInstantiated = !visitedModules.insert(refMod).second;
563 for (
size_t portNum = 0, numPorts = inst.getNumResults();
564 portNum < numPorts; ++portNum) {
565 auto instanceResult = inst.getResult(portNum);
566 if (!isa<RefType>(instanceResult.getType()))
569 return inst.emitOpError(
"cannot lower ext modules with RefType ports");
571 setPortToRemove(inst, portNum, numPorts);
573 if (instanceResult.use_empty() ||
574 isZeroWidth(type_cast<RefType>(instanceResult.getType()).getType()))
576 auto refModuleArg = refMod.getArgument(portNum);
581 auto remoteOpPath = getRemoteRefSend(refModuleArg);
593 if (multiplyInstantiated)
594 return refMod.emitOpError(
595 "multiply instantiated module with input RefType port '")
596 << refMod.getPortName(portNum) <<
"'";
597 dataFlowClasses->unionSets(
598 dataFlowClasses->getOrInsertLeaderValue(refModuleArg),
599 dataFlowClasses->getOrInsertLeaderValue(instanceResult));
606 auto *body = getOperation().getBodyBlock();
609 SmallString<128> circuitRefPrefix;
610 SmallVector<std::tuple<StringAttr, StringAttr, ArrayAttr>> ports;
612 ImplicitLocOpBuilder::atBlockBegin(module.getLoc(), body);
613 for (
size_t portIndex = 0, numPorts = module.getNumPorts();
614 portIndex != numPorts; ++portIndex) {
615 auto refType = type_dyn_cast<RefType>(module.getPortType(portIndex));
616 if (!refType || isZeroWidth(refType.getType()) ||
620 module.getArgument(portIndex).cast<mlir::TypedValue<RefType>>();
621 mlir::FlatSymbolRefAttr ref;
622 SmallString<128> stringLeaf;
623 if (failed(resolveReferencePath(portValue, declBuilder, ref, stringLeaf)))
626 SmallString<128> formatString;
628 formatString +=
"{{0}}";
629 formatString += stringLeaf;
633 if (circuitRefPrefix.empty())
634 getRefABIPrefix(module, circuitRefPrefix);
636 getRefABIMacroForPort(module, portIndex, circuitRefPrefix);
637 declBuilder.create<sv::MacroDeclOp>(macroName, ArrayAttr(), StringAttr());
638 ports.emplace_back(macroName, declBuilder.getStringAttr(formatString),
639 ref ? declBuilder.getArrayAttr({ref}) : ArrayAttr{});
648 auto fileBuilder = ImplicitLocOpBuilder(module.getLoc(), module);
649 fileBuilder.create<emit::FileOp>(circuitRefPrefix +
".sv", [&] {
650 for (
auto [macroName, formatString, symbols] : ports) {
652 formatString, symbols);
661 return moduleNamespaces.try_emplace(module, module).first->second;
665 if (
auto arg = dyn_cast<BlockArgument>(val))
667 cast<FModuleLike>(arg.getParentBlock()->getParentOp()),
669 [&](FModuleLike mod) -> hw::InnerSymbolNamespace & {
670 return getModuleNamespace(mod);
677 [&](FModuleLike mod) -> hw::InnerSymbolNamespace & {
678 return getModuleNamespace(mod);
685 bool errorIfNotFound =
true) {
686 auto iter = dataflowAt.find(dataFlowClasses->getOrInsertLeaderValue(val));
687 if (iter != dataflowAt.end())
688 return iter->getSecond();
689 if (!errorIfNotFound)
693 if (BlockArgument arg = dyn_cast<BlockArgument>(val))
694 arg.getOwner()->getParentOp()->emitError(
695 "reference dataflow cannot be traced back to the remote read op "
697 << dyn_cast<FModuleOp>(arg.getOwner()->getParentOp())
698 .getPortName(arg.getArgNumber())
701 val.getDefiningOp()->emitOpError(
702 "reference dataflow cannot be traced back to the remote read op");
709 std::optional<size_t> continueFrom = std::nullopt) {
710 auto leader = dataFlowClasses->getOrInsertLeaderValue(atRefVal);
711 auto indx = refSendPathList.size();
712 dataflowAt[leader] = indx;
713 refSendPathList.push_back({info, continueFrom});
721 for (Operation *op : llvm::reverse(opsToRemove))
723 for (
auto iter : refPortsToRemoveMap)
724 if (
auto mod = dyn_cast<FModuleOp>(iter.getFirst()))
725 mod.erasePorts(iter.getSecond());
726 else if (
auto mod = dyn_cast<FExtModuleOp>(iter.getFirst()))
727 mod.erasePorts(iter.getSecond());
728 else if (
auto inst = dyn_cast<InstanceOp>(iter.getFirst())) {
729 ImplicitLocOpBuilder b(inst.getLoc(), inst);
730 inst.erasePorts(b, iter.getSecond());
732 }
else if (
auto mem = dyn_cast<MemOp>(iter.getFirst())) {
734 ImplicitLocOpBuilder
builder(mem.getLoc(), mem);
735 SmallVector<Attribute, 4> resultNames;
736 SmallVector<Type, 4> resultTypes;
737 SmallVector<Attribute, 4> portAnnotations;
738 SmallVector<Value, 4> oldResults;
739 for (
const auto &res : llvm::enumerate(mem.getResults())) {
740 if (isa<RefType>(mem.getResult(res.index()).getType()))
742 resultNames.push_back(mem.getPortName(res.index()));
743 resultTypes.push_back(res.value().getType());
744 portAnnotations.push_back(mem.getPortAnnotation(res.index()));
745 oldResults.push_back(res.value());
747 auto newMem =
builder.create<MemOp>(
748 resultTypes, mem.getReadLatency(), mem.getWriteLatency(),
749 mem.getDepth(), RUWAttr::Undefined,
750 builder.getArrayAttr(resultNames), mem.getNameAttr(),
751 mem.getNameKind(), mem.getAnnotations(),
752 builder.getArrayAttr(portAnnotations), mem.getInnerSymAttr(),
753 mem.getInitAttr(), mem.getPrefixAttr());
754 for (
const auto &res : llvm::enumerate(oldResults))
755 res.value().replaceAllUsesWith(newMem.getResult(res.index()));
759 refPortsToRemoveMap.clear();
761 refSendPathList.clear();
769 ImplicitLocOpBuilder &
builder) {
770 assert(pathArray && !pathArray.empty());
772 auto pathIter = pathCache.find(pathArray);
773 if (pathIter != pathCache.end())
774 return pathIter->second;
777 OpBuilder::InsertionGuard guard(
builder);
781 if (pathInsertPoint.isSet())
782 builder.restoreInsertionPoint(pathInsertPoint);
784 builder.setInsertionPointToStart(getOperation().getBodyBlock());
787 hw::HierPathOp path =
790 builder.create<hw::HierPathOp>(
791 circuitNamespace->newName(
"xmrPath"), pathArray)})
793 path.setVisibility(SymbolTable::Visibility::Private);
797 pathInsertPoint =
builder.saveInsertionPoint();
827 return lhs.getImpl() < rhs.getImpl();
848 OpBuilder::InsertPoint pathInsertPoint = {};
852 return std::make_unique<LowerXMRPass>();
assert(baseType &&"element must be base type")
static std::vector< mlir::Value > toVector(mlir::ValueRange range)
hw::InnerSymbolNamespace & getModuleNamespace(FModuleLike module)
Get the cached namespace for a module.
LogicalResult resolveReference(mlir::TypedValue< RefType > refVal, ImplicitLocOpBuilder &builder, FlatSymbolRefAttr &ref, StringAttr &xmrAttr)
DenseMap< Operation *, hw::InnerSymbolNamespace > moduleNamespaces
Cached module namespaces.
DenseMap< size_t, SmallString< 128 > > xmrPathSuffix
Record the internal path to an external module or a memory.
InnerRefAttr getInnerRefTo(Value val)
size_t addReachingSendsEntry(Value atRefVal, XMRNode::SymOrIndexOp info, std::optional< size_t > continueFrom=std::nullopt)
hw::HierPathOp getOrCreatePath(ArrayAttr pathArray, ImplicitLocOpBuilder &builder)
Return a HierPathOp for the provided pathArray.
LogicalResult resolveReferencePath(mlir::TypedValue< RefType > refVal, ImplicitLocOpBuilder builder, mlir::FlatSymbolRefAttr &ref, SmallString< 128 > &stringLeaf)
DenseMap< Value, size_t > dataflowAt
Map of a reference value to an entry into refSendPathList.
void setPortToRemove(Operation *op, size_t index, size_t numPorts)
llvm::EquivalenceClasses< Value, ValueComparator > * dataFlowClasses
void markForRemoval(Operation *op)
LogicalResult handlePublicModuleRefPorts(FModuleOp module)
void getRefABIPrefix(FModuleLike mod, SmallVectorImpl< char > &prefix)
Generate the ABI ref_<module> prefix string into prefix.
void runOnOperation() override
DenseMap< Attribute, hw::HierPathOp > pathCache
A cache of already created HierPathOps.
LogicalResult handleRefResolve(RefResolveOp resolve)
DenseMap< Operation *, llvm::BitVector > refPortsToRemoveMap
SmallVector< XMRNode > refSendPathList
refSendPathList is used to construct a path to the RefSendOp.
LogicalResult handleInstanceOp(InstanceOp inst, InstanceGraph &instanceGraph)
LogicalResult handleForceReleaseOp(Operation *op)
DenseSet< Operation * > visitedModules
InnerRefAttr getInnerRefTo(Operation *op)
StringAttr getRefABIMacroForPort(FModuleLike mod, size_t portIndex, const Twine &prefix, bool backTick=false)
Get full macro name as StringAttr for the specified ref port.
CircuitNamespace * circuitNamespace
bool isZeroWidth(FIRRTLBaseType t)
std::optional< size_t > getRemoteRefSend(Value val, bool errorIfNotFound=true)
SmallVector< Operation * > opsToRemove
RefResolve, RefSend, and Connects involving them that will be removed.
int32_t getBitWidthOrSentinel()
If this is an IntType, AnalogType, or sugar type for a single bit (Clock, Reset, etc) then return the...
This graph tracks modules and where they are instantiated.
def connect(destination, source)
Direction get(bool isOutput)
Returns an output direction if isOutput is true, otherwise returns an input direction.
FieldRef getFieldRefFromValue(Value value, bool lookThroughCasts=false)
Get the FieldRef from a value.
hw::InnerRefAttr getInnerRefTo(const hw::InnerSymTarget &target, GetNamespaceCallback getNamespace)
Obtain an inner reference to the target (operation or port), adding an inner symbol as necessary.
T & operator<<(T &os, FIRVersion version)
std::pair< std::string, bool > getFieldName(const FieldRef &fieldRef, bool nameSafe=false)
Get a string identifier representing the FieldRef.
std::unique_ptr< mlir::Pass > createLowerXMRPass()
std::optional< int64_t > getBitWidth(FIRRTLBaseType type, bool ignoreFlip=false)
IntegerAttr getIntZerosAttr(Type type)
Utility for generating a constant zero attribute.
The InstanceGraph op interface, see InstanceGraphInterface.td for more details.
llvm::EquivalenceClasses wants comparable elements.
bool operator()(const Value &lhs, const Value &rhs) const
The namespace of a CircuitOp, generally inhabited by modules.