20 #include "mlir/IR/BuiltinOps.h"
21 #include "mlir/IR/ImplicitLocOpBuilder.h"
22 #include "llvm/ADT/BitVector.h"
23 #include "llvm/ADT/DenseMap.h"
24 #include "llvm/ADT/EquivalenceClasses.h"
25 #include "llvm/ADT/PostOrderIterator.h"
26 #include "llvm/Support/Debug.h"
28 #define DEBUG_TYPE "firrtl-lower-xmr"
30 using namespace circt;
31 using namespace firrtl;
32 using hw::InnerRefAttr;
54 using NextNodeOnPath = std::optional<size_t>;
55 using SymOrIndexOp = PointerUnion<Attribute, Operation *>;
59 [[maybe_unused]] llvm::raw_ostream &
operator<<(llvm::raw_ostream &os,
60 const XMRNode &node) {
62 if (
auto attr = dyn_cast<Attribute>(node.info))
63 os <<
"path=" << attr;
65 auto subOp = cast<RefSubOp>(cast<Operation *>(node.info));
66 os <<
"index=" << subOp.getIndex() <<
" (-> " << subOp.getType() <<
")";
68 os <<
", next=" << node.next <<
")";
79 circuitNamespace = &ns;
81 llvm::EquivalenceClasses<Value, ValueComparator> eq;
82 dataFlowClasses = &eq;
85 SmallVector<RefResolveOp> resolveOps;
86 SmallVector<RefSubOp> indexingOps;
87 SmallVector<Operation *> forceAndReleaseOps;
90 auto transferFunc = [&](Operation &op) -> LogicalResult {
91 return TypeSwitch<Operation *, LogicalResult>(&op)
92 .Case<RefSendOp>([&](RefSendOp send) {
95 Value xmrDef = send.getBase();
96 if (isZeroWidth(send.getType().getType())) {
101 if (
auto verbExpr = xmrDef.getDefiningOp<VerbatimExprOp>())
102 if (verbExpr.getSymbolsAttr().empty() && verbExpr->hasOneUse()) {
107 auto inRef = InnerRefAttr();
108 auto ind = addReachingSendsEntry(send.getResult(), inRef);
109 xmrPathSuffix[ind] = verbExpr.getText();
110 markForRemoval(verbExpr);
111 markForRemoval(send);
118 ImplicitLocOpBuilder b(xmrDef.getLoc(), &getContext());
119 b.setInsertionPointAfterValue(xmrDef);
120 SmallString<32> opName;
121 auto nameKind = NameKindEnum::DroppableName;
127 opName = name +
"_probe";
128 nameKind = NameKindEnum::InterestingName;
129 }
else if (
auto *xmrDefOp = xmrDef.getDefiningOp()) {
132 if (
auto name = xmrDefOp->getAttrOfType<StringAttr>(
"name")) {
133 (Twine(name.strref()) +
"_probe").
toVector(opName);
134 nameKind = NameKindEnum::InterestingName;
137 xmrDef = b.create<NodeOp>(xmrDef, opName, nameKind).getResult();
141 addReachingSendsEntry(send.getResult(),
getInnerRefTo(xmrDef));
142 markForRemoval(send);
145 .Case<RWProbeOp>([&](RWProbeOp rwprobe) {
146 if (!isZeroWidth(rwprobe.getType().getType()))
147 addReachingSendsEntry(rwprobe.getResult(), rwprobe.getTarget());
148 markForRemoval(rwprobe);
151 .Case<MemOp>([&](MemOp mem) {
157 for (
const auto &res : llvm::enumerate(mem.getResults()))
158 if (isa<RefType>(mem.getResult(res.index()).getType())) {
160 auto ind = addReachingSendsEntry(res.value(), inRef);
161 xmrPathSuffix[ind] =
"Memory";
164 refPortsToRemoveMap[mem].resize(1);
169 [&](
auto inst) {
return handleInstanceOp(inst, instanceGraph); })
170 .Case<FConnectLike>([&](FConnectLike
connect) {
172 if (!isa<RefType>(
connect.getSrc().getType()))
176 type_cast<RefType>(
connect.getSrc().getType()).getType()))
193 .Case<RefSubOp>([&](RefSubOp op) -> LogicalResult {
195 if (isZeroWidth(op.getType().getType()))
199 indexingOps.push_back(op);
202 .Case<RefResolveOp>([&](RefResolveOp resolve) {
213 markForRemoval(resolve);
214 if (!isZeroWidth(resolve.getType()))
215 dataFlowClasses->unionSets(resolve.getRef(), resolve.getResult());
216 resolveOps.push_back(resolve);
219 .Case<RefCastOp>([&](RefCastOp op) {
221 if (!isZeroWidth(op.getType().getType()))
222 dataFlowClasses->unionSets(op.getInput(), op.getResult());
225 .Case<Forceable>([&](Forceable op) {
227 if (type_isa<RefType>(op.getDataRaw().getType())) {
233 if (!op.isForceable() || op.getDataRef().use_empty() ||
234 isZeroWidth(op.getDataType()))
240 .Case<RefForceOp, RefForceInitialOp, RefReleaseOp,
241 RefReleaseInitialOp>([&](
auto op) {
242 forceAndReleaseOps.push_back(op);
245 .Default([&](
auto) {
return success(); });
248 SmallVector<FModuleOp> publicModules;
251 for (
auto node : llvm::post_order(&instanceGraph)) {
252 auto module = dyn_cast<FModuleOp>(*node->getModule());
256 <<
"Traversing module:" << module.getModuleNameAttr() <<
"\n");
258 if (module.isPublic())
259 publicModules.push_back(module);
261 for (Operation &op : module.getBodyBlock()->getOperations())
262 if (transferFunc(op).failed())
263 return signalPassFailure();
271 while (!indexingOps.empty()) {
273 decltype(indexingOps) worklist;
274 worklist.swap(indexingOps);
276 for (
auto op : worklist) {
278 getRemoteRefSend(op.getInput(),
false);
281 indexingOps.push_back(op);
283 addReachingSendsEntry(op.getResult(), op.getOperation(),
287 if (worklist.size() == indexingOps.size()) {
288 auto op = worklist.front();
289 getRemoteRefSend(op.getInput());
291 "indexing through probe of unknown origin (input probe?)")
292 .attachNote(op.getInput().getLoc())
293 .append(
"indexing through this reference");
294 return signalPassFailure();
299 size_t numPorts = module.getNumPorts();
300 for (
size_t portNum = 0; portNum < numPorts; ++portNum)
301 if (isa<RefType>(module.getPortType(portNum))) {
302 setPortToRemove(module, portNum, numPorts);
307 for (
auto I = dataFlowClasses->begin(), E = dataFlowClasses->end();
312 llvm::interleave(llvm::make_range(dataFlowClasses->member_begin(I),
313 dataFlowClasses->member_end()),
315 llvm::dbgs() <<
"\n dataflow at leader::" << I->getData() <<
"\n =>";
316 auto iter = dataflowAt.find(I->getData());
317 if (iter != dataflowAt.end()) {
318 for (auto init = refSendPathList[iter->getSecond()]; init.next;
319 init = refSendPathList[*init.next])
320 llvm::dbgs() <<
"\n " << init;
325 for (
auto refResolve : resolveOps)
326 if (handleRefResolve(refResolve).failed())
327 return signalPassFailure();
328 for (
auto *op : forceAndReleaseOps)
329 if (failed(handleForceReleaseOp(op)))
330 return signalPassFailure();
331 for (
auto module : publicModules) {
332 if (failed(handlePublicModuleRefPorts(module)))
333 return signalPassFailure();
338 moduleNamespaces.clear();
339 visitedModules.clear();
341 refSendPathList.clear();
342 dataFlowClasses =
nullptr;
343 refPortsToRemoveMap.clear();
345 xmrPathSuffix.clear();
346 circuitNamespace =
nullptr;
348 pathInsertPoint = {};
353 auto modName = mod.getModuleName();
354 auto circuitName = getOperation().getName();
355 if (
auto ext = dyn_cast<FExtModuleOp>(*mod)) {
357 if (
auto defname = ext.getDefname(); defname && !defname->empty())
360 circuitName = modName;
362 (Twine(
"ref_") + circuitName +
"_" + modName).
toVector(prefix);
368 const Twine &prefix,
bool backTick =
false) {
369 return StringAttr::get(&getContext(), Twine(backTick ?
"`" :
"") + prefix +
370 "_" + mod.getPortName(portIndex));
375 mlir::FlatSymbolRefAttr &ref,
376 SmallString<128> &stringLeaf) {
377 assert(stringLeaf.empty());
379 auto remoteOpPath = getRemoteRefSend(refVal);
382 SmallVector<Attribute> refSendPath;
383 SmallVector<RefSubOp> indexing;
385 while (remoteOpPath) {
386 lastIndex = *remoteOpPath;
387 auto entr = refSendPathList[*remoteOpPath];
388 TypeSwitch<XMRNode::SymOrIndexOp>(entr.info)
389 .Case<Attribute>([&](
auto attr) {
393 refSendPath.push_back(attr);
396 [&](
auto *op) { indexing.push_back(cast<RefSubOp>(op)); });
397 remoteOpPath = entr.next;
399 auto iter = xmrPathSuffix.find(lastIndex);
403 if (iter != xmrPathSuffix.end()) {
404 if (!refSendPath.empty())
405 stringLeaf.append(
".");
406 stringLeaf.append(iter->getSecond());
409 assert(!(refSendPath.empty() && stringLeaf.empty()) &&
410 "nothing to index through");
423 for (
auto subOp : llvm::reverse(indexing)) {
424 TypeSwitch<FIRRTLBaseType>(subOp.getInput().getType().getType())
425 .Case<FVectorType, OpenVectorType>([&](
auto vecType) {
426 (Twine(
"[") + Twine(subOp.getIndex()) +
"]").
toVector(stringLeaf);
428 .Case<BundleType, OpenBundleType>([&](
auto bundleType) {
429 auto fieldName = bundleType.getElementName(subOp.getIndex());
430 stringLeaf.append({
".", fieldName});
434 if (!refSendPath.empty())
445 FlatSymbolRefAttr &ref, StringAttr &xmrAttr) {
446 auto remoteOpPath = getRemoteRefSend(refVal);
450 SmallString<128> xmrString;
451 if (failed(resolveReferencePath(refVal,
builder, ref, xmrString)))
454 xmrString.empty() ? StringAttr{} :
builder.getStringAttr(xmrString);
461 return TypeSwitch<Operation *, LogicalResult>(op)
462 .Case<RefForceOp, RefForceInitialOp, RefReleaseOp, RefReleaseInitialOp>(
465 auto destType = op.getDest().getType();
466 if (isZeroWidth(destType.getType())) {
471 ImplicitLocOpBuilder
builder(op.getLoc(), op);
472 FlatSymbolRefAttr ref;
474 if (failed(resolveReference(op.getDest(),
builder, ref, str)))
477 Value xmr =
builder.create<XMRRefOp>(destType, ref, str);
478 op.getDestMutable().assign(xmr);
481 .Default([](
auto *op) {
482 return op->emitError(
"unexpected operation kind");
489 if (resWidth.has_value() && *resWidth == 0) {
491 ImplicitLocOpBuilder
builder(resolve.getLoc(), resolve);
493 auto zeroC =
builder.createOrFold<BitCastOp>(
494 resolve.getType(),
builder.create<ConstantOp>(
496 resolve.getResult().replaceAllUsesWith(zeroC);
500 FlatSymbolRefAttr ref;
502 ImplicitLocOpBuilder
builder(resolve.getLoc(), resolve);
503 if (failed(resolveReference(resolve.getRef(),
builder, ref, str)))
506 Value result =
builder.create<XMRDerefOp>(resolve.getType(), ref, str);
507 resolve.getResult().replaceAllUsesWith(result);
512 if (refPortsToRemoveMap[op].size() < numPorts)
513 refPortsToRemoveMap[op].resize(numPorts);
514 refPortsToRemoveMap[op].set(index);
521 if (
auto extRefMod = dyn_cast<FExtModuleOp>(mod)) {
525 auto internalPaths = extRefMod.getInternalPaths();
526 auto numPorts = inst.getNumResults();
527 SmallString<128> circuitRefPrefix;
530 auto getPath = [&](
size_t portNo) {
534 cast<InternalPathAttr>(internalPaths->getValue()[portNo])
540 if (circuitRefPrefix.empty())
541 getRefABIPrefix(extRefMod, circuitRefPrefix);
543 return getRefABIMacroForPort(extRefMod, portNo, circuitRefPrefix,
true);
546 for (
const auto &res : llvm::enumerate(inst.getResults())) {
547 if (!isa<RefType>(inst.getResult(res.index()).getType()))
551 auto ind = addReachingSendsEntry(res.value(), inRef);
553 xmrPathSuffix[ind] = getPath(res.index());
555 setPortToRemove(inst, res.index(), numPorts);
556 setPortToRemove(extRefMod, res.index(), numPorts);
560 auto refMod = dyn_cast<FModuleOp>(mod);
561 bool multiplyInstantiated = !visitedModules.insert(refMod).second;
562 for (
size_t portNum = 0, numPorts = inst.getNumResults();
563 portNum < numPorts; ++portNum) {
564 auto instanceResult = inst.getResult(portNum);
565 if (!isa<RefType>(instanceResult.getType()))
568 return inst.emitOpError(
"cannot lower ext modules with RefType ports");
570 setPortToRemove(inst, portNum, numPorts);
572 if (instanceResult.use_empty() ||
573 isZeroWidth(type_cast<RefType>(instanceResult.getType()).getType()))
575 auto refModuleArg = refMod.getArgument(portNum);
580 auto remoteOpPath = getRemoteRefSend(refModuleArg);
592 if (multiplyInstantiated)
593 return refMod.emitOpError(
594 "multiply instantiated module with input RefType port '")
595 << refMod.getPortName(portNum) <<
"'";
596 dataFlowClasses->unionSets(
597 dataFlowClasses->getOrInsertLeaderValue(refModuleArg),
598 dataFlowClasses->getOrInsertLeaderValue(instanceResult));
605 auto builder = ImplicitLocOpBuilder::atBlockBegin(
606 module.getLoc(), getOperation().getBodyBlock());
608 SmallString<128> circuitRefPrefix;
609 for (
size_t portIndex = 0, numPorts = module.getNumPorts();
610 portIndex != numPorts; ++portIndex) {
611 auto refType = type_dyn_cast<RefType>(module.getPortType(portIndex));
612 if (!refType || isZeroWidth(refType.getType()) ||
616 module.getArgument(portIndex).cast<mlir::TypedValue<RefType>>();
618 mlir::FlatSymbolRefAttr ref;
619 SmallString<128> stringLeaf;
620 if (failed(resolveReferencePath(portValue,
builder, ref, stringLeaf)))
623 SmallString<128> formatString;
625 formatString +=
"{{0}}";
626 formatString += stringLeaf;
630 if (circuitRefPrefix.empty())
631 getRefABIPrefix(module, circuitRefPrefix);
633 getRefABIMacroForPort(module, portIndex, circuitRefPrefix);
634 builder.create<sv::MacroDeclOp>(macroName, ArrayAttr(), StringAttr());
636 auto macroDefOp =
builder.create<sv::MacroDefOp>(
638 builder.getStringAttr(formatString),
639 builder.getArrayAttr(ref ? ref : ArrayRef<Attribute>{}));
643 macroDefOp->setAttr(
"output_file",
644 hw::OutputFileAttr::getFromFilename(
645 &getContext(), circuitRefPrefix +
".sv"));
653 return moduleNamespaces.try_emplace(module, module).first->second;
657 if (
auto arg = dyn_cast<BlockArgument>(val))
659 cast<FModuleLike>(arg.getParentBlock()->getParentOp()),
661 [&](FModuleLike mod) -> hw::InnerSymbolNamespace & {
662 return getModuleNamespace(mod);
669 [&](FModuleLike mod) -> hw::InnerSymbolNamespace & {
670 return getModuleNamespace(mod);
677 bool errorIfNotFound =
true) {
678 auto iter = dataflowAt.find(dataFlowClasses->getOrInsertLeaderValue(val));
679 if (iter != dataflowAt.end())
680 return iter->getSecond();
681 if (!errorIfNotFound)
685 if (BlockArgument arg = dyn_cast<BlockArgument>(val))
686 arg.getOwner()->getParentOp()->emitError(
687 "reference dataflow cannot be traced back to the remote read op "
689 << dyn_cast<FModuleOp>(arg.getOwner()->getParentOp())
690 .getPortName(arg.getArgNumber())
693 val.getDefiningOp()->emitOpError(
694 "reference dataflow cannot be traced back to the remote read op");
701 std::optional<size_t> continueFrom = std::nullopt) {
702 auto leader = dataFlowClasses->getOrInsertLeaderValue(atRefVal);
703 auto indx = refSendPathList.size();
704 dataflowAt[leader] = indx;
705 refSendPathList.push_back({info, continueFrom});
713 for (Operation *op : llvm::reverse(opsToRemove))
715 for (
auto iter : refPortsToRemoveMap)
716 if (
auto mod = dyn_cast<FModuleOp>(iter.getFirst()))
717 mod.erasePorts(iter.getSecond());
718 else if (
auto mod = dyn_cast<FExtModuleOp>(iter.getFirst()))
719 mod.erasePorts(iter.getSecond());
720 else if (
auto inst = dyn_cast<InstanceOp>(iter.getFirst())) {
721 ImplicitLocOpBuilder b(inst.getLoc(), inst);
722 inst.erasePorts(b, iter.getSecond());
724 }
else if (
auto mem = dyn_cast<MemOp>(iter.getFirst())) {
726 ImplicitLocOpBuilder
builder(mem.getLoc(), mem);
727 SmallVector<Attribute, 4> resultNames;
728 SmallVector<Type, 4> resultTypes;
729 SmallVector<Attribute, 4> portAnnotations;
730 SmallVector<Value, 4> oldResults;
731 for (
const auto &res : llvm::enumerate(mem.getResults())) {
732 if (isa<RefType>(mem.getResult(res.index()).getType()))
734 resultNames.push_back(mem.getPortName(res.index()));
735 resultTypes.push_back(res.value().getType());
736 portAnnotations.push_back(mem.getPortAnnotation(res.index()));
737 oldResults.push_back(res.value());
739 auto newMem =
builder.create<MemOp>(
740 resultTypes, mem.getReadLatency(), mem.getWriteLatency(),
741 mem.getDepth(), RUWAttr::Undefined,
742 builder.getArrayAttr(resultNames), mem.getNameAttr(),
743 mem.getNameKind(), mem.getAnnotations(),
744 builder.getArrayAttr(portAnnotations), mem.getInnerSymAttr(),
745 mem.getInitAttr(), mem.getPrefixAttr());
746 for (
const auto &res : llvm::enumerate(oldResults))
747 res.value().replaceAllUsesWith(newMem.getResult(res.index()));
751 refPortsToRemoveMap.clear();
753 refSendPathList.clear();
761 ImplicitLocOpBuilder &
builder) {
762 assert(pathArray && !pathArray.empty());
764 auto pathIter = pathCache.find(pathArray);
765 if (pathIter != pathCache.end())
766 return pathIter->second;
769 OpBuilder::InsertionGuard guard(
builder);
773 if (pathInsertPoint.isSet())
774 builder.restoreInsertionPoint(pathInsertPoint);
776 builder.setInsertionPointToStart(getOperation().getBodyBlock());
779 hw::HierPathOp path =
782 builder.create<hw::HierPathOp>(
783 circuitNamespace->newName(
"xmrPath"), pathArray)})
785 path.setVisibility(SymbolTable::Visibility::Private);
789 pathInsertPoint =
builder.saveInsertionPoint();
819 return lhs.getImpl() < rhs.getImpl();
840 OpBuilder::InsertPoint pathInsertPoint = {};
844 return std::make_unique<LowerXMRPass>();
assert(baseType &&"element must be base type")
static std::vector< mlir::Value > toVector(mlir::ValueRange range)
hw::InnerSymbolNamespace & getModuleNamespace(FModuleLike module)
Get the cached namespace for a module.
LogicalResult resolveReference(mlir::TypedValue< RefType > refVal, ImplicitLocOpBuilder &builder, FlatSymbolRefAttr &ref, StringAttr &xmrAttr)
DenseMap< Operation *, hw::InnerSymbolNamespace > moduleNamespaces
Cached module namespaces.
DenseMap< size_t, SmallString< 128 > > xmrPathSuffix
Record the internal path to an external module or a memory.
InnerRefAttr getInnerRefTo(Value val)
size_t addReachingSendsEntry(Value atRefVal, XMRNode::SymOrIndexOp info, std::optional< size_t > continueFrom=std::nullopt)
hw::HierPathOp getOrCreatePath(ArrayAttr pathArray, ImplicitLocOpBuilder &builder)
Return a HierPathOp for the provided pathArray.
LogicalResult resolveReferencePath(mlir::TypedValue< RefType > refVal, ImplicitLocOpBuilder builder, mlir::FlatSymbolRefAttr &ref, SmallString< 128 > &stringLeaf)
DenseMap< Value, size_t > dataflowAt
Map of a reference value to an entry into refSendPathList.
void setPortToRemove(Operation *op, size_t index, size_t numPorts)
llvm::EquivalenceClasses< Value, ValueComparator > * dataFlowClasses
void markForRemoval(Operation *op)
LogicalResult handlePublicModuleRefPorts(FModuleOp module)
void getRefABIPrefix(FModuleLike mod, SmallVectorImpl< char > &prefix)
Generate the ABI ref_<circuit>_<module> prefix string into prefix.
void runOnOperation() override
DenseMap< Attribute, hw::HierPathOp > pathCache
A cache of already created HierPathOps.
LogicalResult handleRefResolve(RefResolveOp resolve)
DenseMap< Operation *, llvm::BitVector > refPortsToRemoveMap
SmallVector< XMRNode > refSendPathList
refSendPathList is used to construct a path to the RefSendOp.
LogicalResult handleInstanceOp(InstanceOp inst, InstanceGraph &instanceGraph)
LogicalResult handleForceReleaseOp(Operation *op)
DenseSet< Operation * > visitedModules
InnerRefAttr getInnerRefTo(Operation *op)
StringAttr getRefABIMacroForPort(FModuleLike mod, size_t portIndex, const Twine &prefix, bool backTick=false)
Get full macro name as StringAttr for the specified ref port.
CircuitNamespace * circuitNamespace
bool isZeroWidth(FIRRTLBaseType t)
std::optional< size_t > getRemoteRefSend(Value val, bool errorIfNotFound=true)
SmallVector< Operation * > opsToRemove
RefResolve, RefSend, and Connects involving them that will be removed.
int32_t getBitWidthOrSentinel()
If this is an IntType, AnalogType, or sugar type for a single bit (Clock, Reset, etc) then return the...
This graph tracks modules and where they are instantiated.
auto getReferencedModule(InstanceOpInterface op)
Look up the referenced module from an InstanceOp.
def connect(destination, source)
Direction get(bool isOutput)
Returns an output direction if isOutput is true, otherwise returns an input direction.
FieldRef getFieldRefFromValue(Value value, bool lookThroughCasts=false)
Get the FieldRef from a value.
hw::InnerRefAttr getInnerRefTo(const hw::InnerSymTarget &target, GetNamespaceCallback getNamespace)
Obtain an inner reference to the target (operation or port), adding an inner symbol as necessary.
T & operator<<(T &os, FIRVersion version)
std::pair< std::string, bool > getFieldName(const FieldRef &fieldRef, bool nameSafe=false)
Get a string identifier representing the FieldRef.
std::unique_ptr< mlir::Pass > createLowerXMRPass()
std::optional< int64_t > getBitWidth(FIRRTLBaseType type, bool ignoreFlip=false)
IntegerAttr getIntZerosAttr(Type type)
Utility for generating a constant zero attribute.
This file defines an intermediate representation for circuits acting as an abstraction for constraint...
mlir::raw_indented_ostream & dbgs()
llvm::EquivalenceClasses wants comparable elements.
bool operator()(const Value &lhs, const Value &rhs) const
The namespace of a CircuitOp, generally inhabited by modules.