21 #include "mlir/IR/Dominance.h"
22 #include "mlir/Pass/Pass.h"
23 #include "llvm/ADT/DepthFirstIterator.h"
24 #include "llvm/ADT/STLExtras.h"
25 #include "llvm/ADT/STLFunctionalExtras.h"
26 #include "llvm/ADT/SmallPtrSet.h"
27 #include "llvm/Support/Parallel.h"
33 #define GEN_PASS_DEF_LOWERMEMORY
34 #include "circt/Dialect/FIRRTL/Passes.h.inc"
38 using namespace circt;
39 using namespace firrtl;
43 size_t numReadPorts = 0;
44 size_t numWritePorts = 0;
45 size_t numReadWritePorts = 0;
47 SmallVector<int32_t> writeClockIDs;
49 for (
size_t i = 0, e = op.getNumResults(); i != e; ++i) {
50 auto portKind = op.getPortKind(i);
51 if (portKind == MemOp::PortKind::Read)
53 else if (portKind == MemOp::PortKind::Write) {
54 for (
auto *a : op.getResult(i).getUsers()) {
55 auto subfield = dyn_cast<SubfieldOp>(a);
56 if (!subfield || subfield.getFieldIndex() != 2)
58 auto clockPort = a->getResult(0);
59 for (
auto *b : clockPort.getUsers()) {
60 if (
auto connect = dyn_cast<FConnectLike>(b)) {
61 if (
connect.getDest() == clockPort) {
63 clockToLeader.insert({
connect.getSrc(), numWritePorts});
65 writeClockIDs.push_back(numWritePorts);
67 writeClockIDs.push_back(result.first->second);
79 auto width = op.getDataType().getBitWidthOrSentinel();
81 op.emitError(
"'firrtl.mem' should have simple type and known width");
92 *seq::symbolizeRUW(
unsigned(op.getRuw())),
103 struct LowerMemoryPass
104 :
public circt::firrtl::impl::LowerMemoryBase<LowerMemoryPass> {
107 hw::InnerSymbolNamespace &getModuleNamespace(FModuleLike moduleOp) {
108 return moduleNamespaces.try_emplace(moduleOp, moduleOp).first->second;
111 SmallVector<PortInfo> getMemoryModulePorts(
const FirMemory &mem);
112 FMemModuleOp emitMemoryModule(MemOp op,
const FirMemory &summary,
113 const SmallVectorImpl<PortInfo> &ports);
114 FMemModuleOp getOrCreateMemModule(MemOp op,
const FirMemory &summary,
115 const SmallVectorImpl<PortInfo> &ports,
117 FModuleOp createWrapperModule(MemOp op,
const FirMemory &summary,
119 InstanceOp emitMemoryInstance(MemOp op, FModuleOp moduleOp,
121 void lowerMemory(MemOp mem,
const FirMemory &summary,
bool shouldDedup);
122 LogicalResult runOnModule(FModuleOp moduleOp,
bool shouldDedup);
123 void runOnOperation()
override;
126 DenseMap<Operation *, hw::InnerSymbolNamespace> moduleNamespaces;
128 SymbolTable *symbolTable;
132 std::map<FirMemory, FMemModuleOp> memories;
135 SetVector<Operation *> operationsToErase;
139 SmallVector<PortInfo>
140 LowerMemoryPass::getMemoryModulePorts(
const FirMemory &mem) {
141 auto *context = &getContext();
154 SmallVector<PortInfo> ports;
158 {nameAttr, type, direction, hw::InnerSymAttr{}, loc, annotations});
161 auto makePortCommon = [&](StringRef prefix,
size_t idx,
FIRRTLType addrType) {
162 addPort(prefix + Twine(idx) +
"_addr", addrType,
Direction::In);
164 addPort(prefix + Twine(idx) +
"_clk", clockType,
Direction::In);
168 makePortCommon(
"R", i, addrType);
172 makePortCommon(
"RW", i, addrType);
174 addPort(
"RW" + Twine(i) +
"_wdata", dataType,
Direction::In);
178 addPort(
"RW" + Twine(i) +
"_wmask", maskType,
Direction::In);
182 makePortCommon(
"W", i, addrType);
193 LowerMemoryPass::emitMemoryModule(MemOp op,
const FirMemory &mem,
194 const SmallVectorImpl<PortInfo> &ports) {
196 StringRef prefix =
"";
198 prefix = mem.
prefix.getValue();
200 circuitNamespace.newName(prefix + mem.
modName.getValue(),
"ext");
204 OpBuilder b(op->getParentOfType<FModuleOp>());
205 ++numCreatedMemModules;
206 auto moduleOp = b.create<FMemModuleOp>(
210 SymbolTable::setSymbolVisibility(moduleOp, SymbolTable::Visibility::Private);
215 LowerMemoryPass::getOrCreateMemModule(MemOp op,
const FirMemory &summary,
216 const SmallVectorImpl<PortInfo> &ports,
221 auto it = memories.find(summary);
222 if (it != memories.end())
228 auto moduleOp = emitMemoryModule(op, summary, ports);
233 memories[summary] = moduleOp;
238 void LowerMemoryPass::lowerMemory(MemOp mem,
const FirMemory &summary,
240 auto *context = &getContext();
241 auto ports = getMemoryModulePorts(summary);
244 StringRef prefix =
"";
246 prefix = summary.
prefix.getValue();
247 auto newName = circuitNamespace.newName(prefix + mem.getName());
252 OpBuilder b(mem->getParentOfType<FModuleOp>());
253 auto wrapper = b.create<FModuleOp>(
254 mem->getLoc(), wrapperName,
256 SymbolTable::setSymbolVisibility(wrapper, SymbolTable::Visibility::Private);
260 auto memModule = getOrCreateMemModule(mem, summary, ports, shouldDedup);
261 b.setInsertionPointToStart(wrapper.getBodyBlock());
262 auto memInst = b.create<InstanceOp>(
263 mem->getLoc(), memModule, (mem.getName() +
"_ext").str(),
264 mem.getNameKind(), mem.getAnnotations().getValue());
267 for (
auto [dst, src] : llvm::zip(wrapper.getBodyBlock()->getArguments(),
268 memInst.getResults())) {
269 if (wrapper.getPortDirection(dst.getArgNumber()) ==
Direction::Out)
270 b.create<MatchingConnectOp>(mem->getLoc(), dst, src);
272 b.create<MatchingConnectOp>(mem->getLoc(), src, dst);
277 auto inst = emitMemoryInstance(mem, wrapper, summary);
283 auto leafSym = memModule.getModuleNameAttr();
289 bool nlaUpdated =
false;
290 SmallVector<Annotation> newMemModAnnos;
291 OpBuilder nlaBuilder(context);
295 auto nlaSym = anno.
getMember<FlatSymbolRefAttr>(nonlocalAttr);
299 auto newNLAIter = processedNLAs.find(nlaSym.getAttr());
300 StringAttr newNLAName;
301 if (newNLAIter == processedNLAs.end()) {
305 dyn_cast<hw::HierPathOp>(symbolTable->lookup(nlaSym.getAttr()));
306 auto namepath = nla.getNamepath().getValue();
307 SmallVector<Attribute> newNamepath(namepath.begin(), namepath.end());
308 if (!nla.isComponent())
310 getInnerRefTo(inst, [&](
auto mod) -> hw::InnerSymbolNamespace & {
311 return getModuleNamespace(mod);
313 newNamepath.push_back(leafAttr);
315 nlaBuilder.setInsertionPointAfter(nla);
316 auto newNLA = cast<hw::HierPathOp>(nlaBuilder.clone(*nla));
318 context, circuitNamespace.newName(nla.getNameAttr().getValue())));
320 newNLAName = newNLA.getNameAttr();
321 processedNLAs[nlaSym.getAttr()] = newNLAName;
323 newNLAName = newNLAIter->getSecond();
326 newMemModAnnos.push_back(anno);
332 newAnnos.addAnnotations(newMemModAnnos);
333 newAnnos.applyToOperation(memInst);
335 operationsToErase.insert(mem);
341 SmallVector<SubfieldOp> accesses;
342 for (
auto *op : structValue.getUsers()) {
343 assert(isa<SubfieldOp>(op));
344 auto fieldAccess = cast<SubfieldOp>(op);
346 fieldAccess.getInput().getType().base().getElementIndex(field);
347 if (elemIndex && *elemIndex == fieldAccess.getFieldIndex())
348 accesses.push_back(fieldAccess);
353 InstanceOp LowerMemoryPass::emitMemoryInstance(MemOp op, FModuleOp module,
355 OpBuilder builder(op);
356 auto *context = &getContext();
357 auto memName = op.getName();
362 SmallVector<Type, 8> portTypes;
363 SmallVector<Direction> portDirections;
364 SmallVector<Attribute> portNames;
365 DenseMap<Operation *, size_t> returnHolder;
366 mlir::DominanceInfo domInfo(op->getParentOfType<FModuleOp>());
371 for (
unsigned memportKindIdx = 0; memportKindIdx != 3; ++memportKindIdx) {
372 MemOp::PortKind memportKind = MemOp::PortKind::Read;
373 auto *portLabel =
"R";
374 switch (memportKindIdx) {
378 memportKind = MemOp::PortKind::ReadWrite;
382 memportKind = MemOp::PortKind::Write;
389 unsigned portNumber = 0;
392 auto getType = [&](
size_t width) {
return UIntType::get(context, width); };
393 auto ui1Type = getType(1);
394 auto addressType = getType(std::max(1U, llvm::Log2_64_Ceil(summary.
depth)));
400 for (
size_t i = 0, e = op.getNumResults(); i != e; ++i) {
402 if (memportKind != op.getPortKind(i))
405 auto addPort = [&](
Direction direction, StringRef field, Type portType) {
408 for (
auto a : accesses)
409 returnHolder[a] = portTypes.size();
411 portTypes.push_back(portType);
412 portDirections.push_back(direction);
414 builder.getStringAttr(portLabel + Twine(portNumber) +
"_" + field));
417 auto getDriver = [&](StringRef field) -> Operation * {
419 for (
auto a : accesses) {
420 for (
auto *user : a->getUsers()) {
422 if (
auto connect = dyn_cast<FConnectLike>(user);
433 auto removeMask = [&](StringRef enable, StringRef
mask) {
435 auto *maskConnect = getDriver(mask);
439 auto *enConnect = getDriver(enable);
444 OpBuilder b(maskConnect);
445 if (domInfo.dominates(maskConnect, enConnect))
446 b.setInsertionPoint(enConnect);
448 auto andOp = b.create<AndPrimOp>(
449 op->getLoc(), maskConnect->getOperand(1), enConnect->getOperand(1));
450 enConnect->setOperand(1, andOp);
451 enConnect->moveAfter(andOp);
453 auto *maskField = maskConnect->getOperand(0).getDefiningOp();
454 operationsToErase.insert(maskConnect);
455 operationsToErase.insert(maskField);
458 if (memportKind == MemOp::PortKind::Read) {
463 }
else if (memportKind == MemOp::PortKind::ReadWrite) {
474 removeMask(
"wmode",
"wmask");
484 removeMask(
"en",
"mask");
494 auto inst = builder.create<InstanceOp>(
496 op.getNameKind(), portDirections, portNames,
497 ArrayRef<Attribute>(),
498 ArrayRef<Attribute>(),
499 ArrayRef<Attribute>(),
false,
500 op.getInnerSymAttr());
503 for (
auto [subfield, result] : returnHolder) {
504 subfield->getResult(0).replaceAllUsesWith(inst.getResult(result));
505 operationsToErase.insert(subfield);
511 LogicalResult LowerMemoryPass::runOnModule(FModuleOp moduleOp,
513 assert(operationsToErase.empty() &&
"operationsToErase must be empty");
515 auto result = moduleOp.walk([&](MemOp op) {
517 if (!type_isa<UIntType>(op.getDataType())) {
518 op->emitError(
"memories should be flattened before running LowerMemory");
519 return WalkResult::interrupt();
524 lowerMemory(op, summary, shouldDedup);
526 return WalkResult::advance();
529 if (result.wasInterrupted())
532 for (Operation *op : operationsToErase)
535 operationsToErase.clear();
540 void LowerMemoryPass::runOnOperation() {
541 auto circuit = getOperation();
542 auto &instanceInfo = getAnalysis<InstanceInfo>();
543 symbolTable = &getAnalysis<SymbolTable>();
544 circuitNamespace.add(circuit);
551 for (
auto moduleOp : circuit.getBodyBlock()->getOps<FModuleOp>()) {
552 auto shouldDedup = instanceInfo.anyInstanceInEffectiveDesign(moduleOp);
553 if (failed(runOnModule(moduleOp, shouldDedup)))
554 return signalPassFailure();
557 circuitNamespace.clear();
558 symbolTable =
nullptr;
563 return std::make_unique<LowerMemoryPass>();
assert(baseType &&"element must be base type")
static SmallVector< SubfieldOp > getAllFieldAccesses(Value structValue, StringRef field)
FirMemory getSummary(MemOp op)
This class provides a read-only projection over the MLIR attributes that represent a set of annotatio...
bool removeAnnotations(llvm::function_ref< bool(Annotation)> predicate)
Remove all annotations from this annotation set for which predicate returns true.
This class provides a read-only projection of an annotation.
AttrClass getMember(StringAttr name) const
Return a member of the annotation.
void setMember(StringAttr name, Attribute value)
Add or set a member of the annotation to a value.
def connect(destination, source)
Direction get(bool isOutput)
Returns an output direction if isOutput is true, otherwise returns an input direction.
Direction
This represents the direction of a single port.
hw::InnerRefAttr getInnerRefTo(const hw::InnerSymTarget &target, GetNamespaceCallback getNamespace)
Obtain an inner reference to the target (operation or port), adding an inner symbol as necessary.
std::unique_ptr< mlir::Pass > createLowerMemoryPass()
The InstanceGraph op interface, see InstanceGraphInterface.td for more details.
The namespace of a CircuitOp, generally inhabited by modules.
bool isSeqMem() const
Check whether the memory is a seq mem.
StringAttr getFirMemoryName() const