21 #include "mlir/IR/Dominance.h"
22 #include "mlir/Pass/Pass.h"
23 #include "llvm/ADT/DepthFirstIterator.h"
24 #include "llvm/ADT/STLExtras.h"
25 #include "llvm/ADT/STLFunctionalExtras.h"
26 #include "llvm/ADT/SmallPtrSet.h"
27 #include "llvm/Support/Parallel.h"
33 #define GEN_PASS_DEF_LOWERMEMORY
34 #include "circt/Dialect/FIRRTL/Passes.h.inc"
38 using namespace circt;
39 using namespace firrtl;
43 size_t numReadPorts = 0;
44 size_t numWritePorts = 0;
45 size_t numReadWritePorts = 0;
47 SmallVector<int32_t> writeClockIDs;
49 for (
size_t i = 0, e = op.getNumResults(); i != e; ++i) {
50 auto portKind = op.getPortKind(i);
51 if (portKind == MemOp::PortKind::Read)
53 else if (portKind == MemOp::PortKind::Write) {
54 for (
auto *a : op.getResult(i).getUsers()) {
55 auto subfield = dyn_cast<SubfieldOp>(a);
56 if (!subfield || subfield.getFieldIndex() != 2)
58 auto clockPort = a->getResult(0);
59 for (
auto *b : clockPort.getUsers()) {
60 if (
auto connect = dyn_cast<FConnectLike>(b)) {
61 if (
connect.getDest() == clockPort) {
63 clockToLeader.insert({
connect.getSrc(), numWritePorts});
65 writeClockIDs.push_back(numWritePorts);
67 writeClockIDs.push_back(result.first->second);
79 auto width = op.getDataType().getBitWidthOrSentinel();
81 op.emitError(
"'firrtl.mem' should have simple type and known width");
92 *seq::symbolizeRUW(
unsigned(op.getRuw())),
103 struct LowerMemoryPass
104 :
public circt::firrtl::impl::LowerMemoryBase<LowerMemoryPass> {
107 hw::InnerSymbolNamespace &getModuleNamespace(FModuleLike moduleOp) {
108 return moduleNamespaces.try_emplace(moduleOp, moduleOp).first->second;
111 SmallVector<PortInfo> getMemoryModulePorts(
const FirMemory &mem);
112 FMemModuleOp emitMemoryModule(MemOp op,
const FirMemory &summary,
113 const SmallVectorImpl<PortInfo> &ports);
114 FMemModuleOp getOrCreateMemModule(MemOp op,
const FirMemory &summary,
115 const SmallVectorImpl<PortInfo> &ports,
117 FModuleOp createWrapperModule(MemOp op,
const FirMemory &summary,
119 InstanceOp emitMemoryInstance(MemOp op, FModuleOp moduleOp,
121 void lowerMemory(MemOp mem,
const FirMemory &summary,
bool shouldDedup);
122 LogicalResult runOnModule(FModuleOp moduleOp,
bool shouldDedup);
123 void runOnOperation()
override;
126 DenseMap<Operation *, hw::InnerSymbolNamespace> moduleNamespaces;
128 SymbolTable *symbolTable;
132 std::map<FirMemory, FMemModuleOp> memories;
135 SetVector<Operation *> operationsToErase;
139 SmallVector<PortInfo>
140 LowerMemoryPass::getMemoryModulePorts(
const FirMemory &mem) {
141 auto *context = &getContext();
154 SmallVector<PortInfo> ports;
158 {nameAttr, type, direction, hw::InnerSymAttr{}, loc, annotations});
161 auto makePortCommon = [&](StringRef prefix,
size_t idx,
FIRRTLType addrType) {
162 addPort(prefix + Twine(idx) +
"_addr", addrType,
Direction::In);
164 addPort(prefix + Twine(idx) +
"_clk", clockType,
Direction::In);
168 makePortCommon(
"R", i, addrType);
172 makePortCommon(
"RW", i, addrType);
174 addPort(
"RW" + Twine(i) +
"_wdata", dataType,
Direction::In);
178 addPort(
"RW" + Twine(i) +
"_wmask", maskType,
Direction::In);
182 makePortCommon(
"W", i, addrType);
193 LowerMemoryPass::emitMemoryModule(MemOp op,
const FirMemory &mem,
194 const SmallVectorImpl<PortInfo> &ports) {
196 auto newName = circuitNamespace.newName(mem.
modName.getValue(),
"ext");
200 OpBuilder b(op->getParentOfType<FModuleOp>());
201 ++numCreatedMemModules;
202 auto moduleOp = b.create<FMemModuleOp>(
206 SymbolTable::setSymbolVisibility(moduleOp, SymbolTable::Visibility::Private);
211 LowerMemoryPass::getOrCreateMemModule(MemOp op,
const FirMemory &summary,
212 const SmallVectorImpl<PortInfo> &ports,
217 auto it = memories.find(summary);
218 if (it != memories.end())
224 auto moduleOp = emitMemoryModule(op, summary, ports);
229 memories[summary] = moduleOp;
234 void LowerMemoryPass::lowerMemory(MemOp mem,
const FirMemory &summary,
236 auto *context = &getContext();
237 auto ports = getMemoryModulePorts(summary);
240 auto newName = circuitNamespace.newName(mem.getName());
244 OpBuilder b(mem->getParentOfType<FModuleOp>());
245 auto wrapper = b.create<FModuleOp>(
246 mem->getLoc(), wrapperName,
248 SymbolTable::setSymbolVisibility(wrapper, SymbolTable::Visibility::Private);
252 auto memModule = getOrCreateMemModule(mem, summary, ports, shouldDedup);
253 b.setInsertionPointToStart(wrapper.getBodyBlock());
256 b.create<InstanceOp>(mem->getLoc(), memModule, memModule.getModuleName(),
257 mem.getNameKind(), mem.getAnnotations().getValue());
260 for (
auto [dst, src] : llvm::zip(wrapper.getBodyBlock()->getArguments(),
261 memInst.getResults())) {
262 if (wrapper.getPortDirection(dst.getArgNumber()) ==
Direction::Out)
263 b.create<MatchingConnectOp>(mem->getLoc(), dst, src);
265 b.create<MatchingConnectOp>(mem->getLoc(), src, dst);
270 auto inst = emitMemoryInstance(mem, wrapper, summary);
276 auto leafSym = memModule.getModuleNameAttr();
282 bool nlaUpdated =
false;
283 SmallVector<Annotation> newMemModAnnos;
284 OpBuilder nlaBuilder(context);
288 auto nlaSym = anno.
getMember<FlatSymbolRefAttr>(nonlocalAttr);
292 auto newNLAIter = processedNLAs.find(nlaSym.getAttr());
293 StringAttr newNLAName;
294 if (newNLAIter == processedNLAs.end()) {
298 dyn_cast<hw::HierPathOp>(symbolTable->lookup(nlaSym.getAttr()));
299 auto namepath = nla.getNamepath().getValue();
300 SmallVector<Attribute> newNamepath(namepath.begin(), namepath.end());
301 if (!nla.isComponent())
303 getInnerRefTo(inst, [&](
auto mod) -> hw::InnerSymbolNamespace & {
304 return getModuleNamespace(mod);
306 newNamepath.push_back(leafAttr);
308 nlaBuilder.setInsertionPointAfter(nla);
309 auto newNLA = cast<hw::HierPathOp>(nlaBuilder.clone(*nla));
311 context, circuitNamespace.newName(nla.getNameAttr().getValue())));
313 newNLAName = newNLA.getNameAttr();
314 processedNLAs[nlaSym.getAttr()] = newNLAName;
316 newNLAName = newNLAIter->getSecond();
319 newMemModAnnos.push_back(anno);
325 newAnnos.addAnnotations(newMemModAnnos);
326 newAnnos.applyToOperation(memInst);
328 operationsToErase.insert(mem);
334 SmallVector<SubfieldOp> accesses;
335 for (
auto *op : structValue.getUsers()) {
336 assert(isa<SubfieldOp>(op));
337 auto fieldAccess = cast<SubfieldOp>(op);
339 fieldAccess.getInput().getType().base().getElementIndex(field);
340 if (elemIndex && *elemIndex == fieldAccess.getFieldIndex())
341 accesses.push_back(fieldAccess);
346 InstanceOp LowerMemoryPass::emitMemoryInstance(MemOp op, FModuleOp module,
348 OpBuilder builder(op);
349 auto *context = &getContext();
350 auto memName = op.getName();
355 SmallVector<Type, 8> portTypes;
356 SmallVector<Direction> portDirections;
357 SmallVector<Attribute> portNames;
358 DenseMap<Operation *, size_t> returnHolder;
359 mlir::DominanceInfo domInfo(op->getParentOfType<FModuleOp>());
364 for (
unsigned memportKindIdx = 0; memportKindIdx != 3; ++memportKindIdx) {
365 MemOp::PortKind memportKind = MemOp::PortKind::Read;
366 auto *portLabel =
"R";
367 switch (memportKindIdx) {
371 memportKind = MemOp::PortKind::ReadWrite;
375 memportKind = MemOp::PortKind::Write;
382 unsigned portNumber = 0;
386 auto ui1Type = getType(1);
387 auto addressType = getType(std::max(1U, llvm::Log2_64_Ceil(summary.
depth)));
393 for (
size_t i = 0, e = op.getNumResults(); i != e; ++i) {
395 if (memportKind != op.getPortKind(i))
398 auto addPort = [&](
Direction direction, StringRef field, Type portType) {
401 for (
auto a : accesses)
402 returnHolder[a] = portTypes.size();
404 portTypes.push_back(portType);
405 portDirections.push_back(direction);
407 builder.getStringAttr(portLabel + Twine(portNumber) +
"_" + field));
410 auto getDriver = [&](StringRef field) -> Operation * {
412 for (
auto a : accesses) {
413 for (
auto *user : a->getUsers()) {
415 if (
auto connect = dyn_cast<FConnectLike>(user);
426 auto removeMask = [&](StringRef enable, StringRef
mask) {
428 auto *maskConnect = getDriver(mask);
432 auto *enConnect = getDriver(enable);
437 OpBuilder b(maskConnect);
438 if (domInfo.dominates(maskConnect, enConnect))
439 b.setInsertionPoint(enConnect);
441 auto andOp = b.create<AndPrimOp>(
442 op->getLoc(), maskConnect->getOperand(1), enConnect->getOperand(1));
443 enConnect->setOperand(1, andOp);
444 enConnect->moveAfter(andOp);
446 auto *maskField = maskConnect->getOperand(0).getDefiningOp();
447 operationsToErase.insert(maskConnect);
448 operationsToErase.insert(maskField);
451 if (memportKind == MemOp::PortKind::Read) {
456 }
else if (memportKind == MemOp::PortKind::ReadWrite) {
467 removeMask(
"wmode",
"wmask");
477 removeMask(
"en",
"mask");
487 auto inst = builder.create<InstanceOp>(
489 op.getNameKind(), portDirections, portNames,
490 ArrayRef<Attribute>(),
491 ArrayRef<Attribute>(),
492 ArrayRef<Attribute>(),
false,
493 op.getInnerSymAttr());
496 for (
auto [subfield, result] : returnHolder) {
497 subfield->getResult(0).replaceAllUsesWith(inst.getResult(result));
498 operationsToErase.insert(subfield);
504 LogicalResult LowerMemoryPass::runOnModule(FModuleOp moduleOp,
506 assert(operationsToErase.empty() &&
"operationsToErase must be empty");
508 auto result = moduleOp.walk([&](MemOp op) {
510 if (!type_isa<UIntType>(op.getDataType())) {
511 op->emitError(
"memories should be flattened before running LowerMemory");
512 return WalkResult::interrupt();
517 lowerMemory(op, summary, shouldDedup);
519 return WalkResult::advance();
522 if (result.wasInterrupted())
525 for (Operation *op : operationsToErase)
528 operationsToErase.clear();
533 void LowerMemoryPass::runOnOperation() {
534 auto circuit = getOperation();
535 auto &instanceInfo = getAnalysis<InstanceInfo>();
536 symbolTable = &getAnalysis<SymbolTable>();
537 circuitNamespace.add(circuit);
544 for (
auto moduleOp : circuit.getBodyBlock()->getOps<FModuleOp>()) {
545 auto shouldDedup = instanceInfo.anyInstanceUnderEffectiveDut(moduleOp);
546 if (failed(runOnModule(moduleOp, shouldDedup)))
547 return signalPassFailure();
550 circuitNamespace.clear();
551 symbolTable =
nullptr;
556 return std::make_unique<LowerMemoryPass>();
assert(baseType &&"element must be base type")
static SmallVector< SubfieldOp > getAllFieldAccesses(Value structValue, StringRef field)
FirMemory getSummary(MemOp op)
This class provides a read-only projection over the MLIR attributes that represent a set of annotatio...
bool removeAnnotations(llvm::function_ref< bool(Annotation)> predicate)
Remove all annotations from this annotation set for which predicate returns true.
This class provides a read-only projection of an annotation.
AttrClass getMember(StringAttr name) const
Return a member of the annotation.
void setMember(StringAttr name, Attribute value)
Add or set a member of the annotation to a value.
def connect(destination, source)
Direction get(bool isOutput)
Returns an output direction if isOutput is true, otherwise returns an input direction.
Direction
This represents the direction of a single port.
hw::InnerRefAttr getInnerRefTo(const hw::InnerSymTarget &target, GetNamespaceCallback getNamespace)
Obtain an inner reference to the target (operation or port), adding an inner symbol as necessary.
std::unique_ptr< mlir::Pass > createLowerMemoryPass()
The InstanceGraph op interface, see InstanceGraphInterface.td for more details.
The namespace of a CircuitOp, generally inhabited by modules.
bool isSeqMem() const
Check whether the memory is a seq mem.
StringAttr getFirMemoryName() const