20 #include "mlir/IR/ImplicitLocOpBuilder.h"
21 #include "mlir/IR/OperationSupport.h"
22 #include "mlir/Pass/Pass.h"
23 #include "llvm/ADT/DenseMap.h"
24 #include "llvm/ADT/Hashing.h"
25 #include "llvm/ADT/TypeSwitch.h"
29 #define GEN_PASS_DEF_LOWERCHIRRTLPASS
30 #include "circt/Dialect/FIRRTL/Passes.h.inc"
34 using namespace circt;
35 using namespace firrtl;
36 using namespace chirrtl;
39 struct LowerCHIRRTLPass
40 :
public circt::firrtl::impl::LowerCHIRRTLPassBase<LowerCHIRRTLPass>,
48 void visitCHIRRTL(CombMemOp op);
49 void visitCHIRRTL(SeqMemOp op);
50 void visitCHIRRTL(MemoryPortOp op);
51 void visitCHIRRTL(MemoryDebugPortOp op);
52 void visitCHIRRTL(MemoryPortAccessOp op);
53 void visitExpr(SubaccessOp op);
54 void visitExpr(SubfieldOp op);
55 void visitExpr(SubindexOp op);
56 void visitStmt(ConnectOp op);
57 void visitStmt(MatchingConnectOp op);
58 void visitUnhandledOp(Operation *op);
61 void visitInvalidCHIRRTL(Operation *op) { dispatchVisitor(op); }
62 void visitUnhandledCHIRRTL(Operation *op) { visitUnhandledOp(op); }
66 Value getConst(
unsigned c) {
67 auto &value = constCache[c];
69 auto module = getOperation();
70 auto builder = OpBuilder::atBlockBegin(module.getBodyBlock());
72 value = builder.create<ConstantOp>(module.getLoc(), u1Type, APInt(1, c));
87 void emitInvalid(ImplicitLocOpBuilder &builder, Value value);
89 MemDirAttr inferMemoryPortKind(MemoryPortOp memPort);
91 void replaceMem(Operation *op, StringRef name,
bool isSequential, RUWAttr ruw,
92 ArrayAttr annotations);
94 template <
typename OpType,
typename... T>
95 void cloneSubindexOpForMemory(OpType op, Value input, T... operands);
97 void runOnOperation()
override;
100 DenseMap<unsigned, Value> constCache;
101 DenseMap<Type, Value> invalidCache;
104 SmallVector<Operation *> opsToDelete;
109 DenseMap<Operation *, MemDirAttr> subfieldDirs;
114 DenseMap<Value, Value> rdataValues;
126 DenseMap<Value, WDataInfo> wdataValues;
133 static void forEachLeaf(ImplicitLocOpBuilder &builder, Value value,
134 llvm::function_ref<
void(Value)> func) {
135 auto type = value.getType();
136 if (
auto bundleType = type_dyn_cast<BundleType>(type)) {
137 for (
size_t i = 0, e = bundleType.getNumElements(); i < e; ++i)
138 forEachLeaf(builder, builder.create<SubfieldOp>(value, i), func);
139 }
else if (
auto vectorType = type_dyn_cast<FVectorType>(type)) {
140 for (
size_t i = 0, e = vectorType.getNumElements(); i != e; ++i)
141 forEachLeaf(builder, builder.create<SubindexOp>(value, i), func);
153 [&](Value leaf) {
emitConnect(builder, leaf, value); });
158 void LowerCHIRRTLPass::emitInvalid(ImplicitLocOpBuilder &builder, Value value) {
159 auto type = value.getType();
160 auto &invalid = invalidCache[type];
162 auto builder = OpBuilder::atBlockBegin(getOperation().
getBodyBlock());
163 invalid = builder.create<InvalidValueOp>(getOperation().getLoc(), type);
172 case MemDirAttr::Read:
173 return MemOp::PortKind::Read;
174 case MemDirAttr::Write:
175 return MemOp::PortKind::Write;
176 case MemDirAttr::ReadWrite:
177 return MemOp::PortKind::ReadWrite;
180 "Unhandled MemDirAttr, was the port direction not inferred?");
195 MemDirAttr LowerCHIRRTLPass::inferMemoryPortKind(MemoryPortOp memPort) {
202 struct StackElement {
203 StackElement(Value value, Value::use_iterator iterator, MemDirAttr mode)
204 : value(value), iterator(iterator), mode(mode) {}
206 Value::use_iterator iterator;
210 SmallVector<StackElement> stack;
211 stack.emplace_back(memPort.getData(), memPort.getData().use_begin(),
212 memPort.getDirection());
213 MemDirAttr mode = MemDirAttr::Infer;
215 while (!stack.empty()) {
216 auto *iter = &stack.back().iterator;
217 auto end = stack.back().value.use_end();
218 stack.back().mode |= mode;
220 while (*iter != end) {
221 auto &element = stack.back();
222 auto &use = *(*iter);
223 auto *user = use.getOwner();
225 if (isa<SubindexOp, SubfieldOp>(user)) {
227 auto output = user->getResult(0);
228 stack.emplace_back(output, output.use_begin(), MemDirAttr::Infer);
229 mode = MemDirAttr::Infer;
230 iter = &stack.back().iterator;
231 end = output.use_end();
234 if (
auto subaccessOp = dyn_cast<SubaccessOp>(user)) {
238 auto input = subaccessOp.getInput();
239 if (use.get() == input) {
240 auto output = subaccessOp.getResult();
241 stack.emplace_back(output, output.use_begin(), MemDirAttr::Infer);
242 mode = MemDirAttr::Infer;
243 iter = &stack.back().iterator;
244 end = output.use_end();
248 element.mode |= MemDirAttr::Read;
249 }
else if (
auto connectOp = dyn_cast<ConnectOp>(user)) {
250 if (use.get() == connectOp.getDest()) {
251 element.mode |= MemDirAttr::Write;
253 element.mode |= MemDirAttr::Read;
255 }
else if (
auto connectOp = dyn_cast<MatchingConnectOp>(user)) {
256 if (use.get() == connectOp.getDest()) {
257 element.mode |= MemDirAttr::Write;
259 element.mode |= MemDirAttr::Read;
263 element.mode |= MemDirAttr::Read;
266 mode = stack.back().mode;
271 subfieldDirs[stack.back().value.getDefiningOp()] = mode;
278 void LowerCHIRRTLPass::replaceMem(Operation *cmem, StringRef name,
279 bool isSequential, RUWAttr ruw,
280 ArrayAttr annotations) {
281 assert(isa<CombMemOp>(cmem) || isa<SeqMemOp>(cmem));
285 opsToDelete.push_back(cmem);
288 auto cmemType = type_cast<CMemoryType>(cmem->getResult(0).getType());
289 auto depth = cmemType.getNumElements();
290 auto type = cmemType.getElementType();
296 Attribute annotations;
297 MemOp::PortKind portKind;
300 SmallVector<PortInfo, 4> ports;
301 for (
auto *user : cmem->getUsers()) {
302 MemOp::PortKind portKind;
305 if (
auto cmemoryPort = dyn_cast<MemoryPortOp>(user)) {
307 auto portDirection = inferMemoryPortKind(cmemoryPort);
312 if (portDirection == MemDirAttr::Infer)
315 portName = cmemoryPort.getNameAttr();
316 portAnnos = cmemoryPort.getAnnotationsAttr();
317 }
else if (
auto dPort = dyn_cast<MemoryDebugPortOp>(user)) {
318 portKind = MemOp::PortKind::Debug;
319 portName = dPort.getNameAttr();
320 portAnnos = dPort.getAnnotationsAttr();
322 user->emitOpError(
"unhandled user of chirrtl memory");
327 ports.push_back({portName, MemOp::getTypeForPort(depth, type, portKind),
328 portAnnos, portKind, user});
338 llvm::array_pod_sort(ports.begin(), ports.end(),
340 return lhs->name.getValue().compare(
341 rhs->name.getValue());
344 SmallVector<Attribute, 4> resultNames;
345 SmallVector<Type, 4> resultTypes;
346 SmallVector<Attribute, 4> portAnnotations;
347 for (
auto port : ports) {
348 resultNames.push_back(port.name);
349 resultTypes.push_back(port.type);
350 portAnnotations.push_back(port.annotations);
355 auto readLatency = isSequential ? 1 : 0;
356 auto writeLatency = 1;
359 ImplicitLocOpBuilder memBuilder(cmem->getLoc(), cmem);
360 auto symOp = cast<hw::InnerSymbolOpInterface>(cmem);
361 auto memory = memBuilder.create<MemOp>(
362 resultTypes, readLatency, writeLatency, depth, ruw,
363 memBuilder.getArrayAttr(resultNames), name,
364 cmem->getAttrOfType<firrtl::NameKindEnumAttr>(
"nameKind").getValue(),
365 annotations, memBuilder.getArrayAttr(portAnnotations),
366 symOp.getInnerSymAttr(),
367 cmem->getAttrOfType<firrtl::MemoryInitAttr>(
"init"), StringAttr());
372 for (
unsigned i = 0, e = memory.getNumResults(); i < e; ++i) {
373 auto memoryPort = memory.getResult(i);
374 auto portKind = ports[i].portKind;
375 if (portKind == MemOp::PortKind::Debug) {
376 rdataValues[ports[i].cmemPort->getResult(0)] = memoryPort;
379 auto cmemoryPort = cast<MemoryPortOp>(ports[i].cmemPort);
380 auto cmemoryPortAccess = cmemoryPort.getAccess();
387 ImplicitLocOpBuilder portBuilder(cmemoryPortAccess.getLoc(),
389 auto address = memBuilder.create<SubfieldOp>(memoryPort,
"addr");
390 emitInvalid(memBuilder, address);
391 auto enable = memBuilder.create<SubfieldOp>(memoryPort,
"en");
393 auto clock = memBuilder.create<SubfieldOp>(memoryPort,
"clk");
394 emitInvalid(memBuilder, clock);
397 emitConnect(portBuilder, address, cmemoryPortAccess.getIndex());
399 auto useEnableInference = isSequential && portKind == MemOp::PortKind::Read;
400 auto *addressOp = cmemoryPortAccess.getIndex().getDefiningOp();
403 useEnableInference &=
404 !addressOp || isa<WireOp, NodeOp, RegOp, RegResetOp>(addressOp);
407 if (!useEnableInference)
410 emitConnect(portBuilder, clock, cmemoryPortAccess.getClock());
412 if (portKind == MemOp::PortKind::Read) {
414 auto data = memBuilder.create<SubfieldOp>(memoryPort,
"data");
415 rdataValues[cmemoryPort.getData()] =
data;
416 }
else if (portKind == MemOp::PortKind::Write) {
418 auto data = memBuilder.create<SubfieldOp>(memoryPort,
"data");
419 emitInvalid(memBuilder, data);
420 auto mask = memBuilder.create<SubfieldOp>(memoryPort,
"mask");
421 emitInvalid(memBuilder, mask);
427 wdataValues[cmemoryPort.getData()] = {
data,
mask,
nullptr};
428 }
else if (portKind == MemOp::PortKind::ReadWrite) {
430 auto rdata = memBuilder.create<SubfieldOp>(memoryPort,
"rdata");
431 auto wmode = memBuilder.create<SubfieldOp>(memoryPort,
"wmode");
433 auto wdata = memBuilder.create<SubfieldOp>(memoryPort,
"wdata");
434 emitInvalid(memBuilder, wdata);
435 auto wmask = memBuilder.create<SubfieldOp>(memoryPort,
"wmask");
436 emitInvalid(memBuilder, wmask);
443 rdataValues[cmemoryPort.getData()] =
rdata;
451 if (useEnableInference) {
452 auto *indexOp = cmemoryPortAccess.getIndex().getDefiningOp();
453 bool success =
false;
458 }
else if (isa<WireOp, RegResetOp, RegOp>(indexOp)) {
465 make_filter_range(indexOp->getUsers(), [&](Operation *op) {
466 if (auto connectOp = dyn_cast<ConnectOp>(op)) {
467 if (cmemoryPortAccess.getIndex() == connectOp.getDest())
468 return !dyn_cast_or_null<InvalidValueOp>(
469 connectOp.getSrc().getDefiningOp());
470 }
else if (
auto connectOp = dyn_cast<MatchingConnectOp>(op)) {
471 if (cmemoryPortAccess.getIndex() == connectOp.getDest())
472 return !dyn_cast_or_null<InvalidValueOp>(
473 connectOp.getSrc().getDefiningOp());
479 for (
auto *driver : drivers) {
480 ImplicitLocOpBuilder builder(driver->getLoc(), driver);
484 }
else if (isa<NodeOp>(indexOp)) {
487 ImplicitLocOpBuilder builder(indexOp->getLoc(), indexOp);
494 cmemoryPort.emitWarning(
"memory port is never enabled");
499 void LowerCHIRRTLPass::visitCHIRRTL(CombMemOp combmem) {
500 replaceMem(combmem, combmem.getName(),
false,
501 RUWAttr::Undefined, combmem.getAnnotations());
504 void LowerCHIRRTLPass::visitCHIRRTL(SeqMemOp seqmem) {
505 replaceMem(seqmem, seqmem.getName(),
true, seqmem.getRuw(),
506 seqmem.getAnnotations());
509 void LowerCHIRRTLPass::visitCHIRRTL(MemoryPortOp memPort) {
511 opsToDelete.push_back(memPort);
514 void LowerCHIRRTLPass::visitCHIRRTL(MemoryDebugPortOp memPort) {
516 opsToDelete.push_back(memPort);
519 void LowerCHIRRTLPass::visitCHIRRTL(MemoryPortAccessOp memPortAccess) {
521 opsToDelete.push_back(memPortAccess);
524 void LowerCHIRRTLPass::visitStmt(ConnectOp
connect) {
527 auto writeIt = wdataValues.find(
connect.getDest());
528 if (writeIt != wdataValues.end()) {
529 auto writeData = writeIt->second;
530 connect.getDestMutable().assign(writeData.data);
540 auto readIt = rdataValues.find(
connect.getSrc());
541 if (readIt != rdataValues.end()) {
542 auto newSource = readIt->second;
543 connect.getSrcMutable().assign(newSource);
547 void LowerCHIRRTLPass::visitStmt(MatchingConnectOp
connect) {
550 auto writeIt = wdataValues.find(
connect.getDest());
551 if (writeIt != wdataValues.end()) {
552 auto writeData = writeIt->second;
553 connect.getDestMutable().assign(writeData.data);
563 auto readIt = rdataValues.find(
connect.getSrc());
564 if (readIt != rdataValues.end()) {
565 auto newSource = readIt->second;
566 connect.getSrcMutable().assign(newSource);
577 template <
typename OpType,
typename... T>
578 void LowerCHIRRTLPass::cloneSubindexOpForMemory(OpType op, Value input,
582 auto it = subfieldDirs.find(op);
583 if (it == subfieldDirs.end()) {
586 auto iter = rdataValues.find(input);
587 if (iter != rdataValues.end()) {
588 opsToDelete.push_back(op);
589 ImplicitLocOpBuilder builder(op->getLoc(), op);
590 rdataValues[op] = builder.create<OpType>(rdataValues[input], operands...);
599 opsToDelete.push_back(op);
601 auto direction = it->second;
602 ImplicitLocOpBuilder builder(op->getLoc(), op);
606 if (direction == MemDirAttr::Read || direction == MemDirAttr::ReadWrite) {
607 rdataValues[op] = builder.create<OpType>(rdataValues[input], operands...);
612 if (direction == MemDirAttr::Write || direction == MemDirAttr::ReadWrite) {
613 auto writeData = wdataValues[input];
614 auto write = builder.create<OpType>(writeData.data, operands...);
615 auto mask = builder.create<OpType>(writeData.mask, operands...);
616 wdataValues[op] = {write,
mask, writeData.mode};
620 void LowerCHIRRTLPass::visitExpr(SubaccessOp subaccess) {
623 auto readIt = rdataValues.find(subaccess.getIndex());
624 if (readIt != rdataValues.end()) {
625 subaccess.getIndexMutable().assign(readIt->second);
628 cloneSubindexOpForMemory(subaccess, subaccess.getInput(),
629 subaccess.getIndex());
632 void LowerCHIRRTLPass::visitExpr(SubfieldOp subfield) {
633 cloneSubindexOpForMemory<SubfieldOp>(subfield, subfield.getInput(),
634 subfield.getFieldIndex());
637 void LowerCHIRRTLPass::visitExpr(SubindexOp subindex) {
638 cloneSubindexOpForMemory<SubindexOp>(subindex, subindex.getInput(),
639 subindex.getIndex());
642 void LowerCHIRRTLPass::visitUnhandledOp(Operation *op) {
645 for (
auto &operand : op->getOpOperands()) {
646 auto it = rdataValues.find(operand.get());
647 if (it != rdataValues.end()) {
648 operand.set(it->second);
653 void LowerCHIRRTLPass::runOnOperation() {
657 getOperation().getBodyBlock()->walk(
658 [&](Operation *op) { dispatchCHIRRTLVisitor(op); });
662 if (opsToDelete.empty())
663 markAllAnalysesPreserved();
666 while (!opsToDelete.empty())
667 opsToDelete.pop_back_val()->erase();
674 return std::make_unique<LowerCHIRRTLPass>();
assert(baseType &&"element must be base type")
static MemOp::PortKind memDirAttrToPortKind(MemDirAttr direction)
Converts a CHIRRTL memory port direction to a MemoryOp port type.
static void connectLeafsTo(ImplicitLocOpBuilder &builder, Value bundle, Value value)
Drive a value to all leafs of the input aggregate value.
static void forEachLeaf(ImplicitLocOpBuilder &builder, Value value, llvm::function_ref< void(Value)> func)
Performs the callback for each leaf element of a value.
static Block * getBodyBlock(FModuleLike mod)
CHIRRTLVisitor is a visitor for CHIRRTL operations.
FIRRTLVisitor allows you to visit all of the expr/stmt/decls with one class declaration.
def connect(destination, source)
Direction get(bool isOutput)
Returns an output direction if isOutput is true, otherwise returns an input direction.
std::unique_ptr< mlir::Pass > createLowerCHIRRTLPass()
void emitConnect(OpBuilder &builder, Location loc, Value lhs, Value rhs)
Emit a connect between two values.
The InstanceGraph op interface, see InstanceGraphInterface.td for more details.
This holds the name and type that describes the module's ports.