24 #include "mlir/IR/ImplicitLocOpBuilder.h"
25 #include "mlir/Pass/Pass.h"
26 #include "llvm/ADT/TypeSwitch.h"
27 #include "llvm/Support/Path.h"
29 using namespace circt;
35 #define GEN_PASS_DEF_HWMEMSIMIMPL
36 #include "circt/Dialect/Seq/SeqPasses.h.inc"
49 bool disableMemRandomization;
50 bool disableRegRandomization;
51 bool addVivadoRAMAddressConflictSynthesisBugWorkaround;
53 SmallVector<sv::RegOp> registers;
55 Value addPipelineStages(ImplicitLocOpBuilder &b,
56 hw::InnerSymbolNamespace &moduleNamespace,
57 size_t stages, Value clock, Value
data,
58 const Twine &name, Value gate = {});
59 sv::AlwaysOp lastPipelineAlwaysOp;
65 bool disableMemRandomization,
bool disableRegRandomization,
66 bool addVivadoRAMAddressConflictSynthesisBugWorkaround,
68 : readEnableMode(readEnableMode), addMuxPragmas(addMuxPragmas),
69 disableMemRandomization(disableMemRandomization),
70 disableRegRandomization(disableRegRandomization),
71 addVivadoRAMAddressConflictSynthesisBugWorkaround(
72 addVivadoRAMAddressConflictSynthesisBugWorkaround),
73 mlirModuleNamespace(mlirModuleNamespace) {}
75 void generateMemory(
HWModuleOp op, FirMemory mem);
78 struct HWMemSimImplPass :
public impl::HWMemSimImplBase<HWMemSimImplPass> {
79 using HWMemSimImplBase::HWMemSimImplBase;
81 void runOnOperation()
override;
90 Operation *valueOp = value.getDefiningOp();
92 valueOp ? valueOp->getBlock() : cast<BlockArgument>(value).getOwner();
93 while (op->getBlock() && op->getBlock() != valueBlock)
94 op = op->getParentOp();
95 return valueBlock == op->getBlock() &&
96 (!valueOp || valueOp->isBeforeInBlock(op));
109 static Value
getMemoryRead(ImplicitLocOpBuilder &b, Value memory, Value addr,
110 bool addMuxPragmas) {
112 b.create<
sv::ReadInOutOp>(b.create<sv::ArrayIndexInOutOp>(memory, addr));
114 if (!addMuxPragmas ||
115 cast<hw::UnpackedArrayType>(
116 cast<hw::InOutType>(memory.getType()).getElementType())
117 .getNumElements() <= 1)
122 auto valWire = b.create<
sv::WireOp>(slot.getType());
126 "synopsys infer_mux_override",
132 Value HWMemSimImpl::addPipelineStages(ImplicitLocOpBuilder &b,
133 hw::InnerSymbolNamespace &moduleNamespace,
134 size_t stages, Value clock, Value data,
135 const Twine &name, Value gate) {
141 auto alwaysOp = lastPipelineAlwaysOp;
143 if (alwaysOp.getClocks() != ValueRange{clock} ||
149 alwaysOp = b.create<sv::AlwaysOp>(sv::EventControl::AtPosEdge, clock);
152 auto savedIP = b.saveInsertionPoint();
153 SmallVector<sv::RegOp> regs;
154 b.setInsertionPoint(alwaysOp);
155 for (
unsigned i = 0; i < stages; ++i) {
157 b.getStringAttr(moduleNamespace.newName(
"_" + name +
"_d" + Twine(i)));
161 registers.push_back(
reg);
165 b.setInsertionPointToEnd(alwaysOp.getBodyBlock());
166 for (
unsigned i = 0; i < stages; ++i) {
169 auto emitAssign = [&] { b.
create<sv::PAssignOp>(regs[i],
data); };
171 b.create<sv::IfOp>(gate, [&]() { emitAssign(); });
177 b.restoreInsertionPoint(savedIP);
180 lastPipelineAlwaysOp = alwaysOp;
184 void HWMemSimImpl::generateMemory(
HWModuleOp op, FirMemory mem) {
185 ImplicitLocOpBuilder b(op.getLoc(), op.getBody());
190 if (mem.maskGran == 0)
191 mem.maskGran = mem.dataWidth;
192 auto maskBits = mem.dataWidth / mem.maskGran;
193 bool isMasked = maskBits > 1;
195 auto dataType = b.getIntegerType(mem.dataWidth);
199 mem.numReadPorts + mem.numWritePorts + mem.numReadWritePorts;
205 if (addVivadoRAMAddressConflictSynthesisBugWorkaround) {
206 if (mem.readLatency == 0) {
216 }
else if (mem.readLatency == 1 && numPorts > 1) {
227 SmallVector<Value, 4> outputs;
230 for (
size_t i = 0; i < mem.numReadPorts; ++i) {
231 Value
addr = op.getBody().getArgument(inArg++);
232 Value
en = op.getBody().getArgument(inArg++);
233 Value clock = op.getBody().getArgument(inArg++);
235 if (readEnableMode == ReadEnableMode::Ignore) {
236 for (
size_t j = 0, e = mem.readLatency; j != e; ++j) {
239 en = addPipelineStages(b, moduleNamespace, 1, clock, en,
240 "R" + Twine(i) +
"_en");
241 addr = addPipelineStages(b, moduleNamespace, 1, clock, addr,
242 "R" + Twine(i) +
"_addr", enLast);
245 en = addPipelineStages(b, moduleNamespace, mem.readLatency, clock, en,
246 "R" + Twine(i) +
"_en");
247 addr = addPipelineStages(b, moduleNamespace, mem.readLatency, clock, addr,
248 "R" + Twine(i) +
"_addr");
253 switch (readEnableMode) {
254 case ReadEnableMode::Undefined: {
255 Value x = b.create<sv::ConstantXOp>(
rdata.getType());
259 case ReadEnableMode::Zero: {
264 case ReadEnableMode::Ignore:
267 outputs.push_back(rdata);
270 for (
size_t i = 0; i < mem.numReadWritePorts; ++i) {
271 auto numReadStages = mem.readLatency;
272 auto numWriteStages = mem.writeLatency - 1;
273 auto numCommonStages = std::min(numReadStages, numWriteStages);
274 Value
addr = op.getBody().getArgument(inArg++);
275 Value
en = op.getBody().getArgument(inArg++);
276 Value clock = op.getBody().getArgument(inArg++);
277 Value
wmode = op.getBody().getArgument(inArg++);
278 Value wdataIn = op.getBody().getArgument(inArg++);
283 wmaskBits = op.getBody().getArgument(inArg++);
285 wmaskBits = b.create<
ConstantOp>(b.getIntegerAttr(
en.getType(), 1));
288 addr = addPipelineStages(b, moduleNamespace, numCommonStages, clock, addr,
289 "RW" + Twine(i) +
"_addr");
290 en = addPipelineStages(b, moduleNamespace, numCommonStages, clock, en,
291 "RW" + Twine(i) +
"_en");
292 wmode = addPipelineStages(b, moduleNamespace, numCommonStages, clock, wmode,
293 "RW" + Twine(i) +
"_mode");
296 Value readAddr =
addr;
298 if (readEnableMode == ReadEnableMode::Ignore) {
299 for (
size_t j = 0, e = mem.readLatency; j != e; ++j) {
302 readEn = addPipelineStages(b, moduleNamespace, 1, clock, en,
303 "RW" + Twine(i) +
"_ren");
304 readAddr = addPipelineStages(b, moduleNamespace, 1, clock, addr,
305 "RW" + Twine(i) +
"_raddr", enLast);
309 addPipelineStages(b, moduleNamespace, numReadStages - numCommonStages,
310 clock, addr,
"RW" + Twine(i) +
"_raddr");
312 addPipelineStages(b, moduleNamespace, numReadStages - numCommonStages,
313 clock, en,
"RW" + Twine(i) +
"_ren");
316 addPipelineStages(b, moduleNamespace, numReadStages - numCommonStages,
317 clock, wmode,
"RW" + Twine(i) +
"_rmode");
321 addPipelineStages(b, moduleNamespace, numWriteStages - numCommonStages,
322 clock, addr,
"RW" + Twine(i) +
"_waddr");
324 addPipelineStages(b, moduleNamespace, numWriteStages - numCommonStages,
325 clock, en,
"RW" + Twine(i) +
"_wen");
327 addPipelineStages(b, moduleNamespace, numWriteStages - numCommonStages,
328 clock, wmode,
"RW" + Twine(i) +
"_wmode");
329 wdataIn = addPipelineStages(b, moduleNamespace, numWriteStages, clock,
330 wdataIn,
"RW" + Twine(i) +
"_wdata");
332 wmaskBits = addPipelineStages(b, moduleNamespace, numWriteStages, clock,
333 wmaskBits,
"RW" + Twine(i) +
"_wmask");
335 SmallVector<Value, 4> maskValues(maskBits);
336 SmallVector<Value, 4> dataValues(maskBits);
340 for (
size_t i = 0; i < maskBits; ++i) {
342 dataValues[i] = b.createOrFold<
comb::ExtractOp>(wdataIn, i * mem.maskGran,
353 b.createOrFold<comb::ICmpOp>(
354 comb::ICmpPredicate::eq, readWMode,
355 b.createOrFold<
ConstantOp>(readWMode.getType(), 0),
false),
360 switch (readEnableMode) {
361 case ReadEnableMode::Undefined: {
362 Value x = b.create<sv::ConstantXOp>(val.getType());
366 case ReadEnableMode::Zero: {
371 case ReadEnableMode::Ignore:
377 for (
auto wmask : llvm::enumerate(maskValues)) {
378 b.
create<sv::AlwaysOp>(sv::EventControl::AtPosEdge, clock, [&]() {
383 b.create<sv::IfOp>(wcond, [&]() {
384 Value slotReg = b.create<sv::ArrayIndexInOutOp>(
reg, writeAddr);
385 b.create<sv::PAssignOp>(
386 b.createOrFold<sv::IndexedPartSelectInOutOp>(
388 b.createOrFold<
ConstantOp>(b.getIntegerType(32),
389 wmask.index() * mem.maskGran),
391 dataValues[
wmask.index()]);
395 outputs.push_back(rdata);
398 DenseMap<unsigned, Operation *> writeProcesses;
399 for (
size_t i = 0; i < mem.numWritePorts; ++i) {
400 auto numStages = mem.writeLatency - 1;
401 Value
addr = op.getBody().getArgument(inArg++);
402 Value
en = op.getBody().getArgument(inArg++);
403 Value clock = op.getBody().getArgument(inArg++);
404 Value wdataIn = op.getBody().getArgument(inArg++);
409 wmaskBits = op.getBody().getArgument(inArg++);
411 wmaskBits = b.create<
ConstantOp>(b.getIntegerAttr(
en.getType(), 1));
413 addr = addPipelineStages(b, moduleNamespace, numStages, clock, addr,
414 "W" + Twine(i) +
"addr");
415 en = addPipelineStages(b, moduleNamespace, numStages, clock, en,
416 "W" + Twine(i) +
"en");
417 wdataIn = addPipelineStages(b, moduleNamespace, numStages, clock, wdataIn,
418 "W" + Twine(i) +
"data");
420 wmaskBits = addPipelineStages(b, moduleNamespace, numStages, clock,
421 wmaskBits,
"W" + Twine(i) +
"mask");
423 SmallVector<Value, 4> maskValues(maskBits);
424 SmallVector<Value, 4> dataValues(maskBits);
428 for (
size_t i = 0; i < maskBits; ++i) {
430 dataValues[i] = b.createOrFold<
comb::ExtractOp>(wdataIn, i * mem.maskGran,
434 auto writeLogic = [&] {
437 for (
auto wmask : llvm::enumerate(maskValues)) {
440 b.create<sv::IfOp>(wcond, [&]() {
441 auto slot = b.create<sv::ArrayIndexInOutOp>(
reg,
addr);
442 b.create<sv::PAssignOp>(
443 b.createOrFold<sv::IndexedPartSelectInOutOp>(
445 b.createOrFold<
ConstantOp>(b.getIntegerType(32),
446 wmask.index() * mem.maskGran),
448 dataValues[
wmask.index()]);
454 auto alwaysBlock = [&] {
455 return b.create<sv::AlwaysOp>(sv::EventControl::AtPosEdge, clock,
456 [&]() { writeLogic(); });
459 switch (mem.writeUnderWrite) {
462 case seq::WUW::Undefined:
467 case seq::WUW::PortOrder:
468 if (
auto *existingAlwaysBlock =
469 writeProcesses.lookup(mem.writeClockIDs[i])) {
470 OpBuilder::InsertionGuard guard(b);
471 b.setInsertionPointToEnd(
472 cast<sv::AlwaysOp>(existingAlwaysBlock).
getBodyBlock());
475 writeProcesses[i] = alwaysBlock();
480 auto *outputOp = op.getBodyBlock()->getTerminator();
481 outputOp->setOperands(outputs);
485 if (!mem.initFilename.empty()) {
487 if (!
reg.getInnerSymAttr())
489 b.getStringAttr(moduleNamespace.newName(
reg.getName()))));
491 if (mem.initIsInline) {
492 b.create<
sv::IfDefOp>(
"ENABLE_INITIAL_MEM_", [&]() {
493 b.create<sv::InitialOp>([&]() {
494 b.create<sv::ReadMemOp>(
reg, mem.initFilename,
496 ? MemBaseTypeAttr::MemBaseBin
497 : MemBaseTypeAttr::MemBaseHex);
501 OpBuilder::InsertionGuard guard(b);
504 StringAttr boundModuleName =
505 b.getStringAttr(mlirModuleNamespace.newName(op.getName() +
"_init"));
509 if (
auto fileAttr = op->getAttrOfType<OutputFileAttr>(
"output_file")) {
510 if (!fileAttr.isDirectory()) {
511 SmallString<128> path(fileAttr.getFilename().getValue());
512 llvm::sys::path::remove_filename(path);
514 filename = b.getStringAttr(path);
516 filename = fileAttr.getFilename();
519 filename = b.getStringAttr(boundModuleName.getValue() +
".sv");
523 b.setInsertionPointAfter(op);
525 b.create<
HWModuleOp>(boundModuleName, ArrayRef<PortInfo>());
528 auto path = b.create<hw::HierPathOp>(
529 mlirModuleNamespace.newName(op.getName() +
"_path"),
533 b.setInsertionPointToStart(boundModule.getBodyBlock());
534 b.create<sv::InitialOp>([&]() {
535 auto xmr = b.create<sv::XMRRefOp>(
reg.getType(), path.getSymNameAttr());
536 b.create<sv::ReadMemOp>(xmr, mem.initFilename,
537 mem.initIsBinary ? MemBaseTypeAttr::MemBaseBin
538 : MemBaseTypeAttr::MemBaseHex);
542 b.setInsertionPointAfter(
reg);
543 auto boundInstance = b.create<hw::InstanceOp>(
544 boundModule, boundModule.getName(), ArrayRef<Value>());
545 boundInstance->setAttr(
548 moduleNamespace.newName(boundInstance.getInstanceName()))));
549 boundInstance->setAttr(
"doNotPrint", b.getBoolAttr(
true));
552 b.setInsertionPointAfter(op);
553 b.create<emit::FileOp>(filename, [&] {
556 op.getNameAttr(), boundInstance.getInnerSymAttr().getSymName()));
563 if (disableMemRandomization && disableRegRandomization)
566 constexpr
unsigned randomWidth = 32;
567 b.create<
sv::IfDefOp>(
"ENABLE_INITIAL_MEM_", [&]() {
569 SmallVector<sv::RegOp> randRegs;
570 if (!disableRegRandomization) {
571 b.create<
sv::IfDefOp>(
"RANDOMIZE_REG_INIT", [&]() {
572 signed totalWidth = 0;
574 totalWidth +=
reg.getElementType().getIntOrFloatBitWidth();
575 while (totalWidth > 0) {
576 auto name = b.getStringAttr(moduleNamespace.newName(
"_RANDOM"));
578 randRegs.push_back(b.create<
sv::RegOp>(b.getIntegerType(randomWidth),
580 totalWidth -= randomWidth;
585 b.getIntegerType(llvm::divideCeil(mem.dataWidth, randomWidth) *
587 b.getStringAttr(
"_RANDOM_MEM"));
588 b.create<sv::InitialOp>([&]() {
589 b.create<sv::VerbatimOp>(
"`INIT_RANDOM_PROLOG_");
592 if (!disableMemRandomization) {
593 b.create<sv::IfDefProceduralOp>(
"RANDOMIZE_MEM_INIT", [&]() {
594 auto outerLoopIndVarType =
595 b.getIntegerType(llvm::Log2_64_Ceil(mem.depth + 1));
596 auto innerUpperBoundWidth =
597 cast<IntegerType>(randomMemReg.getType().getElementType())
599 auto innerLoopIndVarType =
600 b.getIntegerType(llvm::Log2_64_Ceil(innerUpperBoundWidth + 1));
609 0, mem.depth, 1, outerLoopIndVarType,
"i",
610 [&](BlockArgument outerIndVar) {
612 0, innerUpperBoundWidth, randomWidth, innerLoopIndVarType,
613 "j", [&](BlockArgument innerIndVar) {
614 auto rhs = b.create<sv::MacroRefExprSEOp>(
615 b.getIntegerType(randomWidth),
"RANDOM");
616 auto lhs = b.create<sv::IndexedPartSelectInOutOp>(
617 randomMemReg, innerIndVar, randomWidth,
false);
618 b.create<sv::BPAssignOp>(lhs, rhs);
621 Value iterValue = outerIndVar;
623 if (!outerIndVar.getType().isInteger(
624 llvm::Log2_64_Ceil(mem.depth)))
626 iterValue, 0, llvm::Log2_64_Ceil(mem.depth));
627 auto lhs = b.
create<sv::ArrayIndexInOutOp>(
reg, iterValue);
630 b.create<sv::BPAssignOp>(lhs, rhs);
640 if (!disableRegRandomization) {
641 b.create<sv::IfDefProceduralOp>(
"RANDOMIZE_REG_INIT", [&]() {
642 unsigned bits = randomWidth;
644 b.create<sv::VerbatimOp>(
645 b.getStringAttr(
"{{0}} = {`RANDOM};"), ValueRange{},
647 reg.getInnerNameAttr())));
650 SmallVector<std::pair<Attribute, std::pair<size_t, size_t>>> values;
651 auto width =
reg.getElementType().getIntOrFloatBitWidth();
652 auto widthRemaining =
width;
653 while (widthRemaining > 0) {
654 if (bits == randomWidth) {
655 randReg = randRegs[randRegIdx++];
659 randReg.getInnerNameAttr());
660 if (widthRemaining <= randomWidth - bits) {
661 values.push_back({innerRef, {bits + widthRemaining - 1, bits}});
662 bits += widthRemaining;
666 values.push_back({innerRef, {randomWidth - 1, bits}});
667 widthRemaining -= (randomWidth - bits);
670 SmallString<32> rhs(
"{{0}} = ");
674 op.getNameAttr(),
reg.getInnerNameAttr())});
675 if (values.size() > 1)
677 for (
auto &v : values) {
680 auto [sym, range] = v;
681 symbols.push_back(sym);
682 rhs.append((
"{{" + Twine(idx++) +
"}}").str());
684 if (range.first == randomWidth - 1 && range.second == 0)
687 if (range.first == range.second) {
688 rhs.append((
"[" + Twine(range.first) +
"]").str());
693 (
"[" + Twine(range.first) +
":" + Twine(range.second) +
"]")
696 if (values.size() > 1)
699 b.create<sv::VerbatimOp>(rhs, ValueRange{},
700 b.getArrayAttr(symbols));
708 void HWMemSimImplPass::runOnOperation() {
709 auto topModule = getOperation();
717 mlirModuleNamespace.
add(symbolCache);
719 SmallVector<HWModuleGeneratedOp> toErase;
720 bool anythingChanged =
false;
723 llvm::make_early_inc_range(topModule.getOps<HWModuleGeneratedOp>())) {
724 auto oldModule = cast<HWModuleGeneratedOp>(op);
725 auto gen = oldModule.getGeneratorKind();
726 auto genOp = cast<HWGeneratorSchemaOp>(
727 SymbolTable::lookupSymbolIn(getOperation(), gen));
729 if (genOp.getDescriptor() ==
"FIRRTL_Memory") {
730 FirMemory mem(oldModule);
732 OpBuilder builder(oldModule);
733 auto nameAttr = builder.getStringAttr(oldModule.getName());
738 if (replSeqMem && ((mem.readLatency == 1 && mem.writeLatency == 1) &&
739 mem.dataWidth > 0)) {
741 oldModule.getPortList());
744 oldModule.getLoc(), nameAttr, oldModule.getPortList());
745 if (
auto outdir = oldModule->getAttr(
"output_file"))
746 newModule->setAttr(
"output_file", outdir);
747 newModule.setCommentAttr(
748 builder.getStringAttr(
"VCS coverage exclude_file"));
749 newModule.setPrivate();
751 HWMemSimImpl(readEnableMode, addMuxPragmas, disableMemRandomization,
752 disableRegRandomization,
753 addVivadoRAMAddressConflictSynthesisBugWorkaround,
755 .generateMemory(newModule, mem);
759 anythingChanged =
true;
763 if (!anythingChanged)
764 markAllAnalysesPreserved();
767 std::unique_ptr<Pass>
769 return std::make_unique<HWMemSimImplPass>(options);
assert(baseType &&"element must be base type")
static bool valueDefinedBeforeOp(Value value, Operation *op)
A helper that returns true if a value definition (or block argument) is visible to another operation,...
static Value getMemoryRead(ImplicitLocOpBuilder &b, Value memory, Value addr, bool addMuxPragmas)
static StringAttr append(StringAttr base, const Twine &suffix)
Return a attribute with the specified suffix appended.
static Block * getBodyBlock(FModuleLike mod)
A namespace that is used to store existing names and generate new names in some scope within the IR.
void add(mlir::ModuleOp module)
void addDefinitions(mlir::Operation *top)
Populate the symbol cache with all symbol-defining operations within the 'top' operation.
Default symbol cache implementation; stores associations between names (StringAttr's) to mlir::Operat...
def create(data_type, name=None, sym_name=None)
Direction get(bool isOutput)
Returns an output direction if isOutput is true, otherwise returns an input direction.
std::unique_ptr< mlir::Pass > createHWMemSimImplPass(const HWMemSimImplOptions &options={})
void setSVAttributes(mlir::Operation *op, mlir::ArrayAttr attrs)
Set the SV attributes of an operation.
The InstanceGraph op interface, see InstanceGraphInterface.td for more details.
def reg(value, clock, reset=None, reset_value=None, name=None, sym_name=None)