12#include "mlir/IR/Threading.h"
13#include "mlir/Transforms/DialectConversion.h"
14#include "llvm/ADT/DenseSet.h"
15#include "llvm/Support/Debug.h"
24#define DEBUG_TYPE "lower-seq-firreg"
26static Value
buildXMRTo(OpBuilder &builder, HierPathOp path, Location loc,
28 auto name = path.getSymNameAttr();
29 auto ref = mlir::FlatSymbolRefAttr::get(name);
30 return sv::XMRRefOp::create(builder, loc, type, ref);
35 if (block->mightHaveTerminator())
36 return Block::iterator(block->getTerminator());
41 [](
const Operation *op) ->
bool {
42 return (isa<comb::MuxOp, ArrayGetOp, ArrayCreateOp>(op));
47 return llvm::any_of(regOp.getResult().getUsers(), [&](Operation *user) {
48 if (!OpUserInfo::opAllowsReachability(user))
50 buildReachabilityFrom(user);
51 return reachableMuxes[user].contains(muxOp);
63 if (
visited.contains(startNode))
68 llvm::SmallVector<OpUserInfo, 16> stk;
70 stk.emplace_back(startNode);
72 while (!stk.empty()) {
73 auto &info = stk.back();
74 Operation *currentNode = info.op;
77 if (info.getAndSetUnvisited())
80 if (info.userIter != info.userEnd) {
81 Operation *child = *info.userIter;
84 stk.emplace_back(child);
90 for (
auto *childOp : llvm::make_filter_range(
99 iter->getSecond().end());
107 const std::function<
void()> &trueSide,
108 const std::function<
void()> &falseSide) {
109 auto op =
ifCache.lookup({builder.getBlock(), cond});
114 sv::IfOp::create(builder, cond.getLoc(), cond, trueSide, falseSide);
115 ifCache.insert({{builder.getBlock(), cond}, newIfOp});
117 OpBuilder::InsertionGuard guard(builder);
118 builder.setInsertionPointToEnd(op.getThenBlock());
120 builder.setInsertionPointToEnd(op.getElseBlock());
129 auto attr =
reg.getInnerSymAttr();
134 if (
auto sym = attr.getSymIfExists(0))
138 auto *context =
reg->getContext();
140 auto hint =
reg.getName();
143 auto sym = StringAttr::get(context, innerSymNS.
newName(hint));
144 auto property = hw::InnerSymPropertiesAttr::get(sym);
148 SmallVector<hw::InnerSymPropertiesAttr> properties = {
property};
150 llvm::append_range(properties, attr.getProps());
153 attr = hw::InnerSymAttr::get(context, properties);
154 reg.setInnerSymAttr(attr);
163 return hw::InnerRefAttr::get(mod, tgt);
179 auto name = SymbolTable::getSymbolName(module);
181 std::vector<BuriedFirReg> result;
182 for (
auto &op : *
module.getBodyBlock()) {
183 for (auto ®ion : op.getRegions()) {
184 region.walk([&](FirRegOp reg) {
185 auto ref = getInnerRefTo(name, isns, reg);
186 result.push_back({
reg, ref});
198 auto *context = top.getContext();
199 std::vector<BuriedFirReg> init;
201 const std::vector<HWModuleOp> modules(ms.begin(), ms.end());
203 [](std::vector<BuriedFirReg> acc,
204 std::vector<BuriedFirReg> &&xs) -> std::vector<BuriedFirReg> {
205 acc.insert(acc.end(), xs.begin(), xs.end());
213 BuriedFirReg entry) {
214 auto modName = entry.ref.getModule().getValue();
215 auto symName = entry.ref.getName().getValue();
216 auto name = ns.
newName(Twine(modName) +
"_" + symName);
219 OpBuilder::InsertionGuard guard(builder);
220 builder.setInsertionPoint(entry.reg->getParentOfType<
HWModuleOp>());
222 auto path = builder.getArrayAttr({entry.ref});
223 return hw::HierPathOp::create(builder, entry.reg.getLoc(), name, path);
227 auto builder = OpBuilder::atBlockBegin(top.getBody());
239 bool disableRegRandomization,
240 bool emitSeparateAlwaysBlocks)
241 : pathTable(pathTable), typeConverter(typeConverter), module(module),
242 disableRegRandomization(disableRegRandomization),
243 emitSeparateAlwaysBlocks(emitSeparateAlwaysBlocks) {
250 module->removeAttr("firrtl.random_init_width");
255 auto cond = ifDefOp.getCond();
261 if (ifDefOp.hasElse()) {
270 for (
auto &op : llvm::make_early_inc_range(*block)) {
271 if (
auto ifDefOp = dyn_cast<sv::IfDefOp>(op)) {
275 if (
auto regOp = dyn_cast<seq::FirRegOp>(op)) {
279 for (
auto ®ion : op.getRegions())
280 for (
auto &block : region.getBlocks())
293 if (
reg.randStart >= 0)
294 maxBit = std::max(maxBit, (uint64_t)
reg.randStart +
reg.width);
297 if (
reg.randStart == -1) {
298 reg.randStart = maxBit;
304 SmallVector<Value> randValues;
305 auto numRandomCalls = (maxBit + 31) / 32;
306 auto logic = sv::LogicOp::create(
308 hw::UnpackedArrayType::get(builder.getIntegerType(32), numRandomCalls),
312 auto inducionVariableWidth = llvm::Log2_64_Ceil(numRandomCalls + 1);
313 auto arrayIndexWith = llvm::Log2_64_Ceil(numRandomCalls);
318 auto forLoop = sv::ForOp::create(
319 builder, loc, lb, ub, step,
"i", [&](BlockArgument iter) {
320 auto rhs = sv::MacroRefExprSEOp::create(
321 builder, loc, builder.getIntegerType(32),
"RANDOM");
322 Value iterValue = iter;
323 if (!iter.getType().isInteger(arrayIndexWith))
327 sv::ArrayIndexInOutOp::create(builder, loc, logic, iterValue);
328 sv::BPAssignOp::create(builder, loc, lhs, rhs);
330 builder.setInsertionPointAfter(forLoop);
331 for (uint64_t x = 0; x < numRandomCalls; ++x) {
332 auto lhs = sv::ArrayIndexInOutOp::create(
335 randValues.push_back(lhs.getResult());
343 sv::MacroIdentAttr::get(builder.getContext(),
"RANDOMIZE_REG_INIT");
346 sv::IfDefProceduralOp::create(builder,
"INIT_RANDOM_PROLOG_", [&] {
347 sv::VerbatimOp::create(builder,
"`INIT_RANDOM_PROLOG_");
350 sv::IfDefProceduralOp::create(builder, randInitRef, [&] {
360 OpBuilder::InsertionGuard guard(builder);
362 auto loc = svReg.reg.getLoc();
363 auto elemTy = svReg.reg.getType().getElementType();
367 if (cst.getType() == elemTy)
373 Value target = svReg.reg;
375 target =
buildXMRTo(builder, svReg.path, svReg.reg.getLoc(),
376 svReg.reg.getType());
378 sv::BPAssignOp::create(builder, loc, target, rhs);
386 ImplicitLocOpBuilder &builder) {
388 OpBuilder::InsertionGuard guard(builder);
393 sv::IfOp::create(builder, reset.first, [&]() {
394 for (auto ® : reset.second) {
395 OpBuilder::InsertionGuard guard(builder);
396 buildRegConditions(builder, reg.reg);
397 Value target = reg.reg;
399 target = buildXMRTo(builder, reg.path, reg.reg.getLoc(),
401 sv::BPAssignOp::create(builder, reg.reg.getLoc(), target,
402 reg.asyncResetValue);
426 auto loc =
module.getLoc();
428 ImplicitLocOpBuilder::atBlockTerminator(loc, module.getBodyBlock());
430 sv::IfDefOp::create(builder,
"ENABLE_INITIAL_REG_", [&] {
431 sv::OrderedOutputOp::create(builder, [&] {
432 sv::IfDefOp::create(builder,
"FIRRTL_BEFORE_INITIAL", [&] {
433 sv::VerbatimOp::create(builder,
"`FIRRTL_BEFORE_INITIAL");
436 sv::InitialOp::create(builder, [&] {
442 sv::IfDefOp::create(builder,
"FIRRTL_AFTER_INITIAL", [&] {
443 sv::VerbatimOp::create(builder,
"`FIRRTL_AFTER_INITIAL");
462 return c1.getType() == c2.getType() &&
463 c1.getValue() == c2.getValue() &&
473 if (!andOp || !andOp.getTwoState()) {
474 llvm::SetVector<Value> ret;
479 return llvm::SetVector<Value>(andOp.getOperands().begin(),
480 andOp.getOperands().end());
484 auto constantIndex = value.template getDefiningOp<hw::ConstantOp>();
486 return constantIndex.getValue();
495std::optional<std::tuple<Value, Value, Value>>
499 SmallVector<Value> muxConditions;
502 SmallVector<Value> reverseOpValues(llvm::reverse(nextRegValue.getOperands()));
503 if (!llvm::all_of(llvm::enumerate(reverseOpValues), [&](
auto idxAndValue) {
505 auto [i, value] = idxAndValue;
506 auto mux = value.template getDefiningOp<comb::MuxOp>();
508 if (!mux || !mux.getTwoState())
511 if (trueVal && trueVal != mux.getTrueValue())
514 trueVal = mux.getTrueValue();
515 muxConditions.push_back(mux.getCond());
519 mux.getFalseValue().template getDefiningOp<hw::ArrayGetOp>();
528 llvm::SetVector<Value> commonConditions =
530 for (
auto condition : ArrayRef(muxConditions).drop_front()) {
532 commonConditions.remove_if([&](
auto v) {
return !cond.contains(v); });
535 for (
auto [idx, condition] : llvm::enumerate(muxConditions)) {
539 extractedConditions.remove_if(
540 [&](
auto v) {
return commonConditions.contains(v); });
541 if (extractedConditions.size() != 1)
545 (*extractedConditions.begin()).getDefiningOp<comb::ICmpOp>();
546 if (!indexCompare || !indexCompare.getTwoState() ||
547 indexCompare.getPredicate() != comb::ICmpPredicate::eq)
550 if (indexValue && indexValue != indexCompare.getLhs())
553 indexValue = indexCompare.getLhs();
558 OpBuilder::InsertionGuard guard(builder);
559 builder.setInsertionPointAfterValue(
reg);
560 Value commonConditionValue;
561 if (commonConditions.empty())
564 commonConditionValue = builder.createOrFold<
comb::AndOp>(
565 reg.getLoc(), builder.getI1Type(), commonConditions.takeVector(),
true);
566 return std::make_tuple(commonConditionValue, indexValue, trueVal);
572 constexpr size_t limit = 1024;
584 auto firReg = term.getDefiningOp<seq::FirRegOp>();
586 std::deque<std::tuple<Block *, Value, Value, Value>> worklist;
587 auto addToWorklist = [&](Value
reg, Value term, Value next) {
588 worklist.emplace_back(builder.getBlock(),
reg, term, next);
591 auto getArrayIndex = [&](Value
reg, Value idx) {
593 OpBuilder::InsertionGuard guard(builder);
594 builder.setInsertionPointAfterValue(
reg);
595 return sv::ArrayIndexInOutOp::create(builder,
reg.getLoc(),
reg, idx);
598 SmallVector<Value, 8> opsToDelete;
599 addToWorklist(
reg, term, next);
600 while (!worklist.empty()) {
601 OpBuilder::InsertionGuard guard(builder);
603 Value
reg, term, next;
604 std::tie(block,
reg, term, next) = worklist.front();
605 worklist.pop_front();
607 builder.setInsertionPointToEnd(block);
614 if (mux && mux.getTwoState() &&
616 if (counter >= limit) {
617 sv::PAssignOp::create(builder, term.getLoc(),
reg, next);
621 builder, mux.getCond(),
622 [&]() { addToWorklist(reg, term, mux.getTrueValue()); },
623 [&]() { addToWorklist(reg, term, mux.getFalseValue()); });
631 if (
auto matchResultOpt =
633 Value cond, index, trueValue;
634 std::tie(cond, index, trueValue) = *matchResultOpt;
638 Value nextReg = getArrayIndex(
reg, index);
644 opsToDelete.push_back(termElement);
645 addToWorklist(nextReg, termElement, trueValue);
655 for (
auto [idx, value] : llvm::enumerate(array.getOperands())) {
656 idx = array.getOperands().size() - idx - 1;
660 APInt(std::max(1u, llvm::Log2_64_Ceil(array.getOperands().size())),
665 index = getArrayIndex(
reg, idxVal);
672 opsToDelete.push_back(termElement);
673 addToWorklist(index, termElement, value);
678 sv::PAssignOp::create(builder, term.getLoc(),
reg, next);
681 while (!opsToDelete.empty()) {
682 auto value = opsToDelete.pop_back_val();
683 assert(value.use_empty());
684 value.getDefiningOp()->erase();
689 Location loc =
reg.getLoc();
695 path = lookup->second;
697 ImplicitLocOpBuilder builder(
reg.getLoc(),
reg);
698 RegLowerInfo svReg{
nullptr, path,
reg.getPresetAttr(),
nullptr,
nullptr,
700 svReg.
reg = sv::RegOp::create(builder, loc, regTy,
reg.getNameAttr());
703 if (
auto attr =
reg->getAttrOfType<IntegerAttr>(
"firrtl.random_init_start"))
704 svReg.randStart = attr.getUInt();
707 reg->removeAttr(
"firrtl.random_init_start");
710 svReg.reg->setDialectAttrs(
reg->getDialectAttrs());
712 if (
auto innerSymAttr =
reg.getInnerSymAttr())
713 svReg.reg.setInnerSymAttr(innerSymAttr);
717 if (
reg.hasReset()) {
719 reg->getBlock(), sv::EventControl::AtPosEdge,
reg.getClk(),
723 if (reg.getIsAsync() && areEquivalentValues(reg, reg.getNext()))
724 sv::PAssignOp::create(b, reg.getLoc(), svReg.reg, reg);
726 createTree(b, svReg.reg, reg, reg.getNext());
728 reg.getIsAsync() ? sv::ResetType::AsyncReset : sv::ResetType::SyncReset,
729 sv::EventControl::AtPosEdge,
reg.getReset(),
730 [&](OpBuilder &builder) {
731 sv::PAssignOp::create(builder, loc, svReg.reg,
reg.getResetValue());
733 if (
reg.getIsAsync()) {
734 svReg.asyncResetSignal =
reg.getReset();
735 svReg.asyncResetValue =
reg.getResetValue();
739 reg->getBlock(), sv::EventControl::AtPosEdge,
reg.getClk(),
740 [&](OpBuilder &b) { createTree(b, svReg.reg, reg, reg.getNext()); });
751 if (svReg.asyncResetSignal)
752 asyncResets[svReg.asyncResetSignal].emplace_back(svReg);
760 reg.replaceAllUsesWith(regVal.getResult());
769 OpBuilder &builder, Value
reg,
772 auto type = cast<sv::InOutType>(
reg.getType()).getElementType();
773 if (
auto intTy = hw::type_dyn_cast<IntegerType>(type)) {
775 pos -= intTy.getWidth();
776 auto elem = builder.createOrFold<
comb::ExtractOp>(loc, randomSource, pos,
778 sv::BPAssignOp::create(builder, loc,
reg, elem);
779 }
else if (
auto array = hw::type_dyn_cast<hw::ArrayType>(type)) {
780 for (
unsigned i = 0, e = array.getNumElements(); i < e; ++i) {
783 loc, builder, sv::ArrayIndexInOutOp::create(builder, loc,
reg, index),
786 }
else if (
auto structType = hw::type_dyn_cast<hw::StructType>(type)) {
787 for (
auto e : structType.getElements())
790 sv::StructFieldInOutOp::create(builder, loc,
reg, e.name),
793 assert(
false &&
"unsupported type");
807 auto kind = condition.getKind();
809 auto ifDef = sv::IfDefProceduralOp::create(b,
reg.getLoc(),
810 condition.getMacro(), []() {});
811 b.setInsertionPointToEnd(ifDef.getThenBlock());
815 auto ifDef = sv::IfDefProceduralOp::create(
816 b,
reg.getLoc(), condition.getMacro(), []() {}, []() {});
818 b.setInsertionPointToEnd(ifDef.getElseBlock());
821 llvm_unreachable(
"unknown reg condition type");
826 ArrayRef<Value> rands) {
827 auto loc =
reg.reg.getLoc();
828 SmallVector<Value> nibbles;
832 OpBuilder::InsertionGuard guard(builder);
842 Value target =
reg.reg;
846 uint64_t width =
reg.width;
847 uint64_t offset =
reg.randStart;
849 auto index = offset / 32;
850 auto start = offset % 32;
851 auto nwidth = std::min(32 - start, width);
855 nibbles.push_back(elem);
860 unsigned pos =
reg.width;
866 Block *block, sv::EventControl clockEdge, Value clock,
867 const std::function<
void(OpBuilder &)> &body, sv::ResetType resetStyle,
868 sv::EventControl resetEdge, Value reset,
869 const std::function<
void(OpBuilder &)> &resetBody) {
870 auto loc = clock.getLoc();
871 ImplicitLocOpBuilder builder(loc, block,
getBlockEnd(block));
873 resetStyle, resetEdge, reset};
875 sv::AlwaysOp alwaysOp;
883 assert(resetStyle != sv::ResetType::NoReset);
896 auto createIfOp = [&]() {
899 insideIfOp = sv::IfOp::create(
900 builder, reset, []() {}, []() {});
902 if (resetStyle == sv::ResetType::AsyncReset) {
903 sv::EventControl events[] = {clockEdge, resetEdge};
904 Value clocks[] = {clock, reset};
906 alwaysOp = sv::AlwaysOp::create(builder, events, clocks, [&]() {
907 if (resetEdge == sv::EventControl::AtNegEdge)
908 llvm_unreachable(
"negative edge for reset is not expected");
912 alwaysOp = sv::AlwaysOp::create(builder, clockEdge, clock, createIfOp);
916 alwaysOp = sv::AlwaysOp::create(builder, clockEdge, clock);
917 insideIfOp =
nullptr;
922 assert(insideIfOp &&
"reset body must be initialized before");
924 ImplicitLocOpBuilder::atBlockEnd(loc, insideIfOp.getThenBlock());
925 resetBody(resetBuilder);
928 ImplicitLocOpBuilder::atBlockEnd(loc, insideIfOp.getElseBlock());
932 ImplicitLocOpBuilder::atBlockEnd(loc, alwaysOp.getBodyBlock());
assert(baseType &&"element must be base type")
static SmallVector< T > concat(const SmallVectorImpl< T > &a, const SmallVectorImpl< T > &b)
Returns a new vector containing the concatenation of vectors a and b.
static bool areEquivalentValues(Value term, Value next)
static InnerRefAttr getInnerRefTo(StringAttr mod, InnerSymbolNamespace &isns, seq::FirRegOp reg)
static StringAttr getInnerSymFor(InnerSymbolNamespace &innerSymNS, seq::FirRegOp reg)
Attach an inner-sym to field-id 0 of the given register, or use an existing inner-sym,...
static std::vector< BuriedFirReg > getAllBuriedRegs(ModuleOp top)
Locate all registers which are not at the top-level of their parent HW module.
static std::vector< BuriedFirReg > getBuriedRegs(HWModuleOp module)
Locate the registers under the given HW module, which are not at the top-level of the module body.
static Block::iterator getBlockEnd(Block *block)
Immediately before the terminator, if present. Otherwise, the block's end.
static Value buildXMRTo(OpBuilder &builder, HierPathOp path, Location loc, Type type)
static hw::HierPathOp getHierPathTo(OpBuilder &builder, Namespace &ns, BuriedFirReg entry)
Construct a hierarchical path op that targets the given register.
static std::optional< APInt > getConstantValue(Value value)
static llvm::SetVector< Value > extractConditions(Value value)
std::unique_ptr< ReachableMuxes > reachableMuxes
void initialize(OpBuilder &builder, RegLowerInfo reg, ArrayRef< Value > rands)
llvm::SmallDenseMap< std::pair< Value, unsigned >, Value > arrayIndexCache
void createAsyncResetInitialization(ImplicitLocOpBuilder &builder)
llvm::SmallDenseMap< IfKeyType, sv::IfOp > ifCache
static PathTable createPaths(mlir::ModuleOp top)
When a register is buried under an ifdef op, the initialization code at the footer of the HW module w...
DenseMap< seq::FirRegOp, hw::HierPathOp > PathTable
A map sending registers to their paths.
void createInitialBlock()
void addToIfBlock(OpBuilder &builder, Value cond, const std::function< void()> &trueSide, const std::function< void()> &falseSide)
FirRegLowering(TypeConverter &typeConverter, hw::HWModuleOp module, const PathTable &pathTable, bool disableRegRandomization=false, bool emitSeparateAlwaysBlocks=false)
std::optional< std::tuple< Value, Value, Value > > tryRestoringSubaccess(OpBuilder &builder, Value reg, Value term, hw::ArrayCreateOp nextRegValue)
void createRandomInitialization(ImplicitLocOpBuilder &builder)
void lowerUnderIfDef(sv::IfDefOp ifDefOp)
void lowerInBlock(Block *block)
void buildRegConditions(OpBuilder &b, sv::RegOp reg)
Recreate the ifdefs under which reg was defined.
const PathTable & pathTable
void lowerReg(seq::FirRegOp reg)
SmallVector< Value > createRandomizationVector(OpBuilder &builder, Location loc)
std::vector< RegCondition > conditions
The ambient ifdef conditions we have encountered while lowering.
void createTree(OpBuilder &builder, Value reg, Value term, Value next)
void createPresetInitialization(ImplicitLocOpBuilder &builder)
unsigned numSubaccessRestored
hw::ConstantOp getOrCreateConstant(Location loc, const APInt &value)
void addToAlwaysBlock(Block *block, sv::EventControl clockEdge, Value clock, const std::function< void(OpBuilder &)> &body, sv::ResetType resetStyle={}, sv::EventControl resetEdge={}, Value reset={}, const std::function< void(OpBuilder &)> &resetBody={})
SmallVector< RegLowerInfo > randomInitRegs
A list of registers discovered, bucketed by initialization style.
std::tuple< Block *, sv::EventControl, Value, sv::ResetType, sv::EventControl, Value > AlwaysKeyType
llvm::MapVector< Value, SmallVector< RegLowerInfo > > asyncResets
A map from async reset signal to the registers that use it.
void initializeRegisterElements(Location loc, OpBuilder &builder, Value reg, Value rand, unsigned &pos)
DenseMap< sv::RegOp, std::vector< RegCondition > > regConditionTable
A map from RegOps to the ifdef conditions under which they are defined.
TypeConverter & typeConverter
hw::HWModuleOp bool disableRegRandomization
bool emitSeparateAlwaysBlocks
SmallVector< RegLowerInfo > presetInitRegs
llvm::SmallDenseMap< AlwaysKeyType, std::pair< sv::AlwaysOp, sv::IfOp > > alwaysBlocks
A namespace that is used to store existing names and generate new names in some scope within the IR.
void add(mlir::ModuleOp module)
StringRef newName(const Twine &name)
Return a unique name, derived from the input name, and add the new name to the internal namespace.
void buildReachabilityFrom(Operation *startNode)
llvm::SmallPtrSet< Operation *, 16 > visited
HWModuleOp llvm::DenseMap< Operation *, llvm::SmallDenseSet< Operation * > > reachableMuxes
bool isMuxReachableFrom(seq::FirRegOp regOp, comb::MuxOp muxOp)
int64_t getBitWidth(mlir::Type type)
Return the hardware bit width of a type.
The InstanceGraph op interface, see InstanceGraphInterface.td for more details.
static ResultTy transformReduce(MLIRContext *context, IterTy begin, IterTy end, ResultTy init, ReduceFuncTy reduce, TransformFuncTy transform)
Wrapper for llvm::parallelTransformReduce that performs the transform_reduce serially when MLIR multi...
reg(value, clock, reset=None, reset_value=None, name=None, sym_name=None)
@ IfDefThen
The register is under an ifdef "then" branch.
@ IfDefElse
The register is under an ifdef "else" branch.
static std::function< bool(const Operation *op)> opAllowsReachability