12#include "mlir/IR/Threading.h"
13#include "mlir/Transforms/DialectConversion.h"
14#include "llvm/ADT/DenseSet.h"
15#include "llvm/Support/Debug.h"
24#define DEBUG_TYPE "lower-seq-firreg"
28 if (block->mightHaveTerminator())
29 return Block::iterator(block->getTerminator());
34 [](
const Operation *op) ->
bool {
35 return (isa<comb::MuxOp, ArrayGetOp, ArrayCreateOp>(op));
40 return llvm::any_of(regOp.getResult().getUsers(), [&](Operation *user) {
41 if (!OpUserInfo::opAllowsReachability(user))
43 buildReachabilityFrom(user);
44 return reachableMuxes[user].contains(muxOp);
56 if (
visited.contains(startNode))
61 llvm::SmallVector<OpUserInfo, 16> stk;
63 stk.emplace_back(startNode);
65 while (!stk.empty()) {
66 auto &info = stk.back();
67 Operation *currentNode = info.op;
70 if (info.getAndSetUnvisited())
73 if (info.userIter != info.userEnd) {
74 Operation *child = *info.userIter;
77 stk.emplace_back(child);
83 for (
auto *childOp : llvm::make_filter_range(
92 iter->getSecond().end());
100 const std::function<
void()> &trueSide,
101 const std::function<
void()> &falseSide) {
102 auto op =
ifCache.lookup({builder.getBlock(), cond});
107 sv::IfOp::create(builder, cond.getLoc(), cond, trueSide, falseSide);
108 ifCache.insert({{builder.getBlock(), cond}, newIfOp});
110 OpBuilder::InsertionGuard guard(builder);
111 builder.setInsertionPointToEnd(op.getThenBlock());
113 builder.setInsertionPointToEnd(op.getElseBlock());
122 auto attr =
reg.getInnerSymAttr();
127 if (
auto sym = attr.getSymIfExists(0))
131 auto *context =
reg->getContext();
133 auto hint =
reg.getName();
136 auto sym = StringAttr::get(context, innerSymNS.
newName(hint));
137 auto property = hw::InnerSymPropertiesAttr::get(sym);
141 SmallVector<hw::InnerSymPropertiesAttr> properties = {
property};
143 llvm::append_range(properties, attr.getProps());
146 attr = hw::InnerSymAttr::get(context, properties);
147 reg.setInnerSymAttr(attr);
156 return hw::InnerRefAttr::get(mod, tgt);
172 auto name = SymbolTable::getSymbolName(module);
174 std::vector<BuriedFirReg> result;
175 for (
auto &op : *
module.getBodyBlock()) {
176 for (auto ®ion : op.getRegions()) {
177 region.walk([&](FirRegOp reg) {
178 auto ref = getInnerRefTo(name, isns, reg);
179 result.push_back({
reg, ref});
191 auto *context = top.getContext();
192 std::vector<BuriedFirReg> init;
194 const std::vector<HWModuleOp> modules(ms.begin(), ms.end());
196 [](std::vector<BuriedFirReg> acc,
197 std::vector<BuriedFirReg> &&xs) -> std::vector<BuriedFirReg> {
198 acc.insert(acc.end(), xs.begin(), xs.end());
206 BuriedFirReg entry) {
207 auto modName = entry.ref.getModule().getValue();
208 auto symName = entry.ref.getName().getValue();
209 auto name = ns.
newName(Twine(modName) +
"_" + symName);
212 OpBuilder::InsertionGuard guard(builder);
213 builder.setInsertionPoint(entry.reg->getParentOfType<
HWModuleOp>());
215 auto path = builder.getArrayAttr({entry.ref});
216 return hw::HierPathOp::create(builder, entry.reg.getLoc(), name, path);
220 auto builder = OpBuilder::atBlockBegin(top.getBody());
232 bool disableRegRandomization,
233 bool emitSeparateAlwaysBlocks)
234 : pathTable(pathTable), typeConverter(typeConverter), module(module),
235 disableRegRandomization(disableRegRandomization),
236 emitSeparateAlwaysBlocks(emitSeparateAlwaysBlocks) {
243 module->removeAttr("firrtl.random_init_width");
248 auto cond = ifDefOp.getCond();
254 if (ifDefOp.hasElse()) {
263 for (
auto &op : llvm::make_early_inc_range(*block)) {
264 if (
auto ifDefOp = dyn_cast<sv::IfDefOp>(op)) {
268 if (
auto regOp = dyn_cast<seq::FirRegOp>(op)) {
272 for (
auto ®ion : op.getRegions())
273 for (
auto &block : region.getBlocks())
286 if (
reg.randStart >= 0)
287 maxBit = std::max(maxBit, (uint64_t)
reg.randStart +
reg.width);
290 if (
reg.randStart == -1) {
291 reg.randStart = maxBit;
297 SmallVector<Value> randValues;
298 auto numRandomCalls = (maxBit + 31) / 32;
299 auto logic = sv::LogicOp::create(
301 hw::UnpackedArrayType::get(builder.getIntegerType(32), numRandomCalls),
305 auto inducionVariableWidth = llvm::Log2_64_Ceil(numRandomCalls + 1);
306 auto arrayIndexWith = llvm::Log2_64_Ceil(numRandomCalls);
311 auto forLoop = sv::ForOp::create(
312 builder, loc, lb, ub, step,
"i", [&](BlockArgument iter) {
313 auto rhs = sv::MacroRefExprSEOp::create(
314 builder, loc, builder.getIntegerType(32),
"RANDOM");
315 Value iterValue = iter;
316 if (!iter.getType().isInteger(arrayIndexWith))
320 sv::ArrayIndexInOutOp::create(builder, loc, logic, iterValue);
321 sv::BPAssignOp::create(builder, loc, lhs, rhs);
323 builder.setInsertionPointAfter(forLoop);
324 for (uint64_t x = 0; x < numRandomCalls; ++x) {
325 auto lhs = sv::ArrayIndexInOutOp::create(
328 randValues.push_back(lhs.getResult());
336 sv::MacroIdentAttr::get(builder.getContext(),
"RANDOMIZE_REG_INIT");
339 sv::IfDefProceduralOp::create(builder,
"INIT_RANDOM_PROLOG_", [&] {
340 sv::VerbatimOp::create(builder,
"`INIT_RANDOM_PROLOG_");
343 sv::IfDefProceduralOp::create(builder, randInitRef, [&] {
353 auto loc = svReg.reg.getLoc();
354 auto elemTy = svReg.reg.getType().getElementType();
358 if (cst.getType() == elemTy)
363 sv::BPAssignOp::create(builder, loc, svReg.reg, rhs);
371 ImplicitLocOpBuilder &builder) {
376 sv::IfOp::create(builder, reset.first, [&] {
377 for (auto ® : reset.second)
378 sv::BPAssignOp::create(builder, reg.reg.getLoc(), reg.reg,
379 reg.asyncResetValue);
402 auto loc =
module.getLoc();
404 ImplicitLocOpBuilder::atBlockTerminator(loc, module.getBodyBlock());
406 sv::IfDefOp::create(builder,
"ENABLE_INITIAL_REG_", [&] {
407 sv::OrderedOutputOp::create(builder, [&] {
408 sv::IfDefOp::create(builder,
"FIRRTL_BEFORE_INITIAL", [&] {
409 sv::VerbatimOp::create(builder,
"`FIRRTL_BEFORE_INITIAL");
412 sv::InitialOp::create(builder, [&] {
418 sv::IfDefOp::create(builder,
"FIRRTL_AFTER_INITIAL", [&] {
419 sv::VerbatimOp::create(builder,
"`FIRRTL_AFTER_INITIAL");
438 return c1.getType() == c2.getType() &&
439 c1.getValue() == c2.getValue() &&
449 if (!andOp || !andOp.getTwoState()) {
450 llvm::SetVector<Value> ret;
455 return llvm::SetVector<Value>(andOp.getOperands().begin(),
456 andOp.getOperands().end());
460 auto constantIndex = value.template getDefiningOp<hw::ConstantOp>();
462 return constantIndex.getValue();
471std::optional<std::tuple<Value, Value, Value>>
475 SmallVector<Value> muxConditions;
478 SmallVector<Value> reverseOpValues(llvm::reverse(nextRegValue.getOperands()));
479 if (!llvm::all_of(llvm::enumerate(reverseOpValues), [&](
auto idxAndValue) {
481 auto [i, value] = idxAndValue;
482 auto mux = value.template getDefiningOp<comb::MuxOp>();
484 if (!mux || !mux.getTwoState())
487 if (trueVal && trueVal != mux.getTrueValue())
490 trueVal = mux.getTrueValue();
491 muxConditions.push_back(mux.getCond());
495 mux.getFalseValue().template getDefiningOp<hw::ArrayGetOp>();
504 llvm::SetVector<Value> commonConditions =
506 for (
auto condition : ArrayRef(muxConditions).drop_front()) {
508 commonConditions.remove_if([&](
auto v) {
return !cond.contains(v); });
511 for (
auto [idx, condition] : llvm::enumerate(muxConditions)) {
515 extractedConditions.remove_if(
516 [&](
auto v) {
return commonConditions.contains(v); });
517 if (extractedConditions.size() != 1)
521 (*extractedConditions.begin()).getDefiningOp<comb::ICmpOp>();
522 if (!indexCompare || !indexCompare.getTwoState() ||
523 indexCompare.getPredicate() != comb::ICmpPredicate::eq)
526 if (indexValue && indexValue != indexCompare.getLhs())
529 indexValue = indexCompare.getLhs();
534 OpBuilder::InsertionGuard guard(builder);
535 builder.setInsertionPointAfterValue(
reg);
536 Value commonConditionValue;
537 if (commonConditions.empty())
540 commonConditionValue = builder.createOrFold<
comb::AndOp>(
541 reg.getLoc(), builder.getI1Type(), commonConditions.takeVector(),
true);
542 return std::make_tuple(commonConditionValue, indexValue, trueVal);
548 constexpr size_t limit = 1024;
560 auto firReg = term.getDefiningOp<seq::FirRegOp>();
562 std::deque<std::tuple<Block *, Value, Value, Value>> worklist;
563 auto addToWorklist = [&](Value
reg, Value term, Value next) {
564 worklist.emplace_back(builder.getBlock(),
reg, term, next);
567 auto getArrayIndex = [&](Value
reg, Value idx) {
569 OpBuilder::InsertionGuard guard(builder);
570 builder.setInsertionPointAfterValue(
reg);
571 return sv::ArrayIndexInOutOp::create(builder,
reg.getLoc(),
reg, idx);
574 SmallVector<Value, 8> opsToDelete;
575 addToWorklist(
reg, term, next);
576 while (!worklist.empty()) {
577 OpBuilder::InsertionGuard guard(builder);
579 Value
reg, term, next;
580 std::tie(block,
reg, term, next) = worklist.front();
581 worklist.pop_front();
583 builder.setInsertionPointToEnd(block);
590 if (mux && mux.getTwoState() &&
592 if (counter >= limit) {
593 sv::PAssignOp::create(builder, term.getLoc(),
reg, next);
597 builder, mux.getCond(),
598 [&]() { addToWorklist(reg, term, mux.getTrueValue()); },
599 [&]() { addToWorklist(reg, term, mux.getFalseValue()); });
607 if (
auto matchResultOpt =
609 Value cond, index, trueValue;
610 std::tie(cond, index, trueValue) = *matchResultOpt;
614 Value nextReg = getArrayIndex(
reg, index);
620 opsToDelete.push_back(termElement);
621 addToWorklist(nextReg, termElement, trueValue);
631 for (
auto [idx, value] : llvm::enumerate(array.getOperands())) {
632 idx = array.getOperands().size() - idx - 1;
636 APInt(std::max(1u, llvm::Log2_64_Ceil(array.getOperands().size())),
641 index = getArrayIndex(
reg, idxVal);
648 opsToDelete.push_back(termElement);
649 addToWorklist(index, termElement, value);
654 sv::PAssignOp::create(builder, term.getLoc(),
reg, next);
657 while (!opsToDelete.empty()) {
658 auto value = opsToDelete.pop_back_val();
659 assert(value.use_empty());
660 value.getDefiningOp()->erase();
665 Location loc =
reg.getLoc();
671 path = lookup->second;
673 ImplicitLocOpBuilder builder(
reg.getLoc(),
reg);
674 RegLowerInfo svReg{
nullptr, path,
reg.getPresetAttr(),
nullptr,
nullptr,
676 svReg.
reg = sv::RegOp::create(builder, loc, regTy,
reg.getNameAttr());
679 if (
auto attr =
reg->getAttrOfType<IntegerAttr>(
"firrtl.random_init_start"))
680 svReg.randStart = attr.getUInt();
683 reg->removeAttr(
"firrtl.random_init_start");
686 svReg.reg->setDialectAttrs(
reg->getDialectAttrs());
688 if (
auto innerSymAttr =
reg.getInnerSymAttr())
689 svReg.reg.setInnerSymAttr(innerSymAttr);
693 if (
reg.hasReset()) {
695 reg->getBlock(), sv::EventControl::AtPosEdge,
reg.getClk(),
699 if (reg.getIsAsync() && areEquivalentValues(reg, reg.getNext()))
700 sv::PAssignOp::create(b, reg.getLoc(), svReg.reg, reg);
702 createTree(b, svReg.reg, reg, reg.getNext());
704 reg.getIsAsync() ? sv::ResetType::AsyncReset : sv::ResetType::SyncReset,
705 sv::EventControl::AtPosEdge,
reg.getReset(),
706 [&](OpBuilder &builder) {
707 sv::PAssignOp::create(builder, loc, svReg.reg,
reg.getResetValue());
709 if (
reg.getIsAsync()) {
710 svReg.asyncResetSignal =
reg.getReset();
711 svReg.asyncResetValue =
reg.getResetValue();
715 reg->getBlock(), sv::EventControl::AtPosEdge,
reg.getClk(),
716 [&](OpBuilder &b) { createTree(b, svReg.reg, reg, reg.getNext()); });
727 if (svReg.asyncResetSignal)
728 asyncResets[svReg.asyncResetSignal].emplace_back(svReg);
736 reg.replaceAllUsesWith(regVal.getResult());
745 OpBuilder &builder, Value
reg,
748 auto type = cast<sv::InOutType>(
reg.getType()).getElementType();
749 if (
auto intTy = hw::type_dyn_cast<IntegerType>(type)) {
751 pos -= intTy.getWidth();
752 auto elem = builder.createOrFold<
comb::ExtractOp>(loc, randomSource, pos,
754 sv::BPAssignOp::create(builder, loc,
reg, elem);
755 }
else if (
auto array = hw::type_dyn_cast<hw::ArrayType>(type)) {
756 for (
unsigned i = 0, e = array.getNumElements(); i < e; ++i) {
759 loc, builder, sv::ArrayIndexInOutOp::create(builder, loc,
reg, index),
762 }
else if (
auto structType = hw::type_dyn_cast<hw::StructType>(type)) {
763 for (
auto e : structType.getElements())
766 sv::StructFieldInOutOp::create(builder, loc,
reg, e.name),
769 assert(
false &&
"unsupported type");
783 auto kind = condition.getKind();
785 auto ifDef = sv::IfDefProceduralOp::create(b,
reg.getLoc(),
786 condition.getMacro(), []() {});
787 b.setInsertionPointToEnd(ifDef.getThenBlock());
791 auto ifDef = sv::IfDefProceduralOp::create(
792 b,
reg.getLoc(), condition.getMacro(), []() {}, []() {});
794 b.setInsertionPointToEnd(ifDef.getElseBlock());
797 llvm_unreachable(
"unknown reg condition type");
801static Value
buildXMRTo(OpBuilder &builder, HierPathOp path, Location loc,
803 auto name = path.getSymNameAttr();
804 auto ref = mlir::FlatSymbolRefAttr::get(name);
805 return sv::XMRRefOp::create(builder, loc, type, ref);
809 ArrayRef<Value> rands) {
810 auto loc =
reg.reg.getLoc();
811 SmallVector<Value> nibbles;
815 OpBuilder::InsertionGuard guard(builder);
825 Value target =
reg.reg;
829 uint64_t width =
reg.width;
830 uint64_t offset =
reg.randStart;
832 auto index = offset / 32;
833 auto start = offset % 32;
834 auto nwidth = std::min(32 - start, width);
838 nibbles.push_back(elem);
843 unsigned pos =
reg.width;
849 Block *block, sv::EventControl clockEdge, Value clock,
850 const std::function<
void(OpBuilder &)> &body, sv::ResetType resetStyle,
851 sv::EventControl resetEdge, Value reset,
852 const std::function<
void(OpBuilder &)> &resetBody) {
853 auto loc = clock.getLoc();
854 ImplicitLocOpBuilder builder(loc, block,
getBlockEnd(block));
856 resetStyle, resetEdge, reset};
858 sv::AlwaysOp alwaysOp;
866 assert(resetStyle != sv::ResetType::NoReset);
879 auto createIfOp = [&]() {
882 insideIfOp = sv::IfOp::create(
883 builder, reset, []() {}, []() {});
885 if (resetStyle == sv::ResetType::AsyncReset) {
886 sv::EventControl events[] = {clockEdge, resetEdge};
887 Value clocks[] = {clock, reset};
889 alwaysOp = sv::AlwaysOp::create(builder, events, clocks, [&]() {
890 if (resetEdge == sv::EventControl::AtNegEdge)
891 llvm_unreachable(
"negative edge for reset is not expected");
895 alwaysOp = sv::AlwaysOp::create(builder, clockEdge, clock, createIfOp);
899 alwaysOp = sv::AlwaysOp::create(builder, clockEdge, clock);
900 insideIfOp =
nullptr;
905 assert(insideIfOp &&
"reset body must be initialized before");
907 ImplicitLocOpBuilder::atBlockEnd(loc, insideIfOp.getThenBlock());
908 resetBody(resetBuilder);
911 ImplicitLocOpBuilder::atBlockEnd(loc, insideIfOp.getElseBlock());
915 ImplicitLocOpBuilder::atBlockEnd(loc, alwaysOp.getBodyBlock());
assert(baseType &&"element must be base type")
static SmallVector< T > concat(const SmallVectorImpl< T > &a, const SmallVectorImpl< T > &b)
Returns a new vector containing the concatenation of vectors a and b.
static bool areEquivalentValues(Value term, Value next)
static InnerRefAttr getInnerRefTo(StringAttr mod, InnerSymbolNamespace &isns, seq::FirRegOp reg)
static StringAttr getInnerSymFor(InnerSymbolNamespace &innerSymNS, seq::FirRegOp reg)
Attach an inner-sym to field-id 0 of the given register, or use an existing inner-sym,...
static std::vector< BuriedFirReg > getAllBuriedRegs(ModuleOp top)
Locate all registers which are not at the top-level of their parent HW module.
static std::vector< BuriedFirReg > getBuriedRegs(HWModuleOp module)
Locate the registers under the given HW module, which are not at the top-level of the module body.
static Block::iterator getBlockEnd(Block *block)
Immediately before the terminator, if present. Otherwise, the block's end.
static Value buildXMRTo(OpBuilder &builder, HierPathOp path, Location loc, Type type)
static hw::HierPathOp getHierPathTo(OpBuilder &builder, Namespace &ns, BuriedFirReg entry)
Construct a hierarchical path op that targets the given register.
static std::optional< APInt > getConstantValue(Value value)
static llvm::SetVector< Value > extractConditions(Value value)
std::unique_ptr< ReachableMuxes > reachableMuxes
void initialize(OpBuilder &builder, RegLowerInfo reg, ArrayRef< Value > rands)
llvm::SmallDenseMap< std::pair< Value, unsigned >, Value > arrayIndexCache
void createAsyncResetInitialization(ImplicitLocOpBuilder &builder)
llvm::SmallDenseMap< IfKeyType, sv::IfOp > ifCache
static PathTable createPaths(mlir::ModuleOp top)
When a register is buried under an ifdef op, the initialization code at the footer of the HW module w...
DenseMap< seq::FirRegOp, hw::HierPathOp > PathTable
A map sending registers to their paths.
void createInitialBlock()
void addToIfBlock(OpBuilder &builder, Value cond, const std::function< void()> &trueSide, const std::function< void()> &falseSide)
FirRegLowering(TypeConverter &typeConverter, hw::HWModuleOp module, const PathTable &pathTable, bool disableRegRandomization=false, bool emitSeparateAlwaysBlocks=false)
std::optional< std::tuple< Value, Value, Value > > tryRestoringSubaccess(OpBuilder &builder, Value reg, Value term, hw::ArrayCreateOp nextRegValue)
void createRandomInitialization(ImplicitLocOpBuilder &builder)
void lowerUnderIfDef(sv::IfDefOp ifDefOp)
void lowerInBlock(Block *block)
void buildRegConditions(OpBuilder &b, sv::RegOp reg)
Recreate the ifdefs under which reg was defined.
const PathTable & pathTable
void lowerReg(seq::FirRegOp reg)
SmallVector< Value > createRandomizationVector(OpBuilder &builder, Location loc)
std::vector< RegCondition > conditions
The ambient ifdef conditions we have encountered while lowering.
void createTree(OpBuilder &builder, Value reg, Value term, Value next)
void createPresetInitialization(ImplicitLocOpBuilder &builder)
unsigned numSubaccessRestored
hw::ConstantOp getOrCreateConstant(Location loc, const APInt &value)
void addToAlwaysBlock(Block *block, sv::EventControl clockEdge, Value clock, const std::function< void(OpBuilder &)> &body, sv::ResetType resetStyle={}, sv::EventControl resetEdge={}, Value reset={}, const std::function< void(OpBuilder &)> &resetBody={})
SmallVector< RegLowerInfo > randomInitRegs
A list of registers discovered, bucketed by initialization style.
std::tuple< Block *, sv::EventControl, Value, sv::ResetType, sv::EventControl, Value > AlwaysKeyType
llvm::MapVector< Value, SmallVector< RegLowerInfo > > asyncResets
A map from async reset signal to the registers that use it.
void initializeRegisterElements(Location loc, OpBuilder &builder, Value reg, Value rand, unsigned &pos)
DenseMap< sv::RegOp, std::vector< RegCondition > > regConditionTable
A map from RegOps to the ifdef conditions under which they are defined.
TypeConverter & typeConverter
hw::HWModuleOp bool disableRegRandomization
bool emitSeparateAlwaysBlocks
SmallVector< RegLowerInfo > presetInitRegs
llvm::SmallDenseMap< AlwaysKeyType, std::pair< sv::AlwaysOp, sv::IfOp > > alwaysBlocks
A namespace that is used to store existing names and generate new names in some scope within the IR.
void add(mlir::ModuleOp module)
StringRef newName(const Twine &name)
Return a unique name, derived from the input name, and add the new name to the internal namespace.
void buildReachabilityFrom(Operation *startNode)
llvm::SmallPtrSet< Operation *, 16 > visited
HWModuleOp llvm::DenseMap< Operation *, llvm::SmallDenseSet< Operation * > > reachableMuxes
bool isMuxReachableFrom(seq::FirRegOp regOp, comb::MuxOp muxOp)
int64_t getBitWidth(mlir::Type type)
Return the hardware bit width of a type.
The InstanceGraph op interface, see InstanceGraphInterface.td for more details.
static ResultTy transformReduce(MLIRContext *context, IterTy begin, IterTy end, ResultTy init, ReduceFuncTy reduce, TransformFuncTy transform)
Wrapper for llvm::parallelTransformReduce that performs the transform_reduce serially when MLIR multi...
reg(value, clock, reset=None, reset_value=None, name=None, sym_name=None)
@ IfDefThen
The register is under an ifdef "then" branch.
@ IfDefElse
The register is under an ifdef "else" branch.
static std::function< bool(const Operation *op)> opAllowsReachability