12#include "mlir/IR/Threading.h"
13#include "mlir/Transforms/DialectConversion.h"
14#include "llvm/ADT/DenseSet.h"
15#include "llvm/Support/Debug.h"
24#define DEBUG_TYPE "lower-seq-firreg"
28 if (block->mightHaveTerminator())
29 return Block::iterator(block->getTerminator());
34 [](
const Operation *op) ->
bool {
35 return (isa<comb::MuxOp, ArrayGetOp, ArrayCreateOp>(op));
40 return llvm::any_of(regOp.getResult().getUsers(), [&](Operation *user) {
41 if (!OpUserInfo::opAllowsReachability(user))
43 buildReachabilityFrom(user);
44 return reachableMuxes[user].contains(muxOp);
56 if (
visited.contains(startNode))
61 llvm::SmallVector<OpUserInfo, 16> stk;
63 stk.emplace_back(startNode);
65 while (!stk.empty()) {
66 auto &info = stk.back();
67 Operation *currentNode = info.op;
70 if (info.getAndSetUnvisited())
73 if (info.userIter != info.userEnd) {
74 Operation *child = *info.userIter;
77 stk.emplace_back(child);
83 for (
auto *childOp : llvm::make_filter_range(
92 iter->getSecond().end());
100 const std::function<
void()> &trueSide,
101 const std::function<
void()> &falseSide) {
102 auto op =
ifCache.lookup({builder.getBlock(), cond});
107 builder.create<sv::IfOp>(cond.getLoc(), cond, trueSide, falseSide);
108 ifCache.insert({{builder.getBlock(), cond}, newIfOp});
110 OpBuilder::InsertionGuard guard(builder);
111 builder.setInsertionPointToEnd(op.getThenBlock());
113 builder.setInsertionPointToEnd(op.getElseBlock());
122 auto attr =
reg.getInnerSymAttr();
127 if (
auto sym = attr.getSymIfExists(0))
131 auto *context =
reg->getContext();
133 auto hint =
reg.getName();
136 auto sym = StringAttr::get(context, innerSymNS.
newName(hint));
137 auto property = hw::InnerSymPropertiesAttr::get(sym);
141 SmallVector<hw::InnerSymPropertiesAttr> properties = {
property};
143 llvm::append_range(properties, attr.getProps());
146 attr = hw::InnerSymAttr::get(context, properties);
147 reg.setInnerSymAttr(attr);
156 return hw::InnerRefAttr::get(mod, tgt);
172 auto name = SymbolTable::getSymbolName(module);
174 std::vector<BuriedFirReg> result;
175 for (
auto &op : *
module.getBodyBlock()) {
176 for (auto ®ion : op.getRegions()) {
177 region.walk([&](FirRegOp reg) {
178 auto ref = getInnerRefTo(name, isns, reg);
179 result.push_back({
reg, ref});
191 auto *context = top.getContext();
192 std::vector<BuriedFirReg> init;
194 const std::vector<HWModuleOp> modules(ms.begin(), ms.end());
196 [](std::vector<BuriedFirReg> acc,
197 std::vector<BuriedFirReg> &&xs) -> std::vector<BuriedFirReg> {
198 acc.insert(acc.end(), xs.begin(), xs.end());
206 BuriedFirReg entry) {
207 auto modName = entry.ref.getModule().getValue();
208 auto symName = entry.ref.getName().getValue();
209 auto name = ns.
newName(Twine(modName) +
"_" + symName);
212 OpBuilder::InsertionGuard guard(builder);
213 builder.setInsertionPoint(entry.reg->getParentOfType<
HWModuleOp>());
215 auto path = builder.getArrayAttr({entry.ref});
216 return builder.create<hw::HierPathOp>(entry.reg.getLoc(), name, path);
220 auto builder = OpBuilder::atBlockBegin(top.getBody());
232 bool disableRegRandomization,
233 bool emitSeparateAlwaysBlocks)
234 : pathTable(pathTable), typeConverter(typeConverter), module(module),
235 disableRegRandomization(disableRegRandomization),
236 emitSeparateAlwaysBlocks(emitSeparateAlwaysBlocks) {
243 module->removeAttr("firrtl.random_init_width");
248 auto cond = ifDefOp.getCond();
254 if (ifDefOp.hasElse()) {
263 for (
auto &op : llvm::make_early_inc_range(*block)) {
264 if (
auto ifDefOp = dyn_cast<sv::IfDefOp>(op)) {
268 if (
auto regOp = dyn_cast<seq::FirRegOp>(op)) {
272 for (
auto ®ion : op.getRegions())
273 for (
auto &block : region.getBlocks())
286 if (
reg.randStart >= 0)
287 maxBit = std::max(maxBit, (uint64_t)
reg.randStart +
reg.width);
290 if (
reg.randStart == -1) {
291 reg.randStart = maxBit;
297 SmallVector<Value> randValues;
298 auto numRandomCalls = (maxBit + 31) / 32;
299 auto logic = builder.create<sv::LogicOp>(
301 hw::UnpackedArrayType::get(builder.getIntegerType(32), numRandomCalls),
305 auto inducionVariableWidth = llvm::Log2_64_Ceil(numRandomCalls + 1);
306 auto arrayIndexWith = llvm::Log2_64_Ceil(numRandomCalls);
311 auto forLoop = builder.create<sv::ForOp>(
312 loc, lb, ub, step,
"i", [&](BlockArgument iter) {
313 auto rhs = builder.create<sv::MacroRefExprSEOp>(
314 loc, builder.getIntegerType(32),
"RANDOM");
315 Value iterValue = iter;
316 if (!iter.getType().isInteger(arrayIndexWith))
319 auto lhs = builder.
create<sv::ArrayIndexInOutOp>(loc, logic, iterValue);
320 builder.create<sv::BPAssignOp>(loc, lhs, rhs);
322 builder.setInsertionPointAfter(forLoop);
323 for (uint64_t x = 0; x < numRandomCalls; ++x) {
324 auto lhs = builder.create<sv::ArrayIndexInOutOp>(
326 randValues.push_back(lhs.getResult());
334 sv::MacroIdentAttr::get(builder.getContext(),
"RANDOMIZE_REG_INIT");
337 builder.create<sv::IfDefProceduralOp>(
"INIT_RANDOM_PROLOG_", [&] {
338 builder.create<sv::VerbatimOp>(
"`INIT_RANDOM_PROLOG_");
341 builder.create<sv::IfDefProceduralOp>(randInitRef, [&] {
351 auto loc = svReg.reg.getLoc();
352 auto elemTy = svReg.reg.getType().getElementType();
356 if (cst.getType() == elemTy)
361 builder.
create<sv::BPAssignOp>(loc, svReg.reg, rhs);
369 ImplicitLocOpBuilder &builder) {
374 builder.create<sv::IfOp>(reset.first, [&] {
375 for (
auto &
reg : reset.second)
376 builder.create<sv::BPAssignOp>(
reg.reg.getLoc(),
reg.reg,
377 reg.asyncResetValue);
400 auto loc =
module.getLoc();
402 ImplicitLocOpBuilder::atBlockTerminator(loc, module.getBodyBlock());
404 builder.create<
sv::IfDefOp>(
"ENABLE_INITIAL_REG_", [&] {
405 builder.create<sv::OrderedOutputOp>([&] {
406 builder.create<
sv::IfDefOp>(
"FIRRTL_BEFORE_INITIAL", [&] {
407 builder.create<sv::VerbatimOp>(
"`FIRRTL_BEFORE_INITIAL");
410 builder.create<sv::InitialOp>([&] {
416 builder.create<
sv::IfDefOp>(
"FIRRTL_AFTER_INITIAL", [&] {
417 builder.create<sv::VerbatimOp>(
"`FIRRTL_AFTER_INITIAL");
436 return c1.getType() == c2.getType() &&
437 c1.getValue() == c2.getValue() &&
447 if (!andOp || !andOp.getTwoState()) {
448 llvm::SetVector<Value> ret;
453 return llvm::SetVector<Value>(andOp.getOperands().begin(),
454 andOp.getOperands().end());
458 auto constantIndex = value.template getDefiningOp<hw::ConstantOp>();
460 return constantIndex.getValue();
469std::optional<std::tuple<Value, Value, Value>>
473 SmallVector<Value> muxConditions;
476 SmallVector<Value> reverseOpValues(llvm::reverse(nextRegValue.getOperands()));
477 if (!llvm::all_of(llvm::enumerate(reverseOpValues), [&](
auto idxAndValue) {
479 auto [i, value] = idxAndValue;
480 auto mux = value.template getDefiningOp<comb::MuxOp>();
482 if (!mux || !mux.getTwoState())
485 if (trueVal && trueVal != mux.getTrueValue())
488 trueVal = mux.getTrueValue();
489 muxConditions.push_back(mux.getCond());
493 mux.getFalseValue().template getDefiningOp<hw::ArrayGetOp>();
502 llvm::SetVector<Value> commonConditions =
504 for (
auto condition : ArrayRef(muxConditions).drop_front()) {
506 commonConditions.remove_if([&](
auto v) {
return !cond.contains(v); });
509 for (
auto [idx, condition] : llvm::enumerate(muxConditions)) {
513 extractedConditions.remove_if(
514 [&](
auto v) {
return commonConditions.contains(v); });
515 if (extractedConditions.size() != 1)
519 (*extractedConditions.begin()).getDefiningOp<comb::ICmpOp>();
520 if (!indexCompare || !indexCompare.getTwoState() ||
521 indexCompare.getPredicate() != comb::ICmpPredicate::eq)
524 if (indexValue && indexValue != indexCompare.getLhs())
527 indexValue = indexCompare.getLhs();
532 OpBuilder::InsertionGuard guard(builder);
533 builder.setInsertionPointAfterValue(
reg);
534 Value commonConditionValue;
535 if (commonConditions.empty())
538 commonConditionValue = builder.createOrFold<
comb::AndOp>(
539 reg.getLoc(), builder.getI1Type(), commonConditions.takeVector(),
true);
540 return std::make_tuple(commonConditionValue, indexValue, trueVal);
546 constexpr size_t limit = 1024;
558 auto firReg = term.getDefiningOp<seq::FirRegOp>();
560 std::deque<std::tuple<Block *, Value, Value, Value>> worklist;
561 auto addToWorklist = [&](Value
reg, Value term, Value next) {
562 worklist.emplace_back(builder.getBlock(),
reg, term, next);
565 auto getArrayIndex = [&](Value
reg, Value idx) {
567 OpBuilder::InsertionGuard guard(builder);
568 builder.setInsertionPointAfterValue(
reg);
569 return builder.create<sv::ArrayIndexInOutOp>(
reg.getLoc(),
reg, idx);
572 SmallVector<Value, 8> opsToDelete;
573 addToWorklist(
reg, term, next);
574 while (!worklist.empty()) {
575 OpBuilder::InsertionGuard guard(builder);
577 Value
reg, term, next;
578 std::tie(block,
reg, term, next) = worklist.front();
579 worklist.pop_front();
581 builder.setInsertionPointToEnd(block);
588 if (mux && mux.getTwoState() &&
590 if (counter >= limit) {
591 builder.create<sv::PAssignOp>(term.getLoc(),
reg, next);
595 builder, mux.getCond(),
596 [&]() { addToWorklist(reg, term, mux.getTrueValue()); },
597 [&]() { addToWorklist(reg, term, mux.getFalseValue()); });
605 if (
auto matchResultOpt =
607 Value cond, index, trueValue;
608 std::tie(cond, index, trueValue) = *matchResultOpt;
612 Value nextReg = getArrayIndex(
reg, index);
618 opsToDelete.push_back(termElement);
619 addToWorklist(nextReg, termElement, trueValue);
629 for (
auto [idx, value] : llvm::enumerate(array.getOperands())) {
630 idx = array.getOperands().size() - idx - 1;
634 APInt(std::max(1u, llvm::Log2_64_Ceil(array.getOperands().size())),
639 index = getArrayIndex(
reg, idxVal);
646 opsToDelete.push_back(termElement);
647 addToWorklist(index, termElement, value);
652 builder.create<sv::PAssignOp>(term.getLoc(),
reg, next);
655 while (!opsToDelete.empty()) {
656 auto value = opsToDelete.pop_back_val();
657 assert(value.use_empty());
658 value.getDefiningOp()->erase();
663 Location loc =
reg.getLoc();
669 path = lookup->second;
671 ImplicitLocOpBuilder builder(
reg.getLoc(),
reg);
672 RegLowerInfo svReg{
nullptr, path,
reg.getPresetAttr(),
nullptr,
nullptr,
677 if (
auto attr =
reg->getAttrOfType<IntegerAttr>(
"firrtl.random_init_start"))
678 svReg.randStart = attr.getUInt();
681 reg->removeAttr(
"firrtl.random_init_start");
684 svReg.reg->setDialectAttrs(
reg->getDialectAttrs());
686 if (
auto innerSymAttr =
reg.getInnerSymAttr())
687 svReg.reg.setInnerSymAttr(innerSymAttr);
691 if (
reg.hasReset()) {
693 reg->getBlock(), sv::EventControl::AtPosEdge,
reg.getClk(),
697 if (reg.getIsAsync() && areEquivalentValues(reg, reg.getNext()))
698 b.create<sv::PAssignOp>(reg.getLoc(), svReg.reg, reg);
700 createTree(b, svReg.reg, reg, reg.getNext());
702 reg.getIsAsync() ? sv::ResetType::AsyncReset : sv::ResetType::SyncReset,
703 sv::EventControl::AtPosEdge,
reg.getReset(),
704 [&](OpBuilder &builder) {
705 builder.create<sv::PAssignOp>(loc, svReg.reg,
reg.getResetValue());
707 if (
reg.getIsAsync()) {
708 svReg.asyncResetSignal =
reg.getReset();
709 svReg.asyncResetValue =
reg.getResetValue();
713 reg->getBlock(), sv::EventControl::AtPosEdge,
reg.getClk(),
714 [&](OpBuilder &b) { createTree(b, svReg.reg, reg, reg.getNext()); });
725 if (svReg.asyncResetSignal)
726 asyncResets[svReg.asyncResetSignal].emplace_back(svReg);
734 reg.replaceAllUsesWith(regVal.getResult());
743 OpBuilder &builder, Value
reg,
746 auto type = cast<sv::InOutType>(
reg.getType()).getElementType();
747 if (
auto intTy = hw::type_dyn_cast<IntegerType>(type)) {
749 pos -= intTy.getWidth();
750 auto elem = builder.createOrFold<
comb::ExtractOp>(loc, randomSource, pos,
752 builder.
create<sv::BPAssignOp>(loc,
reg, elem);
753 }
else if (
auto array = hw::type_dyn_cast<hw::ArrayType>(type)) {
754 for (
unsigned i = 0, e = array.getNumElements(); i < e; ++i) {
757 loc, builder, builder.create<sv::ArrayIndexInOutOp>(loc,
reg, index),
760 }
else if (
auto structType = hw::type_dyn_cast<hw::StructType>(type)) {
761 for (
auto e : structType.getElements())
764 builder.create<sv::StructFieldInOutOp>(loc,
reg, e.name),
767 assert(
false &&
"unsupported type");
781 auto kind = condition.getKind();
783 auto ifDef = b.create<sv::IfDefProceduralOp>(
784 reg.getLoc(), condition.getMacro(), []() {});
785 b.setInsertionPointToEnd(ifDef.getThenBlock());
789 auto ifDef = b.create<sv::IfDefProceduralOp>(
790 reg.getLoc(), condition.getMacro(), []() {}, []() {});
792 b.setInsertionPointToEnd(ifDef.getElseBlock());
795 llvm_unreachable(
"unknown reg condition type");
799static Value
buildXMRTo(OpBuilder &builder, HierPathOp path, Location loc,
801 auto name = path.getSymNameAttr();
802 auto ref = mlir::FlatSymbolRefAttr::get(name);
803 return builder.create<sv::XMRRefOp>(loc, type, ref);
799static Value
buildXMRTo(OpBuilder &builder, HierPathOp path, Location loc, {
…}
807 ArrayRef<Value> rands) {
808 auto loc =
reg.reg.getLoc();
809 SmallVector<Value> nibbles;
813 OpBuilder::InsertionGuard guard(builder);
823 Value target =
reg.reg;
827 uint64_t width =
reg.width;
828 uint64_t offset =
reg.randStart;
830 auto index = offset / 32;
831 auto start = offset % 32;
832 auto nwidth = std::min(32 - start, width);
836 nibbles.push_back(elem);
841 unsigned pos =
reg.width;
847 Block *block, sv::EventControl clockEdge, Value clock,
848 const std::function<
void(OpBuilder &)> &body, sv::ResetType resetStyle,
849 sv::EventControl resetEdge, Value reset,
850 const std::function<
void(OpBuilder &)> &resetBody) {
851 auto loc = clock.getLoc();
852 ImplicitLocOpBuilder builder(loc, block,
getBlockEnd(block));
854 resetStyle, resetEdge, reset};
856 sv::AlwaysOp alwaysOp;
864 assert(resetStyle != sv::ResetType::NoReset);
877 auto createIfOp = [&]() {
880 insideIfOp = builder.create<sv::IfOp>(
881 reset, []() {}, []() {});
883 if (resetStyle == sv::ResetType::AsyncReset) {
884 sv::EventControl events[] = {clockEdge, resetEdge};
885 Value clocks[] = {clock, reset};
887 alwaysOp = builder.create<sv::AlwaysOp>(events, clocks, [&]() {
888 if (resetEdge == sv::EventControl::AtNegEdge)
889 llvm_unreachable(
"negative edge for reset is not expected");
893 alwaysOp = builder.create<sv::AlwaysOp>(clockEdge, clock, createIfOp);
897 alwaysOp = builder.create<sv::AlwaysOp>(clockEdge, clock);
898 insideIfOp =
nullptr;
903 assert(insideIfOp &&
"reset body must be initialized before");
905 ImplicitLocOpBuilder::atBlockEnd(loc, insideIfOp.getThenBlock());
906 resetBody(resetBuilder);
909 ImplicitLocOpBuilder::atBlockEnd(loc, insideIfOp.getElseBlock());
913 ImplicitLocOpBuilder::atBlockEnd(loc, alwaysOp.getBodyBlock());
assert(baseType &&"element must be base type")
static SmallVector< T > concat(const SmallVectorImpl< T > &a, const SmallVectorImpl< T > &b)
Returns a new vector containing the concatenation of vectors a and b.
static bool areEquivalentValues(Value term, Value next)
static InnerRefAttr getInnerRefTo(StringAttr mod, InnerSymbolNamespace &isns, seq::FirRegOp reg)
static StringAttr getInnerSymFor(InnerSymbolNamespace &innerSymNS, seq::FirRegOp reg)
Attach an inner-sym to field-id 0 of the given register, or use an existing inner-sym,...
static std::vector< BuriedFirReg > getAllBuriedRegs(ModuleOp top)
Locate all registers which are not at the top-level of their parent HW module.
static std::vector< BuriedFirReg > getBuriedRegs(HWModuleOp module)
Locate the registers under the given HW module, which are not at the top-level of the module body.
static Block::iterator getBlockEnd(Block *block)
Immediately before the terminator, if present. Otherwise, the block's end.
static Value buildXMRTo(OpBuilder &builder, HierPathOp path, Location loc, Type type)
static hw::HierPathOp getHierPathTo(OpBuilder &builder, Namespace &ns, BuriedFirReg entry)
Construct a hierarchical path op that targets the given register.
static std::optional< APInt > getConstantValue(Value value)
static llvm::SetVector< Value > extractConditions(Value value)
std::unique_ptr< ReachableMuxes > reachableMuxes
void initialize(OpBuilder &builder, RegLowerInfo reg, ArrayRef< Value > rands)
llvm::SmallDenseMap< std::pair< Value, unsigned >, Value > arrayIndexCache
void createAsyncResetInitialization(ImplicitLocOpBuilder &builder)
llvm::SmallDenseMap< IfKeyType, sv::IfOp > ifCache
static PathTable createPaths(mlir::ModuleOp top)
When a register is buried under an ifdef op, the initialization code at the footer of the HW module w...
DenseMap< seq::FirRegOp, hw::HierPathOp > PathTable
A map sending registers to their paths.
void createInitialBlock()
void addToIfBlock(OpBuilder &builder, Value cond, const std::function< void()> &trueSide, const std::function< void()> &falseSide)
FirRegLowering(TypeConverter &typeConverter, hw::HWModuleOp module, const PathTable &pathTable, bool disableRegRandomization=false, bool emitSeparateAlwaysBlocks=false)
std::optional< std::tuple< Value, Value, Value > > tryRestoringSubaccess(OpBuilder &builder, Value reg, Value term, hw::ArrayCreateOp nextRegValue)
void createRandomInitialization(ImplicitLocOpBuilder &builder)
void lowerUnderIfDef(sv::IfDefOp ifDefOp)
void lowerInBlock(Block *block)
void buildRegConditions(OpBuilder &b, sv::RegOp reg)
Recreate the ifdefs under which reg was defined.
const PathTable & pathTable
void lowerReg(seq::FirRegOp reg)
SmallVector< Value > createRandomizationVector(OpBuilder &builder, Location loc)
std::vector< RegCondition > conditions
The ambient ifdef conditions we have encountered while lowering.
void createTree(OpBuilder &builder, Value reg, Value term, Value next)
void createPresetInitialization(ImplicitLocOpBuilder &builder)
unsigned numSubaccessRestored
hw::ConstantOp getOrCreateConstant(Location loc, const APInt &value)
void addToAlwaysBlock(Block *block, sv::EventControl clockEdge, Value clock, const std::function< void(OpBuilder &)> &body, sv::ResetType resetStyle={}, sv::EventControl resetEdge={}, Value reset={}, const std::function< void(OpBuilder &)> &resetBody={})
SmallVector< RegLowerInfo > randomInitRegs
A list of registers discovered, bucketed by initialization style.
std::tuple< Block *, sv::EventControl, Value, sv::ResetType, sv::EventControl, Value > AlwaysKeyType
llvm::MapVector< Value, SmallVector< RegLowerInfo > > asyncResets
A map from async reset signal to the registers that use it.
void initializeRegisterElements(Location loc, OpBuilder &builder, Value reg, Value rand, unsigned &pos)
DenseMap< sv::RegOp, std::vector< RegCondition > > regConditionTable
A map from RegOps to the ifdef conditions under which they are defined.
TypeConverter & typeConverter
hw::HWModuleOp bool disableRegRandomization
bool emitSeparateAlwaysBlocks
SmallVector< RegLowerInfo > presetInitRegs
llvm::SmallDenseMap< AlwaysKeyType, std::pair< sv::AlwaysOp, sv::IfOp > > alwaysBlocks
A namespace that is used to store existing names and generate new names in some scope within the IR.
void add(mlir::ModuleOp module)
StringRef newName(const Twine &name)
Return a unique name, derived from the input name, and add the new name to the internal namespace.
void buildReachabilityFrom(Operation *startNode)
llvm::SmallPtrSet< Operation *, 16 > visited
HWModuleOp llvm::DenseMap< Operation *, llvm::SmallDenseSet< Operation * > > reachableMuxes
bool isMuxReachableFrom(seq::FirRegOp regOp, comb::MuxOp muxOp)
int64_t getBitWidth(mlir::Type type)
Return the hardware bit width of a type.
The InstanceGraph op interface, see InstanceGraphInterface.td for more details.
static ResultTy transformReduce(MLIRContext *context, IterTy begin, IterTy end, ResultTy init, ReduceFuncTy reduce, TransformFuncTy transform)
Wrapper for llvm::parallelTransformReduce that performs the transform_reduce serially when MLIR multi...
reg(value, clock, reset=None, reset_value=None, name=None, sym_name=None)
@ IfDefThen
The register is under an ifdef "then" branch.
@ IfDefElse
The register is under an ifdef "else" branch.
static std::function< bool(const Operation *op)> opAllowsReachability