19#include "mlir/IR/Threading.h"
20#include "mlir/Pass/Pass.h"
21#include "llvm/Support/Debug.h"
23#define DEBUG_TYPE "mem-to-reg-of-vec"
27#define GEN_PASS_DEF_MEMTOREGOFVEC
28#include "circt/Dialect/FIRRTL/Passes.h.inc"
33using namespace firrtl;
36struct MemToRegOfVecPass
37 :
public circt::firrtl::impl::MemToRegOfVecBase<MemToRegOfVecPass> {
40 void runOnOperation()
override {
41 auto circtOp = getOperation();
42 auto &instanceInfo = getAnalysis<InstanceInfo>();
46 return markAllAnalysesPreserved();
48 DenseSet<Operation *> dutModuleSet;
49 for (
auto moduleOp : circtOp.getOps<FModuleOp>())
50 if (instanceInfo.anyInstanceInEffectiveDesign(moduleOp))
51 dutModuleSet.insert(moduleOp);
53 mlir::parallelForEach(circtOp.getContext(), dutModuleSet,
55 if (auto mod = dyn_cast<FModuleOp>(op))
60 void runOnModule(FModuleOp mod) {
62 mod.getBodyBlock()->walk([&](MemOp memOp) {
63 LLVM_DEBUG(llvm::dbgs() <<
"\n Memory op:" << memOp);
65 auto firMem = memOp.getSummary();
73 ((firMem.readLatency == 1 && firMem.writeLatency == 1) &&
74 (firMem.numWritePorts + firMem.numReadWritePorts == 1) &&
75 (firMem.numReadPorts <= 1) && firMem.dataWidth > 0))
78 generateMemory(memOp, firMem);
83 Value addPipelineStages(ImplicitLocOpBuilder &b,
size_t stages, Value clock,
84 Value pipeInput, StringRef name, Value gate = {}) {
89 auto reg = b.create<RegOp>(pipeInput.getType(), clock, name).getResult();
91 b.create<WhenOp>(gate,
false, [&]() {
92 b.create<MatchingConnectOp>(
reg, pipeInput);
95 b.create<MatchingConnectOp>(
reg, pipeInput);
103 Value getClock(ImplicitLocOpBuilder &builder, Value bundle) {
104 return builder.create<SubfieldOp>(bundle,
"clk");
107 Value getAddr(ImplicitLocOpBuilder &builder, Value bundle) {
108 return builder.create<SubfieldOp>(bundle,
"addr");
111 Value getWmode(ImplicitLocOpBuilder &builder, Value bundle) {
112 return builder.create<SubfieldOp>(bundle,
"wmode");
115 Value getEnable(ImplicitLocOpBuilder &builder, Value bundle) {
116 return builder.create<SubfieldOp>(bundle,
"en");
119 Value getMask(ImplicitLocOpBuilder &builder, Value bundle) {
120 auto bType = type_cast<BundleType>(bundle.getType());
121 if (bType.getElement(
"mask"))
122 return builder.create<SubfieldOp>(bundle,
"mask");
123 return builder.create<SubfieldOp>(bundle,
"wmask");
126 Value getData(ImplicitLocOpBuilder &builder, Value bundle,
127 bool getWdata =
false) {
128 auto bType = type_cast<BundleType>(bundle.getType());
129 if (bType.getElement(
"data"))
130 return builder.create<SubfieldOp>(bundle,
"data");
131 if (bType.getElement(
"rdata") && !getWdata)
132 return builder.create<SubfieldOp>(bundle,
"rdata");
133 return builder.create<SubfieldOp>(bundle,
"wdata");
136 void generateRead(
const FirMemory &firMem, Value clock, Value
addr,
137 Value enable, Value
data, Value regOfVec,
138 ImplicitLocOpBuilder &builder) {
139 if (ignoreReadEnable) {
142 for (
size_t j = 0, e = firMem.
readLatency; j != e; ++j) {
143 auto enLast = enable;
145 enable = addPipelineStages(builder, 1, clock, enable,
"en");
146 addr = addPipelineStages(builder, 1, clock,
addr,
"addr", enLast);
152 addPipelineStages(builder, firMem.
readLatency, clock, enable,
"en");
158 Value
rdata = builder.create<SubaccessOp>(regOfVec,
addr);
159 if (!ignoreReadEnable) {
161 builder.create<MatchingConnectOp>(
162 data, builder.create<InvalidValueOp>(
data.getType()));
164 builder.create<WhenOp>(enable,
false, [&]() {
165 builder.create<MatchingConnectOp>(
data,
rdata);
169 builder.create<MatchingConnectOp>(
data,
rdata);
173 void generateWrite(
const FirMemory &firMem, Value clock, Value
addr,
174 Value enable, Value maskBits, Value wdataIn,
175 Value regOfVec, ImplicitLocOpBuilder &builder) {
180 addr = addPipelineStages(builder, numStages, clock,
addr,
"addr");
181 enable = addPipelineStages(builder, numStages, clock, enable,
"en");
182 wdataIn = addPipelineStages(builder, numStages, clock, wdataIn,
"wdata");
183 maskBits = addPipelineStages(builder, numStages, clock, maskBits,
"wmask");
192 SmallVector<std::tuple<Value, Value, Value>, 8> loweredRegDataMaskFields;
207 if (!getFields(
rdata, wdataIn, maskBits, loweredRegDataMaskFields,
209 wdataIn.getDefiningOp()->emitOpError(
210 "Cannot convert memory to bank of registers");
214 builder.create<WhenOp>(enable,
false, [&]() {
216 for (
auto regDataMask : loweredRegDataMaskFields) {
217 auto regField = std::get<0>(regDataMask);
218 auto dataField = std::get<1>(regDataMask);
219 auto maskField = std::get<2>(regDataMask);
221 builder.create<WhenOp>(maskField,
false, [&]() {
222 builder.create<MatchingConnectOp>(regField, dataField);
228 void generateReadWrite(
const FirMemory &firMem, Value clock, Value
addr,
229 Value enable, Value maskBits, Value wdataIn,
230 Value rdataOut, Value
wmode, Value regOfVec,
231 ImplicitLocOpBuilder &builder) {
236 addr = addPipelineStages(builder, numStages, clock,
addr,
"addr");
237 enable = addPipelineStages(builder, numStages, clock, enable,
"en");
238 wdataIn = addPipelineStages(builder, numStages, clock, wdataIn,
"wdata");
239 maskBits = addPipelineStages(builder, numStages, clock, maskBits,
"wmask");
242 Value
rdata = builder.create<SubaccessOp>(regOfVec,
addr);
244 SmallVector<std::tuple<Value, Value, Value>, 8> loweredRegDataMaskFields;
245 if (!getFields(
rdata, wdataIn, maskBits, loweredRegDataMaskFields,
247 wdataIn.getDefiningOp()->emitOpError(
248 "Cannot convert memory to bank of registers");
252 builder.create<MatchingConnectOp>(
253 rdataOut, builder.create<InvalidValueOp>(rdataOut.getType()));
255 builder.create<WhenOp>(enable,
false, [&]() {
257 builder.create<WhenOp>(
262 for (
auto regDataMask : loweredRegDataMaskFields) {
263 auto regField = std::get<0>(regDataMask);
264 auto dataField = std::get<1>(regDataMask);
265 auto maskField = std::get<2>(regDataMask);
267 builder.create<WhenOp>(
268 maskField,
false, [&]() {
269 builder.create<MatchingConnectOp>(regField, dataField);
274 [&]() { builder.create<MatchingConnectOp>(rdataOut,
rdata); });
285 bool getFields(Value reg, Value input, Value
mask,
286 SmallVectorImpl<std::tuple<Value, Value, Value>> &results,
287 ImplicitLocOpBuilder &builder) {
291 if (
auto bundle = type_dyn_cast<BundleType>(inType)) {
292 if (
auto mBundle = type_dyn_cast<BundleType>(maskType))
293 return mBundle.getNumElements() == bundle.getNumElements();
294 }
else if (
auto vec = type_dyn_cast<FVectorType>(inType)) {
295 if (
auto mVec = type_dyn_cast<FVectorType>(maskType))
296 return mVec.getNumElements() == vec.getNumElements();
302 std::function<bool(Value, Value, Value)> flatAccess =
303 [&](Value
reg, Value input, Value
mask) ->
bool {
304 FIRRTLType inType = type_cast<FIRRTLType>(input.getType());
305 if (!isValidMask(inType, type_cast<FIRRTLType>(
mask.getType()))) {
306 input.getDefiningOp()->emitOpError(
"Mask type is not valid");
310 .
Case<BundleType>([&](BundleType bundle) {
311 for (
size_t i = 0, e = bundle.getNumElements(); i != e; ++i) {
312 auto regField = builder.create<SubfieldOp>(
reg, i);
313 auto inputField = builder.create<SubfieldOp>(input, i);
314 auto maskField = builder.create<SubfieldOp>(
mask, i);
315 if (!flatAccess(regField, inputField, maskField))
320 .Case<FVectorType>([&](
auto vector) {
321 for (
size_t i = 0, e = vector.getNumElements(); i != e; ++i) {
322 auto regField = builder.create<SubindexOp>(
reg, i);
323 auto inputField = builder.create<SubindexOp>(input, i);
324 auto maskField = builder.create<SubindexOp>(
mask, i);
325 if (!flatAccess(regField, inputField, maskField))
330 .Case<IntType>([&](
auto iType) {
331 results.push_back({
reg, input,
mask});
332 return iType.getWidth().has_value();
334 .Default([&](
auto) {
return false; });
336 if (flatAccess(reg, input,
mask))
341 void scatterMemTapAnno(RegOp op, ArrayAttr attr,
342 ImplicitLocOpBuilder &builder) {
344 SmallVector<Attribute> regAnnotations;
345 auto vecType = type_cast<FVectorType>(op.getResult().getType());
346 for (
auto anno : annos) {
348 for (
size_t i = 0, e = type_cast<FVectorType>(op.getResult().getType())
351 NamedAttrList newAnno;
352 newAnno.append(
"class", anno.getMember(
"class"));
353 newAnno.append(
"circt.fieldID",
354 builder.getI64IntegerAttr(vecType.getFieldID(i)));
355 newAnno.append(
"id", anno.getMember(
"id"));
356 if (
auto nla = anno.getMember(
"circt.nonlocal"))
357 newAnno.append(
"circt.nonlocal", nla);
360 IntegerAttr::get(IntegerType::get(builder.getContext(), 64), i));
362 regAnnotations.push_back(builder.getDictionaryAttr(newAnno));
365 regAnnotations.push_back(anno.getAttr());
367 op.setAnnotationsAttr(builder.getArrayAttr(regAnnotations));
371 void generateMemory(MemOp memOp,
FirMemory &firMem) {
372 ImplicitLocOpBuilder builder(memOp.getLoc(), memOp);
373 auto dataType = memOp.getDataType();
375 auto innerSym = memOp.getInnerSym();
376 SmallVector<Value> debugPorts;
379 for (
size_t index = 0, rend = memOp.getNumResults(); index < rend;
381 auto result = memOp.getResult(index);
382 if (type_isa<RefType>(result.getType())) {
383 debugPorts.push_back(result);
388 auto wire = builder.create<WireOp>(
390 (memOp.getName() +
"_" + memOp.getPortName(index).getValue()).str(),
391 memOp.getNameKind());
392 result.replaceAllUsesWith(wire.getResult());
393 result = wire.getResult();
395 auto adr = getAddr(builder, result);
396 auto enb = getEnable(builder, result);
397 auto clk = getClock(builder, result);
398 auto dta = getData(builder, result);
402 regOfVec = builder.create<RegOp>(
403 FVectorType::get(dataType, firMem.
depth),
clk, memOp.getNameAttr());
406 if (!memOp.getAnnotationsAttr().empty())
407 scatterMemTapAnno(regOfVec, memOp.getAnnotationsAttr(), builder);
409 regOfVec.setInnerSymAttr(memOp.getInnerSymAttr());
411 auto portKind = memOp.getPortKind(index);
412 if (portKind == MemOp::PortKind::Read) {
413 generateRead(firMem,
clk, adr, enb, dta, regOfVec.getResult(), builder);
414 }
else if (portKind == MemOp::PortKind::Write) {
415 auto mask = getMask(builder, result);
416 generateWrite(firMem,
clk, adr, enb,
mask, dta, regOfVec.getResult(),
419 auto wmode = getWmode(builder, result);
420 auto wDta = getData(builder, result,
true);
421 auto mask = getMask(builder, result);
422 generateReadWrite(firMem,
clk, adr, enb,
mask, wDta, dta,
wmode,
423 regOfVec.getResult(), builder);
430 for (
auto r : debugPorts)
431 r.replaceAllUsesWith(builder.create<RefSendOp>(regOfVec.getResult()));
This class provides a read-only projection over the MLIR attributes that represent a set of annotatio...
bool removeAnnotations(llvm::function_ref< bool(Annotation)> predicate)
Remove all annotations from this annotation set for which predicate returns true.
This class implements the same functionality as TypeSwitch except that it uses firrtl::type_dyn_cast ...
FIRRTLTypeSwitch< T, ResultT > & Case(CallableT &&caseFn)
Add a case on the given type.
mlir::TypedValue< FIRRTLBaseType > FIRRTLBaseValue
constexpr const char * convertMemToRegOfVecAnnoClass
constexpr const char * memTapSourceClass
The InstanceGraph op interface, see InstanceGraphInterface.td for more details.
reg(value, clock, reset=None, reset_value=None, name=None, sym_name=None)