CIRCT 22.0.0git
Loading...
Searching...
No Matches
LowerState.cpp
Go to the documentation of this file.
1//===- LowerState.cpp -----------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
18#include "mlir/Analysis/TopologicalSortUtils.h"
19#include "mlir/Dialect/Func/IR/FuncOps.h"
20#include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
21#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
22#include "mlir/Dialect/SCF/IR/SCF.h"
23#include "mlir/IR/IRMapping.h"
24#include "mlir/IR/ImplicitLocOpBuilder.h"
25#include "mlir/IR/SymbolTable.h"
26#include "mlir/Interfaces/SideEffectInterfaces.h"
27#include "mlir/Pass/Pass.h"
28#include "llvm/ADT/TypeSwitch.h"
29#include "llvm/Support/Debug.h"
30
31#define DEBUG_TYPE "arc-lower-state"
32
33namespace circt {
34namespace arc {
35#define GEN_PASS_DEF_LOWERSTATEPASS
36#include "circt/Dialect/Arc/ArcPasses.h.inc"
37} // namespace arc
38} // namespace circt
39
40using namespace circt;
41using namespace arc;
42using namespace hw;
43using namespace mlir;
44using llvm::SmallDenseSet;
45
46namespace {
47enum class Phase { Initial, Old, New, Final };
48
49template <class OS>
50OS &operator<<(OS &os, Phase phase) {
51 switch (phase) {
52 case Phase::Initial:
53 return os << "initial";
54 case Phase::Old:
55 return os << "old";
56 case Phase::New:
57 return os << "new";
58 case Phase::Final:
59 return os << "final";
60 }
61}
62
63struct ModuleLowering;
64
65/// All state associated with lowering a single operation. Instances of this
66/// struct are kept on a worklist to perform a depth-first traversal of the
67/// module being lowered.
68///
69/// The actual lowering occurs in `lower()`. This function is called exactly
70/// twice. A first time with `initial` being true, where other values and
71/// operations that have to be lowered first may be marked with `addPending`. No
72/// actual lowering or error reporting should occur when `initial` is true. The
73/// worklist then ensures that all `pending` ops are lowered before `lower()` is
74/// called a second time with `initial` being false. At this point the actual
75/// lowering and error reporting should occur.
76///
77/// The `initial` variable is used to allow for a single block of code to mark
78/// values and ops as dependencies and actually do the lowering based on them.
79struct OpLowering {
80 Operation *op;
81 Phase phase;
82 ModuleLowering &module;
83
84 bool initial = true;
85 SmallVector<std::pair<Operation *, Phase>, 2> pending;
86
87 OpLowering(Operation *op, Phase phase, ModuleLowering &module)
88 : op(op), phase(phase), module(module) {}
89
90 // Operation Lowering.
91 LogicalResult lower();
92 LogicalResult lowerDefault();
93 LogicalResult lower(StateOp op);
94 LogicalResult lower(sim::DPICallOp op);
95 LogicalResult
96 lowerStateful(Value clock, Value enable, Value reset, ValueRange inputs,
97 ResultRange results,
98 llvm::function_ref<ValueRange(ValueRange)> createMapping);
99 LogicalResult lower(MemoryOp op);
100 LogicalResult lower(TapOp op);
101 LogicalResult lower(InstanceOp op);
102 LogicalResult lower(hw::OutputOp op);
103 LogicalResult lower(seq::InitialOp op);
104 LogicalResult lower(llhd::FinalOp op);
105
106 scf::IfOp createIfClockOp(Value clock);
107
108 // Value Lowering. These functions are called from the `lower()` functions
109 // above. They handle values used by the `op`. This can generate reads from
110 // state and memory storage on-the-fly, or mark other ops as dependencies to
111 // be lowered first.
112 Value lowerValue(Value value, Phase phase);
113 Value lowerValue(InstanceOp op, OpResult result, Phase phase);
114 Value lowerValue(StateOp op, OpResult result, Phase phase);
115 Value lowerValue(sim::DPICallOp op, OpResult result, Phase phase);
116 Value lowerValue(MemoryReadPortOp op, OpResult result, Phase phase);
117 Value lowerValue(seq::InitialOp op, OpResult result, Phase phase);
118 Value lowerValue(seq::FromImmutableOp op, OpResult result, Phase phase);
119
120 void addPending(Value value, Phase phase);
121 void addPending(Operation *op, Phase phase);
122};
123
124/// All state associated with lowering a single module.
125struct ModuleLowering {
126 /// The module being lowered.
127 HWModuleOp moduleOp;
128 /// The builder for the main body of the model.
129 OpBuilder builder;
130 /// The builder for state allocation ops.
131 OpBuilder allocBuilder;
132 /// The builder for the initial phase.
133 OpBuilder initialBuilder;
134 /// The builder for the final phase.
135 OpBuilder finalBuilder;
136
137 /// The storage value that can be used for `arc.alloc_state` and friends.
138 Value storageArg;
139
140 /// A worklist of pending op lowerings.
141 SmallVector<OpLowering> opsWorklist;
142 /// The set of ops currently in the worklist. Used to detect cycles.
143 SmallDenseSet<std::pair<Operation *, Phase>> opsSeen;
144 /// The ops that have already been lowered.
145 DenseSet<std::pair<Operation *, Phase>> loweredOps;
146 /// The values that have already been lowered.
147 DenseMap<std::pair<Value, Phase>, Value> loweredValues;
148
149 /// The allocated input ports.
150 SmallVector<Value> allocatedInputs;
151 /// The allocated states as a mapping from op results to `arc.alloc_state`
152 /// results.
153 DenseMap<Value, Value> allocatedStates;
154 /// The allocated storage for instance inputs and top module outputs.
155 DenseMap<OpOperand *, Value> allocatedOutputs;
156 /// The allocated storage for values computed during the initial phase.
157 DenseMap<Value, Value> allocatedInitials;
158 /// The allocated storage for taps.
159 DenseMap<Operation *, Value> allocatedTaps;
160
161 /// A mapping from unlowered clocks to a value indicating a posedge. This is
162 /// used to not create an excessive number of posedge detectors.
163 DenseMap<Value, Value> loweredPosedges;
164 /// The previous enable and the value it was lowered to. This is used to reuse
165 /// previous if ops for the same enable value.
166 std::pair<Value, Value> prevEnable;
167 /// The previous reset and the value it was lowered to. This is used to reuse
168 /// previous if ops for the same reset value.
169 std::pair<Value, Value> prevReset;
170
171 ModuleLowering(HWModuleOp moduleOp)
172 : moduleOp(moduleOp), builder(moduleOp), allocBuilder(moduleOp),
173 initialBuilder(moduleOp), finalBuilder(moduleOp) {}
174 LogicalResult run();
175 LogicalResult lowerOp(Operation *op);
176 Value getAllocatedState(OpResult result);
177 Value detectPosedge(Value clock);
178 OpBuilder &getBuilder(Phase phase);
179 Value requireLoweredValue(Value value, Phase phase, Location useLoc);
180};
181} // namespace
182
183//===----------------------------------------------------------------------===//
184// Module Lowering
185//===----------------------------------------------------------------------===//
186
187LogicalResult ModuleLowering::run() {
188 LLVM_DEBUG(llvm::dbgs() << "Lowering module `" << moduleOp.getModuleName()
189 << "`\n");
190
191 // Create the replacement `ModelOp`.
192 auto modelOp =
193 ModelOp::create(builder, moduleOp.getLoc(), moduleOp.getModuleNameAttr(),
194 TypeAttr::get(moduleOp.getModuleType()),
195 FlatSymbolRefAttr{}, FlatSymbolRefAttr{});
196 auto &modelBlock = modelOp.getBody().emplaceBlock();
197 storageArg = modelBlock.addArgument(
198 StorageType::get(builder.getContext(), {}), modelOp.getLoc());
199 builder.setInsertionPointToStart(&modelBlock);
200
201 // Create the `arc.initial` op to contain the ops for the initialization
202 // phase.
203 auto initialOp = InitialOp::create(builder, moduleOp.getLoc());
204 initialBuilder.setInsertionPointToStart(&initialOp.getBody().emplaceBlock());
205
206 // Create the `arc.final` op to contain the ops for the finalization phase.
207 auto finalOp = FinalOp::create(builder, moduleOp.getLoc());
208 finalBuilder.setInsertionPointToStart(&finalOp.getBody().emplaceBlock());
209
210 // Position the alloc builder such that allocation ops get inserted above the
211 // initial op.
212 allocBuilder.setInsertionPoint(initialOp);
213
214 // Allocate storage for the inputs.
215 for (auto arg : moduleOp.getBodyBlock()->getArguments()) {
216 auto name = moduleOp.getArgName(arg.getArgNumber());
217 auto state =
218 RootInputOp::create(allocBuilder, arg.getLoc(),
219 StateType::get(arg.getType()), name, storageArg);
220 allocatedInputs.push_back(state);
221 }
222
223 // Lower the ops.
224 for (auto &op : moduleOp.getOps()) {
225 if (mlir::isMemoryEffectFree(&op) && !isa<hw::OutputOp>(op))
226 continue;
227 if (isa<MemoryReadPortOp, MemoryWritePortOp>(op))
228 continue; // handled as part of `MemoryOp`
229 if (failed(lowerOp(&op)))
230 return failure();
231 }
232
233 // Clean up any dead ops. The lowering inserts a few defensive
234 // `arc.state_read` ops that may remain unused. This cleans them up.
235 for (auto &op : llvm::make_early_inc_range(llvm::reverse(modelBlock)))
236 if (mlir::isOpTriviallyDead(&op))
237 op.erase();
238
239 return success();
240}
241
242/// Lower an op and its entire fan-in cone.
243LogicalResult ModuleLowering::lowerOp(Operation *op) {
244 LLVM_DEBUG(llvm::dbgs() << "- Handling " << *op << "\n");
245
246 // Pick in which phases the given operation has to perform some work.
247 SmallVector<Phase, 2> phases = {Phase::New};
248 if (isa<seq::InitialOp>(op))
249 phases = {Phase::Initial};
250 if (isa<llhd::FinalOp>(op))
251 phases = {Phase::Final};
252 if (isa<StateOp>(op))
253 phases = {Phase::Initial, Phase::New};
254
255 for (auto phase : phases) {
256 if (loweredOps.contains({op, phase}))
257 return success();
258 opsWorklist.push_back(OpLowering(op, phase, *this));
259 opsSeen.insert({op, phase});
260 }
261
262 auto dumpWorklist = [&] {
263 for (auto &opLowering : llvm::reverse(opsWorklist))
264 opLowering.op->emitRemark()
265 << "computing " << opLowering.phase << " phase here";
266 };
267
268 while (!opsWorklist.empty()) {
269 auto &opLowering = opsWorklist.back();
270
271 // Collect an initial list of operands that need to be lowered.
272 if (opLowering.initial) {
273 if (failed(opLowering.lower())) {
274 dumpWorklist();
275 return failure();
276 }
277 std::reverse(opLowering.pending.begin(), opLowering.pending.end());
278 opLowering.initial = false;
279 }
280
281 // Push operands onto the worklist.
282 if (!opLowering.pending.empty()) {
283 auto [defOp, phase] = opLowering.pending.pop_back_val();
284 if (loweredOps.contains({defOp, phase}))
285 continue;
286 if (!opsSeen.insert({defOp, phase}).second) {
287 defOp->emitOpError("is on a combinational loop");
288 dumpWorklist();
289 return failure();
290 }
291 opsWorklist.push_back(OpLowering(defOp, phase, *this));
292 continue;
293 }
294
295 // At this point all operands are available and the op itself can be
296 // lowered.
297 LLVM_DEBUG(llvm::dbgs() << " - Lowering " << opLowering.phase << " "
298 << *opLowering.op << "\n");
299 if (failed(opLowering.lower())) {
300 dumpWorklist();
301 return failure();
302 }
303 loweredOps.insert({opLowering.op, opLowering.phase});
304 opsSeen.erase({opLowering.op, opLowering.phase});
305 opsWorklist.pop_back();
306 }
307
308 return success();
309}
310
311/// Return the `arc.alloc_state` associated with the given state op result.
312/// Creates the allocation op if it does not yet exist.
313Value ModuleLowering::getAllocatedState(OpResult result) {
314 if (auto alloc = allocatedStates.lookup(result))
315 return alloc;
316
317 // Handle memories.
318 if (auto memOp = dyn_cast<MemoryOp>(result.getOwner())) {
319 auto alloc =
320 AllocMemoryOp::create(allocBuilder, memOp.getLoc(), memOp.getType(),
321 storageArg, memOp->getAttrs());
322 allocatedStates.insert({result, alloc});
323 return alloc;
324 }
325
326 // Create the allocation op.
327 auto alloc =
328 AllocStateOp::create(allocBuilder, result.getLoc(),
329 StateType::get(result.getType()), storageArg);
330 allocatedStates.insert({result, alloc});
331
332 // HACK: If the result comes from an instance op, add the instance and port
333 // name as an attribute to the allocation. This will make it show up in the C
334 // headers later. Get rid of this once we have proper debug dialect support.
335 if (auto instOp = dyn_cast<InstanceOp>(result.getOwner()))
336 alloc->setAttr(
337 "name", builder.getStringAttr(
338 instOp.getInstanceName() + "/" +
339 instOp.getOutputName(result.getResultNumber()).getValue()));
340
341 // HACK: If the result comes from an op that has a "names" attribute, use that
342 // as a name for the allocation. This should no longer be necessary once we
343 // properly support the Debug dialect.
344 if (isa<StateOp, sim::DPICallOp>(result.getOwner()))
345 if (auto names = result.getOwner()->getAttrOfType<ArrayAttr>("names"))
346 if (result.getResultNumber() < names.size())
347 alloc->setAttr("name", names[result.getResultNumber()]);
348
349 return alloc;
350}
351
352/// Allocate the necessary storage, reads, writes, and comparisons to detect a
353/// rising edge on a clock value.
354Value ModuleLowering::detectPosedge(Value clock) {
355 auto loc = clock.getLoc();
356 if (isa<seq::ClockType>(clock.getType()))
357 clock = seq::FromClockOp::create(builder, loc, clock);
358
359 // Allocate storage to store the previous clock value.
360 auto oldStorage = AllocStateOp::create(
361 allocBuilder, loc, StateType::get(builder.getI1Type()), storageArg);
362
363 // Read the old clock value from storage and write the new clock value to
364 // storage.
365 auto oldClock = StateReadOp::create(builder, loc, oldStorage);
366 StateWriteOp::create(builder, loc, oldStorage, clock, Value{});
367
368 // Detect a rising edge.
369 auto edge = comb::XorOp::create(builder, loc, oldClock, clock);
370 return comb::AndOp::create(builder, loc, edge, clock);
371}
372
373/// Get the builder appropriate for the given phase.
374OpBuilder &ModuleLowering::getBuilder(Phase phase) {
375 switch (phase) {
376 case Phase::Initial:
377 return initialBuilder;
378 case Phase::Old:
379 case Phase::New:
380 return builder;
381 case Phase::Final:
382 return finalBuilder;
383 }
384}
385
386/// Get the lowered value, or emit a diagnostic and return null.
387Value ModuleLowering::requireLoweredValue(Value value, Phase phase,
388 Location useLoc) {
389 if (auto lowered = loweredValues.lookup({value, phase}))
390 return lowered;
391 auto d = emitError(value.getLoc()) << "value has not been lowered";
392 d.attachNote(useLoc) << "value used here";
393 return {};
394}
395
396//===----------------------------------------------------------------------===//
397// Operation Lowering
398//===----------------------------------------------------------------------===//
399
400/// Create a new `scf.if` operation with the given builder, or reuse a previous
401/// `scf.if` if the builder's insertion point is located right after it.
402static scf::IfOp createOrReuseIf(OpBuilder &builder, Value condition,
403 bool withElse) {
404 if (auto ip = builder.getInsertionPoint(); ip != builder.getBlock()->begin())
405 if (auto ifOp = dyn_cast<scf::IfOp>(*std::prev(ip)))
406 if (ifOp.getCondition() == condition)
407 return ifOp;
408 return scf::IfOp::create(builder, condition.getLoc(), condition, withElse);
409}
410
411/// This function is called from the lowering worklist in order to perform a
412/// depth-first traversal of the surrounding module. These functions call
413/// `lowerValue` to mark their operands as dependencies in the depth-first
414/// traversal, and to map them to the lowered value in one go.
415LogicalResult OpLowering::lower() {
416 return TypeSwitch<Operation *, LogicalResult>(op)
417 // Operations with special lowering.
418 .Case<StateOp, sim::DPICallOp, MemoryOp, TapOp, InstanceOp, hw::OutputOp,
419 seq::InitialOp, llhd::FinalOp>([&](auto op) { return lower(op); })
420
421 // Operations that should be skipped entirely and never land on the
422 // worklist to be lowered.
423 .Case<MemoryWritePortOp, MemoryReadPortOp>([&](auto op) {
424 assert(false && "ports must be lowered by memory op");
425 return failure();
426 })
427
428 // All other ops are simply cloned into the lowered model.
429 .Default([&](auto) { return lowerDefault(); });
430}
431
432/// Called for all operations for which there is no special lowering. Simply
433/// clones the operation.
434LogicalResult OpLowering::lowerDefault() {
435 // Make sure that all operand values are lowered first.
436 IRMapping mapping;
437 auto anyFailed = false;
438 op->walk([&](Operation *nestedOp) {
439 for (auto operand : nestedOp->getOperands()) {
440 if (op->isAncestor(operand.getParentBlock()->getParentOp()))
441 continue;
442 auto lowered = lowerValue(operand, phase);
443 if (!lowered)
444 anyFailed = true;
445 mapping.map(operand, lowered);
446 }
447 });
448 if (initial)
449 return success();
450 if (anyFailed)
451 return failure();
452
453 // Clone the operation.
454 auto *clonedOp = module.getBuilder(phase).clone(*op, mapping);
455
456 // Keep track of the results.
457 for (auto [oldResult, newResult] :
458 llvm::zip(op->getResults(), clonedOp->getResults()))
459 module.loweredValues[{oldResult, phase}] = newResult;
460
461 return success();
462}
463
464/// Lower a state to a corresponding storage allocation and `write` of the
465/// state's new value to it. This function uses the `Old` phase to get the
466/// values at the state input before the current update, and then uses them to
467/// compute the `New` value.
468LogicalResult OpLowering::lower(StateOp op) {
469 // Handle initialization.
470 if (phase == Phase::Initial) {
471 // Ensure the initial values of the register have been lowered before.
472 if (initial) {
473 for (auto initial : op.getInitials())
474 lowerValue(initial, Phase::Initial);
475 return success();
476 }
477
478 // Write the initial values to the allocated storage in the initial block.
479 if (op.getInitials().empty())
480 return success();
481 for (auto [initial, result] :
482 llvm::zip(op.getInitials(), op.getResults())) {
483 auto value = lowerValue(initial, Phase::Initial);
484 if (!value)
485 return failure();
486 auto state = module.getAllocatedState(result);
487 if (!state)
488 return failure();
489 StateWriteOp::create(module.initialBuilder, value.getLoc(), state, value,
490 Value{});
491 }
492 return success();
493 }
494
495 assert(phase == Phase::New);
496
497 if (!initial) {
498 if (!op.getClock())
499 return op.emitOpError() << "must have a clock";
500 if (op.getLatency() > 1)
501 return op.emitOpError("latencies > 1 not supported yet");
502 }
503
504 return lowerStateful(op.getClock(), op.getEnable(), op.getReset(),
505 op.getInputs(), op.getResults(), [&](ValueRange inputs) {
506 return CallOp::create(module.builder, op.getLoc(),
507 op.getResultTypes(), op.getArc(),
508 inputs)
509 .getResults();
510 });
511}
512
513/// Lower a DPI call to a corresponding storage allocation and write of the
514/// state's new value to it. This function uses the `Old` phase to get the
515/// values at the state input before the current update, and then uses them to
516/// compute the `New` value.
517LogicalResult OpLowering::lower(sim::DPICallOp op) {
518 // Handle unclocked DPI calls.
519 if (!op.getClock()) {
520 // Make sure that all operands have been lowered.
521 SmallVector<Value> inputs;
522 for (auto operand : op.getInputs())
523 inputs.push_back(lowerValue(operand, phase));
524 if (initial)
525 return success();
526 if (llvm::is_contained(inputs, Value{}))
527 return failure();
528 if (op.getEnable())
529 return op.emitOpError() << "without clock cannot have an enable";
530
531 // Lower the op to a regular function call.
532 auto callOp =
533 func::CallOp::create(module.getBuilder(phase), op.getLoc(),
534 op.getCalleeAttr(), op.getResultTypes(), inputs);
535 for (auto [oldResult, newResult] :
536 llvm::zip(op.getResults(), callOp.getResults()))
537 module.loweredValues[{oldResult, phase}] = newResult;
538 return success();
539 }
540
541 assert(phase == Phase::New);
542
543 return lowerStateful(op.getClock(), op.getEnable(), /*reset=*/{},
544 op.getInputs(), op.getResults(), [&](ValueRange inputs) {
545 return func::CallOp::create(
546 module.builder, op.getLoc(),
547 op.getCalleeAttr(), op.getResultTypes(),
548 inputs)
549 .getResults();
550 });
551}
552
553/// Lower a state to a corresponding storage allocation and `write` of the
554/// state's new value to it. This function uses the `Old` phase to get the
555/// values at the state input before the current update, and then uses them to
556/// compute the `New` value.
557LogicalResult OpLowering::lowerStateful(
558 Value clock, Value enable, Value reset, ValueRange inputs,
559 ResultRange results,
560 llvm::function_ref<ValueRange(ValueRange)> createMapping) {
561 // Ensure all operands are lowered before we lower the op itself. State ops
562 // are special in that they require the "old" value of their inputs and
563 // enable, in order to compute the updated "new" value. The clock needs to be
564 // the "new" value though, such that other states can act as a clock source.
565 if (initial) {
566 lowerValue(clock, Phase::New);
567 if (enable)
568 lowerValue(enable, Phase::Old);
569 if (reset)
570 lowerValue(reset, Phase::Old);
571 for (auto input : inputs)
572 lowerValue(input, Phase::Old);
573 return success();
574 }
575
576 // Check if we're inserting right after an `if` op for the same clock edge, in
577 // which case we can reuse that op. Otherwise, create the new `if` op.
578 auto ifClockOp = createIfClockOp(clock);
579 if (!ifClockOp)
580 return failure();
581 OpBuilder::InsertionGuard guard(module.builder);
582 module.builder.setInsertionPoint(ifClockOp.thenYield());
583
584 // Make sure we have the state storage available such that we can read and
585 // write from and to them.
586 SmallVector<Value> states;
587 for (auto result : results) {
588 auto state = module.getAllocatedState(result);
589 if (!state)
590 return failure();
591 states.push_back(state);
592 }
593
594 // Handle the reset.
595 if (reset) {
596 // Check if we can reuse a previous reset value.
597 auto &[unloweredReset, loweredReset] = module.prevReset;
598 if (unloweredReset != reset ||
599 loweredReset.getParentBlock() != module.builder.getBlock()) {
600 unloweredReset = reset;
601 loweredReset = lowerValue(reset, Phase::Old);
602 if (!loweredReset)
603 return failure();
604 }
605
606 // Check if we're inserting right after an if op for the same reset, in
607 // which case we can reuse that op. Otherwise create the new if op.
608 auto ifResetOp = createOrReuseIf(module.builder, loweredReset, true);
609 module.builder.setInsertionPoint(ifResetOp.thenYield());
610
611 // Generate the zero value writes.
612 for (auto state : states) {
613 auto type = cast<StateType>(state.getType()).getType();
614 Value value = ConstantOp::create(
615 module.builder, loweredReset.getLoc(),
616 module.builder.getIntegerType(hw::getBitWidth(type)), 0);
617 if (value.getType() != type)
618 value = BitcastOp::create(module.builder, loweredReset.getLoc(), type,
619 value);
620 StateWriteOp::create(module.builder, loweredReset.getLoc(), state, value,
621 Value{});
622 }
623 module.builder.setInsertionPoint(ifResetOp.elseYield());
624 }
625
626 // Handle the enable.
627 if (enable) {
628 // Check if we can reuse a previous enable value.
629 auto &[unloweredEnable, loweredEnable] = module.prevEnable;
630 if (unloweredEnable != enable ||
631 loweredEnable.getParentBlock() != module.builder.getBlock()) {
632 unloweredEnable = enable;
633 loweredEnable = lowerValue(enable, Phase::Old);
634 if (!loweredEnable)
635 return failure();
636 }
637
638 // Check if we're inserting right after an if op for the same enable, in
639 // which case we can reuse that op. Otherwise create the new if op.
640 auto ifEnableOp = createOrReuseIf(module.builder, loweredEnable, false);
641 module.builder.setInsertionPoint(ifEnableOp.thenYield());
642 }
643
644 // Get the transfer function inputs. This potentially inserts read ops.
645 SmallVector<Value> loweredInputs;
646 for (auto input : inputs) {
647 auto lowered = lowerValue(input, Phase::Old);
648 if (!lowered)
649 return failure();
650 loweredInputs.push_back(lowered);
651 }
652
653 // Compute the transfer function and write its results to the state's storage.
654 auto loweredResults = createMapping(loweredInputs);
655 for (auto [state, value] : llvm::zip(states, loweredResults))
656 StateWriteOp::create(module.builder, value.getLoc(), state, value, Value{});
657
658 // Since we just wrote the new state value to storage, insert read ops just
659 // before the if op that keep the old value around for any later ops that
660 // still need it.
661 module.builder.setInsertionPoint(ifClockOp);
662 for (auto [state, result] : llvm::zip(states, results)) {
663 auto oldValue = StateReadOp::create(module.builder, result.getLoc(), state);
664 module.loweredValues[{result, Phase::Old}] = oldValue;
665 }
666
667 return success();
668}
669
670/// Lower a memory and its read and write ports to corresponding
671/// `arc.memory_write` operations. Reads are also executed at this point and
672/// stored in `loweredValues` for later operations to pick up.
673LogicalResult OpLowering::lower(MemoryOp op) {
674 assert(phase == Phase::New);
675
676 // Collect all the reads and writes.
677 SmallVector<MemoryReadPortOp> reads;
678 SmallVector<MemoryWritePortOp> writes;
679
680 for (auto *user : op->getUsers()) {
681 if (auto read = dyn_cast<MemoryReadPortOp>(user)) {
682 reads.push_back(read);
683 } else if (auto write = dyn_cast<MemoryWritePortOp>(user)) {
684 writes.push_back(write);
685 } else {
686 auto d = op.emitOpError()
687 << "users must all be memory read or write port ops";
688 d.attachNote(user->getLoc())
689 << "but found " << user->getName() << " user here";
690 return d;
691 }
692 }
693
694 // Ensure all operands are lowered before we lower the memory itself.
695 if (initial) {
696 for (auto read : reads)
697 lowerValue(read, Phase::Old);
698 for (auto write : writes) {
699 if (write.getClock())
700 lowerValue(write.getClock(), Phase::New);
701 for (auto input : write.getInputs())
702 lowerValue(input, Phase::Old);
703 }
704 return success();
705 }
706
707 // Get the allocated storage for the memory.
708 auto state = module.getAllocatedState(op->getResult(0));
709
710 // Since we are going to write new values into storage, insert read ops that
711 // keep the old values around for any later ops that still need them.
712 for (auto read : reads) {
713 auto oldValue = lowerValue(read, Phase::Old);
714 if (!oldValue)
715 return failure();
716 module.loweredValues[{read, Phase::Old}] = oldValue;
717 }
718
719 // Lower the writes.
720 for (auto write : writes) {
721 if (!write.getClock())
722 return write.emitOpError() << "must have a clock";
723 if (write.getLatency() > 1)
724 return write.emitOpError("latencies > 1 not supported yet");
725
726 // Create the if op for the clock edge.
727 auto ifClockOp = createIfClockOp(write.getClock());
728 if (!ifClockOp)
729 return failure();
730 OpBuilder::InsertionGuard guard(module.builder);
731 module.builder.setInsertionPoint(ifClockOp.thenYield());
732
733 // Call the arc that computes the address, data, and enable.
734 SmallVector<Value> inputs;
735 for (auto input : write.getInputs()) {
736 auto lowered = lowerValue(input, Phase::Old);
737 if (!lowered)
738 return failure();
739 inputs.push_back(lowered);
740 }
741 auto callOp =
742 CallOp::create(module.builder, write.getLoc(),
743 write.getArcResultTypes(), write.getArc(), inputs);
744
745 // If the write has an enable, wrap the remaining logic in an if op.
746 if (write.getEnable()) {
747 auto ifEnableOp = createOrReuseIf(
748 module.builder, callOp.getResult(write.getEnableIdx()), false);
749 module.builder.setInsertionPoint(ifEnableOp.thenYield());
750 }
751
752 // If the write is masked, read the current
753 // value in the memory and merge it with the updated value.
754 auto address = callOp.getResult(write.getAddressIdx());
755 auto data = callOp.getResult(write.getDataIdx());
756 if (write.getMask()) {
757 auto mask = callOp.getResult(write.getMaskIdx(write.getEnable()));
758 auto maskInv = module.builder.createOrFold<comb::XorOp>(
759 write.getLoc(), mask,
760 ConstantOp::create(module.builder, write.getLoc(), mask.getType(),
761 -1),
762 true);
763 auto oldData =
764 MemoryReadOp::create(module.builder, write.getLoc(), state, address);
765 auto oldMasked = comb::AndOp::create(module.builder, write.getLoc(),
766 maskInv, oldData, true);
767 auto newMasked =
768 comb::AndOp::create(module.builder, write.getLoc(), mask, data, true);
769 data = comb::OrOp::create(module.builder, write.getLoc(), oldMasked,
770 newMasked, true);
771 }
772
773 // Actually write to the memory.
774 MemoryWriteOp::create(module.builder, write.getLoc(), state, address,
775 Value{}, data);
776 }
777
778 return success();
779}
780
781/// Lower a tap by allocating state storage for it and writing the current value
782/// observed by the tap to it.
783LogicalResult OpLowering::lower(TapOp op) {
784 assert(phase == Phase::New);
785
786 auto value = lowerValue(op.getValue(), phase);
787 if (initial)
788 return success();
789 if (!value)
790 return failure();
791
792 auto &state = module.allocatedTaps[op];
793 if (!state) {
794 auto alloc = AllocStateOp::create(module.allocBuilder, op.getLoc(),
795 StateType::get(value.getType()),
796 module.storageArg, true);
797 alloc->setAttr("name", op.getNameAttr());
798 state = alloc;
799 }
800 StateWriteOp::create(module.builder, op.getLoc(), state, value, Value{});
801 return success();
802}
803
804/// Lower an instance by allocating state storage for each of its inputs and
805/// writing the current value into that storage. This makes instance inputs
806/// behave like outputs of the top-level module.
807LogicalResult OpLowering::lower(InstanceOp op) {
808 assert(phase == Phase::New);
809
810 // Get the current values flowing into the instance's inputs.
811 SmallVector<Value> values;
812 for (auto operand : op.getOperands())
813 values.push_back(lowerValue(operand, Phase::New));
814 if (initial)
815 return success();
816 if (llvm::is_contained(values, Value{}))
817 return failure();
818
819 // Then allocate storage for each instance input and assign the corresponding
820 // value.
821 for (auto [value, name] : llvm::zip(values, op.getArgNames())) {
822 auto state = AllocStateOp::create(module.allocBuilder, value.getLoc(),
823 StateType::get(value.getType()),
824 module.storageArg);
825 state->setAttr("name", module.builder.getStringAttr(
826 op.getInstanceName() + "/" +
827 cast<StringAttr>(name).getValue()));
828 StateWriteOp::create(module.builder, value.getLoc(), state, value, Value{});
829 }
830
831 // HACK: Also ensure that storage has been allocated for all outputs.
832 // Otherwise only the actually used instance outputs would be allocated, which
833 // would make the optimization user-visible. Remove this once we use the debug
834 // dialect.
835 for (auto result : op.getResults())
836 module.getAllocatedState(result);
837
838 return success();
839}
840
841/// Lower the main module's outputs by allocating storage for each and then
842/// writing the current value into that storage.
843LogicalResult OpLowering::lower(hw::OutputOp op) {
844 assert(phase == Phase::New);
845
846 // First get the current value of all outputs.
847 SmallVector<Value> values;
848 for (auto operand : op.getOperands())
849 values.push_back(lowerValue(operand, Phase::New));
850 if (initial)
851 return success();
852 if (llvm::is_contained(values, Value{}))
853 return failure();
854
855 // Then allocate storage for each output and assign the corresponding value.
856 for (auto [value, name] :
857 llvm::zip(values, module.moduleOp.getOutputNames())) {
858 auto state = RootOutputOp::create(
859 module.allocBuilder, value.getLoc(), StateType::get(value.getType()),
860 cast<StringAttr>(name), module.storageArg);
861 StateWriteOp::create(module.builder, value.getLoc(), state, value, Value{});
862 }
863 return success();
864}
865
866/// Lower `seq.initial` ops by inlining them into the `arc.initial` op.
867LogicalResult OpLowering::lower(seq::InitialOp op) {
868 assert(phase == Phase::Initial);
869
870 // First get the initial value of all operands.
871 SmallVector<Value> operands;
872 for (auto operand : op.getOperands())
873 operands.push_back(lowerValue(operand, Phase::Initial));
874 if (initial)
875 return success();
876 if (llvm::is_contained(operands, Value{}))
877 return failure();
878
879 // Expose the `seq.initial` operands as values for the block arguments.
880 for (auto [arg, operand] : llvm::zip(op.getBody().getArguments(), operands))
881 module.loweredValues[{arg, Phase::Initial}] = operand;
882
883 // Lower each op in the body.
884 for (auto &bodyOp : op.getOps()) {
885 if (isa<seq::YieldOp>(bodyOp))
886 continue;
887
888 // Clone the operation.
889 auto *clonedOp = module.initialBuilder.clone(bodyOp);
890 auto result = clonedOp->walk([&](Operation *nestedClonedOp) {
891 for (auto &operand : nestedClonedOp->getOpOperands()) {
892 if (clonedOp->isAncestor(operand.get().getParentBlock()->getParentOp()))
893 continue;
894 auto value = module.requireLoweredValue(operand.get(), Phase::Initial,
895 nestedClonedOp->getLoc());
896 if (!value)
897 return WalkResult::interrupt();
898 operand.set(value);
899 }
900 return WalkResult::advance();
901 });
902 if (result.wasInterrupted())
903 return failure();
904
905 // Keep track of the results.
906 for (auto [result, lowered] :
907 llvm::zip(bodyOp.getResults(), clonedOp->getResults()))
908 module.loweredValues[{result, Phase::Initial}] = lowered;
909 }
910
911 // Expose the operands of `seq.yield` as results from the initial op.
912 auto *terminator = op.getBodyBlock()->getTerminator();
913 for (auto [result, operand] :
914 llvm::zip(op.getResults(), terminator->getOperands())) {
915 auto value = module.requireLoweredValue(operand, Phase::Initial,
916 terminator->getLoc());
917 if (!value)
918 return failure();
919 module.loweredValues[{result, Phase::Initial}] = value;
920 }
921
922 return success();
923}
924
925/// Lower `llhd.final` ops into `scf.execute_region` ops in the `arc.final` op.
926LogicalResult OpLowering::lower(llhd::FinalOp op) {
927 assert(phase == Phase::Final);
928
929 // Determine the uses of values defined outside the op.
930 SmallVector<Value> externalOperands;
931 op.walk([&](Operation *nestedOp) {
932 for (auto value : nestedOp->getOperands())
933 if (!op->isAncestor(value.getParentBlock()->getParentOp()))
934 externalOperands.push_back(value);
935 });
936
937 // Make sure that all uses of external values are lowered first.
938 IRMapping mapping;
939 for (auto operand : externalOperands) {
940 auto lowered = lowerValue(operand, Phase::Final);
941 if (!initial && !lowered)
942 return failure();
943 mapping.map(operand, lowered);
944 }
945 if (initial)
946 return success();
947
948 // Handle the simple case where the final op contains only one block, which we
949 // can inline directly.
950 if (op.getBody().hasOneBlock()) {
951 for (auto &bodyOp : op.getBody().front().without_terminator())
952 module.finalBuilder.clone(bodyOp, mapping);
953 return success();
954 }
955
956 // Create a new `scf.execute_region` op and clone the entire `llhd.final` body
957 // region into it. Replace `llhd.halt` ops with `scf.yield`.
958 auto executeOp = scf::ExecuteRegionOp::create(module.finalBuilder,
959 op.getLoc(), TypeRange{});
960 module.finalBuilder.cloneRegionBefore(op.getBody(), executeOp.getRegion(),
961 executeOp.getRegion().begin(), mapping);
962 executeOp.walk([&](llhd::HaltOp op) {
963 auto builder = OpBuilder(op);
964 scf::YieldOp::create(builder, op.getLoc());
965 op.erase();
966 });
967
968 return success();
969}
970
971/// Create the operations necessary to detect a posedge on the given clock,
972/// potentially reusing a previous posedge detection, and create an `scf.if`
973/// operation for that posedge. This also tries to reuse an `scf.if` operation
974/// immediately before the builder's insertion point if possible.
975scf::IfOp OpLowering::createIfClockOp(Value clock) {
976 auto &posedge = module.loweredPosedges[clock];
977 if (!posedge) {
978 auto loweredClock = lowerValue(clock, Phase::New);
979 if (!loweredClock)
980 return {};
981 posedge = module.detectPosedge(loweredClock);
982 }
983 return createOrReuseIf(module.builder, posedge, false);
984}
985
986//===----------------------------------------------------------------------===//
987// Value Lowering
988//===----------------------------------------------------------------------===//
989
990/// Lower a value being used by the current operation. This will mark the
991/// defining operation as to be lowered first (through `addPending`) in most
992/// cases. Some operations and values have special handling though. For example,
993/// states and memory reads are immediately materialized as a new read op.
994Value OpLowering::lowerValue(Value value, Phase phase) {
995 // Handle module inputs. They read the same in all phases.
996 if (auto arg = dyn_cast<BlockArgument>(value)) {
997 if (initial)
998 return {};
999 auto state = module.allocatedInputs[arg.getArgNumber()];
1000 return StateReadOp::create(module.getBuilder(phase), arg.getLoc(), state);
1001 }
1002
1003 // Check if the value has already been lowered.
1004 if (auto lowered = module.loweredValues.lookup({value, phase}))
1005 return lowered;
1006
1007 // At this point the value is the result of an op. (Block arguments are
1008 // handled above.)
1009 auto result = cast<OpResult>(value);
1010 auto *op = result.getOwner();
1011
1012 // Special handling for some ops.
1013 if (auto instOp = dyn_cast<InstanceOp>(op))
1014 return lowerValue(instOp, result, phase);
1015 if (auto stateOp = dyn_cast<StateOp>(op))
1016 return lowerValue(stateOp, result, phase);
1017 if (auto dpiOp = dyn_cast<sim::DPICallOp>(op); dpiOp && dpiOp.getClock())
1018 return lowerValue(dpiOp, result, phase);
1019 if (auto readOp = dyn_cast<MemoryReadPortOp>(op))
1020 return lowerValue(readOp, result, phase);
1021 if (auto initialOp = dyn_cast<seq::InitialOp>(op))
1022 return lowerValue(initialOp, result, phase);
1023 if (auto castOp = dyn_cast<seq::FromImmutableOp>(op))
1024 return lowerValue(castOp, result, phase);
1025
1026 // Otherwise we mark the defining operation as to be lowered first. This will
1027 // cause the lookup in `loweredValues` above to return a value the next time
1028 // (i.e. when initial is false).
1029 if (initial) {
1030 addPending(op, phase);
1031 return {};
1032 }
1033 emitError(result.getLoc()) << "value has not been lowered";
1034 return {};
1035}
1036
1037/// Handle instance outputs. They behave essentially like a top-level module
1038/// input, and read the same in all phases.
1039Value OpLowering::lowerValue(InstanceOp op, OpResult result, Phase phase) {
1040 if (initial)
1041 return {};
1042 auto state = module.getAllocatedState(result);
1043 return StateReadOp::create(module.getBuilder(phase), result.getLoc(), state);
1044}
1045
1046/// Handle uses of a state. This creates an `arc.state_read` op to read from the
1047/// state's storage. If the new value after all updates is requested, marks the
1048/// state as to be lowered first (which will perform the writes). If the old
1049/// value is requested, asserts that no new values have been written.
1050Value OpLowering::lowerValue(StateOp op, OpResult result, Phase phase) {
1051 if (initial) {
1052 // Ensure that the new or initial value has been written by the lowering of
1053 // the state op before we attempt to read it.
1054 if (phase == Phase::New || phase == Phase::Initial)
1055 addPending(op, phase);
1056 return {};
1057 }
1058
1059 // If we want to read the old value, no writes must have been lowered yet.
1060 if (phase == Phase::Old)
1061 assert(!module.loweredOps.contains({op, Phase::New}) &&
1062 "need old value but new value already written");
1063
1064 auto state = module.getAllocatedState(result);
1065 return StateReadOp::create(module.getBuilder(phase), result.getLoc(), state);
1066}
1067
1068/// Handle uses of a DPI call. This creates an `arc.state_read` op to read from
1069/// the state's storage. If the new value after all updates is requested, marks
1070/// the state as to be lowered first (which will perform the writes). If the old
1071/// value is requested, asserts that no new values have been written.
1072Value OpLowering::lowerValue(sim::DPICallOp op, OpResult result, Phase phase) {
1073 if (initial) {
1074 // Ensure that the new or initial value has been written by the lowering of
1075 // the state op before we attempt to read it.
1076 if (phase == Phase::New || phase == Phase::Initial)
1077 addPending(op, phase);
1078 return {};
1079 }
1080
1081 // If we want to read the old value, no writes must have been lowered yet.
1082 if (phase == Phase::Old)
1083 assert(!module.loweredOps.contains({op, Phase::New}) &&
1084 "need old value but new value already written");
1085
1086 auto state = module.getAllocatedState(result);
1087 return StateReadOp::create(module.getBuilder(phase), result.getLoc(), state);
1088}
1089
1090/// Handle uses of a memory read operation. This creates an `arc.memory_read` op
1091/// to read from the memory's storage. Similar to the `StateOp` handling
1092/// otherwise.
1093Value OpLowering::lowerValue(MemoryReadPortOp op, OpResult result,
1094 Phase phase) {
1095 auto memOp = op.getMemory().getDefiningOp<MemoryOp>();
1096 if (!memOp) {
1097 if (!initial)
1098 op->emitOpError() << "memory must be defined locally";
1099 return {};
1100 }
1101
1102 auto address = lowerValue(op.getAddress(), phase);
1103 if (initial) {
1104 // Ensure that all new values are written before we attempt to read them.
1105 if (phase == Phase::New)
1106 addPending(memOp.getOperation(), Phase::New);
1107 return {};
1108 }
1109 if (!address)
1110 return {};
1111
1112 if (phase == Phase::Old) {
1113 // If we want to read the old value, no writes must have been lowered yet.
1114 assert(!module.loweredOps.contains({memOp, Phase::New}) &&
1115 "need old memory value but new value already written");
1116 } else {
1117 assert(phase == Phase::New);
1118 }
1119
1120 auto state = module.getAllocatedState(memOp->getResult(0));
1121 return MemoryReadOp::create(module.getBuilder(phase), result.getLoc(), state,
1122 address);
1123}
1124
1125/// Handle uses of `seq.initial` values computed during the initial phase. This
1126/// ensures that the interesting value is stored into storage during the initial
1127/// phase, and then reads it back using an `arc.state_read` op.
1128Value OpLowering::lowerValue(seq::InitialOp op, OpResult result, Phase phase) {
1129 // Ensure the op has been lowered first.
1130 if (initial) {
1131 addPending(op, Phase::Initial);
1132 return {};
1133 }
1134 auto value = module.loweredValues.lookup({result, Phase::Initial});
1135 if (!value) {
1136 emitError(result.getLoc()) << "value has not been lowered";
1137 return {};
1138 }
1139
1140 // If we are using the value of `seq.initial` in the initial phase directly,
1141 // there is no need to write it so any temporary storage.
1142 if (phase == Phase::Initial)
1143 return value;
1144
1145 // If necessary, allocate storage for the computed value and store it in the
1146 // initial phase.
1147 auto &state = module.allocatedInitials[result];
1148 if (!state) {
1149 state = AllocStateOp::create(module.allocBuilder, value.getLoc(),
1150 StateType::get(value.getType()),
1151 module.storageArg);
1152 OpBuilder::InsertionGuard guard(module.initialBuilder);
1153 module.initialBuilder.setInsertionPointAfterValue(value);
1154 StateWriteOp::create(module.initialBuilder, value.getLoc(), state, value,
1155 Value{});
1156 }
1157
1158 // Read back the value computed during the initial phase.
1159 return StateReadOp::create(module.getBuilder(phase), state.getLoc(), state);
1160}
1161
1162/// The `seq.from_immutable` cast is just a passthrough.
1163Value OpLowering::lowerValue(seq::FromImmutableOp op, OpResult result,
1164 Phase phase) {
1165 return lowerValue(op.getInput(), phase);
1166}
1167
1168/// Mark a value as to be lowered before the current op.
1169void OpLowering::addPending(Value value, Phase phase) {
1170 auto *defOp = value.getDefiningOp();
1171 assert(defOp && "block args should never be marked as a dependency");
1172 addPending(defOp, phase);
1173}
1174
1175/// Mark an operation as to be lowered before the current op. This adds that
1176/// operation to the `pending` list if the operation has not yet been lowered.
1177void OpLowering::addPending(Operation *op, Phase phase) {
1178 auto pair = std::make_pair(op, phase);
1179 if (!module.loweredOps.contains(pair))
1180 if (!llvm::is_contained(pending, pair))
1181 pending.push_back(pair);
1182}
1183
1184//===----------------------------------------------------------------------===//
1185// Pass Infrastructure
1186//===----------------------------------------------------------------------===//
1187
1188namespace {
1189struct LowerStatePass : public arc::impl::LowerStatePassBase<LowerStatePass> {
1190 using LowerStatePassBase::LowerStatePassBase;
1191 void runOnOperation() override;
1192};
1193} // namespace
1194
1195void LowerStatePass::runOnOperation() {
1196 auto op = getOperation();
1197 for (auto moduleOp : llvm::make_early_inc_range(op.getOps<HWModuleOp>())) {
1198 if (failed(ModuleLowering(moduleOp).run()))
1199 return signalPassFailure();
1200 moduleOp.erase();
1201 }
1202
1203 SymbolTable symbolTable(op);
1204 for (auto extModuleOp :
1205 llvm::make_early_inc_range(op.getOps<HWModuleExternOp>())) {
1206 // Make sure that we're not leaving behind a dangling reference to this
1207 // module
1208 auto uses = symbolTable.getSymbolUses(extModuleOp, op);
1209 if (!uses->empty()) {
1210 extModuleOp->emitError("Failed to remove external module because it is "
1211 "still referenced/instantiated");
1212 return signalPassFailure();
1213 }
1214 extModuleOp.erase();
1215 }
1216}
assert(baseType &&"element must be base type")
static bool isAncestor(Block *block, Block *other)
Definition LayerSink.cpp:60
static scf::IfOp createOrReuseIf(OpBuilder &builder, Value condition, bool withElse)
Create a new scf.if operation with the given builder, or reuse a previous scf.if if the builder's ins...
static Location getLoc(DefSlot slot)
Definition Mem2Reg.cpp:216
static Block * getBodyBlock(FModuleLike mod)
create(data_type, value)
Definition hw.py:441
create(data_type, value)
Definition hw.py:433
OS & operator<<(OS &os, const InnerSymTarget &target)
Printing InnerSymTarget's.
The InstanceGraph op interface, see InstanceGraphInterface.td for more details.
int run(Type[Generator] generator=CppGenerator, cmdline_args=sys.argv)
Definition codegen.py:121
Definition hw.py:1