ESILowerPhysical.cpp

Go to the documentation of this file.

//===- ESILowerPhysical.cpp - Lower ESI to physical -------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Lower ESI to ESI "physical level" ops conversions and pass.
//
//===----------------------------------------------------------------------===//
 
#include "../PassDetails.h"
 
#include "circt/Dialect/Comb/CombOps.h"
#include "circt/Dialect/ESI/ESIOps.h"
#include "circt/Dialect/HW/HWOps.h"
#include "circt/Dialect/Seq/SeqOps.h"
#include "circt/Support/BackedgeBuilder.h"
#include "circt/Support/LLVM.h"
 
#include "mlir/Transforms/DialectConversion.h"
 
namespace circt {
namespace esi {
#define GEN_PASS_DEF_LOWERESITOPHYSICAL
#include "circt/Dialect/ESI/ESIPasses.h.inc"
} // namespace esi
} // namespace circt
 
using namespace circt;
using namespace circt::esi;
using namespace circt::esi::detail;
using namespace circt::hw;
 
namespace {
/// Lower `ChannelBufferOp`s, breaking out the various options. For now, just
/// replace with the specified number of pipeline stages (since that's the only
/// option).
struct ChannelBufferLowering : public OpConversionPattern<ChannelBufferOp> {
public:
  using OpConversionPattern::OpConversionPattern;
 
  LogicalResult
  matchAndRewrite(ChannelBufferOp buffer, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const final;
};
} // anonymous namespace
 
/// Since as of now the only construct supported by the StageOp, we convert FIFO
/// signaling to/from ValidReady signaling. This will be done opposite way when
/// we implement FIFO-based buffers. Changes in data delay are made after the
/// stages.
LogicalResult ChannelBufferLowering::matchAndRewrite(
    ChannelBufferOp buffer, OpAdaptor adaptor,
    ConversionPatternRewriter &rewriter) const {
  auto loc = buffer.getLoc();
 
  ChannelType inputType = buffer.getInput().getType();
  Value stageInput = buffer.getInput();
  // The stages carry the input data delay.
  auto stageType =
      ChannelType::get(rewriter.getContext(), inputType.getInner(),
                       ChannelSignaling::ValidReady, inputType.getDataDelay());
 
  // FIFO signaling must be converted to ValidReady.
  if (inputType.getSignaling() == ChannelSignaling::FIFO) {
    BackedgeBuilder bb(rewriter, loc);
    Backedge rdEn = bb.get(rewriter.getI1Type());
    Backedge valid = bb.get(rewriter.getI1Type());
 
    auto unwrap = UnwrapFIFOOp::create(rewriter, loc, stageInput, rdEn);
    auto wrap =
        WrapValidReadyOp::create(rewriter, loc, stageType, rewriter.getI1Type(),
                                 unwrap.getData(), valid);
    stageInput = wrap.getChanOutput();
 
    // rdEn = valid && ready
    rdEn.setValue(comb::AndOp::create(rewriter, loc, wrap.getReady(), valid));
    // valid = !empty
    valid.setValue(comb::XorOp::create(
        rewriter, loc, unwrap.getEmpty(),
        hw::ConstantOp::create(rewriter, loc, rewriter.getBoolAttr(true))));
  }
 
  // ValidOnly signaling must be converted to ValidReady (always ready).
  if (inputType.getSignaling() == ChannelSignaling::ValidOnly) {
    auto unwrap = UnwrapValidOnlyOp::create(rewriter, loc, stageInput);
    auto wrap =
        WrapValidReadyOp::create(rewriter, loc, stageType, rewriter.getI1Type(),
                                 unwrap.getRawOutput(), unwrap.getValid());
    stageInput = wrap.getChanOutput();
  }
 
  // Expand 'abstract' buffer into 'physical' stages.
  auto stages = buffer.getStagesAttr();
  uint64_t numStages = 1;
  if (stages)
    // Guaranteed positive by the parser.
    numStages = stages.getValue().getLimitedValue();
  StringAttr bufferName = buffer.getNameAttr();
 
  for (uint64_t i = 0; i < numStages; ++i) {
    // Create the stages, connecting them up as we build.
    auto stage = PipelineStageOp::create(rewriter, loc, buffer.getClk(),
                                         buffer.getRst(), stageInput);
    if (bufferName) {
      SmallString<64> stageName(
          {bufferName.getValue(), "_stage", std::to_string(i)});
      stage->setAttr("name", StringAttr::get(rewriter.getContext(), stageName));
    }
    stageInput = stage;
  }
 
  ChannelType outputType = buffer.getOutput().getType();
  Value output = stageInput;
  uint64_t inDelay = inputType.getDataDelay();
  uint64_t outDelay = outputType.getDataDelay();
  auto i1 = rewriter.getI1Type();
  auto regName = [&](StringRef suffix, uint64_t i) -> std::string {
    std::string name = (suffix + Twine(i)).str();
    if (bufferName)
      name = (bufferName.getValue() + "_" + name).str();
    return name;
  };
 
  if (outDelay > inDelay) {
    // Delay the data so that it arrives 'outDelay' cycles after the transfer.
    BackedgeBuilder bb(rewriter, loc);
    Backedge ready = bb.get(i1);
    auto unwrap = UnwrapValidReadyOp::create(rewriter, loc, output, ready);
    Value data = unwrap.getRawOutput();
    // TODO: if outDelay >>> inDelay it might make sense to use a FIFO with
    // rd_latency.
    for (uint64_t i = 0; i < outDelay - inDelay; ++i)
      data = seq::CompRegOp::create(rewriter, loc, data, buffer.getClk(),
                                    regName("data_dly", i), InnerSymAttr());
    auto wrap = WrapValidReadyOp::create(rewriter, loc, data, unwrap.getValid(),
                                         outDelay);
    ready.setValue(wrap.getReady());
    output = wrap.getChanOutput();
  } else if (outDelay < inDelay &&
             outputType.getSignaling() == ChannelSignaling::ValidOnly) {
    // With no backpressure, tokens never wait. Delay the valid so that the
    // data arrives 'outDelay' cycles after it.
    auto alwaysReady = hw::ConstantOp::create(rewriter, loc, i1, 1);
    auto unwrap =
        UnwrapValidReadyOp::create(rewriter, loc, output, alwaysReady);
    Value valid = unwrap.getValid();
    Value false_ = hw::ConstantOp::create(rewriter, loc, i1, 0);
    for (uint64_t i = 0; i < inDelay - outDelay; ++i)
      valid = seq::CompRegOp::create(rewriter, loc, valid, buffer.getClk(),
                                     buffer.getRst(), false_,
                                     regName("valid_dly", i), InnerSymAttr());
    output = WrapValidOnlyOp::create(rewriter, loc, unwrap.getRawOutput(),
                                     valid, outDelay)
                 .getChanOutput();
  } else if (outDelay < inDelay) {
    // Tokens must wait until their data arrives, so hold them in a FIFO. Its
    // input reserves room for 'inDelay' writes in flight, so it needs that many
    // extra entries to sustain one transfer per cycle after an output stall.
    uint64_t depth = 2 * inDelay + std::max<uint64_t>(outDelay, 1) + 1;
    output = FIFOOp::create(rewriter, loc, outputType, buffer.getClk(),
                            buffer.getRst(), output,
                            rewriter.getI64IntegerAttr(depth));
  }
 
  // If the output is FIFO, we need to wrap it back into a FIFO from ValidReady.
  if (output.getType() != outputType &&
      outputType.getSignaling() == ChannelSignaling::FIFO) {
    BackedgeBuilder bb(rewriter, loc);
    Backedge ready = bb.get(rewriter.getI1Type());
    Backedge empty = bb.get(rewriter.getI1Type());
 
    auto unwrap = UnwrapValidReadyOp::create(rewriter, loc, output, ready);
    auto wrap = WrapFIFOOp::create(rewriter, loc,
                                   TypeRange{outputType, rewriter.getI1Type()},
                                   unwrap.getRawOutput(), empty);
 
    ready.setValue(wrap.getRden());
    empty.setValue(comb::XorOp::create(
        rewriter, loc, unwrap.getValid(),
        hw::ConstantOp::create(rewriter, loc, rewriter.getBoolAttr(true))));
    output = wrap.getChanOutput();
  }
 
  // If the output is ValidOnly, convert back from ValidReady.
  if (output.getType() != outputType &&
      outputType.getSignaling() == ChannelSignaling::ValidOnly) {
    // Always ready since ValidOnly has no backpressure.
    auto alwaysReady =
        hw::ConstantOp::create(rewriter, loc, rewriter.getI1Type(), 1);
    auto unwrap =
        UnwrapValidReadyOp::create(rewriter, loc, output, alwaysReady);
    auto wrap =
        WrapValidOnlyOp::create(rewriter, loc, unwrap.getRawOutput(),
                                unwrap.getValid(), outputType.getDataDelay());
    output = wrap.getChanOutput();
  }
 
  // Replace the buffer.
  rewriter.replaceOp(buffer, output);
  return success();
}
 
namespace {
/// Lower `ChannelBufferOp`s, breaking out the various options. For now, just
/// replace with the specified number of pipeline stages (since that's the only
/// option).
struct FIFOLowering : public OpConversionPattern<FIFOOp> {
public:
  using OpConversionPattern::OpConversionPattern;
 
  LogicalResult
  matchAndRewrite(FIFOOp, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const final;
};
} // anonymous namespace
 
LogicalResult
FIFOLowering::matchAndRewrite(FIFOOp op, OpAdaptor adaptor,
                              ConversionPatternRewriter &rewriter) const {
  auto loc = op.getLoc();
  auto outputType = op.getType();
  BackedgeBuilder bb(rewriter, loc);
  auto i1 = rewriter.getI1Type();
  auto c1 = hw::ConstantOp::create(rewriter, loc, rewriter.getI1Type(),
                                   rewriter.getBoolAttr(true));
  mlir::TypedValue<ChannelType> chanInput = op.getInput();
  uint64_t inputDelay = chanInput.getType().getDataDelay();
 
  Backedge inputEn = bb.get(i1);
  Value rawData;
  Value dataNotAvailable;
  if (chanInput.getType().getSignaling() == ChannelSignaling::ValidReady) {
    auto unwrapValidReady =
        UnwrapValidReadyOp::create(rewriter, loc, chanInput, inputEn);
    rawData = unwrapValidReady.getRawOutput();
    dataNotAvailable =
        comb::createOrFoldNot(rewriter, loc, unwrapValidReady.getValid(),
                              /*twoState=*/true);
    dataNotAvailable.getDefiningOp()->setAttr(
        "sv.namehint", rewriter.getStringAttr("dataNotAvailable"));
  } else if (chanInput.getType().getSignaling() == ChannelSignaling::FIFO) {
    auto unwrapPull = UnwrapFIFOOp::create(rewriter, loc, chanInput, inputEn);
    rawData = unwrapPull.getData();
    dataNotAvailable = unwrapPull.getEmpty();
  } else if (chanInput.getType().getSignaling() ==
             ChannelSignaling::ValidOnly) {
    auto unwrapVO = UnwrapValidOnlyOp::create(rewriter, loc, chanInput);
    rawData = unwrapVO.getRawOutput();
    dataNotAvailable = comb::createOrFoldNot(rewriter, loc, unwrapVO.getValid(),
                                             /*twoState=*/true);
    dataNotAvailable.getDefiningOp()->setAttr(
        "sv.namehint", rewriter.getStringAttr("dataNotAvailable"));
  } else {
    return rewriter.notifyMatchFailure(
        op, "only supports ValidReady, FIFO, and ValidOnly signaling");
  }
 
  // With an input data delay of N, the data for an accepted token arrives N
  // cycles later, so delay the write to match. Since up to N writes may be in
  // flight, only accept input while more than N slots are free. A ValidOnly
  // sender can't be stalled though, so instead every token is written. If the
  // FIFO is full, it overflows, which the `seq.fifo` assertions catch in
  // simulation.
  bool validOnlyInput =
      chanInput.getType().getSignaling() == ChannelSignaling::ValidOnly;
  bool reserveSlots = inputDelay > 0 && !validOnlyInput;
  Backedge wrEn = bb.get(i1);
  Type almostFullType;
  IntegerAttr almostFullThreshold;
  if (reserveSlots) {
    almostFullType = i1;
    almostFullThreshold =
        rewriter.getI64IntegerAttr(op.getDepth() - inputDelay);
  }
 
  Backedge outputRdEn = bb.get(i1);
  auto seqFifo = seq::FIFOOp::create(
      rewriter, loc, outputType.getInner(), i1, i1, almostFullType, Type(),
      rawData, outputRdEn, wrEn, op.getClk(), op.getRst(), op.getDepthAttr(),
      rewriter.getI64IntegerAttr(outputType.getDataDelay()),
      almostFullThreshold, IntegerAttr());
  auto inputNotEmpty = comb::XorOp::create(rewriter, loc, dataNotAvailable, c1);
  inputNotEmpty->setAttr("sv.namehint",
                         rewriter.getStringAttr("inputNotEmpty"));
  auto seqFifoNotFull = comb::XorOp::create(
      rewriter, loc, reserveSlots ? seqFifo.getAlmostFull() : seqFifo.getFull(),
      c1);
  seqFifoNotFull->setAttr("sv.namehint",
                          rewriter.getStringAttr("seqFifoNotFull"));
  inputEn.setValue(
      comb::AndOp::create(rewriter, loc, inputNotEmpty, seqFifoNotFull));
  static_cast<Value>(inputEn).getDefiningOp()->setAttr(
      "sv.namehint", rewriter.getStringAttr("inputEn"));
 
  Value write = validOnlyInput ? Value(inputNotEmpty) : Value(inputEn);
  if (inputDelay > 0) {
    Value false_ = hw::ConstantOp::create(rewriter, loc, i1, 0);
    for (uint64_t i = 0; i < inputDelay; ++i)
      write = seq::CompRegOp::create(
          rewriter, loc, write, op.getClk(), op.getRst(), false_,
          (validOnlyInput ? "valid_dly" : "inputEn_dly") + Twine(i),
          InnerSymAttr());
  }
  wrEn.setValue(write);
 
  Value output;
  if (outputType.getSignaling() == ChannelSignaling::ValidReady) {
    auto wrap = WrapValidReadyOp::create(
        rewriter, loc, mlir::TypeRange{outputType, i1}, seqFifo.getOutput(),
        comb::createOrFoldNot(rewriter, loc, seqFifo.getEmpty(),
                              /*twoState=*/true));
    output = wrap.getChanOutput();
    outputRdEn.setValue(
        comb::AndOp::create(rewriter, loc, wrap.getValid(), wrap.getReady()));
    static_cast<Value>(outputRdEn)
        .getDefiningOp()
        ->setAttr("sv.namehint", rewriter.getStringAttr("outputRdEn"));
  } else if (outputType.getSignaling() == ChannelSignaling::FIFO) {
    auto wrap =
        WrapFIFOOp::create(rewriter, loc, mlir::TypeRange{outputType, i1},
                           seqFifo.getOutput(), seqFifo.getEmpty());
    output = wrap.getChanOutput();
    outputRdEn.setValue(wrap.getRden());
  } else if (outputType.getSignaling() == ChannelSignaling::ValidOnly) {
    // ValidOnly output: wrap with valid signal, always read from FIFO when
    // data is available (no backpressure).
    auto notEmpty = comb::createOrFoldNot(rewriter, loc, seqFifo.getEmpty(),
                                          /*twoState=*/true);
    auto wrap = WrapValidOnlyOp::create(rewriter, loc, seqFifo.getOutput(),
                                        notEmpty, outputType.getDataDelay());
    output = wrap.getChanOutput();
    // Always read when not empty since there's no backpressure.
    outputRdEn.setValue(notEmpty);
  } else {
    return rewriter.notifyMatchFailure(
        op, "only supports ValidReady, FIFO, and ValidOnly");
  }
 
  rewriter.replaceOp(op, output);
  return success();
}
 
namespace {
/// Lower pure modules into hw.modules.
struct PureModuleLowering : public OpConversionPattern<ESIPureModuleOp> {
public:
  using OpConversionPattern::OpConversionPattern;
 
  LogicalResult
  matchAndRewrite(ESIPureModuleOp pureMod, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const final;
};
} // anonymous namespace
 
LogicalResult
PureModuleLowering::matchAndRewrite(ESIPureModuleOp pureMod, OpAdaptor adaptor,
                                    ConversionPatternRewriter &rewriter) const {
  auto loc = pureMod.getLoc();
  Block *body = &pureMod.getBody().front();
 
  // Track existing names (so we can de-dup) and get op result when we want to
  // replace it with the block args.
  DenseMap<StringAttr, ESIPureModuleInputOp> inputPortNames;
  // Build the port list for `hw.module` construction.
  SmallVector<hw::PortInfo> ports;
  // List the input and output ops.
  SmallVector<ESIPureModuleInputOp> inputs;
  SmallVector<ESIPureModuleOutputOp> outputs;
  SmallVector<Attribute> params;
 
  for (Operation &op : llvm::make_early_inc_range(body->getOperations())) {
    if (auto port = dyn_cast<ESIPureModuleInputOp>(op)) {
      // If we already have an input port of the same name, replace the result
      // value with the previous one. Checking that the types match is done in
      // the pure module verifier.
      auto existingPort = inputPortNames.find(port.getNameAttr());
      if (existingPort != inputPortNames.end()) {
        rewriter.replaceOp(port, existingPort->getSecond().getResult());
        continue;
      }
      // Normal port construction.
      ports.push_back(
          hw::PortInfo{{port.getNameAttr(), port.getResult().getType(),
                        hw::ModulePort::Direction::Input},
                       inputs.size(),
                       {},
                       port.getLoc()});
      inputs.push_back(port);
    } else if (auto port = dyn_cast<ESIPureModuleOutputOp>(op)) {
      ports.push_back(
          hw::PortInfo{{port.getNameAttr(), port.getValue().getType(),
                        hw::ModulePort::Direction::Output},
                       outputs.size(),
                       {},
                       port.getLoc()});
      outputs.push_back(port);
    } else if (auto param = dyn_cast<ESIPureModuleParamOp>(op)) {
      params.push_back(
          ParamDeclAttr::get(param.getNameAttr(), param.getType()));
      rewriter.eraseOp(param);
    }
  }
 
  // Create the replacement `hw.module`.
  auto hwMod =
      hw::HWModuleOp::create(rewriter, loc, pureMod.getNameAttr(), ports,
                             ArrayAttr::get(getContext(), params));
  hwMod->setDialectAttrs(pureMod->getDialectAttrs());
  rewriter.eraseBlock(hwMod.getBodyBlock());
  rewriter.inlineRegionBefore(*body->getParent(), hwMod.getBodyRegion(),
                              hwMod.getBodyRegion().end());
  body = hwMod.getBodyBlock();
 
  // Re-wire the inputs and erase them.
  for (auto input : inputs) {
    BlockArgument newArg;
    rewriter.modifyOpInPlace(hwMod, [&]() {
      newArg = body->addArgument(input.getResult().getType(), input.getLoc());
    });
    rewriter.replaceOp(input, newArg);
  }
 
  // Assemble the output values.
  SmallVector<Value> hwOutputOperands;
  for (auto output : outputs) {
    hwOutputOperands.push_back(output.getValue());
    rewriter.eraseOp(output);
  }
  rewriter.setInsertionPointToEnd(body);
  hw::OutputOp::create(rewriter, pureMod.getLoc(), hwOutputOperands);
 
  // Erase the original op.
  rewriter.eraseOp(pureMod);
  return success();
}
 
namespace {
/// Run all the physical lowerings.
struct ESIToPhysicalPass
    : public circt::esi::impl::LowerESIToPhysicalBase<ESIToPhysicalPass> {
  void runOnOperation() override;
};
} // anonymous namespace
 
void ESIToPhysicalPass::runOnOperation() {
  // Set up a conversion and give it a set of laws.
  ConversionTarget target(getContext());
  target.markUnknownOpDynamicallyLegal([](Operation *) { return true; });
  target.addIllegalOp<ChannelBufferOp, ESIPureModuleOp, FIFOOp>();
 
  // Add all the conversion patterns.
  RewritePatternSet patterns(&getContext());
  patterns.insert<ChannelBufferLowering, PureModuleLowering, FIFOLowering>(
      &getContext());
 
  // Run the conversion.
  if (failed(
          applyPartialConversion(getOperation(), target, std::move(patterns))))
    signalPassFailure();
}
 
std::unique_ptr<OperationPass<ModuleOp>>
circt::esi::createESIPhysicalLoweringPass() {
  return std::make_unique<ESIToPhysicalPass>();
}