26 #include "mlir/IR/ImplicitLocOpBuilder.h"
27 #include "mlir/Pass/Pass.h"
28 #include "mlir/Pass/PassManager.h"
29 #include "mlir/Transforms/DialectConversion.h"
30 #include "llvm/ADT/TypeSwitch.h"
31 #include "llvm/Support/MathExtras.h"
36 #define GEN_PASS_DEF_DCTOHW
37 #include "circt/Conversion/Passes.h.inc"
41 using namespace circt;
49 auto *ctx = tuple.getContext();
50 mlir::SmallVector<hw::StructType::FieldInfo, 8> hwfields;
51 for (
auto [i,
innerType] : llvm::enumerate(tuple)) {
53 if (
auto tupleInnerType = dyn_cast<TupleType>(
innerType))
68 return TypeSwitch<Type, Type>(t)
70 .Case([](hw::StructType st) {
71 llvm::SmallVector<hw::StructType::FieldInfo> structFields(
73 for (
auto &field : structFields)
78 .Default([](Type t) {
return t; });
83 llvm::TypeSwitch<Type, Type>(t)
84 .Case([](ValueType vt) {
88 .Case([](TokenType tt) {
92 .Default([](
auto t) {
return toHWType(t); });
101 struct DCLoweringState {
102 ModuleOp parentModule;
109 class ESITypeConverter :
public TypeConverter {
112 addConversion([](Type type) -> Type {
return toESIHWType(type); });
114 addTargetMaterialization(
115 [](mlir::OpBuilder &builder, mlir::Type resultType,
116 mlir::ValueRange inputs,
117 mlir::Location loc) -> std::optional<mlir::Value> {
118 if (inputs.size() != 1)
124 addSourceMaterialization(
125 [](mlir::OpBuilder &builder, mlir::Type resultType,
126 mlir::ValueRange inputs,
127 mlir::Location loc) -> std::optional<mlir::Value> {
128 if (inputs.size() != 1)
146 struct InputHandshake {
149 std::optional<Backedge> ready;
155 struct OutputHandshake {
157 std::optional<Backedge> valid;
159 std::optional<Backedge>
data;
163 static void connect(InputHandshake &input, OutputHandshake &output) {
164 output.valid->setValue(input.valid);
165 input.ready->setValue(output.ready);
168 template <
typename T,
typename TInner>
169 llvm::SmallVector<T> extractValues(llvm::SmallVector<TInner> &container,
170 llvm::function_ref<T(TInner &)> extractor) {
171 llvm::SmallVector<T> result;
172 llvm::transform(container, std::back_inserter(result), extractor);
179 llvm::SmallVector<InputHandshake> inputs;
180 llvm::SmallVector<OutputHandshake> outputs;
182 llvm::SmallVector<Value> getInputValids() {
183 return extractValues<Value, InputHandshake>(
184 inputs, [](
auto &hs) {
return hs.valid; });
186 llvm::SmallVector<std::optional<Backedge>> getInputReadys() {
187 return extractValues<std::optional<Backedge>, InputHandshake>(
188 inputs, [](
auto &hs) {
return hs.ready; });
190 llvm::SmallVector<std::optional<Backedge>> getOutputValids() {
191 return extractValues<std::optional<Backedge>, OutputHandshake>(
192 outputs, [](
auto &hs) {
return hs.valid; });
194 llvm::SmallVector<Value> getInputDatas() {
195 return extractValues<Value, InputHandshake>(
196 inputs, [](
auto &hs) {
return hs.data; });
198 llvm::SmallVector<Value> getOutputReadys() {
199 return extractValues<Value, OutputHandshake>(
200 outputs, [](
auto &hs) {
return hs.ready; });
203 llvm::SmallVector<Value> getOutputChannels() {
204 return extractValues<Value, OutputHandshake>(
205 outputs, [](
auto &hs) {
return hs.channel; });
207 llvm::SmallVector<std::optional<Backedge>> getOutputDatas() {
208 return extractValues<std::optional<Backedge>, OutputHandshake>(
209 outputs, [](
auto &hs) {
return hs.data; });
217 RTLBuilder(Location loc, OpBuilder &builder, Value clk = Value(),
219 : b(builder), loc(loc),
clk(
clk), rst(rst) {}
221 Value constant(
const APInt &apv, StringRef name = {}) {
224 bool isZeroWidth = apv.getBitWidth() == 0;
226 auto it = constants.find(apv);
227 if (it != constants.end())
233 constants[apv] = cval;
237 Value constant(
unsigned width, int64_t value, StringRef name = {}) {
238 return constant(APInt(
width, value));
240 std::pair<Value, Value>
wrap(Value data, Value valid, StringRef name = {}) {
241 auto wrapOp = b.create<esi::WrapValidReadyOp>(loc,
data, valid);
242 return {wrapOp.getResult(0), wrapOp.getResult(1)};
244 std::pair<Value, Value>
unwrap(Value channel, Value ready,
245 StringRef name = {}) {
246 auto unwrapOp = b.create<esi::UnwrapValidReadyOp>(loc, channel, ready);
247 return {unwrapOp.getResult(0), unwrapOp.getResult(1)};
251 Value
reg(StringRef name, Value in, Value rstValue, Value clk = Value(),
252 Value rst = Value()) {
253 Value resolvedClk =
clk ?
clk : this->
clk;
254 Value resolvedRst = rst ? rst : this->rst;
256 "No global clock provided to this RTLBuilder - a clock "
257 "signal must be provided to the reg(...) function.");
259 "No global reset provided to this RTLBuilder - a reset "
260 "signal must be provided to the reg(...) function.");
262 return b.create<
seq::CompRegOp>(loc, in, resolvedClk, resolvedRst, rstValue,
266 Value cmp(Value lhs, Value rhs, comb::ICmpPredicate predicate,
267 StringRef name = {}) {
268 return b.
create<comb::ICmpOp>(loc, predicate, lhs, rhs);
271 Value buildNamedOp(llvm::function_ref<Value()> f, StringRef name) {
274 Operation *op = v.getDefiningOp();
276 op->setAttr(
"sv.namehint", b.getStringAttr(name));
277 nameAttr = b.getStringAttr(name);
283 Value bitAnd(ValueRange values, StringRef name = {}) {
285 [&]() {
return b.create<
comb::AndOp>(loc, values,
false); }, name);
289 Value bitOr(ValueRange values, StringRef name = {}) {
291 [&]() {
return b.create<
comb::OrOp>(loc, values,
false); }, name);
295 Value bitNot(Value value, StringRef name = {}) {
296 auto allOnes = constant(value.getType().getIntOrFloatBitWidth(), -1);
297 std::string inferedName;
301 value.getDefiningOp()->getAttrOfType<StringAttr>(
"sv.namehint")) {
302 inferedName = (
"not_" +
valueName.getValue()).str();
308 [&]() {
return b.create<
comb::XorOp>(loc, value, allOnes); }, name);
311 Value shl(Value value, Value shift, StringRef name = {}) {
313 [&]() {
return b.create<
comb::ShlOp>(loc, value, shift); }, name);
316 Value
concat(ValueRange values, StringRef name = {}) {
317 return buildNamedOp([&]() {
return b.create<
comb::ConcatOp>(loc, values); },
321 llvm::SmallVector<Value>
extractBits(Value v, StringRef name = {}) {
322 llvm::SmallVector<Value> bits;
323 for (
unsigned i = 0, e = v.getType().getIntOrFloatBitWidth(); i != e; ++i)
329 Value reduceOr(Value v, StringRef name = {}) {
330 return buildNamedOp([&]() {
return bitOr(
extractBits(v)); }, name);
334 Value extract(Value v,
unsigned lo,
unsigned hi, StringRef name = {}) {
335 unsigned width = hi - lo + 1;
341 Value truncate(Value value,
unsigned width, StringRef name = {}) {
342 return extract(value, 0,
width - 1, name);
345 Value zext(Value value,
unsigned outWidth, StringRef name = {}) {
346 unsigned inWidth = value.getType().getIntOrFloatBitWidth();
347 assert(inWidth <= outWidth &&
"zext: input width must be <= output width.");
348 if (inWidth == outWidth)
350 auto c0 = constant(outWidth - inWidth, 0);
351 return concat({c0, value}, name);
354 Value sext(Value value,
unsigned outWidth, StringRef name = {}) {
359 Value bit(Value v,
unsigned index, StringRef name = {}) {
360 return extract(v, index, index, name);
364 Value arrayCreate(ValueRange values, StringRef name = {}) {
370 Value arrayGet(Value array, Value index, StringRef name = {}) {
372 [&]() {
return b.create<
hw::ArrayGetOp>(loc, array, index); }, name);
378 Value mux(Value index, ValueRange values, StringRef name = {}) {
379 if (values.size() == 2) {
382 return b.create<
comb::MuxOp>(loc, index, values[1], values[0]);
386 return arrayGet(arrayCreate(values), index, name);
391 Value oneHotMux(Value index, ValueRange inputs) {
393 unsigned numInputs = inputs.size();
394 assert(numInputs == index.getType().getIntOrFloatBitWidth() &&
395 "mismatch between width of one-hot select input and the number of "
396 "inputs to be selected");
399 auto dataType = inputs[0].getType();
401 isa<NoneType>(dataType) ? 0 : dataType.getIntOrFloatBitWidth();
402 Value muxValue = constant(
width, 0);
405 for (
size_t i = numInputs - 1; i != 0; --i) {
406 Value input = inputs[i];
407 Value selectBit = bit(index, i);
408 muxValue = mux(selectBit, {muxValue, input});
417 DenseMap<APInt, Value> constants;
420 static bool isZeroWidthType(Type type) {
421 if (
auto intType = dyn_cast<IntegerType>(type))
422 return intType.getWidth() == 0;
423 return isa<NoneType>(type);
426 static UnwrappedIO unwrapIO(Location loc, ValueRange operands,
428 ConversionPatternRewriter &rewriter,
430 RTLBuilder rtlb(loc, rewriter);
431 UnwrappedIO unwrapped;
432 for (
auto in : operands) {
433 assert(isa<esi::ChannelType>(in.getType()));
434 auto ready = bb.
get(rtlb.b.getI1Type());
435 auto [
data, valid] = rtlb.unwrap(in, ready);
436 unwrapped.inputs.push_back(InputHandshake{in, valid, ready,
data});
438 for (
auto outputType : results) {
447 rewriter.create<
hw::ConstantOp>(loc, rewriter.getIntegerType(0), 0);
451 hs.data = dataBackedge;
454 auto valid = bb.
get(rewriter.getI1Type());
455 auto [dataCh, ready] = rtlb.wrap(data, valid);
459 unwrapped.outputs.push_back(hs);
464 static UnwrappedIO unwrapIO(Operation *op, ValueRange operands,
465 ConversionPatternRewriter &rewriter,
467 return unwrapIO(op->getLoc(), operands, op->getResultTypes(), rewriter, bb);
472 static FailureOr<std::pair<Value, Value>> getClockAndReset(Operation *op) {
473 auto *parent = op->getParentOp();
474 auto parentFuncOp = dyn_cast<HWModuleLike>(parent);
476 return parent->emitOpError(
"parent op does not implement HWModuleLike");
478 auto argAttrs = parentFuncOp.getAllInputAttrs();
480 std::optional<size_t> clockIdx, resetIdx;
482 for (
auto [idx, battrs] : llvm::enumerate(argAttrs)) {
483 auto attrs = cast<DictionaryAttr>(battrs);
484 if (attrs.get(
"dc.clock")) {
486 return parent->emitOpError(
487 "multiple arguments contains a 'dc.clock' attribute");
491 if (attrs.get(
"dc.reset")) {
493 return parent->emitOpError(
494 "multiple arguments contains a 'dc.reset' attribute");
500 return parent->emitOpError(
"no argument contains a 'dc.clock' attribute");
503 return parent->emitOpError(
"no argument contains a 'dc.reset' attribute");
505 return {std::make_pair(parentFuncOp.getArgumentForInput(*clockIdx),
506 parentFuncOp.getArgumentForInput(*resetIdx))};
513 matchAndRewrite(ForkOp op, OpAdaptor operands,
514 ConversionPatternRewriter &rewriter)
const override {
516 auto crRes = getClockAndReset(op);
519 auto [clock, reset] = *crRes;
520 RTLBuilder rtlb(op.getLoc(), rewriter, clock, reset);
521 UnwrappedIO io = unwrapIO(op, operands.getOperands(), rewriter, bb);
523 auto &input = io.inputs[0];
525 Value c0I1 = rtlb.constant(1, 0);
526 llvm::SmallVector<Value> doneWires;
527 for (
auto [i, output] : llvm::enumerate(io.outputs)) {
529 Value emitted = rtlb.bitAnd({doneBE, rtlb.bitNot(*input.ready)});
531 rtlb.reg(
"emitted_" + std::to_string(i), emitted, c0I1);
532 Value outValid = rtlb.bitAnd({rtlb.bitNot(emittedReg), input.valid});
533 output.valid->setValue(outValid);
534 Value validReady = rtlb.bitAnd({output.ready, outValid});
536 rtlb.bitOr({validReady, emittedReg},
"done" + std::to_string(i));
538 doneWires.push_back(done);
540 input.ready->setValue(rtlb.bitAnd(doneWires,
"allDone"));
542 rewriter.replaceOp(op, io.getOutputChannels());
552 matchAndRewrite(JoinOp op, OpAdaptor operands,
553 ConversionPatternRewriter &rewriter)
const override {
555 UnwrappedIO io = unwrapIO(op, operands.getOperands(), rewriter, bb);
556 RTLBuilder rtlb(op.getLoc(), rewriter);
557 auto &output = io.outputs[0];
559 Value allValid = rtlb.bitAnd(io.getInputValids());
560 output.valid->setValue(allValid);
562 auto validAndReady = rtlb.bitAnd({output.ready, allValid});
563 for (
auto &input : io.inputs)
564 input.ready->setValue(validAndReady);
566 rewriter.replaceOp(op, io.outputs[0].channel);
576 matchAndRewrite(SelectOp op, OpAdaptor operands,
577 ConversionPatternRewriter &rewriter)
const override {
579 UnwrappedIO io = unwrapIO(op, operands.getOperands(), rewriter, bb);
580 RTLBuilder rtlb(op.getLoc(), rewriter);
583 auto select = io.inputs[0];
584 io.inputs.erase(io.inputs.begin());
585 buildMuxLogic(rtlb, io, select);
587 rewriter.replaceOp(op, io.outputs[0].channel);
594 void buildMuxLogic(RTLBuilder &rtlb, UnwrappedIO &unwrapped,
595 InputHandshake &select)
const {
598 size_t numInputs = unwrapped.inputs.size();
599 size_t selectWidth = llvm::Log2_64_Ceil(numInputs);
600 Value truncatedSelect =
601 select.data.getType().getIntOrFloatBitWidth() > selectWidth
602 ? rtlb.truncate(select.data, selectWidth)
606 auto selectZext = rtlb.zext(truncatedSelect, numInputs);
607 auto select1h = rtlb.shl(rtlb.constant(numInputs, 1), selectZext);
608 auto &res = unwrapped.outputs[0];
611 auto selectedInputValid =
612 rtlb.mux(truncatedSelect, unwrapped.getInputValids());
614 auto selAndInputValid = rtlb.bitAnd({selectedInputValid, select.valid});
615 res.valid->setValue(selAndInputValid);
616 auto resValidAndReady = rtlb.bitAnd({selAndInputValid, res.ready});
619 select.ready->setValue(resValidAndReady);
622 for (
auto [inIdx, in] : llvm::enumerate(unwrapped.inputs)) {
624 auto isSelected = rtlb.bit(select1h, inIdx);
628 auto activeAndResultValidAndReady =
629 rtlb.bitAnd({isSelected, resValidAndReady});
630 in.ready->setValue(activeAndResultValidAndReady);
637 using OpConversionPattern::OpConversionPattern;
639 matchAndRewrite(BranchOp op, OpAdaptor operands,
640 ConversionPatternRewriter &rewriter)
const override {
642 UnwrappedIO io = unwrapIO(op, operands.getOperands(), rewriter, bb);
643 RTLBuilder rtlb(op.getLoc(), rewriter);
644 auto cond = io.inputs[0];
645 auto trueRes = io.outputs[0];
646 auto falseRes = io.outputs[1];
649 trueRes.valid->setValue(rtlb.bitAnd({cond.data, cond.valid}));
650 falseRes.valid->setValue(rtlb.bitAnd({rtlb.bitNot(cond.data), cond.valid}));
653 Value selectedResultReady =
654 rtlb.mux(cond.data, {falseRes.ready, trueRes.ready});
655 Value condReady = rtlb.bitAnd({selectedResultReady, cond.valid});
656 cond.ready->setValue(condReady);
658 rewriter.replaceOp(op,
659 SmallVector<Value>{trueRes.channel, falseRes.channel});
666 using OpConversionPattern::OpConversionPattern;
668 matchAndRewrite(MergeOp op, OpAdaptor operands,
669 ConversionPatternRewriter &rewriter)
const override {
671 UnwrappedIO io = unwrapIO(op, operands.getOperands(), rewriter, bb);
672 auto output = io.outputs[0];
673 RTLBuilder rtlb(op.getLoc(), rewriter);
676 Value hasWin = rtlb.bitOr(io.getInputValids());
682 Value winWasFirst = io.inputs[0].valid;
683 Value winWasSecond = rtlb.bitNot(winWasFirst);
684 Value winIndex = winWasSecond;
686 output.valid->setValue(hasWin);
687 output.data->setValue(winIndex);
691 Value outValidAndReady = rtlb.bitAnd({hasWin, output.ready});
695 io.inputs[0].ready->setValue(rtlb.bitAnd({outValidAndReady, winWasFirst}));
696 io.inputs[1].ready->setValue(rtlb.bitAnd({outValidAndReady, winWasSecond}));
698 rewriter.replaceOp(op, output.channel);
708 using OpConversionPattern::OpConversionPattern;
711 matchAndRewrite(ToESIOp op, OpAdaptor operands,
712 ConversionPatternRewriter &rewriter)
const override {
713 rewriter.replaceOp(op, operands.getOperands());
722 using OpConversionPattern::OpConversionPattern;
725 matchAndRewrite(FromESIOp op, OpAdaptor operands,
726 ConversionPatternRewriter &rewriter)
const override {
727 rewriter.replaceOp(op, operands.getOperands());
737 matchAndRewrite(SinkOp op, OpAdaptor operands,
738 ConversionPatternRewriter &rewriter)
const override {
740 UnwrappedIO io = unwrapIO(op, operands.getOperands(), rewriter, bb);
741 io.inputs[0].ready->setValue(
742 RTLBuilder(op.getLoc(), rewriter).constant(1, 1));
743 rewriter.eraseOp(op);
752 matchAndRewrite(SourceOp op, OpAdaptor operands,
753 ConversionPatternRewriter &rewriter)
const override {
755 UnwrappedIO io = unwrapIO(op, operands.getOperands(), rewriter, bb);
756 RTLBuilder rtlb(op.getLoc(), rewriter);
757 io.outputs[0].valid->setValue(rtlb.constant(1, 1));
758 rewriter.replaceOp(op, io.outputs[0].channel);
767 matchAndRewrite(PackOp op, OpAdaptor operands,
768 ConversionPatternRewriter &rewriter)
const override {
770 UnwrappedIO io = unwrapIO(op, llvm::SmallVector<Value>{operands.getToken()},
772 RTLBuilder rtlb(op.getLoc(), rewriter);
773 auto &input = io.inputs[0];
774 auto &output = io.outputs[0];
775 output.data->setValue(operands.getInput());
777 rewriter.replaceOp(op, output.channel);
786 matchAndRewrite(UnpackOp op, OpAdaptor operands,
787 ConversionPatternRewriter &rewriter)
const override {
789 UnwrappedIO io = unwrapIO(
790 op.getLoc(), llvm::SmallVector<Value>{operands.getInput()},
792 llvm::SmallVector<Type>{op.getToken().getType()}, rewriter, bb);
793 RTLBuilder rtlb(op.getLoc(), rewriter);
794 auto &input = io.inputs[0];
795 auto &output = io.outputs[0];
797 llvm::SmallVector<Value> unpackedValues;
798 unpackedValues.push_back(input.data);
801 llvm::SmallVector<Value> outputs;
802 outputs.push_back(output.channel);
803 outputs.append(unpackedValues.begin(), unpackedValues.end());
804 rewriter.replaceOp(op, outputs);
814 matchAndRewrite(BufferOp op, OpAdaptor operands,
815 ConversionPatternRewriter &rewriter)
const override {
816 auto crRes = getClockAndReset(op);
819 auto [clock, reset] = *crRes;
823 Type channelType = operands.getInput().getType();
824 rewriter.replaceOpWithNewOp<esi::ChannelBufferOp>(
825 op, channelType, clock, reset, operands.getInput(), op.getSizeAttr(),
833 static bool isDCType(Type type) {
return isa<TokenType, ValueType>(type); }
838 if (
auto funcOp = dyn_cast<HWModuleLike>(op)) {
839 return llvm::none_of(funcOp.getPortTypes(),
isDCType) &&
840 llvm::none_of(funcOp.getBodyBlock()->getArgumentTypes(),
isDCType);
843 bool operandsOK = llvm::none_of(op->getOperandTypes(),
isDCType);
844 bool resultsOK = llvm::none_of(op->getResultTypes(),
isDCType);
845 return operandsOK && resultsOK;
853 class DCToHWPass :
public circt::impl::DCToHWBase<DCToHWPass> {
855 void runOnOperation()
override {
856 Operation *parent = getOperation();
860 auto walkRes = parent->walk([&](Operation *op) {
861 for (
auto res : op->getResults()) {
862 if (isa<dc::TokenType, dc::ValueType>(res.getType())) {
863 if (res.use_empty()) {
864 op->emitOpError() <<
"DCToHW: value " << res <<
" is unused.";
865 return WalkResult::interrupt();
867 if (!res.hasOneUse()) {
869 <<
"DCToHW: value " << res <<
" has multiple uses.";
870 return WalkResult::interrupt();
874 return WalkResult::advance();
877 if (walkRes.wasInterrupted()) {
878 parent->emitOpError()
879 <<
"DCToHW: failed to verify that all values "
880 "are used exactly once. Remember to run the "
881 "fork/sink materialization pass before HW lowering.";
886 ESITypeConverter typeConverter;
887 ConversionTarget target(getContext());
888 target.markUnknownOpDynamicallyLegal(
isLegalOp);
892 target.addIllegalDialect<dc::DCDialect>();
894 RewritePatternSet
patterns(parent->getContext());
897 ForkConversionPattern, JoinConversionPattern, SelectConversionPattern,
898 BranchConversionPattern, PackConversionPattern, UnpackConversionPattern,
899 BufferConversionPattern, SourceConversionPattern, SinkConversionPattern,
901 FromESIConversionPattern>(typeConverter, parent->getContext());
903 if (failed(applyPartialConversion(parent, target, std::move(
patterns))))
910 return std::make_unique<DCToHWPass>();
assert(baseType &&"element must be base type")
return wrap(CMemoryType::get(unwrap(ctx), baseType, numElements))
static std::string valueName(Operation *scopeOp, Value v)
Convenience function for getting the SSA name of v under the scope of operation scopeOp.
static SmallVector< T > concat(const SmallVectorImpl< T > &a, const SmallVectorImpl< T > &b)
Returns a new vector containing the concatenation of vectors a and b.
static Type toHWType(Type t)
Converts any type 't' into a hw-compatible type.
static bool isDCType(Type type)
static Type tupleToStruct(TupleType tuple)
static bool isLegalOp(Operation *op)
Returns true if the given op is considered as legal - i.e.
std::function< std::string(Operation *)> NameUniquer
static Type toESIHWType(Type t)
static Value extractBits(OpBuilder &builder, Location loc, Value value, unsigned startBit, unsigned bitWidth)
static EvaluatorValuePtr unwrap(OMEvaluatorValue c)
Instantiate one of these and use it to build typed backedges.
Backedge get(mlir::Type resultType, mlir::LocationAttr optionalLoc={})
Create a typed backedge.
Backedge is a wrapper class around a Value.
void setValue(mlir::Value)
Channels are the basic communication primitives.
const Type * getInner() const
def create(cls, result_type, reset=None, reset_value=None, name=None, sym_name=None, **kwargs)
def connect(destination, source)
Direction get(bool isOutput)
Returns an output direction if isOutput is true, otherwise returns an input direction.
Value createOrFoldSExt(Location loc, Value value, Type destTy, OpBuilder &builder)
Create a sign extension operation from a value of integer type to an equal or larger integer type.
mlir::Type innerType(mlir::Type type)
The InstanceGraph op interface, see InstanceGraphInterface.td for more details.
std::unique_ptr< mlir::Pass > createDCToHWPass()
def reg(value, clock, reset=None, reset_value=None, name=None, sym_name=None)
Generic pattern which replaces an operation by one of the same operation name, but with converted att...