13#include "mlir/Dialect/Arith/IR/Arith.h"
14#include "mlir/Dialect/Func/IR/FuncOps.h"
15#include "mlir/IR/DialectImplementation.h"
16#include "mlir/IR/Matchers.h"
17#include "mlir/IR/Operation.h"
18#include "mlir/IR/RegionGraphTraits.h"
19#include "mlir/IR/SymbolTable.h"
20#include "mlir/IR/TypeUtilities.h"
21#include "mlir/Interfaces/DataLayoutInterfaces.h"
22#include "mlir/Interfaces/FoldInterfaces.h"
23#include "mlir/Transforms/InliningUtils.h"
24#include "llvm/ADT/SCCIterator.h"
25#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/DenseMap.h"
28#include "llvm/ADT/TypeSwitch.h"
29#include "llvm/Support/CheckedArithmetic.h"
30#include "llvm/Support/MathExtras.h"
31#include "llvm/Support/TypeSize.h"
40#include "aie/Dialect/AIEX/IR/AIEXDialect.cpp.inc"
42#include "aie/Dialect/AIEX/IR/AIEXEnums.cpp.inc"
44#include "aie/Dialect/AIEX/IR/AIEXInterfaces.cpp.inc"
46#define GET_TYPEDEF_CLASSES
47#include "aie/Dialect/AIEX/IR/AIEXTypes.cpp.inc"
49#define GET_ATTRDEF_CLASSES
50#include "aie/Dialect/AIEX/IR/AIEXAttrDefs.cpp.inc"
55void AIEXDialect::initialize() {
58#include "aie/Dialect/AIEX/IR/AIEX.cpp.inc"
61#define GET_TYPEDEF_LIST
62#include "aie/Dialect/AIEX/IR/AIEXTypes.cpp.inc"
65#define GET_ATTRDEF_LIST
66#include "aie/Dialect/AIEX/IR/AIEXAttrDefs.cpp.inc"
76#include "aie/Dialect/AIEX/IR/AIEX.cpp.inc"
113 mlir::BaseMemRefType referencedBufType,
114 llvm::SmallVector<int64_t, 4> inputSizes,
115 llvm::SmallVector<int64_t, 4> inputStrides,
116 llvm::SmallVector<int64_t, 4> &sizes,
117 llvm::SmallVector<int64_t, 4> &strides) {
118 assert(inputSizes.size() == inputStrides.size());
119 assert(sizes.size() == 4);
120 assert(strides.size() == 4);
122 DataLayout dataLayout = DataLayout::closest(op);
124 dataLayout.getTypeSizeInBits(referencedBufType.getElementType());
128 llvm::fill(sizes, 0);
129 llvm::fill(strides, 0);
131 if (inputSizes[0] == 0) {
138 int64_t inS[4] = {inputSizes[0], inputSizes[1], inputSizes[2], inputSizes[3]};
139 int64_t inT[4] = {inputStrides[0], inputStrides[1], inputStrides[2],
141 int64_t outS[4], outT[4];
144 for (
int i = 0; i < 4; i++) {
146 strides[i] = outT[i];
151 mlir::Operation *forOp,
152 llvm::function_ref<mlir::InFlightDiagnostic()> emitError,
153 mlir::BaseMemRefType referencedBufType,
int tileCol,
int tileRow,
154 llvm::SmallVector<int64_t, 4> inputSizes,
155 llvm::SmallVector<int64_t, 4> inputStrides,
156 llvm::SmallVector<int64_t, 4> hardwareSizes,
157 llvm::SmallVector<int64_t, 4> hardwareStrides,
158 bool skipTransformationChecks) {
160 auto addressGranularity = targetModel.getAddressGenGranularity();
161 DataLayout dataLayout = DataLayout::closest(forOp);
163 dataLayout.getTypeSizeInBits(referencedBufType.getElementType());
166 if (!targetModel.isCoreTile(tileCol, tileRow) &&
167 !targetModel.isMemTile(tileCol, tileRow) &&
168 !targetModel.isShimNOCTile(tileCol, tileRow))
169 return emitError() <<
"Unsupported tile type at (" << tileCol <<
", "
170 << tileRow <<
") Must be ShimNOC, Mem or Core.";
172 uint32_t wrap_bits = targetModel.getDmaBdWrapBits(tileCol, tileRow);
173 uint32_t step_bits = targetModel.getDmaBdStepBits(tileCol, tileRow);
175 for (
int i = 0; i < 4; i++) {
176 if (inputSizes[i] <= 0) {
177 return emitError() <<
"Size " << i <<
" must be a positive integer.";
182 std::stringstream msg;
183 msg <<
"Transfer sizes must be multiples of " << (addressGranularity / 8)
184 <<
" bytes. " << inputSizes[0] <<
" elements at " << (elemWidth / 8)
185 <<
" bytes each equal " << (inputSizes[0] * elemWidth / 8)
186 <<
" bytes, which is not divisible by " << (addressGranularity / 8)
188 return emitError() << msg.str();
191 for (
int i = 0; i < 3; i++) {
192 if (inputSizes[i] > 1 && inputStrides[i] < 1) {
196 return emitError() <<
"Stride " << i <<
" must be a positive integer.";
201 if (inputSizes[3] > 1 && inputStrides[3] < 0) {
202 return emitError() <<
"Stride 3 must be a non-negative integer.";
205 for (
int i = 0; i < 4; i++) {
208 if (i == 0 && inputStrides[i] == 1)
211 addressGranularity)) {
212 std::stringstream msg;
213 msg <<
"Stride " << i <<
" is " << inputStrides[i] <<
" elements * "
214 << (elemWidth / 8) <<
" bytes = " << (inputStrides[i] * elemWidth / 8)
215 <<
" bytes, which is not divisible by " << (addressGranularity / 8)
217 return emitError() << msg.str();
222 if (elemWidth != addressGranularity && inputSizes[0] > 1 &&
223 inputStrides[0] != 1)
224 return emitError() <<
"Stride 0 is " << inputStrides[0]
225 <<
" elements, but must be 1 for " << (elemWidth / 8)
226 <<
"-byte elements: the DMA moves whole "
227 << (addressGranularity / 8) <<
"-byte words.";
229 auto outOfRange = [&](
const char *what,
int dim, int64_t written,
230 int64_t encoded, int64_t lo, int64_t hi) {
231 return emitError() << what <<
" " << dim <<
" is " << written
232 <<
" (encoded as " << encoded <<
"), which exceeds the ["
233 << lo <<
":" << hi <<
"] range.";
236 int64_t maxWrap = (1 << wrap_bits) - 1;
237 int64_t maxStep = 1 << step_bits;
239 for (
int dim : {0, 1}) {
240 if (!skipTransformationChecks && hardwareSizes[dim] > maxWrap)
241 return outOfRange(
"Size", dim, inputSizes[dim], hardwareSizes[dim], 0,
246 bool pureRepeat = inputStrides[3] == 0;
248 pureRepeat ? targetModel.getMaxRepeatCount() + 1
249 : 1LL << targetModel.getDmaBdIterBits(tileCol, tileRow);
250 if (inputSizes[3] > maxCount)
251 return emitError() << (pureRepeat ?
"repeat count " :
"iteration count ")
252 << inputSizes[3] <<
" exceeds the [1:" << maxCount
254 for (
int dim : {0, 1, 2}) {
255 if (hardwareStrides[dim] > maxStep - 1)
256 return outOfRange(
"Stride", dim, inputStrides[dim], hardwareStrides[dim],
261 if (hardwareStrides[3] > maxStep - 1 && hardwareSizes[3] > 0)
262 return outOfRange(
"Stride", 3, inputStrides[3], hardwareStrides[3], 1,
275LogicalResult AIEX::CertWrite32DOp::verify() {
276 if (getAddressIsReg() && getAddress() > 23)
277 return emitOpError(
"has address_is_reg, so address ")
278 << getAddress() <<
" must be a register index in [0, 23]";
279 if (getValueIsReg() && getValue() > 23)
280 return emitOpError(
"has value_is_reg, so value ")
281 << getValue() <<
" must be a register index in [0, 23]";
285LogicalResult AIEX::CertApplyOffset57Op::verify() {
286 if (getNumEntries() == 0)
287 return emitOpError(
"num_entries must be at least 1");
291LogicalResult AIEX::UseTokenOp::verify() {
292 auto *parentOp = (*this)->getParentOp();
293 if (isa<func::FuncOp>(parentOp) || isa<AIE::CoreOp>(parentOp) ||
294 isa<AIE::MemOp>(parentOp) || isa<AIE::ShimDMAOp>(parentOp))
303LogicalResult AIEX::MulticastOp::verify() {
304 Region &body = getPorts();
305 assert(getOperation()->getNumRegions());
306 assert(!body.empty());
307 for (
auto &ops : body.front())
308 if (!isa<MultiDestOp, AIE::EndOp>(ops))
309 return ops.emitOpError(
"cannot be contained in a Multicast op");
318LogicalResult AIEX::BroadcastPacketOp::verify() {
319 Region &body = getPorts();
320 assert(getOperation()->getNumRegions());
321 assert(!body.empty());
322 for (
auto &ops : body.front())
323 if (!isa<BPIDOp, AIE::EndOp>(ops))
324 return ops.emitOpError(
"cannot be contained in a BroadcastPacket op");
335uint32_t AIEX::NpuDmaMemcpyNdOp::getAxcacheOrDefault() {
339int64_t AIEX::NpuDmaMemcpyNdOp::getOffsetInBytes() {
340 llvm::SmallVector<int64_t, 4> offsets =
341 llvm::map_to_vector(llvm::reverse(getMixedOffsets()), [](OpFoldResult s) {
342 return getConstantIntValue(s).value();
344 auto strides = llvm::to_vector<4>(llvm::reverse(getMixedStrides()));
346 size_t R = offsets.size();
347 size_t el_bit_width = getElementTypeBitwidth();
348 assert(el_bit_width % 8 == 0 &&
349 "Expected Memref element bitwidth to be multiple of 8.");
350 size_t S = el_bit_width / 8;
355 for (
size_t i = 0; i < R; i++) {
358 auto strideConst = getConstantIntValue(strides[i]);
359 assert(strideConst &&
360 "verifier requires a stride paired with a non-zero offset to be "
362 offset += offsets[i] * (*strideConst) * S;
371 llvm::ArrayRef<int64_t> strides) {
372 return sizes[1] == 1 && sizes[2] == 1 && strides[0] == 1 && strides[1] == 0 &&
383 llvm::ArrayRef<int64_t> strides) {
386 if (sizes[1] > 1 && strides[1] != sizes[0])
388 if (sizes[2] > 1 && strides[2] != sizes[0] * sizes[1])
394 int row, llvm::ArrayRef<int64_t> sizes,
395 llvm::ArrayRef<int64_t> strides) {
405bool AIEX::NpuDmaMemcpyNdOp::isLinearTransferWithoutTransformation() {
406 llvm::SmallVector<int64_t, 4> inputSizes =
407 llvm::map_to_vector(llvm::reverse(getMixedSizes()), [](OpFoldResult s) {
408 return getConstantIntValue(s).value();
410 llvm::SmallVector<int64_t, 4> inputStrides =
411 llvm::map_to_vector(llvm::reverse(getMixedStrides()), [](OpFoldResult s) {
412 return getConstantIntValue(s).value();
440struct LinearizeContiguousTransfer
441 :
public mlir::OpRewritePattern<AIEX::NpuDmaMemcpyNdOp> {
442 using OpRewritePattern::OpRewritePattern;
445 matchAndRewrite(AIEX::NpuDmaMemcpyNdOp op,
446 mlir::PatternRewriter &rewriter)
const override {
448 if (!llvm::all_of(op.getMixedSizes(), [](mlir::OpFoldResult s) {
449 return mlir::getConstantIntValue(s).has_value();
451 return mlir::failure();
452 if (!llvm::all_of(op.getMixedStrides(), [](mlir::OpFoldResult s) {
453 return mlir::getConstantIntValue(s).has_value();
455 return mlir::failure();
456 if (!llvm::all_of(op.getMixedOffsets(), [](mlir::OpFoldResult s) {
457 return mlir::getConstantIntValue(s).has_value();
459 return mlir::failure();
462 if (op.isLinearTransferWithoutTransformation())
463 return mlir::failure();
467 llvm::SmallVector<int64_t, 4> sizes = llvm::map_to_vector(
468 llvm::reverse(op.getMixedSizes()), [](mlir::OpFoldResult s) {
469 return mlir::getConstantIntValue(s).value();
471 llvm::SmallVector<int64_t, 4> strides = llvm::map_to_vector(
472 llvm::reverse(op.getMixedStrides()), [](mlir::OpFoldResult s) {
473 return mlir::getConstantIntValue(s).value();
475 llvm::SmallVector<int64_t, 4> offsets = llvm::map_to_vector(
476 llvm::reverse(op.getMixedOffsets()), [](mlir::OpFoldResult s) {
477 return mlir::getConstantIntValue(s).value();
482 return mlir::failure();
486 int64_t N = sizes[0] * sizes[1] * sizes[2];
487 llvm::SmallVector<int64_t, 4> newSizesOuter = {sizes[3], 1, 1, N};
488 llvm::SmallVector<int64_t, 4> newStridesOuter = {strides[3], 0, 0, 1};
495 int64_t linearOffset = offsets[0] * strides[0] + offsets[1] * strides[1] +
496 offsets[2] * strides[2];
497 llvm::SmallVector<int64_t, 4> newOffsetsOuter = {offsets[3], 0, 0,
500 rewriter.replaceOpWithNewOp<AIEX::NpuDmaMemcpyNdOp>(
501 op, op, newOffsetsOuter, newSizesOuter, newStridesOuter);
502 return mlir::success();
507void AIEX::NpuDmaMemcpyNdOp::getCanonicalizationPatterns(
508 mlir::RewritePatternSet &results, mlir::MLIRContext *context) {
509 results.add<LinearizeContiguousTransfer>(context);
515static std::optional<std::string>
517 uint32_t requestedBurstLength) {
518 if (requestedBurstLength != 0) {
520 auto pair = llvm::find_if(bel, [=](
const std::pair<uint32_t, uint32_t> &p) {
521 return p.second == requestedBurstLength;
524 if (pair == bel.end()) {
525 std::string errorMessage =
526 "Requested burst length is not supported by the target. "
527 "Supported burst lengths:";
530 std::accumulate(bel.begin(), bel.end(), errorMessage,
531 [](
const std::string &a,
auto b) {
532 return a +
" " + std::to_string(b.second);
546LogicalResult AIEX::NpuDmaMemcpyNdOp::verifyDynamicSizesStrides(
548 AIE::ShimDMAAllocationOp allocOp) {
552 "runtime sizes/strides require a shim_dma_allocation to resolve the "
553 "tile; none found.");
554 AIE::TileOp tile = allocOp.getTileOp();
558 if (!targetModel.
isShimNOCTile(tile.getCol(), tile.getRow()))
560 "runtime sizes/strides are only supported for shim NOC tile DMAs.");
563 if (getD0ZeroBefore() || getD1ZeroBefore() || getD2ZeroBefore() ||
564 getD0ZeroAfter() || getD1ZeroAfter() || getD2ZeroAfter())
566 "zero padding is not supported with runtime sizes/strides.");
573 auto sizes = getMixedSizes();
574 auto strides = getMixedStrides();
592 DataLayout dataLayout = DataLayout::closest(getOperation());
593 uint64_t elemWidth = dataLayout.getTypeSizeInBits(buffer.getElementType());
595 llvm::SmallVector<mlir::OpFoldResult, 4> sizesRev(llvm::reverse(sizes));
596 auto checkSize = [&](mlir::OpFoldResult ofr, int64_t hwVal, int64_t hi,
597 llvm::StringRef what) -> LogicalResult {
598 if (!getConstantIntValue(ofr))
600 if (hwVal < 0 || hwVal > hi)
601 return emitOpError(what) <<
" hardware value " << hwVal
602 <<
" exceeds hardware range [0:" << hi <<
"].";
605 auto hwSize = [&](mlir::OpFoldResult ofr) {
606 return getConstantIntValue(ofr).value_or(0);
608 int64_t d0Hw = (int64_t)(hwSize(sizesRev[0]) * elemWidth / gran);
609 int64_t d1Hw = hwSize(sizesRev[1]);
610 int64_t iterRaw = hwSize(sizesRev[3]);
611 int64_t iterHw = iterRaw > 1 ? iterRaw - 1 : 0;
612 std::optional<int64_t> outerStride = getConstantIntValue(strides.front());
613 bool pureRepeat = outerStride && *outerStride == 0;
614 AIE::AIETileType tileType =
615 targetModel.
getTileType(tile.getCol(), tile.getRow());
619 if (failed(checkSize(sizesRev[0], d0Hw, wrapMax,
"d0 size")) ||
620 failed(checkSize(sizesRev[1], d1Hw, wrapMax,
"d1 size")) ||
621 failed(checkSize(sizesRev[3], iterHw, maxCount - 1,
622 pureRepeat ?
"repeat count" :
"iteration size")))
628 llvm::SmallVector<mlir::OpFoldResult, 4> stridesRev(llvm::reverse(strides));
635 auto errorMessage = checkBurstLength(targetModel, getBurstLength());
636 if (errorMessage.has_value())
637 return emitOpError(errorMessage.value());
642LogicalResult AIEX::NpuDmaMemcpyNdOp::verify() {
643 BaseMemRefType buffer = getMemref().getType();
647 if (getOffsetParameterAttr() || getOffsetStateTableIdxAttr()) {
648 uint64_t elemBitWidth = buffer.getElementTypeBitWidth();
649 if (elemBitWidth == 0 || (elemBitWidth % 8) != 0)
650 return emitOpError(
"offset_parameter requires a whole-byte element type");
653 if (getElementTypeBitwidth() > addressGranularity) {
654 return emitOpError(
"Maximum element bit width allowed is ")
655 << addressGranularity <<
"bits. ";
657 if (buffer.hasStaticShape() &&
658 (buffer.getNumElements() * getElementTypeBitwidth()) <
659 addressGranularity) {
660 return emitOpError(
"Minimum data transfer size required is ")
661 << addressGranularity <<
"bits. ";
663 bool allStridesConstant = llvm::all_of(getMixedStrides(), [](OpFoldResult s) {
664 return getConstantIntValue(s).has_value();
666 bool allSizesConstant = llvm::all_of(getMixedSizes(), [](OpFoldResult s) {
667 return getConstantIntValue(s).has_value();
669 bool allOffsetsConstant = llvm::all_of(getMixedOffsets(), [](OpFoldResult s) {
670 return getConstantIntValue(s).has_value();
673 bool hasLengthParameter =
674 getLengthParameterAttr() || getLengthStateTableIdxAttr();
677 if (hasLengthParameter &&
678 (!allStridesConstant || !allSizesConstant || !allOffsetsConstant))
680 "length_parameter requires constant offsets, sizes and strides");
683 if (!allStridesConstant || !allSizesConstant || !allOffsetsConstant)
686 llvm::SmallVector<int64_t, 4> inputSizes =
687 llvm::map_to_vector(llvm::reverse(getMixedSizes()), [](OpFoldResult s) {
688 return getConstantIntValue(s).value();
690 llvm::SmallVector<int64_t, 4> inputStrides =
691 llvm::map_to_vector(llvm::reverse(getMixedStrides()), [](OpFoldResult s) {
692 return getConstantIntValue(s).value();
694 int64_t offset = getOffsetInBytes();
696 auto errorMessage = checkBurstLength(targetModel, getBurstLength());
697 if (errorMessage.has_value()) {
698 return emitOpError(errorMessage.value());
701 if (hasLengthParameter) {
703 *
this, getLengthUnit(), buffer,
704 inputSizes[0] * inputSizes[1] * inputSizes[2],
706 llvm::ArrayRef<int64_t>(inputSizes).take_front(3))))
714 if (offset % 4 != 0) {
715 return emitOpError(
"Offset must be 4-byte-aligned.");
719 if (
auto packetInfo = getPacket()) {
722 if (packetInfo->getPktType() > 7)
723 return emitOpError(
"Packet type field can only hold 3 bits.");
724 if (packetInfo->assignedId() > 31)
725 return emitOpError(
"Packet ID field can only hold 5 bits.");
731static LogicalResult verifyTransferCoversWholeObjects(
732 Operation *transfer, SymbolRefAttr alloc, std::optional<int64_t> moved,
733 AIE::ObjectFifoCreateOp fifo, SymbolTableCollection &symbols);
738AIEX::NpuDmaMemcpyNdOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
739 BaseMemRefType buffer = getMemref().getType();
741 bool allStridesConstant = llvm::all_of(getMixedStrides(), [](OpFoldResult s) {
742 return getConstantIntValue(s).has_value();
744 bool allSizesConstant = llvm::all_of(getMixedSizes(), [](OpFoldResult s) {
745 return getConstantIntValue(s).has_value();
747 bool allOffsetsConstant = llvm::all_of(getMixedOffsets(), [](OpFoldResult s) {
748 return getConstantIntValue(s).has_value();
751 !allStridesConstant || !allSizesConstant || !allOffsetsConstant;
753 bool hasLengthParameter =
754 getLengthParameterAttr() || getLengthStateTableIdxAttr();
755 bool isAIE1 = targetModel.
getTargetArch() == AIE::AIEArch::AIE1;
756 AIE::DeviceOp dev = getOperation()->getParentOfType<AIE::DeviceOp>();
757 if (!dev && isAIE1 && !isDynamic && !hasLengthParameter)
760 if (
auto fifo = symbolTable.lookupSymbolIn<AIE::ObjectFifoCreateOp>(
761 dev, getMetadata().getRootReference());
763 failed(verifyTransferCoversWholeObjects(
764 *
this, getMetadata(), getStaticBytesStreamed(), fifo, symbolTable)))
766 auto allocOp = symbolTable.lookupSymbolIn<AIE::ShimDMAAllocationOp>(
767 dev, getMetadata().getRootReference());
768 if (AIE::TileOp tile = allocOp ? allocOp.getTileOp() :
nullptr) {
769 uint32_t numBds = targetModel.
getNumBDs(tile.getCol(), tile.getRow());
770 if (getId() >= numBds)
771 return emitOpError(
"programs buffer descriptor ID ")
772 << getId() <<
", but tile (" << tile.getCol() <<
", "
773 << tile.getRow() <<
") has " << numBds <<
" buffer descriptors";
776 return verifyDynamicSizesStrides(targetModel, buffer, allocOp);
782 auto tile = allocOp.getTileOp();
785 if (isAIE1 && hasLengthParameter)
786 return emitOpError(
"length_parameter is only supported on AIE2 and "
790 int col = tile.getCol();
791 int row = tile.getRow();
792 if (hasLengthParameter &&
796 llvm::SmallVector<int64_t, 4> inputSizes =
797 llvm::map_to_vector(llvm::reverse(getMixedSizes()), [](OpFoldResult s) {
798 return getConstantIntValue(s).value();
800 llvm::SmallVector<int64_t, 4> inputStrides =
801 llvm::map_to_vector(llvm::reverse(getMixedStrides()), [](OpFoldResult s) {
802 return getConstantIntValue(s).value();
804 llvm::SmallVector<int64_t, 4> hardwareSizes(4);
805 llvm::SmallVector<int64_t, 4> hardwareStrides(4);
807 inputStrides, hardwareSizes, hardwareStrides);
819 targetModel, col, row, inputSizes, inputStrides);
826 llvm::SmallVector<int64_t, 4> inputOffsets =
827 llvm::map_to_vector(llvm::reverse(getMixedOffsets()), [](OpFoldResult s) {
828 return getConstantIntValue(s).value();
834 if (!isAIE1 && !hasLengthParameter &&
836 row, inputOffsets, inputSizes,
840 *
this, [&] {
return emitOpError(); }, buffer, col, row,
841 inputSizes, inputStrides, hardwareSizes, hardwareStrides,
842 skipTransformationChecks))) {
853LogicalResult AIEX::NpuDmaWaitOp::verify() {
854 AIE::DeviceOp dev = (*this)->getParentOfType<AIE::DeviceOp>();
857 if (dev && !dev.lookupSymbol(getSymbol()))
858 return emitOpError(
"couldn't find symbol in parent device");
866LogicalResult AIEX::NpuPushQueueOp::verify() {
868 auto numBds = targetModel.
getNumBDs(getColumn(), getRow());
873 bdId && *bdId >= numBds)
874 return emitOpError(
"BD ID exceeds the maximum ID.");
878 return emitOpError(
"BD ID ") << *bdId <<
" is not accessible from channel ("
879 << AIE::stringifyDMAChannelDir(getDirection())
880 <<
", " << getChannel() <<
")";
882 if (std::optional<uint64_t> repeatCount =
884 repeatCount && *repeatCount > maxRepeat)
885 return emitOpError(
"Repeat count exceeds the [0:")
886 << maxRepeat <<
"] range.";
887 if (getRepeatStateTableIdxAttr()) {
889 if (arch != AIE::AIEArch::AIE2 && arch != AIE::AIEArch::AIE2p)
890 return emitOpError(
"repeat_parameter is only supported on AIE2 and "
893 !targetModel.
isMemTile(getColumn(), getRow()))
894 return emitOpError(
"repeat_parameter is only supported on shim NOC "
895 "and mem tiles, got tile (")
896 << getColumn() <<
", " << getRow() <<
")";
898 return emitOpError(
"takes its repeat count from the scratchpad, so its "
899 "repeat_count operand must be 0");
902 return emitOpError(
"takes its repeat count from the scratchpad, so its "
903 "bd_id must be a constant");
905 return emitOpError(
"takes its repeat count from the scratchpad, whose "
906 "update clears the low two bits of the queue word, "
907 "so its bd_id must be a multiple of 4, got ")
917LogicalResult AIEX::NpuSyncOp::verify() {
918 for (
auto [name, count] : {std::pair{
"column_num", getColumnNum()},
919 std::pair{
"row_num", getRowNum()}})
920 if (std::optional<int64_t> value = getConstantIntValue(count);
922 return emitOpError() << name <<
" must cover at least one, got "
927LogicalResult AIEX::NpuWriteBdOp::verify() {
929 auto numBds = targetModel.
getNumBDs(getColumn(), getRow());
931 (getD0Size() >= 1) && (getD1Size() == 1) && (getIterationSize() == 0);
932 if (getBdId() >= numBds)
933 return emitOpError(
"BD ID exceeds the maximum ID.");
934 if (getPacketId() > 31)
935 return emitOpError(
"Packet ID exceeds the maximum supported by 5 bits.");
936 if (getPacketType() > 7)
937 return emitOpError(
"Packet Type exceeds the maximum supported by 3 bits.");
938 int64_t oooId = getOutOfOrderId();
941 return emitOpError(
"out_of_order_id must be in [0, ")
943 if (oooId != 0 && getEnablePacket() == 0)
944 return emitOpError(
"out_of_order_id requires a packet-enabled BD");
949 AIE::AIETileType tileType = targetModel.
getTileType(getColumn(), getRow());
953 uint32_t maxWrap = wrapBits > 0 ? (1u << wrapBits) - 1 : 0;
954 uint32_t maxStep = stepBits > 0 ? (1u << stepBits) - 1 : 0;
955 uint32_t maxIter = iterBits > 0 ? (1u << iterBits) - 1 : 0;
957 if (!isLinearTransfer && getD0Size() > maxWrap)
958 return emitOpError() <<
"D0 Size exceeds the [0:" << maxWrap <<
"] range.";
959 if (getD0Stride() > maxStep)
960 return emitOpError() <<
"D0 Stride exceeds the [0:" << maxStep
962 if (getD1Size() > maxWrap)
963 return emitOpError() <<
"D1 Size exceeds the [0:" << maxWrap <<
"] range.";
964 if (getD1Stride() > maxStep)
965 return emitOpError() <<
"D1 Stride exceeds the [0:" << maxStep
967 if (getD2Stride() > maxStep)
968 return emitOpError() <<
"D2 Stride exceeds the [0:" << maxStep
970 if (getIterationSize() > maxIter)
971 return emitOpError() <<
"Iteration Size exceeds the [0:" << maxIter
973 if (
static_cast<uint32_t
>(getIterationStride()) > maxStep)
974 return emitOpError() <<
"Iteration Stride exceeds the [0:" << maxStep
979 if (getBufferLength() > maxLen)
980 return emitOpError() <<
"Buffer length exceeds the [0:" << maxLen
982 if (targetModel.
isShimNOCTile(getColumn(), getRow()) && getD2Size() != 0)
983 return emitOpError(
"ShimTile only supports 3 dimensions of sizes.");
987 llvm::SmallVector<std::pair<AIE::DmaBdField, AIE::DmaBdField>, 3> padFields;
990 uint64_t pads[3][2] = {{getD0ZeroBefore(), getD0ZeroAfter()},
991 {getD1ZeroBefore(), getD1ZeroAfter()},
992 {getD2ZeroBefore(), getD2ZeroAfter()}};
993 for (
auto [d, pad] : llvm::enumerate(pads)) {
994 if (d >= padFields.size()) {
995 if (pad[0] || pad[1])
997 <<
"D" << d <<
" zero padding is not supported on this tile.";
1000 uint32_t maxBefore = padFields[d].first.mask();
1001 uint32_t maxAfter = padFields[d].second.mask();
1002 if (pad[0] > maxBefore || pad[1] > maxAfter)
1003 return emitOpError() <<
"D" << d
1004 <<
" pad_before/pad_after exceeds the [0:"
1005 << (pad[0] > maxBefore ? maxBefore : maxAfter)
1010 getBurstLength() != 0)
1011 return emitOpError(
"Only ShimTiles support burst length.");
1012 auto errorMessage = checkBurstLength(targetModel, getBurstLength());
1013 if (errorMessage.has_value()) {
1014 return emitOpError(errorMessage.value());
1016 if (!targetModel.
isShimNOCTile(getColumn(), getRow()) && getAxcache())
1017 return emitOpError(
"Only ShimTiles support AxCACHE configuration.");
1022uint32_t AIEX::NpuWriteBdOp::getAxcacheOrDefault() {
1028 return static_cast<uint32_t
>(*cst);
1029 return std::nullopt;
1036 if (
auto constant = v.getDefiningOp<arith::ConstantOp>())
1037 if (
auto attr = dyn_cast<IntegerAttr>(constant.getValue()))
1038 return attr.getValue().getZExtValue();
1040 if (!mlir::matchPattern(v, mlir::m_ConstantInt(&cst)))
1041 return std::nullopt;
1042 return cst.getZExtValue();
1046 mlir::Location loc, uint32_t value) {
1047 return arith::ConstantOp::create(
1048 builder, loc, builder.getI32IntegerAttr(
static_cast<int32_t
>(value)));
1052 mlir::Location loc, uint64_t value) {
1053 if (value <= std::numeric_limits<uint32_t>::max())
1055 return arith::ConstantOp::create(
1057 IntegerAttr::get(builder.getIntegerType(64),
1058 static_cast<int64_t
>(value)));
1068static bool hasQueryableStorageLayout(Type type) {
1069 if (type.isIntOrFloat() || isa<IndexType, DataLayoutTypeInterface>(type))
1071 if (
auto vector = dyn_cast<VectorType>(type))
1072 return hasQueryableStorageLayout(
vector.getElementType());
1073 if (
auto complex = dyn_cast<ComplexType>(type))
1074 return hasQueryableStorageLayout(complex.getElementType());
1080static std::optional<uint64_t>
1081getStaticMemRefStorageBytes(MemRefType type,
const DataLayout &layout) {
1082 if (!type || !type.hasStaticShape() ||
1083 !hasQueryableStorageLayout(type.getElementType()))
1084 return std::nullopt;
1085 if (
auto vector = dyn_cast<VectorType>(type.getElementType())) {
1089 return std::nullopt;
1090 int64_t elements = 1;
1091 for (int64_t dim :
vector.getShape()) {
1092 auto product = llvm::checkedMul(elements, dim);
1094 return std::nullopt;
1095 elements = *product;
1097 uint64_t inner =
vector.getShape().back();
1098 auto padded = llvm::checkedMulUnsigned(
1099 static_cast<uint64_t
>(elements) / inner, llvm::PowerOf2Ceil(inner));
1100 llvm::TypeSize bits = layout.getTypeSizeInBits(
vector.getElementType());
1101 if (!padded || bits.isScalable() ||
1102 !llvm::checkedMulUnsigned(*padded, bits.getFixedValue()))
1103 return std::nullopt;
1105 llvm::TypeSize elementBytes = layout.getTypeSize(type.getElementType());
1106 if (elementBytes.isScalable())
1107 return std::nullopt;
1108 if (llvm::is_contained(type.getShape(), int64_t{0}))
1110 uint64_t bytes = elementBytes.getFixedValue();
1111 for (int64_t dim : type.getShape()) {
1112 auto product = llvm::checkedMulUnsigned(bytes,
static_cast<uint64_t
>(dim));
1114 return std::nullopt;
1120template <
typename OpTy>
1121static LogicalResult verifyBufferWriteScope(OpTy op) {
1122 if (op.getBuffer() && !op->template getParentOfType<AIE::DeviceOp>())
1123 return op.emitOpError(
"must be inside a device when buffer is present");
1127template <
typename OpTy>
1129verifyBufferWrite(OpTy op, std::optional<uint32_t> wordOffset,
1131 auto bufferAttr = op.getBufferAttr();
1135 return op.emitOpError(
"address must be constant when buffer is present");
1136 auto device = op->template getParentOfType<AIE::DeviceOp>();
1138 return op.emitOpError(
"must be inside a device when buffer is present");
1141 auto buffer = names.
lookupIn<AIE::BufferOp>(device, bufferAttr.getAttr());
1143 return op.emitOpError()
1144 <<
"buffer '" << bufferAttr.getValue() <<
"' not found in device";
1145 auto bufferBytes = getStaticMemRefStorageBytes(buffer.getType(),
1146 DataLayout::closest(buffer));
1148 return op.emitOpError(
1149 "referenced buffer must have a static, fixed-size storage extent "
1150 "representable in 64 bits");
1152 uint64_t offsetBytes = uint64_t{*wordOffset} *
sizeof(uint32_t);
1153 auto end = llvm::checkedAddUnsigned(offsetBytes, writeBytes);
1155 return op.emitOpError(
"buffer-relative write range overflows 64-bit bytes");
1156 if (*end > *bufferBytes)
1157 return op.emitOpError() <<
"write range [" << offsetBytes <<
", " << *end
1158 <<
") exceeds buffer @" << bufferAttr.getValue()
1159 <<
" size " << *bufferBytes <<
" bytes";
1161 auto tile = buffer.getTileLike();
1162 const auto &targetModel = device.getTargetModel();
1165 uint64_t start = offsetBytes;
1166 if (
auto base = buffer.getAddress()) {
1168 return op.emitOpError(
"referenced buffer address must be nonnegative");
1169 start +=
static_cast<uint64_t
>(*base);
1170 if (start %
sizeof(uint32_t) != 0)
1171 return op.emitOpError() <<
"resolved buffer write address " << start
1172 <<
" is not 4-byte aligned";
1176 auto memoryEnd = llvm::checkedAddUnsigned(start, writeBytes);
1178 return op.emitOpError(
"resolved buffer write range overflows 64-bit bytes");
1179 if (*memoryEnd > memoryBytes)
1180 return op.emitOpError()
1181 <<
"write range [" << start <<
", " << *memoryEnd
1182 <<
") exceeds tile data memory size " << memoryBytes <<
" bytes";
1190LogicalResult AIEX::NpuWrite32Op::verify() {
1191 return verifyBufferWriteScope(*
this);
1199 sizeof(uint32_t), names);
1202template <
typename T>
1203static std::optional<uint32_t>
1204getAbsoluteAddress(T *op, uint32_t addressOffset,
1206 AIE::DeviceOp device =
1207 op->getOperation()->template getParentOfType<AIE::DeviceOp>();
1209 op->emitError(
"Must be inside a device.");
1210 return std::nullopt;
1214 uint32_t address = 0;
1218 if (
auto bufferSym = op->getBuffer()) {
1219 AIE::BufferOp buffer =
1220 names ? names->
lookup<AIE::BufferOp>(op->getBufferAttr().getAttr())
1223 op->emitError() <<
"buffer '" << *bufferSym <<
"' not found in device";
1224 return std::nullopt;
1227 auto bufferAddress = buffer.getAddress();
1228 if (!bufferAddress) {
1229 mlir::InFlightDiagnostic err =
1230 op->emitError(
"referenced buffer must have address assigned");
1231 err.attachNote(buffer.getLoc()) <<
"This buffer must have an address.";
1232 return std::nullopt;
1235 uint32_t col = buffer.getTileOp().getCol();
1236 uint32_t row = buffer.getTileOp().getRow();
1237 address =
static_cast<uint32_t
>(*bufferAddress) +
1238 addressOffset *
sizeof(uint32_t);
1240 ((row & 0xff) << tm.
getRowShift()) | (address & 0xfffff);
1242 address = addressOffset;
1243 std::optional<uint32_t> col = op->getColumn();
1244 std::optional<uint32_t> row = op->getRow();
1249 ((*row & 0xff) << tm.
getRowShift()) | (address & 0xfffff);
1256std::optional<uint32_t>
1260 return std::nullopt;
1261 return ::getAbsoluteAddress(
this, *addressOffset, names);
1268LogicalResult AIEX::NpuAddressPatchOp::verify() {
1272 if (getAddrVal() && !getAddrVal().getType().isInteger(32))
1273 return emitOpError(
"addr_val must be an i32 value");
1274 if (getArgIdx().has_value() ==
static_cast<bool>(getBuffer()))
1275 return emitOpError(
"must name the host buffer either by the 'arg_idx' "
1276 "attribute or by the 'buffer' operand, not both and not "
1285std::optional<uint32_t> AIEX::NpuUpdateFromScratchpadOp::getAbsoluteAddress(
1287 return ::getAbsoluteAddress(
this, getAddress(), names);
1290LogicalResult AIEX::NpuUpdateFromScratchpadOp::verify() {
1292 constexpr uint32_t kMaxStateTableEntries = 32;
1293 if (getStateTableIdx() >= kMaxStateTableEntries)
1294 return emitOpError(
"state_table_idx ")
1295 <<
static_cast<uint32_t
>(getStateTableIdx())
1296 <<
" exceeds maximum StateTable index ("
1297 << (kMaxStateTableEntries - 1) <<
").";
1307LogicalResult AIEX::NpuCreateScratchpadOp::verify() {
1309 if (getUsageType() != 0) {
1310 return emitOpError(
"usage_type must be 0 (got ")
1311 <<
static_cast<uint32_t
>(getUsageType())
1312 <<
"); other layouts are not supported.";
1317 constexpr uint32_t kMaxScratchpadSizeBytes = 128;
1318 if (getSize() == 0) {
1319 return emitOpError(
"size must be greater than 0.");
1321 if (getSize() % 4 != 0) {
1322 return emitOpError(
"size (")
1323 << getSize() <<
") must be a multiple of 4 bytes.";
1325 if (getSize() > kMaxScratchpadSizeBytes) {
1326 return emitOpError(
"size (")
1327 << getSize() <<
" bytes) exceeds maximum scratchpad size of "
1328 << kMaxScratchpadSizeBytes <<
" bytes.";
1333 uint32_t numEntries = getSize() / 4;
1334 if (Block *block = (*this)->getBlock()) {
1335 for (
auto updateOp : block->getOps<AIEX::NpuUpdateFromScratchpadOp>()) {
1336 if (updateOp.getStateTableIdx() < numEntries)
1338 return updateOp.emitOpError(
"state_table_idx ")
1339 <<
static_cast<uint32_t
>(updateOp.getStateTableIdx())
1340 <<
" is out of bounds for scratchpad of size " << getSize()
1341 <<
" bytes (" << numEntries <<
" entries) created by "
1342 << (*this)->getName() <<
".";
1350 auto runtimeSeq = getOperation()->getParentOfType<AIE::RuntimeSequenceOp>();
1355 NpuCreateScratchpadOp firstSeen;
1356 runtimeSeq.walk([&](NpuCreateScratchpadOp op) {
1358 return WalkResult::interrupt();
1360 if (firstSeen != *
this) {
1361 InFlightDiagnostic diag =
1362 emitOpError(
"only one 'aiex.npu.create_scratchpad' is allowed per "
1363 "runtime sequence");
1364 diag.attachNote(firstSeen.getLoc())
1365 <<
"previous 'aiex.npu.create_scratchpad' here";
1376LogicalResult AIEX::NpuMaskWrite32Op::verify() {
1377 return verifyBufferWriteScope(*
this);
1385 sizeof(uint32_t), names);
1388std::optional<uint32_t>
1392 return std::nullopt;
1393 return ::getAbsoluteAddress(
this, *addressOffset, names);
1400LogicalResult AIEX::NpuMaskPollOp::verify() {
1401 return verifyBufferWriteScope(*
this);
1409 sizeof(uint32_t), names);
1412std::optional<uint32_t>
1416 return std::nullopt;
1417 return ::getAbsoluteAddress(
this, *addressOffset, names);
1424LogicalResult AIEX::NpuBlockWriteOp::verify() {
1425 return verifyBufferWriteScope(*
this);
1432 auto type = dyn_cast<MemRefType>(getData().getType());
1433 DataLayout layout = DataLayout::closest(*
this);
1434 auto bytes = getStaticMemRefStorageBytes(type, layout);
1436 return emitOpError(
"payload must have a static, fixed-size storage extent "
1437 "representable in 64 bits when buffer is present");
1438 llvm::TypeSize elementBits = layout.getTypeSizeInBits(type.getElementType());
1439 if (elementBits.isScalable() || elementBits.getFixedValue() != 32)
1441 "payload must have 32-bit elements when buffer is present");
1442 return verifyBufferWrite(*
this, getAddress(), *bytes, names);
1445std::optional<uint32_t>
1447 return ::getAbsoluteAddress(
this, getAddress(), names);
1450DenseIntElementsAttr AIEX::NpuBlockWriteOp::getDataWords() {
1451 Value
memref = this->getData();
1452 DataLayout dataLayout = DataLayout::closest(*
this);
1453 int64_t width = dataLayout.getTypeSizeInBits(
1454 cast<MemRefType>(
memref.getType()).getElementType());
1456 emitWarning(
"Only 32-bit data type is supported for now");
1460 memref::GetGlobalOp getGlobal =
memref.getDefiningOp<memref::GetGlobalOp>();
1462 emitError(
"Only MemRefs from memref.get_global are supported");
1466 auto global = dyn_cast_if_present<memref::GlobalOp>(
1467 (*this)->getParentOfType<AIE::DeviceOp>().lookupSymbol(
1468 getGlobal.getName()));
1470 emitError(
"Global symbol not found");
1474 auto initVal = global.getInitialValue();
1476 emitError(
"Global symbol has no initial value");
1480 auto data = dyn_cast<DenseIntElementsAttr>(*initVal);
1482 emitError(
"Global symbol initial value is not a dense int array");
1493AIE::DMABDOp AIEX::DMAConfigureTaskOp::getFirstBd() {
1494 Region &body = getBody();
1498 auto bd_ops = body.front().getOps<AIE::DMABDOp>();
1499 if (bd_ops.empty() && body.front().getNumSuccessors() == 1) {
1503 Block &chain_entry = *body.front().getSuccessor(0);
1504 bd_ops = chain_entry.getOps<AIE::DMABDOp>();
1506 if (bd_ops.empty()) {
1509 return *bd_ops.begin();
1513AIEX::DMAConfigureTaskOp::canonicalize(AIEX::DMAConfigureTaskOp op,
1514 PatternRewriter &rewriter) {
1516 Region &body = op.getBody();
1517 bool did_rewrite =
false;
1518 for (
auto &block : body) {
1519 if (block.empty()) {
1522 auto ops_it = block.without_terminator();
1523 if (std::distance(ops_it.begin(), ops_it.end()) == 0) {
1524 rewriter.eraseOp(block.getTerminator());
1563 LogicalResult result = success();
1564 body.walk([&](AIE::DMABDOp bd) {
1567 if (failed(bd.verifyMixedSizesAndStrides())) {
1576 size_t numDims = bd.getMixedSizes().size();
1577 if (bd.getIteration()) {
1578 if (numDims + 1 > maxNDims) {
1579 bd.emitOpError() <<
"Cannot give more than "
1580 << std::to_string(maxNDims - 1)
1581 <<
" dimensions for step sizes and wraps alongside "
1582 "the iteration attribute on this tile (got "
1583 << std::to_string(numDims) <<
" dimensions).";
1588 auto isConstant = [](OpFoldResult v) {
1589 return getConstantIntValue(v).has_value();
1591 if (numDims > maxNDims && llvm::all_of(bd.getMixedSizes(), isConstant) &&
1592 llvm::all_of(bd.getMixedStrides(), isConstant))
1594 if (numDims > maxNDims) {
1595 bd.emitOpError() <<
"Cannot give more than " << std::to_string(maxNDims)
1596 <<
" dimensions for step sizes and wraps on this tile "
1598 << std::to_string(numDims) <<
" dimensions).";
1605LogicalResult AIEX::DMAConfigureTaskOp::verify() {
1607 getDirection(), getChannel())))
1613 std::optional<int> col = getTileLike().tryGetCol();
1614 std::optional<int> row = getTileLike().tryGetRow();
1616 failed(verifyTaskBDDimensions(targetModel, *col, *row, getBody())))
1618 Region &body = getBody();
1625 LogicalResult result = success();
1626 bool taskHasPacket = getPacket().has_value();
1627 llvm::SmallVector<AIE::DMABDOp> bds;
1628 for (
auto &block : body) {
1629 if (block.empty()) {
1632 if (block.hasNoPredecessors() && !block.isEntryBlock()) {
1633 auto error = block.getTerminator()->emitError(
1634 "Block ending in this terminator does not form a chain with "
1642 block.walk([&](AIE::DMABDOp bd) {
1643 if (bd.getBurstLength() != 0 &&
1644 !targetModel.
isShimNOCTile(getTileID().col, getTileID().row)) {
1645 bd.emitOpError(
"Burst length is only supported in Shim NOC tiles that "
1646 "are connected to the memory-mapped NOC.");
1652 bd, bd.getPacket().value_or(AIE::PacketInfoAttr()))) ||
1654 failed(bd.verifyTransferInBounds()))
1661 bool onMemTile = col && row && targetModel.
isMemTile(*col, *row);
1662 uint32_t adjacentChannels =
1665 if (onMemTile &&
static_cast<uint32_t
>(getChannel()) >= adjacentChannels) {
1666 auto isNonLocal = [&](AIE::TileLike tile) {
1669 auto resourceCol = tile.tryGetCol();
1670 auto resourceRow = tile.tryGetRow();
1671 return resourceCol && resourceRow &&
1672 (*resourceCol != *col || *resourceRow != *row);
1674 body.walk([&](Operation *op) {
1675 Operation *resource =
nullptr;
1676 AIE::TileLike resourceTile;
1678 if (
auto bd = dyn_cast<AIE::DMABDOp>(op)) {
1679 auto buffer = bd.getBuffer().getDefiningOp<AIE::BufferOp>();
1683 resourceTile = buffer.getTileLike();
1685 }
else if (
auto useLock = dyn_cast<AIE::UseLockOp>(op)) {
1686 auto lock = useLock.getLock().getDefiningOp<AIE::LockOp>();
1688 useLock.emitOpError()
1689 <<
"requires a lock defined by aie.lock in a task on memtile DMA "
1696 resourceTile = lock.getTileLike();
1704 if (!resourceTile) {
1705 op->emitOpError() <<
"uses a " << what
1706 <<
" whose tile operand is not a tile";
1710 if (!isNonLocal(resourceTile))
1712 InFlightDiagnostic err = op->emitOpError()
1713 <<
"is in a task on memtile DMA channel "
1715 <<
" and attempts to access a non-local " << what
1716 <<
"; only channels below " << adjacentChannels
1717 <<
" reach neighboring tiles";
1718 err.attachNote(resource->getLoc()) << what;
1722 if (getOutOfOrder()) {
1724 if (getIssueToken()) {
1726 emitOpError(
"out_of_order channel cannot issue a completion token");
1727 err.attachNote() <<
"set issue_token = false on this out-of-order task";
1731 getOperation(), getDirection(), getOutOfOrder(),
1732 llvm::ArrayRef<AIE::DMABDOp>(bds),
1739LogicalResult AIEX::DMAStartTaskOp::verify() {
1740 if (getRepeatCountAttr() && getRepeatCountVal())
1742 "takes repeat_count or a runtime repeat count, not both");
1743 if (IntegerAttr rc = getRepeatCountAttr(); rc && rc.getInt() < 0)
1744 return emitOpError(
"repeat_count must be non-negative, got ")
1746 if (hasRepeatParameter() && (getRepeatCountAttr() || getRepeatCountVal()))
1747 return emitOpError(
"takes a repeat_parameter or a repeat count, not both");
1748 if (getRepeatParameterAttr() && getRepeatStateTableIdxAttr())
1749 return emitOpError(
"takes repeat_parameter or repeat_state_table_idx, "
1754std::optional<int64_t> AIEX::DMAConfigureTaskForOp::getStaticBytesStreamed() {
1756 for (Operation *user : getResult().getUsers()) {
1757 auto start = dyn_cast<AIEX::DMAStartTaskOp>(user);
1759 if (isa<AIEX::DMAAwaitTaskOp, AIEX::DMAFreeTaskOp>(user))
1763 return std::nullopt;
1767 if (start->getBlock() != (*this)->getBlock() ||
1768 start.getRepeatCountAttr() || start.getRepeatCountVal() ||
1769 start.hasRepeatParameter())
1770 return std::nullopt;
1775 std::optional<int64_t> repeat = getConstantIntValue(getRepeatCountValue());
1776 if (!repeat || *repeat < 0)
1777 return std::nullopt;
1779 Region &body = getBody();
1781 return std::nullopt;
1785 std::optional<int64_t> moved = 0;
1786 for (
auto scc = llvm::scc_begin(&body.front()); !scc.isAtEnd(); ++scc) {
1788 return std::nullopt;
1789 for (AIE::DMABDOp bd : scc->front()->getOps<AIE::DMABDOp>()) {
1792 std::optional<int64_t> len =
1793 bd.getLengthParameterAttr() || bd.getLengthStateTableIdxAttr()
1795 : bd.getStaticLenInBytes();
1796 if (!len || bd.getIteration() || bd.getNextBdId())
1797 return std::nullopt;
1798 moved = llvm::checkedAdd(*moved, *len);
1800 return std::nullopt;
1803 std::optional<int64_t> runs = llvm::checkedAdd(*repeat, int64_t{1});
1805 runs = llvm::checkedMul(*runs, starts);
1806 return runs ? llvm::checkedMul(*moved, *runs) : std::nullopt;
1813 SmallVector<int64_t> segmentBytes;
1815 int64_t objectBytes;
1818 static std::optional<BdChain>
get(ArrayRef<int64_t> segmentBytes,
1820 if (segmentBytes.size() == 1)
1822 std::optional<int64_t> objectBytes = 0;
1823 for (int64_t bytes : segmentBytes) {
1824 std::optional<int64_t> run = llvm::checkedMul(bytes, copies);
1825 if (bytes <= 0 || copies <= 0 || !run)
1826 return std::nullopt;
1827 objectBytes = llvm::checkedAdd(*objectBytes, *run);
1829 return std::nullopt;
1831 if (segmentBytes.empty())
1832 return std::nullopt;
1833 return BdChain{SmallVector<int64_t>(segmentBytes), copies, *objectBytes};
1837 bool endsOnBd(int64_t bytes)
const {
1838 int64_t rest = bytes % objectBytes;
1840 for (int64_t segment : segmentBytes) {
1841 if (rest < start + segment * copies)
1842 return (rest - start) % segment == 0;
1843 start += segment * copies;
1848 bool operator==(
const BdChain &other)
const {
1849 return segmentBytes == other.segmentBytes && copies == other.copies;
1854std::optional<int64_t> AIEX::NpuDmaMemcpyNdOp::getStaticBytesStreamed() {
1855 if (getLengthParameterAttr() || getLengthStateTableIdxAttr())
1856 return std::nullopt;
1857 uint64_t bits = getElementTypeBitwidth();
1859 return std::nullopt;
1860 std::optional<int64_t> bytes =
static_cast<int64_t
>(bits / 8);
1861 for (OpFoldResult size : getMixedSizes()) {
1862 std::optional<int64_t> extent = getConstantIntValue(size);
1863 bytes = bytes && extent ? llvm::checkedMul(*bytes, *extent) : std::nullopt;
1874static LogicalResult verifyTransferCoversWholeObjects(
1875 Operation *transfer, SymbolRefAttr alloc, std::optional<int64_t> moved,
1876 AIE::ObjectFifoCreateOp fifo, SymbolTableCollection &symbols) {
1877 if (fifo.getPadDimensions() || fifo.usesStream())
1879 if (!moved || *moved == 0)
1881 DataLayout layout = DataLayout::closest(transfer);
1883 auto objectBytesOf = [&](Type fifoType) -> std::optional<int64_t> {
1884 MemRefType elem = cast<AIE::AIEObjectFifoType>(fifoType).getElementType();
1885 return llvm::checkedMul(
1886 elem.getNumElements(),
1887 static_cast<int64_t
>(layout.getTypeSize(elem.getElementType())));
1897 SmallVector<Value> linkTiles;
1898 SmallVector<BdChain> chains;
1899 auto device = fifo->getParentOfType<AIE::DeviceOp>();
1900 auto resolve = [&](ArrayAttr refs) {
1901 std::vector<AIE::ObjectFifoCreateOp> fifos;
1902 for (Attribute ref : refs)
1903 if (
auto name = dyn_cast<FlatSymbolRefAttr>(ref))
1904 if (
auto linked = symbols.lookupSymbolIn<AIE::ObjectFifoCreateOp>(
1905 device, name.getAttr()))
1906 fifos.push_back(linked);
1909 auto self = FlatSymbolRefAttr::get(fifo.getSymNameAttr());
1911 for (AIE::ObjectFifoLinkOp link : device.getOps<AIE::ObjectFifoLinkOp>()) {
1912 if (!llvm::is_contained(link.getFifoIns(), self) &&
1913 !llvm::is_contained(link.getFifoOuts(), self))
1915 std::vector<AIE::ObjectFifoCreateOp> ins = resolve(link.getFifoIns());
1916 std::vector<AIE::ObjectFifoCreateOp> outs = resolve(link.getFifoOuts());
1917 std::optional<Value> sharedTile = link.getOptionalSharedTile();
1918 if (ins.empty() || outs.empty() || !sharedTile)
1920 linkTiles.push_back(*sharedTile);
1921 AIE::ObjectFifoCreateOp owner = link.getPoolOwner();
1923 cast<AIE::AIEObjectFifoType>(owner.getElemType()).getElementType();
1925 static_cast<int64_t
>(layout.getTypeSize(pool.getElementType()));
1926 SmallVector<std::pair<int64_t, int64_t>> segments =
1927 link.getPoolSegments();
1928 if (fifo != owner && (link.isJoin() || link.isDistribute())) {
1929 std::vector<AIE::ObjectFifoCreateOp> &many = link.isJoin() ? ins : outs;
1930 segments = {segments[llvm::find(many, fifo) - many.begin()]};
1932 SmallVector<int64_t> segmentBytes;
1933 for (
auto [offset, size] : segments) {
1934 std::optional<int64_t> bytes = llvm::checkedMul(size, width);
1937 segmentBytes.push_back(*bytes);
1939 int64_t copies = llvm::is_contained(outs, fifo)
1940 ? link.getRepeatCount().value_or(1)
1942 std::optional<BdChain> chain;
1943 if (segmentBytes.size() == segments.size())
1944 chain = BdChain::get(segmentBytes, copies);
1945 if (chain && !llvm::is_contained(chains, *chain))
1946 chains.push_back(*chain);
1953 auto addChain = [&](Value tile, Type fifoType) {
1954 if (llvm::is_contained(linkTiles, tile))
1956 std::optional<BdChain> chain;
1957 if (std::optional<int64_t> bytes = objectBytesOf(fifoType))
1958 chain = BdChain::get({*bytes}, 1);
1959 if (chain && !llvm::is_contained(chains, *chain))
1960 chains.push_back(*chain);
1963 dyn_cast_or_null<AIE::TileLike>(fifo.getProducerTile().getDefiningOp());
1966 if (!producer.isShimTile())
1967 addChain(fifo.getProducerTile(), fifo.getElemType());
1968 for (Value consumer : fifo.getConsumerTiles()) {
1969 auto tile = dyn_cast_or_null<AIE::TileLike>(consumer.getDefiningOp());
1970 if (!tile || !tile.isShimTile())
1971 addChain(consumer, fifo.getConsumerElemTypeOrDefault());
1977 auto sequence = transfer->getParentOfType<AIE::RuntimeSequenceOp>();
1978 if (!sequence || transfer->getBlock() != &sequence.getBody().front())
1983 if (llvm::any_of(transfer->getBlock()->getOps<AIEX::DmaChannelResetForOp>(),
1984 [&](AIEX::DmaChannelResetForOp reset) {
1985 return reset.getObjfifoAttr() == alloc;
1990 auto through = [&](Operation *op) -> std::optional<int64_t> {
1991 if (
auto memcpy = dyn_cast<AIEX::NpuDmaMemcpyNdOp>(op))
1992 return memcpy.getMetadata() == alloc ? memcpy.getStaticBytesStreamed()
1994 if (
auto task = dyn_cast<AIEX::DMAConfigureTaskForOp>(op))
1995 return task.getAlloc() == alloc ? task.getStaticBytesStreamed() : 0;
1996 if (
auto chain = dyn_cast<AIEX::DMAStartBdChainForOp>(op))
1997 return chain.getAllocAttr() == alloc ? std::nullopt
1998 : std::optional<int64_t>(0);
2004 for (Operation *prev = transfer->getPrevNode(); prev;
2005 prev = prev->getPrevNode())
2006 if (through(prev) != 0)
2008 std::optional<int64_t> total = 0;
2009 int64_t sharing = 0;
2010 WalkResult walk = sequence.walk([&](Operation *op) {
2011 std::optional<int64_t> bytes = through(op);
2014 return WalkResult::advance();
2015 if (!bytes || op->getBlock() != transfer->getBlock())
2016 return WalkResult::interrupt();
2017 total = llvm::checkedAdd(*total, *bytes);
2019 return WalkResult::interrupt();
2021 return WalkResult::advance();
2023 if (walk.wasInterrupted())
2025 const BdChain *partial = llvm::find_if(
2026 chains, [&](
const BdChain &chain) {
return !chain.endsOnBd(*total); });
2027 if (partial == chains.end())
2030 InFlightDiagnostic error = transfer->emitOpError()
2031 <<
"moves " << *moved
2032 <<
" bytes through objectFIFO @"
2033 << fifo.getSymName() <<
", which";
2035 error <<
", with this runtime sequence's other transfers through it ("
2036 << *total <<
" bytes in all),";
2037 error <<
" is not a whole number of its " << partial->objectBytes
2039 if (partial->segmentBytes.size() == 1)
2040 return error <<
"; the far end's buffer descriptor is sized by the "
2041 "object, so a partial one never completes";
2042 std::string segments;
2043 llvm::raw_string_ostream os(segments);
2044 llvm::interleaveComma(partial->segmentBytes, os);
2045 error <<
" and doesn't end on a segment boundary (segments of " << segments
2047 if (partial->copies > 1)
2048 error <<
", each sent " << partial->copies <<
" times in a row";
2049 return error <<
"); the far end's buffer descriptors are sized by the "
2050 "segments, so a partial one never completes";
2054AIEX::DMAConfigureTaskOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2055 FlatSymbolRefAttr alloc = getAllocAttr();
2056 AIE::DeviceOp dev = getOperation()->getParentOfType<AIE::DeviceOp>();
2059 auto allocation = dyn_cast_or_null<AIE::ShimDMAAllocationOp>(
2060 symbolTable.lookupSymbolIn(dev, alloc.getAttr()));
2062 return emitOpError(
"alloc ")
2063 << alloc <<
" names no aie.shim_dma_allocation";
2064 if (allocation.getTile() != getTile() ||
2065 allocation.getChannelDir() != getDirection() ||
2066 allocation.getChannelIndex() !=
static_cast<int>(getChannel()))
2067 return emitOpError(
"runs on another channel than its alloc ") << alloc;
2074LogicalResult AIEX::DMAConfigureTaskForOp::verifySymbolUses(
2075 SymbolTableCollection &symbolTable) {
2076 AIE::DeviceOp dev = getOperation()->getParentOfType<AIE::DeviceOp>();
2080 symbolTable.lookupSymbolIn(dev, getAlloc().getRootReference());
2082 return emitOpError() << getAlloc() <<
" does not name a symbol";
2088 LogicalResult named =
2089 TypeSwitch<Operation *, LogicalResult>(target)
2090 .Case([&](AIE::ShimDMAAllocationOp alloc) {
2091 tileValue = alloc.getTile();
2094 .Case([&](AIE::RouteEndpointOp endpoint) -> LogicalResult {
2095 tileValue = endpoint.getTile();
2096 if (endpoint.getBundle() == AIE::WireBundle::DMA)
2098 return emitOpError() << getAlloc() <<
" names a "
2099 << stringifyWireBundle(endpoint.getBundle())
2100 <<
" port, not a DMA channel";
2102 .Case([&](AIE::ObjectFifoCreateOp fifo) {
2103 return verifyTransferCoversWholeObjects(
2104 *
this, getAlloc(), getStaticBytesStreamed(), fifo, symbolTable);
2106 .Default([&](Operation *) -> LogicalResult {
2107 return emitOpError() << getAlloc()
2108 <<
" must name an aie.shim_dma_allocation, an "
2109 "aie.route_endpoint or an aie.objectfifo";
2115 auto tile = tileValue ? tileValue.getDefiningOp<AIE::TileOp>() : nullptr;
2119 return verifyTaskBDDimensions(targetModel, tile.getCol(), tile.getRow(),
2127AIE::BDChainOp AIEX::DMAStartBdChainOp::getBDChainOp() {
2128 AIE::DeviceOp device = (*this)->getParentOfType<AIE::DeviceOp>();
2129 AIE::BDChainOp chain = device.lookupSymbol<AIE::BDChainOp>(getSymbol());
2133LogicalResult AIEX::DMAStartBdChainOp::verify() {
2134 AIE::BDChainOp chain = getBDChainOp();
2136 return emitOpError(
"symbol does not reference valid BD chain");
2139 auto actualArgTypes = getArgs().getTypes();
2140 auto expectedArgTypes = chain.getRegion().getArgumentTypes();
2141 if (actualArgTypes.size() != expectedArgTypes.size()) {
2142 return emitOpError(
"Number of arguments mismatches.");
2144 for (
unsigned i = 0, n = expectedArgTypes.size(); i < n; i++) {
2145 if (actualArgTypes[i] != expectedArgTypes[i]) {
2146 return emitOpError(
"Argument ") << (i + 1) <<
" types mismatch: "
2147 <<
"expected " << expectedArgTypes[i]
2148 <<
" but got " << actualArgTypes[i];
2158uint32_t AIEX::NpuControlPacketOp::getRowFromAddr() {
2160 uint32_t addr = getAddress();
2161 uint32_t rowInt = (addr >> targetModel.
getRowShift()) & 0x1f;
2165uint32_t AIEX::NpuControlPacketOp::getColumnFromAddr() {
2167 uint32_t addr = getAddress();
2172LogicalResult AIEX::NpuControlPacketOp::verify() {
2175 uint32_t opcode = getOpcode();
2177 return emitOpError(
"opcode ")
2179 <<
" is reserved; expected 0 (write), 1 (read), or 2 (write with "
2182 bool isRead = opcode == 1;
2183 if (isRead && getData())
2184 return emitOpError(
"read control packet must not carry a data payload; "
2185 "the requested word count goes in the length attribute");
2186 if (!isRead && !getData())
2188 "control packet with a write opcode requires a data payload");
2191 std::optional<uint32_t> words;
2192 if (
auto data = getData())
2193 words = data->size();
2195 words = getLength();
2197 return emitOpError(
"read control packet requires a length attribute");
2199 return emitOpError(
"control packet payload must be at least 1 word");
2202 if (isRead && *words > getMaxDataWords())
2203 return emitOpError(
"read control packet length ")
2204 << *words <<
" exceeds the maximum of " << getMaxDataWords()
2214LogicalResult AIEX::SetLockOp::verify() {
2218 return emitOpError(
"SetLockOp is not supported on AIE1.");
2220 if (std::optional<int64_t> value = getConstantIntValue(getValue())) {
2222 return emitOpError(
"Lock value must be non-negative");
2224 return emitOpError(
"Lock value exceeds the maximum value of " +
2228 auto lockOp = getLockOp();
2229 auto lockIDOpt = getLockOp().getLockID();
2238 std::optional<int> col = lockOp.getTileLike().tryGetCol();
2239 std::optional<int> row = lockOp.getTileLike().tryGetRow();
2243 uint32_t lockID = lockOp.getLockIDValue();
2245 if (lockID >= targetModel.
getNumLocks(*col, *row)) {
2246 return emitOpError(
"Lock ID out of range for given tile. Max ID: " +
2247 std::to_string(targetModel.
getNumLocks(*col, *row) - 1));
2251 return emitOpError(
"Invalid lock ID and tile combination when trying to "
2252 "retrieve the local lock address.");
2262static LogicalResult verifyBdPoolPartition(Operation *op,
int col,
int row,
2263 uint32_t begin, uint32_t end) {
2265 if (begin >= end || end > numBds)
2266 return op->emitOpError()
2267 <<
"partition [" << begin <<
", " << end
2268 <<
") is not a nonempty range of the " << numBds
2269 <<
" buffer descriptors on tile (" << col <<
", " << row <<
")";
2273LogicalResult AIEX::DMABdPoolPopOp::verify() {
2274 if (failed(verifyBdPoolPartition(*
this, getColumn(), getRow(), getBdBegin(),
2277 std::optional<uint32_t>
id = getId();
2278 if (
id && (*
id < getBdBegin() || *
id >= getBdEnd()))
2279 return emitOpError() <<
"takes buffer descriptor ID " << *
id
2280 <<
", which is outside its partition [" << getBdBegin()
2281 <<
", " << getBdEnd() <<
")";
2285LogicalResult AIEX::DMABdPoolPushOp::verify() {
2286 return verifyBdPoolPartition(*
this, getColumn(), getRow(), getBdBegin(),
2294LogicalResult AIEX::DmaChannelResetOp::verify() {
2297 auto tile = dyn_cast_or_null<AIE::TileOp>(getTile().getDefiningOp());
2299 return emitOpError() <<
"tile operand must be produced by an aie.tile op";
2300 int col = tile.getCol();
2301 int row = tile.getRow();
2308 return emitOpError() <<
"tile (" << col <<
", " << row
2309 <<
") has no DMA channel reset (only core and mem "
2310 "tiles do; shim NOC DMA has no reset bit)";
2315 uint32_t numChannels = getDirection() == AIE::DMAChannelDir::S2MM
2317 col, row, AIE::WireBundle::DMA)
2319 col, row, AIE::WireBundle::DMA);
2320 if (getChannel() >= numChannels)
2321 return emitOpError() <<
"channel " << getChannel()
2322 <<
" out of range for this tile and direction (tile "
2324 << numChannels <<
" DMA channel(s) in this direction)";
2329LogicalResult AIEX::DmaChannelResetForOp::verify() {
2335 AIE::DeviceOp dev = getOperation()->getParentOfType<AIE::DeviceOp>();
2343 return emitOpError() <<
"is not supported on AIE1 devices (the resident "
2344 "re-arm relies on the aie2p start-queue push)";
2345 Operation *target = dev.lookupSymbol(getObjfifo());
2348 if (!isa<AIE::ObjectFifoCreateOp, AIE::RouteEndpoint,
2349 AIE::ObjectFifoRearmBindingOp>(target))
2350 return emitOpError() <<
"'" << getObjfifo()
2351 <<
"' must reference an aie.objectfifo (or its "
2352 "aie.objectfifo_rearm_binding)";
2360LogicalResult AIEX::CoreResetOp::verify() {
2366 return emitOpError(
"aiex.core_reset is not supported on AIE1.");
2368 auto tile = dyn_cast_or_null<AIE::TileOp>(getTile().getDefiningOp());
2370 return emitOpError() <<
"tile operand must be produced by an aie.tile op";
2371 int col = tile.getCol();
2372 int row = tile.getRow();
2381 return emitOpError() <<
"tile (" << col <<
", " << row
2382 <<
") has no core to reset (only core tiles have a "
2383 "CORE_CONTROL register)";
2391uint64_t AIEX::BlockFloatType::getTotalSizeInBits()
const {
2392 return getBlockSize() * getMantissaBits() + getExponentBits() +
2393 getSubtileShiftBits();
2396llvm::TypeSize AIEX::BlockFloatType::getTypeSizeInBits(
2397 const mlir::DataLayout &dataLayout,
2398 mlir::DataLayoutEntryListRef params)
const {
2399 return llvm::TypeSize::getFixed(getTotalSizeInBits());
2402uint64_t AIEX::BlockFloatType::getABIAlignment(
2403 const mlir::DataLayout &dataLayout,
2404 mlir::DataLayoutEntryListRef params)
const {
2410std::optional<AIEX::BlockFloatType::BlockFormat>
2411AIEX::BlockFloatType::getBlockFormat(StringRef blockType) {
2412 static const llvm::StringMap<AIEX::BlockFloatType::BlockFormat>
2414 {
"v8bfp16ebs8", {8, 8, 8, 0}},
2415 {
"v16bfp16ebs16", {16, 8, 8, 0}},
2418 auto it = blockFormatsMap.find(blockType);
2419 if (it != blockFormatsMap.end()) {
2423 return std::nullopt;
2427AIEX::BlockFloatType::verify(function_ref<InFlightDiagnostic()> emitError,
2428 StringRef block_type) {
2429 if (!getBlockFormat(block_type))
2430 return emitError() <<
"Invalid block type: " << block_type
2431 <<
". Known types are: v8bfp16ebs8, v16bfp16ebs16.";
2440AIE::DeviceOp AIEX::ConfigureOp::getReferencedDeviceOp() {
2441 ModuleOp moduleOp = this->getOperation()->getParentOfType<ModuleOp>();
2443 emitError(
"aiex.configure must be inside of a module");
2446 Operation *maybeReferencedDevice =
2447 SymbolTable::lookupSymbolIn(moduleOp.getOperation(), getSymbolAttr());
2448 if (!maybeReferencedDevice) {
2449 emitError(
"No such device: '") << getSymbolAttr() <<
"'";
2452 AIE::DeviceOp referencedDevice =
2453 llvm::dyn_cast<AIE::DeviceOp>(maybeReferencedDevice);
2454 if (!referencedDevice) {
2455 emitError(
"Not a device: '") << getSymbolAttr() <<
"'";
2458 return referencedDevice;
2465LogicalResult AIEX::ReadScratchpadParameterOp::verify() {
2466 auto device = (*this)->getParentOfType<AIE::DeviceOp>();
2468 return emitOpError(
"must be inside an aie.device");
2470 if (!(*this)->getParentOfType<AIE::CoreOp>()) {
2471 return emitOpError(
"must be inside an aie.core");
2473 auto moduleOp = (*this)->getParentOfType<ModuleOp>();
2475 !moduleOp.lookupSymbol<AIEX::ScratchpadParameterOp>(getParameter())) {
2476 return emitOpError(
"references unknown parameter '")
2478 <<
"' (aiex.scratchpad_parameter ops are declared at module scope)";
2480 if (getResult().getType().isF32()) {
2482 "f32 parameters are not supported: the scratchpad encoding zeroes "
2483 "the top 2 bits, which clobbers the sign bit and top exponent bit "
2484 "of an f32. Use bf16 or an integer type up to i32 instead.");
2489LogicalResult AIEX::WaitScratchpadParametersOp::verify() {
2490 if (!(*this)->getParentOfType<AIE::CoreOp>()) {
2491 return emitOpError(
"must be inside an aie.core");
2496LogicalResult AIEX::ConfigureOp::verify() {
2497 AIE::DeviceOp parentDev = getOperation()->getParentOfType<AIE::DeviceOp>();
2498 AIE::DeviceOp referencedDev = getReferencedDeviceOp();
2499 if (!referencedDev) {
2502 if (parentDev.getDevice() != referencedDev.getDevice()) {
2503 emitError(
"Device types do not match: '")
2504 << AIE::stringifyAIEDevice(parentDev.getDevice()) <<
"' vs. '"
2505 << AIE::stringifyAIEDevice(referencedDev.getDevice()) <<
"'";
2515AIE::DeviceOp AIEX::RunOp::getCalleeDeviceOp() {
2516 AIEX::ConfigureOp configureOp =
2517 getOperation()->getParentOfType<AIEX::ConfigureOp>();
2521 AIE::DeviceOp referencedDevice = configureOp.getReferencedDeviceOp();
2522 return referencedDevice;
2525AIE::RuntimeSequenceOp
2527 AIEX::ConfigureOp configureOp =
2528 getOperation()->getParentOfType<AIEX::ConfigureOp>();
2532 AIE::DeviceOp referencedDevice = configureOp.getReferencedDeviceOp();
2533 if (!referencedDevice) {
2537 StringAttr name = getRuntimeSequenceSymbolAttr().getAttr();
2541 return tables->
lookupIn<AIE::RuntimeSequenceOp>(referencedDevice, name);
2548LogicalResult AIEX::NpuLoadPdiOp::canonicalize(AIEX::NpuLoadPdiOp op,
2549 PatternRewriter &rewriter) {
2551 Operation *nextOp = op->getNextNode();
2556 auto nextLoadPdi = dyn_cast<AIEX::NpuLoadPdiOp>(nextOp);
2561 if (op.getDeviceRefAttr() == nextLoadPdi.getDeviceRefAttr() &&
2562 op.getId() == nextLoadPdi.getId() &&
2563 op.getSize() == nextLoadPdi.getSize() &&
2564 op.getAddress() == nextLoadPdi.getAddress()) {
2566 rewriter.eraseOp(op);
2577static bool writesConfig(Operation *op) {
2578 return isa<AIEX::NpuLoadPdiOp, AIEX::NpuWriteConfigOp,
2579 AIEX::NpuWriteTeardownOp>(op);
2586static Operation *findConfigOpBefore(Operation *op,
2587 function_ref<
bool(Operation *)> visit) {
2588 for (Operation *prev = op->getPrevNode(); prev; prev = prev->getPrevNode())
2589 if (writesConfig(prev) && visit(prev))
2591 for (Operation &next : llvm::reverse(*op->getBlock())) {
2594 if (writesConfig(&next) && visit(&next))
2597 return visit(op) ? op :
nullptr;
2602static Operation *precedingReset(Operation *op) {
2603 return findConfigOpBefore(op, [&](Operation *prev) {
2604 return prev != op &&
2605 isa<AIEX::NpuLoadPdiOp, AIEX::NpuWriteTeardownOp>(prev);
2609bool AIEX::NpuLoadPdiOp::loadsSamePdi(NpuLoadPdiOp other) {
2610 return getDeviceRefAttr() == other.getDeviceRefAttr() &&
2611 getId() == other.getId() && getSize() == other.getSize() &&
2612 getAddress() == other.getAddress();
2615LogicalResult AIEX::NpuWriteConfigOp::verify() {
2616 Operation *found = precedingReset(*
this);
2618 return emitOpError(
"must follow the load_pdi that resets the array");
2619 auto reset = dyn_cast<AIEX::NpuLoadPdiOp>(found);
2624 bool tornDown =
true;
2625 Operation *later = reset;
2627 cast<AIEX::NpuLoadPdiOp>(findConfigOpBefore(reset, [&](Operation *op) {
2628 if (isa<AIEX::NpuWriteConfigOp>(op) &&
2629 !isa<AIEX::NpuWriteTeardownOp>(later))
2632 return isa<AIEX::NpuLoadPdiOp>(op);
2634 if (!before.loadsSamePdi(reset) || tornDown)
2636 InFlightDiagnostic diag = emitOpError(
2637 "follows a load_pdi the firmware skips, as it loads the PDI loaded "
2638 "just before it, so the array is not reset");
2639 diag.attachNote(before.getLoc()) <<
"the PDI is loaded here";
2644AIEX::NpuLoadPdiOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2645 FlatSymbolRefAttr ref = getDeviceRefAttr();
2648 auto device = (*this)->getParentOfType<AIE::DeviceOp>();
2649 Operation *scope = device ? device->getParentOp()
2650 : SymbolTable::getNearestSymbolTable(*
this);
2651 if (!isa_and_present<AIE::DeviceOp>(symbolTable.lookupSymbolIn(scope, ref)))
2652 return emitOpError() << ref <<
" is not a device";
2657AIEX::NpuWriteConfigOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2658 Operation *scope = (*this)->getParentOfType<AIE::DeviceOp>()->getParentOp();
2659 if (!isa_and_present<AIE::DeviceOp>(
2660 symbolTable.lookupSymbolIn(scope, getDeviceRefAttr())))
2661 return emitOpError() << getDeviceRefAttr() <<
" is not a device";
2662 auto reset = dyn_cast_or_null<AIEX::NpuLoadPdiOp>(precedingReset(*
this));
2663 FlatSymbolRefAttr resetRef = reset ? reset.getDeviceRefAttr() :
nullptr;
2664 auto empty = resetRef ? dyn_cast_or_null<AIE::DeviceOp>(
2665 symbolTable.lookupSymbolIn(scope, resetRef))
2671 !llvm::all_of(empty.getBody()->without_terminator(), [](Operation &op) {
2672 return isa<AIE::RuntimeSequenceOp>(op) &&
2673 llvm::all_of(op.getRegion(0),
2674 [](Block &block) { return block.empty(); });
2676 return emitOpError(
"must follow a load_pdi of an empty device, which "
2677 "resets the array");
2685LogicalResult AIEX::NpuWriteTeardownOp::verify() {
2686 Operation *prev = (*this)->getPrevNode();
2687 while (prev && !writesConfig(prev))
2688 prev = prev->getPrevNode();
2689 auto config = dyn_cast_or_null<AIEX::NpuWriteConfigOp>(prev);
2690 if (!config || config.getDeviceRefAttr() != getDeviceRefAttr())
2691 return emitOpError(
"must follow the write_config of ")
2692 << getDeviceRefAttr() <<
", with no load_pdi between";
2697AIEX::NpuWriteTeardownOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2698 Operation *scope = (*this)->getParentOfType<AIE::DeviceOp>()->getParentOp();
2699 auto device = dyn_cast_or_null<AIE::DeviceOp>(
2700 symbolTable.lookupSymbolIn(scope, getDeviceRefAttr()));
2702 return emitOpError() << getDeviceRefAttr() <<
" is not a device";
2706 InFlightDiagnostic diag =
2707 emitOpError() << getDeviceRefAttr()
2708 <<
" starts a shim DMA channel, which teardown cannot stop";
2709 diag.attachNote(shimStart->getLoc()) <<
"started here";
2714 Operation *found =
nullptr;
2715 device.walk([&](Operation *op) {
2716 if (!isa<AIE::DMAStartOp, AIE::DMAOp>(op) ||
2717 !isa<AIE::ShimDMAOp>(op->getParentOp()))
2718 return WalkResult::advance();
2720 return WalkResult::interrupt();
bool isCoreTile(int col, int row) const
Return true if the given tile is a Core tile.
virtual uint64_t getDmaBdMaxLen(AIETileType tileType) const =0
Return the maximum transfer length, measured in address-generation granules (see getAddressGenGranula...
virtual const DmaBdLayout * getDmaBdLayout(AIETileType tileType) const
Return the register layout of a buffer descriptor on the given tile type, or null if the target has n...
virtual AIEArch getTargetArch() const =0
Return the target architecture.
bool isMemTile(int col, int row) const
Return true if the given tile is a Mem tile.
virtual uint32_t getDmaBdWrapBits(AIETileType tileType) const =0
Return the bit width of the wrap (size) field in a DMA buffer descriptor for the given tile type.
virtual uint32_t getLocalMemorySize() const =0
Return the size (in bytes) of the local data memory of a core.
virtual uint32_t getNumLocks(AIETileType tileType) const =0
Return the number of lock objects for a given tile type.
virtual uint32_t getDmaBdIterBits(AIETileType tileType) const =0
Return the bit width of the iteration (repeat) wrap/stride fields in a DMA buffer descriptor for the ...
virtual uint32_t getBDMaxDims(AIETileType tileType) const =0
Return the maximum number of addressing dimensions (wraps/strides) a single buffer descriptor support...
virtual std::vector< std::pair< uint32_t, uint32_t > > getShimBurstEncodingsAndLengths() const =0
virtual uint32_t getDmaBdStepBits(AIETileType tileType) const =0
Return the bit width of the step (stride) field in a DMA buffer descriptor for the given tile type.
virtual bool isBdChannelAccessible(int col, int row, uint32_t bd_id, int channel) const =0
Return true iff buffer descriptor bd_id on tile (col, row) can be submitted on channel channel.
virtual std::optional< uint32_t > getLocalLockAddress(uint32_t lockId, TileID tile) const =0
virtual uint32_t getMaxChannelNumForAdjacentMemTile(int col, int row) const =0
virtual AIETileType getTileType(int col, int row) const =0
Return the tile type for the given tile coordinates.
virtual uint32_t getMaxRepeatCount() const =0
Return the largest DMA task repeat count (unsupported = 0).
bool isShimNOCTile(int col, int row) const
Return true if the given tile is a ShimNOC tile.
virtual uint32_t getNumBDs(AIETileType tileType) const =0
Return the number of buffer descriptors for a given tile type.
virtual uint32_t getMaxLockValue() const =0
Return the maximum value that can be stored in a lock register.
virtual uint32_t getMaxOutOfOrderId() const =0
Return the largest out-of-order BD id (unsupported = 0).
virtual uint32_t getColumnShift() const =0
virtual uint32_t getNumDestSwitchboxConnections(int col, int row, WireBundle bundle) const =0
Return the number of destinations of connections inside a switchbox.
virtual uint32_t getRowShift() const =0
virtual uint32_t getAddressGenGranularity() const =0
Return the data bus width of the device.
virtual uint32_t getMemTileSize() const =0
Return the size (in bytes) of a MemTile.
virtual uint32_t getNumSourceSwitchboxConnections(int col, int row, WireBundle bundle) const =0
Return the number of sources of connections inside a switchbox.
OpTy lookupIn(mlir::Operation *symbolTableOp, mlir::StringAttr name)
mlir::Operation * lookup(mlir::StringAttr name) const
const V & get(const std::map< K, V > &pa, K k)
mlir::Operation * findShimDmaStart(AIE::DeviceOp device)
bool isContiguousTransfer(llvm::ArrayRef< int64_t > sizes, llvm::ArrayRef< int64_t > strides)
void encodeHardwareStridesWraps(Policy &p, uint64_t elemWidth, uint32_t addressGranularity, typename Policy::V inputSizes[4], typename Policy::V inputStrides[4], typename Policy::V sizes[4], typename Policy::V strides[4])
mlir::LogicalResult verifyStridesWraps(mlir::Operation *forOp, llvm::function_ref< mlir::InFlightDiagnostic()> emitError, mlir::BaseMemRefType referencedBufType, int tileCol, int tileRow, llvm::SmallVector< int64_t, 4 > inputSizes, llvm::SmallVector< int64_t, 4 > inputStrides, llvm::SmallVector< int64_t, 4 > hardwareSizes, llvm::SmallVector< int64_t, 4 > hardwareStrides, bool skipTransformationChecks=false)
std::optional< uint64_t > getConstantInt64Operand(mlir::Value v)
bool isDecomposableNdDmaPattern(mlir::Operation *forOp, mlir::BaseMemRefType referencedBufType, const xilinx::AIE::AIETargetModel &targetModel, int tileCol, int tileRow, llvm::ArrayRef< int64_t > offsetsInnermostFirst, llvm::ArrayRef< int64_t > sizesInnermostFirst, llvm::ArrayRef< int64_t > stridesInnermostFirst)
True when the pattern is fully constant, non-contiguous, currently illegal, and decomposeNdDmaPattern...
std::optional< uint32_t > getConstantIntOperand(mlir::Value v)
mlir::Value createConstantArgPlus(mlir::OpBuilder &builder, mlir::Location loc, uint64_t value)
void getHardwareStridesWraps(const AIE::AIETargetModel &targetModel, mlir::Operation *op, mlir::BaseMemRefType referencedBufType, llvm::SmallVector< int64_t, 4 > inputSizes, llvm::SmallVector< int64_t, 4 > inputStrides, llvm::SmallVector< int64_t, 4 > &sizes, llvm::SmallVector< int64_t, 4 > &strides)
bool isLinearModeTransfer(const AIE::AIETargetModel &targetModel, int col, int row, llvm::ArrayRef< int64_t > sizes, llvm::ArrayRef< int64_t > strides)
bool isConstMultipleOfGranule(int64_t value, uint64_t elemWidth, uint32_t addressGranularity)
bool isLinearTransfer(llvm::ArrayRef< int64_t > sizes, llvm::ArrayRef< int64_t > strides)
mlir::LogicalResult verifyConstBdRealizability(mlir::Operation *op, llvm::ArrayRef< mlir::OpFoldResult > sizes, llvm::ArrayRef< mlir::OpFoldResult > strides, uint64_t elemWidth, uint32_t gran)
mlir::Value createConstantI32(mlir::OpBuilder &builder, mlir::Location loc, uint32_t value)
void printTypedDynamicIndexList(mlir::OpAsmPrinter &printer, mlir::Operation *op, mlir::OperandRange values, llvm::ArrayRef< int64_t > integers, mlir::TypeRange types)
mlir::LogicalResult verifyLengthParameterTile(mlir::Operation *op, const AIETargetModel &targetModel, int col, int row)
mlir::LogicalResult verifyAssignedPacket(mlir::Operation *op, PacketInfoAttr packet, llvm::StringRef what="packet")
mlir::ParseResult parseTypedDynamicIndexList(mlir::OpAsmParser &parser, llvm::SmallVectorImpl< mlir::OpAsmParser::UnresolvedOperand > &values, mlir::DenseI64ArrayAttr &integers, llvm::SmallVectorImpl< mlir::Type > &types)
mlir::Operation * lookupNamedOpIn(mlir::Operation *symbolTableOp, mlir::StringAttr name)
mlir::LogicalResult verifyDMABDOutOfOrderId(DMABDOp bd, bool packetEnabledByContext=false)
mlir::LogicalResult verifyOutOfOrderChannel(mlir::Operation *op, DMAChannelDir dir, bool outOfOrder, llvm::ArrayRef< DMABDOp > bds, bool packetEnabledByContext=false)
mlir::LogicalResult verifyLengthParameter(mlir::Operation *op, std::optional< int64_t > lengthUnit, mlir::BaseMemRefType buffer, std::optional< int64_t > lenElems, bool contiguous, llvm::ArrayRef< int64_t > innerSizes)
mlir::LogicalResult verifyDMAChannelIndex(mlir::Operation *op, TileLike tile, DMAChannelDir dir, int64_t channel)
struct TileID { friend std::ostream &operator<<(std::ostream &os, const TileID &s) { os<< "TileID("<< s.col<< ", "<< s.row<< ")"; return os; } friend std::string to_string(const TileID &s) { std::ostringstream ss; ss<< s; return ss.str(); } friend llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const TileID &s) { os<< to_string(s); return os; } inline bool operator<(const TileID &rhs) const { return std::tie(col, row)< std::tie(rhs.col, rhs.row); } bool operator==(const TileID &rhs) const { return std::tie(col, row)==std::tie(rhs.col, rhs.row); } bool operator!=(const TileID &rhs) const { return !(*this==rhs);} int col, row;} TileID
const AIETargetModel & getTargetModel(mlir::Operation *op)
mlir::LogicalResult verifyDMABDIteration(DMABDOp bd, AIETileType tileType)
The register layout of one tile type's DMA buffer descriptor.
llvm::SmallVector< std::pair< DmaBdField, DmaBdField >, 3 > padFields() const
The (before, after) zero-padding fields, innermost dimension first.