MLIR-AIE 1.4.6.dev
AIEXDialect.cpp
Go to the documentation of this file.
1//===- AIEXDialect.cpp ------------------------------------------*- C++ -*-===//
2//
3// Copyright (C) 2019-2022 Xilinx, Inc.
4// Copyright (C) 2022-2026 Advanced Micro Devices, Inc.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
12
13#include "mlir/Dialect/Arith/IR/Arith.h"
14#include "mlir/Dialect/Func/IR/FuncOps.h"
15#include "mlir/IR/DialectImplementation.h"
16#include "mlir/IR/Matchers.h"
17#include "mlir/IR/Operation.h"
18#include "mlir/IR/RegionGraphTraits.h"
19#include "mlir/IR/SymbolTable.h"
20#include "mlir/IR/TypeUtilities.h"
21#include "mlir/Interfaces/DataLayoutInterfaces.h"
22#include "mlir/Interfaces/FoldInterfaces.h"
23#include "mlir/Transforms/InliningUtils.h"
24#include "llvm/ADT/SCCIterator.h"
25#include "llvm/ADT/STLExtras.h"
26
27#include "llvm/ADT/DenseMap.h"
28#include "llvm/ADT/TypeSwitch.h"
29#include "llvm/Support/CheckedArithmetic.h"
30#include "llvm/Support/MathExtras.h"
31#include "llvm/Support/TypeSize.h"
32
33#include <cstdint>
34#include <limits>
35#include <numeric>
36
37using namespace mlir;
38using namespace xilinx;
39
40#include "aie/Dialect/AIEX/IR/AIEXDialect.cpp.inc"
41
42#include "aie/Dialect/AIEX/IR/AIEXEnums.cpp.inc"
43
44#include "aie/Dialect/AIEX/IR/AIEXInterfaces.cpp.inc"
45
46#define GET_TYPEDEF_CLASSES
47#include "aie/Dialect/AIEX/IR/AIEXTypes.cpp.inc"
48
49#define GET_ATTRDEF_CLASSES
50#include "aie/Dialect/AIEX/IR/AIEXAttrDefs.cpp.inc"
51
52namespace xilinx::AIEX {
53
54// FIXME: use Tablegen'd dialect class
55void AIEXDialect::initialize() {
56 addOperations<
57#define GET_OP_LIST
58#include "aie/Dialect/AIEX/IR/AIEX.cpp.inc"
59 >();
60 addTypes<
61#define GET_TYPEDEF_LIST
62#include "aie/Dialect/AIEX/IR/AIEXTypes.cpp.inc"
63 >();
64 addAttributes<
65#define GET_ATTRDEF_LIST
66#include "aie/Dialect/AIEX/IR/AIEXAttrDefs.cpp.inc"
67 >();
68}
69
70} // namespace xilinx::AIEX
71
74
75#define GET_OP_CLASSES
76#include "aie/Dialect/AIEX/IR/AIEX.cpp.inc"
77
78/* Return the correct values to write to the hardware registers to configure
79 strides and wraps given the input user-facing strides and wraps.
80
81 In the IR, we express strides in units of element data type, but the hardware
82 requires it in units of address granularity. Address granularity currently is
83 4 bytes for all hardware.
84
85
86 User-facing strides/wraps relate to hardware as follows:
87
88 - By default, stride 0 and size 1 is assumed if unspecified.
89 - If only N strides/wraps are defined, those define the lowest N dimensions.
90
91 inputStride[3] == iteration_stride / elemSizeFac + 1
92 inputWrap[3] == iteration_size + 1
93 Highest-dimension stride/wrap is iteration count / iteration stride.
94 inputStride[2] == d2_stride / elemSizeFac + 1
95 Note: d2_size is not specified in hardware as it is
96 implicit from the total buffer transfer length
97 inputStride[1] == d1_stride / elemSizeFac + 1
98 inputSize[1] == d1_size
99 inputStride[0] == d0_stride / elemSizeFac + 1
100 inputSize[0] == d0_size / elemSizeFac
101
102 where elemSizeFac == bufferElementSize / addressGranularity
103 where bufferElementSize == size in bytes of elements in buffer,
104 e.g. 4 for int32
105 where addressGranularity == transfer granularity in hardware, which is
106 4 bytes for all current hardware
107
108 Note: strides are expressed offset by one from user input strides, because the
109 hardware does not support a 0 stride (repeat).
110 */
112 mlir::Operation *op,
113 mlir::BaseMemRefType referencedBufType,
114 llvm::SmallVector<int64_t, 4> inputSizes,
115 llvm::SmallVector<int64_t, 4> inputStrides,
116 llvm::SmallVector<int64_t, 4> &sizes,
117 llvm::SmallVector<int64_t, 4> &strides) {
118 assert(inputSizes.size() == inputStrides.size());
119 assert(sizes.size() == 4);
120 assert(strides.size() == 4);
121
122 DataLayout dataLayout = DataLayout::closest(op);
123 auto elemWidth =
124 dataLayout.getTypeSizeInBits(referencedBufType.getElementType());
125 auto addressGranularity = targetModel.getAddressGenGranularity();
126
127 // Output strides and sizes are default-initialized to 0
128 llvm::fill(sizes, 0);
129 llvm::fill(strides, 0);
130
131 if (inputSizes[0] == 0) {
132 // Illegal input, this won't transfer anything at all.
133 // Leave it to the verification functions to complain to the user.
134 return;
135 }
136
137 ConstStridePolicy policy;
138 int64_t inS[4] = {inputSizes[0], inputSizes[1], inputSizes[2], inputSizes[3]};
139 int64_t inT[4] = {inputStrides[0], inputStrides[1], inputStrides[2],
140 inputStrides[3]};
141 int64_t outS[4], outT[4];
142 encodeHardwareStridesWraps(policy, elemWidth, addressGranularity, inS, inT,
143 outS, outT);
144 for (int i = 0; i < 4; i++) {
145 sizes[i] = outS[i];
146 strides[i] = outT[i];
147 }
148}
149
150mlir::LogicalResult AIEX::verifyStridesWraps(
151 mlir::Operation *forOp,
152 llvm::function_ref<mlir::InFlightDiagnostic()> emitError,
153 mlir::BaseMemRefType referencedBufType, int tileCol, int tileRow,
154 llvm::SmallVector<int64_t, 4> inputSizes,
155 llvm::SmallVector<int64_t, 4> inputStrides,
156 llvm::SmallVector<int64_t, 4> hardwareSizes,
157 llvm::SmallVector<int64_t, 4> hardwareStrides,
158 bool skipTransformationChecks) {
159 const auto &targetModel = AIE::getTargetModel(forOp);
160 auto addressGranularity = targetModel.getAddressGenGranularity();
161 DataLayout dataLayout = DataLayout::closest(forOp);
162 auto elemWidth =
163 dataLayout.getTypeSizeInBits(referencedBufType.getElementType());
164
165 // ShimPLTiles have no ShimDMA, so BD field widths are meaningless here.
166 if (!targetModel.isCoreTile(tileCol, tileRow) &&
167 !targetModel.isMemTile(tileCol, tileRow) &&
168 !targetModel.isShimNOCTile(tileCol, tileRow))
169 return emitError() << "Unsupported tile type at (" << tileCol << ", "
170 << tileRow << ") Must be ShimNOC, Mem or Core.";
171
172 uint32_t wrap_bits = targetModel.getDmaBdWrapBits(tileCol, tileRow);
173 uint32_t step_bits = targetModel.getDmaBdStepBits(tileCol, tileRow);
174
175 for (int i = 0; i < 4; i++) {
176 if (inputSizes[i] <= 0) {
177 return emitError() << "Size " << i << " must be a positive integer.";
178 }
179 }
180
181 if (!isConstMultipleOfGranule(inputSizes[0], elemWidth, addressGranularity)) {
182 std::stringstream msg;
183 msg << "Transfer sizes must be multiples of " << (addressGranularity / 8)
184 << " bytes. " << inputSizes[0] << " elements at " << (elemWidth / 8)
185 << " bytes each equal " << (inputSizes[0] * elemWidth / 8)
186 << " bytes, which is not divisible by " << (addressGranularity / 8)
187 << ". ";
188 return emitError() << msg.str();
189 }
190
191 for (int i = 0; i < 3; i++) {
192 if (inputSizes[i] > 1 && inputStrides[i] < 1) {
193 // If inputSize[i] == 1, anything is allowable in the stride, since that
194 // stride will never be applied. For any larger size, we must verify that
195 // the stride is positive.
196 return emitError() << "Stride " << i << " must be a positive integer.";
197 }
198 }
199 // A value of zero is allowable for the fourth-dimension stride
200 // (this indicates an interation stride for the repeat of 0)
201 if (inputSizes[3] > 1 && inputStrides[3] < 0) {
202 return emitError() << "Stride 3 must be a non-negative integer.";
203 }
204
205 for (int i = 0; i < 4; i++) {
206 // strides[0] == 1 is ok iff the transfer size is a multiple of
207 // addressGranularity, which is checked below
208 if (i == 0 && inputStrides[i] == 1)
209 continue;
210 if (!isConstMultipleOfGranule(inputStrides[i], elemWidth,
211 addressGranularity)) {
212 std::stringstream msg;
213 msg << "Stride " << i << " is " << inputStrides[i] << " elements * "
214 << (elemWidth / 8) << " bytes = " << (inputStrides[i] * elemWidth / 8)
215 << " bytes, which is not divisible by " << (addressGranularity / 8)
216 << ". ";
217 return emitError() << msg.str();
218 }
219 }
220 // The innermost dimension steps whole granules, so for elements of another
221 // width it can only be contiguous.
222 if (elemWidth != addressGranularity && inputSizes[0] > 1 &&
223 inputStrides[0] != 1)
224 return emitError() << "Stride 0 is " << inputStrides[0]
225 << " elements, but must be 1 for " << (elemWidth / 8)
226 << "-byte elements: the DMA moves whole "
227 << (addressGranularity / 8) << "-byte words.";
228
229 auto outOfRange = [&](const char *what, int dim, int64_t written,
230 int64_t encoded, int64_t lo, int64_t hi) {
231 return emitError() << what << " " << dim << " is " << written
232 << " (encoded as " << encoded << "), which exceeds the ["
233 << lo << ":" << hi << "] range.";
234 };
235
236 int64_t maxWrap = (1 << wrap_bits) - 1;
237 int64_t maxStep = 1 << step_bits;
238
239 for (int dim : {0, 1}) {
240 if (!skipTransformationChecks && hardwareSizes[dim] > maxWrap)
241 return outOfRange("Size", dim, inputSizes[dim], hardwareSizes[dim], 0,
242 maxWrap);
243 }
244 // A zero d3 stride is a pure repeat: the lowerings leave the iteration fields
245 // 0 and carry the count on the queue push, so the repeat limit applies.
246 bool pureRepeat = inputStrides[3] == 0;
247 int64_t maxCount =
248 pureRepeat ? targetModel.getMaxRepeatCount() + 1
249 : 1LL << targetModel.getDmaBdIterBits(tileCol, tileRow);
250 if (inputSizes[3] > maxCount)
251 return emitError() << (pureRepeat ? "repeat count " : "iteration count ")
252 << inputSizes[3] << " exceeds the [1:" << maxCount
253 << "] range.";
254 for (int dim : {0, 1, 2}) {
255 if (hardwareStrides[dim] > maxStep - 1)
256 return outOfRange("Stride", dim, inputStrides[dim], hardwareStrides[dim],
257 1, maxStep);
258 }
259 // strides[3] exceeding the range is ok iff the sizes[3] is one, which is
260 // checked below
261 if (hardwareStrides[3] > maxStep - 1 && hardwareSizes[3] > 0)
262 return outOfRange("Stride", 3, inputStrides[3], hardwareStrides[3], 1,
263 maxStep);
264
265 return success();
266}
267
268//===----------------------------------------------------------------------===//
269// UseTokenOp
270//===----------------------------------------------------------------------===//
271
272// A field flagged as a register must name one of r0..r23. The bound cannot be
273// an attribute constraint, because the same field holds a full 32-bit
274// immediate when unflagged.
275LogicalResult AIEX::CertWrite32DOp::verify() {
276 if (getAddressIsReg() && getAddress() > 23)
277 return emitOpError("has address_is_reg, so address ")
278 << getAddress() << " must be a register index in [0, 23]";
279 if (getValueIsReg() && getValue() > 23)
280 return emitOpError("has value_is_reg, so value ")
281 << getValue() << " must be a register index in [0, 23]";
282 return success();
283}
284
285LogicalResult AIEX::CertApplyOffset57Op::verify() {
286 if (getNumEntries() == 0)
287 return emitOpError("num_entries must be at least 1");
288 return success();
289}
290
291LogicalResult AIEX::UseTokenOp::verify() {
292 auto *parentOp = (*this)->getParentOp();
293 if (isa<func::FuncOp>(parentOp) || isa<AIE::CoreOp>(parentOp) ||
294 isa<AIE::MemOp>(parentOp) || isa<AIE::ShimDMAOp>(parentOp))
295 return success();
296 return failure();
297}
298
299//===----------------------------------------------------------------------===//
300// MulticastOp
301//===----------------------------------------------------------------------===//
302
303LogicalResult AIEX::MulticastOp::verify() {
304 Region &body = getPorts();
305 assert(getOperation()->getNumRegions());
306 assert(!body.empty());
307 for (auto &ops : body.front())
308 if (!isa<MultiDestOp, AIE::EndOp>(ops))
309 return ops.emitOpError("cannot be contained in a Multicast op");
310
311 return success();
312}
313
314//===----------------------------------------------------------------------===//
315// BroadcastPacketOp
316//===----------------------------------------------------------------------===//
317
318LogicalResult AIEX::BroadcastPacketOp::verify() {
319 Region &body = getPorts();
320 assert(getOperation()->getNumRegions());
321 assert(!body.empty());
322 for (auto &ops : body.front())
323 if (!isa<BPIDOp, AIE::EndOp>(ops))
324 return ops.emitOpError("cannot be contained in a BroadcastPacket op");
325
326 return success();
327}
328
329//===----------------------------------------------------------------------===//
330// NpuDmaMemcpyNdOp
331//===----------------------------------------------------------------------===//
332
333/* Calculates the offset value to be written to the
334 */
335uint32_t AIEX::NpuDmaMemcpyNdOp::getAxcacheOrDefault() {
336 return getAxcache().value_or(AIE::getTargetModel(*this).getDefaultAxCache());
337}
338
339int64_t AIEX::NpuDmaMemcpyNdOp::getOffsetInBytes() {
340 llvm::SmallVector<int64_t, 4> offsets =
341 llvm::map_to_vector(llvm::reverse(getMixedOffsets()), [](OpFoldResult s) {
342 return getConstantIntValue(s).value();
343 });
344 auto strides = llvm::to_vector<4>(llvm::reverse(getMixedStrides()));
345 size_t offset = 0;
346 size_t R = offsets.size();
347 size_t el_bit_width = getElementTypeBitwidth();
348 assert(el_bit_width % 8 == 0 &&
349 "Expected Memref element bitwidth to be multiple of 8.");
350 size_t S = el_bit_width / 8;
351 // A dimension only contributes to the byte offset when its (constant) offset
352 // is non-zero; a runtime stride paired with a zero offset is fine and must
353 // not be forced to a constant. The verifier requires any stride multiplied by
354 // a non-zero offset to be constant.
355 for (size_t i = 0; i < R; i++) {
356 if (offsets[i] == 0)
357 continue;
358 auto strideConst = getConstantIntValue(strides[i]);
359 assert(strideConst &&
360 "verifier requires a stride paired with a non-zero offset to be "
361 "constant");
362 offset += offsets[i] * (*strideConst) * S;
363 }
364 return offset;
365}
366
367// Returns true when sizes/strides describe a plain contiguous transfer with
368// no data layout transformation (d1/d2 sizes == 1, d0 stride == 1).
369// d3 (repeat) is intentionally excluded.
370bool AIEX::isLinearTransfer(llvm::ArrayRef<int64_t> sizes,
371 llvm::ArrayRef<int64_t> strides) {
372 return sizes[1] == 1 && sizes[2] == 1 && strides[0] == 1 && strides[1] == 0 &&
373 strides[2] == 0;
374}
375
376// Returns true when sizes/strides (innermost-first) describe a contiguous
377// row-major scan: innermost stride == 1 and each outer stride equals the
378// product of all inner sizes. The repeat dimension (index 3) is excluded.
379// Size-1 dimensions are allowed to carry any stride value because that stride
380// is never applied during the transfer (the loop runs only once).
381// This is the vector-form counterpart of AIE::isContiguousBDTransfer.
382bool AIEX::isContiguousTransfer(llvm::ArrayRef<int64_t> sizes,
383 llvm::ArrayRef<int64_t> strides) {
384 if (strides[0] != 1)
385 return false;
386 if (sizes[1] > 1 && strides[1] != sizes[0])
387 return false;
388 if (sizes[2] > 1 && strides[2] != sizes[0] * sizes[1])
389 return false;
390 return true;
391}
392
393bool AIEX::isLinearModeTransfer(const AIE::AIETargetModel &targetModel, int col,
394 int row, llvm::ArrayRef<int64_t> sizes,
395 llvm::ArrayRef<int64_t> strides) {
396 return isLinearTransfer(sizes, strides) ||
397 (targetModel.isShimNOCTile(col, row) &&
398 isContiguousTransfer(sizes, strides));
399}
400
401// dma_memcpy_nd transfers of the form [*, 1, 1, len][*, 0, 0, 1] do not
402// specify any data layout transformation, but simply express a contiguous
403// transfer of `len`. The 4th dimension is excluded because a repeat count
404// is still compatible with a linear transfer.
405bool AIEX::NpuDmaMemcpyNdOp::isLinearTransferWithoutTransformation() {
406 llvm::SmallVector<int64_t, 4> inputSizes =
407 llvm::map_to_vector(llvm::reverse(getMixedSizes()), [](OpFoldResult s) {
408 return getConstantIntValue(s).value();
409 });
410 llvm::SmallVector<int64_t, 4> inputStrides =
411 llvm::map_to_vector(llvm::reverse(getMixedStrides()), [](OpFoldResult s) {
412 return getConstantIntValue(s).value();
413 });
414 return isLinearTransfer(inputSizes, inputStrides);
415}
416
417// Canonicalization pattern: rewrite a contiguous row-major access pattern to
418// the canonical linear form [s3, 1, 1, N][st3, 0, 0, 1].
419//
420// Using outermost-first index notation (matching the IR syntax), a 4D access
421// [s3, s2, s1, s0][st3, st2, st1, st0] is a contiguous linear scan when:
422// st0 == 1
423// s1 == 1 || st1 == s0 (stride irrelevant when size is 1)
424// s2 == 1 || st2 == s0 * s1
425// yielding a total of N = s0 * s1 * s2 contiguous elements. The repeat
426// dimension s3 / stride st3 is unchanged by the fold.
427//
428// Note: this pattern applies only to NpuDmaMemcpyNdOp. The analogous
429// dimensionsToStream / dimensionsFromStreamPerConsumer attributes on
430// ObjectFifoCreateOp are not canonicalized here; they are lowered separately
431// by the ObjectFifo stateful transform pass.
432//
433// This fold is always semantically valid and never introduces new hardware
434// limit violations: in the resulting linear form, isLinearTransferWithout-
435// Transformation() returns true, so verifyStridesWraps() skips the 10-bit
436// d0 wrap-size check. The hardware uses a wider buffer_length register in
437// linear mode (32-bit on shim tiles, 17-bit on mem tiles, 14-bit on core
438// tiles), so N can be much larger than the 10-bit ND wrap limit.
439namespace {
440struct LinearizeContiguousTransfer
441 : public mlir::OpRewritePattern<AIEX::NpuDmaMemcpyNdOp> {
442 using OpRewritePattern::OpRewritePattern;
443
444 mlir::LogicalResult
445 matchAndRewrite(AIEX::NpuDmaMemcpyNdOp op,
446 mlir::PatternRewriter &rewriter) const override {
447 // Only constant sizes/strides/offsets can be analysed statically.
448 if (!llvm::all_of(op.getMixedSizes(), [](mlir::OpFoldResult s) {
449 return mlir::getConstantIntValue(s).has_value();
450 }))
451 return mlir::failure();
452 if (!llvm::all_of(op.getMixedStrides(), [](mlir::OpFoldResult s) {
453 return mlir::getConstantIntValue(s).has_value();
454 }))
455 return mlir::failure();
456 if (!llvm::all_of(op.getMixedOffsets(), [](mlir::OpFoldResult s) {
457 return mlir::getConstantIntValue(s).has_value();
458 }))
459 return mlir::failure();
460
461 // Skip ops that are already in canonical linear form.
462 if (op.isLinearTransferWithoutTransformation())
463 return mlir::failure();
464
465 // getMixedSizes/Strides/Offsets return outermost-first; reverse to
466 // innermost-first so index 0 = d0 (innermost) and index 3 = repeat.
467 llvm::SmallVector<int64_t, 4> sizes = llvm::map_to_vector(
468 llvm::reverse(op.getMixedSizes()), [](mlir::OpFoldResult s) {
469 return mlir::getConstantIntValue(s).value();
470 });
471 llvm::SmallVector<int64_t, 4> strides = llvm::map_to_vector(
472 llvm::reverse(op.getMixedStrides()), [](mlir::OpFoldResult s) {
473 return mlir::getConstantIntValue(s).value();
474 });
475 llvm::SmallVector<int64_t, 4> offsets = llvm::map_to_vector(
476 llvm::reverse(op.getMixedOffsets()), [](mlir::OpFoldResult s) {
477 return mlir::getConstantIntValue(s).value();
478 });
479
480 // Require a contiguous row-major scan.
481 if (!AIEX::isContiguousTransfer(sizes, strides))
482 return mlir::failure();
483
484 // Fold d0/d1/d2 into one linear count; keep the repeat dimension intact.
485 // Build directly in outermost-first order for the replacement op.
486 int64_t N = sizes[0] * sizes[1] * sizes[2];
487 llvm::SmallVector<int64_t, 4> newSizesOuter = {sizes[3], 1, 1, N};
488 llvm::SmallVector<int64_t, 4> newStridesOuter = {strides[3], 0, 0, 1};
489
490 // getOffsetInBytes() computes: sum(offsets[i] * strides[i] * elemSize).
491 // After folding, the intermediate strides become 0, so any non-zero offset
492 // in those dimensions would silently contribute 0 bytes. Preserve the
493 // correct start address by collapsing the innermost three offset/stride
494 // pairs into a single linear element index at d0.
495 int64_t linearOffset = offsets[0] * strides[0] + offsets[1] * strides[1] +
496 offsets[2] * strides[2];
497 llvm::SmallVector<int64_t, 4> newOffsetsOuter = {offsets[3], 0, 0,
498 linearOffset};
499
500 rewriter.replaceOpWithNewOp<AIEX::NpuDmaMemcpyNdOp>(
501 op, op, newOffsetsOuter, newSizesOuter, newStridesOuter);
502 return mlir::success();
503 }
504};
505} // namespace
506
507void AIEX::NpuDmaMemcpyNdOp::getCanonicalizationPatterns(
508 mlir::RewritePatternSet &results, mlir::MLIRContext *context) {
509 results.add<LinearizeContiguousTransfer>(context);
510}
511
512// Helper method to check if a requested burst length is supported by the target
513// model. Returns an error message if the burst length is not supported or an
514// empty option otherwise.
515static std::optional<std::string>
516checkBurstLength(const xilinx::AIE::AIETargetModel &targetModel,
517 uint32_t requestedBurstLength) {
518 if (requestedBurstLength != 0) {
519 auto bel = targetModel.getShimBurstEncodingsAndLengths();
520 auto pair = llvm::find_if(bel, [=](const std::pair<uint32_t, uint32_t> &p) {
521 return p.second == requestedBurstLength;
522 });
523
524 if (pair == bel.end()) {
525 std::string errorMessage =
526 "Requested burst length is not supported by the target. "
527 "Supported burst lengths:";
528
529 errorMessage =
530 std::accumulate(bel.begin(), bel.end(), errorMessage,
531 [](const std::string &a, auto b) {
532 return a + " " + std::to_string(b.second);
533 });
534
535 return errorMessage;
536 }
537 }
538
539 return std::nullopt;
540}
541
542// Verify the supported scope for a dma_memcpy_nd carrying runtime (SSA)
543// offsets/sizes/strides, hard-erroring on any statically-provable violation.
544// The scope here is exactly what the dynamic BD encoder (AIEDmaToNpu.cpp
545// lowerDynamic) can lower; runtime values are never silently masked.
546LogicalResult AIEX::NpuDmaMemcpyNdOp::verifyDynamicSizesStrides(
547 const AIE::AIETargetModel &targetModel, mlir::BaseMemRefType buffer,
548 AIE::ShimDMAAllocationOp allocOp) {
549 // Shim NOC only.
550 if (!allocOp)
551 return emitOpError(
552 "runtime sizes/strides require a shim_dma_allocation to resolve the "
553 "tile; none found.");
554 AIE::TileOp tile = allocOp.getTileOp();
555 // Coordinate-dependent checks must wait until logical tiles are placed.
556 if (!tile)
557 return success();
558 if (!targetModel.isShimNOCTile(tile.getCol(), tile.getRow()))
559 return emitOpError(
560 "runtime sizes/strides are only supported for shim NOC tile DMAs.");
561
562 // No zero-padding with runtime sizes/strides.
563 if (getD0ZeroBefore() || getD1ZeroBefore() || getD2ZeroBefore() ||
564 getD0ZeroAfter() || getD1ZeroAfter() || getD2ZeroAfter())
565 return emitOpError(
566 "zero padding is not supported with runtime sizes/strides.");
567
568 // Per-field runtime values are allowed: any individual size/stride may be an
569 // SSA value while others stay constant. Only the runtime-dependent fields
570 // need runtime handling; the encoder produces the same word for a constant
571 // operand either way, so the static ≡ dynamic byte-identity holds field by
572 // field.
573 auto sizes = getMixedSizes();
574 auto strides = getMixedStrides();
575
576 // The innermost stride may be runtime like any other dimension: the encoder
577 // resolves its collapse-to-zero case with a select, and its realizability
578 // (unit stride, or granule-aligned) is enforced below for constants and by a
579 // host-side check (buildBdWords) for runtime values.
580
581 // (The memref must also trace to a runtime-sequence block argument through
582 // static subview/cast offsets; that structural check, with the same clean
583 // diagnostic, is enforced by the lowering in AIEDmaToNpu.cpp, which already
584 // owns the trace utility. It is not duplicated here to keep the dialect
585 // verifier free of the analysis-layer dependency.)
586
587 // A constant size must fit its hardware wrap field (an out-of-range constant
588 // is a hard error, never a silent truncation); runtime sizes are guarded at
589 // lowering. Bounds are checked on the ENCODED value, matching
590 // verifyStridesWraps: d0 wrap scaled to granules (size * elemWidth / gran),
591 // iteration wrap biased by -1, d1 a raw element count. Sizes outermost-first.
592 DataLayout dataLayout = DataLayout::closest(getOperation());
593 uint64_t elemWidth = dataLayout.getTypeSizeInBits(buffer.getElementType());
594 uint32_t gran = targetModel.getAddressGenGranularity();
595 llvm::SmallVector<mlir::OpFoldResult, 4> sizesRev(llvm::reverse(sizes));
596 auto checkSize = [&](mlir::OpFoldResult ofr, int64_t hwVal, int64_t hi,
597 llvm::StringRef what) -> LogicalResult {
598 if (!getConstantIntValue(ofr))
599 return success();
600 if (hwVal < 0 || hwVal > hi)
601 return emitOpError(what) << " hardware value " << hwVal
602 << " exceeds hardware range [0:" << hi << "].";
603 return success();
604 };
605 auto hwSize = [&](mlir::OpFoldResult ofr) {
606 return getConstantIntValue(ofr).value_or(0);
607 };
608 int64_t d0Hw = (int64_t)(hwSize(sizesRev[0]) * elemWidth / gran);
609 int64_t d1Hw = hwSize(sizesRev[1]);
610 int64_t iterRaw = hwSize(sizesRev[3]);
611 int64_t iterHw = iterRaw > 1 ? iterRaw - 1 : 0;
612 std::optional<int64_t> outerStride = getConstantIntValue(strides.front());
613 bool pureRepeat = outerStride && *outerStride == 0;
614 AIE::AIETileType tileType =
615 targetModel.getTileType(tile.getCol(), tile.getRow());
616 int64_t wrapMax = (1LL << targetModel.getDmaBdWrapBits(tileType)) - 1;
617 int64_t maxCount = pureRepeat ? targetModel.getMaxRepeatCount() + 1
618 : 1LL << targetModel.getDmaBdIterBits(tileType);
619 if (failed(checkSize(sizesRev[0], d0Hw, wrapMax, "d0 size")) ||
620 failed(checkSize(sizesRev[1], d1Hw, wrapMax, "d1 size")) ||
621 failed(checkSize(sizesRev[3], iterHw, maxCount - 1,
622 pureRepeat ? "repeat count" : "iteration size")))
623 return failure();
624
625 // Realizability of the CONSTANT size/stride operands (divisibility +
626 // positivity, innermost-first). Runtime operands get a host-side check at
627 // lowering time. Shared with the dma_task path.
628 llvm::SmallVector<mlir::OpFoldResult, 4> stridesRev(llvm::reverse(strides));
629 if (failed(verifyConstBdRealizability(getOperation(), sizesRev, stridesRev,
630 elemWidth, gran)))
631 return failure();
632
633 // Runtime fields are checked at dispatch instead; see buildBdWords.
634
635 auto errorMessage = checkBurstLength(targetModel, getBurstLength());
636 if (errorMessage.has_value())
637 return emitOpError(errorMessage.value());
638
639 return success();
640}
641
642LogicalResult AIEX::NpuDmaMemcpyNdOp::verify() {
643 BaseMemRefType buffer = getMemref().getType();
644 const auto &targetModel = AIE::getTargetModel(*this);
645 auto addressGranularity = targetModel.getAddressGenGranularity();
646
647 if (getOffsetParameterAttr() || getOffsetStateTableIdxAttr()) {
648 uint64_t elemBitWidth = buffer.getElementTypeBitWidth();
649 if (elemBitWidth == 0 || (elemBitWidth % 8) != 0)
650 return emitOpError("offset_parameter requires a whole-byte element type");
651 }
652
653 if (getElementTypeBitwidth() > addressGranularity) {
654 return emitOpError("Maximum element bit width allowed is ")
655 << addressGranularity << "bits. ";
656 }
657 if (buffer.hasStaticShape() &&
658 (buffer.getNumElements() * getElementTypeBitwidth()) <
659 addressGranularity) {
660 return emitOpError("Minimum data transfer size required is ")
661 << addressGranularity << "bits. ";
662 }
663 bool allStridesConstant = llvm::all_of(getMixedStrides(), [](OpFoldResult s) {
664 return getConstantIntValue(s).has_value();
665 });
666 bool allSizesConstant = llvm::all_of(getMixedSizes(), [](OpFoldResult s) {
667 return getConstantIntValue(s).has_value();
668 });
669 bool allOffsetsConstant = llvm::all_of(getMixedOffsets(), [](OpFoldResult s) {
670 return getConstantIntValue(s).has_value();
671 });
672
673 bool hasLengthParameter =
674 getLengthParameterAttr() || getLengthStateTableIdxAttr();
675 // Runtime offsets, sizes and strides lower only through EmitC, which has no
676 // scratchpad.
677 if (hasLengthParameter &&
678 (!allStridesConstant || !allSizesConstant || !allOffsetsConstant))
679 return emitOpError(
680 "length_parameter requires constant offsets, sizes and strides");
681
682 // The dynamic scope check needs the allocation resolved by verifySymbolUses.
683 if (!allStridesConstant || !allSizesConstant || !allOffsetsConstant)
684 return success();
685
686 llvm::SmallVector<int64_t, 4> inputSizes =
687 llvm::map_to_vector(llvm::reverse(getMixedSizes()), [](OpFoldResult s) {
688 return getConstantIntValue(s).value();
689 });
690 llvm::SmallVector<int64_t, 4> inputStrides =
691 llvm::map_to_vector(llvm::reverse(getMixedStrides()), [](OpFoldResult s) {
692 return getConstantIntValue(s).value();
693 });
694 int64_t offset = getOffsetInBytes();
695
696 auto errorMessage = checkBurstLength(targetModel, getBurstLength());
697 if (errorMessage.has_value()) {
698 return emitOpError(errorMessage.value());
699 }
700
701 if (hasLengthParameter) {
703 *this, getLengthUnit(), buffer,
704 inputSizes[0] * inputSizes[1] * inputSizes[2],
705 AIEX::isContiguousTransfer(inputSizes, inputStrides),
706 llvm::ArrayRef<int64_t>(inputSizes).take_front(3))))
707 return failure();
708 }
709
710 // AIE1 does not use the NPU alignment and packet-header checks.
711 if (targetModel.getTargetArch() == AIE::AIEArch::AIE1)
712 return success();
713
714 if (offset % 4 != 0) {
715 return emitOpError("Offset must be 4-byte-aligned.");
716 }
717
718 // packet header
719 if (auto packetInfo = getPacket()) {
720 if (failed(AIE::verifyAssignedPacket(*this, *packetInfo)))
721 return failure();
722 if (packetInfo->getPktType() > 7)
723 return emitOpError("Packet type field can only hold 3 bits.");
724 if (packetInfo->assignedId() > 31)
725 return emitOpError("Packet ID field can only hold 5 bits.");
726 }
727
728 return success();
729}
730
731static LogicalResult verifyTransferCoversWholeObjects(
732 Operation *transfer, SymbolRefAttr alloc, std::optional<int64_t> moved,
733 AIE::ObjectFifoCreateOp fifo, SymbolTableCollection &symbols);
734
735// The verifier-owned collection shares the device symbol table across memcpy
736// checks without retaining allocation pointers between verification calls.
737LogicalResult
738AIEX::NpuDmaMemcpyNdOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
739 BaseMemRefType buffer = getMemref().getType();
740 const auto &targetModel = AIE::getTargetModel(*this);
741 bool allStridesConstant = llvm::all_of(getMixedStrides(), [](OpFoldResult s) {
742 return getConstantIntValue(s).has_value();
743 });
744 bool allSizesConstant = llvm::all_of(getMixedSizes(), [](OpFoldResult s) {
745 return getConstantIntValue(s).has_value();
746 });
747 bool allOffsetsConstant = llvm::all_of(getMixedOffsets(), [](OpFoldResult s) {
748 return getConstantIntValue(s).has_value();
749 });
750 bool isDynamic =
751 !allStridesConstant || !allSizesConstant || !allOffsetsConstant;
752
753 bool hasLengthParameter =
754 getLengthParameterAttr() || getLengthStateTableIdxAttr();
755 bool isAIE1 = targetModel.getTargetArch() == AIE::AIEArch::AIE1;
756 AIE::DeviceOp dev = getOperation()->getParentOfType<AIE::DeviceOp>();
757 if (!dev && isAIE1 && !isDynamic && !hasLengthParameter)
758 return success();
759 if (dev)
760 if (auto fifo = symbolTable.lookupSymbolIn<AIE::ObjectFifoCreateOp>(
761 dev, getMetadata().getRootReference());
762 fifo &&
763 failed(verifyTransferCoversWholeObjects(
764 *this, getMetadata(), getStaticBytesStreamed(), fifo, symbolTable)))
765 return failure();
766 auto allocOp = symbolTable.lookupSymbolIn<AIE::ShimDMAAllocationOp>(
767 dev, getMetadata().getRootReference());
768 if (AIE::TileOp tile = allocOp ? allocOp.getTileOp() : nullptr) {
769 uint32_t numBds = targetModel.getNumBDs(tile.getCol(), tile.getRow());
770 if (getId() >= numBds)
771 return emitOpError("programs buffer descriptor ID ")
772 << getId() << ", but tile (" << tile.getCol() << ", "
773 << tile.getRow() << ") has " << numBds << " buffer descriptors";
774 }
775 if (isDynamic)
776 return verifyDynamicSizesStrides(targetModel, buffer, allocOp);
777
778 // Static transfers can precede allocation lowering, so defer tile checks
779 // until metadata names a shim DMA allocation.
780 if (!allocOp)
781 return success();
782 auto tile = allocOp.getTileOp();
783 // Defer coordinate-dependent checks until logical tiles are placed.
784 if (!tile) {
785 if (isAIE1 && hasLengthParameter)
786 return emitOpError("length_parameter is only supported on AIE2 and "
787 "AIE2P devices");
788 return success();
789 }
790 int col = tile.getCol();
791 int row = tile.getRow();
792 if (hasLengthParameter &&
793 failed(AIE::verifyLengthParameterTile(*this, targetModel, col, row)))
794 return failure();
795
796 llvm::SmallVector<int64_t, 4> inputSizes =
797 llvm::map_to_vector(llvm::reverse(getMixedSizes()), [](OpFoldResult s) {
798 return getConstantIntValue(s).value();
799 });
800 llvm::SmallVector<int64_t, 4> inputStrides =
801 llvm::map_to_vector(llvm::reverse(getMixedStrides()), [](OpFoldResult s) {
802 return getConstantIntValue(s).value();
803 });
804 llvm::SmallVector<int64_t, 4> hardwareSizes(4);
805 llvm::SmallVector<int64_t, 4> hardwareStrides(4);
806 getHardwareStridesWraps(targetModel, getOperation(), buffer, inputSizes,
807 inputStrides, hardwareSizes, hardwareStrides);
808
809 // dma_memcpy_nd transfers of the form [1, 1, 1, len][0, 0, 0, 1] do not
810 // specify any data layout transformation, but simply express a contiguous
811 // transfer of `len`. For backwards compatibility, we allow this to proceed
812 // even if it exceeds the maximum stride/wrap size of any one dimension,
813 // and simply do not lower any data layout transformations, since there is
814 // no other way to express this at the dma_memcpy_nd interface otherwise.
815 // A contiguous row-major ND access is also exempt from the ND wrap-size
816 // limit: aie-dma-to-npu lowers it to linear mode (d0_size=d1_size=0),
817 // and LinearizeContiguousTransfer canonicalizes it to explicit linear form.
818 bool skipTransformationChecks = AIEX::isLinearModeTransfer(
819 targetModel, col, row, inputSizes, inputStrides);
820 // An oversized non-contiguous pattern that aie-decompose-large-dma-bd can
821 // split into hardware-legal sub-transfers is also allowed to verify: the
822 // pass rewrites it before BD lowering. Truly undecomposable patterns (e.g.
823 // oversized strides) still fail below. isDecomposableNdDmaPattern
824 // suppresses its own diagnostics, so no stray error is emitted for the
825 // accepted case.
826 llvm::SmallVector<int64_t, 4> inputOffsets =
827 llvm::map_to_vector(llvm::reverse(getMixedOffsets()), [](OpFoldResult s) {
828 return getConstantIntValue(s).value();
829 });
830 // aie-decompose-large-dma-bd leaves a transfer with a runtime length
831 // alone, so it must be legal as written.
832 // AIE1 patterns must fit directly, even when the decomposition pass is
833 // scheduled: verification must not rely on it to legalize AIE1 transfers.
834 if (!isAIE1 && !hasLengthParameter &&
835 AIEX::isDecomposableNdDmaPattern(getOperation(), buffer, targetModel, col,
836 row, inputOffsets, inputSizes,
837 inputStrides)) {
838 // ok: will be decomposed before lowering
839 } else if (failed(verifyStridesWraps(
840 *this, [&] { return emitOpError(); }, buffer, col, row,
841 inputSizes, inputStrides, hardwareSizes, hardwareStrides,
842 skipTransformationChecks))) {
843 return failure();
844 }
845
846 return success();
847}
848
849//===----------------------------------------------------------------------===//
850// NpuDmaWaitOp
851//===----------------------------------------------------------------------===//
852
853LogicalResult AIEX::NpuDmaWaitOp::verify() {
854 AIE::DeviceOp dev = (*this)->getParentOfType<AIE::DeviceOp>();
855 // Some passes (e.g. aie-standard-lowering) use aiex ops outside a DeviceOp,
856 // so we can't expect the device to always exist.
857 if (dev && !dev.lookupSymbol(getSymbol()))
858 return emitOpError("couldn't find symbol in parent device");
859 return success();
860}
861
862//===----------------------------------------------------------------------===//
863// NpuPushQueueOp
864//===----------------------------------------------------------------------===//
865
866LogicalResult AIEX::NpuPushQueueOp::verify() {
867 const auto &targetModel = AIE::getTargetModel(*this);
868 auto numBds = targetModel.getNumBDs(getColumn(), getRow());
869 // bd_id and repeat_count are SSA operands; range-check them only when they
870 // are compile-time constants. A runtime repeat_count is checked on the host
871 // when lowered (AIEDmaToNpu); a runtime bd_id comes from the tile's BD pool.
872 if (std::optional<uint32_t> bdId = getConstantIntOperand(getBdId());
873 bdId && *bdId >= numBds)
874 return emitOpError("BD ID exceeds the maximum ID.");
875 if (std::optional<uint32_t> bdId = getConstantIntOperand(getBdId());
876 bdId && !targetModel.isBdChannelAccessible(getColumn(), getRow(), *bdId,
877 getChannel()))
878 return emitOpError("BD ID ") << *bdId << " is not accessible from channel ("
879 << AIE::stringifyDMAChannelDir(getDirection())
880 << ", " << getChannel() << ")";
881 uint32_t maxRepeat = targetModel.getMaxRepeatCount();
882 if (std::optional<uint64_t> repeatCount =
883 getConstantInt64Operand(getRepeatCount());
884 repeatCount && *repeatCount > maxRepeat)
885 return emitOpError("Repeat count exceeds the [0:")
886 << maxRepeat << "] range.";
887 if (getRepeatStateTableIdxAttr()) {
888 AIE::AIEArch arch = targetModel.getTargetArch();
889 if (arch != AIE::AIEArch::AIE2 && arch != AIE::AIEArch::AIE2p)
890 return emitOpError("repeat_parameter is only supported on AIE2 and "
891 "AIE2P devices");
892 if (!targetModel.isShimNOCTile(getColumn(), getRow()) &&
893 !targetModel.isMemTile(getColumn(), getRow()))
894 return emitOpError("repeat_parameter is only supported on shim NOC "
895 "and mem tiles, got tile (")
896 << getColumn() << ", " << getRow() << ")";
897 if (getConstantInt64Operand(getRepeatCount()) != uint64_t{0})
898 return emitOpError("takes its repeat count from the scratchpad, so its "
899 "repeat_count operand must be 0");
900 std::optional<uint32_t> bdId = getConstantIntOperand(getBdId());
901 if (!bdId)
902 return emitOpError("takes its repeat count from the scratchpad, so its "
903 "bd_id must be a constant");
904 if (*bdId % 4 != 0)
905 return emitOpError("takes its repeat count from the scratchpad, whose "
906 "update clears the low two bits of the queue word, "
907 "so its bd_id must be a multiple of 4, got ")
908 << *bdId;
909 }
910 return success();
911}
912
913//===----------------------------------------------------------------------===//
914// NpuWriteBdOp
915//===----------------------------------------------------------------------===//
916
917LogicalResult AIEX::NpuSyncOp::verify() {
918 for (auto [name, count] : {std::pair{"column_num", getColumnNum()},
919 std::pair{"row_num", getRowNum()}})
920 if (std::optional<int64_t> value = getConstantIntValue(count);
921 value && *value < 1)
922 return emitOpError() << name << " must cover at least one, got "
923 << *value;
924 return success();
925}
926
927LogicalResult AIEX::NpuWriteBdOp::verify() {
928 const auto &targetModel = AIE::getTargetModel(*this);
929 auto numBds = targetModel.getNumBDs(getColumn(), getRow());
930 bool isLinearTransfer =
931 (getD0Size() >= 1) && (getD1Size() == 1) && (getIterationSize() == 0);
932 if (getBdId() >= numBds)
933 return emitOpError("BD ID exceeds the maximum ID.");
934 if (getPacketId() > 31)
935 return emitOpError("Packet ID exceeds the maximum supported by 5 bits.");
936 if (getPacketType() > 7)
937 return emitOpError("Packet Type exceeds the maximum supported by 3 bits.");
938 int64_t oooId = getOutOfOrderId();
939 if (oooId < 0 ||
940 static_cast<uint64_t>(oooId) > targetModel.getMaxOutOfOrderId())
941 return emitOpError("out_of_order_id must be in [0, ")
942 << targetModel.getMaxOutOfOrderId() << "].";
943 if (oooId != 0 && getEnablePacket() == 0)
944 return emitOpError("out_of_order_id requires a packet-enabled BD");
945
946 // Every value on this op is already the hardware-encoded field value (wrap
947 // fields unbiased, stepsize/iteration fields biased actual-1), so the
948 // legal encoded range for a B-bit field is simply [0, 2^B - 1].
949 AIE::AIETileType tileType = targetModel.getTileType(getColumn(), getRow());
950 uint32_t wrapBits = targetModel.getDmaBdWrapBits(tileType);
951 uint32_t stepBits = targetModel.getDmaBdStepBits(tileType);
952 uint32_t iterBits = targetModel.getDmaBdIterBits(tileType);
953 uint32_t maxWrap = wrapBits > 0 ? (1u << wrapBits) - 1 : 0;
954 uint32_t maxStep = stepBits > 0 ? (1u << stepBits) - 1 : 0;
955 uint32_t maxIter = iterBits > 0 ? (1u << iterBits) - 1 : 0;
956
957 if (!isLinearTransfer && getD0Size() > maxWrap)
958 return emitOpError() << "D0 Size exceeds the [0:" << maxWrap << "] range.";
959 if (getD0Stride() > maxStep)
960 return emitOpError() << "D0 Stride exceeds the [0:" << maxStep
961 << "] range.";
962 if (getD1Size() > maxWrap)
963 return emitOpError() << "D1 Size exceeds the [0:" << maxWrap << "] range.";
964 if (getD1Stride() > maxStep)
965 return emitOpError() << "D1 Stride exceeds the [0:" << maxStep
966 << "] range.";
967 if (getD2Stride() > maxStep)
968 return emitOpError() << "D2 Stride exceeds the [0:" << maxStep
969 << "] range.";
970 if (getIterationSize() > maxIter)
971 return emitOpError() << "Iteration Size exceeds the [0:" << maxIter
972 << "] range.";
973 if (static_cast<uint32_t>(getIterationStride()) > maxStep)
974 return emitOpError() << "Iteration Stride exceeds the [0:" << maxStep
975 << "] range.";
976 // buffer_length is the full 32-bit word on a shim NOC tile but only 17 bits
977 // on a mem tile and 14 on a core tile, where the packer masks it.
978 uint64_t maxLen = targetModel.getDmaBdMaxLen(tileType);
979 if (getBufferLength() > maxLen)
980 return emitOpError() << "Buffer length exceeds the [0:" << maxLen
981 << "] range.";
982 if (targetModel.isShimNOCTile(getColumn(), getRow()) && getD2Size() != 0)
983 return emitOpError("ShimTile only supports 3 dimensions of sizes.");
984 // DMABDOp::verify() skips BDs inside aiex.dma_configure_task, so their pads
985 // are first checked here.
986 const AIE::DmaBdLayout *layout = targetModel.getDmaBdLayout(tileType);
987 llvm::SmallVector<std::pair<AIE::DmaBdField, AIE::DmaBdField>, 3> padFields;
988 if (layout)
989 padFields = layout->padFields();
990 uint64_t pads[3][2] = {{getD0ZeroBefore(), getD0ZeroAfter()},
991 {getD1ZeroBefore(), getD1ZeroAfter()},
992 {getD2ZeroBefore(), getD2ZeroAfter()}};
993 for (auto [d, pad] : llvm::enumerate(pads)) {
994 if (d >= padFields.size()) {
995 if (pad[0] || pad[1])
996 return emitOpError()
997 << "D" << d << " zero padding is not supported on this tile.";
998 continue;
999 }
1000 uint32_t maxBefore = padFields[d].first.mask();
1001 uint32_t maxAfter = padFields[d].second.mask();
1002 if (pad[0] > maxBefore || pad[1] > maxAfter)
1003 return emitOpError() << "D" << d
1004 << " pad_before/pad_after exceeds the [0:"
1005 << (pad[0] > maxBefore ? maxBefore : maxAfter)
1006 << "] range.";
1007 }
1008
1009 if (!targetModel.isShimNOCTile(getColumn(), getRow()) &&
1010 getBurstLength() != 0)
1011 return emitOpError("Only ShimTiles support burst length.");
1012 auto errorMessage = checkBurstLength(targetModel, getBurstLength());
1013 if (errorMessage.has_value()) {
1014 return emitOpError(errorMessage.value());
1015 }
1016 if (!targetModel.isShimNOCTile(getColumn(), getRow()) && getAxcache())
1017 return emitOpError("Only ShimTiles support AxCACHE configuration.");
1018
1019 return success();
1020}
1021
1022uint32_t AIEX::NpuWriteBdOp::getAxcacheOrDefault() {
1023 return getAxcache().value_or(AIE::getTargetModel(*this).getDefaultAxCache());
1024}
1025
1026std::optional<uint32_t> AIEX::getConstantIntOperand(mlir::Value v) {
1027 if (std::optional<uint64_t> cst = getConstantInt64Operand(v))
1028 return static_cast<uint32_t>(*cst);
1029 return std::nullopt;
1030}
1031
1032// Widthwise counterpart of getConstantIntOperand; see createConstantArgPlus.
1033std::optional<uint64_t> AIEX::getConstantInt64Operand(mlir::Value v) {
1034 // An arith.constant is read in place: the generic match folds the op, which
1035 // dominates translating a sequence of a million writes.
1036 if (auto constant = v.getDefiningOp<arith::ConstantOp>())
1037 if (auto attr = dyn_cast<IntegerAttr>(constant.getValue()))
1038 return attr.getValue().getZExtValue();
1039 mlir::APInt cst;
1040 if (!mlir::matchPattern(v, mlir::m_ConstantInt(&cst)))
1041 return std::nullopt;
1042 return cst.getZExtValue();
1043}
1044
1045mlir::Value AIEX::createConstantI32(mlir::OpBuilder &builder,
1046 mlir::Location loc, uint32_t value) {
1047 return arith::ConstantOp::create(
1048 builder, loc, builder.getI32IntegerAttr(static_cast<int32_t>(value)));
1049}
1050
1051mlir::Value AIEX::createConstantArgPlus(mlir::OpBuilder &builder,
1052 mlir::Location loc, uint64_t value) {
1053 if (value <= std::numeric_limits<uint32_t>::max())
1054 return createConstantI32(builder, loc, static_cast<uint32_t>(value));
1055 return arith::ConstantOp::create(
1056 builder, loc,
1057 IntegerAttr::get(builder.getIntegerType(64),
1058 static_cast<int64_t>(value)));
1059}
1060
1061//===----------------------------------------------------------------------===//
1062// Buffer-relative runtime writes
1063//===----------------------------------------------------------------------===//
1064
1065// DataLayout's default queries abort for types without a builtin layout or a
1066// type interface; a legal memref element (such as another memref) need not have
1067// either. Check component types before querying composite builtin layouts too.
1068static bool hasQueryableStorageLayout(Type type) {
1069 if (type.isIntOrFloat() || isa<IndexType, DataLayoutTypeInterface>(type))
1070 return true;
1071 if (auto vector = dyn_cast<VectorType>(type))
1072 return hasQueryableStorageLayout(vector.getElementType());
1073 if (auto complex = dyn_cast<ComplexType>(type))
1074 return hasQueryableStorageLayout(complex.getElementType());
1075 return false;
1076}
1077
1078// Match BufferOp's allocation convention without an unchecked shape product or
1079// a fixed-size conversion of scalable element storage.
1080static std::optional<uint64_t>
1081getStaticMemRefStorageBytes(MemRefType type, const DataLayout &layout) {
1082 if (!type || !type.hasStaticShape() ||
1083 !hasQueryableStorageLayout(type.getElementType()))
1084 return std::nullopt;
1085 if (auto vector = dyn_cast<VectorType>(type.getElementType())) {
1086 // The default vector layout uses a signed getNumElements(), then unsigned
1087 // innermost-dimension padding and bit products. Guard that query itself.
1088 if (vector.isScalable() || vector.getRank() == 0)
1089 return std::nullopt;
1090 int64_t elements = 1;
1091 for (int64_t dim : vector.getShape()) {
1092 auto product = llvm::checkedMul(elements, dim);
1093 if (!product)
1094 return std::nullopt;
1095 elements = *product;
1096 }
1097 uint64_t inner = vector.getShape().back();
1098 auto padded = llvm::checkedMulUnsigned(
1099 static_cast<uint64_t>(elements) / inner, llvm::PowerOf2Ceil(inner));
1100 llvm::TypeSize bits = layout.getTypeSizeInBits(vector.getElementType());
1101 if (!padded || bits.isScalable() ||
1102 !llvm::checkedMulUnsigned(*padded, bits.getFixedValue()))
1103 return std::nullopt;
1104 }
1105 llvm::TypeSize elementBytes = layout.getTypeSize(type.getElementType());
1106 if (elementBytes.isScalable())
1107 return std::nullopt;
1108 if (llvm::is_contained(type.getShape(), int64_t{0}))
1109 return 0;
1110 uint64_t bytes = elementBytes.getFixedValue();
1111 for (int64_t dim : type.getShape()) {
1112 auto product = llvm::checkedMulUnsigned(bytes, static_cast<uint64_t>(dim));
1113 if (!product)
1114 return std::nullopt;
1115 bytes = *product;
1116 }
1117 return bytes;
1118}
1119
1120template <typename OpTy>
1121static LogicalResult verifyBufferWriteScope(OpTy op) {
1122 if (op.getBuffer() && !op->template getParentOfType<AIE::DeviceOp>())
1123 return op.emitOpError("must be inside a device when buffer is present");
1124 return success();
1125}
1126
1127template <typename OpTy>
1128static LogicalResult
1129verifyBufferWrite(OpTy op, std::optional<uint32_t> wordOffset,
1130 uint64_t writeBytes, AIE::NamedOpTableCollection &names) {
1131 auto bufferAttr = op.getBufferAttr();
1132 if (!bufferAttr)
1133 return success();
1134 if (!wordOffset)
1135 return op.emitOpError("address must be constant when buffer is present");
1136 auto device = op->template getParentOfType<AIE::DeviceOp>();
1137 if (!device)
1138 return op.emitOpError("must be inside a device when buffer is present");
1139 // The device's region verifier owns the index and has verified all child
1140 // invariants before dispatching these graph-dependent checks.
1141 auto buffer = names.lookupIn<AIE::BufferOp>(device, bufferAttr.getAttr());
1142 if (!buffer)
1143 return op.emitOpError()
1144 << "buffer '" << bufferAttr.getValue() << "' not found in device";
1145 auto bufferBytes = getStaticMemRefStorageBytes(buffer.getType(),
1146 DataLayout::closest(buffer));
1147 if (!bufferBytes)
1148 return op.emitOpError(
1149 "referenced buffer must have a static, fixed-size storage extent "
1150 "representable in 64 bits");
1151
1152 uint64_t offsetBytes = uint64_t{*wordOffset} * sizeof(uint32_t);
1153 auto end = llvm::checkedAddUnsigned(offsetBytes, writeBytes);
1154 if (!end)
1155 return op.emitOpError("buffer-relative write range overflows 64-bit bytes");
1156 if (*end > *bufferBytes)
1157 return op.emitOpError() << "write range [" << offsetBytes << ", " << *end
1158 << ") exceeds buffer @" << bufferAttr.getValue()
1159 << " size " << *bufferBytes << " bytes";
1160
1161 auto tile = buffer.getTileLike();
1162 const auto &targetModel = device.getTargetModel();
1163 uint64_t memoryBytes = tile.isMemTile() ? targetModel.getMemTileSize()
1164 : targetModel.getLocalMemorySize();
1165 uint64_t start = offsetBytes;
1166 if (auto base = buffer.getAddress()) {
1167 if (*base < 0)
1168 return op.emitOpError("referenced buffer address must be nonnegative");
1169 start += static_cast<uint64_t>(*base);
1170 if (start % sizeof(uint32_t) != 0)
1171 return op.emitOpError() << "resolved buffer write address " << start
1172 << " is not 4-byte aligned";
1173 }
1174 // An unassigned base can only move the interval upward. Check this necessary
1175 // bound now, and the resolved interval when address assignment runs.
1176 auto memoryEnd = llvm::checkedAddUnsigned(start, writeBytes);
1177 if (!memoryEnd)
1178 return op.emitOpError("resolved buffer write range overflows 64-bit bytes");
1179 if (*memoryEnd > memoryBytes)
1180 return op.emitOpError()
1181 << "write range [" << start << ", " << *memoryEnd
1182 << ") exceeds tile data memory size " << memoryBytes << " bytes";
1183 return success();
1184}
1185
1186//===----------------------------------------------------------------------===//
1187// NpuWrite32Op
1188//===----------------------------------------------------------------------===//
1189
1190LogicalResult AIEX::NpuWrite32Op::verify() {
1191 return verifyBufferWriteScope(*this);
1192}
1193
1194LogicalResult
1195AIEX::NpuWrite32Op::verifyNamedOpUses(AIE::NamedOpTableCollection &names) {
1196 if (!getBuffer())
1197 return success();
1198 return verifyBufferWrite(*this, getConstantIntOperand(getAddress()),
1199 sizeof(uint32_t), names);
1200}
1201
1202template <typename T>
1203static std::optional<uint32_t>
1204getAbsoluteAddress(T *op, uint32_t addressOffset,
1205 const AIE::NamedOpTable *names) {
1206 AIE::DeviceOp device =
1207 op->getOperation()->template getParentOfType<AIE::DeviceOp>();
1208 if (!device) {
1209 op->emitError("Must be inside a device.");
1210 return std::nullopt;
1211 }
1212 const AIE::AIETargetModel &tm = device.getTargetModel();
1213
1214 uint32_t address = 0;
1215
1216 // If blockwrite references a buffer, the given address is understood to be
1217 // relative to the buffer's start address.
1218 if (auto bufferSym = op->getBuffer()) {
1219 AIE::BufferOp buffer =
1220 names ? names->lookup<AIE::BufferOp>(op->getBufferAttr().getAttr())
1221 : AIE::lookupNamedOpIn<AIE::BufferOp>(device, *bufferSym);
1222 if (!buffer) {
1223 op->emitError() << "buffer '" << *bufferSym << "' not found in device";
1224 return std::nullopt;
1225 }
1226
1227 auto bufferAddress = buffer.getAddress();
1228 if (!bufferAddress) {
1229 mlir::InFlightDiagnostic err =
1230 op->emitError("referenced buffer must have address assigned");
1231 err.attachNote(buffer.getLoc()) << "This buffer must have an address.";
1232 return std::nullopt;
1233 }
1234
1235 uint32_t col = buffer.getTileOp().getCol();
1236 uint32_t row = buffer.getTileOp().getRow();
1237 address = static_cast<uint32_t>(*bufferAddress) +
1238 addressOffset * sizeof(uint32_t);
1239 address = ((col & 0xff) << tm.getColumnShift()) |
1240 ((row & 0xff) << tm.getRowShift()) | (address & 0xfffff);
1241 } else { // otherwise, the given address is absolute
1242 address = addressOffset;
1243 std::optional<uint32_t> col = op->getColumn();
1244 std::optional<uint32_t> row = op->getRow();
1245 if (col && row) {
1246 // If col and row are set, only the lower 20 bits of the address are
1247 // used, and col and row dictate the upper bits (ignored)
1248 address = ((*col & 0xff) << tm.getColumnShift()) |
1249 ((*row & 0xff) << tm.getRowShift()) | (address & 0xfffff);
1250 }
1251 }
1252
1253 return address;
1254}
1255
1256std::optional<uint32_t>
1257AIEX::NpuWrite32Op::getAbsoluteAddress(const AIE::NamedOpTable *names) {
1258 std::optional<uint32_t> addressOffset = getConstantIntOperand(getAddress());
1259 if (!addressOffset)
1260 return std::nullopt;
1261 return ::getAbsoluteAddress(this, *addressOffset, names);
1262}
1263
1264//===----------------------------------------------------------------------===//
1265// NpuAddressPatchOp
1266//===----------------------------------------------------------------------===//
1267
1268LogicalResult AIEX::NpuAddressPatchOp::verify() {
1269 // A runtime register address (addr_val) is meaningful only on the EmitC (C++
1270 // TXN) target; the constant `addr` still carries the fallback / static value.
1271 // The static binary target checks for the operand and diagnoses it there.
1272 if (getAddrVal() && !getAddrVal().getType().isInteger(32))
1273 return emitOpError("addr_val must be an i32 value");
1274 if (getArgIdx().has_value() == static_cast<bool>(getBuffer()))
1275 return emitOpError("must name the host buffer either by the 'arg_idx' "
1276 "attribute or by the 'buffer' operand, not both and not "
1277 "neither");
1278 return success();
1279}
1280
1281//===----------------------------------------------------------------------===//
1282// NpuUpdateFromScratchpadOp
1283//===----------------------------------------------------------------------===//
1284
1285std::optional<uint32_t> AIEX::NpuUpdateFromScratchpadOp::getAbsoluteAddress(
1286 const AIE::NamedOpTable *names) {
1287 return ::getAbsoluteAddress(this, getAddress(), names);
1288}
1289
1290LogicalResult AIEX::NpuUpdateFromScratchpadOp::verify() {
1291 // StateTable has at most 32 entries (32-bit words).
1292 constexpr uint32_t kMaxStateTableEntries = 32;
1293 if (getStateTableIdx() >= kMaxStateTableEntries)
1294 return emitOpError("state_table_idx ")
1295 << static_cast<uint32_t>(getStateTableIdx())
1296 << " exceeds maximum StateTable index ("
1297 << (kMaxStateTableEntries - 1) << ").";
1298
1299 // NpuCreateScratchpadOp::verify() bounds the state_table_idx.
1300 return success();
1301}
1302
1303//===----------------------------------------------------------------------===//
1304// NpuCreateScratchpadOp
1305//===----------------------------------------------------------------------===//
1306
1307LogicalResult AIEX::NpuCreateScratchpadOp::verify() {
1308 // Only usage_type == 0 is currently supported by firmware.
1309 if (getUsageType() != 0) {
1310 return emitOpError("usage_type must be 0 (got ")
1311 << static_cast<uint32_t>(getUsageType())
1312 << "); other layouts are not supported.";
1313 }
1314
1315 // The StateTable layout (usage_type 0) has a max of 32 32-bit-word entries,
1316 // i.e. a maximum total scratchpad size of 128 bytes.
1317 constexpr uint32_t kMaxScratchpadSizeBytes = 128;
1318 if (getSize() == 0) {
1319 return emitOpError("size must be greater than 0.");
1320 }
1321 if (getSize() % 4 != 0) {
1322 return emitOpError("size (")
1323 << getSize() << ") must be a multiple of 4 bytes.";
1324 }
1325 if (getSize() > kMaxScratchpadSizeBytes) {
1326 return emitOpError("size (")
1327 << getSize() << " bytes) exceeds maximum scratchpad size of "
1328 << kMaxScratchpadSizeBytes << " bytes.";
1329 }
1330
1331 // Scratchpads are far rarer than the updates they bound, and an op verifier
1332 // runs after every pass, so the block scan belongs on this side.
1333 uint32_t numEntries = getSize() / 4;
1334 if (Block *block = (*this)->getBlock()) {
1335 for (auto updateOp : block->getOps<AIEX::NpuUpdateFromScratchpadOp>()) {
1336 if (updateOp.getStateTableIdx() < numEntries)
1337 continue;
1338 return updateOp.emitOpError("state_table_idx ")
1339 << static_cast<uint32_t>(updateOp.getStateTableIdx())
1340 << " is out of bounds for scratchpad of size " << getSize()
1341 << " bytes (" << numEntries << " entries) created by "
1342 << (*this)->getName() << ".";
1343 }
1344 }
1345
1346 // At most one create_scratchpad may appear per runtime sequence. Walk the
1347 // parent RuntimeSequenceOp to check; only report from the duplicate (i.e.
1348 // the op that is NOT the first occurrence) to avoid emitting the same error
1349 // twice.
1350 auto runtimeSeq = getOperation()->getParentOfType<AIE::RuntimeSequenceOp>();
1351 if (!runtimeSeq) {
1352 return success();
1353 }
1354
1355 NpuCreateScratchpadOp firstSeen;
1356 runtimeSeq.walk([&](NpuCreateScratchpadOp op) {
1357 firstSeen = op;
1358 return WalkResult::interrupt();
1359 });
1360 if (firstSeen != *this) {
1361 InFlightDiagnostic diag =
1362 emitOpError("only one 'aiex.npu.create_scratchpad' is allowed per "
1363 "runtime sequence");
1364 diag.attachNote(firstSeen.getLoc())
1365 << "previous 'aiex.npu.create_scratchpad' here";
1366 return diag;
1367 }
1368
1369 return success();
1370}
1371
1372//===----------------------------------------------------------------------===//
1373// NpuMaskWrite32Op
1374//===----------------------------------------------------------------------===//
1375
1376LogicalResult AIEX::NpuMaskWrite32Op::verify() {
1377 return verifyBufferWriteScope(*this);
1378}
1379
1380LogicalResult
1381AIEX::NpuMaskWrite32Op::verifyNamedOpUses(AIE::NamedOpTableCollection &names) {
1382 if (!getBuffer())
1383 return success();
1384 return verifyBufferWrite(*this, getConstantIntOperand(getAddress()),
1385 sizeof(uint32_t), names);
1386}
1387
1388std::optional<uint32_t>
1389AIEX::NpuMaskWrite32Op::getAbsoluteAddress(const AIE::NamedOpTable *names) {
1390 std::optional<uint32_t> addressOffset = getConstantIntOperand(getAddress());
1391 if (!addressOffset)
1392 return std::nullopt;
1393 return ::getAbsoluteAddress(this, *addressOffset, names);
1394}
1395
1396//===----------------------------------------------------------------------===//
1397// NpuMaskPollOp
1398//===----------------------------------------------------------------------===//
1399
1400LogicalResult AIEX::NpuMaskPollOp::verify() {
1401 return verifyBufferWriteScope(*this);
1402}
1403
1404LogicalResult
1405AIEX::NpuMaskPollOp::verifyNamedOpUses(AIE::NamedOpTableCollection &names) {
1406 if (!getBuffer())
1407 return success();
1408 return verifyBufferWrite(*this, getConstantIntOperand(getAddress()),
1409 sizeof(uint32_t), names);
1410}
1411
1412std::optional<uint32_t>
1413AIEX::NpuMaskPollOp::getAbsoluteAddress(const AIE::NamedOpTable *names) {
1414 std::optional<uint32_t> addressOffset = getConstantIntOperand(getAddress());
1415 if (!addressOffset)
1416 return std::nullopt;
1417 return ::getAbsoluteAddress(this, *addressOffset, names);
1418}
1419
1420//===----------------------------------------------------------------------===//
1421// NpuBlockWriteOp
1422//===----------------------------------------------------------------------===//
1423
1424LogicalResult AIEX::NpuBlockWriteOp::verify() {
1425 return verifyBufferWriteScope(*this);
1426}
1427
1428LogicalResult
1429AIEX::NpuBlockWriteOp::verifyNamedOpUses(AIE::NamedOpTableCollection &names) {
1430 if (!getBuffer())
1431 return success();
1432 auto type = dyn_cast<MemRefType>(getData().getType());
1433 DataLayout layout = DataLayout::closest(*this);
1434 auto bytes = getStaticMemRefStorageBytes(type, layout);
1435 if (!bytes)
1436 return emitOpError("payload must have a static, fixed-size storage extent "
1437 "representable in 64 bits when buffer is present");
1438 llvm::TypeSize elementBits = layout.getTypeSizeInBits(type.getElementType());
1439 if (elementBits.isScalable() || elementBits.getFixedValue() != 32)
1440 return emitOpError(
1441 "payload must have 32-bit elements when buffer is present");
1442 return verifyBufferWrite(*this, getAddress(), *bytes, names);
1443}
1444
1445std::optional<uint32_t>
1446AIEX::NpuBlockWriteOp::getAbsoluteAddress(const AIE::NamedOpTable *names) {
1447 return ::getAbsoluteAddress(this, getAddress(), names);
1448}
1449
1450DenseIntElementsAttr AIEX::NpuBlockWriteOp::getDataWords() {
1451 Value memref = this->getData();
1452 DataLayout dataLayout = DataLayout::closest(*this);
1453 int64_t width = dataLayout.getTypeSizeInBits(
1454 cast<MemRefType>(memref.getType()).getElementType());
1455 if (width != 32) {
1456 emitWarning("Only 32-bit data type is supported for now");
1457 return nullptr;
1458 }
1459
1460 memref::GetGlobalOp getGlobal = memref.getDefiningOp<memref::GetGlobalOp>();
1461 if (!getGlobal) {
1462 emitError("Only MemRefs from memref.get_global are supported");
1463 return nullptr;
1464 }
1465
1466 auto global = dyn_cast_if_present<memref::GlobalOp>(
1467 (*this)->getParentOfType<AIE::DeviceOp>().lookupSymbol(
1468 getGlobal.getName()));
1469 if (!global) {
1470 emitError("Global symbol not found");
1471 return nullptr;
1472 }
1473
1474 auto initVal = global.getInitialValue();
1475 if (!initVal) {
1476 emitError("Global symbol has no initial value");
1477 return nullptr;
1478 }
1479
1480 auto data = dyn_cast<DenseIntElementsAttr>(*initVal);
1481 if (!data) {
1482 emitError("Global symbol initial value is not a dense int array");
1483 return nullptr;
1484 }
1485
1486 return data;
1487}
1488
1489//===----------------------------------------------------------------------===//
1490// DMAConfigureTaskOp
1491//===----------------------------------------------------------------------===//
1492
1493AIE::DMABDOp AIEX::DMAConfigureTaskOp::getFirstBd() {
1494 Region &body = getBody();
1495 if (body.empty()) {
1496 return nullptr;
1497 }
1498 auto bd_ops = body.front().getOps<AIE::DMABDOp>();
1499 if (bd_ops.empty() && body.front().getNumSuccessors() == 1) {
1500 // Allow the first block to be empty and point to the entry point of the
1501 // chain. This allows for specifying cyclying BD chains (infinite loops)
1502 // within the constraints of MLIR syntax.
1503 Block &chain_entry = *body.front().getSuccessor(0);
1504 bd_ops = chain_entry.getOps<AIE::DMABDOp>();
1505 }
1506 if (bd_ops.empty()) {
1507 return nullptr;
1508 }
1509 return *bd_ops.begin();
1510}
1511
1512LogicalResult
1513AIEX::DMAConfigureTaskOp::canonicalize(AIEX::DMAConfigureTaskOp op,
1514 PatternRewriter &rewriter) {
1515 // Remove blocks that contain nothing but a terminator
1516 Region &body = op.getBody();
1517 bool did_rewrite = false;
1518 for (auto &block : body) {
1519 if (block.empty()) {
1520 continue;
1521 }
1522 auto ops_it = block.without_terminator();
1523 if (std::distance(ops_it.begin(), ops_it.end()) == 0) {
1524 rewriter.eraseOp(block.getTerminator());
1525 did_rewrite = true;
1526 }
1527 }
1528 if (did_rewrite) {
1529 return success();
1530 }
1531 return failure();
1532}
1533
1534// Enforce the per-BD ND access-pattern limit for BDs nested inside a
1535// runtime-sequence DMA task. The AIE::DMABDOp verifier skips these BDs (their
1536// parent is a DMA task op, not a *DMAOp), so this is the only check of the BD
1537// dimension count on the runtime-sequence path.
1538//
1539// Every AIE2/AIE2P DMA BD register file carries getBDMaxDims ND address
1540// dimensions (D0..) plus one separate iteration/repeat dimension: a core/shim
1541// BD has D0..D2 + iteration, a MemTile BD has D0..D3 + iteration. On this path
1542// aiex.shim_dma_single_bd_task hoists the leading tap dimension into that
1543// iteration register, so a shim/core BD may carry one dimension beyond its ND
1544// access limit (3 + 1). A MemTile is not given the +1: AIEDMATasksToNPU maps
1545// the 4th task dimension onto the iteration register for every tile type and
1546// caps the total at 4, which the MemTile's 4 ND dimensions already reach. Both
1547// branches therefore land on the same uniform 4-dimension cap enforced later by
1548// AIEDMATasksToNPU.
1549//
1550// A BD whose sizes and strides are all constant may give more: every dimension
1551// past the third is an iteration dimension, and aie-decompose-large-dma-bd
1552// splits them off into descriptors of 4. With a runtime value among them it
1553// cannot, so the cap applies here.
1554//
1555// A BD's `iteration` attribute claims that iteration register itself, so its
1556// sizes/strides must leave it free: one dimension fewer, with no splitting.
1557static LogicalResult
1558verifyTaskBDDimensions(const AIE::AIETargetModel &targetModel, int col, int row,
1559 Region &body) {
1560 size_t maxNDims = targetModel.getBDMaxDims(col, row);
1561 if (!targetModel.isMemTile(col, row))
1562 ++maxNDims; // leading dim is hoisted into the iteration/repeat register
1563 LogicalResult result = success();
1564 body.walk([&](AIE::DMABDOp bd) {
1565 // The BD's own verifier skips it here, so nothing has yet established that
1566 // its mixed sizes/strides lists are safe to read.
1567 if (failed(bd.verifyMixedSizesAndStrides())) {
1568 result = failure();
1569 return;
1570 }
1571 if (failed(
1572 AIE::verifyDMABDIteration(bd, targetModel.getTileType(col, row)))) {
1573 result = failure();
1574 return;
1575 }
1576 size_t numDims = bd.getMixedSizes().size();
1577 if (bd.getIteration()) {
1578 if (numDims + 1 > maxNDims) {
1579 bd.emitOpError() << "Cannot give more than "
1580 << std::to_string(maxNDims - 1)
1581 << " dimensions for step sizes and wraps alongside "
1582 "the iteration attribute on this tile (got "
1583 << std::to_string(numDims) << " dimensions).";
1584 result = failure();
1585 }
1586 return;
1587 }
1588 auto isConstant = [](OpFoldResult v) {
1589 return getConstantIntValue(v).has_value();
1590 };
1591 if (numDims > maxNDims && llvm::all_of(bd.getMixedSizes(), isConstant) &&
1592 llvm::all_of(bd.getMixedStrides(), isConstant))
1593 return;
1594 if (numDims > maxNDims) {
1595 bd.emitOpError() << "Cannot give more than " << std::to_string(maxNDims)
1596 << " dimensions for step sizes and wraps on this tile "
1597 "(got "
1598 << std::to_string(numDims) << " dimensions).";
1599 result = failure();
1600 }
1601 });
1602 return result;
1603}
1604
1605LogicalResult AIEX::DMAConfigureTaskOp::verify() {
1606 if (failed(AIE::verifyDMAChannelIndex(getOperation(), getTileLike(),
1607 getDirection(), getChannel())))
1608 return failure();
1609 const AIE::AIETargetModel &targetModel = AIE::getTargetModel(getOperation());
1610 // Skip the per-BD dimension check on an unplaced (logical) tile: the ND limit
1611 // is a function of the tile's placed coordinates, which are not yet known.
1612 // The verifier runs again on the concrete tile once placement resolves it.
1613 std::optional<int> col = getTileLike().tryGetCol();
1614 std::optional<int> row = getTileLike().tryGetRow();
1615 if (col && row &&
1616 failed(verifyTaskBDDimensions(targetModel, *col, *row, getBody())))
1617 return failure();
1618 Region &body = getBody();
1619 // This is a layering violation on the DMABDOps, but they are never verified
1620 // otherwise Because DMAConfigureTaskOps are not yet merged into the AIE
1621 // dialect. The normal DMABDOp verify operation will skip over any BD inside
1622 // a DMAConfigureTaskOp
1623 if (failed(AIE::verifyAssignedPacket(*this, getPacketAttr())))
1624 return failure();
1625 LogicalResult result = success();
1626 bool taskHasPacket = getPacket().has_value();
1627 llvm::SmallVector<AIE::DMABDOp> bds;
1628 for (auto &block : body) {
1629 if (block.empty()) {
1630 continue;
1631 }
1632 if (block.hasNoPredecessors() && !block.isEntryBlock()) {
1633 auto error = block.getTerminator()->emitError(
1634 "Block ending in this terminator does not form a chain with "
1635 "entry block.");
1636 return failure();
1637 }
1638
1639 const AIE::AIETargetModel &targetModel =
1640 AIE::getTargetModel(getOperation());
1641
1642 block.walk([&](AIE::DMABDOp bd) {
1643 if (bd.getBurstLength() != 0 &&
1644 !targetModel.isShimNOCTile(getTileID().col, getTileID().row)) {
1645 bd.emitOpError("Burst length is only supported in Shim NOC tiles that "
1646 "are connected to the memory-mapped NOC.");
1647 result = failure();
1648 }
1649 // DMABDOp::verify skips task BDs, so validate their packet header and
1650 // out_of_order_id here too.
1651 if (failed(AIE::verifyAssignedPacket(
1652 bd, bd.getPacket().value_or(AIE::PacketInfoAttr()))) ||
1653 failed(AIE::verifyDMABDOutOfOrderId(bd, taskHasPacket)) ||
1654 failed(bd.verifyTransferInBounds()))
1655 result = failure();
1656 bds.push_back(bd);
1657 });
1658 }
1659 // Only a memtile's low-numbered channels reach its neighbors' buffers and
1660 // locks; the rest are local only, as in aie.memtile_dma.
1661 bool onMemTile = col && row && targetModel.isMemTile(*col, *row);
1662 uint32_t adjacentChannels =
1663 onMemTile ? targetModel.getMaxChannelNumForAdjacentMemTile(*col, *row)
1664 : 0;
1665 if (onMemTile && static_cast<uint32_t>(getChannel()) >= adjacentChannels) {
1666 auto isNonLocal = [&](AIE::TileLike tile) {
1667 // Pinned logical and physical tiles can denote the same location. Defer
1668 // the check until placement if the resource's coordinates are unknown.
1669 auto resourceCol = tile.tryGetCol();
1670 auto resourceRow = tile.tryGetRow();
1671 return resourceCol && resourceRow &&
1672 (*resourceCol != *col || *resourceRow != *row);
1673 };
1674 body.walk([&](Operation *op) {
1675 Operation *resource = nullptr;
1676 AIE::TileLike resourceTile;
1677 StringRef what;
1678 if (auto bd = dyn_cast<AIE::DMABDOp>(op)) {
1679 auto buffer = bd.getBuffer().getDefiningOp<AIE::BufferOp>();
1680 if (!buffer)
1681 return;
1682 resource = buffer;
1683 resourceTile = buffer.getTileLike();
1684 what = "buffer";
1685 } else if (auto useLock = dyn_cast<AIE::UseLockOp>(op)) {
1686 auto lock = useLock.getLock().getDefiningOp<AIE::LockOp>();
1687 if (!lock) {
1688 useLock.emitOpError()
1689 << "requires a lock defined by aie.lock in a task on memtile DMA "
1690 "channel "
1691 << getChannel();
1692 result = failure();
1693 return;
1694 }
1695 resource = lock;
1696 resourceTile = lock.getTileLike();
1697 what = "lock";
1698 } else {
1699 return;
1700 }
1701 // aie.buffer and aie.lock reject a tile operand that is not a tile, but
1702 // they may be verified after this task: they can follow the runtime
1703 // sequence, and a buffer can sit inside the task body.
1704 if (!resourceTile) {
1705 op->emitOpError() << "uses a " << what
1706 << " whose tile operand is not a tile";
1707 result = failure();
1708 return;
1709 }
1710 if (!isNonLocal(resourceTile))
1711 return;
1712 InFlightDiagnostic err = op->emitOpError()
1713 << "is in a task on memtile DMA channel "
1714 << getChannel()
1715 << " and attempts to access a non-local " << what
1716 << "; only channels below " << adjacentChannels
1717 << " reach neighboring tiles";
1718 err.attachNote(resource->getLoc()) << what;
1719 result = failure();
1720 });
1721 }
1722 if (getOutOfOrder()) {
1723 // Out-of-order mode rejects task completion token (bits are aliased).
1724 if (getIssueToken()) {
1725 auto err =
1726 emitOpError("out_of_order channel cannot issue a completion token");
1727 err.attachNote() << "set issue_token = false on this out-of-order task";
1728 result = failure();
1729 }
1731 getOperation(), getDirection(), getOutOfOrder(),
1732 llvm::ArrayRef<AIE::DMABDOp>(bds),
1733 /*packetEnabledByContext=*/taskHasPacket)))
1734 result = failure();
1735 }
1736 return result;
1737}
1738
1739LogicalResult AIEX::DMAStartTaskOp::verify() {
1740 if (getRepeatCountAttr() && getRepeatCountVal())
1741 return emitOpError(
1742 "takes repeat_count or a runtime repeat count, not both");
1743 if (IntegerAttr rc = getRepeatCountAttr(); rc && rc.getInt() < 0)
1744 return emitOpError("repeat_count must be non-negative, got ")
1745 << rc.getInt();
1746 if (hasRepeatParameter() && (getRepeatCountAttr() || getRepeatCountVal()))
1747 return emitOpError("takes a repeat_parameter or a repeat count, not both");
1748 if (getRepeatParameterAttr() && getRepeatStateTableIdxAttr())
1749 return emitOpError("takes repeat_parameter or repeat_state_table_idx, "
1750 "not both");
1751 return success();
1752}
1753
1754std::optional<int64_t> AIEX::DMAConfigureTaskForOp::getStaticBytesStreamed() {
1755 int64_t starts = 0;
1756 for (Operation *user : getResult().getUsers()) {
1757 auto start = dyn_cast<AIEX::DMAStartTaskOp>(user);
1758 if (!start) {
1759 if (isa<AIEX::DMAAwaitTaskOp, AIEX::DMAFreeTaskOp>(user))
1760 continue;
1761 // The task value is forwarded (a branch operand, a call, ...), so its
1762 // starts elsewhere can't be counted.
1763 return std::nullopt;
1764 }
1765 // A start outside the task's own block may run under a loop or a
1766 // mutually-exclusive branch, either of which this static count can't see.
1767 if (start->getBlock() != (*this)->getBlock() ||
1768 start.getRepeatCountAttr() || start.getRepeatCountVal() ||
1769 start.hasRepeatParameter())
1770 return std::nullopt;
1771 ++starts;
1772 }
1773 if (starts == 0)
1774 return 0;
1775 std::optional<int64_t> repeat = getConstantIntValue(getRepeatCountValue());
1776 if (!repeat || *repeat < 0)
1777 return std::nullopt;
1778
1779 Region &body = getBody();
1780 if (body.empty())
1781 return std::nullopt;
1782 // The queue starts the chain at the entry block, so only blocks reachable
1783 // from it stream. A cycle runs indefinitely, and a next_bd_id continues the
1784 // chain outside the body.
1785 std::optional<int64_t> moved = 0;
1786 for (auto scc = llvm::scc_begin(&body.front()); !scc.isAtEnd(); ++scc) {
1787 if (scc.hasCycle())
1788 return std::nullopt;
1789 for (AIE::DMABDOp bd : scc->front()->getOps<AIE::DMABDOp>()) {
1790 // A scratchpad parameter lengthens the BD at run time (AIEOps.td's
1791 // "Runtime length").
1792 std::optional<int64_t> len =
1793 bd.getLengthParameterAttr() || bd.getLengthStateTableIdxAttr()
1794 ? std::nullopt
1795 : bd.getStaticLenInBytes();
1796 if (!len || bd.getIteration() || bd.getNextBdId())
1797 return std::nullopt;
1798 moved = llvm::checkedAdd(*moved, *len);
1799 if (!moved)
1800 return std::nullopt;
1801 }
1802 }
1803 std::optional<int64_t> runs = llvm::checkedAdd(*repeat, int64_t{1});
1804 if (runs)
1805 runs = llvm::checkedMul(*runs, starts);
1806 return runs ? llvm::checkedMul(*moved, *runs) : std::nullopt;
1807}
1808
1809namespace {
1810// A far end's cyclic chain of BDs over one object: each segment's BD, in
1811// order, sent `copies` times in a row.
1812struct BdChain {
1813 SmallVector<int64_t> segmentBytes;
1814 int64_t copies;
1815 int64_t objectBytes;
1816
1817 // Nullopt when the object's bytes overflow or a segment is empty.
1818 static std::optional<BdChain> get(ArrayRef<int64_t> segmentBytes,
1819 int64_t copies) {
1820 if (segmentBytes.size() == 1)
1821 copies = 1;
1822 std::optional<int64_t> objectBytes = 0;
1823 for (int64_t bytes : segmentBytes) {
1824 std::optional<int64_t> run = llvm::checkedMul(bytes, copies);
1825 if (bytes <= 0 || copies <= 0 || !run)
1826 return std::nullopt;
1827 objectBytes = llvm::checkedAdd(*objectBytes, *run);
1828 if (!objectBytes)
1829 return std::nullopt;
1830 }
1831 if (segmentBytes.empty())
1832 return std::nullopt;
1833 return BdChain{SmallVector<int64_t>(segmentBytes), copies, *objectBytes};
1834 }
1835
1836 // True when `bytes` through the chain stop where one of its BDs ends.
1837 bool endsOnBd(int64_t bytes) const {
1838 int64_t rest = bytes % objectBytes;
1839 int64_t start = 0;
1840 for (int64_t segment : segmentBytes) {
1841 if (rest < start + segment * copies)
1842 return (rest - start) % segment == 0;
1843 start += segment * copies;
1844 }
1845 return true;
1846 }
1847
1848 bool operator==(const BdChain &other) const {
1849 return segmentBytes == other.segmentBytes && copies == other.copies;
1850 }
1851};
1852} // namespace
1853
1854std::optional<int64_t> AIEX::NpuDmaMemcpyNdOp::getStaticBytesStreamed() {
1855 if (getLengthParameterAttr() || getLengthStateTableIdxAttr())
1856 return std::nullopt;
1857 uint64_t bits = getElementTypeBitwidth();
1858 if (bits % 8)
1859 return std::nullopt;
1860 std::optional<int64_t> bytes = static_cast<int64_t>(bits / 8);
1861 for (OpFoldResult size : getMixedSizes()) {
1862 std::optional<int64_t> extent = getConstantIntValue(size);
1863 bytes = bytes && extent ? llvm::checkedMul(*bytes, *extent) : std::nullopt;
1864 }
1865 return bytes;
1866}
1867
1868// The far end of a shim transfer on an objectFIFO receives (or sends) whole
1869// objects through BDs sized by the object type, or by its link pool's
1870// segments, so transfers ending mid-BD never complete it and the fifo hangs.
1871// Several transfers may share an object (concatenating host buffers into it, or
1872// splitting it across them), so it is the runtime sequence's static total
1873// through the fifo that must stop at a BD boundary.
1874static LogicalResult verifyTransferCoversWholeObjects(
1875 Operation *transfer, SymbolRefAttr alloc, std::optional<int64_t> moved,
1876 AIE::ObjectFifoCreateOp fifo, SymbolTableCollection &symbols) {
1877 if (fifo.getPadDimensions() || fifo.usesStream())
1878 return success();
1879 if (!moved || *moved == 0)
1880 return success();
1881 DataLayout layout = DataLayout::closest(transfer);
1882 // Bytes an object holds, or nullopt when that overflows.
1883 auto objectBytesOf = [&](Type fifoType) -> std::optional<int64_t> {
1884 MemRefType elem = cast<AIE::AIEObjectFifoType>(fifoType).getElementType();
1885 return llvm::checkedMul(
1886 elem.getNumElements(),
1887 static_cast<int64_t>(layout.getTypeSize(elem.getElementType())));
1888 };
1889 // A linked fifo's far end on the link's shared tile is a chain of BDs over
1890 // the link's pool, one per pool segment, sized by the pool owner whatever
1891 // this fifo's own (or consumer) type. A join's or distribute's combined end
1892 // steps through every segment in order, each small end through its own, and
1893 // a plain link's single segment is the whole pool object. The link's
1894 // repeat_count sends each of its output's segment BDs that many times in a
1895 // row. Every link the fifo meets has an end the transfer must complete,
1896 // one per tile when links on several tiles read a broadcast.
1897 SmallVector<Value> linkTiles;
1898 SmallVector<BdChain> chains;
1899 auto device = fifo->getParentOfType<AIE::DeviceOp>();
1900 auto resolve = [&](ArrayAttr refs) {
1901 std::vector<AIE::ObjectFifoCreateOp> fifos;
1902 for (Attribute ref : refs)
1903 if (auto name = dyn_cast<FlatSymbolRefAttr>(ref))
1904 if (auto linked = symbols.lookupSymbolIn<AIE::ObjectFifoCreateOp>(
1905 device, name.getAttr()))
1906 fifos.push_back(linked);
1907 return fifos;
1908 };
1909 auto self = FlatSymbolRefAttr::get(fifo.getSymNameAttr());
1910 if (device)
1911 for (AIE::ObjectFifoLinkOp link : device.getOps<AIE::ObjectFifoLinkOp>()) {
1912 if (!llvm::is_contained(link.getFifoIns(), self) &&
1913 !llvm::is_contained(link.getFifoOuts(), self))
1914 continue;
1915 std::vector<AIE::ObjectFifoCreateOp> ins = resolve(link.getFifoIns());
1916 std::vector<AIE::ObjectFifoCreateOp> outs = resolve(link.getFifoOuts());
1917 std::optional<Value> sharedTile = link.getOptionalSharedTile();
1918 if (ins.empty() || outs.empty() || !sharedTile)
1919 return success();
1920 linkTiles.push_back(*sharedTile);
1921 AIE::ObjectFifoCreateOp owner = link.getPoolOwner();
1922 MemRefType pool =
1923 cast<AIE::AIEObjectFifoType>(owner.getElemType()).getElementType();
1924 auto width =
1925 static_cast<int64_t>(layout.getTypeSize(pool.getElementType()));
1926 SmallVector<std::pair<int64_t, int64_t>> segments =
1927 link.getPoolSegments();
1928 if (fifo != owner && (link.isJoin() || link.isDistribute())) {
1929 std::vector<AIE::ObjectFifoCreateOp> &many = link.isJoin() ? ins : outs;
1930 segments = {segments[llvm::find(many, fifo) - many.begin()]};
1931 }
1932 SmallVector<int64_t> segmentBytes;
1933 for (auto [offset, size] : segments) {
1934 std::optional<int64_t> bytes = llvm::checkedMul(size, width);
1935 if (!bytes)
1936 break;
1937 segmentBytes.push_back(*bytes);
1938 }
1939 int64_t copies = llvm::is_contained(outs, fifo)
1940 ? link.getRepeatCount().value_or(1)
1941 : 1;
1942 std::optional<BdChain> chain;
1943 if (segmentBytes.size() == segments.size())
1944 chain = BdChain::get(segmentBytes, copies);
1945 if (chain && !llvm::is_contained(chains, *chain))
1946 chains.push_back(*chain);
1947 }
1948 // Every far-end BD chain this transfer must leave at a BD boundary: the
1949 // producer's, for a drain, and each consumer's other than the shim's own. A
1950 // broadcast stalls on its slowest consumer, so a drain bounds what the
1951 // others receive too. An end off every link tile is one BD over its own
1952 // object.
1953 auto addChain = [&](Value tile, Type fifoType) {
1954 if (llvm::is_contained(linkTiles, tile))
1955 return;
1956 std::optional<BdChain> chain;
1957 if (std::optional<int64_t> bytes = objectBytesOf(fifoType))
1958 chain = BdChain::get({*bytes}, 1);
1959 if (chain && !llvm::is_contained(chains, *chain))
1960 chains.push_back(*chain);
1961 };
1962 auto producer =
1963 dyn_cast_or_null<AIE::TileLike>(fifo.getProducerTile().getDefiningOp());
1964 if (!producer)
1965 return success();
1966 if (!producer.isShimTile())
1967 addChain(fifo.getProducerTile(), fifo.getElemType());
1968 for (Value consumer : fifo.getConsumerTiles()) {
1969 auto tile = dyn_cast_or_null<AIE::TileLike>(consumer.getDefiningOp());
1970 if (!tile || !tile.isShimTile())
1971 addChain(consumer, fifo.getConsumerElemTypeOrDefault());
1972 }
1973 if (chains.empty())
1974 return success();
1975
1976 // Transfers in a loop or branch have no static count to sum.
1977 auto sequence = transfer->getParentOfType<AIE::RuntimeSequenceOp>();
1978 if (!sequence || transfer->getBlock() != &sequence.getBody().front())
1979 return success();
1980 // A dma_channel_reset_for re-arms the fifo's far ends and drops a partly
1981 // filled object. It doesn't block, so which bytes land before it isn't
1982 // static, and the transfers around it can't be summed.
1983 if (llvm::any_of(transfer->getBlock()->getOps<AIEX::DmaChannelResetForOp>(),
1984 [&](AIEX::DmaChannelResetForOp reset) {
1985 return reset.getObjfifoAttr() == alloc;
1986 }))
1987 return success();
1988 // Bytes `op` moves through the fifo: zero when it moves none or names
1989 // another, nullopt when its count isn't static.
1990 auto through = [&](Operation *op) -> std::optional<int64_t> {
1991 if (auto memcpy = dyn_cast<AIEX::NpuDmaMemcpyNdOp>(op))
1992 return memcpy.getMetadata() == alloc ? memcpy.getStaticBytesStreamed()
1993 : 0;
1994 if (auto task = dyn_cast<AIEX::DMAConfigureTaskForOp>(op))
1995 return task.getAlloc() == alloc ? task.getStaticBytesStreamed() : 0;
1996 if (auto chain = dyn_cast<AIEX::DMAStartBdChainForOp>(op))
1997 return chain.getAllocAttr() == alloc ? std::nullopt
1998 : std::optional<int64_t>(0);
1999 return 0;
2000 };
2001 // Segment boundaries don't add up the way whole objects do, so the
2002 // sequence's total is checked whatever this transfer moves, once: by the
2003 // first transfer through the fifo.
2004 for (Operation *prev = transfer->getPrevNode(); prev;
2005 prev = prev->getPrevNode())
2006 if (through(prev) != 0)
2007 return success();
2008 std::optional<int64_t> total = 0;
2009 int64_t sharing = 0;
2010 WalkResult walk = sequence.walk([&](Operation *op) {
2011 std::optional<int64_t> bytes = through(op);
2012 // A transfer that moves nothing, or is never started, shares no object.
2013 if (bytes == 0)
2014 return WalkResult::advance();
2015 if (!bytes || op->getBlock() != transfer->getBlock())
2016 return WalkResult::interrupt();
2017 total = llvm::checkedAdd(*total, *bytes);
2018 if (!total)
2019 return WalkResult::interrupt();
2020 ++sharing;
2021 return WalkResult::advance();
2022 });
2023 if (walk.wasInterrupted())
2024 return success();
2025 const BdChain *partial = llvm::find_if(
2026 chains, [&](const BdChain &chain) { return !chain.endsOnBd(*total); });
2027 if (partial == chains.end())
2028 return success();
2029
2030 InFlightDiagnostic error = transfer->emitOpError()
2031 << "moves " << *moved
2032 << " bytes through objectFIFO @"
2033 << fifo.getSymName() << ", which";
2034 if (sharing > 1)
2035 error << ", with this runtime sequence's other transfers through it ("
2036 << *total << " bytes in all),";
2037 error << " is not a whole number of its " << partial->objectBytes
2038 << "-byte objects";
2039 if (partial->segmentBytes.size() == 1)
2040 return error << "; the far end's buffer descriptor is sized by the "
2041 "object, so a partial one never completes";
2042 std::string segments;
2043 llvm::raw_string_ostream os(segments);
2044 llvm::interleaveComma(partial->segmentBytes, os);
2045 error << " and doesn't end on a segment boundary (segments of " << segments
2046 << " bytes";
2047 if (partial->copies > 1)
2048 error << ", each sent " << partial->copies << " times in a row";
2049 return error << "); the far end's buffer descriptors are sized by the "
2050 "segments, so a partial one never completes";
2051}
2052
2053LogicalResult
2054AIEX::DMAConfigureTaskOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2055 FlatSymbolRefAttr alloc = getAllocAttr();
2056 AIE::DeviceOp dev = getOperation()->getParentOfType<AIE::DeviceOp>();
2057 if (!alloc || !dev)
2058 return success();
2059 auto allocation = dyn_cast_or_null<AIE::ShimDMAAllocationOp>(
2060 symbolTable.lookupSymbolIn(dev, alloc.getAttr()));
2061 if (!allocation)
2062 return emitOpError("alloc ")
2063 << alloc << " names no aie.shim_dma_allocation";
2064 if (allocation.getTile() != getTile() ||
2065 allocation.getChannelDir() != getDirection() ||
2066 allocation.getChannelIndex() != static_cast<int>(getChannel()))
2067 return emitOpError("runs on another channel than its alloc ") << alloc;
2068 return success();
2069}
2070
2071// Resolving the allocation symbol through the collection keeps the lookup off
2072// the device's linear symbol scan, which a per-op verifier repeats after every
2073// pass.
2074LogicalResult AIEX::DMAConfigureTaskForOp::verifySymbolUses(
2075 SymbolTableCollection &symbolTable) {
2076 AIE::DeviceOp dev = getOperation()->getParentOfType<AIE::DeviceOp>();
2077 if (!dev)
2078 return success();
2079 Operation *target =
2080 symbolTable.lookupSymbolIn(dev, getAlloc().getRootReference());
2081 if (!target)
2082 return emitOpError() << getAlloc() << " does not name a symbol";
2083 // The tile is known once the name is a shim DMA allocation or a route
2084 // endpoint, so the per-BD dimension limit can be enforced before the task
2085 // becomes a concrete dma_configure_task. Before the objectFIFO lowering
2086 // the name is the fifo's own, and the check waits.
2087 Value tileValue;
2088 LogicalResult named =
2089 TypeSwitch<Operation *, LogicalResult>(target)
2090 .Case([&](AIE::ShimDMAAllocationOp alloc) {
2091 tileValue = alloc.getTile();
2092 return success();
2093 })
2094 .Case([&](AIE::RouteEndpointOp endpoint) -> LogicalResult {
2095 tileValue = endpoint.getTile();
2096 if (endpoint.getBundle() == AIE::WireBundle::DMA)
2097 return success();
2098 return emitOpError() << getAlloc() << " names a "
2099 << stringifyWireBundle(endpoint.getBundle())
2100 << " port, not a DMA channel";
2101 })
2102 .Case([&](AIE::ObjectFifoCreateOp fifo) {
2103 return verifyTransferCoversWholeObjects(
2104 *this, getAlloc(), getStaticBytesStreamed(), fifo, symbolTable);
2105 })
2106 .Default([&](Operation *) -> LogicalResult {
2107 return emitOpError() << getAlloc()
2108 << " must name an aie.shim_dma_allocation, an "
2109 "aie.route_endpoint or an aie.objectfifo";
2110 });
2111 if (failed(named))
2112 return failure();
2113 // An allocation or endpoint on an unplaced (logical) tile waits for
2114 // placement to substitute a real one.
2115 auto tile = tileValue ? tileValue.getDefiningOp<AIE::TileOp>() : nullptr;
2116 if (!tile)
2117 return success();
2118 const AIE::AIETargetModel &targetModel = AIE::getTargetModel(getOperation());
2119 return verifyTaskBDDimensions(targetModel, tile.getCol(), tile.getRow(),
2120 getBody());
2121}
2122
2123//===----------------------------------------------------------------------===//
2124// DMAStartBdChainOp
2125//===----------------------------------------------------------------------===//
2126
2127AIE::BDChainOp AIEX::DMAStartBdChainOp::getBDChainOp() {
2128 AIE::DeviceOp device = (*this)->getParentOfType<AIE::DeviceOp>();
2129 AIE::BDChainOp chain = device.lookupSymbol<AIE::BDChainOp>(getSymbol());
2130 return chain;
2131}
2132
2133LogicalResult AIEX::DMAStartBdChainOp::verify() {
2134 AIE::BDChainOp chain = getBDChainOp();
2135 if (!chain) {
2136 return emitOpError("symbol does not reference valid BD chain");
2137 }
2138
2139 auto actualArgTypes = getArgs().getTypes();
2140 auto expectedArgTypes = chain.getRegion().getArgumentTypes();
2141 if (actualArgTypes.size() != expectedArgTypes.size()) {
2142 return emitOpError("Number of arguments mismatches.");
2143 }
2144 for (unsigned i = 0, n = expectedArgTypes.size(); i < n; i++) {
2145 if (actualArgTypes[i] != expectedArgTypes[i]) {
2146 return emitOpError("Argument ") << (i + 1) << " types mismatch: "
2147 << "expected " << expectedArgTypes[i]
2148 << " but got " << actualArgTypes[i];
2149 }
2150 }
2151 return success();
2152}
2153
2154//===----------------------------------------------------------------------===//
2155// NpuControlPacketOp
2156//===----------------------------------------------------------------------===//
2157
2158uint32_t AIEX::NpuControlPacketOp::getRowFromAddr() {
2159 const auto &targetModel = AIE::getTargetModel(*this);
2160 uint32_t addr = getAddress();
2161 uint32_t rowInt = (addr >> targetModel.getRowShift()) & 0x1f;
2162 return rowInt;
2163}
2164
2165uint32_t AIEX::NpuControlPacketOp::getColumnFromAddr() {
2166 const auto &targetModel = AIE::getTargetModel(*this);
2167 uint32_t addr = getAddress();
2168 uint32_t colInt = (addr >> targetModel.getColumnShift()) & 0x1f;
2169 return colInt;
2170}
2171
2172LogicalResult AIEX::NpuControlPacketOp::verify() {
2173 // The 2-bit operation field: 0 write, 1 read, 2 write with response, 3
2174 // reserved. A wider value would also spill into the stream ID.
2175 uint32_t opcode = getOpcode();
2176 if (opcode > 2)
2177 return emitOpError("opcode ")
2178 << opcode
2179 << " is reserved; expected 0 (write), 1 (read), or 2 (write with "
2180 "response)";
2181
2182 bool isRead = opcode == 1;
2183 if (isRead && getData())
2184 return emitOpError("read control packet must not carry a data payload; "
2185 "the requested word count goes in the length attribute");
2186 if (!isRead && !getData())
2187 return emitOpError(
2188 "control packet with a write opcode requires a data payload");
2189
2190 // The word count is encoded as (words - 1), so 0 cannot be represented.
2191 std::optional<uint32_t> words;
2192 if (auto data = getData())
2193 words = data->size();
2194 else
2195 words = getLength();
2196 if (!words)
2197 return emitOpError("read control packet requires a length attribute");
2198 if (*words < 1)
2199 return emitOpError("control packet payload must be at least 1 word");
2200
2201 // Reads are never split, so their length must already fit one packet.
2202 if (isRead && *words > getMaxDataWords())
2203 return emitOpError("read control packet length ")
2204 << *words << " exceeds the maximum of " << getMaxDataWords()
2205 << " 32-bit words";
2206
2207 return success();
2208}
2209
2210//===----------------------------------------------------------------------===//
2211// SetLockOp
2212//===----------------------------------------------------------------------===//
2213
2214LogicalResult AIEX::SetLockOp::verify() {
2215 const auto &targetModel = AIE::getTargetModel(*this);
2216
2217 if (targetModel.getTargetArch() == AIE::AIEArch::AIE1)
2218 return emitOpError("SetLockOp is not supported on AIE1.");
2219
2220 if (std::optional<int64_t> value = getConstantIntValue(getValue())) {
2221 if (*value < 0)
2222 return emitOpError("Lock value must be non-negative");
2223 if (*value > targetModel.getMaxLockValue())
2224 return emitOpError("Lock value exceeds the maximum value of " +
2225 std::to_string(targetModel.getMaxLockValue()));
2226 }
2227
2228 auto lockOp = getLockOp();
2229 auto lockIDOpt = getLockOp().getLockID();
2230 // Note that the lockID may not be assigned initially, so lets wait until it
2231 // is to verify the lockID dependent conditions
2232 if (!lockIDOpt) {
2233 return success();
2234 }
2235 // A logical tile with a coordinate still unpinned waits for the
2236 // post-placement re-verify for its range and address checks, as
2237 // UsesAreAccessible's do.
2238 std::optional<int> col = lockOp.getTileLike().tryGetCol();
2239 std::optional<int> row = lockOp.getTileLike().tryGetRow();
2240 if (!col || !row)
2241 return success();
2242
2243 uint32_t lockID = lockOp.getLockIDValue();
2244
2245 if (lockID >= targetModel.getNumLocks(*col, *row)) {
2246 return emitOpError("Lock ID out of range for given tile. Max ID: " +
2247 std::to_string(targetModel.getNumLocks(*col, *row) - 1));
2248 }
2249
2250 if (!targetModel.getLocalLockAddress(lockID, AIE::TileID{*col, *row})) {
2251 return emitOpError("Invalid lock ID and tile combination when trying to "
2252 "retrieve the local lock address.");
2253 }
2254
2255 return success();
2256}
2257
2258//===----------------------------------------------------------------------===//
2259// DMABdPoolPopOp / DMABdPoolPushOp
2260//===----------------------------------------------------------------------===//
2261
2262static LogicalResult verifyBdPoolPartition(Operation *op, int col, int row,
2263 uint32_t begin, uint32_t end) {
2264 uint32_t numBds = AIE::getTargetModel(op).getNumBDs(col, row);
2265 if (begin >= end || end > numBds)
2266 return op->emitOpError()
2267 << "partition [" << begin << ", " << end
2268 << ") is not a nonempty range of the " << numBds
2269 << " buffer descriptors on tile (" << col << ", " << row << ")";
2270 return success();
2271}
2272
2273LogicalResult AIEX::DMABdPoolPopOp::verify() {
2274 if (failed(verifyBdPoolPartition(*this, getColumn(), getRow(), getBdBegin(),
2275 getBdEnd())))
2276 return failure();
2277 std::optional<uint32_t> id = getId();
2278 if (id && (*id < getBdBegin() || *id >= getBdEnd()))
2279 return emitOpError() << "takes buffer descriptor ID " << *id
2280 << ", which is outside its partition [" << getBdBegin()
2281 << ", " << getBdEnd() << ")";
2282 return success();
2283}
2284
2285LogicalResult AIEX::DMABdPoolPushOp::verify() {
2286 return verifyBdPoolPartition(*this, getColumn(), getRow(), getBdBegin(),
2287 getBdEnd());
2288}
2289
2290//===----------------------------------------------------------------------===//
2291// DmaChannelResetOp
2292//===----------------------------------------------------------------------===//
2293
2294LogicalResult AIEX::DmaChannelResetOp::verify() {
2295 const auto &targetModel = AIE::getTargetModel(*this);
2296
2297 auto tile = dyn_cast_or_null<AIE::TileOp>(getTile().getDefiningOp());
2298 if (!tile)
2299 return emitOpError() << "tile operand must be produced by an aie.tile op";
2300 int col = tile.getCol();
2301 int row = tile.getRow();
2302 // Only core and mem tiles have a per-channel DMA reset bit. The shim NOC DMA
2303 // control register has no reset field (aie-codegen's Aie2PShimDmaChProp sets
2304 // Reset.Mask = 0); bit 1 there is PAUSE_MEM, not RESET. Rejecting shim keeps
2305 // the op honest about what it can lower, matching aie-codegen's own
2306 // XAie_DmaChannelReset, which errors on SHIMNOC/SHIMPL tiles.
2307 if (!targetModel.isCoreTile(col, row) && !targetModel.isMemTile(col, row))
2308 return emitOpError() << "tile (" << col << ", " << row
2309 << ") has no DMA channel reset (only core and mem "
2310 "tiles do; shim NOC DMA has no reset bit)";
2311
2312 // Number of DMA channels on this tile in this direction. Mirrors
2313 // TileOp::getNumSource/DestConnections(WireBundle::DMA): the switchbox
2314 // direction is reversed relative to the DMA direction.
2315 uint32_t numChannels = getDirection() == AIE::DMAChannelDir::S2MM
2316 ? targetModel.getNumDestSwitchboxConnections(
2317 col, row, AIE::WireBundle::DMA)
2319 col, row, AIE::WireBundle::DMA);
2320 if (getChannel() >= numChannels)
2321 return emitOpError() << "channel " << getChannel()
2322 << " out of range for this tile and direction (tile "
2323 "has "
2324 << numChannels << " DMA channel(s) in this direction)";
2325
2326 return success();
2327}
2328
2329LogicalResult AIEX::DmaChannelResetForOp::verify() {
2330 // Deferring verifier, in the shape of DMAConfigureTaskForOp::verify: the
2331 // referenced symbol is only resolvable once the objectFIFO lowering has run.
2332 // It names an aie.objectfifo, then that fifo's shim endpoint while the
2333 // lowering is in flight, and finally its aie.objectfifo_rearm_binding. If
2334 // none is resolvable yet, defer -- a later pass will resolve it.
2335 AIE::DeviceOp dev = getOperation()->getParentOfType<AIE::DeviceOp>();
2336 if (!dev)
2337 return success();
2338 // The resident re-arm relies on the aie2p behavior that a DMA channel has no
2339 // enable bit, so the only way to restart it is a START_QUEUE push. AIE1 DMA
2340 // channels have an enable bit and are armed differently, so the trio this op
2341 // lowers to would not re-arm them correctly.
2342 if (AIE::getTargetModel(*this).getTargetArch() == AIE::AIEArch::AIE1)
2343 return emitOpError() << "is not supported on AIE1 devices (the resident "
2344 "re-arm relies on the aie2p start-queue push)";
2345 Operation *target = dev.lookupSymbol(getObjfifo());
2346 if (!target)
2347 return success(); // symbol resolved during a later pass; defer the check
2348 if (!isa<AIE::ObjectFifoCreateOp, AIE::RouteEndpoint,
2349 AIE::ObjectFifoRearmBindingOp>(target))
2350 return emitOpError() << "'" << getObjfifo()
2351 << "' must reference an aie.objectfifo (or its "
2352 "aie.objectfifo_rearm_binding)";
2353 return success();
2354}
2355
2356//===----------------------------------------------------------------------===//
2357// CoreResetOp
2358//===----------------------------------------------------------------------===//
2359
2360LogicalResult AIEX::CoreResetOp::verify() {
2361 const auto &targetModel = AIE::getTargetModel(*this);
2362
2363 // The op lowers to an NPU control-packet write; the runtime sequence has no
2364 // meaning on AIE1. Reject it explicitly, as SetLockOp does.
2365 if (targetModel.getTargetArch() == AIE::AIEArch::AIE1)
2366 return emitOpError("aiex.core_reset is not supported on AIE1.");
2367
2368 auto tile = dyn_cast_or_null<AIE::TileOp>(getTile().getDefiningOp());
2369 if (!tile)
2370 return emitOpError() << "tile operand must be produced by an aie.tile op";
2371 int col = tile.getCol();
2372 int row = tile.getRow();
2373 // The tile coordinates are bounded by aie.tile's own verifier, so this op
2374 // does not re-check them for range.
2375
2376 // Only core tiles have a CORE_CONTROL register with a reset bit. Mem and shim
2377 // tiles have no compute core, so there is nothing valid to lower to. This
2378 // matches aie-codegen's XAie_CoreReset, which errors on any tile that is not
2379 // an AIE (core) tile.
2380 if (!targetModel.isCoreTile(col, row))
2381 return emitOpError() << "tile (" << col << ", " << row
2382 << ") has no core to reset (only core tiles have a "
2383 "CORE_CONTROL register)";
2384
2385 return success();
2386}
2387
2388//===----------------------------------------------------------------------===//
2389// BlockFloatingPointType
2390//===----------------------------------------------------------------------===//
2391uint64_t AIEX::BlockFloatType::getTotalSizeInBits() const {
2392 return getBlockSize() * getMantissaBits() + getExponentBits() +
2393 getSubtileShiftBits();
2394}
2395
2396llvm::TypeSize AIEX::BlockFloatType::getTypeSizeInBits(
2397 const mlir::DataLayout &dataLayout,
2398 mlir::DataLayoutEntryListRef params) const {
2399 return llvm::TypeSize::getFixed(getTotalSizeInBits());
2400}
2401
2402uint64_t AIEX::BlockFloatType::getABIAlignment(
2403 const mlir::DataLayout &dataLayout,
2404 mlir::DataLayoutEntryListRef params) const {
2405 // For the purposes of the data movement operations, we want all types to be
2406 // packed <=> ABI alignment is 1.
2407 return 1;
2408}
2409
2410std::optional<AIEX::BlockFloatType::BlockFormat>
2411AIEX::BlockFloatType::getBlockFormat(StringRef blockType) {
2412 static const llvm::StringMap<AIEX::BlockFloatType::BlockFormat>
2413 blockFormatsMap = {
2414 {"v8bfp16ebs8", {8, 8, 8, 0}},
2415 {"v16bfp16ebs16", {16, 8, 8, 0}},
2416 };
2417
2418 auto it = blockFormatsMap.find(blockType);
2419 if (it != blockFormatsMap.end()) {
2420 return it->second;
2421 }
2422
2423 return std::nullopt;
2424}
2425
2426LogicalResult
2427AIEX::BlockFloatType::verify(function_ref<InFlightDiagnostic()> emitError,
2428 StringRef block_type) {
2429 if (!getBlockFormat(block_type))
2430 return emitError() << "Invalid block type: " << block_type
2431 << ". Known types are: v8bfp16ebs8, v16bfp16ebs16.";
2432
2433 return success();
2434}
2435
2436//===----------------------------------------------------------------------===//
2437// ConfigureOp
2438//===----------------------------------------------------------------------===//
2439
2440AIE::DeviceOp AIEX::ConfigureOp::getReferencedDeviceOp() {
2441 ModuleOp moduleOp = this->getOperation()->getParentOfType<ModuleOp>();
2442 if (!moduleOp) {
2443 emitError("aiex.configure must be inside of a module");
2444 return nullptr;
2445 }
2446 Operation *maybeReferencedDevice =
2447 SymbolTable::lookupSymbolIn(moduleOp.getOperation(), getSymbolAttr());
2448 if (!maybeReferencedDevice) {
2449 emitError("No such device: '") << getSymbolAttr() << "'";
2450 return nullptr;
2451 }
2452 AIE::DeviceOp referencedDevice =
2453 llvm::dyn_cast<AIE::DeviceOp>(maybeReferencedDevice);
2454 if (!referencedDevice) {
2455 emitError("Not a device: '") << getSymbolAttr() << "'";
2456 return nullptr;
2457 }
2458 return referencedDevice;
2459}
2460
2461//===----------------------------------------------------------------------===//
2462// ReadScratchpadParameterOp
2463//===----------------------------------------------------------------------===//
2464
2465LogicalResult AIEX::ReadScratchpadParameterOp::verify() {
2466 auto device = (*this)->getParentOfType<AIE::DeviceOp>();
2467 if (!device) {
2468 return emitOpError("must be inside an aie.device");
2469 }
2470 if (!(*this)->getParentOfType<AIE::CoreOp>()) {
2471 return emitOpError("must be inside an aie.core");
2472 }
2473 auto moduleOp = (*this)->getParentOfType<ModuleOp>();
2474 if (!moduleOp ||
2475 !moduleOp.lookupSymbol<AIEX::ScratchpadParameterOp>(getParameter())) {
2476 return emitOpError("references unknown parameter '")
2477 << getParameter()
2478 << "' (aiex.scratchpad_parameter ops are declared at module scope)";
2479 }
2480 if (getResult().getType().isF32()) {
2481 return emitOpError(
2482 "f32 parameters are not supported: the scratchpad encoding zeroes "
2483 "the top 2 bits, which clobbers the sign bit and top exponent bit "
2484 "of an f32. Use bf16 or an integer type up to i32 instead.");
2485 }
2486 return success();
2487}
2488
2489LogicalResult AIEX::WaitScratchpadParametersOp::verify() {
2490 if (!(*this)->getParentOfType<AIE::CoreOp>()) {
2491 return emitOpError("must be inside an aie.core");
2492 }
2493 return success();
2494}
2495
2496LogicalResult AIEX::ConfigureOp::verify() {
2497 AIE::DeviceOp parentDev = getOperation()->getParentOfType<AIE::DeviceOp>();
2498 AIE::DeviceOp referencedDev = getReferencedDeviceOp();
2499 if (!referencedDev) {
2500 return failure();
2501 }
2502 if (parentDev.getDevice() != referencedDev.getDevice()) {
2503 emitError("Device types do not match: '")
2504 << AIE::stringifyAIEDevice(parentDev.getDevice()) << "' vs. '"
2505 << AIE::stringifyAIEDevice(referencedDev.getDevice()) << "'";
2506 return failure();
2507 }
2508 return success();
2509}
2510
2511//===----------------------------------------------------------------------===//
2512// RunOp
2513//===----------------------------------------------------------------------===//
2514
2515AIE::DeviceOp AIEX::RunOp::getCalleeDeviceOp() {
2516 AIEX::ConfigureOp configureOp =
2517 getOperation()->getParentOfType<AIEX::ConfigureOp>();
2518 if (!configureOp) {
2519 return nullptr;
2520 }
2521 AIE::DeviceOp referencedDevice = configureOp.getReferencedDeviceOp();
2522 return referencedDevice;
2523}
2524
2525AIE::RuntimeSequenceOp
2526AIEX::RunOp::getCalleeRuntimeSequenceOp(AIE::NamedOpTableCollection *tables) {
2527 AIEX::ConfigureOp configureOp =
2528 getOperation()->getParentOfType<AIEX::ConfigureOp>();
2529 if (!configureOp) {
2530 return nullptr;
2531 }
2532 AIE::DeviceOp referencedDevice = configureOp.getReferencedDeviceOp();
2533 if (!referencedDevice) {
2534 return nullptr;
2535 }
2536
2537 StringAttr name = getRuntimeSequenceSymbolAttr().getAttr();
2538 if (!tables) {
2539 return AIE::lookupNamedOpIn<AIE::RuntimeSequenceOp>(referencedDevice, name);
2540 }
2541 return tables->lookupIn<AIE::RuntimeSequenceOp>(referencedDevice, name);
2542}
2543
2544//===----------------------------------------------------------------------===//
2545// NpuLoadPdiOp
2546//===----------------------------------------------------------------------===//
2547
2548LogicalResult AIEX::NpuLoadPdiOp::canonicalize(AIEX::NpuLoadPdiOp op,
2549 PatternRewriter &rewriter) {
2550 // Check for back-to-back identical load_pdi ops and remove duplicates
2551 Operation *nextOp = op->getNextNode();
2552 if (!nextOp)
2553 return failure();
2554
2555 // Check if next op is also a NpuLoadPdiOp
2556 auto nextLoadPdi = dyn_cast<AIEX::NpuLoadPdiOp>(nextOp);
2557 if (!nextLoadPdi)
2558 return failure();
2559
2560 // Check if they are identical (all attributes match)
2561 if (op.getDeviceRefAttr() == nextLoadPdi.getDeviceRefAttr() &&
2562 op.getId() == nextLoadPdi.getId() &&
2563 op.getSize() == nextLoadPdi.getSize() &&
2564 op.getAddress() == nextLoadPdi.getAddress()) {
2565 // Erase the first one, keeping the second
2566 rewriter.eraseOp(op);
2567 return success();
2568 }
2569
2570 return failure();
2571}
2572
2573//===----------------------------------------------------------------------===//
2574// NpuWriteConfigOp
2575//===----------------------------------------------------------------------===//
2576
2577static bool writesConfig(Operation *op) {
2578 return isa<AIEX::NpuLoadPdiOp, AIEX::NpuWriteConfigOp,
2579 AIEX::NpuWriteTeardownOp>(op);
2580}
2581
2582// Visits the load_pdi, write_config and write_teardown ops before `op` in its
2583// block, nearest first, then those after it from the block's end, as the host
2584// runs the sequence again on each dispatch, then `op` itself. Returns the first
2585// for which `visit` returns true.
2586static Operation *findConfigOpBefore(Operation *op,
2587 function_ref<bool(Operation *)> visit) {
2588 for (Operation *prev = op->getPrevNode(); prev; prev = prev->getPrevNode())
2589 if (writesConfig(prev) && visit(prev))
2590 return prev;
2591 for (Operation &next : llvm::reverse(*op->getBlock())) {
2592 if (&next == op)
2593 break;
2594 if (writesConfig(&next) && visit(&next))
2595 return &next;
2596 }
2597 return visit(op) ? op : nullptr;
2598}
2599
2600// The load_pdi that resets the array for `op`, or the write_teardown that
2601// leaves it so, or null.
2602static Operation *precedingReset(Operation *op) {
2603 return findConfigOpBefore(op, [&](Operation *prev) {
2604 return prev != op &&
2605 isa<AIEX::NpuLoadPdiOp, AIEX::NpuWriteTeardownOp>(prev);
2606 });
2607}
2608
2609bool AIEX::NpuLoadPdiOp::loadsSamePdi(NpuLoadPdiOp other) {
2610 return getDeviceRefAttr() == other.getDeviceRefAttr() &&
2611 getId() == other.getId() && getSize() == other.getSize() &&
2612 getAddress() == other.getAddress();
2613}
2614
2615LogicalResult AIEX::NpuWriteConfigOp::verify() {
2616 Operation *found = precedingReset(*this);
2617 if (!found)
2618 return emitOpError("must follow the load_pdi that resets the array");
2619 auto reset = dyn_cast<AIEX::NpuLoadPdiOp>(found);
2620 if (!reset)
2621 return success();
2622 // The firmware skips the load if it loaded the same PDI last. That leaves
2623 // the array as a reset does only if every configuration since is torn down.
2624 bool tornDown = true;
2625 Operation *later = reset;
2626 auto before =
2627 cast<AIEX::NpuLoadPdiOp>(findConfigOpBefore(reset, [&](Operation *op) {
2628 if (isa<AIEX::NpuWriteConfigOp>(op) &&
2629 !isa<AIEX::NpuWriteTeardownOp>(later))
2630 tornDown = false;
2631 later = op;
2632 return isa<AIEX::NpuLoadPdiOp>(op);
2633 }));
2634 if (!before.loadsSamePdi(reset) || tornDown)
2635 return success();
2636 InFlightDiagnostic diag = emitOpError(
2637 "follows a load_pdi the firmware skips, as it loads the PDI loaded "
2638 "just before it, so the array is not reset");
2639 diag.attachNote(before.getLoc()) << "the PDI is loaded here";
2640 return diag;
2641}
2642
2643LogicalResult
2644AIEX::NpuLoadPdiOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2645 FlatSymbolRefAttr ref = getDeviceRefAttr();
2646 if (!ref)
2647 return success();
2648 auto device = (*this)->getParentOfType<AIE::DeviceOp>();
2649 Operation *scope = device ? device->getParentOp()
2650 : SymbolTable::getNearestSymbolTable(*this);
2651 if (!isa_and_present<AIE::DeviceOp>(symbolTable.lookupSymbolIn(scope, ref)))
2652 return emitOpError() << ref << " is not a device";
2653 return success();
2654}
2655
2656LogicalResult
2657AIEX::NpuWriteConfigOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2658 Operation *scope = (*this)->getParentOfType<AIE::DeviceOp>()->getParentOp();
2659 if (!isa_and_present<AIE::DeviceOp>(
2660 symbolTable.lookupSymbolIn(scope, getDeviceRefAttr())))
2661 return emitOpError() << getDeviceRefAttr() << " is not a device";
2662 auto reset = dyn_cast_or_null<AIEX::NpuLoadPdiOp>(precedingReset(*this));
2663 FlatSymbolRefAttr resetRef = reset ? reset.getDeviceRefAttr() : nullptr;
2664 auto empty = resetRef ? dyn_cast_or_null<AIE::DeviceOp>(
2665 symbolTable.lookupSymbolIn(scope, resetRef))
2666 : nullptr;
2667 // The transaction conversion gives every device a `@configure` sequence; an
2668 // empty one still configures nothing.
2669 if (reset &&
2670 (!empty ||
2671 !llvm::all_of(empty.getBody()->without_terminator(), [](Operation &op) {
2672 return isa<AIE::RuntimeSequenceOp>(op) &&
2673 llvm::all_of(op.getRegion(0),
2674 [](Block &block) { return block.empty(); });
2675 })))
2676 return emitOpError("must follow a load_pdi of an empty device, which "
2677 "resets the array");
2678 return success();
2679}
2680
2681//===----------------------------------------------------------------------===//
2682// NpuWriteTeardownOp
2683//===----------------------------------------------------------------------===//
2684
2685LogicalResult AIEX::NpuWriteTeardownOp::verify() {
2686 Operation *prev = (*this)->getPrevNode();
2687 while (prev && !writesConfig(prev))
2688 prev = prev->getPrevNode();
2689 auto config = dyn_cast_or_null<AIEX::NpuWriteConfigOp>(prev);
2690 if (!config || config.getDeviceRefAttr() != getDeviceRefAttr())
2691 return emitOpError("must follow the write_config of ")
2692 << getDeviceRefAttr() << ", with no load_pdi between";
2693 return success();
2694}
2695
2696LogicalResult
2697AIEX::NpuWriteTeardownOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2698 Operation *scope = (*this)->getParentOfType<AIE::DeviceOp>()->getParentOp();
2699 auto device = dyn_cast_or_null<AIE::DeviceOp>(
2700 symbolTable.lookupSymbolIn(scope, getDeviceRefAttr()));
2701 if (!device)
2702 return emitOpError() << getDeviceRefAttr() << " is not a device";
2703 Operation *shimStart = AIEX::findShimDmaStart(device);
2704 if (!shimStart)
2705 return success();
2706 InFlightDiagnostic diag =
2707 emitOpError() << getDeviceRefAttr()
2708 << " starts a shim DMA channel, which teardown cannot stop";
2709 diag.attachNote(shimStart->getLoc()) << "started here";
2710 return diag;
2711}
2712
2713Operation *AIEX::findShimDmaStart(AIE::DeviceOp device) {
2714 Operation *found = nullptr;
2715 device.walk([&](Operation *op) {
2716 if (!isa<AIE::DMAStartOp, AIE::DMAOp>(op) ||
2717 !isa<AIE::ShimDMAOp>(op->getParentOp()))
2718 return WalkResult::advance();
2719 found = op;
2720 return WalkResult::interrupt();
2721 });
2722 return found;
2723}
bool isCoreTile(int col, int row) const
Return true if the given tile is a Core tile.
virtual uint64_t getDmaBdMaxLen(AIETileType tileType) const =0
Return the maximum transfer length, measured in address-generation granules (see getAddressGenGranula...
virtual const DmaBdLayout * getDmaBdLayout(AIETileType tileType) const
Return the register layout of a buffer descriptor on the given tile type, or null if the target has n...
virtual AIEArch getTargetArch() const =0
Return the target architecture.
bool isMemTile(int col, int row) const
Return true if the given tile is a Mem tile.
virtual uint32_t getDmaBdWrapBits(AIETileType tileType) const =0
Return the bit width of the wrap (size) field in a DMA buffer descriptor for the given tile type.
virtual uint32_t getLocalMemorySize() const =0
Return the size (in bytes) of the local data memory of a core.
virtual uint32_t getNumLocks(AIETileType tileType) const =0
Return the number of lock objects for a given tile type.
virtual uint32_t getDmaBdIterBits(AIETileType tileType) const =0
Return the bit width of the iteration (repeat) wrap/stride fields in a DMA buffer descriptor for the ...
virtual uint32_t getBDMaxDims(AIETileType tileType) const =0
Return the maximum number of addressing dimensions (wraps/strides) a single buffer descriptor support...
virtual std::vector< std::pair< uint32_t, uint32_t > > getShimBurstEncodingsAndLengths() const =0
virtual uint32_t getDmaBdStepBits(AIETileType tileType) const =0
Return the bit width of the step (stride) field in a DMA buffer descriptor for the given tile type.
virtual bool isBdChannelAccessible(int col, int row, uint32_t bd_id, int channel) const =0
Return true iff buffer descriptor bd_id on tile (col, row) can be submitted on channel channel.
virtual std::optional< uint32_t > getLocalLockAddress(uint32_t lockId, TileID tile) const =0
virtual uint32_t getMaxChannelNumForAdjacentMemTile(int col, int row) const =0
virtual AIETileType getTileType(int col, int row) const =0
Return the tile type for the given tile coordinates.
virtual uint32_t getMaxRepeatCount() const =0
Return the largest DMA task repeat count (unsupported = 0).
bool isShimNOCTile(int col, int row) const
Return true if the given tile is a ShimNOC tile.
virtual uint32_t getNumBDs(AIETileType tileType) const =0
Return the number of buffer descriptors for a given tile type.
virtual uint32_t getMaxLockValue() const =0
Return the maximum value that can be stored in a lock register.
virtual uint32_t getMaxOutOfOrderId() const =0
Return the largest out-of-order BD id (unsupported = 0).
virtual uint32_t getColumnShift() const =0
virtual uint32_t getNumDestSwitchboxConnections(int col, int row, WireBundle bundle) const =0
Return the number of destinations of connections inside a switchbox.
virtual uint32_t getRowShift() const =0
virtual uint32_t getAddressGenGranularity() const =0
Return the data bus width of the device.
virtual uint32_t getMemTileSize() const =0
Return the size (in bytes) of a MemTile.
virtual uint32_t getNumSourceSwitchboxConnections(int col, int row, WireBundle bundle) const =0
Return the number of sources of connections inside a switchbox.
OpTy lookupIn(mlir::Operation *symbolTableOp, mlir::StringAttr name)
Definition AIEDialect.h:179
mlir::Operation * lookup(mlir::StringAttr name) const
Definition AIEDialect.h:153
const V & get(const std::map< K, V > &pa, K k)
Definition d_ary_heap.h:70
mlir::Operation * findShimDmaStart(AIE::DeviceOp device)
bool isContiguousTransfer(llvm::ArrayRef< int64_t > sizes, llvm::ArrayRef< int64_t > strides)
void encodeHardwareStridesWraps(Policy &p, uint64_t elemWidth, uint32_t addressGranularity, typename Policy::V inputSizes[4], typename Policy::V inputStrides[4], typename Policy::V sizes[4], typename Policy::V strides[4])
Definition BdLowering.h:145
mlir::LogicalResult verifyStridesWraps(mlir::Operation *forOp, llvm::function_ref< mlir::InFlightDiagnostic()> emitError, mlir::BaseMemRefType referencedBufType, int tileCol, int tileRow, llvm::SmallVector< int64_t, 4 > inputSizes, llvm::SmallVector< int64_t, 4 > inputStrides, llvm::SmallVector< int64_t, 4 > hardwareSizes, llvm::SmallVector< int64_t, 4 > hardwareStrides, bool skipTransformationChecks=false)
std::optional< uint64_t > getConstantInt64Operand(mlir::Value v)
bool isDecomposableNdDmaPattern(mlir::Operation *forOp, mlir::BaseMemRefType referencedBufType, const xilinx::AIE::AIETargetModel &targetModel, int tileCol, int tileRow, llvm::ArrayRef< int64_t > offsetsInnermostFirst, llvm::ArrayRef< int64_t > sizesInnermostFirst, llvm::ArrayRef< int64_t > stridesInnermostFirst)
True when the pattern is fully constant, non-contiguous, currently illegal, and decomposeNdDmaPattern...
std::optional< uint32_t > getConstantIntOperand(mlir::Value v)
mlir::Value createConstantArgPlus(mlir::OpBuilder &builder, mlir::Location loc, uint64_t value)
void getHardwareStridesWraps(const AIE::AIETargetModel &targetModel, mlir::Operation *op, mlir::BaseMemRefType referencedBufType, llvm::SmallVector< int64_t, 4 > inputSizes, llvm::SmallVector< int64_t, 4 > inputStrides, llvm::SmallVector< int64_t, 4 > &sizes, llvm::SmallVector< int64_t, 4 > &strides)
bool isLinearModeTransfer(const AIE::AIETargetModel &targetModel, int col, int row, llvm::ArrayRef< int64_t > sizes, llvm::ArrayRef< int64_t > strides)
bool isConstMultipleOfGranule(int64_t value, uint64_t elemWidth, uint32_t addressGranularity)
Definition BdLowering.h:65
bool isLinearTransfer(llvm::ArrayRef< int64_t > sizes, llvm::ArrayRef< int64_t > strides)
mlir::LogicalResult verifyConstBdRealizability(mlir::Operation *op, llvm::ArrayRef< mlir::OpFoldResult > sizes, llvm::ArrayRef< mlir::OpFoldResult > strides, uint64_t elemWidth, uint32_t gran)
Definition BdLowering.h:77
mlir::Value createConstantI32(mlir::OpBuilder &builder, mlir::Location loc, uint32_t value)
void printTypedDynamicIndexList(mlir::OpAsmPrinter &printer, mlir::Operation *op, mlir::OperandRange values, llvm::ArrayRef< int64_t > integers, mlir::TypeRange types)
mlir::LogicalResult verifyLengthParameterTile(mlir::Operation *op, const AIETargetModel &targetModel, int col, int row)
mlir::LogicalResult verifyAssignedPacket(mlir::Operation *op, PacketInfoAttr packet, llvm::StringRef what="packet")
mlir::ParseResult parseTypedDynamicIndexList(mlir::OpAsmParser &parser, llvm::SmallVectorImpl< mlir::OpAsmParser::UnresolvedOperand > &values, mlir::DenseI64ArrayAttr &integers, llvm::SmallVectorImpl< mlir::Type > &types)
mlir::Operation * lookupNamedOpIn(mlir::Operation *symbolTableOp, mlir::StringAttr name)
mlir::LogicalResult verifyDMABDOutOfOrderId(DMABDOp bd, bool packetEnabledByContext=false)
mlir::LogicalResult verifyOutOfOrderChannel(mlir::Operation *op, DMAChannelDir dir, bool outOfOrder, llvm::ArrayRef< DMABDOp > bds, bool packetEnabledByContext=false)
mlir::LogicalResult verifyLengthParameter(mlir::Operation *op, std::optional< int64_t > lengthUnit, mlir::BaseMemRefType buffer, std::optional< int64_t > lenElems, bool contiguous, llvm::ArrayRef< int64_t > innerSizes)
mlir::LogicalResult verifyDMAChannelIndex(mlir::Operation *op, TileLike tile, DMAChannelDir dir, int64_t channel)
struct TileID { friend std::ostream &operator<<(std::ostream &os, const TileID &s) { os<< "TileID("<< s.col<< ", "<< s.row<< ")"; return os; } friend std::string to_string(const TileID &s) { std::ostringstream ss; ss<< s; return ss.str(); } friend llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const TileID &s) { os<< to_string(s); return os; } inline bool operator<(const TileID &rhs) const { return std::tie(col, row)< std::tie(rhs.col, rhs.row); } bool operator==(const TileID &rhs) const { return std::tie(col, row)==std::tie(rhs.col, rhs.row); } bool operator!=(const TileID &rhs) const { return !(*this==rhs);} int col, row;} TileID
const AIETargetModel & getTargetModel(mlir::Operation *op)
mlir::LogicalResult verifyDMABDIteration(DMABDOp bd, AIETileType tileType)
The register layout of one tile type's DMA buffer descriptor.
llvm::SmallVector< std::pair< DmaBdField, DmaBdField >, 3 > padFields() const
The (before, after) zero-padding fields, innermost dimension first.