MLIR-AIE 1.4.6.dev
AIEDmaToNpu.cpp
Go to the documentation of this file.
1//===- AIEDmaToNpu.cpp ------------------------------------------*- C++ -*-===//
2//
3// Copyright (C) 2023 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7
15
16#include "mlir/Dialect/Arith/IR/Arith.h"
17#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
18#include "mlir/Dialect/MemRef/IR/MemRef.h"
19#include "mlir/Dialect/SCF/IR/SCF.h"
20#include "mlir/Pass/Pass.h"
21#include "mlir/Transforms/DialectConversion.h"
22#include "llvm/Support/MathExtras.h"
23#include <algorithm>
24#include <cstdint>
25
26namespace xilinx::AIEX {
27#define GEN_PASS_DEF_AIEDMATONPU
28#include "aie/Dialect/AIEX/Transforms/AIEXPasses.h.inc"
29} // namespace xilinx::AIEX
30
31using namespace mlir;
32using namespace xilinx;
33using namespace xilinx::AIEX;
34
35namespace {
36
37struct Write32SymToAddr : OpConversionPattern<NpuWrite32Op> {
38 using OpConversionPattern::OpConversionPattern;
39
40 Write32SymToAddr(MLIRContext *context, const AIE::NamedOpTable &names)
41 : OpConversionPattern(context), names(names) {}
42
43 const AIE::NamedOpTable &names;
44
45 LogicalResult
46 matchAndRewrite(NpuWrite32Op op, OpAdaptor adaptor,
47 ConversionPatternRewriter &rewriter) const override {
48
49 if (!op.getBuffer())
50 return failure();
51
52 std::optional<uint32_t> address = op.getAbsoluteAddress(&names);
53 if (!address.has_value()) {
54 return failure();
55 }
56
57 Value addressVal = createConstantI32(rewriter, op->getLoc(), *address);
58 rewriter.replaceOpWithNewOp<NpuWrite32Op>(
59 op, addressVal, adaptor.getValue(), nullptr, nullptr, nullptr);
60 return success();
61 }
62};
63
64struct BlockWriteSymToAddr : OpConversionPattern<NpuBlockWriteOp> {
65 using OpConversionPattern::OpConversionPattern;
66
67 BlockWriteSymToAddr(MLIRContext *context, const AIE::NamedOpTable &names)
68 : OpConversionPattern(context), names(names) {}
69
70 const AIE::NamedOpTable &names;
71
72 LogicalResult
73 matchAndRewrite(NpuBlockWriteOp op, OpAdaptor adaptor,
74 ConversionPatternRewriter &rewriter) const override {
75
76 if (!op.getBuffer())
77 return failure();
78
79 std::optional<uint32_t> address = op.getAbsoluteAddress(&names);
80 if (!address.has_value()) {
81 return failure();
82 }
83 rewriter.replaceOpWithNewOp<NpuBlockWriteOp>(op, *address, op.getData(),
84 nullptr, nullptr, nullptr);
85 return success();
86 }
87};
88
89struct MaskWrite32SymToAddr : OpConversionPattern<NpuMaskWrite32Op> {
90 using OpConversionPattern::OpConversionPattern;
91
92 MaskWrite32SymToAddr(MLIRContext *context, const AIE::NamedOpTable &names)
93 : OpConversionPattern(context), names(names) {}
94
95 const AIE::NamedOpTable &names;
96
97 LogicalResult
98 matchAndRewrite(NpuMaskWrite32Op op, OpAdaptor adaptor,
99 ConversionPatternRewriter &rewriter) const override {
100
101 if (!op.getBuffer())
102 return failure();
103
104 std::optional<uint32_t> absoluteAddress = op.getAbsoluteAddress(&names);
105 if (!absoluteAddress.has_value()) {
106 return failure();
107 }
108
109 Value addressVal =
110 createConstantI32(rewriter, op->getLoc(), *absoluteAddress);
111 rewriter.replaceOpWithNewOp<NpuMaskWrite32Op>(
112 op, addressVal, adaptor.getValue(), adaptor.getMask(), nullptr, nullptr,
113 nullptr);
114 return success();
115 }
116};
117
118struct MaskPollSymToAddr : OpConversionPattern<NpuMaskPollOp> {
119 using OpConversionPattern::OpConversionPattern;
120
121 MaskPollSymToAddr(MLIRContext *context, const AIE::NamedOpTable &names)
122 : OpConversionPattern(context), names(names) {}
123
124 const AIE::NamedOpTable &names;
125
126 LogicalResult
127 matchAndRewrite(NpuMaskPollOp op, OpAdaptor adaptor,
128 ConversionPatternRewriter &rewriter) const override {
129 if (!op.getBuffer())
130 return failure();
131
132 std::optional<uint32_t> absoluteAddress = op.getAbsoluteAddress(&names);
133 if (!absoluteAddress)
134 return failure();
135
136 Value addressVal =
137 createConstantI32(rewriter, op->getLoc(), *absoluteAddress);
138 rewriter.replaceOpWithNewOp<NpuMaskPollOp>(
139 op, addressVal, adaptor.getValue(), adaptor.getMask(), nullptr, nullptr,
140 nullptr);
141 return success();
142 }
143};
144
145struct RtpToWrite32Pattern : OpConversionPattern<NpuWriteRTPOp> {
146 using OpConversionPattern::OpConversionPattern;
147
148 RtpToWrite32Pattern(MLIRContext *context, const AIE::NamedOpTable &names)
149 : OpConversionPattern(context), names(names) {}
150
151 const AIE::NamedOpTable &names;
152
153 LogicalResult
154 matchAndRewrite(NpuWriteRTPOp op, OpAdaptor adaptor,
155 ConversionPatternRewriter &rewriter) const override {
156
157 auto buffer = names.lookup<AIE::BufferOp>(op.getBufferAttr().getAttr());
158 if (!buffer) {
159 op->emitError("buffer '" + op.getBuffer() + "' not found in device");
160 return failure();
161 }
162
163 auto bufferAddress = buffer.getAddress();
164 if (!bufferAddress) {
165 op->emitError("buffer must have address assigned");
166 return failure();
167 }
168 AIE::TileOp tile = buffer.getTileOp();
169
170 uint32_t idx = op.getIndex() * sizeof(uint32_t);
171 uint32_t address = *bufferAddress + idx;
172
173 NpuWrite32Op::create(rewriter, op->getLoc(),
174 createConstantI32(rewriter, op->getLoc(), address),
175 adaptor.getValue(), nullptr,
176 rewriter.getI32IntegerAttr(tile.getCol()),
177 rewriter.getI32IntegerAttr(tile.getRow()));
178
179 rewriter.eraseOp(op);
180 return success();
181 }
182};
183
184struct PushQueuetoWrite32Pattern : OpConversionPattern<NpuPushQueueOp> {
185
186public:
187 using OpConversionPattern::OpConversionPattern;
188
189 PushQueuetoWrite32Pattern(MLIRContext *context, PatternBenefit benefit = 1)
190 : OpConversionPattern(context, benefit) {}
191
192 LogicalResult
193 matchAndRewrite(NpuPushQueueOp op, OpAdaptor adaptor,
194 ConversionPatternRewriter &rewriter) const override {
195
196 const auto &tm = AIE::getTargetModel(op);
197 uint32_t ctrl_offset = tm.getDmaControlAddress(
198 op.getColumn(), op.getRow(), op.getChannel(), op.getDirection());
199
200 // control packet for issuing token
201 if (op.getIssueToken()) {
202 // set the task-complete-token controller ID field in the dma control
203 // register
204 AIE::TileOp shimTile = AIE::TileOp::getOrCreate(
205 rewriter, op->getParentOfType<AIE::DeviceOp>(), op.getColumn(),
206 op.getRow());
207 if (shimTile->hasAttr("controller_id")) {
208 AIE::PacketInfoAttr controller_id_attr =
209 shimTile->getAttrOfType<AIE::PacketInfoAttr>("controller_id");
210 uint32_t data = controller_id_attr.assignedId() << 8;
211 uint32_t mask = 0x00001F00;
212 NpuMaskWrite32Op::create(
213 rewriter, op->getLoc(),
214 createConstantI32(rewriter, op->getLoc(), ctrl_offset),
215 createConstantI32(rewriter, op->getLoc(), data),
216 createConstantI32(rewriter, op->getLoc(), mask), nullptr, nullptr,
217 nullptr);
218 }
219 }
220
221 uint32_t queue_offset = tm.getDmaQueueAddress(
222 op.getColumn(), op.getRow(), op.getChannel(), op.getDirection());
223
224 // Command word: bd_id in the START_BD_ID field, repeat_count [23:16],
225 // issue-token bit [31]. START_BD_ID is as wide as the tile's BD ids.
226 // bd_id and repeat_count may be runtime SSA; all-constant folds to one
227 // constant (byte-identical to the static path), else built with arith.
228 Location loc = op->getLoc();
229 auto i32ty = rewriter.getIntegerType(32);
230 uint32_t bdIdMask =
231 (1u << llvm::Log2_32_Ceil(tm.getNumBDs(op.getColumn(), op.getRow()))) -
232 1;
233 std::optional<uint32_t> bd_id = getConstantIntOperand(op.getBdId());
234 std::optional<uint64_t> repeat_cnt =
235 getConstantInt64Operand(op.getRepeatCount());
236 uint32_t issueBit = op.getIssueToken() ? 0x80000000 : 0;
237
238 // A scratchpad repeat count: the host writes it already shifted into the
239 // repeat field, and the update adds it to the rest of the command word.
240 // The verifier holds bd_id to a constant multiple of 4, which the
241 // update's clearing of bits [1:0] keeps.
242 if (IntegerAttr idx = op.getRepeatStateTableIdxAttr()) {
243 NpuUpdateFromScratchpadOp::create(
244 rewriter, loc, static_cast<uint8_t>(idx.getUInt()),
245 StateTableFunc::Incr,
246 // NOLINTNEXTLINE(bugprone-unchecked-optional-access)
247 /*func_arg=*/(*bd_id & bdIdMask) | issueBit,
248 /*address=*/queue_offset, /*buffer=*/nullptr, /*column=*/nullptr,
249 /*row=*/nullptr);
250 rewriter.eraseOp(op);
251 return success();
252 }
253
254 Value cmdVal;
255 if (bd_id && repeat_cnt) {
256 cmdVal = createConstantI32(rewriter, loc,
257 (*bd_id & bdIdMask) |
258 ((*repeat_cnt & 0xFF) << 16) | issueBit);
259 } else {
260 // (bd_id & bdIdMask) | ((repeat & 0xFF) << 16) | issueBit, as arith over
261 // the runtime operands (a constant field folds to its contribution).
262 // A runtime repeat_count is masked to its 8-bit field below, so an
263 // out-of-range value would silently wrap to a different number of
264 // executions. Refuse the dispatch instead (host-side guard); the
265 // unsigned compare also rejects a negative count (zero executions).
266 Value repeat;
267 if (repeat_cnt) {
268 repeat = createConstantI32(rewriter, loc, *repeat_cnt);
269 } else {
270 Value repeat64 =
271 getAsI64(rewriter, loc, OpFoldResult(op.getRepeatCount()));
272 if (!repeat64)
273 return failure();
274 uint32_t maxRepeat = tm.getMaxRepeatCount();
275 Value inRange = arith::CmpIOp::create(
276 rewriter, loc, arith::CmpIPredicate::ule, repeat64,
277 arith::ConstantOp::create(rewriter, loc,
278 rewriter.getI64IntegerAttr(maxRepeat)));
279 if (failed(emitRuntimeCheck(
280 rewriter, loc, inRange,
281 "a runtime DMA repeat count exceeds the task queue's [0:" +
282 Twine(maxRepeat) + "] range (at most " +
283 Twine(maxRepeat + 1) + " executions)")))
284 return failure();
285 repeat = rewriter.createOrFold<arith::TruncIOp>(loc, i32ty, repeat64);
286 }
287 Value cmd = createConstantI32(rewriter, loc, issueBit);
288 Value bdField = rewriter.createOrFold<arith::AndIOp>(
289 loc, getAsValue(rewriter, loc, op.getBdId(), i32ty),
290 createConstantI32(rewriter, loc, bdIdMask));
291 cmd = rewriter.createOrFold<arith::OrIOp>(loc, cmd, bdField);
292 Value masked = rewriter.createOrFold<arith::AndIOp>(
293 loc, repeat, createConstantI32(rewriter, loc, 0xFF));
294 Value shifted = rewriter.createOrFold<arith::ShLIOp>(
295 loc, masked, createConstantI32(rewriter, loc, 16));
296 cmdVal = rewriter.createOrFold<arith::OrIOp>(loc, cmd, shifted);
297 }
298
299 NpuWrite32Op::create(
300 rewriter, op->getLoc(),
301 createConstantI32(rewriter, op->getLoc(), queue_offset), cmdVal,
302 nullptr, nullptr, nullptr);
303 rewriter.eraseOp(op);
304 return success();
305 }
306};
307
308struct DmaToNpuPattern : OpConversionPattern<NpuDmaMemcpyNdOp> {
309 using OpConversionPattern::OpConversionPattern;
310
311public:
312 DmaToNpuPattern(MLIRContext *context, PatternBenefit benefit = 1)
313 : OpConversionPattern(context, benefit) {}
314
315 LogicalResult
316 matchAndRewrite(NpuDmaMemcpyNdOp op, OpAdaptor adaptor,
317 ConversionPatternRewriter &rewriter) const override {
318 // Any runtime (SSA) offset/size/stride takes the dynamic path; a
319 // fully-constant descriptor takes the static path below. The op verifier
320 // has already enforced the supported scope for the dynamic case (shim NOC,
321 // no padding, realizable/in-range constants).
322 bool allOffsetsConstant =
323 llvm::all_of(op.getMixedOffsets(),
324 [](OpFoldResult s) { return getConstantIntValue(s); });
325 bool allSizesConstant =
326 llvm::all_of(op.getMixedSizes(),
327 [](OpFoldResult s) { return getConstantIntValue(s); });
328 bool allStridesConstant =
329 llvm::all_of(op.getMixedStrides(),
330 [](OpFoldResult s) { return getConstantIntValue(s); });
331 if (!allOffsetsConstant || !allSizesConstant || !allStridesConstant)
332 return lowerDynamic(op, adaptor, rewriter);
333
334 const auto &targetModel = AIE::getTargetModel(op);
335 BaseMemRefType bufferType = op.getMemref().getType();
336 auto *ctx = op->getContext();
337 auto i32ty = IntegerType::get(ctx, 32);
338 auto zero = IntegerAttr::get(i32ty, 0);
339
340 auto dev = op->getParentOfType<AIE::DeviceOp>();
341 if (!dev)
342 return failure();
343
344 auto infoOp = AIE::ShimDMAAllocationOp::getForSymbol(
345 dev, op.getMetadata().getRootReference());
346 if (!infoOp) {
347 return op->emitOpError("couldn't find shim_dma_allocation op.");
348 }
349
350 AIE::TileOp shimTile = infoOp.getTileOp();
351 if (!shimTile) {
352 return op->emitOpError(
353 "shim_dma_allocation op must reference a valid TileOp.");
354 }
355
356 auto channelDir = infoOp.getChannelDir();
357 bool isMM2S = channelDir == AIE::DMAChannelDir::MM2S;
358 int tileCol = shimTile.getCol();
359 int tileRow = shimTile.getRow();
360
361 // initialize fields to zero
362 auto column = zero;
363 auto bd_id = zero;
364 auto buffer_length = zero;
365 auto buffer_offset = zero;
366 auto enable_packet = zero;
367 auto out_of_order_id = zero;
368 auto packet_id = zero;
369 auto packet_type = zero;
370 auto d0_size = zero;
371 auto d0_stride = zero;
372 auto d1_size = zero;
373 auto d1_stride = zero;
374 auto d2_size = zero;
375 auto d2_stride = zero;
376 auto iteration_current = zero;
377 auto iteration_size = zero;
378 auto iteration_stride = zero;
379 auto next_bd = zero;
380 auto row = zero;
381 auto use_next_bd = zero;
382 auto valid_bd = zero;
383 auto lock_rel_val = zero;
384 auto lock_rel_id = zero;
385 auto lock_acq_enable = zero;
386 auto lock_acq_val = zero;
387 auto lock_acq_id = zero;
388 auto d0_zero_before = zero;
389 auto d1_zero_before = zero;
390 auto d2_zero_before = zero;
391 auto d0_zero_after = zero;
392 auto d1_zero_after = zero;
393 auto d2_zero_after = zero;
394 auto burst_length = zero;
395 auto axcache = zero;
396
397 auto issue_token = BoolAttr::get(ctx, false);
398 auto repeat_count = zero;
399 llvm::SmallVector<int64_t, 4> inputSizes = llvm::map_to_vector(
400 llvm::reverse(op.getMixedSizes()),
401 [](OpFoldResult s) { return getConstantIntValue(s).value(); });
402 llvm::SmallVector<int64_t, 4> inputStrides = llvm::map_to_vector(
403 llvm::reverse(op.getMixedStrides()),
404 [](OpFoldResult s) { return getConstantIntValue(s).value(); });
405 // A contiguous row-major ND access on a shim NOC tile is lowered to linear
406 // mode (d0_size=d1_size=0) just like an already-canonical linear transfer.
407 // This allows naturally-expressed multidimensional transfers (e.g., a 2D
408 // image as [height, width]) without hitting the 10-bit ND wrap-size limit.
409 bool isLinear = isLinearModeTransfer(targetModel, tileCol, tileRow,
410 inputSizes, inputStrides);
411 if (!isLinear && op.getLengthStateTableIdxAttr())
412 AIE::placeRuntimeLengthDimension(inputSizes, inputStrides);
413 llvm::SmallVector<int64_t, 4> sizes(4);
414 llvm::SmallVector<int64_t, 4> strides(4);
415 getHardwareStridesWraps(targetModel, op, bufferType, inputSizes,
416 inputStrides, sizes, strides);
417
418 // column
419 column = IntegerAttr::get(i32ty, tileCol);
420
421 // row
422 row = IntegerAttr::get(i32ty, tileRow);
423
424 if (failed(verifyStridesWraps(
425 op, [&] { return op->emitOpError(); }, bufferType, tileCol, tileRow,
426 inputSizes, inputStrides, sizes, strides, isLinear))) {
427 return failure();
428 }
429
430 // bd_id
431 bd_id = IntegerAttr::get(i32ty, op.getId());
432
433 // buffer_length
434 uint64_t buffer_length_val = inputSizes[0] * op.getElementTypeBitwidth() /
435 targetModel.getAddressGenGranularity();
436 if (inputSizes.size() > 1) {
437 for (size_t i = 1; i < std::min(inputSizes.size(), (size_t)3); i++) {
438 buffer_length_val *= inputSizes[i];
439 }
440 }
441 buffer_length = IntegerAttr::get(i32ty, buffer_length_val);
442
443 // buffer_offset - zero because the complete address is set by the patch op
444 buffer_offset = IntegerAttr::get(i32ty, 0);
445
446 // enable_packet
447 if (auto packetInfo =
448 op.getPacket() ? op.getPacket() : infoOp.getPacket()) {
449 enable_packet = IntegerAttr::get(i32ty, 1);
450 packet_type = IntegerAttr::get(i32ty, packetInfo->getPktType());
451 packet_id = IntegerAttr::get(i32ty, packetInfo->assignedId());
452 }
453
454 // out_of_order_id - stays 0; senders stamp it via aie.dma_bd,
455 // dma_configure_task, or npu.writebd
456
457 if (!isLinear) {
458 // d0_size, d0_stride
459 d0_size = IntegerAttr::get(i32ty, sizes[0]);
460 d0_stride = IntegerAttr::get(i32ty, strides[0]);
461
462 // d1_size, d1_stride
463 d1_size = IntegerAttr::get(i32ty, sizes[1]);
464 d1_stride = IntegerAttr::get(i32ty, strides[1]);
465
466 // d2_stride
467 d2_stride = IntegerAttr::get(i32ty, strides[2]);
468
469 // d2_size
470 if (targetModel.isMemTile(tileCol, 0)) // Need to be any row
471 d2_size = IntegerAttr::get(i32ty, sizes[2]);
472 else
473 d2_size = IntegerAttr::get(i32ty, 0);
474 }
475 // iteration_current, iteration_size, iteration_stride, repeat_count
476 if (inputSizes[3] > 1) {
477 if (inputStrides[3] > 0) {
478 iteration_size = IntegerAttr::get(i32ty, sizes[3]);
479 iteration_stride = IntegerAttr::get(i32ty, strides[3]);
480 } else {
481 // We allow users to encode the repeat_count as a dimension 3 stride
482 // of 0. This must lower to a iteration wrap of 0, so no stride is
483 // ever added. We then repeat the BD using the repeat_count in
484 // NpuPushQueueOp.
485 iteration_size = zero;
486 iteration_stride = zero;
487 }
488 }
489 repeat_count = IntegerAttr::get(i32ty, sizes[3]);
490
491 // next_bd
492
493 // use_next_bd
494
495 // valid_bd
496 valid_bd = IntegerAttr::get(i32ty, 1);
497
498 // lock_rel_val
499
500 // lock_rel_id
501
502 // lock_acq_enable
503
504 // lock_acq_val
505
506 // lock_acq_id
507
508 // d0_zero_before
509 d0_zero_before = IntegerAttr::get(i32ty, op.getD0ZeroBefore());
510
511 // d1_zero_before
512 d1_zero_before = IntegerAttr::get(i32ty, op.getD1ZeroBefore());
513
514 // d2_zero_before
515 d2_zero_before = IntegerAttr::get(i32ty, op.getD2ZeroBefore());
516
517 // d0_zero_after
518 d0_zero_after = IntegerAttr::get(i32ty, op.getD0ZeroAfter());
519
520 // d1_zero_after
521 d1_zero_after = IntegerAttr::get(i32ty, op.getD1ZeroAfter());
522
523 // d2_zero_after
524 d2_zero_after = IntegerAttr::get(i32ty, op.getD2ZeroAfter());
525
526 // burst_size
527 burst_length = IntegerAttr::get(i32ty, op.getBurstLength());
528
529 // axcache; only meaningful on the AXI-MM side, so left unset elsewhere to
530 // match the dma_task path (and NpuWriteBdOp's verifier).
531 if (targetModel.isShimNOCTile(tileCol, tileRow))
532 axcache = IntegerAttr::get(i32ty, op.getAxcacheOrDefault());
533 else
534 axcache = IntegerAttr();
535
536 // Set the issue_token
537 issue_token = BoolAttr::get(ctx, op.getIssueToken());
538 // Earlier, all S2MM channels were implicitly assumed to issue a token.
539 // This logic is kept for now for backward compatibility.
540 if (!isMM2S)
541 issue_token = BoolAttr::get(ctx, true);
542
543 if (targetModel.isMemTile(tileCol, tileRow) && (!isMM2S) &&
544 (op.getD0ZeroBefore() != 0 || op.getD0ZeroAfter() != 0 ||
545 op.getD1ZeroBefore() != 0 || op.getD1ZeroAfter() != 0 ||
546 op.getD2ZeroBefore() != 0 || op.getD2ZeroAfter() != 0)) {
547 op->emitOpError("MemTile supports zero padding only on MM2S direction");
548 return failure();
549 }
550
551 // write the buffer descriptor to the array
552 NpuWriteBdOp::create(
553 rewriter, op->getLoc(), column, bd_id, buffer_length, buffer_offset,
554 enable_packet, out_of_order_id, packet_id, packet_type, d0_size,
555 d0_stride, d1_size, d1_stride, d2_size, d2_stride, iteration_current,
556 iteration_size, iteration_stride, next_bd, row, use_next_bd, valid_bd,
557 lock_rel_val, lock_rel_id, lock_acq_enable, lock_acq_val, lock_acq_id,
558 d0_zero_before, d1_zero_before, d2_zero_before, d0_zero_after,
559 d1_zero_after, d2_zero_after, burst_length, axcache);
560
561 // Resolve the buffer's runtime-sequence arg and emit the address patch
562 // (plus any offset-state update).
563 int arg_idx = -1;
564 if (failed(emitBufferAddressPatch(op, adaptor, rewriter, tileCol, tileRow,
565 arg_idx)))
566 return failure();
567
568 // A length_state_table_idx adds the runtime length to the BD's
569 // Buffer_Length, after the BD write above has set the static length. The
570 // verifier requires a length_unit with it.
571 auto lengthUnit = op.getLengthUnit();
572 if (op.getLengthStateTableIdxAttr() && lengthUnit) {
573 if (failed(emitUpdateBdLengthFromParameter(rewriter, op, bufferType,
574 *lengthUnit, targetModel,
575 tileCol, tileRow, op.getId())))
576 return failure();
577 }
578
579 // push the patched bd onto the dma task queue. bd_id and repeat_count are
580 // SSA operands; materialize them as constants here (the static path).
581 NpuPushQueueOp::create(
582 rewriter, op->getLoc(), column, row, infoOp.getChannelDirAttr(),
583 infoOp.getChannelIndexAttr(), issue_token,
584 createConstantI32(rewriter, op->getLoc(),
585 static_cast<uint32_t>(repeat_count.getInt())),
586 createConstantI32(rewriter, op->getLoc(),
587 static_cast<uint32_t>(bd_id.getInt())),
588 /*repeat_state_table_idx=*/nullptr);
589
590 rewriter.eraseOp(op);
591 return success();
592 }
593
594 // Resolve the runtime-sequence argument index the descriptor's buffer traces
595 // back to, and emit the address-patch (plus an offset-state update, if the op
596 // carries one) that binds the runtime buffer pointer into the BD. Shared by
597 // the static and dynamic lowering paths, which are otherwise identical here.
598 // A walk with any runtime offset, size or stride also gets a host-side check
599 // that it stays inside the host buffer. On success `argIdx` receives the
600 // resolved index.
601 LogicalResult emitBufferAddressPatch(NpuDmaMemcpyNdOp op, OpAdaptor adaptor,
602 ConversionPatternRewriter &rewriter,
603 int tileCol, int tileRow,
604 int &argIdx) const {
605 AIE::RuntimeSequenceOp seqOp =
606 op->getParentOfType<AIE::RuntimeSequenceOp>();
607 if (!seqOp)
608 return op->emitOpError("NpuDmaMemcpyNdOps must have RuntimeSequenceOp "
609 "parent at time of lowering.");
610 auto traceResult = traceSubviewToBlockArgument(adaptor.getMemref());
611 if (!traceResult)
612 return op->emitOpError(
613 "memref must be a block argument or subview/cast/reinterpret_cast of "
614 "a block argument with static offsets, sizes, and strides");
615 std::optional<unsigned> hostIdx =
616 getHostBufferArgIndex(traceResult->rootArg);
617 if (!hostIdx || traceResult->rootArg.getOwner() != &seqOp.getBody().front())
618 return failure();
619 argIdx = static_cast<int>(*hostIdx);
620
621 const auto &targetModel = AIE::getTargetModel(op);
622 uint64_t patchAddr =
623 targetModel.getDmaBdAddress(tileCol, tileRow, op.getId()) +
624 targetModel.getDmaBdAddressOffset(tileCol, tileRow);
625
626 // arg_plus is the buffer byte offset. Constant offsets fold to a constant
627 // (byte-identical to the static path); a runtime offset operand is built
628 // with arith so it flows into the patch instead of being rejected. The
629 // subview trace contributes a constant base byte offset.
630 SmallVector<OpFoldResult> offsets = op.getMixedOffsets();
631 SmallVector<OpFoldResult> strides = op.getMixedStrides();
632 int64_t elemBytes = op.getElementTypeBitwidth() / 8;
633 SmallVector<OpFoldResult> sizes = op.getMixedSizes();
634 bool isRuntime =
635 llvm::any_of(llvm::concat<OpFoldResult>(offsets, sizes, strides),
636 [](OpFoldResult v) { return !getConstantIntValue(v); });
637 if (isRuntime && failed(guardWithinHostBuffer(
638 rewriter, op->getLoc(),
639 cast<BaseMemRefType>(traceResult->rootArg.getType()),
640 traceResult->offsetInBytes, elemBytes, offsets,
641 strides, sizes, strides)))
642 return failure();
643 Value argPlus = buildArgPlusValue(
644 rewriter, op->getLoc(), offsets, strides, elemBytes,
645 traceResult->offsetInBytes, targetModel.getAddressGenGranularity() / 8);
646 if (!argPlus)
647 return failure();
648 NpuAddressPatchOp::create(rewriter, op->getLoc(), patchAddr,
649 /*addr_val=*/Value(), argIdx, argPlus);
650
651 // If this DMA op has an offset_state_table_idx, emit an
652 // update_from_scratchpad to add the runtime offset to the BD address
653 // register (additive; applied after the base patch above).
654 if (op.getOffsetStateTableIdxAttr()) {
655 auto bufType = cast<BaseMemRefType>(op.getMemref().getType());
657 rewriter, op, bufType, patchAddr, targetModel, tileCol, tileRow)))
658 return failure();
659 }
660 return success();
661 }
662
663 // Lower a shim-NOC dma_memcpy_nd carrying runtime (SSA) offsets/sizes/
664 // strides. The encoder runs the same arithmetic as the static path, so a
665 // runtime value equal to a constant yields the same word. aiebu requires the
666 // block-write to precede the address patch and cover the patched word.
667 LogicalResult lowerDynamic(NpuDmaMemcpyNdOp op, OpAdaptor adaptor,
668 ConversionPatternRewriter &rewriter) const {
669 const auto &targetModel = AIE::getTargetModel(op);
670 auto *ctx = op->getContext();
671 auto i32ty = IntegerType::get(ctx, 32);
672 Location loc = op->getLoc();
673
674 auto dev = op->getParentOfType<AIE::DeviceOp>();
675 if (!dev)
676 return failure();
677 auto infoOp = AIE::ShimDMAAllocationOp::getForSymbol(
678 dev, op.getMetadata().getRootReference());
679 if (!infoOp)
680 return op->emitOpError("couldn't find shim_dma_allocation op.");
681 AIE::TileOp shimTile = infoOp.getTileOp();
682 if (!shimTile)
683 return op->emitOpError(
684 "shim_dma_allocation op must reference a valid TileOp.");
685 auto channelDir = infoOp.getChannelDir();
686 bool isMM2S = channelDir == AIE::DMAChannelDir::MM2S;
687 int tileCol = shimTile.getCol();
688 int tileRow = shimTile.getRow();
689
690 // Packet / token setup (identical to the static path).
691 auto column = IntegerAttr::get(i32ty, tileCol);
692 auto row = IntegerAttr::get(i32ty, tileRow);
693 BdTemplateFields fields;
694 if (auto packetInfo =
695 op.getPacket() ? op.getPacket() : infoOp.getPacket()) {
696 fields.enable_packet = 1;
697 fields.packet_type = packetInfo->getPktType();
698 fields.packet_id = packetInfo->assignedId();
699 }
700 auto issue_token = BoolAttr::get(ctx, op.getIssueToken());
701 if (!isMM2S)
702 issue_token = BoolAttr::get(ctx, true);
703
704 // buffer_length is the size-product here, hence no explicit length; the
705 // encoder returns the hw repeat_count for the queue push.
706 SmallVector<Value> words;
707 Value repeatCount;
708 if (failed(buildBdWords(rewriter, loc, targetModel, tileCol, tileRow,
709 fields, op.getMixedSizes(), op.getMixedStrides(),
710 op.getElementTypeBitwidth(), op.getBurstLength(),
711 op.getAxcacheOrDefault(),
712 /*lenElems=*/OpFoldResult(), repeatCount, words)))
713 return failure();
714 Value bdBase =
715 getBdRegisterBase(rewriter, loc, targetModel, tileCol, tileRow,
716 rewriter.getI32IntegerAttr(op.getId()));
717 NpuBlockWriteValuesOp::create(rewriter, loc, bdBase, words);
718
719 // Address patch for the buffer pointer; emitBufferAddressPatch folds a
720 // constant offset or builds a runtime arg_plus with arith as needed.
721 int arg_idx = -1;
722 if (failed(emitBufferAddressPatch(op, adaptor, rewriter, tileCol, tileRow,
723 arg_idx)))
724 return failure();
725
726 NpuPushQueueOp::create(
727 rewriter, loc, column, row, infoOp.getChannelDirAttr(),
728 infoOp.getChannelIndexAttr(), issue_token, repeatCount,
729 createConstantI32(rewriter, loc, op.getId()),
730 /*repeat_state_table_idx=*/nullptr);
731
732 rewriter.eraseOp(op);
733 return success();
734 }
735};
736
737/// Convert NpuDmaWaitOp into NpuSyncOp by retrieving the necessary
738/// information from the ShimDMAAllocationOp referenced through the
739/// symbol argument of this op.
740struct DmaWaitToSyncPattern : OpConversionPattern<NpuDmaWaitOp> {
741
742public:
743 using OpConversionPattern::OpConversionPattern;
744
745 DmaWaitToSyncPattern(MLIRContext *context, PatternBenefit benefit = 1)
746 : OpConversionPattern(context, benefit) {}
747
748 LogicalResult
749 matchAndRewrite(NpuDmaWaitOp op, OpAdaptor adaptor,
750 ConversionPatternRewriter &rewriter) const override {
751 AIE::DeviceOp dev = op->getParentOfType<AIE::DeviceOp>();
752 if (!dev)
753 return op->emitError("couldn't find parent of type DeviceOp");
754
755 AIE::ShimDMAAllocationOp shimDmaAllocOp =
756 AIE::ShimDMAAllocationOp::getForSymbol(dev, op.getSymbol());
757 if (!shimDmaAllocOp) {
758 return op->emitError("couldn't find shim_dma_allocation op");
759 }
760
761 AIE::TileOp shimTile = shimDmaAllocOp.getTileOp();
762 if (!shimTile) {
763 return op->emitError(
764 "shim_dma_allocation op must reference a valid TileOp");
765 }
766
767 // Create with `column_num == 1` and `row_num == 1` to check for a single
768 // column and row.
769 Location loc = op->getLoc();
770 (void)rewriter.replaceOpWithNewOp<NpuSyncOp>(
771 op, createConstantI32(rewriter, loc, shimTile.getCol()),
772 createConstantI32(rewriter, loc, shimTile.getRow()),
774 rewriter, loc,
775 static_cast<uint32_t>(shimDmaAllocOp.getChannelDir())),
776 createConstantI32(rewriter, loc, shimDmaAllocOp.getChannelIndex()),
777 createConstantI32(rewriter, loc, 1),
778 createConstantI32(rewriter, loc, 1));
779
780 return success();
781 }
782};
783
784struct WriteBdToBlockWritePattern : OpConversionPattern<NpuWriteBdOp> {
785 using OpConversionPattern::OpConversionPattern;
786
787public:
788 WriteBdToBlockWritePattern(MLIRContext *context,
789 BlockwriteData &blockwriteData,
790 PatternBenefit benefit = 1)
791 : OpConversionPattern(context, benefit), blockwriteData(blockwriteData) {}
792
793 BlockwriteData &blockwriteData;
794
795 LogicalResult
796 matchAndRewrite(NpuWriteBdOp op, OpAdaptor adaptor,
797 ConversionPatternRewriter &rewriter) const override {
798
799 AIE::DeviceOp dev = op->getParentOfType<AIE::DeviceOp>();
800 const AIE::AIETargetModel &tm = dev.getTargetModel();
801 int col = op.getColumn();
802 int row = op.getRow();
803
804 const AIE::DmaBdLayout *layout = tm.getDmaBdLayout(col, row);
805 if (!layout)
806 return op->emitOpError("has no buffer descriptor layout on this tile");
807 if (!tm.isMemTile(col, row) &&
808 (op.getD0ZeroBefore() || op.getD1ZeroBefore() || op.getD2ZeroBefore() ||
809 op.getD0ZeroAfter() || op.getD1ZeroAfter() || op.getD2ZeroAfter()))
810 return op->emitOpError("Zero padding is only available on MemTile");
811
812 std::vector<uint32_t> words(layout->numWords, 0);
813 auto set = [&](const AIE::DmaBdField &field, uint64_t value) {
814 if (field.exists())
815 words[field.word] |= field.place(value);
816 };
817 uint64_t bufferOffset = op.getBufferOffset();
818 if (tm.isCoreTile(col, row))
819 bufferOffset /= 4;
820 set(layout->bufferLength, op.getBufferLength());
821 set(layout->bufferOffset, bufferOffset);
822 set(layout->enablePacket, op.getEnablePacket());
823 set(layout->packetType, op.getPacketType());
824 set(layout->packetId, op.getPacketId());
825 set(layout->outOfOrderId, op.getOutOfOrderId());
826 set(layout->d0Size, op.getD0Size());
827 set(layout->d0Stride, op.getD0Stride());
828 set(layout->d1Size, op.getD1Size());
829 set(layout->d1Stride, op.getD1Stride());
830 set(layout->d2Stride, op.getD2Stride());
831 set(layout->iterationCurrent, op.getIterationCurrent());
832 set(layout->iterationSize, op.getIterationSize());
833 set(layout->iterationStride, op.getIterationStride());
834 set(layout->d0ZeroBefore, op.getD0ZeroBefore());
835 set(layout->d1ZeroBefore, op.getD1ZeroBefore());
836 set(layout->d2ZeroBefore, op.getD2ZeroBefore());
837 set(layout->d0ZeroAfter, op.getD0ZeroAfter());
838 set(layout->d1ZeroAfter, op.getD1ZeroAfter());
839 set(layout->d2ZeroAfter, op.getD2ZeroAfter());
840 if (layout->burstLength.exists())
841 set(layout->burstLength,
842 getShimBurstLengthEncoding(tm, op.getBurstLength()));
843 set(layout->axcache, op.getAxcacheOrDefault());
844 set(layout->nextBd, op.getNextBd());
845 set(layout->useNextBd, op.getUseNextBd());
846 set(layout->validBd, op.getValidBd());
847 set(layout->lockRelValue, op.getLockRelVal());
848 set(layout->lockRelId, op.getLockRelId());
849 set(layout->lockAcqEnable, op.getLockAcqEnable());
850 set(layout->lockAcqValue, op.getLockAcqVal());
851 set(layout->lockAcqId, op.getLockAcqId());
852
853 uint32_t bd_id = op.getBdId();
854 uint64_t bd_addr = tm.getDmaBdAddress(col, row, bd_id);
855
856 memref::GlobalOp global = nullptr;
857 {
858 OpBuilder::InsertionGuard guard(rewriter);
859 rewriter.setInsertionPoint(op->getParentOfType<AIE::RuntimeSequenceOp>());
860 global = blockwriteData.getOrCreate(rewriter, op.getLoc(), words);
861 }
862 auto memref = memref::GetGlobalOp::create(
863 rewriter, op.getLoc(), global.getType(), global.getName());
864
865 (void)rewriter.replaceOpWithNewOp<NpuBlockWriteOp>(
866 op, rewriter.getUI32IntegerAttr(bd_addr), memref.getResult(), nullptr,
867 nullptr, nullptr);
868 return success();
869 }
870};
871
872// Count all task starts at their common representation, after memcpy lowering
873// but before pushes become register writes. This includes starts lowered by
874// dma-tasks-to-npu and compiler-generated channel rearm pushes, and so every
875// path's tokens: no sequence may leave one on a channel any of them awaits.
876static LogicalResult checkQueueDepth(AIE::DeviceOp device,
877 bool enforceQueueDepth) {
878 const AIE::AIETargetModel &tm = device.getTargetModel();
879 SymbolTable symbols(device);
880 auto queueEffectOf = [&](Operation *op) {
881 return rawQueueEffect(op, symbols);
882 };
883 std::set<DmaQueueModel::ChannelKey> awaited;
884 SmallVector<SyncCover> mayAwait;
885 device.walk([&](Operation *op) {
886 QueueEffect e = queueEffectOf(op);
888 awaited.insert(e.key);
889 else if (auto sync = dyn_cast<NpuSyncOp>(op))
890 mayAwait.push_back(syncCover(sync));
891 });
892 bool consumed = true;
893 device.walk([&](AIE::RuntimeSequenceOp seq) {
894 DmaQueueModel queue;
895 // No dma_free_task survives to this lowering stage; nothing to check here.
896 guardSequenceQueueDepth(seq.getBody(), queue, tm, enforceQueueDepth,
897 /*warnUnsafeBdReuse=*/false, queueEffectOf);
898 seq->removeAttr(queueDiagnosedAttr);
899 consumed &=
900 succeeded(checkTokensConsumed(seq, awaited, mayAwait, queueEffectOf));
901 });
902 return success(consumed);
903}
904
905struct AIEDmaToNpuPass : xilinx::AIEX::impl::AIEDmaToNpuBase<AIEDmaToNpuPass> {
906 using Base = xilinx::AIEX::impl::AIEDmaToNpuBase<AIEDmaToNpuPass>;
907 AIEDmaToNpuPass() = default;
908 AIEDmaToNpuPass(const AIEDmaToNpuOptions &options) : Base(options) {}
909
910 void runOnOperation() override {
911
912 AIE::DeviceOp device = getOperation();
913
914 ConversionTarget target(getContext());
915 target.addLegalDialect<AIEXDialect>();
916 target.addLegalDialect<memref::MemRefDialect>();
917 target.addLegalDialect<arith::ArithDialect>();
918 target.addLegalOp<cf::AssertOp>();
919 target.addLegalOp<AIE::BufferOp>();
920 target.addLegalOp<AIE::ShimDMAAllocationOp>();
921 target.addLegalOp<AIE::TileOp>();
922
923 target.addIllegalOp<NpuDmaMemcpyNdOp>();
924 target.addIllegalOp<NpuDmaWaitOp>();
925 target.addIllegalOp<NpuWriteRTPOp>();
926 target.addIllegalOp<NpuWriteBdOp>();
927 target.addDynamicallyLegalOp<NpuWrite32Op>(
928 [&](NpuWrite32Op op) { return !op.getBuffer(); });
929 target.addDynamicallyLegalOp<NpuBlockWriteOp>(
930 [&](NpuBlockWriteOp op) { return !op.getBuffer(); });
931 target.addDynamicallyLegalOp<NpuMaskWrite32Op>(
932 [&](NpuMaskWrite32Op op) { return !op.getBuffer(); });
933 target.addDynamicallyLegalOp<NpuMaskPollOp>(
934 [&](NpuMaskPollOp op) { return !op.getBuffer(); });
935
936 BlockwriteData blockwriteData(device, "blockwrite_data_");
937 AIE::NamedOpTable names(device);
938
939 RewritePatternSet patterns(&getContext());
940 patterns.insert<BlockWriteSymToAddr>(&getContext(), names);
941 patterns.insert<DmaToNpuPattern>(&getContext());
942 patterns.insert<DmaWaitToSyncPattern>(&getContext());
943 patterns.insert<MaskWrite32SymToAddr>(&getContext(), names);
944 patterns.insert<MaskPollSymToAddr>(&getContext(), names);
945 patterns.insert<RtpToWrite32Pattern>(&getContext(), names);
946 patterns.insert<Write32SymToAddr>(&getContext(), names);
947 patterns.insert<WriteBdToBlockWritePattern>(&getContext(), blockwriteData);
948
949 // The driver converts the illegal ops alone; started from the device, it
950 // would visit every op of the runtime sequences.
951 SmallVector<Operation *> illegal;
952 device.walk([&](Operation *op) {
953 if (target.isIllegal(op))
954 illegal.push_back(op);
955 });
956 ConversionConfig config;
957 config.foldingMode = DialectConversionFoldingMode::Never;
958 config.allowPatternRollback = false;
959 if (failed(applyPartialConversion(illegal, target, std::move(patterns),
960 config))) {
961 signalPassFailure();
962 return;
963 }
964
965 if (failed(checkQueueDepth(device, enforceQueueDepth))) {
966 signalPassFailure();
967 return;
968 }
969
970 target.addIllegalOp<NpuPushQueueOp>();
971 RewritePatternSet pushPatterns(&getContext());
972 pushPatterns.insert<PushQueuetoWrite32Pattern>(&getContext());
973 SmallVector<Operation *> pushes;
974 device.walk([&](NpuPushQueueOp op) { pushes.push_back(op); });
975 if (failed(applyPartialConversion(pushes, target, std::move(pushPatterns),
976 config)))
977 signalPassFailure();
978
979 eraseDeadArith(device);
980 }
981};
982
983} // namespace
984
985std::unique_ptr<OperationPass<AIE::DeviceOp>> AIEX::createAIEDmaToNpuPass() {
986 return std::make_unique<AIEDmaToNpuPass>();
987}
988
989std::unique_ptr<OperationPass<AIE::DeviceOp>>
990AIEX::createAIEDmaToNpuPass(const AIEDmaToNpuOptions &options) {
991 return std::make_unique<AIEDmaToNpuPass>(options);
992}
Tracks, per DMA channel, the pushes not yet known to have completed along one execution path.
mlir::LogicalResult verifyStridesWraps(mlir::Operation *forOp, llvm::function_ref< mlir::InFlightDiagnostic()> emitError, mlir::BaseMemRefType referencedBufType, int tileCol, int tileRow, llvm::SmallVector< int64_t, 4 > inputSizes, llvm::SmallVector< int64_t, 4 > inputStrides, llvm::SmallVector< int64_t, 4 > hardwareSizes, llvm::SmallVector< int64_t, 4 > hardwareStrides, bool skipTransformationChecks=false)
std::unique_ptr< mlir::OperationPass< AIE::DeviceOp > > createAIEDmaToNpuPass()
std::optional< SubviewTraceResult > traceSubviewToBlockArgument(Value value)
Definition AIEUtils.cpp:25
QueueEffect rawQueueEffect(mlir::Operation *op, mlir::SymbolTable &symbols)
What a push or an await that names no task does to its channel: an aiex.npu.push_queue; an aiex....
constexpr llvm::StringLiteral queueDiagnosedAttr
mlir::Value buildArgPlusValue(mlir::OpBuilder &builder, mlir::Location loc, llvm::ArrayRef< mlir::OpFoldResult > elementOffsets, llvm::ArrayRef< mlir::OpFoldResult > strides, int64_t elemWidthBytes, int64_t baseByteOffset, uint32_t granuleBytes)
std::optional< uint64_t > getConstantInt64Operand(mlir::Value v)
mlir::LogicalResult checkTokensConsumed(AIE::RuntimeSequenceOp sequence, const std::set< DmaQueueModel::ChannelKey > &awaited, llvm::ArrayRef< SyncCover > mayAwait, EffectFn effectOf)
Check that sequence ends, on every path, with no task-completion token outstanding on a channel some ...
SyncCover syncCover(NpuSyncOp sync)
mlir::Value getAsValue(mlir::OpBuilder &builder, mlir::Location loc, mlir::OpFoldResult ofr, mlir::Type intType)
mlir::LogicalResult guardWithinHostBuffer(mlir::OpBuilder &builder, mlir::Location loc, mlir::BaseMemRefType hostBufferType, int64_t baseByteOffset, int64_t elemWidthBytes, llvm::ArrayRef< mlir::OpFoldResult > offsets, llvm::ArrayRef< mlir::OpFoldResult > offsetStrides, llvm::ArrayRef< mlir::OpFoldResult > sizes, llvm::ArrayRef< mlir::OpFoldResult > strides)
mlir::LogicalResult buildBdWords(mlir::OpBuilder &builder, mlir::Location loc, const xilinx::AIE::AIETargetModel &targetModel, int tileCol, int tileRow, const BdTemplateFields &fields, llvm::ArrayRef< mlir::OpFoldResult > mixedSizes, llvm::ArrayRef< mlir::OpFoldResult > mixedStrides, uint64_t elemWidth, uint32_t burstLength, uint32_t axcache, mlir::OpFoldResult lenElems, mlir::Value &repeatCountOut, llvm::SmallVectorImpl< mlir::Value > &wordsOut)
LogicalResult emitUpdateBdAddressFromOffsetParameter(OpBuilder &builder, Operation *bdOp, BaseMemRefType bufType, uint64_t registerAddr, const AIE::AIETargetModel &targetModel, int col, int row, std::optional< uint64_t > staticByteAddress=std::nullopt, OpFoldResult lengthField={})
Definition AIEUtils.cpp:214
std::optional< uint32_t > getConstantIntOperand(mlir::Value v)
mlir::Value getAsI64(mlir::OpBuilder &builder, mlir::Location loc, mlir::OpFoldResult ofr)
void eraseDeadArith(mlir::Operation *root)
void guardSequenceQueueDepth(mlir::Region &body, DmaQueueModel &queue, const AIE::AIETargetModel &tm, bool enforce, bool warnUnsafeBdReuse, EffectFn effectOf)
Guard every push in a runtime sequence that can land on a full queue, and (when warnUnsafeBdReuse) wa...
void getHardwareStridesWraps(const AIE::AIETargetModel &targetModel, mlir::Operation *op, mlir::BaseMemRefType referencedBufType, llvm::SmallVector< int64_t, 4 > inputSizes, llvm::SmallVector< int64_t, 4 > inputStrides, llvm::SmallVector< int64_t, 4 > &sizes, llvm::SmallVector< int64_t, 4 > &strides)
bool isLinearModeTransfer(const AIE::AIETargetModel &targetModel, int col, int row, llvm::ArrayRef< int64_t > sizes, llvm::ArrayRef< int64_t > strides)
mlir::Value getBdRegisterBase(mlir::OpBuilder &builder, mlir::Location loc, const xilinx::AIE::AIETargetModel &targetModel, int tileCol, int tileRow, mlir::OpFoldResult bdId)
mlir::LogicalResult emitRuntimeCheck(mlir::OpBuilder &builder, mlir::Location loc, mlir::Value cond, const llvm::Twine &message)
LogicalResult emitUpdateBdLengthFromParameter(OpBuilder &builder, Operation *bdOp, BaseMemRefType bufType, int64_t lengthUnit, const AIE::AIETargetModel &targetModel, int col, int row, int bdId)
Definition AIEUtils.cpp:296
std::optional< unsigned > getHostBufferArgIndex(BlockArgument arg)
Definition AIEUtils.cpp:149
mlir::Value createConstantI32(mlir::OpBuilder &builder, mlir::Location loc, uint32_t value)
uint32_t getShimBurstLengthEncoding(const AIE::AIETargetModel &tm, uint32_t burstLength)
const AIETargetModel & getTargetModel(mlir::Operation *op)
void placeRuntimeLengthDimension(llvm::MutableArrayRef< int64_t > sizes, llvm::MutableArrayRef< int64_t > strides)
What one op in a loop body does to a channel queue.
DmaQueueModel::ChannelKey key
uint32_t place(uint64_t value) const