MLIR-AIE 1.4.6.dev
AIEDMATasksToNPU.cpp
Go to the documentation of this file.
1//===- AIEDMATasksToNPU.cpp -------------------------------------*- C++ -*-===//
2//
3// Copyright (C) 2024 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7
8#include <algorithm>
9#include <iterator>
10#include <limits>
11
17
18#include "mlir/Dialect/Arith/IR/Arith.h"
19#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
20#include "mlir/Dialect/MemRef/IR/MemRef.h"
21#include "mlir/Dialect/SCF/IR/SCF.h"
22#include "mlir/Dialect/UB/IR/UBOps.h"
23#include "mlir/Dialect/Utils/StaticValueUtils.h"
24#include "mlir/Interfaces/ControlFlowInterfaces.h"
25#include "mlir/Interfaces/SideEffectInterfaces.h"
26#include "mlir/Pass/Pass.h"
27#include "mlir/Pass/PassManager.h"
28#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
29#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
30#include "mlir/Transforms/Passes.h"
31#include "mlir/Transforms/WalkPatternRewriteDriver.h"
32#include "llvm/ADT/STLExtras.h"
33#include "llvm/ADT/TypeSwitch.h"
34
35namespace xilinx::AIEX {
36#define GEN_PASS_DEF_AIEDMATASKSTONPU
37#include "aie/Dialect/AIEX/Transforms/AIEXPasses.h.inc"
38} // namespace xilinx::AIEX
39
40using namespace mlir;
41using namespace xilinx;
42using namespace xilinx::AIEX;
43
44// This pass emits absolute tile coordinates, so it cannot run before placement
45// has substituted a concrete `aie.tile` for an `aie.logical_tile`. Report that
46// instead of calling getTileOp(), which fatally errors on an unplaced tile.
47static LogicalResult emitUnplacedTileError(Operation *op,
48 DMAConfigureTaskOp task_op) {
49 auto err = op->emitOpError(
50 "Cannot lower a DMA task whose tile is not placed; run placement first.");
51 if (task_op.getOperation() != op)
52 err.attachNote(task_op.getLoc()) << "Task configured here.";
53 return err;
54}
55
56struct DMAStartTaskOpPattern : OpRewritePattern<DMAStartTaskOp> {
57 DMAStartTaskOpPattern(MLIRContext *context,
58 llvm::DenseMap<Operation *, bool> &onlyPushedOnce)
60
61 // Whether every start of a configure pushes a constant repeat count of 0.
62 llvm::DenseMap<Operation *, bool> &onlyPushedOnce;
63
64 LogicalResult matchAndRewrite(DMAStartTaskOp op,
65 PatternRewriter &rewriter) const override {
66 DMAConfigureTaskOp task_op = op.getTaskOp();
67 if (!task_op && !op.getTask().getDefiningOp<DMAConfigureTaskForOp>()) {
68 // A task carried through runtime control flow. The push names one head
69 // BD, so it lowers only if every value the task can carry comes from
70 // the same configure.
71 task_op = getUniqueReachableConfigure(op.getTask());
72 if (!task_op)
73 return op.emitOpError(
74 "starts a task carried through control flow that does not come "
75 "from exactly one aiex.dma_configure_task, so its queue push has "
76 "no single buffer descriptor ID. Start the task where it is "
77 "configured.");
78 }
79 if (!task_op) {
80 // Cannot rewrite this; probably points to a DMAStartTaskForOp,
81 // which we will lower once it has been rewritten into a DMAStartTaskOp.
82 return failure();
83 }
84 AIE::TileOp tile = task_op.tryGetTileOp();
85 if (!tile)
86 return emitUnplacedTileError(op, task_op);
87 Location loc = op.getLoc();
88
89 // The bd_id for the queue push: the head BD's runtime pool value if it
90 // carries one, else its static value.
91 AIE::DMABDOp head = task_op.getFirstBd();
92 OpFoldResult idOfr = head ? head.getBdIdValue() : OpFoldResult();
93 if (!idOfr) {
94 auto err = op.emitOpError(
95 "First buffer descriptor in chain has not been assigned an ID");
96 err.attachNote() << "Run the `aie-assign-runtime-buffer-descriptor-ids` "
97 "pass first or manually assign an ID.";
98 return failure();
99 }
100 Value bdIdVal = getAsValue(rewriter, loc, idOfr, rewriter.getI32Type());
101 if (op.getRepeatParameterAttr())
102 return op.emitOpError("has a repeat_parameter; run "
103 "--aie-lower-scratchpad-parameters first");
104 // A runtime repeat count keeps its width so the push lowering can
105 // range-check it before narrowing. A scratchpad one is added to a push
106 // of 0.
107 IntegerAttr repeatIdx = op.getRepeatStateTableIdxAttr();
108 OpFoldResult repeatOfr = repeatIdx ? rewriter.getI32IntegerAttr(0)
109 : op.getPushRepeatCount(task_op);
110 onlyPushedOnce.try_emplace(task_op, true).first->second &=
111 !repeatIdx && isConstantIntValue(repeatOfr, 0);
112 Value repeatCount = dyn_cast<Value>(repeatOfr);
113 if (!repeatCount)
114 repeatCount = getAsValue(rewriter, loc, repeatOfr, rewriter.getI32Type());
115 rewriter.replaceOpWithNewOp<NpuPushQueueOp>(
116 op, tile.getCol(), tile.getRow(), task_op.getDirection(),
117 task_op.getChannel(), op.getPushIssueToken(task_op), repeatCount,
118 bdIdVal, repeatIdx);
119 return success();
120 }
121};
122
123// Resolve a task value to a configure that gives the sync its physical channel.
124// The value is usually a configure result, but under the dynamic BD pool path a
125// task threaded through runtime control flow surfaces as a region-branch
126// successor input instead: an scf.for/scf.if result (the task carried out of
127// the construct), or a loop's own iter_arg block argument (when the task is
128// awaited from inside the loop body rather than after it -- e.g. a
129// software-pipelined sequence awaiting the previous iteration's task before
130// reusing its BD). Every reachable configure targets the same physical channel
131// -- the pool pass verified both branches of an scf.if agree, and a loop's
132// init and per-iteration reconfiguration agree -- so the first one found
133// gives the right channel.
134//
135// getReachableConfigures walks such a value back to its configures through
136// RegionBranchOpInterface, at any nesting depth.
137static DMAConfigureTaskOp resolveConfigureThroughCF(Value task) {
138 SmallVector<DMAConfigureTaskOp> configures;
139 (void)getReachableConfigures(task, configures);
140 return configures.empty() ? nullptr : configures.front();
141}
142
143struct DMAAwaitTaskOpPattern : OpRewritePattern<DMAAwaitTaskOp> {
144 using OpRewritePattern::OpRewritePattern;
145
146 LogicalResult matchAndRewrite(DMAAwaitTaskOp op,
147 PatternRewriter &rewriter) const override {
148 DMAConfigureTaskOp task_op = op.getTaskOp();
149 if (!task_op) {
150 // The awaited task threads through runtime control flow (an scf result),
151 // so it has no directly-defining configure; walk the SSA carry to one.
152 task_op = resolveConfigureThroughCF(op.getTask());
153 }
154 if (!task_op) {
155 return failure();
156 }
157 if (!task_op.getIssueToken()) {
158 auto err = op.emitOpError(
159 "Cannot wait on a BD that is not configured to issue a token.");
160 err.attachNote(task_op.getLoc())
161 << "Consider adding attribute `issue_token=true` here.";
162 return err;
163 }
164 AIE::TileOp tile = task_op.tryGetTileOp();
165 if (!tile)
166 return emitUnplacedTileError(op, task_op);
167 Location loc = op.getLoc();
168 rewriter.replaceOpWithNewOp<NpuSyncOp>(
169 op, createConstantI32(rewriter, loc, tile.getCol()),
170 createConstantI32(rewriter, loc, tile.getRow()),
171 createConstantI32(rewriter, loc, (uint32_t)task_op.getDirection()),
172 createConstantI32(rewriter, loc, task_op.getChannel()),
173 createConstantI32(rewriter, loc, 1),
174 createConstantI32(rewriter, loc, 1));
175 return success();
176 }
177};
178
180 : xilinx::AIEX::impl::AIEDMATasksToNPUBase<AIEDMATasksToNPUPass> {
181
182 // A BD whose aie.next_bd successor draws its id from the dynamic pool, mapped
183 // to that runtime id; hoistNextBdOpsIntoAttrs fills it in place of the
184 // next_bd_id attribute.
185 llvm::DenseMap<AIE::DMABDOp, Value> runtimeNextBdId;
186
187 bool shouldSkipBlock(Block &block) {
188 // Allow blocks in the input IR that contain nothing but a next_bd operation
189 // as the entry block. We will skip these blocks and not lower them to
190 // anything.
191 auto it = block.without_terminator();
192 return block.isEntryBlock() && it.begin() == it.end();
193 }
194
195 LogicalResult verifyBdInBlock(Block &block) {
196 auto bd_ops = block.getOps<AIE::DMABDOp>();
197 // Exactly one BD op per block
198 int n_bd_ops = std::distance(bd_ops.begin(), bd_ops.end());
199 if (n_bd_ops < 1) {
200 auto error = block.getTerminator()->emitError(
201 "Block ending in this terminator does not contain a required "
202 "aie.dma_bd operation.");
203 error.attachNote(block.getParentOp()->getLoc())
204 << "Error encountered while lowering this BD configuration.";
205 return failure();
206 }
207 if (n_bd_ops > 1) {
208 auto error = block.getTerminator()->emitOpError(
209 "This block contains multiple aie.dma_bd operations. Exactly one "
210 " is "
211 "required.");
212 auto it = bd_ops.begin();
213 ++it;
214 for (; it != bd_ops.end(); ++it) {
215 error.attachNote((*it)->getLoc()) << "Extra aie.dma_bd operation here.";
216 }
217 return failure();
218 }
219 AIE::DMABDOp bd_op = *bd_ops.begin();
220 bool hasRuntimeBdId = bd_op.getBdIdVal() != nullptr;
221 if (!hasRuntimeBdId && !bd_op.getBdId().has_value()) {
222 auto error = bd_op.emitOpError(
223 "Cannot lower buffer descriptor without assigned ID.");
224 error.attachNote()
225 << "Run the `--aie-assign-runtime-sequence-bd-ids` pass first or "
226 "manually assign an ID to this buffer descriptor.";
227 error.attachNote(block.getParentOp()->getLoc())
228 << "Error encountered while lowering this BD configuration.";
229 return failure();
230 }
231 return success();
232 }
233
234 // A builder that derives a BD operand from runtime scalars (a width cast, a
235 // transfer-length product, a cf.assert guard) emits that arithmetic where it
236 // is called, inside the BD block. None of it depends on the BD, so hoist it
237 // ahead of the task. What stays (an op with other effects, or one reading a
238 // value only the block defines) is left for verifyNoUnsupportedOpsInBlock to
239 // reject.
240 void hoistScalarOpsOutOfBdBlocks(DMAConfigureTaskOp op) {
241 moveLoopInvariantCode(
242 {&op.getBody()},
243 [](Value v, Region *body) {
244 return !body->isAncestor(v.getParentRegion());
245 },
246 [](Operation *inner, Region *) {
247 return inner->getNumRegions() == 0 &&
248 (isPure(inner) || isa<cf::AssertOp>(inner));
249 },
250 [&](Operation *inner, Region *) { inner->moveBefore(op); });
251 }
252
253 LogicalResult verifyNoUnsupportedOpsInBlock(Block &block) {
254 WalkResult unsupported_ops = block.walk([&](Operation *inner_op) {
255 return llvm::TypeSwitch<Operation *, WalkResult>(inner_op)
256 .Case<AIE::DMABDOp>(
257 [&](AIE::DMABDOp bd_op) { return WalkResult::advance(); })
258 .Case<AIE::UseLockOp>(
259 [&](AIE::UseLockOp lock_op) { return WalkResult::advance(); })
260 .Case<arith::ConstantOp>(
261 [&](arith::ConstantOp const_op) { return WalkResult::advance(); })
262 .Case<AIE::NextBDOp>(
263 [&](AIE::NextBDOp lock_op) { return WalkResult::advance(); })
264 .Case<AIE::EndOp>(
265 [&](AIE::EndOp lock_op) { return WalkResult::advance(); })
266 .Default([&](Operation *inner_op) {
267 auto error = block.getParentOp()->emitOpError(
268 "Unsupported operation within BD block.");
269 error.attachNote(inner_op->getLoc())
270 << "No lowering to NPU instructions available for this "
271 "operation.";
272 return WalkResult::interrupt();
273 });
274 });
275 if (unsupported_ops.wasInterrupted()) {
276 return failure();
277 }
278 return success();
279 }
280
281 AIE::DMABDOp getBdForBlock(Block &block) {
282 auto bd_ops = block.getOps<AIE::DMABDOp>();
283 AIE::DMABDOp bd_op = *bd_ops.begin(); // Dereference first (and only, after
284 // previous checks) bd op iterator
285 return bd_op;
286 }
287
288 /// `lengthField` is the value the BD write gave its length field.
289 LogicalResult setAddressForSingleBD(OpBuilder &builder, AIE::DMABDOp &bd_op,
290 AIE::TileOp &tile, OpFoldResult bdId,
291 OpFoldResult lengthField) {
292 const AIE::AIETargetModel &target_model = AIE::getTargetModel(bd_op);
293 auto buf = bd_op.getBuffer();
294 auto col = tile.getCol();
295 auto row = tile.getRow();
296 // A foldable SSA bd_id must name the same register as the block write.
297 auto constBdId = bdId ? getConstantIntValue(bdId) : std::nullopt;
298 uint32_t bd_id = constBdId ? *constBdId : bd_op.getBdId().value_or(0);
299 uint64_t register_addr = target_model.getDmaBdAddress(col, row, bd_id) +
300 target_model.getDmaBdAddressOffset(col, row);
301 std::optional<uint64_t> staticByteAddress;
302 // On the runtime-bd_id path the patched register (BD buffer-address word)
303 // is itself runtime: getBdRegisterBase(bd_id) + getDmaBdAddressOffset.
304 // Emitted as an SSA operand on the address patch; null keeps the static
305 // constant.
306 Value runtimeRegisterAddr;
307 if (bdId && !getConstantIntValue(bdId)) {
308 Value base = getBdRegisterBase(builder, bd_op.getLoc(), target_model, col,
309 row, bdId);
310 runtimeRegisterAddr = arith::AddIOp::create(
311 builder, bd_op.getLoc(), base,
312 createConstantI32(builder, bd_op.getLoc(),
313 target_model.getDmaBdAddressOffset(col, row)));
314 }
315
316 // emitUpdateBdAddressFromOffsetParameter bakes a compile-time register
317 // address into npu.update_from_scratchpad, so pairing it with a BD drawn
318 // from the runtime pool would patch whichever BD that literal names rather
319 // than the one just configured.
320 if (bd_op.getOffsetStateTableIdxAttr() && runtimeRegisterAddr)
321 return bd_op->emitOpError(
322 "offset_state_table_idx is not supported with a runtime bd_id; the "
323 "scratchpad update targets a compile-time BD register address.");
324
325 // A buffer descriptor can refer to a statically allocated aie.buffer, or to
326 // a DDR buffer which will be passed as a runtime argument (block
327 // argument). Try to find the root block argument, either directly or
328 // through supported subview/view/cast chains.
329 mlir::BlockArgument buf_arg = nullptr;
330 int64_t offset = 0;
331
332 if (auto directArg = llvm::dyn_cast<mlir::BlockArgument>(buf)) {
333 buf_arg = directArg;
334 offset = 0;
335 } else if (auto traceResult = traceSubviewToBlockArgument(buf)) {
336 buf_arg = traceResult->rootArg;
337 offset = traceResult->offsetInBytes;
338 }
339
340 if (buf_arg) {
341 if (!target_model.isShimNOCTile(tile.getCol(), tile.getRow())) {
342 return bd_op->emitOpError("DDR memory (runtime input arguments) can "
343 "only be referred to on shim tiles.");
344 }
345
346 std::optional<unsigned> hostIdx = getHostBufferArgIndex(buf_arg);
347 if (!hostIdx)
348 return bd_op->emitOpError(
349 "buffer must resolve to a memref argument of the runtime sequence");
350 unsigned arg_idx = *hostIdx;
351 // arg_plus = buffer byte offset. The dma_bd offset is a single element
352 // offset (stride 1); a constant folds to a constant (byte-identical to
353 // before), a runtime offset operand is built with arith. `offset` here is
354 // the constant subview base in bytes.
355 OpFoldResult offsetOfr =
356 bd_op.getOffset() ? OpFoldResult(bd_op.getOffset())
357 : OpFoldResult(builder.getI32IntegerAttr(
358 bd_op.getConstantOffset().value_or(0)));
359 OpFoldResult oneStride = builder.getI32IntegerAttr(1);
360 Value argPlus =
361 buildArgPlusValue(builder, bd_op.getLoc(), {offsetOfr}, {oneStride},
362 bd_op.getBufferElementTypeWidthInBytes(), offset,
363 target_model.getAddressGenGranularity() / 8);
364 if (!argPlus)
365 return failure();
366 NpuAddressPatchOp::create(builder, bd_op.getLoc(),
367 /*addr*/ register_addr,
368 /*addr_val*/ runtimeRegisterAddr,
369 /*arg_idx*/ arg_idx, argPlus);
370 } else if (AIE::BufferOp buffer =
371 llvm::dyn_cast<AIE::BufferOp>(buf.getDefiningOp())) {
372 std::optional<uint32_t> bufferAddr = buffer.getAddress();
373 if (!bufferAddr.has_value()) {
374 return bd_op->emitOpError(
375 "Cannot lower buffer without associated address. Run pass "
376 "--aie-assign-buffer-addresses first or manually assign an "
377 "address.");
378 }
379 uint64_t buf_addr = *bufferAddr;
380 // A memtile DMA addresses its west neighbour, itself and its east
381 // neighbour as three windows (west=0, own=getMemTileSize, east=2x), as
382 // the static lowering in AIERT.cpp does.
383 if (target_model.isMemTile(col, row)) {
384 auto addrOffset = target_model.getMemLocalBaseAddress(
385 col, row, buffer.getTileOp().getCol(), buffer.getTileOp().getRow());
386 if (addrOffset)
387 buf_addr += addrOffset.value();
388 }
389
390 const AIE::DmaBdLayout *layout = target_model.getDmaBdLayout(col, row);
391 if (!layout)
392 return bd_op->emitOpError(
393 "has no buffer descriptor layout on this tile");
394 const AIE::DmaBdField &addrBits = layout->bufferOffset;
395 uint64_t divisor = getBdAddressUnit(target_model, col, row);
396 uint64_t maxByteAddr = (uint64_t(addrBits.mask()) + 1) * divisor - 1;
397 if (buf_addr > maxByteAddr)
398 return bd_op->emitOpError("buffer address exceeds the tile's DMA "
399 "address field.");
400
401 // getOffsetInBytes() folds a runtime offset to 0, so it may only be read
402 // on the constant path; a runtime offset is added with arith instead.
403 bool constOffset = !bd_op.getOffset() || bd_op.getConstantOffset();
404 Location loc = bd_op.getLoc();
405 Value addrField;
406 if (constOffset) {
407 buf_addr += bd_op.getOffsetInBytes();
408 if (buf_addr > maxByteAddr)
409 return bd_op->emitOpError("buffer address exceeds the tile's DMA "
410 "address field.");
411 staticByteAddress = buf_addr;
412 addrField =
413 createConstantI32(builder, loc, addrBits.place(buf_addr / divisor));
414 } else {
415 Value byteAddr =
416 buildArgPlusValue(builder, loc, {OpFoldResult(bd_op.getOffset())},
417 {OpFoldResult(builder.getI32IntegerAttr(1))},
418 bd_op.getBufferElementTypeWidthInBytes(),
419 (int64_t)buf_addr, divisor);
420 if (!byteAddr)
421 return failure();
422 Type i64 = builder.getI64Type();
423 Value fits = builder.createOrFold<arith::CmpIOp>(
424 loc, arith::CmpIPredicate::ule, byteAddr,
425 arith::ConstantOp::create(builder, loc,
426 IntegerAttr::get(i64, maxByteAddr)));
427 if (failed(emitRuntimeCheck(builder, loc, fits,
428 "a runtime DMA offset puts the buffer "
429 "address past the tile's DMA address "
430 "field")))
431 return failure();
432 if (divisor != 1)
433 byteAddr = arith::DivUIOp::create(
434 builder, loc, byteAddr,
435 arith::ConstantOp::create(builder, loc,
436 IntegerAttr::get(i64, divisor)));
437 addrField = arith::TruncIOp::create(builder, loc, builder.getI32Type(),
438 byteAddr);
439 if (addrBits.shift != 0)
440 addrField = arith::ShLIOp::create(
441 builder, loc, addrField,
442 createConstantI32(builder, loc, addrBits.shift));
443 }
444 // The register holding the address word. A runtime bd_id makes it
445 // runtime too; a pinned one folds to the literal the static path uses.
446 Value regAddrVal =
447 runtimeRegisterAddr
448 ? runtimeRegisterAddr
449 : createConstantI32(builder, loc,
450 static_cast<uint32_t>(register_addr));
451 uint32_t addrMask = addrBits.place(0xFFFFFFFF);
452 if (addrMask == 0xFFFFFFFF)
453 NpuWrite32Op::create(builder, loc, regAddrVal, addrField, nullptr,
454 nullptr, nullptr);
455 else
456 NpuMaskWrite32Op::create(builder, loc, regAddrVal, addrField,
457 createConstantI32(builder, loc, addrMask),
458 nullptr, nullptr, nullptr);
459 } else {
460 return bd_op->emitOpError(
461 "Buffer argument must be a constant aie.buffer, a runtime sequence "
462 "input argument, or a supported chain of memref.subview, "
463 "memref.view, memref.cast, or memref.reinterpret_cast operations "
464 "rooted at a block argument. Subviews must be static and contiguous; "
465 "views must have a constant byte shift and no dynamic result sizes.");
466 }
467
468 // If this BD has an offset_state_table_idx, emit update_from_scratchpad to
469 // add the runtime offset to the BD address register. This is applied after
470 // the base address is set (by either NpuAddressPatchOp for DDR buffers or
471 // NpuMaskWrite32Op/NpuWrite32Op for on-chip buffers), since the hardware
472 // update_from_scratchpad instruction is additive -- it reads the existing
473 // register value and adds a computed delta to it.
474 if (bd_op.getOffsetStateTableIdxAttr()) {
475 auto bufType = llvm::cast<BaseMemRefType>(bd_op.getBuffer().getType());
477 builder, bd_op, bufType, register_addr, target_model, col, row,
478 staticByteAddress, lengthField)))
479 return failure();
480 }
481
482 return success();
483 }
484
485 // The lock / packet / next_bd fields for a BD block. A runtime lock value is
486 // guarded and encoded with `builder`; only the dynamic path reaches that.
487 FailureOr<BdTemplateFields>
488 gatherBdTemplateFields(OpBuilder &builder, Block &block, AIE::DMABDOp bd_op,
489 AIE::TileOp &tile,
490 const AIE::AIETargetModel &target_model,
491 std::optional<xilinx::AIE::PacketInfoAttr> packet,
492 bool outOfOrder = false) {
494 // Out-of-order S2MM ignores Next_BD; the IR chain only lists the receive
495 // BDs. Leave the BDs unchained, as the static AIE-RT lowering does.
496 if (!outOfOrder) {
497 if (std::optional<uint32_t> nextBdId = bd_op.getNextBdId()) {
498 f.next_bd_id = *nextBdId;
499 f.use_next_bd = 1;
500 } else if (Value nextBdIdVal = runtimeNextBdId.lookup(bd_op)) {
501 f.next_bd_id_val = nextBdIdVal;
502 f.use_next_bd = 1;
503 }
504 }
505
506 auto info = bd_op.getPacket().value_or(packet.value_or(nullptr));
507 if (info) {
508 if (!info.isAssigned()) {
509 bd_op.emitOpError("packet has no pkt_id; allocation assigns one, so a "
510 "header at this level must carry it");
511 return failure();
512 }
513 f.enable_packet = 1;
514 f.packet_type = info.getPktType();
515 f.packet_id = info.assignedId();
516 }
517
518 if (std::optional<int32_t> oooId = bd_op.getOutOfOrderId())
519 f.out_of_order_id = *oooId;
520
521 if (std::optional<AIE::BDIterationAttr> iter = bd_op.getIteration())
522 f.iteration_current = iter->getCurrent();
523
524 // AcquireGreaterEqual is encoded negated, so a runtime one must be >= 1:
525 // a field of 0 would mean "acquire == 0".
526 auto lockValue = [&](AIE::UseLockOp op, int32_t &constOut,
527 Value &valueOut) -> LogicalResult {
528 if (std::optional<int64_t> c = getConstantIntValue(op.getValue())) {
529 constOut = op.acquireGE() ? -static_cast<int32_t>(*c)
530 : static_cast<int32_t>(*c);
531 return success();
532 }
533 Location loc = op.getLoc();
534 int32_t minValue = op.acquireGE() ? 1 : 0;
535 int32_t maxValue = target_model.getMaxLockValue();
536 Value fromMin = builder.createOrFold<arith::SubIOp>(
537 loc, op.getValue(), createConstantI32(builder, loc, minValue));
538 Value inRange = builder.createOrFold<arith::CmpIOp>(
539 loc, arith::CmpIPredicate::ule, fromMin,
540 createConstantI32(builder, loc, maxValue - minValue));
541 if (failed(emitRuntimeCheck(builder, loc, inRange,
542 "a runtime lock value must be in [" +
543 Twine(minValue) + ":" + Twine(maxValue) +
544 "]")))
545 return failure();
546 valueOut = op.getValue();
547 if (op.acquireGE())
548 valueOut = arith::SubIOp::create(
549 builder, loc, createConstantI32(builder, loc, 0), valueOut);
550 return success();
551 };
552
553 AIE::UseLockOp acquire_op, release_op;
554 if (failed(
555 AIE::verifyBdLockPair(block, outOfOrder, acquire_op, release_op)))
556 return failure();
557 // A memtile BD may use locks of its neighbours, so each lock ID is offset
558 // by the base index of the tile that owns that lock, as in AIERT.cpp.
559 auto lockIdFor = [&](AIE::LockOp lock, int32_t id) {
560 if (target_model.isMemTile(tile.getCol(), tile.getRow()))
561 if (auto offset = target_model.getLockLocalBaseIndex(
562 tile.getCol(), tile.getRow(), lock.colIndex(), lock.rowIndex()))
563 id += *offset;
564 return id;
565 };
566 if (release_op) {
567 AIE::LockOp rel_lock = release_op.getLockOp();
568
569 // Acquire is optional under out-of-order (release-only completion).
570 if (acquire_op) {
571 AIE::LockOp acq_lock = acquire_op.getLockOp();
572 if (std::optional<int32_t> acqLockId = acq_lock.getLockID()) {
573 f.lock_acq_id = lockIdFor(acq_lock, *acqLockId);
574 if (failed(lockValue(acquire_op, f.lock_acq_val, f.lock_acq_val_val)))
575 return failure();
576 f.lock_acq_enable = 1;
577 }
578 }
579
580 if (std::optional<int32_t> relLockId = rel_lock.getLockID()) {
581 f.lock_rel_id = lockIdFor(rel_lock, *relLockId);
582 if (failed(lockValue(release_op, f.lock_rel_val, f.lock_rel_val_val)))
583 return failure();
584 }
585 }
586 return f;
587 }
588
589 // Dynamic (runtime SSA size/stride/len/bd_id) BD lowering, the dma_task
590 // sibling of DmaToNpuPattern::lowerDynamic. Reaching this function at all
591 // means the design targets the EmitC (C++ TXN) builder, never the static
592 // binary target: a design meant for the binary target is unrolled to
593 // all-constant before this pass runs. Scope (a tile type with a BD register
594 // layout, no padding, realizability) is enforced by the caller;
595 // buildBdWords picks the layout from the tile.
596 LogicalResult
597 rewriteSingleBDDynamic(OpBuilder &builder, Block &block, AIE::DMABDOp bd_op,
598 AIE::TileOp &tile,
599 std::optional<xilinx::AIE::PacketInfoAttr> packet,
600 Value runtimeBdId = nullptr, bool outOfOrder = false) {
601 const auto &target_model = AIE::getTargetModel(bd_op);
602 Location loc = bd_op.getLoc();
603 int col = tile.getCol();
604 int row = tile.getRow();
605
606 auto fieldsOr = gatherBdTemplateFields(builder, block, bd_op, tile,
607 target_model, packet, outOfOrder);
608 if (failed(fieldsOr))
609 return failure();
610 // failed() above guards the deref; FailureOr hides the std::optional base's
611 // has_value(), so the checker cannot see the guard (suppressed below).
612 BdTemplateFields f = *fieldsOr; // NOLINT
613
614 // The bd_id as an OpFoldResult: the runtime pool value if present, else the
615 // pinned constant attribute. On the non-runtime path verifyBdInBlock (run
616 // for every block before any lowering, with hasRuntimeBdId == !runtimeBdId)
617 // has already required the attribute, so the access is guaranteed set here.
618 OpFoldResult bdIdOfr;
619 if (runtimeBdId) {
620 bdIdOfr = runtimeBdId;
621 } else {
622 std::optional<uint32_t> staticBdId = bd_op.getBdId();
623 assert(staticBdId && "static bd_id must be assigned (checked by "
624 "verifyBdInBlock before lowering)");
625 bdIdOfr = builder.getI32IntegerAttr(*staticBdId);
626 }
627
628 // Normalize sizes/strides to a 4-element outermost-first mixed list (the
629 // shared emitter's contract, matching memcpy_nd's always-4D operands),
630 // padding absent leading dims with size 1 / stride 0. dma_bd's mixed lists
631 // are outermost-first and variable-length (0..4 dims). A BD with no dims is
632 // a contiguous scan, so its innermost stride is 1, which keeps it linear
633 // like the static path.
634 SmallVector<OpFoldResult> sizes(bd_op.getMixedSizes());
635 SmallVector<OpFoldResult> strides(bd_op.getMixedStrides());
636 if (sizes.size() > 4 || strides.size() > 4)
637 return bd_op->emitOpError(
638 "At most four data layout transformation dimensions may be "
639 "provided; aie-decompose-large-dma-bd splits a longer one.");
640 OpFoldResult one = builder.getI64IntegerAttr(1);
641 OpFoldResult zeroOfr = builder.getI64IntegerAttr(0);
642 SmallVector<OpFoldResult, 4> sizes4(4, one), strides4(4, zeroOfr);
643 if (strides.empty())
644 strides4.back() = one;
645 for (size_t i = 0; i < sizes.size(); i++) {
646 sizes4[4 - sizes.size() + i] = sizes[i];
647 strides4[4 - strides.size() + i] = strides[i];
648 }
649 // The iteration attribute takes the outermost (iteration) slot, which
650 // verifyTaskBDDimensions has kept free of sizes/strides.
651 if (std::optional<AIE::BDIterationAttr> iter = bd_op.getIteration()) {
652 sizes4.front() = builder.getI64IntegerAttr(iter->getSize());
653 strides4.front() = builder.getI64IntegerAttr(iter->getStride());
654 }
655 // len as OpFoldResult: the runtime operand if present, else the static_len
656 // attr (a constant BD with runtime sizes/strides still reaches here). With
657 // runtime dims and no len, the transfer is their d0*d1*d2 extent; with
658 // constant dims or none, it is the whole buffer, as on the static path.
659 // That default needs a constant offset, or it could run past the buffer.
660 OpFoldResult lenOfr;
661 if (Value lenOperand = bd_op.getLen())
662 lenOfr = lenOperand;
663 else if (std::optional<int32_t> constLen = bd_op.getConstantLen())
664 lenOfr = builder.getI32IntegerAttr(*constLen);
665 else if (bd_op.getSizes().empty() && bd_op.getStrides().empty()) {
666 auto bufTy = cast<BaseMemRefType>(bd_op.getBuffer().getType());
667 if (!bufTy.hasStaticShape() ||
668 (bd_op.getOffset() && !bd_op.getConstantOffset()))
669 return bd_op->emitOpError(
670 "runtime-valued BD requires an explicit transfer length; provide a "
671 "`len` for this buffer descriptor.");
672 lenOfr = builder.getI64IntegerAttr(bufTy.getNumElements());
673 }
674
675 // As on the static path, the lowest three dims (d3 repeats the BD) must
676 // cover a constant length exactly.
677 std::optional<int64_t> constLen =
678 lenOfr ? getConstantIntValue(lenOfr) : std::nullopt;
679 std::optional<int64_t> dimsLen = 1;
680 for (OpFoldResult s : llvm::drop_begin(sizes4)) {
681 std::optional<int64_t> c = getConstantIntValue(s);
682 if (!c) {
683 dimsLen.reset();
684 break;
685 }
686 *dimsLen *= *c;
687 }
688 uint64_t elemWidth =
689 static_cast<uint64_t>(bd_op.getBufferElementTypeWidthInBytes()) * 8;
690 if (!sizes.empty() && constLen && dimsLen && *dimsLen != *constLen) {
691 auto err = bd_op->emitOpError(
692 "Buffer descriptor length does not match length of "
693 "transfer expressed by lowest three dimensions of data "
694 "layout transformation strides/wraps. ")
695 << "BD length is " << (*constLen * elemWidth / 8)
696 << " bytes. Lowest three dimensions of data layout "
697 "transformation would result in transfer of "
698 << (*dimsLen * elemWidth / 8) << " bytes. ";
699 err.attachNote() << "Do not include the highest dimension size in "
700 "transfer length, as this is the BD repeat count.";
701 return failure();
702 }
703
704 // A BD without addressing dims, or whose dims are a unit-stride d0 alone,
705 // is a plain linear transfer of `len` elements (canonicalization strips a
706 // contiguous [1, 1, 1, N] to this form, and the static path encodes it
707 // linear from `len` alone), so give it exactly that shape. With any other
708 // dims, `len` must agree with their d0*d1*d2 extent.
709 bool linear =
710 sizes.empty() ||
711 (lenOfr && isConstantIntValue(strides4[3], 1) &&
712 llvm::all_of(ArrayRef(sizes4).drop_back(),
713 [](OpFoldResult v) { return isConstantIntValue(v, 1); }));
714 if (linear) {
715 sizes4[3] = lenOfr;
716 strides4[3] = one;
717 }
718
719 // The BD-level repeat_count (encoder output) is unused here: the dma_task
720 // queue push is emitted separately by DMAStartTaskOpPattern from the task
721 // op's repeat_count, not the BD's outer dim.
722 SmallVector<Value> bdWords;
723 Value bdRepeatCount;
724 if (failed(buildBdWords(
725 builder, loc, target_model, col, row, f, sizes4, strides4,
726 elemWidth, bd_op.getBurstLength(), bd_op.getAxcacheOrDefault(),
727 linear ? OpFoldResult() : lenOfr, bdRepeatCount, bdWords)))
728 return failure();
729
730 // The walk must stay inside the host buffer it reads or writes.
731 if (auto trace = traceSubviewToBlockArgument(bd_op.getBuffer())) {
732 OpFoldResult offsetOfr =
733 bd_op.getOffset() ? OpFoldResult(bd_op.getOffset())
734 : OpFoldResult(builder.getI32IntegerAttr(
735 bd_op.getConstantOffset().value_or(0)));
736 if (failed(guardWithinHostBuffer(
737 builder, loc, cast<BaseMemRefType>(trace->rootArg.getType()),
738 trace->offsetInBytes, elemWidth / 8, {offsetOfr}, {one}, sizes4,
739 strides4)))
740 return failure();
741 }
742
743 // One blockwrite carries the whole register block for BD configuration.
744 Value bdBase =
745 getBdRegisterBase(builder, loc, target_model, col, row, bdIdOfr);
746 NpuBlockWriteValuesOp::create(builder, loc, bdBase, bdWords);
747 const AIE::DmaBdLayout *layout = target_model.getDmaBdLayout(col, row);
749 builder, bd_op, tile, bdIdOfr,
750 layout ? OpFoldResult(bdWords[layout->bufferLength.word])
751 : OpFoldResult());
752 }
753
754 LogicalResult
755 rewriteSingleBD(OpBuilder &builder, Block &block, AIE::TileOp &tile,
756 AIE::DMAChannelDir channelDir,
757 std::optional<xilinx::AIE::PacketInfoAttr> packet,
758 bool outOfOrder = false) {
759 AIE::DMABDOp bd_op = getBdForBlock(block);
760 Value runtimeBdId = bd_op.getBdIdVal();
761 const auto &target_model = AIE::getTargetModel(bd_op);
762 auto buffer_type = llvm::cast<BaseMemRefType>(bd_op.getBuffer().getType());
763 uint32_t addr_granularity = target_model.getAddressGenGranularity();
764
765 // Runtime (SSA) sizes/strides/len/offset take the dynamic BD-word encoder
766 // path; a runtime bd_id (dynamic free-list pool) also forces it, since the
767 // BD register addresses are then runtime and cannot fold into a blockwrite,
768 // and so does a runtime next_bd, which is a runtime field in the words.
769 // A fully-constant descriptor with a pinned bd_id takes the static path
770 // below unchanged. The encoder covers every tile type that has a BD
771 // register layout (see buildBdWords), so anything the dynamic path can't
772 // represent stays a clean diagnostic.
773 bool runtimeLen = bd_op.getLen() && !bd_op.getConstantLen();
774 bool runtimeOffset = bd_op.getOffset() && !bd_op.getConstantOffset();
775 bool runtimeDims =
776 llvm::any_of(bd_op.getMixedSizes(),
777 [](OpFoldResult s) { return !getConstantIntValue(s); }) ||
778 llvm::any_of(bd_op.getMixedStrides(),
779 [](OpFoldResult s) { return !getConstantIntValue(s); });
780 bool runtimeNextBd = runtimeNextBdId.contains(bd_op);
781 bool runtimeLock =
782 llvm::any_of(block.getOps<AIE::UseLockOp>(), [](AIE::UseLockOp op) {
783 return !getConstantIntValue(op.getValue());
784 });
785 if (runtimeLen || runtimeDims || runtimeOffset || runtimeBdId ||
786 runtimeNextBd || runtimeLock) {
787 // The verifier leaves a runtime offset, bd_id, next_bd or lock value as
788 // the only ways here. An offset that depends on a runtime-sequence loop
789 // is constant by now if the loop was unrolled.
790 if (bd_op.getLengthStateTableIdxAttr())
791 return bd_op->emitOpError("length_parameter requires a constant "
792 "offset, bd_id, next_bd and lock value");
793 int col = tile.getCol(), row = tile.getRow();
794 if (!target_model.isShimNOCTile(col, row) &&
795 !target_model.isMemTile(col, row) &&
796 !target_model.isCoreTile(col, row))
797 return bd_op->emitOpError(
798 "runtime-valued BD size/stride/len/bd_id/next_bd/lock value is "
799 "only supported on tiles with a DMA buffer descriptor layout (shim "
800 "NOC, mem and "
801 "core tiles); use compile-time constants here.");
802 if (bd_op.getPadDimensions().has_value())
803 return bd_op->emitOpError(
804 "zero padding is not supported with runtime sizes/strides/len.");
805 // Realizability of the constant size/stride operands (runtime ones are
806 // guarded at lowering by the shared encoder). Mixed lists are
807 // outermost-first; the helper wants innermost-first.
808 uint64_t elemWidth =
809 static_cast<uint64_t>(bd_op.getBufferElementTypeWidthInBytes()) * 8;
810 SmallVector<OpFoldResult, 4> sizesRev(
811 llvm::reverse(bd_op.getMixedSizes()));
812 SmallVector<OpFoldResult, 4> stridesRev(
813 llvm::reverse(bd_op.getMixedStrides()));
815 bd_op, sizesRev, stridesRev, elemWidth,
816 target_model.getAddressGenGranularity())))
817 return failure();
818 return rewriteSingleBDDynamic(builder, block, bd_op, tile, packet,
819 runtimeBdId, outOfOrder);
820 }
821
822 // Static path: bd_id is a pinned attribute (the dynamic/runtime-bd_id path
823 // returned above) and the offset is constant (runtime offset routed above).
824 // verifyBdInBlock (run for every block before lowering, with the static
825 // path implying hasRuntimeBdId == false) has already required the
826 // attribute.
827 std::optional<uint32_t> bdIdAttr = bd_op.getBdId();
828 assert(bdIdAttr && "static bd_id must be assigned (checked by "
829 "verifyBdInBlock before lowering)");
830 uint32_t bd_id = *bdIdAttr;
831 int64_t offset = bd_op.getOffsetInBytes();
832 uint64_t len = bd_op.getLenInBytes();
833 uint64_t len_addr_granularity = len * 8 / addr_granularity;
834
835 if (offset * 8 % addr_granularity != 0) {
836 return bd_op->emitOpError("Offset must be aligned to ")
837 << (addr_granularity / 8) << " byte boundary.";
838 }
839
840 // A runtime length may start from zero: the update below adds to it.
841 if (len < addr_granularity / 8 && !bd_op.getLengthStateTableIdxAttr()) {
842 return bd_op->emitOpError("Transfer size of ")
843 << len << " bytes falls below minimum hardware transfer unit of "
844 << (addr_granularity / 8) << " bytes.";
845 }
846 // The owning storage must outlive the ArrayRef view below.
847 std::optional<llvm::SmallVector<AIE::BDDimLayoutAttr>> dimsStorage =
848 bd_op.getConstantDimensions();
849 if (!dimsStorage)
850 return bd_op->emitOpError("internal error folding BD dimensions");
851 // The iteration attribute rides in the outermost (iteration) slot of the
852 // dimensioned form, so a BD without dims becomes one contiguous dimension.
853 std::optional<AIE::BDIterationAttr> iter = bd_op.getIteration();
854 if (iter && dimsStorage->empty())
855 dimsStorage->push_back(AIE::BDDimLayoutAttr::get(
856 builder.getContext(), len / bd_op.getBufferElementTypeWidthInBytes(),
857 1));
858 std::optional<llvm::ArrayRef<AIE::BDDimLayoutAttr>> dims;
859 if (!dimsStorage->empty())
860 dims = llvm::ArrayRef<AIE::BDDimLayoutAttr>(*dimsStorage);
861 llvm::SmallVector<int64_t, 4> sizes = llvm::SmallVector<int64_t, 4>(4, 0);
862 llvm::SmallVector<int64_t, 4> strides = llvm::SmallVector<int64_t, 4>(4, 0);
863
864 // Padding
865 std::optional<llvm::ArrayRef<AIE::BDPadLayoutAttr>> padDims =
866 bd_op.getPadDimensions();
867 llvm::SmallVector<int64_t, 4> padBefore =
868 llvm::SmallVector<int64_t, 4>(4, 0);
869 llvm::SmallVector<int64_t, 4> padAfter =
870 llvm::SmallVector<int64_t, 4>(4, 0);
871 llvm::fill(padBefore, 0);
872 llvm::fill(padAfter, 0);
873
874 auto d0size = 0;
875 auto d0stride = 0;
876 auto d1size = 0;
877 auto d1stride = 0;
878 auto d2size = 0;
879 auto d2stride = 0;
880 auto iteration_size = 0;
881 auto iteration_stride = 0;
882
883 if (dims && !dims->empty()) {
884 llvm::SmallVector<int64_t, 4> input_sizes =
885 llvm::SmallVector<int64_t, 4>(4, 1);
886 llvm::SmallVector<int64_t, 4> input_strides =
887 llvm::SmallVector<int64_t, 4>(4, 0);
888 if (dims->size() > 4) {
889 return bd_op->emitOpError(
890 "At most four data layout transformation dimensions may be "
891 "provided; aie-decompose-large-dma-bd splits a longer one.");
892 }
893
894 for (size_t i = 0; i < dims->size(); i++) {
895 // Pass down dimensions in reverse order; in the MLIR, this allows
896 // us to specify step sizes/wraps in the same order as we would
897 // access a multi-dim C array, with the highest dimension first.
898 int j = dims->size() - i - 1;
899 input_sizes[i] = (*dims)[j].getSize();
900 input_strides[i] = (*dims)[j].getStride();
901 }
902 if (iter) {
903 input_sizes[3] = iter->getSize();
904 input_strides[3] = iter->getStride();
905 }
906
907 // d3 (repeat) is excluded; a repeated linear transfer is still linear.
908 // A contiguous row-major ND access on a shim NOC tile is also lowered
909 // using the wide buffer_length register, exempt from the 10-bit ND
910 // wrap-size limit. Canonicalization zeroes size-1 strides before this
911 // pass runs, so isContiguousTransfer is sufficient.
912 // A linear transfer leaves the wraps unset, and the pads apply to them.
913 bool treatAsLinear =
914 !padDims.has_value() &&
915 isLinearModeTransfer(target_model, tile.getCol(), tile.getRow(),
916 input_sizes, input_strides);
917 if (!treatAsLinear && bd_op.getLengthStateTableIdxAttr())
918 AIE::placeRuntimeLengthDimension(input_sizes, input_strides);
919
920 if (padDims.has_value()) {
921 const AIE::DmaBdLayout *layout =
922 target_model.getDmaBdLayout(tile.getCol(), tile.getRow());
923 size_t numPadFields = layout ? layout->padFields().size() : 0;
924 if (numPadFields == 0)
925 return bd_op->emitOpError()
926 << "Padding is only supported by memtile dma bds.";
927 if (padDims->size() > dims->size())
928 return bd_op->emitOpError()
929 << "Mismatch number of dimensions between padding(s)"
930 << " and wrap(s) and stride(s).";
931 for (auto [d, pad] : llvm::enumerate(llvm::reverse(*padDims)))
932 if (d >= numPadFields &&
933 (pad.getConstPadBefore() || pad.getConstPadAfter()))
934 return bd_op->emitOpError()
935 << "D" << d
936 << " zero padding is not supported on this tile.";
937 if (channelDir == AIE::DMAChannelDir::MM2S) {
938 for (size_t i = 0; i < padDims->size(); i++) {
939 int j = padDims->size() - i - 1;
940 padBefore[i] = (*padDims)[j].getConstPadBefore();
941 padAfter[i] = (*padDims)[j].getConstPadAfter();
942 }
943 for (size_t i = padDims->size(); i < dims->size(); i++) {
944 padBefore[i] = 0;
945 padAfter[i] = 0;
946 }
947 int64_t elementBytes = bd_op.getBufferElementTypeWidthInBytes();
948 if ((padBefore[0] * elementBytes) % 4 ||
949 (padAfter[0] * elementBytes) % 4)
950 return bd_op->emitOpError()
951 << "Inner-most padding count must result in padding in "
952 "32-bit words.";
953 padBefore[0] = padBefore[0] * elementBytes / 4;
954 padAfter[0] = padAfter[0] * elementBytes / 4;
955 } else
956 return bd_op->emitOpError()
957 << "supports padding only for MM2S direction on MemTiles.";
958 }
959 getHardwareStridesWraps(target_model, bd_op, buffer_type, input_sizes,
960 input_strides, sizes, strides);
961
962 if (failed(verifyStridesWraps(
963 bd_op, [&] { return bd_op->emitOpError(); }, buffer_type,
964 tile.getCol(), tile.getRow(), input_sizes, input_strides, sizes,
965 strides, treatAsLinear))) {
966 return failure();
967 }
968
969 iteration_size = sizes[3];
970 iteration_stride = strides[3];
971
972 if (!treatAsLinear) {
973 // d0_size, d0_stride
974 d0size = sizes[0];
975 d0stride = strides[0];
976
977 // d1_size, d1_stride
978 d1size = sizes[1];
979 d1stride = strides[1];
980
981 // d2_stride
982 d2stride = strides[2];
983
984 // TODO: d2_size is a dead field; AIEDmaToNpu.cpp memtile word-packing
985 // never writes it (see its `// TODO: D2Size`); the real D2 repeat
986 // count is carried entirely by buffer_length, same as on shim tiles.
987 d2size = (target_model.isMemTile(tile.getCol(), tile.getRow()))
988 ? sizes[2]
989 : 0;
990 }
991 if (input_sizes[3] > 1 && input_strides[3] == 0) {
992 // We allow users to encode the repeat_count as a dimension 3 stride
993 // of 0. This must lower to a iteration wrap of 0, so no stride is
994 // ever added. We then repeat the BD using the repeat_count in
995 // NpuPushQueueOp.
996 iteration_size = 0;
997 iteration_stride = 0;
998 }
999
1000 // Ensure the total transfer length and the padded length expressed in the
1001 // lowest three dimensions of strides/wraps agree. (Fourth dimension is
1002 // iteration/repeat count and repeats the whole BD, so should not be
1003 // incorporated in length of a single BD invocation.)
1004 uint64_t len_dims_addr_granularity = 1;
1005 for (size_t i = 0; i < 3; i++) {
1006 len_dims_addr_granularity *= sizes[i] + padBefore[i] + padAfter[i];
1007 }
1008 // A runtime length from zero keeps the pattern for the units it adds.
1009 bool runtimeLengthFromZero =
1010 len_addr_granularity == 0 && bd_op.getLengthStateTableIdxAttr();
1011 if (len_dims_addr_granularity != len_addr_granularity &&
1012 !runtimeLengthFromZero) {
1013 auto err =
1014 bd_op->emitOpError(
1015 "Buffer descriptor length does not match length of transfer "
1016 "expressed by lowest three dimensions of data layout "
1017 "transformation strides/wraps. ")
1018 << "BD length is " << (len_addr_granularity * addr_granularity / 8)
1019 << " bytes. "
1020 << "Lowest three dimensions of data layout transformation would "
1021 "result in transfer of "
1022 << (len_dims_addr_granularity * addr_granularity / 8) << " bytes. ";
1023 err.attachNote() << "Do not include the highest dimension size in "
1024 "transfer length, as this is the BD repeat count.";
1025 return failure();
1026 }
1027 } else {
1028 if (padDims && target_model.isMemTile(tile.getCol(), tile.getRow()) &&
1029 channelDir == AIE::DMAChannelDir::MM2S) {
1030 return bd_op->emitOpError()
1031 << "Padding requires n-d data layouts expressed as "
1032 << "wrap(s) and stride(s).";
1033 }
1034 if (padDims) {
1035 return bd_op->emitOpError()
1036 << " Padding is supported only on MemTiles.";
1037 }
1038 }
1039 auto fieldsOr = gatherBdTemplateFields(builder, block, bd_op, tile,
1040 target_model, packet, outOfOrder);
1041 if (failed(fieldsOr))
1042 return failure();
1043 // failed() above guards the deref; see note in rewriteSingleBDDynamic.
1044 BdTemplateFields f = *fieldsOr; // NOLINT
1045
1046 NpuWriteBdOp::create(
1047 builder, bd_op.getLoc(), tile.getCol(), bd_id, len_addr_granularity,
1048 offset,
1049 /*enable_packet=*/f.enable_packet,
1050 /*out_of_order_id=*/f.out_of_order_id,
1051 /*packet_id=*/f.packet_id,
1052 /*packet_type=*/f.packet_type,
1053 /*d0_size=*/d0size, /*d0_stride=*/d0stride,
1054 /*d1_size=*/d1size, /*d1_stride=*/d1stride,
1055 /*d2_size=*/d2size, /*d2_stride=*/d2stride,
1056 /*iteration_current=*/f.iteration_current,
1057 /*iteration_size=*/iteration_size,
1058 /*iteration_stride=*/iteration_stride,
1059 /*next_bd=*/f.next_bd_id,
1060 /*row=*/tile.getRow(),
1061 /*use_next_bd=*/f.use_next_bd,
1062 /*valid_bd=*/1,
1063 /*lock_rel_val=*/f.lock_rel_val, /*lock_rel_id=*/f.lock_rel_id,
1064 /*lock_acq_enable=*/f.lock_acq_enable,
1065 /*lock_acq_val=*/f.lock_acq_val, /*lock_acq_id=*/f.lock_acq_id,
1066 /*d0_zero_before=*/padBefore[0],
1067 /*d1_zero_before=*/padBefore[1], /*d2_zero_before=*/padBefore[2],
1068 /*d0_zero_after=*/padAfter[0], /*d1_zero_after=*/padAfter[1],
1069 /*d2_zero_after=*/padAfter[2],
1070 /*burst_length=*/bd_op.getBurstLength(),
1071 /*axcache=*/
1072 target_model.isShimNOCTile(tile.getCol(), tile.getRow())
1073 ? builder.getI32IntegerAttr(bd_op.getAxcacheOrDefault())
1074 : IntegerAttr());
1075 if (failed(setAddressForSingleBD(
1076 builder, bd_op, tile, {},
1077 builder.getI64IntegerAttr(len_addr_granularity))))
1078 return failure();
1079
1080 // A length_state_table_idx adds the runtime length to the BD's
1081 // Buffer_Length, which the BD write has already set to the static length.
1082 // The verifier requires a length_unit with it.
1083 auto lengthUnit = bd_op.getLengthUnit();
1084 if (bd_op.getLengthStateTableIdxAttr() && lengthUnit)
1086 builder, bd_op, buffer_type, *lengthUnit, target_model, tile.getCol(),
1087 tile.getRow(), bd_id);
1088 return success();
1089 }
1090
1091 LogicalResult hoistNextBdOpsIntoAttrs(DMAConfigureTaskOp op) {
1092 Region &body = op.getBody();
1093 for (auto &block : body) {
1094 if (shouldSkipBlock(block)) {
1095 continue;
1096 }
1097 AIE::DMABDOp bd_op = getBdForBlock(block);
1098 if (AIE::NextBDOp next_bd_op =
1099 llvm::dyn_cast<AIE::NextBDOp>(block.getTerminator())) {
1100 if (bd_op.getNextBdId().has_value()) {
1101 auto error =
1102 bd_op.emitOpError("Cannot specify both next_bd_id attribute and "
1103 "aie.next_bd operation.");
1104 error.attachNote(next_bd_op.getLoc())
1105 << "Potentially conflicting next buffer descriptor ID specified "
1106 "here.";
1107 return failure();
1108 }
1109 Block &next_bd_block = *next_bd_op.getDest();
1110 AIE::DMABDOp next_dma_bd_op = getBdForBlock(next_bd_block);
1111 // verifyBdInBlock has required a pinned or a runtime id on every BD.
1112 if (Value nextBdIdVal = next_dma_bd_op.getBdIdVal())
1113 runtimeNextBdId[bd_op] = nextBdIdVal;
1114 else
1115 bd_op.setNextBdId(next_dma_bd_op.getBdId());
1116 OpBuilder builder(next_bd_op);
1117 AIE::EndOp::create(builder, next_bd_op.getLoc());
1118 next_bd_op.erase();
1119 }
1120 }
1121 return success();
1122 }
1123
1124 LogicalResult emitOutOfOrderChannelEnable(OpBuilder &builder,
1125 DMAConfigureTaskOp op,
1126 AIE::TileOp tile) {
1128 int col = tile.getCol();
1129 int row = tile.getRow();
1130 int channel = op.getChannel();
1131
1132 uint32_t ctrlAddrLocal = tm.getLocalDmaControlAddress(
1133 col, row, channel, AIE::DMAChannelDir::S2MM);
1134
1135 std::string ctrlRegName = "DMA_S2MM_" + std::to_string(channel) + "_Ctrl";
1136 // DMA control regs live in the memory module; memtile ignores isMem, shim
1137 // is rejected by the verifier.
1138 const AIE::RegisterInfo *ctrlReg =
1139 tm.lookupRegister(ctrlRegName, tile.getTileID(), /*isMem=*/true);
1140 if (!ctrlReg)
1141 return op.emitOpError("target has no ")
1142 << ctrlRegName << " register in its register database";
1143 const AIE::BitFieldInfo *oooField =
1144 ctrlReg->getField("Enable_Out_of_Order");
1145 if (!oooField)
1146 return op.emitOpError()
1147 << ctrlRegName
1148 << " has no Enable_Out_of_Order field in the register database";
1149 std::optional<uint32_t> oooMask = tm.getFieldMask(*oooField);
1150 if (!oooMask)
1151 return op.emitOpError()
1152 << ctrlRegName
1153 << " Enable_Out_of_Order field does not fit in a 32-bit register";
1154 std::optional<uint32_t> oooValue = tm.encodeFieldValue(*oooField, 1);
1155 if (!oooValue)
1156 return op.emitOpError()
1157 << ctrlRegName
1158 << " Enable_Out_of_Order field value does not fit in its bit "
1159 "width";
1160
1161 Location loc = op.getLoc();
1162 IntegerAttr colAttr = builder.getI32IntegerAttr(col);
1163 IntegerAttr rowAttr = builder.getI32IntegerAttr(row);
1164 Value addr = createConstantI32(builder, loc, ctrlAddrLocal);
1165 Value val = createConstantI32(builder, loc, *oooValue);
1166 Value mask = createConstantI32(builder, loc, *oooMask);
1167 NpuMaskWrite32Op::create(builder, loc, addr, val, mask, nullptr, colAttr,
1168 rowAttr);
1169 return success();
1170 }
1171
1173 DMAConfigureTaskOp op,
1174 const llvm::DenseMap<Operation *, bool> &onlyPushedOnce) {
1175 OpBuilder builder(op);
1176 AIE::TileOp tile = op.tryGetTileOp();
1177 if (!tile)
1178 return emitUnplacedTileError(op, op);
1179
1180 if (!op.use_empty()) {
1181 auto err = op.emitOpError("Cannot lower while op still has uses.");
1182 mlir::Operation::use_range uses = op.getOperation()->getUses();
1183 for (auto &use : uses) {
1184 err.attachNote(use.getOwner()->getLoc()) << "Used here.";
1185 }
1186 return failure();
1187 }
1188
1189 Region &body = op.getBody();
1191
1192 // Verify each BD block first; subsequent functions rely on them being
1193 // well-formed
1194 for (auto &it : body) {
1195 if (shouldSkipBlock(it)) {
1196 continue;
1197 }
1198 if (failed(verifyNoUnsupportedOpsInBlock(it))) {
1199 return failure();
1200 }
1201 if (failed(verifyBdInBlock(it))) {
1202 return failure();
1203 }
1204 AIE::UseLockOp acquire, release;
1205 if (failed(
1206 AIE::verifyBdLockPair(it, op.getOutOfOrder(), acquire, release)))
1207 return failure();
1208 }
1209
1210 if (op.getOutOfOrder()) {
1211 if (failed(emitOutOfOrderChannelEnable(builder, op, tile)))
1212 return failure();
1213 }
1214
1215 // Hoist next_bd operations into next_bd_id attribute of the dma_bd
1216 runtimeNextBdId.clear();
1217 if (failed(hoistNextBdOpsIntoAttrs(op))) {
1218 return failure();
1219 }
1220
1221 auto channelDir = op.getDirection();
1222 auto packet = op.getPacket();
1223
1224 // One push onto the task queue runs one iteration of a BD's iteration
1225 // dimension; only a push's repeat_count runs the rest. Warn once the
1226 // BDs are known to lower, so an invalid one gets only its error.
1227 SmallVector<std::pair<Location, int64_t>> unrepeatedIterations;
1228 if (onlyPushedOnce.lookup(op)) {
1229 for (auto bd_op : body.getOps<AIE::DMABDOp>()) {
1230 SmallVector<OpFoldResult> sizes = bd_op.getMixedSizes();
1231 std::optional<int64_t> iterations;
1232 if (std::optional<AIE::BDIterationAttr> iter = bd_op.getIteration())
1233 iterations = iter->getSize();
1234 else if (sizes.size() == 4)
1235 iterations = getConstantIntValue(sizes.front());
1236 if (iterations && *iterations > 1)
1237 unrepeatedIterations.emplace_back(bd_op.getLoc(), *iterations);
1238 }
1239 }
1240
1241 // Lower all BDs
1242 for (auto &block : body) {
1243 if (shouldSkipBlock(block)) {
1244 continue;
1245 }
1246 if (failed(rewriteSingleBD(builder, block, tile, channelDir, packet,
1247 op.getOutOfOrder()))) {
1248 return failure();
1249 }
1250 }
1251
1252 for (auto [loc, iterations] : unrepeatedIterations)
1253 mlir::emitWarning(loc)
1254 << "iteration dimension of size " << iterations
1255 << " is pushed with repeat_count 0, so only its first iteration "
1256 "runs; set repeat_count = "
1257 << iterations - 1;
1258
1259 op.erase();
1260
1261 return success();
1262 }
1263
1265 AIE::DeviceOp device,
1266 const llvm::DenseMap<Operation *, bool> &onlyPushedOnce) {
1267 WalkResult result = device.walk([&](DMAConfigureTaskOp op) {
1268 if (failed(rewriteSingleDMAConfigureTaskOp(op, onlyPushedOnce))) {
1269 return WalkResult::interrupt();
1270 }
1271 return WalkResult::advance();
1272 });
1273 if (result.wasInterrupted()) {
1274 return failure();
1275 }
1276 return success();
1277 }
1278
1279 // Drop the dead task-index carries left by the dynamic BD pool path once
1280 // awaits have lowered to npu.sync. scf.for/scf.if carry the task value as a
1281 // result purely to hold the await's data dependence on the configure; with
1282 // the await gone that carry is dead but still counts as a use of the
1283 // branch-local configure, blocking its use_empty lowering. scf's own
1284 // canonicalizations remove dead iter-args/results and prune the yields
1285 // feeding them. Run them device-wide but with folding and constant-CSE
1286 // DISABLED, so the static path's byte-golden constant emission is untouched
1287 // -- only the dead scf carries go.
1288 //
1289 // scf's canonicalizations only see one level at a time: a task carried
1290 // through NESTED loops (an outer iter_arg feeding an inner loop's init) is
1291 // never provably dead to either loop in isolation, since each still sees a
1292 // genuine SSA use feeding the other -- even once the whole chain is
1293 // unobserved end to end. RemoveDeadValues performs the cross-region
1294 // liveness analysis (via RegionBranchOpInterface, generic over nesting
1295 // depth) needed to prune that fully; run it as a nested pipeline afterward
1296 // to pick up what the local canonicalizations left behind.
1297 LogicalResult dropDeadTaskCarries(AIE::DeviceOp device) {
1298 // Only a region-carried value can be a dead carry, and a static sequence
1299 // has none; the analysis costs time proportional to the whole device.
1300 if (llvm::none_of(device.getOps<AIE::RuntimeSequenceOp>(), [](auto seq) {
1301 return seq
1302 .walk([](RegionBranchOpInterface) {
1303 return WalkResult::interrupt();
1304 })
1305 .wasInterrupted();
1306 }))
1307 return success();
1308
1309 RewritePatternSet patterns(&getContext());
1310 scf::ForOp::getCanonicalizationPatterns(patterns, &getContext());
1311 scf::IfOp::getCanonicalizationPatterns(patterns, &getContext());
1312 GreedyRewriteConfig config;
1313 config.enableFolding(false).enableConstantCSE(false);
1314 if (failed(applyPatternsGreedily(device, std::move(patterns), config)))
1315 return failure();
1316
1317 OpPassManager pm(AIE::DeviceOp::getOperationName());
1318 pm.addPass(createRemoveDeadValuesPass());
1319 return runPipeline(pm, device);
1320 }
1321
1322 void runOnOperation() override {
1323 AIE::DeviceOp device = getOperation();
1324
1325 // Lower the starts and awaits. Each rewrites one op into ops that need no
1326 // further lowering, so one walk does it.
1327 RewritePatternSet patterns(&getContext());
1328 llvm::DenseMap<Operation *, bool> onlyPushedOnce;
1329 patterns.insert<DMAStartTaskOpPattern>(&getContext(), onlyPushedOnce);
1330 patterns.insert<DMAAwaitTaskOpPattern>(&getContext());
1331 walkAndApplyPatterns(device, std::move(patterns));
1332 // A start or await left unlowered still uses its configure, so lowering
1333 // the configures would only add errors.
1334 bool unlowered = false;
1335 device.walk([&](Operation *op) {
1336 if (isa<DMAStartTaskOp, DMAAwaitTaskOp>(op)) {
1337 op->emitError() << "failed to legalize operation '" << op->getName()
1338 << "'";
1339 unlowered = true;
1340 }
1341 });
1342 if (unlowered)
1343 return signalPassFailure();
1344
1345 // Drop the now-dead task-index carries the awaits held, so the branch-local
1346 // configures they used can reach use_empty and lower below.
1347 if (failed(dropDeadTaskCarries(device)))
1348 signalPassFailure();
1349
1350 // Lower the configuration for the BDs
1351 if (failed(rewriteDMAConfigureTaskOp(device, onlyPushedOnce))) {
1352 signalPassFailure();
1353 }
1354
1355 eraseDeadArith(device);
1356 }
1357};
1358
1359std::unique_ptr<OperationPass<AIE::DeviceOp>>
1361 return std::make_unique<AIEDMATasksToNPUPass>();
1362}
std::optional< uint32_t > encodeFieldValue(const BitFieldInfo &field, uint32_t value) const
Encode a field value with proper bit shifting.
std::optional< uint32_t > getMemLocalBaseAddress(int localCol, int localRow, int memCol, int memRow) const
Return the memory base address (or offset) in the local tile when accessing a neighbor's memory or an...
uint32_t getLocalDmaControlAddress(int col, int row, int channel, AIE::DMAChannelDir direction) const
Return the DMA task-queue register address relative to its tile.
virtual const DmaBdLayout * getDmaBdLayout(AIETileType tileType) const
Return the register layout of a buffer descriptor on the given tile type, or null if the target has n...
bool isMemTile(int col, int row) const
Return true if the given tile is a Mem tile.
std::optional< uint32_t > getLockLocalBaseIndex(int localCol, int localRow, int lockCol, int lockRow) const
Return the lock base index (or offset) in the local tile when accessing a neighbor's lock or an empty...
std::optional< uint32_t > getFieldMask(const BitFieldInfo &field) const
Compute a 32-bit mask for a register field.
const RegisterInfo * lookupRegister(llvm::StringRef name, TileID tile, bool isMem=false) const
Register Database API - provides access to register and event information for trace configuration and...
virtual uint64_t getDmaBdAddress(int col, int row, uint32_t bd_id, int channel=-1, AIE::DMAChannelDir direction=AIE::DMAChannelDir::MM2S) const =0
Return the array address of the dma buffer descriptor for the given col, row, buffer descriptor id,...
virtual uint32_t getDmaBdAddressOffset(int col, int row) const =0
Return the offset of the base address field within the shim dma buffer descriptor.
bool isShimNOCTile(int col, int row) const
Return true if the given tile is a ShimNOC tile.
virtual uint32_t getMaxLockValue() const =0
Return the maximum value that can be stored in a lock register.
virtual uint32_t getAddressGenGranularity() const =0
Return the data bus width of the device.
uint64_t getBdAddressUnit(const AIE::AIETargetModel &targetModel, int col, int row)
Definition AIEUtils.cpp:207
mlir::LogicalResult verifyStridesWraps(mlir::Operation *forOp, llvm::function_ref< mlir::InFlightDiagnostic()> emitError, mlir::BaseMemRefType referencedBufType, int tileCol, int tileRow, llvm::SmallVector< int64_t, 4 > inputSizes, llvm::SmallVector< int64_t, 4 > inputStrides, llvm::SmallVector< int64_t, 4 > hardwareSizes, llvm::SmallVector< int64_t, 4 > hardwareStrides, bool skipTransformationChecks=false)
DMAConfigureTaskOp getUniqueReachableConfigure(Value task)
Definition AIEUtils.cpp:563
std::optional< SubviewTraceResult > traceSubviewToBlockArgument(Value value)
Definition AIEUtils.cpp:25
mlir::Value buildArgPlusValue(mlir::OpBuilder &builder, mlir::Location loc, llvm::ArrayRef< mlir::OpFoldResult > elementOffsets, llvm::ArrayRef< mlir::OpFoldResult > strides, int64_t elemWidthBytes, int64_t baseByteOffset, uint32_t granuleBytes)
mlir::Value getAsValue(mlir::OpBuilder &builder, mlir::Location loc, mlir::OpFoldResult ofr, mlir::Type intType)
mlir::LogicalResult guardWithinHostBuffer(mlir::OpBuilder &builder, mlir::Location loc, mlir::BaseMemRefType hostBufferType, int64_t baseByteOffset, int64_t elemWidthBytes, llvm::ArrayRef< mlir::OpFoldResult > offsets, llvm::ArrayRef< mlir::OpFoldResult > offsetStrides, llvm::ArrayRef< mlir::OpFoldResult > sizes, llvm::ArrayRef< mlir::OpFoldResult > strides)
mlir::LogicalResult buildBdWords(mlir::OpBuilder &builder, mlir::Location loc, const xilinx::AIE::AIETargetModel &targetModel, int tileCol, int tileRow, const BdTemplateFields &fields, llvm::ArrayRef< mlir::OpFoldResult > mixedSizes, llvm::ArrayRef< mlir::OpFoldResult > mixedStrides, uint64_t elemWidth, uint32_t burstLength, uint32_t axcache, mlir::OpFoldResult lenElems, mlir::Value &repeatCountOut, llvm::SmallVectorImpl< mlir::Value > &wordsOut)
LogicalResult emitUpdateBdAddressFromOffsetParameter(OpBuilder &builder, Operation *bdOp, BaseMemRefType bufType, uint64_t registerAddr, const AIE::AIETargetModel &targetModel, int col, int row, std::optional< uint64_t > staticByteAddress=std::nullopt, OpFoldResult lengthField={})
Definition AIEUtils.cpp:214
std::unique_ptr< mlir::OperationPass< AIE::DeviceOp > > createAIEDMATasksToNPUPass()
void eraseDeadArith(mlir::Operation *root)
void getHardwareStridesWraps(const AIE::AIETargetModel &targetModel, mlir::Operation *op, mlir::BaseMemRefType referencedBufType, llvm::SmallVector< int64_t, 4 > inputSizes, llvm::SmallVector< int64_t, 4 > inputStrides, llvm::SmallVector< int64_t, 4 > &sizes, llvm::SmallVector< int64_t, 4 > &strides)
bool isLinearModeTransfer(const AIE::AIETargetModel &targetModel, int col, int row, llvm::ArrayRef< int64_t > sizes, llvm::ArrayRef< int64_t > strides)
mlir::Value getBdRegisterBase(mlir::OpBuilder &builder, mlir::Location loc, const xilinx::AIE::AIETargetModel &targetModel, int tileCol, int tileRow, mlir::OpFoldResult bdId)
mlir::LogicalResult emitRuntimeCheck(mlir::OpBuilder &builder, mlir::Location loc, mlir::Value cond, const llvm::Twine &message)
mlir::LogicalResult verifyConstBdRealizability(mlir::Operation *op, llvm::ArrayRef< mlir::OpFoldResult > sizes, llvm::ArrayRef< mlir::OpFoldResult > strides, uint64_t elemWidth, uint32_t gran)
Definition BdLowering.h:77
bool getReachableConfigures(Value task, SmallVectorImpl< DMAConfigureTaskOp > &configures)
Definition AIEUtils.cpp:527
LogicalResult emitUpdateBdLengthFromParameter(OpBuilder &builder, Operation *bdOp, BaseMemRefType bufType, int64_t lengthUnit, const AIE::AIETargetModel &targetModel, int col, int row, int bdId)
Definition AIEUtils.cpp:296
std::optional< unsigned > getHostBufferArgIndex(BlockArgument arg)
Definition AIEUtils.cpp:149
mlir::Value createConstantI32(mlir::OpBuilder &builder, mlir::Location loc, uint32_t value)
mlir::LogicalResult verifyBdLockPair(mlir::Block &block, bool outOfOrder, UseLockOp &acquire, UseLockOp &release)
const AIETargetModel & getTargetModel(mlir::Operation *op)
void placeRuntimeLengthDimension(llvm::MutableArrayRef< int64_t > sizes, llvm::MutableArrayRef< int64_t > strides)
void runOnOperation() override
LogicalResult dropDeadTaskCarries(AIE::DeviceOp device)
bool shouldSkipBlock(Block &block)
LogicalResult rewriteSingleBDDynamic(OpBuilder &builder, Block &block, AIE::DMABDOp bd_op, AIE::TileOp &tile, std::optional< xilinx::AIE::PacketInfoAttr > packet, Value runtimeBdId=nullptr, bool outOfOrder=false)
void hoistScalarOpsOutOfBdBlocks(DMAConfigureTaskOp op)
LogicalResult rewriteDMAConfigureTaskOp(AIE::DeviceOp device, const llvm::DenseMap< Operation *, bool > &onlyPushedOnce)
LogicalResult verifyNoUnsupportedOpsInBlock(Block &block)
LogicalResult rewriteSingleBD(OpBuilder &builder, Block &block, AIE::TileOp &tile, AIE::DMAChannelDir channelDir, std::optional< xilinx::AIE::PacketInfoAttr > packet, bool outOfOrder=false)
LogicalResult emitOutOfOrderChannelEnable(OpBuilder &builder, DMAConfigureTaskOp op, AIE::TileOp tile)
AIE::DMABDOp getBdForBlock(Block &block)
LogicalResult hoistNextBdOpsIntoAttrs(DMAConfigureTaskOp op)
LogicalResult verifyBdInBlock(Block &block)
llvm::DenseMap< AIE::DMABDOp, Value > runtimeNextBdId
LogicalResult setAddressForSingleBD(OpBuilder &builder, AIE::DMABDOp &bd_op, AIE::TileOp &tile, OpFoldResult bdId, OpFoldResult lengthField)
lengthField is the value the BD write gave its length field.
FailureOr< BdTemplateFields > gatherBdTemplateFields(OpBuilder &builder, Block &block, AIE::DMABDOp bd_op, AIE::TileOp &tile, const AIE::AIETargetModel &target_model, std::optional< xilinx::AIE::PacketInfoAttr > packet, bool outOfOrder=false)
LogicalResult rewriteSingleDMAConfigureTaskOp(DMAConfigureTaskOp op, const llvm::DenseMap< Operation *, bool > &onlyPushedOnce)
LogicalResult matchAndRewrite(DMAAwaitTaskOp op, PatternRewriter &rewriter) const override
llvm::DenseMap< Operation *, bool > & onlyPushedOnce
LogicalResult matchAndRewrite(DMAStartTaskOp op, PatternRewriter &rewriter) const override
DMAStartTaskOpPattern(MLIRContext *context, llvm::DenseMap< Operation *, bool > &onlyPushedOnce)
Bit field information for a register.
Bits [shift, shift + width) of register word in a buffer descriptor's register block.
uint32_t place(uint64_t value) const
uint32_t mask() const
The register layout of one tile type's DMA buffer descriptor.
llvm::SmallVector< std::pair< DmaBdField, DmaBdField >, 3 > padFields() const
The (before, after) zero-padding fields, innermost dimension first.
const BitFieldInfo * getField(llvm::StringRef fieldName) const