MLIR-AIE 1.4.6.dev
AIEDecomposeLargeDmaBd.cpp
Go to the documentation of this file.
1//===- AIEDecomposeLargeDmaBd.cpp -------------------------------*- C++ -*-===//
2//
3// Copyright (C) 2026 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7//
8// Decomposes oversized non-contiguous aiex.npu.dma_memcpy_nd ops and task-path
9// aie.dma_bd ops into one or more hardware-legal ND transfers before
10// aie-dma-to-npu / aie-dma-tasks-to-npu lowering.
11//
12//===----------------------------------------------------------------------===//
13
19
20#include "mlir/Dialect/MemRef/IR/MemRef.h"
21#include "mlir/Dialect/SCF/IR/SCF.h"
22#include "mlir/Dialect/Utils/StaticValueUtils.h"
23#include "mlir/IR/PatternMatch.h"
24#include "mlir/Interfaces/SideEffectInterfaces.h"
25#include "mlir/Pass/Pass.h"
26#include "llvm/ADT/DenseSet.h"
27#include "llvm/ADT/SetVector.h"
28#include "llvm/Support/raw_ostream.h"
29
30#include <limits>
31#include <numeric>
32#include <set>
33#include <string>
34
35namespace xilinx::AIEX {
36#define GEN_PASS_DEF_AIEDECOMPOSELARGEDMABD
37#include "aie/Dialect/AIEX/Transforms/AIEXPasses.h.inc"
38} // namespace xilinx::AIEX
39
40using namespace mlir;
41using namespace xilinx;
42using namespace xilinx::AIEX;
43
44namespace {
45
46// `op`'s pattern, innermost first, when its offsets, sizes and strides are
47// constant.
48static std::optional<NdDmaPattern> constantPattern(NpuDmaMemcpyNdOp op) {
49 std::optional<SmallVector<int64_t>> offsets =
50 getConstantIntValues(op.getMixedOffsets());
51 std::optional<SmallVector<int64_t>> sizes =
52 getConstantIntValues(op.getMixedSizes());
53 std::optional<SmallVector<int64_t>> strides =
54 getConstantIntValues(op.getMixedStrides());
55 if (!offsets || !sizes || !strides)
56 return std::nullopt;
57 NdDmaPattern pattern;
58 pattern.offsets.assign(offsets->rbegin(), offsets->rend());
59 pattern.sizes.assign(sizes->rbegin(), sizes->rend());
60 pattern.strides.assign(strides->rbegin(), strides->rend());
61 return pattern;
62}
63
64static SmallVector<int64_t, 4> toOuter(ArrayRef<int64_t> inner) {
65 return llvm::map_to_vector(llvm::reverse(inner), [](int64_t v) { return v; });
66}
67
68static int64_t flatOffsetFromPattern(int64_t baseFlatOffset,
69 const NdDmaPattern &pattern) {
70 int64_t flat = baseFlatOffset + pattern.baseOffset;
71 for (unsigned k = 0; k < 4; ++k)
72 flat += pattern.offsets[k] * pattern.strides[k];
73 return flat;
74}
75
76static int64_t lenFromInnermost3(ArrayRef<int64_t> sizesInnermostFirst) {
77 int64_t len = 1;
78 for (unsigned i = 0; i < 3; ++i)
79 len *= sizesInnermostFirst[i];
80 return len;
81}
82
83static AIE::BDDimLayoutArrayAttr outerDimsAttr(MLIRContext *ctx,
84 ArrayRef<int64_t> outerSizes,
85 ArrayRef<int64_t> outerStrides) {
86 SmallVector<AIE::BDDimLayoutAttr> dims;
87 dims.reserve(outerSizes.size());
88 for (auto [s, t] : llvm::zip(outerSizes, outerStrides))
89 dims.push_back(AIE::BDDimLayoutAttr::get(ctx, static_cast<uint32_t>(s),
90 static_cast<uint32_t>(t)));
91 return AIE::BDDimLayoutArrayAttr::get(ctx, dims);
92}
93
94static void updateTaskBdInPlace(AIE::DMABDOp bd, int32_t offset, int32_t len,
95 ArrayRef<int64_t> outerSizes,
96 ArrayRef<int64_t> outerStrides) {
97 bd.getOffsetMutable().clear();
98 bd.setStaticOffset(offset);
99 bd.getLenMutable().clear();
100 bd.setStaticLen(len);
101 bd.getSizesMutable().clear();
102 bd.getStridesMutable().clear();
103 bd.setStaticSizes(DenseI64ArrayAttr::get(bd.getContext(), outerSizes));
104 bd.setStaticStrides(DenseI64ArrayAttr::get(bd.getContext(), outerStrides));
105}
106
107static AIE::DMABDOp createTaskBd(RewriterBase &rewriter, Location loc,
108 AIE::DMABDOp tmpl, int32_t offset, int32_t len,
109 ArrayRef<int64_t> outerSizes,
110 ArrayRef<int64_t> outerStrides) {
111 auto dims = outerDimsAttr(rewriter.getContext(), outerSizes, outerStrides);
112 auto bd =
113 AIE::DMABDOp::create(rewriter, loc, tmpl.getBuffer(), offset, len, dims);
114 if (tmpl.getPacketAttr())
115 bd.setPacketAttr(tmpl.getPacketAttr());
116 if (tmpl.getBurstLengthAttr())
117 bd.setBurstLengthAttr(tmpl.getBurstLengthAttr());
118 if (tmpl.getAxcacheAttr())
119 bd.setAxcacheAttr(tmpl.getAxcacheAttr());
120 if (tmpl.getOffsetParameterAttr())
121 bd.setOffsetParameterAttr(tmpl.getOffsetParameterAttr());
122 // out_of_order_id is not copied because slicing an OoO BD is rejected above.
123 return bd;
124}
125
126// How many executions one pass over a task BD's pattern takes: one per index
127// of its iteration dimensions, all of them past d2.
128static int64_t iterationCount(const NdDmaPattern &pattern) {
129 int64_t count = 1;
130 for (int64_t size : llvm::drop_begin(pattern.sizes, 3))
131 count *= size;
132 return count;
133}
134
135// The factor by which decomposition scaled the executions of a pass, num/den
136// in lowest terms: a pass over the original takes `iterations` executions,
137// and over `after` one per index of its iteration dimension. Below one, the
138// executions merged, `after`'s inner dimensions taking in one the original
139// iterated over.
140struct IterationScale {
141 int64_t num = 1;
142 int64_t den = 1;
143
144 bool isOne() const { return num == den; }
145 // Whether `runs` executions of the original are whole executions of the
146 // rewrite.
147 bool divides(int64_t runs) const { return runs % den == 0; }
148 int64_t apply(int64_t runs) const { return runs / den * num; }
149 std::string str() const {
150 return den == 1 ? std::to_string(num)
151 : std::to_string(num) + "/" + std::to_string(den);
152 }
153};
154
155static IterationScale iterationScale(int64_t iterations,
156 const NdDmaPattern &after) {
157 int64_t common = std::gcd(after.sizes[3], iterations);
158 return {after.sizes[3] / common, iterations / common};
159}
160
161static unsigned countTaskBds(DMAConfigureTaskLike taskOp) {
162 unsigned n = 0;
163 taskOp->walk([&](AIE::DMABDOp) { ++n; });
164 return n;
165}
166
167static bool isUnderRuntimeControlFlow(AIE::DMABDOp op) {
168 auto seq = op->getParentOfType<AIE::RuntimeSequenceOp>();
169 if (!seq)
170 return false;
171 for (Operation *parent = op->getParentOp(); parent && parent != seq;
172 parent = parent->getParentOp()) {
173 if (isa<scf::ForOp, scf::WhileOp, scf::IfOp>(parent))
174 return true;
175 }
176 return false;
177}
178
179// The task a runtime descriptor belongs to, with the tile and channel it runs
180// on.
181static std::optional<std::tuple<AIE::TileOp, DMAConfigureTaskLike, int>>
182resolveTaskAndTile(AIE::DMABDOp op) {
183 if (auto cfg = op->getParentOfType<DMAConfigureTaskOp>()) {
184 // Decomposition is a shape rewrite, so an unplaced tile is not an error
185 // here: skip the descriptor until placement has run.
186 AIE::TileOp tile = cfg.tryGetTileOp();
187 if (!tile)
188 return std::nullopt;
189 return std::make_tuple(tile, cast<DMAConfigureTaskLike>(*cfg),
190 static_cast<int>(cfg.getChannel()));
191 }
192 if (auto cfgFor = op->getParentOfType<DMAConfigureTaskForOp>()) {
193 AIE::DeviceOp dev = op->getParentOfType<AIE::DeviceOp>();
194 if (!dev)
195 return std::nullopt;
196 auto allocOp = AIE::ShimDMAAllocationOp::getForSymbol(
197 dev, cfgFor.getAlloc().getRootReference());
198 if (!allocOp)
199 return std::nullopt;
200 AIE::TileOp tile = allocOp.getTileOp();
201 if (!tile)
202 return std::nullopt;
203 return std::make_tuple(tile, cast<DMAConfigureTaskLike>(*cfgFor),
204 static_cast<int>(allocOp.getChannelIndex()));
205 }
206 return std::nullopt;
207}
208
209static NpuDmaMemcpyNdOp createDecomposedOp(RewriterBase &rewriter,
210 NpuDmaMemcpyNdOp op,
211 const NdDmaPattern &pattern,
212 int64_t id, bool issueToken) {
213 assert(pattern.baseOffset == 0 &&
214 "a memcpy pattern has no peeled dimensions");
215 auto newOp = NpuDmaMemcpyNdOp::create(
216 rewriter, op.getLoc(), op, toOuter(pattern.offsets),
217 toOuter(pattern.sizes), toOuter(pattern.strides));
218 newOp.setIdAttr(rewriter.getI64IntegerAttr(id));
219 newOp.setIssueTokenAttr(rewriter.getBoolAttr(issueToken));
220 return newOp;
221}
222
223static int64_t allocateNextId(NpuDmaMemcpyNdOp op, int64_t startId,
224 llvm::DenseSet<int64_t> &usedIds) {
225 int64_t id = startId;
226 while (usedIds.contains(id))
227 ++id;
228 usedIds.insert(id);
229 return id;
230}
231
232// The starts splitIntoTasks emits, until orderSlices has placed them.
233struct Slices {
234 struct Start {
235 int64_t group, index, count;
236 };
237 llvm::DenseMap<Operation *, Start> starts;
238 // The configures of the slices after the first, which orderSlices moves up
239 // to their first start.
240 llvm::DenseSet<Operation *> lateTasks;
241 llvm::SetVector<Block *> blocks;
242 int64_t nextGroup = 0;
243};
244
245// Why the slices of `bd` cannot go out as one task each, or nullopt if they
246// can. Every use of the task has to be one splitIntoTasks rewrites, and every
247// start has to run whole passes over the pattern, a pass being one execution
248// per index of the `iterations`-long iteration dimension: the slices of a pass
249// are issued in turn, so a partial pass has no slice boundary to stop at.
250static std::optional<std::string>
251whyNotSeparateTasks(AIE::DMABDOp bd, DMAConfigureTaskLike taskOp,
252 int64_t iterations) {
253 std::string why;
254 llvm::raw_string_ostream os(why);
255 if (bd.getBdIdVal()) {
256 os << "the descriptor's bd_id is a runtime value";
257 return why;
258 }
259 if (taskOp.getRepeatCountVal()) {
260 os << "the task's repeat count is a runtime value";
261 return why;
262 }
263 for (Operation *user : taskOp->getResult(0).getUsers()) {
264 if (!isa<DMAStartTaskOp, DMAAwaitTaskOp, DMAFreeTaskOp>(user)) {
265 os << "the task is used by '" << user->getName() << "'";
266 return why;
267 }
268 if (user->getBlock() != taskOp->getBlock()) {
269 os << "the task is used outside the block that configures it";
270 return why;
271 }
272 auto start = dyn_cast<DMAStartTaskOp>(user);
273 if (!start)
274 continue;
275 if (start.getRepeatCountVal()) {
276 os << "a start's repeat count is a runtime value";
277 return why;
278 }
279 if (start.hasRepeatParameter()) {
280 os << "a start's repeat count is a scratchpad parameter";
281 return why;
282 }
283 std::optional<uint32_t> rc = start.getRepeatCount();
284 int64_t runs = static_cast<int64_t>(rc ? *rc : taskOp.getRepeatCount());
285 ++runs;
286 if (runs % iterations != 0) {
287 os << "a start runs it " << runs
288 << " times, not a whole number of passes over its " << iterations
289 << "-long iteration dimension";
290 return why;
291 }
292 }
293 return std::nullopt;
294}
295
296// Issue the slices of `bd` as one single-descriptor task each. The task
297// becomes the first slice and the others are configured right after it, until
298// orderSlices moves each up to its first start. Only the last slice issues the
299// task's token, so an await of the task becomes an await of the last slice,
300// and a free frees every slice. Each start becomes a
301// group of consecutive starts, every slice once per pass, which orderSlices
302// then interleaves with the starts around it.
303static void splitIntoTasks(RewriterBase &rewriter, AIE::DMABDOp bd,
304 DMAConfigureTaskLike taskOp,
305 ArrayRef<NdDmaPattern> slices,
306 int64_t baseFlatOffset, int64_t iterations,
307 Slices &state) {
308 bool token = taskOp.getIssueToken();
309 int64_t taskRuns = static_cast<int64_t>(taskOp.getRepeatCount()) + 1;
310 SmallVector<Operation *> users(taskOp->getResult(0).getUsers());
311
312 SmallVector<DMAConfigureTaskLike> tasks{taskOp};
313 rewriter.setInsertionPointAfter(taskOp);
314 for (size_t i = 1; i < slices.size(); ++i) {
315 tasks.push_back(cast<DMAConfigureTaskLike>(rewriter.clone(*taskOp)));
316 state.lateTasks.insert(tasks.back());
317 }
318
319 for (auto it : llvm::enumerate(slices)) {
320 size_t i = it.index();
321 const NdDmaPattern &sub = it.value();
322 DMAConfigureTaskLike task = tasks[i];
323 AIE::DMABDOp sliceBd = bd;
324 if (i > 0)
325 task->walk([&](AIE::DMABDOp b) { sliceBd = b; });
326 int64_t flatOffset = flatOffsetFromPattern(baseFlatOffset, sub);
327 int32_t len = static_cast<int32_t>(lenFromInnermost3(sub.sizes));
328 rewriter.modifyOpInPlace(sliceBd, [&]() {
329 updateTaskBdInPlace(sliceBd, static_cast<int32_t>(flatOffset), len,
330 toOuter(sub.sizes), toOuter(sub.strides));
331 // A pinned id stays with the first slice, as in a chain.
332 if (i > 0)
333 sliceBd.removeBdIdAttr();
334 });
335 rewriter.modifyOpInPlace(task, [&]() {
336 // One pass over the slice is one execution per index of its own
337 // iteration dimension.
338 task.setRepeatCount(static_cast<uint32_t>(sub.sizes[3] - 1));
339 task.setIssueToken(token && i + 1 == slices.size());
340 });
341 }
342
343 for (Operation *user : users) {
344 if (auto await = dyn_cast<DMAAwaitTaskOp>(user)) {
345 rewriter.modifyOpInPlace(await, [&]() {
346 await.getTaskMutable().assign(tasks.back()->getResult(0));
347 });
348 continue;
349 }
350 if (auto free = dyn_cast<DMAFreeTaskOp>(user)) {
351 rewriter.setInsertionPointAfter(free);
352 for (DMAConfigureTaskLike task : llvm::drop_begin(tasks))
353 DMAFreeTaskOp::create(rewriter, free.getLoc(), task->getResult(0));
354 continue;
355 }
356 auto start = cast<DMAStartTaskOp>(user);
357 std::optional<uint32_t> rc = start.getRepeatCount();
358 int64_t runs = rc ? static_cast<int64_t>(*rc) + 1 : taskRuns;
359 int64_t count = runs / iterations * static_cast<int64_t>(slices.size());
360 int64_t group = state.nextGroup++;
361 // Only the last start of the last slice may issue the token: an earlier
362 // pass's would complete an await before the transfer has.
363 bool startToken = token && !start.getNoToken();
364 auto mark = [&](DMAStartTaskOp s, int64_t index) {
365 state.starts[s] = {group, index, count};
366 };
367 rewriter.modifyOpInPlace(start, [&]() { start.removeRepeatCountAttr(); });
368 mark(start, 0);
369 state.blocks.insert(start->getBlock());
370 rewriter.setInsertionPointAfter(start);
371 for (int64_t k = 1; k < count; ++k) {
372 size_t i = static_cast<size_t>(k) % slices.size();
373 bool withholds =
374 token && i + 1 == slices.size() && (!startToken || k + 1 != count);
375 auto s = DMAStartTaskOp::create(
376 rewriter, start.getLoc(), tasks[i]->getResult(0),
377 /*repeat_count=*/nullptr, /*repeat_count_val=*/nullptr,
378 /*no_token=*/withholds ? rewriter.getUnitAttr() : nullptr,
379 /*repeat_parameter=*/nullptr, /*repeat_state_table_idx=*/nullptr);
380 mark(s, k);
381 }
382 }
383}
384
385using ChannelKey = DmaQueueModel::ChannelKey;
386
387// The channel a start pushes onto, or nullopt when its task is not a configure
388// on a placed tile this pass can read.
389static std::optional<ChannelKey> channelOf(DMAStartTaskOp start) {
390 Operation *def = start.getTask().getDefiningOp();
391 if (auto cfg = dyn_cast_or_null<DMAConfigureTaskOp>(def)) {
392 if (cfg.tryGetTileOp())
393 return DmaQueueModel::keyOf(cfg);
394 return std::nullopt;
395 }
396 auto cfgFor = dyn_cast_or_null<DMAConfigureTaskForOp>(def);
397 if (!cfgFor)
398 return std::nullopt;
399 auto dev = cfgFor->getParentOfType<AIE::DeviceOp>();
400 auto alloc = dev ? AIE::ShimDMAAllocationOp::getForSymbol(
401 dev, cfgFor.getAlloc().getRootReference())
402 : AIE::ShimDMAAllocationOp();
403 if (!alloc || !alloc.getTileOp())
404 return std::nullopt;
405 return DmaQueueModel::keyOf(alloc);
406}
407
408// Interleave the slices splitIntoTasks emitted with the starts around them.
409// Later slices wait for earlier ones to finish (see
410// aie-assign-runtime-sequence-bd-ids), so a counterpart transfer left behind
411// all of them (a core's input for an output, say) would deadlock.
412//
413// A round runs from one start up to the next start on a channel it already
414// used. Each start's first slice keeps its place; later ones follow the
415// round's last start, ordered by index / size, ties in program order. Slices
416// move past configures, RTP writes and lock sets, which never wait. Any other
417// effect ends a round: an await, sync, poll, untracked push, free of an
418// unplaced slice, or a write32/blockwrite (either may be a push).
419static void orderSlices(Block &block, Slices &state) {
420 struct Pending {
421 DMAStartTaskOp start;
422 int64_t index, count, head;
423 };
424 SmallVector<Pending> pending;
425 llvm::DenseSet<Value> pendingTasks;
426 std::set<ChannelKey> round;
427 llvm::DenseMap<int64_t, int64_t> headOfGroup;
428 Operation *anchor = nullptr;
429 int64_t heads = 0;
430
431 auto flush = [&]() {
432 llvm::stable_sort(pending, [](const Pending &a, const Pending &b) {
433 int64_t lhs = a.index * b.count;
434 int64_t rhs = b.index * a.count;
435 return lhs != rhs ? lhs < rhs : a.head < b.head;
436 });
437 Operation *after = anchor;
438 for (Pending &p : pending) {
439 p.start->moveAfter(after);
440 after = p.start;
441 // Configure a slice only once it is due, so that the descriptors it
442 // takes are not held while earlier slices wait: the BD-ID pass can take
443 // descriptors back only from started tasks.
444 Operation *task = p.start.getTask().getDefiningOp();
445 if (task && state.lateTasks.erase(task))
446 task->moveBefore(p.start);
447 }
448 pending.clear();
449 pendingTasks.clear();
450 round.clear();
451 headOfGroup.clear();
452 anchor = nullptr;
453 };
454
455 for (Operation &op : llvm::make_early_inc_range(block)) {
456 if (auto start = dyn_cast<DMAStartTaskOp>(op)) {
457 std::optional<ChannelKey> channel = channelOf(start);
458 auto it = state.starts.find(start);
459 std::optional<Slices::Start> slice;
460 if (it != state.starts.end())
461 slice = it->second;
462 if (channel && slice && slice->index > 0) {
463 auto head = headOfGroup.find(slice->group);
464 if (head != headOfGroup.end()) {
465 pending.push_back({start, slice->index, slice->count, head->second});
466 pendingTasks.insert(start.getTask());
467 continue;
468 }
469 }
470 if (!channel || round.count(*channel))
471 flush();
472 if (!channel)
473 continue;
474 round.insert(*channel);
475 if (slice)
476 headOfGroup[slice->group] = heads;
477 ++heads;
478 anchor = &op;
479 continue;
480 }
481 if (auto free = dyn_cast<DMAFreeTaskOp>(op)) {
482 if (pendingTasks.contains(free.getTask()))
483 flush();
484 continue;
485 }
486 if (isa<DMAConfigureTaskOp, DMAConfigureTaskForOp, NpuWriteRTPOp,
487 SetLockOp>(op) ||
488 isMemoryEffectFree(&op))
489 continue;
490 flush();
491 }
492 flush();
493}
494
495static void decomposeMemcpy(RewriterBase &rewriter, NpuDmaMemcpyNdOp op) {
496 std::optional<NdDmaPattern> constant = constantPattern(op);
497 if (!constant)
498 return;
499 // A runtime length stays whole; see DMABDOp::verify.
500 if (op.getLengthParameterAttr() || op.getLengthStateTableIdxAttr())
501 return;
502
503 const NdDmaPattern &pattern = *constant;
504 if (isContiguousTransfer(pattern.sizes, pattern.strides))
505 return;
506
507 AIE::DeviceOp dev = op->getParentOfType<AIE::DeviceOp>();
508 if (!dev)
509 return;
510
511 auto allocOp = AIE::ShimDMAAllocationOp::getForSymbol(
512 dev, op.getMetadata().getRootReference());
513 if (!allocOp)
514 return;
515
516 AIE::TileOp tile = allocOp.getTileOp();
517 if (!tile)
518 return;
519
520 int col = tile.getCol();
521 int row = tile.getRow();
522 const AIE::AIETargetModel &targetModel = AIE::getTargetModel(op);
523 auto bufferType = cast<BaseMemRefType>(op.getMemref().getType());
524
525 if (patternPassesVerification(op, bufferType, targetModel, col, row, pattern))
526 return;
527
528 auto decomposed =
529 decomposeNdDmaPattern(op, bufferType, pattern, targetModel, col, row);
530 // failed() already guards both dereferences below via short-circuit /
531 // prior control flow; the checker just doesn't associate FailureOr's
532 // failed()/succeeded() idiom with the std::optional base it derives from.
533 if (failed(decomposed) ||
534 decomposed->empty()) // NOLINT(bugprone-unchecked-optional-access)
535 return;
536 // Bind a plain reference now that decomposed is known non-failed and
537 // non-empty, so nothing past this point looks like an optional access.
538 SmallVector<NdDmaPattern> &bds =
539 *decomposed; // NOLINT(bugprone-unchecked-optional-access)
540 if (bds.size() > targetModel.getNumBDs(col, row))
541 return;
542
543 rewriter.setInsertionPoint(op);
544 if (bds.size() == 1) {
545 rewriter.replaceOpWithNewOp<NpuDmaMemcpyNdOp>(
546 op, op, toOuter(bds.front().offsets), toOuter(bds.front().sizes),
547 toOuter(bds.front().strides));
548 return;
549 }
550
551 llvm::DenseSet<int64_t> usedIds;
552 if (auto seq = op->getParentOfType<AIE::RuntimeSequenceOp>()) {
553 seq.walk([&](NpuDmaMemcpyNdOp other) {
554 if (other == op)
555 return;
556 if (other.getMetadata() == op.getMetadata())
557 usedIds.insert(other.getId());
558 });
559 }
560
561 int64_t nextId = op.getId();
562 for (auto [idx, subPattern] : llvm::enumerate(bds)) {
563 bool last = idx + 1 == bds.size();
564 int64_t id = allocateNextId(op, nextId, usedIds);
565 nextId = id + 1;
566 createDecomposedOp(rewriter, op, subPattern, id,
567 last && op.getIssueToken());
568 }
569 rewriter.eraseOp(op);
570}
571
572// Rewrites a task's descriptor into legal ones. Fails only after reporting why
573// it cannot.
574static LogicalResult decomposeTaskBd(RewriterBase &rewriter, AIE::DMABDOp op,
575 Slices &slices) {
576 auto taskAndTile = resolveTaskAndTile(op);
577 if (!taskAndTile)
578 return success();
579
580 AIE::TileOp tile = std::get<0>(*taskAndTile);
581 DMAConfigureTaskLike taskOp = std::get<1>(*taskAndTile);
582 int channel = std::get<2>(*taskAndTile);
583
584 // A descriptor with more dimensions than a BD has cannot be lowered as it
585 // is, so where this pass cannot reduce it, it says why.
586 bool tooManyDims = op.getMixedSizes().size() > kNdDmaDims;
587 auto cannotReduce = [&]() {
588 return op.emitOpError()
589 << "has " << op.getMixedSizes().size()
590 << " dimensions, and a buffer descriptor holds " << kNdDmaDims
591 << "; the extra ones can only be split off ";
592 };
593 int col = tile.getCol();
594 int row = tile.getRow();
595 const AIE::AIETargetModel &targetModel = AIE::getTargetModel(op);
596 FailureOr<TaskBdDecomposition> plan =
597 decomposeTaskBdPattern(op, targetModel, col, row);
598 if (failed(plan)) {
599 if (tooManyDims)
600 return cannotReduce()
601 << "a descriptor whose offset, length, sizes and strides are "
602 "all constant and that has no padding";
603 return success();
604 }
605 if (countTaskBds(taskOp) != 1) {
606 if (tooManyDims)
607 return cannotReduce() << "a task's only descriptor";
608 return success();
609 }
610 // A descriptor takes its locks once per execution, which decomposition
611 // regroups, and a slice of its own would take none.
612 bool takesLocks = false;
613 taskOp->walk([&](AIE::UseLockOp) { takesLocks = true; });
614 if (takesLocks) {
615 if (tooManyDims)
616 return cannotReduce() << "a descriptor that takes no locks";
617 return success();
618 }
619 // A runtime length stays whole; see DMABDOp::verify.
620 if (op.getLengthParameterAttr() || op.getLengthStateTableIdxAttr())
621 return success();
622
623 if (plan->legal) // NOLINT(bugprone-unchecked-optional-access)
624 return success();
625
626 // Iteration uses the outermost ND slot, so there is no free dimension to
627 // factor into, and chain-slicing has no defined iteration semantics.
628 if (op.getIteration())
629 return op.emitOpError()
630 << "buffer descriptor with the iteration attribute is too large "
631 "to lower and cannot be decomposed";
632
633 // One pass over the pattern, in executions of the descriptor.
634 int64_t iterations =
635 iterationCount(plan->pass); // NOLINT(bugprone-unchecked-optional-access)
636 // Bind a plain reference now that plan is known non-failed, so nothing past
637 // this point looks like an optional access.
638 SmallVector<NdDmaPattern> &bds =
639 plan->pieces; // NOLINT(bugprone-unchecked-optional-access)
640 if (bds.empty()) {
641 if (tooManyDims)
642 return op.emitOpError()
643 << "has " << op.getMixedSizes().size()
644 << " dimensions, and no split into at most " << kMaxNdDmaPieces
645 << " descriptors of " << kNdDmaDims << " fits this tile";
646 return success();
647 }
648
649 int64_t baseFlatOffset = op.getConstantOffset().value_or(0);
650
651 if (bds.size() == 1) {
652 const NdDmaPattern &sub = bds.front();
653 int64_t flatOffset = flatOffsetFromPattern(baseFlatOffset, sub);
654 auto outerSizes = toOuter(sub.sizes);
655 auto outerStrides = toOuter(sub.strides);
656 // len covers one BD invocation, so it tracks the innermost three
657 // dimensions only. Rewriting in place can move extent into the fourth
658 // (repeat) dimension -- e.g. an innermost run too long for the wrap
659 // field factors out an extra dimension and pushes the outermost one into
660 // the repeat slot -- so len has to be recomputed alongside the shape.
661 int32_t len = static_cast<int32_t>(lenFromInnermost3(sub.sizes));
662
663 IterationScale scale = iterationScale(iterations, sub);
664 int64_t runs = 0;
665 if (!scale.isOne()) {
666 if (taskOp.getRepeatCountVal())
667 return op.emitOpError()
668 << "cannot decompose a buffer descriptor whose repeat count "
669 "is a runtime value: decomposition needs to scale it by "
670 << scale.str();
671 // A scaled count past what one queue push carries is fine: the BD-ID
672 // pass issues it as several starts. Only the attribute width limits
673 // it. Widen before multiplying: the accessor returns int32_t, so the
674 // addition alone would overflow in int and wrap past this check.
675 // Merged executions need every start to run whole ones.
676 auto check = [&](Operation *at, int64_t repeat, StringRef runner,
677 StringRef count) -> LogicalResult {
678 int64_t before = repeat + 1;
679 if (!scale.divides(before))
680 return at->emitOpError()
681 << "cannot decompose: it merges every " << scale.den
682 << " executions of the buffer descriptor into " << scale.num
683 << ", and " << runner << " runs it " << before << " times";
684 int64_t after = scale.apply(before);
685 if (after - 1 > std::numeric_limits<int32_t>::max())
686 return at->emitOpError()
687 << "decomposition scales " << count << " by " << scale.str()
688 << " to " << (after - 1) << ", beyond a 32-bit repeat_count";
689 return success();
690 };
691 runs = taskOp.getRepeatCount() + int64_t{1};
692 if (failed(check(op, runs - 1, "the task", "the repeat count")))
693 return failure();
694 // A start that overrides the task's count repeats the same BD, so it
695 // scales by the same factor.
696 for (Operation *user : taskOp->getResult(0).getUsers()) {
697 auto start = dyn_cast<DMAStartTaskOp>(user);
698 if (!start)
699 continue;
700 if (start.getRepeatCountVal() || start.hasRepeatParameter())
701 return start.emitOpError()
702 << "cannot decompose a buffer descriptor this start repeats "
703 "a runtime number of times: decomposition needs to scale "
704 "the count by "
705 << scale.str();
706 if (IntegerAttr rc = start.getRepeatCountAttr())
707 if (failed(check(start, rc.getInt(), "this start",
708 "this start's repeat count")))
709 return failure();
710 }
711 runs = scale.apply(runs);
712 }
713
714 rewriter.modifyOpInPlace(op, [&]() {
715 updateTaskBdInPlace(op, static_cast<int32_t>(flatOffset), len, outerSizes,
716 outerStrides);
717 });
718 if (!scale.isOne()) {
719 rewriter.modifyOpInPlace(taskOp, [&]() {
720 taskOp.setRepeatCount(static_cast<uint32_t>(runs - 1));
721 });
722 for (Operation *user : taskOp->getResult(0).getUsers())
723 if (auto start = dyn_cast<DMAStartTaskOp>(user))
724 if (IntegerAttr rc = start.getRepeatCountAttr())
725 rewriter.modifyOpInPlace(start, [&]() {
726 start.setRepeatCountAttr(
727 rewriter.getI32IntegerAttr(scale.apply(rc.getInt() + 1) - 1));
728 });
729 }
730 return success();
731 }
732
733 // A split would make every slice reuse the one out_of_order_id slot.
734 // TODO: BD iteration plus padding may enable this feature.
735 if (op.getOutOfOrderId().has_value())
736 return op.emitOpError() << "splitting an out-of-order buffer descriptor "
737 "into multiple descriptors is not implemented";
738
739 // Every member of a chain runs once per task execution, so the one
740 // queue-push repeat count is shared by all of them. A slice that also wants
741 // its own iteration factor needs a private repeat, which only a task of
742 // its own has. A chain also holds all of its descriptors until it
743 // finishes, so past the queue depth the slices go out as separate tasks,
744 // whose descriptors aie-assign-runtime-sequence-bd-ids can recycle one by
745 // one.
746 bool sharedRepeat = llvm::all_of(bds, [&](const NdDmaPattern &sub) {
747 return iterationScale(iterations, sub).isOne();
748 });
749 bool chainFits =
750 sharedRepeat &&
751 bds.size() <= targetModel.getNumBDsForChannel(col, row, channel);
752 uint32_t depth = targetModel.getDmaTaskQueueDepth();
753 if (!chainFits || (depth > 0 && bds.size() > depth)) {
754 // The dynamic BD pool allocates a chain under runtime control flow, but
755 // separate tasks there are left whole.
756 if (isUnderRuntimeControlFlow(op)) {
757 if (tooManyDims)
758 return cannotReduce()
759 << "outside runtime control flow, since its " << bds.size()
760 << " pieces have to be separate tasks";
761 op.emitRemark() << "deferring decomposition into " << bds.size()
762 << " separate tasks under runtime control flow";
763 return success();
764 }
765 std::optional<std::string> why =
766 whyNotSeparateTasks(op, taskOp, iterations);
767 if (!why) {
768 splitIntoTasks(rewriter, op, taskOp, bds, baseFlatOffset, iterations,
769 slices);
770 return success();
771 }
772 if (!sharedRepeat)
773 return op.emitOpError()
774 << "cannot split this buffer descriptor: its slices need "
775 "per-descriptor repeat counts, which a chain shares, and "
776 "they cannot be separate tasks because "
777 << *why;
778 if (!chainFits)
779 return op.emitOpError()
780 << "cannot split this buffer descriptor: its " << bds.size()
781 << " slices outnumber the "
782 << targetModel.getNumBDsForChannel(col, row, channel)
783 << " buffer descriptors its channel reaches, and they cannot be "
784 "separate tasks "
785 "because "
786 << *why;
787 }
788
789 Region *body = &taskOp.getBody();
790 if (body->empty())
791 return success();
792
793 SmallVector<Block *> blocks;
794 blocks.push_back(op->getBlock());
795 for (unsigned i = 1; i < bds.size(); ++i)
796 blocks.push_back(rewriter.createBlock(body));
797
798 for (auto [idx, subPattern] : llvm::enumerate(bds)) {
799 Block *block = blocks[idx];
800 int64_t flatOffset = flatOffsetFromPattern(baseFlatOffset, subPattern);
801 auto outerSizes = toOuter(subPattern.sizes);
802 auto outerStrides = toOuter(subPattern.strides);
803 int32_t len = static_cast<int32_t>(lenFromInnermost3(subPattern.sizes));
804
805 if (idx == 0) {
806 rewriter.modifyOpInPlace(op, [&]() {
807 updateTaskBdInPlace(op, static_cast<int32_t>(flatOffset), len,
808 outerSizes, outerStrides);
809 });
810 Operation *oldTerm = block->getTerminator();
811 rewriter.setInsertionPoint(oldTerm);
812 if (idx + 1 < bds.size())
813 AIE::NextBDOp::create(rewriter, op.getLoc(), blocks[idx + 1]);
814 else
815 AIE::EndOp::create(rewriter, op.getLoc());
816 rewriter.eraseOp(oldTerm);
817 } else {
818 rewriter.setInsertionPointToStart(block);
819 createTaskBd(rewriter, op.getLoc(), op, static_cast<int32_t>(flatOffset),
820 len, outerSizes, outerStrides);
821 rewriter.setInsertionPointToEnd(block);
822 if (idx + 1 < bds.size())
823 AIE::NextBDOp::create(rewriter, op.getLoc(), blocks[idx + 1]);
824 else
825 AIE::EndOp::create(rewriter, op.getLoc());
826 }
827 }
828
829 return success();
830}
831
832struct AIEDecomposeLargeDmaBdPass
833 : xilinx::AIEX::impl::AIEDecomposeLargeDmaBdBase<
834 AIEDecomposeLargeDmaBdPass> {
835 void runOnOperation() override {
836 AIE::DeviceOp device = getOperation();
837 IRRewriter rewriter(&getContext());
838 SmallVector<NpuDmaMemcpyNdOp> memcpys;
839 SmallVector<AIE::DMABDOp> bds;
840 device.walk([&](Operation *op) {
841 if (auto memcpy = dyn_cast<NpuDmaMemcpyNdOp>(op))
842 memcpys.push_back(memcpy);
843 else if (auto bd = dyn_cast<AIE::DMABDOp>(op))
844 bds.push_back(bd);
845 });
846 for (NpuDmaMemcpyNdOp memcpy : memcpys)
847 decomposeMemcpy(rewriter, memcpy);
848 Slices slices;
849 bool failed = false;
850 for (AIE::DMABDOp bd : bds)
851 failed |= mlir::failed(decomposeTaskBd(rewriter, bd, slices));
852 if (failed)
853 return signalPassFailure();
854 for (Block *block : slices.blocks)
855 orderSlices(*block, slices);
856 }
857};
858
859} // namespace
860
861std::unique_ptr<OperationPass<AIE::DeviceOp>>
863 return std::make_unique<AIEDecomposeLargeDmaBdPass>();
864}
std::array< int, 4 > ChannelKey
{col, row, direction, channel}.
static ChannelKey keyOf(AIE::TileOp tile, AIE::DMAChannelDir direction, int64_t channel)
virtual uint32_t getDmaTaskQueueDepth() const =0
Return how many DMA tasks a single channel's task queue holds (unsupported = 0).
virtual uint32_t getNumBDs(AIETileType tileType) const =0
Return the number of buffer descriptors for a given tile type.
uint32_t getNumBDsForChannel(int col, int row, int channel) const
Return the number of buffer descriptors accessible on channel channel for the tile at (col,...
bool isContiguousTransfer(llvm::ArrayRef< int64_t > sizes, llvm::ArrayRef< int64_t > strides)
std::unique_ptr< mlir::OperationPass< AIE::DeviceOp > > createAIEDecomposeLargeDmaBdPass()
mlir::FailureOr< llvm::SmallVector< NdDmaPattern > > decomposeNdDmaPattern(mlir::Operation *forOp, mlir::BaseMemRefType referencedBufType, const NdDmaPattern &pattern, const xilinx::AIE::AIETargetModel &targetModel, int tileCol, int tileRow)
Decompose an illegal pattern into one or more legal sub-patterns that move the same data.
mlir::FailureOr< TaskBdDecomposition > decomposeTaskBdPattern(xilinx::AIE::DMABDOp bd, const xilinx::AIE::AIETargetModel &targetModel, int tileCol, int tileRow)
Plan the split of bd, a runtime task's descriptor, on tile (tileCol, tileRow).
bool patternPassesVerification(mlir::Operation *forOp, mlir::BaseMemRefType referencedBufType, const xilinx::AIE::AIETargetModel &tm, int tileCol, int tileRow, const NdDmaPattern &pattern, bool skipTransformationChecks=false)
Returns true when the pattern passes verifyStridesWraps for the given tile and memref element type.
const AIETargetModel & getTargetModel(mlir::Operation *op)
Innermost-first ND access pattern (d0..d3 / repeat), matching the convention used by verifyStridesWra...
llvm::SmallVector< int64_t, kNdDmaDims > sizes
int64_t baseOffset
Elements added to the address the offsets give.
llvm::SmallVector< int64_t, kNdDmaDims > offsets
llvm::SmallVector< int64_t, kNdDmaDims > strides