MLIR-AIE 1.4.6.dev
AIELowerDynamicBDPool.cpp
Go to the documentation of this file.
1//===- AIELowerDynamicBDPool.cpp --------------------------------*- C++ -*-===//
2//
3// Copyright (C) 2026 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7//
8// The dynamic counterpart to AIEAssignRuntimeSequenceBDIDs: where that pass
9// rejects a runtime-bound scf.for, this keeps the loop rolled and draws bd_ids
10// from a runtime free-list pool per tile and BD partition. Each configure gets
11// a dma_bd_pool_pop (its SSA id feeding bd_id_val); each free gets a
12// dma_bd_pool_push. An await is only a completion sync (npu_sync), never a
13// push.
14//
15// A popped id must be reachable at its push. Since a task value can cross a
16// loop back edge and exit as a result, the ids are carried in lockstep: every
17// task iter_arg gets parallel i32 id iter_args, and a push looks up the paired
18// ids.
19//
20// A multi-BD chain pops one id per BD, in body order, all from the channel's
21// partition. AIEDMATasksToNPU links each BD's next_bd to its successor's
22// runtime id. The ids share the task's lifetime, so a free pushes all of them.
23//
24//===----------------------------------------------------------------------===//
25
30
31#include "mlir/Dialect/Arith/IR/Arith.h"
32#include "mlir/Dialect/SCF/IR/SCF.h"
33#include "mlir/Pass/Pass.h"
34#include "llvm/ADT/DenseMap.h"
35#include "llvm/ADT/DenseSet.h"
36#include "llvm/ADT/SetVector.h"
37
38#include <set>
39
40namespace xilinx::AIEX {
41#define GEN_PASS_DEF_AIELOWERDYNAMICBDPOOL
42#include "aie/Dialect/AIEX/Transforms/AIEXPasses.h.inc"
43} // namespace xilinx::AIEX
44
45using namespace mlir;
46using namespace xilinx;
47using namespace xilinx::AIEX;
48
49namespace {
50
51struct AIELowerDynamicBDPoolPass
52 : xilinx::AIEX::impl::AIELowerDynamicBDPoolBase<AIELowerDynamicBDPoolPass> {
53 using Base =
54 xilinx::AIEX::impl::AIELowerDynamicBDPoolBase<AIELowerDynamicBDPoolPass>;
55 AIELowerDynamicBDPoolPass() = default;
56 AIELowerDynamicBDPoolPass(const AIELowerDynamicBDPoolOptions &options)
57 : Base(options) {}
58
59 // Maps a task value (the Index result of a configure, or any Index the carry
60 // propagates it into) to the i32 pool ids available at that same program
61 // point, one per BD of the task's chain in body order.
62 llvm::DenseMap<Value, SmallVector<Value, 2>> pairedId;
63 // The (col,row) of the tile each task's id belongs to, so a push names the
64 // right pool even when the id is a loop result rather than a direct pop.
65 llvm::DenseMap<Value, std::pair<int, int>> tileForTask;
66 // A representative (loop-invariant) configure per task, so a post-loop await
67 // on a loop result -- which has no defining configure -- can be redirected to
68 // one with the right (tile,dir,channel).
69 llvm::DenseMap<Value, DMAConfigureTaskOp> originConfigure;
70
71 // Every SSA value that carries a DMA task: configure results plus everything
72 // the task flows into (for iter_args/results, if results). Computed by
73 // forward closure from configures, so it is independent of the order in which
74 // control flow is later rewritten.
75 llvm::DenseSet<Value> taskValues;
76 // The number of ids each task value carries (its chain length). Known for
77 // every task value up front, unlike pairedId, whose entry for an scf
78 // result/iter_arg only exists once that op is grown.
79 llvm::DenseMap<Value, unsigned> taskIdCount;
80
81 // A grown scf.for/scf.if whose appended i32 yield operands are still
82 // placeholders. carried[i] is the original task position whose idCount[i]
83 // ids were appended, laid out contiguously from nOrig; the yields are wired
84 // once every position is paired.
85 struct Fixup {
86 Operation *op;
87 SmallVector<unsigned> carried;
88 SmallVector<unsigned> idCount;
89 unsigned nOrig;
90 SmallVector<Operation *> placeholders; // dead consts to erase (scf.if only)
91 };
92 SmallVector<Fixup> fixups;
93
94 // The BD ids a configure's channel can submit. Pools are keyed by this range,
95 // so channels that share a partition share a pool.
96 static std::pair<uint32_t, uint32_t> bdPartition(DMAConfigureTaskOp cfg) {
97 AIE::TileOp tile = cfg.getTileOp();
99 tile.getCol(), tile.getRow(), cfg.getChannel());
100 }
101
102 // The BDs of a configure's body, in block order: the order their ids are
103 // popped and carried.
104 static SmallVector<AIE::DMABDOp, 2> chainBds(DMAConfigureTaskOp cfg) {
105 SmallVector<AIE::DMABDOp, 2> bds;
106 cfg.walk([&](AIE::DMABDOp bd) { bds.push_back(bd); });
107 return bds;
108 }
109
110 void markTask(Value v, unsigned idCount, SmallVectorImpl<Value> &worklist) {
111 if (taskValues.insert(v).second) {
112 taskIdCount[v] = idCount;
113 worklist.push_back(v);
114 }
115 }
116
117 // Forward-propagate task-ness from every configure result through scf.for
118 // (init -> iter_arg, yield -> result) and scf.if (yield -> result). A value
119 // that is never derived from a configure (e.g. an unrelated index yielded on
120 // one branch) stays out of the set, which is what lets the divergent-yield
121 // check below reject it.
122 void computeTaskValues(AIE::RuntimeSequenceOp seq) {
123 taskValues.clear();
124 taskIdCount.clear();
125 SmallVector<Value> worklist;
126 seq.walk([&](DMAConfigureTaskOp c) {
127 markTask(c.getResult(), chainBds(c).size(), worklist);
128 });
129 while (!worklist.empty()) {
130 Value v = worklist.pop_back_val();
131 unsigned idCount = taskIdCount.lookup(v);
132 for (OpOperand &use : v.getUses()) {
133 Operation *user = use.getOwner();
134 if (auto f = dyn_cast<scf::ForOp>(user)) {
135 for (unsigned k = 0; k < f.getInitArgs().size(); ++k)
136 if (f.getInitArgs()[k] == v)
137 markTask(f.getRegionIterArg(k), idCount, worklist);
138 } else if (isa<scf::YieldOp>(user)) {
139 unsigned k = use.getOperandNumber();
140 Operation *parent = user->getParentOp();
141 if (auto pf = dyn_cast<scf::ForOp>(parent))
142 markTask(pf.getResult(k), idCount, worklist);
143 else if (auto pi = dyn_cast<scf::IfOp>(parent))
144 markTask(pi.getResult(k), idCount, worklist);
145 }
146 }
147 }
148 }
149
150 // Every position a task flows through carries one fixed number of ids, so
151 // both branches of an scf.if, and an scf.for's init and per-iteration yield,
152 // must agree on the chain length.
153 LogicalResult verifyChainLengths(AIE::RuntimeSequenceOp seq) {
154 auto agree = [&](Operation *op, Value a, Value b,
155 StringRef what) -> LogicalResult {
156 if (!taskValues.contains(a) || !taskValues.contains(b) ||
157 taskIdCount.lookup(a) == taskIdCount.lookup(b))
158 return success();
159 return op->emitOpError()
160 << what << " a " << taskIdCount.lookup(a) << "-BD chain and a "
161 << taskIdCount.lookup(b)
162 << "-BD chain at the same position; a pool-allocated task "
163 "carries one buffer descriptor id per BD, so every task "
164 "flowing through one position must have the same chain length";
165 };
166 WalkResult wr = seq.walk([&](Operation *op) -> WalkResult {
167 if (auto ifOp = dyn_cast<scf::IfOp>(op)) {
168 if (ifOp.getNumResults() == 0 || ifOp.elseBlock() == nullptr)
169 return WalkResult::advance();
170 auto thenYield = cast<scf::YieldOp>(ifOp.thenBlock()->getTerminator());
171 auto elseYield = cast<scf::YieldOp>(ifOp.elseBlock()->getTerminator());
172 for (unsigned k = 0; k < ifOp.getNumResults(); ++k)
173 if (failed(agree(ifOp, thenYield.getOperand(k),
174 elseYield.getOperand(k),
175 "yields from its branches")))
176 return WalkResult::interrupt();
177 } else if (auto forOp = dyn_cast<scf::ForOp>(op)) {
178 auto yield = cast<scf::YieldOp>(forOp.getBody()->getTerminator());
179 for (unsigned k = 0; k < forOp.getInitArgs().size(); ++k)
180 if (failed(agree(forOp, forOp.getInitArgs()[k], yield.getOperand(k),
181 "carries from its init and its yield")))
182 return WalkResult::interrupt();
183 }
184 return WalkResult::advance();
185 });
186 return failure(wr.wasInterrupted());
187 }
188
189 LogicalResult lowerConfigure(DMAConfigureTaskOp cfg) {
190 SmallVector<AIE::DMABDOp, 2> bds = chainBds(cfg);
191 if (llvm::any_of(bds, [](AIE::DMABDOp bd) { return bd.getBdIdVal(); }))
192 return success(); // already lowered
193
194 // The pool owns all id allocation here, so a hand-pinned bd_id would
195 // collide with a runtime pop. Allocation is all-pool or all-static (for a
196 // straight-line sequence), never mixed.
197 for (AIE::DMABDOp bd : bds)
198 if (bd.getBdId().has_value())
199 return bd.emitOpError(
200 "pins a buffer descriptor ID inside a runtime-bound "
201 "sequence that draws IDs from the dynamic pool; a pinned "
202 "ID would collide with a runtime-allocated one. Allocate "
203 "every BD in this sequence from the pool (drop the "
204 "bd_id), or make the sequence straight-line so the static "
205 "allocator assigns all IDs.");
206
207 AIE::TileOp tile = cfg.getTileOp();
208 auto [begin, end] = bdPartition(cfg);
209 if (bds.size() > end - begin)
210 return cfg.emitOpError()
211 << "chains " << bds.size() << " buffer descriptors, but channel "
212 << cfg.getChannel() << " of tile (" << tile.getCol() << ","
213 << tile.getRow() << ") draws from a pool of only " << end - begin
214 << " ids";
215 OpBuilder b(cfg);
216 SmallVector<Value, 2> ids;
217 for (AIE::DMABDOp bd : bds) {
218 Value bdId =
219 DMABdPoolPopOp::create(b, cfg.getLoc(), b.getI32Type(), tile.getCol(),
220 tile.getRow(), begin, end, /*id=*/nullptr)
221 .getBdId();
222 bd.getBdIdValMutable().assign(bdId);
223 ids.push_back(bdId);
224 }
225 pairedId[cfg.getResult()] = ids;
226 return success();
227 }
228
229 // Grow an scf.for with a parallel i32 iter_arg for every task-carrying
230 // iter_arg, so the popped id can flow in lockstep with the task Index. The
231 // yields are left as placeholders (the new id block arg itself) and wired to
232 // the real per-iteration id later, once every position is paired -- a body
233 // may yield an id defined by a not-yet-grown inner op. Pairs the new
234 // iter_args/results in `pairedId` and records a Fixup.
235 scf::ForOp carryIdsThroughLoop(scf::ForOp forOp) {
236 SmallVector<unsigned> carried; // iter_arg positions carrying a task
237 SmallVector<unsigned> idCount; // ids per carried position (chain length)
238 SmallVector<Value> newInits;
239 for (unsigned k = 0; k < forOp.getInitArgs().size(); ++k) {
240 Value init = forOp.getInitArgs()[k];
241 if (!taskValues.contains(init))
242 continue;
243 SmallVector<Value, 2> ids = pairedId.lookup(init);
244 assert(ids.size() == taskIdCount.lookup(init) &&
245 "for init carries a task but its ids are not all paired");
246 carried.push_back(k);
247 idCount.push_back(ids.size());
248 newInits.append(ids.begin(), ids.end());
249 }
250 if (carried.empty())
251 return forOp;
252
253 unsigned nOrig = forOp.getInitArgs().size();
254 IRRewriter rewriter(forOp);
255 auto newYields = [&](OpBuilder &b, Location loc,
256 ArrayRef<BlockArgument> newBBArgs) {
257 // Placeholder: yield each new id iter_arg straight back. Same type,
258 // dominates the yield, and is a no-op for an identity (task-reuse) carry.
259 // wireFixups overwrites it with the real per-iteration id.
260 return SmallVector<Value>(newBBArgs.begin(), newBBArgs.end());
261 };
262
263 FailureOr<LoopLikeOpInterface> newLoop = forOp.replaceWithAdditionalYields(
264 rewriter, newInits,
265 /*replaceInitOperandUsesInLoop=*/false, newYields);
266 assert(succeeded(newLoop) && "scf.for additional-yields rewrite failed");
267 // The assert above already guards this; the checker just doesn't
268 // associate succeeded() with FailureOr's std::optional base.
269 auto newFor = cast<scf::ForOp>(
270 newLoop->getOperation()); // NOLINT(bugprone-unchecked-optional-access)
271
272 unsigned off = nOrig;
273 for (auto [i, k] : llvm::enumerate(carried)) {
274 // The rewrite gave the loop new block args/results; migrate task-ness
275 // onto them so a later-grown consumer of this loop's result still sees a
276 // task.
277 taskValues.insert(newFor.getRegionIterArg(k));
278 taskValues.insert(newFor.getResult(k));
279 taskIdCount[newFor.getRegionIterArg(k)] = idCount[i];
280 taskIdCount[newFor.getResult(k)] = idCount[i];
281 SmallVector<Value, 2> argIds, resIds;
282 for (unsigned j = 0; j < idCount[i]; ++j) {
283 argIds.push_back(newFor.getRegionIterArg(off + j));
284 resIds.push_back(newFor.getResult(off + j));
285 }
286 pairedId[newFor.getRegionIterArg(k)] = argIds;
287 pairedId[newFor.getResult(k)] = resIds;
288 off += idCount[i];
289 }
290 fixups.push_back({newFor, carried, idCount, nOrig, {}});
291 return newFor;
292 }
293
294 // Grow an scf.if with a parallel i32 result per task it yields out (its
295 // branch-local id), so a task freed after the if finds its id. Yields start
296 // as placeholders, wired later. A result carries a task iff BOTH branches
297 // yield one; a divergent yield (only one branch) is rejected -- no id to push
298 // on the other path.
299 LogicalResult carryIdsThroughIf(scf::IfOp ifOp) {
300 if (ifOp.getNumResults() == 0 || ifOp.elseBlock() == nullptr)
301 return success();
302
303 auto thenYield = cast<scf::YieldOp>(ifOp.thenBlock()->getTerminator());
304 auto elseYield = cast<scf::YieldOp>(ifOp.elseBlock()->getTerminator());
305
306 SmallVector<unsigned> carried; // result positions carrying a task
307 SmallVector<unsigned> idCount; // ids per carried position (chain length)
308 for (unsigned k = 0; k < ifOp.getNumResults(); ++k) {
309 bool thenT = taskValues.contains(thenYield.getOperand(k));
310 bool elseT = taskValues.contains(elseYield.getOperand(k));
311 if (thenT && elseT) {
312 carried.push_back(k);
313 idCount.push_back(taskIdCount.lookup(thenYield.getOperand(k)));
314 } else if (thenT || elseT)
315 return ifOp.emitOpError(
316 "yields a pool-allocated task on only one branch of an scf.if; "
317 "both "
318 "branches must yield the task (and its buffer descriptor id) so it "
319 "can be freed after the if");
320 }
321 if (carried.empty())
322 return success();
323
324 unsigned nOrig = ifOp.getNumResults();
325 OpBuilder b(ifOp);
326 SmallVector<Type> resultTypes(ifOp.getResultTypes());
327 unsigned numIds = llvm::sum_of(idCount);
328 resultTypes.append(numIds, b.getI32Type());
329
330 auto newIf =
331 scf::IfOp::create(b, ifOp.getLoc(), resultTypes, ifOp.getCondition(),
332 /*withElseRegion=*/true);
333 newIf.getThenRegion().takeBody(ifOp.getThenRegion());
334 newIf.getElseRegion().takeBody(ifOp.getElseRegion());
335
336 // Append a placeholder i32 per carried id to each branch's yield;
337 // wireFixups replaces them with the branch's real ids (which may be defined
338 // by a not-yet-grown inner op). The placeholders are dead constants erased
339 // during wiring.
340 SmallVector<Operation *> placeholders;
341 auto appendPlaceholders = [&](scf::YieldOp y) {
342 OpBuilder yb(y);
343 SmallVector<Value> ids;
344 for (unsigned j = 0; j < numIds; ++j) {
345 auto c =
346 arith::ConstantOp::create(yb, y.getLoc(), yb.getI32IntegerAttr(0));
347 placeholders.push_back(c);
348 ids.push_back(c);
349 }
350 y->insertOperands(y->getNumOperands(), ids);
351 };
352 appendPlaceholders(cast<scf::YieldOp>(newIf.thenBlock()->getTerminator()));
353 appendPlaceholders(cast<scf::YieldOp>(newIf.elseBlock()->getTerminator()));
354
355 unsigned off = nOrig;
356 for (auto [i, k] : llvm::enumerate(carried)) {
357 // Migrate task-ness onto the rebuilt result so a later-grown consumer of
358 // this if's result still sees a task.
359 taskValues.insert(newIf.getResult(k));
360 taskIdCount[newIf.getResult(k)] = idCount[i];
361 SmallVector<Value, 2> resIds;
362 for (unsigned j = 0; j < idCount[i]; ++j)
363 resIds.push_back(newIf.getResult(off + j));
364 pairedId[newIf.getResult(k)] = resIds;
365 off += idCount[i];
366 }
367 for (unsigned k = 0; k < nOrig; ++k)
368 ifOp.getResult(k).replaceAllUsesWith(newIf.getResult(k));
369 ifOp.erase();
370 fixups.push_back({newIf, carried, idCount, nOrig, placeholders});
371 return success();
372 }
373
374 // Sweep 2: with every position paired, set each grown op's appended yield
375 // operand(s) to the real id yielded at that position. Runs after all growth,
376 // so an id defined by an inner op (grown later than its enclosing op) is
377 // resolvable, and an identity carry resolves to its own iter_arg's id.
378 void wireFixups() {
379 for (Fixup &fx : fixups) {
380 SmallVector<scf::YieldOp, 2> yields;
381 if (auto forOp = dyn_cast<scf::ForOp>(fx.op)) {
382 yields.push_back(cast<scf::YieldOp>(forOp.getBody()->getTerminator()));
383 } else {
384 auto ifOp = cast<scf::IfOp>(fx.op);
385 for (Block *blk : {ifOp.thenBlock(), ifOp.elseBlock()})
386 yields.push_back(cast<scf::YieldOp>(blk->getTerminator()));
387 }
388 for (scf::YieldOp yield : yields) {
389 unsigned off = fx.nOrig;
390 for (auto [i, k] : llvm::enumerate(fx.carried)) {
391 SmallVector<Value, 2> ids = pairedId.lookup(yield.getOperand(k));
392 assert(ids.size() == fx.idCount[i] &&
393 "carried task yields the wrong number of ids");
394 for (auto [j, id] : llvm::enumerate(ids))
395 yield.setOperand(off + j, id);
396 off += fx.idCount[i];
397 }
398 }
399 }
400 for (Fixup &fx : fixups)
401 for (Operation *p : fx.placeholders)
402 p->erase();
403 }
404
405 // Sweep 3: over the final IR, give every task value its tile (for the push)
406 // and a representative origin configure (for an await that names a loop
407 // result with no defining configure -- a loop's pre-loop init, which
408 // dominates a post-loop await). Order-independent forward closure:
409 // - scf.for: the init's tile/origin flow to the iter_arg AND the result, so
410 // a result takes the loop-invariant init configure (which dominates a
411 // post-loop await), not the per-iteration body configure.
412 // - scf.if: the then-branch yield's tile/origin flow to the result.
413 // A push only reads the tile, so both branches must agree; a genuinely
414 // cross-tile scf.if is rejected, and so is a for-loop whose per-iteration
415 // reconfiguration disagrees with its init.
416 LogicalResult computeMetadata(AIE::RuntimeSequenceOp seq) {
417 tileForTask.clear();
418 originConfigure.clear();
419 SmallVector<Value> worklist;
420 auto setMeta = [&](Value v, std::pair<int, int> tile,
421 DMAConfigureTaskOp origin) {
422 if (tileForTask.count(v))
423 return;
424 tileForTask[v] = tile;
425 originConfigure[v] = origin;
426 worklist.push_back(v);
427 };
428 seq.walk([&](DMAConfigureTaskOp c) {
429 AIE::TileOp t = c.getTileOp();
430 setMeta(c.getResult(), {t.getCol(), t.getRow()}, c);
431 });
432 while (!worklist.empty()) {
433 Value v = worklist.pop_back_val();
434 auto tile = tileForTask[v];
435 DMAConfigureTaskOp origin = originConfigure[v];
436 for (OpOperand &use : v.getUses()) {
437 Operation *user = use.getOwner();
438 if (auto f = dyn_cast<scf::ForOp>(user)) {
439 for (unsigned k = 0; k < f.getInitArgs().size(); ++k)
440 if (f.getInitArgs()[k] == v) {
441 setMeta(f.getRegionIterArg(k), tile, origin);
442 setMeta(f.getResult(k), tile, origin);
443 }
444 } else if (isa<scf::YieldOp>(user)) {
445 auto ifOp = dyn_cast<scf::IfOp>(user->getParentOp());
446 if (ifOp && user->getBlock() == ifOp.thenBlock())
447 setMeta(ifOp.getResult(use.getOperandNumber()), tile, origin);
448 }
449 }
450 }
451 // A task carried out of an scf.if and freed after it resolves to one BD id
452 // and one push, so both branches must agree on the full physical channel
453 // (tile,dir,channel) -- otherwise the single push would be ambiguous.
454 WalkResult wr = seq.walk([&](scf::IfOp ifOp) -> WalkResult {
455 if (ifOp.getNumResults() == 0 || ifOp.elseBlock() == nullptr)
456 return WalkResult::advance();
457 auto thenYield = cast<scf::YieldOp>(ifOp.thenBlock()->getTerminator());
458 auto elseYield = cast<scf::YieldOp>(ifOp.elseBlock()->getTerminator());
459 for (unsigned k = 0; k < ifOp.getNumResults(); ++k)
460 if (failed(checkChannelAgreement(
461 ifOp, thenYield.getOperand(k), elseYield.getOperand(k),
462 "yields tasks on different physical channels "
463 "(tile/direction/channel) from its two branches at "
464 "the same result; a pooled buffer descriptor id and "
465 "its completion sync belong to one channel")))
466 return WalkResult::interrupt();
467 return WalkResult::advance();
468 });
469 if (wr.wasInterrupted())
470 return failure();
471
472 // A task carried around an scf.for must likewise agree on the physical
473 // channel between the loop's init and its per-iteration reconfiguration
474 // -- the push after the loop uses the loop-invariant init's tile
475 // (computeMetadata never adopts the body's), and an await reachable
476 // through the loop (resolveConfigureThroughCF in AIEDMATasksToNPU) may
477 // resolve to either one.
478 WalkResult forWr = seq.walk([&](scf::ForOp forOp) -> WalkResult {
479 auto yield = cast<scf::YieldOp>(forOp.getBody()->getTerminator());
480 for (unsigned k = 0; k < forOp.getInitArgs().size(); ++k)
481 if (failed(checkChannelAgreement(
482 forOp, forOp.getInitArgs()[k], yield.getOperand(k),
483 "carries a task on different physical channels "
484 "(tile/direction/channel) between its init and its "
485 "per-iteration reconfiguration; a pooled buffer "
486 "descriptor id and its completion sync belong to one "
487 "channel")))
488 return WalkResult::interrupt();
489 return WalkResult::advance();
490 });
491 return forWr.wasInterrupted() ? failure() : success();
492 }
493
494 // If both `a` and `b` resolve (via originConfigure) to a configure, and
495 // those configures target different physical channels, emit `msg` on `op`
496 // and return failure. Shared by the scf.if and scf.for channel-agreement
497 // checks in computeMetadata -- only the pair of values being compared
498 // differs between them.
499 LogicalResult checkChannelAgreement(Operation *op, Value a, Value b,
500 StringRef msg) {
501 DMAConfigureTaskOp originA = originConfigure.lookup(a);
502 DMAConfigureTaskOp originB = originConfigure.lookup(b);
503 if (originA && originB &&
504 DmaQueueModel::keyOf(originA) != DmaQueueModel::keyOf(originB)) {
505 op->emitOpError(msg);
506 return failure();
507 }
508 return success();
509 }
510
511 // Every value that may carry `task`'s BD id after it: the region arguments
512 // and results it is forwarded to, including the next iteration of a loop
513 // that yields it. The forward counterpart of getReachableConfigures.
514 static llvm::SetVector<Value> aliasesAfter(Value task) {
515 llvm::SetVector<Value> aliases;
516 aliases.insert(task);
517 for (unsigned i = 0; i < aliases.size(); ++i) {
518 for (OpOperand &use : aliases[i].getUses()) {
519 Operation *user = use.getOwner();
520 auto branch = dyn_cast<RegionBranchOpInterface>(user);
521 if (!branch && user->hasTrait<OpTrait::IsTerminator>())
522 branch =
523 dyn_cast_or_null<RegionBranchOpInterface>(user->getParentOp());
524 if (!branch)
525 continue;
526 RegionBranchSuccessorMapping mapping;
527 branch.getSuccessorOperandInputMapping(mapping);
528 aliases.insert_range(mapping.lookup(&use));
529 }
530 }
531 return aliases;
532 }
533
534 // A start that can run after `freeOp` pushes an id the pool may already
535 // have handed to another task: one later in the block the two share, or one
536 // in a loop around both whose next iteration still names the freed id.
537 static LogicalResult verifyNoStartAfterFree(Value task, Operation *freeOp) {
538 llvm::SetVector<Value> aliases = aliasesAfter(task);
539 auto startsAfterFree = [&](DMAStartTaskOp start) {
540 for (Block *block = freeOp->getBlock(); block;
541 block = block->getParentOp()->getBlock()) {
542 Operation *startAt = block->findAncestorOpInBlock(*start);
543 if (!startAt)
544 continue;
545 Operation *freeAt = block->findAncestorOpInBlock(*freeOp);
546 if (freeAt != startAt && freeAt->isBeforeInBlock(startAt))
547 return true;
548 break;
549 }
550 Value started = start.getTask();
551 for (auto loop = freeOp->getParentOfType<scf::ForOp>(); loop;
552 loop = loop->getParentOfType<scf::ForOp>()) {
553 if (!loop->isAncestor(start))
554 continue;
555 if (!loop.getBody()->getParent()->isAncestor(started.getParentRegion()))
556 return true;
557 auto arg = dyn_cast<BlockArgument>(started);
558 if (arg && arg.getOwner() == loop.getBody() &&
559 arg.getArgNumber() >= loop.getNumInductionVars()) {
560 unsigned k = arg.getArgNumber() - loop.getNumInductionVars();
561 if (aliases.contains(loop.getYieldedValues()[k]))
562 return true;
563 }
564 }
565 return false;
566 };
567 for (Value alias : aliases)
568 for (Operation *user : alias.getUsers()) {
569 auto start = dyn_cast<DMAStartTaskOp>(user);
570 if (!start || !startsAfterFree(start))
571 continue;
572 auto diag = start.emitOpError(
573 "starts a task whose buffer descriptor ID was already returned "
574 "to the runtime pool; the pool may have handed it to another "
575 "task. Free the task after its last start instead.");
576 diag.attachNote(freeOp->getLoc()) << "returned here";
577 return diag;
578 }
579 return success();
580 }
581
582 LogicalResult lowerRelease(Value task, Operation *op) {
583 SmallVector<Value, 2> ids = pairedId.lookup(task);
584 if (ids.empty())
585 return op->emitOpError(
586 "does not resolve to a task allocated from the runtime pool; cannot "
587 "return its buffer descriptor ID");
588 if (failed(verifyNoStartAfterFree(task, op)))
589 return failure();
590 auto tile = tileForTask.lookup(task);
591 auto [begin, end] = bdPartition(originConfigure.lookup(task));
592 OpBuilder b(op);
593 for (Value id : ids)
594 DMABdPoolPushOp::create(b, op->getLoc(), tile.first, tile.second, begin,
595 end, id);
596 return success();
597 }
598
599 // Guard pushes that can land on a full task queue, and (opt-in) warn at a
600 // dma_free_task whose own push is not yet proven complete. Must run before
601 // frees are erased below -- it is the only point where this pass still has
602 // both the dma_free_task ops and the origin-configure metadata they resolve
603 // through.
604 //
605 // A sequence lowered here keeps its scf.for rolled, so the static allocator
606 // skips it. Guard while task metadata is available; aie-dma-to-npu later
607 // checks the combined pushes from tasks, memcpy and channel rearm operations.
608 void guardQueueDepth(AIE::RuntimeSequenceOp seq, SymbolTable &symbols) {
609 const AIE::AIETargetModel &tm =
610 seq->getParentOfType<AIE::DeviceOp>().getTargetModel();
611
612 auto effectOf = [&](Operation *op) -> QueueEffect {
613 if (isa<NpuPushQueueOp, NpuSyncOp, NpuDmaMemcpyNdOp, NpuDmaWaitOp>(op))
614 return rawQueueEffect(op, symbols);
615 DMAConfigureTaskOp cfg;
616 Value task;
617 bool isConfigure = false;
618 bool isPush = false;
619 bool issuesToken = false;
620 if (auto start = dyn_cast<DMAStartTaskOp>(op)) {
621 task = start.getTask();
622 cfg = originConfigure.lookup(task);
623 isPush = true;
624 issuesToken = cfg && start.getPushIssueToken(cfg);
625 } else if (auto configure = dyn_cast<DMAConfigureTaskOp>(op)) {
626 if (!warnUnsafeBdReuse)
627 return {};
628 cfg = configure;
629 task = configure.getResult();
630 isConfigure = true;
631 } else if (auto await = dyn_cast<DMAAwaitTaskOp>(op)) {
632 cfg = originConfigure.lookup(await.getTask());
633 // Only a token-issuing await retires anything.
634 if (cfg && !cfg.getIssueToken())
635 return {};
636 } else if (auto freeOp = dyn_cast<DMAFreeTaskOp>(op)) {
637 if (!warnUnsafeBdReuse)
638 return {};
639 cfg = originConfigure.lookup(freeOp.getTask());
640 if (!cfg)
641 return {};
642 if (!cfg.tryGetTileOp())
643 return {};
644 return QueueEffect::free(DmaQueueModel::keyOf(cfg), freeOp.getTask());
645 } else {
646 return {};
647 }
648 if (!cfg)
649 return {};
650 if (!cfg.tryGetTileOp())
651 return {};
653 if (isConfigure)
654 return QueueEffect::configure(key, task);
655 return isPush ? QueueEffect::push(key, issuesToken,
656 warnUnsafeBdReuse ? task : Value{})
657 : QueueEffect::await(key);
658 };
659
660 DmaQueueModel queue;
661 guardSequenceQueueDepth(seq.getBody(), queue, tm, enforceQueueDepth,
662 warnUnsafeBdReuse, effectOf);
663 }
664
665 // A static DMA program or an aiex.npu.dma_memcpy_nd programs its BD id on
666 // its tile outside the pool, so every pool of that tile holding the id gives
667 // it up for the sequence.
668 void reserveTakenIds(AIE::RuntimeSequenceOp seq, SymbolTable &symbols) {
669 std::set<std::tuple<int, int, uint32_t, uint32_t>> pools;
670 std::set<std::tuple<int, int, uint32_t>> reserved;
671 seq.walk([&](DMABdPoolPopOp pop) {
672 pools.insert(
673 {pop.getColumn(), pop.getRow(), pop.getBdBegin(), pop.getBdEnd()});
674 if (std::optional<uint32_t> id = pop.getId())
675 reserved.insert({pop.getColumn(), pop.getRow(), *id});
676 });
677 if (pools.empty())
678 return;
679 std::set<std::tuple<int, int, uint32_t>> taken;
680 for (AIE::DmaBody program : getOperation().getOps<AIE::DmaBody>())
681 for (uint32_t id : AIE::getAssignedBdIds(program))
682 taken.insert({program.getTileID().col, program.getTileID().row, id});
683 seq.walk([&](NpuDmaMemcpyNdOp copy) {
684 auto allocation = symbols.lookup<AIE::ShimDMAAllocationOp>(
685 copy.getMetadata().getRootReference());
686 if (AIE::TileOp tile = allocation ? allocation.getTileOp() : nullptr)
687 taken.insert({tile.getCol(), tile.getRow(),
688 static_cast<uint32_t>(copy.getId())});
689 });
690 OpBuilder b = OpBuilder::atBlockBegin(&seq.getBody().front());
691 for (auto [col, row, begin, end] : pools)
692 for (auto [takenCol, takenRow, id] : taken)
693 if (takenCol == col && takenRow == row && id >= begin && id < end &&
694 !reserved.count({col, row, id}))
695 DMABdPoolPopOp::create(b, seq.getLoc(), b.getI32Type(), col, row,
696 begin, end, b.getI32IntegerAttr(id));
697 }
698
699 void runOnOperation() override {
700 AIE::DeviceOp device = getOperation();
701 SymbolTable symbols(device);
702
703 WalkResult wr = device.walk([&](AIE::RuntimeSequenceOp seq) -> WalkResult {
704 // A pop with an id withholds a taken BD id from the pool. Every exit
705 // from this sequence's body runs it, whether or not the lowering below
706 // grew new pools (carryIdsThroughLoop, computeMetadata) for it to see.
707 auto finish = [&](WalkResult result) {
708 reserveTakenIds(seq, symbols);
709 return result;
710 };
711
712 bool hasRuntimeCF = false;
713 seq.walk([&](Operation *op) {
714 if (isa<scf::ForOp, scf::IfOp, scf::WhileOp>(op))
715 hasRuntimeCF = true;
716 });
717 if (!hasRuntimeCF)
718 return finish(WalkResult::advance());
719
720 // Idempotency: a sequence already carrying pool pops has been lowered by
721 // a prior run of this pass. Its awaits may have dropped their task
722 // operand (sync_* attrs) and its scf ops already carry i32 ids, so
723 // re-running the carry logic would misfire. Skip it.
724 bool alreadyLowered = false;
725 seq.walk([&](DMABdPoolPopOp) { alreadyLowered = true; });
726 if (alreadyLowered)
727 return finish(WalkResult::advance());
728
729 // scf.while has no bounded trip count, so a BD id popped in its body has
730 // no place to be pushed back deterministically; only scf.for and scf.if
731 // are supported.
732 WalkResult whileCheck = seq.walk([&](scf::WhileOp w) {
733 w.emitOpError("scf.while in a runtime sequence is not supported by the "
734 "dynamic BD pool path; only scf.for and scf.if are "
735 "supported");
736 return WalkResult::interrupt();
737 });
738 if (whileCheck.wasInterrupted())
739 return finish(WalkResult::interrupt());
740
741 pairedId.clear();
742 tileForTask.clear();
743 originConfigure.clear();
744 fixups.clear();
745
746 // 1. Every configure pops an id (program order: a pop dominates every use
747 // of its id).
748 WalkResult r = seq.walk([&](DMAConfigureTaskOp cfg) -> WalkResult {
749 return failed(lowerConfigure(cfg)) ? WalkResult::interrupt()
750 : WalkResult::advance();
751 });
752 if (r.wasInterrupted())
753 return finish(WalkResult::interrupt());
754
755 // 2. Thread the id alongside the task everywhere the task flows. The task
756 // Index already threads (it is valid IR); shadowing every position it
757 // occupies with a parallel i32 makes the id in-scope at every push.
758 // a. Task closure: which SSA values carry a task (order-independent).
759 // b. Grow structure outermost-first (pre-order): an enclosing op's new
760 // iter_arg/result exists before an inner op yields through it. A
761 // divergent scf.if is rejected here.
762 // c. Wire the placeholder yields now that every position is paired --
763 // handles both siblings (producer grown before consumer) and
764 // parent-defines/child-consumes (inner id resolvable after growth).
765 computeTaskValues(seq);
766 if (failed(verifyChainLengths(seq)))
767 return finish(WalkResult::interrupt());
768 SmallVector<Operation *> cf;
769 seq.walk<WalkOrder::PreOrder>([&](Operation *o) {
770 if (isa<scf::ForOp, scf::IfOp>(o))
771 cf.push_back(o);
772 });
773 for (Operation *o : cf) {
774 if (auto f = dyn_cast<scf::ForOp>(o)) {
775 carryIdsThroughLoop(f);
776 } else if (failed(carryIdsThroughIf(cast<scf::IfOp>(o)))) {
777 return finish(WalkResult::interrupt());
778 }
779 }
780 wireFixups();
781
782 // 3. Metadata over the final IR: each task's tile (for the push) and a
783 // dominating origin configure (for an await on a loop/if result).
784 if (failed(computeMetadata(seq)))
785 return finish(WalkResult::interrupt());
786
787 // 4. Return ids to the pool. A free is the ONLY push: it returns the id
788 // to the free-list. An await is a TCT sync (npu_sync), never a release
789 // -- it emits no push. In the rolled pool one physical id is live
790 // across iterations, so the static path's "await = release" (safe only
791 // because unrolling gives each await a distinct id) would push the
792 // same live id once per iteration and corrupt the free-list. Leaving a
793 // task unfreed is valid: the pool is stack-local and re-initialized
794 // every invocation, so an unfreed id merely holds a slot for the rest
795 // of this one run (the await-only idiom, live BDs <= pool size).
796 //
797 // The await keeps its natural SSA operand -- the task value it names,
798 // whether that is a direct configure, a loop result, or an scf.if
799 // result (the phi of the task in flight, whichever branch ran). The
800 // npu_sync lowering (AIEDMATasksToNPU) walks that value to the
801 // physical channel; both branches of an scf.if were verified to agree
802 // on it (computeMetadata). This preserves the sync's data dependence
803 // on the task rather than breaking it.
804 SmallVector<Operation *> toErase;
805 r = seq.walk([&](Operation *op) -> WalkResult {
806 if (auto freeOp = dyn_cast<DMAFreeTaskOp>(op)) {
807 if (failed(lowerRelease(freeOp.getTask(), freeOp)))
808 return WalkResult::interrupt();
809 toErase.push_back(freeOp);
810 }
811 return WalkResult::advance();
812 });
813 if (r.wasInterrupted())
814 return finish(WalkResult::interrupt());
815 // Only once this sequence has lowered cleanly: guarding or reporting a
816 // queue on a design that is already being rejected just buries the
817 // error. Guard before erasing the frees above -- it still needs to see
818 // them, resolved through originConfigure, to check reuse safety.
819 guardQueueDepth(seq, symbols);
820 for (Operation *op : toErase)
821 op->erase();
822 return finish(WalkResult::advance());
823 });
824 if (wr.wasInterrupted())
825 return signalPassFailure();
826 }
827};
828
829} // namespace
830
831std::unique_ptr<OperationPass<AIE::DeviceOp>>
833 return std::make_unique<AIELowerDynamicBDPoolPass>();
834}
835
836std::unique_ptr<OperationPass<AIE::DeviceOp>>
838 const AIELowerDynamicBDPoolOptions &options) {
839 return std::make_unique<AIELowerDynamicBDPoolPass>(options);
840}
std::array< int, 4 > ChannelKey
{col, row, direction, channel}.
static ChannelKey keyOf(AIE::TileOp tile, AIE::DMAChannelDir direction, int64_t channel)
std::pair< uint32_t, uint32_t > getBdIdRangeForChannel(int col, int row, int channel) const
Return the half-open range [first, last) of buffer descriptor ids that channel channel of the tile at...
std::unique_ptr< mlir::OperationPass< AIE::DeviceOp > > createAIELowerDynamicBDPoolPass()
QueueEffect rawQueueEffect(mlir::Operation *op, mlir::SymbolTable &symbols)
What a push or an await that names no task does to its channel: an aiex.npu.push_queue; an aiex....
void guardSequenceQueueDepth(mlir::Region &body, DmaQueueModel &queue, const AIE::AIETargetModel &tm, bool enforce, bool warnUnsafeBdReuse, EffectFn effectOf)
Guard every push in a runtime sequence that can land on a full queue, and (when warnUnsafeBdReuse) wa...
llvm::SmallVector< uint32_t > getAssignedBdIds(DmaBody program)
const AIETargetModel & getTargetModel(mlir::Operation *op)
static QueueEffect await(DmaQueueModel::ChannelKey key)
static QueueEffect configure(DmaQueueModel::ChannelKey key, mlir::Value task)
static QueueEffect free(DmaQueueModel::ChannelKey key, mlir::Value task)
static QueueEffect push(DmaQueueModel::ChannelKey key, bool issuesToken, mlir::Value task={})