MLIR-AIE 1.4.6.dev
DmaQueueModel.h
Go to the documentation of this file.
1//===- DmaQueueModel.h ------------------------------------------*- C++ -*-===//
2//
3// Copyright (C) 2026 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7//
8// Per-channel DMA task queue, shared by the dma_start_task and
9// npu.dma_memcpy_nd paths so the retirement rule cannot drift between them.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef AIE_DIALECT_AIEX_UTILS_DMAQUEUEMODEL_H
14#define AIE_DIALECT_AIEX_UTILS_DMAQUEUEMODEL_H
15
18
19#include "mlir/Dialect/Arith/IR/Arith.h"
20#include "mlir/Dialect/SCF/IR/SCF.h"
21#include "mlir/Dialect/Utils/StaticValueUtils.h"
22#include "mlir/IR/Builders.h"
23#include "llvm/ADT/DenseMap.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/SetVector.h"
26#include "llvm/ADT/SmallVector.h"
27
28#include "llvm/Support/MathExtras.h"
29#include <climits>
30
31#include <algorithm>
32#include <array>
33#include <cstdint>
34#include <functional>
35#include <map>
36#include <set>
37
38namespace xilinx::AIEX {
39
40/// Tracks, per DMA channel, the pushes not yet known to have completed along
41/// one execution path. The sequence analysis below joins control-flow paths.
43public:
44 /// {col, row, direction, channel}. Queues are per channel and per direction,
45 /// and shim tiles are one per column, so the same channel number in two
46 /// columns is two independent queues.
47 using ChannelKey = std::array<int, 4>;
48
49 static ChannelKey keyOf(AIE::TileOp tile, AIE::DMAChannelDir direction,
50 int64_t channel) {
51 return {tile.getCol(), tile.getRow(), static_cast<int>(direction),
52 static_cast<int>(channel)};
53 }
54
55 /// The channel of a shim allocation on a placed tile.
56 static ChannelKey keyOf(AIE::ShimDMAAllocationOp allocation) {
57 return keyOf(allocation.getTileOp(), allocation.getChannelDir(),
58 allocation.getChannelIndex());
59 }
60
61 /// The channel of a configure on a placed tile.
62 static ChannelKey keyOf(DMAConfigureTaskOp cfg) {
63 return keyOf(cfg.getTileOp(), cfg.getDirection(), cfg.getChannel());
64 }
65
66 /// One channel's queue: its pushes not yet known to have completed, oldest
67 /// first, each true when it issues a completion token.
68 struct ChannelState {
69 llvm::SmallVector<bool, 8> pushes;
70 bool unknownTokens = false;
71 bool operator==(const ChannelState &other) const {
72 return pushes == other.pushes && unknownTokens == other.unknownTokens;
73 }
74 };
75
76 /// True when a push would land on a full queue. `depth` of 0 means the
77 /// target has no queued-task model and nothing can overflow.
78 static bool wouldOverflow(const ChannelState &q, uint32_t depth) {
79 return depth != 0 && q.pushes.size() >= depth;
80 }
81
82 /// Record a push. Every push occupies a slot, not just issue_token ones, and
83 /// one push occupies exactly one whatever its repeat_count.
84 static void push(ChannelState &q, bool issuesToken) {
85 q.pushes.push_back(issuesToken && !q.unknownTokens);
86 }
87
88 /// Retire what a task-completion-token await proves has drained. The sync
89 /// pops whichever token the channel produces first -- the oldest outstanding
90 /// token-issuing push, not necessarily the task the await names -- and
91 /// in-order execution means everything queued ahead of it has drained too.
92 /// With no token-issuing push queued, nothing can be retired.
93 static void awaitToken(ChannelState &q) {
94 if (q.unknownTokens)
95 return;
96 auto *tok = llvm::find(q.pushes, true);
97 if (tok != q.pushes.end())
98 q.pushes.erase(q.pushes.begin(), std::next(tok));
99 }
100
101 /// Record that a queue-space poll has guaranteed room. Keeps the newest
102 /// depth-1 entries rather than clearing. Polling does not consume completion
103 /// tokens: if it drops a token-bearing entry, later syncs cannot safely be
104 /// credited to newer pushes.
105 static void noteSpaceGuaranteed(ChannelState &q, uint32_t depth) {
106 if (depth > 0 && q.pushes.size() > depth - 1) {
107 auto end = q.pushes.end() - (depth - 1);
108 q.unknownTokens |=
109 llvm::is_contained(llvm::make_range(q.pushes.begin(), end), true);
110 q.pushes.erase(q.pushes.begin(), end);
111 if (q.unknownTokens)
112 llvm::fill(q.pushes, false);
113 }
114 }
115
116 bool wouldOverflow(const ChannelKey &key, uint32_t depth) const {
117 auto it = queued.find(key);
118 return it != queued.end() && wouldOverflow(it->second, depth);
119 }
120
121 void push(const ChannelKey &key, bool issuesToken) {
122 push(queued[key], issuesToken);
123 }
124
125 void awaitToken(const ChannelKey &key) {
126 if (auto it = queued.find(key); it != queued.end())
127 awaitToken(it->second);
128 }
129
130 void noteSpaceGuaranteed(const ChannelKey &key, uint32_t depth) {
131 noteSpaceGuaranteed(queued[key], depth);
132 }
133
134 /// True the first time a channel is reported. An over-subscribed channel
135 /// usually stays that way for every later push, and repeating the same
136 /// diagnostic per push buries it.
137 bool shouldReport(const ChannelKey &key) {
138 return reported.insert(key).second;
139 }
140
141 /// Outstanding pushes on `key`, for diagnostics.
142 size_t outstanding(const ChannelKey &key) const {
143 auto it = queued.find(key);
144 return it == queued.end() ? 0 : it->second.pushes.size();
145 }
146
147 /// Every channel's queue. `reported` is deliberately not part of it: it is
148 /// diagnostic bookkeeping, not queue state, and folding it in would stop
149 /// two otherwise identical states from comparing equal.
150 using State = std::map<ChannelKey, ChannelState>;
151 const State &state() const { return queued; }
152 void setState(State s) { queued = std::move(s); }
153
154private:
155 State queued;
156 std::set<ChannelKey> reported;
157};
158
159/// The channel an aiex.npu.sync retires on, or nullopt when it cannot be
160/// pinned down. Its six operands are exactly a ChannelKey, with direction
161/// numbered as AIE::DMAChannelDir because DmaWaitToSyncPattern builds that
162/// operand by casting the enum. Unresolvable retires nothing, which
163/// over-reports; guessing the other way would hide a real hang.
164inline std::optional<DmaQueueModel::ChannelKey> syncChannelKey(NpuSyncOp sync) {
165 std::optional<uint32_t> col = getConstantIntOperand(sync.getColumn());
166 std::optional<uint32_t> row = getConstantIntOperand(sync.getRow());
167 std::optional<uint32_t> dir = getConstantIntOperand(sync.getDirection());
168 std::optional<uint32_t> chan = getConstantIntOperand(sync.getChannel());
169 std::optional<uint32_t> colNum = getConstantIntOperand(sync.getColumnNum());
170 std::optional<uint32_t> rowNum = getConstantIntOperand(sync.getRowNum());
171 if (!col || !row || !dir || !chan || !colNum || !rowNum)
172 return std::nullopt;
173 // Nothing in tree settles whether a sync spanning a range of tiles waits for
174 // one token per tile or one for the range -- the cert lowering ignores both
175 // fields -- and crediting nothing is the only reading sound either way.
176 if (*colNum != 1 || *rowNum != 1)
177 return std::nullopt;
179 static_cast<int>(*col), static_cast<int>(*row), static_cast<int>(*dir),
180 static_cast<int>(*chan)};
181}
182
183/// The channels an aiex.npu.sync syncChannelKey cannot pin down may wait on:
184/// its tile range expanded, any value for an operand known only at run time.
185struct SyncCover {
186 int colLo = 0, colHi = INT_MAX, rowLo = 0, rowHi = INT_MAX;
187 std::optional<int> dir, chan;
188
189 bool covers(const DmaQueueModel::ChannelKey &key) const {
190 return key[0] >= colLo && key[0] < colHi && key[1] >= rowLo &&
191 key[1] < rowHi && (!dir || *dir == key[2]) &&
192 (!chan || *chan == key[3]);
193 }
194};
195
196inline SyncCover syncCover(NpuSyncOp sync) {
197 SyncCover cover;
198 auto range = [](std::optional<uint32_t> first, std::optional<uint32_t> num,
199 int &lo, int &hi) {
200 if (!first)
201 return;
202 lo = static_cast<int>(*first);
203 if (num)
204 hi = lo + static_cast<int>(*num);
205 };
206 range(getConstantIntOperand(sync.getColumn()),
207 getConstantIntOperand(sync.getColumnNum()), cover.colLo, cover.colHi);
208 range(getConstantIntOperand(sync.getRow()),
209 getConstantIntOperand(sync.getRowNum()), cover.rowLo, cover.rowHi);
210 if (std::optional<uint32_t> dir = getConstantIntOperand(sync.getDirection()))
211 cover.dir = static_cast<int>(*dir);
212 if (std::optional<uint32_t> chan = getConstantIntOperand(sync.getChannel()))
213 cover.chan = static_cast<int>(*chan);
214 return cover;
215}
216
217/// Retire what an aiex.npu.sync proves has drained. This is the lowered form of
218/// npu.dma_wait and the same hardware event, so it retires by the same rule;
219/// sequences written or lowered down to raw syncs would otherwise look like
220/// they never drain a channel at all.
221inline void awaitSync(DmaQueueModel &queue, NpuSyncOp sync) {
222 if (std::optional<DmaQueueModel::ChannelKey> key = syncChannelKey(sync))
223 queue.awaitToken(*key);
224}
225
226/// Shared wording for the two lowering paths, so the explanation does not
227/// depend on whether the transfer came from dma_start_task or
228/// npu.dma_memcpy_nd.
229inline mlir::InFlightDiagnostic
230emitQueueOverflowWarning(mlir::Operation *op, int col, int row,
231 AIE::DMAChannelDir dir, uint32_t channel,
232 uint32_t depth, size_t outstanding) {
233 // Not op->emitWarning(): its "see current operation" note verifies and
234 // numbers the whole device for each warning, quadratic in the sequence.
235 return mlir::emitWarning(op->getLoc())
236 << "pushes a DMA task onto tile (" << col << "," << row << ") "
237 << AIE::stringifyDMAChannelDir(dir) << " channel " << channel
238 << ", whose task queue is only " << depth << " deep, with "
239 << outstanding
240 << " push(es) not yet known to have completed. A push onto a full "
241 "queue is dropped by hardware and its transfer never runs, so "
242 "whatever waits on it blocks forever. Whether it happens depends "
243 "on how fast the consumer drains, which is why this is a warning "
244 "and not an error. Drain the channel with an await on this tile, "
245 "direction and channel before pushing again";
246}
247
248/// Wording for a dma_free_task that releases a buffer descriptor id the
249/// runtime pool could hand straight back out before hardware has actually
250/// finished the freed transfer -- the rolled-loop counterpart to
251/// aie-assign-runtime-sequence-bd-ids's unsafe-BD-reuse warning.
252inline mlir::InFlightDiagnostic
253emitUnsafeBdReuseWarning(mlir::Operation *freeOp, int col, int row,
254 AIE::DMAChannelDir dir, uint32_t channel) {
255 // Not freeOp->emitWarning(), as in emitQueueOverflowWarning.
256 return mlir::emitWarning(freeOp->getLoc())
257 << "frees a task on tile (" << col << "," << row << ") "
258 << AIE::stringifyDMAChannelDir(dir) << " channel " << channel
259 << " with no completion guarantee, so its buffer descriptor id "
260 "goes back to the runtime pool while the DMA may still be "
261 "running; the pool may hand that id straight to the next task "
262 "drawn on this tile, reprogramming a BD still in flight. Await "
263 "this task, or a later one queued on the same tile, direction "
264 "and channel, before freeing it -- an await consumes the oldest "
265 "outstanding token, regardless of the task it names";
266}
267
268/// Address and mask for a poll that waits for a free task-queue slot.
269/// Unavailable when the target reports no pollable
270/// occupancy register, or when `depth` is not a power of two -- a maskpoll
271/// tests masked equality, not "<", so "occupancy < depth" has to be the single
272/// "depth bit is clear" test (for depth 4, bit 22 of Task_Queue_Size).
273inline std::optional<std::pair<uint32_t, uint32_t>>
274queueSpacePoll(const AIE::AIETargetModel &tm, int col, int row,
275 uint32_t channel, AIE::DMAChannelDir dir, uint32_t depth) {
276 uint32_t fieldMask = tm.getDmaTaskQueueSizeMask();
277 if (!fieldMask || depth == 0 || (depth & (depth - 1)) != 0)
278 return std::nullopt;
279 std::optional<uint32_t> statusAddr =
280 tm.getDmaStatusAddress(col, row, channel, dir);
281 if (!statusAddr)
282 return std::nullopt;
283 uint32_t depthBit = depth << llvm::countr_zero(fieldMask);
284 if ((depthBit & fieldMask) != depthBit)
285 return std::nullopt;
286 return std::make_pair(*statusAddr, depthBit);
287}
288
289inline mlir::LogicalResult
290insertQueueSpaceWait(mlir::Operation *before, const AIE::AIETargetModel &tm,
291 int col, int row, uint32_t channel, AIE::DMAChannelDir dir,
292 uint32_t depth) {
293 auto poll = queueSpacePoll(tm, col, row, channel, dir, depth);
294 if (!poll)
295 return mlir::failure();
296 mlir::OpBuilder b(before);
297 mlir::Location loc = before->getLoc();
298 auto i32 = b.getI32Type();
299 auto cst = [&](uint32_t v) {
300 return mlir::arith::ConstantOp::create(b, loc, i32, b.getI32IntegerAttr(v))
301 .getResult();
302 };
303 NpuMaskPollOp::create(b, loc, cst(poll->first), cst(0), cst(poll->second),
304 /*buffer=*/nullptr, /*column=*/nullptr,
305 /*row=*/nullptr);
306 return mlir::success();
307}
308
309inline constexpr llvm::StringLiteral queueDiagnosedAttr =
310 "aiex.dma_queue_overflow_diagnosed";
311
312/// Handle a push on `key` that would land on a full queue: poll where one can
313/// be emitted, warn where it cannot (see insertQueueSpaceWait above for when
314/// that is). Warning rather than failing keeps a default from rejecting
315/// designs that build today on a target the user can do nothing about; the
316/// note is what keeps it from declining quietly.
317inline void guardQueueOverflow(DmaQueueModel &queue, mlir::Operation *push,
318 const AIE::AIETargetModel &tm,
319 const DmaQueueModel::ChannelKey &key,
320 uint32_t depth, bool enforce) {
321 int col = key[0], row = key[1];
322 auto dir = static_cast<AIE::DMAChannelDir>(key[2]);
323 auto chan = static_cast<uint32_t>(key[3]);
324
325 if (enforce && mlir::succeeded(insertQueueSpaceWait(push, tm, col, row, chan,
326 dir, depth))) {
327 queue.noteSpaceGuaranteed(key, depth);
328 return;
329 }
330 if (!queue.shouldReport(key))
331 return;
332 // Keep diagnostics per sequence/channel across task and combined lowering.
333 // This marker suppresses only warnings, never queue accounting or guards.
334 if (auto seq = push->getParentOfType<AIE::RuntimeSequenceOp>()) {
335 auto channel = mlir::DenseI32ArrayAttr::get(push->getContext(), key);
336 llvm::SmallVector<mlir::Attribute> diagnosed;
337 if (auto previous =
338 seq->getAttrOfType<mlir::ArrayAttr>(queueDiagnosedAttr)) {
339 if (llvm::is_contained(previous, channel))
340 return;
341 llvm::append_range(diagnosed, previous);
342 }
343 diagnosed.push_back(channel);
344 seq->setAttr(queueDiagnosedAttr,
345 mlir::ArrayAttr::get(push->getContext(), diagnosed));
346 }
347 mlir::InFlightDiagnostic diag = emitQueueOverflowWarning(
348 push, col, row, dir, chan, depth, queue.outstanding(key));
349 if (enforce)
350 diag.attachNote() << "the compiler would have waited for a free slot here, "
351 "but this target reports no pollable task-queue "
352 "occupancy register for this channel";
353}
354
355/// What one op in a loop body does to a channel queue. The two lowering paths
356/// spell pushes and awaits with different ops, so the fixed point below takes
357/// the classification from its caller and keeps the FIFO rule to itself.
358///
359/// Free is a read-only check, not a hardware event: a dma_free_task returns a
360/// buffer descriptor id to the compiler's pool without touching the channel,
361/// so it neither pushes nor retires anything here. It asks whether `task`'s
362/// own push -- the one a later pop could still be waiting behind -- has
363/// already been proven complete.
365 enum class Kind { Ignore, Configure, Push, Await, Free };
368 bool issuesToken = false;
369 mlir::Value task;
370
372 mlir::Value task) {
373 return {Kind::Configure, key, false, task};
374 }
376 mlir::Value task = {}) {
377 return {Kind::Push, key, issuesToken, task};
378 }
380 return {Kind::Await, key, false, {}};
381 }
383 return {Kind::Free, key, false, task};
384 }
385};
386
387/// What a push or an await that names no task does to its channel: an
388/// aiex.npu.push_queue; an aiex.npu.dma_memcpy_nd through the placed shim
389/// allocation its metadata names, which always issues a token on S2MM (its
390/// lowering sets one); an aiex.npu.sync syncChannelKey pins down; an
391/// aiex.npu.dma_wait on a placed shim allocation. Ignore for anything else.
392inline QueueEffect rawQueueEffect(mlir::Operation *op,
393 mlir::SymbolTable &symbols) {
394 if (auto push = mlir::dyn_cast<NpuPushQueueOp>(op))
395 return QueueEffect::push({static_cast<int>(push.getColumn()),
396 static_cast<int>(push.getRow()),
397 static_cast<int>(push.getDirection()),
398 static_cast<int>(push.getChannel())},
399 push.getIssueToken());
400 if (auto sync = mlir::dyn_cast<NpuSyncOp>(op)) {
401 if (std::optional<DmaQueueModel::ChannelKey> key = syncChannelKey(sync))
402 return QueueEffect::await(*key);
403 return {};
404 }
405 auto copy = mlir::dyn_cast<NpuDmaMemcpyNdOp>(op);
406 auto wait = mlir::dyn_cast<NpuDmaWaitOp>(op);
407 if (!copy && !wait)
408 return {};
409 auto allocation = symbols.lookup<AIE::ShimDMAAllocationOp>(
410 copy ? copy.getMetadata().getRootReference()
411 : wait.getSymbolAttr().getAttr());
412 if (!allocation || !allocation.getTileOp())
413 return {};
415 if (wait)
416 return QueueEffect::await(key);
417 return QueueEffect::push(key, copy.getIssueToken() ||
418 allocation.getChannelDir() ==
419 AIE::DMAChannelDir::S2MM);
420}
421
422/// Completion is distinct from queue occupancy: Task_Queue_Size excludes the
423/// running task. Task identities are path-local and renamed at joins, so a
424/// configure executed again in a loop is distinct from the carried old task.
426 struct Push {
428 unsigned owner;
429 // Stands in for the newest push of an unknown region, whose owners are
430 // all in forgottenOwners: none of them is proven complete while it is.
431 bool unknownRegion = false;
432 bool operator==(const Push &other) const {
433 return issuesToken == other.issuesToken && owner == other.owner &&
435 }
436 };
437 llvm::SmallVector<unsigned> tasks;
438 llvm::SmallVector<Push, 8> pushes;
439 std::set<unsigned> forgottenOwners;
440 bool unknownTokens = false;
442
443 bool operator==(const DmaCompletionState &other) const {
444 return tasks == other.tasks && pushes == other.pushes &&
446 unknownTokens == other.unknownTokens &&
448 }
449
450 // Canonical identities keep loop fixed points finite. Instances no longer
451 // named by any SSA task need only retain their token and FIFO position.
452 void normalize() {
453 llvm::DenseMap<unsigned, unsigned> names;
454 unsigned next = 1;
455 for (unsigned &task : tasks)
456 if (task) {
457 auto [it, inserted] = names.try_emplace(task, next);
458 next += inserted;
459 task = it->second;
460 }
461 for (Push &push : pushes)
462 push.owner = names.lookup(push.owner);
463 std::set<unsigned> renamed;
464 for (unsigned owner : forgottenOwners)
465 if (unsigned name = names.lookup(owner))
466 renamed.insert(name);
467 forgottenOwners = std::move(renamed);
468 }
469
470 void awaitToken() {
471 if (unknownTokens)
472 return;
473 auto tok =
474 llvm::find_if(pushes, [](const Push &p) { return p.issuesToken; });
475 if (tok != pushes.end()) {
476 pushes.erase(pushes.begin(), std::next(tok));
478 }
479 }
480
481 // Forgotten owners were pushed no later than any remaining unknown-region
482 // stand-in; they are complete once every such stand-in has drained too.
484 if (llvm::none_of(pushes, [](const Push &p) { return p.unknownRegion; }))
485 forgottenOwners.clear();
486 }
487
488 void noteIdle() {
489 // Completion does not consume tokens; later awaits still see these pushes.
490 for (Push &push : pushes) {
491 push.owner = 0;
492 push.unknownRegion = false;
493 }
494 forgottenOwners.clear();
495 }
496
497 // A real space poll leaves at most depth-1 queued tasks AND one running.
498 // Abstract truncation instead retains every lost owner's uncertainty.
499 void bound(unsigned limit, bool spaceGuaranteed) {
500 // Attempted pushes no longer bound FIFO progress after a possible drop.
501 spaceGuaranteed &= !mayHaveDroppedPushes;
502 bool drained = spaceGuaranteed && pushes.size() >= limit;
503 if (pushes.size() > limit) {
504 auto end = pushes.end() - limit;
505 for (const Push &push : llvm::make_range(pushes.begin(), end)) {
506 unknownTokens |= push.issuesToken;
507 if (!spaceGuaranteed && push.owner)
508 forgottenOwners.insert(push.owner);
509 }
510 pushes.erase(pushes.begin(), end);
511 }
512 if (drained)
514 }
515
516 bool isProvenComplete(unsigned owner) const {
517 return owner && !forgottenOwners.count(owner) &&
518 !llvm::any_of(pushes,
519 [&](const Push &p) { return p.owner == owner; });
520 }
521};
522
523/// Per-channel analysis visits only the block each push, await or poll
524/// reaches through, plus the ops enclosing them, in block order, rather than
525/// the whole sequence.
527 llvm::DenseMap<mlir::Block *, llvm::SmallVector<mlir::Operation *>>;
528
529/// Record `op` and each operation enclosing it up to `body`'s own op into
530/// `blocks`, once per block: an ancestor chain's visited block repeats until
531/// the chain's op there changes.
532inline void recordChannelVisit(mlir::Region &body, mlir::Operation *op,
533 ChannelVisits &blocks) {
534 for (mlir::Operation *at = op; at != body.getParentOp();
535 at = at->getParentOp()) {
536 auto &ops = blocks[at->getBlock()];
537 if (!ops.empty() && ops.back() == at)
538 break;
539 ops.push_back(at);
540 }
541}
542
543/// Guard every push in a runtime sequence that can land on a full queue, and
544/// (when `warnUnsafeBdReuse`) warn at every Free effect whose own push is not
545/// yet proven complete along some reachable path.
546///
547/// Retain all reachable states at branches and loop exits, including zero
548/// iterations. Nested loops are analyzed recursively, not walked once. Queue
549/// states are finite because overflowing pushes are capped at the target depth;
550/// targets without a queued-task model must be ignored rather than accumulated.
551/// Analyze each independent channel separately to avoid a Cartesian product
552/// of states across channels.
553template <typename EffectFn>
554inline void guardSequenceQueueDepth(mlir::Region &body, DmaQueueModel &queue,
555 const AIE::AIETargetModel &tm, bool enforce,
556 bool warnUnsafeBdReuse, EffectFn effectOf) {
557 auto depthOf = [&](const DmaQueueModel::ChannelKey &k) {
558 return tm.getDmaTaskQueueDepth(k[0], k[1], k[3],
559 static_cast<AIE::DMAChannelDir>(k[2]));
560 };
561
562 struct PathState {
564 DmaCompletionState completion;
565 bool operator==(const PathState &other) const {
566 return queue == other.queue && completion == other.completion;
567 }
568 };
569 using States = llvm::SmallVector<PathState, 8>;
570 llvm::DenseMap<mlir::Value, unsigned> taskIndices;
571 auto track = [&](mlir::Value value) {
572 if (value && value.getType().isIndex())
573 taskIndices.try_emplace(value, taskIndices.size());
574 };
575 std::set<DmaQueueModel::ChannelKey> keys;
576 for (const auto &[key, channel] : queue.state())
577 if (depthOf(key) != 0)
578 keys.insert(key);
579 llvm::DenseMap<mlir::Operation *, QueueEffect> effects;
580 body.walk([&](mlir::Operation *op) {
581 if (warnUnsafeBdReuse) {
582 for (mlir::Value value : op->getOperands())
583 track(value);
584 for (mlir::Value value : op->getResults())
585 track(value);
586 for (mlir::Region &region : op->getRegions())
587 for (mlir::Block &block : region)
588 for (mlir::Value arg : block.getArguments())
589 track(arg);
590 }
591 QueueEffect e = effectOf(op);
592 if (!warnUnsafeBdReuse) {
593 e.task = {};
596 return;
597 }
598 if (e.kind != QueueEffect::Kind::Ignore && depthOf(e.key) != 0) {
599 keys.insert(e.key);
600 effects[op] = e;
601 }
602 });
603
604 // Each channel's analysis visits only its own pushes, awaits and polls and
605 // the ops enclosing them, in block order, rather than the whole sequence.
606 std::map<uint32_t, DmaQueueModel::ChannelKey> keyByStatus;
607 for (const auto &key : keys)
608 if (auto status = tm.getDmaStatusAddress(
609 key[0], key[1], key[3], static_cast<AIE::DMAChannelDir>(key[2])))
610 keyByStatus[*status] = key;
611 std::map<DmaQueueModel::ChannelKey, ChannelVisits> visits;
612 auto visit = [&](mlir::Operation *op, const DmaQueueModel::ChannelKey &key) {
613 recordChannelVisit(body, op, visits[key]);
614 };
615 body.walk<mlir::WalkOrder::PreOrder>([&](mlir::Operation *op) {
616 if (warnUnsafeBdReuse && mlir::isa<mlir::scf::ForOp, mlir::scf::IfOp>(op))
617 for (const auto &key : keys)
618 visit(op, key);
619 if (auto it = effects.find(op); it != effects.end()) {
620 visit(op, it->second.key);
621 } else if (auto poll = mlir::dyn_cast<NpuMaskPollOp>(op)) {
622 if (auto address = poll.getAbsoluteAddress())
623 if (auto it = keyByStatus.find(*address); it != keyByStatus.end())
624 visit(op, it->second);
625 }
626 });
627
628 DmaQueueModel::State entry = queue.state();
629 DmaQueueModel::State merged = entry;
630 for (const auto &key : keys) {
631 uint32_t depth = depthOf(key);
632 const ChannelVisits &channelVisits = visits[key];
633 auto channelEffectOf = [&](mlir::Operation *op) {
634 auto it = effects.find(op);
635 return it != effects.end() && it->second.key == key ? it->second
636 : QueueEffect{};
637 };
638 llvm::SetVector<mlir::Operation *> overflowing;
639 llvm::SetVector<mlir::Operation *> unsafeFrees;
640 std::map<mlir::Operation *, DmaQueueModel::ChannelState> witnesses;
641 bool trackCompletion = warnUnsafeBdReuse;
642 auto warnUnsafeFree = [&](mlir::Operation *op) {
643 if (warnUnsafeBdReuse && unsafeFrees.insert(op)) {
644 auto diag = emitUnsafeBdReuseWarning(
645 op, key[0], key[1], static_cast<AIE::DMAChannelDir>(key[2]),
646 key[3]);
647 if (!trackCompletion)
648 diag.attachNote() << "task ownership analysis budget exhausted; "
649 "completion is conservatively unproven";
650 }
651 };
652 size_t completionWork = 0;
653 auto equivalent = [&](const PathState &a, const PathState &b) {
654 return a.queue == b.queue &&
655 (!trackCompletion || a.completion == b.completion);
656 };
657 auto contains = [&](const States &states, const PathState &state) {
658 return llvm::any_of(states, [&](const PathState &other) {
659 return equivalent(state, other);
660 });
661 };
662 auto compact = [&](States &states) {
663 States unique;
664 for (const auto &state : states)
665 if (!contains(unique, state))
666 unique.push_back(state);
667 states = std::move(unique);
668 };
669 // Bound optional ownership analysis without weakening queue-depth guards.
670 // Beyond the budget, all frees are uncertain and only occupancy is tracked.
671 auto append = [&](States &to, const States &from) {
672 if (warnUnsafeBdReuse && !trackCompletion)
673 compact(to);
674 for (const auto &state : from)
675 if (!contains(to, state)) {
676 if (trackCompletion && to.size() >= 256) {
677 trackCompletion = false;
678 compact(to);
679 if (contains(to, state))
680 continue;
681 }
682 to.push_back(state);
683 }
684 };
685 auto dedupe = [&](States &states) {
686 if (trackCompletion)
687 for (auto &state : states)
688 state.completion.normalize();
689 if (states.size() < 2)
690 return;
691 States unique;
692 append(unique, states);
693 states = std::move(unique);
694 };
695 auto transfer = [&](States &states, mlir::ValueRange from,
696 mlir::ValueRange to) {
697 if (!trackCompletion)
698 return;
699 for (auto &state : states) {
700 auto previous = state.completion.tasks;
701 for (auto [source, dest] : llvm::zip_equal(from, to)) {
702 auto src = taskIndices.find(source), dst = taskIndices.find(dest);
703 if (src != taskIndices.end() && dst != taskIndices.end())
704 state.completion.tasks[dst->second] = previous[src->second];
705 }
706 }
707 dedupe(states);
708 };
709 auto forgetScope = [&](States &states, mlir::Region &region,
710 bool keepArguments = false) {
711 if (!trackCompletion)
712 return;
713 for (auto [value, index] : taskIndices) {
714 mlir::Region *parent = value.getParentRegion();
715 if (parent != &region && !region.isAncestor(parent))
716 continue;
717 if (keepArguments)
718 if (auto arg = mlir::dyn_cast<mlir::BlockArgument>(value))
719 if (arg.getOwner() == &region.front())
720 continue;
721 for (auto &state : states)
722 state.completion.tasks[index] = 0;
723 }
724 dedupe(states);
725 };
726 bool canPoll =
727 enforce &&
728 queueSpacePoll(tm, key[0], key[1], key[3],
729 static_cast<AIE::DMAChannelDir>(key[2]), depth)
730 .has_value();
731
732 // Earlier passes may already have guarded a start. Credit only polls whose
733 // constant address and mask prove space or idleness on a tracked channel.
734 enum class PollEffect { Ignore, Space, Idle };
735 auto pollEffectOf = [&](NpuMaskPollOp poll) {
736 auto address = poll.getAbsoluteAddress();
737 auto mask = getConstantIntOperand(poll.getMask());
738 auto value = getConstantIntOperand(poll.getValue());
739 if (!address || !mask || !value)
740 return PollEffect::Ignore;
741 auto status = tm.getDmaStatusAddress(
742 key[0], key[1], key[3], static_cast<AIE::DMAChannelDir>(key[2]));
743 if (status != address)
744 return PollEffect::Ignore;
745 uint32_t idleMask = tm.getDmaChannelIdleMask();
746 if (idleMask && (*mask & idleMask) == idleMask &&
747 (*value & idleMask) == 0)
748 return PollEffect::Idle;
749 uint32_t fieldMask = tm.getDmaTaskQueueSizeMask();
750 if (!fieldMask || (depth & (depth - 1)) != 0)
751 return PollEffect::Ignore;
752 uint32_t depthBit = depth << llvm::countr_zero(fieldMask);
753 return depthBit && (depthBit & fieldMask) == depthBit &&
754 (*mask & depthBit) == depthBit && (*value & depthBit) == 0
755 ? PollEffect::Space
756 : PollEffect::Ignore;
757 };
758
759 // Unknown region semantics (including unstructured CFGs) must not credit
760 // conditional waits. Guard their pushes and conservatively forget tokens.
761 auto unknownRegion = [&](mlir::Region &region, States states) {
762 region.walk([&](mlir::Operation *op) {
763 QueueEffect e = channelEffectOf(op);
765 warnUnsafeFree(op);
767 return;
768 for (auto &state : states) {
769 state.queue.pushes.assign(depth, false);
770 state.queue.unknownTokens = true;
771 if (trackCompletion) {
772 state.completion.unknownTokens = true;
773 state.completion.mayHaveDroppedPushes |= !canPoll;
774 for (unsigned owner : state.completion.tasks)
775 if (owner)
776 state.completion.forgottenOwners.insert(owner);
777 // Later polls must count this push before crediting its owners.
778 state.completion.pushes.push_back(
779 {/*issuesToken=*/false, /*owner=*/0, /*unknownRegion=*/true});
780 state.completion.bound(depth + 1, /*spaceGuaranteed=*/false);
781 }
782 if (overflowing.insert(op))
783 witnesses[op] = state.queue;
784 }
785 });
786 dedupe(states);
787 return states;
788 };
789
790 std::function<States(mlir::Region &, States)> analyze;
791 analyze = [&](mlir::Region &region, States states) -> States {
792 if (!region.hasOneBlock())
793 return unknownRegion(region, std::move(states));
794 auto blockVisits = channelVisits.find(&region.front());
795 if (blockVisits == channelVisits.end())
796 return states;
797 for (mlir::Operation *visited : blockVisits->second) {
798 if (trackCompletion) {
799 completionWork += states.size();
800 if (completionWork > 16384) {
801 trackCompletion = false;
802 compact(states);
803 }
804 }
805 mlir::Operation &op = *visited;
806 if (auto loop = mlir::dyn_cast<mlir::scf::ForOp>(&op)) {
807 auto lb = mlir::getConstantIntValue(loop.getLowerBound());
808 auto ub = mlir::getConstantIntValue(loop.getUpperBound());
809 if (lb && ub && *lb >= *ub &&
810 (!loop->hasAttr("unsignedCmp") || (*lb >= 0 && *ub >= 0))) {
811 transfer(states, loop.getInitArgs(), loop.getResults());
812 continue;
813 }
814 // Every loop header state is a possible exit for an unknown trip
815 // count. Include the entry: even an await-only loop might execute
816 // zero times.
817 transfer(states, loop.getInitArgs(), loop.getRegionIterArgs());
818 States reachable = states;
819 for (size_t i = 0; i < reachable.size();) {
820 bool wasTrackingCompletion = trackCompletion;
821 States next = analyze(loop.getRegion(), States{reachable[i]});
822 transfer(next, loop.getYieldedValues(), loop.getRegionIterArgs());
823 forgetScope(next, loop.getRegion(), /*keepArguments=*/true);
824 append(reachable, next);
825 if (wasTrackingCompletion && !trackCompletion) {
826 // Revisit earlier paths with uncertain ownership as well.
827 compact(reachable);
828 i = 0;
829 } else {
830 ++i;
831 }
832 }
833 transfer(reachable, loop.getRegionIterArgs(), loop.getResults());
834 forgetScope(reachable, loop.getRegion());
835 states = std::move(reachable);
836 continue;
837 }
838 if (auto branch = mlir::dyn_cast<mlir::scf::IfOp>(&op)) {
839 States alternatives = analyze(branch.getThenRegion(), states);
840 States otherwise = analyze(branch.getElseRegion(), states);
841 if (branch.getNumResults()) {
842 transfer(alternatives, branch.thenYield().getResults(),
843 branch.getResults());
844 transfer(otherwise, branch.elseYield().getResults(),
845 branch.getResults());
846 }
847 forgetScope(alternatives, branch.getThenRegion());
848 forgetScope(otherwise, branch.getElseRegion());
849 append(alternatives, otherwise);
850 states = std::move(alternatives);
851 continue;
852 }
853 for (mlir::Region &nested : op.getRegions())
854 states = unknownRegion(nested, std::move(states));
855
856 if (auto poll = mlir::dyn_cast<NpuMaskPollOp>(&op))
857 if (auto effect = pollEffectOf(poll); effect != PollEffect::Ignore) {
858 for (auto &state : states) {
859 DmaQueueModel::noteSpaceGuaranteed(state.queue, depth);
860 if (trackCompletion) {
861 if (effect == PollEffect::Idle)
862 state.completion.noteIdle();
863 else
864 state.completion.bound(depth, /*spaceGuaranteed=*/true);
865 }
866 }
867 dedupe(states);
868 }
869
870 QueueEffect e = channelEffectOf(&op);
872 continue;
873 if (e.kind == QueueEffect::Kind::Free) {
874 if (warnUnsafeBdReuse)
875 for (const auto &state : states)
876 if (!trackCompletion ||
877 !state.completion.isProvenComplete(
878 state.completion.tasks[taskIndices.lookup(e.task)]))
879 warnUnsafeFree(&op);
880 continue;
881 }
882 for (auto &state : states) {
884 if (trackCompletion)
885 state.completion.tasks[taskIndices.lookup(e.task)] =
886 taskIndices.size() + 1;
887 continue;
888 }
890 DmaQueueModel::awaitToken(state.queue);
891 if (trackCompletion)
892 state.completion.awaitToken();
893 continue;
894 }
895 if (DmaQueueModel::wouldOverflow(state.queue, depth)) {
896 if (overflowing.insert(&op))
897 witnesses[&op] = state.queue;
898 DmaQueueModel::noteSpaceGuaranteed(state.queue, depth);
899 if (trackCompletion && canPoll)
900 state.completion.bound(depth, /*spaceGuaranteed=*/true);
901 else if (trackCompletion)
902 state.completion.mayHaveDroppedPushes = true;
903 }
904 DmaQueueModel::push(state.queue, e.issuesToken);
905 if (trackCompletion) {
906 unsigned owner =
907 e.task ? state.completion.tasks[taskIndices.lookup(e.task)] : 0;
908 state.completion.pushes.push_back({e.issuesToken, owner});
909 state.completion.bound(depth + 1, /*spaceGuaranteed=*/false);
910 }
911 }
912 dedupe(states);
913 }
914 return states;
915 };
916
917 DmaQueueModel::ChannelState channelEntry;
918 if (auto it = entry.find(key); it != entry.end())
919 channelEntry = it->second;
920 PathState initial{channelEntry, {}};
921 initial.completion.tasks.resize(taskIndices.size());
922 States exits = analyze(body, States{initial});
923 for (mlir::Operation *push : overflowing) {
924 queue.setState({{key, witnesses[push]}});
925 guardQueueOverflow(queue, push, tm, key, depth, enforce);
926 }
927
928 // Preserve exact straight-line state for callers. If control-flow paths
929 // disagree, retain the largest occupancy without crediting ambiguous
930 // tokens.
931 DmaQueueModel::ChannelState channelExit = exits.front().queue;
932 for (const auto &state : exits)
933 if (!(channelExit == state.queue)) {
934 channelExit.pushes.assign(
935 std::max(channelExit.pushes.size(), state.queue.pushes.size()),
936 false);
937 channelExit.unknownTokens = true;
938 }
939 merged[key] = channelExit;
940 }
941 queue.setState(std::move(merged));
942}
943
944/// Check that `sequence` ends, on every path, with no task-completion token
945/// outstanding on a channel some sequence of the device awaits: one in
946/// `awaited`, or one a sync in `mayAwait` may wait on. A token left at the end
947/// outlives the command: the next command's await on that channel takes it and
948/// returns before its own transfer has finished (measured on Strix, NPU
949/// firmware 1.1.2.64), and every completion the compiler derives from an await
950/// assumes the channel started with none. A token on a channel nothing awaits
951/// is never taken. An await consumes the oldest token, so only the count
952/// matters. A loop with an unknown trip count must leave the count where it
953/// found it, or what remains grows with the trip count. Regions the check
954/// cannot follow are not diagnosed, nor channels a sync of the sequence's own
955/// may have waited on.
956template <typename EffectFn>
957inline mlir::LogicalResult
958checkTokensConsumed(AIE::RuntimeSequenceOp sequence,
959 const std::set<DmaQueueModel::ChannelKey> &awaited,
960 llvm::ArrayRef<SyncCover> mayAwait, EffectFn effectOf) {
961 mlir::Region &body = sequence.getBody();
962 auto covered = [](llvm::ArrayRef<SyncCover> covers,
963 const DmaQueueModel::ChannelKey &key) {
964 return llvm::any_of(covers,
965 [&](const SyncCover &c) { return c.covers(key); });
966 };
967 llvm::SmallVector<SyncCover> ownInexact;
968 body.walk([&](NpuSyncOp sync) {
969 if (!syncChannelKey(sync))
970 ownInexact.push_back(syncCover(sync));
971 });
972 // Each channel's analysis visits only its own pushes and syncs and the ops
973 // enclosing them, in block order.
974 llvm::DenseMap<mlir::Operation *, QueueEffect> effects;
975 std::map<DmaQueueModel::ChannelKey, ChannelVisits> visits;
976 body.walk<mlir::WalkOrder::PreOrder>([&](mlir::Operation *op) {
977 QueueEffect e = effectOf(op);
978 if ((e.kind != QueueEffect::Kind::Push &&
980 (!awaited.count(e.key) && !covered(mayAwait, e.key)) ||
981 covered(ownInexact, e.key))
982 return;
983 effects[op] = e;
984 recordChannelVisit(body, op, visits[e.key]);
985 });
986
987 auto describe = [](mlir::InFlightDiagnostic &diag,
988 const DmaQueueModel::ChannelKey &key) {
989 diag << " on tile (" << key[0] << "," << key[1] << ") "
990 << AIE::stringifyDMAChannelDir(static_cast<AIE::DMAChannelDir>(key[2]))
991 << " channel " << key[3];
992 };
993 // Small constant trip counts are followed pass by pass, as unrolling would.
994 constexpr int64_t maxFollowedTrips = 64;
995 bool consumed = true;
996 for (const auto &channel : visits) {
997 const DmaQueueModel::ChannelKey &key = channel.first;
998 const ChannelVisits &blocks = channel.second;
999 using Counts = std::set<int64_t>;
1000 bool unknown = false;
1001 std::function<Counts(mlir::Region &, Counts)> analyze;
1002 analyze = [&](mlir::Region &region, Counts counts) -> Counts {
1003 if (region.empty())
1004 return counts;
1005 if (!region.hasOneBlock()) {
1006 unknown = true;
1007 return counts;
1008 }
1009 auto visited = blocks.find(&region.front());
1010 if (visited == blocks.end())
1011 return counts;
1012 for (mlir::Operation *at : visited->second) {
1013 mlir::Operation &op = *at;
1014 if (unknown)
1015 return counts;
1016 if (auto it = effects.find(&op); it != effects.end()) {
1017 Counts next;
1018 for (int64_t count : counts)
1019 next.insert(it->second.kind == QueueEffect::Kind::Push
1020 ? count + it->second.issuesToken
1021 : std::max<int64_t>(count - 1, 0));
1022 counts = std::move(next);
1023 continue;
1024 }
1025 if (auto loop = mlir::dyn_cast<mlir::scf::ForOp>(&op)) {
1026 auto lb = mlir::getConstantIntValue(loop.getLowerBound());
1027 auto ub = mlir::getConstantIntValue(loop.getUpperBound());
1028 auto step = mlir::getConstantIntValue(loop.getStep());
1029 if (lb && ub && step && *step > 0 && !loop->hasAttr("unsignedCmp")) {
1030 int64_t trips = *ub > *lb ? llvm::divideCeil(*ub - *lb, *step) : 0;
1031 if (trips <= maxFollowedTrips) {
1032 for (int64_t i = 0; i < trips && !unknown; i++)
1033 counts = analyze(loop.getRegion(), counts);
1034 continue;
1035 }
1036 }
1037 for (int64_t count : counts) {
1038 Counts after = analyze(loop.getRegion(), Counts{count});
1039 if (unknown)
1040 return counts;
1041 if (after == Counts{count})
1042 continue;
1043 auto diag = loop.emitOpError("changes how many task-completion "
1044 "tokens are outstanding");
1045 describe(diag, key);
1046 diag << " in each pass, so how many it leaves depends on its trip "
1047 "count; await as many tokens in each pass as it issues";
1048 consumed = false;
1049 unknown = true;
1050 return counts;
1051 }
1052 continue;
1053 }
1054 if (auto branch = mlir::dyn_cast<mlir::scf::IfOp>(&op)) {
1055 Counts taken = analyze(branch.getThenRegion(), counts);
1056 Counts otherwise = analyze(branch.getElseRegion(), counts);
1057 taken.insert(otherwise.begin(), otherwise.end());
1058 counts = std::move(taken);
1059 continue;
1060 }
1061 unknown = true;
1062 return counts;
1063 }
1064 return counts;
1065 };
1066 Counts exits = analyze(body, Counts{0});
1067 if (unknown || *exits.rbegin() == 0)
1068 continue;
1069 int64_t left = *exits.rbegin();
1070 // Not sequence.emitOpError(): that would print the whole sequence.
1071 auto diag = mlir::emitError(sequence.getLoc())
1072 << "ends with " << left << " task-completion token"
1073 << (left > 1 ? "s" : "") << " no await consumes";
1074 describe(diag, key);
1075 diag << "; the next command's await on that channel would return for "
1076 "this sequence's transfer instead of its own. Await every "
1077 "token-issuing transfer before the sequence ends, or start it "
1078 "without issue_token";
1079 consumed = false;
1080 }
1081 return mlir::success(consumed);
1082}
1083
1084} // namespace xilinx::AIEX
1085
1086#endif // AIE_DIALECT_AIEX_UTILS_DMAQUEUEMODEL_H
Tracks, per DMA channel, the pushes not yet known to have completed along one execution path.
static void push(ChannelState &q, bool issuesToken)
Record a push.
void push(const ChannelKey &key, bool issuesToken)
size_t outstanding(const ChannelKey &key) const
Outstanding pushes on key, for diagnostics.
static void awaitToken(ChannelState &q)
Retire what a task-completion-token await proves has drained.
void noteSpaceGuaranteed(const ChannelKey &key, uint32_t depth)
std::array< int, 4 > ChannelKey
{col, row, direction, channel}.
static void noteSpaceGuaranteed(ChannelState &q, uint32_t depth)
Record that a queue-space poll has guaranteed room.
bool shouldReport(const ChannelKey &key)
True the first time a channel is reported.
void awaitToken(const ChannelKey &key)
static ChannelKey keyOf(AIE::ShimDMAAllocationOp allocation)
The channel of a shim allocation on a placed tile.
static bool wouldOverflow(const ChannelState &q, uint32_t depth)
True when a push would land on a full queue.
static ChannelKey keyOf(DMAConfigureTaskOp cfg)
The channel of a configure on a placed tile.
static ChannelKey keyOf(AIE::TileOp tile, AIE::DMAChannelDir direction, int64_t channel)
bool wouldOverflow(const ChannelKey &key, uint32_t depth) const
const State & state() const
std::map< ChannelKey, ChannelState > State
Every channel's queue.
virtual uint32_t getDmaChannelIdleMask() const
Return the bits of the getDmaStatusAddress() register that are all clear exactly when the channel has...
virtual uint32_t getDmaTaskQueueDepth() const =0
Return how many DMA tasks a single channel's task queue holds (unsupported = 0).
virtual uint32_t getDmaTaskQueueSizeMask() const
Return the mask selecting the live task-queue occupancy field within the getDmaStatusAddress() regist...
virtual std::optional< uint32_t > getDmaStatusAddress(int, int, int, AIE::DMAChannelDir) const
Return the address of the DMA status register for a channel, or nullopt when this target's layout has...
void recordChannelVisit(mlir::Region &body, mlir::Operation *op, ChannelVisits &blocks)
Record op and each operation enclosing it up to body's own op into blocks, once per block: an ancesto...
std::optional< std::pair< uint32_t, uint32_t > > queueSpacePoll(const AIE::AIETargetModel &tm, int col, int row, uint32_t channel, AIE::DMAChannelDir dir, uint32_t depth)
Address and mask for a poll that waits for a free task-queue slot.
QueueEffect rawQueueEffect(mlir::Operation *op, mlir::SymbolTable &symbols)
What a push or an await that names no task does to its channel: an aiex.npu.push_queue; an aiex....
mlir::InFlightDiagnostic emitQueueOverflowWarning(mlir::Operation *op, int col, int row, AIE::DMAChannelDir dir, uint32_t channel, uint32_t depth, size_t outstanding)
Shared wording for the two lowering paths, so the explanation does not depend on whether the transfer...
constexpr llvm::StringLiteral queueDiagnosedAttr
mlir::LogicalResult checkTokensConsumed(AIE::RuntimeSequenceOp sequence, const std::set< DmaQueueModel::ChannelKey > &awaited, llvm::ArrayRef< SyncCover > mayAwait, EffectFn effectOf)
Check that sequence ends, on every path, with no task-completion token outstanding on a channel some ...
SyncCover syncCover(NpuSyncOp sync)
std::optional< DmaQueueModel::ChannelKey > syncChannelKey(NpuSyncOp sync)
The channel an aiex.npu.sync retires on, or nullopt when it cannot be pinned down.
llvm::DenseMap< mlir::Block *, llvm::SmallVector< mlir::Operation * > > ChannelVisits
Per-channel analysis visits only the block each push, await or poll reaches through,...
mlir::LogicalResult insertQueueSpaceWait(mlir::Operation *before, const AIE::AIETargetModel &tm, int col, int row, uint32_t channel, AIE::DMAChannelDir dir, uint32_t depth)
void guardQueueOverflow(DmaQueueModel &queue, mlir::Operation *push, const AIE::AIETargetModel &tm, const DmaQueueModel::ChannelKey &key, uint32_t depth, bool enforce)
Handle a push on key that would land on a full queue: poll where one can be emitted,...
mlir::InFlightDiagnostic emitUnsafeBdReuseWarning(mlir::Operation *freeOp, int col, int row, AIE::DMAChannelDir dir, uint32_t channel)
Wording for a dma_free_task that releases a buffer descriptor id the runtime pool could hand straight...
std::optional< uint32_t > getConstantIntOperand(mlir::Value v)
void guardSequenceQueueDepth(mlir::Region &body, DmaQueueModel &queue, const AIE::AIETargetModel &tm, bool enforce, bool warnUnsafeBdReuse, EffectFn effectOf)
Guard every push in a runtime sequence that can land on a full queue, and (when warnUnsafeBdReuse) wa...
void awaitSync(DmaQueueModel &queue, NpuSyncOp sync)
Retire what an aiex.npu.sync proves has drained.
bool operator==(const Push &other) const
Completion is distinct from queue occupancy: Task_Queue_Size excludes the running task.
llvm::SmallVector< Push, 8 > pushes
void bound(unsigned limit, bool spaceGuaranteed)
bool operator==(const DmaCompletionState &other) const
llvm::SmallVector< unsigned > tasks
std::set< unsigned > forgottenOwners
bool isProvenComplete(unsigned owner) const
One channel's queue: its pushes not yet known to have completed, oldest first, each true when it issu...
llvm::SmallVector< bool, 8 > pushes
bool operator==(const ChannelState &other) const
What one op in a loop body does to a channel queue.
static QueueEffect await(DmaQueueModel::ChannelKey key)
static QueueEffect configure(DmaQueueModel::ChannelKey key, mlir::Value task)
static QueueEffect free(DmaQueueModel::ChannelKey key, mlir::Value task)
DmaQueueModel::ChannelKey key
static QueueEffect push(DmaQueueModel::ChannelKey key, bool issuesToken, mlir::Value task={})
The channels an aiex.npu.sync syncChannelKey cannot pin down may wait on: its tile range expanded,...
std::optional< int > dir
std::optional< int > chan
bool covers(const DmaQueueModel::ChannelKey &key) const