31#include "mlir/Dialect/Arith/IR/Arith.h"
32#include "mlir/Dialect/SCF/IR/SCF.h"
33#include "mlir/Pass/Pass.h"
34#include "llvm/ADT/DenseMap.h"
35#include "llvm/ADT/DenseSet.h"
36#include "llvm/ADT/SetVector.h"
41#define GEN_PASS_DEF_AIELOWERDYNAMICBDPOOL
42#include "aie/Dialect/AIEX/Transforms/AIEXPasses.h.inc"
51struct AIELowerDynamicBDPoolPass
52 : xilinx::AIEX::impl::AIELowerDynamicBDPoolBase<AIELowerDynamicBDPoolPass> {
54 xilinx::AIEX::impl::AIELowerDynamicBDPoolBase<AIELowerDynamicBDPoolPass>;
55 AIELowerDynamicBDPoolPass() =
default;
56 AIELowerDynamicBDPoolPass(
const AIELowerDynamicBDPoolOptions &options)
62 llvm::DenseMap<Value, SmallVector<Value, 2>> pairedId;
65 llvm::DenseMap<Value, std::pair<int, int>> tileForTask;
69 llvm::DenseMap<Value, DMAConfigureTaskOp> originConfigure;
75 llvm::DenseSet<Value> taskValues;
79 llvm::DenseMap<Value, unsigned> taskIdCount;
87 SmallVector<unsigned> carried;
88 SmallVector<unsigned> idCount;
90 SmallVector<Operation *> placeholders;
92 SmallVector<Fixup> fixups;
96 static std::pair<uint32_t, uint32_t> bdPartition(DMAConfigureTaskOp cfg) {
97 AIE::TileOp tile = cfg.getTileOp();
99 tile.getCol(), tile.getRow(), cfg.getChannel());
104 static SmallVector<AIE::DMABDOp, 2> chainBds(DMAConfigureTaskOp cfg) {
105 SmallVector<AIE::DMABDOp, 2> bds;
106 cfg.walk([&](AIE::DMABDOp bd) { bds.push_back(bd); });
110 void markTask(Value v,
unsigned idCount, SmallVectorImpl<Value> &worklist) {
111 if (taskValues.insert(v).second) {
112 taskIdCount[v] = idCount;
113 worklist.push_back(v);
122 void computeTaskValues(AIE::RuntimeSequenceOp seq) {
125 SmallVector<Value> worklist;
126 seq.walk([&](DMAConfigureTaskOp c) {
127 markTask(c.getResult(), chainBds(c).size(), worklist);
129 while (!worklist.empty()) {
130 Value v = worklist.pop_back_val();
131 unsigned idCount = taskIdCount.lookup(v);
132 for (OpOperand &use : v.getUses()) {
133 Operation *user = use.getOwner();
134 if (
auto f = dyn_cast<scf::ForOp>(user)) {
135 for (
unsigned k = 0; k < f.getInitArgs().size(); ++k)
136 if (f.getInitArgs()[k] == v)
137 markTask(f.getRegionIterArg(k), idCount, worklist);
138 }
else if (isa<scf::YieldOp>(user)) {
139 unsigned k = use.getOperandNumber();
140 Operation *parent = user->getParentOp();
141 if (
auto pf = dyn_cast<scf::ForOp>(parent))
142 markTask(pf.getResult(k), idCount, worklist);
143 else if (
auto pi = dyn_cast<scf::IfOp>(parent))
144 markTask(pi.getResult(k), idCount, worklist);
153 LogicalResult verifyChainLengths(AIE::RuntimeSequenceOp seq) {
154 auto agree = [&](Operation *op, Value a, Value b,
155 StringRef what) -> LogicalResult {
156 if (!taskValues.contains(a) || !taskValues.contains(b) ||
157 taskIdCount.lookup(a) == taskIdCount.lookup(b))
159 return op->emitOpError()
160 << what <<
" a " << taskIdCount.lookup(a) <<
"-BD chain and a "
161 << taskIdCount.lookup(b)
162 <<
"-BD chain at the same position; a pool-allocated task "
163 "carries one buffer descriptor id per BD, so every task "
164 "flowing through one position must have the same chain length";
166 WalkResult wr = seq.walk([&](Operation *op) -> WalkResult {
167 if (
auto ifOp = dyn_cast<scf::IfOp>(op)) {
168 if (ifOp.getNumResults() == 0 || ifOp.elseBlock() ==
nullptr)
169 return WalkResult::advance();
170 auto thenYield = cast<scf::YieldOp>(ifOp.thenBlock()->getTerminator());
171 auto elseYield = cast<scf::YieldOp>(ifOp.elseBlock()->getTerminator());
172 for (
unsigned k = 0; k < ifOp.getNumResults(); ++k)
173 if (failed(agree(ifOp, thenYield.getOperand(k),
174 elseYield.getOperand(k),
175 "yields from its branches")))
176 return WalkResult::interrupt();
177 }
else if (
auto forOp = dyn_cast<scf::ForOp>(op)) {
178 auto yield = cast<scf::YieldOp>(forOp.getBody()->getTerminator());
179 for (
unsigned k = 0; k < forOp.getInitArgs().size(); ++k)
180 if (failed(agree(forOp, forOp.getInitArgs()[k], yield.getOperand(k),
181 "carries from its init and its yield")))
182 return WalkResult::interrupt();
184 return WalkResult::advance();
186 return failure(wr.wasInterrupted());
189 LogicalResult lowerConfigure(DMAConfigureTaskOp cfg) {
190 SmallVector<AIE::DMABDOp, 2> bds = chainBds(cfg);
191 if (llvm::any_of(bds, [](AIE::DMABDOp bd) {
return bd.getBdIdVal(); }))
197 for (AIE::DMABDOp bd : bds)
198 if (bd.getBdId().has_value())
199 return bd.emitOpError(
200 "pins a buffer descriptor ID inside a runtime-bound "
201 "sequence that draws IDs from the dynamic pool; a pinned "
202 "ID would collide with a runtime-allocated one. Allocate "
203 "every BD in this sequence from the pool (drop the "
204 "bd_id), or make the sequence straight-line so the static "
205 "allocator assigns all IDs.");
207 AIE::TileOp tile = cfg.getTileOp();
208 auto [begin, end] = bdPartition(cfg);
209 if (bds.size() > end - begin)
210 return cfg.emitOpError()
211 <<
"chains " << bds.size() <<
" buffer descriptors, but channel "
212 << cfg.getChannel() <<
" of tile (" << tile.getCol() <<
","
213 << tile.getRow() <<
") draws from a pool of only " << end - begin
216 SmallVector<Value, 2> ids;
217 for (AIE::DMABDOp bd : bds) {
219 DMABdPoolPopOp::create(b, cfg.getLoc(), b.getI32Type(), tile.getCol(),
220 tile.getRow(), begin, end,
nullptr)
222 bd.getBdIdValMutable().assign(bdId);
225 pairedId[cfg.getResult()] = ids;
235 scf::ForOp carryIdsThroughLoop(scf::ForOp forOp) {
236 SmallVector<unsigned> carried;
237 SmallVector<unsigned> idCount;
238 SmallVector<Value> newInits;
239 for (
unsigned k = 0; k < forOp.getInitArgs().size(); ++k) {
240 Value init = forOp.getInitArgs()[k];
241 if (!taskValues.contains(init))
243 SmallVector<Value, 2> ids = pairedId.lookup(init);
244 assert(ids.size() == taskIdCount.lookup(init) &&
245 "for init carries a task but its ids are not all paired");
246 carried.push_back(k);
247 idCount.push_back(ids.size());
248 newInits.append(ids.begin(), ids.end());
253 unsigned nOrig = forOp.getInitArgs().size();
254 IRRewriter rewriter(forOp);
255 auto newYields = [&](OpBuilder &b, Location loc,
256 ArrayRef<BlockArgument> newBBArgs) {
260 return SmallVector<Value>(newBBArgs.begin(), newBBArgs.end());
263 FailureOr<LoopLikeOpInterface> newLoop = forOp.replaceWithAdditionalYields(
266 assert(succeeded(newLoop) &&
"scf.for additional-yields rewrite failed");
269 auto newFor = cast<scf::ForOp>(
270 newLoop->getOperation());
272 unsigned off = nOrig;
273 for (
auto [i, k] : llvm::enumerate(carried)) {
277 taskValues.insert(newFor.getRegionIterArg(k));
278 taskValues.insert(newFor.getResult(k));
279 taskIdCount[newFor.getRegionIterArg(k)] = idCount[i];
280 taskIdCount[newFor.getResult(k)] = idCount[i];
281 SmallVector<Value, 2> argIds, resIds;
282 for (
unsigned j = 0; j < idCount[i]; ++j) {
283 argIds.push_back(newFor.getRegionIterArg(off + j));
284 resIds.push_back(newFor.getResult(off + j));
286 pairedId[newFor.getRegionIterArg(k)] = argIds;
287 pairedId[newFor.getResult(k)] = resIds;
290 fixups.push_back({newFor, carried, idCount, nOrig, {}});
299 LogicalResult carryIdsThroughIf(scf::IfOp ifOp) {
300 if (ifOp.getNumResults() == 0 || ifOp.elseBlock() ==
nullptr)
303 auto thenYield = cast<scf::YieldOp>(ifOp.thenBlock()->getTerminator());
304 auto elseYield = cast<scf::YieldOp>(ifOp.elseBlock()->getTerminator());
306 SmallVector<unsigned> carried;
307 SmallVector<unsigned> idCount;
308 for (
unsigned k = 0; k < ifOp.getNumResults(); ++k) {
309 bool thenT = taskValues.contains(thenYield.getOperand(k));
310 bool elseT = taskValues.contains(elseYield.getOperand(k));
311 if (thenT && elseT) {
312 carried.push_back(k);
313 idCount.push_back(taskIdCount.lookup(thenYield.getOperand(k)));
314 }
else if (thenT || elseT)
315 return ifOp.emitOpError(
316 "yields a pool-allocated task on only one branch of an scf.if; "
318 "branches must yield the task (and its buffer descriptor id) so it "
319 "can be freed after the if");
324 unsigned nOrig = ifOp.getNumResults();
326 SmallVector<Type> resultTypes(ifOp.getResultTypes());
327 unsigned numIds = llvm::sum_of(idCount);
328 resultTypes.append(numIds, b.getI32Type());
331 scf::IfOp::create(b, ifOp.getLoc(), resultTypes, ifOp.getCondition(),
333 newIf.getThenRegion().takeBody(ifOp.getThenRegion());
334 newIf.getElseRegion().takeBody(ifOp.getElseRegion());
340 SmallVector<Operation *> placeholders;
341 auto appendPlaceholders = [&](scf::YieldOp y) {
343 SmallVector<Value> ids;
344 for (
unsigned j = 0; j < numIds; ++j) {
346 arith::ConstantOp::create(yb, y.getLoc(), yb.getI32IntegerAttr(0));
347 placeholders.push_back(c);
350 y->insertOperands(y->getNumOperands(), ids);
352 appendPlaceholders(cast<scf::YieldOp>(newIf.thenBlock()->getTerminator()));
353 appendPlaceholders(cast<scf::YieldOp>(newIf.elseBlock()->getTerminator()));
355 unsigned off = nOrig;
356 for (
auto [i, k] : llvm::enumerate(carried)) {
359 taskValues.insert(newIf.getResult(k));
360 taskIdCount[newIf.getResult(k)] = idCount[i];
361 SmallVector<Value, 2> resIds;
362 for (
unsigned j = 0; j < idCount[i]; ++j)
363 resIds.push_back(newIf.getResult(off + j));
364 pairedId[newIf.getResult(k)] = resIds;
367 for (
unsigned k = 0; k < nOrig; ++k)
368 ifOp.getResult(k).replaceAllUsesWith(newIf.getResult(k));
370 fixups.push_back({newIf, carried, idCount, nOrig, placeholders});
379 for (Fixup &fx : fixups) {
380 SmallVector<scf::YieldOp, 2> yields;
381 if (
auto forOp = dyn_cast<scf::ForOp>(fx.op)) {
382 yields.push_back(cast<scf::YieldOp>(forOp.getBody()->getTerminator()));
384 auto ifOp = cast<scf::IfOp>(fx.op);
385 for (Block *blk : {ifOp.thenBlock(), ifOp.elseBlock()})
386 yields.push_back(cast<scf::YieldOp>(blk->getTerminator()));
388 for (scf::YieldOp yield : yields) {
389 unsigned off = fx.nOrig;
390 for (
auto [i, k] : llvm::enumerate(fx.carried)) {
391 SmallVector<Value, 2> ids = pairedId.lookup(yield.getOperand(k));
392 assert(ids.size() == fx.idCount[i] &&
393 "carried task yields the wrong number of ids");
394 for (
auto [j,
id] : llvm::enumerate(ids))
395 yield.setOperand(off + j,
id);
396 off += fx.idCount[i];
400 for (Fixup &fx : fixups)
401 for (Operation *p : fx.placeholders)
416 LogicalResult computeMetadata(AIE::RuntimeSequenceOp seq) {
418 originConfigure.clear();
419 SmallVector<Value> worklist;
420 auto setMeta = [&](Value v, std::pair<int, int> tile,
421 DMAConfigureTaskOp origin) {
422 if (tileForTask.count(v))
424 tileForTask[v] = tile;
425 originConfigure[v] = origin;
426 worklist.push_back(v);
428 seq.walk([&](DMAConfigureTaskOp c) {
429 AIE::TileOp t = c.getTileOp();
430 setMeta(c.getResult(), {t.getCol(), t.getRow()}, c);
432 while (!worklist.empty()) {
433 Value v = worklist.pop_back_val();
434 auto tile = tileForTask[v];
435 DMAConfigureTaskOp origin = originConfigure[v];
436 for (OpOperand &use : v.getUses()) {
437 Operation *user = use.getOwner();
438 if (
auto f = dyn_cast<scf::ForOp>(user)) {
439 for (
unsigned k = 0; k < f.getInitArgs().size(); ++k)
440 if (f.getInitArgs()[k] == v) {
441 setMeta(f.getRegionIterArg(k), tile, origin);
442 setMeta(f.getResult(k), tile, origin);
444 }
else if (isa<scf::YieldOp>(user)) {
445 auto ifOp = dyn_cast<scf::IfOp>(user->getParentOp());
446 if (ifOp && user->getBlock() == ifOp.thenBlock())
447 setMeta(ifOp.getResult(use.getOperandNumber()), tile, origin);
454 WalkResult wr = seq.walk([&](scf::IfOp ifOp) -> WalkResult {
455 if (ifOp.getNumResults() == 0 || ifOp.elseBlock() ==
nullptr)
456 return WalkResult::advance();
457 auto thenYield = cast<scf::YieldOp>(ifOp.thenBlock()->getTerminator());
458 auto elseYield = cast<scf::YieldOp>(ifOp.elseBlock()->getTerminator());
459 for (
unsigned k = 0; k < ifOp.getNumResults(); ++k)
460 if (failed(checkChannelAgreement(
461 ifOp, thenYield.getOperand(k), elseYield.getOperand(k),
462 "yields tasks on different physical channels "
463 "(tile/direction/channel) from its two branches at "
464 "the same result; a pooled buffer descriptor id and "
465 "its completion sync belong to one channel")))
466 return WalkResult::interrupt();
467 return WalkResult::advance();
469 if (wr.wasInterrupted())
478 WalkResult forWr = seq.walk([&](scf::ForOp forOp) -> WalkResult {
479 auto yield = cast<scf::YieldOp>(forOp.getBody()->getTerminator());
480 for (
unsigned k = 0; k < forOp.getInitArgs().size(); ++k)
481 if (failed(checkChannelAgreement(
482 forOp, forOp.getInitArgs()[k], yield.getOperand(k),
483 "carries a task on different physical channels "
484 "(tile/direction/channel) between its init and its "
485 "per-iteration reconfiguration; a pooled buffer "
486 "descriptor id and its completion sync belong to one "
488 return WalkResult::interrupt();
489 return WalkResult::advance();
491 return forWr.wasInterrupted() ? failure() : success();
499 LogicalResult checkChannelAgreement(Operation *op, Value a, Value b,
501 DMAConfigureTaskOp originA = originConfigure.lookup(a);
502 DMAConfigureTaskOp originB = originConfigure.lookup(b);
503 if (originA && originB &&
505 op->emitOpError(msg);
514 static llvm::SetVector<Value> aliasesAfter(Value task) {
515 llvm::SetVector<Value> aliases;
516 aliases.insert(task);
517 for (
unsigned i = 0; i < aliases.size(); ++i) {
518 for (OpOperand &use : aliases[i].getUses()) {
519 Operation *user = use.getOwner();
520 auto branch = dyn_cast<RegionBranchOpInterface>(user);
521 if (!branch && user->hasTrait<OpTrait::IsTerminator>())
523 dyn_cast_or_null<RegionBranchOpInterface>(user->getParentOp());
526 RegionBranchSuccessorMapping mapping;
527 branch.getSuccessorOperandInputMapping(mapping);
528 aliases.insert_range(mapping.lookup(&use));
537 static LogicalResult verifyNoStartAfterFree(Value task, Operation *freeOp) {
538 llvm::SetVector<Value> aliases = aliasesAfter(task);
539 auto startsAfterFree = [&](DMAStartTaskOp start) {
540 for (Block *block = freeOp->getBlock(); block;
541 block = block->getParentOp()->getBlock()) {
542 Operation *startAt = block->findAncestorOpInBlock(*start);
545 Operation *freeAt = block->findAncestorOpInBlock(*freeOp);
546 if (freeAt != startAt && freeAt->isBeforeInBlock(startAt))
550 Value started = start.getTask();
551 for (
auto loop = freeOp->getParentOfType<scf::ForOp>(); loop;
552 loop = loop->getParentOfType<scf::ForOp>()) {
553 if (!loop->isAncestor(start))
555 if (!loop.getBody()->getParent()->isAncestor(started.getParentRegion()))
557 auto arg = dyn_cast<BlockArgument>(started);
558 if (arg && arg.getOwner() == loop.getBody() &&
559 arg.getArgNumber() >= loop.getNumInductionVars()) {
560 unsigned k = arg.getArgNumber() - loop.getNumInductionVars();
561 if (aliases.contains(loop.getYieldedValues()[k]))
567 for (Value alias : aliases)
568 for (Operation *user : alias.getUsers()) {
569 auto start = dyn_cast<DMAStartTaskOp>(user);
570 if (!start || !startsAfterFree(start))
572 auto diag = start.emitOpError(
573 "starts a task whose buffer descriptor ID was already returned "
574 "to the runtime pool; the pool may have handed it to another "
575 "task. Free the task after its last start instead.");
576 diag.attachNote(freeOp->getLoc()) <<
"returned here";
582 LogicalResult lowerRelease(Value task, Operation *op) {
583 SmallVector<Value, 2> ids = pairedId.lookup(task);
585 return op->emitOpError(
586 "does not resolve to a task allocated from the runtime pool; cannot "
587 "return its buffer descriptor ID");
588 if (failed(verifyNoStartAfterFree(task, op)))
590 auto tile = tileForTask.lookup(task);
591 auto [begin, end] = bdPartition(originConfigure.lookup(task));
594 DMABdPoolPushOp::create(b, op->getLoc(), tile.first, tile.second, begin,
608 void guardQueueDepth(AIE::RuntimeSequenceOp seq, SymbolTable &symbols) {
609 const AIE::AIETargetModel &tm =
612 auto effectOf = [&](Operation *op) -> QueueEffect {
613 if (isa<NpuPushQueueOp, NpuSyncOp, NpuDmaMemcpyNdOp, NpuDmaWaitOp>(op))
615 DMAConfigureTaskOp cfg;
617 bool isConfigure =
false;
619 bool issuesToken =
false;
620 if (
auto start = dyn_cast<DMAStartTaskOp>(op)) {
621 task = start.getTask();
622 cfg = originConfigure.lookup(task);
624 issuesToken = cfg && start.getPushIssueToken(cfg);
625 }
else if (
auto configure = dyn_cast<DMAConfigureTaskOp>(op)) {
626 if (!warnUnsafeBdReuse)
629 task = configure.getResult();
631 }
else if (
auto await = dyn_cast<DMAAwaitTaskOp>(op)) {
632 cfg = originConfigure.lookup(await.getTask());
634 if (cfg && !cfg.getIssueToken())
636 }
else if (
auto freeOp = dyn_cast<DMAFreeTaskOp>(op)) {
637 if (!warnUnsafeBdReuse)
639 cfg = originConfigure.lookup(freeOp.getTask());
642 if (!cfg.tryGetTileOp())
650 if (!cfg.tryGetTileOp())
656 warnUnsafeBdReuse ? task : Value{})
662 warnUnsafeBdReuse, effectOf);
668 void reserveTakenIds(AIE::RuntimeSequenceOp seq, SymbolTable &symbols) {
669 std::set<std::tuple<int, int, uint32_t, uint32_t>> pools;
670 std::set<std::tuple<int, int, uint32_t>> reserved;
671 seq.walk([&](DMABdPoolPopOp pop) {
673 {pop.getColumn(), pop.getRow(), pop.getBdBegin(), pop.getBdEnd()});
674 if (std::optional<uint32_t>
id = pop.getId())
675 reserved.insert({pop.getColumn(), pop.getRow(), *
id});
679 std::set<std::tuple<int, int, uint32_t>> taken;
680 for (AIE::DmaBody program : getOperation().getOps<AIE::DmaBody>())
682 taken.insert({program.getTileID().col, program.getTileID().row,
id});
683 seq.walk([&](NpuDmaMemcpyNdOp copy) {
684 auto allocation = symbols.lookup<AIE::ShimDMAAllocationOp>(
685 copy.getMetadata().getRootReference());
686 if (AIE::TileOp tile = allocation ? allocation.getTileOp() :
nullptr)
687 taken.insert({tile.getCol(), tile.getRow(),
688 static_cast<uint32_t
>(copy.getId())});
690 OpBuilder b = OpBuilder::atBlockBegin(&seq.getBody().front());
691 for (
auto [col, row, begin, end] : pools)
692 for (
auto [takenCol, takenRow,
id] : taken)
693 if (takenCol == col && takenRow == row &&
id >= begin &&
id < end &&
694 !reserved.count({col, row, id}))
695 DMABdPoolPopOp::create(b, seq.getLoc(), b.getI32Type(), col, row,
696 begin, end, b.getI32IntegerAttr(
id));
699 void runOnOperation()
override {
700 AIE::DeviceOp device = getOperation();
701 SymbolTable symbols(device);
703 WalkResult wr = device.walk([&](AIE::RuntimeSequenceOp seq) -> WalkResult {
707 auto finish = [&](WalkResult result) {
708 reserveTakenIds(seq, symbols);
712 bool hasRuntimeCF =
false;
713 seq.walk([&](Operation *op) {
714 if (isa<scf::ForOp, scf::IfOp, scf::WhileOp>(op))
718 return finish(WalkResult::advance());
724 bool alreadyLowered =
false;
725 seq.walk([&](DMABdPoolPopOp) { alreadyLowered =
true; });
727 return finish(WalkResult::advance());
732 WalkResult whileCheck = seq.walk([&](scf::WhileOp w) {
733 w.emitOpError(
"scf.while in a runtime sequence is not supported by the "
734 "dynamic BD pool path; only scf.for and scf.if are "
736 return WalkResult::interrupt();
738 if (whileCheck.wasInterrupted())
739 return finish(WalkResult::interrupt());
743 originConfigure.clear();
748 WalkResult r = seq.walk([&](DMAConfigureTaskOp cfg) -> WalkResult {
749 return failed(lowerConfigure(cfg)) ? WalkResult::interrupt()
750 : WalkResult::advance();
752 if (r.wasInterrupted())
753 return finish(WalkResult::interrupt());
765 computeTaskValues(seq);
766 if (failed(verifyChainLengths(seq)))
767 return finish(WalkResult::interrupt());
768 SmallVector<Operation *> cf;
769 seq.walk<WalkOrder::PreOrder>([&](Operation *o) {
770 if (isa<scf::ForOp, scf::IfOp>(o))
773 for (Operation *o : cf) {
774 if (
auto f = dyn_cast<scf::ForOp>(o)) {
775 carryIdsThroughLoop(f);
776 }
else if (failed(carryIdsThroughIf(cast<scf::IfOp>(o)))) {
777 return finish(WalkResult::interrupt());
784 if (failed(computeMetadata(seq)))
785 return finish(WalkResult::interrupt());
804 SmallVector<Operation *> toErase;
805 r = seq.walk([&](Operation *op) -> WalkResult {
806 if (
auto freeOp = dyn_cast<DMAFreeTaskOp>(op)) {
807 if (failed(lowerRelease(freeOp.getTask(), freeOp)))
808 return WalkResult::interrupt();
809 toErase.push_back(freeOp);
811 return WalkResult::advance();
813 if (r.wasInterrupted())
814 return finish(WalkResult::interrupt());
819 guardQueueDepth(seq, symbols);
820 for (Operation *op : toErase)
822 return finish(WalkResult::advance());
824 if (wr.wasInterrupted())
825 return signalPassFailure();
831std::unique_ptr<OperationPass<AIE::DeviceOp>>
833 return std::make_unique<AIELowerDynamicBDPoolPass>();
836std::unique_ptr<OperationPass<AIE::DeviceOp>>
838 const AIELowerDynamicBDPoolOptions &options) {
839 return std::make_unique<AIELowerDynamicBDPoolPass>(options);
std::array< int, 4 > ChannelKey
{col, row, direction, channel}.
static ChannelKey keyOf(AIE::TileOp tile, AIE::DMAChannelDir direction, int64_t channel)
std::pair< uint32_t, uint32_t > getBdIdRangeForChannel(int col, int row, int channel) const
Return the half-open range [first, last) of buffer descriptor ids that channel channel of the tile at...
std::unique_ptr< mlir::OperationPass< AIE::DeviceOp > > createAIELowerDynamicBDPoolPass()
QueueEffect rawQueueEffect(mlir::Operation *op, mlir::SymbolTable &symbols)
What a push or an await that names no task does to its channel: an aiex.npu.push_queue; an aiex....
void guardSequenceQueueDepth(mlir::Region &body, DmaQueueModel &queue, const AIE::AIETargetModel &tm, bool enforce, bool warnUnsafeBdReuse, EffectFn effectOf)
Guard every push in a runtime sequence that can land on a full queue, and (when warnUnsafeBdReuse) wa...
llvm::SmallVector< uint32_t > getAssignedBdIds(DmaBody program)
const AIETargetModel & getTargetModel(mlir::Operation *op)
static QueueEffect await(DmaQueueModel::ChannelKey key)
static QueueEffect configure(DmaQueueModel::ChannelKey key, mlir::Value task)
static QueueEffect free(DmaQueueModel::ChannelKey key, mlir::Value task)
static QueueEffect push(DmaQueueModel::ChannelKey key, bool issuesToken, mlir::Value task={})