16#include "mlir/Dialect/Arith/IR/Arith.h"
17#include "mlir/Dialect/MemRef/IR/MemRef.h"
18#include "mlir/Dialect/SCF/IR/SCF.h"
19#include "mlir/Dialect/Utils/StaticValueUtils.h"
20#include "mlir/IR/Matchers.h"
21#include "mlir/Interfaces/DataLayoutInterfaces.h"
22#include "mlir/Pass/Pass.h"
23#include "llvm/ADT/MapVector.h"
24#include "llvm/ADT/SetVector.h"
25#include "llvm/ADT/TypeSwitch.h"
26#include "llvm/Support/CheckedArithmetic.h"
27#include "llvm/Support/Debug.h"
28#include "llvm/Support/MathExtras.h"
40#define DEBUG_TYPE "aie-objectfifo-verify"
43#define GEN_PASS_DEF_AIEOBJECTFIFOVERIFY
44#include "aie/Dialect/AIE/Transforms/AIEPasses.h.inc"
50 SmallVector<Operation *> fillers;
51 SmallVector<Operation *> drainers;
56struct TransferBudget {
57 ObjectFifoDmaEndpointOp endpoint;
62static std::optional<int64_t> getStaticTripCount(scf::ForOp forOp) {
63 if (forOp.getSingleLowerBound() && forOp.getSingleUpperBound() &&
64 forOp.getSingleStep()) {
65 if (std::optional<llvm::APInt> tc = forOp.getStaticTripCount()) {
66 return tc->getSExtValue();
74static bool runsForever(Operation *op) {
75 auto whileOp = dyn_cast<scf::WhileOp>(op);
83static std::optional<int64_t> heldAfter(Block &block, StringRef name,
85 for (Operation &op : block) {
86 if (
auto acquire = dyn_cast<ObjectFifoAcquireOp>(op)) {
87 if (acquire.getObjFifoName() == name)
88 held = std::max<int64_t>(held, acquire.acqNumber());
91 if (
auto release = dyn_cast<ObjectFifoReleaseOp>(op)) {
92 if (release.getObjFifoName() == name)
93 held -= release.relNumber();
98 bool touches = op.getNumRegions() != 0 &&
99 op.walk([&](Operation *inner) {
100 auto acquire = dyn_cast<ObjectFifoAcquireOp>(inner);
101 auto release = dyn_cast<ObjectFifoReleaseOp>(inner);
102 if ((acquire && acquire.getObjFifoName() == name) ||
103 (release && release.getObjFifoName() == name))
104 return WalkResult::interrupt();
105 return WalkResult::advance();
109 auto forOp = dyn_cast<scf::ForOp>(op);
110 std::optional<int64_t> trip =
111 forOp ? getStaticTripCount(forOp) : std::nullopt;
116 std::optional<int64_t> once = heldAfter(*forOp.getBody(), name, held);
119 if (*trip > 1 && heldAfter(*forOp.getBody(), name, *once) != once)
133static std::optional<int64_t> tripCount(scf::ForOp forOp, DeviceOp device) {
134 if (std::optional<int64_t> trip = getStaticTripCount(forOp))
136 std::optional<int64_t> lower = getConstantIntValue(forOp.getLowerBound());
137 std::optional<int64_t> step = getConstantIntValue(forOp.getStep());
138 Value bound = forOp.getUpperBound();
139 if (
auto cast = bound.getDefiningOp<arith::IndexCastOp>())
140 bound = cast.getIn();
141 auto load = bound.getDefiningOp<memref::LoadOp>();
142 if (!lower || !step || *step <= 0 || !load || load.getIndices().size() != 1)
144 std::optional<int64_t> index = getConstantIntValue(load.getIndices().front());
145 auto buffer = load.getMemRef().getDefiningOp<BufferOp>();
148 auto type = buffer ? dyn_cast<MemRefType>(buffer.getType()) : MemRefType();
149 if (!index || !type || !type.getElementType().isInteger(32) ||
150 !type.getLayout().isIdentity() || !buffer.getSymName() ||
151 buffer.getInitialValue() ||
152 llvm::any_of(buffer->getUsers(),
153 [](Operation *user) { return !isa<memref::LoadOp>(user); }))
157 for (Operation *prev = load->getPrevNode(); prev && !wait;
158 prev = prev->getPrevNode())
159 if (
auto use = dyn_cast<UseLockOp>(prev); use && !use.release())
161 std::optional<int64_t> waitsFor =
162 wait ? getConstantIntValue(wait.getValue()) : std::nullopt;
163 LockOp lock = wait ? wait.getLock().getDefiningOp<LockOp>() : LockOp();
164 if (!lock || !wait.acquireGE() || !waitsFor || *waitsFor < 1 ||
166 wait.getBlocking().value_or(LockBlocking::Blocking) !=
167 LockBlocking::Blocking ||
168 lock.getInit().value_or(0) != 0)
170 for (Operation *user : lock->getUsers()) {
171 if (isa<AIEX::SetLockOp>(user))
173 auto use = dyn_cast<UseLockOp>(user);
178 std::optional<int64_t> given = getConstantIntValue(use.getValue());
180 for (Operation *prev = use->getPrevNode(); prev && !before;
181 prev = prev->getPrevNode())
182 if (auto other = dyn_cast<UseLockOp>(prev);
183 other && other.getLock() == use.getLock())
185 std::optional<int64_t> taken = before && !before.release()
186 ? getConstantIntValue(before.getValue())
188 if (!given || !taken || *given > *taken)
192 TileOp tile = buffer.getTileOp();
194 auto reachesTile = [&](FlatSymbolRefAttr named, std::optional<int32_t> col,
195 std::optional<int32_t> row,
196 std::optional<int64_t> address) {
198 return named.getValue() == *buffer.getSymName();
202 col = *address >> tm.getColumnShift();
203 row = (*address >> tm.getRowShift()) &
204 ((1 << (tm.getColumnShift() - tm.getRowShift())) - 1);
206 return *col == tile.getCol() && *row == tile.getRow();
209 std::optional<int64_t> value;
210 for (
auto sequence : device.getOps<RuntimeSequenceOp>()) {
211 bool written = false;
212 WalkResult walk = sequence.walk([&](Operation *op) {
214 llvm::TypeSwitch<Operation *, bool>(op)
215 .Case<AIEX::NpuWrite32Op, AIEX::NpuMaskWrite32Op>([&](auto w) {
216 return reachesTile(w.getBufferAttr(), w.getColumn(), w.getRow(),
217 getConstantIntValue(w.getAddress()));
219 .Case([&](AIEX::NpuBlockWriteOp w) {
220 return reachesTile(w.getBufferAttr(), w.getColumn(), w.getRow(),
223 .Case([&](AIEX::NpuPushQueueOp push) {
224 return reachesTile({}, push.getColumn(), push.getRow(), {});
228 return WalkResult::interrupt();
229 if (auto write = dyn_cast<AIEX::NpuWriteRTPOp>(op)) {
230 if (write.getBuffer() != *buffer.getSymName() ||
231 write.getIndex() != *index)
232 return WalkResult::advance();
234 std::optional<int64_t> word = getConstantIntValue(write.getValue());
235 if (write->getParentOp() != sequence || !word ||
236 (value && *value != *word))
237 return WalkResult::interrupt();
240 return WalkResult::advance();
242 auto set = dyn_cast<AIEX::SetLockOp>(op);
243 if (set && set.getLock() == lock.getResult() && !written)
244 return WalkResult::interrupt();
245 return WalkResult::advance();
247 if (walk.wasInterrupted())
252 return *value > *lower ? (*value - *lower + *step - 1) / *step : 0;
268 TakeRatio(DeviceOp device, StringRef in, StringRef out)
269 : device(device), in(in), out(out) {}
271 std::optional<std::pair<int64_t, int64_t>> of(CoreOp core) {
272 core.walk([&](Operation *op) {
273 if (fifoOf(op) != in && fifoOf(op) != out)
275 for (Operation *parent = op->getParentOp(); parent != core;
276 parent = parent->getParentOp())
277 onUsePath.insert(parent);
280 Region &body = core.getBody();
281 if (!used || !body.hasOneBlock())
284 std::optional<Effect> effect = analyze(body.front(), held);
287 std::optional<Pair> least;
288 for (
const Pair &pair : llvm::concat<const Pair>(effect->low, bounds))
289 if (pair.released > 0 && (!least || pair.taken * least->released <
290 least->taken * pair.released))
292 if (!least || least->taken == 0)
294 int64_t common = std::gcd(least->taken, least->released);
295 return std::make_pair(least->taken / common, least->released / common);
300 int64_t taken, released;
303 SmallVector<Pair> end = {{0, 0}}, low = {{0, 0}};
306 int64_t in = 0, out = 0;
307 bool operator==(
const Held &other)
const {
308 return in == other.in && out == other.out;
311 static constexpr size_t maxPairs = 64;
312 static constexpr int64_t maxPeeledPasses = 4096;
314 static StringRef fifoOf(Operation *op) {
315 if (
auto acquire = dyn_cast<ObjectFifoAcquireOp>(op))
316 return acquire.getObjFifoName();
317 if (
auto release = dyn_cast<ObjectFifoReleaseOp>(op))
318 return release.getObjFifoName();
324 std::optional<SmallVector<Pair>> prune(ArrayRef<Pair> pairs) {
325 SmallVector<Pair> kept;
326 for (
auto [index, pair] : llvm::enumerate(pairs)) {
327 bool dominated =
false;
328 for (
auto [other, by] : llvm::enumerate(pairs))
329 dominated |= other != index && by.taken <= pair.taken &&
330 by.released >= pair.released &&
331 (by.taken != pair.taken || by.released != pair.released ||
334 kept.push_back(pair);
336 if (kept.size() > maxPairs)
341 std::optional<SmallVector<Pair>> sum(ArrayRef<Pair> a, ArrayRef<Pair> b) {
342 SmallVector<Pair> sums;
343 for (
const Pair &x : a)
344 for (
const Pair &y : b) {
346 if (llvm::AddOverflow(x.taken, y.taken, both.taken) ||
347 llvm::AddOverflow(x.released, y.released, both.released))
349 sums.push_back(both);
354 std::optional<SmallVector<Pair>> scale(ArrayRef<Pair> pairs, int64_t n) {
355 SmallVector<Pair> scaled;
356 for (
const Pair &pair : pairs) {
358 if (llvm::MulOverflow(pair.taken, n, times.taken) ||
359 llvm::MulOverflow(pair.released, n, times.released))
361 scaled.push_back(times);
366 std::optional<Effect> then(
const Effect &first,
const Effect &second) {
367 std::optional<SmallVector<Pair>> end = sum(first.end, second.end);
368 std::optional<SmallVector<Pair>> late = sum(first.end, second.low);
371 std::optional<SmallVector<Pair>> low =
372 prune(llvm::to_vector(llvm::concat<const Pair>(first.low, *late)));
375 return Effect{*end, *low};
378 std::optional<Effect> either(
const Effect &a,
const Effect &b) {
379 std::optional<SmallVector<Pair>> end =
380 prune(llvm::to_vector(llvm::concat<const Pair>(a.end, b.end)));
381 std::optional<SmallVector<Pair>> low =
382 prune(llvm::to_vector(llvm::concat<const Pair>(a.low, b.low)));
385 return Effect{*end, *low};
391 std::optional<Effect> repeat(
const Effect &pass, int64_t n) {
392 std::optional<SmallVector<Pair>> end = scale(pass.end, n);
393 std::optional<SmallVector<Pair>> before = scale(pass.end, n - 1);
394 std::optional<SmallVector<Pair>> late =
395 before ? sum(*before, pass.low) : std::nullopt;
398 std::optional<SmallVector<Pair>> low =
399 prune(llvm::to_vector(llvm::concat<const Pair>(pass.low, *late)));
400 std::optional<SmallVector<Pair>> ends = prune(*end);
403 return Effect{*ends, *low};
409 Effect anyRepeat(
const Effect &pass) {
410 llvm::append_range(bounds, pass.end);
412 llvm::append_range(any.low, pass.low);
416 std::optional<Effect> analyze(Block &block, Held &held) {
418 for (Operation &op : block) {
419 std::optional<Effect> step = analyzeOp(op, held);
422 std::optional<Effect> next = then(effect, *step);
430 std::optional<Effect> analyzeOp(Operation &op, Held &held) {
431 StringRef fifo = fifoOf(&op);
432 if (fifo == in || fifo == out) {
433 int64_t &count = fifo == in ? held.in : held.out;
435 if (
auto acquire = dyn_cast<ObjectFifoAcquireOp>(op)) {
436 int64_t wanted = acquire.acqNumber();
438 effect.end = {{std::max<int64_t>(0, wanted - count), 0}};
439 count = std::max(count, wanted);
442 int64_t released = cast<ObjectFifoReleaseOp>(op).relNumber();
443 if (released > count)
447 effect.end = {{0, released}};
448 effect.low.push_back({0, released});
452 if (!onUsePath.contains(&op))
454 if (
auto forOp = dyn_cast<scf::ForOp>(op)) {
455 std::optional<int64_t> trip = tripCount(forOp, device);
458 std::optional<Effect> pass = analyze(*forOp.getBody(), held);
459 if (!pass || !(held == entry))
461 return anyRepeat(*pass);
464 for (int64_t done = 0; done < *trip; ++done) {
466 std::optional<Effect> pass = analyze(*forOp.getBody(), held);
467 std::optional<Effect> rest = pass;
469 if (pass && held == before) {
470 rest = repeat(*pass, *trip - done);
472 rest = anyRepeat(*pass);
474 std::optional<Effect> next = rest ? then(total, *rest) : std::nullopt;
475 if (!next || (!(held == before) && done >= maxPeeledPasses))
483 if (
auto ifOp = dyn_cast<scf::IfOp>(op)) {
484 Held otherwise = held;
485 std::optional<Effect> taken = analyze(ifOp.getThenRegion().front(), held);
486 std::optional<Effect> skipped =
487 ifOp.getElseRegion().empty()
489 : analyze(ifOp.getElseRegion().front(), otherwise);
490 if (!taken || !skipped || !(held == otherwise))
492 return either(*taken, *skipped);
494 if (
auto whileOp = dyn_cast<scf::WhileOp>(op)) {
496 std::optional<Effect> check = analyze(*whileOp.getBeforeBody(), held);
497 Held afterCheck = held;
498 std::optional<Effect> body =
499 check ? analyze(*whileOp.getAfterBody(), held) : std::nullopt;
500 if (!body || !(held == entry))
503 std::optional<Effect> pass = then(*body, *check);
504 return pass ? then(*check, anyRepeat(*pass)) : std::nullopt;
511 llvm::SmallPtrSet<Operation *, 16> onUsePath;
514 SmallVector<Pair> bounds;
517struct AIEObjectFifoVerifyPass
518 :
public xilinx::AIE::impl::AIEObjectFifoVerifyBase<
519 AIEObjectFifoVerifyPass> {
523 std::shared_ptr<PoolLocks> poolLocks;
529 LogicalResult verifyActors(ObjectFifoPoolOp pool,
530 ArrayRef<SegmentActors> actors) {
531 bool external = llvm::any_of(pool.getBufferOps(), [](BufferLike buffer) {
532 return isa<ExternalBufferOp>(buffer.getOperation());
534 std::vector<ObjectFifoSegmentOp> segmentOps = pool.getSegmentOps();
535 for (
auto [index, segment] : llvm::enumerate(actors)) {
536 ObjectFifoSegmentOp segmentOp = segmentOps[index];
537 bool manualFill = segmentOp.getManualFill();
538 bool manualDrain = segmentOp.getManualDrain();
539 if (segment.fillers.size() > 1 ||
540 (manualFill && !segment.fillers.empty())) {
541 return pool.emitOpError(
"segment ")
542 << index <<
" is filled by more than one endpoint";
544 if (segment.drainers.size() > 1 ||
545 (manualDrain && !segment.drainers.empty())) {
546 return pool.emitOpError(
"segment ")
547 << index <<
" is drained by more than one endpoint";
551 bool hostDrains = external && !segment.fillers.empty();
552 bool hostFills = external && !segment.drainers.empty();
553 if (segment.drainers.empty() && !hostDrains && !manualDrain) {
554 return pool.emitOpError(
"segment ") << index <<
" has no drainer";
556 if (segment.fillers.empty() && !pool.getInitValues() && !hostFills &&
558 return pool.emitOpError(
"segment ") << index <<
" has no filler";
565 LogicalResult verifyFlows(DeviceOp device,
566 SymbolTableCollection &symbolTables) {
567 DenseMap<StringRef, int> appearances;
568 for (
auto flow : device.getOps<RouteOp>()) {
569 for (StringRef source : flow.getSourceNames()) {
570 appearances[source]++;
572 for (StringRef dest : flow.getDestinationNames()) {
577 for (
auto endpoint : device.getOps<RouteEndpoint>()) {
579 appearances.lookup(cast<SymbolOpInterface>(*endpoint).getName());
581 return endpoint->emitOpError(
"is not connected by any flow");
583 if (
auto dma = dyn_cast<ObjectFifoDmaEndpointOp>(*endpoint);
584 dma && dma.getDispatch()) {
585 if (failed(verifyDispatch(device, dma, symbolTables)))
590 if (count > 1 && !isa<RouteEndpointOp>(endpoint.getOperation())) {
591 return endpoint->emitOpError(
592 "drives one channel, so at most one flow may name it, but "
594 << count <<
" times";
603 LogicalResult verifyDispatch(DeviceOp device,
604 ObjectFifoDmaEndpointOp endpoint,
605 SymbolTableCollection &symbolTables) {
606 StringRef name = endpoint.getSymName();
607 llvm::SmallSetVector<StringRef, 4> turns;
608 for (StringRef dest :
609 endpoint.getDispatchAttr().getAsValueRange<FlatSymbolRefAttr>())
611 llvm::SmallDenseSet<StringRef, 4> reached;
612 for (
auto flow : device.getOps<RouteOp>()) {
613 if (flow.namesAsDestination(name))
614 return flow.emitOpError(
"names dispatching endpoint @")
615 << name <<
" as a destination";
616 if (!flow.namesAsSource(name))
618 if (flow.getSources().size() != 1 || flow.getDestinations().size() != 1)
619 return flow.emitOpError(
"leaves dispatching endpoint @")
621 <<
" for one destination, so it has one source and "
623 if (!flow.getPacket())
624 return flow.emitOpError(
"leaves dispatching endpoint @")
625 << name <<
" and needs a packet header to tell its turns apart";
626 StringRef dest = *flow.getDestinationNames().begin();
627 if (!turns.contains(dest))
628 return flow.emitOpError(
"reaches @")
629 << dest <<
", which dispatching endpoint @" << name
630 <<
" never takes a turn for";
631 reached.insert(dest);
633 for (StringRef dest : turns)
634 if (!reached.contains(dest))
635 return endpoint.emitOpError(
"dispatches to @")
636 << dest <<
", but no route from it reaches there";
641 int64_t elements = transferElements(endpoint);
642 for (StringRef dest : turns) {
643 auto destEndpoint = symbolTables.lookupSymbolIn<ObjectFifoDmaEndpointOp>(
644 device, StringAttr::get(device.getContext(), dest));
647 int64_t destElements = transferElements(destEndpoint);
648 if (destElements != elements)
649 return endpoint.emitOpError(
"dispatches, so every turn moves ")
650 << elements <<
" elements, but @" << dest <<
" moves "
658 static int64_t transferElements(ObjectFifoDmaEndpointOp endpoint) {
659 int64_t elements = 0;
660 std::optional<ArrayRef<BDDimLayoutArrayAttr>> dimensions =
661 endpoint.getDimensions();
662 for (
auto [index, segment] :
663 llvm::enumerate(endpoint.getSelectedSegments())) {
664 BDDimLayoutArrayAttr dims;
665 if (dimensions && index < dimensions->size()) {
666 dims = (*dimensions)[index];
668 if (dims && !dims.empty()) {
670 for (BDDimLayoutAttr dim : dims) {
671 product *= dim.getSize();
675 elements += segment.getSize();
685 LogicalResult verifyFanIn(DeviceOp device,
686 SymbolTableCollection &symbolTables) {
687 for (
auto flow : device.getOps<RouteOp>()) {
688 if (!flow.isFanIn()) {
691 std::optional<std::pair<StringRef, int64_t>> first;
692 auto check = [&](StringRef name) -> LogicalResult {
693 auto endpoint = symbolTables.lookupSymbolIn<ObjectFifoDmaEndpointOp>(
694 device, StringAttr::get(device.getContext(), name));
698 int64_t elements = transferElements(endpoint);
700 first = {name, elements};
703 if (elements != first->second) {
704 return flow.emitOpError(
"fans in, so every end moves one object of "
705 "the same size, but @")
706 << first->first <<
" moves " << first->second
707 <<
" elements and @" << name <<
" moves " << elements;
711 for (StringRef name : flow.getSourceNames()) {
712 if (failed(check(name))) {
716 for (StringRef name : flow.getDestinationNames()) {
717 if (failed(check(name))) {
730 LogicalResult verifyOverRelease(DeviceOp device) {
731 for (
auto coreOp : device.getOps<CoreOp>()) {
732 WalkResult result = coreOp.walk([&](Operation *loop) {
734 return WalkResult::advance();
736 auto directlyIn = [&](Operation *op) {
739 DenseMap<StringRef, int64_t> acquired;
740 DenseMap<StringRef, int64_t> released;
741 DenseMap<StringRef, Operation *> blame;
742 DenseSet<StringRef> spansNestedLoop;
744 loop->walk([&](ObjectFifoAcquireOp a) {
745 StringRef key = a.getObjFifoName();
746 if (!directlyIn(a)) {
747 spansNestedLoop.insert(key);
749 acquired[key] += a.acqNumber();
750 blame.try_emplace(key, a);
753 loop->walk([&](ObjectFifoReleaseOp r) {
754 StringRef key = r.getObjFifoName();
755 if (!directlyIn(r)) {
756 spansNestedLoop.insert(key);
758 released[key] += r.relNumber();
759 blame.try_emplace(key, r);
763 for (
auto &[key, count] : released) {
764 if (spansNestedLoop.contains(key) || count <= acquired.lookup(key)) {
767 blame.lookup(key)->emitOpError(
768 "cannot release more elements than are already acquired");
769 return WalkResult::interrupt();
771 return WalkResult::advance();
773 if (result.wasInterrupted()) {
790 LogicalResult verifyTransferBudgets(DeviceOp device) {
791 DenseMap<Operation *, TransferBudget> budgets;
792 for (
auto endpoint : device.getOps<ObjectFifoDmaEndpointOp>()) {
793 std::optional<int32_t> iterations = endpoint.getIterCount();
794 ObjectFifoPoolOp pool = endpoint.getPoolOp();
795 if (!iterations || !pool) {
798 budgets[pool] = {endpoint, *iterations,
799 int64_t{*iterations} * pool.getDepth()};
801 if (budgets.empty()) {
805 DenseMap<StringRef, int64_t> released;
806 DenseMap<StringRef, ObjectFifoReleaseOp> blame;
807 DenseSet<StringRef> uncounted;
808 device.walk([&](ObjectFifoReleaseOp release) {
809 StringRef key = release.getObjFifoName();
812 uncounted.insert(key);
815 released[key] += *runs * release.relNumber();
816 blame.try_emplace(key, release);
818 DenseSet<StringRef> acquired;
819 device.walk([&](ObjectFifoAcquireOp acquire) {
820 acquired.insert(acquire.getObjFifoName());
823 for (
auto endpoint : device.getOps<ObjectFifoCoreEndpointOp>()) {
825 auto budget = budgets.find(endpoint.getPoolOp());
826 if (budget == budgets.end() || endpoint.getSegments()) {
829 StringRef name = endpoint.getSymName();
830 int64_t count = released.lookup(name);
831 ObjectFifoDmaEndpointOp dma = budget->second.endpoint;
832 ObjectFifoPoolOp pool = endpoint.getPoolOp();
833 int64_t depth = pool.getDepth();
834 if (count > budget->second.objects) {
835 InFlightDiagnostic diag =
836 blame.lookup(name).emitOpError(
"releases ")
837 << count <<
" objects through @" << name
838 <<
" over the run, but the pool's DMA endpoint stops after "
839 << budget->second.objects <<
" (iterCount "
840 << budget->second.iterations <<
" x depth " << depth <<
")";
842 if (!endpoint.drains() && count <= budget->second.objects + depth)
843 diag <<
", leaving " << count - budget->second.objects
844 <<
" in the pool unsent";
846 diag <<
", so the core would wait forever for the rest";
847 diag.attachNote(dma.getLoc()) <<
"the DMA endpoint is here";
851 if (!acquired.contains(name) || uncounted.contains(name) ||
852 dma.getSegments() || pool.getInitValues() ||
853 dma.drains() == endpoint.drains() || !poolLocks->needsLocks(pool)) {
856 int64_t reachable = endpoint.drains() ? count + depth : count;
857 if (budget->second.objects <= reachable) {
860 InFlightDiagnostic diag =
861 dma.emitOpError(dma.drains() ?
"drains " :
"fills ")
862 << budget->second.objects <<
" objects (iterCount "
863 << budget->second.iterations <<
" x depth " << depth
864 <<
") before it stops, but the core releases exactly " << count
865 <<
" through @" << name;
866 if (endpoint.drains())
867 diag <<
", leaving room for only " << reachable;
868 diag <<
", so the DMA endpoint would wait forever for the rest";
869 diag.attachNote(endpoint.getLoc()) <<
"the core endpoint is here";
881 LogicalResult verifyHeldAtEnd(DeviceOp device,
882 SymbolTableCollection &symbolTables) {
886 llvm::DenseMap<StringRef, SmallVector<RuntimeSequenceOp>> launchesByFifo;
887 std::optional<SmallVector<RuntimeSequenceOp>> allLaunches;
888 auto launchesOf = [&](std::optional<StringRef> fifo)
889 ->
const SmallVector<RuntimeSequenceOp> & {
893 llvm::to_vector(device.getOps<RuntimeSequenceOp>()));
896 auto [it, inserted] = launchesByFifo.try_emplace(*fifo);
898 for (
auto seq : device.getOps<RuntimeSequenceOp>())
900 it->second.push_back(seq);
901 if (it->second.empty())
902 llvm::append_range(it->second, device.getOps<RuntimeSequenceOp>());
906 for (
auto core : device.getOps<CoreOp>()) {
907 Region &body = core.getBody();
908 if (!body.hasOneBlock())
911 core.walk([](scf::WhileOp) {
return WalkResult::interrupt(); })
915 llvm::MapVector<StringRef, ObjectFifoAcquireOp> firstAcquire;
916 core.walk([&](ObjectFifoAcquireOp acquire) {
917 firstAcquire.insert({acquire.getObjFifoName(), acquire});
919 for (
auto &[name, acquire] : firstAcquire) {
920 std::optional<int64_t> held = heldAfter(body.front(), name, 0);
921 if (!held || *held == 0)
924 symbolTables.lookupNearestSymbolFrom<ObjectFifoCoreEndpointOp>(
925 device, acquire.getObjFifoNameAttr());
926 ObjectFifoPoolOp pool = endpoint ? endpoint.getPoolOp() :
nullptr;
927 std::optional<StringRef> fifo =
928 pool ? pool.getFifoName() : std::nullopt;
930 fifo ? poolLocks->rearms().find(*fifo) : poolLocks->rearms().end();
931 bool rearmedEveryLaunch =
932 rearms != poolLocks->rearms().end() &&
933 llvm::all_of(launchesOf(fifo), [&](RuntimeSequenceOp seq) {
934 return llvm::any_of(rearms->second, [&](
auto reset) {
935 return reset->getParentOp() == seq;
938 if (pool && (!poolLocks->needsLocks(pool) || rearmedEveryLaunch))
940 InFlightDiagnostic diag =
941 core.emitOpError(
"ends with ")
942 << *held <<
" acquired from @" << name
943 <<
" never released, so their locks stay taken into the next run "
944 "of this configuration";
945 diag.attachNote(acquire.getLoc()) <<
"acquired here";
960 LogicalResult verifyAdvances(DeviceOp device) {
963 auto repeatsForever = [](Operation *loop) {
964 bool forever =
false;
965 for (Operation *op = loop; !isa<CoreOp>(op); op = op->getParentOp()) {
966 Block *block = op->getBlock();
967 if (block != &op->getParentRegion()->front() ||
969 llvm::make_range(block->begin(), op->getIterator()),
970 [&](Operation &before) { return runsForever(&before); }))
972 auto forOp = dyn_cast<scf::ForOp>(op);
973 auto whileOp = dyn_cast<scf::WhileOp>(op);
974 std::optional<int64_t> trip =
975 forOp ? getStaticTripCount(forOp) : std::nullopt;
976 if ((forOp && (!trip || *trip == 0)) ||
979 forever |= runsForever(op);
980 Operation *parent = op->getParentOp();
981 if (
auto ifOp = dyn_cast<scf::IfOp>(parent)) {
982 bool inThen = op->getParentRegion() == &ifOp.getThenRegion();
983 if (!matchPattern(ifOp.getCondition(), inThen ? m_One() : m_Zero()))
985 }
else if (
auto parentWhile = dyn_cast<scf::WhileOp>(parent)) {
986 if (op->getParentRegion() != &parentWhile.getAfter())
988 }
else if (!isa<scf::ForOp, CoreOp>(parent)) {
995 for (
auto endpoint : device.getOps<ObjectFifoCoreEndpointOp>()) {
996 ObjectFifoPoolOp pool = endpoint.getPoolOp();
997 if (!pool || pool.getInitValues() || endpoint.getSegments() ||
998 !poolLocks->needsLocks(pool))
1000 Operation *keepsMoving =
nullptr;
1001 for (
auto dma : device.getOps<ObjectFifoDmaEndpointOp>())
1002 if (dma.getPoolOp() == pool && dma.drains() != endpoint.drains() &&
1003 !dma.getIterCount() && !dma.getSegments())
1005 for (
auto other : device.getOps<ObjectFifoCoreEndpointOp>()) {
1006 if (keepsMoving || other.getPoolOp() != pool ||
1007 other.drains() == endpoint.drains() || other.getSegments())
1009 CoreTurns otherTurns;
1010 std::string otherReason;
1011 if (succeeded(otherTurns.
read({other}, otherReason)) &&
1013 keepsMoving = other;
1020 if (succeeded(turns.
read({endpoint}, reason)) || !turns.
movesNone() ||
1024 dyn_cast_or_null<ObjectFifoAcquireOp>(turns.
periodAcquire());
1027 InFlightDiagnostic diag =
1028 acquire.emitOpError(
"takes from @")
1029 << endpoint.getSymName()
1030 <<
" each time its loop repeats, but the loop releases none of "
1032 if (endpoint.drains())
1033 diag <<
"the core keeps the same objects forever and the end "
1034 "filling the pool stops once it is full";
1036 diag <<
"nothing the core writes there is ever sent";
1037 diag.attachNote(keepsMoving->getLoc()) <<
"the other end is here";
1046 static std::optional<int64_t> hostBytes(RuntimeSequenceOp sequence,
1048 std::optional<SymbolTable::UseRange> uses =
1049 SymbolTable::getSymbolUses(name, &sequence.getBody());
1051 return std::nullopt;
1052 std::optional<int64_t> bytes = 0;
1053 for (
const SymbolTable::SymbolUse &use : *uses) {
1054 auto task = dyn_cast<AIEX::DMAConfigureTaskForOp>(use.getUser());
1055 std::optional<int64_t> runs =
1057 std::optional<int64_t> streamed =
1058 runs ? task.getStaticBytesStreamed() : std::nullopt;
1060 return std::nullopt;
1061 std::optional<int64_t> moved = llvm::checkedMul(*runs, *streamed);
1062 bytes = moved ? llvm::checkedAdd(*bytes, *moved) : std::nullopt;
1064 return std::nullopt;
1071 static int64_t objectBytes(ObjectFifoDmaEndpointOp endpoint) {
1072 auto type = cast<MemRefType>(endpoint.getPoolOp().getElemType());
1073 return transferElements(endpoint) *
1074 static_cast<int64_t
>(DataLayout::closest(endpoint).getTypeSize(
1075 type.getElementType()));
1081 std::optional<int64_t> hostObjects(DeviceOp device,
1082 RuntimeSequenceOp sequence,
1083 ObjectFifoDmaEndpointOp endpoint,
1085 if (endpoint.getIterCount() || endpoint.getDispatch())
1086 return std::nullopt;
1087 StringRef name = endpoint.getSymName();
1088 SmallVector<StringRef> peers;
1089 for (
auto route : device.getOps<RouteOp>()) {
1090 if (endpoint.drains() ? route.namesAsSource(name)
1091 : route.namesAsDestination(name)) {
1092 auto names = endpoint.drains() ? route.getDestinationNames()
1093 : route.getSourceNames();
1094 peers.append(names.begin(), names.end());
1097 if (peers.size() != 1)
1098 return std::nullopt;
1099 Operation *peer = SymbolTable::lookupNearestSymbolFrom(
1100 device, StringAttr::get(device.getContext(), peers.front()));
1102 int64_t sent = objectBytes(endpoint) * endpoint.getRepeat();
1103 int64_t perObject = endpoint.drains() ? sent : objectBytes(endpoint);
1106 if (
auto shim = dyn_cast_or_null<RouteEndpointOp>(peer)) {
1107 std::optional<int64_t> host = hostBytes(sequence, shim.getSymNameAttr());
1109 return std::nullopt;
1111 }
else if (
auto other = dyn_cast_or_null<ObjectFifoDmaEndpointOp>(peer);
1112 other && throughPool && other.drains() != endpoint.drains()) {
1113 ObjectFifoDmaEndpointOp toward;
1114 for (
auto candidate : device.getOps<ObjectFifoDmaEndpointOp>()) {
1115 if (candidate == other || candidate.getPoolOp() != other.getPoolOp() ||
1116 candidate.drains() == other.drains())
1119 return std::nullopt;
1123 return std::nullopt;
1124 std::optional<int64_t> objects =
1125 hostObjects(device, sequence, toward,
false);
1127 return std::nullopt;
1129 *objects * (other.drains() ? objectBytes(other) * other.getRepeat()
1130 : objectBytes(other));
1132 return std::nullopt;
1134 if (perObject == 0 || bytes % perObject)
1135 return std::nullopt;
1136 return bytes / perObject;
1149 LogicalResult verifyDispatchVolumes(DeviceOp device) {
1150 RuntimeSequenceOp sequence;
1151 for (
auto candidate : device.getOps<RuntimeSequenceOp>()) {
1152 auto ops = candidate.getBody()
1153 .getOps<AIEX::SyncScratchpadParametersFromHostOp>();
1154 int64_t count = std::distance(ops.begin(), ops.end());
1158 BoolAttr preamble = candidate.getEmitParameterSyncPreambleAttr();
1159 count = !preamble || preamble.getValue();
1163 if (sequence || count > 1)
1165 sequence = candidate;
1170 for (
auto endpoint : device.getOps<ObjectFifoCoreEndpointOp>()) {
1171 ObjectFifoPoolOp pool = endpoint.getPoolOp();
1172 if (!pool || pool.getInitValues() || endpoint.getSegments() ||
1173 !poolLocks->needsLocks(pool))
1175 ObjectFifoDmaEndpointOp dma;
1177 for (
auto candidate : device.getOps<ObjectFifoDmaEndpointOp>()) {
1178 if (candidate.getPoolOp() != pool ||
1179 candidate.drains() == endpoint.drains())
1184 if (!dma || !single || dma.getSegments())
1189 if (failed(turns.
read({endpoint}, reason)))
1192 std::optional<int64_t> periods = 1;
1193 for (; periods && !isa<CoreOp>(loop) && !runsForever(loop);
1194 loop = loop->getParentOp()) {
1195 auto forOp = dyn_cast<scf::ForOp>(loop);
1196 std::optional<int64_t> trip =
1197 forOp ? getStaticTripCount(forOp) : std::nullopt;
1198 periods = trip && loop->getBlock() == &loop->getParentRegion()->front()
1199 ? llvm::checkedMul(*periods, *trip)
1202 if (!periods || isa<CoreOp>(loop))
1206 auto whileLoop = dyn_cast<scf::WhileOp>(loop);
1207 Block *body = whileLoop ? whileLoop.getAfterBody()
1208 : cast<scf::ForOp>(loop).getBody();
1210 loop->walk([&](AIEX::WaitScratchpadParametersOp) { waits++; });
1212 body->getOps<AIEX::WaitScratchpadParametersOp>().empty())
1214 std::optional<int64_t> perDispatch = llvm::checkedMul(
1215 *periods,
static_cast<int64_t
>(turns.
turns().size()));
1218 std::optional<int64_t> objects = hostObjects(device, sequence, dma,
true);
1219 LLVM_DEBUG(llvm::dbgs()
1220 <<
"dispatch volume of @" << endpoint.getSymName()
1221 <<
": the core moves " << *perDispatch <<
", @"
1222 << sequence.getSymName() <<
" moves "
1223 << (objects ? std::to_string(*objects) :
"an unknown count")
1225 if (!objects || *objects == *perDispatch)
1228 ObjectFifoAcquireOp acquire;
1229 loop->walk([&](ObjectFifoAcquireOp op) {
1230 if (!acquire && op.getObjFifoName() == endpoint.getSymName())
1235 InFlightDiagnostic diag =
1236 acquire.emitOpError()
1237 << (endpoint.drains() ?
"takes " :
"hands over ") << *perDispatch
1238 <<
" objects through @" << endpoint.getSymName()
1239 <<
" for each dispatch, one pass of a loop that waits for each "
1240 "dispatch's parameters, but each run of runtime sequence @"
1241 << sequence.getSymName() <<
" moves " << *objects
1242 <<
", so the core drifts out of step with the dispatches";
1243 diag.attachNote(sequence.getLoc()) <<
"the runtime sequence is here";
1249 using PoolActors = DenseMap<Operation *, SmallVector<SegmentActors>>;
1259 int64_t inBytes, outBytes;
1260 int64_t lagObject = 0, lagIn = 0, lagOut = 0;
1263 static int64_t segmentBytes(ObjectFifoPoolOp pool,
1264 ArrayRef<ObjectFifoSegmentOp> segments) {
1265 int64_t elements = 0;
1266 for (ObjectFifoSegmentOp segment : segments)
1267 elements += segment.getSize();
1268 auto type = cast<MemRefType>(pool.getElemType());
1270 static_cast<int64_t
>(
1271 DataLayout::closest(pool).getTypeSize(type.getElementType()));
1276 static bool passesOn(ObjectFifoDmaEndpointOp dma) {
1277 return !dma.getPadDimensions() && !dma.getDispatch() &&
1278 dma.getRepeat() == 1;
1291 void carries(DeviceOp device, StringAttr from,
const PoolActors &actors,
1292 Reach scale, llvm::SmallPtrSetImpl<Operation *> &onPath,
1293 SmallVectorImpl<Reach> &reaches) {
1295 for (
auto candidate : device.getOps<RouteOp>()) {
1296 if (!candidate.namesAsSource(from.getValue()))
1302 if (!route || route.getSources().size() != 1 || route.getKeepPktHeader())
1304 for (StringRef name : route.getDestinationNames()) {
1305 if (llvm::any_of(device.getOps<RouteOp>(), [&](RouteOp other) {
1306 return other != route && other.namesAsDestination(name);
1309 Operation *dest = SymbolTable::lookupNearestSymbolFrom(
1310 device, StringAttr::get(device.getContext(), name));
1311 if (
auto shim = dyn_cast_or_null<RouteEndpointOp>(dest)) {
1314 if (shim.getBundle() == WireBundle::DMA &&
1315 shim.getTileLike().isShimTile())
1316 reaches.push_back(at);
1319 auto fill = dyn_cast_or_null<ObjectFifoDmaEndpointOp>(dest);
1320 if (!fill || fill.drains() || !passesOn(fill))
1322 ObjectFifoPoolOp pool = fill.getPoolOp();
1323 if (!pool || pool.getInitValues() || pool.getContiguous() ||
1324 !poolLocks->needsLocks(pool) || onPath.contains(pool))
1326 auto it = actors.find(pool);
1327 if (it == actors.end())
1329 std::vector<ObjectFifoSegmentOp> all = pool.getSegmentOps();
1330 std::vector<ObjectFifoSegmentOp> filled = fill.getSelectedSegments();
1331 auto indexOf = [&](ObjectFifoSegmentOp segment) {
1332 return llvm::find(all, segment) - all.begin();
1334 if (filled.empty() || llvm::any_of(filled, [&](
auto segment) {
1335 return it->second[indexOf(segment)].fillers.size() != 1;
1338 llvm::SetVector<Operation *> drainers;
1339 for (
const SegmentActors &segment : it->second)
1340 drainers.insert(segment.drainers.begin(), segment.drainers.end());
1341 onPath.insert(pool);
1342 for (Operation *drainer : drainers) {
1343 auto core = dyn_cast<ObjectFifoCoreEndpointOp>(drainer);
1344 auto dma = dyn_cast<ObjectFifoDmaEndpointOp>(drainer);
1345 if (dma && !passesOn(dma))
1347 std::vector<ObjectFifoSegmentOp> drained =
1348 core ? core.getSelectedSegments() : dma.getSelectedSegments();
1349 if (drained.empty())
1351 bool split = drained.size() == 1 && drained.front() == filled.back();
1352 bool join = filled.size() == 1 && filled.front() == drained.front();
1357 bool laterJoin = dma && filled.size() == 1 && !join &&
1358 llvm::is_contained(drained, filled.front());
1359 if ((drained != filled && !split && !join && !laterJoin) ||
1360 (laterJoin && scale.lagObject))
1362 int64_t in = objectBytes(fill);
1363 int64_t out = core ? segmentBytes(pool, drained) : objectBytes(dma);
1366 if (in <= 0 || out <= 0 ||
1367 llvm::MulOverflow(scale.inBytes, in, next.inBytes) ||
1368 llvm::MulOverflow(scale.outBytes, out, next.outBytes) ||
1370 (llvm::MulOverflow(scale.lagIn, in, next.lagIn) ||
1371 llvm::MulOverflow(scale.lagOut, out, next.lagOut))))
1373 int64_t common = std::gcd(next.inBytes, next.outBytes);
1374 next.inBytes /= common;
1375 next.outBytes /= common;
1377 next = {next.far, next.inBytes, next.outBytes, out, 1, 1};
1378 if (next.lagObject) {
1379 int64_t lagCommon = std::gcd(next.lagIn, next.lagOut);
1380 next.lagIn /= lagCommon;
1381 next.lagOut /= lagCommon;
1384 reaches.push_back(next);
1386 carries(device, dma.getSymNameAttr(), actors, next, onPath, reaches);
1401 void recordConsumption(DeviceOp device,
const PoolActors &actors) {
1402 MLIRContext *ctx = device.getContext();
1404 int64_t in, out, granule;
1408 std::optional<DenseMap<Operation *, unsigned>> rank;
1409 for (
auto input : device.getOps<RouteEndpointOp>()) {
1410 if (input.getBundle() != WireBundle::DMA ||
1411 !input.getTileLike().isShimTile())
1415 llvm::MapVector<Operation *, Share> proofs;
1416 auto prove = [&](RouteEndpointOp output,
const Reach &reach) {
1417 Share share{reach.inBytes, reach.outBytes, 0};
1418 if (reach.lagObject) {
1421 if (llvm::MulOverflow(reach.lagOut, reach.lagObject, whole))
1423 share.granule = whole / std::gcd(whole, reach.lagIn);
1425 auto [it, inserted] = proofs.try_emplace(output, share);
1426 const Share &old = it->second;
1428 if (inserted || llvm::MulOverflow(share.in, old.out, lhs) ||
1429 llvm::MulOverflow(old.in, share.out, rhs))
1431 if ((old.granule && !share.granule) ||
1432 (!old.granule == !share.granule &&
1433 (lhs > rhs || (lhs == rhs && share.granule < old.granule))))
1436 SmallVector<Reach> reaches;
1437 llvm::SmallPtrSet<Operation *, 8> onPath;
1438 carries(device, input.getSymNameAttr(), actors, {nullptr, 1, 1}, onPath,
1442 constexpr int maxCoreHops = 4;
1443 SmallVector<std::pair<Reach, int>> work;
1444 for (
const Reach &reach : reaches)
1445 work.push_back({reach, 0});
1446 while (!work.empty()) {
1447 auto [reach, hops] = work.pop_back_val();
1448 if (
auto output = dyn_cast<RouteEndpointOp>(reach.far)) {
1449 prove(output, reach);
1452 if (hops == maxCoreHops)
1454 auto consumer = cast<ObjectFifoCoreEndpointOp>(reach.far);
1456 for (Operation *user : consumer.getTile().getUsers())
1457 if (
auto candidate = dyn_cast<CoreOp>(user))
1461 int64_t takenBytes =
1462 segmentBytes(consumer.getPoolOp(), consumer.getSelectedSegments());
1463 for (
auto producer : device.getOps<ObjectFifoCoreEndpointOp>()) {
1464 ObjectFifoPoolOp pool = producer.getPoolOp();
1465 if (producer.drains() || producer.getSegments() ||
1466 producer.getTile() != consumer.getTile() || !pool ||
1467 pool.getInitValues() || !poolLocks->needsLocks(pool))
1469 auto it = actors.find(pool);
1470 if (it == actors.end() || it->second.size() != 1 ||
1471 it->second.front().fillers.size() != 1 ||
1472 it->second.front().drainers.size() != 1)
1474 Operation *drainer = it->second.front().drainers.front();
1475 std::optional<std::pair<int64_t, int64_t>> ratio =
1476 TakeRatio(device, consumer.getSymName(), producer.getSymName())
1485 auto dma = dyn_cast<ObjectFifoDmaEndpointOp>(drainer);
1486 int64_t taken, released;
1487 if ((dma && !passesOn(dma)) ||
1488 llvm::MulOverflow(ratio->first, takenBytes, taken) ||
1491 dma ? objectBytes(dma)
1492 : segmentBytes(pool, producer.getSelectedSegments()),
1497 SmallVector<Reach> nexts;
1499 nexts.push_back({drainer, 1, 1, 0, 0, 0});
1501 llvm::SmallPtrSet<Operation *, 8> outPath;
1502 carries(device, dma.getSymNameAttr(), actors, {nullptr, 1, 1},
1505 for (
const Reach &next : nexts) {
1506 if (reach.lagObject && next.lagObject)
1508 Reach composed = next;
1509 if (llvm::MulOverflow(next.inBytes, taken, composed.inBytes) ||
1510 llvm::MulOverflow(composed.inBytes, reach.inBytes,
1511 composed.inBytes) ||
1512 llvm::MulOverflow(next.outBytes, released, composed.outBytes) ||
1513 llvm::MulOverflow(composed.outBytes, reach.outBytes,
1516 int64_t common = std::gcd(composed.inBytes, composed.outBytes);
1517 composed.inBytes /= common;
1518 composed.outBytes /= common;
1521 if (reach.lagObject) {
1522 composed.lagObject = reach.lagObject;
1523 if (llvm::MulOverflow(reach.lagIn, taken, composed.lagIn) ||
1524 llvm::MulOverflow(composed.lagIn, next.inBytes,
1526 llvm::MulOverflow(reach.lagOut, released, composed.lagOut) ||
1527 llvm::MulOverflow(composed.lagOut, next.outBytes,
1530 int64_t lagCommon = std::gcd(composed.lagIn, composed.lagOut);
1531 composed.lagIn /= lagCommon;
1532 composed.lagOut /= lagCommon;
1534 work.push_back({composed, hops + 1});
1538 if (proofs.empty()) {
1539 input.removeConsumedByAttr();
1542 auto ordered = proofs.takeVector();
1543 if (ordered.size() > 1) {
1546 llvm::stable_sort(ordered, [&](
const auto &a,
const auto &b) {
1547 return rank->lookup(a.first) < rank->lookup(b.first);
1550 SmallVector<Attribute> attrs;
1551 for (
auto [output, share] : ordered) {
1552 int64_t common = std::gcd(share.in, share.out);
1553 attrs.push_back(ConsumedByAttr::get(
1555 FlatSymbolRefAttr::get(
1556 cast<RouteEndpointOp>(output).getSymNameAttr()),
1557 share.in / common, share.out / common, share.granule,
1558 FlatSymbolRefAttr()));
1560 input.setConsumedByAttr(ArrayAttr::get(ctx, attrs));
1568 LogicalResult verifyOutOfOrderPools(DeviceOp device) {
1569 auto lookup = [&](FlatSymbolRefAttr name) {
1570 return dyn_cast_or_null<ObjectFifoDmaEndpointOp>(
1571 SymbolTable::lookupNearestSymbolFrom(device, name.getAttr()));
1573 auto objectBytes = [](ObjectFifoPoolOp pool) -> int64_t {
1574 auto type = dyn_cast<MemRefType>(pool.getElemType());
1577 return (type.getNumElements() * type.getElementTypeBitWidth() + 7) / 8;
1579 for (
auto flow : device.getOps<RouteOp>()) {
1580 if (flow.getSources().size() != 1)
1582 ObjectFifoDmaEndpointOp src =
1583 lookup(cast<FlatSymbolRefAttr>(flow.getSources()[0]));
1584 ObjectFifoDmaEndpointOp firstReceiver;
1585 for (
auto destName :
1586 flow.getDestinations().getAsRange<FlatSymbolRefAttr>()) {
1587 ObjectFifoDmaEndpointOp dst = lookup(destName);
1588 if (!dst || !dst.getOutOfOrder())
1591 firstReceiver = dst;
1592 if (firstReceiver.getNumDescriptors() != dst.getNumDescriptors() ||
1593 objectBytes(firstReceiver.getPoolOp()) !=
1594 objectBytes(dst.getPoolOp()))
1595 return flow.emitOpError(
1596 "out-of-order broadcast destinations require matching buffer "
1597 "counts and object sizes");
1598 if (!src || !src.getOutOfOrder())
1600 ObjectFifoPoolOp from = src.getPoolOp(), to = dst.getPoolOp();
1603 if (src.getNumDescriptors() != dst.getNumDescriptors() ||
1604 objectBytes(from) != objectBytes(to))
1605 return src.emitOpError(
"sends out of order from ")
1606 << src.getNumDescriptors() <<
" buffers of "
1607 << objectBytes(from) <<
" bytes into "
1608 << dst.getNumDescriptors() <<
" of " << objectBytes(to)
1609 <<
"; an out-of-order fifo needs the same buffers at both "
1610 "ends, because one descriptor names one whole buffer";
1616 void runOnOperation()
override {
1617 DeviceOp device = getOperation();
1618 poolLocks = std::make_shared<PoolLocks>(device);
1620 DenseMap<Operation *, SmallVector<SegmentActors>> actorsPerPool;
1621 for (
auto pool : device.getOps<ObjectFifoPoolOp>()) {
1622 actorsPerPool[pool].resize(pool.getSegmentOps().size());
1625 auto record = [&](
auto endpoint) {
1626 ObjectFifoPoolOp pool = endpoint.getPoolOp();
1630 auto &actors = actorsPerPool[pool];
1631 std::vector<ObjectFifoSegmentOp> all = pool.getSegmentOps();
1632 for (ObjectFifoSegmentOp segment : endpoint.getSelectedSegments()) {
1633 size_t index = llvm::find(all, segment) - all.begin();
1634 (endpoint.drains() ? actors[index].drainers : actors[index].fillers)
1635 .push_back(endpoint);
1639 for (
auto endpoint : device.getOps<ObjectFifoCoreEndpointOp>()) {
1642 for (
auto endpoint : device.getOps<ObjectFifoDmaEndpointOp>()) {
1646 for (
auto pool : device.getOps<ObjectFifoPoolOp>()) {
1647 if (failed(verifyActors(pool, actorsPerPool[pool]))) {
1648 return signalPassFailure();
1652 SymbolTableCollection symbolTables;
1653 if (failed(verifyFlows(device, symbolTables)) ||
1654 failed(verifyFanIn(device, symbolTables)) ||
1655 failed(verifyOverRelease(device)) ||
1656 failed(verifyTransferBudgets(device)) ||
1657 failed(verifyHeldAtEnd(device, symbolTables)) ||
1658 failed(verifyAdvances(device)) ||
1659 failed(verifyDispatchVolumes(device)) ||
1660 failed(verifyOutOfOrderPools(device))) {
1661 return signalPassFailure();
1663 recordConsumption(device, actorsPerPool);
1669std::unique_ptr<OperationPass<DeviceOp>>
1671 return std::make_unique<AIEObjectFifoVerifyPass>();
mlir::LogicalResult read(llvm::ArrayRef< ObjectFifoCoreEndpointOp > ends, std::string &reason)
Reads the order off the program of the core working through ends, all draining (the channel fills the...
bool movesNone() const
Whether read found a steady state in which the period moves none of the objects: the core keeps what ...
mlir::Operation * periodAcquire() const
The first acquire the period runs, with its branches folded; null when it runs none.
llvm::ArrayRef< unsigned > turns() const
Indices into ends, one per object the channel moves in a period of the core's steady state.
mlir::Operation * periodLoop() const
The loop read took as the core's period, once it found one.
bool namesFifo(Operation *op, StringRef fifo, SymbolTableCollection &symbols)
std::optional< int64_t > staticExecutions(Operation *op)
std::unique_ptr< mlir::OperationPass< DeviceOp > > createAIEObjectFifoVerifyPass()
bool isForeverLoop(mlir::scf::WhileOp whileOp)
Whether whileOp repeats forever: its condition is a constant true, and its condition region does noth...
mlir::Operation * getObjectFifoLoopParent(mlir::Operation *op)
The innermost loop around op that isObjectFifoLoop accepts, or null.
bool isObjectFifoLoop(mlir::Operation *op)
Whether op is a loop that objectFifo lowering unrolls by the rotation period of the objectFifos acces...
llvm::DenseMap< mlir::Operation *, unsigned > rank() const
Rank the nodes by their features and then, a round at a time until no class splits,...
static StructuralGraph of(DeviceOp device)
The ops directly in device but its runtime sequences, each with the ops nested in it,...