MLIR-AIE 1.4.6.dev
AIEObjectFifoLowerDMAs.cpp
Go to the documentation of this file.
1//===- AIEObjectFifoLowerDMAs.cpp -------------------------------*- C++ -*-===//
2//
3// Copyright (C) 2026 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7
10
11#include "mlir/IR/Attributes.h"
12#include "mlir/Pass/Pass.h"
13#include "llvm/Support/MathExtras.h"
14
15#include <map>
16
17using namespace mlir;
18using namespace xilinx;
19using namespace xilinx::AIE;
20
21namespace xilinx::AIE {
22#define GEN_PASS_DEF_AIEOBJECTFIFOLOWERDMAS
23#include "aie/Dialect/AIE/Transforms/AIEPasses.h.inc"
24} // namespace xilinx::AIE
25
26namespace {
27
28/// One buffer descriptor's worth of work: which object, which slice of it, and
29/// the locks guarding that slice.
30struct Descriptor {
31 Value buffer;
32 int64_t offset;
33 int64_t size;
34 BDDimLayoutArrayAttr dimensions;
35 BDPadLayoutArrayAttr padding;
36 FlatSymbolRefAttr acquireLock;
37 FlatSymbolRefAttr releaseLock;
38 /// Which of the pool's objects this moves; also the index of the object's
39 /// own binary lock, where the device uses those.
40 int64_t object;
41 /// Lock units taken and given back, where they are not one object's.
42 std::optional<int64_t> acquireValue = std::nullopt;
43 std::optional<int64_t> releaseValue = std::nullopt;
44 /// How the address steps on each run of a descriptor that walks a
45 /// contiguous pool's objects in turn.
46 BDIterationAttr iteration = nullptr;
47};
48
49Block *findEndOpBlock(Region &region) {
50 Block *endBlock = nullptr;
51 for (Block &block : region) {
52 if (!block.getOps<EndOp>().empty()) {
53 endBlock = &block;
54 }
55 }
56 return endBlock;
57}
58
59struct AIEObjectFifoLowerDMAsPass
60 : public xilinx::AIE::impl::AIEObjectFifoLowerDMAsBase<
61 AIEObjectFifoLowerDMAsPass> {
62
63 DeviceOp device;
64 int32_t nextSendTag = 0;
65 llvm::DenseSet<int32_t> usedSendTags;
66 OpBuilder builder{static_cast<MLIRContext *>(nullptr)};
67
68 /// The DMA program of `tile`, created empty if the tile has none yet. A
69 /// program shared by several endpoints is located at all of them.
70 DmaBody dmaProgramFor(TileLike tile, Location loc) {
71 Value wanted = tile->getResult(0);
72 for (auto program : device.getOps<DmaBody>()) {
73 if (program.getTile() == wanted) {
74 program->setLoc(
75 FusedLoc::get(loc.getContext(), {program->getLoc(), loc}));
76 return program;
77 }
78 }
79
80 OpBuilder::InsertionGuard g(builder);
81 builder.setInsertionPoint(device.getBody()->getTerminator());
82 DmaBody program;
83 if (tile.isShimTile()) {
84 program = ShimDMAOp::create(builder, loc, builder.getIndexType(), wanted);
85 } else if (tile.isMemTile()) {
86 program = MemTileDMAOp::create(builder, loc, wanted);
87 } else {
88 program = MemOp::create(builder, loc, wanted);
89 }
90
91 builder.setInsertionPointToStart(&program.getDmaBody().emplaceBlock());
92 EndOp::create(builder, loc);
93 return program;
94 }
95
96 /// Objects the endpoint moves.
97 SmallVector<Value> buffersOf(ObjectFifoPoolOp pool) {
98 SmallVector<Value> buffers;
99 for (BufferLike buffer : pool.getBufferOps()) {
100 buffers.push_back(buffer.getBuffer());
101 }
102 return buffers;
103 }
104
105 /// A contiguous pool's one buffer holds its objects side by side. Its
106 /// filling end steps through them with one descriptor; its draining end
107 /// walks the stream with one descriptor per output, each taking the objects
108 /// up to the newest it reads and giving back those no later one reads.
109 SmallVector<Descriptor>
110 contiguousDescriptors(ObjectFifoDmaEndpointOp endpoint, Value buffer,
111 bool drains) {
112 ObjectFifoPoolOp pool = endpoint.getPoolOp();
113 ObjectFifoSegmentOp segment = endpoint.getSelectedSegments().front();
114 FlatSymbolRefAttr take =
115 drains ? segment.getConsumeLockAttr() : segment.getProduceLockAttr();
116 FlatSymbolRefAttr give =
117 drains ? segment.getProduceLockAttr() : segment.getConsumeLockAttr();
118 MLIRContext *context = builder.getContext();
119 if (!endpoint.walksStream()) {
120 BDDimLayoutArrayAttr dims;
121 if (auto all = endpoint.getDimensions(); all && !all->empty())
122 dims = (*all)[0];
123 int64_t object = pool.getObjectSize();
124 int64_t first = 0;
125 if (auto zeros = pool.getZeroObjects())
126 first = (*zeros)[0] * object;
127 int64_t walkers = endpoint.getNumDescriptors();
128 SmallVector<Descriptor> descriptors;
129 for (int64_t j = 0; j < walkers; j++)
130 descriptors.push_back(
131 {buffer, first + j * object + segment.getOffset(),
132 segment.getSize(), dims, BDPadLayoutArrayAttr(), take, give, j,
133 std::nullopt, std::nullopt,
134 BDIterationAttr::get(context, pool.getDepth() / walkers,
135 walkers * object, 0)});
136 return descriptors;
137 }
138
139 StreamWindow window;
140 if (failed(endpoint.getStreamWindow(window)))
141 return {};
142 BDDimLayoutArrayAttr dims =
143 BDDimLayoutArrayAttr::get(context, window.inner);
144 SmallVector<Descriptor> descriptors;
145 for (StreamWindowStep step : streamWindowSteps(window))
146 descriptors.push_back({buffer, step.first * window.objectSize,
147 window.transfer(), dims, BDPadLayoutArrayAttr(),
148 take, give, 0, step.acquire, step.release,
149 nullptr});
150 return descriptors;
151 }
152
153 /// Buffer-major, segment-minor: the DMA walks the objects in turn, and within
154 /// each object the slices this endpoint is responsible for.
155 SmallVector<Descriptor> descriptorsFor(ObjectFifoDmaEndpointOp endpoint,
156 ArrayRef<Value> buffers, bool drains) {
157 if (endpoint.getPoolOp().getContiguous())
158 return contiguousDescriptors(endpoint, buffers.front(), drains);
159 std::vector<ObjectFifoSegmentOp> segments = endpoint.getSelectedSegments();
160 SmallVector<BDDimLayoutArrayAttr> dimensions;
161 if (auto allDimensions = endpoint.getDimensions()) {
162 llvm::append_range(dimensions, *allDimensions);
163 }
164 SmallVector<BDPadLayoutArrayAttr> padding;
165 if (auto allPadding = endpoint.getPadDimensions()) {
166 llvm::append_range(padding, *allPadding);
167 }
168
169 SmallVector<Descriptor> descriptors;
170 for (auto [index, buffer] : llvm::enumerate(buffers)) {
171 for (auto [position, segment] : llvm::enumerate(segments)) {
172 descriptors.push_back(
173 {buffer, segment.getOffset(), segment.getSize(),
174 position < dimensions.size() ? dimensions[position]
175 : BDDimLayoutArrayAttr(),
176 position < padding.size() ? padding[position]
177 : BDPadLayoutArrayAttr(),
178 drains ? segment.getConsumeLockAttr()
179 : segment.getProduceLockAttr(),
180 drains ? segment.getProduceLockAttr()
181 : segment.getConsumeLockAttr(),
182 static_cast<int64_t>(index)});
183 }
184 }
185 return descriptors;
186 }
187
188 void emitDescriptor(ObjectFifoDmaEndpointOp endpoint, ObjectFifoPoolOp pool,
189 Descriptor &descriptor, PacketInfoAttr packet,
190 Block *successor, bool binaryLocks, bool drains,
191 int64_t transmittedObject = -1,
192 ArrayRef<int32_t> sendTags = {}) {
193 Location loc = endpoint.getLoc();
194 int count = drains ? 1 : pool.getRepeatCount().value_or(1);
195
196 LockOp acquireLock, releaseLock;
197 int acquireValue = count, releaseValue = count;
198 auto action = LockAction::AcquireGreaterEqual;
199 if (binaryLocks) {
200 // A binary lock's value encodes which side owns the buffer.
201 SmallVector<LockOp> locks = pool.getLockOps();
202 if (descriptor.object < (int64_t)locks.size()) {
203 acquireLock = releaseLock = locks[descriptor.object];
204 acquireValue = drains ? 1 : 0;
205 releaseValue = drains ? 0 : 1;
206 action = LockAction::Acquire;
207 }
208 } else if (descriptor.acquireLock) {
209 acquireLock =
210 lookupNamedOp<LockOp>(device, descriptor.acquireLock.getAttr());
211 releaseLock =
212 lookupNamedOp<LockOp>(device, descriptor.releaseLock.getAttr());
213 }
214
215 if (descriptor.acquireValue)
216 acquireValue = *descriptor.acquireValue;
217 if (descriptor.releaseValue)
218 releaseValue = *descriptor.releaseValue;
219 if (acquireLock) {
220 UseLockOp::create(builder, loc, acquireLock, action, acquireValue);
221 }
222 if (packet) {
223 DMABDPACKETOp::create(builder, loc, packet.getPktType(),
224 packet.assignedId());
225 }
226
227 DMABDOp bd;
228 if (descriptor.dimensions && drains && descriptor.padding) {
229 bd = DMABDOp::create(builder, loc, descriptor.buffer, descriptor.offset,
230 paddedSize(descriptor.dimensions, descriptor.padding,
231 descriptor.size),
232 descriptor.dimensions, descriptor.padding);
233 } else if (descriptor.dimensions) {
234 // An outer dimension of size 1 reads one position, so it is dropped
235 // where the tile's DMA steps through fewer dimensions.
236 ArrayRef<BDDimLayoutAttr> dims = descriptor.dimensions.getValue();
237 size_t maxDims = device.getTargetModel().getBDMaxDims(
238 endpoint.getTileLike().getTileType());
239 while (dims.size() > maxDims && dims.front().getSize() == 1)
240 dims = dims.drop_front();
241 bd = DMABDOp::create(
242 builder, loc, descriptor.buffer, descriptor.offset, descriptor.size,
243 BDDimLayoutArrayAttr::get(builder.getContext(), dims));
244 } else {
245 bd = DMABDOp::create(builder, loc, descriptor.buffer, descriptor.offset,
246 descriptor.size);
247 }
248 if (endpoint.getOutOfOrder()) {
249 auto tags =
250 drains ? endpoint.getOutOfOrderTags() : pool.getOutOfOrderTags();
251 if (tags) {
252 assert(descriptor.object < static_cast<int64_t>(tags->size()) &&
253 "the allocator names every descriptor of an out-of-order pool");
254 int64_t slot = transmittedObject >= 0 && drains
255 ? transmittedObject % tags->size()
256 : descriptor.object;
257 int32_t tag = (*tags)[slot];
258 if (endpoint.getRouteDirection() == DMAChannelDir::MM2S)
259 bd.setOutOfOrderTag(tag);
260 else
261 bd.setBdTag(tag);
262 }
263 }
264 if (descriptor.iteration && descriptor.iteration.getSize() > 1)
265 bd.setIterationAttr(descriptor.iteration);
266 if (!sendTags.empty()) {
267 assert(descriptor.object < static_cast<int64_t>(sendTags.size()) &&
268 "a send tag names every buffer of a core-pushed pool");
269 bd.setBdTag(sendTags[descriptor.object]);
270 }
271
272 if (releaseLock) {
273 UseLockOp::create(builder, loc, releaseLock, LockAction::Release,
274 releaseValue);
275 }
276 NextBDOp::create(builder, loc, successor);
277 }
278
279 /// A `dma_start` of `channel` on the endpoint's tile, chained after the
280 /// tile's others; returns the first BD's block and the program's end block.
281 std::pair<Block *, Block *> startChain(ObjectFifoDmaEndpointOp endpoint,
282 int channel, int taskCount,
283 bool corePushes = false) {
284 Location loc = endpoint.getLoc();
285 DmaBody program = dmaProgramFor(endpoint.getTileLike(), loc);
286 Block *endBlock = findEndOpBlock(program.getDmaBody());
287 Block *lastDmaBlock = endBlock->getSinglePredecessor();
288 Block *dmaBlock = builder.createBlock(endBlock);
289 Block *bdBlock = builder.createBlock(endBlock);
290
291 builder.setInsertionPointToStart(dmaBlock);
292 DMAStartOp::create(builder, loc, endpoint.getRouteDirection(), channel,
293 taskCount, endpoint.getPadValue(),
294 endpoint.getOutOfOrder() &&
295 endpoint.getRouteDirection() == DMAChannelDir::S2MM,
296 corePushes, bdBlock, endBlock);
297 if (lastDmaBlock) {
298 lastDmaBlock->getTerminator()->setSuccessor(dmaBlock, 1);
299 }
300 return {bdBlock, endBlock};
301 }
302
303 /// Endpoints taking turns on one channel share one chain: the k-th BD moves
304 /// the next object of the endpoint `turns` names k-th, through its own pool.
305 LogicalResult lowerTurns(ObjectFifoDmaEndpointOp first, int channel) {
306 SmallVector<ObjectFifoDmaEndpointOp> ends = turnEndpoints(first);
307 SmallVector<SmallVector<Descriptor>> descriptors;
308 for (ObjectFifoDmaEndpointOp end : ends) {
309 descriptors.push_back(
310 descriptorsFor(end, buffersOf(end.getPoolOp()), end.drains()));
311 if (static_cast<int64_t>(descriptors.back().size()) !=
312 end.getNumDescriptors() ||
313 descriptors.back().empty())
314 return end.emitOpError("takes turns, but lowers to ")
315 << descriptors.back().size() << " descriptors per pass for "
316 << end.getNumDescriptors();
317 }
318 SmallVector<unsigned> turns;
319 for (auto name : first.getTurnsAttr().getAsRange<FlatSymbolRefAttr>())
320 turns.push_back(llvm::find_if(ends,
321 [&](ObjectFifoDmaEndpointOp end) {
322 return end.getSymNameAttr() ==
323 name.getAttr();
324 }) -
325 ends.begin());
326
327 auto [bdBlock, endBlock] = startChain(first, channel, /*taskCount=*/0);
328 bool binaryLocks = !device.getTargetModel().hasProperty(
330 SmallVector<size_t> next(ends.size(), 0);
331 int64_t total = first.getNumBDs();
332 Block *current = bdBlock;
333 for (int64_t k = 0; k < total; k++) {
334 unsigned i = turns[k % turns.size()];
335 Block *successor =
336 k + 1 < total ? builder.createBlock(endBlock) : bdBlock;
337 builder.setInsertionPointToStart(current);
338 emitDescriptor(ends[i], ends[i].getPoolOp(), descriptors[i][next[i]],
339 PacketInfoAttr(), successor, binaryLocks,
340 ends[i].drains());
341 next[i] = (next[i] + 1) % descriptors[i].size();
342 current = successor;
343 }
344 return success();
345 }
346
347 LogicalResult lowerEndpoint(ObjectFifoDmaEndpointOp endpoint, int channel) {
348 bool drains = endpoint.drains();
349 ObjectFifoPoolOp pool = endpoint.getPoolOp();
350 SmallVector<Value> buffers = buffersOf(pool);
351 if (buffers.empty()) {
352 return success();
353 }
354
355 SmallVector<Descriptor> descriptors =
356 descriptorsFor(endpoint, buffers, drains);
357 if (descriptors.empty()) {
358 return success();
359 }
360 // Allocation budgeted the channel's BDs by getNumBDs.
361 if (static_cast<int64_t>(descriptors.size()) !=
362 endpoint.getNumDescriptors())
363 return endpoint.emitOpError()
364 << "lowers to " << descriptors.size()
365 << " descriptors per pass, but its channel was allocated for "
366 << endpoint.getNumDescriptors();
367
368 Location loc = endpoint.getLoc();
369 // emitDescriptor scales the filling end's lock count instead.
370 int64_t repeat = endpoint.getRepeat();
371 std::optional<int32_t> iterCount = endpoint.getIterCount();
372
373 bool repeatInHardware = endpoint.repeatsInHardware();
374 // A dispatching chain runs until its turns line up with the pool's
375 // objects again, so one run covers that many passes.
376 int64_t passes = endpoint.getDispatchPasses();
377 int taskCount = iterCount ? *iterCount / passes - 1
378 : (repeatInHardware ? repeat - 1 : 0);
379 int64_t copies = repeatInHardware ? 1 : repeat;
380
381 // With push_queue the configuration pushes nothing: the core pushes
382 // the descriptor of each object it releases. Each descriptor then runs
383 // on its own, so a pool whose buffers do not map one-to-one onto them
384 // leaves the core no descriptor to name.
385 bool corePushes =
386 drains && endpoint.getReleaseStrategy() == ObjectFifoRelease::PushQueue;
387 SmallVector<int32_t> sendTags;
388 if (corePushes) {
389 bool hasRelease = false;
390 ObjectFifoCoreEndpointOp producer;
391 auto sentSegments = endpoint.getSelectedSegments();
392 for (auto coreEnd : device.getOps<ObjectFifoCoreEndpointOp>()) {
393 if (coreEnd.drains() || coreEnd.getPoolOp() != pool ||
394 coreEnd.getTile() != endpoint.getTile())
395 continue;
396 auto filledSegments = coreEnd.getSelectedSegments();
397 if (!llvm::any_of(sentSegments, [&](ObjectFifoSegmentOp segment) {
398 return llvm::is_contained(filledSegments, segment);
399 }))
400 continue;
401 if (producer)
402 return endpoint.emitOpError(
403 "push_queue requires a single core filler for its segment");
404 producer = coreEnd;
405 for (auto core : device.getOps<CoreOp>()) {
406 if (core.getTile() == coreEnd.getTile())
407 core.walk([&](ObjectFifoReleaseOp release) {
408 hasRelease |= release.getObjFifoName() == coreEnd.getSymName();
409 });
410 }
411 }
412 if (!hasRelease)
413 return endpoint.emitOpError(
414 "push_queue requires producer releases in a "
415 "local core to submit send tasks");
416 if (passes != 1)
417 return endpoint.emitOpError(
418 "push_queue requires one dispatch pass per object");
419 // Each release submits one task, with repetitions carried by that task.
420 taskCount = 0;
421 if (static_cast<int64_t>(descriptors.size()) != pool.getDepth())
422 return endpoint.emitOpError("sends ")
423 << descriptors.size() << " descriptors for a pool of "
424 << pool.getDepth()
425 << " objects, so a release names no one of them; push_queue "
426 << "needs one descriptor per object";
427 for (size_t slot = 0; slot < descriptors.size(); ++slot) {
428 while (usedSendTags.contains(nextSendTag))
429 ++nextSendTag;
430 sendTags.push_back(nextSendTag++);
431 }
432 producer.setCorePushAttr(
433 ObjectFifoCorePushAttr::get(pool.getContext(), channel, sendTags));
434 }
435
436 SmallVector<PacketInfoAttr> turns;
437 if (std::optional<ArrayAttr> headers = endpoint.getDispatchPackets()) {
438 llvm::append_range(turns, headers->getAsRange<PacketInfoAttr>());
439 } else if (endpoint.getDispatch()) {
440 return endpoint.emitOpError(
441 "has no dispatchPackets; run --aie-objectfifo-allocate");
442 }
443 // A contiguous pool's descriptor steps through every object, so it has no
444 // one object for a turn to follow.
445 if (!turns.empty() && pool.getContiguous())
446 return endpoint.emitOpError(
447 "dispatches from a contiguous pool, which has no per-object BDs");
448 int64_t depth = pool.getDepth();
449
450 auto [bdBlock, endBlock] =
451 startChain(endpoint, channel, taskCount, corePushes);
452 bool binaryLocks = !device.getTargetModel().hasProperty(
454 size_t total = descriptors.size() * copies * passes;
455 size_t emitted = 0;
456 Block *current = bdBlock;
457 for (int64_t pass = 0; pass < passes; pass++) {
458 for (Descriptor &descriptor : descriptors) {
459 PacketInfoAttr packet = endpoint.getPacketAttr();
460 if (!turns.empty())
461 packet = turns[(pass * depth + descriptor.object) % turns.size()];
462 for (int64_t copy = 0; copy < copies; copy++) {
463 Block *successor;
464 if (emitted + 1 < total) {
465 successor = builder.createBlock(endBlock);
466 } else if (iterCount) {
467 // A bounded chain exits after its final iteration.
468 successor = builder.createBlock(endBlock);
469 builder.setInsertionPointToStart(successor);
470 EndOp::create(builder, loc);
471 } else {
472 successor = bdBlock;
473 }
474 builder.setInsertionPointToStart(current);
475 emitDescriptor(endpoint, pool, descriptor, packet, successor,
476 binaryLocks, drains, emitted, sendTags);
477 current = successor;
478 emitted++;
479 }
480 }
481 }
482 return success();
483 }
484
485 LogicalResult
486 lowerOutOfOrderReceivers(ArrayRef<ObjectFifoDmaEndpointOp> endpoints,
487 int channel) {
488 auto first = endpoints.front();
489 SmallVector<SmallVector<Descriptor>> descriptors;
490 int64_t objects = 0;
491 bool unbounded = false;
492 size_t totalBDs = 0;
493 for (auto endpoint : endpoints) {
494 descriptors.push_back(
495 descriptorsFor(endpoint, buffersOf(endpoint.getPoolOp()), false));
496 if (descriptors.back().empty() ||
497 static_cast<int64_t>(descriptors.back().size()) !=
498 endpoint.getNumDescriptors())
499 return endpoint.emitOpError(
500 "incorrect number of receiving descriptors");
501 totalBDs += descriptors.back().size();
502 if (auto iterations = endpoint.getIterCount())
503 objects += descriptors.back().size() * *iterations;
504 else
505 unbounded = true;
506 }
507 int64_t perTask = device.getTargetModel().getMaxRepeatCount() + 1;
508 if (unbounded)
509 objects = perTask;
510 int64_t tasks = llvm::divideCeil(objects, perTask);
511 if (tasks > device.getTargetModel().getDmaTaskQueueDepth())
512 return first.emitOpError("shared receiving channel needs ")
513 << tasks << " initial tasks for " << objects
514 << " objects, exceeding its queue capacity; use an unbounded "
515 "receiver with core replenishment";
516
517 auto [head, endBlock] =
518 startChain(first, channel, std::min(objects, perTask) - 1);
519 for (objects -= perTask; objects > 0; objects -= perTask) {
520 auto [unusedHead, unusedEnd] =
521 startChain(first, channel, std::min(objects, perTask) - 1);
522 unusedHead->replaceAllUsesWith(head);
523 unusedHead->erase();
524 }
525 bool binaryLocks = !device.getTargetModel().hasProperty(
527 Block *current = head;
528 size_t emitted = 0;
529 for (auto [index, bds] : llvm::enumerate(descriptors)) {
530 ObjectFifoDmaEndpointOp endpoint = endpoints[index];
531 for (auto &descriptor : bds) {
532 Block *successor = builder.createBlock(endBlock);
533 if (++emitted == totalBDs) {
534 builder.setInsertionPointToStart(successor);
535 EndOp::create(builder, endpoint.getLoc());
536 }
537 builder.setInsertionPointToStart(current);
538 emitDescriptor(endpoint, endpoint.getPoolOp(), descriptor,
539 endpoint.getPacketAttr(), successor, binaryLocks, false);
540 current = successor;
541 }
542 }
543 return success();
544 }
545
546 void runOnOperation() override {
547 device = getOperation();
548 builder = OpBuilder(device.getContext());
549
550 // Reserve tags from both pool allocation and hand-written descriptors.
551 nextSendTag = 0;
552 usedSendTags.clear();
553 for (auto pool : device.getOps<ObjectFifoPoolOp>())
554 if (auto tags = pool.getOutOfOrderTags())
555 usedSendTags.insert(tags->begin(), tags->end());
556 for (auto endpoint : device.getOps<ObjectFifoCoreEndpointOp>())
557 if (auto push = endpoint.getCorePushAttr())
558 usedSendTags.insert(push.getBdTags().begin(), push.getBdTags().end());
559 device.walk([&](DMABDOp bd) {
560 if (auto tag = bd.getBdTag())
561 usedSendTags.insert(*tag);
562 });
563
564 SmallVector<ObjectFifoDmaEndpointOp> endpoints(
565 device.getOps<ObjectFifoDmaEndpointOp>());
566 DenseMap<Operation *, ObjectFifoDmaEndpointOp> pushedDrainers;
567 for (ObjectFifoDmaEndpointOp endpoint : endpoints) {
568 if (!endpoint.drains() ||
569 endpoint.getReleaseStrategy() != ObjectFifoRelease::PushQueue)
570 continue;
571 auto [it, inserted] =
572 pushedDrainers.try_emplace(endpoint.getPoolOp(), endpoint);
573 if (!inserted) {
574 auto diag = endpoint.emitOpError(
575 "a pool supports only one core-pushed DMA drainer");
576 diag.attachNote(it->second.getLoc())
577 << "first core-pushed drainer is here";
578 return signalPassFailure();
579 }
580 }
581 // A tile's chains, and so the BD ids they get, follow its channels rather
582 // than the order the endpoints were written in.
583 DenseMap<Value, size_t> tileOrder;
584 for (ObjectFifoDmaEndpointOp endpoint : endpoints)
585 tileOrder.try_emplace(endpoint.getTile(), tileOrder.size());
586 llvm::stable_sort(
587 endpoints, [&](ObjectFifoDmaEndpointOp a, ObjectFifoDmaEndpointOp b) {
588 return std::make_tuple(tileOrder.lookup(a.getTile()),
589 a.getRouteDirection(),
590 a.getChannelIndex().value_or(-1)) <
591 std::make_tuple(tileOrder.lookup(b.getTile()),
592 b.getRouteDirection(),
593 b.getChannelIndex().value_or(-1));
594 });
595 using ChannelKey = std::tuple<int, int, int>;
596 std::map<ChannelKey, SmallVector<ObjectFifoDmaEndpointOp>> receivers;
597 for (auto endpoint : endpoints) {
598 if (endpoint.getOutOfOrder() && !endpoint.drains()) {
599 auto tile = endpoint.getTile().getDefiningOp<TileOp>();
600 receivers[{tile.getCol(), tile.getRow(),
601 endpoint.getChannelIndex().value_or(-1)}]
602 .push_back(endpoint);
603 }
604 }
605 for (ObjectFifoDmaEndpointOp endpoint : endpoints) {
606 std::optional<int> channel = endpoint.getChannelIndex();
607 if (!channel) {
608 endpoint.emitOpError("has no channel; run --aie-objectfifo-allocate");
609 return signalPassFailure();
610 }
611 if (endpoint.getOutOfOrder() && !endpoint.drains()) {
612 auto tile = endpoint.getTile().getDefiningOp<TileOp>();
613 auto &group = receivers.at({tile.getCol(), tile.getRow(), *channel});
614 if (group.front() == endpoint &&
615 failed(lowerOutOfOrderReceivers(group, *channel)))
616 return signalPassFailure();
617 continue;
618 }
619 SmallVector<ObjectFifoDmaEndpointOp> turns = turnEndpoints(endpoint);
620 if (turns.empty() ? failed(lowerEndpoint(endpoint, *channel))
621 : turns.front() == endpoint &&
622 failed(lowerTurns(endpoint, *channel)))
623 return signalPassFailure();
624 }
625 for (ObjectFifoDmaEndpointOp endpoint : endpoints)
626 endpoint.erase();
627
628 // Nothing to program at the far end of these; the routes they asked for
629 // have been drawn.
630 for (auto endpoint :
631 llvm::make_early_inc_range(device.getOps<RouteEndpointOp>())) {
632 endpoint.erase();
633 }
634 }
635};
636
637} // namespace
638
639std::unique_ptr<OperationPass<DeviceOp>>
641 return std::make_unique<AIEObjectFifoLowerDMAsPass>();
642}
llvm::SmallVector< ObjectFifoDmaEndpointOp > turnEndpoints(ObjectFifoDmaEndpointOp end)
llvm::SmallVector< StreamWindowStep > streamWindowSteps(const StreamWindow &w)
mlir::Operation * lookupNamedOp(mlir::Operation *from, mlir::StringAttr name)
std::unique_ptr< mlir::OperationPass< DeviceOp > > createAIEObjectFifoLowerDMAsPass()
int64_t paddedSize(llvm::ArrayRef< BDDimLayoutAttr > dims, llvm::ArrayRef< BDPadLayoutAttr > pads, int64_t fallback)
llvm::SmallVector< BDDimLayoutAttr > inner
Definition AIEDialect.h:414