11#include "mlir/IR/Attributes.h"
12#include "mlir/Pass/Pass.h"
13#include "llvm/Support/MathExtras.h"
22#define GEN_PASS_DEF_AIEOBJECTFIFOLOWERDMAS
23#include "aie/Dialect/AIE/Transforms/AIEPasses.h.inc"
34 BDDimLayoutArrayAttr dimensions;
35 BDPadLayoutArrayAttr padding;
36 FlatSymbolRefAttr acquireLock;
37 FlatSymbolRefAttr releaseLock;
42 std::optional<int64_t> acquireValue = std::nullopt;
43 std::optional<int64_t> releaseValue = std::nullopt;
46 BDIterationAttr iteration =
nullptr;
49Block *findEndOpBlock(Region ®ion) {
50 Block *endBlock =
nullptr;
51 for (Block &block : region) {
52 if (!block.getOps<EndOp>().empty()) {
59struct AIEObjectFifoLowerDMAsPass
60 :
public xilinx::AIE::impl::AIEObjectFifoLowerDMAsBase<
61 AIEObjectFifoLowerDMAsPass> {
64 int32_t nextSendTag = 0;
65 llvm::DenseSet<int32_t> usedSendTags;
66 OpBuilder builder{
static_cast<MLIRContext *
>(
nullptr)};
70 DmaBody dmaProgramFor(TileLike tile, Location loc) {
71 Value wanted = tile->getResult(0);
72 for (
auto program : device.getOps<DmaBody>()) {
73 if (program.getTile() == wanted) {
75 FusedLoc::get(loc.getContext(), {program->getLoc(), loc}));
80 OpBuilder::InsertionGuard g(builder);
81 builder.setInsertionPoint(device.getBody()->getTerminator());
83 if (tile.isShimTile()) {
84 program = ShimDMAOp::create(builder, loc, builder.getIndexType(), wanted);
85 }
else if (tile.isMemTile()) {
86 program = MemTileDMAOp::create(builder, loc, wanted);
88 program = MemOp::create(builder, loc, wanted);
91 builder.setInsertionPointToStart(&program.getDmaBody().emplaceBlock());
92 EndOp::create(builder, loc);
97 SmallVector<Value> buffersOf(ObjectFifoPoolOp pool) {
98 SmallVector<Value> buffers;
99 for (BufferLike buffer : pool.getBufferOps()) {
100 buffers.push_back(buffer.getBuffer());
109 SmallVector<Descriptor>
110 contiguousDescriptors(ObjectFifoDmaEndpointOp endpoint, Value buffer,
112 ObjectFifoPoolOp pool = endpoint.getPoolOp();
113 ObjectFifoSegmentOp segment = endpoint.getSelectedSegments().front();
114 FlatSymbolRefAttr take =
115 drains ? segment.getConsumeLockAttr() : segment.getProduceLockAttr();
116 FlatSymbolRefAttr give =
117 drains ? segment.getProduceLockAttr() : segment.getConsumeLockAttr();
118 MLIRContext *context = builder.getContext();
119 if (!endpoint.walksStream()) {
120 BDDimLayoutArrayAttr dims;
121 if (
auto all = endpoint.getDimensions(); all && !all->empty())
123 int64_t
object = pool.getObjectSize();
125 if (
auto zeros = pool.getZeroObjects())
126 first = (*zeros)[0] * object;
127 int64_t walkers = endpoint.getNumDescriptors();
128 SmallVector<Descriptor> descriptors;
129 for (int64_t j = 0; j < walkers; j++)
130 descriptors.push_back(
131 {buffer, first + j * object + segment.getOffset(),
132 segment.getSize(), dims, BDPadLayoutArrayAttr(), take, give, j,
133 std::nullopt, std::nullopt,
134 BDIterationAttr::get(context, pool.getDepth() / walkers,
135 walkers * object, 0)});
140 if (failed(endpoint.getStreamWindow(window)))
142 BDDimLayoutArrayAttr dims =
143 BDDimLayoutArrayAttr::get(context, window.
inner);
144 SmallVector<Descriptor> descriptors;
146 descriptors.push_back({buffer, step.first * window.
objectSize,
147 window.
transfer(), dims, BDPadLayoutArrayAttr(),
148 take, give, 0, step.acquire, step.release,
155 SmallVector<Descriptor> descriptorsFor(ObjectFifoDmaEndpointOp endpoint,
156 ArrayRef<Value> buffers,
bool drains) {
157 if (endpoint.getPoolOp().getContiguous())
158 return contiguousDescriptors(endpoint, buffers.front(), drains);
159 std::vector<ObjectFifoSegmentOp> segments = endpoint.getSelectedSegments();
160 SmallVector<BDDimLayoutArrayAttr> dimensions;
161 if (
auto allDimensions = endpoint.getDimensions()) {
162 llvm::append_range(dimensions, *allDimensions);
164 SmallVector<BDPadLayoutArrayAttr> padding;
165 if (
auto allPadding = endpoint.getPadDimensions()) {
166 llvm::append_range(padding, *allPadding);
169 SmallVector<Descriptor> descriptors;
170 for (
auto [index, buffer] : llvm::enumerate(buffers)) {
171 for (
auto [position, segment] : llvm::enumerate(segments)) {
172 descriptors.push_back(
173 {buffer, segment.getOffset(), segment.getSize(),
174 position < dimensions.size() ? dimensions[position]
175 : BDDimLayoutArrayAttr(),
176 position < padding.size() ? padding[position]
177 : BDPadLayoutArrayAttr(),
178 drains ? segment.getConsumeLockAttr()
179 : segment.getProduceLockAttr(),
180 drains ? segment.getProduceLockAttr()
181 : segment.getConsumeLockAttr(),
182 static_cast<int64_t
>(index)});
188 void emitDescriptor(ObjectFifoDmaEndpointOp endpoint, ObjectFifoPoolOp pool,
189 Descriptor &descriptor, PacketInfoAttr packet,
190 Block *successor,
bool binaryLocks,
bool drains,
191 int64_t transmittedObject = -1,
192 ArrayRef<int32_t> sendTags = {}) {
193 Location loc = endpoint.getLoc();
194 int count = drains ? 1 : pool.getRepeatCount().value_or(1);
196 LockOp acquireLock, releaseLock;
197 int acquireValue = count, releaseValue = count;
198 auto action = LockAction::AcquireGreaterEqual;
201 SmallVector<LockOp> locks = pool.getLockOps();
202 if (descriptor.object < (int64_t)locks.size()) {
203 acquireLock = releaseLock = locks[descriptor.object];
204 acquireValue = drains ? 1 : 0;
205 releaseValue = drains ? 0 : 1;
206 action = LockAction::Acquire;
208 }
else if (descriptor.acquireLock) {
215 if (descriptor.acquireValue)
216 acquireValue = *descriptor.acquireValue;
217 if (descriptor.releaseValue)
218 releaseValue = *descriptor.releaseValue;
220 UseLockOp::create(builder, loc, acquireLock, action, acquireValue);
223 DMABDPACKETOp::create(builder, loc, packet.getPktType(),
224 packet.assignedId());
228 if (descriptor.dimensions && drains && descriptor.padding) {
229 bd = DMABDOp::create(builder, loc, descriptor.buffer, descriptor.offset,
230 paddedSize(descriptor.dimensions, descriptor.padding,
232 descriptor.dimensions, descriptor.padding);
233 }
else if (descriptor.dimensions) {
236 ArrayRef<BDDimLayoutAttr> dims = descriptor.dimensions.getValue();
237 size_t maxDims = device.getTargetModel().getBDMaxDims(
238 endpoint.getTileLike().getTileType());
239 while (dims.size() > maxDims && dims.front().getSize() == 1)
240 dims = dims.drop_front();
241 bd = DMABDOp::create(
242 builder, loc, descriptor.buffer, descriptor.offset, descriptor.size,
243 BDDimLayoutArrayAttr::get(builder.getContext(), dims));
245 bd = DMABDOp::create(builder, loc, descriptor.buffer, descriptor.offset,
248 if (endpoint.getOutOfOrder()) {
250 drains ? endpoint.getOutOfOrderTags() : pool.getOutOfOrderTags();
252 assert(descriptor.object <
static_cast<int64_t
>(tags->size()) &&
253 "the allocator names every descriptor of an out-of-order pool");
254 int64_t slot = transmittedObject >= 0 && drains
255 ? transmittedObject % tags->size()
257 int32_t tag = (*tags)[slot];
258 if (endpoint.getRouteDirection() == DMAChannelDir::MM2S)
259 bd.setOutOfOrderTag(tag);
264 if (descriptor.iteration && descriptor.iteration.getSize() > 1)
265 bd.setIterationAttr(descriptor.iteration);
266 if (!sendTags.empty()) {
267 assert(descriptor.object <
static_cast<int64_t
>(sendTags.size()) &&
268 "a send tag names every buffer of a core-pushed pool");
269 bd.setBdTag(sendTags[descriptor.object]);
273 UseLockOp::create(builder, loc, releaseLock, LockAction::Release,
276 NextBDOp::create(builder, loc, successor);
281 std::pair<Block *, Block *> startChain(ObjectFifoDmaEndpointOp endpoint,
282 int channel,
int taskCount,
283 bool corePushes =
false) {
284 Location loc = endpoint.getLoc();
285 DmaBody program = dmaProgramFor(endpoint.getTileLike(), loc);
286 Block *endBlock = findEndOpBlock(program.getDmaBody());
287 Block *lastDmaBlock = endBlock->getSinglePredecessor();
288 Block *dmaBlock = builder.createBlock(endBlock);
289 Block *bdBlock = builder.createBlock(endBlock);
291 builder.setInsertionPointToStart(dmaBlock);
292 DMAStartOp::create(builder, loc, endpoint.getRouteDirection(), channel,
293 taskCount, endpoint.getPadValue(),
294 endpoint.getOutOfOrder() &&
295 endpoint.getRouteDirection() == DMAChannelDir::S2MM,
296 corePushes, bdBlock, endBlock);
298 lastDmaBlock->getTerminator()->setSuccessor(dmaBlock, 1);
300 return {bdBlock, endBlock};
305 LogicalResult lowerTurns(ObjectFifoDmaEndpointOp first,
int channel) {
306 SmallVector<ObjectFifoDmaEndpointOp> ends =
turnEndpoints(first);
307 SmallVector<SmallVector<Descriptor>> descriptors;
308 for (ObjectFifoDmaEndpointOp end : ends) {
309 descriptors.push_back(
310 descriptorsFor(end, buffersOf(end.getPoolOp()), end.drains()));
311 if (
static_cast<int64_t
>(descriptors.back().size()) !=
312 end.getNumDescriptors() ||
313 descriptors.back().empty())
314 return end.emitOpError(
"takes turns, but lowers to ")
315 << descriptors.back().size() <<
" descriptors per pass for "
316 << end.getNumDescriptors();
318 SmallVector<unsigned> turns;
319 for (
auto name : first.getTurnsAttr().getAsRange<FlatSymbolRefAttr>())
320 turns.push_back(llvm::find_if(ends,
321 [&](ObjectFifoDmaEndpointOp end) {
322 return end.getSymNameAttr() ==
327 auto [bdBlock, endBlock] = startChain(first, channel, 0);
328 bool binaryLocks = !device.getTargetModel().hasProperty(
330 SmallVector<size_t> next(ends.size(), 0);
331 int64_t total = first.getNumBDs();
332 Block *current = bdBlock;
333 for (int64_t k = 0; k < total; k++) {
334 unsigned i = turns[k % turns.size()];
336 k + 1 < total ? builder.createBlock(endBlock) : bdBlock;
337 builder.setInsertionPointToStart(current);
338 emitDescriptor(ends[i], ends[i].getPoolOp(), descriptors[i][next[i]],
339 PacketInfoAttr(), successor, binaryLocks,
341 next[i] = (next[i] + 1) % descriptors[i].size();
347 LogicalResult lowerEndpoint(ObjectFifoDmaEndpointOp endpoint,
int channel) {
348 bool drains = endpoint.drains();
349 ObjectFifoPoolOp pool = endpoint.getPoolOp();
350 SmallVector<Value> buffers = buffersOf(pool);
351 if (buffers.empty()) {
355 SmallVector<Descriptor> descriptors =
356 descriptorsFor(endpoint, buffers, drains);
357 if (descriptors.empty()) {
361 if (
static_cast<int64_t
>(descriptors.size()) !=
362 endpoint.getNumDescriptors())
363 return endpoint.emitOpError()
364 <<
"lowers to " << descriptors.size()
365 <<
" descriptors per pass, but its channel was allocated for "
366 << endpoint.getNumDescriptors();
368 Location loc = endpoint.getLoc();
370 int64_t repeat = endpoint.getRepeat();
371 std::optional<int32_t> iterCount = endpoint.getIterCount();
373 bool repeatInHardware = endpoint.repeatsInHardware();
376 int64_t passes = endpoint.getDispatchPasses();
377 int taskCount = iterCount ? *iterCount / passes - 1
378 : (repeatInHardware ? repeat - 1 : 0);
379 int64_t copies = repeatInHardware ? 1 : repeat;
386 drains && endpoint.getReleaseStrategy() == ObjectFifoRelease::PushQueue;
387 SmallVector<int32_t> sendTags;
389 bool hasRelease =
false;
390 ObjectFifoCoreEndpointOp producer;
391 auto sentSegments = endpoint.getSelectedSegments();
392 for (
auto coreEnd : device.getOps<ObjectFifoCoreEndpointOp>()) {
393 if (coreEnd.drains() || coreEnd.getPoolOp() != pool ||
394 coreEnd.getTile() != endpoint.getTile())
396 auto filledSegments = coreEnd.getSelectedSegments();
397 if (!llvm::any_of(sentSegments, [&](ObjectFifoSegmentOp segment) {
398 return llvm::is_contained(filledSegments, segment);
402 return endpoint.emitOpError(
403 "push_queue requires a single core filler for its segment");
405 for (
auto core : device.getOps<CoreOp>()) {
406 if (core.getTile() == coreEnd.getTile())
407 core.walk([&](ObjectFifoReleaseOp release) {
408 hasRelease |= release.getObjFifoName() == coreEnd.getSymName();
413 return endpoint.emitOpError(
414 "push_queue requires producer releases in a "
415 "local core to submit send tasks");
417 return endpoint.emitOpError(
418 "push_queue requires one dispatch pass per object");
421 if (
static_cast<int64_t
>(descriptors.size()) != pool.getDepth())
422 return endpoint.emitOpError(
"sends ")
423 << descriptors.size() <<
" descriptors for a pool of "
425 <<
" objects, so a release names no one of them; push_queue "
426 <<
"needs one descriptor per object";
427 for (
size_t slot = 0; slot < descriptors.size(); ++slot) {
428 while (usedSendTags.contains(nextSendTag))
430 sendTags.push_back(nextSendTag++);
432 producer.setCorePushAttr(
433 ObjectFifoCorePushAttr::get(pool.getContext(), channel, sendTags));
436 SmallVector<PacketInfoAttr> turns;
437 if (std::optional<ArrayAttr> headers = endpoint.getDispatchPackets()) {
438 llvm::append_range(turns, headers->getAsRange<PacketInfoAttr>());
439 }
else if (endpoint.getDispatch()) {
440 return endpoint.emitOpError(
441 "has no dispatchPackets; run --aie-objectfifo-allocate");
445 if (!turns.empty() && pool.getContiguous())
446 return endpoint.emitOpError(
447 "dispatches from a contiguous pool, which has no per-object BDs");
448 int64_t depth = pool.getDepth();
450 auto [bdBlock, endBlock] =
451 startChain(endpoint, channel, taskCount, corePushes);
452 bool binaryLocks = !device.getTargetModel().hasProperty(
454 size_t total = descriptors.size() * copies * passes;
456 Block *current = bdBlock;
457 for (int64_t pass = 0; pass < passes; pass++) {
458 for (Descriptor &descriptor : descriptors) {
459 PacketInfoAttr packet = endpoint.getPacketAttr();
461 packet = turns[(pass * depth + descriptor.object) % turns.size()];
462 for (int64_t copy = 0; copy < copies; copy++) {
464 if (emitted + 1 < total) {
465 successor = builder.createBlock(endBlock);
466 }
else if (iterCount) {
468 successor = builder.createBlock(endBlock);
469 builder.setInsertionPointToStart(successor);
470 EndOp::create(builder, loc);
474 builder.setInsertionPointToStart(current);
475 emitDescriptor(endpoint, pool, descriptor, packet, successor,
476 binaryLocks, drains, emitted, sendTags);
486 lowerOutOfOrderReceivers(ArrayRef<ObjectFifoDmaEndpointOp> endpoints,
488 auto first = endpoints.front();
489 SmallVector<SmallVector<Descriptor>> descriptors;
491 bool unbounded =
false;
493 for (
auto endpoint : endpoints) {
494 descriptors.push_back(
495 descriptorsFor(endpoint, buffersOf(endpoint.getPoolOp()),
false));
496 if (descriptors.back().empty() ||
497 static_cast<int64_t
>(descriptors.back().size()) !=
498 endpoint.getNumDescriptors())
499 return endpoint.emitOpError(
500 "incorrect number of receiving descriptors");
501 totalBDs += descriptors.back().size();
502 if (
auto iterations = endpoint.getIterCount())
503 objects += descriptors.back().size() * *iterations;
507 int64_t perTask = device.getTargetModel().getMaxRepeatCount() + 1;
510 int64_t tasks = llvm::divideCeil(objects, perTask);
511 if (tasks > device.getTargetModel().getDmaTaskQueueDepth())
512 return first.emitOpError(
"shared receiving channel needs ")
513 << tasks <<
" initial tasks for " << objects
514 <<
" objects, exceeding its queue capacity; use an unbounded "
515 "receiver with core replenishment";
517 auto [head, endBlock] =
518 startChain(first, channel, std::min(objects, perTask) - 1);
519 for (objects -= perTask; objects > 0; objects -= perTask) {
520 auto [unusedHead, unusedEnd] =
521 startChain(first, channel, std::min(objects, perTask) - 1);
522 unusedHead->replaceAllUsesWith(head);
525 bool binaryLocks = !device.getTargetModel().hasProperty(
527 Block *current = head;
529 for (
auto [index, bds] : llvm::enumerate(descriptors)) {
530 ObjectFifoDmaEndpointOp endpoint = endpoints[index];
531 for (
auto &descriptor : bds) {
532 Block *successor = builder.createBlock(endBlock);
533 if (++emitted == totalBDs) {
534 builder.setInsertionPointToStart(successor);
535 EndOp::create(builder, endpoint.getLoc());
537 builder.setInsertionPointToStart(current);
538 emitDescriptor(endpoint, endpoint.getPoolOp(), descriptor,
539 endpoint.getPacketAttr(), successor, binaryLocks,
false);
546 void runOnOperation()
override {
547 device = getOperation();
548 builder = OpBuilder(device.getContext());
552 usedSendTags.clear();
553 for (
auto pool : device.getOps<ObjectFifoPoolOp>())
554 if (
auto tags = pool.getOutOfOrderTags())
555 usedSendTags.insert(tags->begin(), tags->end());
556 for (
auto endpoint : device.getOps<ObjectFifoCoreEndpointOp>())
557 if (
auto push = endpoint.getCorePushAttr())
558 usedSendTags.insert(push.getBdTags().begin(), push.getBdTags().end());
559 device.walk([&](DMABDOp bd) {
560 if (
auto tag = bd.getBdTag())
561 usedSendTags.insert(*tag);
564 SmallVector<ObjectFifoDmaEndpointOp> endpoints(
565 device.getOps<ObjectFifoDmaEndpointOp>());
566 DenseMap<Operation *, ObjectFifoDmaEndpointOp> pushedDrainers;
567 for (ObjectFifoDmaEndpointOp endpoint : endpoints) {
568 if (!endpoint.drains() ||
569 endpoint.getReleaseStrategy() != ObjectFifoRelease::PushQueue)
571 auto [it, inserted] =
572 pushedDrainers.try_emplace(endpoint.getPoolOp(), endpoint);
574 auto diag = endpoint.emitOpError(
575 "a pool supports only one core-pushed DMA drainer");
576 diag.attachNote(it->second.getLoc())
577 <<
"first core-pushed drainer is here";
578 return signalPassFailure();
583 DenseMap<Value, size_t> tileOrder;
584 for (ObjectFifoDmaEndpointOp endpoint : endpoints)
585 tileOrder.try_emplace(endpoint.getTile(), tileOrder.size());
587 endpoints, [&](ObjectFifoDmaEndpointOp a, ObjectFifoDmaEndpointOp b) {
588 return std::make_tuple(tileOrder.lookup(a.getTile()),
589 a.getRouteDirection(),
590 a.getChannelIndex().value_or(-1)) <
591 std::make_tuple(tileOrder.lookup(b.getTile()),
592 b.getRouteDirection(),
593 b.getChannelIndex().value_or(-1));
595 using ChannelKey = std::tuple<int, int, int>;
596 std::map<ChannelKey, SmallVector<ObjectFifoDmaEndpointOp>> receivers;
597 for (
auto endpoint : endpoints) {
598 if (endpoint.getOutOfOrder() && !endpoint.drains()) {
599 auto tile = endpoint.getTile().getDefiningOp<TileOp>();
600 receivers[{tile.getCol(), tile.getRow(),
601 endpoint.getChannelIndex().value_or(-1)}]
602 .push_back(endpoint);
605 for (ObjectFifoDmaEndpointOp endpoint : endpoints) {
606 std::optional<int> channel = endpoint.getChannelIndex();
608 endpoint.emitOpError(
"has no channel; run --aie-objectfifo-allocate");
609 return signalPassFailure();
611 if (endpoint.getOutOfOrder() && !endpoint.drains()) {
612 auto tile = endpoint.getTile().getDefiningOp<TileOp>();
613 auto &group = receivers.at({tile.getCol(), tile.getRow(), *channel});
614 if (group.front() == endpoint &&
615 failed(lowerOutOfOrderReceivers(group, *channel)))
616 return signalPassFailure();
619 SmallVector<ObjectFifoDmaEndpointOp> turns =
turnEndpoints(endpoint);
620 if (turns.empty() ? failed(lowerEndpoint(endpoint, *channel))
621 : turns.front() == endpoint &&
622 failed(lowerTurns(endpoint, *channel)))
623 return signalPassFailure();
625 for (ObjectFifoDmaEndpointOp endpoint : endpoints)
631 llvm::make_early_inc_range(device.getOps<RouteEndpointOp>())) {
639std::unique_ptr<OperationPass<DeviceOp>>
641 return std::make_unique<AIEObjectFifoLowerDMAsPass>();
llvm::SmallVector< ObjectFifoDmaEndpointOp > turnEndpoints(ObjectFifoDmaEndpointOp end)
llvm::SmallVector< StreamWindowStep > streamWindowSteps(const StreamWindow &w)
mlir::Operation * lookupNamedOp(mlir::Operation *from, mlir::StringAttr name)
std::unique_ptr< mlir::OperationPass< DeviceOp > > createAIEObjectFifoLowerDMAsPass()
int64_t paddedSize(llvm::ArrayRef< BDDimLayoutAttr > dims, llvm::ArrayRef< BDPadLayoutAttr > pads, int64_t fallback)
llvm::SmallVector< BDDimLayoutAttr > inner