MLIR-AIE 1.4.6.dev
AIEObjectFifoSplit.cpp
Go to the documentation of this file.
1//===- AIEObjectFifoSplit.cpp -----------------------------------*- C++ -*-===//
2//
3// Copyright (C) 2026 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7
10
11#include "mlir/IR/Attributes.h"
12#include "mlir/Pass/Pass.h"
13#include "llvm/ADT/DenseMap.h"
14
15namespace xilinx::AIE {
16#define GEN_PASS_DEF_AIEOBJECTFIFOSPLIT
17#include "aie/Dialect/AIE/Transforms/AIEPasses.h.inc"
18} // namespace xilinx::AIE
19
20using namespace mlir;
21using namespace xilinx;
22using namespace xilinx::AIE;
23
24namespace {
25
26/// Where the buffers of a fifo end live, and which of a shared pool's segments
27/// that end occupies.
28struct PoolRef {
29 ObjectFifoPoolOp pool;
30 SmallVector<int32_t> segments;
31};
32
33struct AIEObjectFifoSplitPass
34 : public xilinx::AIE::impl::AIEObjectFifoSplitBase<AIEObjectFifoSplitPass> {
35
36 DeviceOp device;
37 OpBuilder builder{static_cast<MLIRContext *>(nullptr)};
38
39 /// Fill `pool`'s body with one segment per extent, named for its position.
40 void createSegments(ObjectFifoPoolOp pool, Location loc,
41 ArrayRef<std::pair<int64_t, int64_t>> extents) {
42 OpBuilder::InsertionGuard guard(builder);
43 builder.createBlock(&pool.getSegments());
44 for (auto [index, extent] : llvm::enumerate(extents)) {
45 ObjectFifoSegmentOp::create(
46 builder, loc, builder.getStringAttr("s" + std::to_string(index)),
47 builder.getI32IntegerAttr(extent.first),
48 builder.getI32IntegerAttr(extent.second),
49 /*produceLock=*/FlatSymbolRefAttr(),
50 /*consumeLock=*/FlatSymbolRefAttr());
51 }
52 }
53
54 /// The names `ref` selects, empty on a single-segment pool.
55 ArrayAttr selectedSegmentNames(PoolRef ref) {
56 std::vector<ObjectFifoSegmentOp> all = ref.pool.getSegmentOps();
57 if (all.size() <= 1) {
58 return {};
59 }
60 SmallVector<Attribute> names;
61 for (int32_t index : ref.segments) {
62 names.push_back(FlatSymbolRefAttr::get(builder.getContext(),
63 all[index].getSymName()));
64 }
65 return builder.getArrayAttr(names);
66 }
67
68 ObjectFifoPoolOp createPool(Location loc, StringRef name, Value tile,
69 int depth, MemRefType elemType,
70 ObjectFifoCreateOp from,
71 ArrayRef<std::pair<int64_t, int64_t>> extents,
72 bool holdsInitialContents,
73 std::optional<int> repeatCount,
74 bool disableSynchronization = false) {
75 auto pool = ObjectFifoPoolOp::create(
76 builder, loc, name, tile, depth, elemType, /*buffers=*/ArrayAttr(),
77 /*locks=*/ArrayAttr(),
78 repeatCount ? builder.getI32IntegerAttr(*repeatCount) : IntegerAttr(),
79 disableSynchronization || from.getDisableSynchronization(),
80 builder.getStringAttr(from.name().getValue()),
81 holdsInitialContents ? from.getInitValuesAttr() : ArrayAttr(),
82 /*contiguous=*/false, /*zeroObjects=*/nullptr,
83 /*out_of_order_tags=*/DenseI32ArrayAttr());
84 createSegments(pool, loc, extents);
85 return pool;
86 }
87
88 ObjectFifoCoreEndpointOp createCoreEndpoint(Location loc, StringRef name,
89 Value tile, PoolRef ref,
90 ObjectFifoRole role) {
91 return ObjectFifoCoreEndpointOp::create(
92 builder, loc, name, tile, role, ref.pool.getSymName(),
93 selectedSegmentNames(ref),
94 /*core_push=*/ObjectFifoCorePushAttr());
95 }
96
97 ObjectFifoDmaEndpointOp createDmaEndpoint(Location loc, StringRef name,
98 Value tile, PoolRef ref,
99 ObjectFifoRole role,
100 ObjectFifoCreateOp from,
101 BDDimLayoutArrayAttr dims,
102 std::optional<int> channelIndex) {
103 // Both ends of the fifo carry `depth * repeat_count * iter_count` objects.
104 // The draining end spends `repeat_count` on descriptors, replaying each
105 // object in one pass of its chain; the filling end spends it on passes.
106 std::optional<int32_t> iterCount;
107 if (std::optional<int32_t> fifoIterations = from.getIterCount()) {
108 iterCount = role == ObjectFifoRole::Drain
109 ? *fifoIterations
110 : *fifoIterations * from.getRepeatCount().value_or(1);
111 }
112 ArrayAttr segments = selectedSegmentNames(ref);
113
114 BDDimLayoutArrayArrayAttr dimensions;
115 if (dims && !dims.empty()) {
116 SmallVector<BDDimLayoutArrayAttr> perSegment(ref.segments.size(), dims);
117 dimensions =
118 BDDimLayoutArrayArrayAttr::get(builder.getContext(), perSegment);
119 }
120
121 BDPadLayoutArrayArrayAttr padding;
122 std::optional<ArrayRef<BDPadLayoutAttr>> padDims = from.getPadDimensions();
123 if (role == ObjectFifoRole::Drain && padDims && !padDims->empty()) {
124 SmallVector<BDPadLayoutArrayAttr> perSegment(ref.segments.size(),
125 from.getPadDimensionsAttr());
126 padding =
127 BDPadLayoutArrayArrayAttr::get(builder.getContext(), perSegment);
128 }
129
130 return ObjectFifoDmaEndpointOp::create(
131 builder, loc, builder.getStringAttr(name), tile,
132 ObjectFifoRoleAttr::get(builder.getContext(), role),
133 FlatSymbolRefAttr::get(builder.getContext(), ref.pool.getSymName()),
134 segments,
135 channelIndex ? builder.getI32IntegerAttr(*channelIndex) : IntegerAttr(),
136 dimensions, padding,
137 from.getPadValue() ? builder.getI32IntegerAttr(from.getPadValue())
138 : IntegerAttr(),
139 iterCount ? builder.getI32IntegerAttr(*iterCount) : IntegerAttr(),
140 /*packet=*/PacketInfoAttr(),
141 builder.getStringAttr(from.name().getValue()), /*dispatch=*/ArrayAttr(),
142 /*dispatchPackets=*/ArrayAttr(), /*turns=*/ArrayAttr(),
143 from.getOutOfOrder() ? builder.getUnitAttr() : UnitAttr(),
144 /*out_of_order_tags=*/DenseI32ArrayAttr(),
145 role == ObjectFifoRole::Drain ? from.getReleaseStrategyAttr()
146 : ObjectFifoReleaseAttr());
147 }
148
149 /// An end with no pool behind it: a shim whose transfers the runtime issues,
150 /// a PLIO boundary, or a core's raw stream port.
151 RouteEndpointOp createRouteEndpoint(Location loc, StringRef name, Value tile,
152 WireBundle bundle,
153 std::optional<int> channelIndex,
154 ObjectFifoCreateOp from) {
155 auto endpoint = RouteEndpointOp::create(
156 builder, loc, builder.getStringAttr(name), tile,
157 WireBundleAttr::get(builder.getContext(), bundle),
158 channelIndex ? builder.getI32IntegerAttr(*channelIndex) : IntegerAttr(),
159 /*packet=*/PacketInfoAttr(),
160 builder.getStringAttr(from.name().getValue()),
161 /*consumed_by=*/ArrayAttr());
162 if (bundle == WireBundle::DMA &&
163 cast<TileOp>(tile.getDefiningOp()).isShimTile()) {
164 AIEDialect::ShareableWithAttrHelper sharing(builder.getContext());
165 if (auto allowed = sharing.getAttr(from))
166 sharing.setAttr(endpoint, allowed);
167 }
168 return endpoint;
169 }
170
171 /// External buffers registered against a fifo's shim end form a pool in DDR.
172 /// An unregistered end receives its address from the runtime at dispatch.
173 std::optional<PoolRef> registeredPool(ObjectFifoCreateOp fifo, Value tile,
174 StringRef name, MemRefType elemType) {
175 SmallVector<Attribute> names;
176 for (auto regOp : device.getOps<ObjectFifoRegisterExternalBuffersOp>()) {
177 if (regOp.getTile() != tile || regOp.getObjectFifo() != fifo) {
178 continue;
179 }
180 for (Value buffer : regOp.getExternalBuffers()) {
181 auto external = cast<ExternalBufferOp>(buffer.getDefiningOp());
182 names.push_back(SymbolRefAttr::get(external.getSymNameAttr()));
183 // The registered buffer's own extent is what the DMA moves, which need
184 // not be the fifo's object size.
185 elemType = cast<MemRefType>(external.getType());
186 }
187 }
188 if (names.empty()) {
189 return std::nullopt;
190 }
191
192 auto pool = ObjectFifoPoolOp::create(
193 builder, fifo.getLoc(), name, tile, (int)names.size(), elemType,
194 builder.getArrayAttr(names),
195 /*locks=*/ArrayAttr(), /*repeatCount=*/IntegerAttr(),
196 fifo.getDisableSynchronization(),
197 builder.getStringAttr(fifo.name().getValue()),
198 /*initValues=*/ArrayAttr(), /*contiguous=*/false,
199 /*zeroObjects=*/nullptr,
200 /*out_of_order_tags=*/DenseI32ArrayAttr());
201 createSegments(pool, fifo.getLoc(), {{0, elemType.getNumElements()}});
202 return PoolRef{pool, {0}};
203 }
204
205 /// Point a core's accesses at the endpoint it works through.
206 void retargetCoreAccesses(ObjectFifoCreateOp fifo, ObjectFifoPort port,
207 Value tile, StringRef endpointName) {
208 auto name = FlatSymbolRefAttr::get(builder.getContext(), endpointName);
209 // Each access moves to the endpoint for the port it named, so this cannot
210 // be a blanket replaceAllSymbolUses: the two ports of one fifo become two
211 // different symbols.
212 for (auto coreOp : device.getOps<CoreOp>()) {
213 if (coreOp.getTile() != tile) {
214 continue;
215 }
216 coreOp.walk([&](Operation *op) {
217 if (auto acq = dyn_cast<ObjectFifoAcquireOp>(op)) {
218 if (acq.getObjectFifo() == fifo && acq.getPort() == port) {
219 acq.setObjFifoNameAttr(name);
220 acq.removePortAttr();
221 }
222 } else if (auto rel = dyn_cast<ObjectFifoReleaseOp>(op)) {
223 if (rel.getObjectFifo() == fifo && rel.getPort() == port) {
224 rel.setObjFifoNameAttr(name);
225 rel.removePortAttr();
226 }
227 }
228 });
229 }
230 }
231
232 void runOnOperation() override;
233
234 /// Every pool sits on a concrete tile, so the fifo's ends must be placed.
235 LogicalResult verifyTilesArePlaced() {
236 auto placed = [](Value tile) {
237 return isa_and_nonnull<TileOp>(tile.getDefiningOp());
238 };
239 for (auto fifo : device.getOps<ObjectFifoCreateOp>()) {
240 if (!placed(fifo.getProducerTile())) {
241 return fifo.emitOpError("producer tile is not a placed aie.tile; run "
242 "--aie-place-tiles before this pass");
243 }
244 for (Value consumer : fifo.getConsumerTiles()) {
245 if (!placed(consumer)) {
246 return fifo.emitOpError("consumer tile is not a placed aie.tile; run "
247 "--aie-place-tiles before this pass");
248 }
249 }
250 }
251 for (auto alloc : device.getOps<ObjectFifoAllocateOp>())
252 if (!placed(alloc.getDelegateTile()))
253 return alloc.emitOpError("delegate tile is not a placed aie.tile; run "
254 "--aie-place-tiles before this pass");
255 return success();
256 }
257
258 /// Sharing a receiving channel requires the author's explicit opt-in to
259 /// the ordering constraints of out-of-order packet reception.
260 LogicalResult
261 verifyOutOfOrderForSharedChannels(ArrayRef<ObjectFifoCreateOp> fifos) {
262 DenseMap<std::tuple<int, int, int>, SmallVector<ObjectFifoCreateOp>>
263 onChannel;
264 DenseMap<std::tuple<int, int, int>, TileOp> tileAt;
265 for (ObjectFifoCreateOp fifo : fifos) {
267 if (!fifo.placedDmaReason(shared))
268 continue;
269 for (auto [index, consumer] : llvm::enumerate(fifo.getConsumerTiles())) {
270 ObjectFifoEndPortAttr port = fifo.getConsumerPort(index);
271 if (!port || port.isCore())
272 continue;
273 std::optional<int> pinnedChannel = port.channelIndex();
274 if (!pinnedChannel)
275 continue;
276 int channel = *pinnedChannel;
277 auto tile = consumer.getDefiningOp<TileOp>();
278 if (channel < 0 || !tile)
279 continue;
280 onChannel[{tile.getCol(), tile.getRow(), channel}].push_back(fifo);
281 tileAt[{tile.getCol(), tile.getRow(), channel}] = tile;
282 }
283 }
284 for (auto &[channel, sharing] : onChannel) {
285 if (sharing.size() < 2)
286 continue;
287 TileLike tile = tileAt.lookup(channel);
288 for (ObjectFifoCreateOp fifo : sharing) {
289 if (!fifo.getOutOfOrder())
290 return fifo.emitOpError(
291 "shares a receiving channel; explicitly set out_of_order = true "
292 "on its packet transport");
293 if (std::optional<StringRef> why =
294 ObjectFifoCreateOp::sharedReceiveRefusal(tile.isShimTile(),
295 true))
296 return fifo.emitOpError("shares a receiving channel, but ") << *why;
297 }
298 }
299 return success();
300 }
301
302 LogicalResult verifyPushQueueProducers(ArrayRef<ObjectFifoCreateOp> fifos) {
303 for (ObjectFifoCreateOp fifo : fifos) {
304 if (fifo.getReleaseStrategy() != ObjectFifoRelease::PushQueue)
305 continue;
307 if (!fifo.placedDmaReason(shared))
308 continue;
309 if (std::optional<StringRef> why = fifo.pushQueueRefusal())
310 return fifo.emitOpError(*why);
311 }
312 return success();
313 }
314
315 /// A delegate tile can only hold a fifo's objects if both ends reach its
316 /// memory module, and a linked fifo's objects are the link's pool, so both
317 /// ends must reach the link's tile too. A link's participant that does not
318 /// own the link's pool moves by DMA instead.
319 LogicalResult verifyAllocateDelegates() {
320 for (auto fifo : device.getOps<ObjectFifoCreateOp>()) {
321 auto alloc = fifo.getAllocate();
322 if (!alloc || fifo.allocateFallsBackToDMA()) {
323 continue;
324 }
325 std::optional<TileID> linkTile;
326 if (std::optional<ObjectFifoLinkOp> link = fifo.getLink())
327 if (std::optional<Value> point = link->getOptionalSharedTile())
328 linkTile = cast<TileOp>(point->getDefiningOp()).getTileID();
329 TileID producer = fifo.getProducerTileOp().getTileID();
330 TileID consumer =
331 cast<TileOp>(fifo.getConsumerTiles()[0].getDefiningOp()).getTileID();
332 TileID delegate = alloc->getDelegateTileOp().getTileID();
333 if (ObjectFifoCreateOp::allocatedHome(device.getTargetModel(), producer,
334 consumer, delegate, linkTile)) {
335 continue;
336 }
337 if (!ObjectFifoCreateOp::delegateReaches(device.getTargetModel(),
338 delegate, producer, consumer)) {
339 return alloc->emitOpError("objectfifo has no shared memory access to "
340 "delegate tile's memory module");
341 }
342 return alloc->emitOpError("objectfifo has no shared memory access to "
343 "the memory module of its link's tile, which "
344 "holds the link's pool");
345 }
346 return success();
347 }
348
349 /// A fifo end wired to a Core stream port holds no objects, so there is
350 /// nothing for a core to acquire or release there.
351 LogicalResult verifyStreamPortAccesses() {
352 auto check = [&](Operation *op, ObjectFifoCreateOp fifo,
353 std::optional<ObjectFifoPort> port, StringRef verb) {
354 if (!fifo || !port || !fifo.usesStream()) {
355 return success();
356 }
357 bool onStreamPort = *port == ObjectFifoPort::Produce
358 ? fifo.streamsFromProducer()
359 : fifo.streamsToConsumer(0);
360 if (!onStreamPort) {
361 return success();
362 }
363 return LogicalResult(op->emitOpError("cannot ")
364 << verb << " objectfifo stream port");
365 };
366 for (auto coreOp : device.getOps<CoreOp>()) {
367 auto result = coreOp.walk([&](Operation *op) {
368 LogicalResult ok = success();
369 if (auto acq = dyn_cast<ObjectFifoAcquireOp>(op)) {
370 ok = check(op, acq.getObjectFifo(), acq.getPort(), "acquire from");
371 } else if (auto rel = dyn_cast<ObjectFifoReleaseOp>(op)) {
372 ok = check(op, rel.getObjectFifo(), rel.getPort(), "release from");
373 }
374 return failed(ok) ? WalkResult::interrupt() : WalkResult::advance();
375 });
376 if (result.wasInterrupted()) {
377 return failure();
378 }
379 }
380 return success();
381 }
382
383 /// The DMAs on either side of a link own the shared pool between them, so a
384 /// core sharing the link point has no claim on the objects passing through.
385 LogicalResult verifyLinkAccesses() {
386 for (auto linkOp : device.getOps<ObjectFifoLinkOp>()) {
387 auto sharedTile = linkOp.getOptionalSharedTile();
388 if (!sharedTile) {
389 continue;
390 }
391 auto reject = [&](ObjectFifoCreateOp fifo) {
392 return LogicalResult(
393 linkOp.emitOpError("core on the shared tile cannot access '")
394 << fifo.name().getValue() << "': a link moves objects by DMA");
395 };
396 for (ObjectFifoCreateOp in : linkOp.getInputObjectFifos()) {
397 if (hasCoreAccess(device, *sharedTile, in, ObjectFifoPort::Consume)) {
398 return reject(in);
399 }
400 }
401 for (ObjectFifoCreateOp out : linkOp.getOutputObjectFifos()) {
402 if (hasCoreAccess(device, *sharedTile, out, ObjectFifoPort::Produce)) {
403 return reject(out);
404 }
405 }
406 }
407 return success();
408 }
409
410 /// Ends of a linked fifo that resolve to the link's shared pool. Each link
411 /// reading a fifo takes the next of its consumer ends on the link's tile.
412 DenseMap<Operation *, PoolRef> linkedProducerEnd;
413 DenseMap<std::pair<Operation *, Value>, SmallVector<PoolRef>>
414 linkedConsumerEnd;
415 /// Shim endpoint each fifo is driven through, where it has one.
416 DenseMap<Operation *, std::string> shimEndpointName;
417
418 /// The time link each merging input or dispatched output takes turns in.
419 DenseMap<Operation *, ObjectFifoLinkOp> timeLinkOf;
420 /// A time link's one shared endpoint on the link point.
421 DenseMap<Operation *, ObjectFifoDmaEndpointOp> timeLinkEndpoint;
422 /// What each time link gathers while its participants are split: a merge's
423 /// sources, and a dispatch's destination per output.
424 DenseMap<Operation *, SmallVector<Attribute>> mergeSources;
425 DenseMap<Operation *, DenseMap<Operation *, Attribute>> dispatchTurns;
426 /// Each fifo end's DMA endpoint, by the fifo and its consumer index (none
427 /// for the producer), and ObjectFifoCreateOp::plannedDmaEnd for each end
428 /// before the split, by the fifo and consumer index (-1 the producer).
429 SmallVector<std::tuple<ObjectFifoCreateOp, std::optional<unsigned>,
430 ObjectFifoDmaEndpointOp>>
431 dmaEnds;
432 DenseMap<std::pair<Operation *, int>, std::optional<PlannedDmaEnd>>
433 plannedEnds;
434
435 /// The endpoint a time link's turns share, made by whichever participant
436 /// reaches it first.
437 ObjectFifoDmaEndpointOp timeLinkEndpointFor(ObjectFifoLinkOp link,
438 PoolRef ref,
439 std::optional<int> channel);
440 void createTimeLinkRoutes();
441 LogicalResult createLinkPools();
442};
443
444// A link's pool and its shared merge or dispatch endpoint hold every
445// participant's objects, so they are located at the link and at each fifo.
446static Location linkLocation(ObjectFifoLinkOp link) {
447 SmallVector<Location> locs{link.getLoc()};
448 for (ObjectFifoCreateOp fifo : link.getInputObjectFifos())
449 locs.push_back(fifo.getLoc());
450 for (ObjectFifoCreateOp fifo : link.getOutputObjectFifos())
451 locs.push_back(fifo.getLoc());
452 return FusedLoc::get(link.getContext(), locs);
453}
454
455/// A link's participants meet on one pool on the shared tile: the fifo carrying
456/// whole objects owns its size, and each of the other side's fifos occupies one
457/// segment.
458LogicalResult AIEObjectFifoSplitPass::createLinkPools() {
459 for (auto linkOp : device.getOps<ObjectFifoLinkOp>()) {
460 auto sharedTile = linkOp.getOptionalSharedTile();
461 if (!sharedTile) {
462 continue;
463 }
464
465 std::vector<ObjectFifoCreateOp> ins = linkOp.getInputObjectFifos();
466 std::vector<ObjectFifoCreateOp> outs = linkOp.getOutputObjectFifos();
467 bool isJoin = linkOp.isJoin();
468 bool isDistribute = linkOp.isDistribute();
469
470 ObjectFifoCreateOp owner = linkOp.getPoolOwner();
471 if (linkOp.isMerge()) {
472 for (ObjectFifoCreateOp in : ins)
473 timeLinkOf[in] = linkOp;
474 }
475 if (linkOp.isDispatch()) {
476 for (ObjectFifoCreateOp out : outs)
477 timeLinkOf[out] = linkOp;
478 }
479
480 auto elemType = cast<MemRefType>(
481 cast<AIEObjectFifoType>(owner.getElemType()).getElementType());
482 builder.setInsertionPoint(owner);
483
484 SmallVector<std::pair<int64_t, int64_t>> extents = linkOp.getPoolSegments();
485
486 bool ownerIsOutput = llvm::is_contained(outs, owner);
487 std::string name;
488 // A walk over the stream reads across the objects of one image, so the
489 // pool holds the image's objects side by side.
490 bool window = linkOp.walksStream();
491 StreamWindow walk;
492 if (window) {
493 if (failed(linkOp.getStreamWindow(walk)))
494 return failure();
495 name = (owner.name().getValue() + "__cons__pool").str();
496 } else if (std::optional<unsigned> index = linkOp.getOwnerConsumerIndex()) {
497 name = (owner.name().getValue() +
498 (owner.getConsumerTiles().size() > 1
499 ? "__cons" + std::to_string(*index) + "__pool"
500 : "__cons__pool"))
501 .str();
502 } else {
503 name = (owner.name().getValue() + "__pool").str();
504 }
505 int depth = linkOp.getPoolDepth();
506
507 // The pool is shared by every participant, but `owner` is only one of them:
508 // whether its locks are generated must not depend on which fifo that is.
509 auto asksToSkipLocks = [](ObjectFifoCreateOp fifo) {
510 return fifo.getDisableSynchronization();
511 };
512 bool disableSynchronization = llvm::any_of(ins, asksToSkipLocks) ||
513 llvm::any_of(outs, asksToSkipLocks);
514
515 // The pool starts with the initial objects of the fifo whose producer
516 // fills it: the output's, or an input's that shares memory through its
517 // allocate.
518 bool holdsInitialContents = ownerIsOutput;
519 if (std::optional<ObjectFifoAllocateOp> alloc = owner.getAllocate();
520 alloc && !ownerIsOutput)
521 holdsInitialContents = ObjectFifoCreateOp::allocatedHome(
522 device.getTargetModel(), owner.getProducerTileOp().getTileID(),
523 cast<TileOp>(owner.getConsumerTiles()[0].getDefiningOp()).getTileID(),
524 alloc->getDelegateTileOp().getTileID(),
525 cast<TileOp>(sharedTile->getDefiningOp()).getTileID());
526
527 auto pool = createPool(linkLocation(linkOp), name, *sharedTile, depth,
528 elemType, owner, extents, holdsInitialContents,
529 linkOp.getRepeatCount(), disableSynchronization);
530 if (window) {
531 pool.setContiguous(true);
532 // The padding objects sit in the buffer as zeros, since a MemTile
533 // descriptor walking four dimensions cannot pad its outermost one.
534 if (walk.top || walk.bottom)
535 pool.setZeroObjects(SmallVector<int32_t>{
536 static_cast<int32_t>(walk.top), static_cast<int32_t>(walk.bottom)});
537 }
538
539 SmallVector<int32_t> allSegments;
540 for (int32_t index = 0; index < (int32_t)extents.size(); ++index) {
541 allSegments.push_back(index);
542 }
543 for (auto [index, in] : llvm::enumerate(ins)) {
544 linkedConsumerEnd[{in, *sharedTile}].push_back(PoolRef{
545 pool, isJoin ? SmallVector<int32_t>{static_cast<int32_t>(index)}
546 : allSegments});
547 }
548 for (auto [index, out] : llvm::enumerate(outs)) {
549 linkedProducerEnd[out] = PoolRef{
550 pool, isDistribute ? SmallVector<int32_t>{static_cast<int32_t>(index)}
551 : allSegments};
552 }
553 }
554 return success();
555}
556
557ObjectFifoDmaEndpointOp
558AIEObjectFifoSplitPass::timeLinkEndpointFor(ObjectFifoLinkOp link, PoolRef ref,
559 std::optional<int> channel) {
560 ObjectFifoDmaEndpointOp &endpoint = timeLinkEndpoint[link];
561 if (!endpoint) {
562 ObjectFifoCreateOp owner = link.isMerge() ? link.getOutputObjectFifos()[0]
563 : link.getInputObjectFifos()[0];
564 endpoint = createDmaEndpoint(
565 linkLocation(link),
566 (owner.name().getValue() +
567 (link.isMerge() ? "__merge__dma" : "__dispatch__dma"))
568 .str(),
569 ref.pool.getTile(), ref,
570 link.isMerge() ? ObjectFifoRole::Fill : ObjectFifoRole::Drain, owner,
571 BDDimLayoutArrayAttr(), channel);
572 }
573 return endpoint;
574}
575
576/// A merge becomes one fan-in route into its shared endpoint; a dispatch
577/// names its turns on its shared endpoint in the link's output order.
578void AIEObjectFifoSplitPass::createTimeLinkRoutes() {
579 for (auto link : device.getOps<ObjectFifoLinkOp>()) {
580 ObjectFifoDmaEndpointOp endpoint = timeLinkEndpoint.lookup(link);
581 if (!endpoint)
582 continue;
583 MLIRContext *ctx = builder.getContext();
584 if (link.isMerge()) {
585 PacketInfoAttr header = PacketInfoAttr::get(ctx, 0, std::nullopt);
586 for (ObjectFifoCreateOp in : link.getInputObjectFifos())
587 if (PacketInfoAttr pinned = in.packetHeader();
588 pinned && pinned.isAssigned())
589 header = pinned;
590 builder.setInsertionPoint(link);
591 RouteOp::create(builder, link.getLoc(),
592 builder.getArrayAttr(mergeSources[link]),
593 builder.getArrayAttr(
594 {FlatSymbolRefAttr::get(ctx, endpoint.getSymName())}),
595 header);
596 continue;
597 }
598 SmallVector<Attribute> turns;
599 for (ObjectFifoCreateOp out : link.getOutputObjectFifos())
600 turns.push_back(dispatchTurns[link].lookup(out));
601 endpoint.setDispatchAttr(builder.getArrayAttr(turns));
602 }
603}
604
605void AIEObjectFifoSplitPass::runOnOperation() {
606 // One pass instance splits device after device; ops of an earlier one are
607 // gone, and a new op can take a gone one's address.
608 linkedProducerEnd.clear();
609 linkedConsumerEnd.clear();
610 shimEndpointName.clear();
611 timeLinkOf.clear();
612 timeLinkEndpoint.clear();
613 mergeSources.clear();
614 dispatchTurns.clear();
615 dmaEnds.clear();
616 plannedEnds.clear();
617 device = getOperation();
618 builder = OpBuilder(device.getContext());
619 builder.setInsertionPoint(device.getBody()->getTerminator());
620
621 SmallVector<ObjectFifoCreateOp> fifos(device.getOps<ObjectFifoCreateOp>());
622
623 if (failed(verifyTilesArePlaced()) || failed(verifyStreamPortAccesses()) ||
624 failed(verifyLinkAccesses()) ||
625 failed(verifyOutOfOrderForSharedChannels(fifos))) {
626 return signalPassFailure();
627 }
628
629#ifndef NDEBUG
630 for (ObjectFifoCreateOp fifo : fifos) {
631 plannedEnds[{fifo, -1}] = fifo.plannedDmaEnd(std::nullopt);
632 for (unsigned consumer = 0, e = fifo.getConsumerTiles().size();
633 consumer < e; ++consumer)
634 plannedEnds[{fifo, consumer}] = fifo.plannedDmaEnd(consumer);
635 }
636#endif
637
638 for (ObjectFifoCreateOp fifo : fifos)
639 if (fifo.sharesSendChannel())
640 fifo.setReleaseStrategy(ObjectFifoRelease::PushQueue);
641 if (failed(verifyPushQueueProducers(fifos)))
642 return signalPassFailure();
643
644 if (failed(createLinkPools()) || failed(verifyAllocateDelegates())) {
645 return signalPassFailure();
646 }
647
648 for (ObjectFifoCreateOp fifo : fifos) {
649 builder.setInsertionPoint(fifo);
650 Location loc = fifo.getLoc();
651 StringRef fifoName = fifo.name().getValue();
652 auto fifoType = cast<AIEObjectFifoType>(fifo.getElemType());
653 auto elemType = cast<MemRefType>(fifoType.getElementType());
654 auto consElemType = cast<MemRefType>(
655 cast<AIEObjectFifoType>(fifo.getConsumerElemTypeOrDefault())
656 .getElementType());
657
658 auto sharedModule = AIETargetModel::SharedMemory::None;
659 std::optional<StringRef> needsDMAs = fifo.placedDmaReason(sharedModule);
660 if (needsDMAs && fifo.forcesSharedMemory()) {
661 fifo.emitOpError("asks for a shared_mem transport, but ") << *needsDMAs;
662 return signalPassFailure();
663 }
664 bool shared = !needsDMAs;
665
666 if (shared) {
667 PoolRef ref;
668 if (auto linked = linkedProducerEnd.find(fifo);
669 linked != linkedProducerEnd.end()) {
670 ref = linked->second;
671 } else if (auto linked =
672 linkedConsumerEnd.find({fifo, fifo.getConsumerTiles()[0]});
673 linked != linkedConsumerEnd.end()) {
674 ref = linked->second.front();
675 } else {
676 Value tile = sharedModule == AIETargetModel::SharedMemory::Second
677 ? fifo.getConsumerTiles()[0]
678 : fifo.getProducerTile();
679 // An aie.objectfifo.allocate names the tile whose memory is to hold the
680 // objects, in place of either end's own.
681 if (auto alloc = fifo.getAllocate()) {
682 if (!ObjectFifoCreateOp::allocatedHome(
683 device.getTargetModel(), fifo.getProducerTileOp().getTileID(),
684 cast<TileOp>(fifo.getConsumerTiles()[0].getDefiningOp())
685 .getTileID(),
686 alloc->getDelegateTileOp().getTileID(), std::nullopt)) {
687 alloc->emitOpError("objectfifo has no shared memory access to "
688 "delegate tile's memory module");
689 return signalPassFailure();
690 }
691 tile = alloc->getDelegateTile();
692 }
693 // Both ends address the one pool, so it needs the deeper end's depth
694 // even when init_values fill only the producer's share of it.
695 ref = PoolRef{createPool(loc, (fifoName + "__pool").str(), tile,
696 fifo.sharedDepth(), elemType, fifo,
697 {{0, elemType.getNumElements()}},
698 /*holdsInitialContents=*/true,
699 fifo.getRepeatCount()),
700 {0}};
701 }
702
703 if (hasCoreAccess(device, fifo.getProducerTile(), fifo,
704 ObjectFifoPort::Produce)) {
705 auto name = (fifoName + "__prod").str();
706 createCoreEndpoint(loc, name, fifo.getProducerTile(), ref,
707 ObjectFifoRole::Fill);
708 retargetCoreAccesses(fifo, ObjectFifoPort::Produce,
709 fifo.getProducerTile(), name);
710 }
711 if (hasCoreAccess(device, fifo.getConsumerTiles()[0], fifo,
712 ObjectFifoPort::Consume)) {
713 auto name = (fifoName + "__cons").str();
714 createCoreEndpoint(loc, name, fifo.getConsumerTiles()[0], ref,
715 ObjectFifoRole::Drain);
716 retargetCoreAccesses(fifo, ObjectFifoPort::Consume,
717 fifo.getConsumerTiles()[0], name);
718 }
719 continue;
720 }
721
722 Value prodTile = fifo.getProducerTile();
723 // A fifo end wired straight to a Core stream port has no objects of its
724 // own: whatever the core writes goes out on the port.
725 ObjectFifoEndPortAttr prodPort = fifo.getProdPortAttr();
726 std::optional<int> prodChannel =
727 prodPort ? prodPort.channelIndex() : std::nullopt;
728
729 bool prodIsShim = cast<TileOp>(prodTile.getDefiningOp()).isShimTile();
730 std::optional<PoolRef> prodRef;
731 // Objects entering a repeating link arrive that many at a time.
732 if (auto linked = linkedProducerEnd.find(fifo);
733 linked != linkedProducerEnd.end()) {
734 prodRef = linked->second;
735 } else if (prodIsShim) {
736 if (!fifo.getPlio()) {
737 prodRef = registeredPool(fifo, prodTile, (fifoName + "__pool").str(),
738 elemType);
739 }
740 } else if (!fifo.streamsFromProducer()) {
741 int depth = fifo.dmaDepth(std::nullopt);
742 auto pool =
743 createPool(loc, (fifoName + "__pool").str(), prodTile, depth,
744 elemType, fifo, {{0, elemType.getNumElements()}},
745 /*holdsInitialContents=*/true, fifo.getRepeatCount());
746 prodRef = PoolRef{pool, {0}};
747 }
748
749 if (prodRef &&
750 hasCoreAccess(device, prodTile, fifo, ObjectFifoPort::Produce)) {
751 auto name = (fifoName + "__prod").str();
752 createCoreEndpoint(loc, name, prodTile, *prodRef, ObjectFifoRole::Fill);
753 retargetCoreAccesses(fifo, ObjectFifoPort::Produce, prodTile, name);
754 }
755
756 std::string prodDmaName = (fifoName + "__prod__dma").str();
757 ObjectFifoLinkOp timeLink = timeLinkOf.lookup(fifo);
758 if (timeLink && timeLink.isDispatch()) {
759 ObjectFifoDmaEndpointOp shared =
760 timeLinkEndpointFor(timeLink, *prodRef, prodChannel);
761 prodDmaName = shared.getSymName();
762 dmaEnds.push_back({fifo, std::nullopt, shared});
763 } else if (prodRef) {
764 dmaEnds.push_back(
765 {fifo, std::nullopt,
766 createDmaEndpoint(loc, prodDmaName, prodTile, *prodRef,
767 ObjectFifoRole::Drain, fifo,
768 fifo.getDimensionsToStreamAttr(), prodChannel)});
769 } else {
770 createRouteEndpoint(loc, prodDmaName, prodTile,
771 fifo.streamsFromProducer() ? WireBundle::Core
772 : fifo.getPlio() ? WireBundle::PLIO
773 : WireBundle::DMA,
774 prodChannel, fifo);
775 }
776 if (prodIsShim) {
777 shimEndpointName[fifo] = prodDmaName;
778 }
779
780 SmallVector<Attribute> destinations;
781 ArrayRef<BDDimLayoutArrayAttr> consumerDims =
782 fifo.getDimensionsFromStreamPerConsumer();
783 int consumerIndex = 0;
784 DenseMap<Value, unsigned> linkedOn;
785 for (Value consumerTile : fifo.getConsumerTiles()) {
786 std::string suffix = fifo.getConsumerTiles().size() > 1
787 ? ("__cons" + std::to_string(consumerIndex))
788 : "__cons";
789 bool consIsShim = cast<TileOp>(consumerTile.getDefiningOp()).isShimTile();
790
791 std::optional<PoolRef> consRef;
792 if (auto linked = linkedConsumerEnd.find({fifo, consumerTile});
793 linked != linkedConsumerEnd.end() &&
794 linkedOn[consumerTile] < linked->second.size()) {
795 consRef = linked->second[linkedOn[consumerTile]++];
796 } else if (consIsShim) {
797 if (!fifo.getPlio()) {
798 consRef = registeredPool(fifo, consumerTile,
799 (fifoName + suffix + "__pool").str(),
800 consElemType);
801 }
802 } else if (!fifo.streamsToConsumer(consumerIndex)) {
803 int depth = fifo.dmaDepth(consumerIndex);
804 auto pool = createPool(loc, (fifoName + suffix + "__pool").str(),
805 consumerTile, depth, consElemType, fifo,
806 {{0, consElemType.getNumElements()}},
807 /*holdsInitialContents=*/false,
808 /*repeatCount=*/std::nullopt);
809 consRef = PoolRef{pool, {0}};
810 }
811
812 if (consRef &&
813 hasCoreAccess(device, consumerTile, fifo, ObjectFifoPort::Consume)) {
814 auto name = (fifoName + suffix).str();
815 createCoreEndpoint(loc, name, consumerTile, *consRef,
816 ObjectFifoRole::Drain);
817 retargetCoreAccesses(fifo, ObjectFifoPort::Consume, consumerTile, name);
818 }
819
820 ObjectFifoEndPortAttr consPort = fifo.getConsumerPort(consumerIndex);
821 std::optional<int> consChannel =
822 consPort ? consPort.channelIndex() : std::nullopt;
823
824 std::string consDmaName = (fifoName + suffix + "__dma").str();
825 if (timeLink && timeLink.isMerge()) {
826 ObjectFifoDmaEndpointOp shared =
827 timeLinkEndpointFor(timeLink, *consRef, consChannel);
828 consDmaName = shared.getSymName();
829 dmaEnds.push_back({fifo, consumerIndex, shared});
830 } else if (consRef) {
831 dmaEnds.push_back(
832 {fifo, consumerIndex,
833 createDmaEndpoint(loc, consDmaName, consumerTile, *consRef,
834 ObjectFifoRole::Fill, fifo,
835 consumerDims.empty()
836 ? BDDimLayoutArrayAttr()
837 : consumerDims[consumerIndex],
838 consChannel)});
839 } else {
840 createRouteEndpoint(loc, consDmaName, consumerTile,
841 fifo.streamsToConsumer(consumerIndex)
842 ? WireBundle::Core
843 : fifo.getPlio() ? WireBundle::PLIO
844 : WireBundle::DMA,
845 consChannel, fifo);
846 }
847 destinations.push_back(
848 FlatSymbolRefAttr::get(builder.getContext(), consDmaName));
849 if (consIsShim) {
850 shimEndpointName[fifo] = consDmaName;
851 }
852 consumerIndex++;
853 }
854
855 FlatSymbolRefAttr source =
856 FlatSymbolRefAttr::get(builder.getContext(), prodDmaName);
857 if (timeLink && timeLink.isMerge()) {
858 mergeSources[timeLink].push_back(source);
859 continue;
860 }
861 // A dispatch tells its turns apart by header, so each of its routes
862 // carries one.
863 PacketInfoAttr header = fifo.packetHeader();
864 if (timeLink && timeLink.isDispatch()) {
865 dispatchTurns[timeLink][fifo] = destinations.front();
866 if (!header)
867 header = PacketInfoAttr::get(builder.getContext(), 0, std::nullopt);
868 }
869 RouteOp::create(builder, loc, builder.getArrayAttr({source}),
870 builder.getArrayAttr(destinations), header);
871 }
872
873 createTimeLinkRoutes();
874
875#ifndef NDEBUG
876 // The placers plan BD ids from plannedDmaEnd before any fifo is split: each
877 // endpoint off a shim is the one end planned for it, with its BD ids.
878 DenseMap<Operation *, int> planned;
879 for (auto [fifo, consumer, endpoint] : dmaEnds) {
880 std::optional<PlannedDmaEnd> end =
881 plannedEnds.lookup({fifo, consumer ? static_cast<int>(*consumer) : -1});
882 if (!end)
883 continue;
884 ObjectFifoPoolOp pool = endpoint.getPoolOp();
885 assert(end->bds == endpoint.getNumBDs() && end->depth == pool.getDepth() &&
886 end->walkable == pool.canWalkWhole() &&
887 end->objectBytes == pool.getObjectSizeInBytes() &&
888 end->contiguousObjects ==
889 (pool.getContiguous() ? pool.getObjectsPerBuffer() : 0) &&
890 end->channel ==
891 cast<RouteEndpoint>(endpoint.getOperation()).getRouteChannel() &&
892 "plannedDmaEnd disagrees with the endpoint the split made");
893 ++planned[endpoint];
894 }
895 for (auto [fifo, consumer, endpoint] : dmaEnds)
896 assert((endpoint.getTileLike().isShimTile() ||
897 planned.lookup(endpoint) == 1) &&
898 "an endpoint the split made is not planned exactly once");
899#endif
900
901 SmallVector<Operation *> toErase;
902 for (ObjectFifoCreateOp fifo : fifos) {
903 // The runtime sequence drives a fifo through its shim end.
904 auto shim = shimEndpointName.find(fifo);
905 if (shim != shimEndpointName.end()) {
906 (void)SymbolTable::replaceAllSymbolUses(
907 fifo, builder.getStringAttr(shim->second), device);
908 }
909 toErase.push_back(fifo);
910 }
911 for (auto linkOp : device.getOps<ObjectFifoLinkOp>()) {
912 toErase.push_back(linkOp);
913 }
914 for (auto allocOp : device.getOps<ObjectFifoAllocateOp>()) {
915 toErase.push_back(allocOp);
916 }
917 for (auto regOp : device.getOps<ObjectFifoRegisterExternalBuffersOp>()) {
918 toErase.push_back(regOp);
919 }
920 for (Operation *op : toErase) {
921 op->erase();
922 }
923}
924
925} // namespace
926
927std::unique_ptr<OperationPass<DeviceOp>>
929 return std::make_unique<AIEObjectFifoSplitPass>();
930}
@ None
Neither tile can address the other's memory module.
bool hasCoreAccess(DeviceOp device, mlir::Value tile, ObjectFifoCreateOp fifo, ObjectFifoPort port)
struct TileID { friend std::ostream &operator<<(std::ostream &os, const TileID &s) { os<< "TileID("<< s.col<< ", "<< s.row<< ")"; return os; } friend std::string to_string(const TileID &s) { std::ostringstream ss; ss<< s; return ss.str(); } friend llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const TileID &s) { os<< to_string(s); return os; } inline bool operator<(const TileID &rhs) const { return std::tie(col, row)< std::tie(rhs.col, rhs.row); } bool operator==(const TileID &rhs) const { return std::tie(col, row)==std::tie(rhs.col, rhs.row); } bool operator!=(const TileID &rhs) const { return !(*this==rhs);} int col, row;} TileID
std::unique_ptr< mlir::OperationPass< DeviceOp > > createAIEObjectFifoSplitPass()