11#include "mlir/IR/Attributes.h"
12#include "mlir/Pass/Pass.h"
13#include "llvm/ADT/DenseMap.h"
16#define GEN_PASS_DEF_AIEOBJECTFIFOSPLIT
17#include "aie/Dialect/AIE/Transforms/AIEPasses.h.inc"
29 ObjectFifoPoolOp pool;
30 SmallVector<int32_t> segments;
33struct AIEObjectFifoSplitPass
34 :
public xilinx::AIE::impl::AIEObjectFifoSplitBase<AIEObjectFifoSplitPass> {
37 OpBuilder builder{
static_cast<MLIRContext *
>(
nullptr)};
40 void createSegments(ObjectFifoPoolOp pool, Location loc,
41 ArrayRef<std::pair<int64_t, int64_t>> extents) {
42 OpBuilder::InsertionGuard guard(builder);
43 builder.createBlock(&pool.getSegments());
44 for (
auto [index, extent] : llvm::enumerate(extents)) {
45 ObjectFifoSegmentOp::create(
46 builder, loc, builder.getStringAttr(
"s" + std::to_string(index)),
47 builder.getI32IntegerAttr(extent.first),
48 builder.getI32IntegerAttr(extent.second),
55 ArrayAttr selectedSegmentNames(PoolRef ref) {
56 std::vector<ObjectFifoSegmentOp> all = ref.pool.getSegmentOps();
57 if (all.size() <= 1) {
60 SmallVector<Attribute> names;
61 for (int32_t index : ref.segments) {
62 names.push_back(FlatSymbolRefAttr::get(builder.getContext(),
63 all[index].getSymName()));
65 return builder.getArrayAttr(names);
68 ObjectFifoPoolOp createPool(Location loc, StringRef name, Value tile,
69 int depth, MemRefType elemType,
70 ObjectFifoCreateOp from,
71 ArrayRef<std::pair<int64_t, int64_t>> extents,
72 bool holdsInitialContents,
73 std::optional<int> repeatCount,
74 bool disableSynchronization =
false) {
75 auto pool = ObjectFifoPoolOp::create(
76 builder, loc, name, tile, depth, elemType, ArrayAttr(),
78 repeatCount ? builder.getI32IntegerAttr(*repeatCount) : IntegerAttr(),
79 disableSynchronization || from.getDisableSynchronization(),
80 builder.getStringAttr(from.name().getValue()),
81 holdsInitialContents ? from.getInitValuesAttr() : ArrayAttr(),
84 createSegments(pool, loc, extents);
88 ObjectFifoCoreEndpointOp createCoreEndpoint(Location loc, StringRef name,
89 Value tile, PoolRef ref,
90 ObjectFifoRole role) {
91 return ObjectFifoCoreEndpointOp::create(
92 builder, loc, name, tile, role, ref.pool.getSymName(),
93 selectedSegmentNames(ref),
94 ObjectFifoCorePushAttr());
97 ObjectFifoDmaEndpointOp createDmaEndpoint(Location loc, StringRef name,
98 Value tile, PoolRef ref,
100 ObjectFifoCreateOp from,
101 BDDimLayoutArrayAttr dims,
102 std::optional<int> channelIndex) {
106 std::optional<int32_t> iterCount;
107 if (std::optional<int32_t> fifoIterations = from.getIterCount()) {
108 iterCount = role == ObjectFifoRole::Drain
110 : *fifoIterations * from.getRepeatCount().value_or(1);
112 ArrayAttr segments = selectedSegmentNames(ref);
114 BDDimLayoutArrayArrayAttr dimensions;
115 if (dims && !dims.empty()) {
116 SmallVector<BDDimLayoutArrayAttr> perSegment(ref.segments.size(), dims);
118 BDDimLayoutArrayArrayAttr::get(builder.getContext(), perSegment);
121 BDPadLayoutArrayArrayAttr padding;
122 std::optional<ArrayRef<BDPadLayoutAttr>> padDims = from.getPadDimensions();
123 if (role == ObjectFifoRole::Drain && padDims && !padDims->empty()) {
124 SmallVector<BDPadLayoutArrayAttr> perSegment(ref.segments.size(),
125 from.getPadDimensionsAttr());
127 BDPadLayoutArrayArrayAttr::get(builder.getContext(), perSegment);
130 return ObjectFifoDmaEndpointOp::create(
131 builder, loc, builder.getStringAttr(name), tile,
132 ObjectFifoRoleAttr::get(builder.getContext(), role),
133 FlatSymbolRefAttr::get(builder.getContext(), ref.pool.getSymName()),
135 channelIndex ? builder.getI32IntegerAttr(*channelIndex) : IntegerAttr(),
137 from.getPadValue() ? builder.getI32IntegerAttr(from.getPadValue())
139 iterCount ? builder.getI32IntegerAttr(*iterCount) : IntegerAttr(),
141 builder.getStringAttr(from.name().getValue()), ArrayAttr(),
142 ArrayAttr(), ArrayAttr(),
143 from.getOutOfOrder() ? builder.getUnitAttr() : UnitAttr(),
145 role == ObjectFifoRole::Drain ? from.getReleaseStrategyAttr()
146 : ObjectFifoReleaseAttr());
151 RouteEndpointOp createRouteEndpoint(Location loc, StringRef name, Value tile,
153 std::optional<int> channelIndex,
154 ObjectFifoCreateOp from) {
155 auto endpoint = RouteEndpointOp::create(
156 builder, loc, builder.getStringAttr(name), tile,
157 WireBundleAttr::get(builder.getContext(), bundle),
158 channelIndex ? builder.getI32IntegerAttr(*channelIndex) : IntegerAttr(),
160 builder.getStringAttr(from.name().getValue()),
162 if (bundle == WireBundle::DMA &&
163 cast<TileOp>(tile.getDefiningOp()).isShimTile()) {
164 AIEDialect::ShareableWithAttrHelper sharing(builder.getContext());
165 if (
auto allowed = sharing.getAttr(from))
166 sharing.setAttr(endpoint, allowed);
173 std::optional<PoolRef> registeredPool(ObjectFifoCreateOp fifo, Value tile,
174 StringRef name, MemRefType elemType) {
175 SmallVector<Attribute> names;
176 for (
auto regOp : device.getOps<ObjectFifoRegisterExternalBuffersOp>()) {
177 if (regOp.getTile() != tile || regOp.getObjectFifo() != fifo) {
180 for (Value buffer : regOp.getExternalBuffers()) {
181 auto external = cast<ExternalBufferOp>(buffer.getDefiningOp());
182 names.push_back(SymbolRefAttr::get(external.getSymNameAttr()));
185 elemType = cast<MemRefType>(external.getType());
192 auto pool = ObjectFifoPoolOp::create(
193 builder, fifo.getLoc(), name, tile, (
int)names.size(), elemType,
194 builder.getArrayAttr(names),
195 ArrayAttr(), IntegerAttr(),
196 fifo.getDisableSynchronization(),
197 builder.getStringAttr(fifo.name().getValue()),
200 DenseI32ArrayAttr());
201 createSegments(pool, fifo.getLoc(), {{0, elemType.getNumElements()}});
202 return PoolRef{pool, {0}};
206 void retargetCoreAccesses(ObjectFifoCreateOp fifo, ObjectFifoPort port,
207 Value tile, StringRef endpointName) {
208 auto name = FlatSymbolRefAttr::get(builder.getContext(), endpointName);
212 for (
auto coreOp : device.getOps<CoreOp>()) {
213 if (coreOp.getTile() != tile) {
216 coreOp.walk([&](Operation *op) {
217 if (
auto acq = dyn_cast<ObjectFifoAcquireOp>(op)) {
218 if (acq.getObjectFifo() == fifo && acq.getPort() == port) {
219 acq.setObjFifoNameAttr(name);
220 acq.removePortAttr();
222 }
else if (
auto rel = dyn_cast<ObjectFifoReleaseOp>(op)) {
223 if (rel.getObjectFifo() == fifo && rel.getPort() == port) {
224 rel.setObjFifoNameAttr(name);
225 rel.removePortAttr();
232 void runOnOperation()
override;
235 LogicalResult verifyTilesArePlaced() {
236 auto placed = [](Value tile) {
237 return isa_and_nonnull<TileOp>(tile.getDefiningOp());
239 for (
auto fifo : device.getOps<ObjectFifoCreateOp>()) {
240 if (!placed(fifo.getProducerTile())) {
241 return fifo.emitOpError(
"producer tile is not a placed aie.tile; run "
242 "--aie-place-tiles before this pass");
244 for (Value consumer : fifo.getConsumerTiles()) {
245 if (!placed(consumer)) {
246 return fifo.emitOpError(
"consumer tile is not a placed aie.tile; run "
247 "--aie-place-tiles before this pass");
251 for (
auto alloc : device.getOps<ObjectFifoAllocateOp>())
252 if (!placed(alloc.getDelegateTile()))
253 return alloc.emitOpError(
"delegate tile is not a placed aie.tile; run "
254 "--aie-place-tiles before this pass");
261 verifyOutOfOrderForSharedChannels(ArrayRef<ObjectFifoCreateOp> fifos) {
262 DenseMap<std::tuple<int, int, int>, SmallVector<ObjectFifoCreateOp>>
264 DenseMap<std::tuple<int, int, int>, TileOp> tileAt;
265 for (ObjectFifoCreateOp fifo : fifos) {
267 if (!fifo.placedDmaReason(shared))
269 for (
auto [index, consumer] : llvm::enumerate(fifo.getConsumerTiles())) {
270 ObjectFifoEndPortAttr port = fifo.getConsumerPort(index);
271 if (!port || port.isCore())
273 std::optional<int> pinnedChannel = port.channelIndex();
276 int channel = *pinnedChannel;
277 auto tile = consumer.getDefiningOp<TileOp>();
278 if (channel < 0 || !tile)
280 onChannel[{tile.getCol(), tile.getRow(), channel}].push_back(fifo);
281 tileAt[{tile.getCol(), tile.getRow(), channel}] = tile;
284 for (
auto &[channel, sharing] : onChannel) {
285 if (sharing.size() < 2)
287 TileLike tile = tileAt.lookup(channel);
288 for (ObjectFifoCreateOp fifo : sharing) {
289 if (!fifo.getOutOfOrder())
290 return fifo.emitOpError(
291 "shares a receiving channel; explicitly set out_of_order = true "
292 "on its packet transport");
293 if (std::optional<StringRef> why =
294 ObjectFifoCreateOp::sharedReceiveRefusal(tile.isShimTile(),
296 return fifo.emitOpError(
"shares a receiving channel, but ") << *why;
302 LogicalResult verifyPushQueueProducers(ArrayRef<ObjectFifoCreateOp> fifos) {
303 for (ObjectFifoCreateOp fifo : fifos) {
304 if (fifo.getReleaseStrategy() != ObjectFifoRelease::PushQueue)
307 if (!fifo.placedDmaReason(shared))
309 if (std::optional<StringRef> why = fifo.pushQueueRefusal())
310 return fifo.emitOpError(*why);
319 LogicalResult verifyAllocateDelegates() {
320 for (
auto fifo : device.getOps<ObjectFifoCreateOp>()) {
321 auto alloc = fifo.getAllocate();
322 if (!alloc || fifo.allocateFallsBackToDMA()) {
325 std::optional<TileID> linkTile;
326 if (std::optional<ObjectFifoLinkOp> link = fifo.getLink())
327 if (std::optional<Value> point = link->getOptionalSharedTile())
328 linkTile = cast<TileOp>(point->getDefiningOp()).getTileID();
329 TileID producer = fifo.getProducerTileOp().getTileID();
331 cast<TileOp>(fifo.getConsumerTiles()[0].getDefiningOp()).getTileID();
332 TileID delegate = alloc->getDelegateTileOp().getTileID();
333 if (ObjectFifoCreateOp::allocatedHome(device.getTargetModel(), producer,
334 consumer, delegate, linkTile)) {
337 if (!ObjectFifoCreateOp::delegateReaches(device.getTargetModel(),
338 delegate, producer, consumer)) {
339 return alloc->emitOpError(
"objectfifo has no shared memory access to "
340 "delegate tile's memory module");
342 return alloc->emitOpError(
"objectfifo has no shared memory access to "
343 "the memory module of its link's tile, which "
344 "holds the link's pool");
351 LogicalResult verifyStreamPortAccesses() {
352 auto check = [&](Operation *op, ObjectFifoCreateOp fifo,
353 std::optional<ObjectFifoPort> port, StringRef verb) {
354 if (!fifo || !port || !fifo.usesStream()) {
357 bool onStreamPort = *port == ObjectFifoPort::Produce
358 ? fifo.streamsFromProducer()
359 : fifo.streamsToConsumer(0);
363 return LogicalResult(op->emitOpError(
"cannot ")
364 << verb <<
" objectfifo stream port");
366 for (
auto coreOp : device.getOps<CoreOp>()) {
367 auto result = coreOp.walk([&](Operation *op) {
368 LogicalResult ok = success();
369 if (
auto acq = dyn_cast<ObjectFifoAcquireOp>(op)) {
370 ok = check(op, acq.getObjectFifo(), acq.getPort(),
"acquire from");
371 }
else if (
auto rel = dyn_cast<ObjectFifoReleaseOp>(op)) {
372 ok = check(op, rel.getObjectFifo(), rel.getPort(),
"release from");
374 return failed(ok) ? WalkResult::interrupt() : WalkResult::advance();
376 if (result.wasInterrupted()) {
385 LogicalResult verifyLinkAccesses() {
386 for (
auto linkOp : device.getOps<ObjectFifoLinkOp>()) {
387 auto sharedTile = linkOp.getOptionalSharedTile();
391 auto reject = [&](ObjectFifoCreateOp fifo) {
392 return LogicalResult(
393 linkOp.emitOpError(
"core on the shared tile cannot access '")
394 << fifo.name().getValue() <<
"': a link moves objects by DMA");
396 for (ObjectFifoCreateOp in : linkOp.getInputObjectFifos()) {
397 if (
hasCoreAccess(device, *sharedTile, in, ObjectFifoPort::Consume)) {
401 for (ObjectFifoCreateOp out : linkOp.getOutputObjectFifos()) {
402 if (
hasCoreAccess(device, *sharedTile, out, ObjectFifoPort::Produce)) {
412 DenseMap<Operation *, PoolRef> linkedProducerEnd;
413 DenseMap<std::pair<Operation *, Value>, SmallVector<PoolRef>>
416 DenseMap<Operation *, std::string> shimEndpointName;
419 DenseMap<Operation *, ObjectFifoLinkOp> timeLinkOf;
421 DenseMap<Operation *, ObjectFifoDmaEndpointOp> timeLinkEndpoint;
424 DenseMap<Operation *, SmallVector<Attribute>> mergeSources;
425 DenseMap<Operation *, DenseMap<Operation *, Attribute>> dispatchTurns;
429 SmallVector<std::tuple<ObjectFifoCreateOp, std::optional<unsigned>,
430 ObjectFifoDmaEndpointOp>>
432 DenseMap<std::pair<Operation *, int>, std::optional<PlannedDmaEnd>>
437 ObjectFifoDmaEndpointOp timeLinkEndpointFor(ObjectFifoLinkOp link,
439 std::optional<int> channel);
440 void createTimeLinkRoutes();
441 LogicalResult createLinkPools();
446static Location linkLocation(ObjectFifoLinkOp link) {
447 SmallVector<Location> locs{link.getLoc()};
448 for (ObjectFifoCreateOp fifo : link.getInputObjectFifos())
449 locs.push_back(fifo.getLoc());
450 for (ObjectFifoCreateOp fifo : link.getOutputObjectFifos())
451 locs.push_back(fifo.getLoc());
452 return FusedLoc::get(link.getContext(), locs);
458LogicalResult AIEObjectFifoSplitPass::createLinkPools() {
459 for (
auto linkOp : device.getOps<ObjectFifoLinkOp>()) {
460 auto sharedTile = linkOp.getOptionalSharedTile();
465 std::vector<ObjectFifoCreateOp> ins = linkOp.getInputObjectFifos();
466 std::vector<ObjectFifoCreateOp> outs = linkOp.getOutputObjectFifos();
467 bool isJoin = linkOp.isJoin();
468 bool isDistribute = linkOp.isDistribute();
470 ObjectFifoCreateOp owner = linkOp.getPoolOwner();
471 if (linkOp.isMerge()) {
472 for (ObjectFifoCreateOp in : ins)
473 timeLinkOf[in] = linkOp;
475 if (linkOp.isDispatch()) {
476 for (ObjectFifoCreateOp out : outs)
477 timeLinkOf[out] = linkOp;
480 auto elemType = cast<MemRefType>(
481 cast<AIEObjectFifoType>(owner.getElemType()).getElementType());
482 builder.setInsertionPoint(owner);
484 SmallVector<std::pair<int64_t, int64_t>> extents = linkOp.getPoolSegments();
486 bool ownerIsOutput = llvm::is_contained(outs, owner);
490 bool window = linkOp.walksStream();
493 if (failed(linkOp.getStreamWindow(walk)))
495 name = (owner.name().getValue() +
"__cons__pool").str();
496 }
else if (std::optional<unsigned> index = linkOp.getOwnerConsumerIndex()) {
497 name = (owner.name().getValue() +
498 (owner.getConsumerTiles().size() > 1
499 ?
"__cons" + std::to_string(*index) +
"__pool"
503 name = (owner.name().getValue() +
"__pool").str();
505 int depth = linkOp.getPoolDepth();
509 auto asksToSkipLocks = [](ObjectFifoCreateOp fifo) {
510 return fifo.getDisableSynchronization();
512 bool disableSynchronization = llvm::any_of(ins, asksToSkipLocks) ||
513 llvm::any_of(outs, asksToSkipLocks);
518 bool holdsInitialContents = ownerIsOutput;
519 if (std::optional<ObjectFifoAllocateOp> alloc = owner.getAllocate();
520 alloc && !ownerIsOutput)
521 holdsInitialContents = ObjectFifoCreateOp::allocatedHome(
522 device.getTargetModel(), owner.getProducerTileOp().getTileID(),
523 cast<TileOp>(owner.getConsumerTiles()[0].getDefiningOp()).getTileID(),
524 alloc->getDelegateTileOp().getTileID(),
525 cast<TileOp>(sharedTile->getDefiningOp()).getTileID());
527 auto pool = createPool(linkLocation(linkOp), name, *sharedTile, depth,
528 elemType, owner, extents, holdsInitialContents,
529 linkOp.getRepeatCount(), disableSynchronization);
531 pool.setContiguous(
true);
535 pool.setZeroObjects(SmallVector<int32_t>{
536 static_cast<int32_t
>(walk.
top),
static_cast<int32_t
>(walk.
bottom)});
539 SmallVector<int32_t> allSegments;
540 for (int32_t index = 0; index < (int32_t)extents.size(); ++index) {
541 allSegments.push_back(index);
543 for (
auto [index, in] : llvm::enumerate(ins)) {
544 linkedConsumerEnd[{in, *sharedTile}].push_back(PoolRef{
545 pool, isJoin ? SmallVector<int32_t>{
static_cast<int32_t
>(index)}
548 for (
auto [index, out] : llvm::enumerate(outs)) {
549 linkedProducerEnd[out] = PoolRef{
550 pool, isDistribute ? SmallVector<int32_t>{
static_cast<int32_t
>(index)}
557ObjectFifoDmaEndpointOp
558AIEObjectFifoSplitPass::timeLinkEndpointFor(ObjectFifoLinkOp link, PoolRef ref,
559 std::optional<int> channel) {
560 ObjectFifoDmaEndpointOp &endpoint = timeLinkEndpoint[link];
562 ObjectFifoCreateOp owner = link.isMerge() ? link.getOutputObjectFifos()[0]
563 : link.getInputObjectFifos()[0];
564 endpoint = createDmaEndpoint(
566 (owner.name().getValue() +
567 (link.isMerge() ?
"__merge__dma" :
"__dispatch__dma"))
569 ref.pool.getTile(), ref,
570 link.isMerge() ? ObjectFifoRole::Fill : ObjectFifoRole::Drain, owner,
571 BDDimLayoutArrayAttr(), channel);
578void AIEObjectFifoSplitPass::createTimeLinkRoutes() {
579 for (
auto link : device.getOps<ObjectFifoLinkOp>()) {
580 ObjectFifoDmaEndpointOp endpoint = timeLinkEndpoint.lookup(link);
583 MLIRContext *ctx = builder.getContext();
584 if (link.isMerge()) {
585 PacketInfoAttr header = PacketInfoAttr::get(ctx, 0, std::nullopt);
586 for (ObjectFifoCreateOp in : link.getInputObjectFifos())
587 if (PacketInfoAttr pinned = in.packetHeader();
588 pinned && pinned.isAssigned())
590 builder.setInsertionPoint(link);
591 RouteOp::create(builder, link.getLoc(),
592 builder.getArrayAttr(mergeSources[link]),
593 builder.getArrayAttr(
594 {FlatSymbolRefAttr::get(ctx, endpoint.getSymName())}),
598 SmallVector<Attribute> turns;
599 for (ObjectFifoCreateOp out : link.getOutputObjectFifos())
600 turns.push_back(dispatchTurns[link].lookup(out));
601 endpoint.setDispatchAttr(builder.getArrayAttr(turns));
605void AIEObjectFifoSplitPass::runOnOperation() {
608 linkedProducerEnd.clear();
609 linkedConsumerEnd.clear();
610 shimEndpointName.clear();
612 timeLinkEndpoint.clear();
613 mergeSources.clear();
614 dispatchTurns.clear();
617 device = getOperation();
618 builder = OpBuilder(device.getContext());
619 builder.setInsertionPoint(device.getBody()->getTerminator());
621 SmallVector<ObjectFifoCreateOp> fifos(device.getOps<ObjectFifoCreateOp>());
623 if (failed(verifyTilesArePlaced()) || failed(verifyStreamPortAccesses()) ||
624 failed(verifyLinkAccesses()) ||
625 failed(verifyOutOfOrderForSharedChannels(fifos))) {
626 return signalPassFailure();
630 for (ObjectFifoCreateOp fifo : fifos) {
631 plannedEnds[{fifo, -1}] = fifo.plannedDmaEnd(std::nullopt);
632 for (
unsigned consumer = 0, e = fifo.getConsumerTiles().size();
633 consumer < e; ++consumer)
634 plannedEnds[{fifo, consumer}] = fifo.plannedDmaEnd(consumer);
638 for (ObjectFifoCreateOp fifo : fifos)
639 if (fifo.sharesSendChannel())
640 fifo.setReleaseStrategy(ObjectFifoRelease::PushQueue);
641 if (failed(verifyPushQueueProducers(fifos)))
642 return signalPassFailure();
644 if (failed(createLinkPools()) || failed(verifyAllocateDelegates())) {
645 return signalPassFailure();
648 for (ObjectFifoCreateOp fifo : fifos) {
649 builder.setInsertionPoint(fifo);
650 Location loc = fifo.getLoc();
651 StringRef fifoName = fifo.name().getValue();
652 auto fifoType = cast<AIEObjectFifoType>(fifo.getElemType());
653 auto elemType = cast<MemRefType>(fifoType.getElementType());
654 auto consElemType = cast<MemRefType>(
655 cast<AIEObjectFifoType>(fifo.getConsumerElemTypeOrDefault())
658 auto sharedModule = AIETargetModel::SharedMemory::None;
659 std::optional<StringRef> needsDMAs = fifo.placedDmaReason(sharedModule);
660 if (needsDMAs && fifo.forcesSharedMemory()) {
661 fifo.emitOpError(
"asks for a shared_mem transport, but ") << *needsDMAs;
662 return signalPassFailure();
664 bool shared = !needsDMAs;
668 if (
auto linked = linkedProducerEnd.find(fifo);
669 linked != linkedProducerEnd.end()) {
670 ref = linked->second;
671 }
else if (
auto linked =
672 linkedConsumerEnd.find({fifo, fifo.getConsumerTiles()[0]});
673 linked != linkedConsumerEnd.end()) {
674 ref = linked->second.front();
676 Value tile = sharedModule == AIETargetModel::SharedMemory::Second
677 ? fifo.getConsumerTiles()[0]
678 : fifo.getProducerTile();
681 if (
auto alloc = fifo.getAllocate()) {
682 if (!ObjectFifoCreateOp::allocatedHome(
683 device.getTargetModel(), fifo.getProducerTileOp().getTileID(),
684 cast<TileOp>(fifo.getConsumerTiles()[0].getDefiningOp())
686 alloc->getDelegateTileOp().getTileID(), std::nullopt)) {
687 alloc->emitOpError(
"objectfifo has no shared memory access to "
688 "delegate tile's memory module");
689 return signalPassFailure();
691 tile = alloc->getDelegateTile();
695 ref = PoolRef{createPool(loc, (fifoName +
"__pool").str(), tile,
696 fifo.sharedDepth(), elemType, fifo,
697 {{0, elemType.getNumElements()}},
699 fifo.getRepeatCount()),
704 ObjectFifoPort::Produce)) {
705 auto name = (fifoName +
"__prod").str();
706 createCoreEndpoint(loc, name, fifo.getProducerTile(), ref,
707 ObjectFifoRole::Fill);
708 retargetCoreAccesses(fifo, ObjectFifoPort::Produce,
709 fifo.getProducerTile(), name);
712 ObjectFifoPort::Consume)) {
713 auto name = (fifoName +
"__cons").str();
714 createCoreEndpoint(loc, name, fifo.getConsumerTiles()[0], ref,
715 ObjectFifoRole::Drain);
716 retargetCoreAccesses(fifo, ObjectFifoPort::Consume,
717 fifo.getConsumerTiles()[0], name);
722 Value prodTile = fifo.getProducerTile();
725 ObjectFifoEndPortAttr prodPort = fifo.getProdPortAttr();
726 std::optional<int> prodChannel =
727 prodPort ? prodPort.channelIndex() : std::nullopt;
729 bool prodIsShim = cast<TileOp>(prodTile.getDefiningOp()).isShimTile();
730 std::optional<PoolRef> prodRef;
732 if (
auto linked = linkedProducerEnd.find(fifo);
733 linked != linkedProducerEnd.end()) {
734 prodRef = linked->second;
735 }
else if (prodIsShim) {
736 if (!fifo.getPlio()) {
737 prodRef = registeredPool(fifo, prodTile, (fifoName +
"__pool").str(),
740 }
else if (!fifo.streamsFromProducer()) {
741 int depth = fifo.dmaDepth(std::nullopt);
743 createPool(loc, (fifoName +
"__pool").str(), prodTile, depth,
744 elemType, fifo, {{0, elemType.getNumElements()}},
745 true, fifo.getRepeatCount());
746 prodRef = PoolRef{pool, {0}};
750 hasCoreAccess(device, prodTile, fifo, ObjectFifoPort::Produce)) {
751 auto name = (fifoName +
"__prod").str();
752 createCoreEndpoint(loc, name, prodTile, *prodRef, ObjectFifoRole::Fill);
753 retargetCoreAccesses(fifo, ObjectFifoPort::Produce, prodTile, name);
756 std::string prodDmaName = (fifoName +
"__prod__dma").str();
757 ObjectFifoLinkOp timeLink = timeLinkOf.lookup(fifo);
758 if (timeLink && timeLink.isDispatch()) {
759 ObjectFifoDmaEndpointOp shared =
760 timeLinkEndpointFor(timeLink, *prodRef, prodChannel);
761 prodDmaName = shared.getSymName();
762 dmaEnds.push_back({fifo, std::nullopt, shared});
763 }
else if (prodRef) {
766 createDmaEndpoint(loc, prodDmaName, prodTile, *prodRef,
767 ObjectFifoRole::Drain, fifo,
768 fifo.getDimensionsToStreamAttr(), prodChannel)});
770 createRouteEndpoint(loc, prodDmaName, prodTile,
771 fifo.streamsFromProducer() ? WireBundle::Core
772 : fifo.getPlio() ? WireBundle::PLIO
777 shimEndpointName[fifo] = prodDmaName;
780 SmallVector<Attribute> destinations;
781 ArrayRef<BDDimLayoutArrayAttr> consumerDims =
782 fifo.getDimensionsFromStreamPerConsumer();
783 int consumerIndex = 0;
784 DenseMap<Value, unsigned> linkedOn;
785 for (Value consumerTile : fifo.getConsumerTiles()) {
786 std::string suffix = fifo.getConsumerTiles().size() > 1
787 ? (
"__cons" + std::to_string(consumerIndex))
789 bool consIsShim = cast<TileOp>(consumerTile.getDefiningOp()).isShimTile();
791 std::optional<PoolRef> consRef;
792 if (
auto linked = linkedConsumerEnd.find({fifo, consumerTile});
793 linked != linkedConsumerEnd.end() &&
794 linkedOn[consumerTile] < linked->second.size()) {
795 consRef = linked->second[linkedOn[consumerTile]++];
796 }
else if (consIsShim) {
797 if (!fifo.getPlio()) {
798 consRef = registeredPool(fifo, consumerTile,
799 (fifoName + suffix +
"__pool").str(),
802 }
else if (!fifo.streamsToConsumer(consumerIndex)) {
803 int depth = fifo.dmaDepth(consumerIndex);
804 auto pool = createPool(loc, (fifoName + suffix +
"__pool").str(),
805 consumerTile, depth, consElemType, fifo,
806 {{0, consElemType.getNumElements()}},
809 consRef = PoolRef{pool, {0}};
813 hasCoreAccess(device, consumerTile, fifo, ObjectFifoPort::Consume)) {
814 auto name = (fifoName + suffix).str();
815 createCoreEndpoint(loc, name, consumerTile, *consRef,
816 ObjectFifoRole::Drain);
817 retargetCoreAccesses(fifo, ObjectFifoPort::Consume, consumerTile, name);
820 ObjectFifoEndPortAttr consPort = fifo.getConsumerPort(consumerIndex);
821 std::optional<int> consChannel =
822 consPort ? consPort.channelIndex() : std::nullopt;
824 std::string consDmaName = (fifoName + suffix +
"__dma").str();
825 if (timeLink && timeLink.isMerge()) {
826 ObjectFifoDmaEndpointOp shared =
827 timeLinkEndpointFor(timeLink, *consRef, consChannel);
828 consDmaName = shared.getSymName();
829 dmaEnds.push_back({fifo, consumerIndex, shared});
830 }
else if (consRef) {
832 {fifo, consumerIndex,
833 createDmaEndpoint(loc, consDmaName, consumerTile, *consRef,
834 ObjectFifoRole::Fill, fifo,
836 ? BDDimLayoutArrayAttr()
837 : consumerDims[consumerIndex],
840 createRouteEndpoint(loc, consDmaName, consumerTile,
841 fifo.streamsToConsumer(consumerIndex)
843 : fifo.getPlio() ? WireBundle::PLIO
847 destinations.push_back(
848 FlatSymbolRefAttr::get(builder.getContext(), consDmaName));
850 shimEndpointName[fifo] = consDmaName;
855 FlatSymbolRefAttr source =
856 FlatSymbolRefAttr::get(builder.getContext(), prodDmaName);
857 if (timeLink && timeLink.isMerge()) {
858 mergeSources[timeLink].push_back(source);
863 PacketInfoAttr header = fifo.packetHeader();
864 if (timeLink && timeLink.isDispatch()) {
865 dispatchTurns[timeLink][fifo] = destinations.front();
867 header = PacketInfoAttr::get(builder.getContext(), 0, std::nullopt);
869 RouteOp::create(builder, loc, builder.getArrayAttr({source}),
870 builder.getArrayAttr(destinations), header);
873 createTimeLinkRoutes();
878 DenseMap<Operation *, int> planned;
879 for (
auto [fifo, consumer, endpoint] : dmaEnds) {
880 std::optional<PlannedDmaEnd> end =
881 plannedEnds.lookup({fifo, consumer ?
static_cast<int>(*consumer) : -1});
884 ObjectFifoPoolOp pool = endpoint.getPoolOp();
885 assert(end->bds == endpoint.getNumBDs() && end->depth == pool.getDepth() &&
886 end->walkable == pool.canWalkWhole() &&
887 end->objectBytes == pool.getObjectSizeInBytes() &&
888 end->contiguousObjects ==
889 (pool.getContiguous() ? pool.getObjectsPerBuffer() : 0) &&
891 cast<RouteEndpoint>(endpoint.getOperation()).getRouteChannel() &&
892 "plannedDmaEnd disagrees with the endpoint the split made");
895 for (
auto [fifo, consumer, endpoint] : dmaEnds)
896 assert((endpoint.getTileLike().isShimTile() ||
897 planned.lookup(endpoint) == 1) &&
898 "an endpoint the split made is not planned exactly once");
901 SmallVector<Operation *> toErase;
902 for (ObjectFifoCreateOp fifo : fifos) {
904 auto shim = shimEndpointName.find(fifo);
905 if (shim != shimEndpointName.end()) {
906 (void)SymbolTable::replaceAllSymbolUses(
907 fifo, builder.getStringAttr(shim->second), device);
909 toErase.push_back(fifo);
911 for (
auto linkOp : device.getOps<ObjectFifoLinkOp>()) {
912 toErase.push_back(linkOp);
914 for (
auto allocOp : device.getOps<ObjectFifoAllocateOp>()) {
915 toErase.push_back(allocOp);
917 for (
auto regOp : device.getOps<ObjectFifoRegisterExternalBuffersOp>()) {
918 toErase.push_back(regOp);
920 for (Operation *op : toErase) {
927std::unique_ptr<OperationPass<DeviceOp>>
929 return std::make_unique<AIEObjectFifoSplitPass>();
@ None
Neither tile can address the other's memory module.
bool hasCoreAccess(DeviceOp device, mlir::Value tile, ObjectFifoCreateOp fifo, ObjectFifoPort port)
struct TileID { friend std::ostream &operator<<(std::ostream &os, const TileID &s) { os<< "TileID("<< s.col<< ", "<< s.row<< ")"; return os; } friend std::string to_string(const TileID &s) { std::ostringstream ss; ss<< s; return ss.str(); } friend llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const TileID &s) { os<< to_string(s); return os; } inline bool operator<(const TileID &rhs) const { return std::tie(col, row)< std::tie(rhs.col, rhs.row); } bool operator==(const TileID &rhs) const { return std::tie(col, row)==std::tie(rhs.col, rhs.row); } bool operator!=(const TileID &rhs) const { return !(*this==rhs);} int col, row;} TileID
std::unique_ptr< mlir::OperationPass< DeviceOp > > createAIEObjectFifoSplitPass()