AIEObjectFifoLowerDMAs.cpp

Go to the documentation of this file.

//===- AIEObjectFifoLowerDMAs.cpp -------------------------------*- C++ -*-===//
//
// Copyright (C) 2026 Advanced Micro Devices, Inc.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
 
#include "aie/Dialect/AIE/IR/AIEDialect.h"
#include "aie/Dialect/AIE/Transforms/AIEPasses.h"
 
#include "mlir/IR/Attributes.h"
#include "mlir/Pass/Pass.h"
#include "llvm/Support/MathExtras.h"
 
#include <map>
 
using namespace mlir;
using namespace xilinx;
using namespace xilinx::AIE;
 
namespace xilinx::AIE {
#define GEN_PASS_DEF_AIEOBJECTFIFOLOWERDMAS
#include "aie/Dialect/AIE/Transforms/AIEPasses.h.inc"
} // namespace xilinx::AIE
 
namespace {
 
/// One buffer descriptor's worth of work: which object, which slice of it, and
/// the locks guarding that slice.
struct Descriptor {
  Value buffer;
  int64_t offset;
  int64_t size;
  BDDimLayoutArrayAttr dimensions;
  BDPadLayoutArrayAttr padding;
  FlatSymbolRefAttr acquireLock;
  FlatSymbolRefAttr releaseLock;
  /// Which of the pool's objects this moves; also the index of the object's
  /// own binary lock, where the device uses those.
  int64_t object;
  /// Lock units taken and given back, where they are not one object's.
  std::optional<int64_t> acquireValue = std::nullopt;
  std::optional<int64_t> releaseValue = std::nullopt;
  /// How the address steps on each run of a descriptor that walks a
  /// contiguous pool's objects in turn.
  BDIterationAttr iteration = nullptr;
};
 
Block *findEndOpBlock(Region &region) {
  Block *endBlock = nullptr;
  for (Block &block : region) {
    if (!block.getOps<EndOp>().empty()) {
      endBlock = &block;
    }
  }
  return endBlock;
}
 
struct AIEObjectFifoLowerDMAsPass
    : public xilinx::AIE::impl::AIEObjectFifoLowerDMAsBase<
          AIEObjectFifoLowerDMAsPass> {
 
  DeviceOp device;
  int32_t nextSendTag = 0;
  llvm::DenseSet<int32_t> usedSendTags;
  OpBuilder builder{static_cast<MLIRContext *>(nullptr)};
 
  /// The DMA program of `tile`, created empty if the tile has none yet. A
  /// program shared by several endpoints is located at all of them.
  DmaBody dmaProgramFor(TileLike tile, Location loc) {
    Value wanted = tile->getResult(0);
    for (auto program : device.getOps<DmaBody>()) {
      if (program.getTile() == wanted) {
        program->setLoc(
            FusedLoc::get(loc.getContext(), {program->getLoc(), loc}));
        return program;
      }
    }
 
    OpBuilder::InsertionGuard g(builder);
    builder.setInsertionPoint(device.getBody()->getTerminator());
    DmaBody program;
    if (tile.isShimTile()) {
      program = ShimDMAOp::create(builder, loc, builder.getIndexType(), wanted);
    } else if (tile.isMemTile()) {
      program = MemTileDMAOp::create(builder, loc, wanted);
    } else {
      program = MemOp::create(builder, loc, wanted);
    }
 
    builder.setInsertionPointToStart(&program.getDmaBody().emplaceBlock());
    EndOp::create(builder, loc);
    return program;
  }
 
  /// Objects the endpoint moves.
  SmallVector<Value> buffersOf(ObjectFifoPoolOp pool) {
    SmallVector<Value> buffers;
    for (BufferLike buffer : pool.getBufferOps()) {
      buffers.push_back(buffer.getBuffer());
    }
    return buffers;
  }
 
  /// A contiguous pool's one buffer holds its objects side by side. Its
  /// filling end steps through them with one descriptor; its draining end
  /// walks the stream with one descriptor per output, each taking the objects
  /// up to the newest it reads and giving back those no later one reads.
  SmallVector<Descriptor>
  contiguousDescriptors(ObjectFifoDmaEndpointOp endpoint, Value buffer,
                        bool drains) {
    ObjectFifoPoolOp pool = endpoint.getPoolOp();
    ObjectFifoSegmentOp segment = endpoint.getSelectedSegments().front();
    FlatSymbolRefAttr take =
        drains ? segment.getConsumeLockAttr() : segment.getProduceLockAttr();
    FlatSymbolRefAttr give =
        drains ? segment.getProduceLockAttr() : segment.getConsumeLockAttr();
    MLIRContext *context = builder.getContext();
    if (!endpoint.walksStream()) {
      BDDimLayoutArrayAttr dims;
      if (auto all = endpoint.getDimensions(); all && !all->empty())
        dims = (*all)[0];
      int64_t object = pool.getObjectSize();
      int64_t first = 0;
      if (auto zeros = pool.getZeroObjects())
        first = (*zeros)[0] * object;
      int64_t walkers = endpoint.getNumDescriptors();
      SmallVector<Descriptor> descriptors;
      for (int64_t j = 0; j < walkers; j++)
        descriptors.push_back(
            {buffer, first + j * object + segment.getOffset(),
             segment.getSize(), dims, BDPadLayoutArrayAttr(), take, give, j,
             std::nullopt, std::nullopt,
             BDIterationAttr::get(context, pool.getDepth() / walkers,
                                  walkers * object, 0)});
      return descriptors;
    }
 
    StreamWindow window;
    if (failed(endpoint.getStreamWindow(window)))
      return {};
    BDDimLayoutArrayAttr dims =
        BDDimLayoutArrayAttr::get(context, window.inner);
    SmallVector<Descriptor> descriptors;
    for (StreamWindowStep step : streamWindowSteps(window))
      descriptors.push_back({buffer, step.first * window.objectSize,
                             window.transfer(), dims, BDPadLayoutArrayAttr(),
                             take, give, 0, step.acquire, step.release,
                             nullptr});
    return descriptors;
  }
 
  /// Buffer-major, segment-minor: the DMA walks the objects in turn, and within
  /// each object the slices this endpoint is responsible for.
  SmallVector<Descriptor> descriptorsFor(ObjectFifoDmaEndpointOp endpoint,
                                         ArrayRef<Value> buffers, bool drains) {
    if (endpoint.getPoolOp().getContiguous())
      return contiguousDescriptors(endpoint, buffers.front(), drains);
    std::vector<ObjectFifoSegmentOp> segments = endpoint.getSelectedSegments();
    SmallVector<BDDimLayoutArrayAttr> dimensions;
    if (auto allDimensions = endpoint.getDimensions()) {
      llvm::append_range(dimensions, *allDimensions);
    }
    SmallVector<BDPadLayoutArrayAttr> padding;
    if (auto allPadding = endpoint.getPadDimensions()) {
      llvm::append_range(padding, *allPadding);
    }
 
    SmallVector<Descriptor> descriptors;
    for (auto [index, buffer] : llvm::enumerate(buffers)) {
      for (auto [position, segment] : llvm::enumerate(segments)) {
        descriptors.push_back(
            {buffer, segment.getOffset(), segment.getSize(),
             position < dimensions.size() ? dimensions[position]
                                          : BDDimLayoutArrayAttr(),
             position < padding.size() ? padding[position]
                                       : BDPadLayoutArrayAttr(),
             drains ? segment.getConsumeLockAttr()
                    : segment.getProduceLockAttr(),
             drains ? segment.getProduceLockAttr()
                    : segment.getConsumeLockAttr(),
             static_cast<int64_t>(index)});
      }
    }
    return descriptors;
  }
 
  void emitDescriptor(ObjectFifoDmaEndpointOp endpoint, ObjectFifoPoolOp pool,
                      Descriptor &descriptor, PacketInfoAttr packet,
                      Block *successor, bool binaryLocks, bool drains,
                      int64_t transmittedObject = -1,
                      ArrayRef<int32_t> sendTags = {}) {
    Location loc = endpoint.getLoc();
    int count = drains ? 1 : pool.getRepeatCount().value_or(1);
 
    LockOp acquireLock, releaseLock;
    int acquireValue = count, releaseValue = count;
    auto action = LockAction::AcquireGreaterEqual;
    if (binaryLocks) {
      // A binary lock's value encodes which side owns the buffer.
      SmallVector<LockOp> locks = pool.getLockOps();
      if (descriptor.object < (int64_t)locks.size()) {
        acquireLock = releaseLock = locks[descriptor.object];
        acquireValue = drains ? 1 : 0;
        releaseValue = drains ? 0 : 1;
        action = LockAction::Acquire;
      }
    } else if (descriptor.acquireLock) {
      acquireLock =
          lookupNamedOp<LockOp>(device, descriptor.acquireLock.getAttr());
      releaseLock =
          lookupNamedOp<LockOp>(device, descriptor.releaseLock.getAttr());
    }
 
    if (descriptor.acquireValue)
      acquireValue = *descriptor.acquireValue;
    if (descriptor.releaseValue)
      releaseValue = *descriptor.releaseValue;
    if (acquireLock) {
      UseLockOp::create(builder, loc, acquireLock, action, acquireValue);
    }
    if (packet) {
      DMABDPACKETOp::create(builder, loc, packet.getPktType(),
                            packet.assignedId());
    }
 
    DMABDOp bd;
    if (descriptor.dimensions && drains && descriptor.padding) {
      bd = DMABDOp::create(builder, loc, descriptor.buffer, descriptor.offset,
                           paddedSize(descriptor.dimensions, descriptor.padding,
                                      descriptor.size),
                           descriptor.dimensions, descriptor.padding);
    } else if (descriptor.dimensions) {
      bd = DMABDOp::create(builder, loc, descriptor.buffer, descriptor.offset,
                           descriptor.size, descriptor.dimensions);
    } else {
      bd = DMABDOp::create(builder, loc, descriptor.buffer, descriptor.offset,
                           descriptor.size);
    }
    if (endpoint.getOutOfOrder()) {
      auto tags =
          drains ? endpoint.getOutOfOrderTags() : pool.getOutOfOrderTags();
      if (tags) {
        assert(descriptor.object < static_cast<int64_t>(tags->size()) &&
               "the allocator names every descriptor of an out-of-order pool");
        int64_t slot = transmittedObject >= 0 && drains
                           ? transmittedObject % tags->size()
                           : descriptor.object;
        int32_t tag = (*tags)[slot];
        if (endpoint.getRouteDirection() == DMAChannelDir::MM2S)
          bd.setOutOfOrderTag(tag);
        else
          bd.setBdTag(tag);
      }
    }
    if (descriptor.iteration && descriptor.iteration.getSize() > 1)
      bd.setIterationAttr(descriptor.iteration);
    if (!sendTags.empty()) {
      assert(descriptor.object < static_cast<int64_t>(sendTags.size()) &&
             "a send tag names every buffer of a core-pushed pool");
      bd.setBdTag(sendTags[descriptor.object]);
    }
 
    if (releaseLock) {
      UseLockOp::create(builder, loc, releaseLock, LockAction::Release,
                        releaseValue);
    }
    NextBDOp::create(builder, loc, successor);
  }
 
  /// A `dma_start` of `channel` on the endpoint's tile, chained after the
  /// tile's others; returns the first BD's block and the program's end block.
  std::pair<Block *, Block *> startChain(ObjectFifoDmaEndpointOp endpoint,
                                         int channel, int taskCount,
                                         bool corePushes = false) {
    Location loc = endpoint.getLoc();
    DmaBody program = dmaProgramFor(endpoint.getTileLike(), loc);
    Block *endBlock = findEndOpBlock(program.getDmaBody());
    Block *lastDmaBlock = endBlock->getSinglePredecessor();
    Block *dmaBlock = builder.createBlock(endBlock);
    Block *bdBlock = builder.createBlock(endBlock);
 
    builder.setInsertionPointToStart(dmaBlock);
    DMAStartOp::create(builder, loc, endpoint.getRouteDirection(), channel,
                       taskCount, endpoint.getPadValue(),
                       endpoint.getOutOfOrder() &&
                           endpoint.getRouteDirection() == DMAChannelDir::S2MM,
                       corePushes, bdBlock, endBlock);
    if (lastDmaBlock) {
      lastDmaBlock->getTerminator()->setSuccessor(dmaBlock, 1);
    }
    return {bdBlock, endBlock};
  }
 
  /// Endpoints taking turns on one channel share one chain: the k-th BD moves
  /// the next object of the endpoint `turns` names k-th, through its own pool.
  LogicalResult lowerTurns(ObjectFifoDmaEndpointOp first, int channel) {
    SmallVector<ObjectFifoDmaEndpointOp> ends = turnEndpoints(first);
    SmallVector<SmallVector<Descriptor>> descriptors;
    for (ObjectFifoDmaEndpointOp end : ends) {
      descriptors.push_back(
          descriptorsFor(end, buffersOf(end.getPoolOp()), end.drains()));
      if (static_cast<int64_t>(descriptors.back().size()) !=
              end.getNumDescriptors() ||
          descriptors.back().empty())
        return end.emitOpError("takes turns, but lowers to ")
               << descriptors.back().size() << " descriptors per pass for "
               << end.getNumDescriptors();
    }
    SmallVector<unsigned> turns;
    for (auto name : first.getTurnsAttr().getAsRange<FlatSymbolRefAttr>())
      turns.push_back(llvm::find_if(ends,
                                    [&](ObjectFifoDmaEndpointOp end) {
                                      return end.getSymNameAttr() ==
                                             name.getAttr();
                                    }) -
                      ends.begin());
 
    auto [bdBlock, endBlock] = startChain(first, channel, /*taskCount=*/0);
    bool binaryLocks = !device.getTargetModel().hasProperty(
        AIETargetModel::UsesSemaphoreLocks);
    SmallVector<size_t> next(ends.size(), 0);
    int64_t total = first.getNumBDs();
    Block *current = bdBlock;
    for (int64_t k = 0; k < total; k++) {
      unsigned i = turns[k % turns.size()];
      Block *successor =
          k + 1 < total ? builder.createBlock(endBlock) : bdBlock;
      builder.setInsertionPointToStart(current);
      emitDescriptor(ends[i], ends[i].getPoolOp(), descriptors[i][next[i]],
                     PacketInfoAttr(), successor, binaryLocks,
                     ends[i].drains());
      next[i] = (next[i] + 1) % descriptors[i].size();
      current = successor;
    }
    return success();
  }
 
  LogicalResult lowerEndpoint(ObjectFifoDmaEndpointOp endpoint, int channel) {
    bool drains = endpoint.drains();
    ObjectFifoPoolOp pool = endpoint.getPoolOp();
    SmallVector<Value> buffers = buffersOf(pool);
    if (buffers.empty()) {
      return success();
    }
 
    SmallVector<Descriptor> descriptors =
        descriptorsFor(endpoint, buffers, drains);
    if (descriptors.empty()) {
      return success();
    }
    // Allocation budgeted the channel's BDs by getNumBDs.
    if (static_cast<int64_t>(descriptors.size()) !=
        endpoint.getNumDescriptors())
      return endpoint.emitOpError()
             << "lowers to " << descriptors.size()
             << " descriptors per pass, but its channel was allocated for "
             << endpoint.getNumDescriptors();
 
    Location loc = endpoint.getLoc();
    // emitDescriptor scales the filling end's lock count instead.
    int64_t repeat = endpoint.getRepeat();
    std::optional<int32_t> iterCount = endpoint.getIterCount();
 
    bool repeatInHardware = endpoint.repeatsInHardware();
    // A dispatching chain runs until its turns line up with the pool's
    // objects again, so one run covers that many passes.
    int64_t passes = endpoint.getDispatchPasses();
    int taskCount = iterCount ? *iterCount / passes - 1
                              : (repeatInHardware ? repeat - 1 : 0);
    int64_t copies = repeatInHardware ? 1 : repeat;
 
    // With push_queue the configuration pushes nothing: the core pushes
    // the descriptor of each object it releases. Each descriptor then runs
    // on its own, so a pool whose buffers do not map one-to-one onto them
    // leaves the core no descriptor to name.
    bool corePushes =
        drains && endpoint.getReleaseStrategy() == ObjectFifoRelease::PushQueue;
    SmallVector<int32_t> sendTags;
    if (corePushes) {
      bool hasRelease = false;
      ObjectFifoCoreEndpointOp producer;
      auto sentSegments = endpoint.getSelectedSegments();
      for (auto coreEnd : device.getOps<ObjectFifoCoreEndpointOp>()) {
        if (coreEnd.drains() || coreEnd.getPoolOp() != pool ||
            coreEnd.getTile() != endpoint.getTile())
          continue;
        auto filledSegments = coreEnd.getSelectedSegments();
        if (!llvm::any_of(sentSegments, [&](ObjectFifoSegmentOp segment) {
              return llvm::is_contained(filledSegments, segment);
            }))
          continue;
        if (producer)
          return endpoint.emitOpError(
              "push_queue requires a single core filler for its segment");
        producer = coreEnd;
        for (auto core : device.getOps<CoreOp>()) {
          if (core.getTile() == coreEnd.getTile())
            core.walk([&](ObjectFifoReleaseOp release) {
              hasRelease |= release.getObjFifoName() == coreEnd.getSymName();
            });
        }
      }
      if (!hasRelease)
        return endpoint.emitOpError(
            "push_queue requires producer releases in a "
            "local core to submit send tasks");
      if (passes != 1)
        return endpoint.emitOpError(
            "push_queue requires one dispatch pass per object");
      // Each release submits one task, with repetitions carried by that task.
      taskCount = 0;
      if (static_cast<int64_t>(descriptors.size()) != pool.getDepth())
        return endpoint.emitOpError("sends ")
               << descriptors.size() << " descriptors for a pool of "
               << pool.getDepth()
               << " objects, so a release names no one of them; push_queue "
               << "needs one descriptor per object";
      for (size_t slot = 0; slot < descriptors.size(); ++slot) {
        while (usedSendTags.contains(nextSendTag))
          ++nextSendTag;
        sendTags.push_back(nextSendTag++);
      }
      producer.setCorePushAttr(
          ObjectFifoCorePushAttr::get(pool.getContext(), channel, sendTags));
    }
 
    SmallVector<PacketInfoAttr> turns;
    if (std::optional<ArrayAttr> headers = endpoint.getDispatchPackets()) {
      llvm::append_range(turns, headers->getAsRange<PacketInfoAttr>());
    } else if (endpoint.getDispatch()) {
      return endpoint.emitOpError(
          "has no dispatchPackets; run --aie-objectfifo-allocate");
    }
    // A contiguous pool's descriptor steps through every object, so it has no
    // one object for a turn to follow.
    if (!turns.empty() && pool.getContiguous())
      return endpoint.emitOpError(
          "dispatches from a contiguous pool, which has no per-object BDs");
    int64_t depth = pool.getDepth();
 
    auto [bdBlock, endBlock] =
        startChain(endpoint, channel, taskCount, corePushes);
    bool binaryLocks = !device.getTargetModel().hasProperty(
        AIETargetModel::UsesSemaphoreLocks);
    size_t total = descriptors.size() * copies * passes;
    size_t emitted = 0;
    Block *current = bdBlock;
    for (int64_t pass = 0; pass < passes; pass++) {
      for (Descriptor &descriptor : descriptors) {
        PacketInfoAttr packet = endpoint.getPacketAttr();
        if (!turns.empty())
          packet = turns[(pass * depth + descriptor.object) % turns.size()];
        for (int64_t copy = 0; copy < copies; copy++) {
          Block *successor;
          if (emitted + 1 < total) {
            successor = builder.createBlock(endBlock);
          } else if (iterCount) {
            // A bounded chain exits after its final iteration.
            successor = builder.createBlock(endBlock);
            builder.setInsertionPointToStart(successor);
            EndOp::create(builder, loc);
          } else {
            successor = bdBlock;
          }
          builder.setInsertionPointToStart(current);
          emitDescriptor(endpoint, pool, descriptor, packet, successor,
                         binaryLocks, drains, emitted, sendTags);
          current = successor;
          emitted++;
        }
      }
    }
    return success();
  }
 
  LogicalResult
  lowerOutOfOrderReceivers(ArrayRef<ObjectFifoDmaEndpointOp> endpoints,
                           int channel) {
    auto first = endpoints.front();
    SmallVector<SmallVector<Descriptor>> descriptors;
    int64_t objects = 0;
    bool unbounded = false;
    size_t totalBDs = 0;
    for (auto endpoint : endpoints) {
      descriptors.push_back(
          descriptorsFor(endpoint, buffersOf(endpoint.getPoolOp()), false));
      if (descriptors.back().empty() ||
          static_cast<int64_t>(descriptors.back().size()) !=
              endpoint.getNumDescriptors())
        return endpoint.emitOpError(
            "incorrect number of receiving descriptors");
      totalBDs += descriptors.back().size();
      if (auto iterations = endpoint.getIterCount())
        objects += descriptors.back().size() * *iterations;
      else
        unbounded = true;
    }
    int64_t perTask = device.getTargetModel().getMaxRepeatCount() + 1;
    if (unbounded)
      objects = perTask;
    int64_t tasks = llvm::divideCeil(objects, perTask);
    if (tasks > device.getTargetModel().getDmaTaskQueueDepth())
      return first.emitOpError("shared receiving channel needs ")
             << tasks << " initial tasks for " << objects
             << " objects, exceeding its queue capacity; use an unbounded "
                "receiver with core replenishment";
 
    auto [head, endBlock] =
        startChain(first, channel, std::min(objects, perTask) - 1);
    for (objects -= perTask; objects > 0; objects -= perTask) {
      auto [unusedHead, unusedEnd] =
          startChain(first, channel, std::min(objects, perTask) - 1);
      unusedHead->replaceAllUsesWith(head);
      unusedHead->erase();
    }
    bool binaryLocks = !device.getTargetModel().hasProperty(
        AIETargetModel::UsesSemaphoreLocks);
    Block *current = head;
    size_t emitted = 0;
    for (auto [index, bds] : llvm::enumerate(descriptors)) {
      ObjectFifoDmaEndpointOp endpoint = endpoints[index];
      for (auto &descriptor : bds) {
        Block *successor = builder.createBlock(endBlock);
        if (++emitted == totalBDs) {
          builder.setInsertionPointToStart(successor);
          EndOp::create(builder, endpoint.getLoc());
        }
        builder.setInsertionPointToStart(current);
        emitDescriptor(endpoint, endpoint.getPoolOp(), descriptor,
                       endpoint.getPacketAttr(), successor, binaryLocks, false);
        current = successor;
      }
    }
    return success();
  }
 
  void runOnOperation() override {
    device = getOperation();
    builder = OpBuilder(device.getContext());
 
    // Reserve tags from both pool allocation and hand-written descriptors.
    nextSendTag = 0;
    usedSendTags.clear();
    for (auto pool : device.getOps<ObjectFifoPoolOp>())
      if (auto tags = pool.getOutOfOrderTags())
        usedSendTags.insert(tags->begin(), tags->end());
    for (auto endpoint : device.getOps<ObjectFifoCoreEndpointOp>())
      if (auto push = endpoint.getCorePushAttr())
        usedSendTags.insert(push.getBdTags().begin(), push.getBdTags().end());
    device.walk([&](DMABDOp bd) {
      if (auto tag = bd.getBdTag())
        usedSendTags.insert(*tag);
    });
 
    SmallVector<ObjectFifoDmaEndpointOp> endpoints(
        device.getOps<ObjectFifoDmaEndpointOp>());
    DenseMap<Operation *, ObjectFifoDmaEndpointOp> pushedDrainers;
    for (ObjectFifoDmaEndpointOp endpoint : endpoints) {
      if (!endpoint.drains() ||
          endpoint.getReleaseStrategy() != ObjectFifoRelease::PushQueue)
        continue;
      auto [it, inserted] =
          pushedDrainers.try_emplace(endpoint.getPoolOp(), endpoint);
      if (!inserted) {
        auto diag = endpoint.emitOpError(
            "a pool supports only one core-pushed DMA drainer");
        diag.attachNote(it->second.getLoc())
            << "first core-pushed drainer is here";
        return signalPassFailure();
      }
    }
    // A tile's chains, and so the BD ids they get, follow its channels rather
    // than the order the endpoints were written in.
    DenseMap<Value, size_t> tileOrder;
    for (ObjectFifoDmaEndpointOp endpoint : endpoints)
      tileOrder.try_emplace(endpoint.getTile(), tileOrder.size());
    llvm::stable_sort(
        endpoints, [&](ObjectFifoDmaEndpointOp a, ObjectFifoDmaEndpointOp b) {
          return std::make_tuple(tileOrder.lookup(a.getTile()),
                                 a.getRouteDirection(),
                                 a.getChannelIndex().value_or(-1)) <
                 std::make_tuple(tileOrder.lookup(b.getTile()),
                                 b.getRouteDirection(),
                                 b.getChannelIndex().value_or(-1));
        });
    using ChannelKey = std::tuple<int, int, int>;
    std::map<ChannelKey, SmallVector<ObjectFifoDmaEndpointOp>> receivers;
    for (auto endpoint : endpoints) {
      if (endpoint.getOutOfOrder() && !endpoint.drains()) {
        auto tile = endpoint.getTile().getDefiningOp<TileOp>();
        receivers[{tile.getCol(), tile.getRow(),
                   endpoint.getChannelIndex().value_or(-1)}]
            .push_back(endpoint);
      }
    }
    for (ObjectFifoDmaEndpointOp endpoint : endpoints) {
      std::optional<int> channel = endpoint.getChannelIndex();
      if (!channel) {
        endpoint.emitOpError("has no channel; run --aie-objectfifo-allocate");
        return signalPassFailure();
      }
      if (endpoint.getOutOfOrder() && !endpoint.drains()) {
        auto tile = endpoint.getTile().getDefiningOp<TileOp>();
        auto &group = receivers.at({tile.getCol(), tile.getRow(), *channel});
        if (group.front() == endpoint &&
            failed(lowerOutOfOrderReceivers(group, *channel)))
          return signalPassFailure();
        continue;
      }
      SmallVector<ObjectFifoDmaEndpointOp> turns = turnEndpoints(endpoint);
      if (turns.empty() ? failed(lowerEndpoint(endpoint, *channel))
                        : turns.front() == endpoint &&
                              failed(lowerTurns(endpoint, *channel)))
        return signalPassFailure();
    }
    for (ObjectFifoDmaEndpointOp endpoint : endpoints)
      endpoint.erase();
 
    // Nothing to program at the far end of these; the routes they asked for
    // have been drawn.
    for (auto endpoint :
         llvm::make_early_inc_range(device.getOps<RouteEndpointOp>())) {
      endpoint.erase();
    }
  }
};
 
} // namespace
 
std::unique_ptr<OperationPass<DeviceOp>>
xilinx::AIE::createAIEObjectFifoLowerDMAsPass() {
  return std::make_unique<AIEObjectFifoLowerDMAsPass>();
}