AIETargetXAIEV2.cpp

Go to the documentation of this file.

//===- AIETargetXAIEV2.cpp --------------------------------------*- C++ -*-===//
//
// Copyright (C) 2021-2022 Xilinx, Inc.
// Copyright (C) 2021-2023 Advanced Micro Devices, Inc.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "aie/Targets/AIETargetShared.h"
 
#include "aie/Dialect/AIE/IR/AIEDialect.h"
#include "aie/Dialect/AIEX/IR/AIEXDialect.h"
#include "aie/Targets/AIETargets.h"
 
#include "mlir/Dialect/Utils/StaticValueUtils.h"
#include "mlir/IR/Attributes.h"
#include "mlir/IR/IRMapping.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Tools/mlir-translate/MlirTranslateMain.h"
 
#include "llvm/ADT/StringExtras.h"
#include "llvm/IR/Module.h"
 
#include <limits>
 
using namespace mlir;
using namespace xilinx;
using namespace xilinx::AIE;
using namespace xilinx::AIEX;
 
// This string is output at the top of the lowered C++ code.
static const char *xaie_cpp_file_header = R"code(
// This file was auto-generated by aiecc --aie-generate-xaie.
 
#ifndef MLIR_AIE_QUIET
#define __mlir_aie_verbose(x) x
#else
#define __mlir_aie_verbose(x)
#endif
 
// The following is a wrapper for the common "if(call() != 0) return 1" pattern.
// Use this only in functions that return int. If the call this wrapper is used
// on does not succeed, the expanded code will exit out of the function
// containing this macro with an error code.
#define __mlir_aie_try(x) do { \
  AieRC ret = (x); \
  if(ret != XAIE_OK) { \
    return x; \
  } \
} while(0)
 
static XAie_DmaDimDesc *__mlir_aie_alloc_dim_desc(size_t ndims) {
  XAie_DmaDimDesc *ret = NULL;
  ret = (XAie_DmaDimDesc *)calloc(sizeof(XAie_DmaDimDesc), ndims);
  if(NULL == ret) {
    __mlir_aie_verbose(fprintf(stderr, "Allocating DmaDimDesc failed.\n"));
  }
  return ret;
}
 
)code";
 
static std::string tileLockStr(StringRef id, StringRef val) {
  std::string str;
  llvm::raw_string_ostream rss(str);
  // rss << "XAie_Lock(" << id << "," << val << ")";
  rss << "XAie_LockInit(" << id << "," << val << ")";
  return str;
}
 
// Translate mlir-aie WireBundle enum to string of aie-rt StrmSwPortType enum
static std::string wireBundleToPortType(WireBundle bundle) {
  switch (bundle) {
  case WireBundle::PLIO:
    return "PL";
  case WireBundle::TileControl:
    return "CTRL";
  default:
    return stringifyWireBundle(bundle).upper();
  }
}
 
// FIXME: this is a template pending a common DMA-like interface across the op
// types.
// blockMap: A map that gives a unique bd ID assignment for every block.
template <typename OpType>
static mlir::LogicalResult generateDMAConfig(OpType memOp, raw_ostream &output,
                                             const AIETargetModel &targetModel,
                                             DenseMap<Block *, int> blockMap) {
  StringRef enable = "XAIE_ENABLE";
  StringRef disable = "XAIE_DISABLE";
  StringRef deviceInstRef = "ctx->XAieDevInst";
 
  int col = memOp.colIndex();
  int row = memOp.rowIndex();
 
  // Get the region's entry block, then start traversing through the chain of
  // blocks.
  llvm::SetVector<Block *> blockVector =
      getOrderedChainOfBlocks(&memOp.getBody());
 
  llvm::SmallPtrSet<Block *, 8> oooBlocks =
      collectOutOfOrderBlocks(blockVector);
 
  for (auto *block : blockVector) {
    bool foundBdPacket = false;
    int packetType = 0;
    int packetID = 0;
    bool foundBd = false;
    uint64_t lenA = 0;
    int64_t offsetA = 0;
    int BaseAddrA = 0;
    int elementWidthInBytes = 0;
    int ndims = 0;
    int iterSize = 0, iterStride = 0, iterCurr = 0;
    uint32_t axcache = 0;
    std::optional<int> oooId;
    ArrayRef<BDDimLayoutAttr> dims;
    // Owning storage for the folded dims; must outlive `dims` (used far below).
    SmallVector<BDDimLayoutAttr> dimsStorage;
    //      StringRef FifoMode = disable; // FIXME: when to enable FIFO mode?
    // A legacy dma_bd_packet op is read first so that a packet attribute on
    // the dma_bd overrides it, matching AIERT.
    for (auto op : block->getOps<DMABDPACKETOp>()) {
      foundBdPacket = true;
      packetType = op.getPacketType();
      packetID = op.getPacketID();
    }
    for (auto op : block->getOps<DMABDOp>()) {
      foundBd = true;
      if (auto packetInfo = op.getPacket()) {
        foundBdPacket = true;
        packetType = packetInfo->getPktType();
        packetID = packetInfo->assignedId();
      }
      if (!targetModel.isShimNOCTile(col, row)) {
        std::optional<int32_t> bufferAddr = op.getBufferOp().getAddress();
        assert(bufferAddr && "buffer must have address assigned");
        BaseAddrA = *bufferAddr;
        int bufferCol = op.getBufferOp().getTileOp().colIndex();
        int bufferRow = op.getBufferOp().getTileOp().rowIndex();
 
        // Memtile DMAs can access neighboring tiles.
        if (targetModel.isMemTile(col, row)) {
          auto addrOffset = targetModel.getMemLocalBaseAddress(
              col, row, bufferCol, bufferRow);
          if (addrOffset)
            BaseAddrA += addrOffset.value();
        }
      }
 
      lenA = op.getLenInBytes();
      offsetA = op.getOffsetInBytes();
      elementWidthInBytes = op.getBufferElementTypeWidthInBytes();
      if (auto iter = op.getIteration()) {
        iterSize = iter->getSize();
        iterStride = iter->getStride();
        iterCurr = iter->getCurrent();
      }
      axcache = op.getAxcacheOrDefault();
      oooId = op.getOutOfOrderId();
      if (!op.getMixedSizes().empty()) {
        // Runtime-valued sizes/strides are not supported on this static path.
        for (mlir::OpFoldResult s : op.getMixedSizes())
          if (!mlir::getConstantIntValue(s))
            return op->emitOpError(
                "runtime-valued BD size/stride is not supported on the static "
                "XAIE lowering path; use compile-time constant sizes");
        for (mlir::OpFoldResult s : op.getMixedStrides())
          if (!mlir::getConstantIntValue(s))
            return op->emitOpError(
                "runtime-valued BD size/stride is not supported on the static "
                "XAIE lowering path; use compile-time constant sizes");
        std::optional<SmallVector<BDDimLayoutAttr>> folded =
            op.getConstantDimensions();
        if (!folded)
          return op->emitOpError("internal error folding BD dimensions");
        dimsStorage = std::move(*folded);
        dims = dimsStorage;
        ndims = dims.size();
      }
    }
 
    if (0 != ndims)
      if (!targetModel.hasProperty(AIETargetModel::UsesMultiDimensionalBDs))
        return memOp.emitOpError("DMA contains at least one multi-dimensional "
                                 "buffer descriptor. This is currently only "
                                 "supported for AIE-ML and later devices.");
    // Ensure BD length in bytes fits in the u32 `Len` argument of the libxaie
    // DMA APIs (XAie_DmaSetAddrLen / XAie_DmaSetMultiDimAddr).
    if (foundBd && lenA > std::numeric_limits<uint32_t>::max()) {
      return memOp.emitOpError("buffer descriptor length in bytes (")
             << lenA << ") does not fit in 32 bits";
    }
 
    int acqValue = 0, relValue = 0;
    bool hasAcq = false, hasRel = false;
    int acqLockID = 0, relLockID = 0;
    for (auto op : block->getOps<UseLockOp>()) {
      LockOp lock = cast<LockOp>(op.getLock().getDefiningOp());
      int lockCol = lock.colIndex();
      int lockRow = lock.rowIndex();
      int lockID = lock.getLockIDValue();
      // Memtile DMAs can access neighboring tiles.
      if (targetModel.isMemTile(col, row)) {
        auto lockOffset =
            targetModel.getLockLocalBaseIndex(col, row, lockCol, lockRow);
        if (lockOffset)
          lockID += lockOffset.value();
      }
 
      if (op.acquire() || op.acquireGE()) {
        hasAcq = true;
        acqLockID = lockID;
        auto value = op.getConstantValue();
        if (failed(value))
          return failure();
        acqValue = *value; // NOLINT(bugprone-unchecked-optional-access)
        if (op.acquireGE())
          acqValue = -acqValue;
      } else if (op.release()) {
        hasRel = true;
        relLockID = lockID;
        auto value = op.getConstantValue();
        if (failed(value))
          return failure();
        relValue = *value; // NOLINT(bugprone-unchecked-optional-access)
      } else {
        // unreachable for current targets
        return op.emitOpError("unsupported lock action");
      }
    }
 
    int bdNum = blockMap[block];
    if (foundBd) {
      // TODO For now, we are going to name each dma desc with loc and bd
      // which we assume is unique. This is strictly not enforced but in
      // practice, this is true
      output << "XAie_DmaDesc " << tileDMAInstStr(col, row, bdNum) << ";\n";
      output << "__mlir_aie_try(XAie_DmaDescInit(" << deviceInstRef << ", "
             << tileDMAInstRefStr(col, row, bdNum) << ", "
             << tileLocStr(col, row) << "));\n";
      if (hasAcq || hasRel) {
        output << "__mlir_aie_try(XAie_DmaSetLock("
               << tileDMAInstRefStr(col, row, bdNum) << ", "
               << "XAie_LockInit(" << acqLockID << "," << acqValue << "),"
               << "XAie_LockInit(" << relLockID << "," << relValue << ")));\n";
        if (!hasAcq)
          output << tileDMAInstStr(col, row, bdNum)
                 << ".LockDesc.LockAcqEn = " << disable << ";\n";
        if (!hasRel)
          output << tileDMAInstStr(col, row, bdNum)
                 << ".LockDesc.LockRelEn = " << disable << ";\n";
      }
 
      if (0 == ndims) {
        if (targetModel.isShimNOCTile(col, row)) {
          output << "__mlir_aie_try(XAie_DmaSetAddrLen("
                 << tileDMAInstRefStr(col, row, bdNum) << ", /* addrA */ "
                 << "mlir_aie_external_get_addr_myBuffer_" << col << row << "_"
                 << bdNum << "(), "
                 << " /* len */ " << lenA << "));\n";
          output << "__mlir_aie_try(XAie_DmaSetAxi("
                 << tileDMAInstRefStr(col, row, bdNum) << ", "
                 << "/* smid */ 0, "
                 << "/* burstlen */ 4, "
                 << "/* QoS */ 0, "
                 << "/* Cache */ " << axcache << ", "
                 << "/* Secure */ " << enable << "));\n";
        } else {
          if ((BaseAddrA + offsetA) % 4)
            return memOp.emitError("bd address must be 4B (32b) aligned");
          output << "__mlir_aie_try(XAie_DmaSetAddrLen("
                 << tileDMAInstRefStr(col, row, bdNum) << ", /* addrA */ "
                 << "0x" << llvm::utohexstr(BaseAddrA + offsetA) << ", "
                 << " /* len */ " << lenA << "));\n";
        }
      } else
        generateXAieDmaSetMultiDimAddr(output, ndims, dims, col, row, bdNum,
                                       BaseAddrA, offsetA, lenA,
                                       elementWidthInBytes, "1");
 
      if (iterSize > 1 && iterStride > 0) {
        int64_t strideInBytes =
            static_cast<int64_t>(iterStride) * elementWidthInBytes;
        if (strideInBytes % 4)
          return memOp.emitError("iteration_stride must result in a stride "
                                 "aligned to 32-bit words");
        int stepInWords = static_cast<int>(strideInBytes / 4);
        output << "__mlir_aie_try(XAie_DmaSetBdIteration("
               << tileDMAInstRefStr(col, row, bdNum) << ", " << stepInWords
               << ", " << iterSize << ", " << iterCurr << "));\n";
      }
 
      if (block->getNumSuccessors() > 0) {
        Block *nextBlock = block->getSuccessors()[0]; // should have only one
                                                      // successor block
 
        int enableNextBd = 1;
        if (!nextBlock->getOps<EndOp>().empty())
          enableNextBd = 0;
        if (oooBlocks.contains(block))
          enableNextBd = 0;
 
        int nextBdNum = blockMap[nextBlock];
        output << "__mlir_aie_try(XAie_DmaSetNextBd("
               << tileDMAInstRefStr(col, row, bdNum) << ", "
               << " /* nextbd */ " << nextBdNum << ", "
               << " /* enableNextBd */ " << enableNextBd << "));\n";
      }
 
      if (foundBdPacket) {
        output << "__mlir_aie_try(XAie_DmaSetPkt("
               << tileDMAInstRefStr(col, row, bdNum) << ", "
               << packetStr(packetID, packetType) << "));\n";
      }
      if (oooId.has_value()) {
        output << "__mlir_aie_try(XAie_DmaSetOutofOrderBdId("
               << tileDMAInstRefStr(col, row, bdNum) << ", " << *oooId
               << "));\n";
      }
      output << "__mlir_aie_try(XAie_DmaEnableBd("
             << tileDMAInstRefStr(col, row, bdNum) << "));\n";
      output << "__mlir_aie_try(XAie_DmaWriteBd(" << deviceInstRef << ", "
             << tileDMAInstRefStr(col, row, bdNum) << ", "
             << tileLocStr(col, row) << ", "
             << " /* bd */ " << bdNum << "));\n";
    }
  }
 
  for (auto *block : blockVector) {
    for (auto op : block->getOps<DMAStartOp>()) {
      int bdNum = blockMap[op.getDest()];
      StringRef dmaDir = stringifyDMAChannelDir(op.getChannelDir());
      int chNum = op.getChannelIndex();
      const auto &target_model = xilinx::AIE::getTargetModel(op);
      if (target_model.getTargetArch() == AIEArch::AIE1) {
        output << "__mlir_aie_try(XAie_DmaChannelPushBdToQueue("
               << deviceInstRef << ", " << tileLocStr(col, row) << ", "
               << "/* ChNum */" << chNum
               << ", "
               // TODO: interim handling until the physical dialect changes.
               << "/* dmaDir */ DMA_" << dmaDir << ", "
               << "/* BdNum */" << bdNum << "));\n";
      } else if (op.getOutOfOrder()) {
        // in english repeat_count==0 means "do it once" and don't repeat but
        // libxaie treats repeat_count=1 as do it once.
        int repeatCount = op.getRepeatCount() + 1;
        // StartBd is ignored and the in-order validity check is skipped.
        std::string oooSuffix = std::to_string(col) + "_" +
                                std::to_string(row) + "_" +
                                std::to_string(chNum);
        output << "XAie_DmaChannelDesc ooo_desc_" << oooSuffix << ";\n";
        output << "__mlir_aie_try(XAie_DmaChannelDescInit(" << deviceInstRef
               << ", &ooo_desc_" << oooSuffix << ", " << tileLocStr(col, row)
               << "));\n";
        output << "__mlir_aie_try(XAie_DmaChannelEnOutofOrder(&ooo_desc_"
               << oooSuffix << ", " << enable << "));\n";
        output << "__mlir_aie_try(XAie_DmaWriteChannel(" << deviceInstRef
               << ", &ooo_desc_" << oooSuffix << ", " << tileLocStr(col, row)
               << ", /* ChNum */" << chNum << ", /* dmaDir */ DMA_" << dmaDir
               << "));\n";
        output << "XAie_DmaDeclareQueueConfig(ooo_queue_" << oooSuffix
               << ", /* StartBd */ 0, /* Repeat */ " << repeatCount
               << ", /* EnToken */ " << disable << ", /* EnOutOfOrder */ "
               << enable << ");\n";
        output << "__mlir_aie_try(XAie_DmaChannelSetStartQueueGeneric("
               << deviceInstRef << ", " << tileLocStr(col, row)
               << ", /* ChNum */" << chNum << ", /* dmaDir */ DMA_" << dmaDir
               << ", &ooo_queue_" << oooSuffix << "));\n";
      } else {
        // in english repeat_count==0 means "do it once" and don't repeat but
        // libxaie treats repeat_count=1 as do it once.
        int repeatCount = op.getRepeatCount() + 1;
        output << "__mlir_aie_try(XAie_DmaChannelSetStartQueue("
               << deviceInstRef << ", " << tileLocStr(col, row) << ", "
               << "/* ChNum */" << chNum
               << ", "
               // TODO: interim handling until the physical dialect changes.
               << "/* dmaDir */ DMA_" << dmaDir << ", "
               << "/* BdNum */" << bdNum << ", "
               << "/* Repeat */ " << repeatCount << ", "
               << "/* EnToken */ "
               << "XAIE_DISABLE"
               << "));\n";
      }
      output << "__mlir_aie_try(XAie_DmaChannelEnable(" << deviceInstRef << ", "
             << tileLocStr(col, row) << ", "
             << "/* ChNum */ " << chNum
             << ", "
             // TODO: interim handling until the physical dialect changes.
             << "/* dmaDir */ DMA_" << dmaDir << "));\n";
    }
  }
  return success();
}
 
mlir::LogicalResult
xilinx::AIE::AIETranslateToXAIEV2(ModuleOp module, raw_ostream &output,
                                  llvm::StringRef deviceName) {
  StringRef ctx_p = "aie_libxaie_ctx_t* ctx";
  StringRef deviceInstRef = "ctx->XAieDevInst";
 
  DenseMap<TileID, Operation *> tiles;
  DenseMap<Operation *, SmallVector<BufferOp, 4>> buffers;
 
  DeviceOp targetOp = AIE::DeviceOp::getForSymbolInModule(module, deviceName);
  if (!targetOp)
    return module.emitOpError("expected AIE.device operation at toplevel");
  if (failed(verifyDMAChannelsResolved(targetOp)))
    return failure();
  const auto &targetModel = targetOp.getTargetModel();
 
  collectTiles(targetOp, tiles);
  collectBuffers(targetOp, buffers);
 
  //---------------------------------------------------------------------------
  // mlir_aie_init_libxaie
  //---------------------------------------------------------------------------
  output << xaie_cpp_file_header;
  output << "aie_libxaie_ctx_t* mlir_aie_init_libxaie() {\n";
  output << "  aie_libxaie_ctx_t *ctx = new aie_libxaie_ctx_t;\n";
  output << "  if (!ctx)\n";
  output << "    return 0;\n";
  output << "  ctx->XAieConfig = new XAie_Config;\n";
  output << "  if (!ctx->XAieConfig)\n";
  output << "    return 0;\n";
  output << "  ctx->XAieDevInst = new XAie_DevInst;\n";
  output << "  if (!ctx->XAieDevInst)\n";
  output << "    return 0;\n";
  auto arch = targetModel.getTargetArch();
  std::string AIE1_device("XAIE_DEV_GEN_AIE");
  std::string AIE2_device("XAIE_DEV_GEN_AIEML");
  std::string AIE2p_device("XAIE_DEV_GEN_AIE2P");
  std::string device;
  switch (arch) {
  case AIEArch::AIE1:
    device = AIE1_device;
    break;
  case AIEArch::AIE2:
    device = AIE2_device;
    break;
  case AIEArch::AIE2p:
    device = AIE2p_device;
    break;
  case AIEArch::AIE2ps:
    // AIE2PS configures the array through a CERT ELF instead of this libxaie
    // source (see AIE2PSTargetModel). Listed here so that a new AIEArch still
    // raises -Wswitch.
    return targetOp.emitOpError(
        "--aie-generate-xaie is not supported for the AIE2PS architecture");
  }
  output << "  ctx->XAieConfig->AieGen = " << device << ";\n";
  output << "  ctx->XAieConfig->BaseAddr = 0x20000000000;\n";
  output << "  ctx->XAieConfig->ColShift = " << targetModel.getColumnShift()
         << ";\n";
  output << "  ctx->XAieConfig->RowShift = " << targetModel.getRowShift()
         << ";\n";
  output << "  ctx->XAieConfig->NumRows = " << targetModel.rows() << ";\n";
  output << "  ctx->XAieConfig->NumCols = " << targetModel.columns() << ";\n";
  output << "  ctx->XAieConfig->ShimRowNum = 0;\n";
  output << "  ctx->XAieConfig->MemTileRowStart = 1;\n";
  output << "  ctx->XAieConfig->MemTileNumRows = "
         << targetModel.getNumMemTileRows() << ";\n";
  output << "  //  ctx->XAieConfig->ReservedRowStart = "
            "XAIE_RES_TILE_ROW_START;\n";
  output
      << "  //  ctx->XAieConfig->ReservedNumRows  = XAIE_RES_TILE_NUM_ROWS;\n";
  output << "  ctx->XAieConfig->AieTileRowStart = "
         << (1 + targetModel.getNumMemTileRows()) << ";\n";
  output << "  ctx->XAieConfig->AieTileNumRows = "
         << (targetModel.rows() - 1 - targetModel.getNumMemTileRows()) << ";\n";
  output << "  memset(ctx->XAieDevInst, 0, sizeof(XAie_DevInst));\n";
  output << "  return ctx;\n";
  output << "}\n";
  output << "\n";
 
  //---------------------------------------------------------------------------
  // mlir_aie_configure_cores
  //---------------------------------------------------------------------------
  // Defined below. configure_cores calls it at its end, once every ELF is
  // loaded, so a PT_LOAD cannot land on top of an initialized buffer.
  output << "int mlir_aie_initialize_buffers(" << ctx_p << ");\n";
  output << "int mlir_aie_configure_cores(" << ctx_p << ") {\n";
  // Reset each core.  Load the corresponding ELF file, if necessary.
  for (auto tileOp : targetOp.getOps<TileOp>()) {
    int col = tileOp.colIndex();
    int row = tileOp.rowIndex();
    if (tileOp.isShimTile() || tileOp.isMemTile()) {
      // Resets no needed with V2 kernel driver
    } else {
      // Resets no needed with V2 kernel driver
      output << "__mlir_aie_try(XAie_CoreReset(" << deviceInstRef << ", "
             << tileLocStr(col, row) << "));\n";
      output << "__mlir_aie_try(XAie_CoreDisable(" << deviceInstRef << ", "
             << tileLocStr(col, row) << "));\n";
      // Release locks
      int numLocks = targetModel.getNumLocks(col, row);
      output << "for (int l = 0; l < " << numLocks << "; ++l)\n"
             << "  __mlir_aie_try(XAie_LockRelease(" << deviceInstRef << ", "
             << tileLocStr(col, row) << ", XAie_LockInit(l, 0x0), 0));\n";
      if (auto coreOp = tileOp.getCoreOp()) {
        if (coreOp.isEmpty() && coreOp.getElfFile() == std::nullopt) {
          // This core has no MLIR code and references no ELF file -- it is
          // completely empty, so don't generate any code for it.
          continue;
        }
        std::string fileName;
        if (auto fileAttr = coreOp.getElfFile()) {
          fileName = fileAttr.value().str();
        } else {
          return coreOp.emitOpError()
                 << "Expected lowered ELF file to be given as attribute "
                    "`elf_file` for this core. Compile cores first.";
        }
        output << "{\n"
               << "AieRC RC = XAie_LoadElf(" << deviceInstRef << ", "
               << tileLocStr(col, row) << ", "
               << "(const char*)\"" << fileName << "\",0);\n";
        output << "if (RC != XAIE_OK)\n"
               << "    __mlir_aie_verbose(fprintf(stderr, \"Failed to load elf "
                  "for Core[%d,%d], ret is %d\\n\", "
               << std::to_string(col) << ", " << std::to_string(row)
               << ", RC));\n"
               << "assert(RC == XAIE_OK);\n"
               << "}\n";
      }
    }
  }
  // __mlir_aie_try assigns to an AieRC, and the generated entry points return
  // int, which C++ does not convert implicitly.
  output << "{\n";
  output << "  int rc = mlir_aie_initialize_buffers(ctx);\n";
  output << "  if (rc != XAIE_OK)\n";
  output << "    return rc;\n";
  output << "}\n";
  output << "return XAIE_OK;\n";
  output << "} // mlir_aie_configure_cores\n\n";
 
  //---------------------------------------------------------------------------
  // mlir_aie_start_cores
  //---------------------------------------------------------------------------
  output << "int mlir_aie_start_cores(" << ctx_p << ") {\n";
  // Start execution of all the cores.
  for (auto tileOp : targetOp.getOps<TileOp>()) {
    int col = tileOp.colIndex();
    int row = tileOp.rowIndex();
    if (!tileOp.isShimTile() && !tileOp.isMemTile()) {
      output << "__mlir_aie_try(XAie_CoreUnreset(" << deviceInstRef << ", "
             << tileLocStr(col, row) << "));\n";
      output << "__mlir_aie_try(XAie_CoreEnable(" << deviceInstRef << ", "
             << tileLocStr(col, row) << "));\n";
    }
  }
  output << "return XAIE_OK;\n";
  output << "} // mlir_aie_start_cores\n\n";
 
  //---------------------------------------------------------------------------
  // mlir_aie_configure_dmas
  //---------------------------------------------------------------------------
  output << "int mlir_aie_configure_dmas(" << ctx_p << ") {\n";
 
  for (auto memOp : targetOp.getOps<MemOp>()) {
    DenseMap<Block *, int> blockMap;
 
    // Use bd_ids, if specified.
    int bdNum = 0;
    for (auto &block : memOp.getBody()) {
      auto bdOps = block.getOps<DMABDOp>();
      if (bdOps.empty())
        continue;
      std::optional<int32_t> id = (*bdOps.begin()).getBdId();
      blockMap[&block] = id.has_value() ? *id : bdNum;
      bdNum++;
    }
    auto result = generateDMAConfig(memOp, output, targetModel, blockMap);
    if (result.failed())
      return result;
  }
  for (auto memOp : targetOp.getOps<MemTileDMAOp>()) {
    DenseMap<Block *, int> blockMap;
    // Memtiles have restrictions on which channels can access which BDs
    DenseMap<Block *, int> channelMap;
 
    for (auto &block : memOp.getBody()) {
      for (auto op : block.getOps<DMAStartOp>()) {
        int chNum = op.getChannelIndex();
        channelMap[&block] = chNum;
        auto *dest = op.getDest();
        while (dest) {
          channelMap[dest] = chNum;
          if (dest->getSuccessors().empty())
            break;
          dest = dest->getSuccessors()[0];
          if (channelMap.count(dest))
            break;
        }
      }
    }
 
    // Assign each block a BD number, if unassigned
    int evenBdNum = 0;
    int oddBdNum = 24;
    for (auto &block : memOp.getBody()) {
      auto bdOps = block.getOps<DMABDOp>();
      if (bdOps.empty())
        continue;
      assert(channelMap.count(&block));
      if (std::optional<int32_t> id = (*bdOps.begin()).getBdId())
        blockMap[&block] = *id;
      else if (channelMap[&block] & 1)
        blockMap[&block] = oddBdNum++;
      else
        blockMap[&block] = evenBdNum++;
    }
    auto result = generateDMAConfig(memOp, output, targetModel, blockMap);
    if (result.failed())
      return result;
  }
 
  output << "return XAIE_OK;\n";
  output << "} // mlir_aie_configure_dmas\n\n";
 
  for (auto op : targetOp.getOps<ExternalBufferOp>()) {
    if (op.hasName()) {
      output << "static u64 _mlir_aie_external_" << op.name().getValue()
             << ";\n";
      output << "static bool _mlir_aie_external_set_" << op.name().getValue()
             << " = false;\n";
 
      output << "void mlir_aie_external_set_addr_" << op.name().getValue()
             << "(" << ctx_p << ", u64 VA) {\n"
             << "  u64 device_address = mlir_aie_get_device_address(ctx, (void "
                "*)VA);\n"
             << "    _mlir_aie_external_set_" << op.name().getValue()
             << " = true;\n"
             << "    _mlir_aie_external_" << op.name().getValue()
             << " = device_address;\n"
             << "}\n";
    }
  }
 
  // ShimDMA Config
  //  int index = 0;
  for (auto op : targetOp.getOps<ShimDMAOp>()) {
    int col = op.colIndex();
    int row = op.rowIndex();
 
    DenseMap<Block *, int> blockMap;
    {
      // Assign each block a BD number, if unassigned
      int bdNum = 0;
      for (auto &block : op.getBody()) {
        auto bdOps = block.getOps<DMABDOp>();
        if (bdOps.empty())
          continue;
        std::optional<int32_t> id = (*bdOps.begin()).getBdId();
        int slot = id.has_value() ? *id : bdNum;
        blockMap[&block] = slot;
        uint64_t offset = 0;
        for (auto op : block.getOps<DMABDOp>()) {
          offset = op.getOffsetInBytes();
          auto buffer = cast<ExternalBufferOp>(op.getBuffer().getDefiningOp());
 
          output << "u64 mlir_aie_external_get_addr_myBuffer_" << col << row
                 << "_" << slot << "(void) {\n"
                 << "    assert(_mlir_aie_external_set_"
                 << buffer.name().getValue() << ");\n"
                 << "    return _mlir_aie_external_" << buffer.name().getValue()
                 << " + " << llvm::utohexstr(offset) << ";\n"
                 << "}\n";
        }
 
        bdNum++;
      }
    }
 
    output << "int mlir_aie_configure_shimdma_" << col << row << "(" << ctx_p
           << ") {\n";
    auto result = generateDMAConfig(op, output, targetModel, blockMap);
    if (result.failed())
      return result;
    output << "return XAIE_OK;\n";
    output << "} // mlir_aie_configure_shimdma\n\n";
  }
 
  //---------------------------------------------------------------------------
  // mlir_aie_initialize_locks
  //---------------------------------------------------------------------------
  output << "int mlir_aie_initialize_locks(" << ctx_p << ") {\n";
  // Lock configuration
  targetOp.walk<WalkOrder::PreOrder>([&](LockOp lock) {
    TileOp tile = lock.getTileOp();
    int col = tile.colIndex();
    int row = tile.rowIndex();
    int lockID = lock.getLockIDValue();
    auto init = lock.getInit();
    if (init)
      output << "__mlir_aie_try(XAie_LockSetValue(" << deviceInstRef << ", "
             << tileLocStr(col, row) << ", "
             << "XAie_LockInit(" << lockID << ", " << *init << ")));\n";
  });
  output << "return XAIE_OK;\n";
  output << "} // mlir_aie_initialize_locks\n";
 
  //---------------------------------------------------------------------------
  // mlir_aie_initialize_buffers
  //---------------------------------------------------------------------------
  // A buffer's initial_value is device state, written by whoever configures
  // the device: the CDO writer on the NPU path, this function on the simulator
  // path. This walks every tile, because a memtile buffer has no core and so no
  // other route. mlir_aie_configure_cores calls this, so a host that already
  // configures its cores needs no change.
  output << "int mlir_aie_initialize_buffers(" << ctx_p << ") {\n";
  auto initBuffersResult = targetOp.walk<WalkOrder::PreOrder>([&](BufferOp
                                                                      buffer) {
    auto initialValue = buffer.getInitialValue();
    if (!initialValue)
      return WalkResult::advance();
    auto denseInit = llvm::dyn_cast<DenseElementsAttr>(*initialValue);
    if (!denseInit)
      return WalkResult::advance();
    std::optional<std::vector<char>> bytes = denseAttrToBytes(denseInit);
    if (!bytes)
      return WalkResult(buffer.emitOpError(
          "buffer op type not supported for initialization"));
    if (!buffer.getAddress())
      return WalkResult(buffer.emitOpError(
          "buffer has no address; run --aie-assign-buffer-addresses first"));
    TileOp tile = buffer.getTileOp();
    // Emitted as a byte array, so the value lands as data and matches the block
    // write the CDO path performs.
    std::string bufName(buffer.name().getValue());
    output << "{\n";
    output << "static const unsigned char " << bufName << "_init[] = {";
    for (size_t i = 0; i < bytes->size(); ++i)
      output << (i ? "," : "") << (unsigned)(unsigned char)(*bytes)[i];
    output << "};\n";
    output << "__mlir_aie_try(XAie_DataMemBlockWrite(" << deviceInstRef << ", "
           << tileLocStr(tile.colIndex(), tile.rowIndex()) << ", "
           << "0x" << llvm::utohexstr(*buffer.getAddress()) << ", "
           << "(void *)" << bufName << "_init, " << bytes->size() << "));\n";
    output << "}\n";
    return WalkResult::advance();
  });
  if (initBuffersResult.wasInterrupted())
    return failure();
  output << "return XAIE_OK;\n";
  output << "} // mlir_aie_initialize_buffers\n";
 
  //---------------------------------------------------------------------------
  // mlir_aie_configure_switchboxes
  //---------------------------------------------------------------------------
  output << "int mlir_aie_configure_switchboxes(" << ctx_p << ") {\n";
  output << "  int x, y;\n";
 
  // StreamSwitch (switchbox) configuration
  for (auto switchboxOp : targetOp.getOps<SwitchboxOp>()) {
    Region &r = switchboxOp.getConnections();
    Block &b = r.front();
    bool isEmpty = b.getOps<ConnectOp>().empty() &&
                   b.getOps<MasterSetOp>().empty() &&
                   b.getOps<PacketRulesOp>().empty();
    bool isParam = false;
 
    if (isa<TileOp>(switchboxOp.getTile().getDefiningOp())) {
      int col = switchboxOp.colIndex();
      int row = switchboxOp.rowIndex();
      if (!isEmpty) {
        output << "// Core Stream Switch column " << col << " row " << row
               << "\n";
        output << "x = " << col << ";\n";
        output << "y = " << row << ";\n";
      }
    }
 
    for (auto connectOp : b.getOps<ConnectOp>())
      output << "__mlir_aie_try(XAie_StrmConnCctEnable(" << deviceInstRef
             << ", " << tileLocStr("x", "y") << ", "
             << wireBundleToPortType(connectOp.getSourceBundle()) << ", "
             << connectOp.sourceIndex() << ", "
             << wireBundleToPortType(connectOp.getDestBundle()) << ", "
             << connectOp.destIndex() << "));\n";
 
    for (auto connectOp : b.getOps<MasterSetOp>()) {
      int mask = 0;
      int arbiter = -1;
      // MasterSetOp::verify guarantees every amsel names the same arbiter, so
      // folding them into one arbiter plus msel mask is safe here.
      for (auto val : connectOp.getAmsels()) {
        AMSelOp amsel = cast<AMSelOp>(val.getDefiningOp());
        arbiter = amsel.arbiterIndex();
        int msel = amsel.getMselValue();
        mask |= (1 << msel);
      }
      bool isdma = (connectOp.getDestBundle() == WireBundle::DMA);
 
      output << "__mlir_aie_try(XAie_StrmPktSwMstrPortEnable(" << deviceInstRef
             << ", " << tileLocStr("x", "y") << ", "
             << wireBundleToPortType(connectOp.getDestBundle()) << ", "
             << connectOp.destIndex() << ", "
             << "/* drop_header */ "
             << (isdma ? "XAIE_SS_PKT_DROP_HEADER"
                       : "XAIE_SS_PKT_DONOT_DROP_HEADER")
             << ", "
             << "/* arbiter */ " << arbiter << ", "
             << "/* MSelEn */ "
             << "0x" << llvm::utohexstr(mask) << "));\n";
    }
 
    for (auto connectOp : b.getOps<PacketRulesOp>()) {
      int slot = 0;
      Block &block = connectOp.getRules().front();
      for (auto slotOp : block.getOps<PacketRuleOp>()) {
        AMSelOp amselOp = cast<AMSelOp>(slotOp.getAmsel().getDefiningOp());
        int arbiter = amselOp.arbiterIndex();
        int msel = amselOp.getMselValue();
        output << "__mlir_aie_try(XAie_StrmPktSwSlavePortEnable("
               << deviceInstRef << ", " << tileLocStr("x", "y") << ", "
               << wireBundleToPortType(connectOp.getSourceBundle()) << ", "
               << connectOp.sourceIndex() << "));\n";
 
        // TODO Need to better define packet id,type used here
        output << "__mlir_aie_try(XAie_StrmPktSwSlaveSlotEnable("
               << deviceInstRef << ", " << tileLocStr("x", "y") << ", "
               << wireBundleToPortType(connectOp.getSourceBundle()) << ", "
               << connectOp.sourceIndex() << ", "
               << "/* slot */ " << slot << ", "
               << "/* packet */ " << packetStr(slotOp.valueInt(), /*type*/ 0)
               << ", "
               << "/* mask */ "
               << "0x" << llvm::utohexstr(slotOp.maskInt()) << ", "
               << "/* msel */ " << msel << ", "
               << "/* arbiter */ " << arbiter << "));\n";
        slot++;
      }
    }
 
    if (isParam) {
      output << "}\n";
      output << "}\n";
    }
  }
  for (auto op : targetOp.getOps<ShimMuxOp>()) {
    Region &r = op.getConnections();
    Block &b = r.front();
    bool isEmpty = b.getOps<ConnectOp>().empty();
 
    if (isa<TileOp>(op.getTile().getDefiningOp())) {
      int col = op.colIndex();
      int row = op.rowIndex();
      if (!isEmpty) {
        output << "// ShimMux column " << col << " row " << row << "\n";
        output << "// NOTE ShimMux always connects from the south as "
               << "directions are defined relative to the tile stream "
               << "switch\n";
        output << "x = " << col << ";\n";
        output << "y = " << row << ";\n";
      }
    }
 
    for (auto connectOp : b.getOps<ConnectOp>()) {
 
      if (connectOp.getSourceBundle() == WireBundle::DMA ||
          connectOp.getDestBundle() == WireBundle::DMA) {
        if (connectOp.getSourceBundle() == WireBundle::North)
          // demux!
          output << "__mlir_aie_try(XAie_EnableAieToShimDmaStrmPort("
                 << deviceInstRef << ", " << tileLocStr("x", "y") << ", "
                 << connectOp.sourceIndex() << "));\n";
        else if (connectOp.getDestBundle() == WireBundle::North)
          // mux
          output << "__mlir_aie_try(XAie_EnableShimDmaToAieStrmPort("
                 << deviceInstRef << ", " << tileLocStr("x", "y") << ", "
                 << connectOp.destIndex() << "));\n";
      }
 
      else if (connectOp.getSourceBundle() == WireBundle::PLIO ||
               connectOp.getDestBundle() == WireBundle::PLIO) {
        if (connectOp.getSourceBundle() == WireBundle::North) {
          // mux
          output << "__mlir_aie_try(XAie_AieToPlIntfEnable(" << deviceInstRef
                 << ", " << tileLocStr("x", "y") << ", "
                 << connectOp.destIndex() << ", PLIF_WIDTH_64));\n";
        } else if (connectOp.getDestBundle() == WireBundle::North) {
          // mux
          output << "__mlir_aie_try(XAie_PlToAieIntfEnable(" << deviceInstRef
                 << ", " << tileLocStr("x", "y") << ", "
                 << connectOp.destIndex() << ", PLIF_WIDTH_64));\n";
        }
      }
    }
  }
  output << "return XAIE_OK;\n";
  output << "} // mlir_aie_configure_switchboxes\n\n";
 
  //---------------------------------------------------------------------------
  // mlir_aie_configure_cascade
  //---------------------------------------------------------------------------
  output << "int mlir_aie_configure_cascade(" << ctx_p << ") {\n";
  for (auto configOp : targetOp.getOps<ConfigureCascadeOp>()) {
    TileOp tile = cast<TileOp>(configOp.getTile().getDefiningOp());
    int col = tile.colIndex();
    int row = tile.rowIndex();
    output << "XAie_CoreConfigAccumulatorControl(" << deviceInstRef << ", "
           << "XAie_TileLoc(" << col << ", " << row << "), "
           << stringifyCascadeDir(configOp.getInputDir()).upper() << ", "
           << stringifyCascadeDir(configOp.getOutputDir()).upper() << ");\n";
  }
  output << "return XAIE_OK;\n";
  output << "} // mlir_aie_configure_cascade\n\n";
 
  //---------------------------------------------------------------------------
  // Output Buffer Accessors
  //---------------------------------------------------------------------------
  for (auto tile : tiles) {
    Operation *tileOp = tile.second;
    TileID coord = cast<TileOp>(tileOp).getTileID();
    int col = coord.col;
    int row = coord.row;
    auto loc = tileLocStr(col, row);
 
    auto bufferAccessor = [&](BufferOp buf) {
      // int32_t mlir_aie_read_buffer_a13(int index) {
      // void mlir_aie_write_buffer_a13(int index, int32_t value) {
      std::string bufName(buf.name().getValue());
      Type t = buf.getType();
      Type et;
      std::string typestr;
      if (auto memrefType = llvm::dyn_cast<MemRefType>(t)) {
        et = memrefType.getElementType();
        if (et.isInteger(32))
          typestr = "int32_t";
        else if (et.isF32())
          typestr = "float";
        else {
          output << "// buffer " << bufName << " with unsupported type " << t
                 << ";\n";
          return; // Unsupported type
        }
 
      } else {
        output << "// buffer " << bufName << " with unsupported type " << t
               << ";\n";
        return; // Unsupported type
      }
      assert(buf.getAddress().has_value() && "buffer must have address");
      output << "const int " << bufName
             << "_offset = " << buf.getAddress().value() << ";\n";
      output << typestr << " mlir_aie_read_buffer_" << bufName << "(" << ctx_p
             << ", int index) {\n";
      output << "u32 value; auto rc = XAie_DataMemRdWord(" << deviceInstRef
             << ", " << loc << ", " << bufName
             << "_offset + (index*4), &value);\n";
      if (et.isInteger(32))
        output << "  return value;\n";
      else if (et.isF32()) {
        output << "  union caster { int32_t i; float f; };\n";
        output << "  caster c; c.i = value;\n";
        output << "  return c.f;\n";
      }
      output << "}\n";
      output << "int mlir_aie_write_buffer_" << bufName << "(" << ctx_p
             << ", int index, " << typestr << " value) {\n";
      if (et.isInteger(32))
        output << "  int32_t int_value = value;\n";
      else if (et.isF32()) {
        output << "  union caster { int32_t i; float f; };\n";
        output << "  caster c; c.f = value;\n";
        output << "  int32_t int_value = c.i;\n";
      }
      output << "AieRC rc =    XAie_DataMemWrWord(" << deviceInstRef << ", "
             << loc << ", " << bufName << "_offset + (index*4), int_value);\n";
      output << "return rc;\n";
      output << "}\n";
    };
 
    // if(tiles.count(tile.getValue()))
    for (auto buf : buffers[tileOp])
      bufferAccessor(buf);
  }
 
  auto lockAccessor = [&](LockOp lock) {
    int col = lock.colIndex();
    int row = lock.rowIndex();
    if (!lock.hasName())
      return;
    std::string lockName(lock.name().getValue());
    output << "int mlir_aie_acquire_" << lockName << "(" << ctx_p
           << ", int value, int timeout) {\n";
    output << "  const int id = " << lock.getLockIDValue() << ";\n";
    output << "  return XAie_LockAcquire(" << deviceInstRef << ", "
           << tileLocStr(col, row) << ", " << tileLockStr("id", "value")
           << ", timeout);\n";
    output << "}\n";
    output << "int mlir_aie_release_" << lockName << "(" << ctx_p
           << ", int value, int timeout) {\n";
    output << "  const int id = " << lock.getLockIDValue() << ";\n";
    output << "  return XAie_LockRelease(" << deviceInstRef << ", "
           << tileLocStr(col, row) << ", " << tileLockStr("id", "value")
           << ", timeout);\n";
    output << "}\n";
  };
 
  targetOp.walk<WalkOrder::PreOrder>([&](LockOp lock) { lockAccessor(lock); });
 
  return success();
}