AIEXToEmitC.cpp

Go to the documentation of this file.

//===- AIEXToEmitC.cpp - AIEX runtime sequence to EmitC --------*- C++ -*-===//
//
// Copyright (C) 2026 Advanced Micro Devices, Inc.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Converts npu transaction ops in an aie.runtime_sequence into EmitC calls to
// aie/Runtime/TxnEncoding.h; translateToCpp() then yields a standalone C++
// function assembling the same TXN words as the binary emitter
// (AIETargetNPU.cpp), but runtime-parameterizable. A runtime-bound scf.for
// (dynamic BD pool path) is preserved and lowered to emitc.for, keeping the
// loop rolled at runtime.
//
//===----------------------------------------------------------------------===//
 
#include "aie/Conversion/AIEXToEmitC/AIEXToEmitC.h"
#include "aie/Conversion/AIEToConfiguration/AIEToConfiguration.h"
 
#include "aie/Dialect/AIE/IR/AIEDialect.h"
#include "aie/Dialect/AIE/IR/AIETargetModel.h"
#include "aie/Dialect/AIEX/IR/AIEXDialect.h"
#include "aie/Runtime/TxnEncoding.h"
 
#include "mlir/Conversion/ArithToEmitC/ArithToEmitC.h"
#include "mlir/Conversion/SCFToEmitC/SCFToEmitC.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Arith/Transforms/Passes.h"
#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
#include "mlir/Dialect/EmitC/IR/EmitC.h"
#include "mlir/Dialect/EmitC/Transforms/TypeConversions.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/SCF/IR/SCF.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/Interfaces/SideEffectInterfaces.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/DialectConversion.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
#include "mlir/Transforms/RegionUtils.h"
 
#include "llvm/ADT/StringSet.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Support/Format.h"
 
namespace xilinx {
// DECL as well as DEF: the ConvertAIEXToEmitCPass(Options) constructor below
// needs ConvertAIEXToEmitCOptions declared in this translation unit.
#define GEN_PASS_DECL_CONVERTAIEXTOEMITC
#define GEN_PASS_DEF_CONVERTAIEXTOEMITC
#include "aie/Conversion/Passes.h.inc"
} // namespace xilinx
 
using namespace mlir;
 
// The C++ that declines a build with a reason. `why` becomes a string
// literal (quotes and backslashes escaped, line breaks flattened). When the
// snippet goes through an emitc.verbatim with operands, `{}` is a
// placeholder and `{{` its escape, so a `{` in the message is doubled; a
// verbatim without operands is emitted literally.
static std::string refuse(StringRef why, bool inFormat) {
  std::string lit;
  for (char ch : why) {
    if (ch == '\n' || ch == '\r')
      lit += ' ';
    else if (ch == '"' || ch == '\\')
      (lit += '\\') += ch;
    else if (ch == '{' && inFormat)
      lit += "{{";
    else
      lit += ch;
  }
  return "return aie_runtime::txn_refused(\"" + lit + "\");";
}
 
using namespace xilinx;
 
namespace {
 
// The C++ type of the TXN accumulator the generated function builds.
constexpr llvm::StringLiteral kTxnVecType = "std::vector<uint32_t>";
 
emitc::OpaqueType getU32Type(MLIRContext *ctx) {
  return emitc::OpaqueType::get(ctx, "uint32_t");
}
 
// The C++ variable name of a runtime BD pool, declared in the generated
// function's prologue. One pool per tile and BD partition: channels sharing a
// partition must share one pool, or they would hand out the same id twice.
template <typename PoolOp>
std::string bdPoolName(PoolOp op) {
  return "bd_pool_" + std::to_string(op.getColumn()) + "_" +
         std::to_string(op.getRow()) + "_" + std::to_string(op.getBdBegin()) +
         "_" + std::to_string(op.getBdEnd());
}
 
// BD ids the static allocator has already placed on (col, row), read off the
// placed aie.mem / aie.memtile_dma / aie.shim_dma chains. Sorted so the emitted
// stream is deterministic.
llvm::SmallVector<uint32_t, 8> staticBdIdsOnTile(AIE::DeviceOp device,
                                                 uint32_t col, uint32_t row) {
  llvm::SmallVector<uint32_t, 8> ids;
  for (AIE::DmaBody program : device.getOps<AIE::DmaBody>()) {
    auto id = program.getTileID();
    if (id.col != static_cast<int>(col) || id.row != static_cast<int>(row))
      continue;
    llvm::append_range(ids, AIE::getAssignedBdIds(program));
  }
  llvm::sort(ids);
  ids.erase(llvm::unique(ids), ids.end());
  return ids;
}
 
// A uint32_t literal operand (e.g. `42u`) for a txn_append_* argument. Used for
// the compile-time-resolved address fields (buffer/col/row are folded in).
Value u32Literal(OpBuilder &b, Location loc, uint32_t val) {
  return emitc::ConstantOp::create(
      b, loc, getU32Type(b.getContext()),
      emitc::OpaqueAttr::get(b.getContext(), std::to_string(val) + "u"));
}
 
// Emit a call to a TxnEncoding.h free function on the txn vector. Scalar
// operands pass through as-is; convert-arith-to-emitc later lowers them to C++
// literals or function parameters, byte-identical to the static path.
void emitTxnCall(OpBuilder &b, Location loc, StringRef fn, Value txnVec,
                 ValueRange args) {
  SmallVector<Value> callArgs;
  callArgs.push_back(txnVec);
  callArgs.append(args.begin(), args.end());
  emitc::CallOpaqueOp::create(b, loc, TypeRange{}, ("aie_runtime::" + fn).str(),
                              callArgs);
}
 
// Device-relative values (absolute addr, blockwrite data) resolved before the
// ops are cloned out of the aie.device, keyed by their clone in the emitc.func.
struct DeviceResolved {
  llvm::DenseMap<Operation *, uint32_t> absoluteAddr;
  llvm::DenseMap<Operation *, DenseIntElementsAttr> blockWriteData;
};
 
class AIEXToEmitCConverter {
public:
  AIEXToEmitCConverter(emitc::FuncOp funcOp, Value txnVec,
                       const DeviceResolved &resolved, Value opCountVar,
                       bool foldDDRAddrOffset)
      : funcOp(funcOp), txnVec(txnVec), resolved(resolved),
        opCountVar(opCountVar), foldDDRAddrOffset(foldDDRAddrOffset) {}
 
  // Convert every npu/pool op in the body, recursing into scf regions. Returns
  // the compile-time op count for a straight-line body (the literal op_count),
  // or nullopt on error. With a runtime op-count var (a loop is present) each
  // op increments it instead and the returned count is unused.
  std::optional<uint32_t> run() {
    // DMA lowering consumes buffer views but can leave dead subview/cast
    // chains behind (notably after configure/run materialization). They are
    // not transaction operations and must not reach the C++ converter.
    bool erased;
    do {
      erased = false;
      funcOp.walk([&](Operation *op) {
        if (isa<memref::MemRefDialect>(op->getDialect()) &&
            isOpTriviallyDead(op)) {
          op->erase();
          erased = true;
        }
      });
    } while (erased);
    uint32_t count = 0;
    SmallVector<Operation *> consumed;
    convertBlockRecursive(funcOp.getBlocks().front(), count, consumed);
    if (!ok)
      return std::nullopt;
 
    // Erase the converted npu ops, then DCE their now-dead operand clones (a
    // get_global feeding a blockwrite, a folded-away address constant). Region-
    // scoped runRegionDCE, not a module canonicalizer: the latter would also
    // drop the shared memref.global the blockwrite data was inlined from, which
    // other sequences in the module still reference.
    for (Operation *op : llvm::reverse(consumed))
      op->erase();
    IRRewriter rewriter(funcOp.getContext());
    (void)runRegionDCE(rewriter, funcOp.getFunctionBody());
    return count;
  }
 
private:
  // Convert the npu/pool ops in `block`, recursing through scf op regions so a
  // rolled loop's body is lowered in place (the scf.for itself is left for the
  // later convert-scf-to-emitc step). Collects converted ops into `consumed`.
  void convertBlockRecursive(Block &block, uint32_t &count,
                             SmallVector<Operation *> &consumed) {
    SmallVector<Operation *> work;
    for (Operation &op : block)
      work.push_back(&op);
    for (Operation *op : work) {
      if (!ok)
        return;
      // Structural emitc ops and arith (constant / runtime-value math) are left
      // for the later arith-to-emitc step. Recurse into scf ops (and any other
      // region-carrying control flow) to convert their bodies; the control-flow
      // op itself is handled by convert-scf-to-emitc afterwards.
      if (isa<emitc::EmitCDialect, arith::ArithDialect>(op->getDialect()))
        continue;
      if (isa<scf::SCFDialect>(op->getDialect())) {
        for (Region &r : op->getRegions())
          for (Block &b : r)
            convertBlockRecursive(b, count, consumed);
        continue;
      }
      convertOne(op, count);
      if (!isa<memref::GetGlobalOp>(op))
        consumed.push_back(op);
    }
  }
 
private:
  // Record one emitted txn op: bump the compile-time count and, when a runtime
  // op-count variable exists (the sequence has a loop, so the count is not
  // known at compile time), emit a `++__opcount;` so the header gets the true
  // runtime total.
  void countOp(OpBuilder &b, Location loc, uint32_t &count) {
    ++count;
    if (opCountVar)
      emitc::VerbatimOp::create(b, loc, "++{};", ValueRange{opCountVar});
  }
 
  // Convert one op, appending to the txn vector and incrementing `count`.
  // Scalar operands flow through as SSA (lowered later by
  // convert-arith-to-emitc); only the address field is replaced by its
  // device-resolved literal. Unsupported ops set `ok` false.
  void convertOne(Operation *op, uint32_t &count) {
    OpBuilder b(op);
    Location loc = op->getLoc();
 
    llvm::TypeSwitch<Operation *, void>(op)
        .Case<AIEX::NpuWrite32Op>([&](auto w) {
          // A compile-time address folds to a literal; a runtime address (the
          // dynamic BD pool's bdBase + wordIdx, arith over a popped id) is
          // passed through as SSA and lowered by convert-arith-to-emitc.
          Value addrV = runtimeOrResolvedAddr(b, loc, w, w.getAddress());
          if (!addrV)
            return fail(w, "cannot convert a symbolic/unresolved write32 "
                           "address to the C++ TXN target");
          emitTxnCall(b, loc, "txn_append_write32", txnVec,
                      {addrV, w.getValue()});
          countOp(b, loc, count);
        })
        .Case<AIEX::NpuMaskWrite32Op>([&](auto mw) {
          Value addrV = runtimeOrResolvedAddr(b, loc, mw, mw.getAddress());
          if (!addrV)
            return fail(mw, "cannot convert a symbolic/unresolved maskwrite32 "
                            "address to the C++ TXN target");
          emitTxnCall(b, loc, "txn_append_maskwrite32", txnVec,
                      {addrV, mw.getValue(), mw.getMask()});
          countOp(b, loc, count);
        })
        .Case<AIEX::NpuMaskPollOp>([&](auto mp) {
          Value addrV = runtimeOrResolvedAddr(b, loc, mp, mp.getAddress());
          if (!addrV)
            return fail(mp, "cannot convert a symbolic/unresolved maskpoll "
                            "address to the C++ TXN target");
          emitTxnCall(b, loc, "txn_append_maskpoll32", txnVec,
                      {addrV, mp.getValue(), mp.getMask()});
          countOp(b, loc, count);
        })
        .Case<AIEX::NpuSyncOp>([&](auto s) {
          emitTxnCall(b, loc, "txn_append_sync", txnVec,
                      {s.getColumn(), s.getRow(), s.getDirection(),
                       s.getChannel(), s.getColumnNum(), s.getRowNum()});
          countOp(b, loc, count);
        })
        .Case<AIEX::NpuLoadPdiOp>([&](auto pdi) {
          Value address = emitc::ConstantOp::create(
              b, loc, emitc::OpaqueType::get(b.getContext(), "uint64_t"),
              emitc::OpaqueAttr::get(b.getContext(),
                                     std::to_string(pdi.getAddress()) + "ULL"));
          emitTxnCall(b, loc, "txn_append_loadpdi", txnVec,
                      {u32Literal(b, loc, pdi.getId()),
                       u32Literal(b, loc, pdi.getSize()), address});
          countOp(b, loc, count);
        })
        .Case<AIEX::NpuPreemptOp>([&](auto preempt) {
          emitTxnCall(b, loc, "txn_append_preempt", txnVec,
                      {u32Literal(b, loc, preempt.getLevel())});
          countOp(b, loc, count);
        })
        .Case<AIEX::NpuAddressPatchOp>([&](auto ap) {
          // A runtime bd_id makes the patched register address runtime: prefer
          // the SSA addr_val (flows through arith-to-emitc) over the constant.
          Value addrV = ap.getAddrVal() ? ap.getAddrVal()
                                        : u32Literal(b, loc, ap.getAddr());
          std::optional<uint32_t> argIdx = ap.getArgIdx();
          if (!argIdx)
            return fail(ap, "address_patch still names its host buffer by SSA "
                            "value; run -aie-resolve-address-patch-buffers");
          Value idxV = emitc::ConstantOp::create(
              b, loc, emitc::OpaqueType::get(b.getContext(), "int32_t"),
              emitc::OpaqueAttr::get(b.getContext(), std::to_string(*argIdx)));
          // txn_append_arg_patch applies the DDR-aperture fold itself, so the
          // rule lives only in TxnEncoding.h. Both operands are literals here,
          // so -O2 folds the branch away.
          Value foldV = emitc::ConstantOp::create(
              b, loc, emitc::OpaqueType::get(b.getContext(), "bool"),
              emitc::OpaqueAttr::get(b.getContext(),
                                     foldDDRAddrOffset ? "true" : "false"));
          Value argPlus = ap.getArgPlus();
          // i32 offsets are unsigned bit patterns in the static emitter.
          // Cast before widening so bit 31 is not sign-extended to 64 bits.
          if (argPlus.getType().isInteger(32))
            argPlus = emitc::CastOp::create(b, loc, getU32Type(b.getContext()),
                                            argPlus);
          emitTxnCall(b, loc, "txn_append_arg_patch", txnVec,
                      {addrV, idxV, argPlus, foldV});
          countOp(b, loc, count);
        })
        .Case<AIEX::NpuBlockWriteOp>([&](auto bw) {
          convertBlockWrite(b, loc, bw);
          countOp(b, loc, count);
        })
        .Case<AIEX::NpuBlockWriteValuesOp>([&](auto bw) {
          convertBlockWriteValues(b, loc, bw);
          countOp(b, loc, count);
        })
        .Case<cf::AssertOp>([&](auto a) {
          // Host-side guard: a violated condition yields no stream
          // (std::nullopt), and its message is what the host reports. Appends
          // nothing, so not counted.
          emitc::VerbatimOp::create(
              b, loc, "if (!({})) " + refuse(a.getMsg(), /*inFormat=*/true),
              ValueRange{a.getArg()});
        })
        .Case<AIEX::DMABdPoolPopOp>([&](AIEX::DMABdPoolPopOp pop) {
          // Draw a BD id from the tile's runtime pool (declared in the
          // prologue) into a fresh C++ variable; nullopt if the pool is empty.
          // Downstream ops consume the id, so replace the SSA result with a
          // literal naming the variable.
          std::string var = "bd_" + std::to_string(nextPoolVar++);
          emitc::VerbatimOp::create(
              b, loc,
              "uint32_t " + var + "; if (!aie_runtime::bd_pool_pop(" +
                  bdPoolName(pop) + ", " + var + ")) " +
                  refuse("no free buffer descriptor on tile (" +
                             std::to_string(pop.getColumn()) + ", " +
                             std::to_string(pop.getRow()) +
                             "): the sequence keeps more transfers in flight "
                             "than the tile has descriptors",
                         /*inFormat=*/false));
          Value ref =
              emitc::LiteralOp::create(b, loc, pop.getBdId().getType(), var);
          pop.getBdId().replaceAllUsesWith(ref);
        })
        .Case<AIEX::DMABdPoolPushOp>([&](AIEX::DMABdPoolPushOp push) {
          // Return a BD id to the tile's runtime pool.
          emitc::CallOpaqueOp::create(
              b, loc, TypeRange{}, "aie_runtime::bd_pool_push",
              ValueRange{poolRef(b, loc, bdPoolName(push)), push.getBdId()});
        })
        // XRT allocates the scratchpad only for a full-ELF context, which has
        // no per-call instruction buffer to carry a generated stream.
        .Case<AIEX::NpuCreateScratchpadOp, AIEX::NpuUpdateFromScratchpadOp>(
            [&](Operation *sp) {
              fail(sp, "needs the scratchpad of a full-ELF context, which the "
                       "C++ TXN target's per-call streams run without; use a "
                       "runtime sequence argument instead of a scratchpad "
                       "parameter");
            })
        // memref.get_global feeding a blockwrite is consumed by
        // convertBlockWrite (data inlined); the now-dead op is erased later.
        .Case<memref::GetGlobalOp>([&](auto) {})
        .Default([&](Operation *unsupported) {
          if (isa<scf::SCFDialect>(unsupported->getDialect()))
            fail(unsupported,
                 "control flow in runtime sequences is not yet supported by "
                 "the C++ TXN target");
          else
            fail(unsupported, "is not supported by the C++ TXN target");
        });
  }
 
  // The compile-time-resolved absolute address (folding buffer/col/row) as a
  // uint32_t literal, or null if it could not be resolved statically. Address
  // resolution depends on the parent device, so it is precomputed before the op
  // is cloned out (see recordDeviceResolved).
  Value resolvedAddr(OpBuilder &b, Location loc, Operation *clone) {
    auto it = resolved.absoluteAddr.find(clone);
    if (it == resolved.absoluteAddr.end())
      return {};
    return u32Literal(b, loc, it->second);
  }
 
  // The address for a write32/maskwrite32: the device-resolved literal, else a
  // genuine runtime address (arith-derived, e.g. the pool's bdBase + wordIdx).
  // A bare block argument selects no register and is rejected; null for a
  // symbolic/unresolved address.
  Value runtimeOrResolvedAddr(OpBuilder &b, Location loc, Operation *clone,
                              Value addrOperand) {
    if (Value lit = resolvedAddr(b, loc, clone))
      return lit;
    Operation *def = addrOperand ? addrOperand.getDefiningOp() : nullptr;
    if (def && !isa<arith::ConstantOp>(def))
      return addrOperand;
    return {};
  }
 
  // Emit a diagnostic on `op` and mark the conversion failed.
  void fail(Operation *op, const llvm::Twine &msg) {
    op->emitOpError(msg);
    ok = false;
  }
 
  // An emitc.literal referencing a pool variable (for pass-by-reference into
  // bd_pool_push).
  Value poolRef(OpBuilder &b, Location loc, StringRef name) {
    return emitc::LiteralOp::create(
        b, loc, emitc::OpaqueType::get(b.getContext(), "aie_runtime::BdPool"),
        name);
  }
 
  void convertBlockWrite(OpBuilder &b, Location loc, AIEX::NpuBlockWriteOp bw) {
    auto addrIt = resolved.absoluteAddr.find(bw);
    auto dataIt = resolved.blockWriteData.find(bw);
    if (addrIt == resolved.absoluteAddr.end() ||
        dataIt == resolved.blockWriteData.end() || !dataIt->second) {
      bw.emitOpError("cannot convert blockwrite without a constant address and "
                     "data to the C++ TXN target");
      ok = false;
      return;
    }
    uint32_t addr = addrIt->second;
    DenseIntElementsAttr data = dataIt->second;
 
    // Emit the payload as a typed EmitC array variable rather than a verbatim
    // C++ string, so the generated code carries real types.
    MLIRContext *ctx = b.getContext();
    int64_t n = data.size();
    auto arrTy =
        emitc::ArrayType::get(ctx, SmallVector<int64_t>{n}, getU32Type(ctx));
    std::string init = "{";
    llvm::raw_string_ostream ss(init);
    bool first = true;
    for (APInt word : data.getValues<APInt>()) {
      if (!first)
        ss << ", ";
      ss << llvm::format("0x%08xu", static_cast<unsigned>(word.getZExtValue()));
      first = false;
    }
    ss << "}";
    auto arrVar = emitc::VariableOp::create(b, loc, arrTy,
                                            emitc::OpaqueAttr::get(ctx, init));
 
    uint32_t colVal = 0, rowVal = 0;
    if (auto col = bw.getColumn(), row = bw.getRow(); col && row) {
      colVal = *col;
      rowVal = *row;
    }
    emitTxnCall(b, loc, "txn_append_blockwrite", txnVec,
                {u32Literal(b, loc, addr), arrVar.getResult(),
                 u32Literal(b, loc, static_cast<uint32_t>(n)),
                 u32Literal(b, loc, colVal), u32Literal(b, loc, rowVal)});
  }
 
  // A blockwrite whose payload is computed at TXN-build time:
  // stage the words into a local C++ array, then hand that array
  // to one txn_append_blockwrite. Unlike convertBlockWrite there is no constant
  // memref to inline, so the slots are filled by assignment from the operand
  // values, which convert-arith-to-emitc lowers in place afterwards.
  void convertBlockWriteValues(OpBuilder &b, Location loc,
                               AIEX::NpuBlockWriteValuesOp bw) {
    MLIRContext *ctx = b.getContext();
    Value addrV = runtimeOrResolvedAddr(b, loc, bw, bw.getAddress());
    if (!addrV) {
      fail(bw, "cannot convert a symbolic/unresolved blockwrite_values address "
               "to the C++ TXN target");
      return;
    }
    ValueRange words = bw.getValues();
    int64_t n = static_cast<int64_t>(words.size());
 
    // Declare the staging array (zero-initialized; every slot is assigned
    // below). Typed emitc.variable, matching convertBlockWrite's array.
    auto arrTy =
        emitc::ArrayType::get(ctx, SmallVector<int64_t>{n}, getU32Type(ctx));
    auto arrVar = emitc::VariableOp::create(b, loc, arrTy,
                                            emitc::OpaqueAttr::get(ctx, "{}"));
 
    // Fill the slots. Emitted as verbatim assignments (the same mechanism the
    // runtime guards above use to reference a not-yet-lowered SSA value): the
    // i32 words convert to the array's uint32_t elementwise, which is the
    // intended two's-complement reinterpretation of the packed BD fields.
    for (int64_t i = 0; i < n; i++)
      emitc::VerbatimOp::create(b, loc, "{}[" + std::to_string(i) + "] = {};",
                                ValueRange{arrVar.getResult(), words[i]});
 
    // col/row are 0: the address is already flat (as for npu.blockwrite).
    emitTxnCall(b, loc, "txn_append_blockwrite", txnVec,
                {addrV, arrVar.getResult(),
                 u32Literal(b, loc, static_cast<uint32_t>(n)),
                 u32Literal(b, loc, 0), u32Literal(b, loc, 0)});
  }
 
  emitc::FuncOp funcOp;
  Value txnVec;
  const DeviceResolved &resolved;
  // Runtime op-count variable (the C++ `__opcount`), or null when the sequence
  // is straight-line and the count is a compile-time literal.
  Value opCountVar;
  bool ok = true;
  // Counter for unique popped-BD-id C++ variable names.
  unsigned nextPoolVar = 0;
  bool foldDDRAddrOffset;
};
 
struct ConvertAIEXToEmitCPass
    : xilinx::impl::ConvertAIEXToEmitCBase<ConvertAIEXToEmitCPass> {
  ConvertAIEXToEmitCPass() = default;
  // foldDDRAddrOffset is `protected` on the generated base, settable only via
  // a -pass-pipeline string; this constructor is the programmatic equivalent.
  explicit ConvertAIEXToEmitCPass(const ConvertAIEXToEmitCOptions &options)
      : ConvertAIEXToEmitCBase(options) {}
 
  void runOnOperation() override {
    ModuleOp moduleOp = getOperation();
    MLIRContext *ctx = &getContext();
    if (failed(AIE::inlineWriteConfigs(moduleOp)))
      return signalPassFailure();
 
    struct SeqInfo {
      AIE::RuntimeSequenceOp seq;
      AIE::DeviceOp device;
    };
    SmallVector<SeqInfo> sequences;
    moduleOp.walk([&](AIE::RuntimeSequenceOp seq) {
      sequences.push_back({seq, seq->getParentOfType<AIE::DeviceOp>()});
    });
    if (emitDispatchShim && sequences.size() != 1) {
      moduleOp.emitOpError()
          << "emit-dispatch-shim needs exactly one aie.runtime_sequence to "
             "wrap, found "
          << sequences.size();
      return signalPassFailure();
    }
    if (sequences.empty())
      return;
 
    OpBuilder builder(ctx);
    for (auto &[seqOp, deviceOp] : sequences) {
      if (!deviceOp) {
        seqOp.emitOpError("must be nested inside an aie.device");
        return signalPassFailure();
      }
      std::optional<emitc::FuncOp> gen =
          emitFunction(builder, moduleOp, seqOp, deviceOp);
      if (!gen)
        return signalPassFailure();
      if (emitDispatchShim) {
        emitc::FuncOp generated = *gen;
        if (failed(emitDispatchShimFuncs(builder, moduleOp, generated)))
          return signalPassFailure();
      }
    }
 
    // Replace the module body with just the generated emitc funcs + includes.
    SmallVector<Operation *> toErase;
    for (Operation &op : *moduleOp.getBody())
      if (!isa<emitc::FuncOp, emitc::IncludeOp>(op))
        toErase.push_back(&op);
    for (Operation *op : llvm::reverse(toErase))
      op->erase();
 
    // Lower the arith ops (constant scalar fields, runtime-value arithmetic
    // feeding npu ops) and any scf control flow (a rolled dynamic loop) left in
    // the function bodies to emitc.
    if (failed(lowerArithAndScfToEmitC(moduleOp)))
      return signalPassFailure();
 
    builder.setInsertionPointToStart(moduleOp.getBody());
    emitc::IncludeOp::create(builder, moduleOp.getLoc(),
                             "aie/Runtime/TxnEncoding.h",
                             /*is_standard_include=*/false);
    emitc::IncludeOp::create(builder, moduleOp.getLoc(), "cstdint",
                             /*is_standard_include=*/true);
    emitc::IncludeOp::create(builder, moduleOp.getLoc(), "vector",
                             /*is_standard_include=*/true);
    // The builder returns std::optional: a runtime scalar that would overflow a
    // narrow BD field yields std::nullopt instead of a truncated stream.
    emitc::IncludeOp::create(builder, moduleOp.getLoc(), "optional",
                             /*is_standard_include=*/true);
  }
 
  // Run the upstream arith-to-emitc and scf-to-emitc conversions over the
  // generated functions in one pass: a rolled dynamic loop is scf.for, and its
  // bounds / iter_args are arith, so both must lower together.
  LogicalResult lowerArithAndScfToEmitC(ModuleOp moduleOp) {
    // ArithToEmitC has no patterns for the integer ops the Python bindings
    // emit for `//`, ceildiv, min and max on staged scalars. Expand them to
    // the divsi/cmpi/select forms it does lower before converting.
    {
      RewritePatternSet expand(moduleOp.getContext());
      arith::populateCeilFloorDivExpandOpsPatterns(expand);
      arith::populateExpandMinMaxIPatterns(expand);
      // Rewrite only: folding or CSE here would reshape the sequence the
      // per-op conversion below expects (and the C++ its tests check).
      GreedyRewriteConfig config;
      config.enableFolding(false).enableConstantCSE(false);
      if (failed(applyPatternsGreedily(moduleOp, std::move(expand), config)))
        return failure();
    }
 
    TypeConverter typeConverter;
    typeConverter.addConversion([](Type t) { return t; });
    populateEmitCSizeTTypeConversions(typeConverter);
 
    RewritePatternSet patterns(moduleOp.getContext());
    populateArithToEmitCPatterns(typeConverter, patterns);
    populateSCFToEmitCConversionPatterns(patterns, typeConverter);
 
    ConversionTarget target(*moduleOp.getContext());
    target.addLegalDialect<emitc::EmitCDialect>();
    target.addIllegalDialect<arith::ArithDialect>();
    target.addIllegalDialect<scf::SCFDialect>();
    return applyPartialConversion(moduleOp, target, std::move(patterns));
  }
 
private:
  // The C spelling emitc's translator emits for a parameter type, for the ABI
  // string only. Must mirror CppEmitter::emitType down to its signedness rule,
  // or the reported ABI disagrees with the emitted signature.
  static std::optional<std::string> cTypeName(Type t) {
    if (isa<emitc::SizeTType>(t))
      return std::string("size_t");
    auto intTy = dyn_cast<IntegerType>(t);
    if (!intTy || !emitc::isSupportedIntegerType(intTy))
      return std::nullopt;
    if (intTy.getWidth() == 1)
      return std::string("bool");
    return (intTy.isUnsigned() ? "uint" : "int") +
           std::to_string(intTy.getWidth()) + "_t";
  }
 
  // Emit the ctypes-callable entry points: dispatch_abi() reporting the
  // parameter types, and dispatch_generate() handing back a pointer + word
  // count into thread-local storage, or -2 when the builder declined.
  LogicalResult emitDispatchShimFuncs(OpBuilder &builder, ModuleOp moduleOp,
                                      emitc::FuncOp gen) {
    MLIRContext *ctx = moduleOp.getContext();
    Location loc = gen.getLoc();
    TypeRange genParams = gen.getFunctionType().getInputs();
 
    SmallVector<std::string> abiTypes;
    for (Type t : genParams) {
      std::optional<std::string> name = cTypeName(t);
      if (!name)
        return gen.emitOpError()
               << "runtime-sequence parameter type " << t
               << " has no dispatch ABI spelling; it cannot be passed from the "
                  "host at dispatch time";
      abiTypes.push_back(*name);
    }
 
    builder.setInsertionPointToEnd(moduleOp.getBody());
    // Python resolves these two by name out of the built shared library. ELF
    // exports every extern symbol unless told otherwise, but MSVC-target
    // linking exports nothing without an explicit marker, so on Windows the
    // DLL would load and then fail to resolve dispatch_abi. The macro comes
    // from TxnEncoding.h, which the emitted file already includes, and expands
    // to nothing off Windows.
    auto externC =
        builder.getStrArrayAttr({"extern \"C\"", "AIE_DISPATCH_EXPORT"});
 
    // Pointers must be emitc.ptr: an opaque type may not have a pointer as its
    // outer type.
    auto charPtrTy =
        emitc::PointerType::get(emitc::OpaqueType::get(ctx, "const char"));
    auto abiFn = emitc::FuncOp::create(builder, loc, "dispatch_abi",
                                       FunctionType::get(ctx, {}, {charPtrTy}));
    abiFn.setSpecifiersAttr(externC);
    OpBuilder ab = OpBuilder::atBlockBegin(abiFn.addEntryBlock());
    Value abiStr = emitc::LiteralOp::create(
        ab, loc, charPtrTy, "\"" + llvm::join(abiTypes, ",") + "\"");
    emitc::ReturnOp::create(ab, loc, abiStr);
 
    SmallVector<Type> shimParams(genParams);
    shimParams.push_back(
        emitc::PointerType::get(emitc::PointerType::get(getU32Type(ctx))));
    auto i64Ty = emitc::OpaqueType::get(ctx, "int64_t");
    auto shimFn =
        emitc::FuncOp::create(builder, loc, "dispatch_generate",
                              FunctionType::get(ctx, shimParams, {i64Ty}));
    shimFn.setSpecifiersAttr(externC);
    Block *shimBlock = shimFn.addEntryBlock();
    OpBuilder sb = OpBuilder::atBlockBegin(shimBlock);
 
    SmallVector<Value> genArgs(shimBlock->getArguments().drop_back());
    SmallVector<StringRef> holes(genArgs.size(), "{}");
    std::string call = "auto __result = " + gen.getSymName().str() + "(" +
                       llvm::join(holes, ", ") + ");";
    emitc::VerbatimOp::create(
        sb, loc, "static thread_local " + kTxnVecType.str() + " __txn;");
    emitc::VerbatimOp::create(sb, loc, "aie_runtime::txn_refusal = nullptr;");
    emitc::VerbatimOp::create(sb, loc, call, genArgs);
    emitc::VerbatimOp::create(sb, loc, "if (!__result) return -2;");
    emitc::VerbatimOp::create(sb, loc, "__txn = std::move(*__result);");
    emitc::VerbatimOp::create(sb, loc, "*{} = __txn.data();",
                              ValueRange{shimBlock->getArguments().back()});
    emitc::ReturnOp::create(
        sb, loc,
        emitc::LiteralOp::create(sb, loc, i64Ty,
                                 "static_cast<int64_t>(__txn.size())"));
 
    // dispatch_last_refusal(): why the last dispatch_generate() on this
    // thread returned -2 (a string literal in this library), or null.
    auto whyFn = emitc::FuncOp::create(builder, loc, "dispatch_last_refusal",
                                       FunctionType::get(ctx, {}, {charPtrTy}));
    whyFn.setSpecifiersAttr(externC);
    OpBuilder wb = OpBuilder::atBlockBegin(whyFn.addEntryBlock());
    emitc::ReturnOp::create(
        wb, loc,
        emitc::LiteralOp::create(wb, loc, charPtrTy,
                                 "aie_runtime::txn_refusal"));
    return success();
  }
 
  // Resolve `orig`'s device-dependent values (still under the device) keyed by
  // its detached `clone`.
  static void recordDeviceResolved(Operation *orig, Operation *clone,
                                   DeviceResolved &resolved) {
    if (auto w = dyn_cast<AIEX::NpuWrite32Op>(orig)) {
      if (auto a = w.getAbsoluteAddress())
        resolved.absoluteAddr[clone] = *a;
    } else if (auto mw = dyn_cast<AIEX::NpuMaskWrite32Op>(orig)) {
      if (auto a = mw.getAbsoluteAddress())
        resolved.absoluteAddr[clone] = *a;
    } else if (auto mp = dyn_cast<AIEX::NpuMaskPollOp>(orig)) {
      if (auto a = mp.getAbsoluteAddress())
        resolved.absoluteAddr[clone] = *a;
    } else if (auto bw = dyn_cast<AIEX::NpuBlockWriteOp>(orig)) {
      if (auto a = bw.getAbsoluteAddress())
        resolved.absoluteAddr[clone] = *a;
      if (auto d = bw.getDataWords())
        resolved.blockWriteData[clone] = d;
    } else if (auto bwv = dyn_cast<AIEX::NpuBlockWriteValuesOp>(orig)) {
      // Already absolute: no buffer/col/row to fold in.
      if (auto a = AIEX::getConstantIntOperand(bwv.getAddress()))
        resolved.absoluteAddr[clone] = *a;
    }
  }
 
  FailureOr<emitc::FuncOp> emitFunction(OpBuilder &builder, ModuleOp moduleOp,
                                        AIE::RuntimeSequenceOp seqOp,
                                        AIE::DeviceOp deviceOp) {
    Location loc = seqOp.getLoc();
    Block &entry = seqOp.getBody().front();
 
    // The generated function takes the sequence's non-memref args (the runtime
    // scalars) and returns the assembled txn vector. Memref args address host
    // buffers resolved by address_patch; they are not function parameters here.
    SmallVector<Type> paramTypes;
    auto txnVecType =
        emitc::OpaqueType::get(moduleOp.getContext(), kTxnVecType);
    // The builder can fail at TXN-build time (a runtime scalar overflowing a
    // narrow BD field), so it returns std::optional<std::vector<uint32_t>>:
    // std::nullopt on a failed guard, the assembled vector otherwise.
    auto txnRetType = emitc::OpaqueType::get(
        moduleOp.getContext(),
        (llvm::Twine("std::optional<") + kTxnVecType + ">").str());
    // A runtime scalar of type `index` (e.g. a loop trip count) has no C++
    // spelling on its own; emit it as emitc.size_t so it flows straight into a
    // converted scf.for bound with no leftover cast. Other scalar types pass
    // through unchanged.
    auto sizeT = emitc::SizeTType::get(moduleOp.getContext());
    auto paramTypeFor = [&](Type t) -> Type {
      return isa<IndexType>(t) ? Type(sizeT) : t;
    };
    for (BlockArgument arg : entry.getArguments())
      if (!isa<BaseMemRefType>(arg.getType()))
        paramTypes.push_back(paramTypeFor(arg.getType()));
 
    builder.setInsertionPointToEnd(moduleOp.getBody());
    std::string funcName = "generate_txn_" + deviceOp.getSymName().str() + "_" +
                           seqOp.getSymName().str();
    auto funcOp = emitc::FuncOp::create(
        builder, loc, funcName,
        FunctionType::get(moduleOp.getContext(), paramTypes,
                          TypeRange{txnRetType}));
    funcOp.setSpecifiersAttr(builder.getStrArrayAttr({"inline"}));
    Block *funcBlock = funcOp.addEntryBlock();
    OpBuilder fb = OpBuilder::atBlockBegin(funcBlock);
 
    // EmitC has no native std::vector construction, so declare the accumulator
    // with a verbatim statement and reference it by name via an emitc.literal
    // (which translates inline, no extra copy). All txn_append_* calls take
    // this value, so the generated C++ operates on the single `txn` vector.
    emitc::VerbatimOp::create(fb, loc, "std::vector<uint32_t> txn;");
    Value txnVec = emitc::LiteralOp::create(fb, loc, txnVecType, "txn");
    emitTxnCall(fb, loc, "txn_init", txnVec, {});
 
    // Declare each runtime BD free-list pool the sequence draws ids from,
    // with the bounds its ops name.
    llvm::StringSet<> declaredPools;
    auto declarePool = [&](auto poolOp) {
      std::string name = bdPoolName(poolOp);
      if (!declaredPools.insert(name).second)
        return;
      uint32_t lo = poolOp.getBdBegin(), hi = poolOp.getBdEnd();
      emitc::VerbatimOp::create(fb, loc,
                                "aie_runtime::BdPool " + name +
                                    " = aie_runtime::bd_pool_init_range(" +
                                    std::to_string(lo) + ", " +
                                    std::to_string(hi) + ");");
      // Hand back the ids the static allocator already placed on this tile.
      for (uint32_t id :
           staticBdIdsOnTile(deviceOp, poolOp.getColumn(), poolOp.getRow()))
        if (id >= lo && id < hi)
          emitc::VerbatimOp::create(fb, loc,
                                    "aie_runtime::bd_pool_reserve(" + name +
                                        ", " + std::to_string(id) + ");");
    };
    seqOp.walk([&](Operation *op) {
      if (auto pop = dyn_cast<AIEX::DMABdPoolPopOp>(op))
        declarePool(pop);
      else if (auto push = dyn_cast<AIEX::DMABdPoolPushOp>(op))
        declarePool(push);
    });
 
    // A rolled loop makes the op count a runtime quantity: declare a __opcount
    // accumulator the converter bumps per op. A straight-line sequence keeps
    // the compile-time literal count (byte-identical golden).
    bool hasControlFlow = false;
    seqOp.walk([&](Operation *op) {
      if (isa<scf::SCFDialect>(op->getDialect()))
        hasControlFlow = true;
    });
    Value opCountVar;
    if (hasControlFlow) {
      emitc::VerbatimOp::create(fb, loc, "uint32_t __opcount = 0;");
      opCountVar = emitc::LiteralOp::create(
          fb, loc, getU32Type(moduleOp.getContext()), "__opcount");
    }
 
    // Clone the sequence body (whole regions, so a rolled scf.for comes along),
    // mapping non-memref block args to function params. Resolve device-relative
    // values (address, blockwrite data) while the originals are still under the
    // device, keyed by clone, since the accessors don't work once detached.
    IRMapping mapping;
    unsigned p = 0;
    for (BlockArgument arg : entry.getArguments())
      if (!isa<BaseMemRefType>(arg.getType()))
        mapping.map(arg, funcBlock->getArgument(p++));
    DeviceResolved resolved;
    for (Operation &op : entry.without_terminator()) {
      fb.clone(op, mapping);
      // Record device-resolved values for the clone and any nested op (a BD
      // inside a rolled loop resolves the same way as a top-level one).
      op.walk([&](Operation *orig) {
        Operation *c = mapping.lookupOrNull(orig);
        if (c)
          recordDeviceResolved(orig, c, resolved);
      });
    }
 
    AIEXToEmitCConverter conv(funcOp, txnVec, resolved, opCountVar,
                              foldDDRAddrOffset);
    std::optional<uint32_t> count = conv.run();
    if (!count)
      return failure();
 
    // Finalize the header with the op count + device info. The count is the
    // runtime `__opcount` when the sequence has a loop, else the compile-time
    // literal.
    const AIE::AIETargetModel &tm = deviceOp.getTargetModel();
    OpBuilder eb(funcBlock, funcBlock->end());
    std::string countStr =
        hasControlFlow ? "__opcount" : (std::to_string(*count) + "u");
    // txn_device_info assigns TxnDeviceInfo by field name, so a field added or
    // reordered there cannot silently mis-encode this generated header.
    std::string header = "aie_runtime::txn_prepend_header(txn, " + countStr +
                         ", aie_runtime::txn_device_info(" +
                         std::to_string(AIE::txnDeviceGen(tm)) + ", " +
                         std::to_string(tm.rows()) + ", " +
                         std::to_string(tm.columns()) + ", " +
                         std::to_string(tm.getNumMemTileRows()) + "));";
    emitc::VerbatimOp::create(eb, loc, header);
    // Return the vector as the optional result. The literal text differs from
    // the "txn" accumulator literal (so the two are not deduplicated) and is
    // typed as the optional return so emitc's return-type check passes; it
    // relies on the implicit std::vector -> std::optional conversion in C++.
    Value ret = emitc::LiteralOp::create(eb, loc, txnRetType, "std::move(txn)");
    emitc::ReturnOp::create(eb, loc, ret);
    return funcOp;
  }
};
 
} // namespace
 
std::unique_ptr<OperationPass<ModuleOp>>
xilinx::createConvertAIEXToEmitCPass() {
  return std::make_unique<ConvertAIEXToEmitCPass>();
}
 
std::unique_ptr<OperationPass<ModuleOp>>
xilinx::createConvertAIEXToEmitCPass(bool foldDDRAddrOffset,
                                     bool emitDispatchShim) {
  ConvertAIEXToEmitCOptions options;
  options.foldDDRAddrOffset = foldDDRAddrOffset;
  options.emitDispatchShim = emitDispatchShim;
  return std::make_unique<ConvertAIEXToEmitCPass>(options);
}