MLIR-AIE 1.4.6.dev
AIEXToEmitC.cpp
Go to the documentation of this file.
1//===- AIEXToEmitC.cpp - AIEX runtime sequence to EmitC --------*- C++ -*-===//
2//
3// Copyright (C) 2026 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7//
8// Converts npu transaction ops in an aie.runtime_sequence into EmitC calls to
9// aie/Runtime/TxnEncoding.h; translateToCpp() then yields a standalone C++
10// function assembling the same TXN words as the binary emitter
11// (AIETargetNPU.cpp), but runtime-parameterizable. A runtime-bound scf.for
12// (dynamic BD pool path) is preserved and lowered to emitc.for, keeping the
13// loop rolled at runtime.
14//
15//===----------------------------------------------------------------------===//
16
19
24
25#include "mlir/Conversion/ArithToEmitC/ArithToEmitC.h"
26#include "mlir/Conversion/SCFToEmitC/SCFToEmitC.h"
27#include "mlir/Dialect/Arith/IR/Arith.h"
28#include "mlir/Dialect/Arith/Transforms/Passes.h"
29#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
30#include "mlir/Dialect/EmitC/IR/EmitC.h"
31#include "mlir/Dialect/EmitC/Transforms/TypeConversions.h"
32#include "mlir/Dialect/MemRef/IR/MemRef.h"
33#include "mlir/Dialect/SCF/IR/SCF.h"
34#include "mlir/IR/Builders.h"
35#include "mlir/IR/BuiltinOps.h"
36#include "mlir/Interfaces/SideEffectInterfaces.h"
37#include "mlir/Pass/Pass.h"
38#include "mlir/Transforms/DialectConversion.h"
39#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
40#include "mlir/Transforms/RegionUtils.h"
41
42#include "llvm/ADT/StringSet.h"
43#include "llvm/ADT/TypeSwitch.h"
44#include "llvm/Support/Format.h"
45
46namespace xilinx {
47// DECL as well as DEF: the ConvertAIEXToEmitCPass(Options) constructor below
48// needs ConvertAIEXToEmitCOptions declared in this translation unit.
49#define GEN_PASS_DECL_CONVERTAIEXTOEMITC
50#define GEN_PASS_DEF_CONVERTAIEXTOEMITC
51#include "aie/Conversion/Passes.h.inc"
52} // namespace xilinx
53
54using namespace mlir;
55
56// The C++ that declines a build with a reason. `why` becomes a string
57// literal (quotes and backslashes escaped, line breaks flattened). When the
58// snippet goes through an emitc.verbatim with operands, `{}` is a
59// placeholder and `{{` its escape, so a `{` in the message is doubled; a
60// verbatim without operands is emitted literally.
61static std::string refuse(StringRef why, bool inFormat) {
62 std::string lit;
63 for (char ch : why) {
64 if (ch == '\n' || ch == '\r')
65 lit += ' ';
66 else if (ch == '"' || ch == '\\')
67 (lit += '\\') += ch;
68 else if (ch == '{' && inFormat)
69 lit += "{{";
70 else
71 lit += ch;
72 }
73 return "return aie_runtime::txn_refused(\"" + lit + "\");";
74}
75
76using namespace xilinx;
77
78namespace {
79
80// The C++ type of the TXN accumulator the generated function builds.
81constexpr llvm::StringLiteral kTxnVecType = "std::vector<uint32_t>";
82
83emitc::OpaqueType getU32Type(MLIRContext *ctx) {
84 return emitc::OpaqueType::get(ctx, "uint32_t");
85}
86
87// The C++ variable name of a runtime BD pool, declared in the generated
88// function's prologue. One pool per tile and BD partition: channels sharing a
89// partition must share one pool, or they would hand out the same id twice.
90template <typename PoolOp>
91std::string bdPoolName(PoolOp op) {
92 return "bd_pool_" + std::to_string(op.getColumn()) + "_" +
93 std::to_string(op.getRow()) + "_" + std::to_string(op.getBdBegin()) +
94 "_" + std::to_string(op.getBdEnd());
95}
96
97// A uint32_t literal operand (e.g. `42u`) for a txn_append_* argument. Used for
98// the compile-time-resolved address fields (buffer/col/row are folded in).
99Value u32Literal(OpBuilder &b, Location loc, uint32_t val) {
100 return emitc::ConstantOp::create(
101 b, loc, getU32Type(b.getContext()),
102 emitc::OpaqueAttr::get(b.getContext(), std::to_string(val) + "u"));
103}
104
105// Emit a call to a TxnEncoding.h free function on the txn vector. Scalar
106// operands pass through as-is; convert-arith-to-emitc later lowers them to C++
107// literals or function parameters, byte-identical to the static path.
108void emitTxnCall(OpBuilder &b, Location loc, StringRef fn, Value txnVec,
109 ValueRange args) {
110 SmallVector<Value> callArgs;
111 callArgs.push_back(txnVec);
112 callArgs.append(args.begin(), args.end());
113 emitc::CallOpaqueOp::create(b, loc, TypeRange{}, ("aie_runtime::" + fn).str(),
114 callArgs);
115}
116
117// Device-relative values (absolute addr, blockwrite data) resolved before the
118// ops are cloned out of the aie.device, keyed by their clone in the emitc.func.
119struct DeviceResolved {
120 llvm::DenseMap<Operation *, uint32_t> absoluteAddr;
121 llvm::DenseMap<Operation *, DenseIntElementsAttr> blockWriteData;
122};
123
124class AIEXToEmitCConverter {
125public:
126 AIEXToEmitCConverter(emitc::FuncOp funcOp, Value txnVec,
127 const DeviceResolved &resolved, Value opCountVar,
128 bool foldDDRAddrOffset)
129 : funcOp(funcOp), txnVec(txnVec), resolved(resolved),
130 opCountVar(opCountVar), foldDDRAddrOffset(foldDDRAddrOffset) {}
131
132 // Convert every npu/pool op in the body, recursing into scf regions. Returns
133 // the compile-time op count for a straight-line body (the literal op_count),
134 // or nullopt on error. With a runtime op-count var (a loop is present) each
135 // op increments it instead and the returned count is unused.
136 std::optional<uint32_t> run() {
137 // DMA lowering consumes buffer views but can leave dead subview/cast
138 // chains behind (notably after configure/run materialization). They are
139 // not transaction operations and must not reach the C++ converter.
140 bool erased;
141 do {
142 erased = false;
143 funcOp.walk([&](Operation *op) {
144 if (isa<memref::MemRefDialect>(op->getDialect()) &&
145 isOpTriviallyDead(op)) {
146 op->erase();
147 erased = true;
148 }
149 });
150 } while (erased);
151 uint32_t count = 0;
152 SmallVector<Operation *> consumed;
153 convertBlockRecursive(funcOp.getBlocks().front(), count, consumed);
154 if (!ok)
155 return std::nullopt;
156
157 // Erase the converted npu ops, then DCE their now-dead operand clones (a
158 // get_global feeding a blockwrite, a folded-away address constant). Region-
159 // scoped runRegionDCE, not a module canonicalizer: the latter would also
160 // drop the shared memref.global the blockwrite data was inlined from, which
161 // other sequences in the module still reference.
162 for (Operation *op : llvm::reverse(consumed))
163 op->erase();
164 IRRewriter rewriter(funcOp.getContext());
165 (void)runRegionDCE(rewriter, funcOp.getFunctionBody());
166 return count;
167 }
168
169private:
170 // Convert the npu/pool ops in `block`, recursing through scf op regions so a
171 // rolled loop's body is lowered in place (the scf.for itself is left for the
172 // later convert-scf-to-emitc step). Collects converted ops into `consumed`.
173 void convertBlockRecursive(Block &block, uint32_t &count,
174 SmallVector<Operation *> &consumed) {
175 SmallVector<Operation *> work;
176 for (Operation &op : block)
177 work.push_back(&op);
178 for (Operation *op : work) {
179 if (!ok)
180 return;
181 // Structural emitc ops and arith (constant / runtime-value math) are left
182 // for the later arith-to-emitc step. Recurse into scf ops (and any other
183 // region-carrying control flow) to convert their bodies; the control-flow
184 // op itself is handled by convert-scf-to-emitc afterwards.
185 if (isa<emitc::EmitCDialect, arith::ArithDialect>(op->getDialect()))
186 continue;
187 if (isa<scf::SCFDialect>(op->getDialect())) {
188 for (Region &r : op->getRegions())
189 for (Block &b : r)
190 convertBlockRecursive(b, count, consumed);
191 continue;
192 }
193 convertOne(op, count);
194 if (!isa<memref::GetGlobalOp>(op))
195 consumed.push_back(op);
196 }
197 }
198
199private:
200 // Record one emitted txn op: bump the compile-time count and, when a runtime
201 // op-count variable exists (the sequence has a loop, so the count is not
202 // known at compile time), emit a `++__opcount;` so the header gets the true
203 // runtime total.
204 void countOp(OpBuilder &b, Location loc, uint32_t &count) {
205 ++count;
206 if (opCountVar)
207 emitc::VerbatimOp::create(b, loc, "++{};", ValueRange{opCountVar});
208 }
209
210 // Convert one op, appending to the txn vector and incrementing `count`.
211 // Scalar operands flow through as SSA (lowered later by
212 // convert-arith-to-emitc); only the address field is replaced by its
213 // device-resolved literal. Unsupported ops set `ok` false.
214 void convertOne(Operation *op, uint32_t &count) {
215 OpBuilder b(op);
216 Location loc = op->getLoc();
217
218 llvm::TypeSwitch<Operation *, void>(op)
219 .Case<AIEX::NpuWrite32Op>([&](auto w) {
220 // A compile-time address folds to a literal; a runtime address (the
221 // dynamic BD pool's bdBase + wordIdx, arith over a popped id) is
222 // passed through as SSA and lowered by convert-arith-to-emitc.
223 Value addrV = runtimeOrResolvedAddr(b, loc, w, w.getAddress());
224 if (!addrV)
225 return fail(w, "cannot convert a symbolic/unresolved write32 "
226 "address to the C++ TXN target");
227 emitTxnCall(b, loc, "txn_append_write32", txnVec,
228 {addrV, w.getValue()});
229 countOp(b, loc, count);
230 })
231 .Case<AIEX::NpuMaskWrite32Op>([&](auto mw) {
232 Value addrV = runtimeOrResolvedAddr(b, loc, mw, mw.getAddress());
233 if (!addrV)
234 return fail(mw, "cannot convert a symbolic/unresolved maskwrite32 "
235 "address to the C++ TXN target");
236 emitTxnCall(b, loc, "txn_append_maskwrite32", txnVec,
237 {addrV, mw.getValue(), mw.getMask()});
238 countOp(b, loc, count);
239 })
240 .Case<AIEX::NpuMaskPollOp>([&](auto mp) {
241 Value addrV = runtimeOrResolvedAddr(b, loc, mp, mp.getAddress());
242 if (!addrV)
243 return fail(mp, "cannot convert a symbolic/unresolved maskpoll "
244 "address to the C++ TXN target");
245 emitTxnCall(b, loc, "txn_append_maskpoll32", txnVec,
246 {addrV, mp.getValue(), mp.getMask()});
247 countOp(b, loc, count);
248 })
249 .Case<AIEX::NpuSyncOp>([&](auto s) {
250 emitTxnCall(b, loc, "txn_append_sync", txnVec,
251 {s.getColumn(), s.getRow(), s.getDirection(),
252 s.getChannel(), s.getColumnNum(), s.getRowNum()});
253 countOp(b, loc, count);
254 })
255 .Case<AIEX::NpuLoadPdiOp>([&](auto pdi) {
256 Value address = emitc::ConstantOp::create(
257 b, loc, emitc::OpaqueType::get(b.getContext(), "uint64_t"),
258 emitc::OpaqueAttr::get(b.getContext(),
259 std::to_string(pdi.getAddress()) + "ULL"));
260 emitTxnCall(b, loc, "txn_append_loadpdi", txnVec,
261 {u32Literal(b, loc, pdi.getId()),
262 u32Literal(b, loc, pdi.getSize()), address});
263 countOp(b, loc, count);
264 })
265 .Case<AIEX::NpuPreemptOp>([&](auto preempt) {
266 emitTxnCall(b, loc, "txn_append_preempt", txnVec,
267 {u32Literal(b, loc, preempt.getLevel())});
268 countOp(b, loc, count);
269 })
270 .Case<AIEX::NpuAddressPatchOp>([&](auto ap) {
271 // A runtime bd_id makes the patched register address runtime: prefer
272 // the SSA addr_val (flows through arith-to-emitc) over the constant.
273 Value addrV = ap.getAddrVal() ? ap.getAddrVal()
274 : u32Literal(b, loc, ap.getAddr());
275 std::optional<uint32_t> argIdx = ap.getArgIdx();
276 if (!argIdx)
277 return fail(ap, "address_patch still names its host buffer by SSA "
278 "value; run -aie-resolve-address-patch-buffers");
279 Value idxV = emitc::ConstantOp::create(
280 b, loc, emitc::OpaqueType::get(b.getContext(), "int32_t"),
281 emitc::OpaqueAttr::get(b.getContext(), std::to_string(*argIdx)));
282 // txn_append_arg_patch applies the DDR-aperture fold itself, so the
283 // rule lives only in TxnEncoding.h. Both operands are literals here,
284 // so -O2 folds the branch away.
285 Value foldV = emitc::ConstantOp::create(
286 b, loc, emitc::OpaqueType::get(b.getContext(), "bool"),
287 emitc::OpaqueAttr::get(b.getContext(),
288 foldDDRAddrOffset ? "true" : "false"));
289 Value argPlus = ap.getArgPlus();
290 // i32 offsets are unsigned bit patterns in the static emitter.
291 // Cast before widening so bit 31 is not sign-extended to 64 bits.
292 if (argPlus.getType().isInteger(32))
293 argPlus = emitc::CastOp::create(b, loc, getU32Type(b.getContext()),
294 argPlus);
295 emitTxnCall(b, loc, "txn_append_arg_patch", txnVec,
296 {addrV, idxV, argPlus, foldV});
297 countOp(b, loc, count);
298 })
299 .Case<AIEX::NpuBlockWriteOp>([&](auto bw) {
300 convertBlockWrite(b, loc, bw);
301 countOp(b, loc, count);
302 })
303 .Case<AIEX::NpuBlockWriteValuesOp>([&](auto bw) {
304 convertBlockWriteValues(b, loc, bw);
305 countOp(b, loc, count);
306 })
307 .Case<cf::AssertOp>([&](auto a) {
308 // Host-side guard: a violated condition yields no stream
309 // (std::nullopt), and its message is what the host reports. Appends
310 // nothing, so not counted.
311 emitc::VerbatimOp::create(
312 b, loc, "if (!({})) " + refuse(a.getMsg(), /*inFormat=*/true),
313 ValueRange{a.getArg()});
314 })
315 .Case<AIEX::DMABdPoolPopOp>([&](AIEX::DMABdPoolPopOp pop) {
316 if (std::optional<uint32_t> id = pop.getId()) {
317 emitc::VerbatimOp::create(b, loc,
318 "aie_runtime::bd_pool_reserve(" +
319 bdPoolName(pop) + ", " +
320 std::to_string(*id) + ");");
321 pop.getBdId().replaceAllUsesWith(emitc::LiteralOp::create(
322 b, loc, pop.getBdId().getType(), std::to_string(*id)));
323 return;
324 }
325 // Draw a BD id from the tile's runtime pool (declared in the
326 // prologue) into a fresh C++ variable; nullopt if the pool is empty.
327 // Downstream ops consume the id, so replace the SSA result with a
328 // literal naming the variable.
329 std::string var = "bd_" + std::to_string(nextPoolVar++);
330 emitc::VerbatimOp::create(
331 b, loc,
332 "uint32_t " + var + "; if (!aie_runtime::bd_pool_pop(" +
333 bdPoolName(pop) + ", " + var + ")) " +
334 refuse("no free buffer descriptor on tile (" +
335 std::to_string(pop.getColumn()) + ", " +
336 std::to_string(pop.getRow()) +
337 "): the sequence keeps more transfers in flight "
338 "than the tile has descriptors",
339 /*inFormat=*/false));
340 Value ref =
341 emitc::LiteralOp::create(b, loc, pop.getBdId().getType(), var);
342 pop.getBdId().replaceAllUsesWith(ref);
343 })
344 .Case<AIEX::DMABdPoolPushOp>([&](AIEX::DMABdPoolPushOp push) {
345 // Return a BD id to the tile's runtime pool.
346 emitc::CallOpaqueOp::create(
347 b, loc, TypeRange{}, "aie_runtime::bd_pool_push",
348 ValueRange{poolRef(b, loc, bdPoolName(push)), push.getBdId()});
349 })
350 // XRT allocates the scratchpad only for a full-ELF context, which has
351 // no per-call instruction buffer to carry a generated stream.
352 .Case<AIEX::NpuCreateScratchpadOp, AIEX::NpuUpdateFromScratchpadOp>(
353 [&](Operation *sp) {
354 fail(sp, "needs the scratchpad of a full-ELF context, which the "
355 "C++ TXN target's per-call streams run without; use a "
356 "runtime sequence argument instead of a scratchpad "
357 "parameter");
358 })
359 // memref.get_global feeding a blockwrite is consumed by
360 // convertBlockWrite (data inlined); the now-dead op is erased later.
361 .Case<memref::GetGlobalOp>([&](auto) {})
362 .Default([&](Operation *unsupported) {
363 if (isa<scf::SCFDialect>(unsupported->getDialect()))
364 fail(unsupported,
365 "control flow in runtime sequences is not yet supported by "
366 "the C++ TXN target");
367 else
368 fail(unsupported, "is not supported by the C++ TXN target");
369 });
370 }
371
372 // The compile-time-resolved absolute address (folding buffer/col/row) as a
373 // uint32_t literal, or null if it could not be resolved statically. Address
374 // resolution depends on the parent device, so it is precomputed before the op
375 // is cloned out (see recordDeviceResolved).
376 Value resolvedAddr(OpBuilder &b, Location loc, Operation *clone) {
377 auto it = resolved.absoluteAddr.find(clone);
378 if (it == resolved.absoluteAddr.end())
379 return {};
380 return u32Literal(b, loc, it->second);
381 }
382
383 // The address for a write32/maskwrite32: the device-resolved literal, else a
384 // genuine runtime address (arith-derived, e.g. the pool's bdBase + wordIdx).
385 // A bare block argument selects no register and is rejected; null for a
386 // symbolic/unresolved address.
387 Value runtimeOrResolvedAddr(OpBuilder &b, Location loc, Operation *clone,
388 Value addrOperand) {
389 if (Value lit = resolvedAddr(b, loc, clone))
390 return lit;
391 Operation *def = addrOperand ? addrOperand.getDefiningOp() : nullptr;
392 if (def && !isa<arith::ConstantOp>(def))
393 return addrOperand;
394 return {};
395 }
396
397 // Emit a diagnostic on `op` and mark the conversion failed.
398 void fail(Operation *op, const llvm::Twine &msg) {
399 op->emitOpError(msg);
400 ok = false;
401 }
402
403 // An emitc.literal referencing a pool variable (for pass-by-reference into
404 // bd_pool_push).
405 Value poolRef(OpBuilder &b, Location loc, StringRef name) {
406 return emitc::LiteralOp::create(
407 b, loc, emitc::OpaqueType::get(b.getContext(), "aie_runtime::BdPool"),
408 name);
409 }
410
411 void convertBlockWrite(OpBuilder &b, Location loc, AIEX::NpuBlockWriteOp bw) {
412 auto addrIt = resolved.absoluteAddr.find(bw);
413 auto dataIt = resolved.blockWriteData.find(bw);
414 if (addrIt == resolved.absoluteAddr.end() ||
415 dataIt == resolved.blockWriteData.end() || !dataIt->second) {
416 bw.emitOpError("cannot convert blockwrite without a constant address and "
417 "data to the C++ TXN target");
418 ok = false;
419 return;
420 }
421 uint32_t addr = addrIt->second;
422 DenseIntElementsAttr data = dataIt->second;
423
424 // Emit the payload as a typed EmitC array variable rather than a verbatim
425 // C++ string, so the generated code carries real types.
426 MLIRContext *ctx = b.getContext();
427 int64_t n = data.size();
428 auto arrTy =
429 emitc::ArrayType::get(ctx, SmallVector<int64_t>{n}, getU32Type(ctx));
430 std::string init = "{";
431 llvm::raw_string_ostream ss(init);
432 bool first = true;
433 for (APInt word : data.getValues<APInt>()) {
434 if (!first)
435 ss << ", ";
436 ss << llvm::format("0x%08xu", static_cast<unsigned>(word.getZExtValue()));
437 first = false;
438 }
439 ss << "}";
440 auto arrVar = emitc::VariableOp::create(b, loc, arrTy,
441 emitc::OpaqueAttr::get(ctx, init));
442
443 uint32_t colVal = 0, rowVal = 0;
444 if (auto col = bw.getColumn(), row = bw.getRow(); col && row) {
445 colVal = *col;
446 rowVal = *row;
447 }
448 emitTxnCall(b, loc, "txn_append_blockwrite", txnVec,
449 {u32Literal(b, loc, addr), arrVar.getResult(),
450 u32Literal(b, loc, static_cast<uint32_t>(n)),
451 u32Literal(b, loc, colVal), u32Literal(b, loc, rowVal)});
452 }
453
454 // A blockwrite whose payload is computed at TXN-build time:
455 // stage the words into a local C++ array, then hand that array
456 // to one txn_append_blockwrite. Unlike convertBlockWrite there is no constant
457 // memref to inline, so the slots are filled by assignment from the operand
458 // values, which convert-arith-to-emitc lowers in place afterwards.
459 void convertBlockWriteValues(OpBuilder &b, Location loc,
460 AIEX::NpuBlockWriteValuesOp bw) {
461 MLIRContext *ctx = b.getContext();
462 Value addrV = runtimeOrResolvedAddr(b, loc, bw, bw.getAddress());
463 if (!addrV) {
464 fail(bw, "cannot convert a symbolic/unresolved blockwrite_values address "
465 "to the C++ TXN target");
466 return;
467 }
468 ValueRange words = bw.getValues();
469 int64_t n = static_cast<int64_t>(words.size());
470
471 // Declare the staging array (zero-initialized; every slot is assigned
472 // below). Typed emitc.variable, matching convertBlockWrite's array.
473 auto arrTy =
474 emitc::ArrayType::get(ctx, SmallVector<int64_t>{n}, getU32Type(ctx));
475 auto arrVar = emitc::VariableOp::create(b, loc, arrTy,
476 emitc::OpaqueAttr::get(ctx, "{}"));
477
478 // Fill the slots. Emitted as verbatim assignments (the same mechanism the
479 // runtime guards above use to reference a not-yet-lowered SSA value): the
480 // i32 words convert to the array's uint32_t elementwise, which is the
481 // intended two's-complement reinterpretation of the packed BD fields.
482 for (int64_t i = 0; i < n; i++)
483 emitc::VerbatimOp::create(b, loc, "{}[" + std::to_string(i) + "] = {};",
484 ValueRange{arrVar.getResult(), words[i]});
485
486 // col/row are 0: the address is already flat (as for npu.blockwrite).
487 emitTxnCall(b, loc, "txn_append_blockwrite", txnVec,
488 {addrV, arrVar.getResult(),
489 u32Literal(b, loc, static_cast<uint32_t>(n)),
490 u32Literal(b, loc, 0), u32Literal(b, loc, 0)});
491 }
492
493 emitc::FuncOp funcOp;
494 Value txnVec;
495 const DeviceResolved &resolved;
496 // Runtime op-count variable (the C++ `__opcount`), or null when the sequence
497 // is straight-line and the count is a compile-time literal.
498 Value opCountVar;
499 bool ok = true;
500 // Counter for unique popped-BD-id C++ variable names.
501 unsigned nextPoolVar = 0;
502 bool foldDDRAddrOffset;
503};
504
505struct ConvertAIEXToEmitCPass
506 : xilinx::impl::ConvertAIEXToEmitCBase<ConvertAIEXToEmitCPass> {
507 ConvertAIEXToEmitCPass() = default;
508 // foldDDRAddrOffset is `protected` on the generated base, settable only via
509 // a -pass-pipeline string; this constructor is the programmatic equivalent.
510 explicit ConvertAIEXToEmitCPass(const ConvertAIEXToEmitCOptions &options)
511 : ConvertAIEXToEmitCBase(options) {}
512
513 void runOnOperation() override {
514 ModuleOp moduleOp = getOperation();
515 MLIRContext *ctx = &getContext();
516 if (failed(AIE::inlineWriteConfigs(moduleOp)))
517 return signalPassFailure();
518
519 struct SeqInfo {
520 AIE::RuntimeSequenceOp seq;
521 AIE::DeviceOp device;
522 };
523 SmallVector<SeqInfo> sequences;
524 moduleOp.walk([&](AIE::RuntimeSequenceOp seq) {
525 sequences.push_back({seq, seq->getParentOfType<AIE::DeviceOp>()});
526 });
527 if (emitDispatchShim && sequences.size() != 1) {
528 moduleOp.emitOpError()
529 << "emit-dispatch-shim needs exactly one aie.runtime_sequence to "
530 "wrap, found "
531 << sequences.size();
532 return signalPassFailure();
533 }
534 if (sequences.empty())
535 return;
536
537 OpBuilder builder(ctx);
538 for (auto &[seqOp, deviceOp] : sequences) {
539 if (!deviceOp) {
540 seqOp.emitOpError("must be nested inside an aie.device");
541 return signalPassFailure();
542 }
543 std::optional<emitc::FuncOp> gen =
544 emitFunction(builder, moduleOp, seqOp, deviceOp);
545 if (!gen)
546 return signalPassFailure();
547 if (emitDispatchShim) {
548 emitc::FuncOp generated = *gen;
549 if (failed(emitDispatchShimFuncs(builder, moduleOp, generated)))
550 return signalPassFailure();
551 }
552 }
553
554 // Replace the module body with just the generated emitc funcs + includes.
555 SmallVector<Operation *> toErase;
556 for (Operation &op : *moduleOp.getBody())
557 if (!isa<emitc::FuncOp, emitc::IncludeOp>(op))
558 toErase.push_back(&op);
559 for (Operation *op : llvm::reverse(toErase))
560 op->erase();
561
562 // Lower the arith ops (constant scalar fields, runtime-value arithmetic
563 // feeding npu ops) and any scf control flow (a rolled dynamic loop) left in
564 // the function bodies to emitc.
565 if (failed(lowerArithAndScfToEmitC(moduleOp)))
566 return signalPassFailure();
567
568 builder.setInsertionPointToStart(moduleOp.getBody());
569 emitc::IncludeOp::create(builder, moduleOp.getLoc(),
570 "aie/Runtime/TxnEncoding.h",
571 /*is_standard_include=*/false);
572 emitc::IncludeOp::create(builder, moduleOp.getLoc(), "cstdint",
573 /*is_standard_include=*/true);
574 emitc::IncludeOp::create(builder, moduleOp.getLoc(), "vector",
575 /*is_standard_include=*/true);
576 // The builder returns std::optional: a runtime scalar that would overflow a
577 // narrow BD field yields std::nullopt instead of a truncated stream.
578 emitc::IncludeOp::create(builder, moduleOp.getLoc(), "optional",
579 /*is_standard_include=*/true);
580 }
581
582 // Run the upstream arith-to-emitc and scf-to-emitc conversions over the
583 // generated functions in one pass: a rolled dynamic loop is scf.for, and its
584 // bounds / iter_args are arith, so both must lower together.
585 LogicalResult lowerArithAndScfToEmitC(ModuleOp moduleOp) {
586 // ArithToEmitC has no patterns for the integer ops the Python bindings
587 // emit for `//`, ceildiv, min and max on staged scalars. Expand them to
588 // the divsi/cmpi/select forms it does lower before converting.
589 {
590 RewritePatternSet expand(moduleOp.getContext());
591 arith::populateCeilFloorDivExpandOpsPatterns(expand);
592 arith::populateExpandMinMaxIPatterns(expand);
593 // Rewrite only: folding or CSE here would reshape the sequence the
594 // per-op conversion below expects (and the C++ its tests check).
595 GreedyRewriteConfig config;
596 config.enableFolding(false).enableConstantCSE(false);
597 if (failed(applyPatternsGreedily(moduleOp, std::move(expand), config)))
598 return failure();
599 }
600
601 TypeConverter typeConverter;
602 typeConverter.addConversion([](Type t) { return t; });
603 populateEmitCSizeTTypeConversions(typeConverter);
604
605 RewritePatternSet patterns(moduleOp.getContext());
606 populateArithToEmitCPatterns(typeConverter, patterns);
607 populateSCFToEmitCConversionPatterns(patterns, typeConverter);
608
609 ConversionTarget target(*moduleOp.getContext());
610 target.addLegalDialect<emitc::EmitCDialect>();
611 target.addIllegalDialect<arith::ArithDialect>();
612 target.addIllegalDialect<scf::SCFDialect>();
613 return applyPartialConversion(moduleOp, target, std::move(patterns));
614 }
615
616private:
617 // The C spelling emitc's translator emits for a parameter type, for the ABI
618 // string only. Must mirror CppEmitter::emitType down to its signedness rule,
619 // or the reported ABI disagrees with the emitted signature.
620 static std::optional<std::string> cTypeName(Type t) {
621 if (isa<emitc::SizeTType>(t))
622 return std::string("size_t");
623 auto intTy = dyn_cast<IntegerType>(t);
624 if (!intTy || !emitc::isSupportedIntegerType(intTy))
625 return std::nullopt;
626 if (intTy.getWidth() == 1)
627 return std::string("bool");
628 return (intTy.isUnsigned() ? "uint" : "int") +
629 std::to_string(intTy.getWidth()) + "_t";
630 }
631
632 // Emit the ctypes-callable entry points: dispatch_abi() reporting the
633 // parameter types, and dispatch_generate() handing back a pointer + word
634 // count into thread-local storage, or -2 when the builder declined.
635 LogicalResult emitDispatchShimFuncs(OpBuilder &builder, ModuleOp moduleOp,
636 emitc::FuncOp gen) {
637 MLIRContext *ctx = moduleOp.getContext();
638 Location loc = gen.getLoc();
639 TypeRange genParams = gen.getFunctionType().getInputs();
640
641 SmallVector<std::string> abiTypes;
642 for (Type t : genParams) {
643 std::optional<std::string> name = cTypeName(t);
644 if (!name)
645 return gen.emitOpError()
646 << "runtime-sequence parameter type " << t
647 << " has no dispatch ABI spelling; it cannot be passed from the "
648 "host at dispatch time";
649 abiTypes.push_back(*name);
650 }
651
652 builder.setInsertionPointToEnd(moduleOp.getBody());
653 // Python resolves these two by name out of the built shared library. ELF
654 // exports every extern symbol unless told otherwise, but MSVC-target
655 // linking exports nothing without an explicit marker, so on Windows the
656 // DLL would load and then fail to resolve dispatch_abi. The macro comes
657 // from TxnEncoding.h, which the emitted file already includes, and expands
658 // to nothing off Windows.
659 auto externC =
660 builder.getStrArrayAttr({"extern \"C\"", "AIE_DISPATCH_EXPORT"});
661
662 // Pointers must be emitc.ptr: an opaque type may not have a pointer as its
663 // outer type.
664 auto charPtrTy =
665 emitc::PointerType::get(emitc::OpaqueType::get(ctx, "const char"));
666 auto abiFn = emitc::FuncOp::create(builder, loc, "dispatch_abi",
667 FunctionType::get(ctx, {}, {charPtrTy}));
668 abiFn.setSpecifiersAttr(externC);
669 OpBuilder ab = OpBuilder::atBlockBegin(abiFn.addEntryBlock());
670 Value abiStr = emitc::LiteralOp::create(
671 ab, loc, charPtrTy, "\"" + llvm::join(abiTypes, ",") + "\"");
672 emitc::ReturnOp::create(ab, loc, abiStr);
673
674 SmallVector<Type> shimParams(genParams);
675 shimParams.push_back(
676 emitc::PointerType::get(emitc::PointerType::get(getU32Type(ctx))));
677 auto i64Ty = emitc::OpaqueType::get(ctx, "int64_t");
678 auto shimFn =
679 emitc::FuncOp::create(builder, loc, "dispatch_generate",
680 FunctionType::get(ctx, shimParams, {i64Ty}));
681 shimFn.setSpecifiersAttr(externC);
682 Block *shimBlock = shimFn.addEntryBlock();
683 OpBuilder sb = OpBuilder::atBlockBegin(shimBlock);
684
685 SmallVector<Value> genArgs(shimBlock->getArguments().drop_back());
686 SmallVector<StringRef> holes(genArgs.size(), "{}");
687 std::string call = "auto __result = " + gen.getSymName().str() + "(" +
688 llvm::join(holes, ", ") + ");";
689 emitc::VerbatimOp::create(
690 sb, loc, "static thread_local " + kTxnVecType.str() + " __txn;");
691 emitc::VerbatimOp::create(sb, loc, "aie_runtime::txn_refusal = nullptr;");
692 emitc::VerbatimOp::create(sb, loc, call, genArgs);
693 emitc::VerbatimOp::create(sb, loc, "if (!__result) return -2;");
694 emitc::VerbatimOp::create(sb, loc, "__txn = std::move(*__result);");
695 emitc::VerbatimOp::create(sb, loc, "*{} = __txn.data();",
696 ValueRange{shimBlock->getArguments().back()});
697 emitc::ReturnOp::create(
698 sb, loc,
699 emitc::LiteralOp::create(sb, loc, i64Ty,
700 "static_cast<int64_t>(__txn.size())"));
701
702 // dispatch_last_refusal(): why the last dispatch_generate() on this
703 // thread returned -2 (a string literal in this library), or null.
704 auto whyFn = emitc::FuncOp::create(builder, loc, "dispatch_last_refusal",
705 FunctionType::get(ctx, {}, {charPtrTy}));
706 whyFn.setSpecifiersAttr(externC);
707 OpBuilder wb = OpBuilder::atBlockBegin(whyFn.addEntryBlock());
708 emitc::ReturnOp::create(
709 wb, loc,
710 emitc::LiteralOp::create(wb, loc, charPtrTy,
711 "aie_runtime::txn_refusal"));
712 return success();
713 }
714
715 // Resolve `orig`'s device-dependent values (still under the device) keyed by
716 // its detached `clone`.
717 static void recordDeviceResolved(Operation *orig, Operation *clone,
718 DeviceResolved &resolved) {
719 if (auto w = dyn_cast<AIEX::NpuWrite32Op>(orig)) {
720 if (auto a = w.getAbsoluteAddress())
721 resolved.absoluteAddr[clone] = *a;
722 } else if (auto mw = dyn_cast<AIEX::NpuMaskWrite32Op>(orig)) {
723 if (auto a = mw.getAbsoluteAddress())
724 resolved.absoluteAddr[clone] = *a;
725 } else if (auto mp = dyn_cast<AIEX::NpuMaskPollOp>(orig)) {
726 if (auto a = mp.getAbsoluteAddress())
727 resolved.absoluteAddr[clone] = *a;
728 } else if (auto bw = dyn_cast<AIEX::NpuBlockWriteOp>(orig)) {
729 if (auto a = bw.getAbsoluteAddress())
730 resolved.absoluteAddr[clone] = *a;
731 if (auto d = bw.getDataWords())
732 resolved.blockWriteData[clone] = d;
733 } else if (auto bwv = dyn_cast<AIEX::NpuBlockWriteValuesOp>(orig)) {
734 // Already absolute: no buffer/col/row to fold in.
735 if (auto a = AIEX::getConstantIntOperand(bwv.getAddress()))
736 resolved.absoluteAddr[clone] = *a;
737 }
738 }
739
740 FailureOr<emitc::FuncOp> emitFunction(OpBuilder &builder, ModuleOp moduleOp,
741 AIE::RuntimeSequenceOp seqOp,
742 AIE::DeviceOp deviceOp) {
743 Location loc = seqOp.getLoc();
744 Block &entry = seqOp.getBody().front();
745
746 // The generated function takes the sequence's non-memref args (the runtime
747 // scalars) and returns the assembled txn vector. Memref args address host
748 // buffers resolved by address_patch; they are not function parameters here.
749 SmallVector<Type> paramTypes;
750 auto txnVecType =
751 emitc::OpaqueType::get(moduleOp.getContext(), kTxnVecType);
752 // The builder can fail at TXN-build time (a runtime scalar overflowing a
753 // narrow BD field), so it returns std::optional<std::vector<uint32_t>>:
754 // std::nullopt on a failed guard, the assembled vector otherwise.
755 auto txnRetType = emitc::OpaqueType::get(
756 moduleOp.getContext(),
757 (llvm::Twine("std::optional<") + kTxnVecType + ">").str());
758 // A runtime scalar of type `index` (e.g. a loop trip count) has no C++
759 // spelling on its own; emit it as emitc.size_t so it flows straight into a
760 // converted scf.for bound with no leftover cast. Other scalar types pass
761 // through unchanged.
762 auto sizeT = emitc::SizeTType::get(moduleOp.getContext());
763 auto paramTypeFor = [&](Type t) -> Type {
764 return isa<IndexType>(t) ? Type(sizeT) : t;
765 };
766 for (BlockArgument arg : entry.getArguments())
767 if (!isa<BaseMemRefType>(arg.getType()))
768 paramTypes.push_back(paramTypeFor(arg.getType()));
769
770 builder.setInsertionPointToEnd(moduleOp.getBody());
771 std::string funcName = "generate_txn_" + deviceOp.getSymName().str() + "_" +
772 seqOp.getSymName().str();
773 auto funcOp = emitc::FuncOp::create(
774 builder, loc, funcName,
775 FunctionType::get(moduleOp.getContext(), paramTypes,
776 TypeRange{txnRetType}));
777 funcOp.setSpecifiersAttr(builder.getStrArrayAttr({"inline"}));
778 Block *funcBlock = funcOp.addEntryBlock();
779 OpBuilder fb = OpBuilder::atBlockBegin(funcBlock);
780
781 // EmitC has no native std::vector construction, so declare the accumulator
782 // with a verbatim statement and reference it by name via an emitc.literal
783 // (which translates inline, no extra copy). All txn_append_* calls take
784 // this value, so the generated C++ operates on the single `txn` vector.
785 emitc::VerbatimOp::create(fb, loc, "std::vector<uint32_t> txn;");
786 Value txnVec = emitc::LiteralOp::create(fb, loc, txnVecType, "txn");
787 emitTxnCall(fb, loc, "txn_init", txnVec, {});
788
789 // Declare each runtime BD free-list pool the sequence draws ids from,
790 // with the bounds its ops name.
791 llvm::StringSet<> declaredPools;
792 auto declarePool = [&](auto poolOp) {
793 std::string name = bdPoolName(poolOp);
794 if (!declaredPools.insert(name).second)
795 return;
796 uint32_t lo = poolOp.getBdBegin(), hi = poolOp.getBdEnd();
797 emitc::VerbatimOp::create(fb, loc,
798 "aie_runtime::BdPool " + name +
799 " = aie_runtime::bd_pool_init_range(" +
800 std::to_string(lo) + ", " +
801 std::to_string(hi) + ");");
802 };
803 seqOp.walk([&](Operation *op) {
804 if (auto pop = dyn_cast<AIEX::DMABdPoolPopOp>(op))
805 declarePool(pop);
806 else if (auto push = dyn_cast<AIEX::DMABdPoolPushOp>(op))
807 declarePool(push);
808 });
809
810 // A rolled loop makes the op count a runtime quantity: declare a __opcount
811 // accumulator the converter bumps per op. A straight-line sequence keeps
812 // the compile-time literal count (byte-identical golden).
813 bool hasControlFlow = false;
814 seqOp.walk([&](Operation *op) {
815 if (isa<scf::SCFDialect>(op->getDialect()))
816 hasControlFlow = true;
817 });
818 Value opCountVar;
819 if (hasControlFlow) {
820 emitc::VerbatimOp::create(fb, loc, "uint32_t __opcount = 0;");
821 opCountVar = emitc::LiteralOp::create(
822 fb, loc, getU32Type(moduleOp.getContext()), "__opcount");
823 }
824
825 // Clone the sequence body (whole regions, so a rolled scf.for comes along),
826 // mapping non-memref block args to function params. Resolve device-relative
827 // values (address, blockwrite data) while the originals are still under the
828 // device, keyed by clone, since the accessors don't work once detached.
829 IRMapping mapping;
830 unsigned p = 0;
831 for (BlockArgument arg : entry.getArguments())
832 if (!isa<BaseMemRefType>(arg.getType()))
833 mapping.map(arg, funcBlock->getArgument(p++));
834 DeviceResolved resolved;
835 for (Operation &op : entry.without_terminator()) {
836 fb.clone(op, mapping);
837 // Record device-resolved values for the clone and any nested op (a BD
838 // inside a rolled loop resolves the same way as a top-level one).
839 op.walk([&](Operation *orig) {
840 Operation *c = mapping.lookupOrNull(orig);
841 if (c)
842 recordDeviceResolved(orig, c, resolved);
843 });
844 }
845
846 AIEXToEmitCConverter conv(funcOp, txnVec, resolved, opCountVar,
847 foldDDRAddrOffset);
848 std::optional<uint32_t> count = conv.run();
849 if (!count)
850 return failure();
851
852 // Finalize the header with the op count + device info. The count is the
853 // runtime `__opcount` when the sequence has a loop, else the compile-time
854 // literal.
855 const AIE::AIETargetModel &tm = deviceOp.getTargetModel();
856 OpBuilder eb(funcBlock, funcBlock->end());
857 std::string countStr =
858 hasControlFlow ? "__opcount" : (std::to_string(*count) + "u");
859 // txn_device_info assigns TxnDeviceInfo by field name, so a field added or
860 // reordered there cannot silently mis-encode this generated header.
861 std::string header = "aie_runtime::txn_prepend_header(txn, " + countStr +
862 ", aie_runtime::txn_device_info(" +
863 std::to_string(AIE::txnDeviceGen(tm)) + ", " +
864 std::to_string(tm.rows()) + ", " +
865 std::to_string(tm.columns()) + ", " +
866 std::to_string(tm.getNumMemTileRows()) + "));";
867 emitc::VerbatimOp::create(eb, loc, header);
868 // Return the vector as the optional result. The literal text differs from
869 // the "txn" accumulator literal (so the two are not deduplicated) and is
870 // typed as the optional return so emitc's return-type check passes; it
871 // relies on the implicit std::vector -> std::optional conversion in C++.
872 Value ret = emitc::LiteralOp::create(eb, loc, txnRetType, "std::move(txn)");
873 emitc::ReturnOp::create(eb, loc, ret);
874 return funcOp;
875 }
876};
877
878} // namespace
879
880std::unique_ptr<OperationPass<ModuleOp>>
882 return std::make_unique<ConvertAIEXToEmitCPass>();
883}
884
885std::unique_ptr<OperationPass<ModuleOp>>
887 bool emitDispatchShim) {
888 ConvertAIEXToEmitCOptions options;
889 options.foldDDRAddrOffset = foldDDRAddrOffset;
890 options.emitDispatchShim = emitDispatchShim;
891 return std::make_unique<ConvertAIEXToEmitCPass>(options);
892}
virtual int rows() const =0
Return the number of rows in the device.
virtual int columns() const =0
Return the number of columns in the device.
virtual uint32_t getNumMemTileRows() const =0
std::optional< uint32_t > getConstantIntOperand(mlir::Value v)
uint8_t txnDeviceGen(const AIETargetModel &tm)
Device generation ID written into the TXN header (aie_runtime::TxnDeviceInfo::devGen).
mlir::LogicalResult inlineWriteConfigs(mlir::ModuleOp module)
std::unique_ptr< mlir::OperationPass< mlir::ModuleOp > > createConvertAIEXToEmitCPass()