MLIR-AIE 1.4.6.dev
TxnEncoding.h
Go to the documentation of this file.
1//===- TxnEncoding.h - Standalone TXN instruction encoding ------*- C++ -*-===//
2//
3// Copyright (C) 2026 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7//
8// Header-only library for encoding AI Engine TXN (transaction) instructions.
9// This has ZERO dependencies on MLIR or LLVM and can be used standalone in
10// host applications to generate TXN binaries at runtime. It is the single
11// in-tree source of truth for the instruction format, used by both the
12// compiler (AIETargetNPU.cpp) and generated host code.
13//
14//===----------------------------------------------------------------------===//
15
16#ifndef AIE_RUNTIME_TXNENCODING_H
17#define AIE_RUNTIME_TXNENCODING_H
18
19#include <cstddef>
20#include <cstdint>
21#include <optional>
22#include <vector>
23
24// Applied to the emitted entry points; see emitDispatchShimFuncs.
25#ifdef _WIN32
26#define AIE_DISPATCH_EXPORT __declspec(dllexport)
27#else
28#define AIE_DISPATCH_EXPORT
29#endif
30
31namespace aie_runtime {
32
33// Transaction opcodes for the firmware TXN format the compiler currently
34// targets.
35//
36// These values currently agree with the XAie_TxnOpcode enum in the pinned
37// third_party/aie-codegen xaie_txn.h. They are duplicated here, rather than
38// included from the driver, so this header stays independent of it.
39//
40// DRIFT WARNING: when the firmware TXN format changes or the aie-codegen pin
41// is bumped, update these values here and compare them against xaie_txn.h.
69
70// Upper bound on a tile's buffer-descriptor count across all tile types, sizing
71// the pool's fixed storage. This is a standalone header (no MLIR / target model
72// here), so the ACTUAL per-tile count is supplied at runtime by the compiler
73// via bd_pool_init_range -- this constant only bounds the array.
74// AIE2 memtiles have the most BDs (48); shim/core have 16. Keep this >= the
75// largest getNumBDs any target model returns.
76constexpr uint32_t kMaxBDsPerTile = 48;
77
78// Runtime buffer-descriptor free-list pool, for dynamic runtime sequences whose
79// BD IDs are chosen at runtime (a rolled scf.for the compiler cannot unroll).
80// The static allocator assigns BD IDs at compile time; this is its runtime
81// counterpart. `head` is the number of currently-free IDs at the front of
82// `free_ids`; pop takes from the top, push returns to it.
83struct BdPool {
85 int head;
86};
87
88// Initialize a pool over the half-open id range [lo, hi), the ids one BD
89// partition of a tile can submit (the compiler reads it from the target model).
90// A mem tile splits its BD table by channel parity, so each half gets its own
91// pool; one flat list would hand a channel ids it cannot submit. IDs are
92// stacked so that pop() hands out the LOWEST free id first, matching the static
93// allocator's lowest-free-first order -- so a first pop from a fresh pool
94// equals a pinned bd_id = lo.
95inline BdPool bd_pool_init_range(uint32_t lo, uint32_t hi) {
96 BdPool p;
97 p.head = 0;
98 if (hi > kMaxBDsPerTile)
99 hi = kMaxBDsPerTile;
100 for (uint32_t i = hi; i > lo; --i)
101 p.free_ids[p.head++] = i - 1; // top of stack is `lo`
102 return p;
103}
104
105// Initialize a pool with the `n` free IDs [0, n).
106inline BdPool bd_pool_init(uint32_t n) { return bd_pool_init_range(0, n); }
107
108// Withhold `id` from the pool -- a static BD already owns that slot in the
109// tile's shared BD table, and popping it here would silently overwrite it.
110// `free_ids[0..head)` is kept sorted highest-to-lowest (see bd_pool_init_range)
111// so pop keeps handing out the lowest remaining id first; shift the tail down
112// instead of swapping in the top, which would scramble that order.
113inline void bd_pool_reserve(BdPool &p, uint32_t id) {
114 for (int i = 0; i < p.head; ++i)
115 if (p.free_ids[i] == id) {
116 for (int j = i; j < p.head - 1; ++j)
117 p.free_ids[j] = p.free_ids[j + 1];
118 --p.head;
119 return;
120 }
121}
122
123// Pop a free BD ID into `out`. Returns false if the pool is empty -- the
124// generated builder turns that into a `return std::nullopt`, so a runtime
125// working set that exceeds the tile's BD count yields no stream rather than a
126// silently-corrupt one.
127inline bool bd_pool_pop(BdPool &p, uint32_t &out) {
128 if (p.head == 0)
129 return false;
130 out = p.free_ids[--p.head];
131 return true;
132}
133
134// Return a BD ID to the pool for reuse.
135inline void bd_pool_push(BdPool &p, uint32_t id) {
136 if (p.head < static_cast<int>(kMaxBDsPerTile))
137 p.free_ids[p.head++] = id;
138}
139
140// Device information for the TXN header.
142 uint8_t major = 0;
143 uint8_t minor = 1;
144 uint8_t devGen = 3; // 3 = NPU (PHX/HWK), 4 = NPU2 (STX/KRK)
145 uint8_t numRows = 6;
146 uint8_t numCols = 5;
147 uint8_t numMemTileRows = 1;
148};
149
150// Build a TxnDeviceInfo from target-model values. Assigns by field name, so a
151// caller that can only emit an expression (the generated C++ builder) states
152// the field order in exactly one place: here.
153inline TxnDeviceInfo txn_device_info(uint8_t dev_gen, uint8_t num_rows,
154 uint8_t num_cols,
155 uint8_t num_memtile_rows) {
156 TxnDeviceInfo info;
157 info.devGen = dev_gen;
158 info.numRows = num_rows;
159 info.numCols = num_cols;
160 info.numMemTileRows = num_memtile_rows;
161 return info;
162}
163
164// Append a 6-word write32 instruction.
165inline void txn_append_write32(std::vector<uint32_t> &txn, uint32_t addr,
166 uint32_t val) {
167 size_t pos = txn.size();
168 txn.resize(pos + 6, 0);
169 txn[pos + 0] = TXN_OPC_WRITE;
170 // txn[pos + 1] is reserved (0)
171 txn[pos + 2] = addr;
172 txn[pos + 3] = 0; // extra bits for reg offset
173 txn[pos + 4] = val;
174 txn[pos + 5] = 6 * sizeof(uint32_t); // operation size
175}
176
177// Append a 7-word maskwrite32 instruction.
178inline void txn_append_maskwrite32(std::vector<uint32_t> &txn, uint32_t addr,
179 uint32_t val, uint32_t mask) {
180 size_t pos = txn.size();
181 txn.resize(pos + 7, 0);
182 txn[pos + 0] = TXN_OPC_MASKWRITE;
183 // txn[pos + 1] is reserved (0)
184 txn[pos + 2] = addr;
185 txn[pos + 3] = 0;
186 txn[pos + 4] = val;
187 txn[pos + 5] = mask;
188 txn[pos + 6] = 7 * sizeof(uint32_t); // operation size
189}
190
191// Append a 7-word maskpoll instruction: block until (reg & mask) == val.
192// Same word layout as maskwrite32 -- aie-codegen's XAie_MaskPoll32Hdr and
193// XAie_MaskWrite32Hdr agree field for field, and the poll timeout is not
194// serialized (the firmware forces its own default; see xaie_txn.c).
195inline void txn_append_maskpoll32(std::vector<uint32_t> &txn, uint32_t addr,
196 uint32_t val, uint32_t mask) {
197 size_t pos = txn.size();
198 txn.resize(pos + 7, 0);
199 txn[pos + 0] = TXN_OPC_MASKPOLL;
200 // txn[pos + 1] is reserved (0)
201 txn[pos + 2] = addr;
202 txn[pos + 3] = 0;
203 txn[pos + 4] = val;
204 txn[pos + 5] = mask;
205 txn[pos + 6] = 7 * sizeof(uint32_t); // operation size
206}
207
208// Append a 4-word sync (TCT) instruction.
209inline void txn_append_sync(std::vector<uint32_t> &txn, uint32_t col,
210 uint32_t row, uint32_t dir, uint32_t chan,
211 uint32_t ncol, uint32_t nrow) {
212 size_t pos = txn.size();
213 txn.resize(pos + 4, 0);
214 txn[pos + 0] = TXN_OPC_TCT;
215 txn[pos + 1] = 4 * sizeof(uint32_t); // operation size
216 txn[pos + 2] = (dir & 0xff) | ((row & 0xff) << 8) | ((col & 0xff) << 16);
217 txn[pos + 3] =
218 ((nrow & 0xff) << 8) | ((ncol & 0xff) << 16) | ((chan & 0xff) << 24);
219}
220
221// Append a variable-length blockwrite instruction.
222// `data` points to `count` uint32_t words of payload.
223//
224// The destination tile is encoded twice in the firmware blockwrite format: once
225// in the upper bits of the absolute `addr` (column/row folded in by the
226// compiler before this point) and once in the dedicated col/row field at
227// word[1]. Callers that build an absolute address from a tile pass that same
228// (col, row) here so both representations agree; callers with a purely flat
229// address pass 0/0.
230inline void txn_append_blockwrite(std::vector<uint32_t> &txn, uint32_t addr,
231 const uint32_t *data, size_t count,
232 uint32_t col = 0, uint32_t row = 0) {
233 const unsigned headerSize = 4;
234 size_t pos = txn.size();
235 txn.resize(pos + headerSize + count, 0);
236 txn[pos + 0] = TXN_OPC_BLOCKWRITE;
237 txn[pos + 1] = (col & 0xff) | ((row & 0xff) << 8);
238 txn[pos + 2] = addr;
239 txn[pos + 3] = static_cast<uint32_t>((headerSize + count) * sizeof(uint32_t));
240 for (size_t i = 0; i < count; ++i)
241 txn[pos + headerSize + i] = data[i];
242}
243
244// Under the xclbin + insts.bin runtime, NPU firmware adds kDDRAIEAddrOffset to
245// host buffer addresses for only the first kNumFirmwareTranslatedArgs args; a
246// DDR patch for a later arg must fold the offset into arg_plus itself. Full-ELF
247// and HRX translate every arg, so folding there would double-translate the 6th+
248// buffer: those pass fold = false.
249constexpr uint32_t kDDRAIEAddrOffset = 0x80000000;
250constexpr uint32_t kNumFirmwareTranslatedArgs = 5;
251
252// Append a 12-word address_patch (DDR_PATCH) instruction.
253//
254// The consumer casts this byte range to aie-codegen's `XAie_CustomOpHdr`
255// followed by `patch_op_t` (xaie_txn.h), so the layout is that struct pair's
256// under natural alignment: an 8-byte header at words 0-1, op_base and action at
257// 2-5, then regaddr, argidx and argplus as u64 word pairs at 6-7, 8-9 and
258// 10-11, low half first.
259inline void txn_append_address_patch(std::vector<uint32_t> &txn, uint32_t addr,
260 int32_t arg_idx, uint64_t arg_plus) {
261 size_t pos = txn.size();
262 txn.resize(pos + 12, 0);
263 txn[pos + 0] = TXN_OPC_DDR_PATCH; // opcode
264 txn[pos + 1] = 12 * sizeof(uint32_t); // operation size
265 // pos+2..4 are reserved (zero)
266 txn[pos + 5] = 0; // action; the struct puts it at word 4, inert while zero
267 txn[pos + 6] = addr; // register address to patch
268 // pos+7 is reserved (zero)
269 txn[pos + 8] = static_cast<uint32_t>(arg_idx); // buffer argument index
270 // pos+9 is reserved (zero)
271 txn[pos + 10] = static_cast<uint32_t>(arg_plus & 0xFFFFFFFFull);
272 txn[pos + 11] = static_cast<uint32_t>(arg_plus >> 32);
273}
274
275// Append a DDR_PATCH for a host buffer argument, folding the aperture offset
276// when the target runtime needs it. Both the static binary emitter and the
277// generated C++ builder route through here, so the fold rule has one body.
278inline void txn_append_arg_patch(std::vector<uint32_t> &txn, uint32_t addr,
279 int32_t arg_idx, uint64_t arg_plus,
280 bool fold_ddr_addr_offset) {
281 if (fold_ddr_addr_offset &&
282 static_cast<uint32_t>(arg_idx) >= kNumFirmwareTranslatedArgs)
283 arg_plus += kDDRAIEAddrOffset;
284 txn_append_address_patch(txn, addr, arg_idx, arg_plus);
285}
286
287// Append a 4-word loadpdi instruction.
288inline void txn_append_loadpdi(std::vector<uint32_t> &txn, uint32_t id,
289 uint32_t size, uint64_t addr) {
290 size_t pos = txn.size();
291 txn.resize(pos + 4, 0);
292 txn[pos + 0] = TXN_OPC_LOADPDI | (id << 16);
293 txn[pos + 1] = size;
294 txn[pos + 2] = static_cast<uint32_t>(addr);
295 txn[pos + 3] = static_cast<uint32_t>(addr >> 32);
296}
297
298// Append a 1-word preempt instruction.
299inline void txn_append_preempt(std::vector<uint32_t> &txn, uint32_t level) {
300 txn.push_back(TXN_OPC_PREEMPT | (level << 8));
301}
302
303// Why the last generated builder on this thread declined (returned
304// std::nullopt): a shape guard, a buffer-descriptor field overflow or an
305// empty BD pool records a message here before returning, and the dispatch
306// shim reports it through dispatch_last_refusal(). Null after a successful
307// build. The pointer is to a string literal in the generated code.
308inline thread_local const char *txn_refusal = nullptr;
309// `return aie_runtime::txn_refused("why");` declines a build with a reason.
310inline std::nullopt_t txn_refused(const char *why) {
311 txn_refusal = why;
312 return std::nullopt;
313}
314
315// Reserve 4 placeholder words for the TXN header. Call this BEFORE appending
316// any instructions, so that the header space is already allocated.
317inline void txn_init(std::vector<uint32_t> &txn) { txn.resize(4, 0); }
318
319// Finalize the 4-word TXN header in-place. Call this AFTER all instructions
320// are appended. The first 4 words must have been reserved by txn_init().
321// `op_count` is the number of operations appended.
322inline void txn_prepend_header(std::vector<uint32_t> &txn, uint32_t op_count,
323 TxnDeviceInfo info = {}) {
324 if (txn.size() < 4)
325 txn.resize(4, 0);
326 txn[0] = (static_cast<uint32_t>(info.numRows) << 24) |
327 (static_cast<uint32_t>(info.devGen) << 16) |
328 (static_cast<uint32_t>(info.minor) << 8) |
329 static_cast<uint32_t>(info.major);
330 txn[1] = (static_cast<uint32_t>(info.numMemTileRows) << 8) |
331 static_cast<uint32_t>(info.numCols);
332 txn[2] = op_count;
333 txn[3] = static_cast<uint32_t>(txn.size() * sizeof(uint32_t));
334}
335
336} // namespace aie_runtime
337
338#endif // AIE_RUNTIME_TXNENCODING_H
void txn_append_maskpoll32(std::vector< uint32_t > &txn, uint32_t addr, uint32_t val, uint32_t mask)
void txn_append_loadpdi(std::vector< uint32_t > &txn, uint32_t id, uint32_t size, uint64_t addr)
void txn_append_write32(std::vector< uint32_t > &txn, uint32_t addr, uint32_t val)
void txn_init(std::vector< uint32_t > &txn)
std::nullopt_t txn_refused(const char *why)
void txn_append_sync(std::vector< uint32_t > &txn, uint32_t col, uint32_t row, uint32_t dir, uint32_t chan, uint32_t ncol, uint32_t nrow)
void txn_append_preempt(std::vector< uint32_t > &txn, uint32_t level)
void bd_pool_push(BdPool &p, uint32_t id)
void txn_append_address_patch(std::vector< uint32_t > &txn, uint32_t addr, int32_t arg_idx, uint64_t arg_plus)
void txn_append_arg_patch(std::vector< uint32_t > &txn, uint32_t addr, int32_t arg_idx, uint64_t arg_plus, bool fold_ddr_addr_offset)
TxnDeviceInfo txn_device_info(uint8_t dev_gen, uint8_t num_rows, uint8_t num_cols, uint8_t num_memtile_rows)
constexpr uint32_t kNumFirmwareTranslatedArgs
@ TXN_OPC_RECORD_TIMER
Definition TxnEncoding.h:63
@ TXN_OPC_UPDATE_SCRATCH
Definition TxnEncoding.h:56
@ TXN_OPC_CREATE_SCRATCHPAD
Definition TxnEncoding.h:53
@ TXN_OPC_MASKPOLL_BUSY
Definition TxnEncoding.h:50
@ TXN_OPC_UPDATE_STATE_TABLE
Definition TxnEncoding.h:54
@ TXN_OPC_CONFIG_SHIMDMA_DMABUF_BD
Definition TxnEncoding.h:58
@ TXN_OPC_LOAD_PM_END_INTERNAL
Definition TxnEncoding.h:66
@ TXN_OPC_CUSTOM_OP_NEXT
Definition TxnEncoding.h:65
@ TXN_OPC_CUSTOM_OP_MAX
Definition TxnEncoding.h:67
@ TXN_OPC_CONFIG_SHIMDMA_BD
Definition TxnEncoding.h:57
@ TXN_OPC_LOAD_PM_START
Definition TxnEncoding.h:52
@ TXN_OPC_CUSTOM_OP_BEGIN
Definition TxnEncoding.h:59
constexpr uint32_t kDDRAIEAddrOffset
void bd_pool_reserve(BdPool &p, uint32_t id)
void txn_append_maskwrite32(std::vector< uint32_t > &txn, uint32_t addr, uint32_t val, uint32_t mask)
bool bd_pool_pop(BdPool &p, uint32_t &out)
const char * txn_refusal
void txn_append_blockwrite(std::vector< uint32_t > &txn, uint32_t addr, const uint32_t *data, size_t count, uint32_t col=0, uint32_t row=0)
void txn_prepend_header(std::vector< uint32_t > &txn, uint32_t op_count, TxnDeviceInfo info={})
constexpr uint32_t kMaxBDsPerTile
Definition TxnEncoding.h:76
BdPool bd_pool_init(uint32_t n)
BdPool bd_pool_init_range(uint32_t lo, uint32_t hi)
Definition TxnEncoding.h:95
uint32_t free_ids[kMaxBDsPerTile]
Definition TxnEncoding.h:84