MLIR-AIE 1.4.6.dev
AIEToConfiguration.cpp
Go to the documentation of this file.
1//===- AIEToConfiguration.h -------------------------------------*- C++ -*-===//
2//
3// Copyright (C) 2024 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7
8#include "../PassDetail.h"
9#include "TransactionBinary.h"
10
15#include "aie/Targets/AIERT.h"
16
17#include "mlir/Dialect/Arith/IR/Arith.h"
18#include "mlir/Dialect/Func/IR/FuncOps.h"
19#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
20#include "mlir/Dialect/MemRef/IR/MemRef.h"
21#include "mlir/IR/IRMapping.h"
22#include "mlir/IR/Matchers.h"
23#include "mlir/Interfaces/CallInterfaces.h"
24
25#include "llvm/ADT/APInt.h"
26#include "llvm/ADT/DenseMap.h"
27#include "llvm/ADT/DenseSet.h"
28#include "llvm/ADT/SetVector.h"
29#include "llvm/Support/Debug.h"
30
31extern "C" {
32#include "aie_codegen.h"
33}
34
35#include <algorithm>
36#include <array>
37#include <cstring>
38#include <map>
39#include <optional>
40#include <set>
41#include <tuple>
42#include <utility>
43#include <vector>
44
45namespace xilinx {
46#define GEN_PASS_DEF_AIEANNOTATECORELOCKACQUIRES
47#define GEN_PASS_DEF_CONVERTAIETOCONTROLPACKETS
48#define GEN_PASS_DEF_CONVERTAIETOTRANSACTION
49#include "aie/Conversion/Passes.h.inc"
50} // namespace xilinx
51
52#define DEBUG_TYPE "aie-convert-to-config"
53
54using namespace mlir;
55using namespace xilinx;
56using namespace xilinx::AIE;
57
58namespace {
59
60constexpr size_t kTxnHeaderBytes = 16;
61
62struct TxnPreemptHeader {
63 uint8_t opcode;
64 uint8_t level;
65 uint16_t reserved;
66};
67
68struct TxnLoadPdiHeader {
69 uint8_t opcode;
70 uint8_t padding;
71 uint16_t id;
72 uint32_t size;
73 uint64_t address;
74};
75} // namespace
76
77// Parse a TXN binary blob. On success return the number of columns from the
78// header and a vector of parsed operations. On failure return std::nullopt.
80 const std::vector<uint8_t> &data,
81 std::vector<TransactionBinaryOperation> &ops) {
82
83 if (data.size() < kTxnHeaderBytes) {
84 llvm::errs() << "Transaction binary is too small for header\n";
85 return std::nullopt;
86 }
87
88 uint32_t major = data[0];
89 uint32_t minor = data[1];
90 uint32_t num_cols = data[4];
91
92 uint32_t num_ops, txn_size;
93 std::memcpy(&num_ops, &data[8], 4);
94 std::memcpy(&txn_size, &data[12], 4);
95
96 LLVM_DEBUG(llvm::dbgs() << "Major: " << major << "\n");
97 LLVM_DEBUG(llvm::dbgs() << "Minor: " << minor << "\n");
98 LLVM_DEBUG(llvm::dbgs() << "DevGen: " << data[2] << "\n");
99 LLVM_DEBUG(llvm::dbgs() << "NumRows: " << data[3] << "\n");
100 LLVM_DEBUG(llvm::dbgs() << "NumCols: " << num_cols << "\n");
101 LLVM_DEBUG(llvm::dbgs() << "NumMemTileRows: " << data[5] << "\n");
102 LLVM_DEBUG(llvm::dbgs() << "NumOps: " << num_ops << "\n");
103 LLVM_DEBUG(llvm::dbgs() << "TxnSize: " << txn_size << " bytes\n");
104
105 size_t i = kTxnHeaderBytes;
106
107 auto requireBytes = [&](size_t offset, size_t length) -> bool {
108 if (offset + length > data.size()) {
109 llvm::errs() << "Transaction binary truncated while parsing opcode\n";
110 return false;
111 }
112 return true;
113 };
114
115 auto read32 = [&](size_t offset) -> uint32_t {
116 uint32_t value;
117 std::memcpy(&value, data.data() + offset, sizeof(uint32_t));
118 return value;
119 };
120
121 // Convert opcode from uint8 to a validated opcode byte
122 auto convertOpcode = [](uint8_t opc) -> std::optional<uint8_t> {
123 switch (opc) {
124 case static_cast<uint8_t>(XAie_TxnOpcode::XAIE_IO_WRITE):
125 case static_cast<uint8_t>(XAie_TxnOpcode::XAIE_IO_BLOCKWRITE):
126 case static_cast<uint8_t>(XAie_TxnOpcode::XAIE_IO_MASKWRITE):
127 case static_cast<uint8_t>(XAie_TxnOpcode::XAIE_IO_MASKPOLL):
128 case 0x6: // XAie_TxnOpcode::XAIE_IO_PREEMPT
129 case 0x8: // XAie_TxnOpcode::XAIE_IO_LOAD_PDI
130 case static_cast<uint8_t>(XAie_TxnOpcode::XAIE_IO_CUSTOM_OP_TCT):
131 case static_cast<uint8_t>(XAie_TxnOpcode::XAIE_IO_CUSTOM_OP_DDR_PATCH):
132 return opc;
133 default:
134 llvm::errs() << "Unhandled opcode: " << std::to_string(opc) << "\n";
135 return std::nullopt;
136 }
137 };
138
139 // Parse the binary blob. There are two versions supported, 0.1 and 1.0.
140 // For both versions, build a list of TransactionBinaryOperation objects
141 // representing the parsed operations.
142 if (major == 0 && minor == 1) {
143 while (i < data.size()) {
144 auto maybeOpcode = convertOpcode(data[i]);
145 if (!maybeOpcode)
146 return std::nullopt;
147 XAie_TxnOpcode opcode = static_cast<XAie_TxnOpcode>(*maybeOpcode);
148 LLVM_DEBUG(llvm::dbgs() << "opcode: " + std::to_string(opcode) << "\n");
149
151 op.cmd.Opcode = opcode;
152
153 switch (opcode) {
154 case XAie_TxnOpcode::XAIE_IO_WRITE: {
155 LLVM_DEBUG(llvm::dbgs() << "opcode: WRITE (0x00)\n");
156 if (!requireBytes(i, 24))
157 return std::nullopt;
158 uint32_t addrLo = read32(i + 8);
159 uint32_t addrHi = read32(i + 12);
160 uint32_t value = read32(i + 16);
161 uint32_t opSize = read32(i + 20);
162 if (!requireBytes(i, opSize))
163 return std::nullopt;
164 uint64_t addr = (static_cast<uint64_t>(addrHi) << 32) | addrLo;
165 op.cmd.RegOff = addr;
166 op.cmd.Value = value;
167 op.cmd.Size = 0;
168 i += opSize;
169 break;
170 }
171 case XAie_TxnOpcode::XAIE_IO_BLOCKWRITE: {
172 LLVM_DEBUG(llvm::dbgs() << "opcode: BLOCKWRITE (0x01)\n");
173 if (!requireBytes(i, 16))
174 return std::nullopt;
175 uint32_t addr = read32(i + 8);
176 uint32_t opSize = read32(i + 12);
177 if (opSize < 16 || !requireBytes(i, opSize))
178 return std::nullopt;
179 const uint8_t *payload = data.data() + i + 16;
180 uint32_t payloadBytes = opSize - 16;
181 op.cmd.RegOff = addr;
182 op.cmd.DataPtr = reinterpret_cast<uint64_t>(payload);
183 op.cmd.Size = payloadBytes;
184 i += opSize;
185 break;
186 }
187 // MASKPOLL has the same word layout as MASKWRITE (aie-codegen's
188 // XAie_MaskPoll32Hdr matches XAie_MaskWrite32Hdr); the poll timeout is
189 // not serialized.
190 case XAie_TxnOpcode::XAIE_IO_MASKWRITE:
191 case XAie_TxnOpcode::XAIE_IO_MASKPOLL: {
192 LLVM_DEBUG(llvm::dbgs() << "opcode: MASKWRITE (0x03) or MASKPOLL "
193 "(0x04)\n");
194 if (!requireBytes(i, 28))
195 return std::nullopt;
196 uint32_t addrLo = read32(i + 8);
197 uint32_t addrHi = read32(i + 12);
198 uint32_t value = read32(i + 16);
199 uint32_t mask = read32(i + 20);
200 uint32_t opSize = read32(i + 24);
201 if (opSize < 28) {
202 llvm::errs() << "Transaction op size " << opSize
203 << " is smaller than its 28-byte header\n";
204 return std::nullopt;
205 }
206 if (!requireBytes(i, opSize))
207 return std::nullopt;
208 uint64_t addr = (static_cast<uint64_t>(addrHi) << 32) | addrLo;
209 op.cmd.RegOff = addr;
210 op.cmd.Value = value;
211 op.cmd.Mask = mask;
212 op.cmd.Size = opSize;
213 i += opSize;
214 break;
215 }
216 case XAie_TxnOpcode::XAIE_IO_CUSTOM_OP_TCT: {
217 uint32_t opSize = read32(i + 4);
218 if (opSize < 16 || !requireBytes(i, opSize))
219 return std::nullopt;
220 uint32_t descriptor = read32(i + 8);
221 uint32_t config = read32(i + 12);
223 /*column=*/static_cast<int32_t>((descriptor >> 16) & 0xff),
224 /*row=*/static_cast<int32_t>((descriptor >> 8) & 0xff),
225 /*direction=*/static_cast<int32_t>(descriptor & 0xff),
226 /*channel=*/static_cast<int32_t>((config >> 24) & 0xff),
227 /*columnCount=*/static_cast<int32_t>((config >> 16) & 0xff),
228 /*rowCount=*/static_cast<int32_t>((config >> 8) & 0xff)};
229 op.sync = payload;
230 op.cmd.Size = opSize;
231 i += opSize;
232 break;
233 }
234 // Not in the vendored third_party/aie-codegen enum (though the driver
235 // protocol defines it); can't add it without forking upstream.
236 case 0x8: { // NOLINT(clang-diagnostic-switch):
237 // XAie_TxnOpcode::XAIE_IO_LOAD_PDI
238 LLVM_DEBUG(llvm::dbgs() << "opcode: LOAD_PDI (0x08)\n");
239 constexpr size_t opSize = sizeof(TxnLoadPdiHeader);
240 if (!requireBytes(i, opSize))
241 return std::nullopt;
242 TxnLoadPdiHeader header;
243 std::memcpy(&header, data.data() + i, opSize);
245 header.id, header.size, header.address};
246 op.loadPdi = payload;
247 op.cmd.Size = opSize;
248 i += opSize;
249 break;
250 }
251 case XAie_TxnOpcode::XAIE_IO_CUSTOM_OP_DDR_PATCH: {
252 uint32_t opSize = read32(i + 4);
253 if (opSize < 44 || !requireBytes(i, opSize))
254 return std::nullopt;
255 uint32_t action = read32(i + 20);
256 uint32_t addr = read32(i + 24);
257 int32_t argIdx = static_cast<int32_t>(read32(i + 32));
258 uint64_t argPlus = static_cast<uint64_t>(read32(i + 40)) |
259 (static_cast<uint64_t>(read32(i + 44)) << 32);
261 action, addr, argIdx, argPlus};
262 op.addressPatch = payload;
263 op.cmd.Size = opSize;
264 i += opSize;
265 break;
266 }
267 case 0x6: { // XAie_TxnOpcode::XAIE_IO_PREEMPT
268 LLVM_DEBUG(llvm::dbgs() << "opcode: PREEMPT (0x06)\n");
269 constexpr size_t opSize = sizeof(TxnPreemptHeader);
270 if (!requireBytes(i, opSize))
271 return std::nullopt;
272 const auto *header =
273 reinterpret_cast<const TxnPreemptHeader *>(data.data() + i);
274 op.cmd.Value = header->level;
275 op.cmd.Size = opSize;
276 i += opSize;
277 break;
278 }
279 default:
280 llvm::errs() << "Unhandled opcode: " << std::to_string(opcode)
281 << " for v0.1 transaction\n";
282 return std::nullopt;
283 }
284
285 ops.push_back(op);
286 }
287 } else if (major == 1 && minor == 0) {
288 while (i < data.size()) {
289 auto maybeOpcode = convertOpcode(data[i]);
290 if (!maybeOpcode)
291 return std::nullopt;
292 XAie_TxnOpcode opcode = static_cast<XAie_TxnOpcode>(*maybeOpcode);
293 LLVM_DEBUG(llvm::dbgs() << "opcode: " + std::to_string(opcode) << "\n");
294
296 op.cmd.Opcode = opcode;
297
298 switch (opcode) {
299 case XAie_TxnOpcode::XAIE_IO_WRITE: {
300 LLVM_DEBUG(llvm::dbgs() << "opcode: WRITE (0x00)\n");
301 if (!requireBytes(i, 12))
302 return std::nullopt;
303 uint32_t addr = read32(i + 4);
304 uint32_t value = read32(i + 8);
305 op.cmd.RegOff = addr;
306 op.cmd.Value = value;
307 op.cmd.Size = 0;
308 i += 12;
309 break;
310 }
311 case XAie_TxnOpcode::XAIE_IO_BLOCKWRITE: {
312 LLVM_DEBUG(llvm::dbgs() << "opcode: BLOCKWRITE (0x01)\n");
313 if (!requireBytes(i, 12))
314 return std::nullopt;
315 uint32_t addr = read32(i + 4);
316 uint32_t opSize = read32(i + 8);
317 if (opSize < 12 || !requireBytes(i, opSize))
318 return std::nullopt;
319 const uint8_t *payload = data.data() + i + 12;
320 uint32_t payloadBytes = opSize - 12;
321 op.cmd.RegOff = addr;
322 op.cmd.DataPtr = reinterpret_cast<uint64_t>(payload);
323 op.cmd.Size = payloadBytes;
324 i += opSize;
325 break;
326 }
327 case XAie_TxnOpcode::XAIE_IO_MASKWRITE:
328 case XAie_TxnOpcode::XAIE_IO_MASKPOLL: {
329 LLVM_DEBUG(llvm::dbgs() << "opcode: MASKWRITE (0x03) or MASKPOLL "
330 "(0x04)\n");
331 if (!requireBytes(i, 16))
332 return std::nullopt;
333 uint32_t addr = read32(i + 4);
334 uint32_t value = read32(i + 8);
335 uint32_t mask = read32(i + 12);
336 op.cmd.RegOff = addr;
337 op.cmd.Value = value;
338 op.cmd.Mask = mask;
339 op.cmd.Size = 0;
340 i += 16;
341 break;
342 }
343 case XAie_TxnOpcode::XAIE_IO_CUSTOM_OP_TCT: {
344 uint32_t opSize = read32(i + 4);
345 if (opSize < 16 || !requireBytes(i, opSize))
346 return std::nullopt;
347 uint32_t descriptor = read32(i + 8);
348 uint32_t config = read32(i + 12);
350 /*column=*/static_cast<int32_t>((descriptor >> 16) & 0xff),
351 /*row=*/static_cast<int32_t>((descriptor >> 8) & 0xff),
352 /*direction=*/static_cast<int32_t>(descriptor & 0xff),
353 /*channel=*/static_cast<int32_t>((config >> 24) & 0xff),
354 /*columnCount=*/static_cast<int32_t>((config >> 16) & 0xff),
355 /*rowCount=*/static_cast<int32_t>((config >> 8) & 0xff)};
356 op.sync = payload;
357 op.cmd.Size = opSize;
358 i += opSize;
359 break;
360 }
361 // LOAD_PDI is matched by value here too; see the first switch.
362 case 0x8: { // NOLINT(clang-diagnostic-switch):
363 // XAie_TxnOpcode::XAIE_IO_LOAD_PDI
364 LLVM_DEBUG(llvm::dbgs() << "opcode: LOAD_PDI (0x08)\n");
365 constexpr size_t opSize = sizeof(TxnLoadPdiHeader);
366 if (!requireBytes(i, opSize))
367 return std::nullopt;
368 TxnLoadPdiHeader header;
369 std::memcpy(&header, data.data() + i, opSize);
371 header.id, header.size, header.address};
372 op.loadPdi = payload;
373 op.cmd.Size = opSize;
374 i += opSize;
375 break;
376 }
377 case XAie_TxnOpcode::XAIE_IO_CUSTOM_OP_DDR_PATCH: {
378 uint32_t opSize = read32(i + 4);
379 if (opSize < 44 || !requireBytes(i, opSize))
380 return std::nullopt;
381 uint32_t action = read32(i + 20);
382 uint32_t addr = read32(i + 24);
383 int32_t argIdx = static_cast<int32_t>(read32(i + 32));
384 uint64_t argPlus = static_cast<uint64_t>(read32(i + 40)) |
385 (static_cast<uint64_t>(read32(i + 44)) << 32);
387 action, addr, argIdx, argPlus};
388 op.addressPatch = payload;
389 op.cmd.Size = opSize;
390 i += opSize;
391 break;
392 }
393 case 0x6: { // XAie_TxnOpcode::XAIE_IO_PREEMPT
394 LLVM_DEBUG(llvm::dbgs() << "opcode: PREEMPT (0x06)\n");
395 constexpr size_t opSize = sizeof(TxnPreemptHeader);
396 if (!requireBytes(i, opSize))
397 return std::nullopt;
398 const auto *header =
399 reinterpret_cast<const TxnPreemptHeader *>(data.data() + i);
400 op.cmd.Value = header->level;
401 op.cmd.Size = opSize;
402 i += opSize;
403 break;
404 }
405 default:
406 llvm::errs() << "Unhandled opcode: " << std::to_string(opcode)
407 << " for v1.0 transaction\n";
408 return std::nullopt;
409 }
410
411 ops.push_back(op);
412 }
413 } else {
414 llvm::errs() << "Unsupported TXN binary version: " << major << "." << minor
415 << "\n";
416 return std::nullopt;
417 }
418
419 return num_cols;
420}
421
422static LogicalResult generateTransactions(AIERTControl &ctl,
423 const StringRef workDirPath,
424 DeviceOp &targetOp, bool aieSim,
425 bool enableElfs, bool enableInit,
426 bool enableCores,
427 bool skipCtrlPktOverlay = false) {
428 if (enableElfs && !targetOp.getOps<CoreOp>().empty() &&
429 failed(ctl.addAieElfs(targetOp, workDirPath, aieSim)))
430 return failure();
431 if (enableInit && failed(ctl.addInitConfig(targetOp, skipCtrlPktOverlay)))
432 return failure();
433 if (enableCores && !targetOp.getOps<CoreOp>().empty() &&
434 failed(ctl.addCoreEnable(targetOp)))
435 return failure();
436 return success();
437}
438
439// Translate vector of TransactionBinaryOperation to a sequence of transaction
440// ops (npu.write32, npu.maskwrite32, npu.maskpoll, npu.blockwrite). Each
441// emitted op gets the per-op `sourceLoc` from AIERT's instruction-range
442// bracketing if available; otherwise it falls back to `fallbackLoc` (typically
443// the device location).
444static LogicalResult
445emitTransactionOps(OpBuilder &builder, Location fallbackLoc,
446 std::vector<TransactionBinaryOperation> &operations,
447 std::vector<memref::GlobalOp> &global_data) {
448
449 // One constant per distinct value. They all land in the same block through
450 // `builder`, so the first use dominates every later one.
451 //
452 // Because the first use is what creates the op, callers must bind the
453 // operands to locals rather than call this inside an argument list: the
454 // evaluation order of function arguments is unspecified, and it decides the
455 // order the constants are emitted in.
456 llvm::DenseMap<uint32_t, Value> constantCache;
457 auto constant = [&](Location loc, uint32_t value) {
458 Value &cached = constantCache[value];
459 if (!cached)
460 cached = AIEX::createConstantI32(builder, loc, value);
461 return cached;
462 };
463
464 // create the txn ops
465 for (auto [op, payload] : llvm::zip(operations, global_data)) {
466 Location loc = op.sourceLoc.value_or(fallbackLoc);
467
468 if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_WRITE) {
469 Value address = constant(loc, op.cmd.RegOff);
470 Value value = constant(loc, op.cmd.Value);
471 AIEX::NpuWrite32Op::create(builder, loc, address, value, nullptr, nullptr,
472 nullptr);
473 } else if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_BLOCKWRITE) {
474 auto memref = memref::GetGlobalOp::create(builder, loc, payload.getType(),
475 payload.getName());
476 AIEX::NpuBlockWriteOp::create(
477 builder, loc, builder.getUI32IntegerAttr(op.cmd.RegOff),
478 memref.getResult(), nullptr, nullptr, nullptr);
479 } else if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_MASKWRITE) {
480 Value address = constant(loc, op.cmd.RegOff);
481 Value value = constant(loc, op.cmd.Value);
482 Value mask = constant(loc, op.cmd.Mask);
483 AIEX::NpuMaskWrite32Op::create(builder, loc, address, value, mask,
484 nullptr, nullptr, nullptr);
485 } else if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_MASKPOLL) {
486 Value address = constant(loc, op.cmd.RegOff);
487 Value value = constant(loc, op.cmd.Value);
488 Value mask = constant(loc, op.cmd.Mask);
489 AIEX::NpuMaskPollOp::create(builder, loc, address, value, mask,
490 /*buffer=*/nullptr, /*column=*/nullptr,
491 /*row=*/nullptr);
492 } else if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_CUSTOM_OP_TCT) {
493 if (!op.sync) {
494 llvm::errs() << "Missing sync payload while emitting transaction\n";
495 return failure();
496 }
497 const TransactionBinaryOperation::SyncPayload &sync = *op.sync;
498 Value column = constant(loc, sync.column);
499 Value row = constant(loc, sync.row);
500 Value direction = constant(loc, sync.direction);
501 Value channel = constant(loc, sync.channel);
502 Value columnCount = constant(loc, sync.columnCount);
503 Value rowCount = constant(loc, sync.rowCount);
504 AIEX::NpuSyncOp::create(builder, loc, column, row, direction, channel,
505 columnCount, rowCount);
506 } else if (op.cmd.Opcode == 0x8 /* XAie_TxnOpcode::XAIE_IO_LOAD_PDI */) {
507 if (!op.loadPdi) {
508 llvm::errs() << "Missing load_pdi payload while emitting transaction\n";
509 return failure();
510 }
512 *op.loadPdi;
513 auto idAttr =
514 builder.getI32IntegerAttr(static_cast<int32_t>(payloadInfo.id));
515 IntegerAttr sizeAttr =
516 builder.getI32IntegerAttr(static_cast<int32_t>(payloadInfo.size));
517
518 auto ui64Ty =
519 IntegerType::get(builder.getContext(), 64, IntegerType::Unsigned);
520 IntegerAttr addressAttr =
521 IntegerAttr::get(ui64Ty, llvm::APInt(64, payloadInfo.address));
522
523 AIEX::NpuLoadPdiOp::create(builder, loc, /*device_ref=*/nullptr, idAttr,
524 sizeAttr, addressAttr,
525 /*expand_mode=*/nullptr);
526 } else if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_CUSTOM_OP_DDR_PATCH) {
527 if (!op.addressPatch) {
528 llvm::errs()
529 << "Missing address_patch payload while emitting transaction\n";
530 return failure();
531 }
533 *op.addressPatch;
534 AIEX::NpuAddressPatchOp::create(
535 builder, loc, patch.addr,
536 /*addr_val=*/mlir::Value(), static_cast<int32_t>(patch.argIdx),
537 AIEX::createConstantArgPlus(builder, loc, patch.argPlus));
538 } else if (op.cmd.Opcode == 0x6 /* XAie_TxnOpcode::XAIE_IO_PREEMPT */) {
539 auto ui8Ty =
540 IntegerType::get(builder.getContext(), 8, IntegerType::Unsigned);
541 auto levelAttr = IntegerAttr::get(ui8Ty, llvm::APInt(8, op.cmd.Value));
542 AIEX::NpuPreemptOp::create(builder, loc, levelAttr);
543 } else {
544 llvm::errs() << "Unhandled txn opcode: " << op.cmd.Opcode << "\n";
545 return failure();
546 }
547 }
548 return success();
549}
550
551// Scans back from the insertion point to the last PDI load, which reprograms
552// the array.
553static llvm::DenseMap<uint32_t, uint32_t>
554registersWrittenSinceLoad(OpBuilder &builder) {
555 llvm::DenseMap<uint32_t, uint32_t> registers;
556 Block *block = builder.getInsertionBlock();
557 if (!block)
558 return registers;
559 Block::iterator it = builder.getInsertionPoint();
560 while (it != block->begin()) {
561 Operation &op = *--it;
562 if (isa<AIEX::NpuLoadPdiOp>(op))
563 break;
564 auto packet = dyn_cast<AIEX::NpuControlPacketOp>(op);
565 std::optional<ArrayRef<int32_t>> data =
566 packet ? packet.getData() : std::nullopt;
567 if (!data)
568 continue;
569 for (auto [i, word] : llvm::enumerate(*data))
570 registers.try_emplace(packet.getAddress() + i * sizeof(int32_t), word);
571 }
572 return registers;
573}
574
575// Translate vector of TransactionBinaryOperation to a sequence of control
576// packet ops. Each emitted op gets the per-op `sourceLoc` from AIERT's
577// instruction-range bracketing if available; otherwise it falls back to
578// `fallbackLoc`.
579static LogicalResult emitControlPacketOps(
580 OpBuilder &builder, Location fallbackLoc,
581 std::vector<TransactionBinaryOperation> &operations,
582 std::vector<memref::GlobalOp> &global_data,
583 const llvm::DenseMap<uint32_t, uint32_t> &initialRegisters) {
584
585 auto *ctx = builder.getContext();
586
587 // A control packet writes whole words, so a masked write becomes a write of
588 // the value the register would hold after it.
589 llvm::DenseMap<uint32_t, uint32_t> registers =
590 registersWrittenSinceLoad(builder);
591 for (auto [address, value] : initialRegisters)
592 registers.try_emplace(address, value);
593
594 // create the control packet ops
595 for (auto [op, payload] : llvm::zip(operations, global_data)) {
596 Location loc = op.sourceLoc.value_or(fallbackLoc);
597
598 if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_WRITE) {
599 registers[op.cmd.RegOff] = op.cmd.Value;
600 AIEX::NpuControlPacketOp::create(
601 builder, loc, builder.getUI32IntegerAttr(op.cmd.RegOff), nullptr,
602 /*opcode*/ builder.getI32IntegerAttr(0),
603 /*stream_id*/ builder.getI32IntegerAttr(0),
604 DenseI32ArrayAttr::get(ctx, ArrayRef<int32_t>(op.cmd.Value)));
605 } else if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_BLOCKWRITE) {
606 auto initialValue = payload.getInitialValue();
607 if (!initialValue)
608 continue;
609 auto blockWriteData = dyn_cast<DenseIntElementsAttr>(*initialValue);
610 if (!blockWriteData) {
611 payload.emitError(
612 "Global symbol initial value is not a dense int array");
613 break;
614 }
615 auto blockWriteDataValues = blockWriteData.getValues<int32_t>();
616 // Split block write data into beats of 4 or less, in int32_t.
617 int currAddr = op.cmd.RegOff;
618 for (size_t i = 0; i < blockWriteDataValues.size(); i += 4) {
619 auto last = std::min(blockWriteDataValues.size(), i + 4);
620 SmallVector<int32_t> splitData =
621 SmallVector<int32_t>(blockWriteDataValues.begin() + i,
622 blockWriteDataValues.begin() + last);
623 for (auto [j, word] : llvm::enumerate(splitData))
624 registers[currAddr + j * sizeof(int32_t)] = word;
625 AIEX::NpuControlPacketOp::create(
626 builder, loc, builder.getUI32IntegerAttr(currAddr), nullptr,
627 /*opcode*/ builder.getI32IntegerAttr(0),
628 /*stream_id*/ builder.getI32IntegerAttr(0),
629 DenseI32ArrayAttr::get(ctx, ArrayRef<int32_t>(splitData)));
630 currAddr += splitData.size() * sizeof(int32_t);
631 }
632
633 } else if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_MASKWRITE) {
634 uint32_t &reg = registers[op.cmd.RegOff];
635 reg = (reg & ~op.cmd.Mask) | (op.cmd.Value & op.cmd.Mask);
636 int32_t value = reg;
637 AIEX::NpuControlPacketOp::create(
638 builder, loc, builder.getUI32IntegerAttr(op.cmd.RegOff), nullptr,
639 /*opcode*/ builder.getI32IntegerAttr(0),
640 /*stream_id*/ builder.getI32IntegerAttr(0),
641 DenseI32ArrayAttr::get(ctx, ArrayRef<int32_t>(value)));
642 } else if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_MASKPOLL) {
643 llvm::errs() << "control-packet configuration cannot express a mask "
644 "poll\n";
645 return failure();
646 } else {
647 llvm::errs() << "Unhandled txn opcode: " << op.cmd.Opcode << "\n";
648 return failure();
649 }
650 }
651 return success();
652}
653
654// Take transaction operations and insert them at the _current_ insertion point
655// of the supplied builder.
656static LogicalResult convertTransactionOpsToMLIR(
657 OpBuilder builder, AIE::AIEToConfigurationOutputType outputType,
658 std::vector<TransactionBinaryOperation> &operations,
659 AIEX::BlockwriteData &blockwriteData,
660 const llvm::DenseMap<uint32_t, uint32_t> &initialRegisters = {}) {
661
662 // for each blockwrite in the binary, find or create a GlobalOp with the data
663 // at the device level
664 std::vector<memref::GlobalOp> global_data;
665 DeviceOp device = blockwriteData.getDevice();
666 Location loc = device.getLoc();
667 {
668 OpBuilder::InsertionGuard guard(builder);
669 builder.setInsertionPointToStart(device.getBody());
670 for (auto &op : operations) {
671 if (op.cmd.Opcode != XAIE_IO_BLOCKWRITE) {
672 global_data.emplace_back(nullptr);
673 continue;
674 }
675 const uint32_t *d = reinterpret_cast<const uint32_t *>(op.cmd.DataPtr);
676 global_data.push_back(blockwriteData.getOrCreate(
677 builder, loc, ArrayRef<uint32_t>(d, op.cmd.Size / 4)));
678 }
679 }
680
681 // create the txn ops
683 if (failed(emitTransactionOps(builder, loc, operations, global_data)))
684 return failure();
685 } else if (outputType == AIE::AIEToConfigurationOutputType::ControlPacket) {
686 if (failed(emitControlPacketOps(builder, loc, operations, global_data,
687 initialRegisters)))
688 return failure();
689 } else {
690 llvm_unreachable("bad output type");
691 }
692
693 return success();
694}
695
696// Convert (disassemble) a transaction binary to MLIR. On success return a new
697// ModuleOp containing a DeviceOp containing a runtime sequence with the
698// transaction binary encoded as a sequence of npu.write32, npu.maskwrite32,
699// npu.maskpoll and npu.blockwrite operations. On failure return std::nullopt.
700std::optional<mlir::ModuleOp>
702 std::vector<uint8_t> &binary) {
703
704 // parse the binary
705 std::vector<TransactionBinaryOperation> operations;
706 auto c = parseTransactionBinary(binary, operations);
707 if (!c) {
708 llvm::errs() << "Failed to parse binary\n";
709 return std::nullopt;
710 }
711 int columns = *c;
712
713 // write32/maskwrite32/sync/address_patch now carry their integer fields as
714 // SSA operands materialized via arith.constant, so ensure the dialect is
715 // loaded before emitting them.
716 ctx->getOrLoadDialect<mlir::arith::ArithDialect>();
717
718 auto loc = mlir::UnknownLoc::get(ctx);
719
720 // create a new ModuleOp and set the insertion point
721 auto module = ModuleOp::create(loc);
722 OpBuilder builder(module.getBodyRegion());
723 builder.setInsertionPointToStart(module.getBody());
724
725 // create aie.device
726 std::vector<AIEDevice> devices{AIEDevice::npu1_1col, AIEDevice::npu1_2col,
727 AIEDevice::npu1_3col, AIEDevice::npu1};
728 auto device = DeviceOp::create(builder, loc, devices[columns - 1],
729 DeviceOp::getDefaultDeviceName(), nullptr);
730 device.getRegion().emplaceBlock();
731 DeviceOp::ensureTerminator(device.getBodyRegion(), builder, loc);
732 builder.setInsertionPointToStart(device.getBody());
733
734 // convert the parsed ops to MLIR
735 AIEX::BlockwriteData blockwriteData(device, "config_blockwrite_data_");
736 if (failed(convertTransactionOpsToMLIR(
738 blockwriteData)))
739 return std::nullopt;
740
741 return module;
742}
743
744// Insert the ops of the transaction `ctl` recorded, as `outputType`.
745static LogicalResult
746insertRecordedTransaction(AIERTControl &ctl, OpBuilder &builder,
748 AIEX::BlockwriteData &blockwriteData,
749 DeviceOp configured = {}) {
750 // Export the transactions to a binary buffer
751 std::vector<uint8_t> txn_data = ctl.exportSerializedTransaction();
752
753 // parse the binary data
754 std::vector<TransactionBinaryOperation> operations;
755 if (!parseTransactionBinary(txn_data, operations)) {
756 llvm::errs() << "Failed to parse binary\n";
757 return failure();
758 }
759
760 // Attach per-op source locations from AIERT's instruction-range bracketing.
761 // AIERTControl already projected those onto the serialized transaction's
762 // operations, so the indices line up with what the parser reproduced here.
763 // Ops with no bracketed location keep std::nullopt and inherit the device
764 // fallback location at emit time.
765 const std::vector<mlir::Location> &opLocs = ctl.getTxnOpLocs();
766 // Sharp edge: that projection models how aie-codegen's serializer maps
767 // recorded commands to binary operations. A count disagreement means the
768 // model has drifted from the pinned aie-codegen and every index past the
769 // divergence is mislabeled -- fail loudly instead of silently. An empty
770 // vector just means nothing was bracketed.
771 assert((opLocs.empty() || opLocs.size() == operations.size()) &&
772 "txn loc/op count mismatch: AIERTControl's command-to-operation "
773 "projection disagrees with the transaction parser; aie-codegen's "
774 "serializer has drifted from projectCmdLocsOntoSerializedOps");
775 for (size_t i = 0, e = std::min(opLocs.size(), operations.size()); i < e; ++i)
776 if (!isa<UnknownLoc>(opLocs[i]))
777 operations[i].sourceLoc = opLocs[i];
778
779 llvm::DenseMap<uint32_t, uint32_t> initialRegisters;
780 if (outputType == AIEToConfigurationOutputType::ControlPacket && configured &&
781 configured.getOverlayAttr()) {
782 auto overlay = SymbolTable::lookupNearestSymbolFrom<DeviceOp>(
783 configured->getParentOp(), configured.getOverlayAttr());
784 if (!overlay)
785 return configured.emitOpError("references an unknown overlay device");
786 AIERTControl overlayCtl(overlay.getTargetModel());
787 if (failed(overlayCtl.setIOBackend(false, false)))
788 return failure();
789 overlayCtl.startTransaction();
790 if (failed(overlayCtl.configureSwitches(overlay)))
791 return failure();
792 auto overlayData = overlayCtl.exportSerializedTransaction();
793 std::vector<TransactionBinaryOperation> overlayOps;
794 if (!parseTransactionBinary(overlayData, overlayOps))
795 return failure();
796 // Masked writes must preserve fields installed by the overlay's PDI.
797 for (const auto &op : overlayOps) {
798 uint32_t &value = initialRegisters[op.cmd.RegOff];
799 if (op.cmd.Opcode == XAIE_IO_WRITE)
800 value = op.cmd.Value;
801 else if (op.cmd.Opcode == XAIE_IO_MASKWRITE)
802 value = (value & ~op.cmd.Mask) | (op.cmd.Value & op.cmd.Mask);
803 else
804 return overlay.emitOpError("has a non-register switch configuration");
805 }
806 }
807 return convertTransactionOpsToMLIR(builder, outputType, operations,
808 blockwriteData, initialRegisters);
809}
810
812 OpBuilder &builder, xilinx::AIE::DeviceOp device,
813 AIEX::BlockwriteData &blockwriteData, llvm::StringRef clElfDir,
814 AIE::AIEToConfigurationOutputType outputType, bool skipCtrlPktOverlay) {
815 const AIETargetModel &targetModel =
816 (const AIETargetModel &)device.getTargetModel();
817
818 if (!targetModel.hasProperty(AIETargetModel::IsNPU))
819 return failure();
820
821 bool aieSim = false;
822 bool xaieDebug = false;
823
824 AIERTControl ctl(targetModel);
825 if (failed(ctl.setIOBackend(aieSim, xaieDebug)))
826 return failure();
827
828 // start collecting transactions
829 ctl.startTransaction();
830
831 bool generateElfs = true;
832 if (failed(generateTransactions(ctl, clElfDir, device, aieSim, generateElfs,
833 true, true, skipCtrlPktOverlay)))
834 return failure();
835
836 return insertRecordedTransaction(ctl, builder, outputType, blockwriteData,
837 skipCtrlPktOverlay ? device : DeviceOp{});
838}
839
840namespace {
841
842using LockAddress = std::tuple<int32_t, int32_t, int32_t>;
843using LockAcquire = std::pair<LockAction, int32_t>;
844
845static std::optional<LockAddress> getLockAddress(UseLockOp use) {
846 auto lock = use.getLock().getDefiningOp<LockOp>();
847 if (!lock)
848 return std::nullopt;
849 auto lockId = lock.getLockID();
850 if (!lockId)
851 return std::nullopt;
852 auto tile = lock.getTileOp();
853 return LockAddress{tile.colIndex(), tile.rowIndex(), *lockId};
854}
855
856static std::optional<LockAcquire> getLockAcquire(UseLockOp use) {
857 APInt value;
858 if (!matchPattern(use.getValue(), m_ConstantInt(&value)))
859 return std::nullopt;
860 return LockAcquire{use.getAction(), value.getSExtValue()};
861}
862
863static CoreLockAcquiresAttr collectCoreLockAcquires(CoreOp core) {
864 if (core.getElfFile())
865 return core.getLockAcquiresAttr();
866 llvm::SetVector<LockAcquireAttr> acquires;
867 llvm::DenseSet<Operation *> visited;
868 SmallVector<Operation *> pending{core};
869 bool unknown = false;
870 while (!pending.empty()) {
871 Operation *operation = pending.pop_back_val();
872 if (!visited.insert(operation).second)
873 continue;
874 operation->walk([&](UseLockOp use) {
875 if (use.release())
876 return;
877 auto address = getLockAddress(use);
878 auto acquire = getLockAcquire(use);
879 if (!address || !acquire) {
880 unknown = true;
881 return;
882 }
883 auto [column, row, lockId] = *address;
884 auto [action, value] = *acquire;
885 acquires.insert(LockAcquireAttr::get(core.getContext(), column, row,
886 lockId, action, value));
887 });
888 operation->walk([&](func::CallOp call) {
889 auto callee = SymbolTable::lookupNearestSymbolFrom<func::FuncOp>(
890 call, call.getCalleeAttr());
891 if (callee && !callee.isExternal())
892 pending.push_back(callee);
893 else
894 unknown = true;
895 });
896 operation->walk([&](Operation *nested) {
897 if ((isa<CallOpInterface>(nested) && !isa<func::CallOp>(nested)) ||
898 isa<LLVM::CallIntrinsicOp, LLVM::InlineAsmOp>(nested))
899 unknown = true;
900 });
901 }
902 return CoreLockAcquiresAttr::get(core.getContext(), !unknown,
903 acquires.getArrayRef());
904}
905
906struct LockAcquireSummary {
907 std::set<LockAcquire> acquires;
908 SmallVector<CoreOp> cores;
909 bool unknown = false;
910
911 std::optional<int32_t> getSatisfyingValue(uint32_t maximum) const {
912 if (unknown || acquires.empty())
913 return std::nullopt;
914 int32_t lowerBound = 0;
915 std::optional<int32_t> equality;
916 for (auto [action, value] : acquires) {
917 if (value < 0 || static_cast<uint32_t>(value) > maximum)
918 return std::nullopt;
919 if (action == LockAction::AcquireGreaterEqual && value != 0) {
920 lowerBound = std::max(lowerBound, value);
921 } else {
922 if (equality && *equality != value)
923 return std::nullopt;
924 equality = value;
925 }
926 }
927 if (equality && *equality < lowerBound)
928 return std::nullopt;
929 return equality.value_or(lowerBound);
930 }
931};
932
933struct CoreLockAcquireAnalysis {
934 std::map<LockAddress, LockAcquireSummary> locks;
935 llvm::DenseMap<CoreOp, std::set<LockAddress>> coreLocks;
936 llvm::DenseSet<CoreOp> unsupported;
937
938 explicit CoreLockAcquireAnalysis(DeviceOp device) {
939 const auto &model = device.getTargetModel();
940 for (auto core : device.getOps<CoreOp>()) {
941 auto summary = collectCoreLockAcquires(core);
942 if (!summary) {
943 core.emitWarning("teardown cannot determine lock acquires for an ELF "
944 "without lock_acquires; skipping device "
945 "lock draining");
946 unsupported.insert(core);
947 continue;
948 }
949 if (!summary.getComplete()) {
950 core.emitWarning("teardown cannot determine every lock acquire: "
951 "unresolved locks, dynamic values, or opaque callees; "
952 "skipping device lock draining");
953 unsupported.insert(core);
954 continue;
955 }
956 for (LockAcquireAttr acquire : summary.getAcquires()) {
957 LockAddress address{acquire.getColumn(), acquire.getRow(),
958 acquire.getLockId()};
959 auto &lock = locks[address];
960 lock.acquires.emplace(acquire.getAction(), acquire.getValue());
961 if (coreLocks[core].insert(address).second)
962 lock.cores.push_back(core);
963 }
964 }
965 if (!unsupported.empty()) {
966 for (auto core : device.getOps<CoreOp>())
967 unsupported.insert(core);
968 return;
969 }
970 device.walkOutsideRuntimeSequences<UseLockOp>([&](UseLockOp use) {
971 if (use->getParentOfType<CoreOp>() || use.release() || !use.getAcqEn())
972 return;
973 auto address = getLockAddress(use);
974 if (!address)
975 return;
976 auto found = locks.find(*address);
977 if (found == locks.end())
978 return;
979 auto acquire = getLockAcquire(use);
980 if (!acquire)
981 found->second.unknown = true;
982 else
983 found->second.acquires.insert(*acquire);
984 });
985 for (auto &[address, lock] : locks) {
986 auto [column, row, lockId] = address;
987 bool competing = lock.cores.size() > 1;
988 bool incompatible = !lock.getSatisfyingValue(model.getMaxLockValue());
989 if (!competing && !incompatible)
990 continue;
991 for (auto core : lock.cores) {
992 core.emitWarning()
993 << "teardown cannot drain lock " << lockId << " on tile (" << column
994 << ", " << row << "): "
995 << (competing ? "multiple cores may consume the supplied count"
996 : "no single writable value satisfies every acquire")
997 << "; skipping core lock draining";
998 unsupported.insert(core);
999 }
1000 }
1001 }
1002};
1003
1004static LogicalResult
1005insertCoreLockDrain(OpBuilder &builder, DeviceOp device,
1006 AIEX::BlockwriteData &blockwriteData,
1007 AIEToConfigurationOutputType outputType) {
1008 const auto &model = device.getTargetModel();
1009 if (!model.hasProperty(AIETargetModel::UsesSemaphoreLocks))
1010 return success();
1011 CoreLockAcquireAnalysis analysis(device);
1012 std::set<LockAddress> candidates;
1013 SmallVector<CoreOp> cores;
1014 for (auto core : device.getOps<CoreOp>()) {
1015 auto found = analysis.coreLocks.find(core);
1016 if (analysis.unsupported.contains(core) ||
1017 found == analysis.coreLocks.end())
1018 continue;
1019 candidates.insert(found->second.begin(), found->second.end());
1020 cores.push_back(core);
1021 }
1022 auto tileAddress = [&](int32_t column, int32_t row) {
1023 return (static_cast<uint32_t>(column) << model.getColumnShift()) |
1024 (static_cast<uint32_t>(row) << model.getRowShift());
1025 };
1026 auto pollCoreStatus = [&](CoreOp core, uint32_t maskValue) {
1027 auto tile = core.getTileOp();
1028 auto constant = [&](uint32_t value) {
1029 return arith::ConstantOp::create(builder, core.getLoc(),
1030 builder.getI32IntegerAttr(value));
1031 };
1032 Value status =
1033 constant(tileAddress(tile.colIndex(), tile.rowIndex()) | 0x32004);
1034 Value zero = constant(0);
1035 Value mask = constant(maskValue);
1036 AIEX::NpuMaskPollOp::create(builder, core.getLoc(), status, zero, mask,
1037 nullptr, nullptr, nullptr);
1038 };
1039 std::vector<TransactionBinaryOperation> writes;
1040 for (auto address : candidates) {
1041 auto [column, row, lockId] = address;
1042 auto localAddress = model.getLocalLockAddress(lockId, TileID{column, row});
1043 auto value =
1044 analysis.locks.at(address).getSatisfyingValue(model.getMaxLockValue());
1045 if (!localAddress || !value)
1046 return device.emitOpError(
1047 "teardown lock drain requires a valid lock address and value");
1049 write.cmd.Opcode = XAie_TxnOpcode::XAIE_IO_WRITE;
1050 write.cmd.RegOff = tileAddress(column, row) | *localAddress;
1051 write.cmd.Value = *value;
1052 write.sourceLoc = device.getLoc();
1053 writes.push_back(write);
1054 }
1055 if (!writes.empty())
1056 for (auto core : device.getOps<CoreOp>())
1057 pollCoreStatus(core, 0x1);
1058 if (failed(convertTransactionOpsToMLIR(builder, outputType, writes,
1059 blockwriteData, {})))
1060 return failure();
1061 for (auto core : cores)
1062 pollCoreStatus(core, 0x3c0);
1063 return success();
1064}
1065
1066struct AIEAnnotateCoreLockAcquiresPass
1067 : xilinx::impl::AIEAnnotateCoreLockAcquiresBase<
1068 AIEAnnotateCoreLockAcquiresPass> {
1069 void runOnOperation() override {
1070 getOperation().walk([](CoreOp core) {
1071 if (auto summary = collectCoreLockAcquires(core))
1072 core.setLockAcquiresAttr(summary);
1073 });
1074 }
1075};
1076
1077} // namespace
1078
1079std::unique_ptr<mlir::OperationPass<mlir::ModuleOp>>
1081 return std::make_unique<AIEAnnotateCoreLockAcquiresPass>();
1082}
1083
1085 OpBuilder &builder, xilinx::AIE::DeviceOp device,
1086 AIEX::BlockwriteData &blockwriteData,
1087 AIE::AIEToConfigurationOutputType outputType, bool skipCtrlPktOverlay) {
1088 const AIETargetModel &targetModel =
1089 (const AIETargetModel &)device.getTargetModel();
1090 if (!targetModel.hasProperty(AIETargetModel::IsNPU))
1091 return failure();
1092
1093 AIERTControl ctl(targetModel);
1094 if (failed(ctl.setIOBackend(/*aieSim=*/false, /*xaieDebug=*/false)))
1095 return failure();
1096 ctl.startTransaction();
1097 if (failed(ctl.teardownSwitches(device, skipCtrlPktOverlay)) ||
1098 failed(ctl.disableCores(device)) || failed(ctl.resetDmaChannels(device)))
1099 return failure();
1100 if (failed(
1101 insertRecordedTransaction(ctl, builder, outputType, blockwriteData,
1102 skipCtrlPktOverlay ? device : DeviceOp{})))
1103 return failure();
1104 return insertCoreLockDrain(builder, device, blockwriteData, outputType);
1105}
1106
1107LogicalResult
1108xilinx::AIE::insertConfigOps(ArrayRef<std::pair<Operation *, DeviceOp>> sites,
1110 bool ctrlPkt = outputType == AIE::AIEToConfigurationOutputType::ControlPacket;
1111 // Indexed by whether the site is a teardown.
1112 std::array<llvm::DenseMap<DeviceOp, AIEX::BlockwriteData>, 2> blockwriteData;
1113 std::array<llvm::DenseMap<std::pair<Operation *, Operation *>,
1114 std::unique_ptr<Block>>,
1115 2>
1116 configs;
1117 struct ConstantPool {
1118 Operation *last = nullptr;
1119 llvm::DenseMap<Attribute, Value> values;
1120 };
1121 llvm::DenseMap<Operation *, ConstantPool> pools;
1122 for (auto [at, configured] : sites) {
1123 auto device = at->getParentOfType<DeviceOp>();
1124 if (!device)
1125 return at->emitError("expected to be nested under an aie.device op");
1126 bool teardown = isa<AIEX::NpuWriteTeardownOp>(at);
1127 std::unique_ptr<Block> &config = configs[teardown][{configured, device}];
1128 if (!config) {
1129 AIEX::BlockwriteData &payloads =
1130 blockwriteData[teardown]
1131 .try_emplace(device, device, teardown ? "teardown_" : "loadpdi_")
1132 .first->second;
1133 auto generated = std::make_unique<Block>();
1134 OpBuilder configBuilder(at->getContext());
1135 configBuilder.setInsertionPointToEnd(generated.get());
1136 if (teardown ? failed(generateAndInsertTeardownOps(
1137 configBuilder, configured, payloads, outputType))
1139 configBuilder, configured, payloads,
1140 /*clElfDir=*/"", outputType,
1141 /*skipCtrlPktOverlay=*/ctrlPkt)))
1142 return at->emitError(teardown ? "Failed to generate teardown operations"
1143 : "Failed to generate configuration "
1144 "operations");
1145 config = std::move(generated);
1146 }
1147 auto seq = at->getParentOfType<RuntimeSequenceOp>();
1148 ConstantPool *pool = seq ? &pools[seq] : nullptr;
1149 OpBuilder builder(at);
1150 IRMapping mapping;
1151 for (Operation &op : *config) {
1152 Attribute key;
1153 if (auto constant = dyn_cast<arith::ConstantOp>(op))
1154 key = constant.getValue();
1155 else if (auto global = dyn_cast<memref::GetGlobalOp>(op))
1156 key = global.getNameAttr();
1157 if (!pool || !key) {
1158 builder.clone(op, mapping);
1159 continue;
1160 }
1161 Value &pooled = pool->values[key];
1162 if (!pooled) {
1163 OpBuilder::InsertionGuard guard(builder);
1164 if (pool->last)
1165 builder.setInsertionPointAfter(pool->last);
1166 else
1167 builder.setInsertionPointToStart(&seq.getBody().front());
1168 pool->last = builder.clone(op);
1169 pooled = pool->last->getResult(0);
1170 }
1171 mapping.map(op.getResult(0), pooled);
1172 }
1173 }
1174 return success();
1175}
1176
1177LogicalResult xilinx::AIE::inlineWriteConfigs(ModuleOp module) {
1178 SmallVector<std::pair<Operation *, DeviceOp>> sites;
1179 WalkResult walk = module.walk([&](Operation *write) {
1180 FlatSymbolRefAttr ref;
1181 if (auto config = dyn_cast<AIEX::NpuWriteConfigOp>(write))
1182 ref = config.getDeviceRefAttr();
1183 else if (auto teardown = dyn_cast<AIEX::NpuWriteTeardownOp>(write))
1184 ref = teardown.getDeviceRefAttr();
1185 else
1186 return WalkResult::advance();
1187 auto configured = module.lookupSymbol<DeviceOp>(ref);
1188 if (!configured) {
1189 write->emitOpError() << ref << " is not a device";
1190 return WalkResult::interrupt();
1191 }
1192 sites.emplace_back(write, configured);
1193 return WalkResult::advance();
1194 });
1195 if (walk.wasInterrupted() ||
1197 return failure();
1198 for (auto [write, configured] : sites)
1199 write->erase();
1200 return success();
1201}
1202
1203static LogicalResult
1204convertAIEToConfiguration(AIE::DeviceOp device, StringRef clElfDir,
1206
1207 OpBuilder builder(device.getBodyRegion());
1208 // search for aiex.configure ops in runtime sequences by walking the device
1209 // and collect them in a vector. If there are none, create a new runtime
1210 // sequence. Otherwise assume the insertion point is the first
1211 // aiex.configure op.
1212 auto loc = device.getLoc();
1213 SmallVector<AIEX::ConfigureOp> configureOps;
1214 device.walk([&](AIEX::ConfigureOp op) { configureOps.push_back(op); });
1215
1216 if (configureOps.empty()) {
1217 // create aiex.runtime_sequence
1218 int id = 0;
1219 std::string seq_name = "configure";
1220 while (device.lookupSymbol(seq_name))
1221 seq_name = "configure" + std::to_string(id++);
1222 StringAttr seq_sym_name = builder.getStringAttr(seq_name);
1223 auto seq = AIE::RuntimeSequenceOp::create(
1224 builder, loc, seq_sym_name, BoolAttr{}, AIE::TraceBufferAttr{},
1225 ArrayAttr{}, ArrayAttr{});
1226 // A user sequence may already own the name `configure`; consumers find
1227 // the configuration by this tag.
1228 AIE::AIEDialect::GeneratedConfigureAttrHelper(device.getContext())
1229 .setAttr(seq, builder.getUnitAttr());
1230 seq.getBody().push_back(new Block);
1231 builder.setInsertionPointToStart(&seq.getBody().front());
1232 } else {
1233 builder.setInsertionPoint(configureOps.front());
1234 }
1235
1236 // convert the parsed ops to MLIR
1237 AIEX::BlockwriteData blockwriteData(device, "config_blockwrite_data_");
1238 if (failed(generateAndInsertConfigOps(builder, device, blockwriteData,
1239 clElfDir, outputType)))
1240 return failure();
1241
1242 // If we chose the first aiex.configure as insertion point, erase it
1243 // and inline its child operations.
1244 if (!configureOps.empty()) {
1245 // splice the body into the current insertion point
1246 builder.getBlock()->getOperations().splice(
1247 builder.getInsertionPoint(),
1248 configureOps.front().getBody().front().getOperations());
1249 configureOps.front().erase();
1250 }
1251
1252 return success();
1253}
1254
1255namespace {
1256
1257template <typename BaseClass, AIE::AIEToConfigurationOutputType MyOutputType>
1258struct ConvertAIEToConfigurationPass : BaseClass {
1259 std::string &ref_clElfDir;
1260 std::string &ref_clDeviceName;
1261 ConvertAIEToConfigurationPass(std::string &clElfDir,
1262 std::string &clDeviceName)
1263 : ref_clElfDir(clElfDir), ref_clDeviceName(clDeviceName) {}
1264
1265 void getDependentDialects(DialectRegistry &registry) const override {
1266 registry.insert<memref::MemRefDialect, AIEX::AIEXDialect,
1267 arith::ArithDialect>();
1268 }
1269
1270 void runOnOperation() override {
1271 AIE::DeviceOp deviceOp = BaseClass::getOperation();
1272 if (!ref_clDeviceName.empty() &&
1273 deviceOp.getSymName() != ref_clDeviceName) {
1274 return;
1275 }
1276 if (failed(
1277 convertAIEToConfiguration(deviceOp, ref_clElfDir, MyOutputType))) {
1278 return BaseClass::signalPassFailure();
1279 }
1280 }
1281};
1282
1283struct ConvertAIEToTransactionPass
1284 : ConvertAIEToConfigurationPass<
1285 xilinx::impl::ConvertAIEToTransactionBase<
1286 ConvertAIEToTransactionPass>,
1287 AIE::AIEToConfigurationOutputType::Transaction> {
1288 ConvertAIEToTransactionPass()
1289 : ConvertAIEToConfigurationPass<
1290 xilinx::impl::ConvertAIEToTransactionBase<
1291 ConvertAIEToTransactionPass>,
1293 clDeviceName) {}
1294};
1295
1296struct ConvertAIEToControlPacketsPass
1297 : ConvertAIEToConfigurationPass<
1298 xilinx::impl::ConvertAIEToControlPacketsBase<
1299 ConvertAIEToControlPacketsPass>,
1300 AIE::AIEToConfigurationOutputType::ControlPacket> {
1301 ConvertAIEToControlPacketsPass()
1302 : ConvertAIEToConfigurationPass<
1303 xilinx::impl::ConvertAIEToControlPacketsBase<
1304 ConvertAIEToControlPacketsPass>,
1306 clDeviceName) {}
1307};
1308
1309} // end anonymous namespace
1310
1311std::unique_ptr<mlir::OperationPass<xilinx::AIE::DeviceOp>>
1313 return std::make_unique<ConvertAIEToTransactionPass>();
1314}
1315
1316std::unique_ptr<mlir::OperationPass<xilinx::AIE::DeviceOp>>
1318 return std::make_unique<ConvertAIEToControlPacketsPass>();
1319}
memref::GlobalOp getOrCreate(OpBuilder &builder, mlir::Location loc, ArrayRef< uint32_t > words)
Definition AIEUtils.cpp:187
AIE::DeviceOp getDevice() const
Definition AIEUtils.h:40
bool hasProperty(ModelProperty Prop) const
mlir::Value createConstantArgPlus(mlir::OpBuilder &builder, mlir::Location loc, uint64_t value)
mlir::Value createConstantI32(mlir::OpBuilder &builder, mlir::Location loc, uint32_t value)
mlir::LogicalResult insertConfigOps(llvm::ArrayRef< std::pair< mlir::Operation *, xilinx::AIE::DeviceOp > > sites, AIEToConfigurationOutputType outputType)
mlir::LogicalResult inlineWriteConfigs(mlir::ModuleOp module)
std::optional< int > parseTransactionBinary(const std::vector< uint8_t > &data, std::vector< TransactionBinaryOperation > &ops)
Parse an exported transaction.
std::unique_ptr< mlir::OperationPass< xilinx::AIE::DeviceOp > > createConvertAIEToTransactionPass()
mlir::LogicalResult generateAndInsertConfigOps(mlir::OpBuilder &builder, xilinx::AIE::DeviceOp device, xilinx::AIEX::BlockwriteData &blockwriteData, llvm::StringRef clElfDir="", AIEToConfigurationOutputType outputType=AIEToConfigurationOutputType::Transaction, bool skipCtrlPktOverlay=false)
std::unique_ptr< mlir::OperationPass< mlir::ModuleOp > > createAIEAnnotateCoreLockAcquiresPass()
struct TileID { friend std::ostream &operator<<(std::ostream &os, const TileID &s) { os<< "TileID("<< s.col<< ", "<< s.row<< ")"; return os; } friend std::string to_string(const TileID &s) { std::ostringstream ss; ss<< s; return ss.str(); } friend llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const TileID &s) { os<< to_string(s); return os; } inline bool operator<(const TileID &rhs) const { return std::tie(col, row)< std::tie(rhs.col, rhs.row); } bool operator==(const TileID &rhs) const { return std::tie(col, row)==std::tie(rhs.col, rhs.row); } bool operator!=(const TileID &rhs) const { return !(*this==rhs);} int col, row;} TileID
mlir::LogicalResult generateAndInsertTeardownOps(mlir::OpBuilder &builder, xilinx::AIE::DeviceOp device, xilinx::AIEX::BlockwriteData &blockwriteData, AIEToConfigurationOutputType outputType=AIEToConfigurationOutputType::Transaction, bool skipCtrlPktOverlay=false)
std::optional< mlir::ModuleOp > convertTransactionBinaryToMLIR(mlir::MLIRContext *ctx, std::vector< uint8_t > &binary)
std::unique_ptr< mlir::OperationPass< xilinx::AIE::DeviceOp > > createConvertAIEToControlPacketsPass()
mlir::LogicalResult setIOBackend(bool aieSim, bool xaieDebug)
Definition AIERT.cpp:298
std::vector< uint8_t > exportSerializedTransaction()
Definition AIERT.cpp:1327
mlir::LogicalResult disableCores(DeviceOp &targetOp)
Definition AIERT.cpp:911
mlir::LogicalResult addCoreEnable(DeviceOp &targetOp)
Definition AIERT.cpp:1053
const std::vector< mlir::Location > & getTxnOpLocs() const
Definition AIERT.cpp:1346
mlir::LogicalResult addInitConfig(DeviceOp &targetOp, bool skipCtrlPktOverlay=false)
Definition AIERT.cpp:962
mlir::LogicalResult resetDmaChannels(DeviceOp &targetOp)
Definition AIERT.cpp:921
mlir::LogicalResult addAieElfs(DeviceOp &targetOp, const mlir::StringRef elfPath, bool aieSim)
Definition AIERT.cpp:1226
mlir::LogicalResult teardownSwitches(DeviceOp &targetOp, bool skipCtrlPktOverlay=false)
Definition AIERT.cpp:859
std::optional< LoadPdiPayload > loadPdi
std::optional< mlir::Location > sourceLoc
std::optional< AddressPatchPayload > addressPatch