17#include "mlir/Dialect/Arith/IR/Arith.h"
18#include "mlir/Dialect/Func/IR/FuncOps.h"
19#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
20#include "mlir/Dialect/MemRef/IR/MemRef.h"
21#include "mlir/IR/IRMapping.h"
22#include "mlir/IR/Matchers.h"
23#include "mlir/Interfaces/CallInterfaces.h"
25#include "llvm/ADT/APInt.h"
26#include "llvm/ADT/DenseMap.h"
27#include "llvm/ADT/DenseSet.h"
28#include "llvm/ADT/SetVector.h"
29#include "llvm/Support/Debug.h"
32#include "aie_codegen.h"
46#define GEN_PASS_DEF_AIEANNOTATECORELOCKACQUIRES
47#define GEN_PASS_DEF_CONVERTAIETOCONTROLPACKETS
48#define GEN_PASS_DEF_CONVERTAIETOTRANSACTION
49#include "aie/Conversion/Passes.h.inc"
52#define DEBUG_TYPE "aie-convert-to-config"
60constexpr size_t kTxnHeaderBytes = 16;
62struct TxnPreemptHeader {
68struct TxnLoadPdiHeader {
80 const std::vector<uint8_t> &data,
81 std::vector<TransactionBinaryOperation> &ops) {
83 if (data.size() < kTxnHeaderBytes) {
84 llvm::errs() <<
"Transaction binary is too small for header\n";
88 uint32_t major = data[0];
89 uint32_t minor = data[1];
90 uint32_t num_cols = data[4];
92 uint32_t num_ops, txn_size;
93 std::memcpy(&num_ops, &data[8], 4);
94 std::memcpy(&txn_size, &data[12], 4);
96 LLVM_DEBUG(llvm::dbgs() <<
"Major: " << major <<
"\n");
97 LLVM_DEBUG(llvm::dbgs() <<
"Minor: " << minor <<
"\n");
98 LLVM_DEBUG(llvm::dbgs() <<
"DevGen: " << data[2] <<
"\n");
99 LLVM_DEBUG(llvm::dbgs() <<
"NumRows: " << data[3] <<
"\n");
100 LLVM_DEBUG(llvm::dbgs() <<
"NumCols: " << num_cols <<
"\n");
101 LLVM_DEBUG(llvm::dbgs() <<
"NumMemTileRows: " << data[5] <<
"\n");
102 LLVM_DEBUG(llvm::dbgs() <<
"NumOps: " << num_ops <<
"\n");
103 LLVM_DEBUG(llvm::dbgs() <<
"TxnSize: " << txn_size <<
" bytes\n");
105 size_t i = kTxnHeaderBytes;
107 auto requireBytes = [&](
size_t offset,
size_t length) ->
bool {
108 if (offset + length > data.size()) {
109 llvm::errs() <<
"Transaction binary truncated while parsing opcode\n";
115 auto read32 = [&](
size_t offset) -> uint32_t {
117 std::memcpy(&value, data.data() + offset,
sizeof(uint32_t));
122 auto convertOpcode = [](uint8_t opc) -> std::optional<uint8_t> {
124 case static_cast<uint8_t
>(XAie_TxnOpcode::XAIE_IO_WRITE):
125 case static_cast<uint8_t
>(XAie_TxnOpcode::XAIE_IO_BLOCKWRITE):
126 case static_cast<uint8_t
>(XAie_TxnOpcode::XAIE_IO_MASKWRITE):
127 case static_cast<uint8_t
>(XAie_TxnOpcode::XAIE_IO_MASKPOLL):
130 case static_cast<uint8_t
>(XAie_TxnOpcode::XAIE_IO_CUSTOM_OP_TCT):
131 case static_cast<uint8_t
>(XAie_TxnOpcode::XAIE_IO_CUSTOM_OP_DDR_PATCH):
134 llvm::errs() <<
"Unhandled opcode: " << std::to_string(opc) <<
"\n";
142 if (major == 0 && minor == 1) {
143 while (i < data.size()) {
144 auto maybeOpcode = convertOpcode(data[i]);
147 XAie_TxnOpcode opcode =
static_cast<XAie_TxnOpcode
>(*maybeOpcode);
148 LLVM_DEBUG(llvm::dbgs() <<
"opcode: " + std::to_string(opcode) <<
"\n");
151 op.
cmd.Opcode = opcode;
154 case XAie_TxnOpcode::XAIE_IO_WRITE: {
155 LLVM_DEBUG(llvm::dbgs() <<
"opcode: WRITE (0x00)\n");
156 if (!requireBytes(i, 24))
158 uint32_t addrLo = read32(i + 8);
159 uint32_t addrHi = read32(i + 12);
160 uint32_t value = read32(i + 16);
161 uint32_t opSize = read32(i + 20);
162 if (!requireBytes(i, opSize))
164 uint64_t addr = (
static_cast<uint64_t
>(addrHi) << 32) | addrLo;
165 op.
cmd.RegOff = addr;
166 op.
cmd.Value = value;
171 case XAie_TxnOpcode::XAIE_IO_BLOCKWRITE: {
172 LLVM_DEBUG(llvm::dbgs() <<
"opcode: BLOCKWRITE (0x01)\n");
173 if (!requireBytes(i, 16))
175 uint32_t addr = read32(i + 8);
176 uint32_t opSize = read32(i + 12);
177 if (opSize < 16 || !requireBytes(i, opSize))
179 const uint8_t *payload = data.data() + i + 16;
180 uint32_t payloadBytes = opSize - 16;
181 op.
cmd.RegOff = addr;
182 op.
cmd.DataPtr =
reinterpret_cast<uint64_t
>(payload);
183 op.
cmd.Size = payloadBytes;
190 case XAie_TxnOpcode::XAIE_IO_MASKWRITE:
191 case XAie_TxnOpcode::XAIE_IO_MASKPOLL: {
192 LLVM_DEBUG(llvm::dbgs() <<
"opcode: MASKWRITE (0x03) or MASKPOLL "
194 if (!requireBytes(i, 28))
196 uint32_t addrLo = read32(i + 8);
197 uint32_t addrHi = read32(i + 12);
198 uint32_t value = read32(i + 16);
199 uint32_t mask = read32(i + 20);
200 uint32_t opSize = read32(i + 24);
202 llvm::errs() <<
"Transaction op size " << opSize
203 <<
" is smaller than its 28-byte header\n";
206 if (!requireBytes(i, opSize))
208 uint64_t addr = (
static_cast<uint64_t
>(addrHi) << 32) | addrLo;
209 op.
cmd.RegOff = addr;
210 op.
cmd.Value = value;
212 op.
cmd.Size = opSize;
216 case XAie_TxnOpcode::XAIE_IO_CUSTOM_OP_TCT: {
217 uint32_t opSize = read32(i + 4);
218 if (opSize < 16 || !requireBytes(i, opSize))
220 uint32_t descriptor = read32(i + 8);
221 uint32_t config = read32(i + 12);
223 static_cast<int32_t
>((descriptor >> 16) & 0xff),
224 static_cast<int32_t
>((descriptor >> 8) & 0xff),
225 static_cast<int32_t
>(descriptor & 0xff),
226 static_cast<int32_t
>((config >> 24) & 0xff),
227 static_cast<int32_t
>((config >> 16) & 0xff),
228 static_cast<int32_t
>((config >> 8) & 0xff)};
230 op.
cmd.Size = opSize;
238 LLVM_DEBUG(llvm::dbgs() <<
"opcode: LOAD_PDI (0x08)\n");
239 constexpr size_t opSize =
sizeof(TxnLoadPdiHeader);
240 if (!requireBytes(i, opSize))
242 TxnLoadPdiHeader header;
243 std::memcpy(&header, data.data() + i, opSize);
245 header.id, header.size, header.address};
247 op.
cmd.Size = opSize;
251 case XAie_TxnOpcode::XAIE_IO_CUSTOM_OP_DDR_PATCH: {
252 uint32_t opSize = read32(i + 4);
253 if (opSize < 44 || !requireBytes(i, opSize))
255 uint32_t action = read32(i + 20);
256 uint32_t addr = read32(i + 24);
257 int32_t argIdx =
static_cast<int32_t
>(read32(i + 32));
258 uint64_t argPlus =
static_cast<uint64_t
>(read32(i + 40)) |
259 (
static_cast<uint64_t
>(read32(i + 44)) << 32);
261 action, addr, argIdx, argPlus};
263 op.
cmd.Size = opSize;
268 LLVM_DEBUG(llvm::dbgs() <<
"opcode: PREEMPT (0x06)\n");
269 constexpr size_t opSize =
sizeof(TxnPreemptHeader);
270 if (!requireBytes(i, opSize))
273 reinterpret_cast<const TxnPreemptHeader *
>(data.data() + i);
274 op.
cmd.Value = header->level;
275 op.
cmd.Size = opSize;
280 llvm::errs() <<
"Unhandled opcode: " << std::to_string(opcode)
281 <<
" for v0.1 transaction\n";
287 }
else if (major == 1 && minor == 0) {
288 while (i < data.size()) {
289 auto maybeOpcode = convertOpcode(data[i]);
292 XAie_TxnOpcode opcode =
static_cast<XAie_TxnOpcode
>(*maybeOpcode);
293 LLVM_DEBUG(llvm::dbgs() <<
"opcode: " + std::to_string(opcode) <<
"\n");
296 op.
cmd.Opcode = opcode;
299 case XAie_TxnOpcode::XAIE_IO_WRITE: {
300 LLVM_DEBUG(llvm::dbgs() <<
"opcode: WRITE (0x00)\n");
301 if (!requireBytes(i, 12))
303 uint32_t addr = read32(i + 4);
304 uint32_t value = read32(i + 8);
305 op.
cmd.RegOff = addr;
306 op.
cmd.Value = value;
311 case XAie_TxnOpcode::XAIE_IO_BLOCKWRITE: {
312 LLVM_DEBUG(llvm::dbgs() <<
"opcode: BLOCKWRITE (0x01)\n");
313 if (!requireBytes(i, 12))
315 uint32_t addr = read32(i + 4);
316 uint32_t opSize = read32(i + 8);
317 if (opSize < 12 || !requireBytes(i, opSize))
319 const uint8_t *payload = data.data() + i + 12;
320 uint32_t payloadBytes = opSize - 12;
321 op.
cmd.RegOff = addr;
322 op.
cmd.DataPtr =
reinterpret_cast<uint64_t
>(payload);
323 op.
cmd.Size = payloadBytes;
327 case XAie_TxnOpcode::XAIE_IO_MASKWRITE:
328 case XAie_TxnOpcode::XAIE_IO_MASKPOLL: {
329 LLVM_DEBUG(llvm::dbgs() <<
"opcode: MASKWRITE (0x03) or MASKPOLL "
331 if (!requireBytes(i, 16))
333 uint32_t addr = read32(i + 4);
334 uint32_t value = read32(i + 8);
335 uint32_t mask = read32(i + 12);
336 op.
cmd.RegOff = addr;
337 op.
cmd.Value = value;
343 case XAie_TxnOpcode::XAIE_IO_CUSTOM_OP_TCT: {
344 uint32_t opSize = read32(i + 4);
345 if (opSize < 16 || !requireBytes(i, opSize))
347 uint32_t descriptor = read32(i + 8);
348 uint32_t config = read32(i + 12);
350 static_cast<int32_t
>((descriptor >> 16) & 0xff),
351 static_cast<int32_t
>((descriptor >> 8) & 0xff),
352 static_cast<int32_t
>(descriptor & 0xff),
353 static_cast<int32_t
>((config >> 24) & 0xff),
354 static_cast<int32_t
>((config >> 16) & 0xff),
355 static_cast<int32_t
>((config >> 8) & 0xff)};
357 op.
cmd.Size = opSize;
364 LLVM_DEBUG(llvm::dbgs() <<
"opcode: LOAD_PDI (0x08)\n");
365 constexpr size_t opSize =
sizeof(TxnLoadPdiHeader);
366 if (!requireBytes(i, opSize))
368 TxnLoadPdiHeader header;
369 std::memcpy(&header, data.data() + i, opSize);
371 header.id, header.size, header.address};
373 op.
cmd.Size = opSize;
377 case XAie_TxnOpcode::XAIE_IO_CUSTOM_OP_DDR_PATCH: {
378 uint32_t opSize = read32(i + 4);
379 if (opSize < 44 || !requireBytes(i, opSize))
381 uint32_t action = read32(i + 20);
382 uint32_t addr = read32(i + 24);
383 int32_t argIdx =
static_cast<int32_t
>(read32(i + 32));
384 uint64_t argPlus =
static_cast<uint64_t
>(read32(i + 40)) |
385 (
static_cast<uint64_t
>(read32(i + 44)) << 32);
387 action, addr, argIdx, argPlus};
389 op.
cmd.Size = opSize;
394 LLVM_DEBUG(llvm::dbgs() <<
"opcode: PREEMPT (0x06)\n");
395 constexpr size_t opSize =
sizeof(TxnPreemptHeader);
396 if (!requireBytes(i, opSize))
399 reinterpret_cast<const TxnPreemptHeader *
>(data.data() + i);
400 op.
cmd.Value = header->level;
401 op.
cmd.Size = opSize;
406 llvm::errs() <<
"Unhandled opcode: " << std::to_string(opcode)
407 <<
" for v1.0 transaction\n";
414 llvm::errs() <<
"Unsupported TXN binary version: " << major <<
"." << minor
422static LogicalResult generateTransactions(
AIERTControl &ctl,
423 const StringRef workDirPath,
424 DeviceOp &targetOp,
bool aieSim,
425 bool enableElfs,
bool enableInit,
427 bool skipCtrlPktOverlay =
false) {
428 if (enableElfs && !targetOp.getOps<CoreOp>().empty() &&
429 failed(ctl.
addAieElfs(targetOp, workDirPath, aieSim)))
431 if (enableInit && failed(ctl.
addInitConfig(targetOp, skipCtrlPktOverlay)))
433 if (enableCores && !targetOp.getOps<CoreOp>().empty() &&
445emitTransactionOps(OpBuilder &builder, Location fallbackLoc,
446 std::vector<TransactionBinaryOperation> &operations,
447 std::vector<memref::GlobalOp> &global_data) {
456 llvm::DenseMap<uint32_t, Value> constantCache;
457 auto constant = [&](Location loc, uint32_t value) {
458 Value &cached = constantCache[value];
465 for (
auto [op, payload] : llvm::zip(operations, global_data)) {
466 Location loc = op.sourceLoc.value_or(fallbackLoc);
468 if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_WRITE) {
469 Value address = constant(loc, op.cmd.RegOff);
470 Value value = constant(loc, op.cmd.Value);
471 AIEX::NpuWrite32Op::create(builder, loc, address, value,
nullptr,
nullptr,
473 }
else if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_BLOCKWRITE) {
474 auto memref = memref::GetGlobalOp::create(builder, loc, payload.getType(),
476 AIEX::NpuBlockWriteOp::create(
477 builder, loc, builder.getUI32IntegerAttr(op.cmd.RegOff),
478 memref.getResult(),
nullptr,
nullptr,
nullptr);
479 }
else if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_MASKWRITE) {
480 Value address = constant(loc, op.cmd.RegOff);
481 Value value = constant(loc, op.cmd.Value);
482 Value mask = constant(loc, op.cmd.Mask);
483 AIEX::NpuMaskWrite32Op::create(builder, loc, address, value, mask,
484 nullptr,
nullptr,
nullptr);
485 }
else if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_MASKPOLL) {
486 Value address = constant(loc, op.cmd.RegOff);
487 Value value = constant(loc, op.cmd.Value);
488 Value mask = constant(loc, op.cmd.Mask);
489 AIEX::NpuMaskPollOp::create(builder, loc, address, value, mask,
492 }
else if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_CUSTOM_OP_TCT) {
494 llvm::errs() <<
"Missing sync payload while emitting transaction\n";
498 Value column = constant(loc, sync.
column);
499 Value row = constant(loc, sync.
row);
500 Value direction = constant(loc, sync.
direction);
501 Value channel = constant(loc, sync.
channel);
502 Value columnCount = constant(loc, sync.
columnCount);
503 Value rowCount = constant(loc, sync.
rowCount);
504 AIEX::NpuSyncOp::create(builder, loc, column, row, direction, channel,
505 columnCount, rowCount);
506 }
else if (op.cmd.Opcode == 0x8 ) {
508 llvm::errs() <<
"Missing load_pdi payload while emitting transaction\n";
514 builder.getI32IntegerAttr(
static_cast<int32_t
>(payloadInfo.
id));
515 IntegerAttr sizeAttr =
516 builder.getI32IntegerAttr(
static_cast<int32_t
>(payloadInfo.
size));
519 IntegerType::get(builder.getContext(), 64, IntegerType::Unsigned);
520 IntegerAttr addressAttr =
521 IntegerAttr::get(ui64Ty, llvm::APInt(64, payloadInfo.
address));
523 AIEX::NpuLoadPdiOp::create(builder, loc,
nullptr, idAttr,
524 sizeAttr, addressAttr,
526 }
else if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_CUSTOM_OP_DDR_PATCH) {
527 if (!op.addressPatch) {
529 <<
"Missing address_patch payload while emitting transaction\n";
534 AIEX::NpuAddressPatchOp::create(
535 builder, loc, patch.
addr,
536 mlir::Value(),
static_cast<int32_t
>(patch.
argIdx),
538 }
else if (op.cmd.Opcode == 0x6 ) {
540 IntegerType::get(builder.getContext(), 8, IntegerType::Unsigned);
541 auto levelAttr = IntegerAttr::get(ui8Ty, llvm::APInt(8, op.cmd.Value));
542 AIEX::NpuPreemptOp::create(builder, loc, levelAttr);
544 llvm::errs() <<
"Unhandled txn opcode: " << op.cmd.Opcode <<
"\n";
553static llvm::DenseMap<uint32_t, uint32_t>
554registersWrittenSinceLoad(OpBuilder &builder) {
555 llvm::DenseMap<uint32_t, uint32_t> registers;
556 Block *block = builder.getInsertionBlock();
559 Block::iterator it = builder.getInsertionPoint();
560 while (it != block->begin()) {
561 Operation &op = *--it;
562 if (isa<AIEX::NpuLoadPdiOp>(op))
564 auto packet = dyn_cast<AIEX::NpuControlPacketOp>(op);
565 std::optional<ArrayRef<int32_t>> data =
566 packet ? packet.getData() : std::nullopt;
569 for (
auto [i, word] : llvm::enumerate(*data))
570 registers.try_emplace(packet.getAddress() + i *
sizeof(int32_t), word);
579static LogicalResult emitControlPacketOps(
580 OpBuilder &builder, Location fallbackLoc,
581 std::vector<TransactionBinaryOperation> &operations,
582 std::vector<memref::GlobalOp> &global_data,
583 const llvm::DenseMap<uint32_t, uint32_t> &initialRegisters) {
585 auto *ctx = builder.getContext();
589 llvm::DenseMap<uint32_t, uint32_t> registers =
590 registersWrittenSinceLoad(builder);
591 for (
auto [address, value] : initialRegisters)
592 registers.try_emplace(address, value);
595 for (
auto [op, payload] : llvm::zip(operations, global_data)) {
596 Location loc = op.sourceLoc.value_or(fallbackLoc);
598 if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_WRITE) {
599 registers[op.cmd.RegOff] = op.cmd.Value;
600 AIEX::NpuControlPacketOp::create(
601 builder, loc, builder.getUI32IntegerAttr(op.cmd.RegOff),
nullptr,
602 builder.getI32IntegerAttr(0),
603 builder.getI32IntegerAttr(0),
604 DenseI32ArrayAttr::get(ctx, ArrayRef<int32_t>(op.cmd.Value)));
605 }
else if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_BLOCKWRITE) {
606 auto initialValue = payload.getInitialValue();
609 auto blockWriteData = dyn_cast<DenseIntElementsAttr>(*initialValue);
610 if (!blockWriteData) {
612 "Global symbol initial value is not a dense int array");
615 auto blockWriteDataValues = blockWriteData.getValues<int32_t>();
617 int currAddr = op.cmd.RegOff;
618 for (
size_t i = 0; i < blockWriteDataValues.size(); i += 4) {
619 auto last = std::min(blockWriteDataValues.size(), i + 4);
620 SmallVector<int32_t> splitData =
621 SmallVector<int32_t>(blockWriteDataValues.begin() + i,
622 blockWriteDataValues.begin() + last);
623 for (
auto [j, word] : llvm::enumerate(splitData))
624 registers[currAddr + j *
sizeof(int32_t)] = word;
625 AIEX::NpuControlPacketOp::create(
626 builder, loc, builder.getUI32IntegerAttr(currAddr),
nullptr,
627 builder.getI32IntegerAttr(0),
628 builder.getI32IntegerAttr(0),
629 DenseI32ArrayAttr::get(ctx, ArrayRef<int32_t>(splitData)));
630 currAddr += splitData.size() *
sizeof(int32_t);
633 }
else if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_MASKWRITE) {
634 uint32_t ® = registers[op.cmd.RegOff];
635 reg = (reg & ~op.cmd.Mask) | (op.cmd.Value & op.cmd.Mask);
637 AIEX::NpuControlPacketOp::create(
638 builder, loc, builder.getUI32IntegerAttr(op.cmd.RegOff),
nullptr,
639 builder.getI32IntegerAttr(0),
640 builder.getI32IntegerAttr(0),
641 DenseI32ArrayAttr::get(ctx, ArrayRef<int32_t>(value)));
642 }
else if (op.cmd.Opcode == XAie_TxnOpcode::XAIE_IO_MASKPOLL) {
643 llvm::errs() <<
"control-packet configuration cannot express a mask "
647 llvm::errs() <<
"Unhandled txn opcode: " << op.cmd.Opcode <<
"\n";
656static LogicalResult convertTransactionOpsToMLIR(
658 std::vector<TransactionBinaryOperation> &operations,
660 const llvm::DenseMap<uint32_t, uint32_t> &initialRegisters = {}) {
664 std::vector<memref::GlobalOp> global_data;
665 DeviceOp device = blockwriteData.
getDevice();
666 Location loc = device.getLoc();
668 OpBuilder::InsertionGuard guard(builder);
669 builder.setInsertionPointToStart(device.getBody());
670 for (
auto &op : operations) {
671 if (op.cmd.Opcode != XAIE_IO_BLOCKWRITE) {
672 global_data.emplace_back(
nullptr);
675 const uint32_t *d =
reinterpret_cast<const uint32_t *
>(op.cmd.DataPtr);
677 builder, loc, ArrayRef<uint32_t>(d, op.cmd.Size / 4)));
683 if (failed(emitTransactionOps(builder, loc, operations, global_data)))
686 if (failed(emitControlPacketOps(builder, loc, operations, global_data,
690 llvm_unreachable(
"bad output type");
700std::optional<mlir::ModuleOp>
702 std::vector<uint8_t> &binary) {
705 std::vector<TransactionBinaryOperation> operations;
708 llvm::errs() <<
"Failed to parse binary\n";
716 ctx->getOrLoadDialect<mlir::arith::ArithDialect>();
718 auto loc = mlir::UnknownLoc::get(ctx);
721 auto module = ModuleOp::create(loc);
722 OpBuilder builder(module.getBodyRegion());
723 builder.setInsertionPointToStart(module.getBody());
726 std::vector<AIEDevice> devices{AIEDevice::npu1_1col, AIEDevice::npu1_2col,
727 AIEDevice::npu1_3col, AIEDevice::npu1};
728 auto device = DeviceOp::create(builder, loc, devices[columns - 1],
729 DeviceOp::getDefaultDeviceName(),
nullptr);
730 device.getRegion().emplaceBlock();
731 DeviceOp::ensureTerminator(device.getBodyRegion(), builder, loc);
732 builder.setInsertionPointToStart(device.getBody());
736 if (failed(convertTransactionOpsToMLIR(
746insertRecordedTransaction(
AIERTControl &ctl, OpBuilder &builder,
749 DeviceOp configured = {}) {
754 std::vector<TransactionBinaryOperation> operations;
756 llvm::errs() <<
"Failed to parse binary\n";
765 const std::vector<mlir::Location> &opLocs = ctl.
getTxnOpLocs();
771 assert((opLocs.empty() || opLocs.size() == operations.size()) &&
772 "txn loc/op count mismatch: AIERTControl's command-to-operation "
773 "projection disagrees with the transaction parser; aie-codegen's "
774 "serializer has drifted from projectCmdLocsOntoSerializedOps");
775 for (
size_t i = 0, e = std::min(opLocs.size(), operations.size()); i < e; ++i)
776 if (!isa<UnknownLoc>(opLocs[i]))
777 operations[i].sourceLoc = opLocs[i];
779 llvm::DenseMap<uint32_t, uint32_t> initialRegisters;
781 configured.getOverlayAttr()) {
782 auto overlay = SymbolTable::lookupNearestSymbolFrom<DeviceOp>(
783 configured->getParentOp(), configured.getOverlayAttr());
785 return configured.emitOpError(
"references an unknown overlay device");
787 if (failed(overlayCtl.setIOBackend(
false,
false)))
789 overlayCtl.startTransaction();
790 if (failed(overlayCtl.configureSwitches(overlay)))
792 auto overlayData = overlayCtl.exportSerializedTransaction();
793 std::vector<TransactionBinaryOperation> overlayOps;
797 for (
const auto &op : overlayOps) {
798 uint32_t &value = initialRegisters[op.cmd.RegOff];
799 if (op.cmd.Opcode == XAIE_IO_WRITE)
800 value = op.cmd.Value;
801 else if (op.cmd.Opcode == XAIE_IO_MASKWRITE)
802 value = (value & ~op.cmd.Mask) | (op.cmd.Value & op.cmd.Mask);
804 return overlay.emitOpError(
"has a non-register switch configuration");
807 return convertTransactionOpsToMLIR(builder, outputType, operations,
808 blockwriteData, initialRegisters);
812 OpBuilder &builder, xilinx::AIE::DeviceOp device,
822 bool xaieDebug =
false;
831 bool generateElfs =
true;
832 if (failed(generateTransactions(ctl, clElfDir, device, aieSim, generateElfs,
833 true,
true, skipCtrlPktOverlay)))
836 return insertRecordedTransaction(ctl, builder, outputType, blockwriteData,
837 skipCtrlPktOverlay ? device : DeviceOp{});
842using LockAddress = std::tuple<int32_t, int32_t, int32_t>;
843using LockAcquire = std::pair<LockAction, int32_t>;
845static std::optional<LockAddress> getLockAddress(UseLockOp use) {
846 auto lock = use.getLock().getDefiningOp<LockOp>();
849 auto lockId = lock.getLockID();
852 auto tile = lock.getTileOp();
853 return LockAddress{tile.colIndex(), tile.rowIndex(), *lockId};
856static std::optional<LockAcquire> getLockAcquire(UseLockOp use) {
858 if (!matchPattern(use.getValue(), m_ConstantInt(&value)))
860 return LockAcquire{use.getAction(), value.getSExtValue()};
863static CoreLockAcquiresAttr collectCoreLockAcquires(CoreOp core) {
864 if (core.getElfFile())
865 return core.getLockAcquiresAttr();
866 llvm::SetVector<LockAcquireAttr> acquires;
867 llvm::DenseSet<Operation *> visited;
868 SmallVector<Operation *> pending{core};
869 bool unknown =
false;
870 while (!pending.empty()) {
871 Operation *operation = pending.pop_back_val();
872 if (!visited.insert(operation).second)
874 operation->walk([&](UseLockOp use) {
877 auto address = getLockAddress(use);
878 auto acquire = getLockAcquire(use);
879 if (!address || !acquire) {
883 auto [column, row, lockId] = *address;
884 auto [action, value] = *acquire;
885 acquires.insert(LockAcquireAttr::get(core.getContext(), column, row,
886 lockId, action, value));
888 operation->walk([&](func::CallOp call) {
889 auto callee = SymbolTable::lookupNearestSymbolFrom<func::FuncOp>(
890 call, call.getCalleeAttr());
891 if (callee && !callee.isExternal())
892 pending.push_back(callee);
896 operation->walk([&](Operation *nested) {
897 if ((isa<CallOpInterface>(nested) && !isa<func::CallOp>(nested)) ||
898 isa<LLVM::CallIntrinsicOp, LLVM::InlineAsmOp>(nested))
902 return CoreLockAcquiresAttr::get(core.getContext(), !unknown,
903 acquires.getArrayRef());
906struct LockAcquireSummary {
907 std::set<LockAcquire> acquires;
908 SmallVector<CoreOp> cores;
909 bool unknown =
false;
911 std::optional<int32_t> getSatisfyingValue(uint32_t maximum)
const {
912 if (unknown || acquires.empty())
914 int32_t lowerBound = 0;
915 std::optional<int32_t> equality;
916 for (
auto [action, value] : acquires) {
917 if (value < 0 ||
static_cast<uint32_t
>(value) > maximum)
919 if (action == LockAction::AcquireGreaterEqual && value != 0) {
920 lowerBound = std::max(lowerBound, value);
922 if (equality && *equality != value)
927 if (equality && *equality < lowerBound)
929 return equality.value_or(lowerBound);
933struct CoreLockAcquireAnalysis {
934 std::map<LockAddress, LockAcquireSummary> locks;
935 llvm::DenseMap<CoreOp, std::set<LockAddress>> coreLocks;
936 llvm::DenseSet<CoreOp> unsupported;
938 explicit CoreLockAcquireAnalysis(DeviceOp device) {
939 const auto &model = device.getTargetModel();
940 for (
auto core : device.getOps<CoreOp>()) {
941 auto summary = collectCoreLockAcquires(core);
943 core.emitWarning(
"teardown cannot determine lock acquires for an ELF "
944 "without lock_acquires; skipping device "
946 unsupported.insert(core);
949 if (!summary.getComplete()) {
950 core.emitWarning(
"teardown cannot determine every lock acquire: "
951 "unresolved locks, dynamic values, or opaque callees; "
952 "skipping device lock draining");
953 unsupported.insert(core);
956 for (LockAcquireAttr acquire : summary.getAcquires()) {
957 LockAddress address{acquire.getColumn(), acquire.getRow(),
958 acquire.getLockId()};
959 auto &lock = locks[address];
960 lock.acquires.emplace(acquire.getAction(), acquire.getValue());
961 if (coreLocks[core].insert(address).second)
962 lock.cores.push_back(core);
965 if (!unsupported.empty()) {
966 for (
auto core : device.getOps<CoreOp>())
967 unsupported.insert(core);
970 device.walkOutsideRuntimeSequences<UseLockOp>([&](UseLockOp use) {
971 if (use->getParentOfType<CoreOp>() || use.release() || !use.getAcqEn())
973 auto address = getLockAddress(use);
976 auto found = locks.find(*address);
977 if (found == locks.end())
979 auto acquire = getLockAcquire(use);
981 found->second.unknown =
true;
983 found->second.acquires.insert(*acquire);
985 for (
auto &[address, lock] : locks) {
986 auto [column, row, lockId] = address;
987 bool competing = lock.cores.size() > 1;
988 bool incompatible = !lock.getSatisfyingValue(model.getMaxLockValue());
989 if (!competing && !incompatible)
991 for (
auto core : lock.cores) {
993 <<
"teardown cannot drain lock " << lockId <<
" on tile (" << column
994 <<
", " << row <<
"): "
995 << (competing ?
"multiple cores may consume the supplied count"
996 :
"no single writable value satisfies every acquire")
997 <<
"; skipping core lock draining";
998 unsupported.insert(core);
1005insertCoreLockDrain(OpBuilder &builder, DeviceOp device,
1008 const auto &model = device.getTargetModel();
1011 CoreLockAcquireAnalysis analysis(device);
1012 std::set<LockAddress> candidates;
1013 SmallVector<CoreOp> cores;
1014 for (
auto core : device.getOps<CoreOp>()) {
1015 auto found = analysis.coreLocks.find(core);
1016 if (analysis.unsupported.contains(core) ||
1017 found == analysis.coreLocks.end())
1019 candidates.insert(found->second.begin(), found->second.end());
1020 cores.push_back(core);
1022 auto tileAddress = [&](int32_t column, int32_t row) {
1023 return (
static_cast<uint32_t
>(column) << model.getColumnShift()) |
1024 (
static_cast<uint32_t
>(row) << model.getRowShift());
1026 auto pollCoreStatus = [&](CoreOp core, uint32_t maskValue) {
1027 auto tile = core.getTileOp();
1028 auto constant = [&](uint32_t value) {
1029 return arith::ConstantOp::create(builder, core.getLoc(),
1030 builder.getI32IntegerAttr(value));
1033 constant(tileAddress(tile.colIndex(), tile.rowIndex()) | 0x32004);
1034 Value zero = constant(0);
1035 Value mask = constant(maskValue);
1036 AIEX::NpuMaskPollOp::create(builder, core.getLoc(), status, zero, mask,
1037 nullptr,
nullptr,
nullptr);
1039 std::vector<TransactionBinaryOperation> writes;
1040 for (
auto address : candidates) {
1041 auto [column, row, lockId] = address;
1042 auto localAddress = model.getLocalLockAddress(lockId,
TileID{column, row});
1044 analysis.locks.at(address).getSatisfyingValue(model.getMaxLockValue());
1045 if (!localAddress || !value)
1046 return device.emitOpError(
1047 "teardown lock drain requires a valid lock address and value");
1049 write.
cmd.Opcode = XAie_TxnOpcode::XAIE_IO_WRITE;
1050 write.
cmd.RegOff = tileAddress(column, row) | *localAddress;
1051 write.
cmd.Value = *value;
1053 writes.push_back(write);
1055 if (!writes.empty())
1056 for (
auto core : device.getOps<CoreOp>())
1057 pollCoreStatus(core, 0x1);
1058 if (failed(convertTransactionOpsToMLIR(builder, outputType, writes,
1059 blockwriteData, {})))
1061 for (
auto core : cores)
1062 pollCoreStatus(core, 0x3c0);
1066struct AIEAnnotateCoreLockAcquiresPass
1067 : xilinx::impl::AIEAnnotateCoreLockAcquiresBase<
1068 AIEAnnotateCoreLockAcquiresPass> {
1069 void runOnOperation()
override {
1070 getOperation().walk([](CoreOp core) {
1071 if (
auto summary = collectCoreLockAcquires(core))
1072 core.setLockAcquiresAttr(summary);
1079std::unique_ptr<mlir::OperationPass<mlir::ModuleOp>>
1081 return std::make_unique<AIEAnnotateCoreLockAcquiresPass>();
1085 OpBuilder &builder, xilinx::AIE::DeviceOp device,
1101 insertRecordedTransaction(ctl, builder, outputType, blockwriteData,
1102 skipCtrlPktOverlay ? device : DeviceOp{})))
1104 return insertCoreLockDrain(builder, device, blockwriteData, outputType);
1112 std::array<llvm::DenseMap<DeviceOp, AIEX::BlockwriteData>, 2> blockwriteData;
1113 std::array<llvm::DenseMap<std::pair<Operation *, Operation *>,
1114 std::unique_ptr<Block>>,
1117 struct ConstantPool {
1118 Operation *last =
nullptr;
1119 llvm::DenseMap<Attribute, Value> values;
1121 llvm::DenseMap<Operation *, ConstantPool> pools;
1122 for (
auto [at, configured] : sites) {
1123 auto device = at->getParentOfType<DeviceOp>();
1125 return at->emitError(
"expected to be nested under an aie.device op");
1126 bool teardown = isa<AIEX::NpuWriteTeardownOp>(at);
1127 std::unique_ptr<Block> &config = configs[teardown][{configured, device}];
1130 blockwriteData[teardown]
1131 .try_emplace(device, device, teardown ?
"teardown_" :
"loadpdi_")
1133 auto generated = std::make_unique<Block>();
1134 OpBuilder configBuilder(at->getContext());
1135 configBuilder.setInsertionPointToEnd(generated.get());
1137 configBuilder, configured, payloads, outputType))
1139 configBuilder, configured, payloads,
1142 return at->emitError(teardown ?
"Failed to generate teardown operations"
1143 :
"Failed to generate configuration "
1145 config = std::move(generated);
1147 auto seq = at->getParentOfType<RuntimeSequenceOp>();
1148 ConstantPool *pool = seq ? &pools[seq] :
nullptr;
1149 OpBuilder builder(at);
1151 for (Operation &op : *config) {
1153 if (
auto constant = dyn_cast<arith::ConstantOp>(op))
1154 key = constant.getValue();
1155 else if (
auto global = dyn_cast<memref::GetGlobalOp>(op))
1156 key = global.getNameAttr();
1157 if (!pool || !key) {
1158 builder.clone(op, mapping);
1161 Value &pooled = pool->values[key];
1163 OpBuilder::InsertionGuard guard(builder);
1165 builder.setInsertionPointAfter(pool->last);
1167 builder.setInsertionPointToStart(&seq.getBody().front());
1168 pool->last = builder.clone(op);
1169 pooled = pool->last->getResult(0);
1171 mapping.map(op.getResult(0), pooled);
1178 SmallVector<std::pair<Operation *, DeviceOp>> sites;
1179 WalkResult walk =
module.walk([&](Operation *write) {
1180 FlatSymbolRefAttr ref;
1181 if (
auto config = dyn_cast<AIEX::NpuWriteConfigOp>(write))
1182 ref = config.getDeviceRefAttr();
1183 else if (
auto teardown = dyn_cast<AIEX::NpuWriteTeardownOp>(write))
1184 ref = teardown.getDeviceRefAttr();
1186 return WalkResult::advance();
1187 auto configured =
module.lookupSymbol<DeviceOp>(ref);
1189 write->emitOpError() << ref <<
" is not a device";
1190 return WalkResult::interrupt();
1192 sites.emplace_back(write, configured);
1193 return WalkResult::advance();
1195 if (walk.wasInterrupted() ||
1198 for (
auto [write, configured] : sites)
1204convertAIEToConfiguration(AIE::DeviceOp device, StringRef clElfDir,
1207 OpBuilder builder(device.getBodyRegion());
1212 auto loc = device.getLoc();
1213 SmallVector<AIEX::ConfigureOp> configureOps;
1214 device.walk([&](AIEX::ConfigureOp op) { configureOps.push_back(op); });
1216 if (configureOps.empty()) {
1219 std::string seq_name =
"configure";
1220 while (device.lookupSymbol(seq_name))
1221 seq_name =
"configure" + std::to_string(
id++);
1222 StringAttr seq_sym_name = builder.getStringAttr(seq_name);
1223 auto seq = AIE::RuntimeSequenceOp::create(
1224 builder, loc, seq_sym_name, BoolAttr{}, AIE::TraceBufferAttr{},
1225 ArrayAttr{}, ArrayAttr{});
1228 AIE::AIEDialect::GeneratedConfigureAttrHelper(device.getContext())
1229 .setAttr(seq, builder.getUnitAttr());
1230 seq.getBody().push_back(
new Block);
1231 builder.setInsertionPointToStart(&seq.getBody().front());
1233 builder.setInsertionPoint(configureOps.front());
1239 clElfDir, outputType)))
1244 if (!configureOps.empty()) {
1246 builder.getBlock()->getOperations().splice(
1247 builder.getInsertionPoint(),
1248 configureOps.front().getBody().front().getOperations());
1249 configureOps.front().erase();
1257template <
typename BaseClass, AIE::AIEToConfigurationOutputType MyOutputType>
1258struct ConvertAIEToConfigurationPass : BaseClass {
1259 std::string &ref_clElfDir;
1260 std::string &ref_clDeviceName;
1261 ConvertAIEToConfigurationPass(std::string &clElfDir,
1262 std::string &clDeviceName)
1263 : ref_clElfDir(clElfDir), ref_clDeviceName(clDeviceName) {}
1265 void getDependentDialects(DialectRegistry ®istry)
const override {
1266 registry.insert<memref::MemRefDialect, AIEX::AIEXDialect,
1267 arith::ArithDialect>();
1270 void runOnOperation()
override {
1271 AIE::DeviceOp deviceOp = BaseClass::getOperation();
1272 if (!ref_clDeviceName.empty() &&
1273 deviceOp.getSymName() != ref_clDeviceName) {
1277 convertAIEToConfiguration(deviceOp, ref_clElfDir, MyOutputType))) {
1278 return BaseClass::signalPassFailure();
1283struct ConvertAIEToTransactionPass
1284 : ConvertAIEToConfigurationPass<
1285 xilinx::impl::ConvertAIEToTransactionBase<
1286 ConvertAIEToTransactionPass>,
1287 AIE::AIEToConfigurationOutputType::Transaction> {
1288 ConvertAIEToTransactionPass()
1289 : ConvertAIEToConfigurationPass<
1290 xilinx::impl::ConvertAIEToTransactionBase<
1291 ConvertAIEToTransactionPass>,
1296struct ConvertAIEToControlPacketsPass
1297 : ConvertAIEToConfigurationPass<
1298 xilinx::impl::ConvertAIEToControlPacketsBase<
1299 ConvertAIEToControlPacketsPass>,
1300 AIE::AIEToConfigurationOutputType::ControlPacket> {
1301 ConvertAIEToControlPacketsPass()
1302 : ConvertAIEToConfigurationPass<
1303 xilinx::impl::ConvertAIEToControlPacketsBase<
1304 ConvertAIEToControlPacketsPass>,
1311std::unique_ptr<mlir::OperationPass<xilinx::AIE::DeviceOp>>
1313 return std::make_unique<ConvertAIEToTransactionPass>();
1316std::unique_ptr<mlir::OperationPass<xilinx::AIE::DeviceOp>>
1318 return std::make_unique<ConvertAIEToControlPacketsPass>();
memref::GlobalOp getOrCreate(OpBuilder &builder, mlir::Location loc, ArrayRef< uint32_t > words)
AIE::DeviceOp getDevice() const
bool hasProperty(ModelProperty Prop) const
mlir::Value createConstantArgPlus(mlir::OpBuilder &builder, mlir::Location loc, uint64_t value)
mlir::Value createConstantI32(mlir::OpBuilder &builder, mlir::Location loc, uint32_t value)
mlir::LogicalResult insertConfigOps(llvm::ArrayRef< std::pair< mlir::Operation *, xilinx::AIE::DeviceOp > > sites, AIEToConfigurationOutputType outputType)
mlir::LogicalResult inlineWriteConfigs(mlir::ModuleOp module)
std::optional< int > parseTransactionBinary(const std::vector< uint8_t > &data, std::vector< TransactionBinaryOperation > &ops)
Parse an exported transaction.
std::unique_ptr< mlir::OperationPass< xilinx::AIE::DeviceOp > > createConvertAIEToTransactionPass()
mlir::LogicalResult generateAndInsertConfigOps(mlir::OpBuilder &builder, xilinx::AIE::DeviceOp device, xilinx::AIEX::BlockwriteData &blockwriteData, llvm::StringRef clElfDir="", AIEToConfigurationOutputType outputType=AIEToConfigurationOutputType::Transaction, bool skipCtrlPktOverlay=false)
std::unique_ptr< mlir::OperationPass< mlir::ModuleOp > > createAIEAnnotateCoreLockAcquiresPass()
AIEToConfigurationOutputType
struct TileID { friend std::ostream &operator<<(std::ostream &os, const TileID &s) { os<< "TileID("<< s.col<< ", "<< s.row<< ")"; return os; } friend std::string to_string(const TileID &s) { std::ostringstream ss; ss<< s; return ss.str(); } friend llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const TileID &s) { os<< to_string(s); return os; } inline bool operator<(const TileID &rhs) const { return std::tie(col, row)< std::tie(rhs.col, rhs.row); } bool operator==(const TileID &rhs) const { return std::tie(col, row)==std::tie(rhs.col, rhs.row); } bool operator!=(const TileID &rhs) const { return !(*this==rhs);} int col, row;} TileID
mlir::LogicalResult generateAndInsertTeardownOps(mlir::OpBuilder &builder, xilinx::AIE::DeviceOp device, xilinx::AIEX::BlockwriteData &blockwriteData, AIEToConfigurationOutputType outputType=AIEToConfigurationOutputType::Transaction, bool skipCtrlPktOverlay=false)
std::optional< mlir::ModuleOp > convertTransactionBinaryToMLIR(mlir::MLIRContext *ctx, std::vector< uint8_t > &binary)
std::unique_ptr< mlir::OperationPass< xilinx::AIE::DeviceOp > > createConvertAIEToControlPacketsPass()
mlir::LogicalResult setIOBackend(bool aieSim, bool xaieDebug)
std::vector< uint8_t > exportSerializedTransaction()
mlir::LogicalResult disableCores(DeviceOp &targetOp)
mlir::LogicalResult addCoreEnable(DeviceOp &targetOp)
const std::vector< mlir::Location > & getTxnOpLocs() const
mlir::LogicalResult addInitConfig(DeviceOp &targetOp, bool skipCtrlPktOverlay=false)
mlir::LogicalResult resetDmaChannels(DeviceOp &targetOp)
mlir::LogicalResult addAieElfs(DeviceOp &targetOp, const mlir::StringRef elfPath, bool aieSim)
mlir::LogicalResult teardownSwitches(DeviceOp &targetOp, bool skipCtrlPktOverlay=false)
std::optional< LoadPdiPayload > loadPdi
std::optional< mlir::Location > sourceLoc
std::optional< SyncPayload > sync
std::optional< AddressPatchPayload > addressPatch