MLIR-AIE 1.4.6.dev
AIERT.cpp
Go to the documentation of this file.
1//===- AIERT.cpp ------------------------------------------------*- C++ -*-===//
2//
3// Copyright (C) 2024 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7
8#include "aie/Targets/AIERT.h"
10
11#include "mlir/Dialect/Utils/StaticValueUtils.h"
12#include "mlir/Support/LogicalResult.h"
13#include "llvm/ADT/SmallPtrSet.h"
14#include "llvm/ADT/SmallSet.h"
15
16extern "C" {
17#include "aie_codegen.h"
18}
19
20#include <algorithm>
21#include <filesystem>
22
23#define AIERC_STR(x) x, #x
24static const std::map<AieRC, std::string> AIERCTOSTR = {
25 {AIERC_STR(XAIE_OK)},
26 {AIERC_STR(XAIE_ERR)},
27 {AIERC_STR(XAIE_INVALID_DEVICE)},
28 {AIERC_STR(XAIE_INVALID_RANGE)},
29 {AIERC_STR(XAIE_INVALID_ARGS)},
30 {AIERC_STR(XAIE_INVALID_TILE)},
31 {AIERC_STR(XAIE_ERR_STREAM_PORT)},
32 {AIERC_STR(XAIE_INVALID_DMA_TILE)},
33 {AIERC_STR(XAIE_INVALID_BD_NUM)},
34 {AIERC_STR(XAIE_ERR_OUTOFBOUND)},
35 {AIERC_STR(XAIE_INVALID_DATA_MEM_ADDR)},
36 {AIERC_STR(XAIE_INVALID_ELF)},
37 {AIERC_STR(XAIE_CORE_STATUS_TIMEOUT)},
38 {AIERC_STR(XAIE_INVALID_CHANNEL_NUM)},
39 {AIERC_STR(XAIE_INVALID_LOCK)},
40 {AIERC_STR(XAIE_INVALID_DMA_DIRECTION)},
41 {AIERC_STR(XAIE_INVALID_PLIF_WIDTH)},
42 {AIERC_STR(XAIE_INVALID_LOCK_ID)},
43 {AIERC_STR(XAIE_INVALID_LOCK_VALUE)},
44 {AIERC_STR(XAIE_LOCK_RESULT_FAILED)},
45 {AIERC_STR(XAIE_INVALID_DMA_DESC)},
46 {AIERC_STR(XAIE_INVALID_ADDRESS)},
47 {AIERC_STR(XAIE_FEATURE_NOT_SUPPORTED)},
48 {AIERC_STR(XAIE_INVALID_BURST_LENGTH)},
49 {AIERC_STR(XAIE_INVALID_BACKEND)},
50 {AIERC_STR(XAIE_INSUFFICIENT_BUFFER_SIZE)},
51 {AIERC_STR(XAIE_ERR_MAX)}};
52#undef AIERC_STR
53
54static const std::map<xilinx::AIE::WireBundle, StrmSwPortType>
55 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE = {
56 {xilinx::AIE::WireBundle::Core, StrmSwPortType::CORE},
57 {xilinx::AIE::WireBundle::DMA, StrmSwPortType::DMA},
58 {xilinx::AIE::WireBundle::TileControl, StrmSwPortType::CTRL},
59 {xilinx::AIE::WireBundle::FIFO, StrmSwPortType::FIFO},
60 {xilinx::AIE::WireBundle::South, StrmSwPortType::SOUTH},
61 {xilinx::AIE::WireBundle::West, StrmSwPortType::WEST},
62 {xilinx::AIE::WireBundle::North, StrmSwPortType::NORTH},
63 {xilinx::AIE::WireBundle::East, StrmSwPortType::EAST},
64 // missing PLIO from WireBundle
65 // missing NOC from WireBundle
66 {xilinx::AIE::WireBundle::Trace, StrmSwPortType::TRACE},
67};
68
69#ifndef NDEBUG
70
71// https://stackoverflow.com/a/32230306
72template <typename H1>
73static llvm::raw_ostream &showAIEXRTArgs(llvm::raw_ostream &out,
74 const char *label, H1 &&value) {
75 return out << label << "=" << std::forward<H1>(value);
76}
77
78template <typename H1, typename... T>
79static llvm::raw_ostream &showAIEXRTArgs(llvm::raw_ostream &out,
80 const char *label, H1 &&value,
81 T &&...rest) {
82 const char *pcomma = strchr(label, ',');
83 return showAIEXRTArgs(out.write(label, pcomma - label)
84 << "=" << std::forward<H1>(value) << ',',
85 pcomma + 1, std::forward<T>(rest)...);
86}
87
88static llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
89 const XAie_LocType &loc);
90
91static llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
92 const XAie_Lock &lock);
93
94static llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
95 const XAie_Packet &packet);
96
97#define SHOW_AIERT_ARGS(os, ...) showAIEXRTArgs(os, #__VA_ARGS__, __VA_ARGS__)
98
99// So that we can use the pattern if(auto r = TRY_XAIE_API...) { // r is nonzero
100// }
101static_assert(XAIE_OK == 0);
102
103#define TRY_XAIE_API_FATAL_ERROR(API, ...) \
104 do { \
105 LLVM_DEBUG(llvm::dbgs() << "trying XAIE API: " << #API << " with args: "); \
106 LLVM_DEBUG(SHOW_AIERT_ARGS(llvm::dbgs(), __VA_ARGS__)); \
107 LLVM_DEBUG(llvm::dbgs() << "\n"); \
108 if (auto r = API(__VA_ARGS__)) \
109 llvm::report_fatal_error(llvm::Twine(#API " failed with ") + \
110 AIERCTOSTR.at(r)); \
111 } while (0)
112
113#define TRY_XAIE_API_EMIT_ERROR(OP, API, ...) \
114 do { \
115 LLVM_DEBUG(llvm::dbgs() << "trying XAIE API: " << #API << " with args: "); \
116 LLVM_DEBUG(SHOW_AIERT_ARGS(llvm::dbgs(), __VA_ARGS__)); \
117 LLVM_DEBUG(llvm::dbgs() << "\n"); \
118 if (auto r = API(__VA_ARGS__)) \
119 return OP.emitOpError() << #API " failed with " << AIERCTOSTR.at(r); \
120 } while (0)
121
122#define TRY_XAIE_API_LOGICAL_RESULT(API, ...) \
123 do { \
124 LLVM_DEBUG(llvm::dbgs() << "trying XAIE API: " << #API << " with args: "); \
125 LLVM_DEBUG(SHOW_AIERT_ARGS(llvm::dbgs(), __VA_ARGS__)); \
126 LLVM_DEBUG(llvm::dbgs() << "\n"); \
127 if (auto r = API(__VA_ARGS__)) { \
128 llvm::errs() << #API " failed with " << AIERCTOSTR.at(r); \
129 return failure(); \
130 } \
131 } while (0)
132
133#else
134
135#define TRY_XAIE_API_FATAL_ERROR(API, ...) \
136 do { \
137 if (auto r = API(__VA_ARGS__)) \
138 llvm::report_fatal_error(llvm::Twine(#API " failed with ") + \
139 AIERCTOSTR.at(r)); \
140 } while (0)
141
142#define TRY_XAIE_API_EMIT_ERROR(OP, API, ...) \
143 do { \
144 if (auto r = API(__VA_ARGS__)) \
145 return OP.emitOpError() << #API " failed with " << AIERCTOSTR.at(r); \
146 } while (0)
147
148#define TRY_XAIE_API_LOGICAL_RESULT(API, ...) \
149 do { \
150 if (auto r = API(__VA_ARGS__)) { \
151 llvm::errs() << #API " failed with " << AIERCTOSTR.at(r); \
152 return failure(); \
153 } \
154 } while (0)
155
156#endif
157
158using namespace mlir;
159using namespace xilinx;
160
161#define DEBUG_TYPE "aie-aiert"
162
163llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const XAie_LocType &loc) {
164 os << "XAie_LocType(col: " << std::to_string(loc.Col)
165 << ", row: " << std::to_string(loc.Row) << ")";
166 return os;
167}
168
169llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const XAie_Lock &lock) {
170 os << "XAie_Lock(id: " << std::to_string(lock.LockId)
171 << ", val: " << std::to_string(lock.LockVal) << ")";
172 return os;
173}
174
175llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
176 const XAie_Packet &packet) {
177 os << "XAie_Packet(id: " << std::to_string(packet.PktId)
178 << ", type: " << std::to_string(packet.PktType) << ")";
179 return os;
180}
181
182#define XAIE_BASE_ADDR 0x40000000
183#define XAIE_SHIM_ROW 0
184#define XAIE_MEM_TILE_ROW_START 1
185#define XAIE_PARTITION_BASE_ADDR 0x0
186
187#define NPI_ADDR 0x0
188#define NUM_LOCKS 16
189#define EVEN_BD_NUM_START 0
190#define ODD_BD_NUM_START 24
191
193 XAie_Config configPtr;
194 XAie_DevInst devInst;
195
196 // One Location per recorded XAie_TxnCmd, indexed by position in
197 // XAie_TxnInst::CmdBuf. Populated by TxnLocBracket scopes.
198 std::vector<mlir::Location> txnCmdLocs;
199
200 // One Location per operation in the exported transaction binary. Derived
201 // from txnCmdLocs by exportSerializedTransaction().
202 std::vector<mlir::Location> txnOpLocs;
203};
204
205// Walk DevInst->TxnList to find the active XAie_TxnInst (single-threaded
206// compilation: at most one). Returns null if no transaction is being
207// recorded. The XAie_TxnInst struct and TxnList layout are part of
208// aie-codegen's public xaiegbl.h, so this only depends on the public header.
209static XAie_TxnInst *getActiveTxnInst(XAie_DevInst &devInst) {
210 XAie_List *node = devInst.TxnList.Next;
211 if (!node)
212 return nullptr;
213 return reinterpret_cast<XAie_TxnInst *>(reinterpret_cast<char *>(node) -
214 offsetof(XAie_TxnInst, Node));
215}
216
217static uint32_t getActiveTxnNumCmds(XAie_DevInst &devInst) {
218 XAie_TxnInst *inst = getActiveTxnInst(devInst);
219 return inst ? inst->NumCmds : 0;
220}
221
223 : ctl(c), loc(l), startCmds(c.getCurrentTxnNumCmds()) {}
224
226 uint32_t endCmds = ctl.getCurrentTxnNumCmds();
227 if (endCmds <= startCmds)
228 return;
229 // Only record meaningful locations. Falling through (no entry) lets the
230 // downstream emitter use its fallback (device.getLoc()), which preserves
231 // the pre-existing behavior for source ops that don't carry a real loc.
232 if (mlir::isa<mlir::UnknownLoc>(loc))
233 return;
234 ctl.recordTxnLocRange(startCmds, endCmds, loc);
235}
236
237xilinx::AIE::AIERTControl::~AIERTControl() { XAie_Finish(&aiert->devInst); }
238
240 : targetModel(tm), aiert(std::make_unique<AIERtImpl>()) {
241 // The first column in the NPU lacks a shim tile. AIE-RT exposes some of
242 // the internals about how this is modeled in a somewhat awkward way.
243 size_t partitionStartCol =
245 size_t partitionNumCols = tm.columns();
246 size_t deviceRows = tm.rows();
247 size_t deviceCols = tm.columns() + partitionStartCol;
248
249 // Don't put this in the target model, because it's XAIE specific.
250 unsigned char devGen = XAIE_DEV_GEN_AIE;
251 switch (tm.getTargetArch()) {
252 case AIEArch::AIE1: // probably unreachable.
253 devGen = XAIE_DEV_GEN_AIE;
254 break;
255 case AIEArch::AIE2:
256 // FIXME: What if we don't have an IPU? aie-codegen
257 // models non-IPU devices differently.
258 devGen = XAIE_DEV_GEN_AIE2IPU;
259 break;
260 case AIEArch::AIE2p:
261 devGen = XAIE_DEV_GEN_AIE2P_STRIX_B0;
262 break;
263 case AIEArch::AIE2ps:
264 // AIE2PS configures the array through a CERT ELF instead of CDO or an
265 // xclbin (see AIE2PSTargetModel), so it has no aie-codegen device
266 // generation. Listed here so that a new AIEArch still raises -Wswitch.
267 llvm::report_fatal_error(
268 "aie-codegen configuration is not supported for the "
269 "AIE2PS architecture");
270 }
271 aiert->configPtr = XAie_Config{
272 /*AieGen*/ devGen,
273 /*BaseAddr*/ XAIE_BASE_ADDR,
274 /*ColShift*/ static_cast<uint8_t>(tm.getColumnShift()),
275 /*RowShift*/ static_cast<uint8_t>(tm.getRowShift()),
276 /*NumRows*/ static_cast<uint8_t>(deviceRows),
277 /*NumCols*/ static_cast<uint8_t>(deviceCols),
278 /*ShimRowNum*/ XAIE_SHIM_ROW,
279 /*MemTileRowStart*/ XAIE_MEM_TILE_ROW_START,
280 /*MemTileNumRows*/ static_cast<uint8_t>(tm.getNumMemTileRows()),
281 /*AieTileRowStart*/
282 static_cast<uint8_t>(XAIE_MEM_TILE_ROW_START + tm.getNumMemTileRows()),
283 /*AieTileNumRows*/
284 static_cast<uint8_t>(tm.rows() - tm.getNumMemTileRows() - 1),
285 /*PartProp*/ {},
286 /*ShimTileNumRowsNorth*/ 0,
287 /*ShimTileNumRowsSouth*/ 0};
288 XAie_InstDeclare(_devInst, &aiert->configPtr);
289 aiert->devInst = _devInst;
290 TRY_XAIE_API_FATAL_ERROR(XAie_SetupPartitionConfig, &aiert->devInst,
291 XAIE_PARTITION_BASE_ADDR, partitionStartCol,
292 partitionNumCols);
293 TRY_XAIE_API_FATAL_ERROR(XAie_CfgInitialize, &aiert->devInst,
294 &aiert->configPtr);
295 TRY_XAIE_API_FATAL_ERROR(XAie_UpdateNpiAddr, &aiert->devInst, NPI_ADDR);
296}
297
299 bool xaieDebug) {
300 // Quoting: The instance of a device must be always declared using this
301 // macro. In the future, the same macro will be expanded to
302 // allocate more memory from the user application for resource
303 // management.
304 if (aieSim) {
305 TRY_XAIE_API_FATAL_ERROR(XAie_SetIOBackend, &aiert->devInst,
306 XAIE_IO_BACKEND_SIM);
307 } else if (xaieDebug)
308 TRY_XAIE_API_FATAL_ERROR(XAie_SetIOBackend, &aiert->devInst,
309 XAIE_IO_BACKEND_DEBUG);
310 else
311 TRY_XAIE_API_FATAL_ERROR(XAie_SetIOBackend, &aiert->devInst,
312 XAIE_IO_BACKEND_CDO);
313 return success();
314}
315
316static LogicalResult
317configureLocksInBdBlock(const AIE::AIETargetModel &targetModel,
318 XAie_DmaDesc &dmaTileBd, Block &block, int col, int row,
319 bool outOfOrder) {
320 LLVM_DEBUG(llvm::dbgs() << "\nstart configuring bds\n");
321 AIE::UseLockOp acquire, release;
322 if (failed(AIE::verifyBdLockPair(block, outOfOrder, acquire, release)))
323 return failure();
324 std::optional<int> acqValue, relValue, acqLockId, relLockId;
325 bool acqEn = false;
326
327 // A memtile BD may use locks of its neighbours, so each lock ID is offset
328 // by the base index of the tile that owns that lock.
329 auto lockIdFor = [&](AIE::LockOp lock) {
330 int id = lock.getLockIDValue();
331 if (targetModel.isMemTile(col, row))
332 if (auto offset = targetModel.getLockLocalBaseIndex(
333 col, row, lock.colIndex(), lock.rowIndex()))
334 id += *offset;
335 return id;
336 };
337 for (auto op : block.getOps<AIE::UseLockOp>()) {
338 auto lock = cast<AIE::LockOp>(op.getLock().getDefiningOp());
339 switch (op.getAction()) {
340 case AIE::LockAction::Acquire:
341 case AIE::LockAction::AcquireGreaterEqual: {
342 acqEn = op.getAcqEn();
343 acqLockId = lockIdFor(lock);
344 auto value = op.getConstantValue();
345 if (failed(value))
346 return failure();
347 int32_t v = *value; // NOLINT(bugprone-unchecked-optional-access)
348 if (op.acquireGE())
349 v = -v;
350 acqValue = v;
351 break;
352 }
353 case AIE::LockAction::Release: {
354 relLockId = lockIdFor(lock);
355 auto value = op.getConstantValue();
356 if (failed(value))
357 return failure();
358 relValue = value;
359 break;
360 }
361 }
362 }
363
364 // no RelEn in the arch spec even though the API requires you to set it?
365 bool relEn = false;
366 if (outOfOrder && !acqLockId) {
367 // Release-only out-of-order BD: point the disabled acquire lock at the
368 // release lock so XAie_LockInit gets a valid id.
369 acqLockId = relLockId;
370 acqValue = 0;
371 }
372 assert(acqLockId && acqValue && relLockId && relValue &&
373 "lock values must be resolved before init");
374 XAie_Lock acqLock = XAie_LockInit(acqLockId.value(), acqValue.value());
375 XAie_Lock relLock = XAie_LockInit(relLockId.value(), relValue.value());
376 TRY_XAIE_API_EMIT_ERROR((*block.getOps<AIE::UseLockOp>().begin()),
377 dmaTileBd.DmaMod->SetLock, &dmaTileBd, acqLock,
378 relLock, acqEn, relEn);
379 return success();
380}
381
382static LogicalResult configureBdInBlock(const AIE::AIETargetModel &targetModel,
383 XAie_DevInst *devInst,
384 XAie_DmaDesc &dmaTileBd, Block &block,
385 int col, int row, int bdId,
386 std::optional<int> nextBdId) {
387 std::optional<int> packetType;
388 std::optional<int> packetID;
389
390 // Below should go
391 auto maybePacketOps = block.getOps<AIE::DMABDPACKETOp>();
392 if (!maybePacketOps.empty()) {
393 assert(llvm::range_size(maybePacketOps) == 1 &&
394 "expected only one dma_bd_packet");
395 auto packetOp = *maybePacketOps.begin();
396 packetType = packetOp.getPacketType();
397 packetID = packetOp.getPacketID();
398 }
399
400 auto bdOp = *block.getOps<AIE::DMABDOp>().begin();
401
402 if (targetModel.isShimNOCTile(col, row)) {
403 // write them out like this so they show up with names in debug prints
404 uint8_t smid = 0;
405 uint32_t burstLen =
406 getShimBurstLengthBytes(targetModel, bdOp.getBurstLength());
407 uint8_t qOs = 0;
408 uint8_t cache = static_cast<uint8_t>(bdOp.getAxcacheOrDefault());
409 uint8_t secure = 0;
410 TRY_XAIE_API_EMIT_ERROR(bdOp, XAie_DmaSetAxi, &dmaTileBd, smid,
411 burstLen / 16, qOs, cache, secure);
412 }
413
414 // get address from BufferOp (core,mem) or ExternalBufferOp (shim)
415 uint64_t baseAddr = 0;
416 if (targetModel.isShimNOCTile(col, row)) {
417 auto bufferOp =
418 cast<AIE::ExternalBufferOp>(bdOp.getBuffer().getDefiningOp());
419 // external buffers aren't required to have an address here because the
420 // address might get patched later or the default of zero might be a valid
421 // address.
422 if (auto addr = bufferOp.getAddress())
423 baseAddr = *addr; // NOLINT(bugprone-unchecked-optional-access)
424 } else {
425 auto bufferOp = cast<AIE::BufferOp>(bdOp.getBuffer().getDefiningOp());
426 auto addr = bufferOp.getAddress();
427 if (!addr)
428 return bufferOp.emitError("buffer must have address assigned");
429 baseAddr = *addr; // NOLINT(bugprone-unchecked-optional-access)
430 }
431
432 if (targetModel.isMemTile(col, row)) {
433 // check if buffer is allocated on the same memtile, the west, or the east
434 // one
435 auto bufferOp = cast<AIE::BufferOp>(bdOp.getBuffer().getDefiningOp());
436 auto bufferRow = bufferOp.getTileOp().getRow();
437 auto bufferCol = bufferOp.getTileOp().getCol();
438 auto addrOffset =
439 targetModel.getMemLocalBaseAddress(col, row, bufferCol, bufferRow);
440 if (addrOffset)
441 baseAddr += addrOffset.value();
442 }
443
444 // Runtime-valued sizes/strides are not supported on this static lowering
445 // path.
446 for (mlir::OpFoldResult s : bdOp.getMixedSizes())
447 if (!mlir::getConstantIntValue(s))
448 return bdOp->emitOpError(
449 "runtime-valued BD size/stride is not supported on the static XAIE "
450 "lowering path; use compile-time constant sizes");
451 for (mlir::OpFoldResult s : bdOp.getMixedStrides())
452 if (!mlir::getConstantIntValue(s))
453 return bdOp->emitOpError(
454 "runtime-valued BD size/stride is not supported on the static XAIE "
455 "lowering path; use compile-time constant sizes");
456 // The owning storage must outlive the ArrayRef used below.
457 std::optional<llvm::SmallVector<AIE::BDDimLayoutAttr>> dimsStorage =
458 bdOp.getConstantDimensions();
459 if (!dimsStorage)
460 return bdOp->emitOpError("internal error folding BD dimensions");
461 std::optional<llvm::ArrayRef<AIE::BDDimLayoutAttr>> dims;
462 if (!dimsStorage->empty())
463 dims = llvm::ArrayRef<AIE::BDDimLayoutAttr>(*dimsStorage);
464 uint64_t lenInBytes = bdOp.getLenInBytes();
465 uint64_t basePlusOffsetInBytes = baseAddr + bdOp.getOffsetInBytes();
466 if (!dims) {
467 TRY_XAIE_API_EMIT_ERROR(bdOp, XAie_DmaSetAddrLen, &dmaTileBd,
468 basePlusOffsetInBytes, lenInBytes);
469 } else {
470 llvm::SmallVector<XAie_DmaDimDesc, 4> dimDescs(dims->size());
471 XAie_DmaTensor dmaTileBdTensor = {};
472 dmaTileBdTensor.NumDim = dimDescs.size();
473 dmaTileBdTensor.Dim = dimDescs.data();
474 // libxaie requires stride in multiples of 32b
475 double elementWidthIn32bWords =
476 static_cast<double>(bdOp.getBufferElementTypeWidthInBytes()) / 4.0;
477 for (size_t i = 0; i < dims->size(); i++) {
478 // Pass down dimensions in reverse order; in the MLIR, this allows
479 // us to specify step sizes/wraps in the same order as we would
480 // access a multi-dim C array, with the highest dimension first.
481 int j = dims->size() - i - 1;
482 uint16_t size;
483 uint32_t stride;
484 if (j > 0) {
485 stride = static_cast<uint32_t>(dims.value()[i].getStride() *
486 elementWidthIn32bWords);
487 size = dims.value()[i].getSize();
488 } else {
489 stride = dims.value()[i].getStride();
490 size = static_cast<uint16_t>(dims.value()[i].getSize() *
491 elementWidthIn32bWords);
492 }
493 stride = stride > 0 ? stride : 1;
494 // Assume AIE-ML architecture (ie use AieMlDimDesc instead of AieDimDesc);
495 // asserted in AIETranslateToCDODirect).
496 dmaTileBdTensor.Dim[j].AieMlDimDesc = {stride, size};
497 }
498 TRY_XAIE_API_EMIT_ERROR(bdOp, XAie_DmaSetMultiDimAddr, &dmaTileBd,
499 &dmaTileBdTensor, basePlusOffsetInBytes,
500 lenInBytes);
501 }
502
503 // BD iteration.
504 if (auto iter = bdOp.getIteration(); iter && iter->getSize() > 1) {
505 if (iter->getStride() > 0) {
506 double elementWidthIn32bWords =
507 static_cast<double>(bdOp.getBufferElementTypeWidthInBytes()) / 4.0;
508 uint32_t stepInWords =
509 static_cast<uint32_t>(iter->getStride() * elementWidthIn32bWords);
510 uint8_t wrap = static_cast<uint8_t>(iter->getSize());
511 uint8_t iterCurr = static_cast<uint8_t>(iter->getCurrent());
512 TRY_XAIE_API_EMIT_ERROR(bdOp, XAie_DmaSetBdIteration, &dmaTileBd,
513 stepInWords, wrap, iterCurr);
514 }
515 }
516
517 // ND zero padding.
518 std::optional<llvm::ArrayRef<AIE::BDPadLayoutAttr>> padDims =
519 bdOp.getPadDimensions();
520
521 if (padDims) {
522 // The verifier leaves the dims past the tile's pad fields unpadded.
523 llvm::SmallVector<XAie_PadDesc, 3> padDescs(
524 std::min(padDims->size(),
525 targetModel.getDmaBdLayout(col, row)->padFields().size()));
526 XAie_DmaPadTensor dmaPadTensor = {};
527 dmaPadTensor.NumDim = padDescs.size();
528 dmaPadTensor.PadDesc = padDescs.data();
529 // libxaie requires stride in multiples of 32b
530 double elementWidthIn32bWords =
531 static_cast<double>(bdOp.getBufferElementTypeWidthInBytes()) / 4.0;
532 for (size_t i = 0; i < padDims->size(); i++) {
533 // Pass down dimensions in reverse order.
534 int j = padDims->size() - i - 1;
535 if (j >= static_cast<int>(padDescs.size()))
536 continue;
537 uint8_t before;
538 uint8_t after;
539 if (j > 0) {
540 before = static_cast<uint8_t>(padDims.value()[i].getConstPadBefore());
541 after = static_cast<uint8_t>(padDims.value()[i].getConstPadAfter());
542 } else {
543 before = static_cast<uint8_t>(padDims.value()[i].getConstPadBefore() *
544 elementWidthIn32bWords);
545 after = static_cast<uint8_t>(padDims.value()[i].getConstPadAfter() *
546 elementWidthIn32bWords);
547 }
548 dmaPadTensor.PadDesc[j] = {before, after};
549 }
550 TRY_XAIE_API_EMIT_ERROR(bdOp, XAie_DmaSetPadding, &dmaTileBd,
551 &dmaPadTensor);
552 }
553 if (nextBdId) {
554 auto enableNextBd = 1;
555 TRY_XAIE_API_EMIT_ERROR(bdOp, XAie_DmaSetNextBd, &dmaTileBd,
556 nextBdId.value(), enableNextBd);
557 }
558
559 if (auto packetInfo = bdOp.getPacket()) {
560 packetType = packetInfo->getPktType();
561 packetID = packetInfo->assignedId();
562 }
563
564 if (packetID) {
565 if (!packetType)
566 return bdOp.emitError("must have packetType with packetID");
567 // getConstantLen() is nullopt for a runtime len operand, so this guard only
568 // catches a compile-time-zero length. A runtime len that happens to be 0 is
569 // not diagnosed here (runtime len is not yet reachable on this static
570 // path).
571 if (bdOp.getConstantLen() == 0)
572 return bdOp.emitOpError(
573 "For MM2S channels, if Buffer_Length=0 then Enable_Packet must be "
574 "set to 0, otherwise behavior is undefined (3.7.8 arch spec)");
575 TRY_XAIE_API_EMIT_ERROR(bdOp, XAie_DmaSetPkt, &dmaTileBd,
576 XAie_PacketInit(*packetID, *packetType));
577 }
578 if (std::optional<int32_t> oooId = bdOp.getOutOfOrderId())
579 TRY_XAIE_API_EMIT_ERROR(bdOp, XAie_DmaSetOutofOrderBdId, &dmaTileBd,
580 static_cast<uint8_t>(*oooId));
581 TRY_XAIE_API_EMIT_ERROR(bdOp, XAie_DmaEnableBd, &dmaTileBd);
582 auto tileLoc = XAie_TileLoc(col, row);
583 TRY_XAIE_API_EMIT_ERROR(bdOp, XAie_DmaWriteBd, devInst, &dmaTileBd, tileLoc,
584 bdId);
585 LLVM_DEBUG(llvm::dbgs() << "\nend configuring bds\n");
586 return success();
587};
588
590 Operation &op, int col, int row, int chNum, const DMAChannelDir &channelDir,
591 int bdId, int repeatCount, uint32_t padValue, bool outOfOrder,
592 bool configureChannel, bool pushTask) {
593 TxnLocBracket bracket(*this, op.getLoc());
594 XAie_DmaDirection direction =
595 channelDir == DMAChannelDir::S2MM ? DMA_S2MM : DMA_MM2S;
596 auto tileLoc = XAie_TileLoc(col, row);
597 if (configureChannel && padValue != 0 && direction == DMA_MM2S &&
598 targetModel.isMemTile(col, row)) {
599 TRY_XAIE_API_EMIT_ERROR(op, XAie_DmaSetPadValue, &aiert->devInst, tileLoc,
600 static_cast<uint8_t>(chNum), padValue);
601 }
602 // in english repeat_count==0 means "do it once" and don't repeat but
603 // libxaie treats repeat_count=1 as do it once.
604 repeatCount += 1;
605 if (outOfOrder) {
606 if (configureChannel) {
607 XAie_DmaChannelDesc dmaChannelDesc;
608 TRY_XAIE_API_EMIT_ERROR(op, XAie_DmaChannelDescInit, &aiert->devInst,
609 &dmaChannelDesc, tileLoc);
610 TRY_XAIE_API_EMIT_ERROR(op, XAie_DmaChannelEnOutofOrder, &dmaChannelDesc,
611 static_cast<uint8_t>(XAIE_ENABLE));
612 TRY_XAIE_API_EMIT_ERROR(op, XAie_DmaWriteChannel, &aiert->devInst,
613 &dmaChannelDesc, tileLoc,
614 static_cast<uint8_t>(chNum), direction);
615 }
616 // StartBd=0 (ignored) and skip the BD/channel validity check.
617 // Disable TCT because the bd lookup id aliases the token bits.
618 XAie_DmaDeclareQueueConfig(startQueueCfg, /*StartBd=*/0, repeatCount,
619 /*EnToken=*/static_cast<uint8_t>(XAIE_DISABLE),
620 static_cast<uint8_t>(XAIE_ENABLE));
621 if (pushTask)
622 TRY_XAIE_API_EMIT_ERROR(op, XAie_DmaChannelSetStartQueueGeneric,
623 &aiert->devInst, tileLoc, chNum, direction,
624 &startQueueCfg);
625 } else if (pushTask) {
626 bool enTokenIssue = tileLoc.Row == 0 && direction == DMA_S2MM;
627 TRY_XAIE_API_EMIT_ERROR(op, XAie_DmaChannelSetStartQueue, &aiert->devInst,
628 tileLoc, chNum, direction, bdId, repeatCount,
629 enTokenIssue);
630 }
631 if (configureChannel)
632 TRY_XAIE_API_EMIT_ERROR(op, XAie_DmaChannelEnable, &aiert->devInst, tileLoc,
633 chNum, direction);
634 return success();
635};
636
638 int col, int row,
639 bool outOfOrder) {
640 DMABDOp bd = *block.getOps<DMABDOp>().begin();
641 auto bdId = bd.getBdId();
642 assert(bdId.has_value() &&
643 "DMABDOp must have assigned bd_id; did you forget to run "
644 "aie-assign-bd-ids?");
645 TxnLocBracket bracket(*this, bd.getLoc());
646 XAie_DmaDesc dmaTileBd;
647 auto tileLoc = XAie_TileLoc(col, row);
648 TRY_XAIE_API_EMIT_ERROR(bd, XAie_DmaDescInit, &aiert->devInst, &dmaTileBd,
649 tileLoc);
650 if (!block.getOps<UseLockOp>().empty() &&
651 failed(configureLocksInBdBlock(targetModel, dmaTileBd, block, col, row,
652 outOfOrder)))
653 return failure();
654 // Out-of-order ignores Use_Next_BD; the IR chain exists only so every
655 // receive BD gets configured.
656 std::optional<int> nextBdId =
657 outOfOrder ? std::optional<int>() : bd.getNextBdId();
658 if (!block.getOps<DMABDOp>().empty() &&
659 failed(configureBdInBlock(
660 targetModel, &aiert->devInst, dmaTileBd, block, col, row,
661 *bdId, // NOLINT(bugprone-unchecked-optional-access)
662 nextBdId)))
663 return failure();
664 return success();
665}
666
667LogicalResult xilinx::AIE::AIERTControl::initLocks(DeviceOp &targetOp) {
668 for (auto tileOp : targetOp.getOps<TileOp>()) {
669 auto tileLoc = XAie_TileLoc(tileOp.colIndex(), tileOp.rowIndex());
670 if (!tileOp.isShimTile() && tileOp.getCoreOp()) {
671 TxnLocBracket bracket(*this, tileOp.getLoc());
672 TRY_XAIE_API_EMIT_ERROR(tileOp, XAie_CoreReset, &aiert->devInst, tileLoc);
673 TRY_XAIE_API_EMIT_ERROR(tileOp, XAie_CoreUnreset, &aiert->devInst,
674 tileLoc);
675 // Set locks to zero
676 for (uint8_t l = 0; l < NUM_LOCKS; l++) {
677 auto locInit = XAie_LockInit(l, 0);
678 TRY_XAIE_API_EMIT_ERROR(tileOp, XAie_LockSetValue, &aiert->devInst,
679 tileLoc, locInit);
680 }
681 }
682 }
683
684 // Set locks with explicit initializers
685 targetOp.walkOutsideRuntimeSequences<LockOp>([&](LockOp lockOp) {
686 if (lockOp.getLockID() && lockOp.getInit()) {
687 TxnLocBracket bracket(*this, lockOp.getLoc());
688 auto tileLoc = XAie_TileLoc(lockOp.getTileOp().colIndex(),
689 lockOp.getTileOp().rowIndex());
690 auto locInit = XAie_LockInit(*lockOp.getLockID(), *lockOp.getInit());
691 TRY_XAIE_API_FATAL_ERROR(XAie_LockSetValue, &aiert->devInst, tileLoc,
692 locInit);
693 } else
694 LLVM_DEBUG(llvm::dbgs()
695 << "lock op missing either id or init" << lockOp << "\n");
696 });
697 return success();
698}
699
700LogicalResult xilinx::AIE::AIERTControl::initBuffers(DeviceOp &targetOp) {
701 // Set buffers with explicit initializers
702 targetOp.walkOutsideRuntimeSequences<BufferOp>([&](BufferOp bufferOp) {
703 auto initialValue = bufferOp.getInitialValue();
704 if (!initialValue)
705 return;
706 TxnLocBracket bracket(*this, bufferOp.getLoc());
707 mlir::DenseElementsAttr denseInit =
708 dyn_cast<mlir::DenseElementsAttr>(initialValue.value());
709 if (!denseInit)
710 return;
711 auto tileLoc = XAie_TileLoc(bufferOp.getTileOp().colIndex(),
712 bufferOp.getTileOp().rowIndex());
713 std::optional<std::vector<char>> byteVec = denseAttrToBytes(denseInit);
714 if (!byteVec) {
715 llvm::outs() << "buffer op type not supported for initialization "
716 << bufferOp << "\n";
717 return;
718 }
719 TRY_XAIE_API_FATAL_ERROR(XAie_DataMemBlockWrite, &aiert->devInst, tileLoc,
720 bufferOp.getAddress().value(), byteVec->data(),
721 byteVec->size());
722 });
723 return success();
724}
725
726LogicalResult
728 bool skipCtrlPktOverlay) {
729 AIEDialect::IsCtrlPktOverlayAttrHelper overlay(targetOp.getContext());
730
731 // StreamSwitch (switchbox) configuration
732 for (auto switchboxOp : targetOp.getOps<SwitchboxOp>()) {
733 int32_t col = switchboxOp.colIndex();
734 int32_t row = switchboxOp.rowIndex();
735 XAie_LocType tileLoc = XAie_TileLoc(col, row);
736 assert(targetModel.hasProperty(AIETargetModel::IsNPU) &&
737 "Only NPU currently supported");
738
739 Block &b = switchboxOp.getConnections().front();
740 for (auto connectOp : b.getOps<ConnectOp>()) {
741 if (skipCtrlPktOverlay && overlay.isAttrPresent(connectOp))
742 continue;
743 TxnLocBracket bracket(*this, connectOp.getLoc());
745 switchboxOp, XAie_StrmConnCctEnable, &aiert->devInst, tileLoc,
746 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(connectOp.getSourceBundle()),
747 connectOp.sourceIndex(),
748 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(connectOp.getDestBundle()),
749 connectOp.destIndex());
750 }
751
752 for (auto masterSetOp : b.getOps<MasterSetOp>()) {
753 if (skipCtrlPktOverlay && overlay.isAttrPresent(masterSetOp))
754 continue;
755 TxnLocBracket bracket(*this, masterSetOp.getLoc());
756 int mask = 0;
757 int arbiter = -1;
758
759 // MasterSetOp::verify guarantees every amsel names the same arbiter, so
760 // folding them into one arbiter plus msel mask is safe here.
761 for (auto val : masterSetOp.getAmsels()) {
762 AMSelOp amsel = cast<AMSelOp>(val.getDefiningOp());
763 arbiter = amsel.arbiterIndex();
764 int msel = amsel.getMselValue();
765 mask |= (1 << msel);
766 }
767
768 auto dropHeader = masterSetOp.keepsPktHeader()
769 ? XAIE_SS_PKT_DONOT_DROP_HEADER
770 : XAIE_SS_PKT_DROP_HEADER;
772 masterSetOp, XAie_StrmPktSwMstrPortEnable, &aiert->devInst, tileLoc,
773 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(masterSetOp.getDestBundle()),
774 masterSetOp.destIndex(), dropHeader, arbiter, mask);
775 }
776
777 for (auto packetRulesOp : b.getOps<PacketRulesOp>()) {
778 if (skipCtrlPktOverlay && overlay.isAttrPresent(packetRulesOp))
779 continue;
780 TxnLocBracket bracket(*this, packetRulesOp.getLoc());
781 int slot = 0;
782 Block &block = packetRulesOp.getRules().front();
783 // The overlay's rules take the first slots of a port they share with the
784 // design's rules, and the overlay already enabled the port.
785 bool overlayPort =
786 skipCtrlPktOverlay &&
787 llvm::any_of(block.getOps<PacketRuleOp>(), [&](PacketRuleOp rule) {
788 return overlay.isAttrPresent(rule);
789 });
790 for (auto slotOp : block.getOps<PacketRuleOp>()) {
791 if (skipCtrlPktOverlay && overlay.isAttrPresent(slotOp)) {
792 slot++;
793 continue;
794 }
795 AMSelOp amselOp = cast<AMSelOp>(slotOp.getAmsel().getDefiningOp());
796 int arbiter = amselOp.arbiterIndex();
797 int msel = amselOp.getMselValue();
798 if (!overlayPort)
799 TRY_XAIE_API_EMIT_ERROR(packetRulesOp, XAie_StrmPktSwSlavePortEnable,
800 &aiert->devInst, tileLoc,
801 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(
802 packetRulesOp.getSourceBundle()),
803 packetRulesOp.sourceIndex());
804 auto packetInit = XAie_PacketInit(slotOp.valueInt(), /*PktType*/ 0);
805 // TODO Need to better define packet id,type used here
806 TRY_XAIE_API_EMIT_ERROR(packetRulesOp, XAie_StrmPktSwSlaveSlotEnable,
807 &aiert->devInst, tileLoc,
808 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(
809 packetRulesOp.getSourceBundle()),
810 packetRulesOp.sourceIndex(), slot, packetInit,
811 slotOp.maskInt(), msel, arbiter);
812 slot++;
813 }
814 }
815 }
816
817 for (auto muxOp : targetOp.getOps<ShimMuxOp>()) {
818 // NOTE ShimMux always connects from the south as directions are
819 // defined relative to the tile stream switch.
820 auto tileLoc =
821 XAie_TileLoc(muxOp.getTileOp().getCol(), muxOp.getTileOp().getRow());
822 Block &b = muxOp.getConnections().front();
823 for (auto connectOp : b.getOps<ConnectOp>()) {
824 if (skipCtrlPktOverlay && overlay.isAttrPresent(connectOp))
825 continue;
826 TxnLocBracket bracket(*this, connectOp.getLoc());
827 // demux!
828 if (connectOp.getSourceBundle() == WireBundle::North)
829 TRY_XAIE_API_EMIT_ERROR(muxOp, XAie_EnableAieToShimDmaStrmPort,
830 &aiert->devInst, tileLoc,
831 connectOp.sourceIndex());
832 // mux
833 if (connectOp.getDestBundle() == WireBundle::North)
834 TRY_XAIE_API_EMIT_ERROR(muxOp, XAie_EnableShimDmaToAieStrmPort,
835 &aiert->devInst, tileLoc,
836 connectOp.destIndex());
837 }
838 }
839
840 // Cascade configuration
841 if (isa<AIE2TargetModel>(targetModel)) {
842 for (auto configOp : targetOp.getOps<ConfigureCascadeOp>()) {
843 TxnLocBracket bracket(*this, configOp.getLoc());
844 TileOp tile = cast<TileOp>(configOp.getTile().getDefiningOp());
845 auto tileLoc = XAie_TileLoc(tile.getCol(), tile.getRow());
847 targetOp, XAie_CoreConfigAccumulatorControl, &aiert->devInst, tileLoc,
848 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(
849 static_cast<WireBundle>(configOp.getInputDir())),
850 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(
851 static_cast<WireBundle>(configOp.getOutputDir())));
852 }
853 }
854
855 return success();
856}
857
858LogicalResult
860 bool skipCtrlPktOverlay) {
861 AIEDialect::IsCtrlPktOverlayAttrHelper overlay(targetOp.getContext());
862 auto skip = [&](Operation *op) {
863 return skipCtrlPktOverlay && overlay.isAttrPresent(op);
864 };
865 for (auto switchboxOp : targetOp.getOps<SwitchboxOp>()) {
866 XAie_LocType tileLoc =
867 XAie_TileLoc(switchboxOp.colIndex(), switchboxOp.rowIndex());
868 Block &b = switchboxOp.getConnections().front();
869 for (auto connectOp : b.getOps<ConnectOp>()) {
870 if (skip(connectOp))
871 continue;
872 TxnLocBracket bracket(*this, connectOp.getLoc());
874 switchboxOp, XAie_StrmConnCctDisable, &aiert->devInst, tileLoc,
875 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(connectOp.getSourceBundle()),
876 connectOp.sourceIndex(),
877 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(connectOp.getDestBundle()),
878 connectOp.destIndex());
879 }
880 for (auto masterSetOp : b.getOps<MasterSetOp>()) {
881 if (skip(masterSetOp))
882 continue;
883 TxnLocBracket bracket(*this, masterSetOp.getLoc());
885 masterSetOp, XAie_StrmPktSwMstrPortDisable, &aiert->devInst, tileLoc,
886 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(masterSetOp.getDestBundle()),
887 masterSetOp.destIndex());
888 }
889 for (auto packetRulesOp : b.getOps<PacketRulesOp>()) {
890 auto rules = packetRulesOp.getRules().front().getOps<PacketRuleOp>();
891 if (rules.empty() || skip(packetRulesOp))
892 continue;
893 TxnLocBracket bracket(*this, packetRulesOp.getLoc());
894 StrmSwPortType port =
895 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(packetRulesOp.getSourceBundle());
896 if (llvm::none_of(rules, skip))
897 TRY_XAIE_API_EMIT_ERROR(packetRulesOp, XAie_StrmPktSwSlavePortDisable,
898 &aiert->devInst, tileLoc, port,
899 packetRulesOp.sourceIndex());
900 // A slot stays enabled when its port is disabled.
901 for (auto [slot, rule] : llvm::enumerate(rules))
902 if (!skip(rule))
903 TRY_XAIE_API_EMIT_ERROR(packetRulesOp, XAie_StrmPktSwSlaveSlotDisable,
904 &aiert->devInst, tileLoc, port,
905 packetRulesOp.sourceIndex(), slot);
906 }
907 }
908 return success();
909}
910
911LogicalResult xilinx::AIE::AIERTControl::disableCores(DeviceOp &targetOp) {
912 for (auto coreOp : targetOp.getOps<CoreOp>()) {
913 TileID tile = coreOp.getTileID();
914 TxnLocBracket bracket(*this, coreOp.getLoc());
915 TRY_XAIE_API_EMIT_ERROR(coreOp, XAie_CoreDisable, &aiert->devInst,
916 XAie_TileLoc(tile.col, tile.row));
917 }
918 return success();
919}
920
921LogicalResult xilinx::AIE::AIERTControl::resetDmaChannels(DeviceOp &targetOp) {
922 auto memOps = llvm::to_vector_of<TileElement>(targetOp.getOps<MemOp>());
923 llvm::append_range(memOps, targetOp.getOps<MemTileDMAOp>());
924 for (TileElement memOp : memOps) {
925 TileID tile = memOp.getTileID();
926 XAie_LocType tileLoc = XAie_TileLoc(tile.col, tile.row);
927 llvm::SmallSet<std::pair<DMAChannelDir, int>, 4> reset;
928 auto resetChannel = [&](Operation *op, DMAChannelDir dir,
929 int channel) -> LogicalResult {
930 if (!reset.insert({dir, channel}).second)
931 return success();
932 TxnLocBracket bracket(*this, op->getLoc());
933 XAie_DmaDirection direction =
934 dir == DMAChannelDir::S2MM ? DMA_S2MM : DMA_MM2S;
935 TRY_XAIE_API_EMIT_ERROR((*op), XAie_DmaChannelReset, &aiert->devInst,
936 tileLoc, channel, direction,
937 XAie_DmaChReset::DMA_CHANNEL_RESET);
938 // Writing a fresh descriptor ends the reset and returns every field of
939 // the control register, such as out-of-order mode, to its reset value.
940 XAie_DmaChannelDesc defaults;
941 TRY_XAIE_API_EMIT_ERROR((*op), XAie_DmaChannelDescInit, &aiert->devInst,
942 &defaults, tileLoc);
943 TRY_XAIE_API_EMIT_ERROR((*op), XAie_DmaWriteChannel, &aiert->devInst,
944 &defaults, tileLoc, channel, direction);
945 return success();
946 };
947 Region &region = memOp.getOperation()->getRegion(0);
948 for (auto dmaOp : region.getOps<DMAOp>())
949 if (failed(resetChannel(dmaOp, dmaOp.getChannelDir(),
950 dmaOp.getChannelIndex())))
951 return failure();
952 for (Block *block : getOrderedChainOfBlocks(&region))
953 for (auto startOp : block->getOps<DMAStartOp>())
954 if (failed(resetChannel(startOp, startOp.getChannelDir(),
955 startOp.getChannelIndex())))
956 return failure();
957 }
958 return success();
959}
960
961LogicalResult
963 bool skipCtrlPktOverlay) {
964
965 if (failed(verifyDMAChannelsResolved(targetOp)))
966 return failure();
967
968 if (failed(initLocks(targetOp))) {
969 return failure();
970 }
971
972 if (failed(initBuffers(targetOp))) {
973 return failure();
974 }
975
976 // A DMA whose lock is already satisfied streams as soon as it is enabled, so
977 // its route must exist first.
978 if (failed(configureSwitches(targetOp, skipCtrlPktOverlay))) {
979 return failure();
980 }
981
982 auto memOps = llvm::to_vector_of<TileElement>(targetOp.getOps<MemOp>());
983 llvm::append_range(memOps, targetOp.getOps<MemTileDMAOp>());
984 llvm::append_range(memOps, targetOp.getOps<ShimDMAOp>());
985 for (TileElement memOp : memOps) {
986 int col = memOp.getTileID().col;
987 int row = memOp.getTileID().row;
988
989 // Get the region's entry block, then start traversing through the chain of
990 // blocks.
991 llvm::SetVector<Block *> blockVector =
992 getOrderedChainOfBlocks(&memOp.getOperation()->getRegion(0));
993
994 llvm::SmallPtrSet<Block *, 8> unchainedBlocks =
995 collectUnchainedBlocks(blockVector);
996
997 // handle DMA ops separately
998 auto dmaOps = llvm::to_vector_of<DMAOp>(
999 memOp.getOperation()->getRegion(0).getOps<DMAOp>());
1000 if (!dmaOps.empty()) {
1001 for (auto dmaOp : dmaOps)
1002 for (auto &bdRegion : dmaOp.getBds()) {
1003 Block &block = bdRegion.getBlocks().front();
1004 if (failed(
1005 configureLocksAndBd(block, col, row, dmaOp.getOutOfOrder())))
1006 return failure();
1007 }
1008 } else {
1009 for (Block *block : blockVector) {
1010 if (block->getOps<DMABDOp>().empty())
1011 continue;
1012 if (failed(configureLocksAndBd(*block, col, row,
1013 unchainedBlocks.contains(block))))
1014 return failure();
1015 }
1016 }
1017
1018 if (!dmaOps.empty())
1019 for (auto dmaOp : dmaOps) {
1020 auto &block = dmaOp.getBds().front().getBlocks().front();
1021 DMABDOp bd = *block.getOps<DMABDOp>().begin();
1022 auto bdId = bd.getBdId();
1023 assert(bdId.has_value() &&
1024 "bd_id assigned by configureLocksAndBd above");
1025 if (failed(pushToBdQueueAndEnable(
1026 *dmaOp.getOperation(), col, row, dmaOp.getChannelIndex(),
1027 dmaOp.getChannelDir(), bdId.value(), dmaOp.getRepeatCount(),
1028 dmaOp.getPadValue(), dmaOp.getOutOfOrder())))
1029 return failure();
1030 }
1031 else
1032 for (Block *block : blockVector) {
1033 for (auto op : block->getOps<DMAStartOp>()) {
1034 DMABDOp bd = *op.getDest()->getOps<DMABDOp>().begin();
1035 int chNum = op.getChannelIndex();
1036 auto channelDir = op.getChannelDir();
1037 auto bdId = bd.getBdId();
1038 assert(bdId.has_value() &&
1039 "bd_id assigned by configureLocksAndBd above");
1040 if (failed(pushToBdQueueAndEnable(
1041 *bd.getOperation(), col, row, chNum, channelDir, bdId.value(),
1042 op.getRepeatCount(), op.getPadValue(), op.getOutOfOrder(),
1043 /*configureChannel=*/true,
1044 /*pushTask=*/!op.getCorePushes())))
1045 return failure();
1046 }
1047 }
1048 }
1049
1050 return success();
1051}
1052
1053LogicalResult xilinx::AIE::AIERTControl::addCoreEnable(DeviceOp &targetOp) {
1054 // Start execution of all the cores.
1055 for (auto tileOp : targetOp.getOps<TileOp>()) {
1056 auto tileLoc = XAie_TileLoc(tileOp.colIndex(), tileOp.rowIndex());
1057 if (!tileOp.isShimTile() && tileOp.getCoreOp()) {
1058 CoreOp coreOp = tileOp.getCoreOp();
1059 TxnLocBracket bracket(*this, coreOp.getLoc());
1060 // A core reaching its own tile's registers goes through the processor
1061 // bus, which starts disabled: an access made before this stalls the
1062 // core. Only the cores that ask for it pay for it.
1063 if (coreOp.getProcessorBus())
1064 TRY_XAIE_API_EMIT_ERROR(targetOp, XAie_CoreProcessorBusEnable,
1065 &aiert->devInst, tileLoc);
1066 TRY_XAIE_API_EMIT_ERROR(targetOp, XAie_CoreEnable, &aiert->devInst,
1067 tileLoc);
1068 }
1069 }
1070 return success();
1071}
1072
1073LogicalResult xilinx::AIE::AIERTControl::addAieElf(uint8_t col, uint8_t row,
1074 const StringRef elfPath,
1075 bool aieSim) {
1076 TRY_XAIE_API_LOGICAL_RESULT(XAie_CoreDisable, &aiert->devInst,
1077 XAie_TileLoc(col, row));
1078 TRY_XAIE_API_LOGICAL_RESULT(XAie_DmaChannelResetAll, &aiert->devInst,
1079 XAie_TileLoc(col, row),
1080 XAie_DmaChReset::DMA_CHANNEL_RESET);
1081
1082 // loadSym: Load symbols from .map file. This argument is not used when
1083 // __AIESIM__ is not defined.
1084 TRY_XAIE_API_LOGICAL_RESULT(XAie_LoadElf, &aiert->devInst,
1085 XAie_TileLoc(col, row), elfPath.str().c_str(),
1086 /*loadSym*/ aieSim);
1087
1088 TRY_XAIE_API_LOGICAL_RESULT(XAie_DmaChannelResetAll, &aiert->devInst,
1089 XAie_TileLoc(col, row),
1090 XAie_DmaChReset::DMA_CHANNEL_UNRESET);
1091
1092 return success();
1093}
1094
1096 TRY_XAIE_API_LOGICAL_RESULT(XAie_ResetPartition, &aiert->devInst);
1097 return success();
1098}
1099
1100LogicalResult xilinx::AIE::AIERTControl::resetDMA(int col, int row, bool on) {
1101 auto tileLoc = XAie_TileLoc(col, row);
1102 XAie_DmaDesc dmaTileBd;
1103 TRY_XAIE_API_LOGICAL_RESULT(XAie_DmaDescInit, &aiert->devInst, &dmaTileBd,
1104 tileLoc);
1105 TRY_XAIE_API_LOGICAL_RESULT(XAie_DmaDisableBd, &dmaTileBd);
1106 TRY_XAIE_API_LOGICAL_RESULT(XAie_DmaChannelResetAll, &aiert->devInst, tileLoc,
1107 on ? XAie_DmaChReset::DMA_CHANNEL_UNRESET
1108 : XAie_DmaChReset::DMA_CHANNEL_RESET);
1109 return success();
1110}
1111
1112LogicalResult xilinx::AIE::AIERTControl::resetCore(int col, int row) {
1113 auto tileLoc = XAie_TileLoc(col, row);
1114 TRY_XAIE_API_LOGICAL_RESULT(XAie_CoreReset, &aiert->devInst, tileLoc);
1115 return success();
1116}
1117
1118LogicalResult xilinx::AIE::AIERTControl::resetSwitch(int col, int row) {
1119 XAie_LocType tileLoc = XAie_TileLoc(col, row);
1120
1121 // Reset all combinations of input/output routing in the switchbox
1122 for (auto endpoint_a : WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE) {
1123 for (auto endpoint_b : WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE) {
1124 unsigned n_a_connections = targetModel.getNumSourceSwitchboxConnections(
1125 col, row, endpoint_a.first);
1126 unsigned n_b_connections = targetModel.getNumDestSwitchboxConnections(
1127 col, row, endpoint_b.first);
1128 for (unsigned a_index = 0; a_index < n_a_connections; a_index++) {
1129 for (unsigned b_index = 0; b_index < n_b_connections; b_index++) {
1130 if (!targetModel.isLegalTileConnection(col, row, endpoint_a.first,
1131 a_index, endpoint_b.first,
1132 b_index)) {
1133 continue;
1134 }
1135 TRY_XAIE_API_FATAL_ERROR(XAie_StrmConnCctDisable, &aiert->devInst,
1136 tileLoc, endpoint_a.second, a_index,
1137 endpoint_b.second, b_index);
1138 }
1139 }
1140 }
1141 }
1142
1143 return success();
1144}
1145
1146LogicalResult xilinx::AIE::AIERTControl::resetCoreUnreset(int col, int row) {
1147 auto tileLoc = XAie_TileLoc(col, row);
1148 TRY_XAIE_API_LOGICAL_RESULT(XAie_CoreUnreset, &aiert->devInst, tileLoc);
1149 return success();
1150}
1151
1152LogicalResult xilinx::AIE::AIERTControl::resetLock(int col, int row,
1153 int lockId) {
1154 auto tileLoc = XAie_TileLoc(col, row);
1155 // Reset a single lock to value 0
1156 XAie_Lock lock;
1157 lock.LockId = lockId;
1158 lock.LockVal = 0;
1159 TRY_XAIE_API_LOGICAL_RESULT(XAie_LockSetValue, &aiert->devInst, tileLoc,
1160 lock);
1161 return success();
1162}
1163
1165 int col, int row, WireBundle sourceBundle, int sourceChannel,
1166 WireBundle destBundle, int destChannel) {
1167 auto tileLoc = XAie_TileLoc(col, row);
1168
1169 // Helper lambda to map WireBundle to StrmSwPortType
1170 auto mapBundle = [](WireBundle bundle) -> StrmSwPortType {
1171 switch (bundle) {
1172 case WireBundle::Core:
1173 return CORE;
1174 case WireBundle::DMA:
1175 return DMA;
1176 case WireBundle::FIFO:
1177 return FIFO;
1178 case WireBundle::South:
1179 return SOUTH;
1180 case WireBundle::West:
1181 return WEST;
1182 case WireBundle::North:
1183 return NORTH;
1184 case WireBundle::East:
1185 return EAST;
1186 case WireBundle::Trace:
1187 return TRACE;
1188 default:
1189 return SOUTH;
1190 }
1191 };
1192
1193 StrmSwPortType sourcePortType = mapBundle(sourceBundle);
1194 StrmSwPortType destPortType = mapBundle(destBundle);
1195
1196 // Disconnect the specific connection from source to destination
1197 TRY_XAIE_API_LOGICAL_RESULT(XAie_StrmConnCctDisable, &aiert->devInst, tileLoc,
1198 sourcePortType, sourceChannel, destPortType,
1199 destChannel);
1200
1201 return success();
1202}
1203
1204LogicalResult xilinx::AIE::AIERTControl::resetPerfCounters(int col, int row) {
1205 auto tileLoc = XAie_TileLoc(col, row);
1206 // Reset performance counters in all modules
1207 // Try core module counters (if applicable)
1208 for (int counterId = 0; counterId < 4; counterId++) {
1209 // Ignore errors as not all tiles have all counter types
1210 (void)XAie_PerfCounterReset(&aiert->devInst, tileLoc, XAIE_CORE_MOD,
1211 counterId);
1212 }
1213 // Try mem module counters
1214 for (int counterId = 0; counterId < 4; counterId++) {
1215 (void)XAie_PerfCounterReset(&aiert->devInst, tileLoc, XAIE_MEM_MOD,
1216 counterId);
1217 }
1218 // Try PL module counters (for shim tiles)
1219 for (int counterId = 0; counterId < 4; counterId++) {
1220 (void)XAie_PerfCounterReset(&aiert->devInst, tileLoc, XAIE_PL_MOD,
1221 counterId);
1222 }
1223 return success();
1224}
1225
1226LogicalResult xilinx::AIE::AIERTControl::addAieElfs(DeviceOp &targetOp,
1227 const StringRef elfPath,
1228 bool aieSim) {
1229 for (auto tileOp : targetOp.getOps<TileOp>())
1230 if (tileOp.isShimNOCorPLTile()) {
1231 // Resets no needed with V2 kernel driver
1232 } else {
1233 int col = tileOp.colIndex();
1234 int row = tileOp.rowIndex();
1235 if (auto coreOp = tileOp.getCoreOp()) {
1236 std::string fileName;
1237 if (auto fileAttr = coreOp.getElfFile()) {
1238 fileName = fileAttr->str();
1239 } else {
1240 // Skip cores without elf_file (e.g., lightweight reset
1241 // devices that only need DMA/lock reconfiguration).
1242 continue;
1243 }
1244 // Check if fileName is already an absolute path.
1245 // If so, use it directly. Otherwise, concatenate with elfPath.
1246 std::string fullPath;
1247 if (std::filesystem::path(fileName).is_absolute()) {
1248 fullPath = fileName;
1249 } else {
1250 auto ps = std::filesystem::path::preferred_separator;
1251 fullPath =
1252 (llvm::Twine(elfPath) + std::string(1, ps) + fileName).str();
1253 }
1254 if (failed(addAieElf(col, row, fullPath, aieSim)))
1255 return failure();
1256 }
1257 }
1258 return success();
1259}
1260
1261void xilinx::AIE::AIERTControl::dmaUpdateBdAddr(int col, int row, size_t addr,
1262 size_t bdId) {
1263 auto tileLoc = XAie_TileLoc(col, row);
1264 TRY_XAIE_API_FATAL_ERROR(XAie_DmaUpdateBdAddr, &aiert->devInst, tileLoc, addr,
1265 bdId);
1266}
1267
1269 aiert->txnCmdLocs.clear();
1270 aiert->txnOpLocs.clear();
1271 TRY_XAIE_API_FATAL_ERROR(XAie_StartTransaction, &aiert->devInst,
1272 XAIE_TRANSACTION_DISABLE_AUTO_FLUSH);
1273}
1274
1275// Project the per-XAie_TxnCmd locations in `cmdLocs` onto the operations
1276// aie-codegen's serializer actually emits, mirroring
1277// _XAie_TxnExportSerialized's command-to-operation mapping. That mapping is not
1278// 1:1: the serializer buffers block writes and collapses a run of
1279// address-contiguous XAIE_IO_BLOCKWRITE commands into a single block-write
1280// operation, holds that pending run across intervening DDR-patch commands (so a
1281// patch is emitted *before* the block write it follows in CmdBuf), and drops
1282// XAIE_IO_LOAD_PM_END_INTERNAL entirely. Everything else emits one operation
1283// in command order.
1284static std::vector<mlir::Location>
1285projectCmdLocsOntoSerializedOps(const XAie_TxnInst &inst,
1286 const std::vector<mlir::Location> &cmdLocs) {
1287 std::vector<mlir::Location> opLocs;
1288 opLocs.reserve(inst.NumCmds);
1289
1290 mlir::Location unknown = mlir::UnknownLoc::get(cmdLocs.front().getContext());
1291 bool pendingBlockWrite = false;
1292 mlir::Location pendingBlockWriteLoc = unknown;
1293 uint64_t blockWriteEndOff = 0;
1294
1295 for (uint32_t i = 0; i < inst.NumCmds; ++i) {
1296 const XAie_TxnCmd &cmd = inst.CmdBuf[i];
1297 mlir::Location loc = i < cmdLocs.size() ? cmdLocs[i] : unknown;
1298
1299 if (cmd.Opcode == XAIE_IO_BLOCKWRITE) {
1300 // A command that extends the run in flight emits nothing of its own;
1301 // the run keeps the location of the command that started it.
1302 if (!pendingBlockWrite || cmd.RegOff != blockWriteEndOff) {
1303 if (pendingBlockWrite)
1304 opLocs.push_back(pendingBlockWriteLoc);
1305 pendingBlockWrite = true;
1306 pendingBlockWriteLoc = loc;
1307 }
1308 blockWriteEndOff = cmd.RegOff + cmd.Size * sizeof(uint32_t);
1309 continue;
1310 }
1311
1312 if (cmd.Opcode == XAIE_IO_LOAD_PM_END_INTERNAL)
1313 continue;
1314
1315 if (pendingBlockWrite && cmd.Opcode != XAIE_IO_CUSTOM_OP_DDR_PATCH) {
1316 opLocs.push_back(pendingBlockWriteLoc);
1317 pendingBlockWrite = false;
1318 }
1319 opLocs.push_back(loc);
1320 }
1321 if (pendingBlockWrite)
1322 opLocs.push_back(pendingBlockWriteLoc);
1323
1324 return opLocs;
1325}
1326
1328 // Export the transactions to a binary buffer
1329 uint8_t *txn_ptr = XAie_ExportSerializedTransaction(&aiert->devInst, 0, 0);
1330 XAie_TxnHeader *hdr = (XAie_TxnHeader *)txn_ptr;
1331 std::vector<uint8_t> txn_data(txn_ptr, txn_ptr + hdr->TxnSize);
1332 free(txn_ptr);
1333
1334 // Exporting leaves the recorded commands intact, so the loc projection can
1335 // still walk CmdBuf here. With nothing bracketed there is no location to
1336 // project and every emitted op falls back to the device location.
1337 XAie_TxnInst *inst = getActiveTxnInst(aiert->devInst);
1338 if (inst && !aiert->txnCmdLocs.empty())
1339 aiert->txnOpLocs =
1340 projectCmdLocsOntoSerializedOps(*inst, aiert->txnCmdLocs);
1341
1342 return txn_data;
1343}
1344
1345const std::vector<mlir::Location> &
1347 return aiert->txnOpLocs;
1348}
1349
1351 return getActiveTxnNumCmds(aiert->devInst);
1352}
1353
1355 uint32_t endCmds,
1356 mlir::Location loc) {
1357 if (endCmds <= startCmds)
1358 return;
1359 if (aiert->txnCmdLocs.size() < endCmds)
1360 aiert->txnCmdLocs.resize(endCmds, mlir::UnknownLoc::get(loc.getContext()));
1361 for (uint32_t i = startCmds; i < endCmds; ++i)
1362 aiert->txnCmdLocs[i] = loc;
1363}
#define XAIE_PARTITION_BASE_ADDR
Definition AIERT.cpp:185
#define TRY_XAIE_API_LOGICAL_RESULT(API,...)
Definition AIERT.cpp:122
#define NPI_ADDR
Definition AIERT.cpp:187
#define NUM_LOCKS
Definition AIERT.cpp:188
#define TRY_XAIE_API_FATAL_ERROR(API,...)
Definition AIERT.cpp:103
#define TRY_XAIE_API_EMIT_ERROR(OP, API,...)
Definition AIERT.cpp:113
#define AIERC_STR(x)
Definition AIERT.cpp:23
std::optional< uint32_t > getMemLocalBaseAddress(int localCol, int localRow, int memCol, int memRow) const
Return the memory base address (or offset) in the local tile when accessing a neighbor's memory or an...
virtual const DmaBdLayout * getDmaBdLayout(AIETileType tileType) const
Return the register layout of a buffer descriptor on the given tile type, or null if the target has n...
virtual AIEArch getTargetArch() const =0
Return the target architecture.
bool isMemTile(int col, int row) const
Return true if the given tile is a Mem tile.
std::optional< uint32_t > getLockLocalBaseIndex(int localCol, int localRow, int lockCol, int lockRow) const
Return the lock base index (or offset) in the local tile when accessing a neighbor's lock or an empty...
virtual int rows() const =0
Return the number of rows in the device.
bool isShimNOCTile(int col, int row) const
Return true if the given tile is a ShimNOC tile.
bool hasProperty(ModelProperty Prop) const
virtual int columns() const =0
Return the number of columns in the device.
TxnLocBracket(AIERTControl &ctl, mlir::Location loc)
Definition AIERT.cpp:222
uint32_t getShimBurstLengthBytes(const AIE::AIETargetModel &tm, uint32_t burstLength)
mlir::LogicalResult verifyBdLockPair(mlir::Block &block, bool outOfOrder, UseLockOp &acquire, UseLockOp &release)
struct TileID { friend std::ostream &operator<<(std::ostream &os, const TileID &s) { os<< "TileID("<< s.col<< ", "<< s.row<< ")"; return os; } friend std::string to_string(const TileID &s) { std::ostringstream ss; ss<< s; return ss.str(); } friend llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const TileID &s) { os<< to_string(s); return os; } inline bool operator<(const TileID &rhs) const { return std::tie(col, row)< std::tie(rhs.col, rhs.row); } bool operator==(const TileID &rhs) const { return std::tie(col, row)==std::tie(rhs.col, rhs.row); } bool operator!=(const TileID &rhs) const { return !(*this==rhs);} int col, row;} TileID
mlir::LogicalResult verifyDMAChannelsResolved(DeviceOp device)
std::optional< std::vector< char > > denseAttrToBytes(mlir::DenseElementsAttr denseInit)
A buffer's initial_value as the bytes to write to its tile memory, in element order.
llvm::SmallPtrSet< mlir::Block *, 8 > collectUnchainedBlocks(const llvm::SetVector< mlir::Block * > &blockVector)
Collect BD blocks that require use_next_bd=0 during configuration.
llvm::SetVector< mlir::Block * > getOrderedChainOfBlocks(mlir::Region *region)
std::vector< mlir::Location > txnCmdLocs
Definition AIERT.cpp:198
std::vector< mlir::Location > txnOpLocs
Definition AIERT.cpp:202
mlir::LogicalResult configureSwitches(DeviceOp &targetOp, bool skipCtrlPktOverlay=false)
Definition AIERT.cpp:727
mlir::LogicalResult resetPerfCounters(int col, int row)
Definition AIERT.cpp:1204
mlir::LogicalResult configureLocksAndBd(mlir::Block &block, int col, int row, bool outOfOrder=false)
Definition AIERT.cpp:637
mlir::LogicalResult resetCore(int col, int row)
Definition AIERT.cpp:1112
void dmaUpdateBdAddr(int col, int row, size_t addr, size_t bdId)
Definition AIERT.cpp:1261
mlir::LogicalResult resetPartition()
Definition AIERT.cpp:1095
mlir::LogicalResult resetCoreUnreset(int col, int row)
Definition AIERT.cpp:1146
mlir::LogicalResult setIOBackend(bool aieSim, bool xaieDebug)
Definition AIERT.cpp:298
uint32_t getCurrentTxnNumCmds() const
Definition AIERT.cpp:1350
std::vector< uint8_t > exportSerializedTransaction()
Definition AIERT.cpp:1327
AIERTControl(const xilinx::AIE::AIETargetModel &tm)
Definition AIERT.cpp:239
mlir::LogicalResult initLocks(DeviceOp &targetOp)
Definition AIERT.cpp:667
mlir::LogicalResult resetDMA(int col, int row, bool on)
Definition AIERT.cpp:1100
mlir::LogicalResult disableCores(DeviceOp &targetOp)
Definition AIERT.cpp:911
mlir::LogicalResult initBuffers(DeviceOp &targetOp)
Definition AIERT.cpp:700
mlir::LogicalResult resetSwitch(int col, int row)
Definition AIERT.cpp:1118
mlir::LogicalResult addCoreEnable(DeviceOp &targetOp)
Definition AIERT.cpp:1053
mlir::LogicalResult resetSwitchConnection(int col, int row, WireBundle sourceBundle, int sourceChannel, WireBundle destBundle, int destChannel)
Definition AIERT.cpp:1164
const std::vector< mlir::Location > & getTxnOpLocs() const
Definition AIERT.cpp:1346
mlir::LogicalResult addInitConfig(DeviceOp &targetOp, bool skipCtrlPktOverlay=false)
Definition AIERT.cpp:962
mlir::LogicalResult addAieElf(uint8_t col, uint8_t row, const mlir::StringRef elfPath, bool aieSim)
Definition AIERT.cpp:1073
void recordTxnLocRange(uint32_t startCmds, uint32_t endCmds, mlir::Location loc)
Definition AIERT.cpp:1354
mlir::LogicalResult resetDmaChannels(DeviceOp &targetOp)
Definition AIERT.cpp:921
mlir::LogicalResult pushToBdQueueAndEnable(mlir::Operation &op, int col, int row, int chNum, const DMAChannelDir &channelDir, int bdId, int repeatCount, uint32_t padValue=0, bool outOfOrder=false, bool configureChannel=true, bool pushTask=true)
Definition AIERT.cpp:589
mlir::LogicalResult addAieElfs(DeviceOp &targetOp, const mlir::StringRef elfPath, bool aieSim)
Definition AIERT.cpp:1226
mlir::LogicalResult resetLock(int col, int row, int lockId)
Definition AIERT.cpp:1152
mlir::LogicalResult teardownSwitches(DeviceOp &targetOp, bool skipCtrlPktOverlay=false)
Definition AIERT.cpp:859
llvm::SmallVector< std::pair< DmaBdField, DmaBdField >, 3 > padFields() const
The (before, after) zero-padding fields, innermost dimension first.