11#include "mlir/Dialect/Utils/StaticValueUtils.h"
12#include "mlir/Support/LogicalResult.h"
13#include "llvm/ADT/SmallPtrSet.h"
14#include "llvm/ADT/SmallSet.h"
17#include "aie_codegen.h"
23#define AIERC_STR(x) x, #x
24static const std::map<AieRC, std::string> AIERCTOSTR = {
50 {
AIERC_STR(XAIE_INSUFFICIENT_BUFFER_SIZE)},
54static const std::map<xilinx::AIE::WireBundle, StrmSwPortType>
55 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE = {
56 {xilinx::AIE::WireBundle::Core, StrmSwPortType::CORE},
57 {xilinx::AIE::WireBundle::DMA, StrmSwPortType::DMA},
58 {xilinx::AIE::WireBundle::TileControl, StrmSwPortType::CTRL},
59 {xilinx::AIE::WireBundle::FIFO, StrmSwPortType::FIFO},
60 {xilinx::AIE::WireBundle::South, StrmSwPortType::SOUTH},
61 {xilinx::AIE::WireBundle::West, StrmSwPortType::WEST},
62 {xilinx::AIE::WireBundle::North, StrmSwPortType::NORTH},
63 {xilinx::AIE::WireBundle::East, StrmSwPortType::EAST},
66 {xilinx::AIE::WireBundle::Trace, StrmSwPortType::TRACE},
73static llvm::raw_ostream &showAIEXRTArgs(llvm::raw_ostream &out,
74 const char *label, H1 &&value) {
75 return out << label <<
"=" << std::forward<H1>(value);
78template <
typename H1,
typename... T>
79static llvm::raw_ostream &showAIEXRTArgs(llvm::raw_ostream &out,
80 const char *label, H1 &&value,
82 const char *pcomma = strchr(label,
',');
83 return showAIEXRTArgs(out.write(label, pcomma - label)
84 <<
"=" << std::forward<H1>(value) <<
',',
85 pcomma + 1, std::forward<T>(rest)...);
88static llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
89 const XAie_LocType &loc);
91static llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
92 const XAie_Lock &lock);
94static llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
95 const XAie_Packet &packet);
97#define SHOW_AIERT_ARGS(os, ...) showAIEXRTArgs(os, #__VA_ARGS__, __VA_ARGS__)
101static_assert(XAIE_OK == 0);
103#define TRY_XAIE_API_FATAL_ERROR(API, ...) \
105 LLVM_DEBUG(llvm::dbgs() << "trying XAIE API: " << #API << " with args: "); \
106 LLVM_DEBUG(SHOW_AIERT_ARGS(llvm::dbgs(), __VA_ARGS__)); \
107 LLVM_DEBUG(llvm::dbgs() << "\n"); \
108 if (auto r = API(__VA_ARGS__)) \
109 llvm::report_fatal_error(llvm::Twine(#API " failed with ") + \
113#define TRY_XAIE_API_EMIT_ERROR(OP, API, ...) \
115 LLVM_DEBUG(llvm::dbgs() << "trying XAIE API: " << #API << " with args: "); \
116 LLVM_DEBUG(SHOW_AIERT_ARGS(llvm::dbgs(), __VA_ARGS__)); \
117 LLVM_DEBUG(llvm::dbgs() << "\n"); \
118 if (auto r = API(__VA_ARGS__)) \
119 return OP.emitOpError() << #API " failed with " << AIERCTOSTR.at(r); \
122#define TRY_XAIE_API_LOGICAL_RESULT(API, ...) \
124 LLVM_DEBUG(llvm::dbgs() << "trying XAIE API: " << #API << " with args: "); \
125 LLVM_DEBUG(SHOW_AIERT_ARGS(llvm::dbgs(), __VA_ARGS__)); \
126 LLVM_DEBUG(llvm::dbgs() << "\n"); \
127 if (auto r = API(__VA_ARGS__)) { \
128 llvm::errs() << #API " failed with " << AIERCTOSTR.at(r); \
135#define TRY_XAIE_API_FATAL_ERROR(API, ...) \
137 if (auto r = API(__VA_ARGS__)) \
138 llvm::report_fatal_error(llvm::Twine(#API " failed with ") + \
142#define TRY_XAIE_API_EMIT_ERROR(OP, API, ...) \
144 if (auto r = API(__VA_ARGS__)) \
145 return OP.emitOpError() << #API " failed with " << AIERCTOSTR.at(r); \
148#define TRY_XAIE_API_LOGICAL_RESULT(API, ...) \
150 if (auto r = API(__VA_ARGS__)) { \
151 llvm::errs() << #API " failed with " << AIERCTOSTR.at(r); \
161#define DEBUG_TYPE "aie-aiert"
163llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
const XAie_LocType &loc) {
164 os <<
"XAie_LocType(col: " << std::to_string(loc.Col)
165 <<
", row: " << std::to_string(loc.Row) <<
")";
169llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
const XAie_Lock &lock) {
170 os <<
"XAie_Lock(id: " << std::to_string(lock.LockId)
171 <<
", val: " << std::to_string(lock.LockVal) <<
")";
175llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
176 const XAie_Packet &packet) {
177 os <<
"XAie_Packet(id: " << std::to_string(packet.PktId)
178 <<
", type: " << std::to_string(packet.PktType) <<
")";
182#define XAIE_BASE_ADDR 0x40000000
183#define XAIE_SHIM_ROW 0
184#define XAIE_MEM_TILE_ROW_START 1
185#define XAIE_PARTITION_BASE_ADDR 0x0
189#define EVEN_BD_NUM_START 0
190#define ODD_BD_NUM_START 24
209static XAie_TxnInst *getActiveTxnInst(XAie_DevInst &devInst) {
210 XAie_List *node = devInst.TxnList.Next;
213 return reinterpret_cast<XAie_TxnInst *
>(
reinterpret_cast<char *
>(node) -
214 offsetof(XAie_TxnInst, Node));
217static uint32_t getActiveTxnNumCmds(XAie_DevInst &devInst) {
218 XAie_TxnInst *inst = getActiveTxnInst(devInst);
219 return inst ? inst->NumCmds : 0;
223 : ctl(c), loc(l), startCmds(c.getCurrentTxnNumCmds()) {}
226 uint32_t endCmds = ctl.getCurrentTxnNumCmds();
227 if (endCmds <= startCmds)
232 if (mlir::isa<mlir::UnknownLoc>(loc))
234 ctl.recordTxnLocRange(startCmds, endCmds, loc);
240 : targetModel(tm), aiert(std::make_unique<
AIERtImpl>()) {
243 size_t partitionStartCol =
245 size_t partitionNumCols = tm.
columns();
246 size_t deviceRows = tm.
rows();
247 size_t deviceCols = tm.
columns() + partitionStartCol;
250 unsigned char devGen = XAIE_DEV_GEN_AIE;
253 devGen = XAIE_DEV_GEN_AIE;
258 devGen = XAIE_DEV_GEN_AIE2IPU;
261 devGen = XAIE_DEV_GEN_AIE2P_STRIX_B0;
263 case AIEArch::AIE2ps:
267 llvm::report_fatal_error(
268 "aie-codegen configuration is not supported for the "
269 "AIE2PS architecture");
271 aiert->configPtr = XAie_Config{
274 static_cast<uint8_t>(tm.getColumnShift()),
275 static_cast<uint8_t>(tm.getRowShift()),
276 static_cast<uint8_t>(deviceRows),
277 static_cast<uint8_t>(deviceCols),
279 XAIE_MEM_TILE_ROW_START,
280 static_cast<uint8_t>(tm.getNumMemTileRows()),
282 static_cast<uint8_t>(XAIE_MEM_TILE_ROW_START + tm.getNumMemTileRows()),
284 static_cast<uint8_t>(tm.rows() - tm.getNumMemTileRows() - 1),
288 XAie_InstDeclare(_devInst, &aiert->configPtr);
289 aiert->devInst = _devInst;
306 XAIE_IO_BACKEND_SIM);
307 }
else if (xaieDebug)
309 XAIE_IO_BACKEND_DEBUG);
312 XAIE_IO_BACKEND_CDO);
318 XAie_DmaDesc &dmaTileBd, Block &block,
int col,
int row,
320 LLVM_DEBUG(llvm::dbgs() <<
"\nstart configuring bds\n");
321 AIE::UseLockOp acquire, release;
324 std::optional<int> acqValue, relValue, acqLockId, relLockId;
329 auto lockIdFor = [&](AIE::LockOp lock) {
330 int id = lock.getLockIDValue();
333 col, row, lock.colIndex(), lock.rowIndex()))
337 for (
auto op : block.getOps<AIE::UseLockOp>()) {
338 auto lock = cast<AIE::LockOp>(op.getLock().getDefiningOp());
339 switch (op.getAction()) {
340 case AIE::LockAction::Acquire:
341 case AIE::LockAction::AcquireGreaterEqual: {
342 acqEn = op.getAcqEn();
343 acqLockId = lockIdFor(lock);
344 auto value = op.getConstantValue();
353 case AIE::LockAction::Release: {
354 relLockId = lockIdFor(lock);
355 auto value = op.getConstantValue();
366 if (outOfOrder && !acqLockId) {
369 acqLockId = relLockId;
372 assert(acqLockId && acqValue && relLockId && relValue &&
373 "lock values must be resolved before init");
374 XAie_Lock acqLock = XAie_LockInit(acqLockId.value(), acqValue.value());
375 XAie_Lock relLock = XAie_LockInit(relLockId.value(), relValue.value());
377 dmaTileBd.DmaMod->SetLock, &dmaTileBd, acqLock,
378 relLock, acqEn, relEn);
382static LogicalResult configureBdInBlock(
const AIE::AIETargetModel &targetModel,
383 XAie_DevInst *devInst,
384 XAie_DmaDesc &dmaTileBd, Block &block,
385 int col,
int row,
int bdId,
386 std::optional<int> nextBdId) {
387 std::optional<int> packetType;
388 std::optional<int> packetID;
391 auto maybePacketOps = block.getOps<AIE::DMABDPACKETOp>();
392 if (!maybePacketOps.empty()) {
393 assert(llvm::range_size(maybePacketOps) == 1 &&
394 "expected only one dma_bd_packet");
395 auto packetOp = *maybePacketOps.begin();
396 packetType = packetOp.getPacketType();
397 packetID = packetOp.getPacketID();
400 auto bdOp = *block.getOps<AIE::DMABDOp>().begin();
408 uint8_t cache =
static_cast<uint8_t
>(bdOp.getAxcacheOrDefault());
411 burstLen / 16, qOs, cache, secure);
415 uint64_t baseAddr = 0;
418 cast<AIE::ExternalBufferOp>(bdOp.getBuffer().getDefiningOp());
422 if (
auto addr = bufferOp.getAddress())
425 auto bufferOp = cast<AIE::BufferOp>(bdOp.getBuffer().getDefiningOp());
426 auto addr = bufferOp.getAddress();
428 return bufferOp.emitError(
"buffer must have address assigned");
435 auto bufferOp = cast<AIE::BufferOp>(bdOp.getBuffer().getDefiningOp());
436 auto bufferRow = bufferOp.getTileOp().getRow();
437 auto bufferCol = bufferOp.getTileOp().getCol();
441 baseAddr += addrOffset.value();
446 for (mlir::OpFoldResult s : bdOp.getMixedSizes())
447 if (!mlir::getConstantIntValue(s))
448 return bdOp->emitOpError(
449 "runtime-valued BD size/stride is not supported on the static XAIE "
450 "lowering path; use compile-time constant sizes");
451 for (mlir::OpFoldResult s : bdOp.getMixedStrides())
452 if (!mlir::getConstantIntValue(s))
453 return bdOp->emitOpError(
454 "runtime-valued BD size/stride is not supported on the static XAIE "
455 "lowering path; use compile-time constant sizes");
457 std::optional<llvm::SmallVector<AIE::BDDimLayoutAttr>> dimsStorage =
458 bdOp.getConstantDimensions();
460 return bdOp->emitOpError(
"internal error folding BD dimensions");
461 std::optional<llvm::ArrayRef<AIE::BDDimLayoutAttr>> dims;
462 if (!dimsStorage->empty())
463 dims = llvm::ArrayRef<AIE::BDDimLayoutAttr>(*dimsStorage);
464 uint64_t lenInBytes = bdOp.getLenInBytes();
465 uint64_t basePlusOffsetInBytes = baseAddr + bdOp.getOffsetInBytes();
468 basePlusOffsetInBytes, lenInBytes);
470 llvm::SmallVector<XAie_DmaDimDesc, 4> dimDescs(dims->size());
471 XAie_DmaTensor dmaTileBdTensor = {};
472 dmaTileBdTensor.NumDim = dimDescs.size();
473 dmaTileBdTensor.Dim = dimDescs.data();
475 double elementWidthIn32bWords =
476 static_cast<double>(bdOp.getBufferElementTypeWidthInBytes()) / 4.0;
477 for (
size_t i = 0; i < dims->size(); i++) {
481 int j = dims->size() - i - 1;
485 stride =
static_cast<uint32_t
>(dims.value()[i].getStride() *
486 elementWidthIn32bWords);
487 size = dims.value()[i].getSize();
489 stride = dims.value()[i].getStride();
490 size =
static_cast<uint16_t
>(dims.value()[i].getSize() *
491 elementWidthIn32bWords);
493 stride = stride > 0 ? stride : 1;
496 dmaTileBdTensor.Dim[j].AieMlDimDesc = {stride, size};
499 &dmaTileBdTensor, basePlusOffsetInBytes,
504 if (
auto iter = bdOp.getIteration(); iter && iter->getSize() > 1) {
505 if (iter->getStride() > 0) {
506 double elementWidthIn32bWords =
507 static_cast<double>(bdOp.getBufferElementTypeWidthInBytes()) / 4.0;
508 uint32_t stepInWords =
509 static_cast<uint32_t
>(iter->getStride() * elementWidthIn32bWords);
510 uint8_t wrap =
static_cast<uint8_t
>(iter->getSize());
511 uint8_t iterCurr =
static_cast<uint8_t
>(iter->getCurrent());
513 stepInWords, wrap, iterCurr);
518 std::optional<llvm::ArrayRef<AIE::BDPadLayoutAttr>> padDims =
519 bdOp.getPadDimensions();
523 llvm::SmallVector<XAie_PadDesc, 3> padDescs(
524 std::min(padDims->size(),
526 XAie_DmaPadTensor dmaPadTensor = {};
527 dmaPadTensor.NumDim = padDescs.size();
528 dmaPadTensor.PadDesc = padDescs.data();
530 double elementWidthIn32bWords =
531 static_cast<double>(bdOp.getBufferElementTypeWidthInBytes()) / 4.0;
532 for (
size_t i = 0; i < padDims->size(); i++) {
534 int j = padDims->size() - i - 1;
535 if (j >=
static_cast<int>(padDescs.size()))
540 before =
static_cast<uint8_t
>(padDims.value()[i].getConstPadBefore());
541 after =
static_cast<uint8_t
>(padDims.value()[i].getConstPadAfter());
543 before =
static_cast<uint8_t
>(padDims.value()[i].getConstPadBefore() *
544 elementWidthIn32bWords);
545 after =
static_cast<uint8_t
>(padDims.value()[i].getConstPadAfter() *
546 elementWidthIn32bWords);
548 dmaPadTensor.PadDesc[j] = {before, after};
554 auto enableNextBd = 1;
556 nextBdId.value(), enableNextBd);
559 if (
auto packetInfo = bdOp.getPacket()) {
560 packetType = packetInfo->getPktType();
561 packetID = packetInfo->assignedId();
566 return bdOp.emitError(
"must have packetType with packetID");
571 if (bdOp.getConstantLen() == 0)
572 return bdOp.emitOpError(
573 "For MM2S channels, if Buffer_Length=0 then Enable_Packet must be "
574 "set to 0, otherwise behavior is undefined (3.7.8 arch spec)");
576 XAie_PacketInit(*packetID, *packetType));
578 if (std::optional<int32_t> oooId = bdOp.getOutOfOrderId())
580 static_cast<uint8_t
>(*oooId));
582 auto tileLoc = XAie_TileLoc(col, row);
585 LLVM_DEBUG(llvm::dbgs() <<
"\nend configuring bds\n");
590 Operation &op,
int col,
int row,
int chNum,
const DMAChannelDir &channelDir,
591 int bdId,
int repeatCount, uint32_t padValue,
bool outOfOrder,
592 bool configureChannel,
bool pushTask) {
594 XAie_DmaDirection direction =
595 channelDir == DMAChannelDir::S2MM ? DMA_S2MM : DMA_MM2S;
596 auto tileLoc = XAie_TileLoc(col, row);
597 if (configureChannel && padValue != 0 && direction == DMA_MM2S &&
598 targetModel.isMemTile(col, row)) {
600 static_cast<uint8_t
>(chNum), padValue);
606 if (configureChannel) {
607 XAie_DmaChannelDesc dmaChannelDesc;
609 &dmaChannelDesc, tileLoc);
611 static_cast<uint8_t
>(XAIE_ENABLE));
613 &dmaChannelDesc, tileLoc,
614 static_cast<uint8_t
>(chNum), direction);
618 XAie_DmaDeclareQueueConfig(startQueueCfg, 0, repeatCount,
619 static_cast<uint8_t
>(XAIE_DISABLE),
620 static_cast<uint8_t
>(XAIE_ENABLE));
623 &aiert->devInst, tileLoc, chNum, direction,
625 }
else if (pushTask) {
626 bool enTokenIssue = tileLoc.Row == 0 && direction == DMA_S2MM;
628 tileLoc, chNum, direction, bdId, repeatCount,
631 if (configureChannel)
640 DMABDOp bd = *block.getOps<DMABDOp>().begin();
641 auto bdId = bd.getBdId();
642 assert(bdId.has_value() &&
643 "DMABDOp must have assigned bd_id; did you forget to run "
644 "aie-assign-bd-ids?");
646 XAie_DmaDesc dmaTileBd;
647 auto tileLoc = XAie_TileLoc(col, row);
650 if (!block.getOps<UseLockOp>().empty() &&
651 failed(configureLocksInBdBlock(targetModel, dmaTileBd, block, col, row,
656 std::optional<int> nextBdId =
657 outOfOrder ? std::optional<int>() : bd.getNextBdId();
658 if (!block.getOps<DMABDOp>().empty() &&
659 failed(configureBdInBlock(
660 targetModel, &aiert->devInst, dmaTileBd, block, col, row,
668 for (
auto tileOp : targetOp.getOps<TileOp>()) {
669 auto tileLoc = XAie_TileLoc(tileOp.colIndex(), tileOp.rowIndex());
670 if (!tileOp.isShimTile() && tileOp.getCoreOp()) {
676 for (uint8_t l = 0; l <
NUM_LOCKS; l++) {
677 auto locInit = XAie_LockInit(l, 0);
685 targetOp.walkOutsideRuntimeSequences<LockOp>([&](LockOp lockOp) {
686 if (lockOp.getLockID() && lockOp.getInit()) {
688 auto tileLoc = XAie_TileLoc(lockOp.getTileOp().colIndex(),
689 lockOp.getTileOp().rowIndex());
690 auto locInit = XAie_LockInit(*lockOp.getLockID(), *lockOp.getInit());
694 LLVM_DEBUG(llvm::dbgs()
695 <<
"lock op missing either id or init" << lockOp <<
"\n");
702 targetOp.walkOutsideRuntimeSequences<BufferOp>([&](BufferOp bufferOp) {
703 auto initialValue = bufferOp.getInitialValue();
707 mlir::DenseElementsAttr denseInit =
708 dyn_cast<mlir::DenseElementsAttr>(initialValue.value());
711 auto tileLoc = XAie_TileLoc(bufferOp.getTileOp().colIndex(),
712 bufferOp.getTileOp().rowIndex());
715 llvm::outs() <<
"buffer op type not supported for initialization "
720 bufferOp.getAddress().value(), byteVec->data(),
728 bool skipCtrlPktOverlay) {
729 AIEDialect::IsCtrlPktOverlayAttrHelper overlay(targetOp.getContext());
732 for (
auto switchboxOp : targetOp.getOps<SwitchboxOp>()) {
733 int32_t col = switchboxOp.colIndex();
734 int32_t row = switchboxOp.rowIndex();
735 XAie_LocType tileLoc = XAie_TileLoc(col, row);
737 "Only NPU currently supported");
739 Block &b = switchboxOp.getConnections().front();
740 for (
auto connectOp : b.getOps<ConnectOp>()) {
741 if (skipCtrlPktOverlay && overlay.isAttrPresent(connectOp))
745 switchboxOp, XAie_StrmConnCctEnable, &aiert->devInst, tileLoc,
746 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(connectOp.getSourceBundle()),
747 connectOp.sourceIndex(),
748 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(connectOp.getDestBundle()),
749 connectOp.destIndex());
752 for (
auto masterSetOp : b.getOps<MasterSetOp>()) {
753 if (skipCtrlPktOverlay && overlay.isAttrPresent(masterSetOp))
761 for (
auto val : masterSetOp.getAmsels()) {
762 AMSelOp amsel = cast<AMSelOp>(val.getDefiningOp());
763 arbiter = amsel.arbiterIndex();
764 int msel = amsel.getMselValue();
768 auto dropHeader = masterSetOp.keepsPktHeader()
769 ? XAIE_SS_PKT_DONOT_DROP_HEADER
770 : XAIE_SS_PKT_DROP_HEADER;
772 masterSetOp, XAie_StrmPktSwMstrPortEnable, &aiert->devInst, tileLoc,
773 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(masterSetOp.getDestBundle()),
774 masterSetOp.destIndex(), dropHeader, arbiter, mask);
777 for (
auto packetRulesOp : b.getOps<PacketRulesOp>()) {
778 if (skipCtrlPktOverlay && overlay.isAttrPresent(packetRulesOp))
782 Block &block = packetRulesOp.getRules().front();
786 skipCtrlPktOverlay &&
787 llvm::any_of(block.getOps<PacketRuleOp>(), [&](PacketRuleOp rule) {
788 return overlay.isAttrPresent(rule);
790 for (
auto slotOp : block.getOps<PacketRuleOp>()) {
791 if (skipCtrlPktOverlay && overlay.isAttrPresent(slotOp)) {
795 AMSelOp amselOp = cast<AMSelOp>(slotOp.getAmsel().getDefiningOp());
796 int arbiter = amselOp.arbiterIndex();
797 int msel = amselOp.getMselValue();
800 &aiert->devInst, tileLoc,
801 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(
802 packetRulesOp.getSourceBundle()),
803 packetRulesOp.sourceIndex());
804 auto packetInit = XAie_PacketInit(slotOp.valueInt(), 0);
807 &aiert->devInst, tileLoc,
808 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(
809 packetRulesOp.getSourceBundle()),
810 packetRulesOp.sourceIndex(), slot, packetInit,
811 slotOp.maskInt(), msel, arbiter);
817 for (
auto muxOp : targetOp.getOps<ShimMuxOp>()) {
821 XAie_TileLoc(muxOp.getTileOp().getCol(), muxOp.getTileOp().getRow());
822 Block &b = muxOp.getConnections().front();
823 for (
auto connectOp : b.getOps<ConnectOp>()) {
824 if (skipCtrlPktOverlay && overlay.isAttrPresent(connectOp))
828 if (connectOp.getSourceBundle() == WireBundle::North)
830 &aiert->devInst, tileLoc,
831 connectOp.sourceIndex());
833 if (connectOp.getDestBundle() == WireBundle::North)
835 &aiert->devInst, tileLoc,
836 connectOp.destIndex());
841 if (isa<AIE2TargetModel>(targetModel)) {
842 for (
auto configOp : targetOp.getOps<ConfigureCascadeOp>()) {
844 TileOp tile = cast<TileOp>(configOp.getTile().getDefiningOp());
845 auto tileLoc = XAie_TileLoc(tile.getCol(), tile.getRow());
847 targetOp, XAie_CoreConfigAccumulatorControl, &aiert->devInst, tileLoc,
848 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(
849 static_cast<WireBundle
>(configOp.getInputDir())),
850 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(
851 static_cast<WireBundle
>(configOp.getOutputDir())));
860 bool skipCtrlPktOverlay) {
861 AIEDialect::IsCtrlPktOverlayAttrHelper overlay(targetOp.getContext());
862 auto skip = [&](Operation *op) {
863 return skipCtrlPktOverlay && overlay.isAttrPresent(op);
865 for (
auto switchboxOp : targetOp.getOps<SwitchboxOp>()) {
866 XAie_LocType tileLoc =
867 XAie_TileLoc(switchboxOp.colIndex(), switchboxOp.rowIndex());
868 Block &b = switchboxOp.getConnections().front();
869 for (
auto connectOp : b.getOps<ConnectOp>()) {
874 switchboxOp, XAie_StrmConnCctDisable, &aiert->devInst, tileLoc,
875 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(connectOp.getSourceBundle()),
876 connectOp.sourceIndex(),
877 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(connectOp.getDestBundle()),
878 connectOp.destIndex());
880 for (
auto masterSetOp : b.getOps<MasterSetOp>()) {
881 if (skip(masterSetOp))
885 masterSetOp, XAie_StrmPktSwMstrPortDisable, &aiert->devInst, tileLoc,
886 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(masterSetOp.getDestBundle()),
887 masterSetOp.destIndex());
889 for (
auto packetRulesOp : b.getOps<PacketRulesOp>()) {
890 auto rules = packetRulesOp.getRules().front().getOps<PacketRuleOp>();
891 if (rules.empty() || skip(packetRulesOp))
894 StrmSwPortType port =
895 WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE.at(packetRulesOp.getSourceBundle());
896 if (llvm::none_of(rules, skip))
898 &aiert->devInst, tileLoc, port,
899 packetRulesOp.sourceIndex());
901 for (
auto [slot, rule] : llvm::enumerate(rules))
904 &aiert->devInst, tileLoc, port,
905 packetRulesOp.sourceIndex(), slot);
912 for (
auto coreOp : targetOp.getOps<CoreOp>()) {
913 TileID tile = coreOp.getTileID();
916 XAie_TileLoc(tile.col, tile.row));
922 auto memOps = llvm::to_vector_of<TileElement>(targetOp.getOps<MemOp>());
923 llvm::append_range(memOps, targetOp.getOps<MemTileDMAOp>());
924 for (TileElement memOp : memOps) {
925 TileID tile = memOp.getTileID();
926 XAie_LocType tileLoc = XAie_TileLoc(tile.col, tile.row);
927 llvm::SmallSet<std::pair<DMAChannelDir, int>, 4> reset;
928 auto resetChannel = [&](Operation *op, DMAChannelDir dir,
929 int channel) -> LogicalResult {
930 if (!reset.insert({dir, channel}).second)
933 XAie_DmaDirection direction =
934 dir == DMAChannelDir::S2MM ? DMA_S2MM : DMA_MM2S;
936 tileLoc, channel, direction,
937 XAie_DmaChReset::DMA_CHANNEL_RESET);
940 XAie_DmaChannelDesc defaults;
944 &defaults, tileLoc, channel, direction);
947 Region ®ion = memOp.getOperation()->getRegion(0);
948 for (
auto dmaOp : region.getOps<DMAOp>())
949 if (failed(resetChannel(dmaOp, dmaOp.getChannelDir(),
950 dmaOp.getChannelIndex())))
953 for (
auto startOp : block->getOps<DMAStartOp>())
954 if (failed(resetChannel(startOp, startOp.getChannelDir(),
955 startOp.getChannelIndex())))
963 bool skipCtrlPktOverlay) {
982 auto memOps = llvm::to_vector_of<TileElement>(targetOp.getOps<MemOp>());
983 llvm::append_range(memOps, targetOp.getOps<MemTileDMAOp>());
984 llvm::append_range(memOps, targetOp.getOps<ShimDMAOp>());
985 for (TileElement memOp : memOps) {
986 int col = memOp.getTileID().col;
987 int row = memOp.getTileID().row;
991 llvm::SetVector<Block *> blockVector =
994 llvm::SmallPtrSet<Block *, 8> unchainedBlocks =
998 auto dmaOps = llvm::to_vector_of<DMAOp>(
999 memOp.getOperation()->getRegion(0).getOps<DMAOp>());
1000 if (!dmaOps.empty()) {
1001 for (
auto dmaOp : dmaOps)
1002 for (
auto &bdRegion : dmaOp.getBds()) {
1003 Block &block = bdRegion.getBlocks().front();
1009 for (Block *block : blockVector) {
1010 if (block->getOps<DMABDOp>().empty())
1013 unchainedBlocks.contains(block))))
1018 if (!dmaOps.empty())
1019 for (
auto dmaOp : dmaOps) {
1020 auto &block = dmaOp.getBds().front().getBlocks().front();
1021 DMABDOp bd = *block.getOps<DMABDOp>().begin();
1022 auto bdId = bd.getBdId();
1023 assert(bdId.has_value() &&
1024 "bd_id assigned by configureLocksAndBd above");
1026 *dmaOp.getOperation(), col, row, dmaOp.getChannelIndex(),
1027 dmaOp.getChannelDir(), bdId.value(), dmaOp.getRepeatCount(),
1028 dmaOp.getPadValue(), dmaOp.getOutOfOrder())))
1032 for (Block *block : blockVector) {
1033 for (
auto op : block->getOps<DMAStartOp>()) {
1034 DMABDOp bd = *op.getDest()->getOps<DMABDOp>().begin();
1035 int chNum = op.getChannelIndex();
1036 auto channelDir = op.getChannelDir();
1037 auto bdId = bd.getBdId();
1038 assert(bdId.has_value() &&
1039 "bd_id assigned by configureLocksAndBd above");
1041 *bd.getOperation(), col, row, chNum, channelDir, bdId.value(),
1042 op.getRepeatCount(), op.getPadValue(), op.getOutOfOrder(),
1044 !op.getCorePushes())))
1055 for (
auto tileOp : targetOp.getOps<TileOp>()) {
1056 auto tileLoc = XAie_TileLoc(tileOp.colIndex(), tileOp.rowIndex());
1057 if (!tileOp.isShimTile() && tileOp.getCoreOp()) {
1058 CoreOp coreOp = tileOp.getCoreOp();
1063 if (coreOp.getProcessorBus())
1065 &aiert->devInst, tileLoc);
1074 const StringRef elfPath,
1077 XAie_TileLoc(col, row));
1079 XAie_TileLoc(col, row),
1080 XAie_DmaChReset::DMA_CHANNEL_RESET);
1085 XAie_TileLoc(col, row), elfPath.str().c_str(),
1089 XAie_TileLoc(col, row),
1090 XAie_DmaChReset::DMA_CHANNEL_UNRESET);
1101 auto tileLoc = XAie_TileLoc(col, row);
1102 XAie_DmaDesc dmaTileBd;
1107 on ? XAie_DmaChReset::DMA_CHANNEL_UNRESET
1108 : XAie_DmaChReset::DMA_CHANNEL_RESET);
1113 auto tileLoc = XAie_TileLoc(col, row);
1119 XAie_LocType tileLoc = XAie_TileLoc(col, row);
1122 for (
auto endpoint_a : WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE) {
1123 for (
auto endpoint_b : WIRE_BUNDLE_TO_STRM_SW_PORT_TYPE) {
1124 unsigned n_a_connections = targetModel.getNumSourceSwitchboxConnections(
1125 col, row, endpoint_a.first);
1126 unsigned n_b_connections = targetModel.getNumDestSwitchboxConnections(
1127 col, row, endpoint_b.first);
1128 for (
unsigned a_index = 0; a_index < n_a_connections; a_index++) {
1129 for (
unsigned b_index = 0; b_index < n_b_connections; b_index++) {
1130 if (!targetModel.isLegalTileConnection(col, row, endpoint_a.first,
1131 a_index, endpoint_b.first,
1136 tileLoc, endpoint_a.second, a_index,
1137 endpoint_b.second, b_index);
1147 auto tileLoc = XAie_TileLoc(col, row);
1154 auto tileLoc = XAie_TileLoc(col, row);
1157 lock.LockId = lockId;
1165 int col,
int row, WireBundle sourceBundle,
int sourceChannel,
1166 WireBundle destBundle,
int destChannel) {
1167 auto tileLoc = XAie_TileLoc(col, row);
1170 auto mapBundle = [](WireBundle bundle) -> StrmSwPortType {
1172 case WireBundle::Core:
1174 case WireBundle::DMA:
1176 case WireBundle::FIFO:
1178 case WireBundle::South:
1180 case WireBundle::West:
1182 case WireBundle::North:
1184 case WireBundle::East:
1186 case WireBundle::Trace:
1193 StrmSwPortType sourcePortType = mapBundle(sourceBundle);
1194 StrmSwPortType destPortType = mapBundle(destBundle);
1198 sourcePortType, sourceChannel, destPortType,
1205 auto tileLoc = XAie_TileLoc(col, row);
1208 for (
int counterId = 0; counterId < 4; counterId++) {
1210 (void)XAie_PerfCounterReset(&aiert->devInst, tileLoc, XAIE_CORE_MOD,
1214 for (
int counterId = 0; counterId < 4; counterId++) {
1215 (void)XAie_PerfCounterReset(&aiert->devInst, tileLoc, XAIE_MEM_MOD,
1219 for (
int counterId = 0; counterId < 4; counterId++) {
1220 (void)XAie_PerfCounterReset(&aiert->devInst, tileLoc, XAIE_PL_MOD,
1227 const StringRef elfPath,
1229 for (
auto tileOp : targetOp.getOps<TileOp>())
1230 if (tileOp.isShimNOCorPLTile()) {
1233 int col = tileOp.colIndex();
1234 int row = tileOp.rowIndex();
1235 if (
auto coreOp = tileOp.getCoreOp()) {
1236 std::string fileName;
1237 if (
auto fileAttr = coreOp.getElfFile()) {
1238 fileName = fileAttr->str();
1246 std::string fullPath;
1247 if (std::filesystem::path(fileName).is_absolute()) {
1248 fullPath = fileName;
1250 auto ps = std::filesystem::path::preferred_separator;
1252 (llvm::Twine(elfPath) + std::string(1, ps) + fileName).str();
1254 if (failed(
addAieElf(col, row, fullPath, aieSim)))
1263 auto tileLoc = XAie_TileLoc(col, row);
1269 aiert->txnCmdLocs.clear();
1270 aiert->txnOpLocs.clear();
1272 XAIE_TRANSACTION_DISABLE_AUTO_FLUSH);
1284static std::vector<mlir::Location>
1285projectCmdLocsOntoSerializedOps(
const XAie_TxnInst &inst,
1286 const std::vector<mlir::Location> &cmdLocs) {
1287 std::vector<mlir::Location> opLocs;
1288 opLocs.reserve(inst.NumCmds);
1290 mlir::Location unknown = mlir::UnknownLoc::get(cmdLocs.front().getContext());
1291 bool pendingBlockWrite =
false;
1292 mlir::Location pendingBlockWriteLoc = unknown;
1293 uint64_t blockWriteEndOff = 0;
1295 for (uint32_t i = 0; i < inst.NumCmds; ++i) {
1296 const XAie_TxnCmd &cmd = inst.CmdBuf[i];
1297 mlir::Location loc = i < cmdLocs.size() ? cmdLocs[i] : unknown;
1299 if (cmd.Opcode == XAIE_IO_BLOCKWRITE) {
1302 if (!pendingBlockWrite || cmd.RegOff != blockWriteEndOff) {
1303 if (pendingBlockWrite)
1304 opLocs.push_back(pendingBlockWriteLoc);
1305 pendingBlockWrite =
true;
1306 pendingBlockWriteLoc = loc;
1308 blockWriteEndOff = cmd.RegOff + cmd.Size *
sizeof(uint32_t);
1312 if (cmd.Opcode == XAIE_IO_LOAD_PM_END_INTERNAL)
1315 if (pendingBlockWrite && cmd.Opcode != XAIE_IO_CUSTOM_OP_DDR_PATCH) {
1316 opLocs.push_back(pendingBlockWriteLoc);
1317 pendingBlockWrite =
false;
1319 opLocs.push_back(loc);
1321 if (pendingBlockWrite)
1322 opLocs.push_back(pendingBlockWriteLoc);
1329 uint8_t *txn_ptr = XAie_ExportSerializedTransaction(&aiert->devInst, 0, 0);
1330 XAie_TxnHeader *hdr = (XAie_TxnHeader *)txn_ptr;
1331 std::vector<uint8_t> txn_data(txn_ptr, txn_ptr + hdr->TxnSize);
1337 XAie_TxnInst *inst = getActiveTxnInst(aiert->devInst);
1338 if (inst && !aiert->txnCmdLocs.empty())
1340 projectCmdLocsOntoSerializedOps(*inst, aiert->txnCmdLocs);
1345const std::vector<mlir::Location> &
1347 return aiert->txnOpLocs;
1351 return getActiveTxnNumCmds(aiert->devInst);
1356 mlir::Location loc) {
1357 if (endCmds <= startCmds)
1359 if (aiert->txnCmdLocs.size() < endCmds)
1360 aiert->txnCmdLocs.resize(endCmds, mlir::UnknownLoc::get(loc.getContext()));
1361 for (uint32_t i = startCmds; i < endCmds; ++i)
1362 aiert->txnCmdLocs[i] = loc;
#define XAIE_PARTITION_BASE_ADDR
#define TRY_XAIE_API_LOGICAL_RESULT(API,...)
#define TRY_XAIE_API_FATAL_ERROR(API,...)
#define TRY_XAIE_API_EMIT_ERROR(OP, API,...)
std::optional< uint32_t > getMemLocalBaseAddress(int localCol, int localRow, int memCol, int memRow) const
Return the memory base address (or offset) in the local tile when accessing a neighbor's memory or an...
virtual const DmaBdLayout * getDmaBdLayout(AIETileType tileType) const
Return the register layout of a buffer descriptor on the given tile type, or null if the target has n...
virtual AIEArch getTargetArch() const =0
Return the target architecture.
bool isMemTile(int col, int row) const
Return true if the given tile is a Mem tile.
std::optional< uint32_t > getLockLocalBaseIndex(int localCol, int localRow, int lockCol, int lockRow) const
Return the lock base index (or offset) in the local tile when accessing a neighbor's lock or an empty...
virtual int rows() const =0
Return the number of rows in the device.
bool isShimNOCTile(int col, int row) const
Return true if the given tile is a ShimNOC tile.
bool hasProperty(ModelProperty Prop) const
virtual int columns() const =0
Return the number of columns in the device.
TxnLocBracket(AIERTControl &ctl, mlir::Location loc)
uint32_t getShimBurstLengthBytes(const AIE::AIETargetModel &tm, uint32_t burstLength)
mlir::LogicalResult verifyBdLockPair(mlir::Block &block, bool outOfOrder, UseLockOp &acquire, UseLockOp &release)
struct TileID { friend std::ostream &operator<<(std::ostream &os, const TileID &s) { os<< "TileID("<< s.col<< ", "<< s.row<< ")"; return os; } friend std::string to_string(const TileID &s) { std::ostringstream ss; ss<< s; return ss.str(); } friend llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const TileID &s) { os<< to_string(s); return os; } inline bool operator<(const TileID &rhs) const { return std::tie(col, row)< std::tie(rhs.col, rhs.row); } bool operator==(const TileID &rhs) const { return std::tie(col, row)==std::tie(rhs.col, rhs.row); } bool operator!=(const TileID &rhs) const { return !(*this==rhs);} int col, row;} TileID
mlir::LogicalResult verifyDMAChannelsResolved(DeviceOp device)
std::optional< std::vector< char > > denseAttrToBytes(mlir::DenseElementsAttr denseInit)
A buffer's initial_value as the bytes to write to its tile memory, in element order.
llvm::SmallPtrSet< mlir::Block *, 8 > collectUnchainedBlocks(const llvm::SetVector< mlir::Block * > &blockVector)
Collect BD blocks that require use_next_bd=0 during configuration.
llvm::SetVector< mlir::Block * > getOrderedChainOfBlocks(mlir::Region *region)
std::vector< mlir::Location > txnCmdLocs
std::vector< mlir::Location > txnOpLocs
mlir::LogicalResult configureSwitches(DeviceOp &targetOp, bool skipCtrlPktOverlay=false)
mlir::LogicalResult resetPerfCounters(int col, int row)
mlir::LogicalResult configureLocksAndBd(mlir::Block &block, int col, int row, bool outOfOrder=false)
mlir::LogicalResult resetCore(int col, int row)
void dmaUpdateBdAddr(int col, int row, size_t addr, size_t bdId)
mlir::LogicalResult resetPartition()
mlir::LogicalResult resetCoreUnreset(int col, int row)
mlir::LogicalResult setIOBackend(bool aieSim, bool xaieDebug)
uint32_t getCurrentTxnNumCmds() const
std::vector< uint8_t > exportSerializedTransaction()
AIERTControl(const xilinx::AIE::AIETargetModel &tm)
mlir::LogicalResult initLocks(DeviceOp &targetOp)
mlir::LogicalResult resetDMA(int col, int row, bool on)
mlir::LogicalResult disableCores(DeviceOp &targetOp)
mlir::LogicalResult initBuffers(DeviceOp &targetOp)
mlir::LogicalResult resetSwitch(int col, int row)
mlir::LogicalResult addCoreEnable(DeviceOp &targetOp)
mlir::LogicalResult resetSwitchConnection(int col, int row, WireBundle sourceBundle, int sourceChannel, WireBundle destBundle, int destChannel)
const std::vector< mlir::Location > & getTxnOpLocs() const
mlir::LogicalResult addInitConfig(DeviceOp &targetOp, bool skipCtrlPktOverlay=false)
mlir::LogicalResult addAieElf(uint8_t col, uint8_t row, const mlir::StringRef elfPath, bool aieSim)
void recordTxnLocRange(uint32_t startCmds, uint32_t endCmds, mlir::Location loc)
mlir::LogicalResult resetDmaChannels(DeviceOp &targetOp)
mlir::LogicalResult pushToBdQueueAndEnable(mlir::Operation &op, int col, int row, int chNum, const DMAChannelDir &channelDir, int bdId, int repeatCount, uint32_t padValue=0, bool outOfOrder=false, bool configureChannel=true, bool pushTask=true)
mlir::LogicalResult addAieElfs(DeviceOp &targetOp, const mlir::StringRef elfPath, bool aieSim)
mlir::LogicalResult resetLock(int col, int row, int lockId)
mlir::LogicalResult teardownSwitches(DeviceOp &targetOp, bool skipCtrlPktOverlay=false)
llvm::SmallVector< std::pair< DmaBdField, DmaBdField >, 3 > padFields() const
The (before, after) zero-padding fields, innermost dimension first.