15#include "mlir/Dialect/Arith/IR/Arith.h"
16#include "mlir/IR/Attributes.h"
17#include "mlir/Pass/Pass.h"
19#include "llvm/ADT/STLExtras.h"
26#define GEN_PASS_DEF_AIEINSERTTRACEFLOWS
27#include "aie/Dialect/AIE/Transforms/AIEPasses.h.inc"
40 TracePacketType packetType;
45 std::optional<int> stopBroadcast;
49struct ChannelDescriptor {
58 std::vector<TraceInfo> traceSources;
59 std::optional<int> startBroadcast;
60 std::optional<int> stopBroadcast;
61 std::vector<ChannelDescriptor> channels;
62 std::vector<int> traceChannelAssignment;
67struct SequenceTraceConfig {
68 RuntimeSequenceOp seq;
70 bool reuseOutputBuffer;
78static bool isDynamicRuntimeSequence(RuntimeSequenceOp seq) {
79 return llvm::any_of(seq.getBody().getArguments(), [](BlockArgument arg) {
80 return !isa<MemRefType>(arg.getType());
84struct AIEInsertTraceFlowsPass
85 : xilinx::AIE::impl::AIEInsertTraceFlowsBase<AIEInsertTraceFlowsPass> {
87 void runOnOperation()
override {
88 DeviceOp device = getOperation();
89 OpBuilder builder(device);
90 const auto &targetModel = device.getTargetModel();
93 bool hasLogicalTile =
false;
94 device.walk([&](LogicalTileOp op) {
95 op.emitError() <<
"LogicalTileOp must be resolved to TileOp before "
96 "running -aie-insert-trace-flows (run -aie-place-tiles "
98 hasLogicalTile =
true;
101 return signalPassFailure();
104 SmallVector<TraceOp> traces;
105 device.walk([&](TraceOp trace) { traces.push_back(trace); });
112 int shimChannel = clShimChannel;
113 if (shimChannel < 0 || shimChannel > 1) {
114 device.emitError() <<
"shim-channel " << shimChannel
115 <<
" is not a shim S2MM channel; expected 0 or 1";
116 return signalPassFailure();
118 uint32_t numShimBds = targetModel.getNumBDs(AIETileType::ShimNOCTile);
119 int bdId = clDefaultBdId;
120 if (bdId < 0 ||
static_cast<uint32_t
>(bdId) >= numShimBds) {
121 device.emitError() <<
"default-bd-id " << bdId
122 <<
" is not a shim BD; expected [0, " << numShimBds
124 return signalPassFailure();
130 SmallVector<RuntimeSequenceOp> runtimeSeqs;
131 SmallVector<TraceHostConfigOp> hostConfigs;
132 RuntimeSequenceOp firstSeq =
nullptr;
133 for (
auto seq : device.getBody()->getOps<RuntimeSequenceOp>()) {
136 for (
auto hc : seq.getBody().front().getOps<TraceHostConfigOp>()) {
137 runtimeSeqs.push_back(seq);
138 hostConfigs.push_back(hc);
146 <<
"aie.trace ops found but no runtime_sequence defined";
147 return signalPassFailure();
151 if (runtimeSeqs.empty()) {
153 <<
"runtime_sequence with traces requires aie.trace.host_config";
154 return signalPassFailure();
160 TraceHostConfigOp hostConfig = hostConfigs.front();
161 auto routing = hostConfig.getRouting();
162 int egressShimColFromIR = hostConfig.getEgressShimCol();
163 for (
auto other : ArrayRef(hostConfigs).drop_front()) {
164 if (other.getRouting() == routing &&
165 other.getEgressShimCol() == egressShimColFromIR)
167 InFlightDiagnostic err =
169 <<
"aie.trace.host_config routes trace data differently from another "
170 "runtime_sequence in this device; the routes live in the device's "
171 "stream switches, so every sequence must name the same routing "
173 err.attachNote(hostConfig.getLoc()) <<
"first aie.trace.host_config here";
174 return signalPassFailure();
189 SmallVector<SequenceTraceConfig> seqConfigs;
190 for (
auto [seq, hc] : llvm::zip(runtimeSeqs, hostConfigs)) {
191 SequenceTraceConfig cfg;
193 cfg.bufferSizeBytes = hc.getBufferSize();
194 cfg.reuseOutputBuffer = hc.getReuseOutputBuffer();
195 cfg.bufferOffset = 0;
196 if (cfg.reuseOutputBuffer) {
197 auto args = seq.getBody().getArguments();
199 seq.emitError() <<
"trace.host_config reuse_output_buffer "
200 "requires the runtime_sequence to have at "
201 "least one argument to reuse";
202 return signalPassFailure();
204 if (isDynamicRuntimeSequence(seq)) {
206 <<
"trace.host_config reuse_output_buffer=true cannot be used "
207 "with a dynamic (runtime-sized) runtime_sequence: the trace "
208 "offset would be computed from the last tensor's maximum "
209 "static size, not its runtime transfer size, so trace data "
210 "would be written past the output buffer. Use a separate "
211 "trace buffer instead (reuse_output_buffer=false)";
212 return signalPassFailure();
214 auto memrefType = cast<MemRefType>(args.back().getType());
215 cfg.bufferOffset = memrefType.getNumElements() *
216 (memrefType.getElementTypeBitWidth() / 8);
218 seqConfigs.push_back(cfg);
221 for (
auto hc : hostConfigs)
225 std::vector<TraceInfo> traceInfos;
236 llvm::DenseMap<Operation *, int> autoPacketIds;
239 const int kMaxPacketId = device.getTargetModel().getMaxPacketId();
244 SmallVector<TraceOp> autoIdTraces;
245 llvm::DenseMap<int, TracePacketOp> explicitIdOwner;
246 for (
auto trace : traces) {
247 TracePacketOp explicitPacket =
nullptr;
248 for (
auto &op : trace.getBody().getOps()) {
249 if (
auto p = dyn_cast<TracePacketOp>(op)) {
250 if (p.getId().has_value()) {
256 if (!explicitPacket) {
257 autoIdTraces.push_back(trace);
260 int id = *explicitPacket.getId();
261 auto [it, inserted] = explicitIdOwner.try_emplace(
id, explicitPacket);
263 InFlightDiagnostic err =
264 explicitPacket.emitError()
265 <<
"trace packet id " <<
id
266 <<
" is already used by another trace; explicit packet ids "
267 "must be unique across the device";
268 err.attachNote(it->second.getLoc())
269 <<
"previous use of packet id " << id;
270 return signalPassFailure();
276 llvm::stable_sort(autoIdTraces, [](TraceOp a, TraceOp b) {
277 auto ta = cast<TileOp>(a.getTile().getDefiningOp());
278 auto tb = cast<TileOp>(b.getTile().getDefiningOp());
279 if (ta.getCol() != tb.getCol())
280 return ta.getCol() < tb.getCol();
281 return ta.getRow() < tb.getRow();
285 PacketIdSpace held(device);
286 int next = clPacketIdStart;
287 for (
auto trace : autoIdTraces) {
288 while (next <= kMaxPacketId &&
289 (explicitIdOwner.count(next) || held.isTaken(next)))
291 if (next > kMaxPacketId) {
293 <<
"trace overlay needs " << autoIdTraces.size()
294 <<
" auto-allocated packet IDs starting at " << clPacketIdStart
295 <<
" (with " << explicitIdOwner.size()
296 <<
" id(s) reserved by explicit aie.trace.packet ops, and any "
297 "held by packet flows or pinned routes elsewhere in the "
298 "device), but the hardware packet-id field is 5 bits (max "
300 <<
"); reduce the number of traced tiles, free up an explicit "
301 "id, or raise -packet-id-start only if you can spare the "
303 return signalPassFailure();
305 autoPacketIds[trace.getOperation()] = next++;
313 std::set<std::tuple<int, int, int>> seenTraceUnits;
315 for (
auto trace : traces) {
316 auto tile = cast<TileOp>(trace.getTile().getDefiningOp());
319 std::optional<int> packetId;
320 std::optional<TracePacketType> packetType;
321 TracePacketOp existingPacketOp =
nullptr;
322 for (
auto &op : trace.getBody().getOps()) {
323 if (
auto packetOp = dyn_cast<TracePacketOp>(op)) {
324 existingPacketOp = packetOp;
325 if (
auto explicitId = packetOp.getId())
326 packetId = explicitId;
327 packetType = packetOp.getType();
334 if (tile.isShimTile()) {
335 packetType = TracePacketType::ShimTile;
336 }
else if (tile.isMemTile()) {
337 packetType = TracePacketType::MemTile;
340 packetType = TracePacketType::Core;
345 auto unitKey = std::make_tuple(tile.getCol(), tile.getRow(),
346 static_cast<int>(*packetType));
347 if (!seenTraceUnits.insert(unitKey).second) {
348 trace.emitError() <<
"tile (" << tile.getCol() <<
", " << tile.getRow()
349 <<
") is traced more than once on the same trace "
350 "unit; each unit can have only one trace "
352 return signalPassFailure();
358 packetId = autoPacketIds.lookup(trace.getOperation());
363 if (!existingPacketOp) {
364 OpBuilder traceBuilder(&trace.getBody().front(),
365 trace.getBody().front().begin());
366 TracePacketOp::create(traceBuilder, trace.getLoc(),
367 traceBuilder.getI32IntegerAttr(*packetId),
369 }
else if (!existingPacketOp.getId().has_value()) {
370 existingPacketOp.setIdAttr(
371 OpBuilder(existingPacketOp).getI32IntegerAttr(*packetId));
375 WireBundle tracePort = WireBundle::Trace;
376 int traceChannel = 0;
377 if (*packetType == TracePacketType::Mem) {
382 std::optional<int> startBroadcast;
383 std::optional<int> stopBroadcast;
384 bool hasStartConfig =
false;
385 bool hasStopConfig =
false;
386 for (
auto &op : trace.getBody().getOps()) {
387 if (
auto startOp = dyn_cast<TraceStartEventOp>(op)) {
388 hasStartConfig =
true;
389 if (startOp.getBroadcast())
390 startBroadcast = startOp.getBroadcast();
392 if (
auto stopOp = dyn_cast<TraceStopEventOp>(op)) {
393 hasStopConfig =
true;
394 if (stopOp.getBroadcast())
395 stopBroadcast = stopOp.getBroadcast();
400 if (!hasStartConfig) {
401 trace.emitError() <<
"trace is missing 'aie.trace.start'";
402 return signalPassFailure();
404 if (!hasStopConfig) {
405 trace.emitError() <<
"trace is missing 'aie.trace.stop'";
406 return signalPassFailure();
410 info.traceOp = trace;
412 info.packetId = *packetId;
413 info.packetType = *packetType;
414 info.tracePort = tracePort;
415 info.traceChannel = traceChannel;
416 info.startBroadcast = startBroadcast;
417 info.stopBroadcast = stopBroadcast;
418 traceInfos.push_back(info);
422 std::map<int, ShimInfo> shimInfos;
424 if (routing == TraceShimRouting::Single) {
427 int targetCol = egressShimColFromIR;
428 if (targetCol < 0 || targetCol >= targetModel.columns() ||
429 !targetModel.isShimNOCTile(targetCol, 0)) {
430 device.emitError() <<
"egress_shim_col " << targetCol
431 <<
" is not a valid shim NOC tile (device has "
432 << targetModel.columns() <<
" columns)";
433 return signalPassFailure();
435 TileOp shimTile =
nullptr;
436 for (
auto tile : device.getOps<TileOp>()) {
437 if (tile.getCol() == targetCol && tile.getRow() == 0) {
444 builder.setInsertionPointToStart(&device.getRegion().front());
445 shimTile = TileOp::create(builder, device.getLoc(), targetCol, 0);
449 shimInfo.shimTile = shimTile;
450 shimInfo.channel = clShimChannel;
451 shimInfo.bdId = clDefaultBdId;
452 shimInfo.traceSources = traceInfos;
454 for (
auto &trace : traceInfos) {
455 if (trace.startBroadcast && !shimInfo.startBroadcast)
456 shimInfo.startBroadcast = trace.startBroadcast;
457 if (trace.stopBroadcast && !shimInfo.stopBroadcast)
458 shimInfo.stopBroadcast = trace.stopBroadcast;
462 for (
auto &info : traceInfos) {
463 int col = info.tile.getCol();
464 if (shimInfos.find(col) == shimInfos.end()) {
465 shimInfos[col] = shimInfo;
468 if (shimInfos.find(targetCol) == shimInfos.end()) {
469 shimInfos[targetCol] = shimInfo;
477 if (clLateralRouting) {
478 std::set<int> activeColumns;
479 device.walk([&](CoreOp core) {
480 auto coreTile = cast<TileOp>(core.getTile().getDefiningOp());
481 activeColumns.insert(coreTile.getCol());
485 std::map<int, int> redirects;
486 for (
auto &[col, shimInfo] : shimInfos) {
487 int curTarget = shimInfo.shimTile.getCol();
488 if (redirects.count(curTarget))
492 if (clLateralTargetCol >= 0) {
494 if (clLateralTargetCol == curTarget)
497 int numCols = targetModel.columns();
498 if (clLateralTargetCol >= numCols ||
499 !targetModel.isShimNOCTile(clLateralTargetCol, 0)) {
500 device.emitError() <<
"lateral-target-col " << clLateralTargetCol
501 <<
" is not a valid shim NOC tile (device has "
502 << numCols <<
" columns)";
503 return signalPassFailure();
505 spare = clLateralTargetCol;
508 if (activeColumns.count(curTarget) == 0)
510 spare = findNearestSpareColumn(curTarget, activeColumns, targetModel);
513 redirects[curTarget] = spare;
517 if (!redirects.empty()) {
518 std::map<int, ShimInfo> newShimInfos;
519 for (
auto &[col, shimInfo] : shimInfos) {
520 int curTarget = shimInfo.shimTile.getCol();
521 auto it = redirects.find(curTarget);
522 if (it != redirects.end()) {
523 int newTarget = it->second;
524 shimInfo.shimTile = getOrCreateShim(device, builder, newTarget);
526 newShimInfos[col] = shimInfo;
528 shimInfos = std::move(newShimInfos);
535 auto usedChannels = scanUsedS2MMChannels(device);
536 auto s2mmChannels = [&](
int col) {
537 return static_cast<int>(
538 targetModel.getNumDestShimMuxConnections(col, 0, WireBundle::DMA));
541 for (
auto &[col, shimInfo] : shimInfos) {
542 int shimCol = shimInfo.shimTile.getCol();
543 auto usedIt = usedChannels.find(shimCol);
544 if (usedIt == usedChannels.end())
547 const auto &usedSet = usedIt->second;
548 int freeCount = s2mmChannels(shimCol) -
static_cast<int>(usedSet.size());
550 if (freeCount <= 0) {
552 if (clLateralRouting) {
553 std::set<int> activeColumns;
554 device.walk([&](CoreOp core) {
555 auto coreTile = cast<TileOp>(core.getTile().getDefiningOp());
556 activeColumns.insert(coreTile.getCol());
559 for (
auto &[fullCol, channels] : usedChannels) {
560 if (
static_cast<int>(channels.size()) >= s2mmChannels(fullCol))
561 activeColumns.insert(fullCol);
564 findNearestSpareColumn(shimCol, activeColumns, targetModel);
566 shimInfo.shimTile = getOrCreateShim(device, builder, spare);
569 shimInfo.channel = usedChannels[spare].count(clShimChannel)
577 <<
"no S2MM channels available on shim tile at column " << shimCol
578 <<
" (both channels in use by existing flows or objectFifos); "
579 "set egress_shim_col to a column with a free channel, or "
580 "enable lateral-routing to redirect to a spare column";
581 return signalPassFailure();
584 if (freeCount == 1) {
586 int freeChannel = -1;
587 for (
int ch = 0; ch < s2mmChannels(shimCol); ch++) {
588 if (usedSet.count(ch) == 0) {
593 shimInfo.channel = freeChannel;
598 for (
auto &[col, shimInfo] : shimInfos) {
599 int shimCol = shimInfo.shimTile.getCol();
600 std::set<int> available;
601 for (
int ch = 0; ch < s2mmChannels(shimCol); ch++)
602 available.insert(ch);
603 auto usedIt = usedChannels.find(shimCol);
604 if (usedIt != usedChannels.end()) {
605 for (
int ch : usedIt->second)
609 buildChannelDescriptors(shimInfo.traceSources.size(),
610 shimInfo.channel, shimInfo.bdId, available);
612 for (
size_t i = 0; i < shimInfo.traceSources.size(); i++) {
613 shimInfo.traceChannelAssignment.push_back(i % shimInfo.channels.size());
619 for (
auto &info : traceInfos) {
620 if (!info.tile.isShimTile())
623 int col = info.tile.getCol();
624 auto shimIt = shimInfos.find(col);
625 if (shimIt == shimInfos.end())
629 if (shimIt->second.shimTile.getTileID() != info.tile.getTileID())
634 for (
auto &op : info.traceOp.getBody().getOps()) {
635 if (
auto startOp = dyn_cast<TraceStartEventOp>(op)) {
636 if (startOp.getBroadcast()) {
638 startOp->removeAttr(
"broadcast");
639 startOp->setAttr(
"event", TraceEventAttr::get(builder.getContext(),
640 builder.getStringAttr(
644 if (
auto stopOp = dyn_cast<TraceStopEventOp>(op)) {
645 if (stopOp.getBroadcast()) {
647 stopOp->removeAttr(
"broadcast");
648 stopOp->setAttr(
"event", TraceEventAttr::get(builder.getContext(),
649 builder.getStringAttr(
655 info.startBroadcast = std::nullopt;
656 info.stopBroadcast = std::nullopt;
661 Block &deviceBlock = device.getRegion().front();
662 builder.setInsertionPoint(deviceBlock.getTerminator());
664 for (
auto &info : traceInfos) {
666 int col = info.tile.getCol();
667 ShimInfo &shimInfo = shimInfos[col];
671 for (
size_t i = 0; i < shimInfo.traceSources.size(); i++) {
672 if (shimInfo.traceSources[i].packetId == info.packetId) {
673 chanIdx = shimInfo.traceChannelAssignment[i];
677 auto &chanDesc = shimInfo.channels[chanIdx];
680 auto packetFlowOp = PacketFlowOp::create(
681 builder, device.getLoc(), builder.getI8IntegerAttr(info.packetId),
684 Block *flowBody =
new Block();
685 packetFlowOp.getPorts().push_back(flowBody);
686 OpBuilder flowBuilder = OpBuilder::atBlockEnd(flowBody);
688 PacketSourceOp::create(flowBuilder, device.getLoc(),
689 Value(info.tile.getResult()), info.tracePort,
692 PacketDestOp::create(flowBuilder, device.getLoc(),
693 Value(shimInfo.shimTile.getResult()),
694 WireBundle::DMA, chanDesc.channel);
696 EndOp::create(flowBuilder, device.getLoc());
698 packetFlowOp->setAttr(
"keep_pkt_header", builder.getBoolAttr(
true));
702 for (
const SequenceTraceConfig &cfg : seqConfigs) {
703 if (failed(emitSequenceTraceOps(cfg, traceInfos, shimInfos)))
704 return signalPassFailure();
711 LogicalResult emitSequenceTraceOps(
const SequenceTraceConfig &cfg,
712 std::vector<TraceInfo> &traceInfos,
713 std::map<int, ShimInfo> &shimInfos) {
714 DeviceOp device = getOperation();
715 OpBuilder builder(device);
716 const auto &targetModel = device.getTargetModel();
717 RuntimeSequenceOp runtimeSeq = cfg.seq;
718 int bufferSizeBytes = cfg.bufferSizeBytes;
723 size_t maxChannels = 1;
724 for (
auto &[col, shimInfo] : shimInfos)
725 maxChannels = std::max(maxChannels, shimInfo.channels.size());
726 int traceBytesClaimed =
static_cast<int>(maxChannels) * bufferSizeBytes;
731 BlockArgument appendedTraceArg;
732 if (cfg.reuseOutputBuffer) {
733 traceBuffer = runtimeSeq.getBody().getArguments().back();
737 appendedTraceArg = runtimeSeq.getBody().front().addArgument(
738 MemRefType::get({traceBytesClaimed},
739 IntegerType::get(device.getContext(), 8)),
740 runtimeSeq.getLoc());
741 traceBuffer = appendedTraceArg;
744 runtimeSeq.setTraceBufferAttr(TraceBufferAttr::get(
746 appendedTraceArg ? appendedTraceArg.getArgNumber()
747 : runtimeSeq.getBody().getNumArguments() - 1,
748 cfg.bufferOffset, traceBytesClaimed,
bool(appendedTraceArg)));
750 Block &seqBlock = runtimeSeq.getBody().front();
753 Operation *lastStartConfig =
nullptr;
754 for (
auto &op : seqBlock.getOperations()) {
755 if (isa<TraceStartConfigOp>(op)) {
756 lastStartConfig = &op;
761 if (lastStartConfig) {
762 builder.setInsertionPointAfter(lastStartConfig);
764 builder.setInsertionPointToStart(&seqBlock);
768 std::set<std::tuple<int, int, bool>> processedTiles;
769 for (
auto &info : traceInfos) {
770 if (!info.startBroadcast)
773 int col = info.tile.getCol();
774 int row = info.tile.getRow();
775 bool isMemTrace = info.packetType == TracePacketType::Mem;
777 if (processedTiles.count({col, row, isMemTrace}))
779 processedTiles.insert({col, row, isMemTrace});
782 if (info.tile.isShimTile()) {
783 auto shimIt = shimInfos.find(col);
784 if (shimIt != shimInfos.end() &&
785 shimIt->second.shimTile.getTileID() == info.tile.getTileID()) {
790 const RegisterInfo *timerReg = targetModel.lookupRegister(
791 "Timer_Control", info.tile.getTileID(), isMemTrace);
793 return info.traceOp.emitError(
794 "Failed to lookup Timer_Control register");
796 std::string broadcastEventName;
797 if (info.tile.isShimTile()) {
799 "BROADCAST_A_" + std::to_string(*info.startBroadcast);
802 "BROADCAST_" + std::to_string(*info.startBroadcast);
805 auto broadcastEvent = targetModel.lookupEvent(
806 broadcastEventName, info.tile.getTileID(), isMemTrace);
807 if (!broadcastEvent) {
808 info.traceOp.emitError() <<
"Failed to lookup broadcast event '"
809 << broadcastEventName <<
"'";
812 const BitFieldInfo *resetField = timerReg->
getField(
"Reset_Event");
814 return info.traceOp.emitError(
815 "Failed to lookup Reset_Event field in Timer_Control");
816 auto timerCtrlValue =
817 targetModel.encodeFieldValue(*resetField, *broadcastEvent);
819 return info.traceOp.emitError(
"Broadcast event does not fit in "
820 "Timer_Control's Reset_Event field");
822 xilinx::AIEX::NpuWrite32Op::create(
823 builder, runtimeSeq.getLoc(),
828 nullptr, builder.getI32IntegerAttr(col),
829 builder.getI32IntegerAttr(row));
833 std::set<int> configuredShimCols;
834 for (
auto &[col, shimInfo] : shimInfos) {
835 int shimCol = shimInfo.shimTile.getCol();
836 if (!configuredShimCols.insert(shimCol).second)
839 for (
auto [chanIdx, chanDesc] : llvm::enumerate(shimInfo.channels)) {
841 int bufferLengthWords = bufferSizeBytes / 4;
844 xilinx::AIEX::NpuWriteBdOp::create(
845 builder, runtimeSeq.getLoc(),
868 mlir::IntegerAttr());
872 uint32_t bdAddress = computeBDAddress(shimCol, chanDesc.bdId,
873 shimInfo.shimTile, targetModel);
875 cfg.bufferOffset +
static_cast<int>(chanIdx) * bufferSizeBytes;
876 xilinx::AIEX::NpuAddressPatchOp::create(
877 builder, runtimeSeq.getLoc(), bdAddress, mlir::Value(),
882 std::optional<uint32_t> ctrlAddr =
883 computeCtrlAddress(DMAChannelDir::S2MM, chanDesc.channel,
884 shimInfo.shimTile, targetModel);
886 return shimInfo.shimTile.emitError(
887 "Failed to lookup the S2MM control register");
888 ensureControllerIdAttr(shimInfo.shimTile, targetModel);
890 shimInfo.shimTile->getAttrOfType<PacketInfoAttr>(
"controller_id");
891 std::string ctrlRegName =
892 (chanDesc.channel == 0) ?
"DMA_S2MM_0_Ctrl" :
"DMA_S2MM_1_Ctrl";
893 const RegisterInfo *ctrlReg = targetModel.lookupRegister(
894 ctrlRegName, shimInfo.shimTile.getTileID());
896 return shimInfo.shimTile.emitError(llvm::Twine(
"Failed to lookup ") +
898 const BitFieldInfo *ctrlIdField = ctrlReg->
getField(
"Controller_ID");
900 return shimInfo.shimTile.emitError(
901 "Failed to lookup Controller_ID field in " +
902 llvm::Twine(ctrlRegName));
904 targetModel.encodeFieldValue(*ctrlIdField, ctrlIdAttr.assignedId());
906 return shimInfo.shimTile.emitError(
907 "Controller ID does not fit in the Controller_ID field");
908 auto ctrlIdMask = targetModel.getFieldMask(*ctrlIdField);
910 return shimInfo.shimTile.emitError(
911 "Controller_ID field does not fit in 32-bit register");
912 xilinx::AIEX::NpuMaskWrite32Op::create(
913 builder, runtimeSeq.getLoc(),
917 nullptr, builder.getI32IntegerAttr(shimCol),
918 builder.getI32IntegerAttr(0));
921 std::string taskQueueRegName = (chanDesc.channel == 0)
922 ?
"DMA_S2MM_0_Task_Queue"
923 :
"DMA_S2MM_1_Task_Queue";
924 const RegisterInfo *queueReg = targetModel.lookupRegister(
925 taskQueueRegName, shimInfo.shimTile.getTileID());
927 return shimInfo.shimTile.emitError(llvm::Twine(
"Failed to lookup ") +
929 const BitFieldInfo *tokenField =
930 queueReg->
getField(
"Enable_Token_Issue");
931 const BitFieldInfo *bdIdField = queueReg->
getField(
"Start_BD_ID");
932 if (!tokenField || !bdIdField)
933 return shimInfo.shimTile.emitError(
934 "Failed to lookup Enable_Token_Issue or Start_BD_ID fields");
935 auto tokenValue = targetModel.encodeFieldValue(*tokenField, 1);
937 targetModel.encodeFieldValue(*bdIdField, chanDesc.bdId);
939 return shimInfo.shimTile.emitError(
940 "Enable_Token_Issue does not fit in the field of " +
941 llvm::Twine(taskQueueRegName));
943 return shimInfo.shimTile.emitError(
944 "BD ID does not fit in the Start_BD_ID field of " +
945 llvm::Twine(taskQueueRegName));
946 uint32_t queueValue = *tokenValue | *bdIdValue;
947 xilinx::AIEX::NpuWrite32Op::create(
948 builder, runtimeSeq.getLoc(),
952 nullptr, builder.getI32IntegerAttr(shimCol),
953 builder.getI32IntegerAttr(0));
957 if (shimInfo.startBroadcast) {
959 auto userEvent1 = targetModel.lookupEvent(
960 "USER_EVENT_1", shimInfo.shimTile.getTileID(),
false);
962 return shimInfo.shimTile.emitError(
963 "Failed to lookup USER_EVENT_1 event");
965 const RegisterInfo *shimTimerReg = targetModel.lookupRegister(
966 "Timer_Control", shimInfo.shimTile.getTileID(),
false);
968 return shimInfo.shimTile.emitError(
969 "Failed to lookup shim Timer_Control");
970 const BitFieldInfo *shimResetField =
971 shimTimerReg->
getField(
"Reset_Event");
973 return shimInfo.shimTile.emitError(
974 "Failed to lookup Reset_Event in shim Timer_Control");
975 auto shimTimerCtrlValue =
976 targetModel.encodeFieldValue(*shimResetField, *userEvent1);
977 if (!shimTimerCtrlValue)
978 return shimInfo.shimTile.emitError(
979 "USER_EVENT_1 does not fit in shim "
980 "Timer_Control's Reset_Event field");
981 xilinx::AIEX::NpuWrite32Op::create(
982 builder, runtimeSeq.getLoc(),
986 *shimTimerCtrlValue),
987 nullptr, builder.getI32IntegerAttr(shimCol),
988 builder.getI32IntegerAttr(0));
991 std::string broadcastRegName =
992 "Event_Broadcast" + std::to_string(*shimInfo.startBroadcast) +
"_A";
993 const RegisterInfo *broadcastReg = targetModel.lookupRegister(
994 broadcastRegName, shimInfo.shimTile.getTileID());
996 return shimInfo.shimTile.emitError(llvm::Twine(
"Failed to lookup ") +
998 xilinx::AIEX::NpuWrite32Op::create(
999 builder, runtimeSeq.getLoc(),
1003 nullptr, builder.getI32IntegerAttr(shimCol),
1004 builder.getI32IntegerAttr(0));
1007 const RegisterInfo *eventGenReg = targetModel.lookupRegister(
1008 "Event_Generate", shimInfo.shimTile.getTileID());
1010 return shimInfo.shimTile.emitError(
1011 "Failed to lookup Event_Generate register");
1012 xilinx::AIEX::NpuWrite32Op::create(
1013 builder, runtimeSeq.getLoc(),
1017 nullptr, builder.getI32IntegerAttr(shimCol),
1018 builder.getI32IntegerAttr(0));
1023 builder.setInsertionPointToEnd(&seqBlock);
1025 std::set<int> stoppedShimCols;
1026 for (
auto &[col, shimInfo] : shimInfos) {
1027 if (!shimInfo.stopBroadcast)
1030 int shimCol = shimInfo.shimTile.getCol();
1031 if (!stoppedShimCols.insert(shimCol).second)
1034 auto userEvent0 = targetModel.lookupEvent(
1035 "USER_EVENT_0", shimInfo.shimTile.getTileID(),
false);
1037 return shimInfo.shimTile.emitError(
1038 "Failed to lookup USER_EVENT_0 event");
1040 std::string broadcastRegName =
1041 "Event_Broadcast" + std::to_string(*shimInfo.stopBroadcast) +
"_A";
1042 const RegisterInfo *broadcastReg = targetModel.lookupRegister(
1043 broadcastRegName, shimInfo.shimTile.getTileID());
1045 return shimInfo.shimTile.emitError(llvm::Twine(
"Failed to lookup ") +
1047 xilinx::AIEX::NpuWrite32Op::create(
1048 builder, runtimeSeq.getLoc(),
1052 nullptr, builder.getI32IntegerAttr(shimCol),
1053 builder.getI32IntegerAttr(0));
1055 const RegisterInfo *stopEventGenReg = targetModel.lookupRegister(
1056 "Event_Generate", shimInfo.shimTile.getTileID());
1057 if (!stopEventGenReg)
1058 return shimInfo.shimTile.emitError(
1059 "Failed to lookup Event_Generate register");
1060 xilinx::AIEX::NpuWrite32Op::create(
1061 builder, runtimeSeq.getLoc(),
1063 stopEventGenReg->
offset),
1065 nullptr, builder.getI32IntegerAttr(shimCol),
1066 builder.getI32IntegerAttr(0));
1075 void ensureControllerIdAttr(TileOp tile,
const AIETargetModel &tm) {
1076 if (tile->hasAttr(
"controller_id"))
1079 int pktId = idMap[{tile.colIndex(), tile.rowIndex()}];
1080 auto pktInfoAttr = PacketInfoAttr::get(tile->getContext(), 0, pktId);
1081 tile->setAttr(
"controller_id", pktInfoAttr);
1084 uint32_t computeBDAddress(
int col,
int bdId, TileOp shimTile,
1085 const AIETargetModel &tm) {
1086 int row = shimTile.getRow();
1092 std::optional<uint32_t> computeCtrlAddress(DMAChannelDir dir,
int channel,
1094 const AIETargetModel &tm) {
1095 std::string regName;
1096 if (dir == DMAChannelDir::S2MM) {
1097 regName = (channel == 0) ?
"DMA_S2MM_0_Ctrl" :
"DMA_S2MM_1_Ctrl";
1099 regName = (channel == 0) ?
"DMA_MM2S_0_Ctrl" :
"DMA_MM2S_1_Ctrl";
1101 const RegisterInfo *reg = tm.
lookupRegister(regName, shimTile.getTileID());
1103 return std::nullopt;
1110 std::vector<ChannelDescriptor>
1111 buildChannelDescriptors(
size_t numTraces,
int primaryChannel,
int primaryBdId,
1112 const std::set<int> &availableChannels) {
1113 std::vector<ChannelDescriptor> chans;
1114 chans.push_back({primaryChannel, primaryBdId});
1115 if (clDistributeChannels && numTraces > 1 && primaryBdId > 0) {
1116 int ch2 = (primaryChannel == 1) ? 0 : 1;
1119 if (availableChannels.count(ch2)) {
1120 chans.push_back({ch2, primaryBdId - 1});
1127 TileOp getOrCreateShim(DeviceOp device, OpBuilder &builder,
int col) {
1128 for (
auto tile : device.getOps<TileOp>()) {
1129 if (tile.getCol() == col && tile.getRow() == 0)
1132 OpBuilder::InsertionGuard guard(builder);
1133 builder.setInsertionPointToStart(&device.getRegion().front());
1134 return TileOp::create(builder, device.getLoc(), col, 0);
1141 std::map<int, std::set<int>> scanUsedS2MMChannels(DeviceOp device) {
1142 std::map<int, std::set<int>> used;
1144 device.walk([&](FlowOp flow) {
1145 auto destTile = cast<TileOp>(flow.getDest().getDefiningOp());
1146 if (destTile.isShimTile() && flow.getDestBundle() == WireBundle::DMA) {
1147 used[destTile.getCol()].insert(flow.getDestChannel());
1151 device.walk([&](PacketFlowOp pktFlow) {
1152 for (
auto &op : pktFlow.getPorts().front()) {
1153 if (
auto dest = dyn_cast<PacketDestOp>(op)) {
1154 auto destTile = cast<TileOp>(dest.getTile().getDefiningOp());
1155 if (destTile.isShimTile() && dest.getBundle() == WireBundle::DMA) {
1156 used[destTile.getCol()].insert(dest.getChannel());
1162 device.walk([&](ShimDMAAllocationOp alloc) {
1163 if (alloc.getChannelDir() == DMAChannelDir::S2MM) {
1164 auto tile = alloc.getTileOp();
1165 used[tile.getCol()].insert(alloc.getChannelIndex());
1174 SmallVector<int> unpinned;
1175 std::map<int, SmallVector<StringAttr>> unpinnedFifos;
1176 device.walk([&](RouteEndpoint endpoint) {
1177 auto tile = dyn_cast_or_null<TileOp>(endpoint.getTile().getDefiningOp());
1178 if (!tile || !tile.isShimTile() ||
1179 endpoint.getRouteBundle() != WireBundle::DMA ||
1180 endpoint.getRouteDirection() != DMAChannelDir::S2MM)
1182 if (std::optional<int> pin = endpoint.getRouteChannel())
1183 used[tile.getCol()].insert(*pin);
1185 unpinned.push_back(tile.getCol());
1187 device.walk([&](ObjectFifoCreateOp fifo) {
1190 for (
auto [i, consumer] : llvm::enumerate(fifo.getConsumerTiles())) {
1191 auto tile = dyn_cast_or_null<TileOp>(consumer.getDefiningOp());
1192 if (!tile || !tile.isShimTile())
1194 ObjectFifoEndPortAttr port = fifo.getConsumerPort(i);
1195 if (std::optional<int> pin = port ? port.channelIndex() : std::nullopt)
1196 used[tile.getCol()].insert(*pin);
1198 unpinnedFifos[tile.getCol()].push_back(fifo.getSymNameAttr());
1201 std::optional<ShimTransferSpans> spans;
1202 for (
auto &[col, fifos] : unpinnedFifos) {
1203 size_t groups = fifos.size();
1208 [&](StringAttr symbol) {
1212 groups = spans->plan(fifos, fifos, {}).groups.size();
1214 unpinned.append(groups, col);
1216 for (
int col : unpinned) {
1217 auto &channels = used[col];
1219 while (channels.count(ch))
1221 channels.insert(ch);
1229 int findNearestSpareColumn(
int sourceCol,
const std::set<int> &activeColumns,
1230 const AIETargetModel &targetModel) {
1232 int bestDist = INT_MAX;
1233 int numCols = targetModel.
columns();
1234 for (
int c = 0; c < numCols; c++) {
1235 if (activeColumns.count(c) == 0 && targetModel.
isShimNOCTile(c, 0)) {
1236 int dist = std::abs(c - sourceCol);
1237 if (dist > 0 && dist < bestDist) {
1249std::unique_ptr<OperationPass<DeviceOp>>
1251 return std::make_unique<AIEInsertTraceFlowsPass>();
llvm::DenseMap< TileID, int > getTileToControllerIdMap(bool columnWiseUniqueIDs=true) const
Return a mapping from tile coordinates to controller IDs.
const RegisterInfo * lookupRegister(llvm::StringRef name, TileID tile, bool isMem=false) const
Register Database API - provides access to register and event information for trace configuration and...
virtual uint64_t getDmaBdAddress(int col, int row, uint32_t bd_id, int channel=-1, AIE::DMAChannelDir direction=AIE::DMAChannelDir::MM2S) const =0
Return the array address of the dma buffer descriptor for the given col, row, buffer descriptor id,...
virtual uint32_t getDmaBdAddressOffset(int col, int row) const =0
Return the offset of the base address field within the shim dma buffer descriptor.
bool isShimNOCTile(int col, int row) const
Return true if the given tile is a ShimNOC tile.
virtual int columns() const =0
Return the number of columns in the device.
static llvm::DenseMap< mlir::StringAttr, llvm::SmallVector< ShimTransferSpans::Gate > > provenShimGates(DeviceOp device)
static std::optional< End > endOf(DeviceOp device, mlir::StringAttr symbol)
The end a runtime sequence of device transfers through when it names symbol: a shim DMA route end,...
mlir::Value createConstantI32(mlir::OpBuilder &builder, mlir::Location loc, uint32_t value)
std::unique_ptr< mlir::OperationPass< DeviceOp > > createAIEInsertTraceFlowsPass()
const BitFieldInfo * getField(llvm::StringRef fieldName) const