MLIR-AIE 1.4.6.dev
AIEInsertTraceFlows.cpp
Go to the documentation of this file.
1//===- AIEInsertTraceFlows.cpp ----------------------------------*- C++ -*-===//
2//
3// Copyright (C) 2026 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7
14
15#include "mlir/Dialect/Arith/IR/Arith.h"
16#include "mlir/IR/Attributes.h"
17#include "mlir/Pass/Pass.h"
18
19#include "llvm/ADT/STLExtras.h"
20
21#include <climits>
22#include <map>
23#include <set>
24
25namespace xilinx::AIE {
26#define GEN_PASS_DEF_AIEINSERTTRACEFLOWS
27#include "aie/Dialect/AIE/Transforms/AIEPasses.h.inc"
28} // namespace xilinx::AIE
29
30using namespace mlir;
31using namespace xilinx;
32using namespace xilinx::AIE;
33
34namespace {
35
36struct TraceInfo {
37 TraceOp traceOp;
38 TileOp tile;
39 int packetId;
40 TracePacketType packetType;
41 WireBundle tracePort; // Trace:0 (core) or Trace:1 (mem)
42 int traceChannel; // Port number (0 for core, 1 for mem)
43 std::optional<int>
44 startBroadcast; // Broadcast channel for timer sync (if any)
45 std::optional<int> stopBroadcast; // Broadcast channel for trace stop (if any)
46};
47
48/// Per-channel DMA resource allocation.
49struct ChannelDescriptor {
50 int channel; // S2MM channel number
51 int bdId; // Buffer descriptor ID
52};
53
54struct ShimInfo {
55 TileOp shimTile;
56 int channel; // S2MM channel
57 int bdId; // Buffer descriptor ID
58 std::vector<TraceInfo> traceSources; // All traces routed to this shim
59 std::optional<int> startBroadcast; // Broadcast to trigger for start
60 std::optional<int> stopBroadcast; // Broadcast to trigger for stop
61 std::vector<ChannelDescriptor> channels; // Per-channel descriptors
62 std::vector<int> traceChannelAssignment; // Per-trace index into channels
63};
64
65/// The trace buffer of one runtime sequence. The trace units and the routes
66/// belong to the device; the buffer belongs to the sequence.
67struct SequenceTraceConfig {
68 RuntimeSequenceOp seq;
69 int bufferSizeBytes;
70 bool reuseOutputBuffer;
71 int bufferOffset; // byte offset of the trace data in the host buffer
72};
73
74// A dynamic (runtime-sized) runtime sequence carries its transfer sizes as
75// scalar block arguments (the dynamic ABI); its memref args are typed at
76// maximum capacity, not the runtime data size. Any non-memref block argument
77// marks it.
78static bool isDynamicRuntimeSequence(RuntimeSequenceOp seq) {
79 return llvm::any_of(seq.getBody().getArguments(), [](BlockArgument arg) {
80 return !isa<MemRefType>(arg.getType());
81 });
82}
83
84struct AIEInsertTraceFlowsPass
85 : xilinx::AIE::impl::AIEInsertTraceFlowsBase<AIEInsertTraceFlowsPass> {
86
87 void runOnOperation() override {
88 DeviceOp device = getOperation();
89 OpBuilder builder(device);
90 const auto &targetModel = device.getTargetModel();
91
92 // Verify no LogicalTileOps remain — placement must run before this pass
93 bool hasLogicalTile = false;
94 device.walk([&](LogicalTileOp op) {
95 op.emitError() << "LogicalTileOp must be resolved to TileOp before "
96 "running -aie-insert-trace-flows (run -aie-place-tiles "
97 "first)";
98 hasLogicalTile = true;
99 });
100 if (hasLogicalTile)
101 return signalPassFailure();
102
103 // Phase 1: Collect all trace operations
104 SmallVector<TraceOp> traces;
105 device.walk([&](TraceOp trace) { traces.push_back(trace); });
106
107 if (traces.empty())
108 return;
109
110 // The trace DMA program below targets one of the two shim S2MM channels
111 // and a shim BD; reject options it cannot encode.
112 int shimChannel = clShimChannel;
113 if (shimChannel < 0 || shimChannel > 1) {
114 device.emitError() << "shim-channel " << shimChannel
115 << " is not a shim S2MM channel; expected 0 or 1";
116 return signalPassFailure();
117 }
118 uint32_t numShimBds = targetModel.getNumBDs(AIETileType::ShimNOCTile);
119 int bdId = clDefaultBdId;
120 if (bdId < 0 || static_cast<uint32_t>(bdId) >= numShimBds) {
121 device.emitError() << "default-bd-id " << bdId
122 << " is not a shim BD; expected [0, " << numShimBds
123 << ")";
124 return signalPassFailure();
125 }
126
127 // Phase 1b: Collect every runtime_sequence with an aie.trace.host_config.
128 // Each one drives the trace units on its own dispatch, so each one gets a
129 // trace buffer and a shim DMA program.
130 SmallVector<RuntimeSequenceOp> runtimeSeqs;
131 SmallVector<TraceHostConfigOp> hostConfigs;
132 RuntimeSequenceOp firstSeq = nullptr;
133 for (auto seq : device.getBody()->getOps<RuntimeSequenceOp>()) {
134 if (!firstSeq)
135 firstSeq = seq;
136 for (auto hc : seq.getBody().front().getOps<TraceHostConfigOp>()) {
137 runtimeSeqs.push_back(seq);
138 hostConfigs.push_back(hc);
139 break;
140 }
141 }
142
143 // Require runtime_sequence when trace ops are present
144 if (!firstSeq) {
145 device.emitError()
146 << "aie.trace ops found but no runtime_sequence defined";
147 return signalPassFailure();
148 }
149
150 // Require trace.host_config in runtime_sequence
151 if (runtimeSeqs.empty()) {
152 firstSeq.emitError()
153 << "runtime_sequence with traces requires aie.trace.host_config";
154 return signalPassFailure();
155 }
156
157 // The packet flows and the stream switch configuration reach the hardware
158 // through the device, so the trace data of every sequence leaves the array
159 // through the same shim.
160 TraceHostConfigOp hostConfig = hostConfigs.front();
161 auto routing = hostConfig.getRouting();
162 int egressShimColFromIR = hostConfig.getEgressShimCol();
163 for (auto other : ArrayRef(hostConfigs).drop_front()) {
164 if (other.getRouting() == routing &&
165 other.getEgressShimCol() == egressShimColFromIR)
166 continue;
167 InFlightDiagnostic err =
168 other.emitError()
169 << "aie.trace.host_config routes trace data differently from another "
170 "runtime_sequence in this device; the routes live in the device's "
171 "stream switches, so every sequence must name the same routing "
172 "and egress column";
173 err.attachNote(hostConfig.getLoc()) << "first aie.trace.host_config here";
174 return signalPassFailure();
175 }
176
177 // The trace buffer is a host buffer, i.e. an argument of the runtime
178 // sequence.
179 //
180 // - Dedicated (default): append a fresh argument to the runtime sequence
181 // for the trace buffer. It lands at the tail, so enabling trace never
182 // perturbs the indices of the data arguments.
183 // - Reuse-output: trace data is written into the tail of the last
184 // existing argument (an output buffer), saving a host buffer. No new
185 // argument is added; the offset skips past the output data.
186 //
187 // Buffer size and reuse mode reach the hardware through the buffer
188 // descriptor that each sequence writes, so each sequence sets its own.
189 SmallVector<SequenceTraceConfig> seqConfigs;
190 for (auto [seq, hc] : llvm::zip(runtimeSeqs, hostConfigs)) {
191 SequenceTraceConfig cfg;
192 cfg.seq = seq;
193 cfg.bufferSizeBytes = hc.getBufferSize();
194 cfg.reuseOutputBuffer = hc.getReuseOutputBuffer();
195 cfg.bufferOffset = 0;
196 if (cfg.reuseOutputBuffer) {
197 auto args = seq.getBody().getArguments();
198 if (args.empty()) {
199 seq.emitError() << "trace.host_config reuse_output_buffer "
200 "requires the runtime_sequence to have at "
201 "least one argument to reuse";
202 return signalPassFailure();
203 }
204 if (isDynamicRuntimeSequence(seq)) {
205 seq.emitError()
206 << "trace.host_config reuse_output_buffer=true cannot be used "
207 "with a dynamic (runtime-sized) runtime_sequence: the trace "
208 "offset would be computed from the last tensor's maximum "
209 "static size, not its runtime transfer size, so trace data "
210 "would be written past the output buffer. Use a separate "
211 "trace buffer instead (reuse_output_buffer=false)";
212 return signalPassFailure();
213 }
214 auto memrefType = cast<MemRefType>(args.back().getType());
215 cfg.bufferOffset = memrefType.getNumElements() *
216 (memrefType.getElementTypeBitWidth() / 8);
217 }
218 seqConfigs.push_back(cfg);
219 }
220
221 for (auto hc : hostConfigs)
222 hc.erase();
223
224 // Phase 2: Analyze traces and allocate resources
225 std::vector<TraceInfo> traceInfos;
226
227 // Precompute auto-allocated packet IDs in (col, row) order. Walking
228 // `traces` in IR order would make IDs depend on the order trace ops
229 // were emitted, which in turn depends on the placer's worker-to-tile
230 // mapping. Two placements that produce the same tile set would then
231 // get different trace overlays and exercise different routing-rule
232 // layouts -- a coupling that has produced false-match routing
233 // failures (e.g. mask=17 matching id 4 on a column-0 switch port).
234 // Sorting by (col, row) makes the id-to-tile binding a pure
235 // function of the active trace tile set.
236 llvm::DenseMap<Operation *, int> autoPacketIds;
237 {
238 // Wrapping past the max would alias two traces onto the same id.
239 const int kMaxPacketId = device.getTargetModel().getMaxPacketId();
240
241 // First pass: bucket traces and collect explicit ids. An explicit
242 // id reserves a slot the auto-allocator must avoid, and two
243 // explicit traces pinning the same id is a silent alias today.
244 SmallVector<TraceOp> autoIdTraces;
245 llvm::DenseMap<int, TracePacketOp> explicitIdOwner;
246 for (auto trace : traces) {
247 TracePacketOp explicitPacket = nullptr;
248 for (auto &op : trace.getBody().getOps()) {
249 if (auto p = dyn_cast<TracePacketOp>(op)) {
250 if (p.getId().has_value()) {
251 explicitPacket = p;
252 break;
253 }
254 }
255 }
256 if (!explicitPacket) {
257 autoIdTraces.push_back(trace);
258 continue;
259 }
260 int id = *explicitPacket.getId();
261 auto [it, inserted] = explicitIdOwner.try_emplace(id, explicitPacket);
262 if (!inserted) {
263 InFlightDiagnostic err =
264 explicitPacket.emitError()
265 << "trace packet id " << id
266 << " is already used by another trace; explicit packet ids "
267 "must be unique across the device";
268 err.attachNote(it->second.getLoc())
269 << "previous use of packet id " << id;
270 return signalPassFailure();
271 }
272 }
273 // stable_sort so that multiple traces on the same tile (e.g. core
274 // + mem trace on the same core tile) get ids in their IR-emission
275 // order rather than an unspecified order.
276 llvm::stable_sort(autoIdTraces, [](TraceOp a, TraceOp b) {
277 auto ta = cast<TileOp>(a.getTile().getDefiningOp());
278 auto tb = cast<TileOp>(b.getTile().getDefiningOp());
279 if (ta.getCol() != tb.getCol())
280 return ta.getCol() < tb.getCol();
281 return ta.getRow() < tb.getRow();
282 });
283 // From clPacketIdStart up, past ids user traces pinned and ids the
284 // device holds (PacketIdSpace).
285 PacketIdSpace held(device);
286 int next = clPacketIdStart;
287 for (auto trace : autoIdTraces) {
288 while (next <= kMaxPacketId &&
289 (explicitIdOwner.count(next) || held.isTaken(next)))
290 ++next;
291 if (next > kMaxPacketId) {
292 device.emitError()
293 << "trace overlay needs " << autoIdTraces.size()
294 << " auto-allocated packet IDs starting at " << clPacketIdStart
295 << " (with " << explicitIdOwner.size()
296 << " id(s) reserved by explicit aie.trace.packet ops, and any "
297 "held by packet flows or pinned routes elsewhere in the "
298 "device), but the hardware packet-id field is 5 bits (max "
299 << kMaxPacketId
300 << "); reduce the number of traced tiles, free up an explicit "
301 "id, or raise -packet-id-start only if you can spare the "
302 "lower IDs";
303 return signalPassFailure();
304 }
305 autoPacketIds[trace.getOperation()] = next++;
306 }
307 }
308
309 // Each (tile, trace unit) pair may be configured at most once: a hardware
310 // trace unit emits a single packet stream, so two aie.trace ops targeting
311 // the same unit would race on one packet id and corrupt routing. The
312 // packet type identifies the unit (Core/Mem/MemTile/ShimTile).
313 std::set<std::tuple<int, int, int>> seenTraceUnits; // (col, row, pktType)
314
315 for (auto trace : traces) {
316 auto tile = cast<TileOp>(trace.getTile().getDefiningOp());
317
318 // Find packet ID and type from trace body
319 std::optional<int> packetId;
320 std::optional<TracePacketType> packetType;
321 TracePacketOp existingPacketOp = nullptr;
322 for (auto &op : trace.getBody().getOps()) {
323 if (auto packetOp = dyn_cast<TracePacketOp>(op)) {
324 existingPacketOp = packetOp;
325 if (auto explicitId = packetOp.getId())
326 packetId = explicitId;
327 packetType = packetOp.getType();
328 break;
329 }
330 }
331
332 // Determine packet type from tile type if not specified
333 if (!packetType) {
334 if (tile.isShimTile()) {
335 packetType = TracePacketType::ShimTile;
336 } else if (tile.isMemTile()) {
337 packetType = TracePacketType::MemTile;
338 } else {
339 // Core tile defaults to core type
340 packetType = TracePacketType::Core;
341 }
342 }
343
344 // Reject a second trace op on the same hardware unit of the same tile.
345 auto unitKey = std::make_tuple(tile.getCol(), tile.getRow(),
346 static_cast<int>(*packetType));
347 if (!seenTraceUnits.insert(unitKey).second) {
348 trace.emitError() << "tile (" << tile.getCol() << ", " << tile.getRow()
349 << ") is traced more than once on the same trace "
350 "unit; each unit can have only one trace "
351 "configuration";
352 return signalPassFailure();
353 }
354
355 // Allocate packet ID if not specified (precomputed in (col, row)
356 // order above).
357 if (!packetId)
358 packetId = autoPacketIds.lookup(trace.getOperation());
359
360 // Make sure the in-IR TracePacketOp carries the (possibly
361 // auto-allocated) id so downstream passes (e.g.,
362 // -aie-trace-to-config) see consistent info.
363 if (!existingPacketOp) {
364 OpBuilder traceBuilder(&trace.getBody().front(),
365 trace.getBody().front().begin());
366 TracePacketOp::create(traceBuilder, trace.getLoc(),
367 traceBuilder.getI32IntegerAttr(*packetId),
368 *packetType);
369 } else if (!existingPacketOp.getId().has_value()) {
370 existingPacketOp.setIdAttr(
371 OpBuilder(existingPacketOp).getI32IntegerAttr(*packetId));
372 }
373
374 // Determine trace port based on packet type
375 WireBundle tracePort = WireBundle::Trace;
376 int traceChannel = 0;
377 if (*packetType == TracePacketType::Mem) {
378 traceChannel = 1; // Mem trace uses port 1
379 }
380
381 // Find start/stop configuration (broadcast or event)
382 std::optional<int> startBroadcast;
383 std::optional<int> stopBroadcast;
384 bool hasStartConfig = false;
385 bool hasStopConfig = false;
386 for (auto &op : trace.getBody().getOps()) {
387 if (auto startOp = dyn_cast<TraceStartEventOp>(op)) {
388 hasStartConfig = true;
389 if (startOp.getBroadcast())
390 startBroadcast = startOp.getBroadcast();
391 }
392 if (auto stopOp = dyn_cast<TraceStopEventOp>(op)) {
393 hasStopConfig = true;
394 if (stopOp.getBroadcast())
395 stopBroadcast = stopOp.getBroadcast();
396 }
397 }
398
399 // Require explicit start/stop configuration
400 if (!hasStartConfig) {
401 trace.emitError() << "trace is missing 'aie.trace.start'";
402 return signalPassFailure();
403 }
404 if (!hasStopConfig) {
405 trace.emitError() << "trace is missing 'aie.trace.stop'";
406 return signalPassFailure();
407 }
408
409 TraceInfo info;
410 info.traceOp = trace;
411 info.tile = tile;
412 info.packetId = *packetId;
413 info.packetType = *packetType;
414 info.tracePort = tracePort;
415 info.traceChannel = traceChannel;
416 info.startBroadcast = startBroadcast;
417 info.stopBroadcast = stopBroadcast;
418 traceInfos.push_back(info);
419 }
420
421 // Phase 2b: Select shim tiles based on routing strategy
422 std::map<int, ShimInfo> shimInfos; // col -> ShimInfo
423
424 if (routing == TraceShimRouting::Single) {
425 // All traces route to a single shim, controlled by the egress_shim_col
426 // parameter (default is 0).
427 int targetCol = egressShimColFromIR;
428 if (targetCol < 0 || targetCol >= targetModel.columns() ||
429 !targetModel.isShimNOCTile(targetCol, 0)) {
430 device.emitError() << "egress_shim_col " << targetCol
431 << " is not a valid shim NOC tile (device has "
432 << targetModel.columns() << " columns)";
433 return signalPassFailure();
434 }
435 TileOp shimTile = nullptr;
436 for (auto tile : device.getOps<TileOp>()) {
437 if (tile.getCol() == targetCol && tile.getRow() == 0) {
438 shimTile = tile;
439 break;
440 }
441 }
442
443 if (!shimTile) {
444 builder.setInsertionPointToStart(&device.getRegion().front());
445 shimTile = TileOp::create(builder, device.getLoc(), targetCol, 0);
446 }
447
448 ShimInfo shimInfo;
449 shimInfo.shimTile = shimTile;
450 shimInfo.channel = clShimChannel;
451 shimInfo.bdId = clDefaultBdId;
452 shimInfo.traceSources = traceInfos;
453 // Collect broadcast channels from traces that use them
454 for (auto &trace : traceInfos) {
455 if (trace.startBroadcast && !shimInfo.startBroadcast)
456 shimInfo.startBroadcast = trace.startBroadcast;
457 if (trace.stopBroadcast && !shimInfo.stopBroadcast)
458 shimInfo.stopBroadcast = trace.stopBroadcast;
459 }
460
461 // Map all source columns to the single shim
462 for (auto &info : traceInfos) {
463 int col = info.tile.getCol();
464 if (shimInfos.find(col) == shimInfos.end()) {
465 shimInfos[col] = shimInfo;
466 }
467 }
468 if (shimInfos.find(targetCol) == shimInfos.end()) {
469 shimInfos[targetCol] = shimInfo;
470 }
471 }
472
473 // Phase 2b-lateral: Optionally redirect shim targets to spare columns.
474 // This is a post-processing step that works with ANY routing strategy.
475 // When per-column routing returns, lateral routing will compose with it
476 // (each column's shim redirects to its nearest spare).
477 if (clLateralRouting) {
478 std::set<int> activeColumns;
479 device.walk([&](CoreOp core) {
480 auto coreTile = cast<TileOp>(core.getTile().getDefiningOp());
481 activeColumns.insert(coreTile.getCol());
482 });
483
484 // Collect all unique shim target columns and find redirects
485 std::map<int, int> redirects; // old target col -> new target col
486 for (auto &[col, shimInfo] : shimInfos) {
487 int curTarget = shimInfo.shimTile.getCol();
488 if (redirects.count(curTarget))
489 continue; // already computed
490
491 int spare = -1;
492 if (clLateralTargetCol >= 0) {
493 // Forced lateral target: always redirect unless it's a no-op.
494 if (clLateralTargetCol == curTarget)
495 continue;
496 // Validate the forced column is in range and is a shim NOC tile.
497 int numCols = targetModel.columns();
498 if (clLateralTargetCol >= numCols ||
499 !targetModel.isShimNOCTile(clLateralTargetCol, 0)) {
500 device.emitError() << "lateral-target-col " << clLateralTargetCol
501 << " is not a valid shim NOC tile (device has "
502 << numCols << " columns)";
503 return signalPassFailure();
504 }
505 spare = clLateralTargetCol;
506 } else {
507 // Auto-detect: only redirect from active columns to a spare.
508 if (activeColumns.count(curTarget) == 0)
509 continue; // already spare, no redirect needed
510 spare = findNearestSpareColumn(curTarget, activeColumns, targetModel);
511 }
512 if (spare >= 0)
513 redirects[curTarget] = spare;
514 }
515
516 // Apply redirects: rebuild shimInfos with new target columns
517 if (!redirects.empty()) {
518 std::map<int, ShimInfo> newShimInfos;
519 for (auto &[col, shimInfo] : shimInfos) {
520 int curTarget = shimInfo.shimTile.getCol();
521 auto it = redirects.find(curTarget);
522 if (it != redirects.end()) {
523 int newTarget = it->second;
524 shimInfo.shimTile = getOrCreateShim(device, builder, newTarget);
525 }
526 newShimInfos[col] = shimInfo;
527 }
528 shimInfos = std::move(newShimInfos);
529 }
530 }
531
532 // Phase 2b-conflict: Check S2MM channel availability on target shims.
533 // If channels are occupied, adjust: use free channel, redirect lateral,
534 // or error.
535 auto usedChannels = scanUsedS2MMChannels(device);
536 auto s2mmChannels = [&](int col) {
537 return static_cast<int>(
538 targetModel.getNumDestShimMuxConnections(col, 0, WireBundle::DMA));
539 };
540
541 for (auto &[col, shimInfo] : shimInfos) {
542 int shimCol = shimInfo.shimTile.getCol();
543 auto usedIt = usedChannels.find(shimCol);
544 if (usedIt == usedChannels.end())
545 continue; // No conflicts on this shim
546
547 const auto &usedSet = usedIt->second;
548 int freeCount = s2mmChannels(shimCol) - static_cast<int>(usedSet.size());
549
550 if (freeCount <= 0) {
551 // No channels free -- try lateral redirect
552 if (clLateralRouting) {
553 std::set<int> activeColumns;
554 device.walk([&](CoreOp core) {
555 auto coreTile = cast<TileOp>(core.getTile().getDefiningOp());
556 activeColumns.insert(coreTile.getCol());
557 });
558 // Also treat columns with full shims as "active" for spare search
559 for (auto &[fullCol, channels] : usedChannels) {
560 if (static_cast<int>(channels.size()) >= s2mmChannels(fullCol))
561 activeColumns.insert(fullCol);
562 }
563 int spare =
564 findNearestSpareColumn(shimCol, activeColumns, targetModel);
565 if (spare >= 0) {
566 shimInfo.shimTile = getOrCreateShim(device, builder, spare);
567 // A spare has no core but can still carry one flow; it is not
568 // full, so its other channel is free.
569 shimInfo.channel = usedChannels[spare].count(clShimChannel)
570 ? 1 - clShimChannel
571 : clShimChannel;
572 continue;
573 }
574 }
575 // No lateral available -- emit error
576 device.emitError()
577 << "no S2MM channels available on shim tile at column " << shimCol
578 << " (both channels in use by existing flows or objectFifos); "
579 "set egress_shim_col to a column with a free channel, or "
580 "enable lateral-routing to redirect to a spare column";
581 return signalPassFailure();
582 }
583
584 if (freeCount == 1) {
585 // One channel free -- use it
586 int freeChannel = -1;
587 for (int ch = 0; ch < s2mmChannels(shimCol); ch++) {
588 if (usedSet.count(ch) == 0) {
589 freeChannel = ch;
590 break;
591 }
592 }
593 shimInfo.channel = freeChannel;
594 }
595 }
596
597 // Build channel descriptors and trace-to-channel assignments
598 for (auto &[col, shimInfo] : shimInfos) {
599 int shimCol = shimInfo.shimTile.getCol();
600 std::set<int> available;
601 for (int ch = 0; ch < s2mmChannels(shimCol); ch++)
602 available.insert(ch);
603 auto usedIt = usedChannels.find(shimCol);
604 if (usedIt != usedChannels.end()) {
605 for (int ch : usedIt->second)
606 available.erase(ch);
607 }
608 shimInfo.channels =
609 buildChannelDescriptors(shimInfo.traceSources.size(),
610 shimInfo.channel, shimInfo.bdId, available);
611 // Round-robin assignment of traces to channels
612 for (size_t i = 0; i < shimInfo.traceSources.size(); i++) {
613 shimInfo.traceChannelAssignment.push_back(i % shimInfo.channels.size());
614 }
615 }
616
617 // Phase 2c: Rewrite broadcast to USER_EVENT for destination shim tiles
618 // (shim can't listen for its own broadcast)
619 for (auto &info : traceInfos) {
620 if (!info.tile.isShimTile())
621 continue;
622
623 int col = info.tile.getCol();
624 auto shimIt = shimInfos.find(col);
625 if (shimIt == shimInfos.end())
626 continue;
627
628 // Check if this traced shim tile IS the destination shim
629 if (shimIt->second.shimTile.getTileID() != info.tile.getTileID())
630 continue;
631
632 // This shim is both traced and the destination - rewrite start/stop ops
633 // to use USER_EVENT_1/0 instead of broadcast
634 for (auto &op : info.traceOp.getBody().getOps()) {
635 if (auto startOp = dyn_cast<TraceStartEventOp>(op)) {
636 if (startOp.getBroadcast()) {
637 // Replace broadcast with USER_EVENT_1
638 startOp->removeAttr("broadcast");
639 startOp->setAttr("event", TraceEventAttr::get(builder.getContext(),
640 builder.getStringAttr(
641 "USER_EVENT_1")));
642 }
643 }
644 if (auto stopOp = dyn_cast<TraceStopEventOp>(op)) {
645 if (stopOp.getBroadcast()) {
646 // Replace broadcast with USER_EVENT_0
647 stopOp->removeAttr("broadcast");
648 stopOp->setAttr("event", TraceEventAttr::get(builder.getContext(),
649 builder.getStringAttr(
650 "USER_EVENT_0")));
651 }
652 }
653 }
654
655 info.startBroadcast = std::nullopt;
656 info.stopBroadcast = std::nullopt;
657 }
658
659 // Phase 3: Insert packet flows
660 // Insert before the device terminator
661 Block &deviceBlock = device.getRegion().front();
662 builder.setInsertionPoint(deviceBlock.getTerminator());
663
664 for (auto &info : traceInfos) {
665 // Find target shim for this trace
666 int col = info.tile.getCol();
667 ShimInfo &shimInfo = shimInfos[col];
668
669 // Find this trace's channel assignment
670 int chanIdx = 0;
671 for (size_t i = 0; i < shimInfo.traceSources.size(); i++) {
672 if (shimInfo.traceSources[i].packetId == info.packetId) {
673 chanIdx = shimInfo.traceChannelAssignment[i];
674 break;
675 }
676 }
677 auto &chanDesc = shimInfo.channels[chanIdx];
678
679 // Create packet flow
680 auto packetFlowOp = PacketFlowOp::create(
681 builder, device.getLoc(), builder.getI8IntegerAttr(info.packetId),
682 nullptr, nullptr);
683
684 Block *flowBody = new Block();
685 packetFlowOp.getPorts().push_back(flowBody);
686 OpBuilder flowBuilder = OpBuilder::atBlockEnd(flowBody);
687
688 PacketSourceOp::create(flowBuilder, device.getLoc(),
689 Value(info.tile.getResult()), info.tracePort,
690 info.traceChannel);
691
692 PacketDestOp::create(flowBuilder, device.getLoc(),
693 Value(shimInfo.shimTile.getResult()),
694 WireBundle::DMA, chanDesc.channel);
695
696 EndOp::create(flowBuilder, device.getLoc());
697
698 packetFlowOp->setAttr("keep_pkt_header", builder.getBoolAttr(true));
699 }
700
701 // Phase 4: Program each runtime sequence to drain its own trace buffer.
702 for (const SequenceTraceConfig &cfg : seqConfigs) {
703 if (failed(emitSequenceTraceOps(cfg, traceInfos, shimInfos)))
704 return signalPassFailure();
705 }
706 }
707
708 /// Give `cfg.seq` a trace buffer and the shim DMA program that drains the
709 /// trace stream into it. `traceInfos` and `shimInfos` describe the
710 /// device-wide overlay, which every sequence of the device drives.
711 LogicalResult emitSequenceTraceOps(const SequenceTraceConfig &cfg,
712 std::vector<TraceInfo> &traceInfos,
713 std::map<int, ShimInfo> &shimInfos) {
714 DeviceOp device = getOperation();
715 OpBuilder builder(device);
716 const auto &targetModel = device.getTargetModel();
717 RuntimeSequenceOp runtimeSeq = cfg.seq;
718 int bufferSizeBytes = cfg.bufferSizeBytes;
719
720 // A second S2MM channel doubles the bytes the DMAs write. The host and
721 // -aie-fuse-trace-buffers read the byte count from the argument type, so
722 // the type must cover every channel.
723 size_t maxChannels = 1;
724 for (auto &[col, shimInfo] : shimInfos)
725 maxChannels = std::max(maxChannels, shimInfo.channels.size());
726 int traceBytesClaimed = static_cast<int>(maxChannels) * bufferSizeBytes;
727
728 // `aiex.npu.address_patch` names the buffer by SSA value. `aiex.run` may
729 // inline this sequence into a caller with a different argument list.
730 Value traceBuffer;
731 BlockArgument appendedTraceArg; // null when reusing the output buffer
732 if (cfg.reuseOutputBuffer) {
733 traceBuffer = runtimeSeq.getBody().getArguments().back();
734 } else {
735 // An i8 memref makes the type state the buffer's byte size, which both
736 // the host and -aie-fuse-trace-buffers read.
737 appendedTraceArg = runtimeSeq.getBody().front().addArgument(
738 MemRefType::get({traceBytesClaimed},
739 IntegerType::get(device.getContext(), 8)),
740 runtimeSeq.getLoc());
741 traceBuffer = appendedTraceArg;
742 }
743
744 runtimeSeq.setTraceBufferAttr(TraceBufferAttr::get(
745 device.getContext(),
746 appendedTraceArg ? appendedTraceArg.getArgNumber()
747 : runtimeSeq.getBody().getNumArguments() - 1,
748 cfg.bufferOffset, traceBytesClaimed, bool(appendedTraceArg)));
749
750 Block &seqBlock = runtimeSeq.getBody().front();
751
752 // Find the last TraceStartConfigOp in the runtime sequence
753 Operation *lastStartConfig = nullptr;
754 for (auto &op : seqBlock.getOperations()) {
755 if (isa<TraceStartConfigOp>(op)) {
756 lastStartConfig = &op;
757 }
758 }
759
760 // Insert after the last start_config op, or at start if none found
761 if (lastStartConfig) {
762 builder.setInsertionPointAfter(lastStartConfig);
763 } else {
764 builder.setInsertionPointToStart(&seqBlock);
765 }
766
767 // 4b. Configure timer sync for tiles using broadcast-based start
768 std::set<std::tuple<int, int, bool>> processedTiles;
769 for (auto &info : traceInfos) {
770 if (!info.startBroadcast)
771 continue;
772
773 int col = info.tile.getCol();
774 int row = info.tile.getRow();
775 bool isMemTrace = info.packetType == TracePacketType::Mem;
776
777 if (processedTiles.count({col, row, isMemTrace}))
778 continue;
779 processedTiles.insert({col, row, isMemTrace});
780
781 // Skip destination shim tiles (handled in 4f)
782 if (info.tile.isShimTile()) {
783 auto shimIt = shimInfos.find(col);
784 if (shimIt != shimInfos.end() &&
785 shimIt->second.shimTile.getTileID() == info.tile.getTileID()) {
786 continue;
787 }
788 }
789
790 const RegisterInfo *timerReg = targetModel.lookupRegister(
791 "Timer_Control", info.tile.getTileID(), isMemTrace);
792 if (!timerReg)
793 return info.traceOp.emitError(
794 "Failed to lookup Timer_Control register");
795
796 std::string broadcastEventName;
797 if (info.tile.isShimTile()) {
798 broadcastEventName =
799 "BROADCAST_A_" + std::to_string(*info.startBroadcast);
800 } else {
801 broadcastEventName =
802 "BROADCAST_" + std::to_string(*info.startBroadcast);
803 }
804
805 auto broadcastEvent = targetModel.lookupEvent(
806 broadcastEventName, info.tile.getTileID(), isMemTrace);
807 if (!broadcastEvent) {
808 info.traceOp.emitError() << "Failed to lookup broadcast event '"
809 << broadcastEventName << "'";
810 return failure();
811 }
812 const BitFieldInfo *resetField = timerReg->getField("Reset_Event");
813 if (!resetField)
814 return info.traceOp.emitError(
815 "Failed to lookup Reset_Event field in Timer_Control");
816 auto timerCtrlValue =
817 targetModel.encodeFieldValue(*resetField, *broadcastEvent);
818 if (!timerCtrlValue)
819 return info.traceOp.emitError("Broadcast event does not fit in "
820 "Timer_Control's Reset_Event field");
821
822 xilinx::AIEX::NpuWrite32Op::create(
823 builder, runtimeSeq.getLoc(),
824 AIEX::createConstantI32(builder, runtimeSeq.getLoc(),
825 timerReg->offset),
826 AIEX::createConstantI32(builder, runtimeSeq.getLoc(),
827 *timerCtrlValue),
828 nullptr, builder.getI32IntegerAttr(col),
829 builder.getI32IntegerAttr(row));
830 }
831
832 // 4c-4f. Insert per-shim configurations
833 std::set<int> configuredShimCols;
834 for (auto &[col, shimInfo] : shimInfos) {
835 int shimCol = shimInfo.shimTile.getCol();
836 if (!configuredShimCols.insert(shimCol).second)
837 continue;
838
839 for (auto [chanIdx, chanDesc] : llvm::enumerate(shimInfo.channels)) {
840 // Convert buffer size (bytes) to 32-bit words for buffer_length
841 int bufferLengthWords = bufferSizeBytes / 4;
842
843 // 4c. Write buffer descriptor
844 xilinx::AIEX::NpuWriteBdOp::create(
845 builder, runtimeSeq.getLoc(),
846 shimCol, // column
847 chanDesc.bdId, // bd_id
848 bufferLengthWords, // buffer_length (in 32-bit words)
849 0, // buffer_offset
850 1, // enable_packet
851 0, // out_of_order_id
852 0, // packet_id (not used for reception)
853 0, // packet_type (not used for reception)
854 0, 0, 0, 0, 0,
855 0, // d0_size, d0_stride, d1_size, d1_stride, d2_size, d2_stride
856 0, 0, 0, // iteration_current, iteration_size, iteration_stride
857 0, // next_bd
858 0, // row
859 0, // use_next_bd
860 1, // valid_bd
861 0, 0, 0, 0, 0, // lock_rel_val, lock_rel_id, lock_acq_enable,
862 // lock_acq_val, lock_acq_id
863 0, 0, 0, 0, 0, 0, // d0_zero_before, d1_zero_before, d2_zero_before,
864 // d0_zero_after, d1_zero_after, d2_zero_after
865 clTraceBurstLength, // burst_length
866 // axcache left unset: the blockwrite lowering resolves the
867 // target default, so baking it here would only duplicate it.
868 mlir::IntegerAttr());
869
870 // 4d. Address patch -- the channels split the buffer between them, so
871 // channel i starts bufferSizeBytes past channel i-1.
872 uint32_t bdAddress = computeBDAddress(shimCol, chanDesc.bdId,
873 shimInfo.shimTile, targetModel);
874 int chanOffset =
875 cfg.bufferOffset + static_cast<int>(chanIdx) * bufferSizeBytes;
876 xilinx::AIEX::NpuAddressPatchOp::create(
877 builder, runtimeSeq.getLoc(), bdAddress, /*addr_val=*/mlir::Value(),
878 traceBuffer,
879 AIEX::createConstantI32(builder, runtimeSeq.getLoc(), chanOffset));
880
881 // 4e. DMA channel configuration — set Controller_ID from tile attribute
882 std::optional<uint32_t> ctrlAddr =
883 computeCtrlAddress(DMAChannelDir::S2MM, chanDesc.channel,
884 shimInfo.shimTile, targetModel);
885 if (!ctrlAddr)
886 return shimInfo.shimTile.emitError(
887 "Failed to lookup the S2MM control register");
888 ensureControllerIdAttr(shimInfo.shimTile, targetModel);
889 auto ctrlIdAttr =
890 shimInfo.shimTile->getAttrOfType<PacketInfoAttr>("controller_id");
891 std::string ctrlRegName =
892 (chanDesc.channel == 0) ? "DMA_S2MM_0_Ctrl" : "DMA_S2MM_1_Ctrl";
893 const RegisterInfo *ctrlReg = targetModel.lookupRegister(
894 ctrlRegName, shimInfo.shimTile.getTileID());
895 if (!ctrlReg)
896 return shimInfo.shimTile.emitError(llvm::Twine("Failed to lookup ") +
897 ctrlRegName);
898 const BitFieldInfo *ctrlIdField = ctrlReg->getField("Controller_ID");
899 if (!ctrlIdField)
900 return shimInfo.shimTile.emitError(
901 "Failed to lookup Controller_ID field in " +
902 llvm::Twine(ctrlRegName));
903 auto ctrlIdValue =
904 targetModel.encodeFieldValue(*ctrlIdField, ctrlIdAttr.assignedId());
905 if (!ctrlIdValue)
906 return shimInfo.shimTile.emitError(
907 "Controller ID does not fit in the Controller_ID field");
908 auto ctrlIdMask = targetModel.getFieldMask(*ctrlIdField);
909 if (!ctrlIdMask)
910 return shimInfo.shimTile.emitError(
911 "Controller_ID field does not fit in 32-bit register");
912 xilinx::AIEX::NpuMaskWrite32Op::create(
913 builder, runtimeSeq.getLoc(),
914 AIEX::createConstantI32(builder, runtimeSeq.getLoc(), *ctrlAddr),
915 AIEX::createConstantI32(builder, runtimeSeq.getLoc(), *ctrlIdValue),
916 AIEX::createConstantI32(builder, runtimeSeq.getLoc(), *ctrlIdMask),
917 nullptr, builder.getI32IntegerAttr(shimCol),
918 builder.getI32IntegerAttr(0));
919
920 // Push BD to task queue
921 std::string taskQueueRegName = (chanDesc.channel == 0)
922 ? "DMA_S2MM_0_Task_Queue"
923 : "DMA_S2MM_1_Task_Queue";
924 const RegisterInfo *queueReg = targetModel.lookupRegister(
925 taskQueueRegName, shimInfo.shimTile.getTileID());
926 if (!queueReg)
927 return shimInfo.shimTile.emitError(llvm::Twine("Failed to lookup ") +
928 taskQueueRegName);
929 const BitFieldInfo *tokenField =
930 queueReg->getField("Enable_Token_Issue");
931 const BitFieldInfo *bdIdField = queueReg->getField("Start_BD_ID");
932 if (!tokenField || !bdIdField)
933 return shimInfo.shimTile.emitError(
934 "Failed to lookup Enable_Token_Issue or Start_BD_ID fields");
935 auto tokenValue = targetModel.encodeFieldValue(*tokenField, 1);
936 auto bdIdValue =
937 targetModel.encodeFieldValue(*bdIdField, chanDesc.bdId);
938 if (!tokenValue)
939 return shimInfo.shimTile.emitError(
940 "Enable_Token_Issue does not fit in the field of " +
941 llvm::Twine(taskQueueRegName));
942 if (!bdIdValue)
943 return shimInfo.shimTile.emitError(
944 "BD ID does not fit in the Start_BD_ID field of " +
945 llvm::Twine(taskQueueRegName));
946 uint32_t queueValue = *tokenValue | *bdIdValue;
947 xilinx::AIEX::NpuWrite32Op::create(
948 builder, runtimeSeq.getLoc(),
949 AIEX::createConstantI32(builder, runtimeSeq.getLoc(),
950 queueReg->offset),
951 AIEX::createConstantI32(builder, runtimeSeq.getLoc(), queueValue),
952 nullptr, builder.getI32IntegerAttr(shimCol),
953 builder.getI32IntegerAttr(0));
954 }
955
956 // 4f. Shim timer and broadcast control (only if start broadcast is used)
957 if (shimInfo.startBroadcast) {
958 // Look up USER_EVENT_1 from target model (trigger event)
959 auto userEvent1 = targetModel.lookupEvent(
960 "USER_EVENT_1", shimInfo.shimTile.getTileID(), false);
961 if (!userEvent1)
962 return shimInfo.shimTile.emitError(
963 "Failed to lookup USER_EVENT_1 event");
964
965 const RegisterInfo *shimTimerReg = targetModel.lookupRegister(
966 "Timer_Control", shimInfo.shimTile.getTileID(), false);
967 if (!shimTimerReg)
968 return shimInfo.shimTile.emitError(
969 "Failed to lookup shim Timer_Control");
970 const BitFieldInfo *shimResetField =
971 shimTimerReg->getField("Reset_Event");
972 if (!shimResetField)
973 return shimInfo.shimTile.emitError(
974 "Failed to lookup Reset_Event in shim Timer_Control");
975 auto shimTimerCtrlValue =
976 targetModel.encodeFieldValue(*shimResetField, *userEvent1);
977 if (!shimTimerCtrlValue)
978 return shimInfo.shimTile.emitError(
979 "USER_EVENT_1 does not fit in shim "
980 "Timer_Control's Reset_Event field");
981 xilinx::AIEX::NpuWrite32Op::create(
982 builder, runtimeSeq.getLoc(),
983 AIEX::createConstantI32(builder, runtimeSeq.getLoc(),
984 shimTimerReg->offset),
985 AIEX::createConstantI32(builder, runtimeSeq.getLoc(),
986 *shimTimerCtrlValue),
987 nullptr, builder.getI32IntegerAttr(shimCol),
988 builder.getI32IntegerAttr(0));
989
990 // Configure broadcast register with USER_EVENT_1
991 std::string broadcastRegName =
992 "Event_Broadcast" + std::to_string(*shimInfo.startBroadcast) + "_A";
993 const RegisterInfo *broadcastReg = targetModel.lookupRegister(
994 broadcastRegName, shimInfo.shimTile.getTileID());
995 if (!broadcastReg)
996 return shimInfo.shimTile.emitError(llvm::Twine("Failed to lookup ") +
997 broadcastRegName);
998 xilinx::AIEX::NpuWrite32Op::create(
999 builder, runtimeSeq.getLoc(),
1000 AIEX::createConstantI32(builder, runtimeSeq.getLoc(),
1001 broadcastReg->offset),
1002 AIEX::createConstantI32(builder, runtimeSeq.getLoc(), *userEvent1),
1003 nullptr, builder.getI32IntegerAttr(shimCol),
1004 builder.getI32IntegerAttr(0));
1005
1006 // Generate USER_EVENT_1 to trigger the broadcast
1007 const RegisterInfo *eventGenReg = targetModel.lookupRegister(
1008 "Event_Generate", shimInfo.shimTile.getTileID());
1009 if (!eventGenReg)
1010 return shimInfo.shimTile.emitError(
1011 "Failed to lookup Event_Generate register");
1012 xilinx::AIEX::NpuWrite32Op::create(
1013 builder, runtimeSeq.getLoc(),
1014 AIEX::createConstantI32(builder, runtimeSeq.getLoc(),
1015 eventGenReg->offset),
1016 AIEX::createConstantI32(builder, runtimeSeq.getLoc(), *userEvent1),
1017 nullptr, builder.getI32IntegerAttr(shimCol),
1018 builder.getI32IntegerAttr(0));
1019 }
1020 }
1021
1022 // Phase 4g: Insert trace stop at end of runtime sequence
1023 builder.setInsertionPointToEnd(&seqBlock);
1024
1025 std::set<int> stoppedShimCols;
1026 for (auto &[col, shimInfo] : shimInfos) {
1027 if (!shimInfo.stopBroadcast)
1028 continue;
1029
1030 int shimCol = shimInfo.shimTile.getCol();
1031 if (!stoppedShimCols.insert(shimCol).second)
1032 continue;
1033
1034 auto userEvent0 = targetModel.lookupEvent(
1035 "USER_EVENT_0", shimInfo.shimTile.getTileID(), false);
1036 if (!userEvent0)
1037 return shimInfo.shimTile.emitError(
1038 "Failed to lookup USER_EVENT_0 event");
1039
1040 std::string broadcastRegName =
1041 "Event_Broadcast" + std::to_string(*shimInfo.stopBroadcast) + "_A";
1042 const RegisterInfo *broadcastReg = targetModel.lookupRegister(
1043 broadcastRegName, shimInfo.shimTile.getTileID());
1044 if (!broadcastReg)
1045 return shimInfo.shimTile.emitError(llvm::Twine("Failed to lookup ") +
1046 broadcastRegName);
1047 xilinx::AIEX::NpuWrite32Op::create(
1048 builder, runtimeSeq.getLoc(),
1049 AIEX::createConstantI32(builder, runtimeSeq.getLoc(),
1050 broadcastReg->offset),
1051 AIEX::createConstantI32(builder, runtimeSeq.getLoc(), *userEvent0),
1052 nullptr, builder.getI32IntegerAttr(shimCol),
1053 builder.getI32IntegerAttr(0));
1054
1055 const RegisterInfo *stopEventGenReg = targetModel.lookupRegister(
1056 "Event_Generate", shimInfo.shimTile.getTileID());
1057 if (!stopEventGenReg)
1058 return shimInfo.shimTile.emitError(
1059 "Failed to lookup Event_Generate register");
1060 xilinx::AIEX::NpuWrite32Op::create(
1061 builder, runtimeSeq.getLoc(),
1062 AIEX::createConstantI32(builder, runtimeSeq.getLoc(),
1063 stopEventGenReg->offset),
1064 AIEX::createConstantI32(builder, runtimeSeq.getLoc(), *userEvent0),
1065 nullptr, builder.getI32IntegerAttr(shimCol),
1066 builder.getI32IntegerAttr(0));
1067 }
1068
1069 return success();
1070 }
1071
1072private:
1073 /// Ensure a tile has a controller_id attribute. If missing, assign one
1074 /// using the target model's mapping.
1075 void ensureControllerIdAttr(TileOp tile, const AIETargetModel &tm) {
1076 if (tile->hasAttr("controller_id"))
1077 return;
1078 auto idMap = tm.getTileToControllerIdMap();
1079 int pktId = idMap[{tile.colIndex(), tile.rowIndex()}];
1080 auto pktInfoAttr = PacketInfoAttr::get(tile->getContext(), 0, pktId);
1081 tile->setAttr("controller_id", pktInfoAttr);
1082 }
1083
1084 uint32_t computeBDAddress(int col, int bdId, TileOp shimTile,
1085 const AIETargetModel &tm) {
1086 int row = shimTile.getRow();
1087 return tm.getDmaBdAddress(col, row, bdId) +
1088 tm.getDmaBdAddressOffset(col, row);
1089 }
1090
1091 // Compute DMA control register address
1092 std::optional<uint32_t> computeCtrlAddress(DMAChannelDir dir, int channel,
1093 TileOp shimTile,
1094 const AIETargetModel &tm) {
1095 std::string regName;
1096 if (dir == DMAChannelDir::S2MM) {
1097 regName = (channel == 0) ? "DMA_S2MM_0_Ctrl" : "DMA_S2MM_1_Ctrl";
1098 } else { // MM2S
1099 regName = (channel == 0) ? "DMA_MM2S_0_Ctrl" : "DMA_MM2S_1_Ctrl";
1100 }
1101 const RegisterInfo *reg = tm.lookupRegister(regName, shimTile.getTileID());
1102 if (!reg)
1103 return std::nullopt;
1104 return reg->offset;
1105 }
1106
1107 /// Build channel descriptors. Always includes the primary channel.
1108 /// Adds a secondary channel when distribute-channels is enabled and there
1109 /// are multiple traces. AIE2 shim tiles have exactly 2 S2MM DMA channels.
1110 std::vector<ChannelDescriptor>
1111 buildChannelDescriptors(size_t numTraces, int primaryChannel, int primaryBdId,
1112 const std::set<int> &availableChannels) {
1113 std::vector<ChannelDescriptor> chans;
1114 chans.push_back({primaryChannel, primaryBdId});
1115 if (clDistributeChannels && numTraces > 1 && primaryBdId > 0) {
1116 int ch2 = (primaryChannel == 1) ? 0 : 1;
1117 // Only add secondary channel if it's available (not claimed by existing
1118 // flows)
1119 if (availableChannels.count(ch2)) {
1120 chans.push_back({ch2, primaryBdId - 1});
1121 }
1122 }
1123 return chans;
1124 }
1125
1126 /// Find or create a shim tile at the given column.
1127 TileOp getOrCreateShim(DeviceOp device, OpBuilder &builder, int col) {
1128 for (auto tile : device.getOps<TileOp>()) {
1129 if (tile.getCol() == col && tile.getRow() == 0)
1130 return tile;
1131 }
1132 OpBuilder::InsertionGuard guard(builder);
1133 builder.setInsertionPointToStart(&device.getRegion().front());
1134 return TileOp::create(builder, device.getLoc(), col, 0);
1135 }
1136
1137 /// Scan the device for existing S2MM channel claims on shim tiles.
1138 /// Checks aie.flow destinations, aie.packet_flow destinations,
1139 /// ShimDMAAllocationOp declarations, aie.route DMA ends, and objectFifos
1140 /// that end at a shim.
1141 std::map<int, std::set<int>> scanUsedS2MMChannels(DeviceOp device) {
1142 std::map<int, std::set<int>> used; // shimCol -> set of used S2MM channels
1143
1144 device.walk([&](FlowOp flow) {
1145 auto destTile = cast<TileOp>(flow.getDest().getDefiningOp());
1146 if (destTile.isShimTile() && flow.getDestBundle() == WireBundle::DMA) {
1147 used[destTile.getCol()].insert(flow.getDestChannel());
1148 }
1149 });
1150
1151 device.walk([&](PacketFlowOp pktFlow) {
1152 for (auto &op : pktFlow.getPorts().front()) {
1153 if (auto dest = dyn_cast<PacketDestOp>(op)) {
1154 auto destTile = cast<TileOp>(dest.getTile().getDefiningOp());
1155 if (destTile.isShimTile() && dest.getBundle() == WireBundle::DMA) {
1156 used[destTile.getCol()].insert(dest.getChannel());
1157 }
1158 }
1159 }
1160 });
1161
1162 device.walk([&](ShimDMAAllocationOp alloc) {
1163 if (alloc.getChannelDir() == DMAChannelDir::S2MM) {
1164 auto tile = alloc.getTileOp();
1165 used[tile.getCol()].insert(alloc.getChannelIndex());
1166 }
1167 });
1168
1169 // This pass runs before objectFifo lowering, so an objectFifo or route
1170 // that ends at a shim has no flow yet. Claim what its lowering will:
1171 // every pin first, then the lowest free S2MM channel for each unpinned
1172 // end, except that unpinned fifo ends take one per group allocation
1173 // shares, by the turns the design's #aie.consumed_by proofs give.
1174 SmallVector<int> unpinned;
1175 std::map<int, SmallVector<StringAttr>> unpinnedFifos;
1176 device.walk([&](RouteEndpoint endpoint) {
1177 auto tile = dyn_cast_or_null<TileOp>(endpoint.getTile().getDefiningOp());
1178 if (!tile || !tile.isShimTile() ||
1179 endpoint.getRouteBundle() != WireBundle::DMA ||
1180 endpoint.getRouteDirection() != DMAChannelDir::S2MM)
1181 return;
1182 if (std::optional<int> pin = endpoint.getRouteChannel())
1183 used[tile.getCol()].insert(*pin);
1184 else
1185 unpinned.push_back(tile.getCol());
1186 });
1187 device.walk([&](ObjectFifoCreateOp fifo) {
1188 if (fifo.getPlio())
1189 return;
1190 for (auto [i, consumer] : llvm::enumerate(fifo.getConsumerTiles())) {
1191 auto tile = dyn_cast_or_null<TileOp>(consumer.getDefiningOp());
1192 if (!tile || !tile.isShimTile())
1193 continue;
1194 ObjectFifoEndPortAttr port = fifo.getConsumerPort(i);
1195 if (std::optional<int> pin = port ? port.channelIndex() : std::nullopt)
1196 used[tile.getCol()].insert(*pin);
1197 else
1198 unpinnedFifos[tile.getCol()].push_back(fifo.getSymNameAttr());
1199 }
1200 });
1201 std::optional<ShimTransferSpans> spans;
1202 for (auto &[col, fifos] : unpinnedFifos) {
1203 size_t groups = fifos.size();
1204 if (groups > 1) {
1205 if (!spans)
1206 spans.emplace(
1207 device,
1208 [&](StringAttr symbol) {
1209 return ShimTransferSpans::endOf(device, symbol);
1210 },
1211 Placer::provenShimGates(device));
1212 groups = spans->plan(fifos, fifos, {}).groups.size();
1213 }
1214 unpinned.append(groups, col);
1215 }
1216 for (int col : unpinned) {
1217 auto &channels = used[col];
1218 int ch = 0;
1219 while (channels.count(ch))
1220 ++ch;
1221 channels.insert(ch);
1222 }
1223
1224 return used;
1225 }
1226
1227 /// Find the nearest NOC shim column without active cores.
1228 /// Returns -1 if no spare column exists.
1229 int findNearestSpareColumn(int sourceCol, const std::set<int> &activeColumns,
1230 const AIETargetModel &targetModel) {
1231 int bestCol = -1;
1232 int bestDist = INT_MAX;
1233 int numCols = targetModel.columns();
1234 for (int c = 0; c < numCols; c++) {
1235 if (activeColumns.count(c) == 0 && targetModel.isShimNOCTile(c, 0)) {
1236 int dist = std::abs(c - sourceCol);
1237 if (dist > 0 && dist < bestDist) {
1238 bestDist = dist;
1239 bestCol = c;
1240 }
1241 }
1242 }
1243 return bestCol;
1244 }
1245};
1246
1247} // namespace
1248
1249std::unique_ptr<OperationPass<DeviceOp>>
1251 return std::make_unique<AIEInsertTraceFlowsPass>();
1252}
llvm::DenseMap< TileID, int > getTileToControllerIdMap(bool columnWiseUniqueIDs=true) const
Return a mapping from tile coordinates to controller IDs.
const RegisterInfo * lookupRegister(llvm::StringRef name, TileID tile, bool isMem=false) const
Register Database API - provides access to register and event information for trace configuration and...
virtual uint64_t getDmaBdAddress(int col, int row, uint32_t bd_id, int channel=-1, AIE::DMAChannelDir direction=AIE::DMAChannelDir::MM2S) const =0
Return the array address of the dma buffer descriptor for the given col, row, buffer descriptor id,...
virtual uint32_t getDmaBdAddressOffset(int col, int row) const =0
Return the offset of the base address field within the shim dma buffer descriptor.
bool isShimNOCTile(int col, int row) const
Return true if the given tile is a ShimNOC tile.
virtual int columns() const =0
Return the number of columns in the device.
static llvm::DenseMap< mlir::StringAttr, llvm::SmallVector< ShimTransferSpans::Gate > > provenShimGates(DeviceOp device)
static std::optional< End > endOf(DeviceOp device, mlir::StringAttr symbol)
The end a runtime sequence of device transfers through when it names symbol: a shim DMA route end,...
mlir::Value createConstantI32(mlir::OpBuilder &builder, mlir::Location loc, uint32_t value)
std::unique_ptr< mlir::OperationPass< DeviceOp > > createAIEInsertTraceFlowsPass()
const BitFieldInfo * getField(llvm::StringRef fieldName) const