MLIR-AIE 1.4.6.dev
AIEUtils.cpp
Go to the documentation of this file.
1//===- AIEUtils.cpp ---------------------------------------------*- C++ -*-===//
2//
3// Copyright (C) 2025 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7
10#include "mlir/Dialect/Arith/IR/Arith.h"
11#include "mlir/Dialect/SCF/IR/SCF.h"
12#include "mlir/Dialect/Utils/StaticValueUtils.h"
13#include "mlir/Interfaces/ControlFlowInterfaces.h"
14#include "llvm/ADT/MapVector.h"
15#include "llvm/ADT/SmallPtrSet.h"
16#include "llvm/Support/CheckedArithmetic.h"
17#include "llvm/Support/MathExtras.h"
18
19#include <set>
20
21using namespace mlir;
22using namespace xilinx;
23
24std::optional<AIEX::SubviewTraceResult>
26 int64_t offsetInBytes = 0;
27 Value current = value;
28
29 // Walk through the chain of operations until we reach a block argument
30 while (current) {
31 // Check if we've reached a block argument
32 if (auto blockArg = dyn_cast<BlockArgument>(current)) {
33 return SubviewTraceResult{blockArg, offsetInBytes};
34 }
35
36 Operation *defOp = current.getDefiningOp();
37 if (!defOp) {
38 return std::nullopt;
39 }
40
41 // Handle memref.cast (just pass through)
42 if (auto castOp = dyn_cast<memref::CastOp>(defOp)) {
43 current = castOp.getSource();
44 continue;
45 }
46
47 // Handle memref.reinterpret_cast (validate and pass through)
48 if (auto reinterpretOp = dyn_cast<memref::ReinterpretCastOp>(defOp)) {
49 auto sourceType =
50 dyn_cast<MemRefType>(reinterpretOp.getSource().getType());
51 if (!sourceType) {
52 return std::nullopt;
53 }
54
55 // Validate that source is contiguous (all strides must be 1)
56 if (auto strided = dyn_cast<StridedLayoutAttr>(sourceType.getLayout())) {
57 for (int64_t stride : strided.getStrides()) {
58 if (stride != 1) {
59 return std::nullopt; // Non-contiguous memory, cannot safely
60 // reinterpret
61 }
62 }
63 }
64
65 current = reinterpretOp.getSource();
66 continue;
67 }
68
69 // Handle memref.subview. Accepts any source rank; byte-offset delta is
70 // (resultOffset - sourceOffset) from the strided layouts. The result
71 // slice must remain row-major contiguous so callers can treat it as
72 // linear from the returned base offset.
73 if (auto subviewOp = dyn_cast<memref::SubViewOp>(defOp)) {
74 auto sourceType = subviewOp.getSourceType();
75 auto resultType = subviewOp.getType();
76
77 if (llvm::any_of(subviewOp.getStaticOffsets(), ShapedType::isDynamic) ||
78 llvm::any_of(subviewOp.getStaticSizes(), ShapedType::isDynamic) ||
79 llvm::any_of(subviewOp.getStaticStrides(), ShapedType::isDynamic))
80 return std::nullopt;
81
82 // No skipping in source.
83 if (llvm::any_of(subviewOp.getStaticStrides(),
84 [](int64_t s) { return s != 1; }))
85 return std::nullopt;
86
87 unsigned elemSizeInBits =
88 sourceType.getElementType().getIntOrFloatBitWidth();
89 if (elemSizeInBits % 8 != 0)
90 return std::nullopt;
91
92 llvm::SmallVector<int64_t> srcStrides, resStrides;
93 int64_t srcOff, resOff;
94 if (failed(sourceType.getStridesAndOffset(srcStrides, srcOff)) ||
95 failed(resultType.getStridesAndOffset(resStrides, resOff)))
96 return std::nullopt;
97 if (srcOff == ShapedType::kDynamic || resOff == ShapedType::kDynamic)
98 return std::nullopt;
99
100 // Result must be row-major contiguous: innermost stride == 1, and each
101 // outer stride equals the product of all inner sizes. (Size-1 dims are
102 // free: their stride is never stepped.) Without this, e.g. a column
103 // slice of a 2D row-major memref would be accepted and patched as if
104 // linear.
105 ArrayRef<int64_t> resShape = resultType.getShape();
106 if (!resShape.empty()) {
107 if (resStrides.back() != 1)
108 return std::nullopt;
109 uint64_t product = 1;
110 for (int d = static_cast<int>(resShape.size()) - 1; d > 0; --d) {
111 product *= static_cast<uint64_t>(resShape[d]);
112 if (resShape[d - 1] > 1 &&
113 static_cast<uint64_t>(resStrides[d - 1]) != product)
114 return std::nullopt;
115 }
116 }
117
118 offsetInBytes += (resOff - srcOff) * (elemSizeInBits / 8);
119
120 current = subviewOp.getSource();
121 continue;
122 }
123
124 // Handle memref.view. The fused full-ELF path materializes every DMA
125 // buffer as a typed, contiguous view sliced out of a single flat byte
126 // arena (a runtime-sequence block argument) at a constant byte offset:
127 // %v = memref.view %arena[%byte_off][] : memref<Nxi8> to memref<...>
128 // A memref.view result always has an identity (contiguous) layout, so only
129 // the byte offset needs to be accumulated before following the source.
130 if (auto viewOp = dyn_cast<memref::ViewOp>(defOp)) {
131 if (!viewOp.getSizes().empty())
132 return std::nullopt; // dynamic result sizes unsupported
133 std::optional<int64_t> byteShift =
134 getConstantIntValue(viewOp.getByteShift());
135 if (!byteShift)
136 return std::nullopt; // non-constant byte offset
137 offsetInBytes += *byteShift;
138 current = viewOp.getSource();
139 continue;
140 }
141
142 // Encountered an unsupported operation
143 return std::nullopt;
144 }
145
146 return std::nullopt;
147}
148
149std::optional<unsigned> AIEX::getHostBufferArgIndex(BlockArgument arg) {
150 if (!isa<BaseMemRefType>(arg.getType()))
151 return std::nullopt;
152 unsigned index = 0;
153 for (BlockArgument other : arg.getOwner()->getArguments()) {
154 if (other == arg)
155 return index;
156 if (isa<BaseMemRefType>(other.getType()))
157 index++;
158 }
159 return std::nullopt;
160}
161
162unsigned AIEX::getHostBufferCount(Block &block) {
163 return llvm::count_if(block.getArguments(), [](BlockArgument arg) {
164 return isa<BaseMemRefType>(arg.getType());
165 });
166}
167
168AIEX::BlockwriteData::BlockwriteData(AIE::DeviceOp dev, StringRef prefix)
169 : dev(dev), prefix(prefix) {
170 for (Operation &op : dev.getBody()->getOperations()) {
171 if (auto global = dyn_cast<memref::GlobalOp>(op))
172 if (auto initVal = global.getInitialValue();
173 initVal && global.getConstant())
174 byValue.try_emplace(*initVal, global);
175 auto symName =
176 op.getAttrOfType<StringAttr>(SymbolTable::getSymbolAttrName());
177 if (!symName)
178 continue;
179 StringRef suffix = symName.getValue();
180 unsigned idx;
181 if (suffix.consume_front(prefix) && !suffix.getAsInteger(10, idx) &&
182 idx >= nextId)
183 nextId = idx + 1;
184 }
185}
186
187memref::GlobalOp AIEX::BlockwriteData::getOrCreate(OpBuilder &builder,
188 Location loc,
189 ArrayRef<uint32_t> words) {
190 int64_t numWords = words.size();
191 MemRefType memrefType = MemRefType::get({numWords}, builder.getI32Type());
192 TensorType tensorType =
193 RankedTensorType::get({numWords}, builder.getI32Type());
194 auto initVal = DenseElementsAttr::get<uint32_t>(tensorType, words);
195
196 // The attribute is uniqued and carries the shape, but not the memref's
197 // layout or memory space.
198 memref::GlobalOp &global = byValue[initVal];
199 if (global && global.getType() == memrefType)
200 return global;
201 global = memref::GlobalOp::create(
202 builder, loc, prefix + std::to_string(nextId++),
203 builder.getStringAttr("private"), memrefType, initVal, true, nullptr);
204 return global;
205}
206
207uint64_t AIEX::getBdAddressUnit(const AIE::AIETargetModel &targetModel, int col,
208 int row) {
209 return targetModel.isShimNOCTile(col, row)
210 ? 1
211 : targetModel.getAddressGenGranularity() / 8;
212}
213
215 OpBuilder &builder, Operation *bdOp, BaseMemRefType bufType,
216 uint64_t registerAddr, const AIE::AIETargetModel &targetModel, int col,
217 int row, std::optional<uint64_t> staticByteAddress,
218 OpFoldResult lengthField) {
219 auto idxAttr = bdOp->getAttrOfType<IntegerAttr>("offset_state_table_idx");
220 assert(idxAttr && "emitUpdateBdAddressFromOffsetParameter called without "
221 "offset_state_table_idx attribute");
222
223 uint8_t stateIdx = static_cast<uint8_t>(idxAttr.getUInt());
224 uint32_t elemBytes = bufType.getElementTypeBitWidth() / 8;
225
226 const AIE::DmaBdLayout *layout = targetModel.getDmaBdLayout(col, row);
227 if (!layout)
228 return bdOp->emitOpError("has no buffer descriptor layout on this tile");
229 const AIE::DmaBdField &addressField = layout->bufferOffset;
230 // The firmware adds StateTable[idx] * func_arg into the register holding the
231 // address field.
232 uint64_t unit = getBdAddressUnit(targetModel, col, row);
233 uint64_t perElement = uint64_t{elemBytes} << addressField.shift;
234 if (perElement % unit || perElement / unit > UINT32_MAX)
235 return bdOp->emitOpError("a runtime offset into ")
236 << elemBytes
237 << "-byte elements cannot be added to this tile's buffer "
238 "descriptor address, which counts "
239 << unit << "-byte units from bit " << int{addressField.shift};
240 // It also clears bits [1:0] of that register. An address field from bit 0
241 // loses its own low bits, so the buffer must start on that step, as the
242 // offset moves by it (offset_multiple); one higher up loses another field's,
243 // restored below.
244 uint64_t step = addressField.shift < 2 ? unit << (2 - addressField.shift) : 0;
245 if (step && unit > 1) {
246 if (!staticByteAddress)
247 return bdOp->emitOpError("an offset_parameter on this tile needs the "
248 "rest of the buffer descriptor's address known "
249 "at compile time, to check it starts on a "
250 "multiple of ")
251 << step << " bytes; this one has a runtime offset";
252 if (*staticByteAddress % step)
253 return bdOp->emitOpError("a runtime offset needs the buffer at a "
254 "multiple of ")
255 << step
256 << " bytes, as the firmware clears the address's low two bits "
257 "when it adds the offset";
258 }
259 Location loc = bdOp->getLoc();
260 AIEX::NpuUpdateFromScratchpadOp::create(
261 builder, loc, stateIdx, AIEX::StateTableFunc::Mul,
262 /*func_arg=*/static_cast<uint32_t>(perElement / unit),
263 /*address=*/static_cast<uint32_t>(registerAddr),
264 /*buffer=*/nullptr, /*column=*/nullptr, /*row=*/nullptr);
265 if (step)
266 return success();
267 const AIE::DmaBdField &lengthBits = layout->bufferLength;
268 if (lengthBits.word != addressField.word || lengthBits.shift != 0 ||
269 !lengthField)
270 return bdOp->emitOpError("a runtime offset would clear bits [1:0] of a "
271 "buffer descriptor register it cannot restore");
272 // Created one at a time: the order of a call's arguments is unspecified, and
273 // the constants' order shows in the output.
274 Value address = arith::ConstantOp::create(
275 builder, loc,
276 builder.getI32IntegerAttr(static_cast<int32_t>(registerAddr)));
277 Value lowBits, mask;
278 if (std::optional<int64_t> length = getConstantIntValue(lengthField)) {
279 lowBits = arith::ConstantOp::create(
280 builder, loc,
281 builder.getI32IntegerAttr(static_cast<int32_t>(
282 lengthBits.place(static_cast<uint64_t>(*length)) & 0x3)));
283 mask =
284 arith::ConstantOp::create(builder, loc, builder.getI32IntegerAttr(0x3));
285 } else {
286 mask =
287 arith::ConstantOp::create(builder, loc, builder.getI32IntegerAttr(0x3));
288 lowBits =
289 arith::AndIOp::create(builder, loc, cast<Value>(lengthField), mask);
290 }
291 AIEX::NpuMaskWrite32Op::create(builder, loc, address, lowBits, mask, nullptr,
292 nullptr, nullptr);
293 return success();
294}
295
297 OpBuilder &builder, Operation *bdOp, BaseMemRefType bufType,
298 int64_t lengthUnit, const AIE::AIETargetModel &targetModel, int col,
299 int row, int bdId) {
300 auto idxAttr = bdOp->getAttrOfType<IntegerAttr>("length_state_table_idx");
301 assert(idxAttr && "emitUpdateBdLengthFromParameter called without "
302 "length_state_table_idx attribute");
303
304 if (failed(AIE::verifyLengthParameterTile(bdOp, targetModel, col, row)))
305 return failure();
306 uint64_t registerAddr =
307 targetModel.getDmaBdAddress(col, row, bdId) +
308 4 * targetModel.getDmaBdLayout(col, row)->bufferLength.word;
309
310 uint8_t stateIdx = static_cast<uint8_t>(idxAttr.getUInt());
311 FailureOr<int64_t> unitBytes =
312 AIE::getLengthUnitBytes(bdOp, lengthUnit, bufType);
313 if (failed(unitBytes))
314 return failure();
315
316 // The length register counts 32-bit words, so func=mul adds n * unitBytes / 4
317 // words: func_arg is unitBytes / 4 for an addr parameter (StateTable[idx]
318 // holds n) and unitBytes / 16 for a core parameter (it holds n << 2).
319 int64_t bytesPerStateUnit = bdOp->hasAttr("length_core_encoded") ? 16 : 4;
320 AIEX::NpuUpdateFromScratchpadOp::create(
321 builder, bdOp->getLoc(), stateIdx, AIEX::StateTableFunc::Mul,
322 // NOLINTNEXTLINE(bugprone-unchecked-optional-access)
323 /*func_arg=*/static_cast<uint32_t>(*unitBytes / bytesPerStateUnit),
324 /*address=*/static_cast<uint32_t>(registerAddr),
325 /*buffer=*/nullptr, /*column=*/nullptr, /*row=*/nullptr);
326 return success();
327}
328
329std::optional<int64_t> AIEX::staticExecutions(Operation *op) {
330 int64_t count = 1;
331 for (Operation *cur = op; !isa<AIE::CoreOp, AIE::RuntimeSequenceOp>(cur);
332 cur = cur->getParentOp()) {
333 if (cur->getBlock() != &cur->getParentRegion()->front())
334 return std::nullopt;
335 Operation *parent = cur->getParentOp();
336 if (isa<AIE::CoreOp, AIE::RuntimeSequenceOp>(parent))
337 break;
338 auto forOp = dyn_cast<scf::ForOp>(parent);
339 std::optional<APInt> trips =
340 forOp ? forOp.getStaticTripCount() : std::nullopt;
341 if (!trips || trips->isNegative())
342 return std::nullopt;
343 std::optional<int64_t> product =
344 llvm::checkedMul(count, trips->getSExtValue());
345 if (!product)
346 return std::nullopt;
347 count = *product;
348 }
349 return count;
350}
351
352// Every parameter is at least 0, so it is enough to compare the terms one by
353// one, a term missing from either side counting as 0.
354bool AIEX::atMost(const Volume &a, int64_t scaleA, const Volume &b,
355 int64_t scaleB) {
356 std::set<int64_t> keys;
357 for (const auto &term :
358 llvm::concat<const std::pair<const int64_t, int64_t>>(a, b))
359 keys.insert(term.first);
360 for (int64_t key : keys) {
361 auto x = a.find(key), y = b.find(key);
362 int64_t lhs = 0, rhs = 0;
363 if ((x != a.end() && llvm::MulOverflow(x->second, scaleA, lhs)) ||
364 (y != b.end() && llvm::MulOverflow(y->second, scaleB, rhs)) ||
365 lhs > rhs)
366 return false;
367 }
368 return true;
369}
370
371bool AIEX::wholeGranules(const Volume &bytes, int64_t granule) {
372 return !granule || llvm::all_of(bytes, [&](const auto &term) {
373 return term.second % granule == 0;
374 });
375}
376
377AIEX::Unbalanced AIEX::balanceUnder(AIE::ConsumedByAttr proof, const Volume &in,
378 const Volume &out) {
379 if (!wholeGranules(out, proof.getGranuleBytes()))
381 if (!atMost(in, proof.getOutputBytes(), out, proof.getInputBytes()))
382 return Unbalanced::Excess;
383 return Unbalanced::No;
384}
385
386bool AIEX::namesFifo(Operation *op, StringRef fifo,
387 SymbolTableCollection &symbols) {
388 auto uses = SymbolTable::getSymbolUses(op);
389 return uses && llvm::any_of(*uses, [&](const SymbolTable::SymbolUse &use) {
390 auto end = dyn_cast_or_null<AIE::RouteEndpoint>(
391 symbols.lookupNearestSymbolFrom(op, use.getSymbolRef()));
392 return end && end.getFifoName() == fifo;
393 });
394}
395
396LogicalResult AIEX::checkResidentsSurvive(AIE::RuntimeSequenceOp seq,
397 ArrayRef<ConfigStep> steps) {
398 struct Cleared {
399 AIE::BufferOp holder;
400 Operation *reset;
401 };
402 llvm::MapVector<FlatSymbolRefAttr, AIE::BufferOp> held;
403 llvm::DenseMap<FlatSymbolRefAttr, Cleared> cleared;
404 for (auto buffer :
405 seq->getParentOfType<AIE::DeviceOp>().getOps<AIE::BufferOp>())
406 if (FlatSymbolRefAttr resident = buffer.getResidentAttr())
407 held[resident] = buffer;
408 NpuLoadPdiOp lastLoad;
409 auto apply = [&](ConfigStep step, bool again) -> LogicalResult {
410 bool resets =
411 step.resets ||
412 (step.load && !(lastLoad && lastLoad.loadsSamePdi(step.load)));
413 if (step.load || step.resets)
414 lastLoad = step.load;
415 if (resets) {
416 for (auto [resident, holder] : held)
417 cleared.try_emplace(resident, Cleared{holder, step.op});
418 held.clear();
419 }
420 if (!step.configures)
421 return success();
422 for (auto buffer : step.configures.getOps<AIE::BufferOp>()) {
423 FlatSymbolRefAttr resident = buffer.getResidentAttr();
424 if (!resident)
425 continue;
426 if (auto it = cleared.find(resident); it != cleared.end()) {
427 auto [holder, reset] = it->second;
428 InFlightDiagnostic diag =
429 reset->emitOpError("resets the array between @")
430 << holder->getParentOfType<AIE::DeviceOp>().getSymName() << " and @"
431 << step.configures.getSymName()
432 << ", which both hold a buffer resident on " << resident
433 << (again ? " when the sequence runs again" : "")
434 << "; needs --reconfig-teardown";
435 diag.attachNote(holder.getLoc()) << "resident here";
436 if (buffer != holder)
437 diag.attachNote(buffer.getLoc()) << "resident here";
438 return diag;
439 }
440 held[resident] = buffer;
441 }
442 return success();
443 };
444 for (bool again : {false, true})
445 for (ConfigStep step : steps)
446 if (failed(apply(step, again)))
447 return failure();
448 return success();
449}
450
451std::string AIEX::scratchpadCommandName(AIE::RuntimeSequenceOp seq) {
452 auto device = seq->getParentOfType<AIE::DeviceOp>();
453 return (device.getSymName() + ":" + seq.getSymName()).str();
454}
455
456void AIEX::emitScratchpadParamsFile(ModuleOp moduleOp, llvm::raw_ostream &os) {
457 SmallVector<AIEX::ScratchpadParameterOp> allParams;
458 moduleOp.walk([&](AIEX::ScratchpadParameterOp p) { allParams.push_back(p); });
459
460 unsigned numSlots = 0;
461 for (auto p : allParams)
462 if (auto idx = p.getStateTableIdx())
463 numSlots = std::max(numSlots, static_cast<unsigned>(*idx) + 1);
464 os << allParams.size() << " " << numSlots << "\n";
465 for (auto p : allParams) {
466 std::string typeStr;
467 llvm::raw_string_ostream ts(typeStr);
468 p.getType().print(ts);
469 ts.flush();
470 auto kind = p.getKind();
471 assert(kind && "expected --aie-lower-scratchpad-parameters to have "
472 "assigned every parameter a kind");
473 StringRef kindStr = AIEX::stringifyScratchpadParameterKind(*kind);
474 os << p.getSymName() << " ";
475 if (std::optional<uint8_t> stateTableIdx = p.getStateTableIdx())
476 os << static_cast<unsigned>(*stateTableIdx);
477 else
478 os << "-";
479 os << " " << typeStr << " " << kindStr;
480 auto minValue = p.getMinValue();
481 auto maxValue = p.getMaxValue();
482 if (minValue && maxValue)
483 os << " " << static_cast<int32_t>(*minValue) << " "
484 << static_cast<int32_t>(*maxValue);
485 else
486 os << " - -";
487 os << "\n";
488 }
489
490 SmallVector<std::pair<StringRef, AIEX::JointBoundAttr>> jointBounds;
491 for (auto p : allParams)
492 if (ArrayAttr bounds = p.getJointBoundsAttr())
493 for (auto bound : bounds.getAsRange<AIEX::JointBoundAttr>())
494 jointBounds.push_back({p.getSymName(), bound});
495 os << jointBounds.size() << "\n";
496 for (auto [length, bound] : jointBounds)
497 os << length << " " << bound.getOffset().getValue() << " "
498 << bound.getLengthStep() << " " << bound.getMax() << "\n";
499
500 SmallVector<AIE::RuntimeSequenceOp> commands;
501 moduleOp.walk([&](AIE::RuntimeSequenceOp seq) {
502 if (seq.getScratchpadParametersAttr())
503 commands.push_back(seq);
504 });
505 os << commands.size() << "\n";
506 for (AIE::RuntimeSequenceOp seq : commands) {
507 ArrayAttr reads = seq.getScratchpadParametersAttr();
508 os << scratchpadCommandName(seq) << " " << reads.size();
509 for (auto ref : reads.getAsRange<FlatSymbolRefAttr>())
510 os << " " << ref.getValue();
511 os << "\n";
512 }
513
514 // Last, and only when there are any, so a reader that predates it stops
515 // before it.
516 SmallVector<std::pair<StringRef, uint32_t>> multiples;
517 for (auto p : allParams)
518 if (std::optional<uint32_t> multiple = p.getOffsetMultiple())
519 multiples.push_back({p.getSymName(), *multiple});
520 if (multiples.empty())
521 return;
522 os << multiples.size() << "\n";
523 for (auto [name, multiple] : multiples)
524 os << name << " " << multiple << "\n";
525}
526
528 Value task, SmallVectorImpl<DMAConfigureTaskOp> &configures) {
529 bool complete = true;
530 llvm::SmallPtrSet<Value, 8> seen;
531 SmallVector<Value> worklist{task};
532 while (!worklist.empty()) {
533 Value v = worklist.pop_back_val();
534 if (!v || !seen.insert(v).second)
535 continue;
536 if (auto cfg = v.getDefiningOp<DMAConfigureTaskOp>()) {
537 configures.push_back(cfg);
538 continue;
539 }
540
541 Operation *regionBranchOp;
542 if (auto res = dyn_cast<OpResult>(v))
543 regionBranchOp = res.getOwner();
544 else
545 regionBranchOp = cast<BlockArgument>(v).getOwner()->getParentOp();
546 auto rbi = dyn_cast_or_null<RegionBranchOpInterface>(regionBranchOp);
547 if (!rbi) {
548 complete = false;
549 continue;
550 }
551
552 RegionBranchInverseSuccessorMapping mapping;
553 rbi.getSuccessorInputOperandMapping(mapping);
554 auto operands = mapping.lookup(v);
555 if (operands.empty())
556 complete = false;
557 for (OpOperand *operand : operands)
558 worklist.push_back(operand->get());
559 }
560 return complete;
561}
562
563AIEX::DMAConfigureTaskOp AIEX::getUniqueReachableConfigure(Value task) {
564 SmallVector<DMAConfigureTaskOp> configures;
565 if (!getReachableConfigures(task, configures) || configures.size() != 1)
566 return nullptr;
567 return configures.front();
568}
569
570static StringAttr runtimeDmaSymbol(Operation *op) {
571 if (auto task = dyn_cast<AIEX::DMAConfigureTaskForOp>(op))
572 return task.getAlloc().getRootReference();
573 if (auto copy = dyn_cast<AIEX::NpuDmaMemcpyNdOp>(op))
574 return copy.getMetadata().getRootReference();
575 return {};
576}
577
578Operation *AIEX::getRuntimeDmaTile(Operation *op, SymbolTable &symbols) {
579 if (auto task = dyn_cast<AIEX::DMAConfigureTaskOp>(op))
580 return task.getTile().getDefiningOp();
581 StringAttr symbol = runtimeDmaSymbol(op);
582 Operation *named = symbol ? symbols.lookup(symbol) : nullptr;
583 if (auto alloc = dyn_cast_or_null<AIE::ShimDMAAllocationOp>(named))
584 return alloc.getTile().getDefiningOp();
585 if (auto endpoint = dyn_cast_or_null<AIE::RouteEndpointOp>(named))
586 return endpoint.getTile().getDefiningOp();
587 auto fifo = dyn_cast_or_null<AIE::ObjectFifoCreateOp>(named);
588 if (!fifo)
589 return nullptr;
590 SmallVector<Value> ends{fifo.getProducerTile()};
591 llvm::append_range(ends, fifo.getConsumerTiles());
592 for (Value end : ends)
593 if (auto tile = dyn_cast_or_null<AIE::TileLike>(end.getDefiningOp());
594 tile && tile.isShimNOCTile())
595 return tile;
596 return nullptr;
597}
598
600 AIE::DeviceOp device,
601 llvm::function_ref<void(Operation *op, Operation *tile,
602 std::optional<int> channel, int32_t id)>
603 pinned) {
604 SymbolTable symbols(device);
605 device.walk([&](Operation *op) {
606 Operation *tile = getRuntimeDmaTile(op, symbols);
607 if (!tile)
608 return;
609 std::optional<int> channel;
610 if (auto task = dyn_cast<AIEX::DMAConfigureTaskOp>(op))
611 channel = task.getChannel();
612 else if (auto alloc =
613 symbols.lookup<AIE::ShimDMAAllocationOp>(runtimeDmaSymbol(op)))
614 channel = alloc.getChannelIndex();
615 if (auto copy = dyn_cast<AIEX::NpuDmaMemcpyNdOp>(op)) {
616 pinned(op, tile, channel, static_cast<int32_t>(copy.getId()));
617 return;
618 }
619 cast<AIEX::DMAConfigureTaskLike>(op).getBody().walk([&](AIE::DMABDOp bd) {
620 if (std::optional<int32_t> id = bd.getBdId())
621 pinned(op, tile, channel, *id);
622 });
623 });
624}
memref::GlobalOp getOrCreate(OpBuilder &builder, mlir::Location loc, ArrayRef< uint32_t > words)
Definition AIEUtils.cpp:187
BlockwriteData(AIE::DeviceOp dev, llvm::StringRef prefix)
Definition AIEUtils.cpp:168
virtual const DmaBdLayout * getDmaBdLayout(AIETileType tileType) const
Return the register layout of a buffer descriptor on the given tile type, or null if the target has n...
virtual uint64_t getDmaBdAddress(int col, int row, uint32_t bd_id, int channel=-1, AIE::DMAChannelDir direction=AIE::DMAChannelDir::MM2S) const =0
Return the array address of the dma buffer descriptor for the given col, row, buffer descriptor id,...
bool isShimNOCTile(int col, int row) const
Return true if the given tile is a ShimNOC tile.
virtual uint32_t getAddressGenGranularity() const =0
Return the data bus width of the device.
unsigned getHostBufferCount(Block &block)
Definition AIEUtils.cpp:162
void forEachRuntimePinnedBdId(AIE::DeviceOp device, llvm::function_ref< void(Operation *op, Operation *tile, std::optional< int > channel, int32_t id)> pinned)
Definition AIEUtils.cpp:599
uint64_t getBdAddressUnit(const AIE::AIETargetModel &targetModel, int col, int row)
Definition AIEUtils.cpp:207
LogicalResult checkResidentsSurvive(AIE::RuntimeSequenceOp seq, ArrayRef< ConfigStep > steps)
Definition AIEUtils.cpp:396
DMAConfigureTaskOp getUniqueReachableConfigure(Value task)
Definition AIEUtils.cpp:563
bool namesFifo(Operation *op, StringRef fifo, SymbolTableCollection &symbols)
Definition AIEUtils.cpp:386
std::optional< SubviewTraceResult > traceSubviewToBlockArgument(Value value)
Definition AIEUtils.cpp:25
std::optional< int64_t > staticExecutions(Operation *op)
Definition AIEUtils.cpp:329
Operation * getRuntimeDmaTile(Operation *op, SymbolTable &symbols)
Definition AIEUtils.cpp:578
bool atMost(const Volume &a, int64_t scaleA, const Volume &b, int64_t scaleB)
Definition AIEUtils.cpp:354
bool wholeGranules(const Volume &bytes, int64_t granule)
Definition AIEUtils.cpp:371
std::string scratchpadCommandName(AIE::RuntimeSequenceOp seq)
Definition AIEUtils.cpp:451
LogicalResult emitUpdateBdAddressFromOffsetParameter(OpBuilder &builder, Operation *bdOp, BaseMemRefType bufType, uint64_t registerAddr, const AIE::AIETargetModel &targetModel, int col, int row, std::optional< uint64_t > staticByteAddress=std::nullopt, OpFoldResult lengthField={})
Definition AIEUtils.cpp:214
void emitScratchpadParamsFile(mlir::ModuleOp moduleOp, llvm::raw_ostream &os)
std::map< int64_t, int64_t > Volume
Definition AIEUtils.h:146
bool getReachableConfigures(Value task, SmallVectorImpl< DMAConfigureTaskOp > &configures)
Definition AIEUtils.cpp:527
LogicalResult emitUpdateBdLengthFromParameter(OpBuilder &builder, Operation *bdOp, BaseMemRefType bufType, int64_t lengthUnit, const AIE::AIETargetModel &targetModel, int col, int row, int bdId)
Definition AIEUtils.cpp:296
std::optional< unsigned > getHostBufferArgIndex(BlockArgument arg)
Definition AIEUtils.cpp:149
Unbalanced balanceUnder(AIE::ConsumedByAttr proof, const Volume &in, const Volume &out)
Definition AIEUtils.cpp:377
mlir::LogicalResult verifyLengthParameterTile(mlir::Operation *op, const AIETargetModel &targetModel, int col, int row)
mlir::FailureOr< int64_t > getLengthUnitBytes(mlir::Operation *op, int64_t lengthUnit, mlir::BaseMemRefType buffer)
AIE::DeviceOp configures
Definition AIEUtils.h:188
Bits [shift, shift + width) of register word in a buffer descriptor's register block.
uint32_t place(uint64_t value) const
The register layout of one tile type's DMA buffer descriptor.