10#include "mlir/Dialect/Arith/IR/Arith.h"
11#include "mlir/Dialect/SCF/IR/SCF.h"
12#include "mlir/Dialect/Utils/StaticValueUtils.h"
13#include "mlir/Interfaces/ControlFlowInterfaces.h"
14#include "llvm/ADT/MapVector.h"
15#include "llvm/ADT/SmallPtrSet.h"
16#include "llvm/Support/CheckedArithmetic.h"
17#include "llvm/Support/MathExtras.h"
24std::optional<AIEX::SubviewTraceResult>
26 int64_t offsetInBytes = 0;
27 Value current = value;
32 if (
auto blockArg = dyn_cast<BlockArgument>(current)) {
36 Operation *defOp = current.getDefiningOp();
42 if (
auto castOp = dyn_cast<memref::CastOp>(defOp)) {
43 current = castOp.getSource();
48 if (
auto reinterpretOp = dyn_cast<memref::ReinterpretCastOp>(defOp)) {
50 dyn_cast<MemRefType>(reinterpretOp.getSource().getType());
56 if (
auto strided = dyn_cast<StridedLayoutAttr>(sourceType.getLayout())) {
57 for (int64_t stride : strided.getStrides()) {
65 current = reinterpretOp.getSource();
73 if (
auto subviewOp = dyn_cast<memref::SubViewOp>(defOp)) {
74 auto sourceType = subviewOp.getSourceType();
75 auto resultType = subviewOp.getType();
77 if (llvm::any_of(subviewOp.getStaticOffsets(), ShapedType::isDynamic) ||
78 llvm::any_of(subviewOp.getStaticSizes(), ShapedType::isDynamic) ||
79 llvm::any_of(subviewOp.getStaticStrides(), ShapedType::isDynamic))
83 if (llvm::any_of(subviewOp.getStaticStrides(),
84 [](int64_t s) { return s != 1; }))
87 unsigned elemSizeInBits =
88 sourceType.getElementType().getIntOrFloatBitWidth();
89 if (elemSizeInBits % 8 != 0)
92 llvm::SmallVector<int64_t> srcStrides, resStrides;
93 int64_t srcOff, resOff;
94 if (failed(sourceType.getStridesAndOffset(srcStrides, srcOff)) ||
95 failed(resultType.getStridesAndOffset(resStrides, resOff)))
97 if (srcOff == ShapedType::kDynamic || resOff == ShapedType::kDynamic)
105 ArrayRef<int64_t> resShape = resultType.getShape();
106 if (!resShape.empty()) {
107 if (resStrides.back() != 1)
109 uint64_t product = 1;
110 for (
int d =
static_cast<int>(resShape.size()) - 1; d > 0; --d) {
111 product *=
static_cast<uint64_t
>(resShape[d]);
112 if (resShape[d - 1] > 1 &&
113 static_cast<uint64_t
>(resStrides[d - 1]) != product)
118 offsetInBytes += (resOff - srcOff) * (elemSizeInBits / 8);
120 current = subviewOp.getSource();
130 if (
auto viewOp = dyn_cast<memref::ViewOp>(defOp)) {
131 if (!viewOp.getSizes().empty())
133 std::optional<int64_t> byteShift =
134 getConstantIntValue(viewOp.getByteShift());
137 offsetInBytes += *byteShift;
138 current = viewOp.getSource();
150 if (!isa<BaseMemRefType>(arg.getType()))
153 for (BlockArgument other : arg.getOwner()->getArguments()) {
156 if (isa<BaseMemRefType>(other.getType()))
163 return llvm::count_if(block.getArguments(), [](BlockArgument arg) {
164 return isa<BaseMemRefType>(arg.getType());
169 : dev(dev), prefix(prefix) {
170 for (Operation &op : dev.getBody()->getOperations()) {
171 if (auto global = dyn_cast<memref::GlobalOp>(op))
172 if (auto initVal = global.getInitialValue();
173 initVal && global.getConstant())
174 byValue.try_emplace(*initVal, global);
176 op.getAttrOfType<StringAttr>(SymbolTable::getSymbolAttrName());
179 StringRef suffix = symName.getValue();
181 if (suffix.consume_front(prefix) && !suffix.getAsInteger(10, idx) &&
189 ArrayRef<uint32_t> words) {
190 int64_t numWords = words.size();
191 MemRefType memrefType = MemRefType::get({numWords}, builder.getI32Type());
192 TensorType tensorType =
193 RankedTensorType::get({numWords}, builder.getI32Type());
194 auto initVal = DenseElementsAttr::get<uint32_t>(tensorType, words);
198 memref::GlobalOp &global = byValue[initVal];
199 if (global && global.getType() == memrefType)
201 global = memref::GlobalOp::create(
202 builder, loc, prefix + std::to_string(nextId++),
203 builder.getStringAttr(
"private"), memrefType, initVal,
true,
nullptr);
215 OpBuilder &builder, Operation *bdOp, BaseMemRefType bufType,
217 int row, std::optional<uint64_t> staticByteAddress,
218 OpFoldResult lengthField) {
219 auto idxAttr = bdOp->getAttrOfType<IntegerAttr>(
"offset_state_table_idx");
220 assert(idxAttr &&
"emitUpdateBdAddressFromOffsetParameter called without "
221 "offset_state_table_idx attribute");
223 uint8_t stateIdx =
static_cast<uint8_t
>(idxAttr.getUInt());
224 uint32_t elemBytes = bufType.getElementTypeBitWidth() / 8;
228 return bdOp->emitOpError(
"has no buffer descriptor layout on this tile");
233 uint64_t perElement = uint64_t{elemBytes} << addressField.
shift;
234 if (perElement % unit || perElement / unit > UINT32_MAX)
235 return bdOp->emitOpError(
"a runtime offset into ")
237 <<
"-byte elements cannot be added to this tile's buffer "
238 "descriptor address, which counts "
239 << unit <<
"-byte units from bit " <<
int{addressField.
shift};
244 uint64_t step = addressField.
shift < 2 ? unit << (2 - addressField.
shift) : 0;
245 if (step && unit > 1) {
246 if (!staticByteAddress)
247 return bdOp->emitOpError(
"an offset_parameter on this tile needs the "
248 "rest of the buffer descriptor's address known "
249 "at compile time, to check it starts on a "
251 << step <<
" bytes; this one has a runtime offset";
252 if (*staticByteAddress % step)
253 return bdOp->emitOpError(
"a runtime offset needs the buffer at a "
256 <<
" bytes, as the firmware clears the address's low two bits "
257 "when it adds the offset";
259 Location loc = bdOp->getLoc();
260 AIEX::NpuUpdateFromScratchpadOp::create(
261 builder, loc, stateIdx, AIEX::StateTableFunc::Mul,
262 static_cast<uint32_t
>(perElement / unit),
263 static_cast<uint32_t
>(registerAddr),
264 nullptr,
nullptr,
nullptr);
268 if (lengthBits.
word != addressField.
word || lengthBits.
shift != 0 ||
270 return bdOp->emitOpError(
"a runtime offset would clear bits [1:0] of a "
271 "buffer descriptor register it cannot restore");
274 Value address = arith::ConstantOp::create(
276 builder.getI32IntegerAttr(
static_cast<int32_t
>(registerAddr)));
278 if (std::optional<int64_t> length = getConstantIntValue(lengthField)) {
279 lowBits = arith::ConstantOp::create(
281 builder.getI32IntegerAttr(
static_cast<int32_t
>(
282 lengthBits.
place(
static_cast<uint64_t
>(*length)) & 0x3)));
284 arith::ConstantOp::create(builder, loc, builder.getI32IntegerAttr(0x3));
287 arith::ConstantOp::create(builder, loc, builder.getI32IntegerAttr(0x3));
289 arith::AndIOp::create(builder, loc, cast<Value>(lengthField), mask);
291 AIEX::NpuMaskWrite32Op::create(builder, loc, address, lowBits, mask,
nullptr,
297 OpBuilder &builder, Operation *bdOp, BaseMemRefType bufType,
300 auto idxAttr = bdOp->getAttrOfType<IntegerAttr>(
"length_state_table_idx");
301 assert(idxAttr &&
"emitUpdateBdLengthFromParameter called without "
302 "length_state_table_idx attribute");
306 uint64_t registerAddr =
310 uint8_t stateIdx =
static_cast<uint8_t
>(idxAttr.getUInt());
311 FailureOr<int64_t> unitBytes =
313 if (failed(unitBytes))
319 int64_t bytesPerStateUnit = bdOp->hasAttr(
"length_core_encoded") ? 16 : 4;
320 AIEX::NpuUpdateFromScratchpadOp::create(
321 builder, bdOp->getLoc(), stateIdx, AIEX::StateTableFunc::Mul,
323 static_cast<uint32_t
>(*unitBytes / bytesPerStateUnit),
324 static_cast<uint32_t
>(registerAddr),
325 nullptr,
nullptr,
nullptr);
331 for (Operation *cur = op; !isa<AIE::CoreOp, AIE::RuntimeSequenceOp>(cur);
332 cur = cur->getParentOp()) {
333 if (cur->getBlock() != &cur->getParentRegion()->front())
335 Operation *parent = cur->getParentOp();
336 if (isa<AIE::CoreOp, AIE::RuntimeSequenceOp>(parent))
338 auto forOp = dyn_cast<scf::ForOp>(parent);
339 std::optional<APInt> trips =
340 forOp ? forOp.getStaticTripCount() : std::nullopt;
341 if (!trips || trips->isNegative())
343 std::optional<int64_t> product =
344 llvm::checkedMul(count, trips->getSExtValue());
356 std::set<int64_t> keys;
357 for (
const auto &term :
358 llvm::concat<
const std::pair<const int64_t, int64_t>>(a, b))
359 keys.insert(term.first);
360 for (int64_t key : keys) {
361 auto x = a.find(key), y = b.find(key);
362 int64_t lhs = 0, rhs = 0;
363 if ((x != a.end() && llvm::MulOverflow(x->second, scaleA, lhs)) ||
364 (y != b.end() && llvm::MulOverflow(y->second, scaleB, rhs)) ||
372 return !granule || llvm::all_of(bytes, [&](
const auto &term) {
373 return term.second % granule == 0;
381 if (!
atMost(in, proof.getOutputBytes(), out, proof.getInputBytes()))
387 SymbolTableCollection &symbols) {
388 auto uses = SymbolTable::getSymbolUses(op);
389 return uses && llvm::any_of(*uses, [&](
const SymbolTable::SymbolUse &use) {
390 auto end = dyn_cast_or_null<AIE::RouteEndpoint>(
391 symbols.lookupNearestSymbolFrom(op, use.getSymbolRef()));
392 return end && end.getFifoName() == fifo;
397 ArrayRef<ConfigStep> steps) {
399 AIE::BufferOp holder;
402 llvm::MapVector<FlatSymbolRefAttr, AIE::BufferOp> held;
403 llvm::DenseMap<FlatSymbolRefAttr, Cleared> cleared;
405 seq->getParentOfType<AIE::DeviceOp>().getOps<AIE::BufferOp>())
406 if (FlatSymbolRefAttr resident = buffer.getResidentAttr())
407 held[resident] = buffer;
408 NpuLoadPdiOp lastLoad;
409 auto apply = [&](
ConfigStep step,
bool again) -> LogicalResult {
412 (step.
load && !(lastLoad && lastLoad.loadsSamePdi(step.
load)));
414 lastLoad = step.
load;
416 for (
auto [resident, holder] : held)
417 cleared.try_emplace(resident, Cleared{holder, step.
op});
422 for (
auto buffer : step.
configures.getOps<AIE::BufferOp>()) {
423 FlatSymbolRefAttr resident = buffer.getResidentAttr();
426 if (
auto it = cleared.find(resident); it != cleared.end()) {
427 auto [holder, reset] = it->second;
428 InFlightDiagnostic diag =
429 reset->emitOpError(
"resets the array between @")
430 << holder->getParentOfType<AIE::DeviceOp>().getSymName() <<
" and @"
432 <<
", which both hold a buffer resident on " << resident
433 << (again ?
" when the sequence runs again" :
"")
434 <<
"; needs --reconfig-teardown";
435 diag.attachNote(holder.getLoc()) <<
"resident here";
436 if (buffer != holder)
437 diag.attachNote(buffer.getLoc()) <<
"resident here";
440 held[resident] = buffer;
444 for (
bool again : {
false,
true})
446 if (failed(apply(step, again)))
452 auto device = seq->getParentOfType<AIE::DeviceOp>();
453 return (device.getSymName() +
":" + seq.getSymName()).str();
457 SmallVector<AIEX::ScratchpadParameterOp> allParams;
458 moduleOp.walk([&](AIEX::ScratchpadParameterOp p) { allParams.push_back(p); });
460 unsigned numSlots = 0;
461 for (
auto p : allParams)
462 if (
auto idx = p.getStateTableIdx())
463 numSlots = std::max(numSlots,
static_cast<unsigned>(*idx) + 1);
464 os << allParams.size() <<
" " << numSlots <<
"\n";
465 for (
auto p : allParams) {
467 llvm::raw_string_ostream ts(typeStr);
468 p.getType().print(ts);
470 auto kind = p.getKind();
471 assert(kind &&
"expected --aie-lower-scratchpad-parameters to have "
472 "assigned every parameter a kind");
473 StringRef kindStr = AIEX::stringifyScratchpadParameterKind(*kind);
474 os << p.getSymName() <<
" ";
475 if (std::optional<uint8_t> stateTableIdx = p.getStateTableIdx())
476 os << static_cast<unsigned>(*stateTableIdx);
479 os <<
" " << typeStr <<
" " << kindStr;
480 auto minValue = p.getMinValue();
481 auto maxValue = p.getMaxValue();
482 if (minValue && maxValue)
483 os <<
" " <<
static_cast<int32_t
>(*minValue) <<
" "
484 <<
static_cast<int32_t
>(*maxValue);
490 SmallVector<std::pair<StringRef, AIEX::JointBoundAttr>> jointBounds;
491 for (
auto p : allParams)
492 if (ArrayAttr bounds = p.getJointBoundsAttr())
493 for (
auto bound : bounds.getAsRange<AIEX::JointBoundAttr>())
494 jointBounds.push_back({p.getSymName(), bound});
495 os << jointBounds.size() <<
"\n";
496 for (
auto [length, bound] : jointBounds)
497 os << length <<
" " << bound.getOffset().getValue() <<
" "
498 << bound.getLengthStep() <<
" " << bound.getMax() <<
"\n";
500 SmallVector<AIE::RuntimeSequenceOp> commands;
501 moduleOp.walk([&](AIE::RuntimeSequenceOp seq) {
502 if (seq.getScratchpadParametersAttr())
503 commands.push_back(seq);
505 os << commands.size() <<
"\n";
506 for (AIE::RuntimeSequenceOp seq : commands) {
507 ArrayAttr reads = seq.getScratchpadParametersAttr();
509 for (
auto ref : reads.getAsRange<FlatSymbolRefAttr>())
510 os <<
" " << ref.getValue();
516 SmallVector<std::pair<StringRef, uint32_t>> multiples;
517 for (
auto p : allParams)
518 if (std::optional<uint32_t> multiple = p.getOffsetMultiple())
519 multiples.push_back({p.getSymName(), *multiple});
520 if (multiples.empty())
522 os << multiples.size() <<
"\n";
523 for (
auto [name, multiple] : multiples)
524 os << name <<
" " << multiple <<
"\n";
528 Value task, SmallVectorImpl<DMAConfigureTaskOp> &configures) {
529 bool complete =
true;
530 llvm::SmallPtrSet<Value, 8> seen;
531 SmallVector<Value> worklist{task};
532 while (!worklist.empty()) {
533 Value v = worklist.pop_back_val();
534 if (!v || !seen.insert(v).second)
536 if (
auto cfg = v.getDefiningOp<DMAConfigureTaskOp>()) {
537 configures.push_back(cfg);
541 Operation *regionBranchOp;
542 if (
auto res = dyn_cast<OpResult>(v))
543 regionBranchOp = res.getOwner();
545 regionBranchOp = cast<BlockArgument>(v).getOwner()->getParentOp();
546 auto rbi = dyn_cast_or_null<RegionBranchOpInterface>(regionBranchOp);
552 RegionBranchInverseSuccessorMapping mapping;
553 rbi.getSuccessorInputOperandMapping(mapping);
554 auto operands = mapping.lookup(v);
555 if (operands.empty())
557 for (OpOperand *operand : operands)
558 worklist.push_back(operand->get());
564 SmallVector<DMAConfigureTaskOp> configures;
567 return configures.front();
570static StringAttr runtimeDmaSymbol(Operation *op) {
571 if (
auto task = dyn_cast<AIEX::DMAConfigureTaskForOp>(op))
572 return task.getAlloc().getRootReference();
573 if (
auto copy = dyn_cast<AIEX::NpuDmaMemcpyNdOp>(op))
574 return copy.getMetadata().getRootReference();
579 if (
auto task = dyn_cast<AIEX::DMAConfigureTaskOp>(op))
580 return task.getTile().getDefiningOp();
581 StringAttr symbol = runtimeDmaSymbol(op);
582 Operation *named = symbol ? symbols.lookup(symbol) :
nullptr;
583 if (
auto alloc = dyn_cast_or_null<AIE::ShimDMAAllocationOp>(named))
584 return alloc.getTile().getDefiningOp();
585 if (
auto endpoint = dyn_cast_or_null<AIE::RouteEndpointOp>(named))
586 return endpoint.getTile().getDefiningOp();
587 auto fifo = dyn_cast_or_null<AIE::ObjectFifoCreateOp>(named);
590 SmallVector<Value> ends{fifo.getProducerTile()};
591 llvm::append_range(ends, fifo.getConsumerTiles());
592 for (Value end : ends)
593 if (
auto tile = dyn_cast_or_null<AIE::TileLike>(end.getDefiningOp());
594 tile && tile.isShimNOCTile())
600 AIE::DeviceOp device,
601 llvm::function_ref<
void(Operation *op, Operation *tile,
602 std::optional<int> channel, int32_t
id)>
604 SymbolTable symbols(device);
605 device.walk([&](Operation *op) {
609 std::optional<int> channel;
610 if (
auto task = dyn_cast<AIEX::DMAConfigureTaskOp>(op))
611 channel = task.getChannel();
612 else if (
auto alloc =
613 symbols.lookup<AIE::ShimDMAAllocationOp>(runtimeDmaSymbol(op)))
614 channel = alloc.getChannelIndex();
615 if (
auto copy = dyn_cast<AIEX::NpuDmaMemcpyNdOp>(op)) {
616 pinned(op, tile, channel,
static_cast<int32_t
>(copy.getId()));
619 cast<AIEX::DMAConfigureTaskLike>(op).getBody().walk([&](AIE::DMABDOp bd) {
620 if (std::optional<int32_t>
id = bd.getBdId())
621 pinned(op, tile, channel, *
id);
memref::GlobalOp getOrCreate(OpBuilder &builder, mlir::Location loc, ArrayRef< uint32_t > words)
BlockwriteData(AIE::DeviceOp dev, llvm::StringRef prefix)
virtual const DmaBdLayout * getDmaBdLayout(AIETileType tileType) const
Return the register layout of a buffer descriptor on the given tile type, or null if the target has n...
virtual uint64_t getDmaBdAddress(int col, int row, uint32_t bd_id, int channel=-1, AIE::DMAChannelDir direction=AIE::DMAChannelDir::MM2S) const =0
Return the array address of the dma buffer descriptor for the given col, row, buffer descriptor id,...
bool isShimNOCTile(int col, int row) const
Return true if the given tile is a ShimNOC tile.
virtual uint32_t getAddressGenGranularity() const =0
Return the data bus width of the device.
unsigned getHostBufferCount(Block &block)
void forEachRuntimePinnedBdId(AIE::DeviceOp device, llvm::function_ref< void(Operation *op, Operation *tile, std::optional< int > channel, int32_t id)> pinned)
uint64_t getBdAddressUnit(const AIE::AIETargetModel &targetModel, int col, int row)
LogicalResult checkResidentsSurvive(AIE::RuntimeSequenceOp seq, ArrayRef< ConfigStep > steps)
DMAConfigureTaskOp getUniqueReachableConfigure(Value task)
bool namesFifo(Operation *op, StringRef fifo, SymbolTableCollection &symbols)
std::optional< SubviewTraceResult > traceSubviewToBlockArgument(Value value)
std::optional< int64_t > staticExecutions(Operation *op)
Operation * getRuntimeDmaTile(Operation *op, SymbolTable &symbols)
bool atMost(const Volume &a, int64_t scaleA, const Volume &b, int64_t scaleB)
bool wholeGranules(const Volume &bytes, int64_t granule)
std::string scratchpadCommandName(AIE::RuntimeSequenceOp seq)
LogicalResult emitUpdateBdAddressFromOffsetParameter(OpBuilder &builder, Operation *bdOp, BaseMemRefType bufType, uint64_t registerAddr, const AIE::AIETargetModel &targetModel, int col, int row, std::optional< uint64_t > staticByteAddress=std::nullopt, OpFoldResult lengthField={})
void emitScratchpadParamsFile(mlir::ModuleOp moduleOp, llvm::raw_ostream &os)
std::map< int64_t, int64_t > Volume
bool getReachableConfigures(Value task, SmallVectorImpl< DMAConfigureTaskOp > &configures)
LogicalResult emitUpdateBdLengthFromParameter(OpBuilder &builder, Operation *bdOp, BaseMemRefType bufType, int64_t lengthUnit, const AIE::AIETargetModel &targetModel, int col, int row, int bdId)
std::optional< unsigned > getHostBufferArgIndex(BlockArgument arg)
Unbalanced balanceUnder(AIE::ConsumedByAttr proof, const Volume &in, const Volume &out)
mlir::LogicalResult verifyLengthParameterTile(mlir::Operation *op, const AIETargetModel &targetModel, int col, int row)
mlir::FailureOr< int64_t > getLengthUnitBytes(mlir::Operation *op, int64_t lengthUnit, mlir::BaseMemRefType buffer)
Bits [shift, shift + width) of register word in a buffer descriptor's register block.
uint32_t place(uint64_t value) const
The register layout of one tile type's DMA buffer descriptor.