13#include "mlir/Analysis/DataFlow/ConstantPropagationAnalysis.h"
14#include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h"
15#include "mlir/Analysis/DataFlow/IntegerRangeAnalysis.h"
16#include "mlir/Analysis/DataFlowFramework.h"
17#include "mlir/Dialect/Arith/IR/Arith.h"
18#include "mlir/Dialect/MemRef/IR/MemRef.h"
19#include "mlir/Dialect/Utils/StaticValueUtils.h"
20#include "mlir/IR/Builders.h"
21#include "mlir/IR/Dominance.h"
22#include "mlir/Interfaces/LoopLikeInterface.h"
23#include "mlir/Pass/Pass.h"
25#include "llvm/ADT/MapVector.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/SetVector.h"
28#include "llvm/Support/raw_ostream.h"
39#define GEN_PASS_DEF_AIELOWERSCRATCHPADPARAMETERS
40#include "aie/Dialect/AIEX/Transforms/AIEXPasses.h.inc"
48static bool shouldEmitParameterSyncPreamble(CoreOp coreOp) {
49 if (
auto attr = coreOp.getEmitParameterSyncPreambleAttr()) {
50 return attr.getValue();
52 WalkResult found = coreOp.getBody().walk([&](Operation *op) {
53 return isa<ReadScratchpadParameterOp, WaitScratchpadParametersOp>(op)
54 ? WalkResult::interrupt()
55 : WalkResult::advance();
57 return found.wasInterrupted();
62struct TransferExtent {
70static std::optional<TransferExtent>
71getTransferExtent(Type bufType, int64_t offset, int64_t lenElems,
72 SmallVector<int64_t> sizes, SmallVector<int64_t> strides,
73 std::optional<int64_t> lengthUnit) {
74 auto buffer = llvm::cast<BaseMemRefType>(bufType);
75 if (!buffer.hasStaticShape())
77 TransferExtent extent{buffer.getNumElements(), offset, offset, 0};
80 extent.last += lenElems - 1;
81 for (
auto [size, stride] : llvm::zip(sizes, strides))
82 extent.last += (size - 1) * stride;
85 bool linear = sizes.empty();
86 sizes.resize(std::max<size_t>(sizes.size(), 3), 1);
87 strides.resize(sizes.size(), 0);
88 for (
size_t i = 3; i < sizes.size(); ++i)
89 extent.last += (sizes[i] - 1) * strides[i];
91 extent.last += lenElems - 1;
92 extent.lengthStep = *lengthUnit;
96 int64_t rowElems = sizes[0] * sizes[1];
99 extent.last += (sizes[0] - 1) * strides[0] + (sizes[1] - 1) * strides[1] +
100 (llvm::divideCeilSigned(lenElems, rowElems) - 1) * strides[2];
101 extent.lengthStep = *lengthUnit / rowElems * strides[2];
105static std::optional<TransferExtent>
106getTransferExtent(NpuDmaMemcpyNdOp op, std::optional<int64_t> lengthUnit) {
107 std::optional<SmallVector<int64_t>> offsets =
108 getConstantIntValues(op.getMixedOffsets());
109 std::optional<SmallVector<int64_t>> sizes =
110 getConstantIntValues(op.getMixedSizes());
111 std::optional<SmallVector<int64_t>> strides =
112 getConstantIntValues(op.getMixedStrides());
113 int64_t elemBits = op.getElementTypeBitwidth();
114 if (!offsets || !sizes || !strides || elemBits % 8 != 0)
116 std::reverse(sizes->begin(), sizes->end());
117 std::reverse(strides->begin(), strides->end());
118 return getTransferExtent(
119 op.getMemref().getType(), op.getOffsetInBytes() / (elemBits / 8),
120 (*sizes)[0] * (*sizes)[1] * (*sizes)[2], *sizes, *strides, lengthUnit);
126static std::optional<std::pair<int64_t, int64_t>>
127getOffsetRange(Value offset, int64_t bufferElems, DataFlowSolver *solver) {
130 auto *state = solver->lookupState<dataflow::IntegerValueRangeLattice>(offset);
131 if (!state || state->getValue().isUninitialized())
133 const ConstantIntRanges &range = state->getValue().getValue();
136 std::optional<int64_t> lo = range.smin().trySExtValue();
137 std::optional<int64_t> hi = range.smax().trySExtValue();
138 if (!lo || !hi || *lo < 0 || *hi >= bufferElems)
140 return std::make_pair(*lo, *hi);
145static std::optional<TransferExtent>
146getTransferExtent(AIE::DMABDOp op, std::optional<int64_t> lengthUnit,
147 DataFlowSolver *solver) {
148 auto buffer = llvm::cast<BaseMemRefType>(op.getBuffer().getType());
149 std::optional<int64_t> offset = 0, offsetMin = 0;
150 if (op.hasOffset()) {
151 offset = offsetMin = op.getConstantOffset();
152 if (!offset && buffer.hasStaticShape()) {
154 getOffsetRange(op.getOffset(), buffer.getNumElements(), solver)) {
155 offsetMin = range->first;
156 offset = range->second;
160 std::optional<int64_t> len;
162 len = op.getConstantLen();
163 else if (buffer.hasStaticShape())
164 len = buffer.getNumElements();
165 std::optional<SmallVector<int64_t>> sizes =
166 getConstantIntValues(op.getMixedSizes());
167 std::optional<SmallVector<int64_t>> strides =
168 getConstantIntValues(op.getMixedStrides());
169 if (!offset || !len || !sizes || !strides)
171 std::reverse(sizes->begin(), sizes->end());
172 std::reverse(strides->begin(), strides->end());
173 std::optional<TransferExtent> extent =
174 getTransferExtent(buffer, *offset, *len, *sizes, *strides, lengthUnit);
176 extent->offset = *offsetMin;
182static int64_t getMaxLengthSteps(Operation *op, int64_t lenBytes,
186 return (maxWords - lenBytes / 4) / (unitBytes / 4);
189static std::optional<int64_t> getMaxLengthSteps(NpuDmaMemcpyNdOp op) {
190 std::optional<SmallVector<int64_t>> sizes =
191 getConstantIntValues(op.getMixedSizes());
192 std::optional<int64_t> lengthUnit = op.getLengthUnit();
193 if (!sizes || !lengthUnit)
195 int64_t elemBytes = op.getElementTypeBitwidth() / 8;
196 return getMaxLengthSteps(op,
197 (*sizes)[1] * (*sizes)[2] * (*sizes)[3] * elemBytes,
198 *lengthUnit * elemBytes);
201static std::optional<int64_t> getMaxLengthSteps(AIE::DMABDOp op) {
202 std::optional<int64_t> lengthUnit = op.getLengthUnit();
205 return getMaxLengthSteps(op, op.getLenInBytes(),
206 *lengthUnit * op.getBufferElementTypeWidthInBytes());
215static bool shouldEmitParameterSyncPreamble(RuntimeSequenceOp seqOp,
217 if (
auto attr = seqOp.getEmitParameterSyncPreambleAttr()) {
218 return attr.getValue();
220 bool hasManualSync =
false;
221 seqOp.walk([&](SyncScratchpadParametersFromHostOp) { hasManualSync =
true; });
228 auto device = seqOp->getParentOfType<DeviceOp>();
233 device.walk([&](Operation *op) {
237 if (llvm::isa<ReadScratchpadParameterOp>(op) ||
238 op->hasAttr(
"offset_parameter") ||
239 op->hasAttr(
"offset_state_table_idx") ||
240 op->hasAttr(
"length_parameter") ||
241 op->hasAttr(
"length_state_table_idx") ||
242 op->hasAttr(
"repeat_parameter") ||
243 op->hasAttr(
"repeat_state_table_idx")) {
251constexpr unsigned kMaxStateTableEntries = 32;
259static llvm::MapVector<RuntimeSequenceOp, llvm::SetVector<StringRef>>
260commandParameters(ModuleOp moduleOp) {
261 DenseMap<Operation *, llvm::SetVector<StringRef>> coreReads;
262 moduleOp.walk([&](ReadScratchpadParameterOp op) {
263 coreReads[op->getParentOfType<DeviceOp>()].insert(op.getParameter());
265 DenseMap<Operation *, llvm::SetVector<StringRef>> uses;
266 DenseMap<Operation *, SmallVector<RuntimeSequenceOp>> callees;
267 DenseSet<Operation *> called;
269 moduleOp.walk([&](RuntimeSequenceOp seq) {
270 seq.walk([&](Operation *op) {
271 for (StringRef attr :
272 {
"offset_parameter",
"length_parameter",
"repeat_parameter"})
273 if (
auto ref = op->getAttrOfType<FlatSymbolRefAttr>(attr))
274 uses[seq].insert(ref.getValue());
275 if (
auto run = dyn_cast<RunOp>(op))
276 if (RuntimeSequenceOp callee = run.getCalleeRuntimeSequenceOp(&names)) {
277 callees[seq].push_back(callee);
278 called.insert(callee);
282 llvm::MapVector<RuntimeSequenceOp, llvm::SetVector<StringRef>> commands;
283 moduleOp.walk([&](RuntimeSequenceOp seq) {
284 if (called.contains(seq))
286 llvm::SetVector<StringRef> &reads = commands[seq];
287 if (ArrayAttr lowered = seq.getScratchpadParametersAttr())
288 for (
auto ref : lowered.getAsRange<FlatSymbolRefAttr>())
289 reads.insert(ref.getValue());
291 llvm::SetVector<RuntimeSequenceOp> reached;
293 for (
unsigned i = 0; i < reached.size(); i++) {
294 RuntimeSequenceOp at = reached[i];
295 reads.insert_range(uses.lookup(at));
296 reads.insert_range(coreReads.lookup(at->getParentOfType<DeviceOp>()));
297 reached.insert_range(callees.lookup(at));
312 enum class Result { Found, Infeasible, GaveUp };
314 explicit SlotColoring(
unsigned numParams)
315 : conflicts(numParams), slots(numParams), wanted(numParams, false),
316 neighbourSlots(numParams), saturation(numParams, 0) {}
318 void addConflict(
unsigned a,
unsigned b) {
319 if (a == b || llvm::is_contained(conflicts[a], b))
321 conflicts[a].push_back(b);
322 conflicts[b].push_back(a);
325 void want(
unsigned param) { wanted[param] =
true; }
327 void pin(
unsigned param,
unsigned slot) {
328 wanted[param] =
false;
330 maxSlot = std::max<int>(maxSlot, slot);
333 Result solve(
unsigned stepLimit) {
338 std::optional<unsigned> slot(
unsigned param)
const {
return slots[param]; }
341 unsigned stuck()
const {
return firstStuck; }
344 SmallVector<SmallVector<unsigned>> conflicts;
345 SmallVector<std::optional<unsigned>> slots;
346 SmallVector<bool> wanted;
348 SmallVector<std::array<unsigned, kMaxStateTableEntries>> neighbourSlots;
349 SmallVector<unsigned> saturation;
352 bool deadEnd =
false;
353 unsigned firstStuck = 0;
355 void assign(
unsigned param,
unsigned slot) {
357 for (
unsigned other : conflicts[param])
358 if (neighbourSlots[other][slot]++ == 0)
362 void unassign(
unsigned param,
unsigned slot) {
363 slots[param].reset();
364 for (
unsigned other : conflicts[param])
365 if (--neighbourSlots[other][slot] == 0)
370 std::optional<unsigned> next;
371 for (
unsigned p = 0; p < slots.size(); p++) {
372 if (!wanted[p] || slots[p])
374 if (!next || saturation[p] > saturation[*next] ||
375 (saturation[p] == saturation[*next] &&
376 conflicts[p].size() > conflicts[*next].size()))
380 return Result::Found;
382 unsigned limit = std::min<unsigned>(maxSlot + 2, kMaxStateTableEntries);
383 for (
unsigned s = 0; s < limit; s++) {
384 if (neighbourSlots[*next][s] != 0)
386 if (deadEnd && steps-- == 0)
387 return Result::GaveUp;
388 int previousMax = maxSlot;
389 maxSlot = std::max<int>(maxSlot, s);
391 Result result = search();
392 if (result != Result::Infeasible)
395 maxSlot = previousMax;
401 return Result::Infeasible;
406constexpr unsigned kMaxSlotSearchSteps = 1 << 17;
408struct AIELowerScratchpadParametersPass
409 :
public xilinx::AIEX::impl::AIELowerScratchpadParametersBase<
410 AIELowerScratchpadParametersPass> {
411 using AIELowerScratchpadParametersBase::AIELowerScratchpadParametersBase;
416 void allocateBuffers(DeviceOp device, OpBuilder &builder) {
417 MLIRContext *ctx = device.getContext();
418 unsigned uniquingCounter = 0;
419 AIE::NamedOpTable names(device);
421 DenseMap<std::pair<StringRef, Operation *>, BufferOp> seen;
423 device.walk([&](ReadScratchpadParameterOp readOp) {
424 auto coreOp = readOp->getParentOfType<CoreOp>();
425 TileOp tile = coreOp.getTileOp();
426 StringRef paramName = readOp.getParameter();
427 auto key = std::make_pair(paramName, tile.getOperation());
429 if (seen.count(key)) {
430 readOp.setBufferAttr(
431 FlatSymbolRefAttr::get(ctx, *seen[key].getSymName()));
435 builder.setInsertionPointAfter(tile);
444 auto bufType = MemRefType::get({2}, builder.getI32Type());
446 (
"__param_" + paramName +
"_" + std::to_string(tile.getCol()) +
"_" +
447 std::to_string(tile.getRow()) +
"_")
449 std::string bufName = names.uniqueName(prefix, uniquingCounter);
451 BufferOp::create(builder, readOp.getLoc(), bufType, tile,
452 builder.getStringAttr(bufName),
nullptr,
456 readOp.setBufferAttr(
457 FlatSymbolRefAttr::get(ctx, *seen[key].getSymName()));
464 void lowerReadParameters(DeviceOp device, OpBuilder &builder) {
465 SmallVector<ReadScratchpadParameterOp> readOps;
466 device.walk([&](ReadScratchpadParameterOp op) { readOps.push_back(op); });
468 for (
auto readOp : readOps) {
469 FlatSymbolRefAttr bufRef = readOp.getBufferAttr();
472 builder.setInsertionPoint(readOp);
473 Value c0 = arith::ConstantIndexOp::create(builder, readOp.getLoc(), 0);
474 Value raw = memref::LoadOp::create(builder, readOp.getLoc(), buf, c0);
475 Value c2 = arith::ConstantOp::create(builder, readOp.getLoc(),
476 builder.getI32IntegerAttr(2));
477 Value decoded = arith::ShRUIOp::create(builder, readOp.getLoc(), raw, c2);
479 Type resultType = readOp.getResult().getType();
480 Value result = decoded;
481 if (resultType.isInteger() && resultType != builder.getI32Type()) {
482 result = arith::TruncIOp::create(builder, readOp.getLoc(), resultType,
484 }
else if (resultType.isBF16()) {
485 Value masked = arith::TruncIOp::create(builder, readOp.getLoc(),
486 builder.getI16Type(), decoded);
487 result = arith::BitcastOp::create(builder, readOp.getLoc(), resultType,
491 readOp.getResult().replaceAllUsesWith(result);
504 LogicalResult emitCorePreambles(DeviceOp device, OpBuilder &builder,
505 SmallVector<Value> &syncLocks) {
506 WalkResult result = device.walk([&](CoreOp coreOp) {
507 if (!shouldEmitParameterSyncPreamble(coreOp)) {
508 return coreOp.walk([](WaitScratchpadParametersOp wait) {
509 wait.emitOpError(
"waits in a core with emit_parameter_sync_preamble "
510 "= false, which gets no lock to wait on");
511 return WalkResult::interrupt();
515 SmallVector<WaitScratchpadParametersOp> waits;
516 coreOp.walk([&](WaitScratchpadParametersOp op) { waits.push_back(op); });
517 if (!waits.empty()) {
518 DominanceInfo dominance(coreOp);
519 WalkResult unordered = coreOp.walk([&](ReadScratchpadParameterOp read) {
522 Operation *outerLoop =
nullptr;
523 for (Operation *op = read->getParentOp(); op != coreOp;
524 op = op->getParentOp())
525 if (isa<LoopLikeOpInterface>(op))
527 bool dominated =
false;
528 for (WaitScratchpadParametersOp wait : waits) {
529 if (!dominance.properlyDominates(wait, read))
532 if (!outerLoop || outerLoop->isProperAncestor(wait))
533 return WalkResult::advance();
536 read.emitOpError(
"reads @")
537 << read.getParameter()
538 <<
" in a loop with no aiex.wait_scratchpad_parameters "
539 "inside it before the read, so after the loop's first "
540 "iteration it may read a previous dispatch's value";
542 read.emitOpError(
"reads @")
543 << read.getParameter()
544 <<
" where no aiex.wait_scratchpad_parameters of its core "
545 "dominates it, so it may read a previous dispatch's value";
546 return WalkResult::interrupt();
548 if (unordered.wasInterrupted())
549 return WalkResult::interrupt();
552 TileOp tile = coreOp.getTileOp();
553 builder.setInsertionPointAfter(tile);
554 auto lockOp = LockOp::create(
555 builder, coreOp.getLoc(), builder.getIndexType(), tile.getResult(),
556 IntegerAttr{}, builder.getI32IntegerAttr(0),
558 syncLocks.push_back(lockOp.getResult());
561 builder.setInsertionPointToStart(&coreOp.getBody().front());
562 UseLockOp::create(builder, coreOp.getLoc(), lockOp.getResult(),
563 LockAction::Acquire, 1);
565 for (WaitScratchpadParametersOp wait : waits) {
566 builder.setInsertionPoint(wait);
567 UseLockOp::create(builder, wait.getLoc(), lockOp.getResult(),
568 LockAction::AcquireGreaterEqual, 1);
573 coreOp.setEmitParameterSyncPreambleAttr(builder.getBoolAttr(
false));
574 return WalkResult::advance();
576 return failure(result.wasInterrupted());
583 LogicalResult emitSequencePreambles(DeviceOp device, OpBuilder &builder,
585 WalkResult result = device.walk([&](RuntimeSequenceOp seqOp) {
586 if (!shouldEmitParameterSyncPreamble(seqOp, coresWait)) {
587 return WalkResult::advance();
590 Region ®ion = seqOp.getBody();
592 region.emplaceBlock();
593 Block &body = region.front();
594 builder.setInsertionPointToStart(&body);
596 for (NpuLoadPdiOp loadPdi : body.getOps<NpuLoadPdiOp>())
597 builder.setInsertionPointAfter(loadPdi);
599 llvm::make_range(body.begin(), builder.getInsertionPoint())) {
600 WalkResult early = op.walk([&](Operation *inner) {
601 auto dmaOp = dyn_cast<NpuDmaMemcpyNdOp>(inner);
602 auto bdOp = dyn_cast<AIE::DMABDOp>(inner);
603 auto startOp = dyn_cast<DMAStartTaskOp>(inner);
604 auto pushOp = dyn_cast<NpuPushQueueOp>(inner);
605 if ((dmaOp && (dmaOp.getOffsetStateTableIdxAttr() ||
606 dmaOp.getLengthStateTableIdxAttr())) ||
607 (bdOp && (bdOp.getOffsetStateTableIdxAttr() ||
608 bdOp.getLengthStateTableIdxAttr())) ||
609 (startOp && startOp.getRepeatStateTableIdxAttr()) ||
610 (pushOp && pushOp.getRepeatStateTableIdxAttr())) {
612 "reads a scratchpad parameter before the last "
613 "aiex.npu.load_pdi of its runtime sequence, which is where "
614 "the scratchpad is created. Load the PDIs first, or place "
615 "aiex.sync_scratchpad_parameters_from_host before this op.");
616 return WalkResult::interrupt();
618 return WalkResult::advance();
620 if (early.wasInterrupted())
621 return WalkResult::interrupt();
623 SyncScratchpadParametersFromHostOp::create(builder, seqOp.getLoc());
626 seqOp.setEmitParameterSyncPreambleAttr(builder.getBoolAttr(
false));
627 return WalkResult::advance();
629 return failure(result.wasInterrupted());
639 void lowerSyncParametersOp(
640 SyncScratchpadParametersFromHostOp syncOp, uint32_t scratchpadSize,
641 const SmallVector<std::pair<uint8_t, FlatSymbolRefAttr>> ¶mEntries,
642 const SmallVector<Value> &syncLocks) {
643 OpBuilder builder(syncOp);
644 Location loc = syncOp.getLoc();
647 if (scratchpadSize > 0)
648 NpuCreateScratchpadOp::create(builder, loc, scratchpadSize);
650 for (
auto &[stateIdx, bufRef] : paramEntries) {
652 NpuWrite32Op::create(builder, loc,
656 NpuUpdateFromScratchpadOp::create(
657 builder, loc, stateIdx, StateTableFunc::Incr,
658 static_cast<uint32_t
>(0),
659 static_cast<uint32_t
>(0), bufRef,
663 for (Value lock : syncLocks) {
672 void lowerSyncParametersOps(
673 DeviceOp device, uint32_t scratchpadSize,
674 const SmallVector<std::pair<uint8_t, FlatSymbolRefAttr>> ¶mEntries,
675 const SmallVector<Value> &syncLocks) {
676 SmallVector<SyncScratchpadParametersFromHostOp> syncOps;
678 [&](SyncScratchpadParametersFromHostOp op) { syncOps.push_back(op); });
679 for (SyncScratchpadParametersFromHostOp syncOp : syncOps) {
680 lowerSyncParametersOp(syncOp, scratchpadSize, paramEntries, syncLocks);
685 LogicalResult emitParamsFile() {
686 if (outputParamsFile.empty())
690 llvm::raw_fd_ostream out(outputParamsFile, ec);
692 return emitError(UnknownLoc::get(&getContext()),
693 "failed to open params output file '")
694 << outputParamsFile <<
"': " << ec.message();
701 void runOnOperation()
override {
702 ModuleOp moduleOp = getOperation();
703 OpBuilder builder(&getContext());
706 std::unique_ptr<DataFlowSolver> solver;
707 WalkResult needRanges = moduleOp.walk([](AIE::DMABDOp bd) {
708 bool param = bd.getLengthParameterAttr() || bd.getOffsetParameterAttr();
709 return param && bd.getOffset() && !bd.getConstantOffset()
710 ? WalkResult::interrupt()
711 : WalkResult::advance();
713 if (needRanges.wasInterrupted()) {
714 solver = std::make_unique<DataFlowSolver>();
715 solver->load<dataflow::DeadCodeAnalysis>();
716 solver->load<dataflow::SparseConstantPropagation>();
717 solver->load<dataflow::IntegerRangeAnalysis>();
718 if (failed(solver->initializeAndRun(moduleOp)))
724 SmallVector<ScratchpadParameterOp> allParams;
725 moduleOp.walk([&](ScratchpadParameterOp p) { allParams.push_back(p); });
726 llvm::MapVector<RuntimeSequenceOp, llvm::SetVector<StringRef>> commands =
727 commandParameters(moduleOp);
728 for (
auto &[seq, reads] : commands) {
729 if (reads.size() <= kMaxStateTableEntries)
731 InFlightDiagnostic diag =
732 seq.emitError(
"runtime sequence '")
734 <<
" parameters but its scratchpad holds at most "
735 << kMaxStateTableEntries
736 <<
": each parameter a command reads takes one of the command "
737 "processor's state-table entries";
738 for (StringRef name : reads)
739 diag.attachNote(seq.getLoc()) <<
"parameter '" << name <<
"'";
740 return signalPassFailure();
749 DenseMap<StringRef, bool> usedAsCore;
750 DenseMap<StringRef, bool> usedAsLength;
751 DenseMap<StringRef, bool> usedAsAddr;
752 moduleOp.walk([&](ReadScratchpadParameterOp op) {
753 usedAsCore[op.getParameter()] =
true;
755 auto markDmaUses = [&](FlatSymbolRefAttr offsetRef,
756 FlatSymbolRefAttr lengthRef) {
758 usedAsAddr[offsetRef.getValue()] =
true;
760 usedAsLength[lengthRef.getValue()] =
true;
762 moduleOp.walk([&](NpuDmaMemcpyNdOp op) {
763 markDmaUses(op.getOffsetParameterAttr(), op.getLengthParameterAttr());
765 moduleOp.walk([&](AIE::DMABDOp op) {
766 markDmaUses(op.getOffsetParameterAttr(), op.getLengthParameterAttr());
770 DenseMap<StringRef, bool> usedAsRepeat;
771 moduleOp.walk([&](DMAStartTaskOp op) {
772 if (FlatSymbolRefAttr ref = op.getRepeatParameterAttr())
773 usedAsRepeat[ref.getValue()] =
true;
776 for (
auto p : allParams) {
777 StringRef name = p.getSymName();
778 bool core = usedAsCore.lookup(name);
779 bool length = usedAsLength.lookup(name);
780 bool addr = usedAsAddr.lookup(name);
781 if (usedAsRepeat.lookup(name)) {
782 if (core || length || addr) {
783 p.emitError(
"parameter '")
785 <<
"' is an aiex.dma_start_task repeat_parameter (repeat) and "
786 "also read by a core or used as a DMA offset or length; a "
787 "repeat parameter must have no other use";
788 return signalPassFailure();
790 p.setKindAttr(ScratchpadParameterKindAttr::get(
791 &getContext(), ScratchpadParameterKind::Repeat));
795 if (!core && !length && !addr && p.getKindAttr())
798 p.emitError(
"parameter '")
800 <<
"' is used both as an aiex.read_scratchpad_parameter source "
801 "(core) and as a DMA offset_parameter (addr); a parameter "
802 "must have a single kind";
803 return signalPassFailure();
805 p.setKindAttr(ScratchpadParameterKindAttr::get(
806 &getContext(), addr || (length && !core)
807 ? ScratchpadParameterKind::Addr
808 : ScratchpadParameterKind::Core));
812 DenseMap<StringRef, unsigned> paramIndex;
813 for (
auto [i, p] : llvm::enumerate(allParams))
814 paramIndex[p.getSymName()] = i;
815 SlotColoring coloring(allParams.size());
816 DenseMap<unsigned, SmallVector<RuntimeSequenceOp>> readers;
817 for (
auto &[seq, reads] : commands) {
818 SmallVector<unsigned> read;
819 for (StringRef name : reads)
820 if (
auto it = paramIndex.find(name); it != paramIndex.end())
821 read.push_back(it->second);
822 DenseMap<unsigned, unsigned> pinnedBy;
823 for (
auto [i, a] : llvm::enumerate(read)) {
824 readers[a].push_back(seq);
826 for (
unsigned b : llvm::drop_begin(read, i + 1))
827 coloring.addConflict(a, b);
828 std::optional<uint8_t> slot = allParams[a].getStateTableIdx();
831 auto [it, inserted] = pinnedBy.try_emplace(*slot, a);
833 allParams[a].emitError(
"parameter '")
834 << allParams[a].getSymName() <<
"' and another parameter "
836 <<
"' reads both have state-table entry " << *slot;
837 return signalPassFailure();
841 for (
auto [i, p] : llvm::enumerate(allParams)) {
842 std::optional<uint8_t> slot = p.getStateTableIdx();
845 if (*slot >= kMaxStateTableEntries) {
846 p.emitError(
"parameter '")
847 << p.getSymName() <<
"' has state-table entry " << *slot
848 <<
", past the " << kMaxStateTableEntries <<
" there are";
849 return signalPassFailure();
851 coloring.pin(i, *slot);
853 SlotColoring::Result colored = coloring.solve(kMaxSlotSearchSteps);
854 if (colored != SlotColoring::Result::Found) {
855 ScratchpadParameterOp p = allParams[coloring.stuck()];
856 InFlightDiagnostic diag = p.emitError(
"parameter '")
858 <<
"' finds no free state-table entry among "
859 << kMaxStateTableEntries <<
": ";
860 if (colored == SlotColoring::Result::Infeasible)
861 diag <<
"no assignment gives the parameters each command reads "
864 diag <<
"a search of " << kMaxSlotSearchSteps
865 <<
" steps found no assignment that gives the parameters each "
866 "command reads distinct entries";
867 for (RuntimeSequenceOp seq : readers.lookup(coloring.stuck()))
868 diag.attachNote(seq.getLoc())
870 return signalPassFailure();
872 unsigned numSlots = 0;
873 for (
auto [i, p] : llvm::enumerate(allParams)) {
874 std::optional<unsigned> slot = coloring.slot(i);
877 numSlots = std::max(numSlots, *slot + 1);
878 p.setStateTableIdxAttr(builder.getIntegerAttr(
879 builder.getIntegerType(8,
false), *slot));
881 for (
auto &[seq, reads] : commands) {
882 SmallVector<Attribute> refs;
883 for (StringRef name : reads)
884 refs.push_back(FlatSymbolRefAttr::get(&getContext(), name));
885 seq.setScratchpadParametersAttr(builder.getArrayAttr(refs));
893 [&](Operation *op, FlatSymbolRefAttr ref, StringRef attrName,
894 StringRef idxAttrName) -> FailureOr<ScratchpadParameterOp> {
896 moduleOp.lookupSymbol<ScratchpadParameterOp>(ref.getAttr());
898 op->emitOpError() << attrName <<
" '" << ref.getValue()
899 <<
"' not found. Declare it at module scope with "
900 "aiex.scratchpad_parameter.";
903 if (!paramOp.getType().isInteger(32)) {
904 auto err = op->emitOpError()
905 << attrName <<
" '" << ref.getValue()
906 <<
"' must have type i32, got " << paramOp.getType() <<
".";
907 err.attachNote(paramOp.getLoc()) <<
"Parameter declared here.";
910 std::optional<uint8_t> stateIdx = paramOp.getStateTableIdx();
912 op->emitOpError() << attrName <<
" '" << ref.getValue()
913 <<
"' has no state-table entry: no command, a "
914 "runtime sequence no aiex.run calls, reaches "
920 builder.getIntegerAttr(builder.getIntegerType(8,
false),
922 op->removeAttr(attrName);
928 DenseMap<StringRef, std::pair<int64_t, int64_t>> bounds;
930 llvm::MapVector<std::tuple<StringRef, StringRef, int64_t>, int64_t>
934 DenseMap<StringRef, int64_t> offsetMultiples;
935 SymbolTableCollection symbolTables;
937 [&](Operation *op, FlatSymbolRefAttr offsetRef,
938 FlatSymbolRefAttr lengthRef, Type bufType,
939 std::optional<TransferExtent> extent,
940 std::optional<int64_t> maxLengthSteps) -> LogicalResult {
941 bool joint = offsetRef == lengthRef;
942 int64_t room = extent ? extent->bufferElems - 1 - extent->last : 0;
944 if (failed(rewriteParam(op, offsetRef,
"offset_parameter",
945 "offset_state_table_idx")))
948 llvm::cast<BaseMemRefType>(bufType).getElementTypeBitWidth() / 8;
949 Operation *task = op->getParentOfType<DMAConfigureTaskOp>();
951 task = op->getParentOfType<DMAConfigureTaskForOp>();
952 auto device = task ? task->getParentOfType<DeviceOp>() : DeviceOp();
954 task, symbolTables.getSymbolTable(device)))
957 isa<NpuDmaMemcpyNdOp>(op) || (tile && !tile.isMemTile()) ? 4 : 16;
958 if (elemBytes && elemBytes % step) {
960 offsetMultiples.try_emplace(offsetRef.getValue(), 1)
963 std::lcm<int64_t>(multiple, step / std::gcd(step, elemBytes));
966 bounds.try_emplace(offsetRef.getValue(), INT32_MIN, INT32_MAX)
969 lo = std::max(lo, -extent->offset);
970 hi = std::min(hi, llvm::divideFloorSigned(
971 room, 1 + (joint ? extent->lengthStep : 0)));
975 FailureOr<ScratchpadParameterOp> lengthParam = rewriteParam(
976 op, lengthRef,
"length_parameter",
"length_state_table_idx");
977 if (failed(lengthParam))
980 bounds.try_emplace(lengthRef.getValue(), INT32_MIN, INT32_MAX)
982 lo = std::max<int64_t>(lo, 0);
984 hi = std::min(hi, *maxLengthSteps);
985 if (lengthParam->getKind() == ScratchpadParameterKind::Core) {
986 op->setAttr(
"length_core_encoded", builder.getUnitAttr());
987 hi = std::min<int64_t>(hi, (1 << 30) - 1);
989 if (extent && !joint && extent->lengthStep > 0)
990 hi = std::min(hi, llvm::divideFloorSigned(room, extent->lengthStep));
991 if (extent && offsetRef && !joint && extent->lengthStep > 0) {
992 auto [it, inserted] = jointBounds.try_emplace(
993 {lengthRef.getValue(), offsetRef.getValue(), extent->lengthStep},
995 it->second = std::min(it->second, room);
998 for (FlatSymbolRefAttr ref : {offsetRef, lengthRef}) {
1001 auto [lo, hi] = bounds.lookup(ref.getValue());
1003 return op->emitOpError(
"no value of parameter '")
1005 <<
"' keeps every transfer using it within its buffer";
1009 WalkResult rewriteResult = moduleOp.walk([&](Operation *op) {
1010 if (
auto dmaOp = dyn_cast<NpuDmaMemcpyNdOp>(op)) {
1011 FlatSymbolRefAttr offsetRef = dmaOp.getOffsetParameterAttr();
1012 FlatSymbolRefAttr lengthRef = dmaOp.getLengthParameterAttr();
1013 if (!offsetRef && !lengthRef)
1014 return WalkResult::advance();
1015 if (failed(rewriteParams(
1016 op, offsetRef, lengthRef, dmaOp.getMemref().getType(),
1017 getTransferExtent(dmaOp, lengthRef ? dmaOp.getLengthUnit()
1019 getMaxLengthSteps(dmaOp))))
1020 return WalkResult::interrupt();
1021 }
else if (
auto bdOp = dyn_cast<AIE::DMABDOp>(op)) {
1022 FlatSymbolRefAttr offsetRef = bdOp.getOffsetParameterAttr();
1023 FlatSymbolRefAttr lengthRef = bdOp.getLengthParameterAttr();
1024 if (!offsetRef && !lengthRef)
1025 return WalkResult::advance();
1026 std::optional<int64_t> lengthUnit;
1028 lengthUnit = bdOp.getLengthUnit();
1029 if (failed(rewriteParams(
1030 op, offsetRef, lengthRef, bdOp.getBuffer().getType(),
1031 getTransferExtent(bdOp, lengthUnit, solver.get()),
1032 getMaxLengthSteps(bdOp))))
1033 return WalkResult::interrupt();
1035 return WalkResult::advance();
1037 if (rewriteResult.wasInterrupted()) {
1038 return signalPassFailure();
1042 WalkResult repeatResult = moduleOp.walk([&](DMAStartTaskOp op) {
1043 FlatSymbolRefAttr ref = op.getRepeatParameterAttr();
1045 return WalkResult::advance();
1046 auto task = op.getTask().getDefiningOp<DMAConfigureTaskLike>();
1048 op.emitOpError(
"has a repeat_parameter, so it must start the task "
1049 "where it is configured: the task's repeat count "
1050 "bounds the parameter");
1051 return WalkResult::interrupt();
1053 if (task.getRepeatCountVal()) {
1054 op.emitOpError(
"has a repeat_parameter, which its task's repeat count "
1055 "bounds, but that is a runtime value");
1056 return WalkResult::interrupt();
1058 if (failed(rewriteParam(op, ref,
"repeat_parameter",
1059 "repeat_state_table_idx")))
1060 return WalkResult::interrupt();
1062 bounds.try_emplace(ref.getValue(), 0, INT32_MAX).first->second;
1063 hi = std::min<int64_t>(hi, task.getRepeatCount());
1064 return WalkResult::advance();
1066 if (repeatResult.wasInterrupted())
1067 return signalPassFailure();
1068 for (ScratchpadParameterOp p : allParams) {
1069 auto it = bounds.find(p.getSymName());
1070 if (it == bounds.end())
1072 p.setMinValueAttr(builder.getI32IntegerAttr(it->second.first));
1073 p.setMaxValueAttr(builder.getI32IntegerAttr(it->second.second));
1075 for (ScratchpadParameterOp p : allParams)
1076 if (int64_t multiple = offsetMultiples.lookup(p.getSymName())) {
1077 p.setOffsetMultipleAttr(builder.getI32IntegerAttr(multiple));
1080 p.emitRemark() <<
"takes offsets in multiples of " << multiple
1081 <<
" elements; the host's scratchpad helpers reject "
1082 "others, and a host writing the scratchpad itself "
1085 llvm::MapVector<StringRef, SmallVector<Attribute>> jointBoundAttrs;
1086 for (
auto [key, max] : jointBounds) {
1087 auto [length, offset, step] = key;
1088 jointBoundAttrs[length].push_back(JointBoundAttr::get(
1089 &getContext(), FlatSymbolRefAttr::get(&getContext(), offset), step,
1092 for (ScratchpadParameterOp p : allParams) {
1093 auto *it = jointBoundAttrs.find(p.getSymName());
1094 if (it != jointBoundAttrs.end())
1095 p.setJointBoundsAttr(builder.getArrayAttr(it->second));
1099 SmallVector<DeviceOp> devices;
1100 moduleOp.walk([&](DeviceOp d) { devices.push_back(d); });
1101 for (
auto d : devices) {
1102 allocateBuffers(d, builder);
1106 SmallVector<std::pair<uint8_t, FlatSymbolRefAttr>> paramEntries;
1107 DenseSet<StringRef> seenBufs;
1108 d.walk([&](ReadScratchpadParameterOp readOp) {
1109 FlatSymbolRefAttr bufRef = readOp.getBufferAttr();
1110 if (!bufRef || !seenBufs.insert(bufRef.getValue()).second) {
1114 moduleOp.lookupSymbol<ScratchpadParameterOp>(readOp.getParameter());
1117 if (std::optional<uint8_t> stateIdx = paramOp.getStateTableIdx())
1118 paramEntries.push_back({*stateIdx, bufRef});
1127 SmallVector<Value> syncLocks;
1128 if (failed(emitCorePreambles(d, builder, syncLocks)) ||
1129 failed(emitSequencePreambles(d, builder, !syncLocks.empty())))
1130 return signalPassFailure();
1131 uint32_t scratchpadSize = numSlots * 4;
1132 lowerSyncParametersOps(d, scratchpadSize, paramEntries, syncLocks);
1134 lowerReadParameters(d, builder);
1138 if (failed(emitParamsFile())) {
1139 return signalPassFailure();
1146std::unique_ptr<OperationPass<ModuleOp>>
1148 return std::make_unique<AIELowerScratchpadParametersPass>();
1151std::unique_ptr<OperationPass<ModuleOp>>
1153 AIELowerScratchpadParametersOptions options) {
1154 return std::make_unique<AIELowerScratchpadParametersPass>(std::move(options));
virtual uint64_t getDmaBdMaxLen(AIETileType tileType) const =0
Return the maximum transfer length, measured in address-generation granules (see getAddressGenGranula...
bool isContiguousTransfer(llvm::ArrayRef< int64_t > sizes, llvm::ArrayRef< int64_t > strides)
Operation * getRuntimeDmaTile(Operation *op, SymbolTable &symbols)
std::unique_ptr< mlir::OperationPass< mlir::ModuleOp > > createAIELowerScratchpadParametersPass()
std::string scratchpadCommandName(AIE::RuntimeSequenceOp seq)
void emitScratchpadParamsFile(mlir::ModuleOp moduleOp, llvm::raw_ostream &os)
mlir::Value createConstantI32(mlir::OpBuilder &builder, mlir::Location loc, uint32_t value)
mlir::Operation * lookupNamedOpIn(mlir::Operation *symbolTableOp, mlir::StringAttr name)
const AIETargetModel & getTargetModel(mlir::Operation *op)
void placeRuntimeLengthDimension(llvm::MutableArrayRef< int64_t > sizes, llvm::MutableArrayRef< int64_t > strides)