MLIR-AIE 1.4.6.dev
AIELowerScratchpadParameters.cpp
Go to the documentation of this file.
1//===- AIELowerScratchpadParameters.cpp - Lower scratchpad parameter ops --===//
2//
3// Copyright (C) 2026 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7
12
13#include "mlir/Analysis/DataFlow/ConstantPropagationAnalysis.h"
14#include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h"
15#include "mlir/Analysis/DataFlow/IntegerRangeAnalysis.h"
16#include "mlir/Analysis/DataFlowFramework.h"
17#include "mlir/Dialect/Arith/IR/Arith.h"
18#include "mlir/Dialect/MemRef/IR/MemRef.h"
19#include "mlir/Dialect/Utils/StaticValueUtils.h"
20#include "mlir/IR/Builders.h"
21#include "mlir/IR/Dominance.h"
22#include "mlir/Interfaces/LoopLikeInterface.h"
23#include "mlir/Pass/Pass.h"
24
25#include "llvm/ADT/MapVector.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/SetVector.h"
28#include "llvm/Support/raw_ostream.h"
29
30#include <array>
31#include <numeric>
32
33using namespace mlir;
34using namespace xilinx;
35using namespace xilinx::AIE;
36using namespace xilinx::AIEX;
37
38namespace xilinx::AIEX {
39#define GEN_PASS_DEF_AIELOWERSCRATCHPADPARAMETERS
40#include "aie/Dialect/AIEX/Transforms/AIEXPasses.h.inc"
41} // namespace xilinx::AIEX
42
43namespace {
44
45/// Returns true iff this core needs a parameter-sync lock. Reads the explicit
46/// attribute if present; otherwise defaults to true iff the core body reads a
47/// scratchpad parameter or waits for them.
48static bool shouldEmitParameterSyncPreamble(CoreOp coreOp) {
49 if (auto attr = coreOp.getEmitParameterSyncPreambleAttr()) {
50 return attr.getValue();
51 }
52 WalkResult found = coreOp.getBody().walk([&](Operation *op) {
53 return isa<ReadScratchpadParameterOp, WaitScratchpadParametersOp>(op)
54 ? WalkResult::interrupt()
55 : WalkResult::advance();
56 });
57 return found.wasInterrupted();
58}
59
60/// A DMA with runtime offset `o` and length `l` touches elements
61/// [offset + o, last + o + lengthStep * l].
62struct TransferExtent {
63 int64_t bufferElems;
64 int64_t offset;
65 int64_t last;
66 int64_t lengthStep;
67};
68
69/// `sizes`/`strides` are innermost-first and empty for a linear BD.
70static std::optional<TransferExtent>
71getTransferExtent(Type bufType, int64_t offset, int64_t lenElems,
72 SmallVector<int64_t> sizes, SmallVector<int64_t> strides,
73 std::optional<int64_t> lengthUnit) {
74 auto buffer = llvm::cast<BaseMemRefType>(bufType);
75 if (!buffer.hasStaticShape())
76 return std::nullopt;
77 TransferExtent extent{buffer.getNumElements(), offset, offset, 0};
78 if (!lengthUnit) {
79 if (sizes.empty())
80 extent.last += lenElems - 1;
81 for (auto [size, stride] : llvm::zip(sizes, strides))
82 extent.last += (size - 1) * stride;
83 return extent;
84 }
85 bool linear = sizes.empty();
86 sizes.resize(std::max<size_t>(sizes.size(), 3), 1);
87 strides.resize(sizes.size(), 0);
88 for (size_t i = 3; i < sizes.size(); ++i)
89 extent.last += (sizes[i] - 1) * strides[i];
90 if (linear || AIEX::isContiguousTransfer(sizes, strides)) {
91 extent.last += lenElems - 1;
92 extent.lengthStep = *lengthUnit;
93 return extent;
94 }
96 int64_t rowElems = sizes[0] * sizes[1];
97 // A static length of 0 counts -1 steps, so that the first added unit's
98 // steps start at 0.
99 extent.last += (sizes[0] - 1) * strides[0] + (sizes[1] - 1) * strides[1] +
100 (llvm::divideCeilSigned(lenElems, rowElems) - 1) * strides[2];
101 extent.lengthStep = *lengthUnit / rowElems * strides[2];
102 return extent;
103}
104
105static std::optional<TransferExtent>
106getTransferExtent(NpuDmaMemcpyNdOp op, std::optional<int64_t> lengthUnit) {
107 std::optional<SmallVector<int64_t>> offsets =
108 getConstantIntValues(op.getMixedOffsets());
109 std::optional<SmallVector<int64_t>> sizes =
110 getConstantIntValues(op.getMixedSizes());
111 std::optional<SmallVector<int64_t>> strides =
112 getConstantIntValues(op.getMixedStrides());
113 int64_t elemBits = op.getElementTypeBitwidth();
114 if (!offsets || !sizes || !strides || elemBits % 8 != 0)
115 return std::nullopt;
116 std::reverse(sizes->begin(), sizes->end());
117 std::reverse(strides->begin(), strides->end());
118 return getTransferExtent(
119 op.getMemref().getType(), op.getOffsetInBytes() / (elemBits / 8),
120 (*sizes)[0] * (*sizes)[1] * (*sizes)[2], *sizes, *strides, lengthUnit);
121}
122
123/// The range of a BD's runtime offset, if it is known to stay within the
124/// buffer. An offset computed from a runtime-sequence loop's induction variable
125/// is only folded once the loop is unrolled, after this pass.
126static std::optional<std::pair<int64_t, int64_t>>
127getOffsetRange(Value offset, int64_t bufferElems, DataFlowSolver *solver) {
128 if (!solver)
129 return std::nullopt;
130 auto *state = solver->lookupState<dataflow::IntegerValueRangeLattice>(offset);
131 if (!state || state->getValue().isUninitialized())
132 return std::nullopt;
133 const ConstantIntRanges &range = state->getValue().getValue();
134 // The offset may be wider than 64 bits; an extremum that does not fit is
135 // outside the buffer anyway.
136 std::optional<int64_t> lo = range.smin().trySExtValue();
137 std::optional<int64_t> hi = range.smax().trySExtValue();
138 if (!lo || !hi || *lo < 0 || *hi >= bufferElems)
139 return std::nullopt;
140 return std::make_pair(*lo, *hi);
141}
142
143/// With a runtime offset, the extent starts at its smallest value and ends
144/// at its largest.
145static std::optional<TransferExtent>
146getTransferExtent(AIE::DMABDOp op, std::optional<int64_t> lengthUnit,
147 DataFlowSolver *solver) {
148 auto buffer = llvm::cast<BaseMemRefType>(op.getBuffer().getType());
149 std::optional<int64_t> offset = 0, offsetMin = 0;
150 if (op.hasOffset()) {
151 offset = offsetMin = op.getConstantOffset();
152 if (!offset && buffer.hasStaticShape()) {
153 if (auto range =
154 getOffsetRange(op.getOffset(), buffer.getNumElements(), solver)) {
155 offsetMin = range->first;
156 offset = range->second;
157 }
158 }
159 }
160 std::optional<int64_t> len;
161 if (op.hasLen())
162 len = op.getConstantLen();
163 else if (buffer.hasStaticShape())
164 len = buffer.getNumElements();
165 std::optional<SmallVector<int64_t>> sizes =
166 getConstantIntValues(op.getMixedSizes());
167 std::optional<SmallVector<int64_t>> strides =
168 getConstantIntValues(op.getMixedStrides());
169 if (!offset || !len || !sizes || !strides)
170 return std::nullopt;
171 std::reverse(sizes->begin(), sizes->end());
172 std::reverse(strides->begin(), strides->end());
173 std::optional<TransferExtent> extent =
174 getTransferExtent(buffer, *offset, *len, *sizes, *strides, lengthUnit);
175 if (extent)
176 extent->offset = *offsetMin;
177 return extent;
178}
179
180/// The largest length parameter value that keeps the shim BD's Buffer_Length,
181/// the static length plus that many units in 32-bit words, within its field.
182static int64_t getMaxLengthSteps(Operation *op, int64_t lenBytes,
183 int64_t unitBytes) {
184 int64_t maxWords =
185 getTargetModel(op).getDmaBdMaxLen(AIETileType::ShimNOCTile);
186 return (maxWords - lenBytes / 4) / (unitBytes / 4);
187}
188
189static std::optional<int64_t> getMaxLengthSteps(NpuDmaMemcpyNdOp op) {
190 std::optional<SmallVector<int64_t>> sizes =
191 getConstantIntValues(op.getMixedSizes());
192 std::optional<int64_t> lengthUnit = op.getLengthUnit();
193 if (!sizes || !lengthUnit)
194 return std::nullopt;
195 int64_t elemBytes = op.getElementTypeBitwidth() / 8;
196 return getMaxLengthSteps(op,
197 (*sizes)[1] * (*sizes)[2] * (*sizes)[3] * elemBytes,
198 *lengthUnit * elemBytes);
199}
200
201static std::optional<int64_t> getMaxLengthSteps(AIE::DMABDOp op) {
202 std::optional<int64_t> lengthUnit = op.getLengthUnit();
203 if (!lengthUnit)
204 return std::nullopt;
205 return getMaxLengthSteps(op, op.getLenInBytes(),
206 *lengthUnit * op.getBufferElementTypeWidthInBytes());
207}
208
209/// Returns true iff the parameter-sync preamble (create_scratchpad + set_lock)
210/// should be emitted into this sequence. Reads the explicit attribute if
211/// present; otherwise defaults to true iff the runtime sequence does not
212/// already contain a `aiex.sync_scratchpad_parameters_from_host` marker and
213/// either a core of the device waits on a sync lock (`coresWait`) or the device
214/// uses a scratchpad parameter from a core, a DMA BD or a task start.
215static bool shouldEmitParameterSyncPreamble(RuntimeSequenceOp seqOp,
216 bool coresWait) {
217 if (auto attr = seqOp.getEmitParameterSyncPreambleAttr()) {
218 return attr.getValue();
219 }
220 bool hasManualSync = false;
221 seqOp.walk([&](SyncScratchpadParametersFromHostOp) { hasManualSync = true; });
222 if (hasManualSync) {
223 return false;
224 }
225 if (coresWait) {
226 return true;
227 }
228 auto device = seqOp->getParentOfType<DeviceOp>();
229 if (!device) {
230 return false;
231 }
232 bool found = false;
233 device.walk([&](Operation *op) {
234 if (found) {
235 return;
236 }
237 if (llvm::isa<ReadScratchpadParameterOp>(op) ||
238 op->hasAttr("offset_parameter") ||
239 op->hasAttr("offset_state_table_idx") ||
240 op->hasAttr("length_parameter") ||
241 op->hasAttr("length_state_table_idx") ||
242 op->hasAttr("repeat_parameter") ||
243 op->hasAttr("repeat_state_table_idx")) {
244 found = true;
245 }
246 });
247 return found;
248}
249
250// The command processor's state table: one entry per parameter a command reads.
251constexpr unsigned kMaxStateTableEntries = 32;
252
253/// The parameters each runtime sequence no `aiex.run` calls reads, in walk
254/// order: those its transfers and task starts use, directly or through the
255/// sequences it runs, and the core parameters of every device it reaches.
256/// Each such sequence is a command, which a host runs with a scratchpad of
257/// its own. The `scratchpad_parameters` of a command a previous run lowered
258/// come first, since that run dropped the uses they were found from.
259static llvm::MapVector<RuntimeSequenceOp, llvm::SetVector<StringRef>>
260commandParameters(ModuleOp moduleOp) {
261 DenseMap<Operation *, llvm::SetVector<StringRef>> coreReads;
262 moduleOp.walk([&](ReadScratchpadParameterOp op) {
263 coreReads[op->getParentOfType<DeviceOp>()].insert(op.getParameter());
264 });
265 DenseMap<Operation *, llvm::SetVector<StringRef>> uses;
266 DenseMap<Operation *, SmallVector<RuntimeSequenceOp>> callees;
267 DenseSet<Operation *> called;
269 moduleOp.walk([&](RuntimeSequenceOp seq) {
270 seq.walk([&](Operation *op) {
271 for (StringRef attr :
272 {"offset_parameter", "length_parameter", "repeat_parameter"})
273 if (auto ref = op->getAttrOfType<FlatSymbolRefAttr>(attr))
274 uses[seq].insert(ref.getValue());
275 if (auto run = dyn_cast<RunOp>(op))
276 if (RuntimeSequenceOp callee = run.getCalleeRuntimeSequenceOp(&names)) {
277 callees[seq].push_back(callee);
278 called.insert(callee);
279 }
280 });
281 });
282 llvm::MapVector<RuntimeSequenceOp, llvm::SetVector<StringRef>> commands;
283 moduleOp.walk([&](RuntimeSequenceOp seq) {
284 if (called.contains(seq))
285 return;
286 llvm::SetVector<StringRef> &reads = commands[seq];
287 if (ArrayAttr lowered = seq.getScratchpadParametersAttr())
288 for (auto ref : lowered.getAsRange<FlatSymbolRefAttr>())
289 reads.insert(ref.getValue());
290 // In call order: a sequence, then those it runs.
291 llvm::SetVector<RuntimeSequenceOp> reached;
292 reached.insert(seq);
293 for (unsigned i = 0; i < reached.size(); i++) {
294 RuntimeSequenceOp at = reached[i];
295 reads.insert_range(uses.lookup(at));
296 reads.insert_range(coreReads.lookup(at->getParentOfType<DeviceOp>()));
297 reached.insert_range(callees.lookup(at));
298 }
299 });
300 return commands;
301}
302
303/// Gives parameters state-table slots such that two a command reads together
304/// differ: a colouring of the graph whose cliques are the commands. The search
305/// is exact, so whether a module fits does not depend on the order its
306/// parameters are declared in. It goes in DSatur order (next the parameter
307/// whose conflicts hold the most distinct slots, then the most conflicts,
308/// then the first declared), lowest slot first, so its first descent is the
309/// DSatur greedy colouring, and it backtracks only when that does not fit.
310class SlotColoring {
311public:
312 enum class Result { Found, Infeasible, GaveUp };
313
314 explicit SlotColoring(unsigned numParams)
315 : conflicts(numParams), slots(numParams), wanted(numParams, false),
316 neighbourSlots(numParams), saturation(numParams, 0) {}
317
318 void addConflict(unsigned a, unsigned b) {
319 if (a == b || llvm::is_contained(conflicts[a], b))
320 return;
321 conflicts[a].push_back(b);
322 conflicts[b].push_back(a);
323 }
324
325 void want(unsigned param) { wanted[param] = true; }
326
327 void pin(unsigned param, unsigned slot) {
328 wanted[param] = false;
329 assign(param, slot);
330 maxSlot = std::max<int>(maxSlot, slot);
331 }
332
333 Result solve(unsigned stepLimit) {
334 steps = stepLimit;
335 return search();
336 }
337
338 std::optional<unsigned> slot(unsigned param) const { return slots[param]; }
339
340 /// The parameter the first descent found no slot for, after a failed solve.
341 unsigned stuck() const { return firstStuck; }
342
343private:
344 SmallVector<SmallVector<unsigned>> conflicts;
345 SmallVector<std::optional<unsigned>> slots;
346 SmallVector<bool> wanted;
347 // How many conflicts hold each slot, and how many slots that is.
348 SmallVector<std::array<unsigned, kMaxStateTableEntries>> neighbourSlots;
349 SmallVector<unsigned> saturation;
350 int maxSlot = -1;
351 unsigned steps = 0;
352 bool deadEnd = false;
353 unsigned firstStuck = 0;
354
355 void assign(unsigned param, unsigned slot) {
356 slots[param] = slot;
357 for (unsigned other : conflicts[param])
358 if (neighbourSlots[other][slot]++ == 0)
359 saturation[other]++;
360 }
361
362 void unassign(unsigned param, unsigned slot) {
363 slots[param].reset();
364 for (unsigned other : conflicts[param])
365 if (--neighbourSlots[other][slot] == 0)
366 saturation[other]--;
367 }
368
369 Result search() {
370 std::optional<unsigned> next;
371 for (unsigned p = 0; p < slots.size(); p++) {
372 if (!wanted[p] || slots[p])
373 continue;
374 if (!next || saturation[p] > saturation[*next] ||
375 (saturation[p] == saturation[*next] &&
376 conflicts[p].size() > conflicts[*next].size()))
377 next = p;
378 }
379 if (!next)
380 return Result::Found;
381 // Every slot above the highest held is alike, so only the first is tried.
382 unsigned limit = std::min<unsigned>(maxSlot + 2, kMaxStateTableEntries);
383 for (unsigned s = 0; s < limit; s++) {
384 if (neighbourSlots[*next][s] != 0)
385 continue;
386 if (deadEnd && steps-- == 0)
387 return Result::GaveUp;
388 int previousMax = maxSlot;
389 maxSlot = std::max<int>(maxSlot, s);
390 assign(*next, s);
391 Result result = search();
392 if (result != Result::Infeasible)
393 return result;
394 unassign(*next, s);
395 maxSlot = previousMax;
396 }
397 if (!deadEnd) {
398 deadEnd = true;
399 firstStuck = *next;
400 }
401 return Result::Infeasible;
402 }
403};
404
405// Slots the search tries after its first dead end before giving up.
406constexpr unsigned kMaxSlotSearchSteps = 1 << 17;
407
408struct AIELowerScratchpadParametersPass
409 : public xilinx::AIEX::impl::AIELowerScratchpadParametersBase<
410 AIELowerScratchpadParametersPass> {
411 using AIELowerScratchpadParametersBase::AIELowerScratchpadParametersBase;
412
413 // For each read_scratchpad_parameter of a unique parameter, create a 2xi32
414 // buffer and store a reference to it on the ReadScratchpadParameterOp as
415 // the `buffer` attribute.
416 void allocateBuffers(DeviceOp device, OpBuilder &builder) {
417 MLIRContext *ctx = device.getContext();
418 unsigned uniquingCounter = 0;
419 AIE::NamedOpTable names(device);
420
421 DenseMap<std::pair<StringRef, Operation *>, BufferOp> seen;
422
423 device.walk([&](ReadScratchpadParameterOp readOp) {
424 auto coreOp = readOp->getParentOfType<CoreOp>();
425 TileOp tile = coreOp.getTileOp();
426 StringRef paramName = readOp.getParameter();
427 auto key = std::make_pair(paramName, tile.getOperation());
428
429 if (seen.count(key)) {
430 readOp.setBufferAttr(
431 FlatSymbolRefAttr::get(ctx, *seen[key].getSymName()));
432 return;
433 }
434
435 builder.setInsertionPointAfter(tile);
436 // Buffer must be 8 bytes: update_from_scratchpad always writes a 48-bit
437 // value across two 32-bit registers at [RegOff] and [RegOff+4]. The
438 // firmware masks Reg[0] with 0xFFFFFFFC (lower 2 bits forced to 0)
439 // because it was designed for 4-byte-aligned DMA BD addresses. The host
440 // library's `ParameterScratchpad::write` left-shifts by 2 before
441 // writing to the scratchpad, and the core right-shifts by 2 after
442 // loading. Note that this limits effective parameter values to 30
443 // bits.
444 auto bufType = MemRefType::get({2}, builder.getI32Type());
445 std::string prefix =
446 ("__param_" + paramName + "_" + std::to_string(tile.getCol()) + "_" +
447 std::to_string(tile.getRow()) + "_")
448 .str();
449 std::string bufName = names.uniqueName(prefix, uniquingCounter);
450 auto buf =
451 BufferOp::create(builder, readOp.getLoc(), bufType, tile,
452 builder.getStringAttr(bufName), /*address=*/nullptr,
453 /*initial_value=*/nullptr, /*mem_bank=*/nullptr);
454 seen[key] = buf;
455
456 readOp.setBufferAttr(
457 FlatSymbolRefAttr::get(ctx, *seen[key].getSymName()));
458 });
459 }
460
461 // Lower each read_scratchpad_parameter to: load from buffer[0], shift right
462 // by 2, and cast to the result type. The buffer to use comes from the
463 // `buffer` attribute set by allocateBuffers().
464 void lowerReadParameters(DeviceOp device, OpBuilder &builder) {
465 SmallVector<ReadScratchpadParameterOp> readOps;
466 device.walk([&](ReadScratchpadParameterOp op) { readOps.push_back(op); });
467
468 for (auto readOp : readOps) {
469 FlatSymbolRefAttr bufRef = readOp.getBufferAttr();
470 auto buf = AIE::lookupNamedOpIn<BufferOp>(device, bufRef.getAttr());
471
472 builder.setInsertionPoint(readOp);
473 Value c0 = arith::ConstantIndexOp::create(builder, readOp.getLoc(), 0);
474 Value raw = memref::LoadOp::create(builder, readOp.getLoc(), buf, c0);
475 Value c2 = arith::ConstantOp::create(builder, readOp.getLoc(),
476 builder.getI32IntegerAttr(2));
477 Value decoded = arith::ShRUIOp::create(builder, readOp.getLoc(), raw, c2);
478
479 Type resultType = readOp.getResult().getType();
480 Value result = decoded;
481 if (resultType.isInteger() && resultType != builder.getI32Type()) {
482 result = arith::TruncIOp::create(builder, readOp.getLoc(), resultType,
483 decoded);
484 } else if (resultType.isBF16()) {
485 Value masked = arith::TruncIOp::create(builder, readOp.getLoc(),
486 builder.getI16Type(), decoded);
487 result = arith::BitcastOp::create(builder, readOp.getLoc(), resultType,
488 masked);
489 }
490
491 readOp.getResult().replaceAllUsesWith(result);
492 readOp.erase();
493 }
494 }
495
496 // For each core in `device` with shouldEmitParameterSyncPreamble()==true,
497 // create an aie.lock (no lockID; AIEAssignLockIDs will assign one later) and
498 // acquire it: in place of each aiex.wait_scratchpad_parameters if the core
499 // has any, else once at the top of the core body. Appends the lock values
500 // created (one per qualifying core, in walk order) to `syncLocks`. Fails if
501 // a core with waits reads a parameter no wait dominates, or no wait inside
502 // the outermost loop around the read, or if a core without the preamble
503 // waits.
504 LogicalResult emitCorePreambles(DeviceOp device, OpBuilder &builder,
505 SmallVector<Value> &syncLocks) {
506 WalkResult result = device.walk([&](CoreOp coreOp) {
507 if (!shouldEmitParameterSyncPreamble(coreOp)) {
508 return coreOp.walk([](WaitScratchpadParametersOp wait) {
509 wait.emitOpError("waits in a core with emit_parameter_sync_preamble "
510 "= false, which gets no lock to wait on");
511 return WalkResult::interrupt();
512 });
513 }
514
515 SmallVector<WaitScratchpadParametersOp> waits;
516 coreOp.walk([&](WaitScratchpadParametersOp op) { waits.push_back(op); });
517 if (!waits.empty()) {
518 DominanceInfo dominance(coreOp);
519 WalkResult unordered = coreOp.walk([&](ReadScratchpadParameterOp read) {
520 // A loop serving several dispatches is the outermost one around the
521 // read; a wait outside it passes once, on the first.
522 Operation *outerLoop = nullptr;
523 for (Operation *op = read->getParentOp(); op != coreOp;
524 op = op->getParentOp())
525 if (isa<LoopLikeOpInterface>(op))
526 outerLoop = op;
527 bool dominated = false;
528 for (WaitScratchpadParametersOp wait : waits) {
529 if (!dominance.properlyDominates(wait, read))
530 continue;
531 dominated = true;
532 if (!outerLoop || outerLoop->isProperAncestor(wait))
533 return WalkResult::advance();
534 }
535 if (dominated)
536 read.emitOpError("reads @")
537 << read.getParameter()
538 << " in a loop with no aiex.wait_scratchpad_parameters "
539 "inside it before the read, so after the loop's first "
540 "iteration it may read a previous dispatch's value";
541 else
542 read.emitOpError("reads @")
543 << read.getParameter()
544 << " where no aiex.wait_scratchpad_parameters of its core "
545 "dominates it, so it may read a previous dispatch's value";
546 return WalkResult::interrupt();
547 });
548 if (unordered.wasInterrupted())
549 return WalkResult::interrupt();
550 }
551
552 TileOp tile = coreOp.getTileOp();
553 builder.setInsertionPointAfter(tile);
554 auto lockOp = LockOp::create(
555 builder, coreOp.getLoc(), builder.getIndexType(), tile.getResult(),
556 /*lockID=*/IntegerAttr{}, builder.getI32IntegerAttr(0),
557 /*sym_name=*/StringAttr{});
558 syncLocks.push_back(lockOp.getResult());
559
560 if (waits.empty()) {
561 builder.setInsertionPointToStart(&coreOp.getBody().front());
562 UseLockOp::create(builder, coreOp.getLoc(), lockOp.getResult(),
563 LockAction::Acquire, 1);
564 }
565 for (WaitScratchpadParametersOp wait : waits) {
566 builder.setInsertionPoint(wait);
567 UseLockOp::create(builder, wait.getLoc(), lockOp.getResult(),
568 LockAction::AcquireGreaterEqual, 1);
569 wait.erase();
570 }
571
572 // Mark as done so the pass is idempotent.
573 coreOp.setEmitParameterSyncPreambleAttr(builder.getBoolAttr(false));
574 return WalkResult::advance();
575 });
576 return failure(result.wasInterrupted());
577 }
578
579 // For each runtime sequence in `device` with shouldEmitParameterSyncPreamble
580 // ()==true, insert a marker SyncScratchpadParametersFromHostOp after the
581 // last top-level load_pdi of the sequence body, or at its start if there is
582 // none.
583 LogicalResult emitSequencePreambles(DeviceOp device, OpBuilder &builder,
584 bool coresWait) {
585 WalkResult result = device.walk([&](RuntimeSequenceOp seqOp) {
586 if (!shouldEmitParameterSyncPreamble(seqOp, coresWait)) {
587 return WalkResult::advance();
588 }
589
590 Region &region = seqOp.getBody();
591 if (region.empty())
592 region.emplaceBlock();
593 Block &body = region.front();
594 builder.setInsertionPointToStart(&body);
595 // Loading a PDI resets the core buffers and locks the sync writes.
596 for (NpuLoadPdiOp loadPdi : body.getOps<NpuLoadPdiOp>())
597 builder.setInsertionPointAfter(loadPdi);
598 for (Operation &op :
599 llvm::make_range(body.begin(), builder.getInsertionPoint())) {
600 WalkResult early = op.walk([&](Operation *inner) {
601 auto dmaOp = dyn_cast<NpuDmaMemcpyNdOp>(inner);
602 auto bdOp = dyn_cast<AIE::DMABDOp>(inner);
603 auto startOp = dyn_cast<DMAStartTaskOp>(inner);
604 auto pushOp = dyn_cast<NpuPushQueueOp>(inner);
605 if ((dmaOp && (dmaOp.getOffsetStateTableIdxAttr() ||
606 dmaOp.getLengthStateTableIdxAttr())) ||
607 (bdOp && (bdOp.getOffsetStateTableIdxAttr() ||
608 bdOp.getLengthStateTableIdxAttr())) ||
609 (startOp && startOp.getRepeatStateTableIdxAttr()) ||
610 (pushOp && pushOp.getRepeatStateTableIdxAttr())) {
611 inner->emitOpError(
612 "reads a scratchpad parameter before the last "
613 "aiex.npu.load_pdi of its runtime sequence, which is where "
614 "the scratchpad is created. Load the PDIs first, or place "
615 "aiex.sync_scratchpad_parameters_from_host before this op.");
616 return WalkResult::interrupt();
617 }
618 return WalkResult::advance();
619 });
620 if (early.wasInterrupted())
621 return WalkResult::interrupt();
622 }
623 SyncScratchpadParametersFromHostOp::create(builder, seqOp.getLoc());
624
625 // Mark as done so the pass is idempotent.
626 seqOp.setEmitParameterSyncPreambleAttr(builder.getBoolAttr(false));
627 return WalkResult::advance();
628 });
629 return failure(result.wasInterrupted());
630 }
631
632 // Lower a sync_scratchpad_parameters_from_host marker op in place to its
633 // component ops, using the parameter information for the device that contains
634 // it:
635 // 1. npu.create_scratchpad(scratchpadSize)
636 // 2. For each (state_idx, buffer_ref) in paramEntries: npu.write32(0) +
637 // npu.update_from_scratchpad
638 // 3. set_lock(lock, 1) for each lock in syncLocks
639 void lowerSyncParametersOp(
640 SyncScratchpadParametersFromHostOp syncOp, uint32_t scratchpadSize,
641 const SmallVector<std::pair<uint8_t, FlatSymbolRefAttr>> &paramEntries,
642 const SmallVector<Value> &syncLocks) {
643 OpBuilder builder(syncOp);
644 Location loc = syncOp.getLoc();
645
646 // A module with no parameters has nothing to stage, and size 0 is invalid.
647 if (scratchpadSize > 0)
648 NpuCreateScratchpadOp::create(builder, loc, scratchpadSize);
649
650 for (auto &[stateIdx, bufRef] : paramEntries) {
651 // Zero the destination before the additive UpdateScratchpad.
652 NpuWrite32Op::create(builder, loc,
653 /*address=*/createConstantI32(builder, loc, 0),
654 /*value=*/createConstantI32(builder, loc, 0), bufRef,
655 /*column=*/nullptr, /*row=*/nullptr);
656 NpuUpdateFromScratchpadOp::create(
657 builder, loc, stateIdx, StateTableFunc::Incr,
658 /*func_arg=*/static_cast<uint32_t>(0),
659 /*address=*/static_cast<uint32_t>(0), bufRef,
660 /*column=*/nullptr, /*row=*/nullptr);
661 }
662
663 for (Value lock : syncLocks) {
664 SetLockOp::create(builder, loc, lock, createConstantI32(builder, loc, 1));
665 }
666
667 syncOp.erase();
668 }
669
670 // Lower every sync_scratchpad_parameters_from_host op in `device` to its
671 // component ops in place, using parameter info for this device only.
672 void lowerSyncParametersOps(
673 DeviceOp device, uint32_t scratchpadSize,
674 const SmallVector<std::pair<uint8_t, FlatSymbolRefAttr>> &paramEntries,
675 const SmallVector<Value> &syncLocks) {
676 SmallVector<SyncScratchpadParametersFromHostOp> syncOps;
677 device.walk(
678 [&](SyncScratchpadParametersFromHostOp op) { syncOps.push_back(op); });
679 for (SyncScratchpadParametersFromHostOp syncOp : syncOps) {
680 lowerSyncParametersOp(syncOp, scratchpadSize, paramEntries, syncLocks);
681 }
682 }
683
684 // Emit a single params.txt for the whole module to `outputParamsFile`.
685 LogicalResult emitParamsFile() {
686 if (outputParamsFile.empty())
687 return success();
688
689 std::error_code ec;
690 llvm::raw_fd_ostream out(outputParamsFile, ec);
691 if (ec) {
692 return emitError(UnknownLoc::get(&getContext()),
693 "failed to open params output file '")
694 << outputParamsFile << "': " << ec.message();
695 }
696
697 emitScratchpadParamsFile(getOperation(), out);
698 return success();
699 }
700
701 void runOnOperation() override {
702 ModuleOp moduleOp = getOperation();
703 OpBuilder builder(&getContext());
704
705 // Ranges are only needed for a parameterized BD with a runtime offset.
706 std::unique_ptr<DataFlowSolver> solver;
707 WalkResult needRanges = moduleOp.walk([](AIE::DMABDOp bd) {
708 bool param = bd.getLengthParameterAttr() || bd.getOffsetParameterAttr();
709 return param && bd.getOffset() && !bd.getConstantOffset()
710 ? WalkResult::interrupt()
711 : WalkResult::advance();
712 });
713 if (needRanges.wasInterrupted()) {
714 solver = std::make_unique<DataFlowSolver>();
715 solver->load<dataflow::DeadCodeAnalysis>();
716 solver->load<dataflow::SparseConstantPropagation>();
717 solver->load<dataflow::IntegerRangeAnalysis>();
718 if (failed(solver->initializeAndRun(moduleOp)))
719 solver.reset();
720 }
721
722 // Step 1: collect every parameter in the module, and the ones each
723 // command reads.
724 SmallVector<ScratchpadParameterOp> allParams;
725 moduleOp.walk([&](ScratchpadParameterOp p) { allParams.push_back(p); });
726 llvm::MapVector<RuntimeSequenceOp, llvm::SetVector<StringRef>> commands =
727 commandParameters(moduleOp);
728 for (auto &[seq, reads] : commands) {
729 if (reads.size() <= kMaxStateTableEntries)
730 continue;
731 InFlightDiagnostic diag =
732 seq.emitError("runtime sequence '")
733 << scratchpadCommandName(seq) << "' reads " << reads.size()
734 << " parameters but its scratchpad holds at most "
735 << kMaxStateTableEntries
736 << ": each parameter a command reads takes one of the command "
737 "processor's state-table entries";
738 for (StringRef name : reads)
739 diag.attachNote(seq.getLoc()) << "parameter '" << name << "'";
740 return signalPassFailure();
741 }
742
743 // Step 2: determine each parameter's kind from its usage, erroring on
744 // mixed use. A parameter is "core" if any aiex.read_scratchpad_parameter
745 // references it; "addr" if any DMA op references it via offset_parameter.
746 // If both, emit an error. A DMA length_parameter works with either kind,
747 // so it takes the kind of the parameter's other uses, and "addr" if it is
748 // the only use; the lowering compensates in the update's multiplier.
749 DenseMap<StringRef, bool> usedAsCore;
750 DenseMap<StringRef, bool> usedAsLength;
751 DenseMap<StringRef, bool> usedAsAddr;
752 moduleOp.walk([&](ReadScratchpadParameterOp op) {
753 usedAsCore[op.getParameter()] = true;
754 });
755 auto markDmaUses = [&](FlatSymbolRefAttr offsetRef,
756 FlatSymbolRefAttr lengthRef) {
757 if (offsetRef)
758 usedAsAddr[offsetRef.getValue()] = true;
759 if (lengthRef)
760 usedAsLength[lengthRef.getValue()] = true;
761 };
762 moduleOp.walk([&](NpuDmaMemcpyNdOp op) {
763 markDmaUses(op.getOffsetParameterAttr(), op.getLengthParameterAttr());
764 });
765 moduleOp.walk([&](AIE::DMABDOp op) {
766 markDmaUses(op.getOffsetParameterAttr(), op.getLengthParameterAttr());
767 });
768 // A repeat count is written shifted into the task queue's repeat field,
769 // an encoding no other use shares.
770 DenseMap<StringRef, bool> usedAsRepeat;
771 moduleOp.walk([&](DMAStartTaskOp op) {
772 if (FlatSymbolRefAttr ref = op.getRepeatParameterAttr())
773 usedAsRepeat[ref.getValue()] = true;
774 });
775
776 for (auto p : allParams) {
777 StringRef name = p.getSymName();
778 bool core = usedAsCore.lookup(name);
779 bool length = usedAsLength.lookup(name);
780 bool addr = usedAsAddr.lookup(name);
781 if (usedAsRepeat.lookup(name)) {
782 if (core || length || addr) {
783 p.emitError("parameter '")
784 << name
785 << "' is an aiex.dma_start_task repeat_parameter (repeat) and "
786 "also read by a core or used as a DMA offset or length; a "
787 "repeat parameter must have no other use";
788 return signalPassFailure();
789 }
790 p.setKindAttr(ScratchpadParameterKindAttr::get(
791 &getContext(), ScratchpadParameterKind::Repeat));
792 continue;
793 }
794 // Step 3b drops the symbol uses, so a lowered parameter keeps its kind.
795 if (!core && !length && !addr && p.getKindAttr())
796 continue;
797 if (core && addr) {
798 p.emitError("parameter '")
799 << name
800 << "' is used both as an aiex.read_scratchpad_parameter source "
801 "(core) and as a DMA offset_parameter (addr); a parameter "
802 "must have a single kind";
803 return signalPassFailure();
804 }
805 p.setKindAttr(ScratchpadParameterKindAttr::get(
806 &getContext(), addr || (length && !core)
807 ? ScratchpadParameterKind::Addr
808 : ScratchpadParameterKind::Core));
809 }
810
811 // Step 3: colour the slots, keeping those a previous run assigned.
812 DenseMap<StringRef, unsigned> paramIndex;
813 for (auto [i, p] : llvm::enumerate(allParams))
814 paramIndex[p.getSymName()] = i;
815 SlotColoring coloring(allParams.size());
816 DenseMap<unsigned, SmallVector<RuntimeSequenceOp>> readers;
817 for (auto &[seq, reads] : commands) {
818 SmallVector<unsigned> read;
819 for (StringRef name : reads)
820 if (auto it = paramIndex.find(name); it != paramIndex.end())
821 read.push_back(it->second);
822 DenseMap<unsigned, unsigned> pinnedBy;
823 for (auto [i, a] : llvm::enumerate(read)) {
824 readers[a].push_back(seq);
825 coloring.want(a);
826 for (unsigned b : llvm::drop_begin(read, i + 1))
827 coloring.addConflict(a, b);
828 std::optional<uint8_t> slot = allParams[a].getStateTableIdx();
829 if (!slot)
830 continue;
831 auto [it, inserted] = pinnedBy.try_emplace(*slot, a);
832 if (!inserted) {
833 allParams[a].emitError("parameter '")
834 << allParams[a].getSymName() << "' and another parameter "
835 << "command '" << scratchpadCommandName(seq)
836 << "' reads both have state-table entry " << *slot;
837 return signalPassFailure();
838 }
839 }
840 }
841 for (auto [i, p] : llvm::enumerate(allParams)) {
842 std::optional<uint8_t> slot = p.getStateTableIdx();
843 if (!slot)
844 continue;
845 if (*slot >= kMaxStateTableEntries) {
846 p.emitError("parameter '")
847 << p.getSymName() << "' has state-table entry " << *slot
848 << ", past the " << kMaxStateTableEntries << " there are";
849 return signalPassFailure();
850 }
851 coloring.pin(i, *slot);
852 }
853 SlotColoring::Result colored = coloring.solve(kMaxSlotSearchSteps);
854 if (colored != SlotColoring::Result::Found) {
855 ScratchpadParameterOp p = allParams[coloring.stuck()];
856 InFlightDiagnostic diag = p.emitError("parameter '")
857 << p.getSymName()
858 << "' finds no free state-table entry among "
859 << kMaxStateTableEntries << ": ";
860 if (colored == SlotColoring::Result::Infeasible)
861 diag << "no assignment gives the parameters each command reads "
862 "distinct entries";
863 else
864 diag << "a search of " << kMaxSlotSearchSteps
865 << " steps found no assignment that gives the parameters each "
866 "command reads distinct entries";
867 for (RuntimeSequenceOp seq : readers.lookup(coloring.stuck()))
868 diag.attachNote(seq.getLoc())
869 << "read by command '" << scratchpadCommandName(seq) << "'";
870 return signalPassFailure();
871 }
872 unsigned numSlots = 0;
873 for (auto [i, p] : llvm::enumerate(allParams)) {
874 std::optional<unsigned> slot = coloring.slot(i);
875 if (!slot)
876 continue;
877 numSlots = std::max(numSlots, *slot + 1);
878 p.setStateTableIdxAttr(builder.getIntegerAttr(
879 builder.getIntegerType(8, /*isSigned=*/false), *slot));
880 }
881 for (auto &[seq, reads] : commands) {
882 SmallVector<Attribute> refs;
883 for (StringRef name : reads)
884 refs.push_back(FlatSymbolRefAttr::get(&getContext(), name));
885 seq.setScratchpadParametersAttr(builder.getArrayAttr(refs));
886 }
887
888 // Step 3b: rewrite DMA `offset_parameter` / `length_parameter` symbol
889 // references to plain `offset_state_table_idx` / `length_state_table_idx`
890 // integer attributes, so downstream `aie`-dialect passes do not need to
891 // resolve `aiex.scratchpad_parameter` symbols.
892 auto rewriteParam =
893 [&](Operation *op, FlatSymbolRefAttr ref, StringRef attrName,
894 StringRef idxAttrName) -> FailureOr<ScratchpadParameterOp> {
895 auto paramOp =
896 moduleOp.lookupSymbol<ScratchpadParameterOp>(ref.getAttr());
897 if (!paramOp) {
898 op->emitOpError() << attrName << " '" << ref.getValue()
899 << "' not found. Declare it at module scope with "
900 "aiex.scratchpad_parameter.";
901 return failure();
902 }
903 if (!paramOp.getType().isInteger(32)) {
904 auto err = op->emitOpError()
905 << attrName << " '" << ref.getValue()
906 << "' must have type i32, got " << paramOp.getType() << ".";
907 err.attachNote(paramOp.getLoc()) << "Parameter declared here.";
908 return failure();
909 }
910 std::optional<uint8_t> stateIdx = paramOp.getStateTableIdx();
911 if (!stateIdx) {
912 op->emitOpError() << attrName << " '" << ref.getValue()
913 << "' has no state-table entry: no command, a "
914 "runtime sequence no aiex.run calls, reaches "
915 "this op";
916 return failure();
917 }
918 op->setAttr(
919 idxAttrName,
920 builder.getIntegerAttr(builder.getIntegerType(8, /*isSigned=*/false),
921 *stateIdx));
922 op->removeAttr(attrName);
923 return paramOp;
924 };
925 // The values of each DMA offset or length parameter that keep every
926 // transfer using it within its buffer, judged with the transfer's other
927 // parameter at 0.
928 DenseMap<StringRef, std::pair<int64_t, int64_t>> bounds;
929 // (length, offset, length step) -> max of offset + length step * length.
930 llvm::MapVector<std::tuple<StringRef, StringRef, int64_t>, int64_t>
931 jointBounds;
932 // Each offset parameter's byte step, 16 where the tile is not known; see
933 // AIEX::emitUpdateBdAddressFromOffsetParameter for why.
934 DenseMap<StringRef, int64_t> offsetMultiples;
935 SymbolTableCollection symbolTables;
936 auto rewriteParams =
937 [&](Operation *op, FlatSymbolRefAttr offsetRef,
938 FlatSymbolRefAttr lengthRef, Type bufType,
939 std::optional<TransferExtent> extent,
940 std::optional<int64_t> maxLengthSteps) -> LogicalResult {
941 bool joint = offsetRef == lengthRef;
942 int64_t room = extent ? extent->bufferElems - 1 - extent->last : 0;
943 if (offsetRef) {
944 if (failed(rewriteParam(op, offsetRef, "offset_parameter",
945 "offset_state_table_idx")))
946 return failure();
947 uint32_t elemBytes =
948 llvm::cast<BaseMemRefType>(bufType).getElementTypeBitWidth() / 8;
949 Operation *task = op->getParentOfType<DMAConfigureTaskOp>();
950 if (!task)
951 task = op->getParentOfType<DMAConfigureTaskForOp>();
952 auto device = task ? task->getParentOfType<DeviceOp>() : DeviceOp();
953 auto tile = device ? dyn_cast_or_null<TileLike>(getRuntimeDmaTile(
954 task, symbolTables.getSymbolTable(device)))
955 : TileLike();
956 uint32_t step =
957 isa<NpuDmaMemcpyNdOp>(op) || (tile && !tile.isMemTile()) ? 4 : 16;
958 if (elemBytes && elemBytes % step) {
959 int64_t &multiple =
960 offsetMultiples.try_emplace(offsetRef.getValue(), 1)
961 .first->second;
962 multiple =
963 std::lcm<int64_t>(multiple, step / std::gcd(step, elemBytes));
964 }
965 auto &[lo, hi] =
966 bounds.try_emplace(offsetRef.getValue(), INT32_MIN, INT32_MAX)
967 .first->second;
968 if (extent) {
969 lo = std::max(lo, -extent->offset);
970 hi = std::min(hi, llvm::divideFloorSigned(
971 room, 1 + (joint ? extent->lengthStep : 0)));
972 }
973 }
974 if (lengthRef) {
975 FailureOr<ScratchpadParameterOp> lengthParam = rewriteParam(
976 op, lengthRef, "length_parameter", "length_state_table_idx");
977 if (failed(lengthParam))
978 return failure();
979 auto &[lo, hi] =
980 bounds.try_emplace(lengthRef.getValue(), INT32_MIN, INT32_MAX)
981 .first->second;
982 lo = std::max<int64_t>(lo, 0);
983 if (maxLengthSteps)
984 hi = std::min(hi, *maxLengthSteps);
985 if (lengthParam->getKind() == ScratchpadParameterKind::Core) {
986 op->setAttr("length_core_encoded", builder.getUnitAttr());
987 hi = std::min<int64_t>(hi, (1 << 30) - 1);
988 }
989 if (extent && !joint && extent->lengthStep > 0)
990 hi = std::min(hi, llvm::divideFloorSigned(room, extent->lengthStep));
991 if (extent && offsetRef && !joint && extent->lengthStep > 0) {
992 auto [it, inserted] = jointBounds.try_emplace(
993 {lengthRef.getValue(), offsetRef.getValue(), extent->lengthStep},
994 room);
995 it->second = std::min(it->second, room);
996 }
997 }
998 for (FlatSymbolRefAttr ref : {offsetRef, lengthRef}) {
999 if (!ref)
1000 continue;
1001 auto [lo, hi] = bounds.lookup(ref.getValue());
1002 if (lo > hi)
1003 return op->emitOpError("no value of parameter '")
1004 << ref.getValue()
1005 << "' keeps every transfer using it within its buffer";
1006 }
1007 return success();
1008 };
1009 WalkResult rewriteResult = moduleOp.walk([&](Operation *op) {
1010 if (auto dmaOp = dyn_cast<NpuDmaMemcpyNdOp>(op)) {
1011 FlatSymbolRefAttr offsetRef = dmaOp.getOffsetParameterAttr();
1012 FlatSymbolRefAttr lengthRef = dmaOp.getLengthParameterAttr();
1013 if (!offsetRef && !lengthRef)
1014 return WalkResult::advance();
1015 if (failed(rewriteParams(
1016 op, offsetRef, lengthRef, dmaOp.getMemref().getType(),
1017 getTransferExtent(dmaOp, lengthRef ? dmaOp.getLengthUnit()
1018 : std::nullopt),
1019 getMaxLengthSteps(dmaOp))))
1020 return WalkResult::interrupt();
1021 } else if (auto bdOp = dyn_cast<AIE::DMABDOp>(op)) {
1022 FlatSymbolRefAttr offsetRef = bdOp.getOffsetParameterAttr();
1023 FlatSymbolRefAttr lengthRef = bdOp.getLengthParameterAttr();
1024 if (!offsetRef && !lengthRef)
1025 return WalkResult::advance();
1026 std::optional<int64_t> lengthUnit;
1027 if (lengthRef)
1028 lengthUnit = bdOp.getLengthUnit();
1029 if (failed(rewriteParams(
1030 op, offsetRef, lengthRef, bdOp.getBuffer().getType(),
1031 getTransferExtent(bdOp, lengthUnit, solver.get()),
1032 getMaxLengthSteps(bdOp))))
1033 return WalkResult::interrupt();
1034 }
1035 return WalkResult::advance();
1036 });
1037 if (rewriteResult.wasInterrupted()) {
1038 return signalPassFailure();
1039 }
1040 // A repeat parameter is bounded by its task's own repeat count, the
1041 // largest the start may push.
1042 WalkResult repeatResult = moduleOp.walk([&](DMAStartTaskOp op) {
1043 FlatSymbolRefAttr ref = op.getRepeatParameterAttr();
1044 if (!ref)
1045 return WalkResult::advance();
1046 auto task = op.getTask().getDefiningOp<DMAConfigureTaskLike>();
1047 if (!task) {
1048 op.emitOpError("has a repeat_parameter, so it must start the task "
1049 "where it is configured: the task's repeat count "
1050 "bounds the parameter");
1051 return WalkResult::interrupt();
1052 }
1053 if (task.getRepeatCountVal()) {
1054 op.emitOpError("has a repeat_parameter, which its task's repeat count "
1055 "bounds, but that is a runtime value");
1056 return WalkResult::interrupt();
1057 }
1058 if (failed(rewriteParam(op, ref, "repeat_parameter",
1059 "repeat_state_table_idx")))
1060 return WalkResult::interrupt();
1061 auto &[lo, hi] =
1062 bounds.try_emplace(ref.getValue(), 0, INT32_MAX).first->second;
1063 hi = std::min<int64_t>(hi, task.getRepeatCount());
1064 return WalkResult::advance();
1065 });
1066 if (repeatResult.wasInterrupted())
1067 return signalPassFailure();
1068 for (ScratchpadParameterOp p : allParams) {
1069 auto it = bounds.find(p.getSymName());
1070 if (it == bounds.end())
1071 continue;
1072 p.setMinValueAttr(builder.getI32IntegerAttr(it->second.first));
1073 p.setMaxValueAttr(builder.getI32IntegerAttr(it->second.second));
1074 }
1075 for (ScratchpadParameterOp p : allParams)
1076 if (int64_t multiple = offsetMultiples.lookup(p.getSymName())) {
1077 p.setOffsetMultipleAttr(builder.getI32IntegerAttr(multiple));
1078 // Another offset corrupts the buffer descriptor, not just rounds.
1079 if (multiple > 1)
1080 p.emitRemark() << "takes offsets in multiples of " << multiple
1081 << " elements; the host's scratchpad helpers reject "
1082 "others, and a host writing the scratchpad itself "
1083 "must too";
1084 }
1085 llvm::MapVector<StringRef, SmallVector<Attribute>> jointBoundAttrs;
1086 for (auto [key, max] : jointBounds) {
1087 auto [length, offset, step] = key;
1088 jointBoundAttrs[length].push_back(JointBoundAttr::get(
1089 &getContext(), FlatSymbolRefAttr::get(&getContext(), offset), step,
1090 max));
1091 }
1092 for (ScratchpadParameterOp p : allParams) {
1093 auto *it = jointBoundAttrs.find(p.getSymName());
1094 if (it != jointBoundAttrs.end())
1095 p.setJointBoundsAttr(builder.getArrayAttr(it->second));
1096 }
1097
1098 // Step 4: per-device lowering.
1099 SmallVector<DeviceOp> devices;
1100 moduleOp.walk([&](DeviceOp d) { devices.push_back(d); });
1101 for (auto d : devices) {
1102 allocateBuffers(d, builder);
1103
1104 // Collect unique (stateIdx, bufferRef) pairs for core-kind parameters in
1105 // this device. Buffer attrs are set by allocateBuffers() above.
1106 SmallVector<std::pair<uint8_t, FlatSymbolRefAttr>> paramEntries;
1107 DenseSet<StringRef> seenBufs;
1108 d.walk([&](ReadScratchpadParameterOp readOp) {
1109 FlatSymbolRefAttr bufRef = readOp.getBufferAttr();
1110 if (!bufRef || !seenBufs.insert(bufRef.getValue()).second) {
1111 return;
1112 }
1113 auto paramOp =
1114 moduleOp.lookupSymbol<ScratchpadParameterOp>(readOp.getParameter());
1115 // No command reaches a core whose parameter has no slot, so no sync
1116 // writes it.
1117 if (std::optional<uint8_t> stateIdx = paramOp.getStateTableIdx())
1118 paramEntries.push_back({*stateIdx, bufRef});
1119 });
1120
1121 // Emit lock + use_lock preambles in cores, then insert a marker
1122 // SyncScratchpadParametersFromHostOp at the start of each qualifying
1123 // runtime sequence. Immediately lower every
1124 // sync_scratchpad_parameters_from_host op in the device (both the
1125 // preamble-inserted ones and any user-written ones) using this device's
1126 // parameter info.
1127 SmallVector<Value> syncLocks;
1128 if (failed(emitCorePreambles(d, builder, syncLocks)) ||
1129 failed(emitSequencePreambles(d, builder, !syncLocks.empty())))
1130 return signalPassFailure();
1131 uint32_t scratchpadSize = numSlots * 4;
1132 lowerSyncParametersOps(d, scratchpadSize, paramEntries, syncLocks);
1133
1134 lowerReadParameters(d, builder);
1135 }
1136
1137 // Step 5: emit the single params.txt for the module.
1138 if (failed(emitParamsFile())) {
1139 return signalPassFailure();
1140 }
1141 }
1142};
1143
1144} // namespace
1145
1146std::unique_ptr<OperationPass<ModuleOp>>
1148 return std::make_unique<AIELowerScratchpadParametersPass>();
1149}
1150
1151std::unique_ptr<OperationPass<ModuleOp>>
1153 AIELowerScratchpadParametersOptions options) {
1154 return std::make_unique<AIELowerScratchpadParametersPass>(std::move(options));
1155}
virtual uint64_t getDmaBdMaxLen(AIETileType tileType) const =0
Return the maximum transfer length, measured in address-generation granules (see getAddressGenGranula...
bool isContiguousTransfer(llvm::ArrayRef< int64_t > sizes, llvm::ArrayRef< int64_t > strides)
Operation * getRuntimeDmaTile(Operation *op, SymbolTable &symbols)
Definition AIEUtils.cpp:578
std::unique_ptr< mlir::OperationPass< mlir::ModuleOp > > createAIELowerScratchpadParametersPass()
std::string scratchpadCommandName(AIE::RuntimeSequenceOp seq)
Definition AIEUtils.cpp:451
void emitScratchpadParamsFile(mlir::ModuleOp moduleOp, llvm::raw_ostream &os)
mlir::Value createConstantI32(mlir::OpBuilder &builder, mlir::Location loc, uint32_t value)
mlir::Operation * lookupNamedOpIn(mlir::Operation *symbolTableOp, mlir::StringAttr name)
const AIETargetModel & getTargetModel(mlir::Operation *op)
void placeRuntimeLengthDimension(llvm::MutableArrayRef< int64_t > sizes, llvm::MutableArrayRef< int64_t > strides)