MLIR-AIE 1.4.6.dev
AIEMaterializeRuntimeSequences.cpp
Go to the documentation of this file.
1//===- AIEMaterializeRuntimeSequences.cpp -----------------------*- C++ -*-===//
2//
3// Copyright (C) 2025 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7
13#include "aie/Targets/AIERT.h"
14
15#include "mlir/Dialect/Arith/IR/Arith.h"
16#include "mlir/Dialect/MemRef/IR/MemRef.h"
17#include "mlir/IR/IRMapping.h"
18#include "mlir/IR/PatternMatch.h"
19#include "mlir/Pass/Pass.h"
20#include "mlir/Transforms/DialectConversion.h"
21#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
22#include "mlir/Transforms/WalkPatternRewriteDriver.h"
23#include "llvm/ADT/STLExtras.h"
24
25#include <set>
26
27namespace xilinx::AIEX {
28#define GEN_PASS_DEF_AIEMATERIALIZERUNTIMESEQUENCES
29#include "aie/Dialect/AIEX/Transforms/AIEXPasses.h.inc"
30} // namespace xilinx::AIEX
31
32#define DEBUG_TYPE "aie-materialize-runtime-sequence"
33
34using namespace mlir;
35using namespace xilinx;
36using namespace xilinx::AIEX;
37
39 AnalysisManager &analysisManager;
40
41 // if invalid, analysis failed and results should not be considered
42 bool isValid = false;
43
44 // Call graph is cyclic
45 bool isCyclic = false;
46
47 RuntimeCallGraphCyclicityAnalysis(Operation *op, AnalysisManager &am)
48 : analysisManager(am) {
49 AIE::RuntimeSequenceOp runtimeSequenceOp =
50 llvm::dyn_cast<AIE::RuntimeSequenceOp>(op);
51 if (!runtimeSequenceOp) {
52 op->emitError("RuntimeCallGraphCyclicityAnalysis can only be called on "
53 "aiex.runtime_sequence operations.");
54 return;
55 }
56
57 // Use DFS with a stack to detect cycles
58 // A cycle exists if we encounter a sequence already on the current path
59 llvm::DenseSet<AIE::RuntimeSequenceOp> callStack;
60 llvm::DenseSet<AIE::RuntimeSequenceOp> visited;
62
63 std::function<bool(AIE::RuntimeSequenceOp)> hasCycle =
64 [&](AIE::RuntimeSequenceOp seq) -> bool {
65 if (callStack.contains(seq)) {
66 return true; // Found a cycle
67 }
68 if (visited.contains(seq)) {
69 return false; // Already checked this sequence
70 }
71
72 callStack.insert(seq);
73 visited.insert(seq);
74
75 // Check all sequences called by this one
76 bool foundCycle = false;
77 seq.walk([&](RunOp runOp) {
78 if (AIE::RuntimeSequenceOp callee =
79 runOp.getCalleeRuntimeSequenceOp(&calleeNames)) {
80 if (hasCycle(callee)) {
81 foundCycle = true;
82 return WalkResult::interrupt();
83 }
84 }
85 return WalkResult::advance();
86 });
87
88 callStack.erase(seq);
89 return foundCycle;
90 };
91
92 if (hasCycle(runtimeSequenceOp)) {
93 isCyclic = true;
94 isValid = true;
95 return;
96 }
97 isCyclic = false;
98 isValid = true;
99 }
100};
101
102// Turn aie.configure @device into aie.run %.. @configure
103// TODO: add check that liveness of two aie.configures do not overlap
104// (i.e., when we configure A, then configure B, cannot call runtime sequence of
105// A after configuring B)
106// TODO: add code to remove repeated @configure ops
107struct InsertLoadPdiForConfigurePattern : RewritePattern {
108
110 PatternBenefit benefit = 1)
111 : RewritePattern(ConfigureOp::getOperationName(), benefit, context) {}
112
113 LogicalResult matchAndRewrite(Operation *op,
114 PatternRewriter &rewriter) const override {
115 ConfigureOp configureOp = llvm::dyn_cast<ConfigureOp>(op);
116 if (!configureOp) {
117 return failure();
118 }
119
120 // LoadPDI resets the whole device, hence cannot do partial reconfiguration;
121 // therefore, this only supports top-level configure ops
122 if (!llvm::isa<AIE::RuntimeSequenceOp>(configureOp->getParentOp())) {
123 return failure();
124 }
125
126 AIE::DeviceOp referencedDevice = configureOp.getReferencedDeviceOp();
127 if (!referencedDevice) {
128 configureOp.emitError("Referenced symbol is not a device");
129 return failure();
130 }
131
132 Block *configureBlock;
133 if (configureOp.getBody().empty()) {
134 configureBlock = rewriter.createBlock(&configureOp.getBody());
135 } else {
136 configureBlock = &configureOp.getBody().front();
137 }
138
139 rewriter.setInsertionPointToStart(configureBlock);
140 AIEX::NpuLoadPdiOp::create(
141 rewriter, configureOp.getLoc(),
142 FlatSymbolRefAttr::get(referencedDevice.getSymNameAttr()),
143 /*id=*/nullptr, /*size=*/nullptr, /*address=*/nullptr,
144 /*expand_mode=*/configureOp.getExpandModeAttr());
145
146 return success();
147 }
148};
149
150// Collects all external SSA values referenced by an operation (and its nested
151// operations).
152// 1. Collects SSA values from the operation's operands.
153// 2. Recursively walks through all operations in the operation's regions.
154// 3. For each nested operation, collects SSA values from its operands.
155// 4. Skips values that are already in argMap or defined within the operation.
156// 5. For memref.subview operations, traces to the root block argument
157static void
158collectReferencedSSAValues(Operation *op, const IRMapping &argMap,
159 llvm::SetVector<Value> &referencedValues) {
160
161 auto processValue = [&](Value operand) {
162 if (argMap.contains(operand)) {
163 return;
164 }
165
166 // If this is a subview, trace to the root block argument
167 if (auto traceResult = traceSubviewToBlockArgument(operand)) {
168 // Check if the root argument is already mapped
169 if (!argMap.contains(traceResult->rootArg)) {
170 referencedValues.insert(traceResult->rootArg);
171 }
172 return;
173 }
174
175 // Not a subview chain leading to block arg, add as-is
176 referencedValues.insert(operand);
177 };
178
179 // Collect SSA values from the operation's direct operands.
180 for (Value operand : op->getOperands()) {
181 processValue(operand);
182 }
183
184 // Recursively collect SSA values from nested operations in all regions.
185 for (Region &region : op->getRegions()) {
186 region.walk([&](Operation *nestedOp) {
187 for (Value operand : nestedOp->getOperands()) {
188 if (argMap.contains(operand)) {
189 continue;
190 }
191
192 // Check if defined within the parent operation.
193 if (Operation *defOp = operand.getDefiningOp()) {
194 if (op->isProperAncestor(defOp)) {
195 continue;
196 }
197 } else if (auto blockArg = llvm::dyn_cast<BlockArgument>(operand)) {
198 // A block argument has no defining op, so the check above cannot see
199 // it. One belonging to a region nested inside `op` -- an scf.for
200 // induction variable, most commonly -- is nonetheless defined within
201 // `op` and must not be collected as an external reference: it would
202 // reach copyReferencedSSAValues, whose getDefiningOp() is null, and
203 // fail with "Referenced value is not defined by an operation".
204 Operation *owner = blockArg.getOwner()->getParentOp();
205 if (owner && (owner == op || op->isProperAncestor(owner))) {
206 continue;
207 }
208 }
209
210 processValue(operand);
211 }
212 });
213 }
214}
215
216// Returns `base`, or `base_<n>` if the caller device already holds `base`, and
217// records the returned name in `allSymbolNames`.
218static std::string
219uniqueSymbolName(MLIRContext *ctx, llvm::StringRef base,
220 llvm::SetVector<SymbolRefAttr> &allSymbolNames) {
221 std::string uniqueName = base.str();
222 unsigned uniquingCounter = 0;
223 while (allSymbolNames.count(SymbolRefAttr::get(ctx, uniqueName))) {
224 uniqueName = base.str() + "_" + std::to_string(uniquingCounter);
225 uniquingCounter++;
226 }
227 allSymbolNames.insert(SymbolRefAttr::get(ctx, uniqueName));
228 return uniqueName;
229}
230
231// Return the operation in the caller device that stands for `op`, cloning `op`
232// on first use. `clonedDefs` spans the aiex.run calls of one caller device, so
233// several calls that name one definition share one clone and one symbol.
234static Operation *
235getOrClone(PatternRewriter &rewriter, Operation *op, IRMapping &argMap,
236 llvm::DenseMap<Operation *, Operation *> &clonedDefs,
237 mlir::OpBuilder::InsertPoint &insertPoint,
238 llvm::SetVector<SymbolRefAttr> &allSymbolNames) {
239 auto it = clonedDefs.find(op);
240 if (it == clonedDefs.end()) {
241 rewriter.restoreInsertionPoint(insertPoint);
242 Operation *clone = rewriter.clone(*op, argMap);
243 if (auto name = clone->getAttrOfType<StringAttr>(
244 SymbolTable::getSymbolAttrName())) {
245 clone->setAttr(SymbolTable::getSymbolAttrName(),
246 rewriter.getStringAttr(uniqueSymbolName(
247 op->getContext(), name.getValue(), allSymbolNames)));
248 }
249 it = clonedDefs.try_emplace(op, clone).first;
250 insertPoint = rewriter.saveInsertionPoint();
251 }
252 argMap.map(op->getResults(), it->second->getResults());
253 return it->second;
254}
255
256// Copies SSA value definitions into the caller device.
257// Supports `aie.tile`, `aie.lock` and `aie.buffer` operations.
258// Updates argMap to map old values to new/existing values.
259static LogicalResult copyReferencedSSAValues(
260 PatternRewriter &rewriter, const llvm::SetVector<Value> &referencedValues,
261 AIE::DeviceOp callerDevice, IRMapping &argMap,
262 llvm::DenseMap<Operation *, Operation *> &clonedDefs,
263 mlir::OpBuilder::InsertPoint &clonedSSAInsertPoint,
264 llvm::SetVector<SymbolRefAttr> &allSymbolNames, Operation *errorReportOp) {
265
266 llvm::SetVector<Value> referencedValuesToVisit = referencedValues;
267 std::vector<Operation *> referencedOpsToClone = {};
268 while (!referencedValuesToVisit.empty()) {
269 Value referencedValue = referencedValuesToVisit.pop_back_val();
270 Operation *definingOp = referencedValue.getDefiningOp();
271 if (!definingOp) {
272 return errorReportOp->emitError()
273 << "Referenced value is not defined by an operation";
274 }
275 if (llvm::find(referencedOpsToClone, definingOp) !=
276 referencedOpsToClone.end()) {
277 continue;
278 }
279
280 if (auto tileOp = llvm::dyn_cast<AIE::TileOp>(definingOp)) {
281 referencedOpsToClone.insert(referencedOpsToClone.begin(), definingOp);
282 } else if (auto lockOp = llvm::dyn_cast<AIE::LockOp>(definingOp)) {
283 Value lockTile = lockOp.getTile();
284 if (lockTile) {
285 referencedValuesToVisit.insert(lockTile);
286 }
287 referencedOpsToClone.push_back(definingOp);
288 } else if (auto bufferOp = llvm::dyn_cast<AIE::BufferOp>(definingOp)) {
289 referencedValuesToVisit.insert(bufferOp.getTile());
290 referencedOpsToClone.push_back(definingOp);
291 } else {
292 return errorReportOp->emitError()
293 << "Referenced SSA value defined by unsupported operation type: "
294 << definingOp->getName().getStringRef()
295 << ". Currently only aie.tile, aie.lock and aie.buffer "
296 "operations are supported.";
297 }
298 }
299
300 for (Operation *definingOp : referencedOpsToClone) {
301 if (auto tileOp = llvm::dyn_cast<AIE::TileOp>(definingOp)) {
302 int col = tileOp.getCol();
303 int row = tileOp.getRow();
304
305 // A tile is its coordinates, so a tile the caller already declares stands
306 // for the callee's tile.
307 AIE::TileOp existingTile = nullptr;
308 for (AIE::TileOp tile : callerDevice.getOps<AIE::TileOp>()) {
309 if (tile.getCol() == col && tile.getRow() == row) {
310 existingTile = tile;
311 break;
312 }
313 }
314
315 if (existingTile) {
316 // Verify that all attributes match
317 if (tileOp->getAttrDictionary() != existingTile->getAttrDictionary()) {
318 // Filter out result type attributes and symbol attributes for
319 // comparison
320 auto filterAttrs = [](DictionaryAttr dict) -> DictionaryAttr {
321 SmallVector<NamedAttribute> filteredAttrs;
322 for (auto namedAttr : dict) {
323 StringRef name = namedAttr.getName().getValue();
324 if (name != "col" && name != "row" &&
325 name != "aiex.reserved_bd_ids") {
326 filteredAttrs.push_back(namedAttr);
327 }
328 }
329 return DictionaryAttr::get(dict.getContext(), filteredAttrs);
330 };
331
332 DictionaryAttr tileAttrs = filterAttrs(tileOp->getAttrDictionary());
333 DictionaryAttr existingAttrs =
334 filterAttrs(existingTile->getAttrDictionary());
335
336 if (tileAttrs != existingAttrs) {
337 return errorReportOp->emitError()
338 << "aie.tile(" << col << ", " << row
339 << ") already exists in the device with different "
340 "attributes";
341 }
342 }
343 // The callee's sequence ops move here, so the ids they pin do too.
344 if (auto calleeIds = tileOp->getAttrOfType<DenseI32ArrayAttr>(
345 "aiex.reserved_bd_ids")) {
346 std::set<int32_t> ids(calleeIds.asArrayRef().begin(),
347 calleeIds.asArrayRef().end());
348 if (auto callerIds = existingTile->getAttrOfType<DenseI32ArrayAttr>(
349 "aiex.reserved_bd_ids"))
350 ids.insert(callerIds.asArrayRef().begin(),
351 callerIds.asArrayRef().end());
352 SmallVector<int32_t> sorted(ids.begin(), ids.end());
353 existingTile->setAttr(
354 "aiex.reserved_bd_ids",
355 DenseI32ArrayAttr::get(existingTile->getContext(), sorted));
356 }
357 clonedDefs[definingOp] = existingTile.getOperation();
358 }
359 }
360
361 Operation *clonedOp = getOrClone(rewriter, definingOp, argMap, clonedDefs,
362 clonedSSAInsertPoint, allSymbolNames);
363 rewriter.replaceOpUsesWithIf(
364 definingOp, clonedOp->getResult(0), [&](OpOperand &operand) {
365 return operand.getOwner()->getParentOfType<AIE::DeviceOp>() ==
366 callerDevice;
367 });
368 }
369
370 return success();
371}
372
373// Inlines the definitions of all symbols referenced in the given operation
374// at the current insertion point in the given rewriter, unless the symbol
375// definition is in the "previouslyInlinedSymbolMap" map. While inlining,
376// symbols will be renamed to have a unique name.
377// Also copies in SSA values referenced by the inlined symbol definitions.
378static LogicalResult inlineReferencedSymbolDefinitions(
379 PatternRewriter &rewriter, Operation *op, Operation *lookupFrom,
380 IRMapping &argMap,
381 llvm::DenseMap<SymbolRefAttr, SymbolRefAttr> &previouslyInlinedSymbolMap,
382 AIE::DeviceOp callerDevice,
383 llvm::DenseMap<Operation *, Operation *> &clonedDefs,
384 AIE::NamedOpTableCollection &calleeNames,
385 mlir::OpBuilder::InsertPoint &clonedDefOpsInsertionPoint,
386 llvm::SetVector<SymbolRefAttr> &allSymbolNames) {
387 for (NamedAttribute namedAttr : op->getAttrs()) {
388 Attribute attr = namedAttr.getValue();
389 auto newAttr = attr.replace([&](SymbolRefAttr oldSymbolRef) {
390 auto inlined = previouslyInlinedSymbolMap.find(oldSymbolRef);
391 if (inlined != previouslyInlinedSymbolMap.end())
392 return std::make_pair(inlined->second, WalkResult::advance());
393
394 // Add the new symbol definition
395 // First try to look up from the lookupFrom operation (e.g., within the
396 // callee device). If not found, try looking up from the module level
397 // (for cross-device references).
398 // The caller device gains clones as calls are inlined; a callee device
399 // keeps its names while the caller's calls are inlined, so it is indexed
400 // once.
401 Operation *symbolDefOp = nullptr;
402 Operation *calleeTable = SymbolTable::getNearestSymbolTable(lookupFrom);
403 if (calleeTable && calleeTable != callerDevice &&
404 oldSymbolRef.getNestedReferences().empty()) {
405 symbolDefOp = calleeNames.getTable(calleeTable)
406 .lookup(oldSymbolRef.getRootReference());
407 } else {
408 symbolDefOp =
409 SymbolTable::lookupNearestSymbolFrom(lookupFrom, oldSymbolRef);
410 if (!symbolDefOp && oldSymbolRef.getNestedReferences().empty()) {
411 symbolDefOp =
412 AIE::lookupNamedOp(lookupFrom, oldSymbolRef.getRootReference());
413 }
414 }
415 if (!symbolDefOp) {
416 if (ModuleOp moduleOp = lookupFrom->getParentOfType<ModuleOp>()) {
417 symbolDefOp = SymbolTable::lookupSymbolIn(moduleOp, oldSymbolRef);
418 }
419 }
420 if (!symbolDefOp) {
421 return std::make_pair(oldSymbolRef, WalkResult::interrupt());
422 }
423
424 // If the symbol is a device, don't clone it - keep the original
425 // reference. Device ops must stay at module level.
426 if (llvm::isa<AIE::DeviceOp>(symbolDefOp)) {
427 previouslyInlinedSymbolMap[oldSymbolRef] = oldSymbolRef;
428 return std::make_pair(oldSymbolRef, WalkResult::advance());
429 }
430
431 // Collect SSA values referenced by the symbol definition operation
432 llvm::SetVector<Value> symbolReferencedValues;
433 collectReferencedSSAValues(symbolDefOp, argMap, symbolReferencedValues);
434
435 // Copy SSA values referenced by the symbol definition
436 // This updates clonedDefOpsInsertionPoint to be after the copied SSA
437 // values
438 if (failed(copyReferencedSSAValues(
439 rewriter, symbolReferencedValues, callerDevice, argMap,
440 clonedDefs, clonedDefOpsInsertionPoint, allSymbolNames, op))) {
441 return std::make_pair(oldSymbolRef, WalkResult::interrupt());
442 }
443
444 // Insert the cloned symbol at the device level, after its SSA
445 // dependencies. A definition also named by SSA shares that clone.
446 bool cloned = !clonedDefs.contains(symbolDefOp);
447 Operation *clonedSymbolDefOp =
448 getOrClone(rewriter, symbolDefOp, argMap, clonedDefs,
449 clonedDefOpsInsertionPoint, allSymbolNames);
450 SymbolRefAttr newSymbolRef =
451 FlatSymbolRefAttr::get(clonedSymbolDefOp->getAttrOfType<StringAttr>(
452 SymbolTable::getSymbolAttrName()));
453 previouslyInlinedSymbolMap[oldSymbolRef] = newSymbolRef;
454 // A definition may itself name others, as a shim allocation's
455 // consumed_by names its outputs, and the clone must name their clones.
456 if (cloned &&
457 failed(inlineReferencedSymbolDefinitions(
458 rewriter, clonedSymbolDefOp, symbolDefOp, argMap,
459 previouslyInlinedSymbolMap, callerDevice, clonedDefs, calleeNames,
460 clonedDefOpsInsertionPoint, allSymbolNames)))
461 return std::make_pair(oldSymbolRef, WalkResult::interrupt());
462 // A copied consumption proof describes the device it came from.
463 auto allocation = dyn_cast<AIE::ShimDMAAllocationOp>(clonedSymbolDefOp);
464 ArrayAttr proofs = allocation ? allocation.getConsumedByAttr() : nullptr;
465 auto callee = symbolDefOp->getParentOfType<AIE::DeviceOp>();
466 if (cloned && proofs && callee && callee.getSymNameAttr()) {
467 SmallVector<Attribute> stamped;
468 for (auto proof : proofs.getAsRange<AIE::ConsumedByAttr>())
469 stamped.push_back(AIE::ConsumedByAttr::get(
470 proof.getContext(), proof.getOutput(), proof.getInputBytes(),
471 proof.getOutputBytes(), proof.getGranuleBytes(),
472 proof.getDevice()
473 ? proof.getDevice()
474 : FlatSymbolRefAttr::get(callee.getSymNameAttr())));
475 allocation.setConsumedByAttr(rewriter.getArrayAttr(stamped));
476 }
477 return std::make_pair(newSymbolRef, WalkResult::advance());
478 });
479 if (!newAttr) {
480 return failure();
481 }
482 op->setAttr(namedAttr.getName(), newAttr);
483 }
484 return success();
485}
486
487struct InlineRuntimeCallsPattern : RewritePattern {
488
489 mlir::OpBuilder::InsertPoint &ssaDefInsertPoint;
490 mlir::OpBuilder::InsertPoint &symbolDefInsertPoint;
491 llvm::SetVector<SymbolRefAttr> &allSymbolNames;
492 llvm::DenseMap<Operation *, Operation *> &clonedDefs;
494
496 MLIRContext *ctx, mlir::OpBuilder::InsertPoint &ssaDefInsertPoint,
497 mlir::OpBuilder::InsertPoint &symbolDefInsertPoint,
498 llvm::SetVector<SymbolRefAttr> &allSymbolNames,
499 llvm::DenseMap<Operation *, Operation *> &clonedDefs,
501 : RewritePattern(RunOp::getOperationName(), PatternBenefit(1), ctx),
506
507 LogicalResult matchAndRewrite(Operation *op,
508 PatternRewriter &rewriter) const override {
509 llvm::DenseMap<SymbolRefAttr, SymbolRefAttr> previouslyInlinedSymbolMap;
510
511 RunOp runOp = llvm::dyn_cast<RunOp>(op);
512 if (!runOp) {
513 return failure();
514 }
515
516 AIE::DeviceOp calleeDevice = runOp.getCalleeDeviceOp();
517 AIE::RuntimeSequenceOp calleeRuntimeSequence =
518 runOp.getCalleeRuntimeSequenceOp(&calleeNames);
519 if (!calleeDevice || !calleeRuntimeSequence) {
520 return failure();
521 }
522
523 // rewrite logic
524
525 // Get caller and callee bodies. The callee body will be inlined into the
526 // caller body at the point of the RunOp.
527 Region &calleeBody = calleeRuntimeSequence.getBody();
528 AIE::DeviceOp callerDevice =
529 runOp.getOperation()->getParentOfType<AIE::DeviceOp>();
530 if (!callerDevice) {
531 runOp.emitError() << "needs to be in a DeviceOp";
532 return failure();
533 }
534
535 // The argMap maps callee arguments to caller SSA values.
536 IRMapping argMap;
537 ValueRange values = runOp.getArgs();
538 for (unsigned i = 0, n = calleeBody.getNumArguments(); i < n; i++) {
539 BlockArgument arg = calleeBody.getArgument(i);
540 Value val = values[i];
541 argMap.map(arg, val);
542 }
543
544 // The callee body may reference SSA values and symbols that are defined
545 // in the callee device (outside the callee runtime sequence). We will
546 // inline a supported set of these and error otherwise.
547
548 // Collect SSA values referenced in the callee not defined by the callee and
549 // not in the argMap.
550 llvm::SetVector<Value> referencedValues;
551 for (Operation &op : calleeBody.getOps()) {
552 collectReferencedSSAValues(&op, argMap, referencedValues);
553 }
554 llvm::SetVector<Value> filteredValues;
555 for (Value val : referencedValues) {
556 if (val.getParentRegion() != &calleeBody) {
557 filteredValues.insert(val);
558 }
559 }
560 referencedValues = std::move(filteredValues);
561
562 // Copy the operations that define these SSA values into the caller device
563 if (failed(copyReferencedSSAValues(rewriter, referencedValues, callerDevice,
565 allSymbolNames, runOp))) {
566 return failure();
567 }
568
569 // Find the calling runtime sequence so we can hoist certain ops to its
570 // start instead of placing them at the call site.
571 AIE::RuntimeSequenceOp callerRuntimeSequence =
572 runOp.getOperation()->getParentOfType<AIE::RuntimeSequenceOp>();
573 if (!callerRuntimeSequence) {
574 runOp.emitError() << "needs to be (transitively) inside a "
575 "aie.runtime_sequence operation";
576 return failure();
577 }
578 Block &callerBodyBlock = callerRuntimeSequence.getBody().front();
579 mlir::Block::iterator hoistPos = callerBodyBlock.begin();
580
581 // Now, also inline symbol definitions referenced in the callee body;
582 // this may pull in additional SSA values referenced by the symbol
583 // definitions.
584 //
585 // npu.create_scratchpad ops are hoisted to the start of the calling
586 // runtime sequence instead of being placed at the call site, so a single
587 // scratchpad is created per sequence regardless of how many callees were
588 // inlined.
589 rewriter.setInsertionPoint(runOp);
590 mlir::OpBuilder::InsertPoint clonedOpInsertionPoint =
591 rewriter.saveInsertionPoint();
592 for (Operation &op : calleeBody.getOps()) {
593 bool shouldHoist = llvm::isa<NpuCreateScratchpadOp>(&op);
594 if (shouldHoist) {
595 rewriter.setInsertionPoint(&callerBodyBlock, hoistPos);
596 rewriter.clone(op, argMap);
597 continue;
598 }
599
600 rewriter.restoreInsertionPoint(clonedOpInsertionPoint);
601 Operation *clonedOp = rewriter.clone(op, argMap);
602 clonedOpInsertionPoint = rewriter.saveInsertionPoint();
603
604 // Inline symbol references in all nested ops.
605 WalkResult symbolWalk = clonedOp->walk([&](Operation *nestedOp) {
606 if (failed(inlineReferencedSymbolDefinitions(
607 rewriter, nestedOp, calleeRuntimeSequence.getOperation(),
608 argMap, previouslyInlinedSymbolMap, callerDevice, clonedDefs,
610 return WalkResult::interrupt();
611 }
612 return WalkResult::advance();
613 });
614 if (symbolWalk.wasInterrupted()) {
615 return failure();
616 }
617 }
618
619 // The aiex.run op has been inlined; erase it.
620 rewriter.eraseOp(runOp);
621
622 return success();
623 }
624};
625
626/// Validate all aiex.run ops inside a aiex.configure op against the referenced
627/// device. This must be called sequentially (before any parallel pass
628/// execution) because it performs cross-DeviceOp symbol table lookups.
629/// Placing this validation in RunOp::verify() is unsafe: MLIR's pass manager
630/// may invoke op verifiers concurrently on sibling DeviceOps, causing a data
631/// race on the referenced device's symbol table.
632static LogicalResult
633verifyRunOpsInConfigureOp(ConfigureOp configureOp, AIE::DeviceOp referencedDev,
634 AIE::NamedOpTableCollection &referencedDevNames) {
635 if (configureOp.getBody().empty())
636 return success();
637 AIE::NamedOpTable &devNames = referencedDevNames.getTable(referencedDev);
638 for (RunOp runOp : configureOp.getBody().front().getOps<RunOp>()) {
639 auto seqName = runOp.getRuntimeSequenceSymbol();
640 Operation *maybeSeq =
641 devNames.lookup(runOp.getRuntimeSequenceSymbolAttr().getAttr());
642 if (!maybeSeq) {
643 auto err = runOp.emitError()
644 << "No such runtime sequence for device '"
645 << referencedDev.getSymName() << "': '" << seqName << "'";
646 err.attachNote(referencedDev.getLoc())
647 << "This device does not have a '" << seqName << "' runtime sequence";
648 return failure();
649 }
650 auto runtimeSeq = llvm::dyn_cast<AIE::RuntimeSequenceOp>(maybeSeq);
651 if (!runtimeSeq) {
652 return runOp.emitError()
653 << "'" << seqName << "' is not a runtime sequence";
654 }
655
656 // Validate argument count and types against the callee signature.
657 // An empty body region (no blocks) means the sequence takes no arguments.
658 Region &calleeRegion = runtimeSeq.getBody();
659 unsigned numCalleeArgs =
660 calleeRegion.empty() ? 0 : calleeRegion.getNumArguments();
661 ValueRange args = runOp.getArgs();
662 if (numCalleeArgs != args.size())
663 return runOp.emitOpError() << "argument count mismatch: expected "
664 << numCalleeArgs << " but got " << args.size();
665 for (unsigned i = 0; i < numCalleeArgs; i++) {
666 Type expected = calleeRegion.front().getArgument(i).getType();
667 Type actual = args[i].getType();
668 if (expected != actual)
669 return runOp.emitOpError()
670 << "argument " << i << " type mismatch: expected " << expected
671 << " but got " << actual;
672 }
673 }
674 return success();
675}
676
679 AIEMaterializeRuntimeSequencesPass> {
680 using Base::Base;
681
682 // After inlining, a runtime sequence may contain multiple
683 // npu.create_scratchpad ops (one from the sequence itself and one from each
684 // inlined callee). Keep only the first; all must agree on size because the
685 // scratchpad size is a module-level property. Returns failure if a size
686 // mismatch is detected.
687 LogicalResult
688 deduplicateCreateScratchpadOps(AIE::RuntimeSequenceOp runtimeSeqOp) {
689 if (runtimeSeqOp.getBody().empty())
690 return success();
691
692 Block &body = runtimeSeqOp.getBody().front();
693 NpuCreateScratchpadOp firstScratchpad;
694 SmallVector<NpuCreateScratchpadOp> duplicates;
695 for (NpuCreateScratchpadOp scratchpadOp :
696 body.getOps<NpuCreateScratchpadOp>()) {
697 if (!firstScratchpad) {
698 firstScratchpad = scratchpadOp;
699 continue;
700 }
701 if (firstScratchpad.getSize() != scratchpadOp.getSize()) {
702 scratchpadOp.emitError(
703 "create_scratchpad size mismatch after inlining: ")
704 << scratchpadOp.getSize() << " != " << firstScratchpad.getSize();
705 return failure();
706 }
707 duplicates.push_back(scratchpadOp);
708 }
709 for (NpuCreateScratchpadOp dup : duplicates)
710 dup.erase();
711
712 return success();
713 }
714
715 void runOnOperation() override {
716 ModuleOp moduleOp = getOperation();
717
718 AIE::NamedOpTableCollection referencedDevNames;
719
720 // Process each device in the module
721 for (AIE::DeviceOp deviceOp : moduleOp.getOps<AIE::DeviceOp>()) {
722
723 // Verify all runtime sequences before materialization
724 for (AIE::RuntimeSequenceOp runtimeSequenceOp :
725 deviceOp.getOps<AIE::RuntimeSequenceOp>()) {
726 if (failed(runtimeSequenceOp.verifyBeforeMaterialization())) {
727 return signalPassFailure();
728 }
729
730 // Validate aiex.run ops inside aiex.configure ops. This cross-device
731 // check is performed here (sequentially) rather than in RunOp::verify()
732 // to avoid a race condition: MLIR's pass manager runs verifiers on
733 // sibling DeviceOps concurrently, and looking up symbols in a sibling
734 // DeviceOp from a verifier causes a data race on its symbol table.
735 for (ConfigureOp configureOp :
736 runtimeSequenceOp.getOps<ConfigureOp>()) {
737 AIE::DeviceOp referencedDev = configureOp.getReferencedDeviceOp();
738 if (!referencedDev) {
739 // ConfigureOp::verify() already reported the error (no such
740 // device, not a device, or device type mismatch) at parse time;
741 // this is a safety net if verification was disabled.
742 return signalPassFailure();
743 }
744 if (failed(verifyRunOpsInConfigureOp(configureOp, referencedDev,
745 referencedDevNames)))
746 return signalPassFailure();
747 }
748 }
749
750 // Check for cycles in runtime sequence calls
751 for (AIE::RuntimeSequenceOp runtimeSequenceOp :
752 deviceOp.getOps<AIE::RuntimeSequenceOp>()) {
753 AnalysisManager am =
754 getAnalysisManager().nest(deviceOp).nest(runtimeSequenceOp);
756 am.getAnalysis<RuntimeCallGraphCyclicityAnalysis>();
757 if (!cyclicity.isValid) {
758 return signalPassFailure();
759 }
760 if (cyclicity.isCyclic) {
761 runtimeSequenceOp.emitError(
762 "Runtime sequence call graph contains a cycle");
763 return signalPassFailure();
764 }
765 }
766
767 // Greedily inline all runtime sequences that can be inlined;
768 // this will start with runtime sequences that do not call other runtime
769 // sequences (leaves); once their callers inline them, the callers can
770 // be inlined as well, and so on
771 mlir::Block &deviceBodyFirstBlock = deviceOp.getBodyRegion().front();
772 auto runtimeSequenceOps = deviceOp.getOps<AIE::RuntimeSequenceOp>();
773 if (runtimeSequenceOps.begin() == runtimeSequenceOps.end()) {
774 // No runtime sequences to materialize
775 continue;
776 }
777 AIE::RuntimeSequenceOp firstRuntimeSequenceOp =
778 *runtimeSequenceOps.begin();
779 mlir::OpBuilder::InsertPoint ssaDefInsertPoint(
780 &deviceBodyFirstBlock, deviceBodyFirstBlock.begin());
781 mlir::OpBuilder::InsertPoint symbolDefInsertPoint(
782 &deviceBodyFirstBlock, mlir::Block::iterator(firstRuntimeSequenceOp));
783 llvm::SetVector<SymbolRefAttr> allSymbolNames = {};
784 for (Operation &op : deviceBodyFirstBlock) {
785 if (auto symbolName = op.getAttrOfType<StringAttr>(
786 SymbolTable::getSymbolAttrName())) {
787 allSymbolNames.insert(SymbolRefAttr::get(symbolName));
788 }
789 }
790
791 MLIRContext *ctx = &getContext();
792 GreedyRewriteConfig rewriter_config = GreedyRewriteConfig();
793 rewriter_config.setRegionSimplificationLevel(
794 GreedySimplifyRegionLevel::Disabled);
795
796 RewritePatternSet patterns_0(ctx);
797 llvm::DenseMap<Operation *, Operation *> clonedDefs;
798 AIE::NamedOpTableCollection calleeNames;
799 patterns_0.insert<InlineRuntimeCallsPattern>(
800 ctx, ssaDefInsertPoint, symbolDefInsertPoint, allSymbolNames,
801 clonedDefs, calleeNames);
802 if (failed(applyPatternsGreedily(deviceOp, std::move(patterns_0),
803 rewriter_config))) {
804 return signalPassFailure();
805 }
806
807 // Deduplicate create_scratchpad ops that were hoisted during inlining.
808 for (AIE::RuntimeSequenceOp runtimeSeqOp :
809 deviceOp.getOps<AIE::RuntimeSequenceOp>()) {
810 if (failed(deduplicateCreateScratchpadOps(runtimeSeqOp)))
811 return signalPassFailure();
812 }
813
814 // Insert LoadPDI ops for each aiex.configure op
815 RewritePatternSet patterns_1(ctx);
816 patterns_1.insert<InsertLoadPdiForConfigurePattern>(ctx);
817 walkAndApplyPatterns(deviceOp, std::move(patterns_1));
818
819 // Canonicalize to remove duplicate back-to-back load_pdi ops, visiting
820 // only the load_pdi ops rather than every op of the sequences.
821 RewritePatternSet canonicalize_patterns(ctx);
822 AIEX::NpuLoadPdiOp::getCanonicalizationPatterns(canonicalize_patterns,
823 ctx);
824 SmallVector<Operation *> loadPdis;
825 deviceOp.walk([&](NpuLoadPdiOp op) { loadPdis.push_back(op); });
826 GreedyRewriteConfig loadPdiConfig = rewriter_config;
827 loadPdiConfig.setStrictness(GreedyRewriteStrictness::ExistingOps);
828 if (failed(applyOpPatternsGreedily(
829 loadPdis, std::move(canonicalize_patterns), loadPdiConfig))) {
830 return signalPassFailure();
831 }
832
833 // Flatten the IR: hoist all operations inside aiex.configure to be direct
834 // children of the runtime sequence, preserving order
835 for (AIE::RuntimeSequenceOp runtimeSequenceOp :
836 deviceOp.getOps<AIE::RuntimeSequenceOp>()) {
837 SmallVector<ConfigureOp> configureOps;
838
839 for (ConfigureOp configureOp :
840 runtimeSequenceOp.getOps<ConfigureOp>()) {
841 configureOps.push_back(configureOp);
842 }
843
844 IRRewriter rewriter(ctx);
845 for (ConfigureOp configureOp : configureOps) {
846 Block &configureBlock = configureOp.getBody().front();
847
848 // Collect all operations in the configure block
849 SmallVector<Operation *> opsToHoist;
850 for (Operation &op : configureBlock) {
851 opsToHoist.push_back(&op);
852 }
853
854 // Hoist operations to be right before the configure op
855 rewriter.setInsertionPoint(configureOp);
856 for (Operation *op : opsToHoist) {
857 op->moveBefore(configureOp);
858 }
859
860 // Erase the now-empty configure op
861 rewriter.eraseOp(configureOp);
862 }
863 }
864
865 } // end for each device
866 }
867};
868
869std::unique_ptr<OperationPass<ModuleOp>>
871 return std::make_unique<AIEMaterializeRuntimeSequencesPass>();
872}
NamedOpTable & getTable(mlir::Operation *symbolTableOp)
Definition AIEDialect.h:175
mlir::Operation * lookup(mlir::StringAttr name) const
Definition AIEDialect.h:153
std::optional< SubviewTraceResult > traceSubviewToBlockArgument(Value value)
Definition AIEUtils.cpp:25
std::unique_ptr< mlir::OperationPass< mlir::ModuleOp > > createAIEMaterializeRuntimeSequencesPass()
mlir::Operation * lookupNamedOp(mlir::Operation *from, mlir::StringAttr name)
LogicalResult deduplicateCreateScratchpadOps(AIE::RuntimeSequenceOp runtimeSeqOp)
LogicalResult matchAndRewrite(Operation *op, PatternRewriter &rewriter) const override
mlir::OpBuilder::InsertPoint & ssaDefInsertPoint
InlineRuntimeCallsPattern(MLIRContext *ctx, mlir::OpBuilder::InsertPoint &ssaDefInsertPoint, mlir::OpBuilder::InsertPoint &symbolDefInsertPoint, llvm::SetVector< SymbolRefAttr > &allSymbolNames, llvm::DenseMap< Operation *, Operation * > &clonedDefs, AIE::NamedOpTableCollection &calleeNames)
llvm::DenseMap< Operation *, Operation * > & clonedDefs
mlir::OpBuilder::InsertPoint & symbolDefInsertPoint
llvm::SetVector< SymbolRefAttr > & allSymbolNames
AIE::NamedOpTableCollection & calleeNames
InsertLoadPdiForConfigurePattern(MLIRContext *context, PatternBenefit benefit=1)
LogicalResult matchAndRewrite(Operation *op, PatternRewriter &rewriter) const override
RuntimeCallGraphCyclicityAnalysis(Operation *op, AnalysisManager &am)