MLIR-AIE 1.4.6.dev
AIENpuToCert.cpp
Go to the documentation of this file.
1//===- AIENpuToCert.cpp ----------------------------------------*- C++ -*-===//
2//
3// Copyright (C) 2025 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7
12
13#include "mlir/Dialect/Arith/IR/Arith.h"
14#include "mlir/Dialect/MemRef/IR/MemRef.h"
15#include "mlir/IR/IRMapping.h"
16#include "mlir/IR/Matchers.h"
17#include "mlir/IR/SymbolTable.h"
18#include "mlir/Pass/Pass.h"
19#include "mlir/Transforms/DialectConversion.h"
20#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
21#include "llvm/ADT/DenseSet.h"
22#include "llvm/ADT/StringMap.h"
23
24#include <bitset>
25#include <map>
26#include <set>
27#include <type_traits>
28#include <utility>
29#include <vector>
30
31namespace xilinx::AIEX {
32#define GEN_PASS_DEF_AIENPUTOCERT
33#define GEN_PASS_DEF_AIECERTPAGES
34#include "aie/Dialect/AIEX/Transforms/AIEXPasses.h.inc"
35} // namespace xilinx::AIEX
36
37using namespace mlir;
38using namespace xilinx;
39
40#define DEBUG_TYPE "npu-to-cert"
41
42namespace {
43
44// Trigger splitting below the 8 KB page limit to allow for estimated padding.
45// An unsplittable page is an error only if it exceeds cert_page_limit.
46static constexpr uint32_t cert_page_size = 8000;
47static constexpr uint32_t cert_page_limit = 8192;
48
49// Per-job control-code overhead: START_JOB (ISA_OPSIZE_START_JOB = 0x08) plus
50// END_JOB (ISA_OPSIZE_END_JOB = 0x04), emitted around every job's body by
51// emitJob.
52static constexpr uint32_t cert_job_start_cost = 8;
53static constexpr uint32_t cert_job_end_cost = 4;
54
55// True if `seq` holds its device's generated configuration, i.e. carries the
56// aie.generated_configure tag, either from convert-aie-to-transaction or from
57// tagConfigureByName.
58static bool isGeneratedConfigure(AIE::RuntimeSequenceOp seq) {
59 return AIE::AIEDialect::GeneratedConfigureAttrHelper(seq.getContext())
60 .isAttrPresent(seq);
61}
62
63// Hand-written IR may have no tag: there the sequence named `configure` is the
64// configuration. Tag it, unless a sequence of the device is already tagged (the
65// generated one was then named `configure0`, and `configure` is the user's).
66// Runs on every device before any sequence is converted, so the choice does
67// not depend on conversion order.
68static void tagConfigureByName(AIE::DeviceOp device) {
69 auto seqs = device.getBody()->getOps<AIE::RuntimeSequenceOp>();
70 if (llvm::any_of(seqs, isGeneratedConfigure))
71 return;
72 for (auto seq : seqs)
73 if (seq.getSymName() == "configure") {
74 AIE::AIEDialect::GeneratedConfigureAttrHelper(device.getContext())
75 .setAttr(seq, UnitAttr::get(device.getContext()));
76 return;
77 }
78}
79
80// The generated configuration sequence of `device`, or null if it has none.
81static AIE::RuntimeSequenceOp findGeneratedConfigure(AIE::DeviceOp device) {
82 for (auto seq : device.getBody()->getOps<AIE::RuntimeSequenceOp>())
83 if (isGeneratedConfigure(seq))
84 return seq;
85 return nullptr;
86}
87
88// Returns the PDI id to use for a cert.load_pdi that references `deviceSymName`
89// within `parentDevice`. If a load_pdi for that symbol already exists, its id
90// is reused so repeated loads of the same PDI share an id; otherwise a fresh id
91// (1-based, one past the current maximum) is assigned.
92static uint32_t getOrAssignPdiId(AIE::DeviceOp parentDevice,
93 StringRef deviceSymName) {
94 uint32_t pdiId = 1;
95 bool foundExisting = false;
96 parentDevice.walk([&](AIEX::CertLoadPdiOp loadPdiOp) {
97 if (loadPdiOp.getSymbol() == deviceSymName) {
98 pdiId = loadPdiOp.getPdiId();
99 foundExisting = true;
100 } else if (!foundExisting) {
101 pdiId = std::max(pdiId, loadPdiOp.getPdiId() + 1);
102 }
103 });
104 return pdiId;
105}
106
107struct RuntimeSequenceToCertJob : OpConversionPattern<AIE::RuntimeSequenceOp> {
108 using OpConversionPattern::OpConversionPattern;
109
110 RuntimeSequenceToCertJob(MLIRContext *context, PatternBenefit benefit = 1)
111 : OpConversionPattern(context, benefit) {}
112
113 LogicalResult
114 matchAndRewrite(AIE::RuntimeSequenceOp op, OpAdaptor adaptor,
115 ConversionPatternRewriter &rewriter) const override {
116
117 bool isConfigure = isGeneratedConfigure(op);
118 uint32_t newJobId = 1;
119 if (!isConfigure) {
120 uint32_t maxJobId = 1;
121 op->getParentOp()->walk([&](AIEX::CertJobOp certJobOp) {
122 maxJobId = std::max(maxJobId, certJobOp.getJobId());
123 });
124 newJobId = maxJobId + 1;
125 }
126
127 // Replace the runtime sequence with a job directly inside aie.device.
128 // CertJobOp lists DeviceOp as an allowed parent in its ParentOneOf trait.
129 // cert-legalize-pages later groups contiguous jobs into implicit pages.
130 rewriter.setInsertionPoint(op);
131 auto jobOp = AIEX::CertJobOp::create(rewriter, op.getLoc(), newJobId);
132
133 // Tag the implicit configuration job so a later step can force it to be the
134 // first page on uC0 and keep it out of user implicit pages.
135 if (isConfigure)
136 jobOp->setAttr("cert.configure", rewriter.getUnitAttr());
137
138 // Clone runtime sequence body into job
139 // Note: This preserves block arguments from the runtime sequence, which
140 // will be present in the MLIR IR but are not emitted in the final assembly
141 IRMapping remap;
142 op.getRegion().cloneInto(&jobOp.getBody(), remap);
143 AIEX::CertJobOp::ensureTerminator(jobOp.getBody(), rewriter, op->getLoc());
144
145 // Erase the original runtime sequence
146 rewriter.eraseOp(op);
147
148 return success();
149 }
150};
151
152struct NpuWrite32ToCertWrite32 : OpConversionPattern<AIEX::NpuWrite32Op> {
153 using OpConversionPattern::OpConversionPattern;
154
155 LogicalResult
156 matchAndRewrite(AIEX::NpuWrite32Op op, OpAdaptor adaptor,
157 ConversionPatternRewriter &rewriter) const override {
158
159 std::optional<uint32_t> address =
160 AIEX::getConstantIntOperand(op.getAddress());
161 std::optional<uint32_t> value = AIEX::getConstantIntOperand(op.getValue());
162 if (!address || !value)
163 return op.emitOpError(
164 "cannot lower to cert.write32 with non-constant address or value");
165
166 // Get the absolute address, which properly handles row/col if present
167 std::optional<uint32_t> absAddress = op.getAbsoluteAddress();
168 if (!absAddress)
169 return failure();
170
171 uint32_t absoluteAddr = *absAddress;
172
173 // If row and col are specified, validate that the address upper bits match
174 std::optional<uint32_t> col = op.getColumn();
175 std::optional<uint32_t> row = op.getRow();
176 if (col && row) {
177 const auto &tm = AIE::getTargetModel(op);
178 uint32_t expectedUpperBits = ((*col & 0xff) << tm.getColumnShift()) |
179 ((*row & 0xff) << tm.getRowShift());
180
181 // Warn if the original address had non-zero upper bits that don't match
182 uint32_t origAddress = *address;
183 uint32_t origUpperBits = origAddress & ~0xfffff;
184 if (origUpperBits != 0 && origUpperBits != expectedUpperBits) {
185 op.emitWarning() << "address upper bits (0x"
186 << llvm::utohexstr(origUpperBits)
187 << ") don't match row=" << *row << " col=" << *col
188 << " computed bits (0x"
189 << llvm::utohexstr(expectedUpperBits) << ")";
190 }
191 }
192
193 rewriter.replaceOpWithNewOp<AIEX::CertWrite32Op>(op, absoluteAddr, *value);
194 return success();
195 }
196};
197
198struct NpuMaskWrite32ToCertMaskWrite32
199 : OpConversionPattern<AIEX::NpuMaskWrite32Op> {
200 using OpConversionPattern::OpConversionPattern;
201
202 LogicalResult
203 matchAndRewrite(AIEX::NpuMaskWrite32Op op, OpAdaptor adaptor,
204 ConversionPatternRewriter &rewriter) const override {
205
206 std::optional<uint32_t> address =
207 AIEX::getConstantIntOperand(op.getAddress());
208 std::optional<uint32_t> mask = AIEX::getConstantIntOperand(op.getMask());
209 std::optional<uint32_t> value = AIEX::getConstantIntOperand(op.getValue());
210 if (!address || !mask || !value)
211 return op.emitOpError(
212 "cannot lower to cert.maskwrite32 with non-constant "
213 "address, mask, or value");
214
215 // Get the absolute address, which properly handles row/col if present
216 std::optional<uint32_t> absAddress = op.getAbsoluteAddress();
217 if (!absAddress)
218 return failure();
219
220 uint32_t absoluteAddr = *absAddress;
221
222 // If row and col are specified, validate that the address upper bits match
223 std::optional<uint32_t> col = op.getColumn();
224 std::optional<uint32_t> row = op.getRow();
225 if (col && row) {
226 const auto &tm = AIE::getTargetModel(op);
227 uint32_t expectedUpperBits = ((*col & 0xff) << tm.getColumnShift()) |
228 ((*row & 0xff) << tm.getRowShift());
229
230 // Warn if the original address had non-zero upper bits that don't match
231 uint32_t origAddress = *address;
232 uint32_t origUpperBits = origAddress & ~0xfffff;
233 if (origUpperBits != 0 && origUpperBits != expectedUpperBits) {
234 op.emitWarning() << "address upper bits (0x"
235 << llvm::utohexstr(origUpperBits)
236 << ") don't match row=" << *row << " col=" << *col
237 << " computed bits (0x"
238 << llvm::utohexstr(expectedUpperBits) << ")";
239 }
240 }
241
242 rewriter.replaceOpWithNewOp<AIEX::CertMaskWrite32Op>(op, absoluteAddr,
243 *mask, *value);
244 return success();
245 }
246};
247
248struct NpuMaskPollToCertMaskPoll32 : OpConversionPattern<AIEX::NpuMaskPollOp> {
249 using OpConversionPattern::OpConversionPattern;
250
251 LogicalResult
252 matchAndRewrite(AIEX::NpuMaskPollOp op, OpAdaptor adaptor,
253 ConversionPatternRewriter &rewriter) const override {
254 std::optional<uint32_t> address =
255 AIEX::getConstantIntOperand(op.getAddress());
256 std::optional<uint32_t> mask = AIEX::getConstantIntOperand(op.getMask());
257 std::optional<uint32_t> value = AIEX::getConstantIntOperand(op.getValue());
258 if (!address || !mask || !value)
259 return op.emitOpError("cannot lower to cert.maskpoll32 with non-constant "
260 "address, mask, or value");
261
262 std::optional<uint32_t> absAddress = op.getAbsoluteAddress();
263 if (!absAddress)
264 return failure();
265
266 rewriter.replaceOpWithNewOp<AIEX::CertMaskPoll32Op>(op, *absAddress, *mask,
267 *value);
268 return success();
269 }
270};
271
272struct NpuBlockWriteToCertUcDma : OpConversionPattern<AIEX::NpuBlockWriteOp> {
273 // nextChainId is owned by the caller and shared by every rewrite in one
274 // conversion, so each name search resumes after the last name allocated.
275 // symbolTable is the device's; each new chain is added to it.
276 NpuBlockWriteToCertUcDma(MLIRContext *ctx, unsigned &nextChainId,
277 SymbolTable &symbolTable)
278 : OpConversionPattern(ctx), nextChainId(nextChainId),
279 symbolTable(symbolTable) {}
280
281 LogicalResult
282 matchAndRewrite(AIEX::NpuBlockWriteOp op, OpAdaptor adaptor,
283 ConversionPatternRewriter &rewriter) const override {
284
285 memref::GetGlobalOp dataOperand =
286 dyn_cast_or_null<memref::GetGlobalOp>(op.getData().getDefiningOp());
287 if (!dataOperand)
288 return failure();
289 MemRefType dataType = cast<MemRefType>(dataOperand.getResult().getType());
290 uint32_t dataSize = dataType.getNumElements();
291 // An empty copy is a no-op, and AIEBU rejects a descriptor whose payload
292 // label has no data.
293 if (dataSize == 0) {
294 rewriter.eraseOp(op);
295 return success();
296 }
297
298 std::string symbolName = "chain_" + std::to_string(nextChainId++);
299 while (symbolTable.lookup(symbolName))
300 symbolName = "chain_" + std::to_string(nextChainId++);
301
302 // Create a new uc_dma_write_des_sync operation
303 rewriter.replaceOpWithNewOp<AIEX::CertUcDmaWriteDesSyncOp>(op, symbolName);
304
305 // Create the uc_dma_chain operation
306 // Find the nearest device to insert the chain
307 auto parentDevice = op->getParentOfType<AIE::DeviceOp>();
308 if (!parentDevice) {
309 // No parent device - this shouldn't happen but handle gracefully
310 return failure();
311 }
312
313 // Insert after the last existing uc_dma_chain (before any pages/jobs),
314 // preserving the order of blockwrite ops. Globals are allowed to precede
315 // the chains, so skip over them too. The previous chain ends that leading
316 // run, so resume the scan there instead of at the start of the body.
317 Block *deviceBody = parentDevice.getBody();
318 Operation *insertAfter = nullptr;
319 Block::iterator it = deviceBody->begin();
320 if (lastChain && lastChain->getBlock() == deviceBody) {
321 insertAfter = lastChain;
322 it = std::next(Block::iterator(lastChain));
323 }
324 for (; it != deviceBody->end() &&
325 isa<AIEX::CertUcDmaChainOp, memref::GlobalOp>(*it);
326 ++it)
327 insertAfter = &*it;
328 if (insertAfter)
329 rewriter.setInsertionPointAfter(insertAfter);
330 else
331 rewriter.setInsertionPointToStart(deviceBody);
332
333 auto symbolAttr = rewriter.getStringAttr(symbolName);
334 auto chainOp =
335 AIEX::CertUcDmaChainOp::create(rewriter, op.getLoc(), symbolAttr);
336 symbolTable.insert(chainOp);
337 lastChain = chainOp;
338
339 Block *bb = new Block();
340 chainOp.getRegion().push_back(bb);
341 rewriter.setInsertionPointToStart(bb);
342 AIEX::CertUcDmaBdOp::create(rewriter, op.getLoc(), dataOperand.getName(),
343 op.getAddress(), dataSize, false);
344
345 AIEX::CertUcDmaChainOp::ensureTerminator(chainOp.getBody(), rewriter,
346 op->getLoc());
347 return success();
348 }
349
350 unsigned &nextChainId;
351 SymbolTable &symbolTable;
352 mutable Operation *lastChain = nullptr;
353};
354
355struct NpuSyncToCertWaitTCTS : OpConversionPattern<AIEX::NpuSyncOp> {
356 using OpConversionPattern::OpConversionPattern;
357
358 LogicalResult
359 matchAndRewrite(AIEX::NpuSyncOp op, OpAdaptor adaptor,
360 ConversionPatternRewriter &rewriter) const override {
361 std::optional<uint32_t> rowOpt = AIEX::getConstantIntOperand(op.getRow());
362 std::optional<uint32_t> colOpt =
363 AIEX::getConstantIntOperand(op.getColumn());
364 std::optional<uint32_t> channelOpt =
365 AIEX::getConstantIntOperand(op.getChannel());
366 std::optional<uint32_t> directionOpt =
367 AIEX::getConstantIntOperand(op.getDirection());
368 if (!rowOpt || !colOpt || !channelOpt || !directionOpt)
369 return op.emitOpError(
370 "cannot lower to cert.wait_tcts with non-constant sync parameters");
371 uint32_t row = *rowOpt;
372 uint32_t col = *colOpt;
373
374 // These are the shift amounts from the tct packet format in the
375 // architecture spec. The firmware expects the row and column packed and
376 // shifted down to zero.
377 const int row_id_shift = 16;
378 const int col_id_shift = 21;
379 uint16_t tile_id = col << (col_id_shift - row_id_shift) | row;
380 uint32_t channel = *channelOpt;
381 uint32_t direction = *directionOpt;
382
383 const std::vector<int> chan2actor_shim_s2mm = {0, 2, 3, 4};
384 const std::vector<int> chan2actor_shim_mm2s = {6, 7, 8, 9, 10, 11, 12, 13};
385 const std::vector<int> chan2actor_mem_s2mm = {1, 2, 3, 4, 5, 6, 7};
386 const std::vector<int> chan2actor_mem_mm2s = {16, 17, 18, 19, 20,
387 22, 23, 24, 25, 26};
388 const std::vector<int> chan2actor_tile_s2mm = {0, 1};
389 const std::vector<int> chan2actor_tile_mm2s = {6};
390
391 const auto &tm = AIE::getTargetModel(op);
392 const bool isS2MM =
393 direction == static_cast<std::underlying_type_t<AIE::DMAChannelDir>>(
394 AIE::DMAChannelDir::S2MM);
395
396 const std::vector<int> *chan2actor = nullptr;
397 if (tm.isCoreTile(col, row))
398 chan2actor = isS2MM ? &chan2actor_tile_s2mm : &chan2actor_tile_mm2s;
399 else if (tm.isMemTile(col, row))
400 chan2actor = isS2MM ? &chan2actor_mem_s2mm : &chan2actor_mem_mm2s;
401 else
402 chan2actor = isS2MM ? &chan2actor_shim_s2mm : &chan2actor_shim_mm2s;
403
404 size_t chanIdx = static_cast<size_t>(channel);
405 if (chanIdx >= chan2actor->size()) {
406 op.emitError("invalid DMA channel ")
407 << channel << " for " << (isS2MM ? "S2MM" : "MM2S")
408 << " direction in NpuSyncToCertWaitTCTS conversion";
409 return failure();
410 }
411
412 uint8_t actor_id = static_cast<uint8_t>((*chan2actor)[chanIdx]);
413 uint8_t num_tcts = 1;
414 rewriter.replaceOpWithNewOp<AIEX::CertWaitTCTSOp>(op, tile_id, actor_id,
415 num_tcts);
416 return success();
417 }
418};
419
420struct NpuLoadPdiToCertLoadPdi : OpConversionPattern<AIEX::NpuLoadPdiOp> {
421 using OpConversionPattern::OpConversionPattern;
422
423 LogicalResult
424 matchAndRewrite(AIEX::NpuLoadPdiOp op, OpAdaptor adaptor,
425 ConversionPatternRewriter &rewriter) const override {
426 // Get the device reference
427 auto deviceRef = op.getDeviceRef();
428 if (!deviceRef)
429 return failure();
430
431 StringRef deviceSymName = *deviceRef;
432
433 // Find parent device to get unique PDI ID
434 auto parentDevice = op->getParentOfType<AIE::DeviceOp>();
435 if (!parentDevice)
436 return failure();
437
438 // Assign a PDI ID, reusing an existing one for this device if present.
439 uint32_t pdiId = getOrAssignPdiId(parentDevice, deviceSymName);
440
441 // Replace with cert.load_pdi
442 rewriter.replaceOpWithNewOp<AIEX::CertLoadPdiOp>(
443 op, rewriter.getUI32IntegerAttr(pdiId),
444 FlatSymbolRefAttr::get(rewriter.getContext(), deviceSymName));
445
446 return success();
447 }
448};
449
450struct RunOpInlining : OpRewritePattern<AIEX::RunOp> {
451 using OpRewritePattern::OpRewritePattern;
452
453 LogicalResult matchAndRewrite(AIEX::RunOp op,
454 PatternRewriter &rewriter) const override {
455 // Get the callee runtime sequence
456 AIE::RuntimeSequenceOp calleeRuntimeSequence =
457 op.getCalleeRuntimeSequenceOp();
458 if (!calleeRuntimeSequence)
459 return failure();
460
461 // Get the callee body region
462 Region &calleeBody = calleeRuntimeSequence.getBody();
463
464 // Create argument mapping from run op arguments to callee parameters
465 IRMapping argMap;
466 ValueRange values = op.getArgs();
467 if (values.size() != calleeBody.getNumArguments()) {
468 op.emitError("number of run op arguments (")
469 << values.size()
470 << ") does not match number of callee runtime sequence "
471 "arguments ("
472 << calleeBody.getNumArguments() << ")";
473 return failure();
474 }
475 for (unsigned i = 0, n = calleeBody.getNumArguments(); i < n; i++) {
476 BlockArgument arg = calleeBody.getArgument(i);
477 Value val = values[i];
478 argMap.map(arg, val);
479 }
480
481 // Clone operations from callee into current location
482 rewriter.setInsertionPoint(op);
483 for (Operation &o : calleeBody.front().getOperations()) {
484 // Skip the terminator
485 if (isa<AIE::EndOp>(o))
486 continue;
487 rewriter.clone(o, argMap);
488 }
489
490 // Erase the run op
491 rewriter.eraseOp(op);
492
493 return success();
494 }
495};
496
497struct ConfigureOpToCertSection : OpRewritePattern<AIEX::ConfigureOp> {
498 using OpRewritePattern::OpRewritePattern;
499
500 LogicalResult matchAndRewrite(AIEX::ConfigureOp op,
501 PatternRewriter &rewriter) const override {
502 // A run resolves its callee through the enclosing configure, so the
503 // configure is lowered only once RunOpInlining has expanded its runs.
504 if (!op.getRegion().getOps<AIEX::RunOp>().empty())
505 return failure();
506
507 // Get the referenced device
508 AIE::DeviceOp referencedDevice = op.getReferencedDeviceOp();
509 if (!referencedDevice)
510 return failure();
511
512 // Get the device's symbol name for the section
513 StringRef deviceSymName = referencedDevice.getSymName();
514
515 // Find the parent DeviceOp to insert the section into
516 auto parentDevice = op->getParentOfType<AIE::DeviceOp>();
517 if (!parentDevice)
518 return failure();
519
520 // Check if section already exists (avoid creating duplicates)
521 auto existingSection = dyn_cast_if_present<AIEX::CertSectionOp>(
522 parentDevice.lookupSymbol(deviceSymName));
523 if (existingSection) {
524 // Section already exists, just create load_pdi at call site, reusing the
525 // PDI ID already assigned to this section (or assigning a new one).
526 uint32_t pdiId = getOrAssignPdiId(parentDevice, deviceSymName);
527
528 rewriter.setInsertionPoint(op);
529 AIEX::CertLoadPdiOp::create(
530 rewriter, op.getLoc(), rewriter.getUI32IntegerAttr(pdiId),
531 FlatSymbolRefAttr::get(rewriter.getContext(), deviceSymName));
532
533 // Clone configure body operations after the load_pdi
534 IRMapping bodyMapper;
535 for (Operation &o : op.getRegion().front().getOperations()) {
536 if (isa<AIE::EndOp>(o))
537 continue;
538 rewriter.clone(o, bodyMapper);
539 }
540
541 rewriter.eraseOp(op);
542 return success();
543 }
544
545 // Create cert.section with the device symbol name
546 rewriter.setInsertionPoint(parentDevice.getBody()->getTerminator());
547 auto sectionOp = AIEX::CertSectionOp::create(
548 rewriter, op.getLoc(), rewriter.getStringAttr(deviceSymName));
549
550 // Create the section body with a page containing a job
551 Block *sectionBlock = new Block();
552 sectionOp.getBody().push_back(sectionBlock);
553 rewriter.setInsertionPointToStart(sectionBlock);
554
555 // Create page within section
556 auto pageOp = AIEX::CertPageOp::create(rewriter, op.getLoc());
557 Block *pageBlock = new Block();
558 pageOp.getBody().push_back(pageBlock);
559 rewriter.setInsertionPointToStart(pageBlock);
560
561 // Create job within page - assign unique job ID across all devices
562 // Find the maximum job ID in the parent device
563 uint32_t maxJobId = 0;
564 parentDevice.walk([&](AIEX::CertJobOp certJobOp) {
565 maxJobId = std::max(maxJobId, certJobOp.getJobId());
566 });
567 uint32_t sectionJobId = maxJobId + 1;
568
569 auto jobOp = AIEX::CertJobOp::create(rewriter, op.getLoc(), sectionJobId);
570 Block *jobBlock = new Block();
571 jobOp.getBody().push_back(jobBlock);
572 rewriter.setInsertionPointToStart(jobBlock);
573
574 // Find the "configure" sequence content from the referenced device
575 // It could be either:
576 // 1. Still the generated configuration sequence (if that device hasn't
577 // been processed yet)
578 // 2. Already converted to cert.page/cert.job (if that device was processed
579 // first)
580
581 AIE::RuntimeSequenceOp configureSeq =
582 findGeneratedConfigure(referencedDevice);
583
584 if (configureSeq) {
585 // Clone operations from the "configure" runtime_sequence
586 IRMapping mapper;
587 for (Operation &o : configureSeq.getRegion().front().getOperations()) {
588 if (isa<AIE::EndOp>(o))
589 continue;
590 rewriter.clone(o, mapper);
591 }
592 } else {
593 // "configure" was already converted - find the first cert.page
594 // (configuration page)
595 AIEX::CertPageOp configPage = nullptr;
596 for (auto &op : referencedDevice.getBody()->getOperations()) {
597 if (auto page = dyn_cast<AIEX::CertPageOp>(op)) {
598 configPage = page;
599 break; // Take first page (configure)
600 }
601 }
602
603 if (configPage) {
604 // Clone operations from the config page's job
605 IRMapping mapper;
606 for (auto &op : configPage.getBody().front().getOperations()) {
607 if (auto job = dyn_cast<AIEX::CertJobOp>(op)) {
608 for (Operation &jobOp : job.getBody().front().getOperations()) {
609 if (isa<AIE::EndOp>(jobOp))
610 continue;
611 rewriter.clone(jobOp, mapper);
612 }
613 break; // Only clone first job
614 }
615 }
616 }
617 }
618
619 // Ensure terminators
620 AIEX::CertJobOp::ensureTerminator(jobOp.getBody(), rewriter, op.getLoc());
621 AIEX::CertPageOp::ensureTerminator(pageOp.getBody(), rewriter, op.getLoc());
622 AIEX::CertSectionOp::ensureTerminator(sectionOp.getBody(), rewriter,
623 op.getLoc());
624
625 // At the configure call site, replace with load_pdi, reusing the PDI ID
626 // already assigned to this device if present (or assigning a new one).
627 uint32_t pdiId = getOrAssignPdiId(parentDevice, deviceSymName);
628
629 rewriter.setInsertionPoint(op);
630 AIEX::CertLoadPdiOp::create(
631 rewriter, op.getLoc(), rewriter.getUI32IntegerAttr(pdiId),
632 FlatSymbolRefAttr::get(rewriter.getContext(), deviceSymName));
633
634 // Clone configure body operations after the load_pdi
635 // Operations that reference external values were not cloned into the
636 // section, so we clone them here into the main control flow
637 IRMapping bodyMapper;
638 for (Operation &o : op.getRegion().front().getOperations()) {
639 if (isa<AIE::EndOp>(o))
640 continue;
641 rewriter.clone(o, bodyMapper);
642 }
643
644 // Erase the original configure op
645 rewriter.eraseOp(op);
646 return success();
647 }
648};
649
650// The blockwrite that loads the BD an address_patch patches: the nearest
651// preceding blockwrite, when it targets the patched BD.
652static AIEX::NpuBlockWriteOp
653findPatchedBlockWrite(AIEX::NpuAddressPatchOp patch) {
654 const auto &tm = AIE::getTargetModel(patch);
655 uint32_t addr = patch.getAddr();
656 int col = (addr >> tm.getColumnShift()) & 0x1f;
657 int row = (addr >> tm.getRowShift()) & 0x1f;
658 if (!tm.isValidTile({col, row}))
659 return {};
660 for (Block::iterator it(patch); it != patch->getBlock()->begin();) {
661 auto blockWrite = dyn_cast<AIEX::NpuBlockWriteOp>(*--it);
662 if (!blockWrite)
663 continue;
664 if (blockWrite.getAddress() + tm.getDmaBdAddressOffset(col, row) != addr)
665 return {};
666 return blockWrite;
667 }
668 return {};
669}
670
671// apply_offset_57 names an argument by its first 32-bit register, 2 * arg_idx,
672// in a 16-bit field, and reserves the index 0xFFFF for the control code's own
673// address. So the largest usable arg_idx is the one whose double still fits.
674static constexpr uint32_t kMaxApplyOffsetArgIdx = 0x7FFF;
675
676// An address_patch writes buffer address + arg_plus into a BD register, but
677// cert.apply_offset_57 adds the buffer address to the BD template in memory.
678// So before conversion the patched template word must hold arg_plus, and the
679// template must belong to its blockwrite alone: dma-to-npu shares identical
680// templates between BDs, and a shared one would accumulate every BD's address.
681static LogicalResult prepareAddressPatchTemplates(AIE::DeviceOp device) {
682 LogicalResult result = success();
683 SymbolTable symbols(device);
684 // Count each global's symbol uses once, then keep the counts current as
685 // getters are added and erased. Asking SymbolTable per patch walks the whole
686 // device each time, which is quadratic in the number of patched BDs. With no
687 // counts (an op with unknown symbol uses), every patched template is copied.
688 std::optional<llvm::DenseMap<StringAttr, unsigned>> symbolUseCounts;
689 if (auto uses = SymbolTable::getSymbolUses(&device.getRegion())) {
690 symbolUseCounts.emplace();
691 for (const SymbolTable::SymbolUse &use : *uses)
692 ++(*symbolUseCounts)[use.getSymbolRef().getRootReference()];
693 }
694 SmallVector<AIEX::NpuAddressPatchOp> patches;
695 device.walk([&](AIEX::NpuAddressPatchOp patch) { patches.push_back(patch); });
696 for (AIEX::NpuAddressPatchOp patch : patches) {
697 std::optional<uint32_t> argIdx = patch.getArgIdx();
698 if (argIdx && *argIdx > kMaxApplyOffsetArgIdx) {
699 patch.emitError("arg_idx ")
700 << *argIdx << " does not fit apply_offset_57's 16-bit argument "
701 << "offset; the largest is " << kMaxApplyOffsetArgIdx;
702 result = failure();
703 continue;
704 }
705 AIEX::NpuBlockWriteOp blockWrite = findPatchedBlockWrite(patch);
706 if (!blockWrite)
707 continue;
708 auto getGlobal = dyn_cast_or_null<memref::GetGlobalOp>(
709 blockWrite.getData().getDefiningOp());
710 if (!getGlobal)
711 continue;
712 auto global = symbols.lookup<memref::GlobalOp>(getGlobal.getName());
713 if (!global)
714 continue;
715 std::optional<Attribute> initialValue = global.getInitialValue();
716 auto words = initialValue ? dyn_cast<DenseIntElementsAttr>(*initialValue)
717 : DenseIntElementsAttr();
718 if (!words)
719 continue;
720
721 if (!patch.getArgIdx()) {
722 patch.emitError("cert lowering requires an address_patch with arg_idx");
723 result = failure();
724 continue;
725 }
726 APInt argPlus;
727 if (!matchPattern(patch.getArgPlus(), m_ConstantInt(&argPlus))) {
728 patch.emitError("cert lowering requires a constant arg_plus: "
729 "apply_offset_57 cannot add a runtime offset");
730 result = failure();
731 continue;
732 }
733 // An i32 arg_plus is unsigned, so any value fits; a negative i64 has 64
734 // active bits and is rejected.
735 if (argPlus.getActiveBits() > 32) {
736 patch.emitError("arg_plus ")
737 << argPlus.getSExtValue()
738 << " does not fit the BD's 32-bit address word";
739 result = failure();
740 continue;
741 }
742 unsigned word = (patch.getAddr() - blockWrite.getAddress()) / 4;
743 if (word >= words.getNumElements())
744 continue;
745
746 // Give this blockwrite a private template if anything else uses it,
747 // whether through another getter or through this one (after CSE).
748 if (!symbolUseCounts ||
749 symbolUseCounts->lookup(global.getSymNameAttr()) != 1 ||
750 !getGlobal->hasOneUse()) {
751 OpBuilder builder(global);
752 auto copy = cast<memref::GlobalOp>(builder.clone(*global));
753 // insert renames the copy, so count it under its new name.
754 StringAttr copyName = symbols.insert(copy);
755 OpBuilder atBlockWrite(blockWrite);
756 auto privateGet =
757 memref::GetGlobalOp::create(atBlockWrite, getGlobal.getLoc(),
758 getGlobal.getType(), copy.getSymName());
759 blockWrite.getDataMutable().assign(privateGet);
760 if (symbolUseCounts)
761 (*symbolUseCounts)[copyName] = 1;
762 if (getGlobal->use_empty()) {
763 if (symbolUseCounts)
764 --(*symbolUseCounts)[global.getSymNameAttr()];
765 getGlobal->erase();
766 }
767 global = copy;
768 }
769
770 auto values = llvm::to_vector(words.getValues<APInt>());
771 values[word] = argPlus.zextOrTrunc(values[word].getBitWidth());
772 global.setInitialValueAttr(
773 DenseIntElementsAttr::get(words.getType(), values));
774 }
775 return result;
776}
777
778struct NpuAddressPatchToCertApplyOffset57
779 : OpConversionPattern<AIEX::NpuAddressPatchOp> {
780 using OpConversionPattern::OpConversionPattern;
781
782 LogicalResult
783 matchAndRewrite(AIEX::NpuAddressPatchOp op, OpAdaptor adaptor,
784 ConversionPatternRewriter &rewriter) const override {
785 AIEX::NpuBlockWriteOp blockWriteOp = findPatchedBlockWrite(op);
786 std::optional<uint32_t> argIdx = op.getArgIdx();
787 if (!blockWriteOp || !argIdx)
788 return failure();
789 auto getGlobalOp =
790 dyn_cast<memref::GetGlobalOp>(blockWriteOp.getData().getDefiningOp());
791 if (!getGlobalOp)
792 return failure();
793 rewriter.setInsertionPoint(blockWriteOp);
794 rewriter.replaceOpWithNewOp<AIEX::CertApplyOffset57Op>(
795 op, getGlobalOp.getName(), 1, *argIdx);
796 return success();
797 }
798};
799
800struct MergeConsecutiveCertUcDmaWriteDesSyncOps
801 : OpRewritePattern<AIEX::CertUcDmaWriteDesSyncOp> {
802 // symbolTable is the device's. useCounts holds the symbol uses in the device,
803 // counted once up front: a merge erases the earlier chain and its one
804 // enqueue and moves no other chain reference.
805 MergeConsecutiveCertUcDmaWriteDesSyncOps(
806 MLIRContext *context, SymbolTable &symbolTable,
807 llvm::DenseMap<StringAttr, unsigned> &useCounts)
808 : OpRewritePattern(context), symbolTable(symbolTable),
809 useCounts(useCounts) {}
810
811 LogicalResult matchAndRewrite(AIEX::CertUcDmaWriteDesSyncOp op,
812 PatternRewriter &rewriter) const override {
813 // Fusing the two enqueues sinks the earlier one to the later one's position
814 // in the job, so every op scanned over is an op the earlier DMA is moved
815 // past. Only ops with no architectural effect at all may be crossed; this
816 // is an allowlist rather than a denylist so a cert op added to this dialect
817 // later blocks the merge by default instead of silently becoming
818 // reorderable with respect to a DMA enqueue. cert.nop is the only such op
819 // today: it emits padding and carries no ordering, sync or side effect.
820 auto isTransparentToMerge = [](Operation *o) {
821 return isa<AIEX::CertNopOp>(o);
822 };
823
824 // Get the previous operation in the block
825 Block::iterator it(op);
826 AIEX::CertUcDmaWriteDesSyncOp prevWriteDesSync = nullptr;
827 while (it != op->getBlock()->begin() && !prevWriteDesSync) {
828 --it;
829 Operation *prevOp = &*it;
830 prevWriteDesSync = dyn_cast<AIEX::CertUcDmaWriteDesSyncOp>(prevOp);
831 if (!prevWriteDesSync && !isTransparentToMerge(prevOp))
832 return failure();
833 }
834 if (!prevWriteDesSync)
835 return failure();
836
837 // find the uc_dma_chain
838 StringRef sym_name = op.getSymbol();
839 StringRef prev_sym_name = prevWriteDesSync.getSymbol();
840 auto chain = symbolTable.lookup<AIEX::CertUcDmaChainOp>(sym_name);
841 auto prevChain = symbolTable.lookup<AIEX::CertUcDmaChainOp>(prev_sym_name);
842 if (!chain || !prevChain)
843 return failure();
844
845 // The merge rewrites @chain's BD list in place and deletes @prevChain, so
846 // both symbols must be private to this pair of enqueues. Cert IR that is
847 // hand-authored rather than lowered from npu ops may enqueue one chain from
848 // several places: folding into a shared @chain would silently change those
849 // other enqueues, and erasing a shared @prevChain would leave a dangling
850 // symbol reference that neither the verifier nor the page cost model
851 // notices. Decline the merge instead.
852 auto isSingleUse = [&](AIEX::CertUcDmaChainOp c) {
853 return useCounts.lookup(c.getSymNameAttr()) == 1;
854 };
855 if (!isSingleUse(chain) || !isSingleUse(prevChain))
856 return failure();
857
858 // Compute the size of the current and previous chains. If their combined
859 // data size is greater than the cert page size, then we cannot merge them.
860 //
861 // A chain costs 16 bytes of descriptor per BD on top of the BD's payload --
862 // the same accounting updateCostForOp (the page splitter's authoritative
863 // cost model) uses. Charging payload only under-reports a chain by 16 bytes
864 // per BD, which for many small BDs is most of its real size.
865 auto chainSize = [](AIEX::CertUcDmaChainOp c) {
866 uint32_t size = 0;
867 for (auto bdOp : c.getBody().front().getOps<AIEX::CertUcDmaBdOp>())
868 size += bdOp.getLength() * sizeof(int) + 16; // payload + bd descriptor
869 return size;
870 };
871 if ((chainSize(chain) + chainSize(prevChain)) >= cert_page_size)
872 return failure();
873
874 IRMapping map;
875 rewriter.setInsertionPointToStart(&chain.getBody().front());
876 for (auto &o : prevChain.getBody().front().getOperations()) {
877 auto bdOp = dyn_cast<AIEX::CertUcDmaBdOp>(o);
878 if (!bdOp)
879 continue;
880 AIEX::CertUcDmaBdOp::create(
881 rewriter, bdOp.getLoc(), bdOp.getRemoteAddress(),
882 bdOp.getLocalAddress(), bdOp.getLength(), true);
883 }
884 useCounts.erase(prevChain.getSymNameAttr());
885 symbolTable.remove(prevChain);
886 rewriter.eraseOp(prevChain);
887 rewriter.eraseOp(prevWriteDesSync);
888 return success();
889 }
890
891 SymbolTable &symbolTable;
892 llvm::DenseMap<StringAttr, unsigned> &useCounts;
893};
894
895struct SplitNpuBlockWriteOpPattern : OpRewritePattern<AIEX::NpuBlockWriteOp> {
896 // The pattern runs over the module, so it keeps a table per device and adds
897 // the globals it creates to it.
898 SplitNpuBlockWriteOpPattern(MLIRContext *context,
899 SymbolTableCollection &symbolTables)
900 : OpRewritePattern(context), symbolTables(symbolTables) {}
901
902 LogicalResult matchAndRewrite(AIEX::NpuBlockWriteOp op,
903 PatternRewriter &rewriter) const override {
904
905 memref::GetGlobalOp dataOperand =
906 dyn_cast_or_null<memref::GetGlobalOp>(op.getData().getDefiningOp());
907 if (!dataOperand)
908 return failure();
909
910 MemRefType dataType = cast<MemRefType>(dataOperand.getResult().getType());
911 uint32_t dataSize = dataType.getNumElements();
912
913 uint32_t dataSizeBytes = dataSize * sizeof(int);
914 if (dataSizeBytes < cert_page_size)
915 return failure();
916
917 auto loc = op.getLoc();
918
919 // Calculate split point (split roughly in half)
920 uint32_t splitElements = dataSize / 2;
921 uint32_t firstChunkSize = splitElements;
922 uint32_t secondChunkSize = dataSize - splitElements;
923
924 // Find the original memref.global operation
925 auto deviceOp = op->getParentOfType<AIE::DeviceOp>();
926 SymbolTable &symbolTable = symbolTables.getSymbolTable(deviceOp);
927 auto originalGlobal =
928 symbolTable.lookup<memref::GlobalOp>(dataOperand.getName());
929 if (!originalGlobal)
930 return failure();
931
932 // Get the original data attribute
933 auto originalData = originalGlobal.getInitialValue();
934 if (!originalData)
935 return failure();
936
937 auto denseData = dyn_cast<DenseIntElementsAttr>(*originalData);
938 if (!denseData)
939 return failure();
940
941 // Split the data into two chunks
942 auto dataValues = denseData.getValues<APInt>();
943 std::vector<APInt> firstChunkData(dataValues.begin(),
944 dataValues.begin() + firstChunkSize);
945 std::vector<APInt> secondChunkData(dataValues.begin() + firstChunkSize,
946 dataValues.end());
947
948 // Create new global operations for the split data
949 auto elementType = rewriter.getI32Type();
950 auto firstChunkType = MemRefType::get({firstChunkSize}, elementType);
951 auto secondChunkType = MemRefType::get({secondChunkSize}, elementType);
952 TensorType firstTensorType =
953 RankedTensorType::get({firstChunkSize}, elementType);
954 TensorType secondTensorType =
955 RankedTensorType::get({secondChunkSize}, elementType);
956
957 auto firstChunkAttr =
958 DenseIntElementsAttr::get(firstTensorType, firstChunkData);
959 auto secondChunkAttr =
960 DenseIntElementsAttr::get(secondTensorType, secondChunkData);
961
962 // Generate unique names for the new globals
963 std::string firstName = dataOperand.getName().str() + "_split_0";
964 std::string secondName = dataOperand.getName().str() + "_split_1";
965
966 // Ensure unique names
967 int counter = 0;
968 while (symbolTable.lookup(firstName)) {
969 firstName =
970 dataOperand.getName().str() + "_split_0_" + std::to_string(counter++);
971 }
972 counter = 0;
973 while (symbolTable.lookup(secondName)) {
974 secondName =
975 dataOperand.getName().str() + "_split_1_" + std::to_string(counter++);
976 }
977
978 // Create the new global operations
979 rewriter.setInsertionPoint(originalGlobal);
980 auto firstGlobal = memref::GlobalOp::create(
981 rewriter, loc, firstName, rewriter.getStringAttr("private"),
982 firstChunkType, firstChunkAttr, true, nullptr);
983 symbolTable.insert(firstGlobal);
984
985 auto secondGlobal = memref::GlobalOp::create(
986 rewriter, loc, secondName, rewriter.getStringAttr("private"),
987 secondChunkType, secondChunkAttr, true, nullptr);
988 symbolTable.insert(secondGlobal);
989
990 // Create get_global operations for the new data
991 rewriter.setInsertionPoint(op);
992
993 auto firstGetGlobal =
994 memref::GetGlobalOp::create(rewriter, loc, firstChunkType, firstName);
995 auto secondGetGlobal =
996 memref::GetGlobalOp::create(rewriter, loc, secondChunkType, secondName);
997
998 uint32_t baseAddr = op.getAddress();
999
1000 AIEX::NpuBlockWriteOp::create(rewriter, loc, baseAddr,
1001 firstGetGlobal.getResult(), nullptr, nullptr,
1002 nullptr);
1003
1004 AIEX::NpuBlockWriteOp::create(rewriter, loc, baseAddr + firstChunkSize * 4,
1005 secondGetGlobal.getResult(), nullptr, nullptr,
1006 nullptr);
1007
1008 // Replace the original operation
1009 rewriter.eraseOp(op);
1010
1011 LLVM_DEBUG(llvm::outs()
1012 << "Split NpuBlockWriteOp with data size: " << dataSizeBytes
1013 << " bytes into chunks of " << firstChunkSize << " and "
1014 << secondChunkSize << " elements\n");
1015
1016 return success();
1017 }
1018
1019 SymbolTableCollection &symbolTables;
1020};
1021
1022// Rewrite each tile- or buffer-relative blockwrite to the absolute address it
1023// targets, so that splitting, address_patch matching and lowering all use the
1024// raw address. This runs before configure sequences are cloned into another
1025// device, where the buffer symbol would no longer resolve.
1026static LogicalResult resolveBlockWriteAddresses(AIE::DeviceOp device) {
1027 AIE::NamedOpTable names(device);
1028 LogicalResult result = success();
1029 device.walk([&](AIEX::NpuBlockWriteOp op) {
1030 if (!op.getBuffer() && !(op.getColumn() && op.getRow()))
1031 return;
1032 std::optional<uint32_t> address = op.getAbsoluteAddress(&names);
1033 if (!address) {
1034 result = failure();
1035 return;
1036 }
1037 op.setAddress(*address);
1038 op.removeBufferAttr();
1039 op.removeColumnAttr();
1040 op.removeRowAttr();
1041 });
1042 return result;
1043}
1044
1045struct AIENpuToCertPass
1046 : xilinx::AIEX::impl::AIENpuToCertBase<AIENpuToCertPass> {
1047 void runOnOperation() override {
1048 auto moduleOp = getOperation();
1049 if (failed(AIE::inlineWriteConfigs(moduleOp)))
1050 return signalPassFailure();
1051
1052 // Collect all devices
1053 llvm::SmallVector<AIE::DeviceOp, 4> devices;
1054 for (auto &op : moduleOp.getBody()->getOperations()) {
1055 if (auto deviceOp = dyn_cast<AIE::DeviceOp>(op)) {
1056 devices.push_back(deviceOp);
1057 }
1058 }
1059
1060 bool resolved = true;
1061 for (auto dev : devices)
1062 resolved &= succeeded(resolveBlockWriteAddresses(dev));
1063 if (!resolved)
1064 return signalPassFailure();
1065
1066 for (auto dev : devices)
1067 tagConfigureByName(dev);
1068
1069 // Process main device first (named "main" or specified by --device-name)
1070 // so referenced devices' "configure" sequences can be found before they
1071 // get converted to cert.job
1072 AIE::DeviceOp mainDevice = nullptr;
1073 for (auto dev : devices) {
1074 if (dev.getSymName() == deviceName) {
1075 mainDevice = dev;
1076 break;
1077 }
1078 }
1079
1080 if (!mainDevice) {
1081 moduleOp.emitError("no device found matching --device-name \"")
1082 << deviceName << "\"";
1083 signalPassFailure();
1084 return;
1085 }
1086
1087 migrateGlobalsIntoMainDevice(mainDevice, devices);
1088 processDevice(mainDevice);
1089
1090 // Then process other devices
1091 for (auto dev : devices) {
1092 if (dev != mainDevice) {
1093 processDevice(dev);
1094 }
1095 }
1096
1097 // Remove absorbed devices (keep only main device)
1098 // Referenced devices have been absorbed as cert.sections, so remove them
1099 llvm::SmallVector<AIE::DeviceOp, 4> devicesToRemove;
1100 for (auto dev : devices) {
1101 if (dev != mainDevice) {
1102 devicesToRemove.push_back(dev);
1103 }
1104 }
1105
1106 for (auto dev : devicesToRemove) {
1107 dev.erase();
1108 }
1109 }
1110
1111 // Move globals into the surviving device and rename payload references before
1112 // cloning configure bodies. Reserve device names for sections, including when
1113 // renaming the main device's globals: processDevice skips names that resolve.
1114 void migrateGlobalsIntoMainDevice(AIE::DeviceOp mainDevice,
1115 ArrayRef<AIE::DeviceOp> devices) {
1116 auto isDeviceName = [&](StringRef candidate) {
1117 return llvm::any_of(devices, [&](AIE::DeviceOp device) {
1118 return device.getSymName() == candidate;
1119 });
1120 };
1121
1122 // Names and payload uses are indexed once per device and kept current
1123 // below; scanning the device per global is quadratic in the globals.
1124 DeviceNames mainNames = collectNames(mainDevice);
1125 std::optional<PayloadUses> mainUses;
1126 for (auto global : llvm::to_vector(mainDevice.getOps<memref::GlobalOp>())) {
1127 StringRef oldName = global.getSymName();
1128 if (!isDeviceName(oldName))
1129 continue;
1130 std::string newName;
1131 unsigned counter = 0;
1132 do
1133 newName = oldName.str() + "_" + std::to_string(counter++);
1134 while (mainNames.contains(newName) || isDeviceName(newName));
1135 if (!mainUses)
1136 mainUses = collectPayloadUses(mainDevice);
1137 renamePayloadUses(*mainUses, oldName, newName);
1138 removeName(mainNames, oldName);
1139 ++mainNames[newName];
1140 global.setSymName(newName);
1141 }
1142
1143 for (auto dev : devices) {
1144 if (dev == mainDevice)
1145 continue;
1146 DeviceNames devNames = collectNames(dev);
1147 PayloadUses devUses = collectPayloadUses(dev);
1148 auto globals = llvm::to_vector(dev.getOps<memref::GlobalOp>());
1149 for (auto global : globals) {
1150 StringRef oldName = global.getSymName();
1151 std::string newName = oldName.str();
1152 unsigned counter = 0;
1153 // Avoid a temporary collision with another source global: rewriting
1154 // its uses later would otherwise redirect both payloads to one symbol.
1155 auto taken = [&](StringRef candidate) {
1156 return mainNames.contains(candidate) || isDeviceName(candidate) ||
1157 (candidate != oldName && devNames.contains(candidate));
1158 };
1159 while (taken(newName))
1160 newName = oldName.str() + "_" + std::to_string(counter++);
1161 if (newName != oldName) {
1162 renamePayloadUses(devUses, oldName, newName);
1163 global.setSymName(newName);
1164 }
1165 removeName(devNames, oldName);
1166 ++mainNames[newName];
1167 global->moveBefore(mainDevice.getBody(), mainDevice.getBody()->begin());
1168 }
1169 }
1170 }
1171
1172 // The sym_names of a device's body ops, with their counts. The device
1173 // verifier rejects duplicates among all of them, including SSA ops such as
1174 // aie.buffer that the symbol table does not index.
1175 using DeviceNames = llvm::StringMap<unsigned>;
1176 static DeviceNames collectNames(AIE::DeviceOp dev) {
1177 DeviceNames names;
1178 for (Operation &op : dev.getBody()->getOperations())
1179 if (auto symName =
1180 op.getAttrOfType<StringAttr>(SymbolTable::getSymbolAttrName()))
1181 ++names[symName.getValue()];
1182 return names;
1183 }
1184 static void removeName(DeviceNames &names, StringRef name) {
1185 auto it = names.find(name);
1186 if (it != names.end() && --it->second == 0)
1187 names.erase(it);
1188 }
1189
1190 // The ops that reference a payload global, by the name they reference. A
1191 // generic symbol-use rewrite would also rename same-named device references,
1192 // such as load_pdi's device_ref, which resolve against the module. Nested
1193 // symbol tables resolve their own names, so their uses are left alone.
1194 using PayloadUses = llvm::StringMap<SmallVector<Operation *>>;
1195 static PayloadUses collectPayloadUses(AIE::DeviceOp dev) {
1196 PayloadUses uses;
1197 dev.walk<WalkOrder::PreOrder>([&](Operation *op) {
1198 if (op != dev && op->hasTrait<OpTrait::SymbolTable>())
1199 return WalkResult::skip();
1200 if (auto get = dyn_cast<memref::GetGlobalOp>(op))
1201 uses[get.getName()].push_back(op);
1202 else if (auto bd = dyn_cast<AIEX::CertUcDmaBdOp>(op))
1203 uses[bd.getRemoteAddress()].push_back(op);
1204 else if (auto patch = dyn_cast<AIEX::CertApplyOffset57Op>(op))
1205 uses[patch.getSymbol()].push_back(op);
1206 return WalkResult::advance();
1207 });
1208 return uses;
1209 }
1210
1211 // Point the indexed references to `oldName` at `newName`.
1212 static void renamePayloadUses(PayloadUses &uses, StringRef oldName,
1213 StringRef newName) {
1214 auto it = uses.find(oldName);
1215 if (it == uses.end())
1216 return;
1217 SmallVector<Operation *> ops = std::move(it->second);
1218 uses.erase(it);
1219 for (Operation *op : ops) {
1220 auto newRef = FlatSymbolRefAttr::get(op->getContext(), newName);
1221 if (auto get = dyn_cast<memref::GetGlobalOp>(op))
1222 get.setNameAttr(newRef);
1223 else if (auto bd = dyn_cast<AIEX::CertUcDmaBdOp>(op))
1224 bd.setRemoteAddressAttr(newRef);
1225 else if (auto patch = dyn_cast<AIEX::CertApplyOffset57Op>(op))
1226 patch.setSymbolAttr(newRef);
1227 }
1228 llvm::append_range(uses[newName], ops);
1229 }
1230
1231 // Absorb each device that an npu.load_pdi in `currentDevice` names as a
1232 // same-named cert.section built from the device's configuration.
1233 LogicalResult absorbLoadedDevices(AIE::DeviceOp currentDevice,
1234 ModuleOp moduleOp) {
1235 // Collect npu.load_pdi operations, before they become cert.load_pdi. An
1236 // absorbed configuration can load further devices, so its loads are
1237 // collected too.
1238 llvm::SmallVector<AIEX::NpuLoadPdiOp, 4> loads;
1239 auto collectLoads = [&](Operation *root) {
1240 root->walk([&](AIEX::NpuLoadPdiOp loadPdiOp) {
1241 if (loadPdiOp.getDeviceRef())
1242 loads.push_back(loadPdiOp);
1243 });
1244 };
1245 collectLoads(currentDevice);
1246
1247 OpBuilder builder(&getContext());
1248 // Worklist in collection order: absorbing a device appends its loads, so
1249 // `loads` can grow (and reallocate) while it is being walked.
1250 size_t next = 0;
1251 while (next < loads.size()) {
1252 AIEX::NpuLoadPdiOp load = loads[next++];
1253 std::optional<StringRef> deviceRef = load.getDeviceRef();
1254 if (!deviceRef)
1255 continue;
1256 StringRef refSymName = *deviceRef;
1257 // Skip if section already exists
1258 Operation *existing = AIE::lookupNamedOpIn(currentDevice, refSymName);
1259 if (isa_and_present<AIEX::CertSectionOp>(existing))
1260 continue;
1261
1262 // Find referenced device
1263 AIE::DeviceOp refDevice = nullptr;
1264 for (auto &op : moduleOp.getBody()->getOperations()) {
1265 if (auto dev = dyn_cast<AIE::DeviceOp>(op)) {
1266 if (dev.getSymName() == refSymName) {
1267 refDevice = dev;
1268 break;
1269 }
1270 }
1271 }
1272
1273 if (!refDevice)
1274 continue;
1275
1276 // Find the generated configuration sequence
1277 AIE::RuntimeSequenceOp configureSeq = findGeneratedConfigure(refDevice);
1278
1279 if (!configureSeq)
1280 continue;
1281
1282 // Any other symbol of that name would leave the load without its
1283 // section.
1284 if (existing) {
1285 load.emitError("cannot absorb device @")
1286 << refSymName << " as a cert.section: @" << refSymName
1287 << " is already defined by '" << existing->getName() << "'";
1288 return failure();
1289 }
1290
1291 // Create cert.section in current device
1292 builder.setInsertionPoint(currentDevice.getBody()->getTerminator());
1293 auto sectionOp = AIEX::CertSectionOp::create(
1294 builder, configureSeq.getLoc(), builder.getStringAttr(refSymName));
1295
1296 Block *sectionBlock = new Block();
1297 sectionOp.getBody().push_back(sectionBlock);
1298 builder.setInsertionPointToStart(sectionBlock);
1299
1300 // Create page
1301 auto pageOp = AIEX::CertPageOp::create(builder, configureSeq.getLoc());
1302 Block *pageBlock = new Block();
1303 pageOp.getBody().push_back(pageBlock);
1304 builder.setInsertionPointToStart(pageBlock);
1305
1306 // Get unique job ID
1307 uint32_t maxJobId = 0;
1308 currentDevice.walk([&](AIEX::CertJobOp certJobOp) {
1309 maxJobId = std::max(maxJobId, certJobOp.getJobId());
1310 });
1311
1312 // Create job
1313 auto jobOp =
1314 AIEX::CertJobOp::create(builder, configureSeq.getLoc(), maxJobId + 1);
1315
1316 // Clone configure sequence body into job
1317 IRMapping mapper;
1318 configureSeq.getRegion().cloneInto(&jobOp.getBody(), mapper);
1319
1320 // Ensure terminators
1321 AIEX::CertJobOp::ensureTerminator(jobOp.getBody(), builder,
1322 configureSeq.getLoc());
1323 AIEX::CertPageOp::ensureTerminator(pageOp.getBody(), builder,
1324 configureSeq.getLoc());
1325 AIEX::CertSectionOp::ensureTerminator(sectionOp.getBody(), builder,
1326 configureSeq.getLoc());
1327 collectLoads(jobOp);
1328 }
1329 return success();
1330 }
1331
1332 void processDevice(AIE::DeviceOp currentDevice) {
1333 auto moduleOp = currentDevice->getParentOfType<ModuleOp>();
1334
1335 // Inline RunOps and convert ConfigureOps to cert.section + cert.load_pdi.
1336 // ConfigureOpToCertSection waits for the runs of a configure to be inlined.
1337 RewritePatternSet p_expand(&getContext());
1338 p_expand.insert<RunOpInlining, ConfigureOpToCertSection>(&getContext());
1339 FrozenRewritePatternSet expandPatterns(std::move(p_expand));
1340
1341 // Absorbing a device can clone in run and configure ops, and expanding
1342 // those can clone in further loads. Repeat until the expansion that follows
1343 // an absorption changes nothing.
1344 bool changed = true;
1345 while (changed) {
1346 // Identify and convert referenced "configure" sequences, before the
1347 // RuntimeSequenceOp conversion runs on other devices.
1348 if (moduleOp && failed(absorbLoadedDevices(currentDevice, moduleOp)))
1349 return signalPassFailure();
1350
1351 if (failed(applyPatternsGreedily(currentDevice, expandPatterns,
1352 GreedyRewriteConfig(), &changed)))
1353 return signalPassFailure();
1354 }
1355
1356 ConversionTarget target(getContext());
1357 target.addIllegalOp<AIE::RuntimeSequenceOp>();
1358
1359 target.addLegalOp<AIEX::CertApplyOffset57Op>();
1360 target.addLegalOp<AIEX::CertJobOp>();
1361 target.addLegalOp<AIEX::CertPageOp>();
1362 target.addLegalOp<AIEX::CertSectionOp>();
1363 target.addLegalOp<AIEX::CertLoadPdiOp>();
1364 target.addLegalOp<AIEX::CertMaskWrite32Op>();
1365 target.addLegalOp<AIEX::CertMaskPoll32Op>();
1366 target.addLegalOp<AIEX::CertUcDmaWriteDesSyncOp>();
1367 target.addLegalOp<AIEX::CertUcDmaChainOp>();
1368 target.addLegalOp<AIEX::CertUcDmaBdOp>();
1369 target.addLegalOp<AIEX::CertWrite32Op>();
1370 target.addLegalOp<AIEX::CertWaitTCTSOp>();
1371 target.addLegalOp<AIEX::ConfigureOp>(); // TODO: Convert in separate pass
1372 target.addLegalDialect<AIE::AIEDialect>();
1373
1374 RewritePatternSet p0(&getContext());
1375 p0.insert<RuntimeSequenceToCertJob>(&getContext());
1376
1377 if (failed(applyPartialConversion(currentDevice, target, std::move(p0))))
1378 return signalPassFailure();
1379
1380 if (failed(prepareAddressPatchTemplates(currentDevice)))
1381 return signalPassFailure();
1382
1383 target.addIllegalOp<AIEX::NpuAddressPatchOp>();
1384
1385 // patch conversion must come before blockwrite conversion
1386 RewritePatternSet p1(&getContext());
1387 p1.insert<NpuAddressPatchToCertApplyOffset57>(&getContext());
1388
1389 if (failed(applyPartialConversion(currentDevice, target, std::move(p1))))
1390 return signalPassFailure();
1391
1392 // Split oversized NpuBlockWriteOps before lowering them to cert ops
1393 {
1394 SymbolTableCollection symbolTables;
1395 RewritePatternSet p(&getContext());
1396 p.insert<SplitNpuBlockWriteOpPattern>(&getContext(), symbolTables);
1397 if (failed(applyPatternsGreedily(getOperation(), std::move(p))))
1398 signalPassFailure();
1399 }
1400
1401 target.addIllegalOp<AIEX::NpuBlockWriteOp>();
1402 target.addIllegalOp<AIEX::NpuBlockWriteValuesOp>();
1403 target.addIllegalOp<AIEX::NpuMaskWrite32Op>();
1404 target.addIllegalOp<AIEX::NpuMaskPollOp>();
1405 target.addIllegalOp<AIEX::NpuSyncOp>();
1406 target.addIllegalOp<AIEX::NpuWrite32Op>();
1407 target.addIllegalOp<AIEX::NpuLoadPdiOp>();
1408
1409 unsigned nextChainId = 0;
1410 SymbolTable symbolTable(currentDevice);
1411 RewritePatternSet p2(&getContext());
1412 p2.insert<NpuBlockWriteToCertUcDma>(&getContext(), nextChainId,
1413 symbolTable);
1414 p2.insert<NpuMaskWrite32ToCertMaskWrite32>(&getContext());
1415 p2.insert<NpuMaskPollToCertMaskPoll32>(&getContext());
1416 p2.insert<NpuWrite32ToCertWrite32>(&getContext());
1417 p2.insert<NpuSyncToCertWaitTCTS>(&getContext());
1418 p2.insert<NpuLoadPdiToCertLoadPdi>(&getContext());
1419
1420 if (failed(applyPartialConversion(currentDevice, target, std::move(p2))))
1421 return signalPassFailure();
1422
1423 // Run the greedy driver with no patterns, purely for its DCE: p2
1424 // leaves each converted blockwrite's memref.get_global (and any constant
1425 // feeding it) dead, since the new op references the global by symbol
1426 // rather than by SSA value. CertJobOp's verifier rejects those ops
1427 RewritePatternSet cleanup(&getContext());
1428 if (failed(applyPatternsGreedily(getOperation(), std::move(cleanup))))
1429 return signalPassFailure();
1430
1431 // Convert referenced devices to cert.sections
1432 // Now that cert.load_pdi ops exist, find referenced devices and convert
1433 // their cert jobs into sections in this device
1434 {
1435 OpBuilder builder(&getContext());
1436
1437 // Find all cert.load_pdi in current device
1438 llvm::SetVector<StringRef> referencedDeviceNames;
1439 currentDevice.walk([&](AIEX::CertLoadPdiOp loadPdiOp) {
1440 StringRef refSymName = loadPdiOp.getSymbol();
1441 if (!currentDevice.lookupSymbol(refSymName))
1442 referencedDeviceNames.insert(refSymName);
1443 });
1444
1445 // For each referenced device, create a section with its cert jobs
1446 for (StringRef refSymName : referencedDeviceNames) {
1447 // Find the referenced device
1448 AIE::DeviceOp refDevice = nullptr;
1449 for (auto &op : moduleOp.getBody()->getOperations()) {
1450 if (auto dev = dyn_cast<AIE::DeviceOp>(op)) {
1451 if (dev.getSymName() == refSymName) {
1452 refDevice = dev;
1453 break;
1454 }
1455 }
1456 }
1457
1458 if (!refDevice)
1459 continue;
1460
1461 // Find all cert.page ops in the referenced device that are direct
1462 // children
1463 llvm::SmallVector<AIEX::CertPageOp, 4> certPages;
1464 for (auto &op : refDevice.getBody()->getOperations()) {
1465 if (auto pageOp = dyn_cast<AIEX::CertPageOp>(op)) {
1466 certPages.push_back(pageOp);
1467 }
1468 }
1469
1470 if (certPages.empty())
1471 continue;
1472
1473 // Create cert.section in current device
1474 builder.setInsertionPoint(currentDevice.getBody()->getTerminator());
1475 auto sectionOp = AIEX::CertSectionOp::create(
1476 builder, refDevice.getLoc(), builder.getStringAttr(refSymName));
1477
1478 Block *sectionBlock = new Block();
1479 sectionOp.getBody().push_back(sectionBlock);
1480
1481 // Clone all cert.page operations into the section
1482 IRMapping mapper;
1483 for (auto pageOp : certPages) {
1484 builder.setInsertionPointToEnd(sectionBlock);
1485 builder.clone(*pageOp.getOperation(), mapper);
1486 }
1487
1488 // Ensure section terminator
1489 AIEX::CertSectionOp::ensureTerminator(sectionOp.getBody(), builder,
1490 refDevice.getLoc());
1491 }
1492 }
1493
1494 // Clean up unused block arguments from cert.job operations
1495 currentDevice.walk([&](AIEX::CertJobOp jobOp) {
1496 Block &jobBlock = jobOp.getBody().front();
1497
1498 // Remove unused block arguments (in reverse order)
1499 for (int i = jobBlock.getNumArguments() - 1; i >= 0; --i) {
1500 BlockArgument arg = jobBlock.getArgument(i);
1501 if (arg.use_empty()) {
1502 jobBlock.eraseArgument(i);
1503 }
1504 }
1505 });
1506 }
1507};
1508
1509} // namespace
1510
1511// The chain a uC DMA push names, synchronous or not.
1512static StringRef pushedChain(Operation &o) {
1513 if (auto push = dyn_cast<AIEX::CertUcDmaWriteDesSyncOp>(o))
1514 return push.getSymbol();
1515 if (auto push = dyn_cast<AIEX::CertUcDmaWriteDesOp>(o))
1516 return push.getSymbol();
1517 return {};
1518}
1519
1520// Registers r0..r7 are local to a job. Splitting a job, or moving some of its
1521// ops into a new job, gives each part its own local registers, so a value
1522// defined in one part is gone in the other. r8..r23 are shared by all jobs and
1523// survive both.
1524static bool isLocalRegister(uint32_t r) { return r < 8; }
1525
1526// The local registers `o` reads (uses) and writes (defs).
1527static void collectLocalRegisterDefsUses(Operation &o,
1528 SmallVectorImpl<uint32_t> &uses,
1529 SmallVectorImpl<uint32_t> &defs) {
1530 auto add = [](SmallVectorImpl<uint32_t> &regs, uint32_t r) {
1531 if (isLocalRegister(r))
1532 regs.push_back(r);
1533 };
1534 if (auto op = dyn_cast<AIEX::CertUcDmaWriteDesOp>(o)) {
1535 add(defs, op.getWaitHandle());
1536 } else if (auto op = dyn_cast<AIEX::CertWaitUcDmaOp>(o)) {
1537 add(uses, op.getWaitHandle());
1538 } else if (auto op = dyn_cast<AIEX::CertRead32Op>(o)) {
1539 add(defs, op.getValue());
1540 } else if (auto op = dyn_cast<AIEX::CertRead32DOp>(o)) {
1541 add(uses, op.getAddress());
1542 add(defs, op.getValue());
1543 } else if (auto op = dyn_cast<AIEX::CertWrite32DOp>(o)) {
1544 if (op.getAddressIsReg())
1545 add(uses, op.getAddress());
1546 if (op.getValueIsReg())
1547 add(uses, op.getValue());
1548 }
1549}
1550
1551// A job touching any local register is never split, which keeps every
1552// definition and use of those registers in one job.
1553static bool jobUsesLocalRegister(AIEX::CertJobOp job) {
1554 SmallVector<uint32_t> uses, defs;
1555 for (Operation &o : job.getBody().front())
1556 collectLocalRegisterDefsUses(o, uses, defs);
1557 return !uses.empty() || !defs.empty();
1558}
1559
1560// The first local register `ops` reads before defining it: its value comes
1561// from earlier in the job, so moving `ops` into a new job would lose it.
1562static std::optional<uint32_t>
1563findLocalRegisterCrossingIn(ArrayRef<Operation *> ops) {
1564 llvm::SmallDenseSet<uint32_t> defined;
1565 SmallVector<uint32_t> uses, defs;
1566 for (Operation *o : ops) {
1567 uses.clear();
1568 defs.clear();
1569 collectLocalRegisterDefsUses(*o, uses, defs);
1570 for (uint32_t r : uses)
1571 if (!defined.contains(r))
1572 return r;
1573 defined.insert(defs.begin(), defs.end());
1574 }
1575 return std::nullopt;
1576}
1577
1578// AIEBU lays down each chain and each payload global once per page, however
1579// many pushes on that page name them, so data is charged once per page.
1581 llvm::DenseSet<Operation *> chains;
1582 llvm::DenseSet<Operation *> payloads;
1583};
1584
1585// A pushed chain's BDs and payloads, charged the first time the page names
1586// them.
1587static void addChainDataCost(StringRef sym_name, const SymbolTable &symbolTable,
1588 uint32_t &data_cost, PageDataSeen &seen) {
1589 auto chain = symbolTable.lookup<AIEX::CertUcDmaChainOp>(sym_name);
1590 if (!chain || !seen.chains.insert(chain).second)
1591 return;
1592 for (auto bdOp : chain.getBody().front().getOps<AIEX::CertUcDmaBdOp>()) {
1593 data_cost += 16; // bd op
1594 auto global = symbolTable.lookup<memref::GlobalOp>(bdOp.getRemoteAddress());
1595 if (!global || !seen.payloads.insert(global).second)
1596 continue;
1597 auto initVal = global.getInitialValue();
1598 if (!initVal)
1599 continue;
1600 auto data = dyn_cast<DenseIntElementsAttr>(*initVal);
1601 if (!data)
1602 continue;
1603 data_cost += data.getNumElements() * 4; // 4 bytes per element
1604 }
1605}
1606
1607// Instruction sizes follow the AIEBU CERT ISA specification so the running
1608// total here matches the bytes emitJob actually produces. Keep the cost model
1609// and emitter in sync when adding a cert op:
1610// an op the emitter can emit but this function ignores is counted as free,
1611// which makes the page look smaller than it is.
1612static void updateCostForOp(Operation &o, const SymbolTable &symbolTable,
1613 uint32_t &text_cost, uint32_t &data_cost,
1614 PageDataSeen &seen) {
1615 // Several distinct op kinds share an encoded instruction size below --
1616 // an instruction-size table, not a copy-paste clone.
1617 // NOLINTBEGIN(bugprone-branch-clone)
1618 if (isa<AIEX::CertLocalBarrierOp>(o)) {
1619 text_cost += 4; // ISA_OPSIZE_LOCAL_BARRIER
1620 } else if (isa<AIEX::CertRemoteBarrierOp>(o)) {
1621 text_cost += 8; // ISA_OPSIZE_REMOTE_BARRIER
1622 } else if (isa<AIEX::CertRelAcqSyncOp>(o)) {
1623 text_cost += 12; // encoded REL_ACQ_SYNC instruction size
1624 } else if (isa<AIEX::CertWaitTCTSOp>(o)) {
1625 text_cost += 8; // wait tct
1626 } else if (isa<AIEX::CertMaskWrite32Op, AIEX::CertMaskPoll32Op>(o)) {
1627 text_cost += 16; // mask write / poll
1628 } else if (isa<AIEX::CertWrite32Op>(o)) {
1629 text_cost += 12; // ISA_OPSIZE_WRITE_32
1630 } else if (isa<AIEX::CertApplyOffset57Op>(o)) {
1631 text_cost += 8; // ISA_OPSIZE_APPLY_OFFSET_57
1632 } else if (isa<AIEX::CertNopOp>(o)) {
1633 text_cost += 4; // ISA_OPSIZE_NOP
1634 } else if (isa<AIEX::CertPreemptOp>(o)) {
1635 text_cost += 8; // ISA_OPSIZE_PREEMPT
1636 } else if (isa<AIEX::CertLoadPdiOp>(o)) {
1637 text_cost += 12; // ISA_OPSIZE_LOAD_PDI
1638 // NOLINTEND(bugprone-branch-clone)
1639 } else if (auto syncOp = dyn_cast<AIEX::CertUcDmaWriteDesSyncOp>(o)) {
1640 text_cost += 4; // ISA_OPSIZE_UC_DMA_WRITE_DES_SYNC
1641 addChainDataCost(syncOp.getSymbol(), symbolTable, data_cost, seen);
1642 } else if (auto desOp = dyn_cast<AIEX::CertUcDmaWriteDesOp>(o)) {
1643 text_cost += 8; // ISA_OPSIZE_UC_DMA_WRITE_DES
1644 addChainDataCost(desOp.getSymbol(), symbolTable, data_cost, seen);
1645 } else if (isa<AIEX::CertWaitUcDmaOp>(o)) {
1646 text_cost += 4; // ISA_OPSIZE_WAIT_UC_DMA
1647 } else if (isa<AIEX::CertRead32Op>(o)) {
1648 text_cost += 8; // ISA_OPSIZE_READ_32
1649 } else if (isa<AIEX::CertRead32DOp>(o)) {
1650 text_cost += 4; // ISA_OPSIZE_READ_32_D
1651 } else if (isa<AIEX::CertWrite32DOp>(o)) {
1652 text_cost += 12; // ISA_OPSIZE_WRITE_32_D
1653 } else if (isa<AIEX::CertSaveTimestampsOp>(o)) {
1654 text_cost += 8; // ISA_OPSIZE_SAVE_TIMESTAMPS
1655 } else if (isa<AIEX::CertSaveRegisterOp>(o)) {
1656 text_cost += 12; // ISA_OPSIZE_SAVE_REGISTER
1657 }
1658}
1659
1660static uint32_t estimateCost(AIEX::CertPageOp op,
1661 const SymbolTable &symbolTable,
1662 uint32_t split_target, AIEX::CertJobOp &split_job,
1663 Block::iterator &split_iter,
1664 bool &found_split_point) {
1665 uint32_t text_cost = 32; // page header: `.align 16` padding plus `.eop`
1666 uint32_t data_cost = 0;
1667 PageDataSeen seen;
1668 found_split_point = false;
1669
1670 for (auto job : op.getBody().front().getOps<AIEX::CertJobOp>()) {
1671 // Cuts inside a job that uses local registers would lose their values.
1672 bool splittable = !jobUsesLocalRegister(job);
1673 // START_JOB precedes the body, so it is already spent at every candidate
1674 // split point inside this job; END_JOB is charged after the body below.
1675 text_cost += cert_job_start_cost;
1676 for (auto &o : job.getBody().front().getOperations()) {
1677 Block::iterator current(&o);
1678 if (splittable && !found_split_point && !isa<AIE::EndOp>(o) &&
1679 current != job.getBody().front().begin() &&
1680 (text_cost + data_cost) >= split_target) {
1681 split_job = job;
1682 split_iter = current;
1683 found_split_point = true;
1684 }
1685
1686 updateCostForOp(o, symbolTable, text_cost, data_cost, seen);
1687
1688 if (splittable && !found_split_point &&
1689 (text_cost + data_cost) >= split_target) {
1690 Block::iterator next = current;
1691 ++next;
1692 if (next != job.getBody().front().end() && !isa<AIE::EndOp>(*next)) {
1693 split_job = job;
1694 split_iter = next;
1695 found_split_point = true;
1696 } else if (!isa<AIE::EndOp>(o) &&
1697 current != job.getBody().front().begin()) {
1698 // Cutting after the last real op would leave an empty second half.
1699 // Cut before it; exclude the terminator to guarantee progress.
1700 split_job = job;
1701 split_iter = current;
1702 found_split_point = true;
1703 }
1704 }
1705 }
1706 text_cost += cert_job_end_cost;
1707 }
1708 return text_cost + data_cost;
1709}
1710
1711// A cut before flat index p separates a span [lo, hi] when lo < p <= hi.
1712// Local barrier participants must share a page (see
1713// checkLocalBarrierColocation). A payload patch must share a page with every
1714// later uC-DMA push of that payload: each page carries its own payload copy,
1715// so a push on another page would use the unpatched addresses.
1717
1722
1723// Index non-terminator ops in textual order and record indivisible spans.
1724static void computeIndivisibleSpans(AIEX::CertPageOp page,
1725 const SymbolTable &symbolTable,
1726 std::map<Operation *, int> &flatIndex,
1727 SmallVectorImpl<IndivisibleSpan> &spans) {
1728 std::map<int, std::pair<int, int>> barriers;
1729 // Patches accumulate and survive pushes, so later pushes depend on the
1730 // earliest patch of their payload.
1731 llvm::StringMap<int> firstPatch;
1732 auto payloadsOf = [&](StringRef chainSym) {
1733 SmallVector<StringRef> payloads;
1734 auto chain = symbolTable.lookup<AIEX::CertUcDmaChainOp>(chainSym);
1735 if (!chain || chain.getBody().empty())
1736 return payloads;
1737 for (auto bd : chain.getBody().front().getOps<AIEX::CertUcDmaBdOp>())
1738 payloads.push_back(bd.getRemoteAddress());
1739 return payloads;
1740 };
1741
1742 int idx = 0;
1743 for (auto job : page.getBody().front().getOps<AIEX::CertJobOp>()) {
1744 for (auto &o : job.getBody().front().getOperations()) {
1745 if (isa<AIE::EndOp>(o))
1746 continue;
1747 flatIndex[&o] = idx;
1748 if (auto bar = dyn_cast<AIEX::CertLocalBarrierOp>(o)) {
1749 int id = static_cast<int>(bar.getLocalBarrierId());
1750 auto it = barriers.find(id);
1751 if (it == barriers.end())
1752 barriers[id] = {idx, idx};
1753 else {
1754 it->second.first = std::min(it->second.first, idx);
1755 it->second.second = std::max(it->second.second, idx);
1756 }
1757 } else if (auto patch = dyn_cast<AIEX::CertApplyOffset57Op>(o)) {
1758 firstPatch.try_emplace(patch.getSymbol(), idx);
1759 } else if (StringRef pushed = pushedChain(o); !pushed.empty()) {
1760 for (StringRef payload : payloadsOf(pushed)) {
1761 auto it = firstPatch.find(payload);
1762 if (it != firstPatch.end())
1763 spans.push_back({it->second, idx, SpanKind::PatchedPayload});
1764 }
1765 }
1766 ++idx;
1767 }
1768 }
1769 for (auto &[id, span] : barriers)
1770 spans.push_back({span.first, span.second, SpanKind::LocalBarrier});
1771}
1772
1773// Return whether the cut separates a span, optionally reporting its kind.
1774static bool cutSeparatesSpan(ArrayRef<IndivisibleSpan> spans, int p,
1775 SpanKind *kind = nullptr) {
1776 const auto *it = llvm::find_if(
1777 spans, [p](const IndivisibleSpan &s) { return s.lo < p && s.hi >= p; });
1778 if (it == spans.end())
1779 return false;
1780 if (kind)
1781 *kind = it->kind;
1782 return true;
1783}
1784
1785// Choose the span-safe cut closest to split_target, allowing both interior
1786// and job-boundary cuts. Each half needs content so repeated splitting
1787// progresses.
1788static bool findLegalSplitPoint(AIEX::CertPageOp page,
1789 const SymbolTable &symbolTable,
1790 uint32_t split_target,
1791 const std::map<Operation *, int> &flatIndex,
1792 ArrayRef<IndivisibleSpan> spans,
1793 AIEX::CertJobOp &split_job,
1794 Block::iterator &split_iter) {
1795 uint32_t text_cost = 32; // page header, mirrors estimateCost
1796 uint32_t data_cost = 0;
1797 PageDataSeen seen;
1798 bool found = false;
1799 uint32_t best_delta = 0;
1800 // Whether some job already visited holds a real op, i.e. whether cutting at
1801 // the current job's first op would leave content on the earlier page.
1802 bool sawOpInEarlierJob = false;
1803 for (auto job : page.getBody().front().getOps<AIEX::CertJobOp>()) {
1804 text_cost += cert_job_start_cost; // mirrors estimateCost
1805 bool splittable = !jobUsesLocalRegister(job); // mirrors estimateCost
1806 bool sawOpInJob = false;
1807 for (auto &o : job.getBody().front().getOperations()) {
1808 if (isa<AIE::EndOp>(o))
1809 continue;
1810 Block::iterator current(&o);
1811 // Candidate: cut before `o`, legal as an interior cut when an op of this
1812 // job precedes it, or as a job-boundary cut when an earlier job does.
1813 // Admitting the latter matters for a page built of small jobs whose sizes
1814 // only sum over the limit: no job has an interior point worth cutting at,
1815 // yet a cut between two whole jobs splits it perfectly well.
1816 if (sawOpInJob ? splittable : sawOpInEarlierJob) {
1817 auto fi = flatIndex.find(&o);
1818 int p = fi == flatIndex.end() ? 0 : fi->second;
1819 if (!cutSeparatesSpan(spans, p)) {
1820 uint32_t cost = text_cost + data_cost;
1821 uint32_t delta =
1822 cost > split_target ? cost - split_target : split_target - cost;
1823 if (!found || delta < best_delta) {
1824 found = true;
1825 best_delta = delta;
1826 split_job = job;
1827 split_iter = current;
1828 }
1829 }
1830 }
1831 updateCostForOp(o, symbolTable, text_cost, data_cost, seen);
1832 sawOpInJob = true;
1833 }
1834 text_cost += cert_job_end_cost; // mirrors estimateCost
1835 sawOpInEarlierJob |= sawOpInJob;
1836 }
1837 return found;
1838}
1839
1840namespace {
1841
1842// Pattern to isolate load_pdi and preempt operations into their own job and
1843// page According to CERT spec: "load_pdi and preempt should take one whole job
1844// which in turn should take one whole page"
1845struct IsolateFullPageOpsPattern : OpRewritePattern<AIEX::CertJobOp> {
1846 // `hadError` is raised with an error diagnostic, as in
1847 // SplitCertPageOpPattern.
1848 IsolateFullPageOpsPattern(MLIRContext *context, bool *hadError)
1849 : OpRewritePattern(context), hadError(hadError) {}
1850
1851 // Create an empty, terminated cert.page directly after `after`, in the same
1852 // block. Staying in `after`'s block is what keeps a page that was moved into
1853 // a cert.attach_to_group by placement lowering on its uC.
1854 static AIEX::CertPageOp makePageAfter(PatternRewriter &rewriter,
1855 Operation *after, Location loc) {
1856 rewriter.setInsertionPointAfter(after);
1857 auto page = AIEX::CertPageOp::create(rewriter, loc);
1858 page.getBody().push_back(new Block());
1859 AIEX::CertPageOp::ensureTerminator(page.getBody(), rewriter, loc);
1860 return page;
1861 }
1862
1863 LogicalResult matchAndRewrite(AIEX::CertJobOp jobOp,
1864 PatternRewriter &rewriter) const override {
1865
1866 // Find first load_pdi or preempt in this job
1867 Operation *fullPageOp = nullptr;
1868 Block::iterator fullPageOpIter;
1869 for (Block::iterator it = jobOp.getBody().front().begin();
1870 it != jobOp.getBody().front().end(); ++it) {
1871 if (isa<AIEX::CertLoadPdiOp, AIEX::CertPreemptOp>(*it)) {
1872 fullPageOp = &*it;
1873 fullPageOpIter = it;
1874 break;
1875 }
1876 }
1877
1878 if (!fullPageOp)
1879 return failure(); // No full-page op in this job
1880
1881 auto parentPage = jobOp->getParentOfType<AIEX::CertPageOp>();
1882 if (!parentPage)
1883 return failure(); // No parent page (unusual but skip)
1884
1885 // Real ops in this job (the terminator does not count).
1886 size_t opCount = 0;
1887 for (Operation &op : jobOp.getBody().front().getOperations())
1888 if (!isa<AIE::EndOp>(op))
1889 opCount++;
1890
1891 // Pages execute in order, so the rewrite has to preserve this job's
1892 // position among its *siblings*, not just relative to the parent page:
1893 // anchoring every new page on the parent page's boundary hoists the jobs
1894 // that surround this one across the isolated load_pdi/preempt. Nothing
1895 // downstream checks op order, so that reordering is silent.
1896 SmallVector<Operation *> leadingSiblings, trailingSiblings;
1897 bool pastSelf = false;
1898 for (Operation &op : parentPage.getBody().front()) {
1899 if (isa<AIE::EndOp>(op))
1900 continue;
1901 if (&op == jobOp.getOperation()) {
1902 pastSelf = true;
1903 continue;
1904 }
1905 (pastSelf ? trailingSiblings : leadingSiblings).push_back(&op);
1906 }
1907
1908 // Already properly isolated: sole op of the sole job of its page.
1909 if (opCount == 1 && leadingSiblings.empty() && trailingSiblings.empty())
1910 return failure();
1911
1912 auto loc = jobOp.getLoc();
1913
1914 // The parent page stays where it is and keeps the leading siblings; every
1915 // page created below is chained after it, in program order.
1916 Operation *anchor = parentPage.getOperation();
1917
1918 if (opCount == 1) {
1919 // The job is exactly the full-page op: it just needs a page of its own.
1920 auto isolatedPage = makePageAfter(rewriter, anchor, loc);
1921 jobOp->moveBefore(isolatedPage.getBody().front().getTerminator());
1922 anchor = isolatedPage;
1923 } else {
1924 // Mixed job: split it into up to three jobs, each on its own page, in
1925 // source order (ops before the full-page op, it, then the ops after).
1926 auto parentDevice = jobOp->getParentOfType<AIE::DeviceOp>();
1927 uint32_t maxJobId = 0;
1928 parentDevice.walk([&](AIEX::CertJobOp certJobOp) {
1929 maxJobId = std::max(maxJobId, certJobOp.getJobId());
1930 });
1931 uint32_t beforeJobId = jobOp.getJobId();
1932 uint32_t fullPageJobId = maxJobId + 1;
1933 uint32_t afterJobId = maxJobId + 2;
1934
1935 SmallVector<Operation *> beforeOps, afterOps;
1936 for (Block::iterator it = jobOp.getBody().front().begin();
1937 it != fullPageOpIter; ++it)
1938 if (!isa<AIE::EndOp>(*it))
1939 beforeOps.push_back(&*it);
1940 Block::iterator afterStart = std::next(fullPageOpIter);
1941 for (Block::iterator it = afterStart; it != jobOp.getBody().front().end();
1942 ++it)
1943 if (!isa<AIE::EndOp>(*it))
1944 afterOps.push_back(&*it);
1945
1946 // Isolation is mandatory, so a local register written before the
1947 // full-page op and read after it cannot be kept; reject the job.
1948 if (auto crossing = findLocalRegisterCrossingIn(afterOps)) {
1949 StringRef name = fullPageOp->getName().getStringRef();
1950 *hadError = true;
1951 return jobOp.emitError()
1952 << "job " << jobOp.getJobId() << " must be split around its "
1953 << name << ", but local register r" << *crossing
1954 << " is written before it and read after it; keep that "
1955 "register's uses on one side of the "
1956 << name << ", or put them in a separate job";
1957 }
1958
1959 auto emitJobPage = [&](uint32_t jobId, ArrayRef<Operation *> ops) {
1960 auto page = makePageAfter(rewriter, anchor, loc);
1961 rewriter.setInsertionPoint(page.getBody().front().getTerminator());
1962 auto job = AIEX::CertJobOp::create(rewriter, loc, jobId);
1963 job.getBody().push_back(new Block());
1964 AIEX::CertJobOp::ensureTerminator(job.getBody(), rewriter, loc);
1965 for (Operation *op : ops)
1966 op->moveBefore(job.getBody().front().getTerminator());
1967 anchor = page;
1968 };
1969
1970 if (!beforeOps.empty())
1971 emitJobPage(beforeJobId, beforeOps);
1972 emitJobPage(fullPageJobId, {fullPageOp});
1973 if (!afterOps.empty())
1974 emitJobPage(afterJobId, afterOps);
1975
1976 rewriter.eraseOp(jobOp);
1977 }
1978
1979 // The trailing siblings cannot stay on the parent page, which now precedes
1980 // the isolated page, so they move to a page of their own behind it.
1981 if (!trailingSiblings.empty()) {
1982 auto tailPage = makePageAfter(rewriter, anchor, loc);
1983 for (Operation *op : trailingSiblings)
1984 op->moveBefore(tailPage.getBody().front().getTerminator());
1985 }
1986
1987 if (llvm::all_of(parentPage.getBody().front(),
1988 [](Operation &op) { return isa<AIE::EndOp>(op); }))
1989 rewriter.eraseOp(parentPage);
1990
1991 return success();
1992 }
1993
1994 bool *hadError;
1995};
1996
1997struct SplitCertPageOpPattern : OpRewritePattern<AIEX::CertPageOp> {
1998 // `hadError` is raised when the pattern emits an error diagnostic. A rewrite
1999 // pattern returning failure() is ordinary control flow -- it does NOT fail
2000 // the pass -- so without this the "cannot split" error below would print and
2001 // the oversized page would still reach codegen. The pass checks the flag
2002 // after applyPatternsGreedily and calls signalPassFailure().
2003 // symbolTable is the device's; splitting only clones jobs, so it stays
2004 // current.
2005 SplitCertPageOpPattern(MLIRContext *context, const SymbolTable &symbolTable,
2006 bool *hadError)
2007 : OpRewritePattern(context), symbolTable(symbolTable),
2008 hadError(hadError) {}
2009
2010 LogicalResult matchAndRewrite(AIEX::CertPageOp op,
2011 PatternRewriter &rewriter) const override {
2012
2013 constexpr uint32_t split_threshold = cert_page_size;
2014
2015 AIEX::CertJobOp split_job;
2016 Block::iterator split_iter;
2017 bool found_split_point = false;
2018 uint32_t cost = estimateCost(op, symbolTable, cert_page_size / 2, split_job,
2019 split_iter, found_split_point);
2020 LLVM_DEBUG(llvm::outs()
2021 << "Estimate cost for page: " << " is " << cost << "\n");
2022
2023 if (cost < split_threshold)
2024 return failure();
2025
2026 bool isImplicit = op->hasAttr("cert.implicit");
2027
2028 std::map<Operation *, int> flatIndex;
2029 SmallVector<IndivisibleSpan> spans;
2030 computeIndivisibleSpans(op, symbolTable, flatIndex, spans);
2031
2032 // The initial estimate can miss a job-boundary cut or an earlier candidate;
2033 // search all positions before declaring the page unsplittable. Spans are
2034 // not applied here: a page whose cuts are all blocked by a span has split
2035 // points, and the span check below reports which constraint blocks them.
2036 if (!found_split_point)
2037 found_split_point =
2038 findLegalSplitPoint(op, symbolTable, cert_page_size / 2, flatIndex,
2039 {}, split_job, split_iter);
2040
2041 // Over the split trigger but with nowhere to cut. Every cut has to leave at
2042 // least one op on each side, so a page whose bulk sits in a single one-op
2043 // job offers none. Leave the page alone -- but if it is also over the hard
2044 // limit, it will not load, so say so instead of emitting a page the
2045 // firmware will reject.
2046 if (!found_split_point) {
2047 if (cost > cert_page_limit) {
2048 auto jobs = op.getBody().front().getOps<AIEX::CertJobOp>();
2049 auto blocker = llvm::find_if(jobs, jobUsesLocalRegister);
2050 if (blocker != jobs.end())
2051 op.emitError() << "cert.page is an estimated " << cost
2052 << " bytes, over the " << cert_page_limit
2053 << "-byte microcontroller page limit, and cannot be "
2054 "split: job "
2055 << (*blocker).getJobId()
2056 << " uses local registers r0..r7, which do not "
2057 "survive a split; break it into smaller jobs "
2058 "that each keep a register's definition and "
2059 "uses together";
2060 else
2061 op.emitError() << "cert.page is an estimated " << cost
2062 << " bytes, over the " << cert_page_limit
2063 << "-byte microcontroller page limit, and offers no "
2064 "split point (a split must leave at least one op "
2065 "on each page); break the oversized job into "
2066 "smaller jobs";
2067 *hadError = true;
2068 }
2069 return failure();
2070 }
2071
2072 // Never split a local_barrier participant set, or a payload patch from a
2073 // later push of that payload, across an .eop. If the cost-chosen split
2074 // point would, retarget to a legal split point; if none exists the page
2075 // cannot be made legal, so fail the compile rather than emit something
2076 // broken. An implicit page (formed by formImplicitPages) groups jobs meant
2077 // to be cooperatively scheduled together; an explicit page is a
2078 // user-authored boundary. This governs the wording of the diagnostics
2079 // below (the remedy differs), not whether they fire.
2080 if (!spans.empty()) {
2081 auto fi = flatIndex.find(&*split_iter);
2082 int p = fi == flatIndex.end() ? 0 : fi->second;
2084 if (cutSeparatesSpan(spans, p, &kind)) {
2085 if (!findLegalSplitPoint(op, symbolTable, cert_page_size / 2, flatIndex,
2086 spans, split_job, split_iter)) {
2087 // Leave the page whole: a severed span hangs the firmware or pushes
2088 // stale addresses, so an oversized page is the better of the two.
2089 // That is only survivable while the page still fits; past
2090 // cert_page_limit, diagnose and raise `hadError` so the pass fails
2091 // rather than only this rewrite.
2092 if (cost > cert_page_limit) {
2093 bool barrier = kind == SpanKind::LocalBarrier;
2094 StringRef what = barrier
2095 ? "a local_barrier participant set"
2096 : "a cert.apply_offset_57 from a uC-DMA push "
2097 "of the payload it patches";
2098 if (isImplicit)
2099 op.emitError()
2100 << "implicit cert.page is an estimated " << cost
2101 << " bytes, over the " << cert_page_limit
2102 << "-byte microcontroller page limit, and cannot be split "
2103 "without separating "
2104 << what
2105 << " across pages; insert an explicit cert.page boundary to "
2106 << (barrier ? "co-locate the barrier participants"
2107 : "co-locate each apply_offset_57 with its "
2108 "pushes");
2109 else
2110 op.emitError()
2111 << "cert.page is an estimated " << cost << " bytes, over the "
2112 << cert_page_limit
2113 << "-byte microcontroller page limit, and cannot be split "
2114 "without separating "
2115 << what << " across pages; reduce the page's contents or "
2116 << (barrier ? "regroup its local_barrier participants"
2117 : "regroup its apply_offset_57 ops and their "
2118 "pushes")
2119 << " so that a legal split point exists";
2120 *hadError = true;
2121 }
2122 return failure();
2123 }
2124 }
2125 }
2126
2127 // splitting an implicit page turns jobs that were meant to run
2128 // cooperatively on one page into strictly sequential pages. Warn so the
2129 // user can insert an explicit boundary if the serialization is unintended.
2130 if (isImplicit)
2131 op.emitRemark()
2132 << "auto-split of implicit cert.page serializes cooperatively-"
2133 "scheduled jobs into separate sequential pages; insert an "
2134 "explicit "
2135 "cert.page boundary to control where the split occurs";
2136
2137 auto loc = op.getLoc();
2138
2139 // Preserve source order across both pages so cross-page dependencies stay
2140 // backward. A job-boundary cut moves split_job whole onto the later page.
2141 bool jobBoundaryCut = split_iter == split_job.getBody().front().begin();
2142 [[maybe_unused]] auto earlierJobHasRealOp = [&]() {
2143 for (auto job : op.getBody().front().getOps<AIEX::CertJobOp>()) {
2144 if (job == split_job)
2145 break;
2146 for (auto &o : job.getBody().front().getOperations())
2147 if (!isa<AIE::EndOp>(o))
2148 return true;
2149 }
2150 return false;
2151 };
2152 assert((!jobBoundaryCut || earlierJobHasRealOp()) &&
2153 "a job-boundary cut needs a preceding job with content, else the "
2154 "earlier page is empty and the split makes no progress");
2155
2156 // The second half of a split job takes the next ID, which may collide
2157 // with another job on its page; allocateJobIds resolves that after all
2158 // splitting is done.
2159 auto cloneJobRange = [&](AIEX::CertJobOp sourceJob, uint32_t jobId,
2160 Block::iterator begin, Block::iterator end) {
2161 auto newJobOp = AIEX::CertJobOp::create(rewriter, loc, jobId);
2162 Block *newJobBlock = new Block();
2163 newJobOp.getBody().push_back(newJobBlock);
2164
2165 IRMapping mapper;
2166 Block &sourceBlock = sourceJob.getBody().front();
2167 for (BlockArgument arg : sourceBlock.getArguments()) {
2168 BlockArgument newArg =
2169 newJobBlock->addArgument(arg.getType(), arg.getLoc());
2170 mapper.map(arg, newArg);
2171 }
2172
2173 rewriter.setInsertionPointToStart(newJobBlock);
2174 for (Block::iterator oi = begin; oi != end; ++oi) {
2175 if (!isa<AIE::EndOp>(*oi))
2176 rewriter.clone(*oi, mapper);
2177 }
2178 AIEX::CertJobOp::ensureTerminator(newJobOp.getBody(), rewriter, loc);
2179 };
2180
2181 rewriter.setInsertionPoint(op);
2182 auto newPageOp0 = AIEX::CertPageOp::create(rewriter, loc);
2183 Block *newPageBlock0 = new Block();
2184 newPageOp0.getBody().push_back(newPageBlock0);
2185
2186 auto newPageOp1 = AIEX::CertPageOp::create(rewriter, loc);
2187 Block *newPageBlock1 = new Block();
2188 newPageOp1.getBody().push_back(newPageBlock1);
2189
2190 // propagate the implicit tag so a later re-split of either fragment
2191 // still knows it came from an implicit page and diagnoses consistently.
2192 if (isImplicit) {
2193 newPageOp0->setAttr("cert.implicit", rewriter.getUnitAttr());
2194 newPageOp1->setAttr("cert.implicit", rewriter.getUnitAttr());
2195 }
2196
2197 for (auto job : op.getBody().front().getOps<AIEX::CertJobOp>()) {
2198 if (job == split_job) {
2199 if (jobBoundaryCut) {
2200 // Whole job to the later page: partitioning it at its own first op
2201 // would leave an empty half behind on the earlier page.
2202 rewriter.setInsertionPointToEnd(newPageBlock1);
2203 cloneJobRange(job, job.getJobId(), job.getBody().front().begin(),
2204 job.getBody().front().end());
2205 continue;
2206 }
2207 rewriter.setInsertionPointToEnd(newPageBlock0);
2208 cloneJobRange(job, job.getJobId(), job.getBody().front().begin(),
2209 split_iter);
2210
2211 rewriter.setInsertionPointToEnd(newPageBlock1);
2212 cloneJobRange(job, job.getJobId() + 1, split_iter,
2213 job.getBody().front().end());
2214 continue;
2215 }
2216
2217 if (job->isBeforeInBlock(split_job)) {
2218 rewriter.setInsertionPointToEnd(newPageBlock0);
2219 cloneJobRange(job, job.getJobId(), job.getBody().front().begin(),
2220 job.getBody().front().end());
2221 } else {
2222 rewriter.setInsertionPointToEnd(newPageBlock1);
2223 cloneJobRange(job, job.getJobId(), job.getBody().front().begin(),
2224 job.getBody().front().end());
2225 }
2226 }
2227
2228 AIEX::CertPageOp::ensureTerminator(newPageOp0.getBody(), rewriter, loc);
2229 AIEX::CertPageOp::ensureTerminator(newPageOp1.getBody(), rewriter, loc);
2230
2231 rewriter.eraseOp(op);
2232 return success();
2233 }
2234
2235 const SymbolTable &symbolTable;
2236 bool *hadError;
2237};
2238
2239// Explicit group 0 shares the default stream. Flatten it before page formation
2240// so it does not introduce an artificial scheduling boundary.
2241static void inlineDefaultGroup(AIE::DeviceOp dev) {
2242 Block *body = dev.getBody();
2243 SmallVector<AIEX::CertAttachToGroupOp> zeroGroups;
2244 for (auto grp : body->getOps<AIEX::CertAttachToGroupOp>())
2245 if (grp.getGroupId() == 0)
2246 zeroGroups.push_back(grp);
2247 for (auto grp : zeroGroups) {
2248 SmallVector<Operation *> inner;
2249 for (Operation &op : grp.getBody().front())
2250 if (!isa<AIE::EndOp>(op))
2251 inner.push_back(&op);
2252 for (Operation *op : inner)
2253 op->moveBefore(grp.getOperation());
2254 grp.erase();
2255 }
2256}
2257
2258// Group a block's maximal runs of contiguous top-level cert.job ops into
2259// implicit cert.page ops (only an explicit cert.page delimits; the
2260// {cert.configure} job stays isolated in its own page). Applied per uC stream.
2261static void formImplicitPagesInBlock(Block *body, OpBuilder &builder) {
2262 // Snapshot top-level ops; the body is mutated while iterating.
2263 SmallVector<Operation *> topOps;
2264 for (Operation &op : *body)
2265 topOps.push_back(&op);
2266
2267 // `cert.implicit` marks a page formed from a run of *user* jobs -- a
2268 // boundary the author could have drawn but didn't. The splitter's diagnostics
2269 // are keyed off it and all advise editing page boundaries, so it must not be
2270 // set on a page the user cannot restructure. The compiler-synthesized
2271 // @configure page is exactly that case, and it is isolated by construction,
2272 // so the tag would buy nothing there anyway.
2273 auto openPage = [&](Location loc, Operation *before,
2274 bool implicit) -> AIEX::CertPageOp {
2275 builder.setInsertionPoint(before);
2276 auto page = AIEX::CertPageOp::create(builder, loc);
2277 if (implicit)
2278 page->setAttr("cert.implicit", builder.getUnitAttr());
2279 Block *b = new Block();
2280 page.getBody().push_back(b);
2281 AIEX::CertPageOp::ensureTerminator(page.getBody(), builder, loc);
2282 return page;
2283 };
2284
2285 // The (tile, channel) actors a job wait_tcts on, keyed exactly as
2286 // AIECertVerify's G-tct check (checkWaitTctsUniqueness) keys them so the two
2287 // stay consistent.
2288 auto waitTctsActors = [](AIEX::CertJobOp job) {
2289 std::set<std::pair<int, int>> actors;
2290 job.walk([&](AIEX::CertWaitTCTSOp wait) {
2291 actors.insert({wait.getTileId(), wait.getChannelId()});
2292 });
2293 return actors;
2294 };
2295
2296 AIEX::CertPageOp currentPage; // null => no open implicit page
2297 // Actors already waited on by the jobs accumulated into `currentPage`.
2298 std::set<std::pair<int, int>> pageActors;
2299 auto closePage = [&]() {
2300 currentPage = nullptr;
2301 pageActors.clear();
2302 };
2303
2304 for (Operation *op : topOps) {
2305 if (auto job = dyn_cast<AIEX::CertJobOp>(op)) {
2306 if (job->hasAttr("cert.configure")) {
2307 // Keep the configuration job isolated in its own page. Not tagged
2308 // implicit: the user did not author this job and cannot add a boundary
2309 // inside it, so the auto-split remark would be noise on every compile
2310 // whose
2311 // config transaction is large enough to split (and it usually is).
2312 auto page = openPage(job.getLoc(), job, /*implicit=*/false);
2313 job->moveBefore(page.getBody().front().getTerminator());
2314 closePage();
2315 continue;
2316 }
2317 // G-tct: at most one job per page may wait on a given actor. Two jobs
2318 // that each legally wait on the same (tile, channel) -- an ordinary shape
2319 // for two runtime sequences sharing a shim channel -- are only legal
2320 // while they sit on separate pages, so grouping them would manufacture a
2321 // verifier error out of valid input. Cut a page boundary instead, exactly
2322 // as an over-size job would.
2323 auto actors = waitTctsActors(job);
2324 if (currentPage &&
2325 llvm::any_of(actors, [&](const std::pair<int, int> &actor) {
2326 return pageActors.count(actor);
2327 }))
2328 closePage();
2329 if (!currentPage)
2330 currentPage = openPage(job.getLoc(), job, /*implicit=*/true);
2331 job->moveBefore(currentPage.getBody().front().getTerminator());
2332 pageActors.insert(actors.begin(), actors.end());
2333 } else if (isa<AIEX::CertPageOp>(op)) {
2334 // An explicit page delimits the current implicit run.
2335 closePage();
2336 }
2337 // All other ops (structural / control containers) do not delimit a run.
2338 }
2339}
2340
2341static void formImplicitPages(AIE::DeviceOp dev) {
2342 OpBuilder builder(dev.getContext());
2343 formImplicitPagesInBlock(dev.getBody(), builder);
2344 // Each other uC (attach_to_group) forms implicit pages within its own stream,
2345 // so adjacent jobs on that uC are cooperatively co-located too.
2346 SmallVector<AIEX::CertAttachToGroupOp> groups;
2347 for (auto grp : dev.getBody()->getOps<AIEX::CertAttachToGroupOp>())
2348 groups.push_back(grp);
2349 for (auto grp : groups)
2350 formImplicitPagesInBlock(&grp.getBody().front(), builder);
2351}
2352
2353// Force the implicit configuration page to be the first page on uC0:
2354// @configure sets up the device (DMA/lock/core config), so it must strictly
2355// precede all other uC0 pages regardless of where it appeared textually. The
2356// config job is already isolated in its own page by formImplicitPages; here we
2357// just hoist that page to the front of the device body (structural ops are not
2358// control code, so position relative to them does not matter).
2359static void moveConfigPageFirst(AIE::DeviceOp dev) {
2360 Block *body = dev.getBody();
2361 AIEX::CertPageOp configPage;
2362 for (auto page : body->getOps<AIEX::CertPageOp>()) {
2363 for (auto job : page.getBody().front().getOps<AIEX::CertJobOp>()) {
2364 if (job->hasAttr("cert.configure")) {
2365 configPage = page;
2366 break;
2367 }
2368 }
2369 if (configPage)
2370 break;
2371 }
2372 if (configPage && &body->front() != configPage.getOperation())
2373 configPage->moveBefore(&body->front());
2374}
2375
2376// Move nonzero placements into their controller containers, retaining order.
2377static void lowerPagePlacementToGroups(AIE::DeviceOp dev) {
2378 Block *body = dev.getBody();
2379
2380 // Collect placed top-level pages by resolved group id, in first-seen order.
2381 std::map<int32_t, SmallVector<AIEX::CertPageOp, 4>> byGroup;
2382 for (auto page : body->getOps<AIEX::CertPageOp>()) {
2383 if (auto placement = page.getPlacementAttr()) {
2384 auto gid = static_cast<int32_t>(placement.getInt());
2385 if (gid != 0)
2386 byGroup[gid].push_back(page);
2387 }
2388 }
2389
2390 OpBuilder builder(dev.getContext());
2391 for (auto &entry : byGroup) {
2392 int32_t gid = entry.first;
2393 SmallVector<AIEX::CertPageOp, 4> &pages = entry.second;
2394
2395 // Create the group op where the first placed page currently lives (avoids
2396 // assuming anything about a device terminator).
2397 builder.setInsertionPoint(pages.front());
2398 auto grp = AIEX::CertAttachToGroupOp::create(builder, dev.getLoc(), gid);
2399 Block *grpBlock = new Block();
2400 grp.getBody().push_back(grpBlock);
2401
2402 // Move the placed pages into the group, in order, dropping the
2403 // now-redundant placement attribute (the group encodes it).
2404 for (AIEX::CertPageOp page : pages) {
2405 page->removeAttr("placement");
2406 page->moveBefore(grpBlock, grpBlock->end());
2407 }
2408 AIEX::CertAttachToGroupOp::ensureTerminator(grp.getBody(), builder,
2409 dev.getLoc());
2410 }
2411}
2412
2413// The firmware indexes job contexts by job_id % 64, and a page's jobs finish
2414// before the next page starts. Keep each page's noncolliding IDs and move the
2415// colliding ones to free slots.
2416static LogicalResult allocateJobIds(AIE::DeviceOp device) {
2417 WalkResult result = device.walk([&](AIEX::CertPageOp page) {
2418 std::bitset<AIEX::CertJobOp::NumSlots> used;
2419 SmallVector<AIEX::CertJobOp> collisions;
2420 unsigned count = 0;
2421 for (auto job : page.getBody().front().getOps<AIEX::CertJobOp>()) {
2422 unsigned slot = job.getJobId() % AIEX::CertJobOp::NumSlots;
2423 if (used.test(slot))
2424 collisions.push_back(job);
2425 used.set(slot);
2426 ++count;
2427 }
2428 if (count > AIEX::CertJobOp::NumSlots) {
2429 page.emitError() << "cert.page has " << count << " jobs, exceeding the "
2430 << AIEX::CertJobOp::NumSlots
2431 << " firmware job slots; reduce the jobs on this page";
2432 return WalkResult::interrupt();
2433 }
2434 unsigned slot = 0;
2435 for (auto job : collisions) {
2436 while (used.test(slot))
2437 ++slot;
2438 job.setJobId(slot);
2439 used.set(slot);
2440 }
2441 return WalkResult::advance();
2442 });
2443 return failure(result.wasInterrupted());
2444}
2445
2446struct AIECertPagesPass
2447 : xilinx::AIEX::impl::AIECertPagesBase<AIECertPagesPass> {
2448 void runOnOperation() override {
2449 inlineDefaultGroup(getOperation());
2450
2451 formImplicitPages(getOperation());
2452
2453 // Force the @configure page to be first on uC0.
2454 moveConfigPageFirst(getOperation());
2455
2456 // Lower cert.page placement to cert.attach_to_group BEFORE isolate
2457 // and split. Those rewrites create replacement pages adjacent to the
2458 // original (same parent block) with the no-placement builder; running the
2459 // placement grouping first means placed pages already live inside their
2460 // cert.attach_to_group, so any pages the later rewrites spawn stay in the
2461 // same group by position. (Doing this last silently dropped placement from
2462 // split/isolated pages, emitting them on group 0.)
2463 lowerPagePlacementToGroups(getOperation());
2464
2465 // The steps below look chains and payloads up many times per op. None of
2466 // them creates a symbol, and the merge removes the chains it erases.
2467 SymbolTable symbolTable(getOperation());
2468
2469 // First, isolate load_pdi and preempt operations into their own job/page
2470 bool isolateError = false;
2471 RewritePatternSet isolatePatterns(&getContext());
2472 isolatePatterns.insert<IsolateFullPageOpsPattern>(&getContext(),
2473 &isolateError);
2474 if (failed(
2475 applyPatternsGreedily(getOperation(), std::move(isolatePatterns))))
2476 signalPassFailure();
2477 if (isolateError)
2478 return signalPassFailure();
2479
2480 // Then apply the page splitting pattern. `splitError` catches diagnostics
2481 // the pattern emits while declining to split (see SplitCertPageOpPattern):
2482 // applyPatternsGreedily reports only whether the rewrite converged, so an
2483 // illegal-to-split page would otherwise print an error and still compile.
2484 bool splitError = false;
2485 RewritePatternSet p1(&getContext());
2486 p1.insert<SplitCertPageOpPattern>(&getContext(), symbolTable, &splitError);
2487 if (failed(applyPatternsGreedily(getOperation(), std::move(p1))))
2488 signalPassFailure();
2489 if (splitError)
2490 signalPassFailure();
2491
2492 if (failed(allocateJobIds(getOperation())))
2493 return signalPassFailure();
2494
2495 // Add the merge pattern for CertUcDmaWriteDesSyncOps. With an op whose
2496 // symbol uses are unknown, no use is counted and nothing merges.
2497 llvm::DenseMap<StringAttr, unsigned> useCounts;
2498 if (auto uses = SymbolTable::getSymbolUses(&getOperation().getRegion()))
2499 for (const SymbolTable::SymbolUse &use : *uses)
2500 ++useCounts[use.getSymbolRef().getRootReference()];
2501 RewritePatternSet p3(&getContext());
2502 p3.insert<MergeConsecutiveCertUcDmaWriteDesSyncOps>(&getContext(),
2503 symbolTable, useCounts);
2504 if (failed(applyPatternsGreedily(getOperation(), std::move(p3))))
2505 return signalPassFailure();
2506 }
2507};
2508
2509} // namespace
2510
2511std::unique_ptr<OperationPass<mlir::ModuleOp>> AIEX::createAIENpuToCertPass() {
2512 return std::make_unique<AIENpuToCertPass>();
2513}
2514
2515std::unique_ptr<OperationPass<AIE::DeviceOp>> AIEX::createAIECertPagesPass() {
2516 return std::make_unique<AIECertPagesPass>();
2517}
const V & get(const std::map< K, V > &pa, K k)
Definition d_ary_heap.h:70
std::unique_ptr< mlir::OperationPass< mlir::ModuleOp > > createAIENpuToCertPass()
std::optional< uint32_t > getConstantIntOperand(mlir::Value v)
std::unique_ptr< mlir::OperationPass< AIE::DeviceOp > > createAIECertPagesPass()
mlir::LogicalResult inlineWriteConfigs(mlir::ModuleOp module)
mlir::Operation * lookupNamedOpIn(mlir::Operation *symbolTableOp, mlir::StringAttr name)
const AIETargetModel & getTargetModel(mlir::Operation *op)
llvm::DenseSet< Operation * > chains
llvm::DenseSet< Operation * > payloads