13#include "mlir/Dialect/Arith/IR/Arith.h"
14#include "mlir/Dialect/MemRef/IR/MemRef.h"
15#include "mlir/IR/IRMapping.h"
16#include "mlir/IR/Matchers.h"
17#include "mlir/IR/SymbolTable.h"
18#include "mlir/Pass/Pass.h"
19#include "mlir/Transforms/DialectConversion.h"
20#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
21#include "llvm/ADT/DenseSet.h"
22#include "llvm/ADT/StringMap.h"
32#define GEN_PASS_DEF_AIENPUTOCERT
33#define GEN_PASS_DEF_AIECERTPAGES
34#include "aie/Dialect/AIEX/Transforms/AIEXPasses.h.inc"
40#define DEBUG_TYPE "npu-to-cert"
46static constexpr uint32_t cert_page_size = 8000;
47static constexpr uint32_t cert_page_limit = 8192;
52static constexpr uint32_t cert_job_start_cost = 8;
53static constexpr uint32_t cert_job_end_cost = 4;
58static bool isGeneratedConfigure(AIE::RuntimeSequenceOp seq) {
59 return AIE::AIEDialect::GeneratedConfigureAttrHelper(seq.getContext())
68static void tagConfigureByName(AIE::DeviceOp device) {
69 auto seqs = device.getBody()->getOps<AIE::RuntimeSequenceOp>();
70 if (llvm::any_of(seqs, isGeneratedConfigure))
73 if (seq.getSymName() ==
"configure") {
74 AIE::AIEDialect::GeneratedConfigureAttrHelper(device.getContext())
75 .setAttr(seq, UnitAttr::get(device.getContext()));
81static AIE::RuntimeSequenceOp findGeneratedConfigure(AIE::DeviceOp device) {
82 for (
auto seq : device.getBody()->getOps<AIE::RuntimeSequenceOp>())
83 if (isGeneratedConfigure(seq))
92static uint32_t getOrAssignPdiId(AIE::DeviceOp parentDevice,
93 StringRef deviceSymName) {
95 bool foundExisting =
false;
96 parentDevice.walk([&](AIEX::CertLoadPdiOp loadPdiOp) {
97 if (loadPdiOp.getSymbol() == deviceSymName) {
98 pdiId = loadPdiOp.getPdiId();
100 }
else if (!foundExisting) {
101 pdiId = std::max(pdiId, loadPdiOp.getPdiId() + 1);
107struct RuntimeSequenceToCertJob : OpConversionPattern<AIE::RuntimeSequenceOp> {
108 using OpConversionPattern::OpConversionPattern;
110 RuntimeSequenceToCertJob(MLIRContext *context, PatternBenefit benefit = 1)
111 : OpConversionPattern(context, benefit) {}
114 matchAndRewrite(AIE::RuntimeSequenceOp op, OpAdaptor adaptor,
115 ConversionPatternRewriter &rewriter)
const override {
117 bool isConfigure = isGeneratedConfigure(op);
118 uint32_t newJobId = 1;
120 uint32_t maxJobId = 1;
121 op->getParentOp()->walk([&](AIEX::CertJobOp certJobOp) {
122 maxJobId = std::max(maxJobId, certJobOp.getJobId());
124 newJobId = maxJobId + 1;
130 rewriter.setInsertionPoint(op);
131 auto jobOp = AIEX::CertJobOp::create(rewriter, op.getLoc(), newJobId);
136 jobOp->setAttr(
"cert.configure", rewriter.getUnitAttr());
142 op.getRegion().cloneInto(&jobOp.getBody(), remap);
143 AIEX::CertJobOp::ensureTerminator(jobOp.getBody(), rewriter, op->getLoc());
146 rewriter.eraseOp(op);
152struct NpuWrite32ToCertWrite32 : OpConversionPattern<AIEX::NpuWrite32Op> {
153 using OpConversionPattern::OpConversionPattern;
156 matchAndRewrite(AIEX::NpuWrite32Op op, OpAdaptor adaptor,
157 ConversionPatternRewriter &rewriter)
const override {
159 std::optional<uint32_t> address =
162 if (!address || !value)
163 return op.emitOpError(
164 "cannot lower to cert.write32 with non-constant address or value");
167 std::optional<uint32_t> absAddress = op.getAbsoluteAddress();
171 uint32_t absoluteAddr = *absAddress;
174 std::optional<uint32_t> col = op.getColumn();
175 std::optional<uint32_t> row = op.getRow();
178 uint32_t expectedUpperBits = ((*col & 0xff) << tm.getColumnShift()) |
179 ((*row & 0xff) << tm.getRowShift());
182 uint32_t origAddress = *address;
183 uint32_t origUpperBits = origAddress & ~0xfffff;
184 if (origUpperBits != 0 && origUpperBits != expectedUpperBits) {
185 op.emitWarning() <<
"address upper bits (0x"
186 << llvm::utohexstr(origUpperBits)
187 <<
") don't match row=" << *row <<
" col=" << *col
188 <<
" computed bits (0x"
189 << llvm::utohexstr(expectedUpperBits) <<
")";
193 rewriter.replaceOpWithNewOp<AIEX::CertWrite32Op>(op, absoluteAddr, *value);
198struct NpuMaskWrite32ToCertMaskWrite32
199 : OpConversionPattern<AIEX::NpuMaskWrite32Op> {
200 using OpConversionPattern::OpConversionPattern;
203 matchAndRewrite(AIEX::NpuMaskWrite32Op op, OpAdaptor adaptor,
204 ConversionPatternRewriter &rewriter)
const override {
206 std::optional<uint32_t> address =
210 if (!address || !mask || !value)
211 return op.emitOpError(
212 "cannot lower to cert.maskwrite32 with non-constant "
213 "address, mask, or value");
216 std::optional<uint32_t> absAddress = op.getAbsoluteAddress();
220 uint32_t absoluteAddr = *absAddress;
223 std::optional<uint32_t> col = op.getColumn();
224 std::optional<uint32_t> row = op.getRow();
227 uint32_t expectedUpperBits = ((*col & 0xff) << tm.getColumnShift()) |
228 ((*row & 0xff) << tm.getRowShift());
231 uint32_t origAddress = *address;
232 uint32_t origUpperBits = origAddress & ~0xfffff;
233 if (origUpperBits != 0 && origUpperBits != expectedUpperBits) {
234 op.emitWarning() <<
"address upper bits (0x"
235 << llvm::utohexstr(origUpperBits)
236 <<
") don't match row=" << *row <<
" col=" << *col
237 <<
" computed bits (0x"
238 << llvm::utohexstr(expectedUpperBits) <<
")";
242 rewriter.replaceOpWithNewOp<AIEX::CertMaskWrite32Op>(op, absoluteAddr,
248struct NpuMaskPollToCertMaskPoll32 : OpConversionPattern<AIEX::NpuMaskPollOp> {
249 using OpConversionPattern::OpConversionPattern;
252 matchAndRewrite(AIEX::NpuMaskPollOp op, OpAdaptor adaptor,
253 ConversionPatternRewriter &rewriter)
const override {
254 std::optional<uint32_t> address =
258 if (!address || !mask || !value)
259 return op.emitOpError(
"cannot lower to cert.maskpoll32 with non-constant "
260 "address, mask, or value");
262 std::optional<uint32_t> absAddress = op.getAbsoluteAddress();
266 rewriter.replaceOpWithNewOp<AIEX::CertMaskPoll32Op>(op, *absAddress, *mask,
272struct NpuBlockWriteToCertUcDma : OpConversionPattern<AIEX::NpuBlockWriteOp> {
276 NpuBlockWriteToCertUcDma(MLIRContext *ctx,
unsigned &nextChainId,
277 SymbolTable &symbolTable)
278 : OpConversionPattern(ctx), nextChainId(nextChainId),
279 symbolTable(symbolTable) {}
282 matchAndRewrite(AIEX::NpuBlockWriteOp op, OpAdaptor adaptor,
283 ConversionPatternRewriter &rewriter)
const override {
285 memref::GetGlobalOp dataOperand =
286 dyn_cast_or_null<memref::GetGlobalOp>(op.getData().getDefiningOp());
289 MemRefType dataType = cast<MemRefType>(dataOperand.getResult().getType());
290 uint32_t dataSize = dataType.getNumElements();
294 rewriter.eraseOp(op);
298 std::string symbolName =
"chain_" + std::to_string(nextChainId++);
299 while (symbolTable.lookup(symbolName))
300 symbolName =
"chain_" + std::to_string(nextChainId++);
303 rewriter.replaceOpWithNewOp<AIEX::CertUcDmaWriteDesSyncOp>(op, symbolName);
307 auto parentDevice = op->getParentOfType<AIE::DeviceOp>();
317 Block *deviceBody = parentDevice.getBody();
318 Operation *insertAfter =
nullptr;
319 Block::iterator it = deviceBody->begin();
320 if (lastChain && lastChain->getBlock() == deviceBody) {
321 insertAfter = lastChain;
322 it = std::next(Block::iterator(lastChain));
324 for (; it != deviceBody->end() &&
325 isa<AIEX::CertUcDmaChainOp, memref::GlobalOp>(*it);
329 rewriter.setInsertionPointAfter(insertAfter);
331 rewriter.setInsertionPointToStart(deviceBody);
333 auto symbolAttr = rewriter.getStringAttr(symbolName);
335 AIEX::CertUcDmaChainOp::create(rewriter, op.getLoc(), symbolAttr);
336 symbolTable.insert(chainOp);
339 Block *bb =
new Block();
340 chainOp.getRegion().push_back(bb);
341 rewriter.setInsertionPointToStart(bb);
342 AIEX::CertUcDmaBdOp::create(rewriter, op.getLoc(), dataOperand.getName(),
343 op.getAddress(), dataSize,
false);
345 AIEX::CertUcDmaChainOp::ensureTerminator(chainOp.getBody(), rewriter,
350 unsigned &nextChainId;
351 SymbolTable &symbolTable;
352 mutable Operation *lastChain =
nullptr;
355struct NpuSyncToCertWaitTCTS : OpConversionPattern<AIEX::NpuSyncOp> {
356 using OpConversionPattern::OpConversionPattern;
359 matchAndRewrite(AIEX::NpuSyncOp op, OpAdaptor adaptor,
360 ConversionPatternRewriter &rewriter)
const override {
362 std::optional<uint32_t> colOpt =
364 std::optional<uint32_t> channelOpt =
366 std::optional<uint32_t> directionOpt =
368 if (!rowOpt || !colOpt || !channelOpt || !directionOpt)
369 return op.emitOpError(
370 "cannot lower to cert.wait_tcts with non-constant sync parameters");
371 uint32_t row = *rowOpt;
372 uint32_t col = *colOpt;
377 const int row_id_shift = 16;
378 const int col_id_shift = 21;
379 uint16_t tile_id = col << (col_id_shift - row_id_shift) | row;
380 uint32_t channel = *channelOpt;
381 uint32_t direction = *directionOpt;
383 const std::vector<int> chan2actor_shim_s2mm = {0, 2, 3, 4};
384 const std::vector<int> chan2actor_shim_mm2s = {6, 7, 8, 9, 10, 11, 12, 13};
385 const std::vector<int> chan2actor_mem_s2mm = {1, 2, 3, 4, 5, 6, 7};
386 const std::vector<int> chan2actor_mem_mm2s = {16, 17, 18, 19, 20,
388 const std::vector<int> chan2actor_tile_s2mm = {0, 1};
389 const std::vector<int> chan2actor_tile_mm2s = {6};
393 direction ==
static_cast<std::underlying_type_t<AIE::DMAChannelDir>
>(
394 AIE::DMAChannelDir::S2MM);
396 const std::vector<int> *chan2actor =
nullptr;
397 if (tm.isCoreTile(col, row))
398 chan2actor = isS2MM ? &chan2actor_tile_s2mm : &chan2actor_tile_mm2s;
399 else if (tm.isMemTile(col, row))
400 chan2actor = isS2MM ? &chan2actor_mem_s2mm : &chan2actor_mem_mm2s;
402 chan2actor = isS2MM ? &chan2actor_shim_s2mm : &chan2actor_shim_mm2s;
404 size_t chanIdx =
static_cast<size_t>(channel);
405 if (chanIdx >= chan2actor->size()) {
406 op.emitError(
"invalid DMA channel ")
407 << channel <<
" for " << (isS2MM ?
"S2MM" :
"MM2S")
408 <<
" direction in NpuSyncToCertWaitTCTS conversion";
412 uint8_t actor_id =
static_cast<uint8_t
>((*chan2actor)[chanIdx]);
413 uint8_t num_tcts = 1;
414 rewriter.replaceOpWithNewOp<AIEX::CertWaitTCTSOp>(op, tile_id, actor_id,
420struct NpuLoadPdiToCertLoadPdi : OpConversionPattern<AIEX::NpuLoadPdiOp> {
421 using OpConversionPattern::OpConversionPattern;
424 matchAndRewrite(AIEX::NpuLoadPdiOp op, OpAdaptor adaptor,
425 ConversionPatternRewriter &rewriter)
const override {
427 auto deviceRef = op.getDeviceRef();
431 StringRef deviceSymName = *deviceRef;
434 auto parentDevice = op->getParentOfType<AIE::DeviceOp>();
439 uint32_t pdiId = getOrAssignPdiId(parentDevice, deviceSymName);
442 rewriter.replaceOpWithNewOp<AIEX::CertLoadPdiOp>(
443 op, rewriter.getUI32IntegerAttr(pdiId),
444 FlatSymbolRefAttr::get(rewriter.getContext(), deviceSymName));
451 using OpRewritePattern::OpRewritePattern;
453 LogicalResult matchAndRewrite(AIEX::RunOp op,
454 PatternRewriter &rewriter)
const override {
456 AIE::RuntimeSequenceOp calleeRuntimeSequence =
457 op.getCalleeRuntimeSequenceOp();
458 if (!calleeRuntimeSequence)
462 Region &calleeBody = calleeRuntimeSequence.getBody();
466 ValueRange values = op.getArgs();
467 if (values.size() != calleeBody.getNumArguments()) {
468 op.emitError(
"number of run op arguments (")
470 <<
") does not match number of callee runtime sequence "
472 << calleeBody.getNumArguments() <<
")";
475 for (
unsigned i = 0, n = calleeBody.getNumArguments(); i < n; i++) {
476 BlockArgument arg = calleeBody.getArgument(i);
477 Value val = values[i];
478 argMap.map(arg, val);
482 rewriter.setInsertionPoint(op);
483 for (Operation &o : calleeBody.front().getOperations()) {
485 if (isa<AIE::EndOp>(o))
487 rewriter.clone(o, argMap);
491 rewriter.eraseOp(op);
498 using OpRewritePattern::OpRewritePattern;
500 LogicalResult matchAndRewrite(AIEX::ConfigureOp op,
501 PatternRewriter &rewriter)
const override {
504 if (!op.getRegion().getOps<AIEX::RunOp>().empty())
508 AIE::DeviceOp referencedDevice = op.getReferencedDeviceOp();
509 if (!referencedDevice)
513 StringRef deviceSymName = referencedDevice.getSymName();
516 auto parentDevice = op->getParentOfType<AIE::DeviceOp>();
521 auto existingSection = dyn_cast_if_present<AIEX::CertSectionOp>(
522 parentDevice.lookupSymbol(deviceSymName));
523 if (existingSection) {
526 uint32_t pdiId = getOrAssignPdiId(parentDevice, deviceSymName);
528 rewriter.setInsertionPoint(op);
529 AIEX::CertLoadPdiOp::create(
530 rewriter, op.getLoc(), rewriter.getUI32IntegerAttr(pdiId),
531 FlatSymbolRefAttr::get(rewriter.getContext(), deviceSymName));
534 IRMapping bodyMapper;
535 for (Operation &o : op.getRegion().front().getOperations()) {
536 if (isa<AIE::EndOp>(o))
538 rewriter.clone(o, bodyMapper);
541 rewriter.eraseOp(op);
546 rewriter.setInsertionPoint(parentDevice.getBody()->getTerminator());
547 auto sectionOp = AIEX::CertSectionOp::create(
548 rewriter, op.getLoc(), rewriter.getStringAttr(deviceSymName));
551 Block *sectionBlock =
new Block();
552 sectionOp.getBody().push_back(sectionBlock);
553 rewriter.setInsertionPointToStart(sectionBlock);
556 auto pageOp = AIEX::CertPageOp::create(rewriter, op.getLoc());
557 Block *pageBlock =
new Block();
558 pageOp.getBody().push_back(pageBlock);
559 rewriter.setInsertionPointToStart(pageBlock);
563 uint32_t maxJobId = 0;
564 parentDevice.walk([&](AIEX::CertJobOp certJobOp) {
565 maxJobId = std::max(maxJobId, certJobOp.getJobId());
567 uint32_t sectionJobId = maxJobId + 1;
569 auto jobOp = AIEX::CertJobOp::create(rewriter, op.getLoc(), sectionJobId);
570 Block *jobBlock =
new Block();
571 jobOp.getBody().push_back(jobBlock);
572 rewriter.setInsertionPointToStart(jobBlock);
581 AIE::RuntimeSequenceOp configureSeq =
582 findGeneratedConfigure(referencedDevice);
587 for (Operation &o : configureSeq.getRegion().front().getOperations()) {
588 if (isa<AIE::EndOp>(o))
590 rewriter.clone(o, mapper);
595 AIEX::CertPageOp configPage =
nullptr;
596 for (
auto &op : referencedDevice.getBody()->getOperations()) {
597 if (
auto page = dyn_cast<AIEX::CertPageOp>(op)) {
606 for (
auto &op : configPage.getBody().front().getOperations()) {
607 if (
auto job = dyn_cast<AIEX::CertJobOp>(op)) {
608 for (Operation &jobOp : job.getBody().front().getOperations()) {
609 if (isa<AIE::EndOp>(jobOp))
611 rewriter.clone(jobOp, mapper);
620 AIEX::CertJobOp::ensureTerminator(jobOp.getBody(), rewriter, op.getLoc());
621 AIEX::CertPageOp::ensureTerminator(pageOp.getBody(), rewriter, op.getLoc());
622 AIEX::CertSectionOp::ensureTerminator(sectionOp.getBody(), rewriter,
627 uint32_t pdiId = getOrAssignPdiId(parentDevice, deviceSymName);
629 rewriter.setInsertionPoint(op);
630 AIEX::CertLoadPdiOp::create(
631 rewriter, op.getLoc(), rewriter.getUI32IntegerAttr(pdiId),
632 FlatSymbolRefAttr::get(rewriter.getContext(), deviceSymName));
637 IRMapping bodyMapper;
638 for (Operation &o : op.getRegion().front().getOperations()) {
639 if (isa<AIE::EndOp>(o))
641 rewriter.clone(o, bodyMapper);
645 rewriter.eraseOp(op);
652static AIEX::NpuBlockWriteOp
653findPatchedBlockWrite(AIEX::NpuAddressPatchOp patch) {
655 uint32_t addr = patch.getAddr();
656 int col = (addr >> tm.getColumnShift()) & 0x1f;
657 int row = (addr >> tm.getRowShift()) & 0x1f;
658 if (!tm.isValidTile({col, row}))
660 for (Block::iterator it(patch); it != patch->getBlock()->begin();) {
661 auto blockWrite = dyn_cast<AIEX::NpuBlockWriteOp>(*--it);
664 if (blockWrite.getAddress() + tm.getDmaBdAddressOffset(col, row) != addr)
674static constexpr uint32_t kMaxApplyOffsetArgIdx = 0x7FFF;
681static LogicalResult prepareAddressPatchTemplates(AIE::DeviceOp device) {
682 LogicalResult result = success();
683 SymbolTable symbols(device);
688 std::optional<llvm::DenseMap<StringAttr, unsigned>> symbolUseCounts;
689 if (
auto uses = SymbolTable::getSymbolUses(&device.getRegion())) {
690 symbolUseCounts.emplace();
691 for (
const SymbolTable::SymbolUse &use : *uses)
692 ++(*symbolUseCounts)[use.getSymbolRef().getRootReference()];
694 SmallVector<AIEX::NpuAddressPatchOp> patches;
695 device.walk([&](AIEX::NpuAddressPatchOp patch) { patches.push_back(patch); });
696 for (AIEX::NpuAddressPatchOp patch : patches) {
697 std::optional<uint32_t> argIdx = patch.getArgIdx();
698 if (argIdx && *argIdx > kMaxApplyOffsetArgIdx) {
699 patch.emitError(
"arg_idx ")
700 << *argIdx <<
" does not fit apply_offset_57's 16-bit argument "
701 <<
"offset; the largest is " << kMaxApplyOffsetArgIdx;
705 AIEX::NpuBlockWriteOp blockWrite = findPatchedBlockWrite(patch);
708 auto getGlobal = dyn_cast_or_null<memref::GetGlobalOp>(
709 blockWrite.getData().getDefiningOp());
712 auto global = symbols.lookup<memref::GlobalOp>(getGlobal.getName());
715 std::optional<Attribute> initialValue = global.getInitialValue();
716 auto words = initialValue ? dyn_cast<DenseIntElementsAttr>(*initialValue)
717 : DenseIntElementsAttr();
721 if (!patch.getArgIdx()) {
722 patch.emitError(
"cert lowering requires an address_patch with arg_idx");
727 if (!matchPattern(patch.getArgPlus(), m_ConstantInt(&argPlus))) {
728 patch.emitError(
"cert lowering requires a constant arg_plus: "
729 "apply_offset_57 cannot add a runtime offset");
735 if (argPlus.getActiveBits() > 32) {
736 patch.emitError(
"arg_plus ")
737 << argPlus.getSExtValue()
738 <<
" does not fit the BD's 32-bit address word";
742 unsigned word = (patch.getAddr() - blockWrite.getAddress()) / 4;
743 if (word >= words.getNumElements())
748 if (!symbolUseCounts ||
749 symbolUseCounts->lookup(global.getSymNameAttr()) != 1 ||
750 !getGlobal->hasOneUse()) {
751 OpBuilder builder(global);
752 auto copy = cast<memref::GlobalOp>(builder.clone(*global));
754 StringAttr copyName = symbols.insert(copy);
755 OpBuilder atBlockWrite(blockWrite);
757 memref::GetGlobalOp::create(atBlockWrite, getGlobal.getLoc(),
758 getGlobal.getType(), copy.getSymName());
759 blockWrite.getDataMutable().assign(privateGet);
761 (*symbolUseCounts)[copyName] = 1;
762 if (getGlobal->use_empty()) {
764 --(*symbolUseCounts)[global.getSymNameAttr()];
770 auto values = llvm::to_vector(words.getValues<APInt>());
771 values[word] = argPlus.zextOrTrunc(values[word].getBitWidth());
772 global.setInitialValueAttr(
773 DenseIntElementsAttr::get(words.getType(), values));
778struct NpuAddressPatchToCertApplyOffset57
779 : OpConversionPattern<AIEX::NpuAddressPatchOp> {
780 using OpConversionPattern::OpConversionPattern;
783 matchAndRewrite(AIEX::NpuAddressPatchOp op, OpAdaptor adaptor,
784 ConversionPatternRewriter &rewriter)
const override {
785 AIEX::NpuBlockWriteOp blockWriteOp = findPatchedBlockWrite(op);
786 std::optional<uint32_t> argIdx = op.getArgIdx();
787 if (!blockWriteOp || !argIdx)
790 dyn_cast<memref::GetGlobalOp>(blockWriteOp.getData().getDefiningOp());
793 rewriter.setInsertionPoint(blockWriteOp);
794 rewriter.replaceOpWithNewOp<AIEX::CertApplyOffset57Op>(
795 op, getGlobalOp.getName(), 1, *argIdx);
800struct MergeConsecutiveCertUcDmaWriteDesSyncOps
805 MergeConsecutiveCertUcDmaWriteDesSyncOps(
806 MLIRContext *context, SymbolTable &symbolTable,
807 llvm::DenseMap<StringAttr, unsigned> &useCounts)
809 useCounts(useCounts) {}
811 LogicalResult matchAndRewrite(AIEX::CertUcDmaWriteDesSyncOp op,
812 PatternRewriter &rewriter)
const override {
820 auto isTransparentToMerge = [](Operation *o) {
821 return isa<AIEX::CertNopOp>(o);
825 Block::iterator it(op);
826 AIEX::CertUcDmaWriteDesSyncOp prevWriteDesSync =
nullptr;
827 while (it != op->getBlock()->begin() && !prevWriteDesSync) {
829 Operation *prevOp = &*it;
830 prevWriteDesSync = dyn_cast<AIEX::CertUcDmaWriteDesSyncOp>(prevOp);
831 if (!prevWriteDesSync && !isTransparentToMerge(prevOp))
834 if (!prevWriteDesSync)
838 StringRef sym_name = op.getSymbol();
839 StringRef prev_sym_name = prevWriteDesSync.getSymbol();
840 auto chain = symbolTable.lookup<AIEX::CertUcDmaChainOp>(sym_name);
841 auto prevChain = symbolTable.lookup<AIEX::CertUcDmaChainOp>(prev_sym_name);
842 if (!chain || !prevChain)
852 auto isSingleUse = [&](AIEX::CertUcDmaChainOp c) {
853 return useCounts.lookup(c.getSymNameAttr()) == 1;
855 if (!isSingleUse(chain) || !isSingleUse(prevChain))
865 auto chainSize = [](AIEX::CertUcDmaChainOp c) {
867 for (
auto bdOp : c.getBody().front().getOps<AIEX::CertUcDmaBdOp>())
868 size += bdOp.getLength() *
sizeof(int) + 16;
871 if ((chainSize(chain) + chainSize(prevChain)) >= cert_page_size)
875 rewriter.setInsertionPointToStart(&chain.getBody().front());
876 for (
auto &o : prevChain.getBody().front().getOperations()) {
877 auto bdOp = dyn_cast<AIEX::CertUcDmaBdOp>(o);
880 AIEX::CertUcDmaBdOp::create(
881 rewriter, bdOp.getLoc(), bdOp.getRemoteAddress(),
882 bdOp.getLocalAddress(), bdOp.getLength(),
true);
884 useCounts.erase(prevChain.getSymNameAttr());
885 symbolTable.remove(prevChain);
886 rewriter.eraseOp(prevChain);
887 rewriter.eraseOp(prevWriteDesSync);
891 SymbolTable &symbolTable;
892 llvm::DenseMap<StringAttr, unsigned> &useCounts;
895struct SplitNpuBlockWriteOpPattern :
OpRewritePattern<AIEX::NpuBlockWriteOp> {
898 SplitNpuBlockWriteOpPattern(MLIRContext *context,
899 SymbolTableCollection &symbolTables)
902 LogicalResult matchAndRewrite(AIEX::NpuBlockWriteOp op,
903 PatternRewriter &rewriter)
const override {
905 memref::GetGlobalOp dataOperand =
906 dyn_cast_or_null<memref::GetGlobalOp>(op.getData().getDefiningOp());
910 MemRefType dataType = cast<MemRefType>(dataOperand.getResult().getType());
911 uint32_t dataSize = dataType.getNumElements();
913 uint32_t dataSizeBytes = dataSize *
sizeof(int);
914 if (dataSizeBytes < cert_page_size)
917 auto loc = op.getLoc();
920 uint32_t splitElements = dataSize / 2;
921 uint32_t firstChunkSize = splitElements;
922 uint32_t secondChunkSize = dataSize - splitElements;
925 auto deviceOp = op->getParentOfType<AIE::DeviceOp>();
926 SymbolTable &symbolTable = symbolTables.getSymbolTable(deviceOp);
927 auto originalGlobal =
928 symbolTable.lookup<memref::GlobalOp>(dataOperand.getName());
933 auto originalData = originalGlobal.getInitialValue();
937 auto denseData = dyn_cast<DenseIntElementsAttr>(*originalData);
942 auto dataValues = denseData.getValues<APInt>();
943 std::vector<APInt> firstChunkData(dataValues.begin(),
944 dataValues.begin() + firstChunkSize);
945 std::vector<APInt> secondChunkData(dataValues.begin() + firstChunkSize,
949 auto elementType = rewriter.getI32Type();
950 auto firstChunkType = MemRefType::get({firstChunkSize}, elementType);
951 auto secondChunkType = MemRefType::get({secondChunkSize}, elementType);
952 TensorType firstTensorType =
953 RankedTensorType::get({firstChunkSize}, elementType);
954 TensorType secondTensorType =
955 RankedTensorType::get({secondChunkSize}, elementType);
957 auto firstChunkAttr =
958 DenseIntElementsAttr::get(firstTensorType, firstChunkData);
959 auto secondChunkAttr =
960 DenseIntElementsAttr::get(secondTensorType, secondChunkData);
963 std::string firstName = dataOperand.getName().str() +
"_split_0";
964 std::string secondName = dataOperand.getName().str() +
"_split_1";
968 while (symbolTable.lookup(firstName)) {
970 dataOperand.getName().str() +
"_split_0_" + std::to_string(counter++);
973 while (symbolTable.lookup(secondName)) {
975 dataOperand.getName().str() +
"_split_1_" + std::to_string(counter++);
979 rewriter.setInsertionPoint(originalGlobal);
980 auto firstGlobal = memref::GlobalOp::create(
981 rewriter, loc, firstName, rewriter.getStringAttr(
"private"),
982 firstChunkType, firstChunkAttr,
true,
nullptr);
983 symbolTable.insert(firstGlobal);
985 auto secondGlobal = memref::GlobalOp::create(
986 rewriter, loc, secondName, rewriter.getStringAttr(
"private"),
987 secondChunkType, secondChunkAttr,
true,
nullptr);
988 symbolTable.insert(secondGlobal);
991 rewriter.setInsertionPoint(op);
993 auto firstGetGlobal =
994 memref::GetGlobalOp::create(rewriter, loc, firstChunkType, firstName);
995 auto secondGetGlobal =
996 memref::GetGlobalOp::create(rewriter, loc, secondChunkType, secondName);
998 uint32_t baseAddr = op.getAddress();
1000 AIEX::NpuBlockWriteOp::create(rewriter, loc, baseAddr,
1001 firstGetGlobal.getResult(),
nullptr,
nullptr,
1004 AIEX::NpuBlockWriteOp::create(rewriter, loc, baseAddr + firstChunkSize * 4,
1005 secondGetGlobal.getResult(),
nullptr,
nullptr,
1009 rewriter.eraseOp(op);
1011 LLVM_DEBUG(llvm::outs()
1012 <<
"Split NpuBlockWriteOp with data size: " << dataSizeBytes
1013 <<
" bytes into chunks of " << firstChunkSize <<
" and "
1014 << secondChunkSize <<
" elements\n");
1019 SymbolTableCollection &symbolTables;
1026static LogicalResult resolveBlockWriteAddresses(AIE::DeviceOp device) {
1028 LogicalResult result = success();
1029 device.walk([&](AIEX::NpuBlockWriteOp op) {
1030 if (!op.getBuffer() && !(op.getColumn() && op.getRow()))
1032 std::optional<uint32_t> address = op.getAbsoluteAddress(&names);
1037 op.setAddress(*address);
1038 op.removeBufferAttr();
1039 op.removeColumnAttr();
1045struct AIENpuToCertPass
1046 : xilinx::AIEX::impl::AIENpuToCertBase<AIENpuToCertPass> {
1047 void runOnOperation()
override {
1048 auto moduleOp = getOperation();
1050 return signalPassFailure();
1053 llvm::SmallVector<AIE::DeviceOp, 4> devices;
1054 for (
auto &op : moduleOp.getBody()->getOperations()) {
1055 if (
auto deviceOp = dyn_cast<AIE::DeviceOp>(op)) {
1056 devices.push_back(deviceOp);
1060 bool resolved =
true;
1061 for (
auto dev : devices)
1062 resolved &= succeeded(resolveBlockWriteAddresses(dev));
1064 return signalPassFailure();
1066 for (
auto dev : devices)
1067 tagConfigureByName(dev);
1072 AIE::DeviceOp mainDevice =
nullptr;
1073 for (
auto dev : devices) {
1074 if (dev.getSymName() == deviceName) {
1081 moduleOp.emitError(
"no device found matching --device-name \"")
1082 << deviceName <<
"\"";
1083 signalPassFailure();
1087 migrateGlobalsIntoMainDevice(mainDevice, devices);
1088 processDevice(mainDevice);
1091 for (
auto dev : devices) {
1092 if (dev != mainDevice) {
1099 llvm::SmallVector<AIE::DeviceOp, 4> devicesToRemove;
1100 for (
auto dev : devices) {
1101 if (dev != mainDevice) {
1102 devicesToRemove.push_back(dev);
1106 for (
auto dev : devicesToRemove) {
1114 void migrateGlobalsIntoMainDevice(AIE::DeviceOp mainDevice,
1115 ArrayRef<AIE::DeviceOp> devices) {
1116 auto isDeviceName = [&](StringRef candidate) {
1117 return llvm::any_of(devices, [&](AIE::DeviceOp device) {
1118 return device.getSymName() == candidate;
1124 DeviceNames mainNames = collectNames(mainDevice);
1125 std::optional<PayloadUses> mainUses;
1126 for (
auto global : llvm::to_vector(mainDevice.getOps<memref::GlobalOp>())) {
1127 StringRef oldName = global.getSymName();
1128 if (!isDeviceName(oldName))
1130 std::string newName;
1131 unsigned counter = 0;
1133 newName = oldName.str() +
"_" + std::to_string(counter++);
1134 while (mainNames.contains(newName) || isDeviceName(newName));
1136 mainUses = collectPayloadUses(mainDevice);
1137 renamePayloadUses(*mainUses, oldName, newName);
1138 removeName(mainNames, oldName);
1139 ++mainNames[newName];
1140 global.setSymName(newName);
1143 for (
auto dev : devices) {
1144 if (dev == mainDevice)
1146 DeviceNames devNames = collectNames(dev);
1147 PayloadUses devUses = collectPayloadUses(dev);
1148 auto globals = llvm::to_vector(dev.getOps<memref::GlobalOp>());
1149 for (
auto global : globals) {
1150 StringRef oldName = global.getSymName();
1151 std::string newName = oldName.str();
1152 unsigned counter = 0;
1155 auto taken = [&](StringRef candidate) {
1156 return mainNames.contains(candidate) || isDeviceName(candidate) ||
1157 (candidate != oldName && devNames.contains(candidate));
1159 while (taken(newName))
1160 newName = oldName.str() +
"_" + std::to_string(counter++);
1161 if (newName != oldName) {
1162 renamePayloadUses(devUses, oldName, newName);
1163 global.setSymName(newName);
1165 removeName(devNames, oldName);
1166 ++mainNames[newName];
1167 global->moveBefore(mainDevice.getBody(), mainDevice.getBody()->begin());
1175 using DeviceNames = llvm::StringMap<unsigned>;
1176 static DeviceNames collectNames(AIE::DeviceOp dev) {
1178 for (Operation &op : dev.getBody()->getOperations())
1180 op.getAttrOfType<StringAttr>(SymbolTable::getSymbolAttrName()))
1181 ++names[symName.getValue()];
1184 static void removeName(DeviceNames &names, StringRef name) {
1185 auto it = names.find(name);
1186 if (it != names.end() && --it->second == 0)
1194 using PayloadUses = llvm::StringMap<SmallVector<Operation *>>;
1195 static PayloadUses collectPayloadUses(AIE::DeviceOp dev) {
1197 dev.walk<WalkOrder::PreOrder>([&](Operation *op) {
1198 if (op != dev && op->hasTrait<OpTrait::SymbolTable>())
1199 return WalkResult::skip();
1200 if (
auto get = dyn_cast<memref::GetGlobalOp>(op))
1201 uses[
get.getName()].push_back(op);
1202 else if (
auto bd = dyn_cast<AIEX::CertUcDmaBdOp>(op))
1203 uses[bd.getRemoteAddress()].push_back(op);
1204 else if (
auto patch = dyn_cast<AIEX::CertApplyOffset57Op>(op))
1205 uses[patch.getSymbol()].push_back(op);
1206 return WalkResult::advance();
1212 static void renamePayloadUses(PayloadUses &uses, StringRef oldName,
1213 StringRef newName) {
1214 auto it = uses.find(oldName);
1215 if (it == uses.end())
1217 SmallVector<Operation *> ops = std::move(it->second);
1219 for (Operation *op : ops) {
1220 auto newRef = FlatSymbolRefAttr::get(op->getContext(), newName);
1221 if (
auto get = dyn_cast<memref::GetGlobalOp>(op))
1222 get.setNameAttr(newRef);
1223 else if (
auto bd = dyn_cast<AIEX::CertUcDmaBdOp>(op))
1224 bd.setRemoteAddressAttr(newRef);
1225 else if (
auto patch = dyn_cast<AIEX::CertApplyOffset57Op>(op))
1226 patch.setSymbolAttr(newRef);
1228 llvm::append_range(uses[newName], ops);
1233 LogicalResult absorbLoadedDevices(AIE::DeviceOp currentDevice,
1234 ModuleOp moduleOp) {
1238 llvm::SmallVector<AIEX::NpuLoadPdiOp, 4> loads;
1239 auto collectLoads = [&](Operation *root) {
1240 root->walk([&](AIEX::NpuLoadPdiOp loadPdiOp) {
1241 if (loadPdiOp.getDeviceRef())
1242 loads.push_back(loadPdiOp);
1245 collectLoads(currentDevice);
1247 OpBuilder builder(&getContext());
1251 while (next < loads.size()) {
1252 AIEX::NpuLoadPdiOp load = loads[next++];
1253 std::optional<StringRef> deviceRef = load.getDeviceRef();
1256 StringRef refSymName = *deviceRef;
1259 if (isa_and_present<AIEX::CertSectionOp>(existing))
1263 AIE::DeviceOp refDevice =
nullptr;
1264 for (
auto &op : moduleOp.getBody()->getOperations()) {
1265 if (
auto dev = dyn_cast<AIE::DeviceOp>(op)) {
1266 if (dev.getSymName() == refSymName) {
1277 AIE::RuntimeSequenceOp configureSeq = findGeneratedConfigure(refDevice);
1285 load.emitError(
"cannot absorb device @")
1286 << refSymName <<
" as a cert.section: @" << refSymName
1287 <<
" is already defined by '" << existing->getName() <<
"'";
1292 builder.setInsertionPoint(currentDevice.getBody()->getTerminator());
1293 auto sectionOp = AIEX::CertSectionOp::create(
1294 builder, configureSeq.getLoc(), builder.getStringAttr(refSymName));
1296 Block *sectionBlock =
new Block();
1297 sectionOp.getBody().push_back(sectionBlock);
1298 builder.setInsertionPointToStart(sectionBlock);
1301 auto pageOp = AIEX::CertPageOp::create(builder, configureSeq.getLoc());
1302 Block *pageBlock =
new Block();
1303 pageOp.getBody().push_back(pageBlock);
1304 builder.setInsertionPointToStart(pageBlock);
1307 uint32_t maxJobId = 0;
1308 currentDevice.walk([&](AIEX::CertJobOp certJobOp) {
1309 maxJobId = std::max(maxJobId, certJobOp.getJobId());
1314 AIEX::CertJobOp::create(builder, configureSeq.getLoc(), maxJobId + 1);
1318 configureSeq.getRegion().cloneInto(&jobOp.getBody(), mapper);
1321 AIEX::CertJobOp::ensureTerminator(jobOp.getBody(), builder,
1322 configureSeq.getLoc());
1323 AIEX::CertPageOp::ensureTerminator(pageOp.getBody(), builder,
1324 configureSeq.getLoc());
1325 AIEX::CertSectionOp::ensureTerminator(sectionOp.getBody(), builder,
1326 configureSeq.getLoc());
1327 collectLoads(jobOp);
1332 void processDevice(AIE::DeviceOp currentDevice) {
1333 auto moduleOp = currentDevice->getParentOfType<ModuleOp>();
1337 RewritePatternSet p_expand(&getContext());
1338 p_expand.insert<RunOpInlining, ConfigureOpToCertSection>(&getContext());
1339 FrozenRewritePatternSet expandPatterns(std::move(p_expand));
1344 bool changed =
true;
1348 if (moduleOp && failed(absorbLoadedDevices(currentDevice, moduleOp)))
1349 return signalPassFailure();
1351 if (failed(applyPatternsGreedily(currentDevice, expandPatterns,
1352 GreedyRewriteConfig(), &changed)))
1353 return signalPassFailure();
1356 ConversionTarget target(getContext());
1357 target.addIllegalOp<AIE::RuntimeSequenceOp>();
1359 target.addLegalOp<AIEX::CertApplyOffset57Op>();
1360 target.addLegalOp<AIEX::CertJobOp>();
1361 target.addLegalOp<AIEX::CertPageOp>();
1362 target.addLegalOp<AIEX::CertSectionOp>();
1363 target.addLegalOp<AIEX::CertLoadPdiOp>();
1364 target.addLegalOp<AIEX::CertMaskWrite32Op>();
1365 target.addLegalOp<AIEX::CertMaskPoll32Op>();
1366 target.addLegalOp<AIEX::CertUcDmaWriteDesSyncOp>();
1367 target.addLegalOp<AIEX::CertUcDmaChainOp>();
1368 target.addLegalOp<AIEX::CertUcDmaBdOp>();
1369 target.addLegalOp<AIEX::CertWrite32Op>();
1370 target.addLegalOp<AIEX::CertWaitTCTSOp>();
1371 target.addLegalOp<AIEX::ConfigureOp>();
1372 target.addLegalDialect<AIE::AIEDialect>();
1374 RewritePatternSet p0(&getContext());
1375 p0.insert<RuntimeSequenceToCertJob>(&getContext());
1377 if (failed(applyPartialConversion(currentDevice, target, std::move(p0))))
1378 return signalPassFailure();
1380 if (failed(prepareAddressPatchTemplates(currentDevice)))
1381 return signalPassFailure();
1383 target.addIllegalOp<AIEX::NpuAddressPatchOp>();
1386 RewritePatternSet p1(&getContext());
1387 p1.insert<NpuAddressPatchToCertApplyOffset57>(&getContext());
1389 if (failed(applyPartialConversion(currentDevice, target, std::move(p1))))
1390 return signalPassFailure();
1394 SymbolTableCollection symbolTables;
1395 RewritePatternSet p(&getContext());
1396 p.insert<SplitNpuBlockWriteOpPattern>(&getContext(), symbolTables);
1397 if (failed(applyPatternsGreedily(getOperation(), std::move(p))))
1398 signalPassFailure();
1401 target.addIllegalOp<AIEX::NpuBlockWriteOp>();
1402 target.addIllegalOp<AIEX::NpuBlockWriteValuesOp>();
1403 target.addIllegalOp<AIEX::NpuMaskWrite32Op>();
1404 target.addIllegalOp<AIEX::NpuMaskPollOp>();
1405 target.addIllegalOp<AIEX::NpuSyncOp>();
1406 target.addIllegalOp<AIEX::NpuWrite32Op>();
1407 target.addIllegalOp<AIEX::NpuLoadPdiOp>();
1409 unsigned nextChainId = 0;
1410 SymbolTable symbolTable(currentDevice);
1411 RewritePatternSet p2(&getContext());
1412 p2.insert<NpuBlockWriteToCertUcDma>(&getContext(), nextChainId,
1414 p2.insert<NpuMaskWrite32ToCertMaskWrite32>(&getContext());
1415 p2.insert<NpuMaskPollToCertMaskPoll32>(&getContext());
1416 p2.insert<NpuWrite32ToCertWrite32>(&getContext());
1417 p2.insert<NpuSyncToCertWaitTCTS>(&getContext());
1418 p2.insert<NpuLoadPdiToCertLoadPdi>(&getContext());
1420 if (failed(applyPartialConversion(currentDevice, target, std::move(p2))))
1421 return signalPassFailure();
1427 RewritePatternSet cleanup(&getContext());
1428 if (failed(applyPatternsGreedily(getOperation(), std::move(cleanup))))
1429 return signalPassFailure();
1435 OpBuilder builder(&getContext());
1438 llvm::SetVector<StringRef> referencedDeviceNames;
1439 currentDevice.walk([&](AIEX::CertLoadPdiOp loadPdiOp) {
1440 StringRef refSymName = loadPdiOp.getSymbol();
1441 if (!currentDevice.lookupSymbol(refSymName))
1442 referencedDeviceNames.insert(refSymName);
1446 for (StringRef refSymName : referencedDeviceNames) {
1448 AIE::DeviceOp refDevice =
nullptr;
1449 for (
auto &op : moduleOp.getBody()->getOperations()) {
1450 if (
auto dev = dyn_cast<AIE::DeviceOp>(op)) {
1451 if (dev.getSymName() == refSymName) {
1463 llvm::SmallVector<AIEX::CertPageOp, 4> certPages;
1464 for (
auto &op : refDevice.getBody()->getOperations()) {
1465 if (
auto pageOp = dyn_cast<AIEX::CertPageOp>(op)) {
1466 certPages.push_back(pageOp);
1470 if (certPages.empty())
1474 builder.setInsertionPoint(currentDevice.getBody()->getTerminator());
1475 auto sectionOp = AIEX::CertSectionOp::create(
1476 builder, refDevice.getLoc(), builder.getStringAttr(refSymName));
1478 Block *sectionBlock =
new Block();
1479 sectionOp.getBody().push_back(sectionBlock);
1483 for (
auto pageOp : certPages) {
1484 builder.setInsertionPointToEnd(sectionBlock);
1485 builder.clone(*pageOp.getOperation(), mapper);
1489 AIEX::CertSectionOp::ensureTerminator(sectionOp.getBody(), builder,
1490 refDevice.getLoc());
1495 currentDevice.walk([&](AIEX::CertJobOp jobOp) {
1496 Block &jobBlock = jobOp.getBody().front();
1499 for (
int i = jobBlock.getNumArguments() - 1; i >= 0; --i) {
1500 BlockArgument arg = jobBlock.getArgument(i);
1501 if (arg.use_empty()) {
1502 jobBlock.eraseArgument(i);
1512static StringRef pushedChain(Operation &o) {
1513 if (
auto push = dyn_cast<AIEX::CertUcDmaWriteDesSyncOp>(o))
1514 return push.getSymbol();
1515 if (
auto push = dyn_cast<AIEX::CertUcDmaWriteDesOp>(o))
1516 return push.getSymbol();
1524static bool isLocalRegister(uint32_t r) {
return r < 8; }
1527static void collectLocalRegisterDefsUses(Operation &o,
1528 SmallVectorImpl<uint32_t> &uses,
1529 SmallVectorImpl<uint32_t> &defs) {
1530 auto add = [](SmallVectorImpl<uint32_t> ®s, uint32_t r) {
1531 if (isLocalRegister(r))
1534 if (
auto op = dyn_cast<AIEX::CertUcDmaWriteDesOp>(o)) {
1535 add(defs, op.getWaitHandle());
1536 }
else if (
auto op = dyn_cast<AIEX::CertWaitUcDmaOp>(o)) {
1537 add(uses, op.getWaitHandle());
1538 }
else if (
auto op = dyn_cast<AIEX::CertRead32Op>(o)) {
1539 add(defs, op.getValue());
1540 }
else if (
auto op = dyn_cast<AIEX::CertRead32DOp>(o)) {
1541 add(uses, op.getAddress());
1542 add(defs, op.getValue());
1543 }
else if (
auto op = dyn_cast<AIEX::CertWrite32DOp>(o)) {
1544 if (op.getAddressIsReg())
1545 add(uses, op.getAddress());
1546 if (op.getValueIsReg())
1547 add(uses, op.getValue());
1553static bool jobUsesLocalRegister(AIEX::CertJobOp job) {
1554 SmallVector<uint32_t> uses, defs;
1555 for (Operation &o : job.getBody().front())
1556 collectLocalRegisterDefsUses(o, uses, defs);
1557 return !uses.empty() || !defs.empty();
1562static std::optional<uint32_t>
1563findLocalRegisterCrossingIn(ArrayRef<Operation *> ops) {
1564 llvm::SmallDenseSet<uint32_t> defined;
1565 SmallVector<uint32_t> uses, defs;
1566 for (Operation *o : ops) {
1569 collectLocalRegisterDefsUses(*o, uses, defs);
1570 for (uint32_t r : uses)
1571 if (!defined.contains(r))
1573 defined.insert(defs.begin(), defs.end());
1575 return std::nullopt;
1587static void addChainDataCost(StringRef sym_name,
const SymbolTable &symbolTable,
1589 auto chain = symbolTable.lookup<AIEX::CertUcDmaChainOp>(sym_name);
1590 if (!chain || !seen.
chains.insert(chain).second)
1592 for (
auto bdOp : chain.getBody().front().getOps<AIEX::CertUcDmaBdOp>()) {
1594 auto global = symbolTable.lookup<memref::GlobalOp>(bdOp.getRemoteAddress());
1595 if (!global || !seen.
payloads.insert(global).second)
1597 auto initVal = global.getInitialValue();
1600 auto data = dyn_cast<DenseIntElementsAttr>(*initVal);
1603 data_cost += data.getNumElements() * 4;
1612static void updateCostForOp(Operation &o,
const SymbolTable &symbolTable,
1613 uint32_t &text_cost, uint32_t &data_cost,
1618 if (isa<AIEX::CertLocalBarrierOp>(o)) {
1620 }
else if (isa<AIEX::CertRemoteBarrierOp>(o)) {
1622 }
else if (isa<AIEX::CertRelAcqSyncOp>(o)) {
1624 }
else if (isa<AIEX::CertWaitTCTSOp>(o)) {
1626 }
else if (isa<AIEX::CertMaskWrite32Op, AIEX::CertMaskPoll32Op>(o)) {
1628 }
else if (isa<AIEX::CertWrite32Op>(o)) {
1630 }
else if (isa<AIEX::CertApplyOffset57Op>(o)) {
1632 }
else if (isa<AIEX::CertNopOp>(o)) {
1634 }
else if (isa<AIEX::CertPreemptOp>(o)) {
1636 }
else if (isa<AIEX::CertLoadPdiOp>(o)) {
1639 }
else if (
auto syncOp = dyn_cast<AIEX::CertUcDmaWriteDesSyncOp>(o)) {
1641 addChainDataCost(syncOp.getSymbol(), symbolTable, data_cost, seen);
1642 }
else if (
auto desOp = dyn_cast<AIEX::CertUcDmaWriteDesOp>(o)) {
1644 addChainDataCost(desOp.getSymbol(), symbolTable, data_cost, seen);
1645 }
else if (isa<AIEX::CertWaitUcDmaOp>(o)) {
1647 }
else if (isa<AIEX::CertRead32Op>(o)) {
1649 }
else if (isa<AIEX::CertRead32DOp>(o)) {
1651 }
else if (isa<AIEX::CertWrite32DOp>(o)) {
1653 }
else if (isa<AIEX::CertSaveTimestampsOp>(o)) {
1655 }
else if (isa<AIEX::CertSaveRegisterOp>(o)) {
1660static uint32_t estimateCost(AIEX::CertPageOp op,
1661 const SymbolTable &symbolTable,
1662 uint32_t split_target, AIEX::CertJobOp &split_job,
1663 Block::iterator &split_iter,
1664 bool &found_split_point) {
1665 uint32_t text_cost = 32;
1666 uint32_t data_cost = 0;
1668 found_split_point =
false;
1670 for (
auto job : op.getBody().front().getOps<AIEX::CertJobOp>()) {
1672 bool splittable = !jobUsesLocalRegister(job);
1675 text_cost += cert_job_start_cost;
1676 for (
auto &o : job.getBody().front().getOperations()) {
1677 Block::iterator current(&o);
1678 if (splittable && !found_split_point && !isa<AIE::EndOp>(o) &&
1679 current != job.getBody().front().begin() &&
1680 (text_cost + data_cost) >= split_target) {
1682 split_iter = current;
1683 found_split_point =
true;
1686 updateCostForOp(o, symbolTable, text_cost, data_cost, seen);
1688 if (splittable && !found_split_point &&
1689 (text_cost + data_cost) >= split_target) {
1690 Block::iterator next = current;
1692 if (next != job.getBody().front().end() && !isa<AIE::EndOp>(*next)) {
1695 found_split_point =
true;
1696 }
else if (!isa<AIE::EndOp>(o) &&
1697 current != job.getBody().front().begin()) {
1701 split_iter = current;
1702 found_split_point =
true;
1706 text_cost += cert_job_end_cost;
1708 return text_cost + data_cost;
1724static void computeIndivisibleSpans(AIEX::CertPageOp page,
1725 const SymbolTable &symbolTable,
1726 std::map<Operation *, int> &flatIndex,
1727 SmallVectorImpl<IndivisibleSpan> &spans) {
1728 std::map<int, std::pair<int, int>> barriers;
1731 llvm::StringMap<int> firstPatch;
1732 auto payloadsOf = [&](StringRef chainSym) {
1733 SmallVector<StringRef> payloads;
1734 auto chain = symbolTable.lookup<AIEX::CertUcDmaChainOp>(chainSym);
1735 if (!chain || chain.getBody().empty())
1737 for (
auto bd : chain.getBody().front().getOps<AIEX::CertUcDmaBdOp>())
1738 payloads.push_back(bd.getRemoteAddress());
1743 for (
auto job : page.getBody().front().getOps<AIEX::CertJobOp>()) {
1744 for (
auto &o : job.getBody().front().getOperations()) {
1745 if (isa<AIE::EndOp>(o))
1747 flatIndex[&o] = idx;
1748 if (
auto bar = dyn_cast<AIEX::CertLocalBarrierOp>(o)) {
1749 int id =
static_cast<int>(bar.getLocalBarrierId());
1750 auto it = barriers.find(
id);
1751 if (it == barriers.end())
1752 barriers[id] = {idx, idx};
1754 it->second.first = std::min(it->second.first, idx);
1755 it->second.second = std::max(it->second.second, idx);
1757 }
else if (
auto patch = dyn_cast<AIEX::CertApplyOffset57Op>(o)) {
1758 firstPatch.try_emplace(patch.getSymbol(), idx);
1759 }
else if (StringRef pushed = pushedChain(o); !pushed.empty()) {
1760 for (StringRef payload : payloadsOf(pushed)) {
1761 auto it = firstPatch.find(payload);
1762 if (it != firstPatch.end())
1769 for (
auto &[
id, span] : barriers)
1774static bool cutSeparatesSpan(ArrayRef<IndivisibleSpan> spans,
int p,
1776 const auto *it = llvm::find_if(
1778 if (it == spans.end())
1788static bool findLegalSplitPoint(AIEX::CertPageOp page,
1789 const SymbolTable &symbolTable,
1790 uint32_t split_target,
1791 const std::map<Operation *, int> &flatIndex,
1792 ArrayRef<IndivisibleSpan> spans,
1793 AIEX::CertJobOp &split_job,
1794 Block::iterator &split_iter) {
1795 uint32_t text_cost = 32;
1796 uint32_t data_cost = 0;
1799 uint32_t best_delta = 0;
1802 bool sawOpInEarlierJob =
false;
1803 for (
auto job : page.getBody().front().getOps<AIEX::CertJobOp>()) {
1804 text_cost += cert_job_start_cost;
1805 bool splittable = !jobUsesLocalRegister(job);
1806 bool sawOpInJob =
false;
1807 for (
auto &o : job.getBody().front().getOperations()) {
1808 if (isa<AIE::EndOp>(o))
1810 Block::iterator current(&o);
1816 if (sawOpInJob ? splittable : sawOpInEarlierJob) {
1817 auto fi = flatIndex.find(&o);
1818 int p = fi == flatIndex.end() ? 0 : fi->second;
1819 if (!cutSeparatesSpan(spans, p)) {
1820 uint32_t cost = text_cost + data_cost;
1822 cost > split_target ? cost - split_target : split_target - cost;
1823 if (!found || delta < best_delta) {
1827 split_iter = current;
1831 updateCostForOp(o, symbolTable, text_cost, data_cost, seen);
1834 text_cost += cert_job_end_cost;
1835 sawOpInEarlierJob |= sawOpInJob;
1848 IsolateFullPageOpsPattern(MLIRContext *context,
bool *hadError)
1849 : OpRewritePattern(context), hadError(hadError) {}
1854 static AIEX::CertPageOp makePageAfter(PatternRewriter &rewriter,
1855 Operation *after, Location loc) {
1856 rewriter.setInsertionPointAfter(after);
1857 auto page = AIEX::CertPageOp::create(rewriter, loc);
1858 page.getBody().push_back(
new Block());
1859 AIEX::CertPageOp::ensureTerminator(page.getBody(), rewriter, loc);
1863 LogicalResult matchAndRewrite(AIEX::CertJobOp jobOp,
1864 PatternRewriter &rewriter)
const override {
1867 Operation *fullPageOp =
nullptr;
1868 Block::iterator fullPageOpIter;
1869 for (Block::iterator it = jobOp.getBody().front().begin();
1870 it != jobOp.getBody().front().end(); ++it) {
1871 if (isa<AIEX::CertLoadPdiOp, AIEX::CertPreemptOp>(*it)) {
1873 fullPageOpIter = it;
1881 auto parentPage = jobOp->getParentOfType<AIEX::CertPageOp>();
1887 for (Operation &op : jobOp.getBody().front().getOperations())
1888 if (!isa<AIE::EndOp>(op))
1896 SmallVector<Operation *> leadingSiblings, trailingSiblings;
1897 bool pastSelf =
false;
1898 for (Operation &op : parentPage.getBody().front()) {
1899 if (isa<AIE::EndOp>(op))
1901 if (&op == jobOp.getOperation()) {
1905 (pastSelf ? trailingSiblings : leadingSiblings).push_back(&op);
1909 if (opCount == 1 && leadingSiblings.empty() && trailingSiblings.empty())
1912 auto loc = jobOp.getLoc();
1916 Operation *anchor = parentPage.getOperation();
1920 auto isolatedPage = makePageAfter(rewriter, anchor, loc);
1921 jobOp->moveBefore(isolatedPage.getBody().front().getTerminator());
1922 anchor = isolatedPage;
1926 auto parentDevice = jobOp->getParentOfType<AIE::DeviceOp>();
1927 uint32_t maxJobId = 0;
1928 parentDevice.walk([&](AIEX::CertJobOp certJobOp) {
1929 maxJobId = std::max(maxJobId, certJobOp.getJobId());
1931 uint32_t beforeJobId = jobOp.getJobId();
1932 uint32_t fullPageJobId = maxJobId + 1;
1933 uint32_t afterJobId = maxJobId + 2;
1935 SmallVector<Operation *> beforeOps, afterOps;
1936 for (Block::iterator it = jobOp.getBody().front().begin();
1937 it != fullPageOpIter; ++it)
1938 if (!isa<AIE::EndOp>(*it))
1939 beforeOps.push_back(&*it);
1940 Block::iterator afterStart = std::next(fullPageOpIter);
1941 for (Block::iterator it = afterStart; it != jobOp.getBody().front().end();
1943 if (!isa<AIE::EndOp>(*it))
1944 afterOps.push_back(&*it);
1948 if (
auto crossing = findLocalRegisterCrossingIn(afterOps)) {
1949 StringRef name = fullPageOp->getName().getStringRef();
1951 return jobOp.emitError()
1952 <<
"job " << jobOp.getJobId() <<
" must be split around its "
1953 << name <<
", but local register r" << *crossing
1954 <<
" is written before it and read after it; keep that "
1955 "register's uses on one side of the "
1956 << name <<
", or put them in a separate job";
1959 auto emitJobPage = [&](uint32_t jobId, ArrayRef<Operation *> ops) {
1960 auto page = makePageAfter(rewriter, anchor, loc);
1961 rewriter.setInsertionPoint(page.getBody().front().getTerminator());
1962 auto job = AIEX::CertJobOp::create(rewriter, loc, jobId);
1963 job.getBody().push_back(
new Block());
1964 AIEX::CertJobOp::ensureTerminator(job.getBody(), rewriter, loc);
1965 for (Operation *op : ops)
1966 op->moveBefore(job.getBody().front().getTerminator());
1970 if (!beforeOps.empty())
1971 emitJobPage(beforeJobId, beforeOps);
1972 emitJobPage(fullPageJobId, {fullPageOp});
1973 if (!afterOps.empty())
1974 emitJobPage(afterJobId, afterOps);
1976 rewriter.eraseOp(jobOp);
1981 if (!trailingSiblings.empty()) {
1982 auto tailPage = makePageAfter(rewriter, anchor, loc);
1983 for (Operation *op : trailingSiblings)
1984 op->moveBefore(tailPage.getBody().front().getTerminator());
1987 if (llvm::all_of(parentPage.getBody().front(),
1988 [](Operation &op) { return isa<AIE::EndOp>(op); }))
1989 rewriter.eraseOp(parentPage);
2005 SplitCertPageOpPattern(MLIRContext *context,
const SymbolTable &symbolTable,
2007 : OpRewritePattern(context), symbolTable(symbolTable),
2008 hadError(hadError) {}
2010 LogicalResult matchAndRewrite(AIEX::CertPageOp op,
2011 PatternRewriter &rewriter)
const override {
2013 constexpr uint32_t split_threshold = cert_page_size;
2015 AIEX::CertJobOp split_job;
2016 Block::iterator split_iter;
2017 bool found_split_point =
false;
2018 uint32_t cost = estimateCost(op, symbolTable, cert_page_size / 2, split_job,
2019 split_iter, found_split_point);
2020 LLVM_DEBUG(llvm::outs()
2021 <<
"Estimate cost for page: " <<
" is " << cost <<
"\n");
2023 if (cost < split_threshold)
2026 bool isImplicit = op->hasAttr(
"cert.implicit");
2028 std::map<Operation *, int> flatIndex;
2029 SmallVector<IndivisibleSpan> spans;
2030 computeIndivisibleSpans(op, symbolTable, flatIndex, spans);
2036 if (!found_split_point)
2038 findLegalSplitPoint(op, symbolTable, cert_page_size / 2, flatIndex,
2039 {}, split_job, split_iter);
2046 if (!found_split_point) {
2047 if (cost > cert_page_limit) {
2048 auto jobs = op.getBody().front().getOps<AIEX::CertJobOp>();
2049 auto blocker = llvm::find_if(jobs, jobUsesLocalRegister);
2050 if (blocker != jobs.end())
2051 op.emitError() <<
"cert.page is an estimated " << cost
2052 <<
" bytes, over the " << cert_page_limit
2053 <<
"-byte microcontroller page limit, and cannot be "
2055 << (*blocker).getJobId()
2056 <<
" uses local registers r0..r7, which do not "
2057 "survive a split; break it into smaller jobs "
2058 "that each keep a register's definition and "
2061 op.emitError() <<
"cert.page is an estimated " << cost
2062 <<
" bytes, over the " << cert_page_limit
2063 <<
"-byte microcontroller page limit, and offers no "
2064 "split point (a split must leave at least one op "
2065 "on each page); break the oversized job into "
2080 if (!spans.empty()) {
2081 auto fi = flatIndex.find(&*split_iter);
2082 int p = fi == flatIndex.end() ? 0 : fi->second;
2084 if (cutSeparatesSpan(spans, p, &kind)) {
2085 if (!findLegalSplitPoint(op, symbolTable, cert_page_size / 2, flatIndex,
2086 spans, split_job, split_iter)) {
2092 if (cost > cert_page_limit) {
2094 StringRef what = barrier
2095 ?
"a local_barrier participant set"
2096 :
"a cert.apply_offset_57 from a uC-DMA push "
2097 "of the payload it patches";
2100 <<
"implicit cert.page is an estimated " << cost
2101 <<
" bytes, over the " << cert_page_limit
2102 <<
"-byte microcontroller page limit, and cannot be split "
2103 "without separating "
2105 <<
" across pages; insert an explicit cert.page boundary to "
2106 << (barrier ?
"co-locate the barrier participants"
2107 :
"co-locate each apply_offset_57 with its "
2111 <<
"cert.page is an estimated " << cost <<
" bytes, over the "
2113 <<
"-byte microcontroller page limit, and cannot be split "
2114 "without separating "
2115 << what <<
" across pages; reduce the page's contents or "
2116 << (barrier ?
"regroup its local_barrier participants"
2117 :
"regroup its apply_offset_57 ops and their "
2119 <<
" so that a legal split point exists";
2132 <<
"auto-split of implicit cert.page serializes cooperatively-"
2133 "scheduled jobs into separate sequential pages; insert an "
2135 "cert.page boundary to control where the split occurs";
2137 auto loc = op.getLoc();
2141 bool jobBoundaryCut = split_iter == split_job.getBody().front().begin();
2142 [[maybe_unused]]
auto earlierJobHasRealOp = [&]() {
2143 for (
auto job : op.getBody().front().getOps<AIEX::CertJobOp>()) {
2144 if (job == split_job)
2146 for (
auto &o : job.getBody().front().getOperations())
2147 if (!isa<AIE::EndOp>(o))
2152 assert((!jobBoundaryCut || earlierJobHasRealOp()) &&
2153 "a job-boundary cut needs a preceding job with content, else the "
2154 "earlier page is empty and the split makes no progress");
2159 auto cloneJobRange = [&](AIEX::CertJobOp sourceJob, uint32_t jobId,
2160 Block::iterator begin, Block::iterator end) {
2161 auto newJobOp = AIEX::CertJobOp::create(rewriter, loc, jobId);
2162 Block *newJobBlock =
new Block();
2163 newJobOp.getBody().push_back(newJobBlock);
2166 Block &sourceBlock = sourceJob.getBody().front();
2167 for (BlockArgument arg : sourceBlock.getArguments()) {
2168 BlockArgument newArg =
2169 newJobBlock->addArgument(arg.getType(), arg.getLoc());
2170 mapper.map(arg, newArg);
2173 rewriter.setInsertionPointToStart(newJobBlock);
2174 for (Block::iterator oi = begin; oi != end; ++oi) {
2175 if (!isa<AIE::EndOp>(*oi))
2176 rewriter.clone(*oi, mapper);
2178 AIEX::CertJobOp::ensureTerminator(newJobOp.getBody(), rewriter, loc);
2181 rewriter.setInsertionPoint(op);
2182 auto newPageOp0 = AIEX::CertPageOp::create(rewriter, loc);
2183 Block *newPageBlock0 =
new Block();
2184 newPageOp0.getBody().push_back(newPageBlock0);
2186 auto newPageOp1 = AIEX::CertPageOp::create(rewriter, loc);
2187 Block *newPageBlock1 =
new Block();
2188 newPageOp1.getBody().push_back(newPageBlock1);
2193 newPageOp0->setAttr(
"cert.implicit", rewriter.getUnitAttr());
2194 newPageOp1->setAttr(
"cert.implicit", rewriter.getUnitAttr());
2197 for (
auto job : op.getBody().front().getOps<AIEX::CertJobOp>()) {
2198 if (job == split_job) {
2199 if (jobBoundaryCut) {
2202 rewriter.setInsertionPointToEnd(newPageBlock1);
2203 cloneJobRange(job, job.getJobId(), job.getBody().front().begin(),
2204 job.getBody().front().end());
2207 rewriter.setInsertionPointToEnd(newPageBlock0);
2208 cloneJobRange(job, job.getJobId(), job.getBody().front().begin(),
2211 rewriter.setInsertionPointToEnd(newPageBlock1);
2212 cloneJobRange(job, job.getJobId() + 1, split_iter,
2213 job.getBody().front().end());
2217 if (job->isBeforeInBlock(split_job)) {
2218 rewriter.setInsertionPointToEnd(newPageBlock0);
2219 cloneJobRange(job, job.getJobId(), job.getBody().front().begin(),
2220 job.getBody().front().end());
2222 rewriter.setInsertionPointToEnd(newPageBlock1);
2223 cloneJobRange(job, job.getJobId(), job.getBody().front().begin(),
2224 job.getBody().front().end());
2228 AIEX::CertPageOp::ensureTerminator(newPageOp0.getBody(), rewriter, loc);
2229 AIEX::CertPageOp::ensureTerminator(newPageOp1.getBody(), rewriter, loc);
2231 rewriter.eraseOp(op);
2235 const SymbolTable &symbolTable;
2241static void inlineDefaultGroup(AIE::DeviceOp dev) {
2242 Block *body = dev.getBody();
2243 SmallVector<AIEX::CertAttachToGroupOp> zeroGroups;
2244 for (
auto grp : body->getOps<AIEX::CertAttachToGroupOp>())
2245 if (grp.getGroupId() == 0)
2246 zeroGroups.push_back(grp);
2247 for (
auto grp : zeroGroups) {
2248 SmallVector<Operation *> inner;
2249 for (Operation &op : grp.getBody().front())
2250 if (!isa<AIE::EndOp>(op))
2251 inner.push_back(&op);
2252 for (Operation *op : inner)
2253 op->moveBefore(grp.getOperation());
2261static void formImplicitPagesInBlock(Block *body, OpBuilder &builder) {
2263 SmallVector<Operation *> topOps;
2264 for (Operation &op : *body)
2265 topOps.push_back(&op);
2273 auto openPage = [&](Location loc, Operation *before,
2274 bool implicit) -> AIEX::CertPageOp {
2275 builder.setInsertionPoint(before);
2276 auto page = AIEX::CertPageOp::create(builder, loc);
2278 page->setAttr(
"cert.implicit", builder.getUnitAttr());
2279 Block *b =
new Block();
2280 page.getBody().push_back(b);
2281 AIEX::CertPageOp::ensureTerminator(page.getBody(), builder, loc);
2288 auto waitTctsActors = [](AIEX::CertJobOp job) {
2289 std::set<std::pair<int, int>> actors;
2290 job.walk([&](AIEX::CertWaitTCTSOp wait) {
2291 actors.insert({wait.getTileId(), wait.getChannelId()});
2296 AIEX::CertPageOp currentPage;
2298 std::set<std::pair<int, int>> pageActors;
2299 auto closePage = [&]() {
2300 currentPage =
nullptr;
2304 for (Operation *op : topOps) {
2305 if (
auto job = dyn_cast<AIEX::CertJobOp>(op)) {
2306 if (job->hasAttr(
"cert.configure")) {
2312 auto page = openPage(job.getLoc(), job,
false);
2313 job->moveBefore(page.getBody().front().getTerminator());
2323 auto actors = waitTctsActors(job);
2325 llvm::any_of(actors, [&](
const std::pair<int, int> &actor) {
2326 return pageActors.count(actor);
2330 currentPage = openPage(job.getLoc(), job,
true);
2331 job->moveBefore(currentPage.getBody().front().getTerminator());
2332 pageActors.insert(actors.begin(), actors.end());
2333 }
else if (isa<AIEX::CertPageOp>(op)) {
2341static void formImplicitPages(AIE::DeviceOp dev) {
2342 OpBuilder builder(dev.getContext());
2343 formImplicitPagesInBlock(dev.getBody(), builder);
2346 SmallVector<AIEX::CertAttachToGroupOp> groups;
2347 for (
auto grp : dev.getBody()->getOps<AIEX::CertAttachToGroupOp>())
2348 groups.push_back(grp);
2349 for (
auto grp : groups)
2350 formImplicitPagesInBlock(&grp.getBody().front(), builder);
2359static void moveConfigPageFirst(AIE::DeviceOp dev) {
2360 Block *body = dev.getBody();
2361 AIEX::CertPageOp configPage;
2362 for (
auto page : body->getOps<AIEX::CertPageOp>()) {
2363 for (
auto job : page.getBody().front().getOps<AIEX::CertJobOp>()) {
2364 if (job->hasAttr(
"cert.configure")) {
2372 if (configPage && &body->front() != configPage.getOperation())
2373 configPage->moveBefore(&body->front());
2377static void lowerPagePlacementToGroups(AIE::DeviceOp dev) {
2378 Block *body = dev.getBody();
2381 std::map<int32_t, SmallVector<AIEX::CertPageOp, 4>> byGroup;
2382 for (
auto page : body->getOps<AIEX::CertPageOp>()) {
2383 if (
auto placement = page.getPlacementAttr()) {
2384 auto gid =
static_cast<int32_t
>(placement.getInt());
2386 byGroup[gid].push_back(page);
2390 OpBuilder builder(dev.getContext());
2391 for (
auto &entry : byGroup) {
2392 int32_t gid = entry.first;
2393 SmallVector<AIEX::CertPageOp, 4> &pages = entry.second;
2397 builder.setInsertionPoint(pages.front());
2398 auto grp = AIEX::CertAttachToGroupOp::create(builder, dev.getLoc(), gid);
2399 Block *grpBlock =
new Block();
2400 grp.getBody().push_back(grpBlock);
2404 for (AIEX::CertPageOp page : pages) {
2405 page->removeAttr(
"placement");
2406 page->moveBefore(grpBlock, grpBlock->end());
2408 AIEX::CertAttachToGroupOp::ensureTerminator(grp.getBody(), builder,
2416static LogicalResult allocateJobIds(AIE::DeviceOp device) {
2417 WalkResult result = device.walk([&](AIEX::CertPageOp page) {
2418 std::bitset<AIEX::CertJobOp::NumSlots> used;
2419 SmallVector<AIEX::CertJobOp> collisions;
2421 for (
auto job : page.getBody().front().getOps<AIEX::CertJobOp>()) {
2422 unsigned slot = job.getJobId() % AIEX::CertJobOp::NumSlots;
2423 if (used.test(slot))
2424 collisions.push_back(job);
2428 if (count > AIEX::CertJobOp::NumSlots) {
2429 page.emitError() <<
"cert.page has " << count <<
" jobs, exceeding the "
2430 << AIEX::CertJobOp::NumSlots
2431 <<
" firmware job slots; reduce the jobs on this page";
2432 return WalkResult::interrupt();
2435 for (
auto job : collisions) {
2436 while (used.test(slot))
2441 return WalkResult::advance();
2443 return failure(result.wasInterrupted());
2446struct AIECertPagesPass
2447 : xilinx::AIEX::impl::AIECertPagesBase<AIECertPagesPass> {
2448 void runOnOperation()
override {
2449 inlineDefaultGroup(getOperation());
2451 formImplicitPages(getOperation());
2454 moveConfigPageFirst(getOperation());
2463 lowerPagePlacementToGroups(getOperation());
2467 SymbolTable symbolTable(getOperation());
2470 bool isolateError =
false;
2471 RewritePatternSet isolatePatterns(&getContext());
2472 isolatePatterns.insert<IsolateFullPageOpsPattern>(&getContext(),
2475 applyPatternsGreedily(getOperation(), std::move(isolatePatterns))))
2476 signalPassFailure();
2478 return signalPassFailure();
2484 bool splitError =
false;
2485 RewritePatternSet p1(&getContext());
2486 p1.insert<SplitCertPageOpPattern>(&getContext(), symbolTable, &splitError);
2487 if (failed(applyPatternsGreedily(getOperation(), std::move(p1))))
2488 signalPassFailure();
2490 signalPassFailure();
2492 if (failed(allocateJobIds(getOperation())))
2493 return signalPassFailure();
2497 llvm::DenseMap<StringAttr, unsigned> useCounts;
2498 if (
auto uses = SymbolTable::getSymbolUses(&getOperation().getRegion()))
2499 for (
const SymbolTable::SymbolUse &use : *uses)
2500 ++useCounts[use.getSymbolRef().getRootReference()];
2501 RewritePatternSet p3(&getContext());
2502 p3.insert<MergeConsecutiveCertUcDmaWriteDesSyncOps>(&getContext(),
2503 symbolTable, useCounts);
2504 if (failed(applyPatternsGreedily(getOperation(), std::move(p3))))
2505 return signalPassFailure();
2512 return std::make_unique<AIENpuToCertPass>();
2516 return std::make_unique<AIECertPagesPass>();
const V & get(const std::map< K, V > &pa, K k)
std::unique_ptr< mlir::OperationPass< mlir::ModuleOp > > createAIENpuToCertPass()
std::optional< uint32_t > getConstantIntOperand(mlir::Value v)
std::unique_ptr< mlir::OperationPass< AIE::DeviceOp > > createAIECertPagesPass()
mlir::LogicalResult inlineWriteConfigs(mlir::ModuleOp module)
mlir::Operation * lookupNamedOpIn(mlir::Operation *symbolTableOp, mlir::StringAttr name)
const AIETargetModel & getTargetModel(mlir::Operation *op)
llvm::DenseSet< Operation * > chains
llvm::DenseSet< Operation * > payloads