31#include "mlir/Dialect/Arith/IR/Arith.h"
32#include "mlir/Dialect/MemRef/IR/MemRef.h"
33#include "mlir/Pass/Pass.h"
35#include "llvm/ADT/DenseMap.h"
36#include "llvm/ADT/MapVector.h"
37#include "llvm/ADT/StringRef.h"
38#include "llvm/Support/FormatVariadic.h"
39#include "llvm/Support/raw_ostream.h"
46#define GEN_PASS_DEF_AIEEXPANDLOADPDI
47#include "aie/Dialect/AIEX/Transforms/AIEXPasses.h.inc"
50#define DEBUG_TYPE "aie-expand-load-pdi"
59static AIE::DeviceOp controlOverlayOf(AIE::DeviceOp device) {
60 auto ref = device.getOverlayAttr();
61 return ref ? device->getParentOfType<ModuleOp>().lookupSymbol<AIE::DeviceOp>(
66using OverlayCopies = llvm::DenseMap<AIE::DeviceOp, AIE::DeviceOp>;
69static AIE::DeviceOp overlayPreload(AIE::DeviceOp overlay,
unsigned parity,
70 OverlayCopies &copies) {
73 auto [it, inserted] = copies.try_emplace(overlay);
76 auto copy = cast<AIE::DeviceOp>(overlay->clone());
77 copy.setSymName((overlay.getSymName() +
"_copy").str());
78 SymbolTable(overlay->getParentOfType<ModuleOp>()).insert(copy);
79 return it->second = copy;
85verifyOverlayReachesTiles(NpuLoadPdiOp loadPdiOp, AIE::DeviceOp overlay,
86 iterator_range<Block::iterator> packets) {
87 std::set<AIE::TileID> reached;
88 overlay.walk([&](AIE::SwitchboxOp sb) {
89 for (Operation &op : sb.getConnections().front()) {
90 std::optional<AIE::Port> dest;
91 if (
auto master = dyn_cast<AIE::MasterSetOp>(op))
92 dest = master.destPort();
93 else if (
auto connect = dyn_cast<AIE::ConnectOp>(op))
94 dest = connect.destPort();
95 if (dest ==
AIE::Port{AIE::WireBundle::TileControl, 0})
96 reached.insert({sb.colIndex(), sb.rowIndex()});
99 for (Operation &op : packets) {
100 auto packet = dyn_cast<NpuControlPacketOp>(op);
103 AIE::TileID tile{
static_cast<int>(packet.getColumnFromAddr()),
104 static_cast<int>(packet.getRowFromAddr())};
105 if (reached.count(tile))
107 return loadPdiOp.emitError()
108 <<
"a control-packet reload configures tile (" << tile.col <<
", "
109 << tile.row <<
"), but @" << overlay.getSymName()
110 <<
" routes no control packets to it";
118static AIE::DeviceOp getOrCreateEmptyDevice(ModuleOp moduleOp,
119 AIE::AIEDevice deviceType,
120 unsigned parity, Location loc) {
121 std::string emptyName =
"empty_" + std::to_string(parity);
122 if (
auto existing = moduleOp.lookupSymbol<AIE::DeviceOp>(emptyName)) {
124 loc = FusedLoc::get(moduleOp.getContext(), {existing.getLoc(), loc});
125 existing->setLoc(loc);
126 existing.getRegion().front().getTerminator()->setLoc(loc);
130 OpBuilder builder(moduleOp.getContext());
131 builder.setInsertionPointToStart(moduleOp.getBody());
132 auto emptyDevice = AIE::DeviceOp::create(
133 builder, loc, deviceType, builder.getStringAttr(emptyName),
nullptr);
134 emptyDevice.getRegion().emplaceBlock();
135 builder.setInsertionPointToEnd(&emptyDevice.getRegion().front());
136 AIE::EndOp::create(builder, loc);
142struct CtrlPktReload {
144 AIE::DeviceOp device;
145 NpuLoadPdiOp preload;
146 AIE::DeviceOp overlay;
150static void setInsertionPointToSegmentEnd(OpBuilder &builder,
151 NpuLoadPdiOp loadPdiOp) {
152 Operation *op = loadPdiOp->getNextNode();
153 while (op && !isa<NpuLoadPdiOp>(op) && !op->hasTrait<OpTrait::IsTerminator>())
154 op = op->getNextNode();
156 builder.setInsertionPoint(op);
158 builder.setInsertionPointToEnd(loadPdiOp->getBlock());
168static LogicalResult transformLoadPdi(
169 NpuLoadPdiOp loadPdiOp, ModuleOp moduleOp,
unsigned index,
170 AIEX::ExpandMode defaultMode,
bool keepPreload,
bool teardown,
172 SmallVectorImpl<CtrlPktReload> &ctrlPktReloads, OverlayCopies &copies) {
173 OpBuilder builder(loadPdiOp);
176 auto deviceRefAttr = loadPdiOp.getDeviceRefAttr();
177 if (!deviceRefAttr) {
182 AIEX::ExpandMode mode = loadPdiOp.getExpandMode().value_or(defaultMode);
183 if (mode == AIEX::ExpandMode::none)
185 bool ctrlPkt = (mode == AIEX::ExpandMode::ctrlpkt);
187 auto referencedDevice = moduleOp.lookupSymbol<AIE::DeviceOp>(deviceRefAttr);
188 if (!referencedDevice) {
189 loadPdiOp.emitError(
"Referenced symbol '")
190 << deviceRefAttr.getValue() <<
"' is not a device";
194 AIE::DeviceOp overlay;
196 overlay = controlOverlayOf(referencedDevice);
198 return referencedDevice.emitOpError(
199 "requires an overlay symbol reference for control-packet "
206 FlatSymbolRefAttr preloadRef;
208 preloadRef = FlatSymbolRefAttr::get(
209 overlayPreload(overlay, index % 2, copies).getSymNameAttr());
211 AIE::DeviceOp emptyDevice =
212 getOrCreateEmptyDevice(moduleOp, referencedDevice.getDevice(),
213 index % 2, loadPdiOp.getLoc());
214 preloadRef = FlatSymbolRefAttr::get(emptyDevice.getSymNameAttr());
217 AIEX::ExpandModeAttr::get(builder.getContext(), AIEX::ExpandMode::none);
219 NpuLoadPdiOp::create(builder, loadPdiOp.getLoc(), preloadRef,
221 nullptr, expandNone);
223 NpuLoadPdiOp::create(builder, loadPdiOp.getLoc(), preloadRef,
224 loadPdiOp.getIdAttr(), loadPdiOp.getSizeAttr(),
225 loadPdiOp.getAddressAttr(), expandNone);
227 Operation *beforeConfig = loadPdiOp->getPrevNode();
230 NpuWriteConfigOp::create(builder, loadPdiOp.getLoc(), deviceRefAttr);
232 setInsertionPointToSegmentEnd(builder, loadPdiOp);
233 NpuWriteTeardownOp::create(builder, loadPdiOp.getLoc(), deviceRefAttr);
239 ctrlPktReloads.push_back({loadPdiOp, referencedDevice,
240 cast<NpuLoadPdiOp>(beforeConfig), overlay});
246 builder, referencedDevice, *configData,
"", outputType,
248 loadPdiOp.emitError(
"Failed to generate configuration operations");
251 if (failed(verifyOverlayReachesTiles(
253 {beforeConfig ? std::next(beforeConfig->getIterator())
254 : loadPdiOp->getBlock()->begin(),
255 loadPdiOp->getIterator()})))
260 Location loc = loadPdiOp.getLoc();
261 setInsertionPointToSegmentEnd(builder, loadPdiOp);
264 builder, referencedDevice, *teardownData, outputType,
266 emitError(loc,
"Failed to generate teardown operations");
275verifyExpansion(ModuleOp module, ArrayRef<NpuLoadPdiOp> loadPdiOps,
276 function_ref<AIEX::ExpandMode(NpuLoadPdiOp)> expandMode,
277 ArrayRef<bool> keepPreloads,
bool teardown,
bool initEntry) {
278 if (initEntry && !teardown)
279 return module.emitError(
280 "aie-expand-load-pdi: init-entry requires teardown");
282 for (
auto device : module.getOps<AIE::DeviceOp>())
284 return existing->emitOpError()
286 <<
"', which aie-expand-load-pdi reserves under init-entry";
289 llvm::MapVector<AIE::RuntimeSequenceOp, SmallVector<AIEX::ConfigStep>> steps;
290 for (
auto [load, keepPreload] : llvm::zip_equal(loadPdiOps, keepPreloads)) {
291 NpuLoadPdiOp loadPdiOp = load;
292 if (
auto seq = loadPdiOp->getParentOfType<AIE::RuntimeSequenceOp>()) {
293 FlatSymbolRefAttr ref = loadPdiOp.getDeviceRefAttr();
294 bool expanded = expandMode(loadPdiOp) != AIEX::ExpandMode::none;
295 steps[seq].push_back(
296 {loadPdiOp, expanded ? NpuLoadPdiOp() : loadPdiOp,
297 expanded && keepPreload,
298 ref ? module.lookupSymbol<AIE::DeviceOp>(ref) :
nullptr});
301 for (
auto &[seq, seqSteps] : steps)
306 for (NpuLoadPdiOp loadPdiOp : loadPdiOps) {
307 if (expandMode(loadPdiOp) == AIEX::ExpandMode::none)
311 auto dev =
module.lookupSymbol<AIE::DeviceOp>(loadPdiOp.getDeviceRefAttr());
313 return start->emitOpError(
314 "starts a shim DMA channel, which teardown cannot stop");
319struct AIEExpandLoadPdiPass
320 :
public xilinx::AIEX::impl::AIEExpandLoadPdiBase<AIEExpandLoadPdiPass> {
321 using AIEExpandLoadPdiBase::AIEExpandLoadPdiBase;
323 void getDependentDialects(DialectRegistry ®istry)
const override {
324 registry.insert<arith::ArithDialect, memref::MemRefDialect, AIE::AIEDialect,
325 AIEX::AIEXDialect>();
328 void runOnOperation()
override {
329 auto module = getOperation();
335 SmallVector<NpuLoadPdiOp> loadPdiOps;
338 [&](NpuLoadPdiOp loadPdiOp) { loadPdiOps.push_back(loadPdiOp); });
341 AIEX::ExpandMode defaultMode =
342 clCtrlPkt ? AIEX::ExpandMode::ctrlpkt : AIEX::ExpandMode::write32;
344 auto expandMode = [&](NpuLoadPdiOp loadPdiOp) {
345 return loadPdiOp.getDeviceRefAttr()
346 ? loadPdiOp.getExpandMode().value_or(defaultMode)
347 : AIEX::ExpandMode::none;
357 using ArrayState = std::pair<AIEX::ExpandMode, FlatSymbolRefAttr>;
358 struct PreloadParity {
359 unsigned firstParity = 0;
360 unsigned lastParity = 0;
362 AIE::AIEDevice device = {};
363 AIE::DeviceOp firstOverlay, lastOverlay;
365 struct SequenceLoads {
366 std::array<PreloadParity, 2> preloads;
367 ArrayState last = {AIEX::ExpandMode::none, {}};
369 llvm::MapVector<AIE::RuntimeSequenceOp, SequenceLoads> sequences;
370 for (
auto [index, loadPdiOp] : llvm::enumerate(loadPdiOps)) {
371 auto seq = loadPdiOp->getParentOfType<AIE::RuntimeSequenceOp>();
374 SequenceLoads &loads = sequences[seq];
375 AIEX::ExpandMode mode = expandMode(loadPdiOp);
376 loads.last = {mode, {}};
377 if (mode == AIEX::ExpandMode::none)
380 module.lookupSymbol<AIE::DeviceOp>(loadPdiOp.getDeviceRefAttr());
383 auto overlay = mode == AIEX::ExpandMode::ctrlpkt ? controlOverlayOf(dev)
385 if (mode == AIEX::ExpandMode::ctrlpkt && !overlay) {
386 dev.emitOpError(
"requires an overlay symbol reference for "
387 "control-packet configuration");
388 return signalPassFailure();
390 loads.last.second = overlay ? dev.getOverlayAttr() : FlatSymbolRefAttr{};
391 auto &entry = loads.preloads[mode == AIEX::ExpandMode::ctrlpkt];
393 entry.firstParity = index % 2;
395 entry.firstOverlay = overlay;
397 entry.lastParity = index % 2;
398 entry.lastOverlay = overlay;
399 entry.device = dev.getDevice();
409 llvm::MapVector<AIE::DeviceOp, ArrayState> initEntries;
411 for (
auto &[seq, loads] : sequences)
412 initEntries.try_emplace(seq->getParentOfType<AIE::DeviceOp>(),
414 for (
auto &[seq, loads] : sequences) {
415 auto device = seq->getParentOfType<AIE::DeviceOp>();
416 auto *it = initEntries.find(device);
417 if (it != initEntries.end() &&
418 (loads.last.first == AIEX::ExpandMode::none ||
419 loads.last != it->second))
420 initEntries.erase(it);
426 SmallVector<bool> keepPreloads;
427 llvm::DenseMap<Operation *, std::optional<ArrayState>> arrayState;
428 for (NpuLoadPdiOp loadPdiOp : loadPdiOps) {
429 auto seq = loadPdiOp->getParentOfType<AIE::RuntimeSequenceOp>();
430 std::optional<ArrayState> start;
432 if (
auto *it = initEntries.find(seq->getParentOfType<AIE::DeviceOp>());
433 it != initEntries.end())
435 std::optional<ArrayState> &state =
436 arrayState.try_emplace(seq, start).first->second;
438 module.lookupSymbol<AIE::DeviceOp>(loadPdiOp.getDeviceRefAttr());
439 auto mode = expandMode(loadPdiOp);
440 ArrayState next = {mode, dev && mode == AIEX::ExpandMode::ctrlpkt
441 ? dev.getOverlayAttr()
442 : FlatSymbolRefAttr{}};
443 keepPreloads.push_back(!clTeardown || state != next);
447 if (failed(verifyExpansion(module, loadPdiOps, expandMode, keepPreloads,
448 clTeardown, clInitEntry)))
449 return signalPassFailure();
451 OverlayCopies copies;
452 SmallVector<CtrlPktReload> ctrlPktReloads;
453 llvm::DenseMap<AIE::DeviceOp, AIEX::BlockwriteData> configData,
455 for (
auto [idx, loadPdiOp] : llvm::enumerate(loadPdiOps)) {
456 bool keepPreload = keepPreloads[idx];
458 if (clTeardown && expandMode(loadPdiOp) == AIEX::ExpandMode::ctrlpkt) {
459 auto device = loadPdiOp->getParentOfType<AIE::DeviceOp>();
461 &configData.try_emplace(device, device,
"loadpdi_").first->second;
462 teardown = &teardownData.try_emplace(device, device,
"teardown_")
465 if (failed(transformLoadPdi(loadPdiOp, module, idx, defaultMode,
466 keepPreload, clTeardown, config, teardown,
467 ctrlPktReloads, copies))) {
473 SmallVector<std::pair<Operation *, AIE::DeviceOp>> ctrlPktSites;
474 for (CtrlPktReload &reload : ctrlPktReloads)
475 ctrlPktSites.emplace_back(reload.load, reload.device);
481 for (CtrlPktReload &reload : ctrlPktReloads) {
482 if (failed(verifyOverlayReachesTiles(
483 reload.load, reload.overlay,
484 {std::next(reload.preload->getIterator()),
485 reload.load->getIterator()}))) {
492 for (
auto [device, kind] : initEntries) {
493 FlatSymbolRefAttr preloadRef =
494 kind.first == AIEX::ExpandMode::ctrlpkt
496 : FlatSymbolRefAttr::get(
497 getOrCreateEmptyDevice(module, device.getDevice(), 0,
500 OpBuilder builder(device.getContext());
501 builder.setInsertionPoint(
502 *device.getOps<AIE::RuntimeSequenceOp>().begin());
503 auto init = AIE::RuntimeSequenceOp::create(
504 builder, device.getLoc(),
506 AIE::TraceBufferAttr{}, ArrayAttr{}, ArrayAttr{});
507 builder.setInsertionPointToStart(&init.getBody().emplaceBlock());
508 NpuLoadPdiOp::create(builder, device.getLoc(), preloadRef,
512 AIEX::ExpandModeAttr::get(builder.getContext(),
513 AIEX::ExpandMode::none));
517 return signalPassFailure();
538 for (
auto &[seq, loads] : sequences) {
539 for (
auto [ctrlPkt, info] : llvm::enumerate(loads.preloads)) {
540 if (!info.seen || info.firstParity != info.lastParity ||
541 info.firstOverlay != info.lastOverlay)
543 unsigned parity = 1 - info.lastParity;
544 FlatSymbolRefAttr preloadRef;
546 preloadRef = FlatSymbolRefAttr::get(
547 overlayPreload(info.lastOverlay, parity, copies)
550 AIE::DeviceOp emptyDevice =
551 getOrCreateEmptyDevice(module, info.device, parity, seq.getLoc());
552 preloadRef = FlatSymbolRefAttr::get(emptyDevice.getSymNameAttr());
554 OpBuilder builder(seq.getContext());
555 Block &body = seq.getBody().front();
556 if (!body.empty() && body.back().hasTrait<OpTrait::IsTerminator>())
557 builder.setInsertionPoint(&body.back());
559 builder.setInsertionPointToEnd(&body);
560 NpuLoadPdiOp::create(builder, seq.getLoc(), preloadRef,
564 AIEX::ExpandModeAttr::get(seq.getContext(),
565 AIEX::ExpandMode::none));
573std::unique_ptr<mlir::OperationPass<mlir::ModuleOp>>
575 return std::make_unique<AIEExpandLoadPdiPass>();
constexpr llvm::StringLiteral initEntrySequenceName
Name of the runtime sequence aie-expand-load-pdi{init-entry} adds.
mlir::Operation * findShimDmaStart(AIE::DeviceOp device)
LogicalResult checkResidentsSurvive(AIE::RuntimeSequenceOp seq, ArrayRef< ConfigStep > steps)
std::unique_ptr< mlir::OperationPass< mlir::ModuleOp > > createAIEExpandLoadPdiPass()
mlir::LogicalResult insertConfigOps(llvm::ArrayRef< std::pair< mlir::Operation *, xilinx::AIE::DeviceOp > > sites, AIEToConfigurationOutputType outputType)
mlir::LogicalResult inlineWriteConfigs(mlir::ModuleOp module)
mlir::LogicalResult generateAndInsertConfigOps(mlir::OpBuilder &builder, xilinx::AIE::DeviceOp device, xilinx::AIEX::BlockwriteData &blockwriteData, llvm::StringRef clElfDir="", AIEToConfigurationOutputType outputType=AIEToConfigurationOutputType::Transaction, bool skipCtrlPktOverlay=false)
struct TileID { friend std::ostream &operator<<(std::ostream &os, const TileID &s) { os<< "TileID("<< s.col<< ", "<< s.row<< ")"; return os; } friend std::string to_string(const TileID &s) { std::ostringstream ss; ss<< s; return ss.str(); } friend llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const TileID &s) { os<< to_string(s); return os; } inline bool operator<(const TileID &rhs) const { return std::tie(col, row)< std::tie(rhs.col, rhs.row); } bool operator==(const TileID &rhs) const { return std::tie(col, row)==std::tie(rhs.col, rhs.row); } bool operator!=(const TileID &rhs) const { return !(*this==rhs);} int col, row;} TileID
mlir::LogicalResult generateAndInsertTeardownOps(mlir::OpBuilder &builder, xilinx::AIE::DeviceOp device, xilinx::AIEX::BlockwriteData &blockwriteData, AIEToConfigurationOutputType outputType=AIEToConfigurationOutputType::Transaction, bool skipCtrlPktOverlay=false)
struct Port { WireBundle bundle; int channel; bool operator==(const Port &rhs) const { return std::tie(bundle, channel)==std::tie(rhs.bundle, rhs.channel); } bool operator!=(const Port &rhs) const { return !(*this==rhs);} bool operator<(const Port &rhs) const { return std::tie(bundle, channel)< std::tie(rhs.bundle, rhs.channel); } friend std::ostream &operator<<(std::ostream &os, const Port &port) { os<< "("; os<< stringifyWireBundle(port.bundle).str(); os<< ": "<< std::to_string(port.channel)<< ")"; return os; } friend llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const Port &port) { os<< to_string(port); return os; } } Port