MLIR-AIE 1.4.6.dev
AIECoreToStandard.cpp
Go to the documentation of this file.
1//===- AIECoreToStandard.cpp ------------------------------------*- C++ -*-===//
2//
3// Copyright (C) 2019-2022 Xilinx, Inc.
4// Copyright (C) 2022-2026 Advanced Micro Devices, Inc.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
13
14#include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
15#include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h"
16#include "mlir/Dialect/Arith/IR/Arith.h"
17#include "mlir/Dialect/ControlFlow/IR/ControlFlow.h"
18#include "mlir/Dialect/DLTI/DLTI.h"
19#include "mlir/Dialect/Index/IR/IndexDialect.h"
20#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
21#include "mlir/Dialect/Math/IR/Math.h"
22#include "mlir/Dialect/Ptr/IR/PtrOps.h"
23#include "mlir/Dialect/UB/IR/UBOps.h"
24#include "mlir/Dialect/Vector/IR/VectorOps.h"
25#include "mlir/IR/Attributes.h"
26#include "mlir/IR/IRMapping.h"
27#include "mlir/IR/PatternMatch.h"
28#include "mlir/Pass/Pass.h"
29#include "mlir/Tools/mlir-translate/MlirTranslateMain.h"
30#include "mlir/Transforms/DialectConversion.h"
31
32namespace xilinx::AIE {
33#define GEN_PASS_DEF_AIECORETOSTANDARD
34#include "aie/Dialect/AIE/Transforms/AIEPasses.h.inc"
35} // namespace xilinx::AIE
36
37using namespace mlir;
38using namespace mlir::vector;
39using namespace xilinx;
40using namespace xilinx::AIE;
41
42static StringRef getArchTargetTriple(AIEArch arch) {
43 switch (arch) {
44 case AIEArch::AIE1:
45 return "aie";
46 case AIEArch::AIE2:
47 return "aie2";
48 case AIEArch::AIE2p:
49 return "aie2p";
50 case AIEArch::AIE2ps:
51 return "aie2ps";
52 }
53 llvm::report_fatal_error("unsupported arch");
54}
55
56// AIE2PS has no core intrinsic set (see declareAIEIntrinsics), so an op that
57// lowers to an intrinsic call is diagnosed rather than left behind.
58static LogicalResult checkCoreIntrinsics(Operation *op) {
59 if (!isa<AIE2PSTargetModel>(op->getParentOfType<DeviceOp>().getTargetModel()))
60 return success();
61 return op->emitOpError("has no core lowering for AIE2ps");
62}
63
65 std::tuple<const char *, std::vector<Type>, std::vector<Type>>;
66using IntrinsicDecls = std::vector<IntrinsicDecl>;
67
68static auto getAIE1Intrinsics(OpBuilder &builder) {
69 Type int32Type = IntegerType::get(builder.getContext(), 32);
70 Type int128Type = IntegerType::get(builder.getContext(), 128);
71 Type int384Type = IntegerType::get(builder.getContext(), 384);
72 Type floatType = Float32Type::get(builder.getContext());
73
74 // Note that not all of these are valid for a particular design, or needed.
75 // For right now, we will just accept the noise.
76 IntrinsicDecls functions = {
77 {"debug_i32", {int32Type}, {}},
78 {"llvm.aie.event0", {}, {}},
79 {"llvm.aie.event1", {}, {}},
80 {"llvm.aie.put.ms",
81 {int32Type, int32Type},
82 {}}, //(%channel, %value) -> ()
83 {"llvm.aie.put.wms",
84 {int32Type, int128Type},
85 {}}, //(%channel, %value) -> ()
86 {"llvm.aie.put.fms",
87 {int32Type, floatType},
88 {}}, //(%channel, %value) -> ()
89 {"llvm.aie.get.ss", {int32Type}, {int32Type}}, //(%channel, %value) -> ()
90 {"llvm.aie.get.wss",
91 {int32Type},
92 {int128Type}}, //(%channel, %value) -> ()
93 {"llvm.aie.get.fss", {int32Type}, {floatType}}, //(%channel, %value) -> ()
94 {"llvm.aie.put.mcd", {int384Type}, {}},
95 {"llvm.aie.get.scd", {}, {int384Type}},
96 {"llvm.aie.lock.acquire.reg",
97 {int32Type, int32Type},
98 {}}, //(%lock_id, %lock_val) -> ()
99 {"llvm.aie.lock.release.reg",
100 {int32Type, int32Type},
101 {}}, //(%lock_id, %lock_val) -> ()
102 };
103 return functions;
104}
105
106static auto getAIE2Intrinsics(OpBuilder &builder) {
107 Type int32Type = IntegerType::get(builder.getContext(), 32);
108 Type accType = VectorType::get({16}, int32Type);
109 IntrinsicDecls functions = {
110 {"debug_i32", {int32Type}, {}},
111 {"llvm.aie2.event", {int32Type}, {}},
112 {"llvm.aie2.put.ms", {int32Type, int32Type}, {}}, //(%value, %tlast) -> ()
113 {"llvm.aie2.get.ss", {}, {int32Type, int32Type}}, //() -> (%value, %tlast)
114 {"llvm.aie2.mcd.write.vec",
115 {accType, int32Type},
116 {}}, // (%value, %enable) -> ()
117 {"llvm.aie2.scd.read.vec",
118 {int32Type},
119 {accType}}, // (%enable) -> (%value)
120 {"llvm.aie2.acquire",
121 {int32Type, int32Type},
122 {}}, //(%lock_id, %lock_val) -> ()
123 {"llvm.aie2.release",
124 {int32Type, int32Type},
125 {}}, //(%lock_id, %lock_val) -> ()
126 {"llvm.aie2.set.ctrl.reg",
127 {int32Type, int32Type},
128 {}}, //(%reg_id, %value) -> ()
129 };
130 return functions;
131}
132
133static auto getAIE2pIntrinsics(OpBuilder &builder) {
134 Type int32Type = IntegerType::get(builder.getContext(), 32);
135 Type accType = VectorType::get({16}, int32Type);
136 IntrinsicDecls functions = {
137 {"debug_i32", {int32Type}, {}},
138 {"llvm.aie2p.event", {int32Type}, {}},
139 {"llvm.aie2p.put.ms",
140 {int32Type, int32Type},
141 {}}, //(%value, %tlast) -> ()
142 {"llvm.aie2p.get.ss",
143 {},
144 {int32Type, int32Type}}, //() -> (%value, %tlast)
145 {"llvm.aie2p.mcd.write.vec",
146 {accType, int32Type},
147 {}}, // (%value, %enable) -> ()
148 {"llvm.aie2p.scd.read.vec",
149 {int32Type},
150 {accType}}, // (%enable) -> (%value)
151 {"llvm.aie2p.acquire",
152 {int32Type, int32Type},
153 {}}, //(%lock_id, %lock_val) -> ()
154 {"llvm.aie2p.release",
155 {int32Type, int32Type},
156 {}}, //(%lock_id, %lock_val) -> ()
157 {"llvm.aie2p.set.ctrl.reg",
158 {int32Type, int32Type},
159 {}}, //(%reg_id, %value) -> ()
160 };
161 return functions;
162}
163
164static void declareAIEIntrinsics(AIEArch arch, OpBuilder &builder) {
165 auto registerIntrinsics = [&builder](const IntrinsicDecls &functions) {
166 for (auto &i : functions) {
167 auto [name, argTypes, retTypes] = i;
168 func::FuncOp::create(
169 builder, builder.getUnknownLoc(), name,
170 FunctionType::get(builder.getContext(), argTypes, retTypes))
171 .setPrivate();
172 }
173 };
174 switch (arch) {
175 case AIEArch::AIE1:
176 registerIntrinsics(getAIE1Intrinsics(builder));
177 return;
178 case AIEArch::AIE2:
179 registerIntrinsics(getAIE2Intrinsics(builder));
180 return;
181 case AIEArch::AIE2p:
182 registerIntrinsics(getAIE2pIntrinsics(builder));
183 return;
184 case AIEArch::AIE2ps:
185 // No intrinsic set: checkCoreIntrinsics rejects ops that need one.
186 return;
187 }
188 llvm::report_fatal_error("unsupported arch");
189}
190
191// Move all the ops with OpTy inside device, to just before the device.
192template <typename OpTy>
193static void outlineOps(DeviceOp device) {
194 SmallVector<OpTy, 16> ops;
195 for (const auto &op : device.getOps<OpTy>())
196 ops.push_back(op);
197
198 for (const auto &op : ops)
199 op->moveBefore(device);
200}
201
202// The outlined core function must not use values of the device, which is
203// erased. Give the core its own copy of each outside constant it uses and
204// diagnose any other outside value.
205static LogicalResult sinkOutsideValuesIntoCore(CoreOp core) {
206 Region &body = core.getBody();
207 OpBuilder builder = OpBuilder::atBlockBegin(&body.front());
208 DenseMap<Value, Value> copies;
209 WalkResult result = body.walk([&](Operation *op) {
210 for (OpOperand &operand : op->getOpOperands()) {
211 Value value = operand.get();
212 if (body.isAncestor(value.getParentRegion()))
213 continue;
214 Operation *def = value.getDefiningOp();
215 if (!def || !def->hasTrait<OpTrait::ConstantLike>()) {
216 InFlightDiagnostic diag =
217 op->emitOpError("uses a value defined outside its aie.core, "
218 "which cannot be outlined");
219 diag.attachNote(value.getLoc()) << "value defined here";
220 return WalkResult::interrupt();
221 }
222 Value &copy = copies[value];
223 if (!copy) {
224 Operation *clone = builder.clone(*def);
225 builder.setInsertionPointAfter(clone);
226 copy = clone->getResult(cast<OpResult>(value).getResultNumber());
227 }
228 operand.set(copy);
229 }
230 return WalkResult::advance();
231 });
232 return failure(result.wasInterrupted());
233}
234
235namespace {
236template <typename MyAIEOp>
237struct AIEOpRemoval : OpConversionPattern<MyAIEOp> {
238 using OpConversionPattern<MyAIEOp>::OpConversionPattern;
239 using OpAdaptor = typename MyAIEOp::Adaptor;
240 ModuleOp &module;
241
242 AIEOpRemoval(MLIRContext *context, ModuleOp &m, PatternBenefit benefit = 1)
243 : OpConversionPattern<MyAIEOp>(context, benefit), module(m) {}
244
245 LogicalResult
246 matchAndRewrite(MyAIEOp op, OpAdaptor adaptor,
247 ConversionPatternRewriter &rewriter) const override {
248 rewriter.eraseOp(op);
249 return success();
250 }
251};
252
253struct AIEDebugOpToStdLowering : OpConversionPattern<DebugOp> {
254 using OpConversionPattern::OpConversionPattern;
255 ModuleOp &module;
256
257 AIEDebugOpToStdLowering(MLIRContext *context, ModuleOp &m,
258 PatternBenefit benefit = 1)
259 : OpConversionPattern(context, benefit), module(m) {}
260
261 LogicalResult
262 matchAndRewrite(DebugOp op, OpAdaptor adaptor,
263 ConversionPatternRewriter &rewriter) const override {
264 if (failed(checkCoreIntrinsics(op)))
265 return failure();
266 std::string funcName = "debug_i32";
267 auto func = module.lookupSymbol<func::FuncOp>(funcName);
268 if (!func)
269 return op.emitOpError("Could not find the intrinsic function ")
270 << funcName;
271 SmallVector<Value, 1> args;
272 args.push_back(op.getArg());
273 func::CallOp::create(rewriter, op.getLoc(), func, args);
274 rewriter.eraseOp(op);
275 return success();
276 }
277};
278
279struct AIEPutStreamToStdLowering : OpConversionPattern<PutStreamOp> {
280 using OpConversionPattern::OpConversionPattern;
281 ModuleOp &module;
282
283 AIEPutStreamToStdLowering(MLIRContext *context, ModuleOp &m,
284 PatternBenefit benefit = 1)
285 : OpConversionPattern(context, benefit), module(m) {}
286
287 LogicalResult
288 matchAndRewrite(PutStreamOp op, OpAdaptor adaptor,
289 ConversionPatternRewriter &rewriter) const override {
290 if (failed(checkCoreIntrinsics(op)))
291 return failure();
292 auto device = op->getParentOfType<DeviceOp>();
293 const auto &targetModel = device.getTargetModel();
294 std::string funcName;
295 if (targetModel.getTargetArch() == AIEArch::AIE1)
296 funcName = "llvm.aie.put.";
297 else if (targetModel.getTargetArch() == AIEArch::AIE2)
298 funcName = "llvm.aie2.put.";
299 else
300 funcName = "llvm.aie2p.put.";
301
302 if (op.isWideStream())
303 funcName += "wms";
304 else if (op.isFloatStream())
305 funcName += "fms";
306 else
307 funcName += "ms";
308
309 auto putMSFunc = module.lookupSymbol<func::FuncOp>(funcName);
310 if (!putMSFunc)
311 return op.emitOpError("Could not find the intrinsic function ")
312 << funcName;
313 SmallVector<Value, 2> args;
314 if (targetModel.getTargetArch() == AIEArch::AIE1) {
315 args.push_back(op.getChannel());
316 args.push_back(op.getStreamValue());
317 } else {
318 args.push_back(op.getStreamValue());
319 args.push_back(arith::ConstantOp::create(
320 rewriter, op.getLoc(), IntegerType::get(rewriter.getContext(), 32),
321 rewriter.getI32IntegerAttr(0))); // tlast
322 }
323 func::CallOp::create(rewriter, op.getLoc(), putMSFunc, args);
324 rewriter.eraseOp(op);
325 return success();
326 }
327};
328
329struct AIEGetStreamToStdLowering : OpConversionPattern<GetStreamOp> {
330 using OpConversionPattern::OpConversionPattern;
331 ModuleOp &module;
332
333 AIEGetStreamToStdLowering(MLIRContext *context, ModuleOp &m,
334 PatternBenefit benefit = 1)
335 : OpConversionPattern(context, benefit), module(m) {}
336
337 LogicalResult
338 matchAndRewrite(GetStreamOp op, OpAdaptor adaptor,
339 ConversionPatternRewriter &rewriter) const override {
340 if (failed(checkCoreIntrinsics(op)))
341 return failure();
342 auto device = op->getParentOfType<DeviceOp>();
343 const auto &targetModel = device.getTargetModel();
344 std::string funcName;
345 if (targetModel.getTargetArch() == AIEArch::AIE1)
346 funcName = "llvm.aie.get.";
347 else if (targetModel.getTargetArch() == AIEArch::AIE2)
348 funcName = "llvm.aie2.get.";
349 else
350 funcName = "llvm.aie2p.get.";
351
352 if (op.isWideStream())
353 funcName += "wss";
354 else if (op.isFloatStream())
355 funcName += "fss";
356 else
357 funcName += "ss";
358
359 auto getSSFunc = module.lookupSymbol<func::FuncOp>(funcName);
360 if (!getSSFunc)
361 return op.emitOpError("Could not find the intrinsic function ")
362 << funcName;
363 SmallVector<Value, 2> args;
364 if (targetModel.getTargetArch() == AIEArch::AIE1)
365 args.push_back(op.getChannel());
366 auto getSSCall =
367 func::CallOp::create(rewriter, op.getLoc(), getSSFunc, args);
368 rewriter.replaceOp(op, getSSCall.getResult(0));
369 // Capture TLAST in AIEv2?
370 return success();
371 }
372};
373
374struct AIEPutCascadeToStdLowering : OpConversionPattern<PutCascadeOp> {
375 using OpConversionPattern::OpConversionPattern;
376 ModuleOp &module;
377
378 AIEPutCascadeToStdLowering(MLIRContext *context, ModuleOp &m,
379 PatternBenefit benefit = 1)
380 : OpConversionPattern(context, benefit), module(m) {}
381
382 LogicalResult
383 matchAndRewrite(PutCascadeOp op, OpAdaptor adaptor,
384 ConversionPatternRewriter &rewriter) const override {
385 if (failed(checkCoreIntrinsics(op)))
386 return failure();
387 auto device = op->getParentOfType<DeviceOp>();
388 const auto &targetModel = device.getTargetModel();
389 std::string funcName;
390 if (targetModel.getTargetArch() == AIEArch::AIE1)
391 funcName = "llvm.aie.put.mcd";
392 else if (targetModel.getTargetArch() == AIEArch::AIE2)
393 funcName = "llvm.aie2.mcd.write.vec";
394 else
395 funcName = "llvm.aie2p.mcd.write.vec";
396 auto putMCDFunc = module.lookupSymbol<func::FuncOp>(funcName);
397 if (!putMCDFunc)
398 return op.emitOpError("Could not find the intrinsic function ")
399 << funcName;
400 SmallVector<Value, 2> args;
401 Value cascadeValue = op.getCascadeValue();
402
403 // Check if we need a bitcast for the input value
404 Type expectedInputType = putMCDFunc.getFunctionType().getInput(0);
405 Type actualInputType = cascadeValue.getType();
406
407 if (expectedInputType != actualInputType) {
408 // Create a bitcast operation to convert from actual input type to
409 // expected type
410 cascadeValue = vector::BitCastOp::create(rewriter, op.getLoc(),
411 expectedInputType, cascadeValue);
412 }
413
414 args.push_back(cascadeValue);
415 if (isa<AIE2TargetModel>(targetModel))
416 args.push_back(arith::ConstantOp::create(
417 rewriter, op.getLoc(), IntegerType::get(rewriter.getContext(), 32),
418 rewriter.getI32IntegerAttr(1))); // enable
419
420 func::CallOp::create(rewriter, op.getLoc(), putMCDFunc, args);
421 rewriter.eraseOp(op);
422 return success();
423 }
424};
425
426struct AIEGetCascadeToStdLowering : OpConversionPattern<GetCascadeOp> {
427 using OpConversionPattern::OpConversionPattern;
428 ModuleOp &module;
429
430 AIEGetCascadeToStdLowering(MLIRContext *context, ModuleOp &m,
431 PatternBenefit benefit = 1)
432 : OpConversionPattern(context, benefit), module(m) {}
433
434 LogicalResult
435 matchAndRewrite(GetCascadeOp op, OpAdaptor adaptor,
436 ConversionPatternRewriter &rewriter) const override {
437 if (failed(checkCoreIntrinsics(op)))
438 return failure();
439 auto device = op->getParentOfType<DeviceOp>();
440 const auto &targetModel = device.getTargetModel();
441 std::string funcName;
442 if (targetModel.getTargetArch() == AIEArch::AIE1)
443 funcName = "llvm.aie.get.scd";
444 else if (targetModel.getTargetArch() == AIEArch::AIE2)
445 funcName = "llvm.aie2.scd.read.vec";
446 else
447 funcName = "llvm.aie2p.scd.read.vec";
448 auto getSCDFunc = module.lookupSymbol<func::FuncOp>(funcName);
449 if (!getSCDFunc)
450 return op.emitOpError("Could not find the intrinsic function ")
451 << funcName;
452 SmallVector<Value, 2> args;
453 if (isa<AIE2TargetModel>(targetModel))
454 args.push_back(arith::ConstantOp::create(
455 rewriter, op.getLoc(), IntegerType::get(rewriter.getContext(), 32),
456 rewriter.getI32IntegerAttr(1))); // enable
457
458 auto getSCDCall =
459 func::CallOp::create(rewriter, op.getLoc(), getSCDFunc, args);
460 Value result = getSCDCall.getResult(0);
461
462 // Check if we need a bitcast
463 Type expectedType = op.getResult().getType();
464 Type intrinsicReturnType = result.getType();
465
466 if (expectedType != intrinsicReturnType) {
467 // Create a bitcast operation to convert from intrinsic return type to
468 // expected type
469 result = vector::BitCastOp::create(rewriter, op.getLoc(), expectedType,
470 result);
471 }
472
473 rewriter.replaceOp(op, result);
474 return success();
475 }
476};
477
478struct AIEUseLockToStdLowering : OpConversionPattern<UseLockOp> {
479 using OpConversionPattern::OpConversionPattern;
480 ModuleOp &module;
481
482 AIEUseLockToStdLowering(MLIRContext *context, ModuleOp &m,
483 PatternBenefit benefit = 1)
484 : OpConversionPattern(context, benefit), module(m) {}
485 LogicalResult
486 matchAndRewrite(UseLockOp useLock, OpAdaptor adaptor,
487 ConversionPatternRewriter &rewriter) const override {
488 if (!isa<DeviceOp>(useLock->getParentOp())) {
489 auto device = useLock->getParentOfType<DeviceOp>();
490 if (!device) {
491 return module.emitOpError("Device Not found!");
492 }
493 const auto &targetModel = device.getTargetModel();
494 // A lock in a core or in any function needs an intrinsic, whether or
495 // not an exported core calls the function. Any other lock (DMA
496 // regions, runtime sequences) is erased with its parent.
497 if (isa<AIE2PSTargetModel>(targetModel)) {
498 if (useLock->getParentOfType<CoreOp>() ||
499 useLock->getParentOfType<func::FuncOp>())
500 return checkCoreIntrinsics(useLock);
501 rewriter.eraseOp(useLock);
502 return success();
503 }
504
505 // Generate the intrinsic name
506 std::string funcName;
507 if (targetModel.getTargetArch() == AIEArch::AIE1)
508 funcName = "llvm.aie.lock.";
509 else if (targetModel.getTargetArch() == AIEArch::AIE2)
510 funcName = "llvm.aie2.";
511 else
512 funcName = "llvm.aie2p.";
513 if (useLock.acquire() || useLock.acquireGE())
514 funcName += "acquire";
515 else if (useLock.release())
516 funcName += "release";
517 if (targetModel.getTargetArch() == AIEArch::AIE1)
518 funcName += ".reg";
519
520 auto useLockFunc = module.lookupSymbol<func::FuncOp>(funcName);
521 if (!useLockFunc)
522 return useLock.emitOpError("Could not find the intrinsic function!");
523
524 SmallVector<Value, 2> args;
525 auto i32Ty = IntegerType::get(rewriter.getContext(), 32);
526 args.push_back(arith::IndexCastOp::create(rewriter, useLock.getLoc(),
527 i32Ty, useLock.getLock()));
528
529 Value value = adaptor.getValue();
530 // AIE2 acquire-greater-equal is encoded as a negative value, so negate
531 // it at runtime.
532 if (useLock.acquireGE()) {
533 Value zero = arith::ConstantOp::create(
534 rewriter, useLock.getLoc(), i32Ty, rewriter.getI32IntegerAttr(0));
535 value = arith::SubIOp::create(rewriter, useLock.getLoc(), zero, value);
536 }
537 args.push_back(value);
538
539 func::CallOp::create(rewriter, useLock.getLoc(), useLockFunc, args);
540 }
541 rewriter.eraseOp(useLock);
542 return success();
543 }
544};
545
546struct AIEBufferToStandard : OpConversionPattern<BufferOp> {
547 using OpConversionPattern::OpConversionPattern;
548 ModuleOp &module;
549 AIEBufferToStandard(MLIRContext *context, ModuleOp &m,
550 PatternBenefit benefit = 1)
551 : OpConversionPattern(context, benefit), module(m) {}
552 LogicalResult
553 matchAndRewrite(BufferOp buffer, OpAdaptor adaptor,
554 ConversionPatternRewriter &rewriter) const override {
555 // The core's own sections fill this extent; nothing refers to it by name.
556 if (buffer.getCoreData()) {
557 rewriter.eraseOp(buffer);
558 return success();
559 }
560 rewriter.setInsertionPointToStart(module.getBody());
561 auto t = llvm::cast<MemRefType>(buffer.getType());
562 auto symName = buffer.name().getValue();
563 // A declaration, never a definition: a buffer lives at the address the
564 // allocator chose, and the linker supplies that address from the generated
565 // script or BCF. `initial_value` is device state, written by whoever
566 // configures the device -- the CDO writer, or mlir_aie_initialize_buffers
567 // on the simulator path. A definition here would place the bytes wherever
568 // `*(.data*)` lands, which costs the core's data region a copy it never
569 // reads.
570 memref::GlobalOp::create(rewriter, buffer.getLoc(), symName,
571 rewriter.getStringAttr("public"), buffer.getType(),
572 /*initial_value=*/nullptr, /*constant*/ false,
573 /*alignment*/ nullptr);
574
575 for (auto &use : make_early_inc_range(buffer.getResult().getUses())) {
576 Operation *user = use.getOwner();
577 rewriter.setInsertionPoint(user);
578 auto allocated =
579 memref::GetGlobalOp::create(rewriter, buffer.getLoc(), t, symName);
580 // Assume that buffers are aligned so they can be vectorized.
581 memref::AssumeAlignmentOp::create(rewriter, buffer.getLoc(), allocated,
582 32);
583
584 use.set(allocated.getResult());
585 }
586
587 rewriter.eraseOp(buffer);
588 return success();
589 }
590};
591
592struct AIECoreToStandardFunc : OpConversionPattern<CoreOp> {
593 using OpConversionPattern::OpConversionPattern;
594 ModuleOp &module;
595 IRMapping &mapper;
596 DenseMap<Operation *, SmallVector<BufferOp, 4>> &tileToBuffers;
597 int tileCol = 0;
598 int tileRow = 0;
599
600 AIECoreToStandardFunc(
601 MLIRContext *context, ModuleOp &m, IRMapping &mapper,
602 DenseMap<Operation *, SmallVector<BufferOp, 4>> &tileToBuffers,
603 PatternBenefit benefit = 1, int tileCol = 1, int tileRow = 1)
604 : OpConversionPattern(context, benefit), module(m), mapper(mapper),
605 tileToBuffers(tileToBuffers), tileCol(tileCol), tileRow(tileRow) {}
606
607 LogicalResult
608 matchAndRewrite(CoreOp op, OpAdaptor adaptor,
609 ConversionPatternRewriter &rewriter) const override {
610
611 int col = op.colIndex();
612 int row = op.rowIndex();
613
614 // Only pull code for the indicated function
615 if ((tileRow != row && tileRow != -1) ||
616 (tileCol != col && tileCol != -1)) {
617 rewriter.eraseOp(op);
618 return success();
619 }
620
621 // The parent should be an AIE.device op.
622 rewriter.setInsertionPointAfter(op->getParentOp());
623
624 std::string coreName = coreFrameSymbolName(col, row);
625 auto coreFunc =
626 func::FuncOp::create(rewriter, op.getLoc(), coreName,
627 FunctionType::get(rewriter.getContext(), {}, {}));
628
629 rewriter.cloneRegionBefore(op.getBody(), coreFunc.getBody(),
630 coreFunc.getBody().begin(), mapper);
631
632 // Set saturation and rounding modes at core entry for AIE2/AIE2p, but
633 // only if the core body contains aievec.srs or bf16 aievec.matmul ops.
634 // Skip for cores with only lock/stream ops to avoid breaking existing
635 // test SSA naming.
636 bool hasSRS = false;
637 bool hasIntegerSRS = false;
638 bool hasBF16Matmul = false;
639 auto scan = [&](Operation *childOp) {
640 StringRef opName = childOp->getName().getStringRef();
641 if (opName == "aievec.srs") {
642 hasSRS = true;
643 // Check if this is an integer SRS (e.g., i32→i8) vs float SRS
644 // (e.g., f32→bf16). Integer SRS needs positive_inf rounding to
645 // match C++ kernel behavior; float SRS works better with floor.
646 if (childOp->getNumResults() > 0) {
647 auto resultType = childOp->getResult(0).getType();
648 if (auto vecType = dyn_cast<VectorType>(resultType)) {
649 if (vecType.getElementType().isInteger())
650 hasIntegerSRS = true;
651 }
652 }
653 }
654 // Detect bf16 matmul ops — these need conv_even rounding to avoid
655 // systematic negative bias from floor rounding in BFP16 arithmetic.
656 // On AIE2P a bf16 matmul is a bfp16 one: aievec.to_bfp16 converts its
657 // bf16 operands, and aievec.matmul_aie2p_bfp16 may take them loaded.
658 if (opName == "aievec.to_bfp16" || opName == "aievec.matmul_aie2p_bfp16")
659 hasBF16Matmul = true;
660 if (opName == "aievec.matmul" || opName == "aievec.matmul_aie2p") {
661 if (childOp->getNumOperands() > 0) {
662 auto lhsType = childOp->getOperand(0).getType();
663 if (auto vecType = dyn_cast<VectorType>(lhsType))
664 if (vecType.getElementType().isBF16())
665 hasBF16Matmul = true;
666 }
667 }
668 };
669 coreFunc.walk(scan);
670 auto device = op->getParentOfType<DeviceOp>();
671 // AIE2ps cannot set the modes anywhere, so the functions the core calls
672 // must be free of these ops too.
673 if (device && isa<AIE2PSTargetModel>(device.getTargetModel())) {
674 SmallVector<func::FuncOp> worklist;
675 DenseSet<Operation *> visited;
676 auto addCallees = [&](Operation *root) {
677 root->walk([&](func::CallOp call) {
678 auto callee = SymbolTable::lookupNearestSymbolFrom<func::FuncOp>(
679 device, call.getCalleeAttr());
680 if (callee && visited.insert(callee).second)
681 worklist.push_back(callee);
682 });
683 };
684 addCallees(coreFunc);
685 while (!worklist.empty()) {
686 func::FuncOp callee = worklist.pop_back_val();
687 callee.walk(scan);
688 addCallees(callee);
689 }
690 }
691 if (hasSRS || hasBF16Matmul) {
692 if (device) {
693 if (isa<AIE2PSTargetModel>(device.getTargetModel()))
694 return op.emitOpError("needs the rounding and saturation modes set "
695 "for aievec.srs or bf16 matmul, which has no "
696 "lowering for AIE2ps");
697 AIEArch arch = device.getTargetModel().getTargetArch();
698 if (arch == AIEArch::AIE2 || arch == AIEArch::AIE2p) {
699 std::string ctrlRegFuncName = (arch == AIEArch::AIE2p)
700 ? "llvm.aie2p.set.ctrl.reg"
701 : "llvm.aie2.set.ctrl.reg";
702 auto ctrlRegFunc = module.lookupSymbol<func::FuncOp>(ctrlRegFuncName);
703 if (ctrlRegFunc) {
704 Block &entryBlock = coreFunc.getBody().front();
705 rewriter.setInsertionPointToStart(&entryBlock);
706 Location loc = op.getLoc();
707 // Rounding register index differs between AIE2 and AIE2P:
708 // AIE2: crRnd=6
709 // AIE2P: crRnd=1
710 // Saturation register uses AIE2 index (9) for both architectures.
711 // On AIE2P, index 9 maps to crPackSize (no-op for saturation),
712 // preserving the pre-existing behavior. The AIE2P crSat fix
713 // (index 0) requires updating downstream tests and is tracked
714 // separately.
715 int satRegIdx = 9;
716 int rndRegIdx = (arch == AIEArch::AIE2p) ? 1 : 6;
717 // saturation_mode::saturate = 1
718 auto cSatIdx = arith::ConstantOp::create(
719 rewriter, loc, rewriter.getI32IntegerAttr(satRegIdx));
720 auto c1 = arith::ConstantOp::create(rewriter, loc,
721 rewriter.getI32IntegerAttr(1));
722 func::CallOp::create(rewriter, loc, ctrlRegFunc,
723 ValueRange{cSatIdx, c1});
724 // Rounding mode:
725 // - conv_even (12) for bf16 matmul: eliminates systematic
726 // negative bias from floor rounding in BFP16 arithmetic,
727 // matching ::aie::set_rounding(aie::rounding_mode::conv_even)
728 // used in external C++ matmul kernels.
729 // - positive_inf (9) for integer SRS (shift-round-saturate on
730 // integer data, e.g., i32→i8), unless the core has a bf16
731 // matmul.
732 // - conv_even (12) for float-only SRS (f32→bf16), matching the
733 // round-to-nearest-even semantics of arith.truncf.
734 int roundingMode = hasIntegerSRS && !hasBF16Matmul ? 9 : 12;
735 auto cRndIdx = arith::ConstantOp::create(
736 rewriter, loc, rewriter.getI32IntegerAttr(rndRegIdx));
737 auto cRoundingMode = arith::ConstantOp::create(
738 rewriter, loc, rewriter.getI32IntegerAttr(roundingMode));
739 func::CallOp::create(rewriter, loc, ctrlRegFunc,
740 ValueRange{cRndIdx, cRoundingMode});
741 }
742 }
743 }
744 }
745
746 // Rewrite the AIE.end() op
747 coreFunc.getBody().walk([&](Operation *childOp) {
748 rewriter.setInsertionPointAfter(childOp);
749
750 if (isa<EndOp>(childOp)) {
751 func::ReturnOp::create(rewriter, childOp->getLoc(), ValueRange({}));
752 rewriter.eraseOp(childOp);
753 }
754 });
755
756 rewriter.eraseOp(op);
757 return success();
758 }
759};
760
761// Lower AIE.event to llvm.aie.event intrinsic
762struct AIEEventOpToStdLowering : OpConversionPattern<EventOp> {
763 using OpConversionPattern::OpConversionPattern;
764 ModuleOp &module;
765
766 AIEEventOpToStdLowering(MLIRContext *context, ModuleOp &m,
767 PatternBenefit benefit = 1)
768 : OpConversionPattern(context, benefit), module(m) {}
769
770 LogicalResult
771 matchAndRewrite(EventOp op, OpAdaptor adaptor,
772 ConversionPatternRewriter &rewriter) const override {
773 auto device = op->getParentOfType<DeviceOp>();
774 std::string funcName;
775 SmallVector<Value, 1> args;
776 switch (device.getTargetModel().getTargetArch()) {
777 case AIEArch::AIE1:
778 funcName = "llvm.aie.event" + std::to_string(op.getVal());
779 break;
780 case AIEArch::AIE2:
781 funcName = "llvm.aie2.event";
782 args.push_back(arith::ConstantOp::create(
783 rewriter, op.getLoc(), rewriter.getI32Type(),
784 rewriter.getI32IntegerAttr(op.getVal())));
785 break;
786 case AIEArch::AIE2p:
787 funcName = "llvm.aie2p.event";
788 args.push_back(arith::ConstantOp::create(
789 rewriter, op.getLoc(), rewriter.getI32Type(),
790 rewriter.getI32IntegerAttr(op.getVal())));
791 break;
792 case AIEArch::AIE2ps:
793 return op.emitOpError("has no core lowering for AIE2ps");
794 }
795 auto eventFunc = module.lookupSymbol<func::FuncOp>(funcName);
796 if (!eventFunc)
797 return op.emitOpError("Could not find the intrinsic function ")
798 << funcName;
799 func::CallOp::create(rewriter, op.getLoc(), eventFunc, args);
800 rewriter.eraseOp(op);
801 return success();
802 }
803};
804
805struct AIECoreToStandardPass
806 : xilinx::AIE::impl::AIECoreToStandardBase<AIECoreToStandardPass> {
807 AIECoreToStandardPass() = default;
808 AIECoreToStandardPass(const AIECoreToStandardOptions &options) {
809 deviceName = options.deviceName;
810 tileCol = options.tileCol;
811 tileRow = options.tileRow;
812 }
813
814 void runOnOperation() override {
815
816 ModuleOp m = getOperation();
817 OpBuilder builder = OpBuilder::atBlockEnd(m.getBody());
818
819 DeviceOp deviceOp = DeviceOp::getForSymbolInModuleOrError(m, deviceName);
820 if (!deviceOp) {
821 return signalPassFailure();
822 }
823 const auto &targetModel = deviceOp.getTargetModel();
824
825 // Copy data layout attribute from DeviceOp to ModuleOp if present
826 if (auto dlAttr = deviceOp->getAttr(DLTIDialect::kDataLayoutAttrName)) {
827 m->setAttr(DLTIDialect::kDataLayoutAttrName, dlAttr);
828 }
829
830 // Ensure that we don't have an incorrect target triple. This may override
831 // some bogus target triple in the original mlir.
832 m->setAttr(LLVM::LLVMDialect::getTargetTripleAttrName(),
833 builder.getStringAttr(
834 getArchTargetTriple(targetModel.getTargetArch())));
835
836 DenseMap<Operation *, SmallVector<BufferOp, 4>> tileToBuffers;
837
838 // Populate intrinsic functions
839 // Intrinsic information:
840 // peano/llvm-project/llvm/lib/Target/AIE/AIEInstrInfo.td Also take a look
841 // at the tests: peano/llvm-project/llvm/test/CodeGen/AIE
842 builder.setInsertionPointToStart(m.getBody());
843 declareAIEIntrinsics(targetModel.getTargetArch(), builder);
844
845 IRMapping mapper;
846 ConversionTarget target(getContext());
847 target.addLegalDialect<func::FuncDialect>();
848 target.addLegalDialect<cf::ControlFlowDialect>();
849 target.addLegalDialect<memref::MemRefDialect>();
850 target.addLegalDialect<VectorDialect>();
851 target.addLegalDialect<aievec::AIEVecDialect>();
852 target.addLegalDialect<arith::ArithDialect>();
853 target.addLegalDialect<ub::UBDialect>();
854 target.addLegalDialect<math::MathDialect>();
855 target.addLegalDialect<index::IndexDialect>();
856 target.addLegalDialect<ptr::PtrDialect>();
857 // A core body may already hold LLVM ops: aie.dma_push_queue lowers to the
858 // stores that push a task before the body is outlined.
859 target.addLegalDialect<LLVM::LLVMDialect>();
860 target.addLegalOp<func::FuncOp, ModuleOp, UnrealizedConversionCastOp>();
861 // Ops this pass lowers must be illegal: a failed rewrite of an op with
862 // unknown legality is silently left in place.
863 // A core that is not exported is erased, so its ops are left alone.
864 target.addIllegalOp<BufferOp>();
865 target.addDynamicallyLegalOp<PutStreamOp, GetStreamOp, PutCascadeOp,
866 GetCascadeOp, DebugOp, UseLockOp, EventOp>(
867 [col = static_cast<int>(tileCol),
868 row = static_cast<int>(tileRow)](Operation *op) {
869 auto core = op->getParentOfType<CoreOp>();
870 return core && ((row != core.rowIndex() && row != -1) ||
871 (col != core.colIndex() && col != -1));
872 });
873
874 RewritePatternSet patterns(&getContext());
875 patterns.add<AIEPutStreamToStdLowering, AIEGetStreamToStdLowering,
876 AIEPutCascadeToStdLowering, AIEGetCascadeToStdLowering,
877 AIEDebugOpToStdLowering, AIEUseLockToStdLowering,
878 AIEEventOpToStdLowering>(m.getContext(), m);
879
880 patterns.add<AIEBufferToStandard>(m.getContext(), m, /*benefit*/ 1);
881 if (failed(applyPartialConversion(deviceOp, target, std::move(patterns))))
882 return signalPassFailure();
883
884 // Unselected cores are erased, not outlined.
885 int col = static_cast<int>(tileCol);
886 int row = static_cast<int>(tileRow);
887 for (CoreOp core : deviceOp.getOps<CoreOp>()) {
888 if ((row != core.rowIndex() && row != -1) ||
889 (col != core.colIndex() && col != -1))
890 continue;
891 if (failed(sinkOutsideValuesIntoCore(core)))
892 return signalPassFailure();
893 }
894
895 RewritePatternSet outlinePatterns(&getContext());
896 outlinePatterns.add<AIECoreToStandardFunc>(m.getContext(), m, mapper,
897 tileToBuffers, /*benefit*/ 1,
898 tileCol, tileRow);
899 // Otherwise a core whose outlined body does not legalize is dropped.
900 target.addIllegalOp<CoreOp>();
901 if (failed(applyPartialConversion(deviceOp, target,
902 std::move(outlinePatterns))))
903 return signalPassFailure();
904
905 // Move all the func.func ops and memref.globals from the device to the
906 // module
907 outlineOps<memref::GlobalOp>(deviceOp);
908 outlineOps<func::FuncOp>(deviceOp);
909
910 RewritePatternSet removepatterns(&getContext());
911 removepatterns.add<
912 AIEOpRemoval<DeviceOp>, AIEOpRemoval<TileOp>, AIEOpRemoval<FlowOp>,
913 AIEOpRemoval<MemOp>, AIEOpRemoval<ShimDMAOp>, AIEOpRemoval<ShimMuxOp>,
914 AIEOpRemoval<SwitchboxOp>, AIEOpRemoval<LockOp>, AIEOpRemoval<BufferOp>,
915 AIEOpRemoval<ExternalBufferOp>, AIEOpRemoval<ShimDMAAllocationOp>,
916 AIEOpRemoval<CascadeFlowOp>, AIEOpRemoval<ConfigureCascadeOp>>(
917 m.getContext(), m);
918
919 if (failed(applyPartialConversion(m, target, std::move(removepatterns))))
920 return signalPassFailure();
921 }
922};
923} // namespace
924
925std::unique_ptr<OperationPass<ModuleOp>> AIE::createAIECoreToStandardPass() {
926 return std::make_unique<AIECoreToStandardPass>();
927}
928
929std::unique_ptr<OperationPass<ModuleOp>>
930AIE::createAIECoreToStandardPass(const AIECoreToStandardOptions &options) {
931 return std::make_unique<AIECoreToStandardPass>(options);
932}
std::tuple< const char *, std::vector< Type >, std::vector< Type > > IntrinsicDecl
std::vector< IntrinsicDecl > IntrinsicDecls
std::string coreFrameSymbolName(int col, int row)
std::unique_ptr< mlir::OperationPass< mlir::ModuleOp > > createAIECoreToStandardPass()
AIEOpRemoval(MLIRContext *context, PatternBenefit benefit=1)
LogicalResult matchAndRewrite(MyOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override