MLIR-AIE 1.4.6.dev
AIEVecToLLVM.cpp
Go to the documentation of this file.
1//===- AIEVecToLLVM.cpp - AIEVec to LLVM dialect conversion ---------------===//
2//
3// Copyright (C) 2022 Xilinx, Inc.
4// Copyright (C) 2024 Advanced Micro Devices, Inc.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "../PassDetail.h"
10
20#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
21#include "mlir/Conversion/LLVMCommon/Pattern.h"
22#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
23#include "mlir/Dialect/Math/IR/Math.h"
24#include "mlir/Dialect/MemRef/IR/MemRef.h"
25#include "mlir/Dialect/UB/IR/UBOps.h"
26#include "mlir/IR/TypeUtilities.h"
27#include <numeric>
28#include <sstream>
29
30namespace xilinx {
31using namespace mlir; // For LLVM::LLVMDialect in generated getDependentDialects
32#define GEN_PASS_DEF_CONVERTAIEVECTOLLVM
33#include "aie/Conversion/Passes.h.inc"
34} // namespace xilinx
35
36using namespace mlir;
37
38namespace xilinx::aievec {
39
40inline static Value bitcastValueToType(OpBuilder &builder, Location loc,
41 Value val, Type dstTy) {
42 return LLVM::BitcastOp::create(builder, loc, dstTy, val).getResult();
43}
44
45// This function emits the instructions required to widen a 128b input vector
46// into a 512b encoded as a vector<16xi32>. It first bitcasts it to a
47// vector<4xi32> to respect the intrinsic signature.
48inline static Value widen128bVectorValueTo512b(OpBuilder &builder, Location loc,
49 Value val) {
50 return xllvm::VectorSetI512I128IntrOp::create(
51 builder, loc, VectorType::get({16}, builder.getI32Type()),
52 bitcastValueToType(builder, loc, val,
53 VectorType::get({4}, builder.getI32Type())))
54 .getResult();
55}
56
57// This function emits the instructions required to widen a 256b input vector
58// into a 512b encoded as a vector<16xi32>. It first bitcasts it to a
59// vector<8xi32> to respect the intrinsic signature. It will also materialize
60// a constant 0, used as an insertion index.
61inline static Value widen256bVectorValueTo512b(OpBuilder &builder, Location loc,
62 Value val) {
63 auto cst0 =
64 LLVM::ConstantOp::create(builder, loc, builder.getI32Type(), (int32_t)0);
65 return xllvm::VectorSetI512I256IntrOp::create(
66 builder, loc, VectorType::get({16}, builder.getI32Type()),
67 bitcastValueToType(builder, loc, val,
68 VectorType::get({8}, builder.getI32Type())),
69 cst0)
70 .getResult();
71}
72
73// This function emits the sequence of operations that forces a value into a
74// specific type. This may include widening vectors to match a specific bit
75// length.
76static Value forceCastValueToType(OpBuilder &builder, Location loc, Value val,
77 Type type) {
78 auto valTy = val.getType();
79 if (valTy == type)
80 return val;
81 auto srcVecTy = dyn_cast<VectorType>(valTy);
82 auto dstVecTy = dyn_cast<VectorType>(type);
83
84 if (srcVecTy) {
85 assert(dstVecTy && "vector values cannot be forced into a non-vector type");
86
87 // Flatten source vector if it's not rank-1
88 auto flatSrcVecTy = getFlattenedVectorType(srcVecTy);
89 if (srcVecTy != flatSrcVecTy)
90 val = vector::ShapeCastOp::create(builder, loc, flatSrcVecTy, val);
91
92 // Flatten destination type if it's not rank-1
93 auto flatDstVecTy = getFlattenedVectorType(dstVecTy);
94
95 int64_t dstVecLength =
96 flatDstVecTy.getElementTypeBitWidth() * flatDstVecTy.getShape()[0];
97 int64_t srcVecLength =
98 flatSrcVecTy.getElementTypeBitWidth() * flatSrcVecTy.getShape()[0];
99 if (srcVecLength != dstVecLength) {
100 assert(srcVecLength < dstVecLength &&
101 "only widening forced casts are supported");
102 assert(dstVecLength == 512 &&
103 (srcVecLength == 128 || srcVecLength == 256) &&
104 "only 128b to 512b and 256b to 512b forced casts are supported");
105 if (srcVecLength == 128)
106 val = widen128bVectorValueTo512b(builder, loc, val);
107 else
108 val = widen256bVectorValueTo512b(builder, loc, val);
109 }
110
111 // Bitcast to flat destination type (bitcast only supports flat vectors)
112 val = bitcastValueToType(builder, loc, val, flatDstVecTy);
113
114 // Reshape back to original destination shape if needed
115 if (flatDstVecTy != dstVecTy)
116 val = vector::ShapeCastOp::create(builder, loc, dstVecTy, val);
117
118 return val;
119 }
120
121 // Non-vector types can be bitcast directly
122 assert(!dstVecTy && "cannot force cast scalar to vector type");
123 return bitcastValueToType(builder, loc, val, type);
124}
125
126// This function emits the sequence of operations that forces a range of values
127// to match the signature specified by the TypeRange. It can be used to convert
128// the parameters of an op being converted to the types accepted by an
129// intrinsic with a fixed signature that treats its inputs as "bags of bits".
130static SmallVector<Value> forceCastOperandsToSignature(OpBuilder &builder,
131 Location loc,
132 ValueRange operands,
133 TypeRange signature) {
134 return llvm::to_vector(llvm::map_range(
135 llvm::zip_equal(operands, signature), [&](auto &&vt) -> Value {
136 return forceCastValueToType(builder, loc, std::get<0>(vt),
137 std::get<1>(vt));
138 }));
139}
140
141// Utility function to get or create a noinline scalar helper function.
142// This is used to create optimization barriers that prevent LLVM from
143// re-vectorizing unrolled scalar operations.
144//
145// Parameters:
146// - module: The parent module to insert the function into
147// - rewriter: The pattern rewriter
148// - opName: Base name of the operation (e.g., "fdiv", "addf", "mulf")
149// - device: Target device ("aie2", "aie2p", etc.)
150// - argTypes: Input argument types
151// - resultType: Return type
152// - bodyBuilder: Lambda that builds the function body given (builder, loc,
153// args)
154//
155// Returns: The helper function (created or existing)
156//
157// Function naming convention: __<device>_scalar_<opName>
158// Example: __aie2p_scalar_fdiv, __aie2_scalar_addf
159static LLVM::LLVMFuncOp getOrCreateScalarHelperFunc(
160 ModuleOp module, OpBuilder &rewriter, StringRef opName, StringRef device,
161 TypeRange argTypes, Type resultType,
162 const std::function<void(OpBuilder &, Location, ValueRange)> &bodyBuilder) {
163
164 // Build function name: __<device>_scalar_<opName>
165 std::string funcName = "__" + device.str() + "_scalar_" + opName.str();
166
167 // Check if function already exists
168 auto helperFunc = module.lookupSymbol<LLVM::LLVMFuncOp>(funcName);
169 if (helperFunc)
170 return helperFunc;
171
172 // Create new function
173 OpBuilder::InsertionGuard guard(rewriter);
174 rewriter.setInsertionPointToStart(module.getBody());
175
176 // Convert TypeRange to SmallVector<Type> for LLVM::LLVMFunctionType::get
177 SmallVector<Type> argTypesVec(argTypes.begin(), argTypes.end());
178
179 helperFunc = LLVM::LLVMFuncOp::create(
180 rewriter, rewriter.getUnknownLoc(), funcName,
181 LLVM::LLVMFunctionType::get(resultType, argTypesVec));
182
183 // Mark as noinline to act as optimization barrier
184 helperFunc->setAttr("passthrough", rewriter.getArrayAttr(
185 {rewriter.getStringAttr("noinline")}));
186
187 // Add function body
188 auto *entryBlock = helperFunc.addEntryBlock(rewriter);
189 OpBuilder::InsertionGuard bodyGuard(rewriter);
190 rewriter.setInsertionPointToStart(entryBlock);
191
192 // Collect function arguments
193 SmallVector<Value> args;
194 for (unsigned i = 0; i < argTypes.size(); ++i)
195 args.push_back(entryBlock->getArgument(i));
196
197 // Call the body builder with the function arguments
198 bodyBuilder(rewriter, rewriter.getUnknownLoc(), args);
199
200 return helperFunc;
201}
202
204 uint32_t start;
205 uint32_t offsets;
206 uint32_t offsets_hi;
207 uint32_t step;
208 uint32_t square;
209};
210
211// sgn_x: Sign mask of matrix X. If it is one matrix X is interpreted as
212// signed, else it treated as unsigned.
213// sgn_y: Sign mask of matrix Y. If it is one matrix Y is interpreted as
214// signed, else it treated as unsigned.
215// amode/bmode/variant: config acc width, mul precision, and mul mode
216// zero_acc: Zeroing of acc1. If it is one then acc1 is zeroed.
217// shift16: Shift mask of acc1. If a bit is set the <<16 operation will be
218// executed on acc1.
219// sub_mul: Negation mask of the matrix multiplication result. If it is
220// one the result of the operation will be negated.
221// sub_acc1: Negation mask of acc1. If it is one acc1 will be negated.
222// sub_acc2: Negation mask of acc2. If it is one acc2 will be negated.
223// sub_mask: Negation mask of complex multiplications. Negates a term of a
224// complex multiplication.
225static inline int aiev2_vmac_compute_control(int sgn_x, int sgn_y, int amode,
226 int bmode, int variant,
227 int zero_acc, int shift16,
228 int sub_mul, int sub_acc1,
229 int sub_acc2, int sub_mask) {
230 return ((unsigned)sub_mask << 16) | ((unsigned)shift16 << 10) |
231 ((unsigned)sub_mul << 11) | ((unsigned)sub_acc1 << 12) |
232 ((unsigned)sub_acc2 << 13) | ((unsigned)amode << 1) |
233 ((unsigned)bmode << 3) | ((unsigned)variant << 5) |
234 (((unsigned)sgn_x << 9) | ((unsigned)sgn_y << 8)) |
235 ((unsigned)zero_acc << 0);
236}
237
238static std::string getVectorTypeString(VectorType type, bool abbrev = false,
239 bool acc = false) {
240 std::stringstream ss;
241 auto size = getVectorLaneSize(type);
242 ss << "v" << size;
243 if (auto intType = dyn_cast<IntegerType>(type.getElementType())) {
244 ss << (acc ? "acc" : abbrev ? "i" : "int") << intType.getWidth();
245 } else if (dyn_cast<FloatType>(type.getElementType())) {
246 ss << (abbrev ? "f" : "float");
247 }
248 return ss.str();
249}
250
251static std::string getMulOrFMAIntrinsicName(Operation *op) {
252 std::string baseName;
253 Value lhs, result;
254 if (auto mulOp = dyn_cast<aievec::aie1::MulOp>(op)) {
255 baseName = "mul";
256 lhs = mulOp.getLhs();
257 result = mulOp.getResult();
258 } else if (auto fmaOp = dyn_cast<aievec::aie1::FMAOp>(op)) {
259 baseName = "mac";
260 lhs = fmaOp.getLhs();
261 result = fmaOp.getResult();
262 }
263 VectorType resultType = cast<VectorType>(result.getType());
264 int resultSize = getVectorLaneSize(resultType);
265 std::stringstream ss;
266 ss << "llvm.aie.";
267 if (dyn_cast<IntegerType>(resultType.getElementType())) {
268 ss << baseName;
269 ss << resultSize << "."
270 << getVectorTypeString(cast<VectorType>(lhs.getType()));
271 } else if (dyn_cast<FloatType>(resultType.getElementType())) {
272 ss << "vfp" << baseName;
273 }
274 return ss.str();
275}
276
277// Squashes the easy-to-read 16-bit square encoding into
278// the 8-bit encoding the configuration register uses
279static uint32_t encodeSquare(uint32_t square) {
280 uint32_t out = 0;
281 out |= ((square >> 0) & 0x3) << 0;
282 out |= ((square >> 4) & 0x3) << 2;
283 out |= ((square >> 8) & 0x3) << 4;
284 out |= ((square >> 12) & 0x3) << 6;
285 return out & 0xFF;
286}
287
288// Encode the configuration register with buffer parameters and options
289// TODO: struct to handle this?
290static void encodeConf(uint32_t conf[2], const BufferParams &x,
291 const BufferParams &z, bool sub) {
292 conf[0] |= ((x.step & 0x3F) << 0) | ((z.step & 0x3F) << 8);
293 conf[1] |= (encodeSquare(x.square) << 0) | (encodeSquare(z.square) << 8);
294 conf[1] |= sub << 17;
295}
296
298 : public mlir::ConvertOpToLLVMPattern<aievec::aie1::AddOp> {
299public:
300 using ConvertOpToLLVMPattern<aievec::aie1::AddOp>::ConvertOpToLLVMPattern;
301
302 LogicalResult
303 matchAndRewrite(aievec::aie1::AddOp op, OpAdaptor adaptor,
304 ConversionPatternRewriter &rewriter) const override {
305 op.emitWarning() << "aie.add conversion is not implemented\n";
306 return failure();
307 }
308};
309
310// AIE2 version of AddElemOp conversion
312 : public mlir::ConvertOpToLLVMPattern<aievec::AddElemOp> {
313public:
314 using ConvertOpToLLVMPattern<aievec::AddElemOp>::ConvertOpToLLVMPattern;
315
321
322 static DecodedAddElemOp decodeAddElemOp(OpAdaptor op) {
323 auto lhs = op.getLhs();
324 auto lhsVecTy = cast<VectorType>(lhs.getType());
325 auto lhsScaTy = lhsVecTy.getElementType();
326 unsigned lhsBitWidth = lhsScaTy.getIntOrFloatBitWidth();
327
328 // Integer types
329 if (llvm::isa<IntegerType>(lhsScaTy)) {
331 } // Float types
332 if (lhsBitWidth == 32) {
333 // FP32 add_elem
335 }
336
338 }
339
340 LogicalResult
341 matchAndRewrite(aievec::AddElemOp op, OpAdaptor adaptor,
342 ConversionPatternRewriter &rewriter) const override {
343 Location loc = op.getLoc();
344 auto decodedAddElemOp = decodeAddElemOp(adaptor);
345
346 if (decodedAddElemOp.kind == DecodedAddElemOp::Kind::UNSUPPORTED) {
347 op.emitWarning() << "aievec.add_elem conversion is not supported.\n";
348 return failure();
349 }
350
351 // Handle the FP32 add_elem for AIE2 - uses packed I64 representation
352 if (decodedAddElemOp.kind ==
354 auto confCst = LLVM::ConstantOp::create(
355 rewriter, loc, rewriter.getI32Type(),
356 rewriter.getI32IntegerAttr(decodedAddElemOp.conf));
357 SmallVector<Value> operands(
358 {adaptor.getLhs(), adaptor.getRhs(), confCst});
359
360 auto addElemOp = xllvm::AddAccFloatAIE2IntrOp::create(
361 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()),
362 forceCastOperandsToSignature(
363 rewriter, loc, operands,
364 {VectorType::get({8}, rewriter.getI64Type()),
365 VectorType::get({8}, rewriter.getI64Type()),
366 rewriter.getI32Type()}));
367
368 // create bitcast/shape_cast for result
369 auto resultVal = forceCastValueToType(rewriter, loc, addElemOp,
370 op.getResult().getType());
371 rewriter.replaceOp(op, resultVal);
372 return success();
373 }
374
375 op.emitWarning() << "aievec.add_elem conversion is not supported.\n";
376 return failure();
377 }
378};
379
380// AIE2 version of SubElemOp conversion
382 : public mlir::ConvertOpToLLVMPattern<aievec::SubElemOp> {
383public:
384 using ConvertOpToLLVMPattern<aievec::SubElemOp>::ConvertOpToLLVMPattern;
385
391
392 static DecodedSubElemOp decodeSubElemOp(OpAdaptor op) {
393 auto lhs = op.getLhs();
394 auto lhsVecTy = cast<VectorType>(lhs.getType());
395 auto lhsScaTy = lhsVecTy.getElementType();
396 unsigned lhsBitWidth = lhsScaTy.getIntOrFloatBitWidth();
397
398 // Integer types
399 if (llvm::isa<IntegerType>(lhsScaTy)) {
401 } // Float types
402 if (lhsBitWidth == 32) {
403 // FP32 sub_elem
405 }
406
408 }
409
410 LogicalResult
411 matchAndRewrite(aievec::SubElemOp op, OpAdaptor adaptor,
412 ConversionPatternRewriter &rewriter) const override {
413 Location loc = op.getLoc();
414 auto decodedSubElemOp = decodeSubElemOp(adaptor);
415
416 if (decodedSubElemOp.kind == DecodedSubElemOp::Kind::UNSUPPORTED) {
417 op.emitWarning() << "aievec.sub_elem conversion is not supported.\n";
418 return failure();
419 }
420
421 // Handle the FP32 sub_elem for AIE2 - uses packed I64 representation
422 if (decodedSubElemOp.kind ==
424 auto confCst = LLVM::ConstantOp::create(
425 rewriter, loc, rewriter.getI32Type(),
426 rewriter.getI32IntegerAttr(decodedSubElemOp.conf));
427 SmallVector<Value> operands(
428 {adaptor.getLhs(), adaptor.getRhs(), confCst});
429
430 auto subElemOp = xllvm::SubAccFloatAIE2IntrOp::create(
431 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()),
432 forceCastOperandsToSignature(
433 rewriter, loc, operands,
434 {VectorType::get({8}, rewriter.getI64Type()),
435 VectorType::get({8}, rewriter.getI64Type()),
436 rewriter.getI32Type()}));
437
438 // create bitcast/shape_cast for result
439 auto resultVal = forceCastValueToType(rewriter, loc, subElemOp,
440 op.getResult().getType());
441 rewriter.replaceOp(op, resultVal);
442 return success();
443 }
444
445 op.emitWarning() << "aievec.sub_elem conversion is not supported.\n";
446 return failure();
447 }
448};
449
450// AIE2p version of AddElemOp and SubElemOp conversion for f32.
451//
452// AIE2p adds and subtracts f32 only on the 2048-bit accumulator, which holds
453// 64 lanes: the ACC2048 accfloat intrinsic. An operand with fewer lanes that
454// divide 64 is padded with poison, and the low lanes of the result are kept.
455// The intrinsic is used even at 64 lanes, as Peano's generic fsub on
456// <64 x float> gives wrong results.
457template <typename SrcOpTy, typename IntrOpTy>
459 : public mlir::ConvertOpToLLVMPattern<SrcOpTy> {
460public:
461 using mlir::ConvertOpToLLVMPattern<SrcOpTy>::ConvertOpToLLVMPattern;
462 using OpAdaptor = typename SrcOpTy::Adaptor;
463
464 LogicalResult
465 matchAndRewrite(SrcOpTy op, OpAdaptor adaptor,
466 ConversionPatternRewriter &rewriter) const override {
467 constexpr int64_t accLanes = 64;
468 Location loc = op.getLoc();
469 auto type = cast<VectorType>(adaptor.getLhs().getType());
470 int64_t lanes = type.getNumElements();
471 if (!type.getElementType().isF32() || type.getRank() != 1 ||
472 accLanes % lanes != 0 || lanes % 2 != 0) {
473 op.emitWarning() << op->getName() << " conversion is not supported.\n";
474 return failure();
475 }
476
477 // The padding and the extraction shuffle pairs of lanes as i64, which
478 // Peano lowers to fewer register moves than f32 shuffles.
479 auto accType = VectorType::get({accLanes}, rewriter.getF32Type());
480 auto accI64Type = VectorType::get({accLanes / 2}, rewriter.getI64Type());
481 auto i64Type = VectorType::get({lanes / 2}, rewriter.getI64Type());
482 auto lowLanes = [](int64_t n, int64_t size) {
483 SmallVector<int64_t> mask(size, -1);
484 std::iota(mask.begin(), mask.begin() + std::min(n, size), 0);
485 return mask;
486 };
487 Value lhs = adaptor.getLhs(), rhs = adaptor.getRhs();
488 if (lanes != accLanes) {
489 lhs = LLVM::BitcastOp::create(rewriter, loc, i64Type, lhs);
490 rhs = LLVM::BitcastOp::create(rewriter, loc, i64Type, rhs);
491 SmallVector<int64_t> padMask = lowLanes(lanes / 2, accLanes / 2);
492 lhs = vector::ShuffleOp::create(rewriter, loc, lhs, lhs, padMask);
493 rhs = vector::ShuffleOp::create(rewriter, loc, rhs, rhs, padMask);
494 lhs = LLVM::BitcastOp::create(rewriter, loc, accType, lhs);
495 rhs = LLVM::BitcastOp::create(rewriter, loc, accType, rhs);
496 }
497 // conf 60 selects the fp32 accumulator datapath.
498 auto confCst = LLVM::ConstantOp::create(
499 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(60));
500 Value result = IntrOpTy::create(rewriter, loc, accType, lhs, rhs, confCst);
501 if (lanes != accLanes) {
502 result = LLVM::BitcastOp::create(rewriter, loc, accI64Type, result);
503 result = vector::ShuffleOp::create(rewriter, loc, result, result,
504 lowLanes(lanes / 2, lanes / 2));
505 result = LLVM::BitcastOp::create(rewriter, loc, type, result);
506 }
507 rewriter.replaceOp(op, result);
508 return success();
509 }
510};
511
513 AccFloatAddSubAIE2pConversion<aievec::AddElemOp,
514 xllvm::AddACC2048AccFloatAIE2pIntrOp>;
515
516// AIE2p version of AddElemOp conversion for i32: the 2048-bit accumulator's
517// integer add, whose 64 lanes of 32 bits are carried as 32 of i64. An operand
518// with fewer lanes is padded with poison, and the low lanes of the result are
519// kept, as for f32.
521 : public mlir::ConvertOpToLLVMPattern<aievec::AddElemOp> {
522public:
523 using ConvertOpToLLVMPattern<aievec::AddElemOp>::ConvertOpToLLVMPattern;
524
525 LogicalResult
526 matchAndRewrite(aievec::AddElemOp op, OpAdaptor adaptor,
527 ConversionPatternRewriter &rewriter) const override {
528 constexpr int64_t accLanes = 64;
529 Location loc = op.getLoc();
530 auto type = cast<VectorType>(adaptor.getLhs().getType());
531 int64_t lanes = type.getNumElements();
532 if (!type.getElementType().isInteger(32) || type.getRank() != 1 ||
533 accLanes % lanes != 0 || lanes % 2 != 0)
534 return failure();
535 auto accI64Type = VectorType::get({accLanes / 2}, rewriter.getI64Type());
536 auto i64Type = VectorType::get({lanes / 2}, rewriter.getI64Type());
537 auto lowLanes = [](int64_t n, int64_t size) {
538 SmallVector<int64_t> mask(size, -1);
539 std::iota(mask.begin(), mask.begin() + std::min(n, size), 0);
540 return mask;
541 };
542 Value lhs =
543 LLVM::BitcastOp::create(rewriter, loc, i64Type, adaptor.getLhs());
544 Value rhs =
545 LLVM::BitcastOp::create(rewriter, loc, i64Type, adaptor.getRhs());
546 if (lanes != accLanes) {
547 SmallVector<int64_t> padMask = lowLanes(lanes / 2, accLanes / 2);
548 lhs = vector::ShuffleOp::create(rewriter, loc, lhs, lhs, padMask);
549 rhs = vector::ShuffleOp::create(rewriter, loc, rhs, rhs, padMask);
550 }
551 // conf 0 selects the 32-bit integer datapath.
552 auto confCst = LLVM::ConstantOp::create(
553 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(0));
554 Value result = xllvm::AddACC2048AIE2pIntrOp::create(
555 rewriter, loc, accI64Type, lhs, rhs, confCst);
556 if (lanes != accLanes)
557 result = vector::ShuffleOp::create(rewriter, loc, result, result,
558 lowLanes(lanes / 2, lanes / 2));
559 rewriter.replaceOp(op,
560 LLVM::BitcastOp::create(rewriter, loc, type, result));
561 return success();
562 }
563};
565 AccFloatAddSubAIE2pConversion<aievec::SubElemOp,
566 xllvm::SubACC2048AccFloatAIE2pIntrOp>;
567
568// AIE2 version of NegOp conversion.
569//
570// aievec.neg operates on an accumulator. For floats the accumulator is
571// v16accfloat, which is carried as vector<16xf32> in AIEVec and as <8 x i64>
572// at the intrinsic boundary, so this maps directly onto the ACC512 accfloat
573// negate.
574class NegOpAIE2Conversion : public mlir::ConvertOpToLLVMPattern<aievec::NegOp> {
575public:
576 using ConvertOpToLLVMPattern<aievec::NegOp>::ConvertOpToLLVMPattern;
577
578 LogicalResult
579 matchAndRewrite(aievec::NegOp op, OpAdaptor adaptor,
580 ConversionPatternRewriter &rewriter) const override {
581 Location loc = op.getLoc();
582 auto srcVecTy = cast<VectorType>(adaptor.getSource().getType());
583 auto srcScaTy = srcVecTy.getElementType();
584
585 // Only the float accumulator (v16accfloat) is wired up. Integer
586 // accumulators would need the ACC1024 acc32/acc64 negates instead.
587 if (!isa<FloatType>(srcScaTy) || srcScaTy.getIntOrFloatBitWidth() != 32 ||
588 srcVecTy.getNumElements() != 16) {
589 op.emitWarning() << "aievec.neg conversion is not supported.\n";
590 return failure();
591 }
592
593 auto v8i64Ty = VectorType::get({8}, rewriter.getI64Type());
594 // conf selects the fp32 accumulator datapath. Matches the AIE API's
595 // neg(v16accfloat): aiev2_compute_control(0, 0, /*amode=*/2, /*bmode=*/3,
596 // 0, 0, 0, 0, 0, 0, 0) == (2 << 1) | (3 << 3) == 28.
597 auto confCst = LLVM::ConstantOp::create(
598 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(28));
599 SmallVector<Value> operands({adaptor.getSource(), confCst});
600
601 auto negOp = xllvm::NegACC512AccFloatAIE2IntrOp::create(
602 rewriter, loc, v8i64Ty,
603 forceCastOperandsToSignature(rewriter, loc, operands,
604 {v8i64Ty, rewriter.getI32Type()}));
605
606 auto resultVal =
607 forceCastValueToType(rewriter, loc, negOp, op.getResult().getType());
608 rewriter.replaceOp(op, resultVal);
609 return success();
610 }
611};
612
613// AIE2p version of NegOp conversion.
614//
615// AIE2p only exposes the negate on the 2048-bit accumulator, so a
616// v16accfloat operand is widened to <64 x float>, negated, and the low 16
617// lanes extracted back out. This mirrors AddElemOpAIE2pConversion.
619 : public mlir::ConvertOpToLLVMPattern<aievec::NegOp> {
620public:
621 using ConvertOpToLLVMPattern<aievec::NegOp>::ConvertOpToLLVMPattern;
622
623 LogicalResult
624 matchAndRewrite(aievec::NegOp op, OpAdaptor adaptor,
625 ConversionPatternRewriter &rewriter) const override {
626 Location loc = op.getLoc();
627 auto srcVecTy = cast<VectorType>(adaptor.getSource().getType());
628 auto srcScaTy = srcVecTy.getElementType();
629 unsigned laneSize = srcVecTy.getNumElements();
630
631 if (!isa<FloatType>(srcScaTy) || srcScaTy.getIntOrFloatBitWidth() != 32 ||
632 (laneSize != 16 && laneSize != 32)) {
633 op.emitWarning() << "aievec.neg conversion is not supported.\n";
634 return failure();
635 }
636
637 auto v64f32Ty = VectorType::get({64}, rewriter.getF32Type());
638
639 // The masks below address individual lanes, but vector.shuffle indexes
640 // the leading dimension, so an n-D operand has to be flattened first.
641 Value srcVal = adaptor.getSource();
642 auto fltSrcVecTy = getFlattenedVectorType(srcVecTy);
643 if (srcVecTy != fltSrcVecTy)
644 srcVal = vector::ShapeCastOp::create(rewriter, loc, fltSrcVecTy, srcVal)
645 .getResult();
646
647 // Widen the accumulator to ACC2048, filling the unused lanes with poison.
648 SmallVector<int64_t> expandMask;
649 for (unsigned i = 0; i < laneSize; ++i)
650 expandMask.push_back(i);
651 for (unsigned i = laneSize; i < 64; ++i)
652 expandMask.push_back(-1);
653 auto srcExpanded =
654 vector::ShuffleOp::create(rewriter, loc, srcVal, srcVal, expandMask);
655
656 // conf selects the fp32 accumulator datapath, matching the ACC2048
657 // accfloat add/sub lowerings and the AIE API's neg(v64accfloat):
658 // aie2p_compute_control(0, 0, /*amode=*/2, /*bmode=*/3, /*variant=*/1,
659 // 0, 0, 0, 0, 0, 0) == (2 << 1) | (3 << 3) | (1 << 5) == 60.
660 auto confCst = LLVM::ConstantOp::create(
661 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(60));
662 auto negResult = xllvm::NegACC2048AccFloatAIE2pIntrOp::create(
663 rewriter, loc, v64f32Ty, srcExpanded, confCst);
664
665 SmallVector<int64_t> extractMask;
666 for (unsigned i = 0; i < laneSize; ++i)
667 extractMask.push_back(i);
668 Value finalResult = vector::ShuffleOp::create(rewriter, loc, negResult,
669 negResult, extractMask)
670 .getResult();
671
672 // Restore the operand's shape.
673 if (srcVecTy != fltSrcVecTy)
674 finalResult =
675 vector::ShapeCastOp::create(rewriter, loc, srcVecTy, finalResult)
676 .getResult();
677
678 rewriter.replaceOp(op, finalResult);
679 return success();
680 }
681};
682
684 : public mlir::ConvertOpToLLVMPattern<aievec::aie1::SubOp> {
685public:
686 using ConvertOpToLLVMPattern<aievec::aie1::SubOp>::ConvertOpToLLVMPattern;
687
688 LogicalResult
689 matchAndRewrite(aievec::aie1::SubOp op, OpAdaptor adaptor,
690 ConversionPatternRewriter &rewriter) const override {
691 op.emitWarning() << "aie.sub conversion is not implemented\n";
692 return failure();
693 }
694};
695
697 : public mlir::ConvertOpToLLVMPattern<aievec::aie1::FMAOp> {
698public:
699 using ConvertOpToLLVMPattern<aievec::aie1::FMAOp>::ConvertOpToLLVMPattern;
700
701 LogicalResult
702 matchAndRewrite(aievec::aie1::FMAOp op, OpAdaptor adaptor,
703 ConversionPatternRewriter &rewriter) const override {
704 auto module = op->getParentOfType<ModuleOp>();
705 MLIRContext *context = rewriter.getContext();
706
707 auto startType = IntegerType::get(context, 32);
708 auto offsetsType = VectorType::get({2}, IntegerType::get(context, 32));
709 auto confType = VectorType::get({2}, IntegerType::get(context, 32));
710
711 // If the intrinsic declaration doesn't exist, create it
712 std::string intrinsicName = getMulOrFMAIntrinsicName(op);
713 auto func = module.lookupSymbol<LLVM::LLVMFuncOp>(
714 StringAttr::get(context, intrinsicName));
715
716 if (!func) {
717 OpBuilder::InsertionGuard guard(rewriter);
718 rewriter.setInsertionPointToStart(module.getBody());
719 func = LLVM::LLVMFuncOp::create(
720 rewriter, rewriter.getUnknownLoc(), intrinsicName,
721 LLVM::LLVMFunctionType::get(
722 op.getResult().getType(),
723 {op.getLhs().getType(), op.getRhs().getType(),
724 op.getAcc().getType(), startType, /* xstart */
725 startType, /* ystart */
726 startType, /* zstart */
727 offsetsType, /* xoffsets */
728 offsetsType, /* zoffsets */
729 confType}));
730 }
731
732 // Parse the string attribute values
733 BufferParams x = {};
734 BufferParams z = {};
735 op.getXstart().getAsInteger(0, x.start);
736 op.getXoffsets().getAsInteger(0, x.offsets);
737 op.getXoffsetsHi().getAsInteger(0, x.offsets_hi);
738 op.getXstep().getAsInteger(0, x.step);
739 op.getXsquare().getAsInteger(0, x.square);
740 op.getZstart().getAsInteger(0, z.start);
741 op.getZoffsets().getAsInteger(0, z.offsets);
742 op.getZoffsetsHi().getAsInteger(0, z.offsets_hi);
743 op.getZstep().getAsInteger(0, z.step);
744 op.getZsquare().getAsInteger(0, z.square);
745
746 // Encode the configuration register
747 uint32_t conf[2] = {0, 0};
748 encodeConf(conf, x, z, op.getFmsub());
749
750 // Create the constants and replace the op
751 auto xstartVal = LLVM::ConstantOp::create(
752 rewriter, op->getLoc(), startType, rewriter.getI32IntegerAttr(x.start));
753 auto ystartVal = LLVM::ConstantOp::create(rewriter, op->getLoc(), startType,
754 rewriter.getI32IntegerAttr(0));
755 auto zstartVal = LLVM::ConstantOp::create(
756 rewriter, op->getLoc(), startType, rewriter.getI32IntegerAttr(z.start));
757 auto xoffsetsVal = LLVM::ConstantOp::create(
758 rewriter, op->getLoc(), offsetsType,
759 rewriter.getI32VectorAttr({(int32_t)x.offsets, (int32_t)x.offsets_hi}));
760 auto zoffsetsVal = LLVM::ConstantOp::create(
761 rewriter, op->getLoc(), offsetsType,
762 rewriter.getI32VectorAttr({(int32_t)z.offsets, (int32_t)z.offsets_hi}));
763 auto confVal = LLVM::ConstantOp::create(
764 rewriter, op->getLoc(), confType,
765 rewriter.getI32VectorAttr({(int32_t)conf[0], (int32_t)conf[1]}));
766 rewriter.replaceOpWithNewOp<LLVM::CallOp>(
767 op, func,
768 ValueRange{op.getLhs(), op.getRhs(), op.getAcc(), xstartVal, ystartVal,
769 zstartVal, xoffsetsVal, zoffsetsVal, confVal});
770 return success();
771 }
772};
773
775 : public mlir::ConvertOpToLLVMPattern<aievec::aie1::MulOp> {
776public:
777 using ConvertOpToLLVMPattern<aievec::aie1::MulOp>::ConvertOpToLLVMPattern;
778
779 LogicalResult
780 matchAndRewrite(aievec::aie1::MulOp op, OpAdaptor adaptor,
781 ConversionPatternRewriter &rewriter) const override {
782 auto module = op->getParentOfType<ModuleOp>();
783 MLIRContext *context = rewriter.getContext();
784
785 auto startType = IntegerType::get(context, 32);
786 auto offsetsType = VectorType::get({2}, IntegerType::get(context, 32));
787 auto confType = VectorType::get({2}, IntegerType::get(context, 32));
788
789 // If the intrinsic declaration doesn't exist, create it
790 std::string intrinsicName = getMulOrFMAIntrinsicName(op);
791 auto func = module.lookupSymbol<LLVM::LLVMFuncOp>(
792 StringAttr::get(context, intrinsicName));
793
794 if (!func) {
795 OpBuilder::InsertionGuard guard(rewriter);
796 rewriter.setInsertionPointToStart(module.getBody());
797 func = LLVM::LLVMFuncOp::create(
798 rewriter, rewriter.getUnknownLoc(), intrinsicName,
799 LLVM::LLVMFunctionType::get(op.getResult().getType(),
800 {op.getLhs().getType(),
801 op.getRhs().getType(),
802 startType, /* xstart */
803 startType, /* ystart */
804 startType, /* zstart */
805 offsetsType, /* xoffsets */
806 offsetsType, /* zoffsets */
807 confType}));
808 }
809
810 // Parse the string attribute values
811 BufferParams x = {};
812 BufferParams z = {};
813 op.getXstart().getAsInteger(0, x.start);
814 op.getXoffsets().getAsInteger(0, x.offsets);
815 op.getXoffsetsHi().getAsInteger(0, x.offsets_hi);
816 op.getXstep().getAsInteger(0, x.step);
817 op.getXsquare().getAsInteger(0, x.square);
818 op.getZstart().getAsInteger(0, z.start);
819 op.getZoffsets().getAsInteger(0, z.offsets);
820 op.getZoffsetsHi().getAsInteger(0, z.offsets_hi);
821 op.getZstep().getAsInteger(0, z.step);
822 op.getZsquare().getAsInteger(0, z.square);
823
824 // Encode the configuration register
825 uint32_t conf[2] = {0, 0};
826 encodeConf(conf, x, z, false);
827
828 // Create the constants and replace the op
829 auto xstartVal = LLVM::ConstantOp::create(
830 rewriter, op->getLoc(), startType, rewriter.getI32IntegerAttr(x.start));
831 auto ystartVal = LLVM::ConstantOp::create(rewriter, op->getLoc(), startType,
832 rewriter.getI32IntegerAttr(0));
833 auto zstartVal = LLVM::ConstantOp::create(
834 rewriter, op->getLoc(), startType, rewriter.getI32IntegerAttr(z.start));
835 auto xoffsetsVal = LLVM::ConstantOp::create(
836 rewriter, op->getLoc(), offsetsType,
837 rewriter.getI32VectorAttr({(int32_t)x.offsets, (int32_t)x.offsets_hi}));
838 auto zoffsetsVal = LLVM::ConstantOp::create(
839 rewriter, op->getLoc(), offsetsType,
840 rewriter.getI32VectorAttr({(int32_t)z.offsets, (int32_t)z.offsets_hi}));
841 auto confVal = LLVM::ConstantOp::create(
842 rewriter, op->getLoc(), confType,
843 rewriter.getI32VectorAttr({(int32_t)conf[0], (int32_t)conf[1]}));
844 rewriter.replaceOpWithNewOp<LLVM::CallOp>(
845 op, func,
846 ValueRange{op.getLhs(), op.getRhs(), xstartVal, ystartVal, zstartVal,
847 xoffsetsVal, zoffsetsVal, confVal});
848 return success();
849 }
850};
851
853 : public mlir::ConvertOpToLLVMPattern<aievec::MulElemOp> {
854public:
855 using ConvertOpToLLVMPattern<aievec::MulElemOp>::ConvertOpToLLVMPattern;
856
857 MulElemOpConversion(const LLVMTypeConverter &typeConverter,
858 Aie2Fp32Emulation aie2Fp32EmulationOption)
859 : ConvertOpToLLVMPattern(typeConverter),
861
862 Aie2Fp32Emulation aie2Fp32EmulationOption;
863
865 enum class Kind {
866 // DtIn0_DtIn1_DtRes_CxMxKxN
873 // TODO: I16_I16_I64_16x1x2x1
874 };
875
877 int conf;
878 };
879
880 static DecodedMulElemOp decodeMulElemOp(OpAdaptor op) {
881 auto lhs = op.getLhs();
882 auto lhsVecTy = cast<VectorType>(lhs.getType());
883 auto lhsScaTy = lhsVecTy.getElementType();
884 unsigned lhsBitWidth = lhsScaTy.getIntOrFloatBitWidth();
885
886 // Integer types
887 if (llvm::isa<IntegerType>(lhsScaTy)) {
888 if (lhsBitWidth == 8) {
890 aiev2_vmac_compute_control(
891 /*sgn_x=*/1, /*sgn_y=*/1, /*amode=*/0, /*bmode=*/1,
892 /*variant=*/1, /*zero_acc=*/0, /*shift16=*/0,
893 /*sub_mul=*/0, /*sub_acc1=*/0, /*sub_acc2=*/0,
894 /*sub_mask=*/0)};
895 }
896 if (lhsBitWidth == 16) {
898 aiev2_vmac_compute_control(
899 /*sgn_x=*/1, /*sgn_y=*/1, /*amode=*/0, /*bmode=*/3,
900 /*variant=*/1, /*zero_acc=*/0, /*shift16=*/0,
901 /*sub_mul=*/0, /*sub_acc1=*/0, /*sub_acc2=*/0,
902 /*sub_mask=*/0)};
903 }
904 if (lhsBitWidth == 32) {
905 // emulated I32 mul_elem
907 }
908 } else {
909 // Float types
910 if (lhsBitWidth == 16) {
912 aiev2_vmac_compute_control(
913 /*sgn_x=*/0, /*sgn_y=*/0, /*amode=*/2, /*bmode=*/3,
914 /*variant=*/1, /*zero_acc=*/0, /*shift16=*/0,
915 /*sub_mul=*/0, /*sub_acc1=*/0, /*sub_acc2=*/0,
916 /*sub_mask=*/0)};
917 }
918 if (lhsBitWidth == 32) {
919 // emulated FP32 mul_elem
921 }
922 }
923
925 }
926
927 // This conversion pattern implements the below CPP emulated I32 mul_elem.
928 // INTRINSIC(v16acc64)
929 // mul_elem_16_2(v16int32 a0, v16int32 a1, v16int32 b0, v16int32 b1) {
930 // v32uint16 a_lo = (v32uint16)shuffle(a0, a1, 2);
931 // v32int16 a_hi = (v32int16)shuffle(a0, a1, 3);
932 // v32uint16 b_lo = (v32uint16)shuffle(b0, b1, 2);
933 // v32int16 b_hi = (v32int16)shuffle(b0, b1, 3);
934 // v16acc64 acc = ::mul_elem_16_2(a_hi, b_hi);
935 // acc = mac_elem_16_2_conf(a_hi, 1, b_lo, false, acc, 0, 1, 0, 0);
936 // acc = mac_elem_16_2_conf(a_lo, false, b_hi, 1, acc, 0, 0, 0, 0);
937 // acc = mac_elem_16_2_conf(a_lo, false, b_lo, false, acc, 0, 1, 0, 0);
938 // return acc;
939 // }
940 // Caller example when handling the elementwise mul of two v16int32 vectors.
941 // v16int32 v1 = LHS();
942 // v16int32 v2 = RHS();
943 // v16acc64 v3 = mul_elem_16_2(v1, broadcast_zero_s32(), v2,
944 // undef_v16int32());
945 // Explantion:
946 // a_lo = low_part(a0[0]--a0[15], a1[0]--a1[15])
947 // a_hi = high_part(a0[0]--a0[15], a1[0]--a1[15])
948 // b_lo = low_part(b0[0]--b0[15], b1[0]--b1[15])
949 // b_hi = high_part(b0[0]--b0[15], b1[0]--b1[15])
950 // The firt `acc` is from mul_elem_16_2(a_hi, b_hi), which performs 16 channel
951 // of 1x2x1 matmul, acc[0] = a_hi[0]*b_hi[0]+a_hi[16]*b_hi[16], ... , acc[15]
952 // = a_hi[15]*b_hi[15]+a_hi[31]*b_hi[31]. Then, the first MAC performs `acc`
953 // left shift 16bit, and then 16 channel of 1x2x1 matmul (a_hi, b_lo)
954 // accumulating to `acc`. The second MAC performs 16 channel of 1x2x1 matmul
955 // (a_lo, b_hi) accumulating to `acc`. Finally, the third MAC performs 16
956 // channel of 1x2x1 matmul (a_lo, b_hi) accumulating to `acc`.
957 LogicalResult
958 convertToEmulatedI32MulElem(aievec::MulElemOp op, OpAdaptor adaptor,
959 ConversionPatternRewriter &rewriter) const {
960
961 Location loc = op.getLoc();
962 auto zeroCst = LLVM::ConstantOp::create(
963 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(0));
964 auto a0 = adaptor.getLhs();
965 auto a1 = xllvm::VectorBroadcast32I512IntrOp::create(
966 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()), zeroCst);
967 auto b0 = adaptor.getRhs();
968 auto b1 = xllvm::UndefV16I32IntrOp::create(
969 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()));
970
971 // 4* Shuffle
972 auto a_lo = xllvm::VectorShuffleIntrOp::create(
973 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()), a0, a1,
974 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
975 rewriter.getI32IntegerAttr(2)));
976 auto a_hi = xllvm::VectorShuffleIntrOp::create(
977 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()), a0, a1,
978 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
979 rewriter.getI32IntegerAttr(3)));
980 auto b_lo = xllvm::VectorShuffleIntrOp::create(
981 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()), b0, b1,
982 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
983 rewriter.getI32IntegerAttr(2)));
984 auto b_hi = xllvm::VectorShuffleIntrOp::create(
985 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()), b0, b1,
986 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
987 rewriter.getI32IntegerAttr(3)));
988 // MUL + 3 * MAC
989 auto mulConfCst = LLVM::ConstantOp::create(
990 rewriter, loc, rewriter.getI32Type(),
991 rewriter.getI32IntegerAttr(aiev2_vmac_compute_control(
992 /*sgn_x=*/1, /*sgn_y=*/1, /*amode=*/1, /*bmode=*/3,
993 /*variant=*/2, /*zero_acc=*/0, /*shift16=*/0,
994 /*sub_mul=*/0, /*sub_acc1=*/0, /*sub_acc2=*/0, /*sub_mask=*/0)));
995 auto mulConfOp = xllvm::MulConfAcc64IntrOp::create(
996 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()),
997 forceCastOperandsToSignature(
998 rewriter, loc,
999 /*operands=*/{a_hi, b_hi, mulConfCst},
1000 /*signature=*/
1001 {VectorType::get({64}, rewriter.getI8Type()),
1002 VectorType::get({16}, rewriter.getI32Type()),
1003 rewriter.getI32Type()}));
1004
1005 auto createMacConfOp = [&](SmallVector<Value> operands,
1006 int macConf) -> Value {
1007 operands.push_back(
1008 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1009 rewriter.getI32IntegerAttr(macConf)));
1010 return xllvm::MacConfAcc64IntrOp::create(
1011 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()),
1012 forceCastOperandsToSignature(
1013 rewriter, loc,
1014 /*operands=*/operands,
1015 /*signature=*/
1016 {VectorType::get({64}, rewriter.getI8Type()),
1017 VectorType::get({16}, rewriter.getI32Type()),
1018 VectorType::get({16}, rewriter.getI64Type()),
1019 rewriter.getI32Type()}))
1020 .getResult();
1021 };
1022 auto acc64Val = mulConfOp.getResult();
1023 acc64Val = createMacConfOp(
1024 SmallVector<Value>{a_hi, b_lo, acc64Val},
1025 aiev2_vmac_compute_control(
1026 /*sgn_x=*/1, /*sgn_y=*/0, /*amode=*/1, /*bmode=*/3,
1027 /*variant=*/2, /*zero_acc=*/0, /*shift16=*/1,
1028 /*sub_mul=*/0, /*sub_acc1=*/0, /*sub_acc2=*/0, /*sub_mask=*/0));
1029 acc64Val = createMacConfOp(
1030 SmallVector<Value>{a_lo, b_hi, acc64Val},
1031 aiev2_vmac_compute_control(
1032 /*sgn_x=*/0, /*sgn_y=*/1, /*amode=*/1, /*bmode=*/3,
1033 /*variant=*/2, /*zero_acc=*/0, /*shift16=*/0,
1034 /*sub_mul=*/0, /*sub_acc1=*/0, /*sub_acc2=*/0, /*sub_mask=*/0));
1035 acc64Val = createMacConfOp(
1036 SmallVector<Value>{a_lo, b_lo, acc64Val},
1037 aiev2_vmac_compute_control(
1038 /*sgn_x=*/0, /*sgn_y=*/0, /*amode=*/1, /*bmode=*/3,
1039 /*variant=*/2, /*zero_acc=*/0, /*shift16=*/1,
1040 /*sub_mul=*/0, /*sub_acc1=*/0, /*sub_acc2=*/0, /*sub_mask=*/0));
1041
1042 // create bitcast/shape_cast for result
1043 auto resultVal =
1044 forceCastValueToType(rewriter, loc, acc64Val, op.getResult().getType());
1045 rewriter.replaceOp(op, resultVal);
1046 return success();
1047 }
1048
1049 // This conversion pattern implements the below CPP emulated FP32 mul_elem.
1050 // inline v16accfloat mul_elem_16_accuracy_safe(v16float v1, v16float v2) {
1051 // v32bfloat16 a = broadcast_zero_to_v32bfloat16();
1052 // v32bfloat16 b = broadcast_zero_to_v32bfloat16();
1053 // v32bfloat16 c = broadcast_zero_to_v32bfloat16();
1054 // v32bfloat16 d = broadcast_zero_to_v32bfloat16();
1055 // v32bfloat16 e = broadcast_zero_to_v32bfloat16();
1056 // v32bfloat16 f = broadcast_zero_to_v32bfloat16();
1057 // v32bfloat16 dummy0 = broadcast_one_to_v32bfloat16();
1058 // a = insert(a,0,to_v16bfloat16((v16accfloat)v1));
1059 // v16accfloat acc0 = msc_elem_16_2(a, dummy0, (v16accfloat)v1);
1060 // b = insert(b,0,to_v16bfloat16(acc0));
1061 // c = insert(c,0,to_v16bfloat16(msc_elem_16_2(b, dummy0, acc0)));
1062 // d = insert(d,0,to_v16bfloat16((v16accfloat)v2));
1063 // v16accfloat acc1 = msc_elem_16_2(d, dummy0, (v16accfloat)v2);
1064 // e = insert(e,0,to_v16bfloat16(acc1));
1065 // f = insert(f,0,to_v16bfloat16(msc_elem_16_2(e, dummy0, acc1)));
1066 // return
1067 // mac_elem_16_2(a,d,mac_elem_16_2(a,e,mac_elem_16_2(b,d,mac_elem_16_2(
1068 // d,c,mac_elem_16_2(b,e,mac_elem_16_2(a,f,mac_elem_16_2(
1069 // b,f,mac_elem_16_2(c,e,mul_elem_16_2(c,f)))))))));
1070 // }
1071 // Caller example when handling the elementwise mul of two v16float vectors.
1072 // v16float v1 = LHS(); v16float v2 = RHS();
1073 // v16accfloat v3 = mul_elem_16(v1, v2);
1074 // Explantion: For v32bfloat16 `a`, the first half v16bf16 contains `most
1075 // significant 7 bits of mantissa` from v1, and the second half v16bf16 are
1076 // zeros. For v16accfloat `acc0`, the MSC equals to "(original `v1` with 23
1077 // bits of mantissa) - (`a` with MSB 7 bits of mantissa from v1)". For
1078 // v32bfloat16 `b`, the first half v16bf16 contains `[7:13] bits of mantissa
1079 // from v1` from v1, and the second half v16bf16 are zeros. For v32bfloat16
1080 // `c`, the first half v16bf16 contains `[14:20] bits of mantissa from v1`
1081 // from v1, and the second half v16bf16 are zeros. Hence, we can represent
1082 // v16float in three v32bfloat16 and then perform 9 MUL/MAC in v32bfloat16 to
1083 // get the final elementwise multiplication result.
1084
1085 LogicalResult
1086 convertToEmulatedFP32MulElem(aievec::MulElemOp op, OpAdaptor adaptor,
1087 ConversionPatternRewriter &rewriter) const {
1088 Location loc = op.getLoc();
1089 auto zeroCst =
1090 LLVM::ConstantOp::create(rewriter, loc, rewriter.getBF16Type(),
1091 rewriter.getZeroAttr(rewriter.getBF16Type()));
1092 auto aZeros = xllvm::VectorBroadcast16BF512IntrOp::create(
1093 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), zeroCst);
1094 auto bZeros = xllvm::VectorBroadcast16BF512IntrOp::create(
1095 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), zeroCst);
1096 auto cZeros = xllvm::VectorBroadcast16BF512IntrOp::create(
1097 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), zeroCst);
1098 auto dZeros = xllvm::VectorBroadcast16BF512IntrOp::create(
1099 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), zeroCst);
1100 auto eZeros = xllvm::VectorBroadcast16BF512IntrOp::create(
1101 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), zeroCst);
1102 auto fZeros = xllvm::VectorBroadcast16BF512IntrOp::create(
1103 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), zeroCst);
1104 auto oneCst =
1105 LLVM::ConstantOp::create(rewriter, loc, rewriter.getBF16Type(),
1106 rewriter.getOneAttr(rewriter.getBF16Type()));
1107 auto dummy0 = xllvm::VectorBroadcast16BF512IntrOp::create(
1108 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), oneCst);
1109 auto zeroCstI32 = LLVM::ConstantOp::create(
1110 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(0));
1111 auto mscMacMulConfCst = LLVM::ConstantOp::create(
1112 rewriter, loc, rewriter.getI32Type(),
1113 rewriter.getI32IntegerAttr(aiev2_vmac_compute_control(
1114 /*sgn_x=*/0, /*sgn_y=*/0, /*amode=*/2, /*bmode=*/3,
1115 /*variant=*/1, /*zero_acc=*/0, /*shift16=*/0,
1116 /*sub_mul=*/0, /*sub_acc1=*/0, /*sub_acc2=*/0, /*sub_mask=*/0)));
1117
1118 auto extractV16FP32ToThreeV16BF16 =
1119 [&](Value inputV16FP32, Value aZeros, Value bZeros,
1120 Value cZeros) -> std::tuple<Value, Value, Value> {
1121 // a = insert(a,0,to_v16bfloat16((v16accfloat)v1));
1122 auto inputBitCasted =
1123 forceCastValueToType(rewriter, loc, inputV16FP32,
1124 VectorType::get({8}, rewriter.getI64Type()));
1125 auto v1ToBF16 = xllvm::Vector16AccFloatToV16BF16AIE2IntrOp::create(
1126 rewriter, loc, VectorType::get({16}, rewriter.getBF16Type()),
1127 inputBitCasted);
1128 auto a = xllvm::UpdBF512BF256IntrOp::create(
1129 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), aZeros,
1130 v1ToBF16, zeroCstI32);
1131
1132 // v16accfloat acc0 = msc_elem_16_2(a, dummy0, (v16accfloat)v1);
1133 auto acc0 = xllvm::MscConfBF16IntrOp::create(
1134 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()), a, dummy0,
1135 inputBitCasted, mscMacMulConfCst);
1136
1137 // b = insert(b,0,to_v16bfloat16(acc0));
1138 auto acc0ToBF16 = xllvm::Vector16AccFloatToV16BF16AIE2IntrOp::create(
1139 rewriter, loc, VectorType::get({16}, rewriter.getBF16Type()), acc0);
1140 auto b = xllvm::UpdBF512BF256IntrOp::create(
1141 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), bZeros,
1142 acc0ToBF16, zeroCstI32);
1143
1144 // c = insert(c,0,to_v16bfloat16(msc_elem_16_2(b, dummy0, acc0)));
1145 auto acc0Mscb = xllvm::MscConfBF16IntrOp::create(
1146 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()), b, dummy0,
1147 acc0, mscMacMulConfCst);
1148 auto acc0MscbToBF16 = xllvm::Vector16AccFloatToV16BF16AIE2IntrOp::create(
1149 rewriter, loc, VectorType::get({16}, rewriter.getBF16Type()),
1150 acc0Mscb);
1151 auto c = xllvm::UpdBF512BF256IntrOp::create(
1152 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), cZeros,
1153 acc0MscbToBF16, zeroCstI32);
1154 return std::make_tuple(a.getResult(), b.getResult(), c.getResult());
1155 };
1156
1157 // Get v16vfloat16 a, b, c for representing v16float v1
1158 auto [a, b, c] =
1159 extractV16FP32ToThreeV16BF16(adaptor.getLhs(), aZeros, bZeros, cZeros);
1160 // Get v16vfloat16 d, e, f for representing v16float v2
1161 auto [d, e, f] =
1162 extractV16FP32ToThreeV16BF16(adaptor.getRhs(), dZeros, eZeros, fZeros);
1163
1164 // Create 1 MUL and 2/5/8 MACs depending on the Aie2Fp32EmulationOption
1165 auto createMacOps = [&](Value lhs, Value rhs, Value acc) -> Value {
1166 return xllvm::MacConfBF16IntrOp::create(
1167 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()),
1168 lhs, rhs, acc, mscMacMulConfCst)
1169 .getResult();
1170 };
1171
1172 Value finalMacVal;
1173 if (aie2Fp32EmulationOption == Aie2Fp32Emulation::AccuracyFast) {
1174 // Fast and Accurate option. float a*b would require 6 mac operations.
1175 // Input fp32 number is split in to 3 bfloat16 numbers to extract all the
1176 // bits of the mantissa. float a,b; both a and b are split in to 3
1177 // bfloat16 numbers each. Hence there would be 9 mac operations in
1178 // multiplication of a and b. In the 9 mac operations to emulate fp32 mul,
1179 // mac operations with LSBs are ignored. (3 last terms). This helps
1180 // improve cycle count of mul and has least impact on accuracy of result.
1181 // This is the default option to the aiecompiler
1182 auto afMul = xllvm::MulConfBF16IntrOp::create(
1183 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()), a, f,
1184 mscMacMulConfCst);
1185 finalMacVal = createMacOps(
1186 a, d,
1187 createMacOps(
1188 a, e,
1189 createMacOps(b, d,
1190 createMacOps(d, c, createMacOps(b, e, afMul)))));
1191 } else if (aie2Fp32EmulationOption == Aie2Fp32Emulation::AccuracyLow) {
1192 // Fast and least accurate option. float a*b would require 3 mac
1193 // operations.
1194 // Input fp32 number is split in to 2 bfloat16 numbers. Hence not all the
1195 // bits from mantissa can be used. float a,b; Both a and b are split in to
1196 // 2 bfloat16 numbers each. Hence there would be 4 mac operations in
1197 // multiplication of a and b. In the 4 mac operations to emulate fp32 mul,
1198 // mac operations with LSBs are ignored. (1 last term). This helps improve
1199 // cycle count of mul float a, b;
1200 auto bdMul = xllvm::MulConfBF16IntrOp::create(
1201 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()), b, d,
1202 mscMacMulConfCst);
1203 finalMacVal = createMacOps(a, d, createMacOps(a, e, bdMul));
1204 } else {
1205 // aie2Fp32EmulationOption == Aie2Fp32Emulation::AccuracySafe
1206 // Most accurate option since input fp32 number is split in to 3 bfloat16
1207 // numbers to extract all the bits of the mantissa. float a*b would
1208 // require 9 mac operations due to 3 bfloat16 splits each.
1209 auto cfMul = xllvm::MulConfBF16IntrOp::create(
1210 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()), c, f,
1211 mscMacMulConfCst);
1212 finalMacVal = createMacOps(
1213 a, d,
1214 createMacOps(
1215 a, e,
1216 createMacOps(
1217 b, d,
1218 createMacOps(
1219 d, c,
1220 createMacOps(
1221 b, e,
1222 createMacOps(
1223 a, f,
1224 createMacOps(b, f,
1225 createMacOps(c, e, cfMul))))))));
1226 }
1227
1228 // create bitcast/shape_cast for result
1229 auto resultVal = forceCastValueToType(rewriter, loc, finalMacVal,
1230 op.getResult().getType());
1231 rewriter.replaceOp(op, resultVal);
1232 return success();
1233 }
1234
1235 LogicalResult
1236 matchAndRewrite(aievec::MulElemOp op, OpAdaptor adaptor,
1237 ConversionPatternRewriter &rewriter) const override {
1238 Location loc = op.getLoc();
1239 auto decodedMulElemOp = decodeMulElemOp(adaptor);
1240
1241 if (decodedMulElemOp.kind == DecodedMulElemOp::Kind::UNSUPPORTED) {
1242 op.emitWarning() << "aievec.mul_elem conversion is not supported.\n";
1243 return failure();
1244 }
1245
1246 // Handle the emulated I32/FP32 mul_elem
1247 if (decodedMulElemOp.kind == DecodedMulElemOp::Kind::I32_I32_I64_32x1x2x1) {
1248 return convertToEmulatedI32MulElem(op, adaptor, rewriter);
1249 }
1250 if (decodedMulElemOp.kind ==
1252 return convertToEmulatedFP32MulElem(op, adaptor, rewriter);
1253 }
1254
1255 // create constant for config
1256 auto confCst = LLVM::ConstantOp::create(
1257 rewriter, loc, rewriter.getI32Type(),
1258 rewriter.getI32IntegerAttr(decodedMulElemOp.conf));
1259 Value mulElemOp = nullptr;
1260 SmallVector<Value> operands({adaptor.getLhs(), adaptor.getRhs(), confCst});
1261
1262 // create xllvm intrinsic
1263 if (decodedMulElemOp.kind == DecodedMulElemOp::Kind::I16_I16_I32_32x1x1x1 ||
1264 decodedMulElemOp.kind == DecodedMulElemOp::Kind::I8_I8_I32_32x1x2x1) {
1265 mulElemOp = xllvm::MulConfAcc32IntrOp::create(
1266 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()),
1267 forceCastOperandsToSignature(
1268 rewriter, loc, operands,
1269 {VectorType::get({64}, rewriter.getI8Type()),
1270 VectorType::get({16}, rewriter.getI32Type()),
1271 rewriter.getI32Type()}));
1272 } else if (decodedMulElemOp.kind ==
1274 // Create zero vector using the exact pattern from working reference:
1275 // vbroadcast16.I512(0) -> bitcast to bf16 -> extract lower 256 bits
1276 auto zero32 = LLVM::ConstantOp::create(
1277 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(0));
1278 auto zeros_i16 = xllvm::VectorBroadcast16I512IntrOp::create(
1279 rewriter, loc, VectorType::get({32}, rewriter.getI16Type()), zero32);
1280 auto zeros_bf16 = LLVM::BitcastOp::create(
1281 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()),
1282 zeros_i16);
1283 auto zeroVec = xllvm::ExtBF256BF512IntrOp::create(
1284 rewriter, loc, VectorType::get({16}, rewriter.getBF16Type()),
1285 zeros_bf16, zero32);
1286
1287 // Use set+upd pattern to match working reference
1288 auto idx1 = LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1289 rewriter.getI32IntegerAttr(1));
1290
1291 // Set lhs at lower 256 bits, then update upper 256 bits with zeros
1292 auto lhsSet = xllvm::VectorSetBF512BF256IntrOp::create(
1293 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()),
1294 adaptor.getLhs(), zero32);
1295 auto lhsConcat = xllvm::UpdBF512BF256IntrOp::create(
1296 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), lhsSet,
1297 zeroVec, idx1);
1298
1299 // Set rhs at lower 256 bits, then update upper 256 bits with zeros
1300 auto rhsSet = xllvm::VectorSetBF512BF256IntrOp::create(
1301 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()),
1302 adaptor.getRhs(), zero32);
1303 auto rhsConcat = xllvm::UpdBF512BF256IntrOp::create(
1304 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), rhsSet,
1305 zeroVec, idx1);
1306
1307 // Call bf.mul16.conf with padded vectors
1308 mulElemOp = xllvm::MulConfBF16IntrOp::create(
1309 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()), lhsConcat,
1310 rhsConcat, confCst);
1311 }
1312
1313 // create bitcast/shape_cast for result
1314 auto resultVal = forceCastValueToType(rewriter, loc, mulElemOp,
1315 op.getResult().getType());
1316 rewriter.replaceOp(op, resultVal);
1317 return success();
1318 }
1319};
1320
1321// AIE2p version of MulElemOp conversion
1323 : public mlir::ConvertOpToLLVMPattern<aievec::MulElemOp> {
1324public:
1325 using ConvertOpToLLVMPattern<aievec::MulElemOp>::ConvertOpToLLVMPattern;
1326
1328 enum class Kind {
1329 BF16_BF16_FP32_16x1x1x1, // 16-lane bf16 -> 16-lane f32
1330 BF16_BF16_FP32_32x1x2x1, // 32-lane bf16 -> 32-lane f32
1331 BF16_BF16_FP32_64x1x2x1, // 64-lane bf16 -> 64-lane f32
1333 };
1335 int conf;
1336 };
1337
1338 static DecodedMulElemOp decodeMulElemOp(OpAdaptor op) {
1339 auto lhs = op.getLhs();
1340 auto lhsVecTy = cast<VectorType>(lhs.getType());
1341 auto lhsScaTy = lhsVecTy.getElementType();
1342 unsigned lhsBitWidth = lhsScaTy.getIntOrFloatBitWidth();
1343 int lhsLanes = getVectorLaneSize(lhsVecTy);
1344
1345 // Integer types - not supported for AIE2p elementwise mul
1346 if (llvm::isa<IntegerType>(lhsScaTy)) {
1348 } // Float types
1349 if (lhsBitWidth == 16) {
1350 // BF16 mul_elem
1351 if (lhsLanes == 16) {
1352 // 16-lane bfloat16 uses I512.I512.ACC512 intrinsic
1354 }
1355 if (lhsLanes == 32) {
1356 // 32-lane bfloat16 uses I512.I512.ACC1024 intrinsic
1358 }
1359 if (lhsLanes == 64) {
1360 // 64-lane bfloat16 uses I1024.I1024.ACC2048 intrinsic
1362 }
1363 }
1364
1366 }
1367
1368 LogicalResult
1369 matchAndRewrite(aievec::MulElemOp op, OpAdaptor adaptor,
1370 ConversionPatternRewriter &rewriter) const override {
1371 Location loc = op.getLoc();
1372 auto decodedMulElemOp = decodeMulElemOp(adaptor);
1373
1374 if (decodedMulElemOp.kind == DecodedMulElemOp::Kind::UNSUPPORTED) {
1375 op.emitWarning() << "aievec.mul_elem conversion is not supported for "
1376 "AIE2p.\n";
1377 return failure();
1378 }
1379
1380 // Create constant for config
1381 auto confCst = LLVM::ConstantOp::create(
1382 rewriter, loc, rewriter.getI32Type(),
1383 rewriter.getI32IntegerAttr(decodedMulElemOp.conf));
1384
1385 Value mulElemOp = nullptr;
1386
1387 // Handle BF16 mul_elem for AIE2p
1388 if (decodedMulElemOp.kind ==
1390 // 16-lane bfloat16: <16 x bfloat> x <16 x bfloat> -> <16 x float>
1391 // The intrinsic requires <32 x bfloat> inputs, so we need to pad
1392
1393 // Pad LHS from 16 to 32 bfloat16 using shuffle
1394 SmallVector<int64_t> padMask;
1395 for (int i = 0; i < 16; ++i)
1396 padMask.push_back(i);
1397 for (int i = 16; i < 32; ++i)
1398 padMask.push_back(-1); // poison/undef
1399
1400 auto lhsPadded = vector::ShuffleOp::create(
1401 rewriter, loc, adaptor.getLhs(), adaptor.getLhs(), padMask);
1402 auto rhsPadded = vector::ShuffleOp::create(
1403 rewriter, loc, adaptor.getRhs(), adaptor.getRhs(), padMask);
1404
1405 SmallVector<Value> operands({lhsPadded, rhsPadded, confCst});
1406
1407 // Call I512.I512.ACC512 intrinsic
1408 mulElemOp = xllvm::MulConfBF16I512ACC512AIE2pIntrOp::create(
1409 rewriter, loc, VectorType::get({16}, rewriter.getF32Type()),
1410 forceCastOperandsToSignature(
1411 rewriter, loc, operands,
1412 {VectorType::get({32}, rewriter.getBF16Type()),
1413 VectorType::get({32}, rewriter.getBF16Type()),
1414 rewriter.getI32Type()}));
1415 } else if (decodedMulElemOp.kind ==
1417 // 32-lane bfloat16: <32 x bfloat> x <32 x bfloat> -> <32 x float>
1418 SmallVector<Value> operands(
1419 {adaptor.getLhs(), adaptor.getRhs(), confCst});
1420 mulElemOp = xllvm::MulConfBF16I512ACC1024AIE2pIntrOp::create(
1421 rewriter, loc, VectorType::get({32}, rewriter.getF32Type()),
1422 forceCastOperandsToSignature(
1423 rewriter, loc, operands,
1424 {VectorType::get({32}, rewriter.getBF16Type()),
1425 VectorType::get({32}, rewriter.getBF16Type()),
1426 rewriter.getI32Type()}));
1427 } else if (decodedMulElemOp.kind ==
1429 // 64-lane bfloat16: <64 x bfloat> x <64 x bfloat> -> <64 x float>
1430 SmallVector<Value> operands(
1431 {adaptor.getLhs(), adaptor.getRhs(), confCst});
1432 mulElemOp = xllvm::MulConfBF16I1024ACC2048AIE2pIntrOp::create(
1433 rewriter, loc, VectorType::get({64}, rewriter.getF32Type()),
1434 forceCastOperandsToSignature(
1435 rewriter, loc, operands,
1436 {VectorType::get({64}, rewriter.getBF16Type()),
1437 VectorType::get({64}, rewriter.getBF16Type()),
1438 rewriter.getI32Type()}));
1439 }
1440
1441 // create bitcast/shape_cast for result
1442 auto resultVal = forceCastValueToType(rewriter, loc, mulElemOp,
1443 op.getResult().getType());
1444 rewriter.replaceOp(op, resultVal);
1445 return success();
1446 }
1447};
1448
1449// AIE2p version of FMAElemOp conversion. Uses native F32 accumulators
1450// and AIE2p-specific MAC intrinsics.
1452 : public mlir::ConvertOpToLLVMPattern<aievec::FMAElemOp> {
1453public:
1454 using ConvertOpToLLVMPattern<aievec::FMAElemOp>::ConvertOpToLLVMPattern;
1455
1456 LogicalResult
1457 matchAndRewrite(aievec::FMAElemOp fmaOp, OpAdaptor adaptor,
1458 ConversionPatternRewriter &rewriter) const override {
1459 auto loc = fmaOp.getLoc();
1460 auto lhs = adaptor.getLhs();
1461 auto rhs = adaptor.getRhs();
1462 auto acc = adaptor.getAcc();
1463 auto lhsTy = cast<VectorType>(lhs.getType());
1464 auto accTy = cast<VectorType>(acc.getType());
1465 auto flatLhsTy = getFlattenedVectorType(lhsTy);
1466 auto flatAccTy = getFlattenedVectorType(accTy);
1467
1468 // Flatten operands, if needed
1469 if (lhsTy != flatLhsTy)
1470 lhs = vector::ShapeCastOp::create(rewriter, loc, flatLhsTy, lhs);
1471 if (cast<VectorType>(rhs.getType()) != flatLhsTy)
1472 rhs = vector::ShapeCastOp::create(rewriter, loc, flatLhsTy, rhs);
1473 if (accTy != flatAccTy)
1474 acc = vector::ShapeCastOp::create(rewriter, loc, flatAccTy, acc);
1475
1476 if (!flatLhsTy.getElementType().isBF16()) {
1477 fmaOp.emitWarning()
1478 << "aievec.mac_elem AIE2p conversion only supports bf16 inputs.\n";
1479 return failure();
1480 }
1481
1482 Type i32ty = rewriter.getI32Type();
1483 auto confCst = LLVM::ConstantOp::create(
1484 rewriter, loc, i32ty,
1485 rewriter.getI32IntegerAttr(aiev2_vmac_compute_control(
1486 /*sgn_x=*/0, /*sgn_y=*/0, /*amode=*/2, /*bmode=*/3,
1487 /*variant=*/1, /*zero_acc=*/0, /*shift16=*/0,
1488 /*sub_mul=*/0, /*sub_acc1=*/0, /*sub_acc2=*/0,
1489 /*sub_mask=*/0)));
1490
1491 unsigned lhsLanes = flatLhsTy.getNumElements();
1492 Value macIntrOp = nullptr;
1493
1494 if (lhsLanes == 16) {
1495 // 16-lane bf16: pad to v32bf16, use I512.I512.ACC512 intrinsic
1496 SmallVector<int64_t> padMask;
1497 for (int i = 0; i < 16; ++i)
1498 padMask.push_back(i);
1499 for (int i = 16; i < 32; ++i)
1500 padMask.push_back(-1); // poison
1501
1502 auto lhsPadded =
1503 vector::ShuffleOp::create(rewriter, loc, lhs, lhs, padMask);
1504 auto rhsPadded =
1505 vector::ShuffleOp::create(rewriter, loc, rhs, rhs, padMask);
1506
1507 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
1508 auto v16f32Ty = VectorType::get({16}, rewriter.getF32Type());
1509 macIntrOp = xllvm::MacConfBF16I512ACC512AIE2pIntrOp::create(
1510 rewriter, loc, v16f32Ty,
1511 forceCastOperandsToSignature(
1512 rewriter, loc, {lhsPadded, rhsPadded, acc, confCst},
1513 {v32bf16Ty, v32bf16Ty, v16f32Ty, i32ty}));
1514 } else if (lhsLanes == 32) {
1515 // 32-lane bf16: direct, use I512.I512.ACC1024 intrinsic
1516 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
1517 auto v32f32Ty = VectorType::get({32}, rewriter.getF32Type());
1518 macIntrOp = xllvm::MacConfBF16I512ACC1024AIE2pIntrOp::create(
1519 rewriter, loc, v32f32Ty,
1520 forceCastOperandsToSignature(
1521 rewriter, loc, {lhs, rhs, acc, confCst},
1522 {v32bf16Ty, v32bf16Ty, v32f32Ty, i32ty}));
1523 } else {
1524 fmaOp.emitWarning()
1525 << "aievec.mac_elem AIE2p conversion: unsupported lane count "
1526 << lhsLanes << ".\n";
1527 return failure();
1528 }
1529
1530 // Recast/Reshape result
1531 auto resVal = forceCastValueToType(rewriter, loc, macIntrOp, flatAccTy);
1532 if (flatAccTy != accTy)
1533 resVal = vector::ShapeCastOp::create(rewriter, loc, accTy, resVal);
1534
1535 rewriter.replaceOp(fmaOp, resVal);
1536 return success();
1537 }
1538};
1539
1540class UPSOpAIE2Conversion : public mlir::ConvertOpToLLVMPattern<aievec::UPSOp> {
1541public:
1542 using ConvertOpToLLVMPattern<aievec::UPSOp>::ConvertOpToLLVMPattern;
1543
1544 LogicalResult
1545 matchAndRewrite(aievec::UPSOp op, OpAdaptor adaptor,
1546 ConversionPatternRewriter &rewriter) const override {
1547 Location loc = op.getLoc();
1548
1549 Value result = op.getResult();
1550 VectorType resultType = cast<VectorType>(result.getType());
1551 VectorType flatResTy = getFlattenedVectorType(resultType);
1552 Type resultScaTy = resultType.getElementType();
1553 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
1554 int resultLanes = getVectorLaneSize(resultType);
1555 int resultVectorSize = resultBitWidth * resultLanes;
1556
1557 Value opSrcVal = adaptor.getSource();
1558 auto srcVecTy = cast<VectorType>(opSrcVal.getType());
1559 auto fltSrcVecTy = getFlattenedVectorType(srcVecTy);
1560 if (srcVecTy != fltSrcVecTy)
1561 opSrcVal = vector::ShapeCastOp::create(rewriter, op.getLoc(), fltSrcVecTy,
1562 opSrcVal)
1563 .getResult();
1564
1565 // create xllvm intrinsic
1566 // Integer types
1567 Value upsIntrOp = nullptr;
1568 if (llvm::isa<IntegerType>(resultScaTy)) {
1569 // create constant for sign
1570 auto signCst = LLVM::ConstantOp::create(
1571 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(1));
1572 auto shiftCst =
1573 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1574 rewriter.getI32IntegerAttr(op.getShift()));
1575
1576 SmallVector<Value> operands({opSrcVal, shiftCst, signCst});
1577 if (resultVectorSize == 512) {
1578 if (resultBitWidth == 32) {
1579 // v16int16 -> v16acc32
1580 upsIntrOp = xllvm::Acc32V16I256UpsAIE2IntrOp::create(
1581 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()),
1582 forceCastOperandsToSignature(
1583 rewriter, loc, operands,
1584 {VectorType::get({16}, rewriter.getI16Type()),
1585 rewriter.getI32Type(), rewriter.getI32Type()}));
1586 } else if (resultBitWidth == 64) {
1587 // v8int32 -> v8acc64
1588 upsIntrOp = xllvm::Acc64V8I256UpsAIE2IntrOp::create(
1589 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()),
1590 forceCastOperandsToSignature(
1591 rewriter, loc, operands,
1592 {VectorType::get({8}, rewriter.getI32Type()),
1593 rewriter.getI32Type(), rewriter.getI32Type()}));
1594 }
1595 } else if (resultVectorSize == 1024) {
1596 Value src = opSrcVal;
1597 VectorType srcType = cast<VectorType>(src.getType());
1598 Type srcScaType = srcType.getElementType();
1599 unsigned srcBitWidth = srcScaType.getIntOrFloatBitWidth();
1600
1601 if (resultBitWidth == 32 && srcBitWidth == 16) {
1602 // v32int16 -> v32acc32
1603 upsIntrOp = xllvm::Acc32V32I512UpsAIE2IntrOp::create(
1604 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()),
1605 forceCastOperandsToSignature(
1606 rewriter, loc, operands,
1607 {VectorType::get({32}, rewriter.getI16Type()),
1608 rewriter.getI32Type(), rewriter.getI32Type()}));
1609 } else if (resultBitWidth == 64 && srcBitWidth == 32) {
1610 // v16int32 -> v16acc64
1611 upsIntrOp = xllvm::Acc64V16I512UpsAIE2IntrOp::create(
1612 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()),
1613 forceCastOperandsToSignature(
1614 rewriter, loc, operands,
1615 {VectorType::get({16}, rewriter.getI32Type()),
1616 rewriter.getI32Type(), rewriter.getI32Type()}));
1617 } else if (resultBitWidth == 64 && srcBitWidth == 16) {
1618 // v16int16 -> v16acc64
1619 upsIntrOp = xllvm::Acc64V16I256UpsAIE2IntrOp::create(
1620 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()),
1621 forceCastOperandsToSignature(
1622 rewriter, loc, operands,
1623 {VectorType::get({16}, rewriter.getI16Type()),
1624 rewriter.getI32Type(), rewriter.getI32Type()}));
1625 } else if (resultBitWidth == 32 && srcBitWidth == 8) {
1626 // v32int8 -> v32acc32
1627 upsIntrOp = xllvm::Acc32V32I256UpsAIE2IntrOp::create(
1628 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()),
1629 forceCastOperandsToSignature(
1630 rewriter, loc, operands,
1631 {VectorType::get({32}, rewriter.getI8Type()),
1632 rewriter.getI32Type(), rewriter.getI32Type()}));
1633 }
1634 }
1635 } else {
1636 // Float types
1637 // AIE2p uses native F32 types, AIE2 uses packed I64 types
1638 if (resultVectorSize == 512) {
1639 // v16bfloat16 -> v16accfloat
1640 upsIntrOp = xllvm::Vector16BF16ToV16AccFloatAIE2IntrOp::create(
1641 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()),
1642 forceCastOperandsToSignature(
1643 rewriter, loc, {opSrcVal},
1644 {VectorType::get({16}, rewriter.getBF16Type())}));
1645 } else if (resultVectorSize == 1024) {
1646 // v32bfloat16 -> v32accfloat
1647 // The CPP example of the implementation is below:
1648 // INTRINSIC(v32accfloat) ups_to_v32accfloat(v32bfloat16 a) {
1649 // v16accfloat x0 = ups_to_v16accfloat(extract_v16bfloat16(a, 0));
1650 // v16accfloat x1 = ups_to_v16accfloat(extract_v16bfloat16(a, 1));
1651 // return concat(x0, x1);
1652 // }
1653 auto indexZeroCst =
1654 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1655 rewriter.getI32IntegerAttr(0));
1656 auto indexOneCst =
1657 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1658 rewriter.getI32IntegerAttr(1));
1659 auto extractUps = [&](Value source, Value index) -> Value {
1660 auto extOp = xllvm::ExtI256I512IntrOp::create(
1661 rewriter, loc, VectorType::get({8}, rewriter.getI32Type()),
1662 forceCastOperandsToSignature(
1663 rewriter, loc, {source, index},
1664 {VectorType::get({16}, rewriter.getI32Type()),
1665 rewriter.getI32Type()}));
1666 return xllvm::Vector16BF16ToV16AccFloatAIE2IntrOp::create(
1667 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()),
1668 forceCastOperandsToSignature(
1669 rewriter, loc, {extOp},
1670 {VectorType::get({16}, rewriter.getBF16Type())}));
1671 };
1672 auto resLo = extractUps(opSrcVal, indexZeroCst);
1673 auto resHi = extractUps(opSrcVal, indexOneCst);
1674 // Concat the two 512-bit vector to a 1024-bit vector.
1675 // Note that given sources a0 and a1, the result is [a1; a0].
1676 upsIntrOp = xllvm::ConcatI1024I512IntrOp::create(
1677 rewriter, loc, VectorType::get({32}, rewriter.getI32Type()),
1678 forceCastOperandsToSignature(
1679 rewriter, loc, {resLo, resHi},
1680 {VectorType::get({16}, rewriter.getI32Type()),
1681 VectorType::get({16}, rewriter.getI32Type())}));
1682 }
1683 }
1684
1685 if (!upsIntrOp) {
1686 op.emitWarning() << "aievec.ups is not supported.\n";
1687 return failure();
1688 }
1689
1690 // create bitcast for result if needed
1691 if (flatResTy != upsIntrOp.getType())
1692 upsIntrOp = LLVM::BitcastOp::create(rewriter, loc, flatResTy, upsIntrOp);
1693
1694 if (flatResTy != resultType)
1695 upsIntrOp =
1696 vector::ShapeCastOp::create(rewriter, loc, resultType, upsIntrOp);
1697
1698 rewriter.replaceOp(op, upsIntrOp);
1699
1700 return success();
1701 }
1702};
1703
1704// TODO: Split the op at AIEVec dialect level
1706 : public mlir::ConvertOpToLLVMPattern<aievec::UPSOp> {
1707public:
1708 using ConvertOpToLLVMPattern<aievec::UPSOp>::ConvertOpToLLVMPattern;
1709
1710 LogicalResult
1711 matchAndRewrite(aievec::UPSOp op, OpAdaptor adaptor,
1712 ConversionPatternRewriter &rewriter) const override {
1713 Location loc = op.getLoc();
1714
1715 Value result = op.getResult();
1716 VectorType resultType = cast<VectorType>(result.getType());
1717 VectorType flatResTy = getFlattenedVectorType(resultType);
1718 Type resultScaTy = resultType.getElementType();
1719 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
1720 int resultLanes = getVectorLaneSize(resultType);
1721 int resultVectorSize = resultBitWidth * resultLanes;
1722
1723 Value opSrcVal = adaptor.getSource();
1724 auto srcVecTy = cast<VectorType>(opSrcVal.getType());
1725 auto fltSrcVecTy = getFlattenedVectorType(srcVecTy);
1726 if (srcVecTy != fltSrcVecTy)
1727 opSrcVal = vector::ShapeCastOp::create(rewriter, op.getLoc(), fltSrcVecTy,
1728 opSrcVal)
1729 .getResult();
1730
1731 // create xllvm intrinsic
1732 // Integer types
1733 Value upsIntrOp = nullptr;
1734 if (llvm::isa<IntegerType>(resultScaTy)) {
1735 // create constant for sign
1736 auto signCst = LLVM::ConstantOp::create(
1737 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(1));
1738 auto shiftCst =
1739 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1740 rewriter.getI32IntegerAttr(op.getShift()));
1741
1742 SmallVector<Value> operands({opSrcVal, shiftCst, signCst});
1743 if (resultVectorSize == 512) {
1744 if (resultBitWidth == 32) {
1745 // v16int16 -> v16acc32
1746 upsIntrOp = xllvm::Acc32V16I256UpsAIE2pIntrOp::create(
1747 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
1748 forceCastOperandsToSignature(
1749 rewriter, loc, operands,
1750 {VectorType::get({16}, rewriter.getI16Type()),
1751 rewriter.getI32Type(), rewriter.getI32Type()}));
1752 } else if (resultBitWidth == 64) {
1753 // v8int32 -> v8acc64
1754 upsIntrOp = xllvm::Acc64V8I256UpsAIE2pIntrOp::create(
1755 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()),
1756 forceCastOperandsToSignature(
1757 rewriter, loc, operands,
1758 {VectorType::get({8}, rewriter.getI32Type()),
1759 rewriter.getI32Type(), rewriter.getI32Type()}));
1760 }
1761 } else if (resultVectorSize == 1024) {
1762 Value src = opSrcVal;
1763 VectorType srcType = cast<VectorType>(src.getType());
1764 Type srcScaType = srcType.getElementType();
1765 unsigned srcBitWidth = srcScaType.getIntOrFloatBitWidth();
1766 int srcLanes = getVectorLaneSize(srcType);
1767 int srcVectorSize = srcBitWidth * srcLanes;
1768
1769 if (resultBitWidth == 32 && srcBitWidth == 16 && srcVectorSize == 512) {
1770 // v32int16 -> v32acc32
1771 upsIntrOp = xllvm::Acc32V32I512UpsAIE2pIntrOp::create(
1772 rewriter, loc, VectorType::get({32}, rewriter.getI32Type()),
1773 forceCastOperandsToSignature(
1774 rewriter, loc, operands,
1775 {VectorType::get({32}, rewriter.getI16Type()),
1776 rewriter.getI32Type(), rewriter.getI32Type()}));
1777 } else if (resultBitWidth == 64 && srcBitWidth == 32 &&
1778 srcVectorSize == 512) {
1779 // v16int32 -> v16acc64
1780 upsIntrOp = xllvm::Acc64V16I512UpsAIE2pIntrOp::create(
1781 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()),
1782 forceCastOperandsToSignature(
1783 rewriter, loc, operands,
1784 {VectorType::get({16}, rewriter.getI32Type()),
1785 rewriter.getI32Type(), rewriter.getI32Type()}));
1786 } else if (resultBitWidth == 64 && srcBitWidth == 16 &&
1787 srcVectorSize == 256) {
1788 // v16int16 -> v16acc64
1789 upsIntrOp = xllvm::Acc64V16I256UpsAIE2pIntrOp::create(
1790 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()),
1791 forceCastOperandsToSignature(
1792 rewriter, loc, operands,
1793 {VectorType::get({16}, rewriter.getI16Type()),
1794 rewriter.getI32Type(), rewriter.getI32Type()}));
1795 } else if (resultBitWidth == 32 && srcBitWidth == 8 &&
1796 srcVectorSize == 256) {
1797 // v32int8 -> v32acc32
1798 upsIntrOp = xllvm::Acc32V32I256UpsAIE2pIntrOp::create(
1799 rewriter, loc, VectorType::get({32}, rewriter.getI32Type()),
1800 forceCastOperandsToSignature(
1801 rewriter, loc, operands,
1802 {VectorType::get({32}, rewriter.getI8Type()),
1803 rewriter.getI32Type(), rewriter.getI32Type()}));
1804 }
1805 } else if (resultVectorSize == 2048) {
1806 Value src = opSrcVal;
1807 VectorType srcType = cast<VectorType>(src.getType());
1808 Type srcScaType = srcType.getElementType();
1809 unsigned srcBitWidth = srcScaType.getIntOrFloatBitWidth();
1810 int srcLanes = getVectorLaneSize(srcType);
1811 int srcVectorSize = srcBitWidth * srcLanes;
1812
1813 if (resultBitWidth == 32 && srcBitWidth == 8 && srcVectorSize == 512) {
1814 // v64int8 -> v64acc32
1815 upsIntrOp = xllvm::Acc32V64I512UpsAIE2pIntrOp::create(
1816 rewriter, loc, VectorType::get({64}, rewriter.getI32Type()),
1817 forceCastOperandsToSignature(
1818 rewriter, loc, operands,
1819 {VectorType::get({64}, rewriter.getI8Type()),
1820 rewriter.getI32Type(), rewriter.getI32Type()}));
1821 } else if (resultBitWidth == 64 && srcBitWidth == 16 &&
1822 srcVectorSize == 512) {
1823 // v32int16 -> v32acc64
1824 upsIntrOp = xllvm::Acc64V32I512UpsAIE2pIntrOp::create(
1825 rewriter, loc, VectorType::get({32}, rewriter.getI64Type()),
1826 forceCastOperandsToSignature(
1827 rewriter, loc, operands,
1828 {VectorType::get({32}, rewriter.getI16Type()),
1829 rewriter.getI32Type(), rewriter.getI32Type()}));
1830 } else if (resultBitWidth == 32 && srcBitWidth == 16 &&
1831 srcVectorSize == 1024) {
1832 // v64int16 -> v64acc32
1833 // Extract 2 chunks of v32int16 and convert each to v32acc32
1834 auto index0Cst =
1835 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1836 rewriter.getI32IntegerAttr(0));
1837 auto index1Cst =
1838 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1839 rewriter.getI32IntegerAttr(1));
1840
1841 auto extractUps2048 = [&](Value source, Value index, Value shiftCst,
1842 Value signCst) -> Value {
1843 // Use vector::ShuffleOp to extract 512-bit from 1024-bit
1844 // Cast source to v32xi32 for shuffling
1845 auto v32i32Source = forceCastValueToType(
1846 rewriter, loc, source,
1847 VectorType::get({32}, rewriter.getI32Type()));
1848
1849 // Determine shuffle mask based on index
1850 // index 0: elements [0-15]
1851 // index 1: elements [16-31]
1852 SmallVector<int64_t> shuffleMask;
1853 if (auto constIndex = index.getDefiningOp<LLVM::ConstantOp>()) {
1854 auto indexAttr = cast<IntegerAttr>(constIndex.getValue());
1855 int64_t idxVal = indexAttr.getInt();
1856 int startIdx = idxVal * 16;
1857 for (int i = 0; i < 16; ++i) {
1858 shuffleMask.push_back(startIdx + i);
1859 }
1860 } else {
1861 // Default to index 0 if not constant
1862 for (int i = 0; i < 16; ++i) {
1863 shuffleMask.push_back(i);
1864 }
1865 }
1866
1867 auto extOp = vector::ShuffleOp::create(rewriter, loc, v32i32Source,
1868 v32i32Source, shuffleMask);
1869
1870 return xllvm::Acc32V32I512UpsAIE2pIntrOp::create(
1871 rewriter, loc, VectorType::get({32}, rewriter.getI32Type()),
1872 forceCastOperandsToSignature(
1873 rewriter, loc, {extOp, shiftCst, signCst},
1874 {VectorType::get({32}, rewriter.getI16Type()),
1875 rewriter.getI32Type(), rewriter.getI32Type()}));
1876 };
1877
1878 auto res0 = extractUps2048(opSrcVal, index0Cst, shiftCst, signCst);
1879 auto res1 = extractUps2048(opSrcVal, index1Cst, shiftCst, signCst);
1880
1881 // Concat two 1024-bit vectors to a 2048-bit vector using
1882 // vector::ShuffleOp
1883 SmallVector<int64_t> concatMask;
1884 for (int i = 0; i < 64; ++i) {
1885 concatMask.push_back(i);
1886 }
1887 upsIntrOp =
1888 vector::ShuffleOp::create(rewriter, loc, res0, res1, concatMask);
1889 }
1890 }
1891 } else {
1892 // Float types
1893 // AIE2p uses native F32 types, AIE2 uses packed I64 types
1894 if (resultVectorSize == 512) {
1895 // v16bfloat16 -> v16accfloat
1896 upsIntrOp = xllvm::Vector16BF16ToV16AccFloatAIE2pIntrOp::create(
1897 rewriter, loc, VectorType::get({16}, rewriter.getF32Type()),
1898 forceCastOperandsToSignature(
1899 rewriter, loc, {opSrcVal},
1900 {VectorType::get({16}, rewriter.getBF16Type())}));
1901 } else if (resultVectorSize == 1024) {
1902 // v32bfloat16 -> v32accfloat
1903 upsIntrOp = xllvm::Vector32BF16ToV32AccFloatAIE2pIntrOp::create(
1904 rewriter, loc, VectorType::get({32}, rewriter.getF32Type()),
1905 forceCastOperandsToSignature(
1906 rewriter, loc, {opSrcVal},
1907 {VectorType::get({32}, rewriter.getBF16Type())}));
1908 } else if (resultVectorSize == 2048) {
1909 // v64bfloat16 -> v64accfloat
1910 // Extract 2 chunks of v32bfloat16 and convert each to v32accfloat
1911 auto index0Cst =
1912 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1913 rewriter.getI32IntegerAttr(0));
1914 auto index1Cst =
1915 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1916 rewriter.getI32IntegerAttr(1));
1917
1918 auto extractUps2048 = [&](Value source, Value index) -> Value {
1919 // Use vector::ShuffleOp to extract 512-bit from 1024-bit
1920 // Cast source to v32xi32 for shuffling
1921 auto v32i32Source = forceCastValueToType(
1922 rewriter, loc, source,
1923 VectorType::get({32}, rewriter.getI32Type()));
1924
1925 // Determine shuffle mask based on index
1926 // index 0: elements [0-15]
1927 // index 1: elements [16-31]
1928 SmallVector<int64_t> shuffleMask;
1929 if (auto constIndex = index.getDefiningOp<LLVM::ConstantOp>()) {
1930 auto indexAttr = cast<IntegerAttr>(constIndex.getValue());
1931 int64_t idxVal = indexAttr.getInt();
1932 int startIdx = idxVal * 16;
1933 for (int i = 0; i < 16; ++i) {
1934 shuffleMask.push_back(startIdx + i);
1935 }
1936 } else {
1937 // Default to index 0 if not constant
1938 for (int i = 0; i < 16; ++i) {
1939 shuffleMask.push_back(i);
1940 }
1941 }
1942
1943 auto extOp = vector::ShuffleOp::create(rewriter, loc, v32i32Source,
1944 v32i32Source, shuffleMask);
1945
1946 return xllvm::Vector32BF16ToV32AccFloatAIE2pIntrOp::create(
1947 rewriter, loc, VectorType::get({32}, rewriter.getF32Type()),
1948 forceCastOperandsToSignature(
1949 rewriter, loc, {extOp},
1950 {VectorType::get({32}, rewriter.getBF16Type())}));
1951 };
1952
1953 auto res0 = extractUps2048(opSrcVal, index0Cst);
1954 auto res1 = extractUps2048(opSrcVal, index1Cst);
1955
1956 // Concat two 1024-bit vectors to a 2048-bit vector using
1957 // vector::ShuffleOp
1958 auto v32i32Res0 = forceCastValueToType(
1959 rewriter, loc, res0, VectorType::get({32}, rewriter.getI32Type()));
1960 auto v32i32Res1 = forceCastValueToType(
1961 rewriter, loc, res1, VectorType::get({32}, rewriter.getI32Type()));
1962
1963 SmallVector<int64_t> concatMask;
1964 for (int i = 0; i < 64; ++i) {
1965 concatMask.push_back(i);
1966 }
1967 upsIntrOp = vector::ShuffleOp::create(rewriter, loc, v32i32Res0,
1968 v32i32Res1, concatMask);
1969 }
1970 }
1971
1972 if (!upsIntrOp) {
1973 op.emitWarning() << "aievec.ups is not supported.\n";
1974 return failure();
1975 }
1976
1977 // create bitcast for result if needed
1978 if (flatResTy != upsIntrOp.getType())
1979 upsIntrOp = LLVM::BitcastOp::create(rewriter, loc, flatResTy, upsIntrOp);
1980
1981 if (flatResTy != resultType)
1982 upsIntrOp =
1983 vector::ShapeCastOp::create(rewriter, loc, resultType, upsIntrOp);
1984
1985 rewriter.replaceOp(op, upsIntrOp);
1986
1987 return success();
1988 }
1989};
1990
1991class SRSOpAIE2Conversion : public mlir::ConvertOpToLLVMPattern<aievec::SRSOp> {
1992public:
1993 using ConvertOpToLLVMPattern<aievec::SRSOp>::ConvertOpToLLVMPattern;
1994
1995 LogicalResult
1996 matchAndRewrite(aievec::SRSOp op, OpAdaptor adaptor,
1997 ConversionPatternRewriter &rewriter) const override {
1998 Location loc = op.getLoc();
1999
2000 Value result = op.getResult();
2001 VectorType resultType = cast<VectorType>(result.getType());
2002 Type resultScaTy = resultType.getElementType();
2003 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
2004 int resultLanes = getVectorLaneSize(resultType);
2005 int resultVectorSize = resultBitWidth * resultLanes;
2006
2007 // Integer types
2008 Value srsIntrOp = nullptr;
2009 if (llvm::isa<IntegerType>(resultScaTy)) {
2010 // create constant for sign from the op's sign attribute
2011 auto signCst =
2012 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2013 rewriter.getI32IntegerAttr(op.getSign()));
2014
2015 // create xllvm intrinsic
2016 SmallVector<Value> operands(
2017 {adaptor.getSource(), adaptor.getShift(), signCst});
2018 if (resultVectorSize == 512) {
2019 if (resultBitWidth == 16) {
2020 srsIntrOp = xllvm::I512V32Acc32SrsAIE2IntrOp::create(
2021 rewriter, loc, VectorType::get({32}, rewriter.getI16Type()),
2022 forceCastOperandsToSignature(
2023 rewriter, loc, operands,
2024 {VectorType::get({16}, rewriter.getI64Type()),
2025 rewriter.getI32Type(), rewriter.getI32Type()}));
2026 } else if (resultBitWidth == 32) {
2027 srsIntrOp = xllvm::I512V16Acc64SrsAIE2IntrOp::create(
2028 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
2029 forceCastOperandsToSignature(
2030 rewriter, loc, operands,
2031 {VectorType::get({16}, rewriter.getI64Type()),
2032 rewriter.getI32Type(), rewriter.getI32Type()}));
2033 }
2034 } else if (resultVectorSize == 256) {
2035 Value src = adaptor.getSource();
2036 VectorType srcType = cast<VectorType>(src.getType());
2037 Type srcScaType = srcType.getElementType();
2038 unsigned srcBitWidth = srcScaType.getIntOrFloatBitWidth();
2039
2040 if (resultBitWidth == 16 && srcBitWidth == 32) {
2041 srsIntrOp = xllvm::I256V16Acc32SrsAIE2IntrOp::create(
2042 rewriter, loc, VectorType::get({16}, rewriter.getI16Type()),
2043 forceCastOperandsToSignature(
2044 rewriter, loc, operands,
2045 {VectorType::get({8}, rewriter.getI64Type()),
2046 rewriter.getI32Type(), rewriter.getI32Type()}));
2047 } else if (resultBitWidth == 8 && srcBitWidth == 32) {
2048 srsIntrOp = xllvm::I256V32Acc32SrsAIE2IntrOp::create(
2049 rewriter, loc, VectorType::get({32}, rewriter.getI8Type()),
2050 forceCastOperandsToSignature(
2051 rewriter, loc, operands,
2052 {VectorType::get({16}, rewriter.getI64Type()),
2053 rewriter.getI32Type(), rewriter.getI32Type()}));
2054 } else if (resultBitWidth == 16 && srcBitWidth == 64) {
2055 srsIntrOp = xllvm::I256V16Acc64SrsAIE2IntrOp::create(
2056 rewriter, loc, VectorType::get({16}, rewriter.getI16Type()),
2057 forceCastOperandsToSignature(
2058 rewriter, loc, operands,
2059 {VectorType::get({16}, rewriter.getI64Type()),
2060 rewriter.getI32Type(), rewriter.getI32Type()}));
2061 } else if (resultBitWidth == 32 && srcBitWidth == 64) {
2062 srsIntrOp = xllvm::I256V8Acc64SrsAIE2IntrOp::create(
2063 rewriter, loc, VectorType::get({8}, rewriter.getI32Type()),
2064 forceCastOperandsToSignature(
2065 rewriter, loc, operands,
2066 {VectorType::get({8}, rewriter.getI64Type()),
2067 rewriter.getI32Type(), rewriter.getI32Type()}));
2068 }
2069 }
2070 } else {
2071 // Float types
2072 if (resultVectorSize == 256) {
2073 srsIntrOp = xllvm::Vector16AccFloatToV16BF16AIE2IntrOp::create(
2074 rewriter, loc, VectorType::get({16}, rewriter.getBF16Type()),
2075 forceCastOperandsToSignature(
2076 rewriter, loc, {adaptor.getSource()},
2077 {VectorType::get({8}, rewriter.getI64Type())}));
2078 } else if (resultVectorSize == 512) {
2079 // v32accfloat -> v32bfloat16
2080 // The CPP example of the implementation is below:
2081 // v32bfloat16 to_v32bfloat16(v32accfloat acc) {
2082 // v16bfloat16 x0 = to_v16bfloat16(extract_v16accfloat(acc, 0));
2083 // v16bfloat16 x1 = to_v16bfloat16(extract_v16accfloat(acc, 1));
2084 // return concat(x0, x1);
2085 // }
2086 auto indexZeroCst =
2087 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2088 rewriter.getI32IntegerAttr(0));
2089 auto indexOneCst =
2090 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2091 rewriter.getI32IntegerAttr(1));
2092 auto extractSrs = [&](Value source, Value index) -> Value {
2093 auto extOp = xllvm::ExtI512I1024IntrOp::create(
2094 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
2095 forceCastOperandsToSignature(
2096 rewriter, loc, {source, index},
2097 {VectorType::get({32}, rewriter.getI32Type()),
2098 rewriter.getI32Type()}));
2099 return xllvm::Vector16AccFloatToV16BF16AIE2IntrOp::create(
2100 rewriter, loc, VectorType::get({16}, rewriter.getBF16Type()),
2101 forceCastOperandsToSignature(
2102 rewriter, loc, {extOp},
2103 {VectorType::get({8}, rewriter.getI64Type())}));
2104 };
2105 auto resLo = extractSrs(adaptor.getSource(), indexZeroCst);
2106 auto resHi = extractSrs(adaptor.getSource(), indexOneCst);
2107 // Concat the two 256-bit vector to a 512-bit vector.
2108 // Note that given sources a0 and a1, the result is [a1; a0].
2109 srsIntrOp = xllvm::ConcatI512I256IntrOp::create(
2110 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
2111 forceCastOperandsToSignature(
2112 rewriter, loc, {resLo, resHi},
2113 {VectorType::get({8}, rewriter.getI32Type()),
2114 VectorType::get({8}, rewriter.getI32Type())}));
2115 }
2116 }
2117
2118 if (!srsIntrOp) {
2119 op.emitWarning() << "aievec.srs is not supported.\n";
2120 return failure();
2121 }
2122
2123 // create bitcast/shape_cast for result if needed
2124 auto resultVal = forceCastValueToType(rewriter, loc, srsIntrOp,
2125 op.getResult().getType());
2126 rewriter.replaceOp(op, resultVal);
2127
2128 return success();
2129 }
2130};
2131
2132// TODO: Split the op at AIEVec dialect level
2134 : public mlir::ConvertOpToLLVMPattern<aievec::SRSOp> {
2135public:
2136 using ConvertOpToLLVMPattern<aievec::SRSOp>::ConvertOpToLLVMPattern;
2137
2138 LogicalResult
2139 matchAndRewrite(aievec::SRSOp op, OpAdaptor adaptor,
2140 ConversionPatternRewriter &rewriter) const override {
2141 Location loc = op.getLoc();
2142
2143 Value result = op.getResult();
2144 VectorType resultType = cast<VectorType>(result.getType());
2145 Type resultScaTy = resultType.getElementType();
2146 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
2147 int resultLanes = getVectorLaneSize(resultType);
2148 int resultVectorSize = resultBitWidth * resultLanes;
2149
2150 // Integer types
2151 Value srsIntrOp = nullptr;
2152 if (llvm::isa<IntegerType>(resultScaTy)) {
2153 // create constant for sign from the op's sign attribute
2154 auto signCst =
2155 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2156 rewriter.getI32IntegerAttr(op.getSign()));
2157
2158 // create xllvm intrinsic
2159 SmallVector<Value> operands(
2160 {adaptor.getSource(), adaptor.getShift(), signCst});
2161 if (resultVectorSize == 512) {
2162 Value src = adaptor.getSource();
2163 VectorType srcType = cast<VectorType>(src.getType());
2164 Type srcScaType = srcType.getElementType();
2165 unsigned srcBitWidth = srcScaType.getIntOrFloatBitWidth();
2166
2167 if (resultBitWidth == 16 && srcBitWidth == 32) {
2168 // v32acc32 -> v32int16
2169 srsIntrOp = xllvm::I512V32Acc32SrsAIE2pIntrOp::create(
2170 rewriter, loc, VectorType::get({32}, rewriter.getI16Type()),
2171 forceCastOperandsToSignature(
2172 rewriter, loc, operands,
2173 {VectorType::get({32}, rewriter.getI32Type()),
2174 rewriter.getI32Type(), rewriter.getI32Type()}));
2175 } else if (resultBitWidth == 16 && srcBitWidth == 64) {
2176 // v32acc64 -> v32int16
2177 srsIntrOp = xllvm::I512V32Acc64SrsAIE2pIntrOp::create(
2178 rewriter, loc, VectorType::get({32}, rewriter.getI16Type()),
2179 forceCastOperandsToSignature(
2180 rewriter, loc, operands,
2181 {VectorType::get({32}, rewriter.getI64Type()),
2182 rewriter.getI32Type(), rewriter.getI32Type()}));
2183 } else if (resultBitWidth == 32 && srcBitWidth == 64) {
2184 // v16acc64 -> v16int32
2185 srsIntrOp = xllvm::I512V16Acc64SrsAIE2pIntrOp::create(
2186 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
2187 forceCastOperandsToSignature(
2188 rewriter, loc, operands,
2189 {VectorType::get({16}, rewriter.getI64Type()),
2190 rewriter.getI32Type(), rewriter.getI32Type()}));
2191 } else if (resultBitWidth == 8 && srcBitWidth == 32) {
2192 // v64acc32 -> v64int8
2193 srsIntrOp = xllvm::I512V64Acc32SrsAIE2pIntrOp::create(
2194 rewriter, loc, VectorType::get({64}, rewriter.getI8Type()),
2195 forceCastOperandsToSignature(
2196 rewriter, loc, operands,
2197 {VectorType::get({64}, rewriter.getI32Type()),
2198 rewriter.getI32Type(), rewriter.getI32Type()}));
2199 }
2200 } else if (resultVectorSize == 256) {
2201 Value src = adaptor.getSource();
2202 VectorType srcType = cast<VectorType>(src.getType());
2203 Type srcScaType = srcType.getElementType();
2204 unsigned srcBitWidth = srcScaType.getIntOrFloatBitWidth();
2205
2206 if (resultBitWidth == 16 && srcBitWidth == 32) {
2207 // v16acc32 -> v16int16
2208 srsIntrOp = xllvm::I256V16Acc32SrsAIE2pIntrOp::create(
2209 rewriter, loc, VectorType::get({16}, rewriter.getI16Type()),
2210 forceCastOperandsToSignature(
2211 rewriter, loc, operands,
2212 {VectorType::get({16}, rewriter.getI32Type()),
2213 rewriter.getI32Type(), rewriter.getI32Type()}));
2214 } else if (resultBitWidth == 8 && srcBitWidth == 32) {
2215 // v32acc32 -> v32int8
2216 srsIntrOp = xllvm::I256V32Acc32SrsAIE2pIntrOp::create(
2217 rewriter, loc, VectorType::get({32}, rewriter.getI8Type()),
2218 forceCastOperandsToSignature(
2219 rewriter, loc, operands,
2220 {VectorType::get({32}, rewriter.getI32Type()),
2221 rewriter.getI32Type(), rewriter.getI32Type()}));
2222 } else if (resultBitWidth == 16 && srcBitWidth == 64) {
2223 // v16acc64 -> v16int16
2224 srsIntrOp = xllvm::I256V16Acc64SrsAIE2pIntrOp::create(
2225 rewriter, loc, VectorType::get({16}, rewriter.getI16Type()),
2226 forceCastOperandsToSignature(
2227 rewriter, loc, operands,
2228 {VectorType::get({16}, rewriter.getI64Type()),
2229 rewriter.getI32Type(), rewriter.getI32Type()}));
2230 } else if (resultBitWidth == 32 && srcBitWidth == 64) {
2231 // v8acc64 -> v8int32
2232 srsIntrOp = xllvm::I256V8Acc64SrsAIE2pIntrOp::create(
2233 rewriter, loc, VectorType::get({8}, rewriter.getI32Type()),
2234 forceCastOperandsToSignature(
2235 rewriter, loc, operands,
2236 {VectorType::get({8}, rewriter.getI64Type()),
2237 rewriter.getI32Type(), rewriter.getI32Type()}));
2238 }
2239 } else if (resultVectorSize == 1024) {
2240 Value src = adaptor.getSource();
2241 VectorType srcType = cast<VectorType>(src.getType());
2242 Type srcScaType = srcType.getElementType();
2243 unsigned srcBitWidth = srcScaType.getIntOrFloatBitWidth();
2244
2245 if (resultBitWidth == 16 && srcBitWidth == 32) {
2246 // v64acc32 -> v64int16
2247 // Extract 2 chunks of v32acc32 and convert each to v32int16
2248 auto index0Cst =
2249 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2250 rewriter.getI32IntegerAttr(0));
2251 auto index1Cst =
2252 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2253 rewriter.getI32IntegerAttr(1));
2254
2255 auto extractSrs1024 = [&](Value source, Value index, Value shiftCst,
2256 Value signCst) -> Value {
2257 // Use vector::ShuffleOp to extract 1024-bit from 2048-bit
2258 // Cast source to v64xi32 for shuffling
2259 auto v64i32Source = forceCastValueToType(
2260 rewriter, loc, source,
2261 VectorType::get({64}, rewriter.getI32Type()));
2262
2263 // Determine shuffle mask based on index
2264 // index 0: elements [0-31]
2265 // index 1: elements [32-63]
2266 SmallVector<int64_t> shuffleMask;
2267 if (auto constIndex = index.getDefiningOp<LLVM::ConstantOp>()) {
2268 auto indexAttr = cast<IntegerAttr>(constIndex.getValue());
2269 int64_t idxVal = indexAttr.getInt();
2270 int startIdx = idxVal * 32;
2271 for (int i = 0; i < 32; ++i) {
2272 shuffleMask.push_back(startIdx + i);
2273 }
2274 } else {
2275 // Default to index 0 if not constant
2276 for (int i = 0; i < 32; ++i) {
2277 shuffleMask.push_back(i);
2278 }
2279 }
2280
2281 auto extOp = vector::ShuffleOp::create(rewriter, loc, v64i32Source,
2282 v64i32Source, shuffleMask);
2283
2284 return xllvm::I512V32Acc32SrsAIE2pIntrOp::create(
2285 rewriter, loc, VectorType::get({32}, rewriter.getI16Type()),
2286 forceCastOperandsToSignature(
2287 rewriter, loc, {extOp, shiftCst, signCst},
2288 {VectorType::get({32}, rewriter.getI32Type()),
2289 rewriter.getI32Type(), rewriter.getI32Type()}));
2290 };
2291
2292 auto res0 =
2293 extractSrs1024(src, index0Cst, adaptor.getShift(), signCst);
2294 auto res1 =
2295 extractSrs1024(src, index1Cst, adaptor.getShift(), signCst);
2296
2297 // Concat two 512-bit vectors to a 1024-bit vector using
2298 // vector::ShuffleOp
2299 auto v16i32Res0 = forceCastValueToType(
2300 rewriter, loc, res0,
2301 VectorType::get({16}, rewriter.getI32Type()));
2302 auto v16i32Res1 = forceCastValueToType(
2303 rewriter, loc, res1,
2304 VectorType::get({16}, rewriter.getI32Type()));
2305
2306 SmallVector<int64_t> concatMask;
2307 for (int i = 0; i < 32; ++i) {
2308 concatMask.push_back(i);
2309 }
2310 srsIntrOp = vector::ShuffleOp::create(rewriter, loc, v16i32Res0,
2311 v16i32Res1, concatMask);
2312 }
2313 }
2314 } else {
2315 // Float types
2316 // AIE2p uses native F32 types, AIE2 uses packed I64 types
2317 if (resultVectorSize == 256) {
2318 // v16accfloat -> v16bfloat16
2319 srsIntrOp = xllvm::Vector16AccFloatToV16BF16AIE2pIntrOp::create(
2320 rewriter, loc, VectorType::get({16}, rewriter.getBF16Type()),
2321 forceCastOperandsToSignature(
2322 rewriter, loc, {adaptor.getSource()},
2323 {VectorType::get({16}, rewriter.getF32Type())}));
2324 } else if (resultVectorSize == 512) {
2325 // v32accfloat -> v32bfloat16
2326 srsIntrOp = xllvm::Vector32AccFloatToV32BF16AIE2pIntrOp::create(
2327 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()),
2328 forceCastOperandsToSignature(
2329 rewriter, loc, {adaptor.getSource()},
2330 {VectorType::get({32}, rewriter.getF32Type())}));
2331 } else if (resultVectorSize == 1024) {
2332 // v64accfloat -> v64bfloat16
2333 // Extract 2 chunks of v32accfloat and convert each to v32bfloat16
2334 auto index0Cst =
2335 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2336 rewriter.getI32IntegerAttr(0));
2337 auto index1Cst =
2338 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2339 rewriter.getI32IntegerAttr(1));
2340
2341 auto extractSrs1024 = [&](Value source, Value index) -> Value {
2342 // Use vector::ShuffleOp to extract 1024-bit from 2048-bit
2343 // Cast source to v64xi32 for shuffling
2344 auto v64i32Source = forceCastValueToType(
2345 rewriter, loc, source,
2346 VectorType::get({64}, rewriter.getI32Type()));
2347
2348 // Determine shuffle mask based on index
2349 // index 0: elements [0-31]
2350 // index 1: elements [32-63]
2351 SmallVector<int64_t> shuffleMask;
2352 if (auto constIndex = index.getDefiningOp<LLVM::ConstantOp>()) {
2353 auto indexAttr = cast<IntegerAttr>(constIndex.getValue());
2354 int64_t idxVal = indexAttr.getInt();
2355 int startIdx = idxVal * 32;
2356 for (int i = 0; i < 32; ++i) {
2357 shuffleMask.push_back(startIdx + i);
2358 }
2359 } else {
2360 // Default to index 0 if not constant
2361 for (int i = 0; i < 32; ++i) {
2362 shuffleMask.push_back(i);
2363 }
2364 }
2365
2366 auto extOp = vector::ShuffleOp::create(rewriter, loc, v64i32Source,
2367 v64i32Source, shuffleMask);
2368
2369 return xllvm::Vector32AccFloatToV32BF16AIE2pIntrOp::create(
2370 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()),
2371 forceCastOperandsToSignature(
2372 rewriter, loc, {extOp},
2373 {VectorType::get({32}, rewriter.getF32Type())}));
2374 };
2375
2376 auto res0 = extractSrs1024(adaptor.getSource(), index0Cst);
2377 auto res1 = extractSrs1024(adaptor.getSource(), index1Cst);
2378
2379 // Concat two 512-bit vectors to a 1024-bit vector using
2380 // vector::ShuffleOp
2381 auto v16i32Res0 = forceCastValueToType(
2382 rewriter, loc, res0, VectorType::get({16}, rewriter.getI32Type()));
2383 auto v16i32Res1 = forceCastValueToType(
2384 rewriter, loc, res1, VectorType::get({16}, rewriter.getI32Type()));
2385
2386 SmallVector<int64_t> concatMask;
2387 for (int i = 0; i < 32; ++i) {
2388 concatMask.push_back(i);
2389 }
2390 srsIntrOp = vector::ShuffleOp::create(rewriter, loc, v16i32Res0,
2391 v16i32Res1, concatMask);
2392 }
2393 }
2394
2395 if (!srsIntrOp) {
2396 op.emitWarning() << "aievec.srs is not supported.\n";
2397 return failure();
2398 }
2399
2400 // create bitcast/shape_cast for result if needed
2401 auto resultVal = forceCastValueToType(rewriter, loc, srsIntrOp,
2402 op.getResult().getType());
2403 rewriter.replaceOp(op, resultVal);
2404
2405 return success();
2406 }
2407};
2408
2409class UPDOpConversion : public mlir::ConvertOpToLLVMPattern<aievec::UPDOp> {
2410public:
2411 using ConvertOpToLLVMPattern<aievec::UPDOp>::ConvertOpToLLVMPattern;
2412
2413 static std::string getIntrinsicName(aievec::UPDOp op, int loadSize) {
2414 auto resultType = cast<VectorType>(op.getResult().getType());
2415 std::stringstream ss;
2416 ss << "llvm.aie.upd.";
2417 ss << (loadSize == 128 ? 'v' : loadSize == 256 ? 'w' : 'x') << ".";
2418 ss << getVectorTypeString(resultType) << ".";
2419 // The index affects which intrinsic to call
2420 ss << (op.getIndex() == 0 ? "lo" : "hi");
2421 return ss.str();
2422 }
2423
2424 LogicalResult
2425 matchAndRewrite(aievec::UPDOp op, OpAdaptor adaptor,
2426 ConversionPatternRewriter &rewriter) const override {
2427 auto module = op->getParentOfType<ModuleOp>();
2428 MLIRContext *context = rewriter.getContext();
2429
2430 // A bit more complicated: load the vector, then update result vector
2431 // AIE1 is capable of 128-bit on one bank and 256-bit loads on even-odd
2432 // banks Identify size of update
2433 int vecSizeInBits =
2434 getVectorSizeInBits(cast<VectorType>(op.getResult().getType()));
2435
2436 auto ptr = this->getStridedElementPtr(
2437 rewriter, op->getLoc(), cast<MemRefType>(op.getSource().getType()),
2438 adaptor.getSource(), adaptor.getIndices());
2439
2440 // TODO: handle the offset field
2441
2442 if (vecSizeInBits <= 256) {
2443 // Total <=256-bit updates are much simpler:
2444 // we can do a direct load into the vector register
2445 // look at the indices to calculate the address
2446 auto vectorPtrType = LLVM::LLVMPointerType::get(
2447 getContext(),
2448 cast<MemRefType>(op.getSource().getType()).getMemorySpaceAsInt());
2449 auto castedPtr =
2450 LLVM::BitcastOp::create(rewriter, op->getLoc(), vectorPtrType, ptr);
2451 auto vecType = cast<VectorType>(op.getResult().getType());
2452 rewriter.replaceOpWithNewOp<LLVM::LoadOp>(op, vecType, castedPtr, 1);
2453 } else {
2454 // Total >256-bit updates will require upd ops to fill the whole vector
2455 // each UDP op represents one of these 256-bit loads and updates
2456
2457 // Determine the load size
2458 // TODO: no examples of 1024-bit output vectors: doesn't feel right
2459 // to attempt a 512-bit load to do an update like this
2460 int loadSize = vecSizeInBits == 256 ? 128
2461 : vecSizeInBits == 512 ? 256
2462 : 512;
2463
2464 // Create a vectorType for the load proper
2465 // Load half of the final result vector
2466 auto resultType = cast<VectorType>(op.getResult().getType());
2467 int lanes = getVectorLaneSize(resultType);
2468 auto loadType =
2469 VectorType::get({(int64_t)lanes / 2}, resultType.getElementType());
2470
2471 // Load the vector
2472 auto vectorPtrType = LLVM::LLVMPointerType::get(
2473 getContext(),
2474 cast<MemRefType>(op.getSource().getType()).getMemorySpaceAsInt());
2475 auto castedPtr =
2476 LLVM::BitcastOp::create(rewriter, op->getLoc(), vectorPtrType, ptr);
2477 auto loadValue =
2478 LLVM::LoadOp::create(rewriter, op->getLoc(), loadType, castedPtr, 1);
2479
2480 // Get set up for the intrinsic
2481 std::string intrinsicName = getIntrinsicName(op, loadSize);
2482
2483 // If the intrinsic declaration doesn't exist, create it
2484 auto func = module.lookupSymbol<LLVM::LLVMFuncOp>(
2485 StringAttr::get(context, intrinsicName));
2486
2487 if (!func) {
2488 OpBuilder::InsertionGuard guard(rewriter);
2489 rewriter.setInsertionPointToStart(module.getBody());
2490 func = LLVM::LLVMFuncOp::create(
2491 rewriter, rewriter.getUnknownLoc(), intrinsicName,
2492 LLVM::LLVMFunctionType::get(resultType, {resultType, loadType}));
2493 }
2494
2495 // Determine what the destination is
2496 Value destValue;
2497 if (adaptor.getVector()) {
2498 // This UPD is using an existing destination vector
2499 destValue = adaptor.getVector();
2500 } else {
2501 // If this UPD is not working off of an existing destination vector,
2502 // create an undefined vector as the destination
2503
2504 // TODO: determine if the undef intrinsic is needed or if an LLVM
2505 // undef suffices destValue =
2506 // LLVM::UndefOp::create(rewriter, op->getLoc(), resultType);
2507
2508 std::stringstream ss;
2509 ss << "llvm.aie." << getVectorTypeString(resultType) << ".undef";
2510 std::string intrinsicName = ss.str();
2511
2512 auto func = module.lookupSymbol<LLVM::LLVMFuncOp>(
2513 StringAttr::get(rewriter.getContext(), intrinsicName));
2514
2515 if (!func) {
2516 OpBuilder::InsertionGuard guard(rewriter);
2517 rewriter.setInsertionPointToStart(module.getBody());
2518 func = LLVM::LLVMFuncOp::create(
2519 rewriter, rewriter.getUnknownLoc(), intrinsicName,
2520 LLVM::LLVMFunctionType::get(resultType, {}));
2521 }
2522 destValue =
2523 LLVM::CallOp::create(rewriter, op->getLoc(), func, ValueRange{})
2524 ->getOpResult(0);
2525 }
2526
2527 // Create our call
2528 rewriter.replaceOpWithNewOp<LLVM::CallOp>(
2529 op, func, ValueRange{destValue, loadValue});
2530 }
2531
2532 return success();
2533 }
2534};
2535
2537 : public mlir::ConvertOpToLLVMPattern<aievec::ConcatOp> {
2538public:
2539 using ConvertOpToLLVMPattern<aievec::ConcatOp>::ConvertOpToLLVMPattern;
2540
2541 LogicalResult
2542 matchAndRewrite(aievec::ConcatOp op, OpAdaptor adaptor,
2543 ConversionPatternRewriter &rewriter) const override {
2544 Location loc = op.getLoc();
2545
2546 SmallVector<Value> sources = adaptor.getSources();
2547 Value src = sources.front();
2548 VectorType srcType = cast<VectorType>(src.getType());
2549 Type srcScalarType = srcType.getElementType();
2550 unsigned srcBitWidth = srcScalarType.getIntOrFloatBitWidth();
2551 int srcLanes = getVectorLaneSize(srcType);
2552 int srcVectorSize = srcBitWidth * srcLanes;
2553
2554 Value result = op.getResult();
2555 VectorType resultType = cast<VectorType>(result.getType());
2556 Type resultScaTy = resultType.getElementType();
2557 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
2558 int resultLanes = getVectorLaneSize(resultType);
2559 int resultVectorSize = resultBitWidth * resultLanes;
2560
2561 if (sources.size() != 2 && sources.size() != 4) {
2562 op.emitWarning() << "aievec.concat with " << sources.size()
2563 << " operands is not supported.\n";
2564 return failure();
2565 }
2566
2567 // create xllvm intrinsic
2568 Value concatOp = nullptr;
2569 if (srcVectorSize == 256 && resultVectorSize == 512) {
2570 concatOp = xllvm::ConcatI512I256IntrOp::create(
2571 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
2572 forceCastOperandsToSignature(
2573 rewriter, loc, adaptor.getSources(),
2574 {VectorType::get({8}, rewriter.getI32Type()),
2575 VectorType::get({8}, rewriter.getI32Type())}));
2576 } else if (srcVectorSize == 256 && resultVectorSize == 1024) {
2577 concatOp = xllvm::ConcatI1024I256IntrOp::create(
2578 rewriter, loc, VectorType::get({32}, rewriter.getI32Type()),
2579 forceCastOperandsToSignature(
2580 rewriter, loc, adaptor.getSources(),
2581 {VectorType::get({8}, rewriter.getI32Type()),
2582 VectorType::get({8}, rewriter.getI32Type()),
2583 VectorType::get({8}, rewriter.getI32Type()),
2584 VectorType::get({8}, rewriter.getI32Type())}));
2585 } else if (srcVectorSize == 512 && resultVectorSize == 1024) {
2586 concatOp = xllvm::ConcatI1024I512IntrOp::create(
2587 rewriter, loc, VectorType::get({32}, rewriter.getI32Type()),
2588 forceCastOperandsToSignature(
2589 rewriter, loc, adaptor.getSources(),
2590 {VectorType::get({16}, rewriter.getI32Type()),
2591 VectorType::get({16}, rewriter.getI32Type())}));
2592 } else {
2593 op.emitWarning() << "aievec.concat with " << srcVectorSize
2594 << "-bit operands, and " << resultVectorSize
2595 << "-bit result is not supported.\n";
2596 return failure();
2597 }
2598
2599 // create bitcast/shape_cast for result
2600 auto resultVal =
2601 forceCastValueToType(rewriter, loc, concatOp, op.getResult().getType());
2602 rewriter.replaceOp(op, resultVal);
2603
2604 return success();
2605 }
2606};
2607
2608class ExtOpConversion : public mlir::ConvertOpToLLVMPattern<aievec::ExtOp> {
2609public:
2610 using ConvertOpToLLVMPattern<aievec::ExtOp>::ConvertOpToLLVMPattern;
2611
2612 LogicalResult
2613 matchAndRewrite(aievec::ExtOp op, OpAdaptor adaptor,
2614 ConversionPatternRewriter &rewriter) const override {
2615 Location loc = op.getLoc();
2616
2617 Value src = adaptor.getSource();
2618 VectorType srcType = cast<VectorType>(src.getType());
2619 Type srcScalarType = srcType.getElementType();
2620 unsigned srcBitWidth = srcScalarType.getIntOrFloatBitWidth();
2621 int srcLanes = getVectorLaneSize(srcType);
2622 int srcVectorSize = srcBitWidth * srcLanes;
2623
2624 Value result = op.getResult();
2625 VectorType resultType = cast<VectorType>(result.getType());
2626 Type resultScaTy = resultType.getElementType();
2627 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
2628 int resultLanes = getVectorLaneSize(resultType);
2629 int resultVectorSize = resultBitWidth * resultLanes;
2630
2631 // create constant for index
2632 auto indexCst =
2633 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2634 rewriter.getI32IntegerAttr(op.getIndex()));
2635
2636 // create xllvm intrinsic
2637 SmallVector<Value> operands({adaptor.getSource(), indexCst});
2638 Value extOp = nullptr;
2639 // Integer types
2640 if (resultVectorSize == 256 && srcVectorSize == 512) {
2641 extOp = xllvm::ExtI256I512IntrOp::create(
2642 rewriter, loc, VectorType::get({8}, rewriter.getI32Type()),
2643 forceCastOperandsToSignature(
2644 rewriter, loc, operands,
2645 {VectorType::get({16}, rewriter.getI32Type()),
2646 rewriter.getI32Type()}));
2647 } else if (resultVectorSize == 512 && srcVectorSize == 1024) {
2648 extOp = xllvm::ExtI512I1024IntrOp::create(
2649 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
2650 forceCastOperandsToSignature(
2651 rewriter, loc, operands,
2652 {VectorType::get({32}, rewriter.getI32Type()),
2653 rewriter.getI32Type()}));
2654 } else if (resultVectorSize == 256 && srcVectorSize == 1024) {
2655 extOp = xllvm::ExtI256I1024IntrOp::create(
2656 rewriter, loc, VectorType::get({8}, rewriter.getI32Type()),
2657 forceCastOperandsToSignature(
2658 rewriter, loc, operands,
2659 {VectorType::get({32}, rewriter.getI32Type()),
2660 rewriter.getI32Type()}));
2661 } else if (resultVectorSize == 128 && srcVectorSize == 512) {
2662 auto shiftOp = adaptor.getSource();
2663 if (op.getIndex() > 0) {
2664 auto undefOp = xllvm::UndefV16I32IntrOp::create(
2665 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()));
2666 auto stepCst =
2667 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2668 rewriter.getI32IntegerAttr(0));
2669 auto shiftCst = LLVM::ConstantOp::create(
2670 rewriter, loc, rewriter.getI32Type(),
2671 rewriter.getI32IntegerAttr(op.getIndex() * 16));
2672 SmallVector<Value> shiftOperands{adaptor.getSource(), undefOp, stepCst,
2673 shiftCst};
2674 // Right shift the source vector in index * 16 bytes (i.e. in index *
2675 // 128 bits). The integer index is expected to be 0 to 3.
2676 shiftOp = xllvm::VectorShiftI512I512IntrOp::create(
2677 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
2678 forceCastOperandsToSignature(
2679 rewriter, loc, shiftOperands,
2680 {VectorType::get({16}, rewriter.getI32Type()),
2681 VectorType::get({16}, rewriter.getI32Type()),
2682 rewriter.getI32Type(), rewriter.getI32Type()}));
2683 }
2684 // The underlying intrinsic takes a source vector and extract the lowest
2685 // 128-bit. i.e. it always extracts the input vector with index = 0.
2686 extOp = xllvm::ExtI128I512IntrOp::create(
2687 rewriter, loc, VectorType::get({4}, rewriter.getI32Type()),
2688 forceCastOperandsToSignature(
2689 rewriter, loc, /*operands=*/{shiftOp},
2690 {VectorType::get({16}, rewriter.getI32Type())}));
2691 } else {
2692 op.emitWarning() << "aievec.ext with " << srcVectorSize
2693 << "-bit source, and " << resultVectorSize
2694 << "-bit result is not supported.\n";
2695 return failure();
2696 }
2697
2698 // create bitcast/shape_cast for result
2699 auto resultVal =
2700 forceCastValueToType(rewriter, loc, extOp, op.getResult().getType());
2701 rewriter.replaceOp(op, resultVal);
2702
2703 return success();
2704 }
2705};
2706
2707// AIE2p version of ExtOp conversion using vector.shuffle
2709 : public mlir::ConvertOpToLLVMPattern<aievec::ExtOp> {
2710public:
2711 using ConvertOpToLLVMPattern<aievec::ExtOp>::ConvertOpToLLVMPattern;
2712
2713 LogicalResult
2714 matchAndRewrite(aievec::ExtOp op, OpAdaptor adaptor,
2715 ConversionPatternRewriter &rewriter) const override {
2716 Location loc = op.getLoc();
2717
2718 Value src = adaptor.getSource();
2719 VectorType srcType = cast<VectorType>(src.getType());
2720 VectorType resultType = cast<VectorType>(op.getResult().getType());
2721
2722 int srcLanes = getVectorLaneSize(srcType);
2723 int resultLanes = getVectorLaneSize(resultType);
2724
2725 // The result is one of the equal parts the source divides into.
2726 if (srcLanes % resultLanes != 0) {
2727 op.emitWarning() << "aievec.ext of a part that does not divide the "
2728 "source is not supported for AIE2p.\n";
2729 return failure();
2730 }
2731
2732 // Part `index`: [index * resultLanes, (index + 1) * resultLanes)
2733 SmallVector<int64_t> shuffleMask;
2734 int startIdx = op.getIndex() * resultLanes;
2735 for (int i = 0; i < resultLanes; ++i) {
2736 shuffleMask.push_back(startIdx + i);
2737 }
2738
2739 auto extracted =
2740 vector::ShuffleOp::create(rewriter, loc, src, src, shuffleMask);
2741
2742 rewriter.replaceOp(op, extracted);
2743 return success();
2744 }
2745};
2746
2748 : public mlir::ConvertOpToLLVMPattern<aievec::aie1::SelectOp> {
2749public:
2750 using ConvertOpToLLVMPattern<aievec::aie1::SelectOp>::ConvertOpToLLVMPattern;
2751
2752 static std::string getIntrinsicName(aievec::aie1::SelectOp op) {
2753 auto xbuffType = cast<VectorType>(op.getXbuff().getType());
2754 std::stringstream ss;
2755 ss << "llvm.aie.prim." << getVectorTypeString(xbuffType) << ".select";
2756 return ss.str();
2757 }
2758
2759 LogicalResult
2760 matchAndRewrite(aievec::aie1::SelectOp op, OpAdaptor adaptor,
2761 ConversionPatternRewriter &rewriter) const override {
2762 auto module = op->getParentOfType<ModuleOp>();
2763 MLIRContext *context = rewriter.getContext();
2764
2765 auto selectType = IntegerType::get(context, 32);
2766 auto startType = IntegerType::get(context, 32);
2767 auto offsetsType = VectorType::get({2}, IntegerType::get(context, 32));
2768 auto confType = VectorType::get({2}, IntegerType::get(context, 32));
2769
2770 // If the intrinsic declaration doesn't exist, create it
2771 std::string intrinsicName = getIntrinsicName(op);
2772 auto func = module.lookupSymbol<LLVM::LLVMFuncOp>(
2773 StringAttr::get(context, intrinsicName));
2774
2775 if (!func) {
2776 OpBuilder::InsertionGuard guard(rewriter);
2777 rewriter.setInsertionPointToStart(module.getBody());
2778 func = LLVM::LLVMFuncOp::create(
2779 rewriter, rewriter.getUnknownLoc(), intrinsicName,
2780 LLVM::LLVMFunctionType::get(op.getResult().getType(),
2781 {op.getXbuff().getType(), selectType,
2782 startType, /* xstart */
2783 startType, /* ystart */
2784 offsetsType, /* xoffsets */
2785 offsetsType, /* yoffsets */
2786 confType}));
2787 }
2788
2789 // Parse the string attribute values
2790 uint32_t select = 0;
2791 BufferParams x = {};
2792 BufferParams y = {};
2793 BufferParams z = {};
2794
2795 op.getSelect().getAsInteger(0, select);
2796 op.getXstart().getAsInteger(0, x.start);
2797 op.getXoffsets().getAsInteger(0, x.offsets);
2798 op.getXoffsetsHi().getAsInteger(0, x.offsets_hi);
2799 op.getXsquare().getAsInteger(0, x.square);
2800 op.getYstart().getAsInteger(0, y.start);
2801 op.getYoffsets().getAsInteger(0, y.offsets);
2802 op.getYoffsetsHi().getAsInteger(0, y.offsets_hi);
2803 op.getYsquare().getAsInteger(0, y.square);
2804
2805 // Encode the configuration register
2806 uint32_t conf[2] = {0, 0};
2807 encodeConf(conf, x, z, false);
2808 conf[1] |= encodeSquare(y.square) << 21;
2809
2810 // Create the constants and replace the op
2811 auto selectVal = LLVM::ConstantOp::create(
2812 rewriter, op->getLoc(), selectType, rewriter.getI32IntegerAttr(select));
2813 auto xstartVal = LLVM::ConstantOp::create(
2814 rewriter, op->getLoc(), startType, rewriter.getI32IntegerAttr(x.start));
2815 auto ystartVal = LLVM::ConstantOp::create(
2816 rewriter, op->getLoc(), startType, rewriter.getI32IntegerAttr(y.start));
2817 auto xoffsetsVal = LLVM::ConstantOp::create(
2818 rewriter, op->getLoc(), offsetsType,
2819 rewriter.getI32VectorAttr({(int32_t)x.offsets, (int32_t)x.offsets_hi}));
2820 auto yoffsetsVal = LLVM::ConstantOp::create(
2821 rewriter, op->getLoc(), offsetsType,
2822 rewriter.getI32VectorAttr({(int32_t)y.offsets, (int32_t)y.offsets_hi}));
2823 auto confVal = LLVM::ConstantOp::create(
2824 rewriter, op->getLoc(), confType,
2825 rewriter.getI32VectorAttr({(int32_t)conf[0], (int32_t)conf[1]}));
2826 rewriter.replaceOpWithNewOp<LLVM::CallOp>(
2827 op, func,
2828 ValueRange{op.getXbuff(), selectVal, xstartVal, ystartVal, xoffsetsVal,
2829 yoffsetsVal, confVal});
2830 return success();
2831 }
2832};
2833
2834class PackOpConversion : public mlir::ConvertOpToLLVMPattern<aievec::PackOp> {
2835public:
2836 using ConvertOpToLLVMPattern<aievec::PackOp>::ConvertOpToLLVMPattern;
2837
2838 static std::string getIntrinsicName(aievec::PackOp op) {
2839 auto sourceType = cast<VectorType>(op.getSource().getType());
2840 std::stringstream ss;
2841 ss << "llvm.aie.pack." << getVectorTypeString(sourceType);
2842 return ss.str();
2843 }
2844
2845 LogicalResult
2846 matchAndRewrite(aievec::PackOp op, OpAdaptor adaptor,
2847 ConversionPatternRewriter &rewriter) const override {
2848 auto module = op->getParentOfType<ModuleOp>();
2849 MLIRContext *context = rewriter.getContext();
2850
2851 // If the intrinsic declaration doesn't exist, create it
2852 std::string intrinsicName = getIntrinsicName(op);
2853 auto func = module.lookupSymbol<LLVM::LLVMFuncOp>(
2854 StringAttr::get(context, intrinsicName));
2855
2856 if (!func) {
2857 OpBuilder::InsertionGuard guard(rewriter);
2858 rewriter.setInsertionPointToStart(module.getBody());
2859 func = LLVM::LLVMFuncOp::create(
2860 rewriter, rewriter.getUnknownLoc(), intrinsicName,
2861 LLVM::LLVMFunctionType::get(op.getResult().getType(),
2862 {op.getSource().getType()}));
2863 }
2864
2865 rewriter.replaceOpWithNewOp<LLVM::CallOp>(op, func,
2866 ValueRange{op.getSource()});
2867 return success();
2868 }
2869};
2870
2872 : public mlir::ConvertOpToLLVMPattern<aievec::UnpackOp> {
2873public:
2874 using ConvertOpToLLVMPattern<aievec::UnpackOp>::ConvertOpToLLVMPattern;
2875
2876 LogicalResult
2877 matchAndRewrite(aievec::UnpackOp op, OpAdaptor adaptor,
2878 ConversionPatternRewriter &rewriter) const override {
2879 op.emitWarning() << "aie.unpack conversion is not implemented\n";
2880 return failure();
2881 }
2882};
2883
2884// AIE2P unpack int4-packed: source is vector<N/2 x i8> carrying N packed
2885// nibbles, result is vector<N x i8> (one byte per nibble). Maps directly
2886// to llvm.aie2p.unpack.I512.I8.I4 (N=64) or .I1024.I8.I4 (N=128).
2887// AWQ packed weights are unsigned; the signed offset is folded into the
2888// per-group zero point so the intrinsic is invoked with sign=0.
2890 : public mlir::ConvertOpToLLVMPattern<aievec::UnpackOp> {
2891public:
2892 using ConvertOpToLLVMPattern<aievec::UnpackOp>::ConvertOpToLLVMPattern;
2893
2894 LogicalResult
2895 matchAndRewrite(aievec::UnpackOp op, OpAdaptor adaptor,
2896 ConversionPatternRewriter &rewriter) const override {
2897 Location loc = op.getLoc();
2898 auto srcTy = cast<VectorType>(op.getSource().getType());
2899 auto dstTy = cast<VectorType>(op.getResult().getType());
2900 if (!srcTy.getElementType().isInteger(8) ||
2901 !dstTy.getElementType().isInteger(8))
2902 return failure();
2903
2904 unsigned srcLanes = getVectorLaneSize(srcTy);
2905 unsigned dstLanes = getVectorLaneSize(dstTy);
2906 if (dstLanes != 2 * srcLanes)
2907 return failure();
2908 if (dstLanes != 64 && dstLanes != 128)
2909 return failure();
2910
2911 auto signCst = LLVM::ConstantOp::create(
2912 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(0));
2913
2914 Value unpacked;
2915 if (dstLanes == 64) {
2916 unpacked = xllvm::UnpackI512I8I4AIE2pIntrOp::create(
2917 rewriter, loc, VectorType::get({64}, rewriter.getI8Type()),
2918 adaptor.getSource(), signCst)
2919 ->getResult(0);
2920 } else {
2921 unpacked =
2922 xllvm::UnpackI1024I8I4AIE2pIntrOp::create(
2923 rewriter, loc, VectorType::get({128}, rewriter.getI8Type()),
2924 adaptor.getSource(), signCst)
2925 ->getResult(0);
2926 }
2927 rewriter.replaceOp(op, unpacked);
2928 return success();
2929 }
2930};
2931
2933 : public mlir::ConvertOpToLLVMPattern<aievec::BroadcastOp> {
2934public:
2935 using ConvertOpToLLVMPattern<aievec::BroadcastOp>::ConvertOpToLLVMPattern;
2936
2937 LogicalResult
2938 matchAndRewrite(aievec::BroadcastOp op, OpAdaptor adaptor,
2939 ConversionPatternRewriter &rewriter) const override {
2940 op.emitWarning() << "aie.broadcast conversion is not implemented\n";
2941 return failure();
2942 }
2943};
2944
2945// Helper to pad a vector from N lanes to dstLanes using vector.shuffle
2946// with poison (-1) for the upper portion.
2947static Value padVectorWithPoison(ConversionPatternRewriter &rewriter,
2948 Location loc, Value vec, int srcLanes,
2949 int dstLanes) {
2950 SmallVector<int64_t> padMask;
2951 for (int i = 0; i < srcLanes; ++i)
2952 padMask.push_back(i);
2953 for (int i = srcLanes; i < dstLanes; ++i)
2954 padMask.push_back(-1);
2955 return vector::ShuffleOp::create(rewriter, loc, vec, vec, padMask);
2956}
2957
2958// Helper to extract the first N lanes from a wider vector using
2959// vector.shuffle.
2960static Value extractLowerLanes(ConversionPatternRewriter &rewriter,
2961 Location loc, Value vec, int lanes) {
2962 SmallVector<int64_t> extractMask;
2963 for (int i = 0; i < lanes; ++i)
2964 extractMask.push_back(i);
2965 return vector::ShuffleOp::create(rewriter, loc, vec, vec, extractMask);
2966}
2967
2968class MaxOpConversion : public mlir::ConvertOpToLLVMPattern<aievec::MaxOp> {
2969public:
2970 using ConvertOpToLLVMPattern<aievec::MaxOp>::ConvertOpToLLVMPattern;
2971
2972 LogicalResult
2973 matchAndRewrite(aievec::MaxOp op, OpAdaptor adaptor,
2974 ConversionPatternRewriter &rewriter) const override {
2975 Location loc = op.getLoc();
2976
2977 VectorType resultType = cast<VectorType>(op.getResult().getType());
2978 Type resultScaTy = resultType.getElementType();
2979 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
2980 int resultLanes = getVectorLaneSize(resultType);
2981 int resultVectorSize = resultBitWidth * resultLanes;
2982
2983 // aievec.max op has the AllTypesMatch constraint on lhs/rhs/res
2984 if (resultVectorSize != 512 && resultVectorSize != 256) {
2985 op.emitWarning() << "aievec.max conversion with " << resultVectorSize
2986 << "-bit result is not supported.\n";
2987 return failure();
2988 }
2989
2990 // create xllvm intrinsic
2991 Value maxOp = nullptr;
2992 if (llvm::isa<IntegerType>(resultScaTy)) {
2993 // create constant for third operand `cmp`
2994 // Note: `cmp` is implicitly treated as `sign` to the vmax intrinsic
2995 auto cmpCst = LLVM::ConstantOp::create(
2996 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(1));
2997 SmallVector<Value> operands{adaptor.getLhs(), adaptor.getRhs(), cmpCst};
2998 if (resultBitWidth == 8) {
2999 maxOp = xllvm::VectorMaxLt8IntrOp::create(
3000 rewriter, loc,
3001 mlir::LLVM::LLVMStructType::getLiteral(
3002 rewriter.getContext(),
3003 {VectorType::get({64}, rewriter.getI8Type()),
3004 VectorType::get({2}, rewriter.getI32Type())}),
3005 forceCastOperandsToSignature(
3006 rewriter, loc, operands,
3007 {VectorType::get({64}, rewriter.getI8Type()),
3008 VectorType::get({64}, rewriter.getI8Type()),
3009 rewriter.getI32Type()}));
3010 } else if (resultBitWidth == 16) {
3011 maxOp = xllvm::VectorMaxLt16IntrOp::create(
3012 rewriter, loc,
3013 mlir::LLVM::LLVMStructType::getLiteral(
3014 rewriter.getContext(),
3015 {VectorType::get({32}, rewriter.getI16Type()),
3016 rewriter.getI32Type()}),
3017 forceCastOperandsToSignature(
3018 rewriter, loc, operands,
3019 {VectorType::get({32}, rewriter.getI16Type()),
3020 VectorType::get({32}, rewriter.getI16Type()),
3021 rewriter.getI32Type()}));
3022 } else if (resultBitWidth == 32) {
3023 maxOp = xllvm::VectorMaxLt32IntrOp::create(
3024 rewriter, loc,
3025 mlir::LLVM::LLVMStructType::getLiteral(
3026 rewriter.getContext(),
3027 {VectorType::get({16}, rewriter.getI32Type()),
3028 rewriter.getI32Type()}),
3029 forceCastOperandsToSignature(
3030 rewriter, loc, operands,
3031 {VectorType::get({16}, rewriter.getI32Type()),
3032 VectorType::get({16}, rewriter.getI32Type()),
3033 rewriter.getI32Type()}));
3034 }
3035 } else {
3036 if (resultBitWidth == 16) {
3037 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
3038 Value lhs = adaptor.getLhs(), rhs = adaptor.getRhs();
3039
3040 // Pad 16-lane to 32-lane if needed
3041 if (resultLanes == 16) {
3042 lhs = padVectorWithPoison(rewriter, loc, lhs, 16, 32);
3043 rhs = padVectorWithPoison(rewriter, loc, rhs, 16, 32);
3044 }
3045
3046 maxOp = xllvm::VectorMaxLtBf16IntrOp::create(
3047 rewriter, loc,
3048 mlir::LLVM::LLVMStructType::getLiteral(
3049 rewriter.getContext(), {v32bf16Ty, rewriter.getI32Type()}),
3050 forceCastOperandsToSignature(rewriter, loc, {lhs, rhs},
3051 {v32bf16Ty, v32bf16Ty}));
3052 }
3053 }
3054
3055 if (!maxOp) {
3056 op.emitWarning() << "aievec.max conversion fails due to unsupported "
3057 "element data type.\n";
3058 return failure();
3059 }
3060
3061 // Extract the vector result from the struct
3062 Value resultVec = LLVM::ExtractValueOp::create(rewriter, loc, maxOp,
3063 /*position=*/0);
3064 // Truncate back to 16 lanes if padded
3065 if (resultLanes == 16 && !llvm::isa<IntegerType>(resultScaTy))
3066 resultVec = extractLowerLanes(rewriter, loc, resultVec, 16);
3067
3068 rewriter.replaceOp(op, resultVec);
3069
3070 return success();
3071 }
3072};
3073
3074class MinOpConversion : public mlir::ConvertOpToLLVMPattern<aievec::MinOp> {
3075public:
3076 using ConvertOpToLLVMPattern<aievec::MinOp>::ConvertOpToLLVMPattern;
3077
3078 LogicalResult
3079 matchAndRewrite(aievec::MinOp op, OpAdaptor adaptor,
3080 ConversionPatternRewriter &rewriter) const override {
3081 Location loc = op.getLoc();
3082
3083 VectorType resultType = cast<VectorType>(op.getResult().getType());
3084 Type resultScaTy = resultType.getElementType();
3085 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
3086 int resultLanes = getVectorLaneSize(resultType);
3087 int resultVectorSize = resultBitWidth * resultLanes;
3088
3089 // aievec.min op has the AllTypesMatch constraint on lhs/rhs/res
3090 if (resultVectorSize != 512 && resultVectorSize != 256) {
3091 op.emitWarning() << "aievec.min conversion with " << resultVectorSize
3092 << "-bit result is not supported.\n";
3093 return failure();
3094 }
3095
3096 // create xllvm intrinsic
3097 Value minOp = nullptr;
3098 if (llvm::isa<IntegerType>(resultScaTy)) {
3099 // create constant for third operand `cmp`
3100 // Note: `cmp` is implicitly treated as `sign` to the vmin intrinsic
3101 auto cmpCst = LLVM::ConstantOp::create(
3102 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(1));
3103 SmallVector<Value> operands{adaptor.getLhs(), adaptor.getRhs(), cmpCst};
3104 if (resultBitWidth == 8) {
3105 minOp = xllvm::VectorMinGe8IntrOp::create(
3106 rewriter, loc,
3107 mlir::LLVM::LLVMStructType::getLiteral(
3108 rewriter.getContext(),
3109 {VectorType::get({64}, rewriter.getI8Type()),
3110 VectorType::get({2}, rewriter.getI32Type())}),
3111 forceCastOperandsToSignature(
3112 rewriter, loc, operands,
3113 {VectorType::get({64}, rewriter.getI8Type()),
3114 VectorType::get({64}, rewriter.getI8Type()),
3115 rewriter.getI32Type()}));
3116 } else if (resultBitWidth == 16) {
3117 minOp = xllvm::VectorMinGe16IntrOp::create(
3118 rewriter, loc,
3119 mlir::LLVM::LLVMStructType::getLiteral(
3120 rewriter.getContext(),
3121 {VectorType::get({32}, rewriter.getI16Type()),
3122 rewriter.getI32Type()}),
3123 forceCastOperandsToSignature(
3124 rewriter, loc, operands,
3125 {VectorType::get({32}, rewriter.getI16Type()),
3126 VectorType::get({32}, rewriter.getI16Type()),
3127 rewriter.getI32Type()}));
3128 } else if (resultBitWidth == 32) {
3129 minOp = xllvm::VectorMinGe32IntrOp::create(
3130 rewriter, loc,
3131 mlir::LLVM::LLVMStructType::getLiteral(
3132 rewriter.getContext(),
3133 {VectorType::get({16}, rewriter.getI32Type()),
3134 rewriter.getI32Type()}),
3135 forceCastOperandsToSignature(
3136 rewriter, loc, operands,
3137 {VectorType::get({16}, rewriter.getI32Type()),
3138 VectorType::get({16}, rewriter.getI32Type()),
3139 rewriter.getI32Type()}));
3140 }
3141 } else {
3142 if (resultBitWidth == 16) {
3143 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
3144 Value lhs = adaptor.getLhs(), rhs = adaptor.getRhs();
3145
3146 // Pad 16-lane to 32-lane if needed
3147 if (resultLanes == 16) {
3148 lhs = padVectorWithPoison(rewriter, loc, lhs, 16, 32);
3149 rhs = padVectorWithPoison(rewriter, loc, rhs, 16, 32);
3150 }
3151
3152 minOp = xllvm::VectorMinGeBf16IntrOp::create(
3153 rewriter, loc,
3154 mlir::LLVM::LLVMStructType::getLiteral(
3155 rewriter.getContext(), {v32bf16Ty, rewriter.getI32Type()}),
3156 forceCastOperandsToSignature(rewriter, loc, {lhs, rhs},
3157 {v32bf16Ty, v32bf16Ty}));
3158 }
3159 }
3160
3161 if (!minOp) {
3162 op.emitWarning() << "aievec.min conversion fails due to unsupported "
3163 "element data type.\n";
3164 return failure();
3165 }
3166
3167 // Extract the vector result from the struct
3168 Value resultVec = LLVM::ExtractValueOp::create(rewriter, loc, minOp,
3169 /*position=*/0);
3170 // Truncate back to 16 lanes if padded
3171 if (resultLanes == 16 && !llvm::isa<IntegerType>(resultScaTy))
3172 resultVec = extractLowerLanes(rewriter, loc, resultVec, 16);
3173
3174 rewriter.replaceOp(op, resultVec);
3175
3176 return success();
3177 }
3178};
3179
3180// AIE2p version of MaxOp conversion
3182 : public mlir::ConvertOpToLLVMPattern<aievec::MaxOp> {
3183public:
3184 using ConvertOpToLLVMPattern<aievec::MaxOp>::ConvertOpToLLVMPattern;
3185
3186 LogicalResult
3187 matchAndRewrite(aievec::MaxOp op, OpAdaptor adaptor,
3188 ConversionPatternRewriter &rewriter) const override {
3189 Location loc = op.getLoc();
3190
3191 VectorType resultType = cast<VectorType>(op.getResult().getType());
3192 Type resultScaTy = resultType.getElementType();
3193 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
3194 int resultLanes = getVectorLaneSize(resultType);
3195 int resultVectorSize = resultBitWidth * resultLanes;
3196
3197 // aievec.max op has the AllTypesMatch constraint on lhs/rhs/res
3198 if (resultVectorSize != 512 && resultVectorSize != 256) {
3199 op.emitWarning() << "aievec.max conversion with " << resultVectorSize
3200 << "-bit result is not supported.\n";
3201 return failure();
3202 }
3203
3204 // create xllvm intrinsic
3205 Value maxOp = nullptr;
3206 if (llvm::isa<IntegerType>(resultScaTy)) {
3207 // create constant for third operand `cmp`
3208 // Note: `cmp` is implicitly treated as `sign` to the vmax intrinsic
3209 auto cmpCst = LLVM::ConstantOp::create(
3210 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(1));
3211 SmallVector<Value> operands{adaptor.getLhs(), adaptor.getRhs(), cmpCst};
3212 if (resultBitWidth == 8) {
3213 maxOp = xllvm::VectorMaxLt8AIE2pIntrOp::create(
3214 rewriter, loc,
3215 mlir::LLVM::LLVMStructType::getLiteral(
3216 rewriter.getContext(),
3217 {VectorType::get({64}, rewriter.getI8Type()),
3218 VectorType::get({2}, rewriter.getI32Type())}),
3219 forceCastOperandsToSignature(
3220 rewriter, loc, operands,
3221 {VectorType::get({64}, rewriter.getI8Type()),
3222 VectorType::get({64}, rewriter.getI8Type()),
3223 rewriter.getI32Type()}));
3224 } else if (resultBitWidth == 16) {
3225 maxOp = xllvm::VectorMaxLt16AIE2pIntrOp::create(
3226 rewriter, loc,
3227 mlir::LLVM::LLVMStructType::getLiteral(
3228 rewriter.getContext(),
3229 {VectorType::get({32}, rewriter.getI16Type()),
3230 rewriter.getI32Type()}),
3231 forceCastOperandsToSignature(
3232 rewriter, loc, operands,
3233 {VectorType::get({32}, rewriter.getI16Type()),
3234 VectorType::get({32}, rewriter.getI16Type()),
3235 rewriter.getI32Type()}));
3236 } else if (resultBitWidth == 32) {
3237 maxOp = xllvm::VectorMaxLt32AIE2pIntrOp::create(
3238 rewriter, loc,
3239 mlir::LLVM::LLVMStructType::getLiteral(
3240 rewriter.getContext(),
3241 {VectorType::get({16}, rewriter.getI32Type()),
3242 rewriter.getI32Type()}),
3243 forceCastOperandsToSignature(
3244 rewriter, loc, operands,
3245 {VectorType::get({16}, rewriter.getI32Type()),
3246 VectorType::get({16}, rewriter.getI32Type()),
3247 rewriter.getI32Type()}));
3248 }
3249 } else {
3250 if (resultBitWidth == 16) {
3251 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
3252 Value lhs = adaptor.getLhs(), rhs = adaptor.getRhs();
3253
3254 if (resultLanes == 16) {
3255 lhs = padVectorWithPoison(rewriter, loc, lhs, 16, 32);
3256 rhs = padVectorWithPoison(rewriter, loc, rhs, 16, 32);
3257 }
3258
3259 maxOp = xllvm::VectorMaxLtBf16AIE2pIntrOp::create(
3260 rewriter, loc,
3261 mlir::LLVM::LLVMStructType::getLiteral(
3262 rewriter.getContext(), {v32bf16Ty, rewriter.getI32Type()}),
3263 forceCastOperandsToSignature(rewriter, loc, {lhs, rhs},
3264 {v32bf16Ty, v32bf16Ty}));
3265 }
3266 }
3267
3268 if (!maxOp) {
3269 op.emitWarning() << "aievec.max conversion fails due to unsupported "
3270 "element data type.\n";
3271 return failure();
3272 }
3273
3274 Value resultVec = LLVM::ExtractValueOp::create(rewriter, loc, maxOp,
3275 /*position=*/0);
3276 if (resultLanes == 16 && !llvm::isa<IntegerType>(resultScaTy))
3277 resultVec = extractLowerLanes(rewriter, loc, resultVec, 16);
3278
3279 rewriter.replaceOp(op, resultVec);
3280
3281 return success();
3282 }
3283};
3284
3285// AIE2p version of MinOp conversion
3287 : public mlir::ConvertOpToLLVMPattern<aievec::MinOp> {
3288public:
3289 using ConvertOpToLLVMPattern<aievec::MinOp>::ConvertOpToLLVMPattern;
3290
3291 LogicalResult
3292 matchAndRewrite(aievec::MinOp op, OpAdaptor adaptor,
3293 ConversionPatternRewriter &rewriter) const override {
3294 Location loc = op.getLoc();
3295
3296 VectorType resultType = cast<VectorType>(op.getResult().getType());
3297 Type resultScaTy = resultType.getElementType();
3298 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
3299 int resultLanes = getVectorLaneSize(resultType);
3300 int resultVectorSize = resultBitWidth * resultLanes;
3301
3302 // aievec.min op has the AllTypesMatch constraint on lhs/rhs/res
3303 if (resultVectorSize != 512 && resultVectorSize != 256) {
3304 op.emitWarning() << "aievec.min conversion with " << resultVectorSize
3305 << "-bit result is not supported.\n";
3306 return failure();
3307 }
3308
3309 // create xllvm intrinsic
3310 Value minOp = nullptr;
3311 if (llvm::isa<IntegerType>(resultScaTy)) {
3312 // create constant for third operand `cmp`
3313 // Note: `cmp` is implicitly treated as `sign` to the vmin intrinsic
3314 auto cmpCst = LLVM::ConstantOp::create(
3315 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(1));
3316 SmallVector<Value> operands{adaptor.getLhs(), adaptor.getRhs(), cmpCst};
3317 if (resultBitWidth == 8) {
3318 minOp = xllvm::VectorMinGe8AIE2pIntrOp::create(
3319 rewriter, loc,
3320 mlir::LLVM::LLVMStructType::getLiteral(
3321 rewriter.getContext(),
3322 {VectorType::get({64}, rewriter.getI8Type()),
3323 VectorType::get({2}, rewriter.getI32Type())}),
3324 forceCastOperandsToSignature(
3325 rewriter, loc, operands,
3326 {VectorType::get({64}, rewriter.getI8Type()),
3327 VectorType::get({64}, rewriter.getI8Type()),
3328 rewriter.getI32Type()}));
3329 } else if (resultBitWidth == 16) {
3330 minOp = xllvm::VectorMinGe16AIE2pIntrOp::create(
3331 rewriter, loc,
3332 mlir::LLVM::LLVMStructType::getLiteral(
3333 rewriter.getContext(),
3334 {VectorType::get({32}, rewriter.getI16Type()),
3335 rewriter.getI32Type()}),
3336 forceCastOperandsToSignature(
3337 rewriter, loc, operands,
3338 {VectorType::get({32}, rewriter.getI16Type()),
3339 VectorType::get({32}, rewriter.getI16Type()),
3340 rewriter.getI32Type()}));
3341 } else if (resultBitWidth == 32) {
3342 minOp = xllvm::VectorMinGe32AIE2pIntrOp::create(
3343 rewriter, loc,
3344 mlir::LLVM::LLVMStructType::getLiteral(
3345 rewriter.getContext(),
3346 {VectorType::get({16}, rewriter.getI32Type()),
3347 rewriter.getI32Type()}),
3348 forceCastOperandsToSignature(
3349 rewriter, loc, operands,
3350 {VectorType::get({16}, rewriter.getI32Type()),
3351 VectorType::get({16}, rewriter.getI32Type()),
3352 rewriter.getI32Type()}));
3353 }
3354 } else {
3355 if (resultBitWidth == 16) {
3356 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
3357 Value lhs = adaptor.getLhs(), rhs = adaptor.getRhs();
3358
3359 if (resultLanes == 16) {
3360 lhs = padVectorWithPoison(rewriter, loc, lhs, 16, 32);
3361 rhs = padVectorWithPoison(rewriter, loc, rhs, 16, 32);
3362 }
3363
3364 minOp = xllvm::VectorMinGeBf16AIE2pIntrOp::create(
3365 rewriter, loc,
3366 mlir::LLVM::LLVMStructType::getLiteral(
3367 rewriter.getContext(), {v32bf16Ty, rewriter.getI32Type()}),
3368 forceCastOperandsToSignature(rewriter, loc, {lhs, rhs},
3369 {v32bf16Ty, v32bf16Ty}));
3370 }
3371 }
3372
3373 if (!minOp) {
3374 op.emitWarning() << "aievec.min conversion fails due to unsupported "
3375 "element data type.\n";
3376 return failure();
3377 }
3378
3379 Value resultVec = LLVM::ExtractValueOp::create(rewriter, loc, minOp,
3380 /*position=*/0);
3381 if (resultLanes == 16 && !llvm::isa<IntegerType>(resultScaTy))
3382 resultVec = extractLowerLanes(rewriter, loc, resultVec, 16);
3383
3384 rewriter.replaceOp(op, resultVec);
3385
3386 return success();
3387 }
3388};
3389
3390// The i32 bitmask of the lanes where `lhs rel rhs` holds, with `rel` one of
3391// lt, ge, gt, le, eq and ne. `LtOp` and `GeOp` are vmax.lt and vmin.ge
3392// intrinsics, which return the lanes where a < b and a >= b in field 1 of their
3393// struct result. gt and le swap the operands; eq and ne combine the compares in
3394// both directions. `extraOperands` follow the two vectors in each call.
3395template <typename LtOp, typename GeOp>
3396static Value buildCmpBitmask(OpBuilder &builder, Location loc, StringRef rel,
3397 Value lhs, Value rhs, ValueRange extraOperands) {
3398 auto structTy = LLVM::LLVMStructType::getLiteral(
3399 builder.getContext(), {lhs.getType(), builder.getI32Type()});
3400 auto compare = [&](bool lessThan, Value a, Value b) -> Value {
3401 SmallVector<Value> operands{a, b};
3402 llvm::append_range(operands, extraOperands);
3403 Value result = lessThan
3404 ? Value(LtOp::create(builder, loc, structTy, operands))
3405 : Value(GeOp::create(builder, loc, structTy, operands));
3406 return LLVM::ExtractValueOp::create(builder, loc, result, 1);
3407 };
3408 if (rel == "lt")
3409 return compare(/*lessThan=*/true, lhs, rhs);
3410 if (rel == "ge")
3411 return compare(/*lessThan=*/false, lhs, rhs);
3412 if (rel == "gt")
3413 return compare(/*lessThan=*/true, rhs, lhs);
3414 if (rel == "le")
3415 return compare(/*lessThan=*/false, rhs, lhs);
3416 assert((rel == "eq" || rel == "ne") && "unexpected compare predicate");
3417 // Separate statements fix the order in which the two compares are emitted.
3418 bool isEq = rel == "eq";
3419 Value forward = compare(/*lessThan=*/!isEq, lhs, rhs);
3420 Value backward = compare(/*lessThan=*/!isEq, rhs, lhs);
3421 if (isEq)
3422 return LLVM::AndOp::create(builder, loc, forward, backward);
3423 return LLVM::OrOp::create(builder, loc, forward, backward);
3424}
3425
3426// ----- CmpOp conversion for AIE2 and AIE2P -----
3427// Implements aievec.cmp using vmax.lt / vmin.ge intrinsics. The integer
3428// intrinsics take a last operand that selects a signed (1) or unsigned (0)
3429// compare, which follows the predicate.
3430template <typename MaxLtBf16IntrOp, typename MinGeBf16IntrOp,
3431 typename MaxLt32IntrOp, typename MinGe32IntrOp,
3432 typename MaxLt16IntrOp, typename MinGe16IntrOp>
3433class CmpOpConversionBase : public mlir::ConvertOpToLLVMPattern<aievec::CmpOp> {
3434public:
3435 using ConvertOpToLLVMPattern<aievec::CmpOp>::ConvertOpToLLVMPattern;
3436
3437 LogicalResult
3438 matchAndRewrite(aievec::CmpOp op, OpAdaptor adaptor,
3439 ConversionPatternRewriter &rewriter) const override {
3440 Location loc = op.getLoc();
3441 auto vecTy = cast<VectorType>(op.getLhs().getType());
3442 auto elTy = vecTy.getElementType();
3443 unsigned elWidth = elTy.getIntOrFloatBitWidth();
3444 unsigned lanes = getVectorLaneSize(vecTy);
3445 StringRef pred = op.getPred();
3446
3447 bool isUnsigned = llvm::is_contained({"ult", "uge", "ugt", "ule"}, pred);
3448 if (!isUnsigned &&
3449 !llvm::is_contained({"slt", "sge", "sgt", "sle", "eq", "ne"}, pred))
3450 return failure();
3451 // The relation without its signedness prefix: slt -> lt, uge -> ge.
3452 StringRef rel = pred.size() == 3 ? pred.drop_front() : pred;
3453
3454 Value lhs = adaptor.getLhs(), rhs = adaptor.getRhs();
3455 Value bitmask;
3456 if (elWidth == 16 && isa<FloatType>(elTy) && (lanes == 16 || lanes == 32)) {
3457 // bf16 vectors (16 or 32 lanes); 16 lanes are padded to 32
3458 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
3459 if (lanes == 16) {
3460 lhs = padVectorWithPoison(rewriter, loc, lhs, 16, 32);
3461 rhs = padVectorWithPoison(rewriter, loc, rhs, 16, 32);
3462 }
3463 SmallVector<Value> operands = forceCastOperandsToSignature(
3464 rewriter, loc, {lhs, rhs}, {v32bf16Ty, v32bf16Ty});
3465 bitmask = buildCmpBitmask<MaxLtBf16IntrOp, MinGeBf16IntrOp>(
3466 rewriter, loc, rel, operands[0], operands[1], {});
3467 // Mask off upper bits for 16-lane inputs
3468 if (lanes == 16) {
3469 auto mask =
3470 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
3471 rewriter.getI32IntegerAttr(0xFFFF));
3472 bitmask = LLVM::AndOp::create(rewriter, loc, bitmask, mask);
3473 }
3474 } else if (isa<IntegerType>(elTy) && elWidth * lanes == 512 &&
3475 (elWidth == 32 || elWidth == 16)) {
3476 // v16i32 and v32i16 vectors. eq and ne do not depend on the signedness
3477 // and use the signed form.
3478 auto intVecTy =
3479 VectorType::get({lanes}, rewriter.getIntegerType(elWidth));
3480 Value sign = LLVM::ConstantOp::create(
3481 rewriter, loc, rewriter.getI32Type(),
3482 rewriter.getI32IntegerAttr(isUnsigned ? 0 : 1));
3483 SmallVector<Value> operands = forceCastOperandsToSignature(
3484 rewriter, loc, {lhs, rhs}, {intVecTy, intVecTy});
3485 bitmask = elWidth == 32
3486 ? buildCmpBitmask<MaxLt32IntrOp, MinGe32IntrOp>(
3487 rewriter, loc, rel, operands[0], operands[1], sign)
3488 : buildCmpBitmask<MaxLt16IntrOp, MinGe16IntrOp>(
3489 rewriter, loc, rel, operands[0], operands[1], sign);
3490 } else {
3491 return failure();
3492 }
3493
3494 // Cast i32 bitmask to the op's unsigned integer result type
3495 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
3496 op, op.getResult().getType(), bitmask);
3497 return success();
3498 }
3499};
3500
3502 CmpOpConversionBase<xllvm::VectorMaxLtBf16IntrOp,
3503 xllvm::VectorMinGeBf16IntrOp,
3504 xllvm::VectorMaxLt32IntrOp, xllvm::VectorMinGe32IntrOp,
3505 xllvm::VectorMaxLt16IntrOp, xllvm::VectorMinGe16IntrOp>;
3506
3508 xllvm::VectorMaxLtBf16AIE2pIntrOp, xllvm::VectorMinGeBf16AIE2pIntrOp,
3509 xllvm::VectorMaxLt32AIE2pIntrOp, xllvm::VectorMinGe32AIE2pIntrOp,
3510 xllvm::VectorMaxLt16AIE2pIntrOp, xllvm::VectorMinGe16AIE2pIntrOp>;
3511
3512// ----- SelOp conversion for AIE2/AIE2p -----
3513// Implements aievec.sel using vsel16/vsel32 intrinsics.
3514// For bf16 vectors, bitcasts to i16, calls vsel16, bitcasts back.
3515template <typename Sel16IntrOp, typename Sel32IntrOp>
3516class SelOpConversionBase : public mlir::ConvertOpToLLVMPattern<aievec::SelOp> {
3517public:
3518 using ConvertOpToLLVMPattern<aievec::SelOp>::ConvertOpToLLVMPattern;
3519
3520 LogicalResult
3521 matchAndRewrite(aievec::SelOp op, OpAdaptor adaptor,
3522 ConversionPatternRewriter &rewriter) const override {
3523 Location loc = op.getLoc();
3524 auto resultType = cast<VectorType>(op.getResult().getType());
3525 auto elTy = resultType.getElementType();
3526 unsigned elWidth = elTy.getIntOrFloatBitWidth();
3527 unsigned lanes = getVectorLaneSize(resultType);
3528 auto i32Ty = rewriter.getI32Type();
3529
3530 // Cast the sel bitmask from unsigned integer to i32
3531 Value selMask = adaptor.getSel();
3532 if (selMask.getType() != i32Ty)
3533 selMask =
3534 UnrealizedConversionCastOp::create(rewriter, loc, i32Ty, selMask)
3535 .getResult(0);
3536
3537 // Handle bf16 vectors (16 or 32 lanes) via bitcast to i16
3538 if (elWidth == 16 && isa<FloatType>(elTy)) {
3539 auto v32i16Ty = VectorType::get({32}, rewriter.getI16Type());
3540 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
3541
3542 Value lhs = adaptor.getLhs(), rhs = adaptor.getRhs();
3543 bool needExtract = false;
3544
3545 if (lanes == 16) {
3546 lhs = padVectorWithPoison(rewriter, loc, lhs, 16, 32);
3547 rhs = padVectorWithPoison(rewriter, loc, rhs, 16, 32);
3548 needExtract = true;
3549 } else if (lanes != 32) {
3550 return failure();
3551 }
3552
3553 // Bitcast bf16 -> i16
3554 auto lhsCast = forceCastValueToType(rewriter, loc, lhs, v32bf16Ty);
3555 auto lhsI16 = LLVM::BitcastOp::create(rewriter, loc, v32i16Ty, lhsCast);
3556 auto rhsCast = forceCastValueToType(rewriter, loc, rhs, v32bf16Ty);
3557 auto rhsI16 = LLVM::BitcastOp::create(rewriter, loc, v32i16Ty, rhsCast);
3558
3559 auto selResult = Sel16IntrOp::create(
3560 rewriter, loc, v32i16Ty,
3561 forceCastOperandsToSignature(rewriter, loc, {lhsI16, rhsI16, selMask},
3562 {v32i16Ty, v32i16Ty, i32Ty}));
3563
3564 // Bitcast i16 -> bf16
3565 Value result =
3566 LLVM::BitcastOp::create(rewriter, loc, v32bf16Ty, selResult);
3567
3568 if (needExtract)
3569 result = extractLowerLanes(rewriter, loc, result, 16);
3570
3571 rewriter.replaceOp(op, result);
3572 return success();
3573 }
3574
3575 // Handle i32 vectors (16 lanes)
3576 if (elWidth == 32 && isa<IntegerType>(elTy) && lanes == 16) {
3577 auto v16i32Ty = VectorType::get({16}, rewriter.getI32Type());
3578 auto selResult = Sel32IntrOp::create(
3579 rewriter, loc, v16i32Ty,
3580 forceCastOperandsToSignature(
3581 rewriter, loc, {adaptor.getLhs(), adaptor.getRhs(), selMask},
3582 {v16i32Ty, v16i32Ty, i32Ty}));
3583 rewriter.replaceOp(op, selResult->getResult(0));
3584 return success();
3585 }
3586
3587 // Handle i16 vectors (32 lanes)
3588 if (elWidth == 16 && isa<IntegerType>(elTy) && lanes == 32) {
3589 auto v32i16Ty = VectorType::get({32}, rewriter.getI16Type());
3590 auto selResult = Sel16IntrOp::create(
3591 rewriter, loc, v32i16Ty,
3592 forceCastOperandsToSignature(
3593 rewriter, loc, {adaptor.getLhs(), adaptor.getRhs(), selMask},
3594 {v32i16Ty, v32i16Ty, i32Ty}));
3595 rewriter.replaceOp(op, selResult->getResult(0));
3596 return success();
3597 }
3598
3599 return failure();
3600 }
3601};
3602
3605
3606using SelOpAIE2pConversion = SelOpConversionBase<xllvm::VectorSel16AIE2pIntrOp,
3607 xllvm::VectorSel32AIE2pIntrOp>;
3608
3610 : public mlir::ConvertOpToLLVMPattern<aievec::BroadcastScalarOp> {
3611public:
3612 using ConvertOpToLLVMPattern<
3613 aievec::BroadcastScalarOp>::ConvertOpToLLVMPattern;
3614
3615 LogicalResult
3616 matchAndRewrite(aievec::BroadcastScalarOp op, OpAdaptor adaptor,
3617 ConversionPatternRewriter &rewriter) const override {
3618 Location loc = op.getLoc();
3619
3620 Value result = op.getResult();
3621 VectorType resultType = cast<VectorType>(result.getType());
3622 Type resultScaTy = resultType.getElementType();
3623 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
3624 int resultLanes = getVectorLaneSize(resultType);
3625 int resultVectorSize = resultBitWidth * resultLanes;
3626
3627 if (resultVectorSize != 512) {
3628 op.emitWarning()
3629 << "aievec.broadcast_scalar conversion with result vector size "
3630 << resultVectorSize << " is not implemented.\n";
3631 return failure();
3632 }
3633
3634 // Integer types
3635 if (llvm::isa<IntegerType>(resultScaTy)) {
3636 Value src = adaptor.getSource();
3637 Type srcType = src.getType();
3638 unsigned srcBitWidth = srcType.getIntOrFloatBitWidth();
3639
3640 if (srcBitWidth < 32) {
3641 src = LLVM::SExtOp::create(rewriter, loc, rewriter.getI32Type(), src);
3642 }
3643
3644 if (resultBitWidth == 8) {
3645 rewriter.replaceOpWithNewOp<xllvm::VectorBroadcast8I512IntrOp>(
3646 op, VectorType::get({64}, rewriter.getI8Type()), src);
3647 } else if (resultBitWidth == 16) {
3648 rewriter.replaceOpWithNewOp<xllvm::VectorBroadcast16I512IntrOp>(
3649 op, VectorType::get({32}, rewriter.getI16Type()), src);
3650 } else if (resultBitWidth == 32) {
3651 rewriter.replaceOpWithNewOp<xllvm::VectorBroadcast32I512IntrOp>(
3652 op, VectorType::get({16}, rewriter.getI32Type()), src);
3653 } else {
3654 op.emitWarning()
3655 << "aievec.broadcast_scalar conversion with result bitwidth "
3656 << resultBitWidth << " is not implemented.\n";
3657 return failure();
3658 }
3659 } else {
3660 // Float types
3661 if (resultBitWidth == 16) {
3662 rewriter.replaceOpWithNewOp<xllvm::VectorBroadcast16BF512IntrOp>(
3663 op, VectorType::get({32}, rewriter.getBF16Type()),
3664 adaptor.getSource());
3665 } else if (resultBitWidth == 32) {
3666 // Use llvm.aie2.vbroadcast32.I512 with bitcasts (float -> i32 -> float)
3667 // Following the pattern: %0 = bitcast float %b to i32
3668 // %1 = tail call <16 x i32>
3669 // @llvm.aie2.vbroadcast32.I512(i32 %0) %2 =
3670 // bitcast <16 x i32> %1 to <16 x float>
3671 auto srcAsI32 = bitcastValueToType(rewriter, loc, adaptor.getSource(),
3672 rewriter.getI32Type());
3673 auto broadcastI32 = xllvm::VectorBroadcast32I512IntrOp::create(
3674 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
3675 srcAsI32);
3676 auto resultF32 =
3677 bitcastValueToType(rewriter, loc, broadcastI32,
3678 VectorType::get({16}, rewriter.getF32Type()));
3679 rewriter.replaceOp(op, resultF32);
3680 } else {
3681 op.emitWarning()
3682 << "aievec.broadcast_scalar conversion with result bitwidth "
3683 << resultBitWidth << " is not implemented.\n";
3684 return failure();
3685 }
3686 }
3687
3688 return success();
3689 }
3690};
3691
3692// AIE2p version of BroadcastScalarOp conversion using insertelement +
3693// shufflevector
3695 : public mlir::ConvertOpToLLVMPattern<aievec::BroadcastScalarOp> {
3696public:
3697 using ConvertOpToLLVMPattern<
3698 aievec::BroadcastScalarOp>::ConvertOpToLLVMPattern;
3699
3700 LogicalResult
3701 matchAndRewrite(aievec::BroadcastScalarOp op, OpAdaptor adaptor,
3702 ConversionPatternRewriter &rewriter) const override {
3703 Location loc = op.getLoc();
3704
3705 Value result = op.getResult();
3706 VectorType resultType = cast<VectorType>(result.getType());
3707 Type resultScaTy = resultType.getElementType();
3708 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
3709 int resultLanes = getVectorLaneSize(resultType);
3710 int resultVectorSize = resultBitWidth * resultLanes;
3711
3712 // Support both 256-bit and 512-bit vectors for AIE2p
3713 if (resultVectorSize != 256 && resultVectorSize != 512) {
3714 op.emitWarning()
3715 << "aievec.broadcast_scalar conversion with result vector size "
3716 << resultVectorSize << " is not implemented for AIE2p.\n";
3717 return failure();
3718 }
3719
3720 Value src = adaptor.getSource();
3721 Type srcType = src.getType();
3722
3723 // For integer types, extend or truncate to match result element type
3724 if (llvm::isa<IntegerType>(resultScaTy)) {
3725 unsigned srcBitWidth = srcType.getIntOrFloatBitWidth();
3726 if (srcBitWidth < resultBitWidth) {
3727 src = LLVM::SExtOp::create(rewriter, loc, resultScaTy, src);
3728 } else if (srcBitWidth > resultBitWidth) {
3729 src = LLVM::TruncOp::create(rewriter, loc, resultScaTy, src);
3730 }
3731 }
3732
3733 // Create poison vector of the result type
3734 auto poisonVec = LLVM::PoisonOp::create(rewriter, loc, resultType);
3735
3736 // Insert scalar at position 0
3737 auto idx0 = LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(),
3738 rewriter.getI64IntegerAttr(0));
3739 auto insertedVec = LLVM::InsertElementOp::create(rewriter, loc, resultType,
3740 poisonVec, src, idx0);
3741
3742 // Create shufflevector mask with all zeros (broadcast position 0 to all
3743 // lanes)
3744 SmallVector<int64_t> broadcastMask(resultLanes, 0);
3745 auto broadcastVec = vector::ShuffleOp::create(rewriter, loc, insertedVec,
3746 insertedVec, broadcastMask);
3747
3748 rewriter.replaceOp(op, broadcastVec);
3749 return success();
3750 }
3751};
3752
3753class ShiftOpConversion : public mlir::ConvertOpToLLVMPattern<aievec::ShiftOp> {
3754public:
3755 using ConvertOpToLLVMPattern<aievec::ShiftOp>::ConvertOpToLLVMPattern;
3756
3757 LogicalResult
3758 matchAndRewrite(aievec::ShiftOp op, OpAdaptor adaptor,
3759 ConversionPatternRewriter &rewriter) const override {
3760 Location loc = op.getLoc();
3761
3762 Value result = op.getResult();
3763 VectorType resultType = cast<VectorType>(result.getType());
3764 Type resultScaTy = resultType.getElementType();
3765 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
3766 int resultLanes = getVectorLaneSize(resultType);
3767 int resultVectorSize = resultBitWidth * resultLanes;
3768
3769 if (resultVectorSize != 512) {
3770 op.emitWarning() << "aievec.shift conversion with result vector size "
3771 << resultVectorSize << " is not implemented.\n";
3772 return failure();
3773 }
3774
3775 // assume step is always zero
3776 auto stepCst = LLVM::ConstantOp::create(
3777 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(0));
3778
3779 // create xllvm intrinsic
3780 Value shiftOp = nullptr;
3781 SmallVector<Value> operands(
3782 {adaptor.getLhs(), adaptor.getRhs(), stepCst, adaptor.getShift()});
3783 if (llvm::isa<IntegerType>(resultScaTy)) {
3784 // Integer types
3785 shiftOp = xllvm::VectorShiftI512I512IntrOp::create(
3786 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
3787 forceCastOperandsToSignature(
3788 rewriter, loc, operands,
3789 {VectorType::get({16}, rewriter.getI32Type()),
3790 VectorType::get({16}, rewriter.getI32Type()),
3791 rewriter.getI32Type(), rewriter.getI32Type()}));
3792 } else {
3793 // Float types
3794 shiftOp = xllvm::VectorShiftBF512BF512IntrOp::create(
3795 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()),
3796 forceCastOperandsToSignature(
3797 rewriter, loc, operands,
3798 {VectorType::get({32}, rewriter.getBF16Type()),
3799 VectorType::get({32}, rewriter.getBF16Type()),
3800 rewriter.getI32Type(), rewriter.getI32Type()}));
3801 }
3802
3803 // create bitcast/shape_cast for result
3804 auto resultVal =
3805 forceCastValueToType(rewriter, loc, shiftOp, op.getResult().getType());
3806 rewriter.replaceOp(op, resultVal);
3807
3808 return success();
3809 }
3810};
3811
3812// AIE2p version of ShiftOp conversion
3814 : public mlir::ConvertOpToLLVMPattern<aievec::ShiftOp> {
3815public:
3816 using ConvertOpToLLVMPattern<aievec::ShiftOp>::ConvertOpToLLVMPattern;
3817
3818 LogicalResult
3819 matchAndRewrite(aievec::ShiftOp op, OpAdaptor adaptor,
3820 ConversionPatternRewriter &rewriter) const override {
3821 Location loc = op.getLoc();
3822
3823 Value result = op.getResult();
3824 VectorType resultType = cast<VectorType>(result.getType());
3825 Type resultScaTy = resultType.getElementType();
3826 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
3827 int resultLanes = getVectorLaneSize(resultType);
3828 int resultVectorSize = resultBitWidth * resultLanes;
3829
3830 if (resultVectorSize != 512) {
3831 op.emitWarning() << "aievec.shift conversion with result vector size "
3832 << resultVectorSize << " is not implemented.\n";
3833 return failure();
3834 }
3835
3836 // assume step is always zero
3837 auto stepCst = LLVM::ConstantOp::create(
3838 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(0));
3839
3840 // create xllvm intrinsic
3841 Value shiftOp = nullptr;
3842 SmallVector<Value> operands(
3843 {adaptor.getLhs(), adaptor.getRhs(), stepCst, adaptor.getShift()});
3844 if (llvm::isa<IntegerType>(resultScaTy)) {
3845 // Integer types - use AIE2p intrinsic
3846 shiftOp = xllvm::VectorShiftI512I512AIE2pIntrOp::create(
3847 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
3848 forceCastOperandsToSignature(
3849 rewriter, loc, operands,
3850 {VectorType::get({16}, rewriter.getI32Type()),
3851 VectorType::get({16}, rewriter.getI32Type()),
3852 rewriter.getI32Type(), rewriter.getI32Type()}));
3853 } else {
3854 // Float types - use AIE2p intrinsic
3855 shiftOp = xllvm::VectorShiftBF512BF512AIE2pIntrOp::create(
3856 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()),
3857 forceCastOperandsToSignature(
3858 rewriter, loc, operands,
3859 {VectorType::get({32}, rewriter.getBF16Type()),
3860 VectorType::get({32}, rewriter.getBF16Type()),
3861 rewriter.getI32Type(), rewriter.getI32Type()}));
3862 }
3863
3864 // create bitcast/shape_cast for result
3865 auto resultVal =
3866 forceCastValueToType(rewriter, loc, shiftOp, op.getResult().getType());
3867 rewriter.replaceOp(op, resultVal);
3868
3869 return success();
3870 }
3871};
3872
3874 : public mlir::ConvertOpToLLVMPattern<aievec::ExtElemOp> {
3875public:
3876 using ConvertOpToLLVMPattern<aievec::ExtElemOp>::ConvertOpToLLVMPattern;
3877
3878 LogicalResult
3879 matchAndRewrite(aievec::ExtElemOp op, OpAdaptor adaptor,
3880 ConversionPatternRewriter &rewriter) const override {
3881 Location loc = op.getLoc();
3882
3883 Type resultType = op.getResult().getType();
3884 unsigned resultBitWidth = resultType.getIntOrFloatBitWidth();
3885
3886 Value src = adaptor.getSource();
3887 VectorType srcType = cast<VectorType>(src.getType());
3888 Type srcScalarType = srcType.getElementType();
3889 unsigned srcBitWidth = srcScalarType.getIntOrFloatBitWidth();
3890 int srcLanes = getVectorLaneSize(srcType);
3891 int srcVectorSize = srcBitWidth * srcLanes;
3892
3893 if (srcVectorSize != 512) {
3894 op.emitWarning() << "aievec.ext_elem conversion with source vector size "
3895 << srcVectorSize << " is not supported.\n";
3896 return failure();
3897 }
3898
3899 // create constant for sign
3900 auto signCst = LLVM::ConstantOp::create(
3901 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(1));
3902
3903 // create xllvm intrinsic
3904 Value extElemOp = nullptr;
3905 SmallVector<Value> operands(
3906 {adaptor.getSource(), adaptor.getIndex(), signCst});
3907 if (resultBitWidth == 8) {
3908 extElemOp = xllvm::VectorExtractElem8I512IntrOp::create(
3909 rewriter, loc, rewriter.getI32Type(),
3910 forceCastOperandsToSignature(
3911 rewriter, loc, operands,
3912 {VectorType::get({64}, rewriter.getI8Type()),
3913 rewriter.getI32Type(), rewriter.getI32Type()}));
3914 } else if (resultBitWidth == 16) {
3915 extElemOp = xllvm::VectorExtractElem16I512IntrOp::create(
3916 rewriter, loc, rewriter.getI32Type(),
3917 forceCastOperandsToSignature(
3918 rewriter, loc, operands,
3919 {VectorType::get({32}, rewriter.getI16Type()),
3920 rewriter.getI32Type(), rewriter.getI32Type()}));
3921 } else if (resultBitWidth == 32) {
3922 extElemOp = xllvm::VectorExtractElem32I512IntrOp::create(
3923 rewriter, loc, rewriter.getI32Type(),
3924 forceCastOperandsToSignature(
3925 rewriter, loc, operands,
3926 {VectorType::get({16}, rewriter.getI32Type()),
3927 rewriter.getI32Type(), rewriter.getI32Type()}));
3928 } else {
3929 op.emitWarning() << "aievec.ext_elem conversion with result bit width "
3930 << resultBitWidth << " is not implemented.\n";
3931 return failure();
3932 }
3933
3934 // create truncation op (and bitcast op)
3935 if (llvm::isa<IntegerType>(resultType)) {
3936 if (resultBitWidth < 32) {
3937 // Two-step truncation to avoid direct i32→i8 which the AIE2
3938 // backend cannot legalize after SLP vectorization.
3939 if (resultBitWidth < 16) {
3940 auto i16Ty = rewriter.getI16Type();
3941 auto trunc16 = LLVM::TruncOp::create(rewriter, loc, i16Ty, extElemOp);
3942 rewriter.replaceOpWithNewOp<LLVM::TruncOp>(op, resultType,
3943 trunc16.getResult());
3944 } else {
3945 rewriter.replaceOpWithNewOp<LLVM::TruncOp>(op, resultType, extElemOp);
3946 }
3947 } else {
3948 rewriter.replaceOp(op, extElemOp);
3949 }
3950 } else {
3951 // Float types
3952 if (resultBitWidth == 16) {
3953 extElemOp = LLVM::TruncOp::create(rewriter, loc, rewriter.getI16Type(),
3954 extElemOp);
3955 }
3956 rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, resultType, extElemOp);
3957 }
3958
3959 return success();
3960 }
3961};
3962
3964 : public mlir::ConvertOpToLLVMPattern<aievec::FMAElemOp> {
3965public:
3966 using ConvertOpToLLVMPattern<aievec::FMAElemOp>::ConvertOpToLLVMPattern;
3967
3968 LogicalResult
3969 matchAndRewrite(aievec::FMAElemOp fmaOp, OpAdaptor adaptor,
3970 ConversionPatternRewriter &rewriter) const override {
3971 auto loc = fmaOp.getLoc();
3972 auto lhs = adaptor.getLhs();
3973 auto rhs = adaptor.getRhs();
3974 auto acc = adaptor.getAcc();
3975 auto lhsTy = cast<VectorType>(lhs.getType());
3976 auto rhsTy = cast<VectorType>(rhs.getType());
3977 auto accTy = cast<VectorType>(acc.getType());
3978 auto flatLhsTy = getFlattenedVectorType(lhsTy);
3979 auto flatRhsTy = getFlattenedVectorType(rhsTy);
3980 auto flatAccTy = getFlattenedVectorType(accTy);
3981
3982 // Flatten operands, if needed
3983 if (lhsTy != flatLhsTy)
3984 lhs = vector::ShapeCastOp::create(rewriter, loc, flatLhsTy, lhs);
3985 if (rhsTy != flatRhsTy)
3986 rhs = vector::ShapeCastOp::create(rewriter, loc, flatRhsTy, rhs);
3987 if (accTy != flatAccTy)
3988 acc = vector::ShapeCastOp::create(rewriter, loc, flatAccTy, acc);
3989
3990 // Build vmac configuration constant
3991 Type i32ty = rewriter.getI32Type();
3992 auto confCst = LLVM::ConstantOp::create(
3993 rewriter, loc, i32ty,
3994 rewriter.getI32IntegerAttr(aiev2_vmac_compute_control(
3995 /*sgn_x=*/0, /*sgn_y=*/0, /*amode=*/2, /*bmode=*/3,
3996 /*variant=*/1, /*zero_acc=*/0, /*shift16=*/0,
3997 /*sub_mul=*/0, /*sub_acc1=*/0, /*sub_acc2=*/0,
3998 /*sub_mask=*/0)));
3999
4000 // Pad 16-lane bf16 operands to 32-lane using set+upd intrinsics.
4001 // forceCastOperandsToSignature only does bitwise reinterpretation, which
4002 // leaves garbage in the upper lanes. The MAC intrinsic requires properly
4003 // zero-padded v32bf16 inputs.
4004 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
4005 if (flatLhsTy.getElementType().isBF16() &&
4006 flatLhsTy.getNumElements() < 32) {
4007 auto zero32 = LLVM::ConstantOp::create(rewriter, loc, i32ty,
4008 rewriter.getI32IntegerAttr(0));
4009 auto zeros_i16 = xllvm::VectorBroadcast16I512IntrOp::create(
4010 rewriter, loc, VectorType::get({32}, rewriter.getI16Type()), zero32);
4011 auto zeros_bf16 =
4012 LLVM::BitcastOp::create(rewriter, loc, v32bf16Ty, zeros_i16);
4013 auto zeroVec = xllvm::ExtBF256BF512IntrOp::create(
4014 rewriter, loc, VectorType::get({16}, rewriter.getBF16Type()),
4015 zeros_bf16, zero32);
4016
4017 auto idx1 = LLVM::ConstantOp::create(rewriter, loc, i32ty,
4018 rewriter.getI32IntegerAttr(1));
4019
4020 auto lhsSet = xllvm::VectorSetBF512BF256IntrOp::create(
4021 rewriter, loc, v32bf16Ty, lhs, zero32);
4022 lhs = xllvm::UpdBF512BF256IntrOp::create(rewriter, loc, v32bf16Ty, lhsSet,
4023 zeroVec, idx1);
4024
4025 auto rhsSet = xllvm::VectorSetBF512BF256IntrOp::create(
4026 rewriter, loc, v32bf16Ty, rhs, zero32);
4027 rhs = xllvm::UpdBF512BF256IntrOp::create(rewriter, loc, v32bf16Ty, rhsSet,
4028 zeroVec, idx1);
4029 }
4030
4031 // Insert vmac intrinsic
4032 auto v8i64Ty = VectorType::get({8}, rewriter.getI64Type());
4033 auto macIntrOp = xllvm::MacConfBF16IntrOp::create(
4034 rewriter, loc, v8i64Ty,
4035 forceCastOperandsToSignature(rewriter, loc, {lhs, rhs, acc, confCst},
4036 {v32bf16Ty, v32bf16Ty, v8i64Ty, i32ty}));
4037
4038 // Recast/Reshape result
4039 auto resVal =
4040 forceCastValueToType(rewriter, loc, macIntrOp.getResult(), flatAccTy);
4041 if (flatAccTy != accTy)
4042 resVal = vector::ShapeCastOp::create(rewriter, loc, accTy, resVal);
4043
4044 rewriter.replaceOp(fmaOp, resVal);
4045 return success();
4046 }
4047};
4048
4049// Look through vector.shape_cast ops to find the defining op. The
4050// VectorToAIEVec pass inserts shape_casts (via reshapeLeadingUnitDims) between
4051// the extension ops and the matmul, which would otherwise hide the signedness.
4052static Value lookThroughShapeCasts(Value v) {
4053 while (auto castOp = v.getDefiningOp<vector::ShapeCastOp>())
4054 v = castOp.getSource();
4055 return v;
4056}
4057
4058// If `v` is defined by a signedness-carrying unrealized_conversion_cast --
4059// which is how VectorToAIEVec attaches si*/ui* to an otherwise signless
4060// operand -- return the signedness it carries and advance `v` past it, so the
4061// intrinsic consumes the signless input and the cast is left dead for
4062// canonicalization rather than leaking to LLVM translation.
4063//
4064// A cast whose result element type is not explicitly signed or unsigned is
4065// left in place: it is some other type conversion, and dropping it would
4066// silently change the operand.
4067static std::optional<bool> peelSignednessCast(Value &v) {
4068 auto castOp = v.getDefiningOp<UnrealizedConversionCastOp>();
4069 if (!castOp)
4070 return std::nullopt;
4071 auto vecTy = dyn_cast<VectorType>(v.getType());
4072 auto intTy =
4073 vecTy ? dyn_cast<IntegerType>(vecTy.getElementType()) : IntegerType();
4074 if (!intTy || intTy.isSignless())
4075 return std::nullopt;
4076 v = castOp.getInputs()[0];
4077 return intTy.isSigned();
4078}
4079
4080// The result of resolving one integer matmul operand.
4082 // The value the MAC intrinsic should consume, with any widening op and any
4083 // signedness-carrying cast peeled off.
4085 // Whether that value is to be treated as signed, i.e. whether the MAC
4086 // configuration word's signX (lhs) / signY (rhs) bit should be set.
4088};
4089
4090// Recover the signedness of an integer matmul operand. MLIR integers are
4091// signless, so it has to come from context; in priority order:
4092//
4093// 1. an arith.extsi / arith.extui feeding the operand -- peel it and take
4094// its signedness;
4095// 2. an unrealized_conversion_cast to a signed/unsigned element type, which
4096// is how VectorToAIEVec carries signedness across the peel it performs.
4097// Peel it too, so the intrinsic consumes the signless value and the cast
4098// is left dead for canonicalization rather than leaking to LLVM;
4099// 3. an explicitly signed or unsigned element type;
4100// 4. otherwise the element type is signless -- treat it as signed.
4101//
4102// Case 4 matches the conventional reading of a signless integer, and is the
4103// more useful default: the shapes that reach it are the ones whose operand
4104// type is already a legal AIE2 narrow type (so nothing was extended), which in
4105// practice means i16/i32 operands carrying signed data.
4106//
4107// Both aievec.matmul and aievec.matmul_aie2p resolve signedness through this
4108// function. They previously each had their own copy and had drifted apart.
4109static ResolvedMatMulOperand resolveMatMulOperand(Value v) {
4110 std::optional<bool> castSigned = peelSignednessCast(v);
4111
4112 Value orig = lookThroughShapeCasts(v);
4113 if (auto extSIOp = orig.getDefiningOp<arith::ExtSIOp>())
4114 return {lookThroughShapeCasts(extSIOp.getIn()), true};
4115 if (auto extUIOp = orig.getDefiningOp<arith::ExtUIOp>())
4116 return {lookThroughShapeCasts(extUIOp.getIn()), false};
4117 if (castSigned)
4118 return {v, *castSigned};
4119
4120 auto vecTy = dyn_cast<VectorType>(v.getType());
4121 auto intTy =
4122 vecTy ? dyn_cast<IntegerType>(vecTy.getElementType()) : IntegerType();
4123 return {v, !(intTy && intTy.isUnsigned())};
4124}
4125
4126// Transposes the rows x cols tile held row-major in the flat vector `v` with
4127// the shuffle modes getTileTransposeModes picks for it. A 256-bit tile is
4128// padded to 512 bits for the shuffle, and a 1024-bit one is shuffled from its
4129// two 512-bit halves into the two halves of the result.
4130static Value transposeTile(OpBuilder &b, Location loc, Value v, int64_t rows,
4131 int64_t cols, bool aie2p) {
4132 auto vecTy = cast<VectorType>(v.getType());
4133 std::optional<SmallVector<aievec::ShuffleMode, 2>> modes =
4134 aievec::getTileTransposeModes(aie2p, vecTy.getElementType(), rows, cols);
4135 assert(modes && "the matmul verifier admits only transposable tiles");
4136 if (modes->empty())
4137 return v;
4138 Type i32ty = b.getI32Type();
4139 auto v16i32Ty = VectorType::get({16}, i32ty);
4140 SmallVector<Value, 2> modeCsts;
4141 auto createModeConstants = [&]() {
4142 for (aievec::ShuffleMode mode : *modes)
4143 modeCsts.push_back(LLVM::ConstantOp::create(
4144 b, loc, i32ty, b.getI32IntegerAttr(static_cast<int32_t>(mode))));
4145 };
4146 auto shuffle = [&](Value x, Value y, Value modeCst) -> Value {
4147 if (aie2p)
4148 return xllvm::VectorShuffleAIE2pIntrOp::create(b, loc, v16i32Ty, x, y,
4149 modeCst);
4150 return xllvm::VectorShuffleIntrOp::create(b, loc, v16i32Ty, x, y, modeCst);
4151 };
4152 int64_t numElems = vecTy.getNumElements();
4153 if (modes->size() == 2) {
4154 auto halves = forceCastValueToType(b, loc, v, VectorType::get({32}, i32ty));
4155 auto lo = vector::ShuffleOp::create(
4156 b, loc, halves, halves, llvm::to_vector(llvm::seq<int64_t>(0, 16)));
4157 auto hi = vector::ShuffleOp::create(
4158 b, loc, halves, halves, llvm::to_vector(llvm::seq<int64_t>(16, 32)));
4159 createModeConstants();
4160 Value resLo = shuffle(lo, hi, modeCsts[0]);
4161 Value resHi = shuffle(lo, hi, modeCsts[1]);
4162 auto res = vector::ShuffleOp::create(
4163 b, loc, resLo, resHi, llvm::to_vector(llvm::seq<int64_t>(0, 32)));
4164 return forceCastValueToType(b, loc, res, vecTy);
4165 }
4166 bool half = numElems * vecTy.getElementTypeBitWidth() == 256;
4167 Value x = v;
4168 if (half) {
4169 SmallVector<int64_t> padMask =
4170 llvm::to_vector(llvm::seq<int64_t>(0, numElems));
4171 padMask.append(numElems, vector::ShuffleOp::kPoisonIndex);
4172 x = vector::ShuffleOp::create(b, loc, v, v, padMask);
4173 }
4174 Value xi = forceCastValueToType(b, loc, x, v16i32Ty);
4175 Value y =
4176 aie2p ? xi
4177 : xllvm::UndefV16I32IntrOp::create(b, loc, v16i32Ty).getResult();
4178 createModeConstants();
4179 Value res =
4180 forceCastValueToType(b, loc, shuffle(xi, y, modeCsts[0]), x.getType());
4181 if (half)
4182 res = vector::ShuffleOp::create(
4183 b, loc, res, res, llvm::to_vector(llvm::seq<int64_t>(0, numElems)));
4184 return res;
4185}
4186
4187// Brings the flattened operand `flat`, stored as `storedTy` (transposed when
4188// `stored` is set), into the layout the recipe reads it in.
4189static Value materializeOperandLayout(OpBuilder &b, Location loc, Value flat,
4190 VectorType storedTy, bool stored,
4191 bool native, bool aie2p) {
4192 if (stored == native)
4193 return flat;
4194 return transposeTile(b, loc, flat, storedTy.getDimSize(0),
4195 storedTy.getDimSize(1), aie2p);
4196}
4197
4199 : public mlir::ConvertOpToLLVMPattern<aievec::MatMulOp> {
4200 using ConvertOpToLLVMPattern<aievec::MatMulOp>::ConvertOpToLLVMPattern;
4201
4202 enum class Accumulator { I32, I64, BF16 };
4203
4204 // The accumulator kind and `mac.conf` modes of each AIE2 recipe.
4205 static std::tuple<Accumulator, int, int, int>
4206 getAccAndModes(aievec::MatMulRecipeKind kind) {
4207 using K = aievec::MatMulRecipeKind;
4208 switch (kind) {
4209 case K::AIE2_I8xI4_4x16x8:
4210 return {Accumulator::I32, 0, 0, 0};
4211 case K::AIE2_I8xI8_4x8x8:
4212 return {Accumulator::I32, 0, 1, 0};
4213 case K::AIE2_I16xI8_4x4x8:
4214 return {Accumulator::I32, 0, 2, 0};
4215 case K::AIE2_I16xI16_4x2x8:
4216 return {Accumulator::I32, 0, 3, 0};
4217 case K::AIE2_I16xI8_2x8x8_Acc64:
4218 return {Accumulator::I64, 1, 2, 0};
4219 case K::AIE2_I16xI8_4x8x4_Acc64:
4220 return {Accumulator::I64, 1, 2, 1};
4221 case K::AIE2_I16xI16_2x4x8_Acc64:
4222 return {Accumulator::I64, 1, 3, 0};
4223 case K::AIE2_I16xI16_4x4x4_Acc64:
4224 return {Accumulator::I64, 1, 3, 1};
4225 case K::AIE2_I32xI16_4x2x4_Acc64:
4226 return {Accumulator::I64, 1, 0, 0};
4227 case K::AIE2_BF16_4x8x4:
4228 return {Accumulator::BF16, 2, 3, 0};
4229 default:
4230 llvm_unreachable("not an AIE2 recipe");
4231 }
4232 }
4233
4234 LogicalResult
4235 matchAndRewrite(aievec::MatMulOp op, OpAdaptor adaptor,
4236 ConversionPatternRewriter &rewriter) const override {
4237 auto lhsTy = cast<VectorType>(op.getLhs().getType());
4238 auto rhsTy = cast<VectorType>(op.getRhs().getType());
4239 auto accTy = cast<VectorType>(op.getAcc().getType());
4240 std::optional<aievec::MatMulRecipe> recipe =
4241 aievec::checkMatMulTile(/*aie2p=*/false, lhsTy, rhsTy, accTy,
4242 op.getLhsTransposed(), op.getRhsTransposed());
4243 if (!recipe)
4244 return rewriter.notifyMatchFailure(op, "not an AIE2 matmul tile");
4245 auto [accKind, amode, bmode, variant] = getAccAndModes(recipe->kind);
4246
4247 // Recover operand signedness, and narrow each operand to the value the MAC
4248 // should consume. An operand is only narrowed to a value of its own
4249 // element width, as the recipe was chosen for that width.
4250 Value lhs = adaptor.getLhs();
4251 Value rhs = adaptor.getRhs();
4252 int signX = 0, signY = 0;
4253 if (accKind != Accumulator::BF16) {
4254 auto narrow = [](Value v, VectorType ty, int &sign) {
4255 ResolvedMatMulOperand resolved = resolveMatMulOperand(v);
4256 sign = resolved.isSigned ? 1 : 0;
4257 if (getElementTypeOrSelf(resolved.narrowed).getIntOrFloatBitWidth() ==
4258 ty.getElementTypeBitWidth())
4259 return resolved.narrowed;
4260 return v;
4261 };
4262 lhs = narrow(lhs, lhsTy, signX);
4263 rhs = narrow(rhs, rhsTy, signY);
4264 }
4265 int conf = aiev2_vmac_compute_control(
4266 /*sgn_x=*/signX, /*sgn_y=*/signY, amode, bmode, variant,
4267 /*zero_acc=*/0, /*shift16=*/0, /*sub_mul=*/0, /*sub_acc1=*/0,
4268 /*sub_acc2=*/0, /*sub_mask=*/0);
4269
4270 Location loc = op.getLoc();
4271 // Flatten the inputs, and transpose those the multiply reads the other
4272 // way.
4273 auto flatten = [&](Value v) -> Value {
4274 auto flatTy = getFlattenedVectorType(cast<VectorType>(v.getType()));
4275 if (v.getType() == flatTy)
4276 return v;
4277 return vector::ShapeCastOp::create(rewriter, loc, flatTy, v);
4278 };
4279 lhs = flatten(lhs);
4280 rhs = flatten(rhs);
4281 auto accFlattenedVecTy = getFlattenedVectorType(accTy);
4282 Value acc = vector::ShapeCastOp::create(rewriter, loc, accFlattenedVecTy,
4283 adaptor.getAcc());
4284 lhs = materializeOperandLayout(rewriter, loc, lhs, lhsTy,
4285 op.getLhsTransposed(), recipe->lhsKM,
4286 /*aie2p=*/false);
4287 rhs = materializeOperandLayout(rewriter, loc, rhs, rhsTy,
4288 op.getRhsTransposed(), recipe->rhsNK,
4289 /*aie2p=*/false);
4290
4291 Type i32ty = rewriter.getI32Type();
4292 auto confCst = LLVM::ConstantOp::create(rewriter, loc, i32ty,
4293 rewriter.getI32IntegerAttr(conf));
4294 SmallVector<Value> operands({lhs, rhs, acc, confCst});
4295 Value matMulResVal;
4296 if (accKind == Accumulator::BF16)
4297 matMulResVal =
4298 xllvm::MacConfBF16IntrOp::create(
4299 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()),
4300 forceCastOperandsToSignature(
4301 rewriter, loc, operands,
4302 {VectorType::get({32}, rewriter.getBF16Type()),
4303 VectorType::get({32}, rewriter.getBF16Type()),
4304 VectorType::get({8}, rewriter.getI64Type()), i32ty}))
4305 .getResult();
4306 else {
4307 SmallVector<Type> intrFuncSig(
4308 {VectorType::get({64}, rewriter.getI8Type()),
4309 VectorType::get({16}, i32ty),
4310 VectorType::get({16}, rewriter.getI64Type()), i32ty});
4311 VectorType v16xi64ty = VectorType::get({16}, rewriter.getI64Type());
4312 if (accKind == Accumulator::I32)
4313 matMulResVal = xllvm::MacConfAcc32IntrOp::create(
4314 rewriter, loc, v16xi64ty,
4315 forceCastOperandsToSignature(rewriter, loc, operands,
4316 intrFuncSig))
4317 .getResult();
4318 else
4319 matMulResVal = xllvm::MacConfAcc64IntrOp::create(
4320 rewriter, loc, v16xi64ty,
4321 forceCastOperandsToSignature(rewriter, loc, operands,
4322 intrFuncSig))
4323 .getResult();
4324 }
4325
4326 auto castFromAcc =
4327 bitcastValueToType(rewriter, loc, matMulResVal, accFlattenedVecTy);
4328
4329 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, op.getType(),
4330 castFromAcc);
4331
4332 return success();
4333 }
4334};
4335
4336// Widens a v64bf16 to v64accfloat by multiplying it by 1.0, which is exact
4337// and runs on the multiplier instead of the conversion unit.
4338static Value widenBF16ToAccFloat(OpBuilder &rewriter, Location loc, Type i32ty,
4339 Value v64bf16) {
4340 auto v64bf16Ty = VectorType::get({64}, rewriter.getBF16Type());
4341 auto ones = LLVM::ConstantOp::create(
4342 rewriter, loc, v64bf16Ty,
4343 DenseElementsAttr::get(
4344 v64bf16Ty, rewriter.getFloatAttr(rewriter.getBF16Type(), 1.0)));
4345 // bf16 x bf16 elementwise multiply into an fp32 accumulator.
4346 auto conf = LLVM::ConstantOp::create(rewriter, loc, i32ty,
4347 rewriter.getI32IntegerAttr(60));
4348 return xllvm::MulConfBF16I1024ACC2048AIE2pIntrOp::create(
4349 rewriter, loc, VectorType::get({64}, rewriter.getF32Type()), v64bf16,
4350 ones, conf);
4351}
4352
4353// Helper function to perform 8×8×4 BF16 matmul following mac_8x8_8x4_bf16
4354// LHS: 64 bfloat16 (8×8 matrix stored [k, m]), RHS: 32 bfloat16 (8×4 matrix)
4355// ACC: 32 float (8×4 result)
4356static Value perform8x8x4MatMul(OpBuilder &rewriter, Location loc, Type i32ty,
4357 Value lhsKM64bf16, Value rhs32bf16,
4358 Value acc32f32) {
4359 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
4360 auto v32f32Ty = VectorType::get({32}, rewriter.getF32Type());
4361
4362 // Rows 0-3 and 4-7 of the [k, m] lhs: one column of A per 8 values.
4363 auto xaBF16 =
4364 vector::ShuffleOp::create(rewriter, loc, lhsKM64bf16, lhsKM64bf16,
4365 llvm::to_vector(llvm::seq<int64_t>(0, 32)));
4366 auto xbBF16 =
4367 vector::ShuffleOp::create(rewriter, loc, lhsKM64bf16, lhsKM64bf16,
4368 llvm::to_vector(llvm::seq<int64_t>(32, 64)));
4369
4370 // Helper to extract and broadcast 8 elements to 32, then shuffle with T16_4x8
4371 auto extractBroadcastShuffle = [&](Value src, int idx) -> Value {
4372 SmallVector<int64_t> extractMask;
4373 int startIdx = idx * 8;
4374 for (int i = 0; i < 8; ++i)
4375 extractMask.push_back(startIdx + i);
4376 // Broadcast by repeating 4 times to get 32 elements
4377 for (int rep = 0; rep < 3; ++rep) {
4378 for (int i = 0; i < 8; ++i)
4379 extractMask.push_back(startIdx + i);
4380 }
4381 auto broadcasted =
4382 vector::ShuffleOp::create(rewriter, loc, src, src, extractMask);
4383
4384 // Apply T16_4x8 shuffle pattern (mode 29)
4385 auto broadI32 = forceCastValueToType(rewriter, loc, broadcasted,
4386 VectorType::get({16}, i32ty));
4387 auto shuffleMode4x8 = LLVM::ConstantOp::create(
4388 rewriter, loc, i32ty, rewriter.getI32IntegerAttr(29));
4389 auto shuffled = xllvm::VectorShuffleAIE2pIntrOp::create(
4390 rewriter, loc, VectorType::get({16}, i32ty), broadI32, broadI32,
4391 shuffleMode4x8);
4392
4393 return forceCastValueToType(rewriter, loc, shuffled, v32bf16Ty);
4394 };
4395
4396 // Prepare 8 row vectors from xa and xb
4397 SmallVector<Value> rowVectors;
4398 for (int i = 0; i < 4; ++i)
4399 rowVectors.push_back(extractBroadcastShuffle(xaBF16, i));
4400 for (int i = 0; i < 4; ++i)
4401 rowVectors.push_back(extractBroadcastShuffle(xbBF16, i));
4402
4403 // Helper to extract and broadcast 4 elements to 32 (for RHS columns)
4404 auto extractBroadcast4 = [&](Value src, int idx) -> Value {
4405 SmallVector<int64_t> mask;
4406 int startIdx = idx * 4;
4407 // Repeat the 4 elements 8 times to get 32 elements
4408 for (int rep = 0; rep < 8; ++rep) {
4409 for (int i = 0; i < 4; ++i)
4410 mask.push_back(startIdx + i);
4411 }
4412 return vector::ShuffleOp::create(rewriter, loc, src, src, mask);
4413 };
4414
4415 // Prepare 8 column vectors from RHS
4416 SmallVector<Value> colVectors;
4417 for (int i = 0; i < 8; ++i)
4418 colVectors.push_back(extractBroadcast4(rhs32bf16, i));
4419
4420 // Perform 8 MAC operations with conf=60 (no zero_acc)
4421 auto conf60 = LLVM::ConstantOp::create(rewriter, loc, i32ty,
4422 rewriter.getI32IntegerAttr(60));
4423
4424 Value acc = acc32f32;
4425 for (int i = 0; i < 8; ++i) {
4426 acc = xllvm::MacConfBF16I512ACC1024AIE2pIntrOp::create(
4427 rewriter, loc, v32f32Ty, rowVectors[i], colVectors[i], acc, conf60);
4428 }
4429
4430 return acc;
4431}
4432
4433// Expands a bf16 aievec.matmul_aie2p with a bfp16 recipe into
4434// aievec.to_bfp16 and aievec.matmul_aie2p_bfp16, lowered by the patterns
4435// below.
4437 : public mlir::ConvertOpToLLVMPattern<aievec::MatMulOp_AIE2P> {
4438 using ConvertOpToLLVMPattern<aievec::MatMulOp_AIE2P>::ConvertOpToLLVMPattern;
4439
4440 LogicalResult
4441 matchAndRewrite(aievec::MatMulOp_AIE2P op, OpAdaptor adaptor,
4442 ConversionPatternRewriter &rewriter) const override {
4443 return aievec::expandBFP16MatMul(rewriter, op);
4444 }
4445};
4446
4447// Lowers aievec.to_bfp16: widens the tile to 64 fp32 accumulator lanes (the
4448// upper 32 lanes are undefined for a 4-row tile) as described at
4449// isBFP16MatMulRhsOnly, then converts them to bfp16ebs8.
4451 : public mlir::ConvertOpToLLVMPattern<aievec::ToBFP16Op> {
4452 using ConvertOpToLLVMPattern<aievec::ToBFP16Op>::ConvertOpToLLVMPattern;
4453
4454 LogicalResult
4455 matchAndRewrite(aievec::ToBFP16Op op, OpAdaptor adaptor,
4456 ConversionPatternRewriter &rewriter) const override {
4457 Location loc = op.getLoc();
4458 VectorType srcTy = op.getSource().getType();
4459 Value flat = vector::ShapeCastOp::create(
4460 rewriter, loc, getFlattenedVectorType(srcTy), adaptor.getSource());
4461 if (op.getTranspose())
4462 flat = transposeTile(rewriter, loc, flat, srcTy.getDimSize(0),
4463 srcTy.getDimSize(1), /*aie2p=*/true);
4464 Type i32ty = rewriter.getI32Type();
4465 Value f64;
4466 auto v32f32Ty = VectorType::get({32}, rewriter.getF32Type());
4467 if (srcTy.getNumElements() == 64 && aievec::isBFP16MatMulRhsOnly(op)) {
4468 f64 = widenBF16ToAccFloat(rewriter, loc, i32ty, flat);
4469 } else if (srcTy.getNumElements() == 64) {
4470 Value half[2];
4471 for (int64_t h = 0; h < 2; ++h) {
4472 Value v = vector::ShuffleOp::create(
4473 rewriter, loc, flat, flat,
4474 llvm::to_vector(llvm::seq<int64_t>(32 * h, 32 * h + 32)));
4475 half[h] = xllvm::Vector32BF16ToV32AccFloatAIE2pIntrOp::create(
4476 rewriter, loc, v32f32Ty, v);
4477 }
4478 f64 =
4479 vector::ShuffleOp::create(rewriter, loc, half[0], half[1],
4480 llvm::to_vector(llvm::seq<int64_t>(0, 64)));
4481 } else {
4482 SmallVector<int64_t> padMask = llvm::to_vector(llvm::seq<int64_t>(0, 32));
4483 padMask.append(32, vector::ShuffleOp::kPoisonIndex);
4484 Value f32 = xllvm::Vector32BF16ToV32AccFloatAIE2pIntrOp::create(
4485 rewriter, loc, v32f32Ty, flat);
4486 f64 = vector::ShuffleOp::create(rewriter, loc, f32, f32, padMask);
4487 }
4488 auto bfpStructTy = LLVM::LLVMStructType::getLiteral(
4489 rewriter.getContext(), {VectorType::get({64}, rewriter.getI8Type()),
4490 VectorType::get({8}, rewriter.getI8Type())});
4491 Value bfp = xllvm::Vector64AccFloatToV64BFP16EBS8AIE2pIntrOp::create(
4492 rewriter, loc, bfpStructTy, f64);
4493 rewriter.replaceOp(op,
4494 {LLVM::ExtractValueOp::create(rewriter, loc, bfp, 0),
4495 LLVM::ExtractValueOp::create(rewriter, loc, bfp, 1)});
4496 return success();
4497 }
4498};
4499
4500// Builds a pointer to byte `offset` of the i8 memref `base`. Memrefs are
4501// lowered to LLVM after this pass, so the address is computed with memref
4502// ops.
4503static Value bytePointer(OpBuilder &b, Location loc, Value base, Value offset) {
4504 auto meta = memref::ExtractStridedMetadataOp::create(b, loc, base);
4505 Value start = memref::ExtractAlignedPointerAsIndexOp::create(
4506 b, loc, meta.getBaseBuffer());
4507 Value off = arith::AddIOp::create(b, loc, meta.getOffset(), offset);
4508 Value addr = arith::IndexCastOp::create(
4509 b, loc, b.getI32Type(), arith::AddIOp::create(b, loc, start, off));
4510 return LLVM::IntToPtrOp::create(
4511 b, loc, LLVM::LLVMPointerType::get(b.getContext()), addr);
4512}
4513
4514// Returns the FIFO state type { ptr, state, count }, extended with the
4515// block's mantissas and exponents if `popped` is set.
4516static LLVM::LLVMStructType fifoLdType(MLIRContext *ctx, bool popped) {
4517 Type i32 = IntegerType::get(ctx, 32), i8 = IntegerType::get(ctx, 8);
4518 SmallVector<Type> fields = {LLVM::LLVMPointerType::get(ctx),
4519 VectorType::get({32}, i32), i32};
4520 if (popped) {
4521 fields.push_back(VectorType::get({64}, i8));
4522 fields.push_back(VectorType::get({8}, i8));
4523 }
4524 return LLVM::LLVMStructType::getLiteral(ctx, fields);
4525}
4526
4527// Fills the load FIFO at `ptr` and returns its state. The state is undefined
4528// until the fill sets it.
4529static Value fifoLdFill(OpBuilder &b, Location loc, Value ptr) {
4530 Type i32 = b.getI32Type();
4531 auto stateTy = VectorType::get({32}, i32);
4532 Value state = LLVM::UndefOp::create(b, loc, stateTy);
4533 Value count = LLVM::ConstantOp::create(b, loc, i32, b.getI32IntegerAttr(0));
4534 return xllvm::FifoLdFillAIE2pIntrOp::create(
4535 b, loc, fifoLdType(b.getContext(), false), ptr, state, count);
4536}
4537
4538// Lowers aievec.load_bfp16 to a FIFO fill at the block followed by one pop.
4540 : public mlir::ConvertOpToLLVMPattern<aievec::LoadBFP16Op> {
4541 using ConvertOpToLLVMPattern<aievec::LoadBFP16Op>::ConvertOpToLLVMPattern;
4542
4543 LogicalResult
4544 matchAndRewrite(aievec::LoadBFP16Op op, OpAdaptor adaptor,
4545 ConversionPatternRewriter &rewriter) const override {
4546 Location loc = op.getLoc();
4547 Value filled =
4548 fifoLdFill(rewriter, loc,
4549 bytePointer(rewriter, loc, op.getBase(), op.getOffset()));
4550 Value popped = xllvm::FifoLdPopBFP576AIE2pIntrOp::create(
4551 rewriter, loc, fifoLdType(rewriter.getContext(), true),
4552 LLVM::ExtractValueOp::create(rewriter, loc, filled, 0),
4553 LLVM::ExtractValueOp::create(rewriter, loc, filled, 1),
4554 LLVM::ExtractValueOp::create(rewriter, loc, filled, 2));
4555 rewriter.replaceOp(
4556 op, {LLVM::ExtractValueOp::create(rewriter, loc, popped, 3),
4557 LLVM::ExtractValueOp::create(rewriter, loc, popped, 4)});
4558 return success();
4559 }
4560};
4561
4562// Lowers aievec.fifo_start to the byte's address and an empty FIFO state.
4564 : public mlir::ConvertOpToLLVMPattern<aievec::FifoStartOp> {
4565 using ConvertOpToLLVMPattern<aievec::FifoStartOp>::ConvertOpToLLVMPattern;
4566
4567 LogicalResult
4568 matchAndRewrite(aievec::FifoStartOp op, OpAdaptor adaptor,
4569 ConversionPatternRewriter &rewriter) const override {
4570 Location loc = op.getLoc();
4571 Type i32 = rewriter.getI32Type();
4572 auto stateTy = VectorType::get({32}, i32);
4573 Value next = bytePointer(rewriter, loc, op.getBase(), op.getOffset());
4574 // The state is undefined until the first fill sets it.
4575 Value state = LLVM::UndefOp::create(rewriter, loc, stateTy);
4576 Value count = LLVM::ConstantOp::create(rewriter, loc, i32,
4577 rewriter.getI32IntegerAttr(0));
4578 rewriter.replaceOp(op, {next, state, count});
4579 return success();
4580 }
4581};
4582
4583// Lowers aievec.fifo_st_push_bfp16 to a push followed by a flush.
4585 : public mlir::ConvertOpToLLVMPattern<aievec::FifoStPushBFP16Op> {
4586 using ConvertOpToLLVMPattern<
4587 aievec::FifoStPushBFP16Op>::ConvertOpToLLVMPattern;
4588
4589 LogicalResult
4590 matchAndRewrite(aievec::FifoStPushBFP16Op op, OpAdaptor adaptor,
4591 ConversionPatternRewriter &rewriter) const override {
4592 Location loc = op.getLoc();
4593 MLIRContext *ctx = rewriter.getContext();
4594 Value ptr = adaptor.getAddr();
4595 Value pushed = xllvm::FifoStPushBFP576AIE2pIntrOp::create(
4596 rewriter, loc, fifoLdType(ctx, false), ptr, adaptor.getMantissas(),
4597 adaptor.getExponents(), adaptor.getState(), adaptor.getCount());
4598 Value flushed = xllvm::FifoStFlushAIE2pIntrOp::create(
4599 rewriter, loc, fifoLdType(ctx, false),
4600 LLVM::ExtractValueOp::create(rewriter, loc, pushed, 0),
4601 LLVM::ExtractValueOp::create(rewriter, loc, pushed, 1),
4602 LLVM::ExtractValueOp::create(rewriter, loc, pushed, 2));
4603 Value next = LLVM::ExtractValueOp::create(rewriter, loc, flushed, 0);
4604 rewriter.replaceOp(
4605 op, {next, LLVM::ExtractValueOp::create(rewriter, loc, flushed, 1),
4606 LLVM::ExtractValueOp::create(rewriter, loc, flushed, 2)});
4607 return success();
4608 }
4609};
4610
4611// Lowers aievec.fifo_ld_pop_bfp16 to a pop, preceded by a fill unless
4612// `no_fill` is set.
4614 : public mlir::ConvertOpToLLVMPattern<aievec::FifoLdPopBFP16Op> {
4615 using ConvertOpToLLVMPattern<
4616 aievec::FifoLdPopBFP16Op>::ConvertOpToLLVMPattern;
4617
4618 LogicalResult
4619 matchAndRewrite(aievec::FifoLdPopBFP16Op op, OpAdaptor adaptor,
4620 ConversionPatternRewriter &rewriter) const override {
4621 Location loc = op.getLoc();
4622 MLIRContext *ctx = rewriter.getContext();
4623 Value ptr = adaptor.getAddr();
4624 Value state = adaptor.getState(), count = adaptor.getCount();
4625 // A pop consumes 576 bits and loads 512, so the FIFO is refilled first.
4626 if (!op.getNoFill()) {
4627 Value filled = xllvm::FifoLdFillAIE2pIntrOp::create(
4628 rewriter, loc, fifoLdType(ctx, false), ptr, state, count);
4629 ptr = LLVM::ExtractValueOp::create(rewriter, loc, filled, 0);
4630 state = LLVM::ExtractValueOp::create(rewriter, loc, filled, 1);
4631 count = LLVM::ExtractValueOp::create(rewriter, loc, filled, 2);
4632 }
4633 Value popped = xllvm::FifoLdPopBFP576AIE2pIntrOp::create(
4634 rewriter, loc, fifoLdType(ctx, true), ptr, state, count);
4635 SmallVector<Value> results;
4636 for (int64_t i = 0; i < 5; ++i)
4637 results.push_back(LLVM::ExtractValueOp::create(rewriter, loc, popped, i));
4638 rewriter.replaceOp(op, results);
4639 return success();
4640 }
4641};
4642
4643// Lowers aievec.matmul_aie2p_bfp16 to mac_8x8_8x8T_conf. A 4-row
4644// accumulator uses the first 32 of the 64 lanes.
4646 : public mlir::ConvertOpToLLVMPattern<aievec::MatMulBFP16Op_AIE2P> {
4647 using ConvertOpToLLVMPattern<
4648 aievec::MatMulBFP16Op_AIE2P>::ConvertOpToLLVMPattern;
4649
4650 LogicalResult
4651 matchAndRewrite(aievec::MatMulBFP16Op_AIE2P op, OpAdaptor adaptor,
4652 ConversionPatternRewriter &rewriter) const override {
4653 Location loc = op.getLoc();
4654 auto accTy = cast<VectorType>(op.getAcc().getType());
4655 auto accFlatTy = getFlattenedVectorType(accTy);
4656 Type i32ty = rewriter.getI32Type();
4657 int64_t lanes = accTy.getNumElements();
4658 Value acc = forceCastValueToType(
4659 rewriter, loc,
4660 vector::ShapeCastOp::create(rewriter, loc, accFlatTy, adaptor.getAcc()),
4661 VectorType::get({lanes}, i32ty));
4662 if (lanes == 32) {
4663 SmallVector<int64_t> padMask = llvm::to_vector(llvm::seq<int64_t>(0, 32));
4664 padMask.append(32, vector::ShuffleOp::kPoisonIndex);
4665 acc = vector::ShuffleOp::create(rewriter, loc, acc, acc, padMask);
4666 }
4667 auto conf = LLVM::ConstantOp::create(rewriter, loc, i32ty,
4668 rewriter.getI32IntegerAttr(780));
4669 Value res = xllvm::MacConfBFP576ACC2048AIE2pIntrOp::create(
4670 rewriter, loc, VectorType::get({64}, i32ty), adaptor.getLhsMantissas(),
4671 adaptor.getLhsExponents(), adaptor.getRhsMantissas(),
4672 adaptor.getRhsExponents(), acc, conf);
4673 if (lanes == 32)
4674 res = vector::ShuffleOp::create(
4675 rewriter, loc, res, res, llvm::to_vector(llvm::seq<int64_t>(0, 32)));
4676 res = forceCastValueToType(rewriter, loc, res, accFlatTy);
4677 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, accTy, res);
4678 return success();
4679 }
4680};
4681
4683 : public mlir::ConvertOpToLLVMPattern<aievec::MatMulOp_AIE2P> {
4684 using ConvertOpToLLVMPattern<aievec::MatMulOp_AIE2P>::ConvertOpToLLVMPattern;
4685
4686 LogicalResult
4687 matchAndRewrite(aievec::MatMulOp_AIE2P op, OpAdaptor adaptor,
4688 ConversionPatternRewriter &rewriter) const override {
4689 using K = aievec::MatMulRecipeKind;
4690 auto lhsTy = cast<VectorType>(op.getLhs().getType());
4691 auto rhsTy = cast<VectorType>(op.getRhs().getType());
4692 auto accTy = cast<VectorType>(op.getAcc().getType());
4693 std::optional<aievec::MatMulRecipe> recipe =
4694 aievec::checkMatMulTile(/*aie2p=*/true, lhsTy, rhsTy, accTy,
4695 op.getLhsTransposed(), op.getRhsTransposed());
4696 if (!recipe)
4697 return rewriter.notifyMatchFailure(op, "not an AIE2P matmul tile");
4698
4699 // Signedness follows the same rule as aievec.matmul; see
4700 // resolveMatMulOperand. Only the signedness is taken from it here; the
4701 // operand values are only advanced past a signedness-carrying cast, not
4702 // past a widening op, as the recipe was chosen for the operand types.
4703 Value lhsOperand = adaptor.getLhs();
4704 Value rhsOperand = adaptor.getRhs();
4705 bool lhsIsSigned = resolveMatMulOperand(lhsOperand).isSigned;
4706 bool rhsIsSigned = resolveMatMulOperand(rhsOperand).isSigned;
4707 peelSignednessCast(lhsOperand);
4708 peelSignednessCast(rhsOperand);
4709
4710 Location loc = op.getLoc();
4711 // Flatten the inputs. An operand the multiply reads the other way is
4712 // transposed where the recipe first uses it.
4713 Value lhsFlat = vector::ShapeCastOp::create(
4714 rewriter, loc,
4715 getFlattenedVectorType(cast<VectorType>(lhsOperand.getType())),
4716 lhsOperand);
4717 Value rhsFlat = vector::ShapeCastOp::create(
4718 rewriter, loc,
4719 getFlattenedVectorType(cast<VectorType>(rhsOperand.getType())),
4720 rhsOperand);
4721 auto accFlattenedVecTy = getFlattenedVectorType(accTy);
4722 Value acc = vector::ShapeCastOp::create(rewriter, loc, accFlattenedVecTy,
4723 adaptor.getAcc());
4724 auto getLhs = [&]() {
4725 return materializeOperandLayout(rewriter, loc, lhsFlat, lhsTy,
4726 op.getLhsTransposed(), recipe->lhsKM,
4727 /*aie2p=*/true);
4728 };
4729 auto getRhs = [&]() {
4730 return materializeOperandLayout(rewriter, loc, rhsFlat, rhsTy,
4731 op.getRhsTransposed(), recipe->rhsNK,
4732 /*aie2p=*/true);
4733 };
4734 Type i32ty = rewriter.getI32Type();
4735
4736 Value matMulResVal;
4737 switch (recipe->kind) {
4738 case K::AIE2P_I8_8x8x8: {
4739 // <8x8xi8> x <8x8xi8> + <8x8xi32>
4740 // Signature: <32 x i64> @llvm.aie2p.I512.I512.ACC2048.mac.conf(
4741 // <16 x i32>, <32 x i16>, <32 x i64>, i32)
4742 // Base conf for amode=0/bmode=1 is bmode<<3 = 8; signedness lives in
4743 // signX (bit 9) / signY (bit 8).
4744 int conf =
4745 8 | ((lhsIsSigned ? 1 : 0) << 9) | ((rhsIsSigned ? 1 : 0) << 8);
4746 auto confCst = LLVM::ConstantOp::create(rewriter, loc, i32ty,
4747 rewriter.getI32IntegerAttr(conf));
4748 matMulResVal =
4749 xllvm::MacConfI512ACC2048AIE2pIntrOp::create(
4750 rewriter, loc, VectorType::get({32}, rewriter.getI64Type()),
4751 forceCastOperandsToSignature(
4752 rewriter, loc, {getLhs(), getRhs(), acc, confCst},
4753 {VectorType::get({16}, rewriter.getI32Type()),
4754 VectorType::get({32}, rewriter.getI16Type()),
4755 VectorType::get({32}, rewriter.getI64Type()), i32ty}))
4756 .getResult();
4757 break;
4758 }
4759 case K::AIE2P_I16_8x2x8: {
4760 // <8x2xi16> x <2x8xi16> + <8x8xi32>
4761 // Input vectors are 16 lanes each, need to pad to 64 lanes for intrinsic
4762 // Signature: <32 x i64> @llvm.aie2p.I1024.I1024.ACC2048.mac.conf(
4763 // <32 x i32>, <64 x i16>, <32 x i64>, i32)
4764 SmallVector<int64_t> padMask = llvm::to_vector(llvm::seq<int64_t>(0, 16));
4765 padMask.append(48, vector::ShuffleOp::kPoisonIndex);
4766 Value lhs = getLhs(), rhs = getRhs();
4767 auto lhsPadded =
4768 vector::ShuffleOp::create(rewriter, loc, lhs, lhs, padMask);
4769 auto rhsPadded =
4770 vector::ShuffleOp::create(rewriter, loc, rhs, rhs, padMask);
4771 auto confCst = LLVM::ConstantOp::create(rewriter, loc, i32ty,
4772 rewriter.getI32IntegerAttr(24));
4773 matMulResVal =
4774 xllvm::MacConfI1024ACC2048AIE2pIntrOp::create(
4775 rewriter, loc, VectorType::get({32}, rewriter.getI64Type()),
4776 forceCastOperandsToSignature(
4777 rewriter, loc, {lhsPadded, rhsPadded, acc, confCst},
4778 {VectorType::get({32}, rewriter.getI32Type()),
4779 VectorType::get({64}, rewriter.getI16Type()),
4780 VectorType::get({32}, rewriter.getI64Type()), i32ty}))
4781 .getResult();
4782 break;
4783 }
4784 case K::AIE2P_BF16Outer_8x1x8: {
4785 // <8x1xbf16> x <1x8xbf16> + <8x8xf32>
4786 // Outer product: grow_replicate both to 64, transpose LHS, use
4787 // mac_elem_64_conf
4788 auto v64f32Ty = VectorType::get({64}, rewriter.getF32Type());
4789
4790 // Replicate LHS and RHS from 8 to 64 elements (8 times each)
4791 SmallVector<int64_t> replicateMask;
4792 for (int rep = 0; rep < 8; ++rep)
4793 for (int i = 0; i < 8; ++i)
4794 replicateMask.push_back(i);
4795 Value lhs = lhsFlat, rhs = rhsFlat;
4796 auto lhs64bf16 =
4797 vector::ShuffleOp::create(rewriter, loc, lhs, lhs, replicateMask);
4798
4799 // Transpose LHS as 8×8 matrix
4800 SmallVector<int64_t> transposeMask;
4801 for (int c = 0; c < 8; ++c)
4802 for (int r = 0; r < 8; ++r)
4803 transposeMask.push_back(r * 8 + c);
4804 auto lhs64bf16Transposed = vector::ShuffleOp::create(
4805 rewriter, loc, lhs64bf16, lhs64bf16, transposeMask);
4806 auto rhs64bf16 =
4807 vector::ShuffleOp::create(rewriter, loc, rhs, rhs, replicateMask);
4808
4809 // mac_elem_64_conf: 64 bf16 lanes, so the I1024 operand form.
4810 auto confCst = LLVM::ConstantOp::create(rewriter, loc, i32ty,
4811 rewriter.getI32IntegerAttr(60));
4812 matMulResVal = xllvm::MacConfBF16I1024ACC2048AIE2pIntrOp::create(
4813 rewriter, loc, v64f32Ty, lhs64bf16Transposed, rhs64bf16, acc,
4814 confCst);
4815 break;
4816 }
4817 case K::AIE2P_BF16Elem_4x8x4: {
4818 // 4×8×4 matmul: <4x8xbf16> x <8x4xbf16> + <4x4xf32>
4819 // Following the reference pattern: a.grow<64>(), b,
4820 // acc.grow<32>().extract<16>(0). Pad the [m, k] LHS to 8 rows, transpose
4821 // it to [k, m] and pad ACC 16→32 for the 8×8×4 multiply, then extract
4822 // 32→16.
4823 SmallVector<int64_t> lhsPadMask =
4824 llvm::to_vector(llvm::seq<int64_t>(0, 32));
4825 lhsPadMask.append(32, vector::ShuffleOp::kPoisonIndex);
4826 Value lhs = getLhs();
4827 Value lhsPadded =
4828 vector::ShuffleOp::create(rewriter, loc, lhs, lhs, lhsPadMask);
4829 lhsPadded = transposeTile(rewriter, loc, lhsPadded, 8, 8,
4830 /*aie2p=*/true);
4831
4832 SmallVector<int64_t> accPadMask =
4833 llvm::to_vector(llvm::seq<int64_t>(0, 16));
4834 accPadMask.append(16, vector::ShuffleOp::kPoisonIndex);
4835 auto accPadded =
4836 vector::ShuffleOp::create(rewriter, loc, acc, acc, accPadMask);
4837
4838 Value acc32 = perform8x8x4MatMul(rewriter, loc, i32ty, lhsPadded,
4839 getRhs(), accPadded);
4840
4841 matMulResVal =
4842 vector::ShuffleOp::create(rewriter, loc, acc32, acc32,
4843 llvm::to_vector(llvm::seq<int64_t>(0, 16)));
4844 break;
4845 }
4846 case K::AIE2P_BF16Elem_8x8x4: {
4847 // 8×8×4 matmul: <8x8xbf16> [k, m] x <8x4xbf16> + <8x4xf32>
4848 Value lhs = getLhs();
4849 Value rhs = getRhs();
4850 matMulResVal = perform8x8x4MatMul(rewriter, loc, i32ty, lhs, rhs, acc);
4851 break;
4852 }
4853 default:
4854 return rewriter.notifyMatchFailure(op, "not an AIE2P recipe");
4855 }
4856
4857 // Cast from flattened result back to original accumulator shape
4858 auto castFromAcc =
4859 forceCastValueToType(rewriter, loc, matMulResVal, accFlattenedVecTy);
4860 // Reshape back to original shape
4861 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, op.getType(),
4862 castFromAcc);
4863 return success();
4864 }
4865};
4866
4867// This pattern folds aievec.cast op. For AIE2, the accumulators are in 32/64
4868// bits, and the vectors are in 4/8/16/32 bits. Hence, we don't have to
4869// explicitly express the casting between accumulators and vectors at the LLVM
4870// dialect level. The backend LLVM compiler will decide the correct accumulator
4871// or vector registers given the ops and intrinsics.
4872class FoldAIECastOps : public mlir::ConvertOpToLLVMPattern<aievec::CastOp> {
4873 using ConvertOpToLLVMPattern<aievec::CastOp>::ConvertOpToLLVMPattern;
4874
4875 // Helper to check if a value is a constant zero
4876 static bool isConstantZero(Value val) {
4877 DenseElementsAttr denseAttr;
4878
4879 // Check for both arith.constant and llvm.mlir.constant
4880 if (auto arithConstOp = val.getDefiningOp<arith::ConstantOp>()) {
4881 denseAttr = dyn_cast<DenseElementsAttr>(arithConstOp.getValue());
4882 } else if (auto llvmConstOp = val.getDefiningOp<LLVM::ConstantOp>()) {
4883 denseAttr = dyn_cast<DenseElementsAttr>(llvmConstOp.getValue());
4884 }
4885
4886 if (!denseAttr || !denseAttr.isSplat())
4887 return false;
4888
4889 auto splatAttr = denseAttr.getSplatValue<Attribute>();
4890 if (auto floatAttr = dyn_cast<FloatAttr>(splatAttr))
4891 return floatAttr.getValue().isZero();
4892 if (auto intAttr = dyn_cast<IntegerAttr>(splatAttr))
4893 return intAttr.getValue().isZero();
4894
4895 return false;
4896 }
4897
4898 LogicalResult
4899 matchAndRewrite(aievec::CastOp castOp, OpAdaptor adaptor,
4900 ConversionPatternRewriter &rewriter) const override {
4901 // Special handling for isResAcc=true with zero constant source
4902 // The backend cannot handle zeroinitializer for accumulator types,
4903 // so we must use the vbroadcast.zero.acc1024 intrinsic instead
4904 if (!castOp.getIsResAcc() || !isConstantZero(adaptor.getSource())) {
4905 // Default behavior: fold the cast
4906 rewriter.replaceOp(castOp, adaptor.getSource());
4907 return success();
4908 }
4909
4910 Location loc = castOp.getLoc();
4911 auto srcVecType = cast<VectorType>(castOp.getSource().getType());
4912 Type srcElemType = srcVecType.getElementType();
4913 int lanes = getVectorLaneSize(srcVecType);
4914
4915 // For f32 vectors (accfloat), use vbroadcast.zero.acc1024
4916 if (srcElemType.isF32() && lanes == 16) {
4917 // Call vbroadcast.zero.acc1024 to get vector<16xi64>
4918 auto zeroAcc1024 = xllvm::VectorBroadcastZeroAcc1024IntrOp::create(
4919 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()));
4920
4921 // Extract lower 8 elements to get vector<8xi64> (512-bit accumulator)
4922 SmallVector<int64_t> extractMask = {0, 1, 2, 3, 4, 5, 6, 7};
4923 auto zeroAcc512 = vector::ShuffleOp::create(rewriter, loc, zeroAcc1024,
4924 zeroAcc1024, extractMask);
4925
4926 // Bitcast back to vector<16xf32> to match the cast result type
4927 auto result = LLVM::BitcastOp::create(
4928 rewriter, loc, VectorType::get({16}, rewriter.getF32Type()),
4929 zeroAcc512);
4930
4931 rewriter.replaceOp(castOp, result);
4932 return success();
4933 }
4934
4935 // Fallback: fold the cast (should not reach here for supported cases)
4936 rewriter.replaceOp(castOp, adaptor.getSource());
4937 return success();
4938 }
4939};
4940
4941// AIE2p version of FoldAIECastOps.
4942// On AIE2P the int/float accumulators share the same physical register, so
4943// aievec.cast between e.g. v32i32 (acc32) and v32f32 (accfloat) is a no-op
4944// at hardware level. LLVM IR still needs a typed value, so emit an
4945// llvm.bitcast when the source/destination LLVM types differ but have the
4946// same bitwidth. Replace with the source value directly when types already
4947// match (the cast becomes a pure type alias).
4949 : public mlir::ConvertOpToLLVMPattern<aievec::CastOp> {
4950 using ConvertOpToLLVMPattern<aievec::CastOp>::ConvertOpToLLVMPattern;
4951
4952 LogicalResult
4953 matchAndRewrite(aievec::CastOp castOp, OpAdaptor adaptor,
4954 ConversionPatternRewriter &rewriter) const override {
4955 Value src = adaptor.getSource();
4956 Type srcTy = src.getType();
4957 Type dstTy = getTypeConverter()->convertType(castOp.getResult().getType());
4958 if (srcTy == dstTy) {
4959 rewriter.replaceOp(castOp, src);
4960 return success();
4961 }
4962 rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(castOp, dstTy, src);
4963 return success();
4964 }
4965};
4966
4968 : public mlir::ConvertOpToLLVMPattern<aievec::ShuffleOp> {
4969 using ConvertOpToLLVMPattern<aievec::ShuffleOp>::ConvertOpToLLVMPattern;
4970
4971 LogicalResult
4972 matchAndRewrite(aievec::ShuffleOp shuffleOp, OpAdaptor adaptor,
4973 ConversionPatternRewriter &rewriter) const override {
4974 if (isAIE2POnlyShuffleMode(shuffleOp.getMode()))
4975 return shuffleOp.emitOpError()
4976 << "shuffle mode '" << stringifyShuffleMode(shuffleOp.getMode())
4977 << "' is not available on AIE2";
4978 auto loc = shuffleOp.getLoc();
4979 auto lhs = adaptor.getLhs();
4980 auto rhs = adaptor.getRhs();
4981 auto i32ty = rewriter.getI32Type();
4982 auto v16xi32ty = VectorType::get({16}, i32ty);
4983 if (!rhs)
4984 rhs = xllvm::UndefV16I32IntrOp::create(rewriter, loc, v16xi32ty);
4985
4986 auto modeAttrVal =
4987 LLVM::ConstantOp::create(rewriter, loc, i32ty,
4988 static_cast<int32_t>(shuffleOp.getMode()))
4989 .getResult();
4990 auto vShuffleVal = xllvm::VectorShuffleIntrOp::create(
4991 rewriter, loc, v16xi32ty,
4992 forceCastOperandsToSignature(
4993 rewriter, loc,
4994 /*operands=*/{lhs, rhs, modeAttrVal},
4995 /*signature=*/{v16xi32ty, v16xi32ty, i32ty}))
4996 .getResult();
4997
4998 vShuffleVal = forceCastValueToType(rewriter, loc, vShuffleVal,
4999 shuffleOp.getResult().getType());
5000
5001 rewriter.replaceOp(shuffleOp, vShuffleVal);
5002
5003 return success();
5004 }
5005};
5006
5007// aievec.shuffle on AIE2P: AIE2P keeps AIE2's mode numbers and adds its own;
5008// the intrinsic is AIE2P's.
5010 : public mlir::ConvertOpToLLVMPattern<aievec::ShuffleOp> {
5011 using ConvertOpToLLVMPattern<aievec::ShuffleOp>::ConvertOpToLLVMPattern;
5012
5013 LogicalResult
5014 matchAndRewrite(aievec::ShuffleOp shuffleOp, OpAdaptor adaptor,
5015 ConversionPatternRewriter &rewriter) const override {
5016 auto loc = shuffleOp.getLoc();
5017 auto i32ty = rewriter.getI32Type();
5018 auto v16xi32ty = VectorType::get({16}, i32ty);
5019 Value lhs = adaptor.getLhs();
5020 Value rhs = adaptor.getRhs();
5021 if (!rhs)
5022 rhs = lhs;
5023 auto mode = LLVM::ConstantOp::create(
5024 rewriter, loc, i32ty, static_cast<int32_t>(shuffleOp.getMode()));
5025 Value v = xllvm::VectorShuffleAIE2pIntrOp::create(
5026 rewriter, loc, v16xi32ty,
5027 forceCastOperandsToSignature(rewriter, loc, {lhs, rhs, mode},
5028 {v16xi32ty, v16xi32ty, i32ty}))
5029 .getResult();
5030 rewriter.replaceOp(shuffleOp,
5031 forceCastValueToType(rewriter, loc, v,
5032 shuffleOp.getResult().getType()));
5033 return success();
5034 }
5035};
5036
5037// Convert aievec.inv to xllvm.intr.aie2p.inv intrinsic for AIE2P
5038// Scalar f32: direct conversion to xllvm.intr.aie2p.inv
5039// Vector f32: unroll into scalar xllvm.intr.aie2p.inv operations
5041 : public mlir::ConvertOpToLLVMPattern<aievec::InvOp> {
5042public:
5043 using ConvertOpToLLVMPattern<aievec::InvOp>::ConvertOpToLLVMPattern;
5044
5045 LogicalResult
5046 matchAndRewrite(aievec::InvOp invOp, OpAdaptor adaptor,
5047 ConversionPatternRewriter &rewriter) const override {
5048 auto loc = invOp.getLoc();
5049 auto operandType = adaptor.getSource().getType();
5050
5051 // Handle scalar f32 inverse
5052 if (operandType.isF32()) {
5053 auto invResult = xllvm::InvAIE2pIntrOp::create(
5054 rewriter, loc, rewriter.getF32Type(), adaptor.getSource());
5055 rewriter.replaceOp(invOp, invResult);
5056 return success();
5057 }
5058
5059 // Handle vector<N x f32> inverse
5060 auto vecType = dyn_cast<VectorType>(operandType);
5061 if (!vecType || !vecType.getElementType().isF32())
5062 return failure();
5063
5064 // Unroll vector inverse into scalar operations
5065 int numElements = getVectorLaneSize(vecType);
5066 Value result = LLVM::PoisonOp::create(rewriter, loc, vecType);
5067
5068 for (int i = 0; i < numElements; ++i) {
5069 // Extract element i
5070 auto indexCst = LLVM::ConstantOp::create(
5071 rewriter, loc, rewriter.getI64Type(), rewriter.getI64IntegerAttr(i));
5072 auto extractedElem = LLVM::ExtractElementOp::create(
5073 rewriter, loc, adaptor.getSource(), indexCst);
5074
5075 // Call xllvm.intr.aie2p.inv on the scalar
5076 auto invResult = xllvm::InvAIE2pIntrOp::create(
5077 rewriter, loc, rewriter.getF32Type(), extractedElem);
5078
5079 // Insert result back into vector
5080 result = LLVM::InsertElementOp::create(rewriter, loc, vecType, result,
5081 invResult, indexCst);
5082 }
5083
5084 rewriter.replaceOp(invOp, result);
5085 return success();
5086 }
5087};
5088
5089// Apply `IntrOpTy`, an AIE2P intrinsic from v16 accfloat to v16 bf16 such as
5090// exp2 or tanh, to a 16- or 32-lane bf16 or f32 vector. A 32-lane vector is
5091// processed as two 16-lane halves. bf16 lanes are widened to accfloat before
5092// the intrinsic, and the bf16 results are widened again when `resultElemType`
5093// is f32.
5094template <typename IntrOpTy>
5095static Value applyAccFloatUnaryIntrinsic(ConversionPatternRewriter &rewriter,
5096 Location loc, Value source,
5097 Type resultElemType) {
5098 auto srcType = cast<VectorType>(source.getType());
5099 unsigned laneSize = getVectorLaneSize(srcType);
5100 auto v16bf16Ty = VectorType::get({16}, rewriter.getBF16Type());
5101 auto v16f32Ty = VectorType::get({16}, rewriter.getF32Type());
5102 auto toF32 = [&](Value v16bf16) -> Value {
5103 return xllvm::Vector16BF16ToV16AccFloatAIE2pIntrOp::create(
5104 rewriter, loc, v16f32Ty, v16bf16);
5105 };
5106
5107 SmallVector<Value, 2> parts;
5108 if (laneSize == 16) {
5109 parts.push_back(source);
5110 } else {
5111 for (int64_t half = 0; half < 2; ++half) {
5112 SmallVector<int64_t> mask =
5113 llvm::to_vector(llvm::seq<int64_t>(16 * half, 16 * (half + 1)));
5114 parts.push_back(
5115 vector::ShuffleOp::create(rewriter, loc, source, source, mask));
5116 }
5117 }
5118 if (srcType.getElementType().isBF16())
5119 for (Value &part : parts)
5120 part = toF32(part);
5121 for (Value &part : parts)
5122 part = IntrOpTy::create(rewriter, loc, v16bf16Ty, part);
5123 if (resultElemType.isF32())
5124 for (Value &part : parts)
5125 part = toF32(part);
5126
5127 if (parts.size() == 1)
5128 return parts.front();
5129 SmallVector<int64_t> combineMask = llvm::to_vector(llvm::seq<int64_t>(0, 32));
5130 return vector::ShuffleOp::create(rewriter, loc, parts[0], parts[1],
5131 combineMask);
5132}
5133
5134// Convert aievec.exp to xllvm.exp2 intrinsic for AIE2P
5135// Uses the identity: exp(x) = exp2(x * log2(e))
5136// Supports both lane-16 and lane-32 bf16 vectors
5138 : public mlir::ConvertOpToLLVMPattern<aievec::ExpOp> {
5139public:
5140 using ConvertOpToLLVMPattern<aievec::ExpOp>::ConvertOpToLLVMPattern;
5141
5142 LogicalResult
5143 matchAndRewrite(aievec::ExpOp expOp, OpAdaptor adaptor,
5144 ConversionPatternRewriter &rewriter) const override {
5145 auto loc = expOp.getLoc();
5146 auto srcType = cast<VectorType>(adaptor.getSource().getType());
5147 auto srcElemType = srcType.getElementType();
5148 unsigned laneSize = getVectorLaneSize(srcType);
5149
5150 // Support v16bfloat16 and v32bfloat16
5151 if ((laneSize != 16 && laneSize != 32) || !srcElemType.isBF16())
5152 return expOp.emitWarning()
5153 << "aievec.exp conversion only supports v16bfloat16 and "
5154 "v32bfloat16.\n";
5155
5156 // Step 1: Create bf16 constant for log2(e) ≈ 1.442695
5157 auto log2eBF16Const = LLVM::ConstantOp::create(
5158 rewriter, loc, rewriter.getBF16Type(),
5159 rewriter.getFloatAttr(rewriter.getBF16Type(), 1.442695));
5160
5161 // Broadcast log2(e) to match input lane size
5162 SmallVector<int64_t> broadcastMask;
5163 for (unsigned i = 0; i < laneSize; ++i)
5164 broadcastMask.push_back(0);
5165
5166 auto v1bf16 = LLVM::UndefOp::create(
5167 rewriter, loc, VectorType::get({1}, rewriter.getBF16Type()));
5168 auto v1bf16Inserted = LLVM::InsertElementOp::create(
5169 rewriter, loc, v1bf16, log2eBF16Const,
5170 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), 0));
5171
5172 auto log2eVec = vector::ShuffleOp::create(rewriter, loc, v1bf16Inserted,
5173 v1bf16Inserted, broadcastMask);
5174
5175 // Step 2: Multiply input by log2(e) in bf16 domain using MulElemOp
5176 // For lane-16: uses I512.I512.ACC512
5177 // For lane-32: uses I512.I512.ACC1024
5178 auto resultF32Ty =
5179 VectorType::get({(int64_t)laneSize}, rewriter.getF32Type());
5180 auto mulResult = aievec::MulElemOp::create(rewriter, loc, resultF32Ty,
5181 adaptor.getSource(), log2eVec);
5182
5183 // Step 3: exp2 of the product
5184 Value exp2Result = applyAccFloatUnaryIntrinsic<xllvm::Exp2AIE2pIntrOp>(
5185 rewriter, loc, mulResult, rewriter.getBF16Type());
5186
5187 rewriter.replaceOp(expOp, exp2Result);
5188
5189 return success();
5190 }
5191};
5192
5193// Convert aievec.tanh or aievec.exp2 to the matching xllvm intrinsic for
5194// AIE2P. Supports lane-16 and lane-32 bf16 and f32 vectors.
5195template <typename SrcOpTy, typename IntrOpTy>
5197 : public mlir::ConvertOpToLLVMPattern<SrcOpTy> {
5198public:
5199 using ConvertOpToLLVMPattern<SrcOpTy>::ConvertOpToLLVMPattern;
5200 using OpAdaptor = typename SrcOpTy::Adaptor;
5201
5202 LogicalResult
5203 matchAndRewrite(SrcOpTy op, OpAdaptor adaptor,
5204 ConversionPatternRewriter &rewriter) const override {
5205 auto srcType = cast<VectorType>(adaptor.getSource().getType());
5206 Type srcElemType = srcType.getElementType();
5207 unsigned laneSize = getVectorLaneSize(srcType);
5208
5209 if ((laneSize != 16 && laneSize != 32) ||
5210 (!srcElemType.isBF16() && !srcElemType.isF32()))
5211 return op.emitWarning()
5212 << op->getName()
5213 << " conversion only supports v16 and v32 of bfloat16 or float.\n";
5214
5215 rewriter.replaceOp(
5216 op, applyAccFloatUnaryIntrinsic<IntrOpTy>(
5217 rewriter, op.getLoc(), adaptor.getSource(), srcElemType));
5218 return success();
5219 }
5220};
5221
5222// Convert math.rsqrt (scalar f32 or vector f32) to xllvm.intr.aie2p.invsqrt
5223// Scalar f32: direct conversion to xllvm.intr.aie2p.invsqrt
5224// Vector f32: unroll into scalar xllvm.intr.aie2p.invsqrt operations
5226 : public mlir::ConvertOpToLLVMPattern<math::RsqrtOp> {
5227public:
5228 using ConvertOpToLLVMPattern<math::RsqrtOp>::ConvertOpToLLVMPattern;
5229
5230 LogicalResult
5231 matchAndRewrite(math::RsqrtOp rsqrtOp, OpAdaptor adaptor,
5232 ConversionPatternRewriter &rewriter) const override {
5233 auto loc = rsqrtOp.getLoc();
5234 auto operandType = adaptor.getOperand().getType();
5235
5236 // Handle scalar f32 rsqrt
5237 if (operandType.isF32()) {
5238 auto rsqrtResult = xllvm::InvsqrtAIE2pIntrOp::create(
5239 rewriter, loc, rewriter.getF32Type(), adaptor.getOperand());
5240 rewriter.replaceOp(rsqrtOp, rsqrtResult);
5241 return success();
5242 }
5243
5244 // Handle vector<N x f32> rsqrt
5245 auto vecType = dyn_cast<VectorType>(operandType);
5246 if (!vecType || !vecType.getElementType().isF32())
5247 return failure();
5248
5249 // Unroll vector rsqrt into scalar operations
5250 int numElements = getVectorLaneSize(vecType);
5251 Value result = LLVM::PoisonOp::create(rewriter, loc, vecType);
5252
5253 for (int i = 0; i < numElements; ++i) {
5254 // Extract element i
5255 auto indexCst = LLVM::ConstantOp::create(
5256 rewriter, loc, rewriter.getI64Type(), rewriter.getI64IntegerAttr(i));
5257 auto extractedElem = LLVM::ExtractElementOp::create(
5258 rewriter, loc, adaptor.getOperand(), indexCst);
5259
5260 // Call xllvm.intr.aie2p.invsqrt on the scalar
5261 auto rsqrtResult = xllvm::InvsqrtAIE2pIntrOp::create(
5262 rewriter, loc, rewriter.getF32Type(), extractedElem);
5263
5264 // Insert result back into vector
5265 result = LLVM::InsertElementOp::create(rewriter, loc, vecType, result,
5266 rsqrtResult, indexCst);
5267 }
5268
5269 rewriter.replaceOp(rsqrtOp, result);
5270 return success();
5271 }
5272};
5273
5274// A one-element vector built from a scalar is lowered to llvm.bitcast. As an
5275// insertelement, LLVM folds it with the extract of a lane into a shufflevector
5276// with a one-element result, which Peano's AIE combines do not handle.
5277static bool isOneLaneVectorOfScalar(Type resultType, Type sourceType) {
5278 auto vecType = dyn_cast<VectorType>(resultType);
5279 return vecType && !vecType.isScalable() && vecType.getRank() == 1 &&
5280 vecType.getNumElements() == 1 && !isa<VectorType>(sourceType);
5281}
5282
5283static bool isOneLaneBroadcastOfScalar(vector::BroadcastOp op) {
5284 return isOneLaneVectorOfScalar(op.getType(), op.getSourceType());
5285}
5286
5287static bool isOneLaneInsertOfScalar(vector::InsertOp op) {
5288 return isOneLaneVectorOfScalar(op.getType(),
5289 op.getValueToStore().getType()) &&
5290 !op.hasDynamicPosition();
5291}
5292
5294 : public mlir::ConvertOpToLLVMPattern<vector::BroadcastOp> {
5295public:
5296 using ConvertOpToLLVMPattern<vector::BroadcastOp>::ConvertOpToLLVMPattern;
5297
5298 LogicalResult
5299 matchAndRewrite(vector::BroadcastOp bcastOp, OpAdaptor adaptor,
5300 ConversionPatternRewriter &rewriter) const override {
5301 if (!isOneLaneBroadcastOfScalar(bcastOp))
5302 return failure();
5303 rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(bcastOp, bcastOp.getType(),
5304 adaptor.getSource());
5305 return success();
5306 }
5307};
5308
5310 : public mlir::ConvertOpToLLVMPattern<vector::InsertOp> {
5311public:
5312 using ConvertOpToLLVMPattern<vector::InsertOp>::ConvertOpToLLVMPattern;
5313
5314 LogicalResult
5315 matchAndRewrite(vector::InsertOp insertOp, OpAdaptor adaptor,
5316 ConversionPatternRewriter &rewriter) const override {
5317 if (!isOneLaneInsertOfScalar(insertOp))
5318 return failure();
5319 rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(insertOp, insertOp.getType(),
5320 adaptor.getValueToStore());
5321 return success();
5322 }
5323};
5324
5325// Convert arith.divf for vector<N x f32> to unrolled scalar divisions
5326// Uses a noinline helper function call as a barrier to prevent LLVM
5327// re-vectorization. Scalar f32 divisions are handled by downstream passes.
5328class FdivOpConversion : public mlir::ConvertOpToLLVMPattern<arith::DivFOp> {
5329public:
5330 using ConvertOpToLLVMPattern<arith::DivFOp>::ConvertOpToLLVMPattern;
5331
5332 FdivOpConversion(const LLVMTypeConverter &typeConverter, StringRef device)
5333 : ConvertOpToLLVMPattern(typeConverter), deviceName(device.str()) {}
5334
5335 std::string deviceName;
5336
5337 LogicalResult
5338 matchAndRewrite(arith::DivFOp divOp, OpAdaptor adaptor,
5339 ConversionPatternRewriter &rewriter) const override {
5340 auto loc = divOp.getLoc();
5341 auto lhsType = adaptor.getLhs().getType();
5342
5343 // Only handle vector<N x f32> fdiv
5344 // Scalar f32 fdiv is handled by downstream passes
5345 auto vecType = dyn_cast<VectorType>(lhsType);
5346 if (!vecType || !vecType.getElementType().isF32())
5347 return failure();
5348
5349 auto rhsType = adaptor.getRhs().getType();
5350 auto rhsVecType = dyn_cast<VectorType>(rhsType);
5351 if (!rhsVecType || rhsVecType != vecType)
5352 return failure();
5353
5354 // For AIE2P, implement a/b as a * inv(b) using the hardware reciprocal
5355 // intrinsic. For AIE2, use scalar fdiv helper function.
5356 auto module = divOp->getParentOfType<ModuleOp>();
5357 auto f32Ty = rewriter.getF32Type();
5358
5359 // Select device-specific body for the scalar helper function.
5360 // AIE2P: use inv(b) * a (hardware reciprocal intrinsic is reliable).
5361 // AIE2: use scalar fdiv directly.
5362 std::function<void(OpBuilder &, Location, ValueRange)> bodyBuilder;
5363 if (deviceName == "aie2p") {
5364 bodyBuilder = [](OpBuilder &builder, Location loc, ValueRange args) {
5365 auto invResult = xllvm::InvAIE2pIntrOp::create(
5366 builder, loc, builder.getF32Type(), args[1]);
5367 auto mulResult =
5368 arith::MulFOp::create(builder, loc, args[0], invResult);
5369 LLVM::ReturnOp::create(builder, loc, ValueRange{mulResult});
5370 };
5371 } else {
5372 bodyBuilder = [](OpBuilder &builder, Location loc, ValueRange args) {
5373 auto divResult = arith::DivFOp::create(builder, loc, args[0], args[1]);
5374 LLVM::ReturnOp::create(builder, loc, ValueRange{divResult});
5375 };
5376 }
5377
5378 // Get or create the noinline scalar helper (acts as barrier to prevent
5379 // LLVM from re-vectorizing the scalar ops).
5380 auto helperFunc =
5381 getOrCreateScalarHelperFunc(module, rewriter, "fdiv", deviceName,
5382 /*argTypes=*/{f32Ty, f32Ty},
5383 /*resultType=*/f32Ty, bodyBuilder);
5384
5385 // Unroll vector fdiv into scalar helper function calls.
5386 int numElements = getVectorLaneSize(vecType);
5387 Value result = LLVM::PoisonOp::create(rewriter, loc, vecType);
5388
5389 for (int i = 0; i < numElements; ++i) {
5390 auto indexCst = LLVM::ConstantOp::create(
5391 rewriter, loc, rewriter.getI64Type(), rewriter.getI64IntegerAttr(i));
5392 auto lhsElem = LLVM::ExtractElementOp::create(rewriter, loc,
5393 adaptor.getLhs(), indexCst);
5394 auto rhsElem = LLVM::ExtractElementOp::create(rewriter, loc,
5395 adaptor.getRhs(), indexCst);
5396
5397 auto divResult = LLVM::CallOp::create(rewriter, loc, helperFunc,
5398 ValueRange{lhsElem, rhsElem})
5399 ->getResult(0);
5400
5401 result = LLVM::InsertElementOp::create(rewriter, loc, vecType, result,
5402 divResult, indexCst);
5403 }
5404
5405 rewriter.replaceOp(divOp, result);
5406 return success();
5407 }
5408};
5409
5411 mlir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns) {
5412 // clang-format off
5413 // Patterns that work for all backends (AIE1, AIE2, AIE2p)
5414 patterns.add<AddOpConversion,
5425 ShuffleOpConversion>(converter);
5426 // clang-format on
5427}
5428
5429static void populateAIEVecToLLVMAIE2ConversionPatterns(
5430 mlir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns,
5431 Aie2Fp32Emulation aie2Fp32EmulationOption) {
5432 // Patterns specific to AIE2 backend
5433 patterns.add<AddElemOpAIE2Conversion, SubElemOpAIE2Conversion>(converter);
5434 patterns.add<NegOpAIE2Conversion>(converter);
5435 patterns.add<MulElemOpConversion>(converter, aie2Fp32EmulationOption);
5436 patterns.add<UPSOpAIE2Conversion, SRSOpAIE2Conversion>(converter);
5437 patterns.add<ShiftOpConversion>(converter);
5438 patterns.add<MaxOpConversion, MinOpConversion>(converter);
5439 patterns.add<CmpOpConversion, SelOpConversion>(converter);
5440 patterns.add<ExtOpConversion>(converter);
5441 patterns.add<ExtractElemOpConversion>(converter);
5442 patterns.add<ConcatOpConversion>(converter);
5443 patterns.add<FMAElemOpConversion>(converter);
5444 patterns.add<FoldAIECastOps>(converter);
5445 patterns.add<FdivOpConversion>(converter, "aie2");
5446 patterns.add<OneLaneBroadcastOpConversion, OneLaneInsertOpConversion>(
5447 converter);
5448}
5449
5450// AIE2p version of ExtractElemOp conversion using LLVM extractelement
5452 : public mlir::ConvertOpToLLVMPattern<aievec::ExtElemOp> {
5453public:
5454 using ConvertOpToLLVMPattern<aievec::ExtElemOp>::ConvertOpToLLVMPattern;
5455
5456 LogicalResult
5457 matchAndRewrite(aievec::ExtElemOp op, OpAdaptor adaptor,
5458 ConversionPatternRewriter &rewriter) const override {
5459 Location loc = op.getLoc();
5460
5461 // AIE2p doesn't have dedicated vextract intrinsics, so use LLVM
5462 // extractelement
5463 Value extracted = LLVM::ExtractElementOp::create(
5464 rewriter, loc, adaptor.getSource(), adaptor.getIndex());
5465
5466 rewriter.replaceOp(op, extracted);
5467 return success();
5468 }
5469};
5470
5471// AIE2p version of ConcatOp conversion using vector.shuffle
5473 : public mlir::ConvertOpToLLVMPattern<aievec::ConcatOp> {
5474public:
5475 using ConvertOpToLLVMPattern<aievec::ConcatOp>::ConvertOpToLLVMPattern;
5476
5477 // AIE2p has no concat intrinsic: the sources are joined two at a time with
5478 // vector.shuffle. When a round has an odd number, the last one is first
5479 // widened with poison to the width of the others.
5480 LogicalResult
5481 matchAndRewrite(aievec::ConcatOp op, OpAdaptor adaptor,
5482 ConversionPatternRewriter &rewriter) const override {
5483 Location loc = op.getLoc();
5484 SmallVector<Value> parts(adaptor.getSources());
5485 if (parts.empty()) {
5486 op.emitWarning() << "aievec.concat with no sources is not supported.\n";
5487 return failure();
5488 }
5489 auto lanesOf = [](Value v) {
5490 return cast<VectorType>(v.getType()).getNumElements();
5491 };
5492 while (parts.size() > 1) {
5493 SmallVector<Value> next;
5494 for (size_t i = 0; i + 1 < parts.size(); i += 2) {
5495 Value lhs = parts[i], rhs = parts[i + 1];
5496 int64_t lhsLanes = lanesOf(lhs), rhsLanes = lanesOf(rhs);
5497 if (rhsLanes < lhsLanes) {
5498 SmallVector<int64_t> padMask(lhsLanes, -1);
5499 std::iota(padMask.begin(), padMask.begin() + rhsLanes, 0);
5500 rhs = vector::ShuffleOp::create(rewriter, loc, rhs, rhs, padMask);
5501 }
5502 SmallVector<int64_t> mask(lhsLanes + rhsLanes);
5503 std::iota(mask.begin(), mask.end(), 0);
5504 next.push_back(
5505 vector::ShuffleOp::create(rewriter, loc, lhs, rhs, mask));
5506 }
5507 if (parts.size() % 2)
5508 next.push_back(parts.back());
5509 parts = std::move(next);
5510 }
5511 rewriter.replaceOp(op, parts.front());
5512 return success();
5513 }
5514};
5515
5517 mlir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns) {
5518 // Takes precedence over the common pattern, which emits the AIE2 intrinsic.
5519 patterns.add<ShuffleOpAIE2pConversion>(converter, /*benefit=*/2);
5520 // Patterns specific to AIE2p backend
5521 patterns.add<AddElemOpI32AIE2pConversion>(converter, /*benefit=*/2);
5522 patterns.add<AddElemOpAIE2pConversion, SubElemOpAIE2pConversion>(converter);
5523 patterns.add<NegOpAIE2pConversion>(converter);
5524 patterns.add<MulElemOpAIE2pConversion>(converter);
5525 patterns.add<FMAElemOpAIE2pConversion>(converter);
5526 patterns.add<UPSOpAIE2pConversion, SRSOpAIE2pConversion>(converter);
5527 patterns.add<MatMulOpAIE2pConversion>(converter);
5528 // Expand bfp16 recipes into the ops below before the generic recipe
5529 // lowering applies.
5530 patterns.add<ExpandBFP16MatMulAIE2pConversion>(converter, /*benefit=*/2);
5531 patterns
5535 converter);
5536 patterns.add<ShiftOpAIE2pConversion>(converter);
5537 patterns.add<MaxOpAIE2pConversion, MinOpAIE2pConversion>(converter);
5538 patterns.add<CmpOpAIE2pConversion, SelOpAIE2pConversion>(converter);
5539 patterns.add<ExtOpAIE2pConversion>(converter);
5540 patterns.add<ExtractElemOpAIE2pConversion>(converter);
5541 patterns.add<ConcatOpAIE2pConversion>(converter);
5542 // Higher benefit so it matches before the generic stub UnpackOpConversion
5543 // returns failure for i4->i8 inputs.
5544 patterns.add<UnpackI4OpAIE2pConversion>(converter, /*benefit=*/2);
5545 patterns.add<ExpOpAIE2pConversion>(converter);
5546 patterns.add<
5549 converter);
5550 patterns.add<InvOpAIE2pConversion>(converter);
5551 patterns.add<BroadcastScalarOpAIE2pConversion>(converter);
5552 patterns.add<RsqrtOpAIE2pConversion>(converter);
5553 patterns.add<FdivOpConversion>(converter, "aie2p");
5554 patterns.add<FoldAIECastOpsAIE2p>(converter);
5556 converter);
5557}
5558
5560 mlir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns,
5561 Aie2Fp32Emulation aie2Fp32EmulationOption, AIEVecTargetInfo aieTarget) {
5562 assert(aieTarget.hasAIEVecLowering() && "no AIEVec lowering for target");
5564 // AIE1 and AIE2 both use the AIE2 patterns. The common patterns cover the
5565 // aievec_aie1 ops.
5566 if (aieTarget.isAIE2P())
5568 else
5570 aie2Fp32EmulationOption);
5571}
5572
5573// Configure legalization rules shared by AIE2 and AIE2p
5574static void configureAIEVecToLLVMLegalizations(LLVMConversionTarget &target) {
5575 // Vector f32 divf is illegal (needs unrolling to scalar divf)
5576 // Scalar f32 divf is legal (handled by downstream passes)
5577 target.addDynamicallyLegalOp<arith::DivFOp>([](arith::DivFOp divOp) {
5578 auto resultType = divOp.getType();
5579 if (auto vecType = dyn_cast<VectorType>(resultType)) {
5580 // Vector f32 divf is illegal and needs conversion
5581 return !vecType.getElementType().isF32();
5582 }
5583 // Scalar divf is legal
5584 return true;
5585 });
5586
5587 // One-element vectors built from a scalar are illegal (lowered to bitcast)
5588 target.addDynamicallyLegalOp<vector::BroadcastOp>(
5589 [](vector::BroadcastOp op) { return !isOneLaneBroadcastOfScalar(op); });
5590 target.addDynamicallyLegalOp<vector::InsertOp>(
5591 [](vector::InsertOp op) { return !isOneLaneInsertOfScalar(op); });
5592}
5593
5595 : xilinx::impl::ConvertAIEVecToLLVMBase<ConvertAIEVecToLLVMPass> {
5597 ConvertAIEVecToLLVMPass(const xilinx::ConvertAIEVecToLLVMOptions &options) {
5598 aieTarget = options.aieTarget;
5599 aie2Fp32Emulation = options.aie2Fp32Emulation;
5600 }
5601
5602 void runOnOperation() override {
5603 ModuleOp module = getOperation();
5604 std::optional<AIEVecTargetInfo> targetInfo =
5605 parseAIEVecTarget(aieTarget, module);
5606 if (!targetInfo)
5607 return signalPassFailure();
5608
5609 // Without an AIEVec lowering for the target, the pass leaves the module
5610 // unchanged, and fails if the module has an AIEVec op to lower.
5611 if (!targetInfo->hasAIEVecLowering()) {
5612 WalkResult result = module.walk([&](Operation *op) {
5613 if (isa_and_present<AIEVecDialect, aie1::AIEVecAIE1Dialect>(
5614 op->getDialect())) {
5615 op->emitOpError() << "has no LLVM lowering for target '"
5616 << getAIEVecTargetName(*targetInfo) << "'";
5617 return WalkResult::interrupt();
5618 }
5619 return WalkResult::advance();
5620 });
5621 if (result.wasInterrupted())
5622 signalPassFailure();
5623 return;
5624 }
5625
5626 RewritePatternSet patterns(&getContext());
5627 LLVMTypeConverter converter(&getContext());
5628
5629 // Don't convert vector types, we want to handle multi-dimensional
5630 // vector on our own.
5631 converter.addConversion(
5632 [&](VectorType type) -> std::optional<Type> { return type; });
5633
5634 populateAIEVecToLLVMConversionPatterns(converter, patterns,
5635 aie2Fp32Emulation, *targetInfo);
5636
5637 LLVMConversionTarget target(getContext());
5638 target.addIllegalDialect<xilinx::aievec::AIEVecDialect,
5639 xilinx::aievec::aie1::AIEVecAIE1Dialect>();
5640 target.addLegalDialect<arith::ArithDialect, vector::VectorDialect,
5641 memref::MemRefDialect, xilinx::xllvm::XLLVMDialect,
5642 ub::UBDialect>();
5643
5644 // Configure legalizations for AIE2/AIE2p
5645 configureAIEVecToLLVMLegalizations(target);
5646
5647 if (failed(applyPartialConversion(getOperation(), target,
5648 std::move(patterns))))
5649 signalPassFailure();
5650 }
5651};
5652
5653std::unique_ptr<mlir::OperationPass<mlir::ModuleOp>>
5655 return std::make_unique<ConvertAIEVecToLLVMPass>();
5656}
5657
5658std::unique_ptr<mlir::OperationPass<mlir::ModuleOp>>
5660 const xilinx::ConvertAIEVecToLLVMOptions &options) {
5661 return std::make_unique<ConvertAIEVecToLLVMPass>(options);
5662}
5663
5664} // namespace xilinx::aievec
LogicalResult matchAndRewrite(SrcOpTy op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(SrcOpTy op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
static DecodedAddElemOp decodeAddElemOp(OpAdaptor op)
LogicalResult matchAndRewrite(aievec::AddElemOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::AddElemOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::aie1::AddOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::BroadcastOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::BroadcastScalarOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::BroadcastScalarOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::CmpOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::ConcatOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::ConcatOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::ExpOp expOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::ExtOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::ExtOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::ExtElemOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::ExtElemOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::FMAElemOp fmaOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::FMAElemOp fmaOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::aie1::FMAOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::DivFOp divOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
FdivOpConversion(const LLVMTypeConverter &typeConverter, StringRef device)
LogicalResult matchAndRewrite(aievec::InvOp invOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::MaxOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::MaxOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::MinOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::MinOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
static DecodedMulElemOp decodeMulElemOp(OpAdaptor op)
LogicalResult matchAndRewrite(aievec::MulElemOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult convertToEmulatedFP32MulElem(aievec::MulElemOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const
LogicalResult convertToEmulatedI32MulElem(aievec::MulElemOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const
MulElemOpConversion(const LLVMTypeConverter &typeConverter, Aie2Fp32Emulation aie2Fp32EmulationOption)
LogicalResult matchAndRewrite(aievec::MulElemOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
static DecodedMulElemOp decodeMulElemOp(OpAdaptor op)
LogicalResult matchAndRewrite(aievec::aie1::MulOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::NegOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::NegOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::BroadcastOp bcastOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::InsertOp insertOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::PackOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
static std::string getIntrinsicName(aievec::PackOp op)
LogicalResult matchAndRewrite(math::RsqrtOp rsqrtOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::SRSOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::SRSOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::SelOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
static std::string getIntrinsicName(aievec::aie1::SelectOp op)
LogicalResult matchAndRewrite(aievec::aie1::SelectOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::ShiftOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::ShiftOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::SubElemOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
static DecodedSubElemOp decodeSubElemOp(OpAdaptor op)
LogicalResult matchAndRewrite(aievec::aie1::SubOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
static std::string getIntrinsicName(aievec::UPDOp op, int loadSize)
LogicalResult matchAndRewrite(aievec::UPDOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::UPSOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::UPSOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::UnpackOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::UnpackOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
mlir::VectorType getFlattenedVectorType(mlir::VectorType vecTy)
std::optional< MatMulRecipe > checkMatMulTile(bool aie2p, mlir::VectorType lhsTy, mlir::VectorType rhsTy, mlir::VectorType accTy, bool lhsTransposed, bool rhsTransposed, llvm::function_ref< mlir::InFlightDiagnostic()> emitError=nullptr)
int32_t getVectorSizeInBits(mlir::VectorType type)
Definition AIEVecUtils.h:65
unsigned getVectorLaneSize(mlir::VectorType type)
Definition AIEVecUtils.h:54
bool isAIE2POnlyShuffleMode(ShuffleMode mode)
void populateAIEVecToLLVMCommonConversionPatterns(mlir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns)
void populateAIEVecToLLVMConversionPatterns(mlir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns, Aie2Fp32Emulation aie2Fp32EmulationOption, AIEVecTargetInfo aieTarget)
std::optional< AIEVecTargetInfo > parseAIEVecTarget(llvm::StringRef spelling)
bool isBFP16MatMulRhsOnly(ToBFP16Op op)
SelOpConversionBase< xllvm::VectorSel16AIE2pIntrOp, xllvm::VectorSel32AIE2pIntrOp > SelOpAIE2pConversion
AccFloatAddSubAIE2pConversion< aievec::AddElemOp, xllvm::AddACC2048AccFloatAIE2pIntrOp > AddElemOpAIE2pConversion
mlir::LogicalResult expandBFP16MatMul(mlir::RewriterBase &rewriter, MatMulOp_AIE2P op)
CmpOpConversionBase< xllvm::VectorMaxLtBf16AIE2pIntrOp, xllvm::VectorMinGeBf16AIE2pIntrOp, xllvm::VectorMaxLt32AIE2pIntrOp, xllvm::VectorMinGe32AIE2pIntrOp, xllvm::VectorMaxLt16AIE2pIntrOp, xllvm::VectorMinGe16AIE2pIntrOp > CmpOpAIE2pConversion
void populateAIEVecToLLVMAIE2ConversionPatterns(mlir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns)
void populateAIEVecToLLVMAIE2pConversionPatterns(mlir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns)
SelOpConversionBase< xllvm::VectorSel16IntrOp, xllvm::VectorSel32IntrOp > SelOpConversion
CmpOpConversionBase< xllvm::VectorMaxLtBf16IntrOp, xllvm::VectorMinGeBf16IntrOp, xllvm::VectorMaxLt32IntrOp, xllvm::VectorMinGe32IntrOp, xllvm::VectorMaxLt16IntrOp, xllvm::VectorMinGe16IntrOp > CmpOpConversion
AccFloatAddSubAIE2pConversion< aievec::SubElemOp, xllvm::SubACC2048AccFloatAIE2pIntrOp > SubElemOpAIE2pConversion
std::unique_ptr< mlir::OperationPass< mlir::ModuleOp > > createConvertAIEVecToLLVMPass()
ConvertAIEVecToLLVMPass(const xilinx::ConvertAIEVecToLLVMOptions &options)