20#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
21#include "mlir/Conversion/LLVMCommon/Pattern.h"
22#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
23#include "mlir/Dialect/Math/IR/Math.h"
24#include "mlir/Dialect/MemRef/IR/MemRef.h"
25#include "mlir/Dialect/UB/IR/UBOps.h"
26#include "mlir/IR/TypeUtilities.h"
32#define GEN_PASS_DEF_CONVERTAIEVECTOLLVM
33#include "aie/Conversion/Passes.h.inc"
40inline static Value bitcastValueToType(OpBuilder &builder, Location loc,
41 Value val, Type dstTy) {
42 return LLVM::BitcastOp::create(builder, loc, dstTy, val).getResult();
48inline static Value widen128bVectorValueTo512b(OpBuilder &builder, Location loc,
50 return xllvm::VectorSetI512I128IntrOp::create(
51 builder, loc, VectorType::get({16}, builder.getI32Type()),
52 bitcastValueToType(builder, loc, val,
53 VectorType::get({4}, builder.getI32Type())))
61inline static Value widen256bVectorValueTo512b(OpBuilder &builder, Location loc,
64 LLVM::ConstantOp::create(builder, loc, builder.getI32Type(), (int32_t)0);
65 return xllvm::VectorSetI512I256IntrOp::create(
66 builder, loc, VectorType::get({16}, builder.getI32Type()),
67 bitcastValueToType(builder, loc, val,
68 VectorType::get({8}, builder.getI32Type())),
76static Value forceCastValueToType(OpBuilder &builder, Location loc, Value val,
78 auto valTy = val.getType();
81 auto srcVecTy = dyn_cast<VectorType>(valTy);
82 auto dstVecTy = dyn_cast<VectorType>(type);
85 assert(dstVecTy &&
"vector values cannot be forced into a non-vector type");
89 if (srcVecTy != flatSrcVecTy)
90 val = vector::ShapeCastOp::create(builder, loc, flatSrcVecTy, val);
95 int64_t dstVecLength =
96 flatDstVecTy.getElementTypeBitWidth() * flatDstVecTy.getShape()[0];
97 int64_t srcVecLength =
98 flatSrcVecTy.getElementTypeBitWidth() * flatSrcVecTy.getShape()[0];
99 if (srcVecLength != dstVecLength) {
100 assert(srcVecLength < dstVecLength &&
101 "only widening forced casts are supported");
102 assert(dstVecLength == 512 &&
103 (srcVecLength == 128 || srcVecLength == 256) &&
104 "only 128b to 512b and 256b to 512b forced casts are supported");
105 if (srcVecLength == 128)
106 val = widen128bVectorValueTo512b(builder, loc, val);
108 val = widen256bVectorValueTo512b(builder, loc, val);
112 val = bitcastValueToType(builder, loc, val, flatDstVecTy);
115 if (flatDstVecTy != dstVecTy)
116 val = vector::ShapeCastOp::create(builder, loc, dstVecTy, val);
122 assert(!dstVecTy &&
"cannot force cast scalar to vector type");
123 return bitcastValueToType(builder, loc, val, type);
130static SmallVector<Value> forceCastOperandsToSignature(OpBuilder &builder,
133 TypeRange signature) {
134 return llvm::to_vector(llvm::map_range(
135 llvm::zip_equal(operands, signature), [&](
auto &&vt) -> Value {
136 return forceCastValueToType(builder, loc, std::get<0>(vt),
159static LLVM::LLVMFuncOp getOrCreateScalarHelperFunc(
160 ModuleOp module, OpBuilder &rewriter, StringRef opName, StringRef device,
161 TypeRange argTypes, Type resultType,
162 const std::function<
void(OpBuilder &, Location, ValueRange)> &bodyBuilder) {
165 std::string funcName =
"__" + device.str() +
"_scalar_" + opName.str();
168 auto helperFunc =
module.lookupSymbol<LLVM::LLVMFuncOp>(funcName);
173 OpBuilder::InsertionGuard guard(rewriter);
174 rewriter.setInsertionPointToStart(module.getBody());
177 SmallVector<Type> argTypesVec(argTypes.begin(), argTypes.end());
179 helperFunc = LLVM::LLVMFuncOp::create(
180 rewriter, rewriter.getUnknownLoc(), funcName,
181 LLVM::LLVMFunctionType::get(resultType, argTypesVec));
184 helperFunc->setAttr(
"passthrough", rewriter.getArrayAttr(
185 {rewriter.getStringAttr(
"noinline")}));
188 auto *entryBlock = helperFunc.addEntryBlock(rewriter);
189 OpBuilder::InsertionGuard bodyGuard(rewriter);
190 rewriter.setInsertionPointToStart(entryBlock);
193 SmallVector<Value> args;
194 for (
unsigned i = 0; i < argTypes.size(); ++i)
195 args.push_back(entryBlock->getArgument(i));
198 bodyBuilder(rewriter, rewriter.getUnknownLoc(), args);
225static inline int aiev2_vmac_compute_control(
int sgn_x,
int sgn_y,
int amode,
226 int bmode,
int variant,
227 int zero_acc,
int shift16,
228 int sub_mul,
int sub_acc1,
229 int sub_acc2,
int sub_mask) {
230 return ((
unsigned)sub_mask << 16) | ((unsigned)shift16 << 10) |
231 ((
unsigned)sub_mul << 11) | ((unsigned)sub_acc1 << 12) |
232 ((
unsigned)sub_acc2 << 13) | ((unsigned)amode << 1) |
233 ((
unsigned)bmode << 3) | ((unsigned)variant << 5) |
234 (((
unsigned)sgn_x << 9) | ((
unsigned)sgn_y << 8)) |
235 ((unsigned)zero_acc << 0);
238static std::string getVectorTypeString(VectorType type,
bool abbrev =
false,
240 std::stringstream ss;
243 if (
auto intType = dyn_cast<IntegerType>(type.getElementType())) {
244 ss << (acc ?
"acc" : abbrev ?
"i" :
"int") << intType.getWidth();
245 }
else if (dyn_cast<FloatType>(type.getElementType())) {
246 ss << (abbrev ?
"f" :
"float");
251static std::string getMulOrFMAIntrinsicName(Operation *op) {
252 std::string baseName;
254 if (
auto mulOp = dyn_cast<aievec::aie1::MulOp>(op)) {
256 lhs = mulOp.getLhs();
257 result = mulOp.getResult();
258 }
else if (
auto fmaOp = dyn_cast<aievec::aie1::FMAOp>(op)) {
260 lhs = fmaOp.getLhs();
261 result = fmaOp.getResult();
263 VectorType resultType = cast<VectorType>(result.getType());
265 std::stringstream ss;
267 if (dyn_cast<IntegerType>(resultType.getElementType())) {
269 ss << resultSize <<
"."
270 << getVectorTypeString(cast<VectorType>(lhs.getType()));
271 }
else if (dyn_cast<FloatType>(resultType.getElementType())) {
272 ss <<
"vfp" << baseName;
279static uint32_t encodeSquare(uint32_t square) {
281 out |= ((square >> 0) & 0x3) << 0;
282 out |= ((square >> 4) & 0x3) << 2;
283 out |= ((square >> 8) & 0x3) << 4;
284 out |= ((square >> 12) & 0x3) << 6;
290static void encodeConf(uint32_t conf[2],
const BufferParams &x,
292 conf[0] |= ((x.step & 0x3F) << 0) | ((z.step & 0x3F) << 8);
293 conf[1] |= (encodeSquare(x.square) << 0) | (encodeSquare(z.square) << 8);
294 conf[1] |= sub << 17;
298 :
public mlir::ConvertOpToLLVMPattern<aievec::aie1::AddOp> {
300 using ConvertOpToLLVMPattern<aievec::aie1::AddOp>::ConvertOpToLLVMPattern;
304 ConversionPatternRewriter &rewriter)
const override {
305 op.emitWarning() <<
"aie.add conversion is not implemented\n";
312 :
public mlir::ConvertOpToLLVMPattern<aievec::AddElemOp> {
314 using ConvertOpToLLVMPattern<aievec::AddElemOp>::ConvertOpToLLVMPattern;
323 auto lhs = op.getLhs();
324 auto lhsVecTy = cast<VectorType>(lhs.getType());
325 auto lhsScaTy = lhsVecTy.getElementType();
326 unsigned lhsBitWidth = lhsScaTy.getIntOrFloatBitWidth();
329 if (llvm::isa<IntegerType>(lhsScaTy)) {
332 if (lhsBitWidth == 32) {
342 ConversionPatternRewriter &rewriter)
const override {
343 Location loc = op.getLoc();
347 op.emitWarning() <<
"aievec.add_elem conversion is not supported.\n";
352 if (decodedAddElemOp.kind ==
354 auto confCst = LLVM::ConstantOp::create(
355 rewriter, loc, rewriter.getI32Type(),
356 rewriter.getI32IntegerAttr(decodedAddElemOp.conf));
357 SmallVector<Value> operands(
358 {adaptor.getLhs(), adaptor.getRhs(), confCst});
360 auto addElemOp = xllvm::AddAccFloatAIE2IntrOp::create(
361 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()),
362 forceCastOperandsToSignature(
363 rewriter, loc, operands,
364 {VectorType::get({8}, rewriter.getI64Type()),
365 VectorType::get({8}, rewriter.getI64Type()),
366 rewriter.getI32Type()}));
369 auto resultVal = forceCastValueToType(rewriter, loc, addElemOp,
370 op.getResult().getType());
371 rewriter.replaceOp(op, resultVal);
375 op.emitWarning() <<
"aievec.add_elem conversion is not supported.\n";
382 :
public mlir::ConvertOpToLLVMPattern<aievec::SubElemOp> {
384 using ConvertOpToLLVMPattern<aievec::SubElemOp>::ConvertOpToLLVMPattern;
393 auto lhs = op.getLhs();
394 auto lhsVecTy = cast<VectorType>(lhs.getType());
395 auto lhsScaTy = lhsVecTy.getElementType();
396 unsigned lhsBitWidth = lhsScaTy.getIntOrFloatBitWidth();
399 if (llvm::isa<IntegerType>(lhsScaTy)) {
402 if (lhsBitWidth == 32) {
412 ConversionPatternRewriter &rewriter)
const override {
413 Location loc = op.getLoc();
417 op.emitWarning() <<
"aievec.sub_elem conversion is not supported.\n";
422 if (decodedSubElemOp.kind ==
424 auto confCst = LLVM::ConstantOp::create(
425 rewriter, loc, rewriter.getI32Type(),
426 rewriter.getI32IntegerAttr(decodedSubElemOp.conf));
427 SmallVector<Value> operands(
428 {adaptor.getLhs(), adaptor.getRhs(), confCst});
430 auto subElemOp = xllvm::SubAccFloatAIE2IntrOp::create(
431 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()),
432 forceCastOperandsToSignature(
433 rewriter, loc, operands,
434 {VectorType::get({8}, rewriter.getI64Type()),
435 VectorType::get({8}, rewriter.getI64Type()),
436 rewriter.getI32Type()}));
439 auto resultVal = forceCastValueToType(rewriter, loc, subElemOp,
440 op.getResult().getType());
441 rewriter.replaceOp(op, resultVal);
445 op.emitWarning() <<
"aievec.sub_elem conversion is not supported.\n";
457template <
typename SrcOpTy,
typename IntrOpTy>
459 :
public mlir::ConvertOpToLLVMPattern<SrcOpTy> {
461 using mlir::ConvertOpToLLVMPattern<SrcOpTy>::ConvertOpToLLVMPattern;
466 ConversionPatternRewriter &rewriter)
const override {
467 constexpr int64_t accLanes = 64;
468 Location loc = op.getLoc();
469 auto type = cast<VectorType>(adaptor.getLhs().getType());
470 int64_t lanes = type.getNumElements();
471 if (!type.getElementType().isF32() || type.getRank() != 1 ||
472 accLanes % lanes != 0 || lanes % 2 != 0) {
473 op.emitWarning() << op->getName() <<
" conversion is not supported.\n";
479 auto accType = VectorType::get({accLanes}, rewriter.getF32Type());
480 auto accI64Type = VectorType::get({accLanes / 2}, rewriter.getI64Type());
481 auto i64Type = VectorType::get({lanes / 2}, rewriter.getI64Type());
482 auto lowLanes = [](int64_t n, int64_t size) {
483 SmallVector<int64_t> mask(size, -1);
484 std::iota(mask.begin(), mask.begin() + std::min(n, size), 0);
487 Value lhs = adaptor.getLhs(), rhs = adaptor.getRhs();
488 if (lanes != accLanes) {
489 lhs = LLVM::BitcastOp::create(rewriter, loc, i64Type, lhs);
490 rhs = LLVM::BitcastOp::create(rewriter, loc, i64Type, rhs);
491 SmallVector<int64_t> padMask = lowLanes(lanes / 2, accLanes / 2);
492 lhs = vector::ShuffleOp::create(rewriter, loc, lhs, lhs, padMask);
493 rhs = vector::ShuffleOp::create(rewriter, loc, rhs, rhs, padMask);
494 lhs = LLVM::BitcastOp::create(rewriter, loc, accType, lhs);
495 rhs = LLVM::BitcastOp::create(rewriter, loc, accType, rhs);
498 auto confCst = LLVM::ConstantOp::create(
499 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(60));
500 Value result = IntrOpTy::create(rewriter, loc, accType, lhs, rhs, confCst);
501 if (lanes != accLanes) {
502 result = LLVM::BitcastOp::create(rewriter, loc, accI64Type, result);
503 result = vector::ShuffleOp::create(rewriter, loc, result, result,
504 lowLanes(lanes / 2, lanes / 2));
505 result = LLVM::BitcastOp::create(rewriter, loc, type, result);
507 rewriter.replaceOp(op, result);
514 xllvm::AddACC2048AccFloatAIE2pIntrOp>;
521 :
public mlir::ConvertOpToLLVMPattern<aievec::AddElemOp> {
523 using ConvertOpToLLVMPattern<aievec::AddElemOp>::ConvertOpToLLVMPattern;
527 ConversionPatternRewriter &rewriter)
const override {
528 constexpr int64_t accLanes = 64;
529 Location loc = op.getLoc();
530 auto type = cast<VectorType>(adaptor.getLhs().getType());
531 int64_t lanes = type.getNumElements();
532 if (!type.getElementType().isInteger(32) || type.getRank() != 1 ||
533 accLanes % lanes != 0 || lanes % 2 != 0)
535 auto accI64Type = VectorType::get({accLanes / 2}, rewriter.getI64Type());
536 auto i64Type = VectorType::get({lanes / 2}, rewriter.getI64Type());
537 auto lowLanes = [](int64_t n, int64_t size) {
538 SmallVector<int64_t> mask(size, -1);
539 std::iota(mask.begin(), mask.begin() + std::min(n, size), 0);
543 LLVM::BitcastOp::create(rewriter, loc, i64Type, adaptor.getLhs());
545 LLVM::BitcastOp::create(rewriter, loc, i64Type, adaptor.getRhs());
546 if (lanes != accLanes) {
547 SmallVector<int64_t> padMask = lowLanes(lanes / 2, accLanes / 2);
548 lhs = vector::ShuffleOp::create(rewriter, loc, lhs, lhs, padMask);
549 rhs = vector::ShuffleOp::create(rewriter, loc, rhs, rhs, padMask);
552 auto confCst = LLVM::ConstantOp::create(
553 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(0));
554 Value result = xllvm::AddACC2048AIE2pIntrOp::create(
555 rewriter, loc, accI64Type, lhs, rhs, confCst);
556 if (lanes != accLanes)
557 result = vector::ShuffleOp::create(rewriter, loc, result, result,
558 lowLanes(lanes / 2, lanes / 2));
559 rewriter.replaceOp(op,
560 LLVM::BitcastOp::create(rewriter, loc, type, result));
566 xllvm::SubACC2048AccFloatAIE2pIntrOp>;
576 using ConvertOpToLLVMPattern<aievec::NegOp>::ConvertOpToLLVMPattern;
580 ConversionPatternRewriter &rewriter)
const override {
581 Location loc = op.getLoc();
582 auto srcVecTy = cast<VectorType>(adaptor.getSource().getType());
583 auto srcScaTy = srcVecTy.getElementType();
587 if (!isa<FloatType>(srcScaTy) || srcScaTy.getIntOrFloatBitWidth() != 32 ||
588 srcVecTy.getNumElements() != 16) {
589 op.emitWarning() <<
"aievec.neg conversion is not supported.\n";
593 auto v8i64Ty = VectorType::get({8}, rewriter.getI64Type());
597 auto confCst = LLVM::ConstantOp::create(
598 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(28));
599 SmallVector<Value> operands({adaptor.getSource(), confCst});
601 auto negOp = xllvm::NegACC512AccFloatAIE2IntrOp::create(
602 rewriter, loc, v8i64Ty,
603 forceCastOperandsToSignature(rewriter, loc, operands,
604 {v8i64Ty, rewriter.getI32Type()}));
607 forceCastValueToType(rewriter, loc, negOp, op.getResult().getType());
608 rewriter.replaceOp(op, resultVal);
619 :
public mlir::ConvertOpToLLVMPattern<aievec::NegOp> {
621 using ConvertOpToLLVMPattern<aievec::NegOp>::ConvertOpToLLVMPattern;
625 ConversionPatternRewriter &rewriter)
const override {
626 Location loc = op.getLoc();
627 auto srcVecTy = cast<VectorType>(adaptor.getSource().getType());
628 auto srcScaTy = srcVecTy.getElementType();
629 unsigned laneSize = srcVecTy.getNumElements();
631 if (!isa<FloatType>(srcScaTy) || srcScaTy.getIntOrFloatBitWidth() != 32 ||
632 (laneSize != 16 && laneSize != 32)) {
633 op.emitWarning() <<
"aievec.neg conversion is not supported.\n";
637 auto v64f32Ty = VectorType::get({64}, rewriter.getF32Type());
641 Value srcVal = adaptor.getSource();
643 if (srcVecTy != fltSrcVecTy)
644 srcVal = vector::ShapeCastOp::create(rewriter, loc, fltSrcVecTy, srcVal)
648 SmallVector<int64_t> expandMask;
649 for (
unsigned i = 0; i < laneSize; ++i)
650 expandMask.push_back(i);
651 for (
unsigned i = laneSize; i < 64; ++i)
652 expandMask.push_back(-1);
654 vector::ShuffleOp::create(rewriter, loc, srcVal, srcVal, expandMask);
660 auto confCst = LLVM::ConstantOp::create(
661 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(60));
662 auto negResult = xllvm::NegACC2048AccFloatAIE2pIntrOp::create(
663 rewriter, loc, v64f32Ty, srcExpanded, confCst);
665 SmallVector<int64_t> extractMask;
666 for (
unsigned i = 0; i < laneSize; ++i)
667 extractMask.push_back(i);
668 Value finalResult = vector::ShuffleOp::create(rewriter, loc, negResult,
669 negResult, extractMask)
673 if (srcVecTy != fltSrcVecTy)
675 vector::ShapeCastOp::create(rewriter, loc, srcVecTy, finalResult)
678 rewriter.replaceOp(op, finalResult);
684 :
public mlir::ConvertOpToLLVMPattern<aievec::aie1::SubOp> {
686 using ConvertOpToLLVMPattern<aievec::aie1::SubOp>::ConvertOpToLLVMPattern;
690 ConversionPatternRewriter &rewriter)
const override {
691 op.emitWarning() <<
"aie.sub conversion is not implemented\n";
697 :
public mlir::ConvertOpToLLVMPattern<aievec::aie1::FMAOp> {
699 using ConvertOpToLLVMPattern<aievec::aie1::FMAOp>::ConvertOpToLLVMPattern;
703 ConversionPatternRewriter &rewriter)
const override {
704 auto module = op->getParentOfType<ModuleOp>();
705 MLIRContext *context = rewriter.getContext();
707 auto startType = IntegerType::get(context, 32);
708 auto offsetsType = VectorType::get({2}, IntegerType::get(context, 32));
709 auto confType = VectorType::get({2}, IntegerType::get(context, 32));
712 std::string intrinsicName = getMulOrFMAIntrinsicName(op);
713 auto func =
module.lookupSymbol<LLVM::LLVMFuncOp>(
714 StringAttr::get(context, intrinsicName));
717 OpBuilder::InsertionGuard guard(rewriter);
718 rewriter.setInsertionPointToStart(module.getBody());
719 func = LLVM::LLVMFuncOp::create(
720 rewriter, rewriter.getUnknownLoc(), intrinsicName,
721 LLVM::LLVMFunctionType::get(
722 op.getResult().getType(),
723 {op.getLhs().getType(), op.getRhs().getType(),
724 op.getAcc().getType(), startType,
735 op.getXstart().getAsInteger(0, x.
start);
736 op.getXoffsets().getAsInteger(0, x.
offsets);
737 op.getXoffsetsHi().getAsInteger(0, x.
offsets_hi);
738 op.getXstep().getAsInteger(0, x.
step);
739 op.getXsquare().getAsInteger(0, x.
square);
740 op.getZstart().getAsInteger(0, z.
start);
741 op.getZoffsets().getAsInteger(0, z.
offsets);
742 op.getZoffsetsHi().getAsInteger(0, z.
offsets_hi);
743 op.getZstep().getAsInteger(0, z.
step);
744 op.getZsquare().getAsInteger(0, z.
square);
747 uint32_t conf[2] = {0, 0};
748 encodeConf(conf, x, z, op.getFmsub());
751 auto xstartVal = LLVM::ConstantOp::create(
752 rewriter, op->getLoc(), startType, rewriter.getI32IntegerAttr(x.
start));
753 auto ystartVal = LLVM::ConstantOp::create(rewriter, op->getLoc(), startType,
754 rewriter.getI32IntegerAttr(0));
755 auto zstartVal = LLVM::ConstantOp::create(
756 rewriter, op->getLoc(), startType, rewriter.getI32IntegerAttr(z.
start));
757 auto xoffsetsVal = LLVM::ConstantOp::create(
758 rewriter, op->getLoc(), offsetsType,
759 rewriter.getI32VectorAttr({(int32_t)x.offsets, (int32_t)x.offsets_hi}));
760 auto zoffsetsVal = LLVM::ConstantOp::create(
761 rewriter, op->getLoc(), offsetsType,
762 rewriter.getI32VectorAttr({(int32_t)z.offsets, (int32_t)z.offsets_hi}));
763 auto confVal = LLVM::ConstantOp::create(
764 rewriter, op->getLoc(), confType,
765 rewriter.getI32VectorAttr({(int32_t)conf[0], (int32_t)conf[1]}));
766 rewriter.replaceOpWithNewOp<LLVM::CallOp>(
768 ValueRange{op.getLhs(), op.getRhs(), op.getAcc(), xstartVal, ystartVal,
769 zstartVal, xoffsetsVal, zoffsetsVal, confVal});
775 :
public mlir::ConvertOpToLLVMPattern<aievec::aie1::MulOp> {
777 using ConvertOpToLLVMPattern<aievec::aie1::MulOp>::ConvertOpToLLVMPattern;
781 ConversionPatternRewriter &rewriter)
const override {
782 auto module = op->getParentOfType<ModuleOp>();
783 MLIRContext *context = rewriter.getContext();
785 auto startType = IntegerType::get(context, 32);
786 auto offsetsType = VectorType::get({2}, IntegerType::get(context, 32));
787 auto confType = VectorType::get({2}, IntegerType::get(context, 32));
790 std::string intrinsicName = getMulOrFMAIntrinsicName(op);
791 auto func =
module.lookupSymbol<LLVM::LLVMFuncOp>(
792 StringAttr::get(context, intrinsicName));
795 OpBuilder::InsertionGuard guard(rewriter);
796 rewriter.setInsertionPointToStart(module.getBody());
797 func = LLVM::LLVMFuncOp::create(
798 rewriter, rewriter.getUnknownLoc(), intrinsicName,
799 LLVM::LLVMFunctionType::get(op.getResult().getType(),
800 {op.getLhs().getType(),
801 op.getRhs().getType(),
813 op.getXstart().getAsInteger(0, x.
start);
814 op.getXoffsets().getAsInteger(0, x.
offsets);
815 op.getXoffsetsHi().getAsInteger(0, x.
offsets_hi);
816 op.getXstep().getAsInteger(0, x.
step);
817 op.getXsquare().getAsInteger(0, x.
square);
818 op.getZstart().getAsInteger(0, z.
start);
819 op.getZoffsets().getAsInteger(0, z.
offsets);
820 op.getZoffsetsHi().getAsInteger(0, z.
offsets_hi);
821 op.getZstep().getAsInteger(0, z.
step);
822 op.getZsquare().getAsInteger(0, z.
square);
825 uint32_t conf[2] = {0, 0};
826 encodeConf(conf, x, z,
false);
829 auto xstartVal = LLVM::ConstantOp::create(
830 rewriter, op->getLoc(), startType, rewriter.getI32IntegerAttr(x.
start));
831 auto ystartVal = LLVM::ConstantOp::create(rewriter, op->getLoc(), startType,
832 rewriter.getI32IntegerAttr(0));
833 auto zstartVal = LLVM::ConstantOp::create(
834 rewriter, op->getLoc(), startType, rewriter.getI32IntegerAttr(z.
start));
835 auto xoffsetsVal = LLVM::ConstantOp::create(
836 rewriter, op->getLoc(), offsetsType,
837 rewriter.getI32VectorAttr({(int32_t)x.offsets, (int32_t)x.offsets_hi}));
838 auto zoffsetsVal = LLVM::ConstantOp::create(
839 rewriter, op->getLoc(), offsetsType,
840 rewriter.getI32VectorAttr({(int32_t)z.offsets, (int32_t)z.offsets_hi}));
841 auto confVal = LLVM::ConstantOp::create(
842 rewriter, op->getLoc(), confType,
843 rewriter.getI32VectorAttr({(int32_t)conf[0], (int32_t)conf[1]}));
844 rewriter.replaceOpWithNewOp<LLVM::CallOp>(
846 ValueRange{op.getLhs(), op.getRhs(), xstartVal, ystartVal, zstartVal,
847 xoffsetsVal, zoffsetsVal, confVal});
853 :
public mlir::ConvertOpToLLVMPattern<aievec::MulElemOp> {
855 using ConvertOpToLLVMPattern<aievec::MulElemOp>::ConvertOpToLLVMPattern;
859 : ConvertOpToLLVMPattern(typeConverter),
881 auto lhs = op.getLhs();
882 auto lhsVecTy = cast<VectorType>(lhs.getType());
883 auto lhsScaTy = lhsVecTy.getElementType();
884 unsigned lhsBitWidth = lhsScaTy.getIntOrFloatBitWidth();
887 if (llvm::isa<IntegerType>(lhsScaTy)) {
888 if (lhsBitWidth == 8) {
890 aiev2_vmac_compute_control(
896 if (lhsBitWidth == 16) {
898 aiev2_vmac_compute_control(
904 if (lhsBitWidth == 32) {
910 if (lhsBitWidth == 16) {
912 aiev2_vmac_compute_control(
918 if (lhsBitWidth == 32) {
959 ConversionPatternRewriter &rewriter)
const {
961 Location loc = op.getLoc();
962 auto zeroCst = LLVM::ConstantOp::create(
963 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(0));
964 auto a0 = adaptor.getLhs();
965 auto a1 = xllvm::VectorBroadcast32I512IntrOp::create(
966 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()), zeroCst);
967 auto b0 = adaptor.getRhs();
968 auto b1 = xllvm::UndefV16I32IntrOp::create(
969 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()));
972 auto a_lo = xllvm::VectorShuffleIntrOp::create(
973 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()), a0, a1,
974 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
975 rewriter.getI32IntegerAttr(2)));
976 auto a_hi = xllvm::VectorShuffleIntrOp::create(
977 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()), a0, a1,
978 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
979 rewriter.getI32IntegerAttr(3)));
980 auto b_lo = xllvm::VectorShuffleIntrOp::create(
981 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()), b0, b1,
982 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
983 rewriter.getI32IntegerAttr(2)));
984 auto b_hi = xllvm::VectorShuffleIntrOp::create(
985 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()), b0, b1,
986 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
987 rewriter.getI32IntegerAttr(3)));
989 auto mulConfCst = LLVM::ConstantOp::create(
990 rewriter, loc, rewriter.getI32Type(),
991 rewriter.getI32IntegerAttr(aiev2_vmac_compute_control(
995 auto mulConfOp = xllvm::MulConfAcc64IntrOp::create(
996 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()),
997 forceCastOperandsToSignature(
999 {a_hi, b_hi, mulConfCst},
1001 {VectorType::get({64}, rewriter.getI8Type()),
1002 VectorType::get({16}, rewriter.getI32Type()),
1003 rewriter.getI32Type()}));
1005 auto createMacConfOp = [&](SmallVector<Value> operands,
1006 int macConf) -> Value {
1008 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1009 rewriter.getI32IntegerAttr(macConf)));
1010 return xllvm::MacConfAcc64IntrOp::create(
1011 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()),
1012 forceCastOperandsToSignature(
1016 {VectorType::get({64}, rewriter.getI8Type()),
1017 VectorType::get({16}, rewriter.getI32Type()),
1018 VectorType::get({16}, rewriter.getI64Type()),
1019 rewriter.getI32Type()}))
1022 auto acc64Val = mulConfOp.getResult();
1023 acc64Val = createMacConfOp(
1024 SmallVector<Value>{a_hi, b_lo, acc64Val},
1025 aiev2_vmac_compute_control(
1029 acc64Val = createMacConfOp(
1030 SmallVector<Value>{a_lo, b_hi, acc64Val},
1031 aiev2_vmac_compute_control(
1035 acc64Val = createMacConfOp(
1036 SmallVector<Value>{a_lo, b_lo, acc64Val},
1037 aiev2_vmac_compute_control(
1044 forceCastValueToType(rewriter, loc, acc64Val, op.getResult().getType());
1045 rewriter.replaceOp(op, resultVal);
1087 ConversionPatternRewriter &rewriter)
const {
1088 Location loc = op.getLoc();
1090 LLVM::ConstantOp::create(rewriter, loc, rewriter.getBF16Type(),
1091 rewriter.getZeroAttr(rewriter.getBF16Type()));
1092 auto aZeros = xllvm::VectorBroadcast16BF512IntrOp::create(
1093 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), zeroCst);
1094 auto bZeros = xllvm::VectorBroadcast16BF512IntrOp::create(
1095 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), zeroCst);
1096 auto cZeros = xllvm::VectorBroadcast16BF512IntrOp::create(
1097 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), zeroCst);
1098 auto dZeros = xllvm::VectorBroadcast16BF512IntrOp::create(
1099 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), zeroCst);
1100 auto eZeros = xllvm::VectorBroadcast16BF512IntrOp::create(
1101 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), zeroCst);
1102 auto fZeros = xllvm::VectorBroadcast16BF512IntrOp::create(
1103 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), zeroCst);
1105 LLVM::ConstantOp::create(rewriter, loc, rewriter.getBF16Type(),
1106 rewriter.getOneAttr(rewriter.getBF16Type()));
1107 auto dummy0 = xllvm::VectorBroadcast16BF512IntrOp::create(
1108 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), oneCst);
1109 auto zeroCstI32 = LLVM::ConstantOp::create(
1110 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(0));
1111 auto mscMacMulConfCst = LLVM::ConstantOp::create(
1112 rewriter, loc, rewriter.getI32Type(),
1113 rewriter.getI32IntegerAttr(aiev2_vmac_compute_control(
1118 auto extractV16FP32ToThreeV16BF16 =
1119 [&](Value inputV16FP32, Value aZeros, Value bZeros,
1120 Value cZeros) -> std::tuple<Value, Value, Value> {
1122 auto inputBitCasted =
1123 forceCastValueToType(rewriter, loc, inputV16FP32,
1124 VectorType::get({8}, rewriter.getI64Type()));
1125 auto v1ToBF16 = xllvm::Vector16AccFloatToV16BF16AIE2IntrOp::create(
1126 rewriter, loc, VectorType::get({16}, rewriter.getBF16Type()),
1128 auto a = xllvm::UpdBF512BF256IntrOp::create(
1129 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), aZeros,
1130 v1ToBF16, zeroCstI32);
1133 auto acc0 = xllvm::MscConfBF16IntrOp::create(
1134 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()), a, dummy0,
1135 inputBitCasted, mscMacMulConfCst);
1138 auto acc0ToBF16 = xllvm::Vector16AccFloatToV16BF16AIE2IntrOp::create(
1139 rewriter, loc, VectorType::get({16}, rewriter.getBF16Type()), acc0);
1140 auto b = xllvm::UpdBF512BF256IntrOp::create(
1141 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), bZeros,
1142 acc0ToBF16, zeroCstI32);
1145 auto acc0Mscb = xllvm::MscConfBF16IntrOp::create(
1146 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()), b, dummy0,
1147 acc0, mscMacMulConfCst);
1148 auto acc0MscbToBF16 = xllvm::Vector16AccFloatToV16BF16AIE2IntrOp::create(
1149 rewriter, loc, VectorType::get({16}, rewriter.getBF16Type()),
1151 auto c = xllvm::UpdBF512BF256IntrOp::create(
1152 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), cZeros,
1153 acc0MscbToBF16, zeroCstI32);
1154 return std::make_tuple(a.getResult(), b.getResult(), c.getResult());
1159 extractV16FP32ToThreeV16BF16(adaptor.getLhs(), aZeros, bZeros, cZeros);
1162 extractV16FP32ToThreeV16BF16(adaptor.getRhs(), dZeros, eZeros, fZeros);
1165 auto createMacOps = [&](Value lhs, Value rhs, Value acc) -> Value {
1166 return xllvm::MacConfBF16IntrOp::create(
1167 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()),
1168 lhs, rhs, acc, mscMacMulConfCst)
1182 auto afMul = xllvm::MulConfBF16IntrOp::create(
1183 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()), a, f,
1185 finalMacVal = createMacOps(
1190 createMacOps(d, c, createMacOps(b, e, afMul)))));
1200 auto bdMul = xllvm::MulConfBF16IntrOp::create(
1201 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()), b, d,
1203 finalMacVal = createMacOps(a, d, createMacOps(a, e, bdMul));
1209 auto cfMul = xllvm::MulConfBF16IntrOp::create(
1210 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()), c, f,
1212 finalMacVal = createMacOps(
1225 createMacOps(c, e, cfMul))))))));
1229 auto resultVal = forceCastValueToType(rewriter, loc, finalMacVal,
1230 op.getResult().getType());
1231 rewriter.replaceOp(op, resultVal);
1237 ConversionPatternRewriter &rewriter)
const override {
1238 Location loc = op.getLoc();
1242 op.emitWarning() <<
"aievec.mul_elem conversion is not supported.\n";
1250 if (decodedMulElemOp.kind ==
1256 auto confCst = LLVM::ConstantOp::create(
1257 rewriter, loc, rewriter.getI32Type(),
1258 rewriter.getI32IntegerAttr(decodedMulElemOp.conf));
1259 Value mulElemOp =
nullptr;
1260 SmallVector<Value> operands({adaptor.getLhs(), adaptor.getRhs(), confCst});
1265 mulElemOp = xllvm::MulConfAcc32IntrOp::create(
1266 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()),
1267 forceCastOperandsToSignature(
1268 rewriter, loc, operands,
1269 {VectorType::get({64}, rewriter.getI8Type()),
1270 VectorType::get({16}, rewriter.getI32Type()),
1271 rewriter.getI32Type()}));
1272 }
else if (decodedMulElemOp.kind ==
1276 auto zero32 = LLVM::ConstantOp::create(
1277 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(0));
1278 auto zeros_i16 = xllvm::VectorBroadcast16I512IntrOp::create(
1279 rewriter, loc, VectorType::get({32}, rewriter.getI16Type()), zero32);
1280 auto zeros_bf16 = LLVM::BitcastOp::create(
1281 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()),
1283 auto zeroVec = xllvm::ExtBF256BF512IntrOp::create(
1284 rewriter, loc, VectorType::get({16}, rewriter.getBF16Type()),
1285 zeros_bf16, zero32);
1288 auto idx1 = LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1289 rewriter.getI32IntegerAttr(1));
1292 auto lhsSet = xllvm::VectorSetBF512BF256IntrOp::create(
1293 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()),
1294 adaptor.getLhs(), zero32);
1295 auto lhsConcat = xllvm::UpdBF512BF256IntrOp::create(
1296 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), lhsSet,
1300 auto rhsSet = xllvm::VectorSetBF512BF256IntrOp::create(
1301 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()),
1302 adaptor.getRhs(), zero32);
1303 auto rhsConcat = xllvm::UpdBF512BF256IntrOp::create(
1304 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()), rhsSet,
1308 mulElemOp = xllvm::MulConfBF16IntrOp::create(
1309 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()), lhsConcat,
1310 rhsConcat, confCst);
1314 auto resultVal = forceCastValueToType(rewriter, loc, mulElemOp,
1315 op.getResult().getType());
1316 rewriter.replaceOp(op, resultVal);
1323 :
public mlir::ConvertOpToLLVMPattern<aievec::MulElemOp> {
1325 using ConvertOpToLLVMPattern<aievec::MulElemOp>::ConvertOpToLLVMPattern;
1339 auto lhs = op.getLhs();
1340 auto lhsVecTy = cast<VectorType>(lhs.getType());
1341 auto lhsScaTy = lhsVecTy.getElementType();
1342 unsigned lhsBitWidth = lhsScaTy.getIntOrFloatBitWidth();
1346 if (llvm::isa<IntegerType>(lhsScaTy)) {
1349 if (lhsBitWidth == 16) {
1351 if (lhsLanes == 16) {
1355 if (lhsLanes == 32) {
1359 if (lhsLanes == 64) {
1370 ConversionPatternRewriter &rewriter)
const override {
1371 Location loc = op.getLoc();
1375 op.emitWarning() <<
"aievec.mul_elem conversion is not supported for "
1381 auto confCst = LLVM::ConstantOp::create(
1382 rewriter, loc, rewriter.getI32Type(),
1383 rewriter.getI32IntegerAttr(decodedMulElemOp.conf));
1385 Value mulElemOp =
nullptr;
1388 if (decodedMulElemOp.kind ==
1394 SmallVector<int64_t> padMask;
1395 for (
int i = 0; i < 16; ++i)
1396 padMask.push_back(i);
1397 for (
int i = 16; i < 32; ++i)
1398 padMask.push_back(-1);
1400 auto lhsPadded = vector::ShuffleOp::create(
1401 rewriter, loc, adaptor.getLhs(), adaptor.getLhs(), padMask);
1402 auto rhsPadded = vector::ShuffleOp::create(
1403 rewriter, loc, adaptor.getRhs(), adaptor.getRhs(), padMask);
1405 SmallVector<Value> operands({lhsPadded, rhsPadded, confCst});
1408 mulElemOp = xllvm::MulConfBF16I512ACC512AIE2pIntrOp::create(
1409 rewriter, loc, VectorType::get({16}, rewriter.getF32Type()),
1410 forceCastOperandsToSignature(
1411 rewriter, loc, operands,
1412 {VectorType::get({32}, rewriter.getBF16Type()),
1413 VectorType::get({32}, rewriter.getBF16Type()),
1414 rewriter.getI32Type()}));
1415 }
else if (decodedMulElemOp.kind ==
1418 SmallVector<Value> operands(
1419 {adaptor.getLhs(), adaptor.getRhs(), confCst});
1420 mulElemOp = xllvm::MulConfBF16I512ACC1024AIE2pIntrOp::create(
1421 rewriter, loc, VectorType::get({32}, rewriter.getF32Type()),
1422 forceCastOperandsToSignature(
1423 rewriter, loc, operands,
1424 {VectorType::get({32}, rewriter.getBF16Type()),
1425 VectorType::get({32}, rewriter.getBF16Type()),
1426 rewriter.getI32Type()}));
1427 }
else if (decodedMulElemOp.kind ==
1430 SmallVector<Value> operands(
1431 {adaptor.getLhs(), adaptor.getRhs(), confCst});
1432 mulElemOp = xllvm::MulConfBF16I1024ACC2048AIE2pIntrOp::create(
1433 rewriter, loc, VectorType::get({64}, rewriter.getF32Type()),
1434 forceCastOperandsToSignature(
1435 rewriter, loc, operands,
1436 {VectorType::get({64}, rewriter.getBF16Type()),
1437 VectorType::get({64}, rewriter.getBF16Type()),
1438 rewriter.getI32Type()}));
1442 auto resultVal = forceCastValueToType(rewriter, loc, mulElemOp,
1443 op.getResult().getType());
1444 rewriter.replaceOp(op, resultVal);
1452 :
public mlir::ConvertOpToLLVMPattern<aievec::FMAElemOp> {
1454 using ConvertOpToLLVMPattern<aievec::FMAElemOp>::ConvertOpToLLVMPattern;
1458 ConversionPatternRewriter &rewriter)
const override {
1459 auto loc = fmaOp.getLoc();
1460 auto lhs = adaptor.getLhs();
1461 auto rhs = adaptor.getRhs();
1462 auto acc = adaptor.getAcc();
1463 auto lhsTy = cast<VectorType>(lhs.getType());
1464 auto accTy = cast<VectorType>(acc.getType());
1469 if (lhsTy != flatLhsTy)
1470 lhs = vector::ShapeCastOp::create(rewriter, loc, flatLhsTy, lhs);
1471 if (cast<VectorType>(rhs.getType()) != flatLhsTy)
1472 rhs = vector::ShapeCastOp::create(rewriter, loc, flatLhsTy, rhs);
1473 if (accTy != flatAccTy)
1474 acc = vector::ShapeCastOp::create(rewriter, loc, flatAccTy, acc);
1476 if (!flatLhsTy.getElementType().isBF16()) {
1478 <<
"aievec.mac_elem AIE2p conversion only supports bf16 inputs.\n";
1482 Type i32ty = rewriter.getI32Type();
1483 auto confCst = LLVM::ConstantOp::create(
1484 rewriter, loc, i32ty,
1485 rewriter.getI32IntegerAttr(aiev2_vmac_compute_control(
1491 unsigned lhsLanes = flatLhsTy.getNumElements();
1492 Value macIntrOp =
nullptr;
1494 if (lhsLanes == 16) {
1496 SmallVector<int64_t> padMask;
1497 for (
int i = 0; i < 16; ++i)
1498 padMask.push_back(i);
1499 for (
int i = 16; i < 32; ++i)
1500 padMask.push_back(-1);
1503 vector::ShuffleOp::create(rewriter, loc, lhs, lhs, padMask);
1505 vector::ShuffleOp::create(rewriter, loc, rhs, rhs, padMask);
1507 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
1508 auto v16f32Ty = VectorType::get({16}, rewriter.getF32Type());
1509 macIntrOp = xllvm::MacConfBF16I512ACC512AIE2pIntrOp::create(
1510 rewriter, loc, v16f32Ty,
1511 forceCastOperandsToSignature(
1512 rewriter, loc, {lhsPadded, rhsPadded, acc, confCst},
1513 {v32bf16Ty, v32bf16Ty, v16f32Ty, i32ty}));
1514 }
else if (lhsLanes == 32) {
1516 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
1517 auto v32f32Ty = VectorType::get({32}, rewriter.getF32Type());
1518 macIntrOp = xllvm::MacConfBF16I512ACC1024AIE2pIntrOp::create(
1519 rewriter, loc, v32f32Ty,
1520 forceCastOperandsToSignature(
1521 rewriter, loc, {lhs, rhs, acc, confCst},
1522 {v32bf16Ty, v32bf16Ty, v32f32Ty, i32ty}));
1525 <<
"aievec.mac_elem AIE2p conversion: unsupported lane count "
1526 << lhsLanes <<
".\n";
1531 auto resVal = forceCastValueToType(rewriter, loc, macIntrOp, flatAccTy);
1532 if (flatAccTy != accTy)
1533 resVal = vector::ShapeCastOp::create(rewriter, loc, accTy, resVal);
1535 rewriter.replaceOp(fmaOp, resVal);
1542 using ConvertOpToLLVMPattern<aievec::UPSOp>::ConvertOpToLLVMPattern;
1546 ConversionPatternRewriter &rewriter)
const override {
1547 Location loc = op.getLoc();
1549 Value result = op.getResult();
1550 VectorType resultType = cast<VectorType>(result.getType());
1552 Type resultScaTy = resultType.getElementType();
1553 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
1555 int resultVectorSize = resultBitWidth * resultLanes;
1557 Value opSrcVal = adaptor.getSource();
1558 auto srcVecTy = cast<VectorType>(opSrcVal.getType());
1560 if (srcVecTy != fltSrcVecTy)
1561 opSrcVal = vector::ShapeCastOp::create(rewriter, op.getLoc(), fltSrcVecTy,
1567 Value upsIntrOp =
nullptr;
1568 if (llvm::isa<IntegerType>(resultScaTy)) {
1570 auto signCst = LLVM::ConstantOp::create(
1571 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(1));
1573 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1574 rewriter.getI32IntegerAttr(op.getShift()));
1576 SmallVector<Value> operands({opSrcVal, shiftCst, signCst});
1577 if (resultVectorSize == 512) {
1578 if (resultBitWidth == 32) {
1580 upsIntrOp = xllvm::Acc32V16I256UpsAIE2IntrOp::create(
1581 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()),
1582 forceCastOperandsToSignature(
1583 rewriter, loc, operands,
1584 {VectorType::get({16}, rewriter.getI16Type()),
1585 rewriter.getI32Type(), rewriter.getI32Type()}));
1586 }
else if (resultBitWidth == 64) {
1588 upsIntrOp = xllvm::Acc64V8I256UpsAIE2IntrOp::create(
1589 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()),
1590 forceCastOperandsToSignature(
1591 rewriter, loc, operands,
1592 {VectorType::get({8}, rewriter.getI32Type()),
1593 rewriter.getI32Type(), rewriter.getI32Type()}));
1595 }
else if (resultVectorSize == 1024) {
1596 Value src = opSrcVal;
1597 VectorType srcType = cast<VectorType>(src.getType());
1598 Type srcScaType = srcType.getElementType();
1599 unsigned srcBitWidth = srcScaType.getIntOrFloatBitWidth();
1601 if (resultBitWidth == 32 && srcBitWidth == 16) {
1603 upsIntrOp = xllvm::Acc32V32I512UpsAIE2IntrOp::create(
1604 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()),
1605 forceCastOperandsToSignature(
1606 rewriter, loc, operands,
1607 {VectorType::get({32}, rewriter.getI16Type()),
1608 rewriter.getI32Type(), rewriter.getI32Type()}));
1609 }
else if (resultBitWidth == 64 && srcBitWidth == 32) {
1611 upsIntrOp = xllvm::Acc64V16I512UpsAIE2IntrOp::create(
1612 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()),
1613 forceCastOperandsToSignature(
1614 rewriter, loc, operands,
1615 {VectorType::get({16}, rewriter.getI32Type()),
1616 rewriter.getI32Type(), rewriter.getI32Type()}));
1617 }
else if (resultBitWidth == 64 && srcBitWidth == 16) {
1619 upsIntrOp = xllvm::Acc64V16I256UpsAIE2IntrOp::create(
1620 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()),
1621 forceCastOperandsToSignature(
1622 rewriter, loc, operands,
1623 {VectorType::get({16}, rewriter.getI16Type()),
1624 rewriter.getI32Type(), rewriter.getI32Type()}));
1625 }
else if (resultBitWidth == 32 && srcBitWidth == 8) {
1627 upsIntrOp = xllvm::Acc32V32I256UpsAIE2IntrOp::create(
1628 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()),
1629 forceCastOperandsToSignature(
1630 rewriter, loc, operands,
1631 {VectorType::get({32}, rewriter.getI8Type()),
1632 rewriter.getI32Type(), rewriter.getI32Type()}));
1638 if (resultVectorSize == 512) {
1640 upsIntrOp = xllvm::Vector16BF16ToV16AccFloatAIE2IntrOp::create(
1641 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()),
1642 forceCastOperandsToSignature(
1643 rewriter, loc, {opSrcVal},
1644 {VectorType::get({16}, rewriter.getBF16Type())}));
1645 }
else if (resultVectorSize == 1024) {
1654 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1655 rewriter.getI32IntegerAttr(0));
1657 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1658 rewriter.getI32IntegerAttr(1));
1659 auto extractUps = [&](Value source, Value index) -> Value {
1660 auto extOp = xllvm::ExtI256I512IntrOp::create(
1661 rewriter, loc, VectorType::get({8}, rewriter.getI32Type()),
1662 forceCastOperandsToSignature(
1663 rewriter, loc, {source, index},
1664 {VectorType::get({16}, rewriter.getI32Type()),
1665 rewriter.getI32Type()}));
1666 return xllvm::Vector16BF16ToV16AccFloatAIE2IntrOp::create(
1667 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()),
1668 forceCastOperandsToSignature(
1669 rewriter, loc, {extOp},
1670 {VectorType::get({16}, rewriter.getBF16Type())}));
1672 auto resLo = extractUps(opSrcVal, indexZeroCst);
1673 auto resHi = extractUps(opSrcVal, indexOneCst);
1676 upsIntrOp = xllvm::ConcatI1024I512IntrOp::create(
1677 rewriter, loc, VectorType::get({32}, rewriter.getI32Type()),
1678 forceCastOperandsToSignature(
1679 rewriter, loc, {resLo, resHi},
1680 {VectorType::get({16}, rewriter.getI32Type()),
1681 VectorType::get({16}, rewriter.getI32Type())}));
1686 op.emitWarning() <<
"aievec.ups is not supported.\n";
1691 if (flatResTy != upsIntrOp.getType())
1692 upsIntrOp = LLVM::BitcastOp::create(rewriter, loc, flatResTy, upsIntrOp);
1694 if (flatResTy != resultType)
1696 vector::ShapeCastOp::create(rewriter, loc, resultType, upsIntrOp);
1698 rewriter.replaceOp(op, upsIntrOp);
1706 :
public mlir::ConvertOpToLLVMPattern<aievec::UPSOp> {
1708 using ConvertOpToLLVMPattern<aievec::UPSOp>::ConvertOpToLLVMPattern;
1712 ConversionPatternRewriter &rewriter)
const override {
1713 Location loc = op.getLoc();
1715 Value result = op.getResult();
1716 VectorType resultType = cast<VectorType>(result.getType());
1718 Type resultScaTy = resultType.getElementType();
1719 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
1721 int resultVectorSize = resultBitWidth * resultLanes;
1723 Value opSrcVal = adaptor.getSource();
1724 auto srcVecTy = cast<VectorType>(opSrcVal.getType());
1726 if (srcVecTy != fltSrcVecTy)
1727 opSrcVal = vector::ShapeCastOp::create(rewriter, op.getLoc(), fltSrcVecTy,
1733 Value upsIntrOp =
nullptr;
1734 if (llvm::isa<IntegerType>(resultScaTy)) {
1736 auto signCst = LLVM::ConstantOp::create(
1737 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(1));
1739 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1740 rewriter.getI32IntegerAttr(op.getShift()));
1742 SmallVector<Value> operands({opSrcVal, shiftCst, signCst});
1743 if (resultVectorSize == 512) {
1744 if (resultBitWidth == 32) {
1746 upsIntrOp = xllvm::Acc32V16I256UpsAIE2pIntrOp::create(
1747 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
1748 forceCastOperandsToSignature(
1749 rewriter, loc, operands,
1750 {VectorType::get({16}, rewriter.getI16Type()),
1751 rewriter.getI32Type(), rewriter.getI32Type()}));
1752 }
else if (resultBitWidth == 64) {
1754 upsIntrOp = xllvm::Acc64V8I256UpsAIE2pIntrOp::create(
1755 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()),
1756 forceCastOperandsToSignature(
1757 rewriter, loc, operands,
1758 {VectorType::get({8}, rewriter.getI32Type()),
1759 rewriter.getI32Type(), rewriter.getI32Type()}));
1761 }
else if (resultVectorSize == 1024) {
1762 Value src = opSrcVal;
1763 VectorType srcType = cast<VectorType>(src.getType());
1764 Type srcScaType = srcType.getElementType();
1765 unsigned srcBitWidth = srcScaType.getIntOrFloatBitWidth();
1767 int srcVectorSize = srcBitWidth * srcLanes;
1769 if (resultBitWidth == 32 && srcBitWidth == 16 && srcVectorSize == 512) {
1771 upsIntrOp = xllvm::Acc32V32I512UpsAIE2pIntrOp::create(
1772 rewriter, loc, VectorType::get({32}, rewriter.getI32Type()),
1773 forceCastOperandsToSignature(
1774 rewriter, loc, operands,
1775 {VectorType::get({32}, rewriter.getI16Type()),
1776 rewriter.getI32Type(), rewriter.getI32Type()}));
1777 }
else if (resultBitWidth == 64 && srcBitWidth == 32 &&
1778 srcVectorSize == 512) {
1780 upsIntrOp = xllvm::Acc64V16I512UpsAIE2pIntrOp::create(
1781 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()),
1782 forceCastOperandsToSignature(
1783 rewriter, loc, operands,
1784 {VectorType::get({16}, rewriter.getI32Type()),
1785 rewriter.getI32Type(), rewriter.getI32Type()}));
1786 }
else if (resultBitWidth == 64 && srcBitWidth == 16 &&
1787 srcVectorSize == 256) {
1789 upsIntrOp = xllvm::Acc64V16I256UpsAIE2pIntrOp::create(
1790 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()),
1791 forceCastOperandsToSignature(
1792 rewriter, loc, operands,
1793 {VectorType::get({16}, rewriter.getI16Type()),
1794 rewriter.getI32Type(), rewriter.getI32Type()}));
1795 }
else if (resultBitWidth == 32 && srcBitWidth == 8 &&
1796 srcVectorSize == 256) {
1798 upsIntrOp = xllvm::Acc32V32I256UpsAIE2pIntrOp::create(
1799 rewriter, loc, VectorType::get({32}, rewriter.getI32Type()),
1800 forceCastOperandsToSignature(
1801 rewriter, loc, operands,
1802 {VectorType::get({32}, rewriter.getI8Type()),
1803 rewriter.getI32Type(), rewriter.getI32Type()}));
1805 }
else if (resultVectorSize == 2048) {
1806 Value src = opSrcVal;
1807 VectorType srcType = cast<VectorType>(src.getType());
1808 Type srcScaType = srcType.getElementType();
1809 unsigned srcBitWidth = srcScaType.getIntOrFloatBitWidth();
1811 int srcVectorSize = srcBitWidth * srcLanes;
1813 if (resultBitWidth == 32 && srcBitWidth == 8 && srcVectorSize == 512) {
1815 upsIntrOp = xllvm::Acc32V64I512UpsAIE2pIntrOp::create(
1816 rewriter, loc, VectorType::get({64}, rewriter.getI32Type()),
1817 forceCastOperandsToSignature(
1818 rewriter, loc, operands,
1819 {VectorType::get({64}, rewriter.getI8Type()),
1820 rewriter.getI32Type(), rewriter.getI32Type()}));
1821 }
else if (resultBitWidth == 64 && srcBitWidth == 16 &&
1822 srcVectorSize == 512) {
1824 upsIntrOp = xllvm::Acc64V32I512UpsAIE2pIntrOp::create(
1825 rewriter, loc, VectorType::get({32}, rewriter.getI64Type()),
1826 forceCastOperandsToSignature(
1827 rewriter, loc, operands,
1828 {VectorType::get({32}, rewriter.getI16Type()),
1829 rewriter.getI32Type(), rewriter.getI32Type()}));
1830 }
else if (resultBitWidth == 32 && srcBitWidth == 16 &&
1831 srcVectorSize == 1024) {
1835 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1836 rewriter.getI32IntegerAttr(0));
1838 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1839 rewriter.getI32IntegerAttr(1));
1841 auto extractUps2048 = [&](Value source, Value index, Value shiftCst,
1842 Value signCst) -> Value {
1845 auto v32i32Source = forceCastValueToType(
1846 rewriter, loc, source,
1847 VectorType::get({32}, rewriter.getI32Type()));
1852 SmallVector<int64_t> shuffleMask;
1853 if (
auto constIndex = index.getDefiningOp<LLVM::ConstantOp>()) {
1854 auto indexAttr = cast<IntegerAttr>(constIndex.getValue());
1855 int64_t idxVal = indexAttr.getInt();
1856 int startIdx = idxVal * 16;
1857 for (
int i = 0; i < 16; ++i) {
1858 shuffleMask.push_back(startIdx + i);
1862 for (
int i = 0; i < 16; ++i) {
1863 shuffleMask.push_back(i);
1867 auto extOp = vector::ShuffleOp::create(rewriter, loc, v32i32Source,
1868 v32i32Source, shuffleMask);
1870 return xllvm::Acc32V32I512UpsAIE2pIntrOp::create(
1871 rewriter, loc, VectorType::get({32}, rewriter.getI32Type()),
1872 forceCastOperandsToSignature(
1873 rewriter, loc, {extOp, shiftCst, signCst},
1874 {VectorType::get({32}, rewriter.getI16Type()),
1875 rewriter.getI32Type(), rewriter.getI32Type()}));
1878 auto res0 = extractUps2048(opSrcVal, index0Cst, shiftCst, signCst);
1879 auto res1 = extractUps2048(opSrcVal, index1Cst, shiftCst, signCst);
1883 SmallVector<int64_t> concatMask;
1884 for (
int i = 0; i < 64; ++i) {
1885 concatMask.push_back(i);
1888 vector::ShuffleOp::create(rewriter, loc, res0, res1, concatMask);
1894 if (resultVectorSize == 512) {
1896 upsIntrOp = xllvm::Vector16BF16ToV16AccFloatAIE2pIntrOp::create(
1897 rewriter, loc, VectorType::get({16}, rewriter.getF32Type()),
1898 forceCastOperandsToSignature(
1899 rewriter, loc, {opSrcVal},
1900 {VectorType::get({16}, rewriter.getBF16Type())}));
1901 }
else if (resultVectorSize == 1024) {
1903 upsIntrOp = xllvm::Vector32BF16ToV32AccFloatAIE2pIntrOp::create(
1904 rewriter, loc, VectorType::get({32}, rewriter.getF32Type()),
1905 forceCastOperandsToSignature(
1906 rewriter, loc, {opSrcVal},
1907 {VectorType::get({32}, rewriter.getBF16Type())}));
1908 }
else if (resultVectorSize == 2048) {
1912 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1913 rewriter.getI32IntegerAttr(0));
1915 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
1916 rewriter.getI32IntegerAttr(1));
1918 auto extractUps2048 = [&](Value source, Value index) -> Value {
1921 auto v32i32Source = forceCastValueToType(
1922 rewriter, loc, source,
1923 VectorType::get({32}, rewriter.getI32Type()));
1928 SmallVector<int64_t> shuffleMask;
1929 if (
auto constIndex = index.getDefiningOp<LLVM::ConstantOp>()) {
1930 auto indexAttr = cast<IntegerAttr>(constIndex.getValue());
1931 int64_t idxVal = indexAttr.getInt();
1932 int startIdx = idxVal * 16;
1933 for (
int i = 0; i < 16; ++i) {
1934 shuffleMask.push_back(startIdx + i);
1938 for (
int i = 0; i < 16; ++i) {
1939 shuffleMask.push_back(i);
1943 auto extOp = vector::ShuffleOp::create(rewriter, loc, v32i32Source,
1944 v32i32Source, shuffleMask);
1946 return xllvm::Vector32BF16ToV32AccFloatAIE2pIntrOp::create(
1947 rewriter, loc, VectorType::get({32}, rewriter.getF32Type()),
1948 forceCastOperandsToSignature(
1949 rewriter, loc, {extOp},
1950 {VectorType::get({32}, rewriter.getBF16Type())}));
1953 auto res0 = extractUps2048(opSrcVal, index0Cst);
1954 auto res1 = extractUps2048(opSrcVal, index1Cst);
1958 auto v32i32Res0 = forceCastValueToType(
1959 rewriter, loc, res0, VectorType::get({32}, rewriter.getI32Type()));
1960 auto v32i32Res1 = forceCastValueToType(
1961 rewriter, loc, res1, VectorType::get({32}, rewriter.getI32Type()));
1963 SmallVector<int64_t> concatMask;
1964 for (
int i = 0; i < 64; ++i) {
1965 concatMask.push_back(i);
1967 upsIntrOp = vector::ShuffleOp::create(rewriter, loc, v32i32Res0,
1968 v32i32Res1, concatMask);
1973 op.emitWarning() <<
"aievec.ups is not supported.\n";
1978 if (flatResTy != upsIntrOp.getType())
1979 upsIntrOp = LLVM::BitcastOp::create(rewriter, loc, flatResTy, upsIntrOp);
1981 if (flatResTy != resultType)
1983 vector::ShapeCastOp::create(rewriter, loc, resultType, upsIntrOp);
1985 rewriter.replaceOp(op, upsIntrOp);
1993 using ConvertOpToLLVMPattern<aievec::SRSOp>::ConvertOpToLLVMPattern;
1997 ConversionPatternRewriter &rewriter)
const override {
1998 Location loc = op.getLoc();
2000 Value result = op.getResult();
2001 VectorType resultType = cast<VectorType>(result.getType());
2002 Type resultScaTy = resultType.getElementType();
2003 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
2005 int resultVectorSize = resultBitWidth * resultLanes;
2008 Value srsIntrOp =
nullptr;
2009 if (llvm::isa<IntegerType>(resultScaTy)) {
2012 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2013 rewriter.getI32IntegerAttr(op.getSign()));
2016 SmallVector<Value> operands(
2017 {adaptor.getSource(), adaptor.getShift(), signCst});
2018 if (resultVectorSize == 512) {
2019 if (resultBitWidth == 16) {
2020 srsIntrOp = xllvm::I512V32Acc32SrsAIE2IntrOp::create(
2021 rewriter, loc, VectorType::get({32}, rewriter.getI16Type()),
2022 forceCastOperandsToSignature(
2023 rewriter, loc, operands,
2024 {VectorType::get({16}, rewriter.getI64Type()),
2025 rewriter.getI32Type(), rewriter.getI32Type()}));
2026 }
else if (resultBitWidth == 32) {
2027 srsIntrOp = xllvm::I512V16Acc64SrsAIE2IntrOp::create(
2028 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
2029 forceCastOperandsToSignature(
2030 rewriter, loc, operands,
2031 {VectorType::get({16}, rewriter.getI64Type()),
2032 rewriter.getI32Type(), rewriter.getI32Type()}));
2034 }
else if (resultVectorSize == 256) {
2035 Value src = adaptor.getSource();
2036 VectorType srcType = cast<VectorType>(src.getType());
2037 Type srcScaType = srcType.getElementType();
2038 unsigned srcBitWidth = srcScaType.getIntOrFloatBitWidth();
2040 if (resultBitWidth == 16 && srcBitWidth == 32) {
2041 srsIntrOp = xllvm::I256V16Acc32SrsAIE2IntrOp::create(
2042 rewriter, loc, VectorType::get({16}, rewriter.getI16Type()),
2043 forceCastOperandsToSignature(
2044 rewriter, loc, operands,
2045 {VectorType::get({8}, rewriter.getI64Type()),
2046 rewriter.getI32Type(), rewriter.getI32Type()}));
2047 }
else if (resultBitWidth == 8 && srcBitWidth == 32) {
2048 srsIntrOp = xllvm::I256V32Acc32SrsAIE2IntrOp::create(
2049 rewriter, loc, VectorType::get({32}, rewriter.getI8Type()),
2050 forceCastOperandsToSignature(
2051 rewriter, loc, operands,
2052 {VectorType::get({16}, rewriter.getI64Type()),
2053 rewriter.getI32Type(), rewriter.getI32Type()}));
2054 }
else if (resultBitWidth == 16 && srcBitWidth == 64) {
2055 srsIntrOp = xllvm::I256V16Acc64SrsAIE2IntrOp::create(
2056 rewriter, loc, VectorType::get({16}, rewriter.getI16Type()),
2057 forceCastOperandsToSignature(
2058 rewriter, loc, operands,
2059 {VectorType::get({16}, rewriter.getI64Type()),
2060 rewriter.getI32Type(), rewriter.getI32Type()}));
2061 }
else if (resultBitWidth == 32 && srcBitWidth == 64) {
2062 srsIntrOp = xllvm::I256V8Acc64SrsAIE2IntrOp::create(
2063 rewriter, loc, VectorType::get({8}, rewriter.getI32Type()),
2064 forceCastOperandsToSignature(
2065 rewriter, loc, operands,
2066 {VectorType::get({8}, rewriter.getI64Type()),
2067 rewriter.getI32Type(), rewriter.getI32Type()}));
2072 if (resultVectorSize == 256) {
2073 srsIntrOp = xllvm::Vector16AccFloatToV16BF16AIE2IntrOp::create(
2074 rewriter, loc, VectorType::get({16}, rewriter.getBF16Type()),
2075 forceCastOperandsToSignature(
2076 rewriter, loc, {adaptor.getSource()},
2077 {VectorType::get({8}, rewriter.getI64Type())}));
2078 }
else if (resultVectorSize == 512) {
2087 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2088 rewriter.getI32IntegerAttr(0));
2090 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2091 rewriter.getI32IntegerAttr(1));
2092 auto extractSrs = [&](Value source, Value index) -> Value {
2093 auto extOp = xllvm::ExtI512I1024IntrOp::create(
2094 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
2095 forceCastOperandsToSignature(
2096 rewriter, loc, {source, index},
2097 {VectorType::get({32}, rewriter.getI32Type()),
2098 rewriter.getI32Type()}));
2099 return xllvm::Vector16AccFloatToV16BF16AIE2IntrOp::create(
2100 rewriter, loc, VectorType::get({16}, rewriter.getBF16Type()),
2101 forceCastOperandsToSignature(
2102 rewriter, loc, {extOp},
2103 {VectorType::get({8}, rewriter.getI64Type())}));
2105 auto resLo = extractSrs(adaptor.getSource(), indexZeroCst);
2106 auto resHi = extractSrs(adaptor.getSource(), indexOneCst);
2109 srsIntrOp = xllvm::ConcatI512I256IntrOp::create(
2110 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
2111 forceCastOperandsToSignature(
2112 rewriter, loc, {resLo, resHi},
2113 {VectorType::get({8}, rewriter.getI32Type()),
2114 VectorType::get({8}, rewriter.getI32Type())}));
2119 op.emitWarning() <<
"aievec.srs is not supported.\n";
2124 auto resultVal = forceCastValueToType(rewriter, loc, srsIntrOp,
2125 op.getResult().getType());
2126 rewriter.replaceOp(op, resultVal);
2134 :
public mlir::ConvertOpToLLVMPattern<aievec::SRSOp> {
2136 using ConvertOpToLLVMPattern<aievec::SRSOp>::ConvertOpToLLVMPattern;
2140 ConversionPatternRewriter &rewriter)
const override {
2141 Location loc = op.getLoc();
2143 Value result = op.getResult();
2144 VectorType resultType = cast<VectorType>(result.getType());
2145 Type resultScaTy = resultType.getElementType();
2146 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
2148 int resultVectorSize = resultBitWidth * resultLanes;
2151 Value srsIntrOp =
nullptr;
2152 if (llvm::isa<IntegerType>(resultScaTy)) {
2155 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2156 rewriter.getI32IntegerAttr(op.getSign()));
2159 SmallVector<Value> operands(
2160 {adaptor.getSource(), adaptor.getShift(), signCst});
2161 if (resultVectorSize == 512) {
2162 Value src = adaptor.getSource();
2163 VectorType srcType = cast<VectorType>(src.getType());
2164 Type srcScaType = srcType.getElementType();
2165 unsigned srcBitWidth = srcScaType.getIntOrFloatBitWidth();
2167 if (resultBitWidth == 16 && srcBitWidth == 32) {
2169 srsIntrOp = xllvm::I512V32Acc32SrsAIE2pIntrOp::create(
2170 rewriter, loc, VectorType::get({32}, rewriter.getI16Type()),
2171 forceCastOperandsToSignature(
2172 rewriter, loc, operands,
2173 {VectorType::get({32}, rewriter.getI32Type()),
2174 rewriter.getI32Type(), rewriter.getI32Type()}));
2175 }
else if (resultBitWidth == 16 && srcBitWidth == 64) {
2177 srsIntrOp = xllvm::I512V32Acc64SrsAIE2pIntrOp::create(
2178 rewriter, loc, VectorType::get({32}, rewriter.getI16Type()),
2179 forceCastOperandsToSignature(
2180 rewriter, loc, operands,
2181 {VectorType::get({32}, rewriter.getI64Type()),
2182 rewriter.getI32Type(), rewriter.getI32Type()}));
2183 }
else if (resultBitWidth == 32 && srcBitWidth == 64) {
2185 srsIntrOp = xllvm::I512V16Acc64SrsAIE2pIntrOp::create(
2186 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
2187 forceCastOperandsToSignature(
2188 rewriter, loc, operands,
2189 {VectorType::get({16}, rewriter.getI64Type()),
2190 rewriter.getI32Type(), rewriter.getI32Type()}));
2191 }
else if (resultBitWidth == 8 && srcBitWidth == 32) {
2193 srsIntrOp = xllvm::I512V64Acc32SrsAIE2pIntrOp::create(
2194 rewriter, loc, VectorType::get({64}, rewriter.getI8Type()),
2195 forceCastOperandsToSignature(
2196 rewriter, loc, operands,
2197 {VectorType::get({64}, rewriter.getI32Type()),
2198 rewriter.getI32Type(), rewriter.getI32Type()}));
2200 }
else if (resultVectorSize == 256) {
2201 Value src = adaptor.getSource();
2202 VectorType srcType = cast<VectorType>(src.getType());
2203 Type srcScaType = srcType.getElementType();
2204 unsigned srcBitWidth = srcScaType.getIntOrFloatBitWidth();
2206 if (resultBitWidth == 16 && srcBitWidth == 32) {
2208 srsIntrOp = xllvm::I256V16Acc32SrsAIE2pIntrOp::create(
2209 rewriter, loc, VectorType::get({16}, rewriter.getI16Type()),
2210 forceCastOperandsToSignature(
2211 rewriter, loc, operands,
2212 {VectorType::get({16}, rewriter.getI32Type()),
2213 rewriter.getI32Type(), rewriter.getI32Type()}));
2214 }
else if (resultBitWidth == 8 && srcBitWidth == 32) {
2216 srsIntrOp = xllvm::I256V32Acc32SrsAIE2pIntrOp::create(
2217 rewriter, loc, VectorType::get({32}, rewriter.getI8Type()),
2218 forceCastOperandsToSignature(
2219 rewriter, loc, operands,
2220 {VectorType::get({32}, rewriter.getI32Type()),
2221 rewriter.getI32Type(), rewriter.getI32Type()}));
2222 }
else if (resultBitWidth == 16 && srcBitWidth == 64) {
2224 srsIntrOp = xllvm::I256V16Acc64SrsAIE2pIntrOp::create(
2225 rewriter, loc, VectorType::get({16}, rewriter.getI16Type()),
2226 forceCastOperandsToSignature(
2227 rewriter, loc, operands,
2228 {VectorType::get({16}, rewriter.getI64Type()),
2229 rewriter.getI32Type(), rewriter.getI32Type()}));
2230 }
else if (resultBitWidth == 32 && srcBitWidth == 64) {
2232 srsIntrOp = xllvm::I256V8Acc64SrsAIE2pIntrOp::create(
2233 rewriter, loc, VectorType::get({8}, rewriter.getI32Type()),
2234 forceCastOperandsToSignature(
2235 rewriter, loc, operands,
2236 {VectorType::get({8}, rewriter.getI64Type()),
2237 rewriter.getI32Type(), rewriter.getI32Type()}));
2239 }
else if (resultVectorSize == 1024) {
2240 Value src = adaptor.getSource();
2241 VectorType srcType = cast<VectorType>(src.getType());
2242 Type srcScaType = srcType.getElementType();
2243 unsigned srcBitWidth = srcScaType.getIntOrFloatBitWidth();
2245 if (resultBitWidth == 16 && srcBitWidth == 32) {
2249 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2250 rewriter.getI32IntegerAttr(0));
2252 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2253 rewriter.getI32IntegerAttr(1));
2255 auto extractSrs1024 = [&](Value source, Value index, Value shiftCst,
2256 Value signCst) -> Value {
2259 auto v64i32Source = forceCastValueToType(
2260 rewriter, loc, source,
2261 VectorType::get({64}, rewriter.getI32Type()));
2266 SmallVector<int64_t> shuffleMask;
2267 if (
auto constIndex = index.getDefiningOp<LLVM::ConstantOp>()) {
2268 auto indexAttr = cast<IntegerAttr>(constIndex.getValue());
2269 int64_t idxVal = indexAttr.getInt();
2270 int startIdx = idxVal * 32;
2271 for (
int i = 0; i < 32; ++i) {
2272 shuffleMask.push_back(startIdx + i);
2276 for (
int i = 0; i < 32; ++i) {
2277 shuffleMask.push_back(i);
2281 auto extOp = vector::ShuffleOp::create(rewriter, loc, v64i32Source,
2282 v64i32Source, shuffleMask);
2284 return xllvm::I512V32Acc32SrsAIE2pIntrOp::create(
2285 rewriter, loc, VectorType::get({32}, rewriter.getI16Type()),
2286 forceCastOperandsToSignature(
2287 rewriter, loc, {extOp, shiftCst, signCst},
2288 {VectorType::get({32}, rewriter.getI32Type()),
2289 rewriter.getI32Type(), rewriter.getI32Type()}));
2293 extractSrs1024(src, index0Cst, adaptor.getShift(), signCst);
2295 extractSrs1024(src, index1Cst, adaptor.getShift(), signCst);
2299 auto v16i32Res0 = forceCastValueToType(
2300 rewriter, loc, res0,
2301 VectorType::get({16}, rewriter.getI32Type()));
2302 auto v16i32Res1 = forceCastValueToType(
2303 rewriter, loc, res1,
2304 VectorType::get({16}, rewriter.getI32Type()));
2306 SmallVector<int64_t> concatMask;
2307 for (
int i = 0; i < 32; ++i) {
2308 concatMask.push_back(i);
2310 srsIntrOp = vector::ShuffleOp::create(rewriter, loc, v16i32Res0,
2311 v16i32Res1, concatMask);
2317 if (resultVectorSize == 256) {
2319 srsIntrOp = xllvm::Vector16AccFloatToV16BF16AIE2pIntrOp::create(
2320 rewriter, loc, VectorType::get({16}, rewriter.getBF16Type()),
2321 forceCastOperandsToSignature(
2322 rewriter, loc, {adaptor.getSource()},
2323 {VectorType::get({16}, rewriter.getF32Type())}));
2324 }
else if (resultVectorSize == 512) {
2326 srsIntrOp = xllvm::Vector32AccFloatToV32BF16AIE2pIntrOp::create(
2327 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()),
2328 forceCastOperandsToSignature(
2329 rewriter, loc, {adaptor.getSource()},
2330 {VectorType::get({32}, rewriter.getF32Type())}));
2331 }
else if (resultVectorSize == 1024) {
2335 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2336 rewriter.getI32IntegerAttr(0));
2338 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2339 rewriter.getI32IntegerAttr(1));
2341 auto extractSrs1024 = [&](Value source, Value index) -> Value {
2344 auto v64i32Source = forceCastValueToType(
2345 rewriter, loc, source,
2346 VectorType::get({64}, rewriter.getI32Type()));
2351 SmallVector<int64_t> shuffleMask;
2352 if (
auto constIndex = index.getDefiningOp<LLVM::ConstantOp>()) {
2353 auto indexAttr = cast<IntegerAttr>(constIndex.getValue());
2354 int64_t idxVal = indexAttr.getInt();
2355 int startIdx = idxVal * 32;
2356 for (
int i = 0; i < 32; ++i) {
2357 shuffleMask.push_back(startIdx + i);
2361 for (
int i = 0; i < 32; ++i) {
2362 shuffleMask.push_back(i);
2366 auto extOp = vector::ShuffleOp::create(rewriter, loc, v64i32Source,
2367 v64i32Source, shuffleMask);
2369 return xllvm::Vector32AccFloatToV32BF16AIE2pIntrOp::create(
2370 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()),
2371 forceCastOperandsToSignature(
2372 rewriter, loc, {extOp},
2373 {VectorType::get({32}, rewriter.getF32Type())}));
2376 auto res0 = extractSrs1024(adaptor.getSource(), index0Cst);
2377 auto res1 = extractSrs1024(adaptor.getSource(), index1Cst);
2381 auto v16i32Res0 = forceCastValueToType(
2382 rewriter, loc, res0, VectorType::get({16}, rewriter.getI32Type()));
2383 auto v16i32Res1 = forceCastValueToType(
2384 rewriter, loc, res1, VectorType::get({16}, rewriter.getI32Type()));
2386 SmallVector<int64_t> concatMask;
2387 for (
int i = 0; i < 32; ++i) {
2388 concatMask.push_back(i);
2390 srsIntrOp = vector::ShuffleOp::create(rewriter, loc, v16i32Res0,
2391 v16i32Res1, concatMask);
2396 op.emitWarning() <<
"aievec.srs is not supported.\n";
2401 auto resultVal = forceCastValueToType(rewriter, loc, srsIntrOp,
2402 op.getResult().getType());
2403 rewriter.replaceOp(op, resultVal);
2411 using ConvertOpToLLVMPattern<aievec::UPDOp>::ConvertOpToLLVMPattern;
2414 auto resultType = cast<VectorType>(op.getResult().getType());
2415 std::stringstream ss;
2416 ss <<
"llvm.aie.upd.";
2417 ss << (loadSize == 128 ?
'v' : loadSize == 256 ?
'w' :
'x') <<
".";
2418 ss << getVectorTypeString(resultType) <<
".";
2420 ss << (op.getIndex() == 0 ?
"lo" :
"hi");
2426 ConversionPatternRewriter &rewriter)
const override {
2427 auto module = op->getParentOfType<ModuleOp>();
2428 MLIRContext *context = rewriter.getContext();
2436 auto ptr = this->getStridedElementPtr(
2437 rewriter, op->getLoc(), cast<MemRefType>(op.getSource().getType()),
2438 adaptor.getSource(), adaptor.getIndices());
2442 if (vecSizeInBits <= 256) {
2446 auto vectorPtrType = LLVM::LLVMPointerType::get(
2448 cast<MemRefType>(op.getSource().getType()).getMemorySpaceAsInt());
2450 LLVM::BitcastOp::create(rewriter, op->getLoc(), vectorPtrType, ptr);
2451 auto vecType = cast<VectorType>(op.getResult().getType());
2452 rewriter.replaceOpWithNewOp<LLVM::LoadOp>(op, vecType, castedPtr, 1);
2460 int loadSize = vecSizeInBits == 256 ? 128
2461 : vecSizeInBits == 512 ? 256
2466 auto resultType = cast<VectorType>(op.getResult().getType());
2469 VectorType::get({(int64_t)lanes / 2}, resultType.getElementType());
2472 auto vectorPtrType = LLVM::LLVMPointerType::get(
2474 cast<MemRefType>(op.getSource().getType()).getMemorySpaceAsInt());
2476 LLVM::BitcastOp::create(rewriter, op->getLoc(), vectorPtrType, ptr);
2478 LLVM::LoadOp::create(rewriter, op->getLoc(), loadType, castedPtr, 1);
2484 auto func =
module.lookupSymbol<LLVM::LLVMFuncOp>(
2485 StringAttr::get(context, intrinsicName));
2488 OpBuilder::InsertionGuard guard(rewriter);
2489 rewriter.setInsertionPointToStart(module.getBody());
2490 func = LLVM::LLVMFuncOp::create(
2491 rewriter, rewriter.getUnknownLoc(), intrinsicName,
2492 LLVM::LLVMFunctionType::get(resultType, {resultType, loadType}));
2497 if (adaptor.getVector()) {
2499 destValue = adaptor.getVector();
2508 std::stringstream ss;
2509 ss <<
"llvm.aie." << getVectorTypeString(resultType) <<
".undef";
2510 std::string intrinsicName = ss.str();
2512 auto func =
module.lookupSymbol<LLVM::LLVMFuncOp>(
2513 StringAttr::get(rewriter.getContext(), intrinsicName));
2516 OpBuilder::InsertionGuard guard(rewriter);
2517 rewriter.setInsertionPointToStart(module.getBody());
2518 func = LLVM::LLVMFuncOp::create(
2519 rewriter, rewriter.getUnknownLoc(), intrinsicName,
2520 LLVM::LLVMFunctionType::get(resultType, {}));
2523 LLVM::CallOp::create(rewriter, op->getLoc(),
func, ValueRange{})
2528 rewriter.replaceOpWithNewOp<LLVM::CallOp>(
2529 op,
func, ValueRange{destValue, loadValue});
2537 :
public mlir::ConvertOpToLLVMPattern<aievec::ConcatOp> {
2539 using ConvertOpToLLVMPattern<aievec::ConcatOp>::ConvertOpToLLVMPattern;
2543 ConversionPatternRewriter &rewriter)
const override {
2544 Location loc = op.getLoc();
2546 SmallVector<Value> sources = adaptor.getSources();
2547 Value src = sources.front();
2548 VectorType srcType = cast<VectorType>(src.getType());
2549 Type srcScalarType = srcType.getElementType();
2550 unsigned srcBitWidth = srcScalarType.getIntOrFloatBitWidth();
2552 int srcVectorSize = srcBitWidth * srcLanes;
2554 Value result = op.getResult();
2555 VectorType resultType = cast<VectorType>(result.getType());
2556 Type resultScaTy = resultType.getElementType();
2557 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
2559 int resultVectorSize = resultBitWidth * resultLanes;
2561 if (sources.size() != 2 && sources.size() != 4) {
2562 op.emitWarning() <<
"aievec.concat with " << sources.size()
2563 <<
" operands is not supported.\n";
2568 Value concatOp =
nullptr;
2569 if (srcVectorSize == 256 && resultVectorSize == 512) {
2570 concatOp = xllvm::ConcatI512I256IntrOp::create(
2571 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
2572 forceCastOperandsToSignature(
2573 rewriter, loc, adaptor.getSources(),
2574 {VectorType::get({8}, rewriter.getI32Type()),
2575 VectorType::get({8}, rewriter.getI32Type())}));
2576 }
else if (srcVectorSize == 256 && resultVectorSize == 1024) {
2577 concatOp = xllvm::ConcatI1024I256IntrOp::create(
2578 rewriter, loc, VectorType::get({32}, rewriter.getI32Type()),
2579 forceCastOperandsToSignature(
2580 rewriter, loc, adaptor.getSources(),
2581 {VectorType::get({8}, rewriter.getI32Type()),
2582 VectorType::get({8}, rewriter.getI32Type()),
2583 VectorType::get({8}, rewriter.getI32Type()),
2584 VectorType::get({8}, rewriter.getI32Type())}));
2585 }
else if (srcVectorSize == 512 && resultVectorSize == 1024) {
2586 concatOp = xllvm::ConcatI1024I512IntrOp::create(
2587 rewriter, loc, VectorType::get({32}, rewriter.getI32Type()),
2588 forceCastOperandsToSignature(
2589 rewriter, loc, adaptor.getSources(),
2590 {VectorType::get({16}, rewriter.getI32Type()),
2591 VectorType::get({16}, rewriter.getI32Type())}));
2593 op.emitWarning() <<
"aievec.concat with " << srcVectorSize
2594 <<
"-bit operands, and " << resultVectorSize
2595 <<
"-bit result is not supported.\n";
2601 forceCastValueToType(rewriter, loc, concatOp, op.getResult().getType());
2602 rewriter.replaceOp(op, resultVal);
2610 using ConvertOpToLLVMPattern<aievec::ExtOp>::ConvertOpToLLVMPattern;
2614 ConversionPatternRewriter &rewriter)
const override {
2615 Location loc = op.getLoc();
2617 Value src = adaptor.getSource();
2618 VectorType srcType = cast<VectorType>(src.getType());
2619 Type srcScalarType = srcType.getElementType();
2620 unsigned srcBitWidth = srcScalarType.getIntOrFloatBitWidth();
2622 int srcVectorSize = srcBitWidth * srcLanes;
2624 Value result = op.getResult();
2625 VectorType resultType = cast<VectorType>(result.getType());
2626 Type resultScaTy = resultType.getElementType();
2627 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
2629 int resultVectorSize = resultBitWidth * resultLanes;
2633 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2634 rewriter.getI32IntegerAttr(op.getIndex()));
2637 SmallVector<Value> operands({adaptor.getSource(), indexCst});
2638 Value extOp =
nullptr;
2640 if (resultVectorSize == 256 && srcVectorSize == 512) {
2641 extOp = xllvm::ExtI256I512IntrOp::create(
2642 rewriter, loc, VectorType::get({8}, rewriter.getI32Type()),
2643 forceCastOperandsToSignature(
2644 rewriter, loc, operands,
2645 {VectorType::get({16}, rewriter.getI32Type()),
2646 rewriter.getI32Type()}));
2647 }
else if (resultVectorSize == 512 && srcVectorSize == 1024) {
2648 extOp = xllvm::ExtI512I1024IntrOp::create(
2649 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
2650 forceCastOperandsToSignature(
2651 rewriter, loc, operands,
2652 {VectorType::get({32}, rewriter.getI32Type()),
2653 rewriter.getI32Type()}));
2654 }
else if (resultVectorSize == 256 && srcVectorSize == 1024) {
2655 extOp = xllvm::ExtI256I1024IntrOp::create(
2656 rewriter, loc, VectorType::get({8}, rewriter.getI32Type()),
2657 forceCastOperandsToSignature(
2658 rewriter, loc, operands,
2659 {VectorType::get({32}, rewriter.getI32Type()),
2660 rewriter.getI32Type()}));
2661 }
else if (resultVectorSize == 128 && srcVectorSize == 512) {
2662 auto shiftOp = adaptor.getSource();
2663 if (op.getIndex() > 0) {
2664 auto undefOp = xllvm::UndefV16I32IntrOp::create(
2665 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()));
2667 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
2668 rewriter.getI32IntegerAttr(0));
2669 auto shiftCst = LLVM::ConstantOp::create(
2670 rewriter, loc, rewriter.getI32Type(),
2671 rewriter.getI32IntegerAttr(op.getIndex() * 16));
2672 SmallVector<Value> shiftOperands{adaptor.getSource(), undefOp, stepCst,
2676 shiftOp = xllvm::VectorShiftI512I512IntrOp::create(
2677 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
2678 forceCastOperandsToSignature(
2679 rewriter, loc, shiftOperands,
2680 {VectorType::get({16}, rewriter.getI32Type()),
2681 VectorType::get({16}, rewriter.getI32Type()),
2682 rewriter.getI32Type(), rewriter.getI32Type()}));
2686 extOp = xllvm::ExtI128I512IntrOp::create(
2687 rewriter, loc, VectorType::get({4}, rewriter.getI32Type()),
2688 forceCastOperandsToSignature(
2689 rewriter, loc, {shiftOp},
2690 {VectorType::get({16}, rewriter.getI32Type())}));
2692 op.emitWarning() <<
"aievec.ext with " << srcVectorSize
2693 <<
"-bit source, and " << resultVectorSize
2694 <<
"-bit result is not supported.\n";
2700 forceCastValueToType(rewriter, loc, extOp, op.getResult().getType());
2701 rewriter.replaceOp(op, resultVal);
2709 :
public mlir::ConvertOpToLLVMPattern<aievec::ExtOp> {
2711 using ConvertOpToLLVMPattern<aievec::ExtOp>::ConvertOpToLLVMPattern;
2715 ConversionPatternRewriter &rewriter)
const override {
2716 Location loc = op.getLoc();
2718 Value src = adaptor.getSource();
2719 VectorType srcType = cast<VectorType>(src.getType());
2720 VectorType resultType = cast<VectorType>(op.getResult().getType());
2726 if (srcLanes % resultLanes != 0) {
2727 op.emitWarning() <<
"aievec.ext of a part that does not divide the "
2728 "source is not supported for AIE2p.\n";
2733 SmallVector<int64_t> shuffleMask;
2734 int startIdx = op.getIndex() * resultLanes;
2735 for (
int i = 0; i < resultLanes; ++i) {
2736 shuffleMask.push_back(startIdx + i);
2740 vector::ShuffleOp::create(rewriter, loc, src, src, shuffleMask);
2742 rewriter.replaceOp(op, extracted);
2748 :
public mlir::ConvertOpToLLVMPattern<aievec::aie1::SelectOp> {
2750 using ConvertOpToLLVMPattern<aievec::aie1::SelectOp>::ConvertOpToLLVMPattern;
2753 auto xbuffType = cast<VectorType>(op.getXbuff().getType());
2754 std::stringstream ss;
2755 ss <<
"llvm.aie.prim." << getVectorTypeString(xbuffType) <<
".select";
2761 ConversionPatternRewriter &rewriter)
const override {
2762 auto module = op->getParentOfType<ModuleOp>();
2763 MLIRContext *context = rewriter.getContext();
2765 auto selectType = IntegerType::get(context, 32);
2766 auto startType = IntegerType::get(context, 32);
2767 auto offsetsType = VectorType::get({2}, IntegerType::get(context, 32));
2768 auto confType = VectorType::get({2}, IntegerType::get(context, 32));
2772 auto func =
module.lookupSymbol<LLVM::LLVMFuncOp>(
2773 StringAttr::get(context, intrinsicName));
2776 OpBuilder::InsertionGuard guard(rewriter);
2777 rewriter.setInsertionPointToStart(module.getBody());
2778 func = LLVM::LLVMFuncOp::create(
2779 rewriter, rewriter.getUnknownLoc(), intrinsicName,
2780 LLVM::LLVMFunctionType::get(op.getResult().getType(),
2781 {op.getXbuff().getType(), selectType,
2790 uint32_t select = 0;
2795 op.getSelect().getAsInteger(0, select);
2796 op.getXstart().getAsInteger(0, x.
start);
2797 op.getXoffsets().getAsInteger(0, x.
offsets);
2798 op.getXoffsetsHi().getAsInteger(0, x.
offsets_hi);
2799 op.getXsquare().getAsInteger(0, x.
square);
2800 op.getYstart().getAsInteger(0, y.
start);
2801 op.getYoffsets().getAsInteger(0, y.
offsets);
2802 op.getYoffsetsHi().getAsInteger(0, y.
offsets_hi);
2803 op.getYsquare().getAsInteger(0, y.
square);
2806 uint32_t conf[2] = {0, 0};
2807 encodeConf(conf, x, z,
false);
2808 conf[1] |= encodeSquare(y.
square) << 21;
2811 auto selectVal = LLVM::ConstantOp::create(
2812 rewriter, op->getLoc(), selectType, rewriter.getI32IntegerAttr(select));
2813 auto xstartVal = LLVM::ConstantOp::create(
2814 rewriter, op->getLoc(), startType, rewriter.getI32IntegerAttr(x.
start));
2815 auto ystartVal = LLVM::ConstantOp::create(
2816 rewriter, op->getLoc(), startType, rewriter.getI32IntegerAttr(y.
start));
2817 auto xoffsetsVal = LLVM::ConstantOp::create(
2818 rewriter, op->getLoc(), offsetsType,
2819 rewriter.getI32VectorAttr({(int32_t)x.offsets, (int32_t)x.offsets_hi}));
2820 auto yoffsetsVal = LLVM::ConstantOp::create(
2821 rewriter, op->getLoc(), offsetsType,
2822 rewriter.getI32VectorAttr({(int32_t)y.offsets, (int32_t)y.offsets_hi}));
2823 auto confVal = LLVM::ConstantOp::create(
2824 rewriter, op->getLoc(), confType,
2825 rewriter.getI32VectorAttr({(int32_t)conf[0], (int32_t)conf[1]}));
2826 rewriter.replaceOpWithNewOp<LLVM::CallOp>(
2828 ValueRange{op.getXbuff(), selectVal, xstartVal, ystartVal, xoffsetsVal,
2829 yoffsetsVal, confVal});
2836 using ConvertOpToLLVMPattern<aievec::PackOp>::ConvertOpToLLVMPattern;
2839 auto sourceType = cast<VectorType>(op.getSource().getType());
2840 std::stringstream ss;
2841 ss <<
"llvm.aie.pack." << getVectorTypeString(sourceType);
2847 ConversionPatternRewriter &rewriter)
const override {
2848 auto module = op->getParentOfType<ModuleOp>();
2849 MLIRContext *context = rewriter.getContext();
2853 auto func =
module.lookupSymbol<LLVM::LLVMFuncOp>(
2854 StringAttr::get(context, intrinsicName));
2857 OpBuilder::InsertionGuard guard(rewriter);
2858 rewriter.setInsertionPointToStart(module.getBody());
2859 func = LLVM::LLVMFuncOp::create(
2860 rewriter, rewriter.getUnknownLoc(), intrinsicName,
2861 LLVM::LLVMFunctionType::get(op.getResult().getType(),
2862 {op.getSource().getType()}));
2865 rewriter.replaceOpWithNewOp<LLVM::CallOp>(op,
func,
2866 ValueRange{op.getSource()});
2872 :
public mlir::ConvertOpToLLVMPattern<aievec::UnpackOp> {
2874 using ConvertOpToLLVMPattern<aievec::UnpackOp>::ConvertOpToLLVMPattern;
2878 ConversionPatternRewriter &rewriter)
const override {
2879 op.emitWarning() <<
"aie.unpack conversion is not implemented\n";
2890 :
public mlir::ConvertOpToLLVMPattern<aievec::UnpackOp> {
2892 using ConvertOpToLLVMPattern<aievec::UnpackOp>::ConvertOpToLLVMPattern;
2896 ConversionPatternRewriter &rewriter)
const override {
2897 Location loc = op.getLoc();
2898 auto srcTy = cast<VectorType>(op.getSource().getType());
2899 auto dstTy = cast<VectorType>(op.getResult().getType());
2900 if (!srcTy.getElementType().isInteger(8) ||
2901 !dstTy.getElementType().isInteger(8))
2906 if (dstLanes != 2 * srcLanes)
2908 if (dstLanes != 64 && dstLanes != 128)
2911 auto signCst = LLVM::ConstantOp::create(
2912 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(0));
2915 if (dstLanes == 64) {
2916 unpacked = xllvm::UnpackI512I8I4AIE2pIntrOp::create(
2917 rewriter, loc, VectorType::get({64}, rewriter.getI8Type()),
2918 adaptor.getSource(), signCst)
2922 xllvm::UnpackI1024I8I4AIE2pIntrOp::create(
2923 rewriter, loc, VectorType::get({128}, rewriter.getI8Type()),
2924 adaptor.getSource(), signCst)
2927 rewriter.replaceOp(op, unpacked);
2933 :
public mlir::ConvertOpToLLVMPattern<aievec::BroadcastOp> {
2935 using ConvertOpToLLVMPattern<aievec::BroadcastOp>::ConvertOpToLLVMPattern;
2939 ConversionPatternRewriter &rewriter)
const override {
2940 op.emitWarning() <<
"aie.broadcast conversion is not implemented\n";
2947static Value padVectorWithPoison(ConversionPatternRewriter &rewriter,
2948 Location loc, Value vec,
int srcLanes,
2950 SmallVector<int64_t> padMask;
2951 for (
int i = 0; i < srcLanes; ++i)
2952 padMask.push_back(i);
2953 for (
int i = srcLanes; i < dstLanes; ++i)
2954 padMask.push_back(-1);
2955 return vector::ShuffleOp::create(rewriter, loc, vec, vec, padMask);
2960static Value extractLowerLanes(ConversionPatternRewriter &rewriter,
2961 Location loc, Value vec,
int lanes) {
2962 SmallVector<int64_t> extractMask;
2963 for (
int i = 0; i < lanes; ++i)
2964 extractMask.push_back(i);
2965 return vector::ShuffleOp::create(rewriter, loc, vec, vec, extractMask);
2970 using ConvertOpToLLVMPattern<aievec::MaxOp>::ConvertOpToLLVMPattern;
2974 ConversionPatternRewriter &rewriter)
const override {
2975 Location loc = op.getLoc();
2977 VectorType resultType = cast<VectorType>(op.getResult().getType());
2978 Type resultScaTy = resultType.getElementType();
2979 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
2981 int resultVectorSize = resultBitWidth * resultLanes;
2984 if (resultVectorSize != 512 && resultVectorSize != 256) {
2985 op.emitWarning() <<
"aievec.max conversion with " << resultVectorSize
2986 <<
"-bit result is not supported.\n";
2991 Value maxOp =
nullptr;
2992 if (llvm::isa<IntegerType>(resultScaTy)) {
2995 auto cmpCst = LLVM::ConstantOp::create(
2996 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(1));
2997 SmallVector<Value> operands{adaptor.getLhs(), adaptor.getRhs(), cmpCst};
2998 if (resultBitWidth == 8) {
2999 maxOp = xllvm::VectorMaxLt8IntrOp::create(
3001 mlir::LLVM::LLVMStructType::getLiteral(
3002 rewriter.getContext(),
3003 {VectorType::get({64}, rewriter.getI8Type()),
3004 VectorType::get({2}, rewriter.getI32Type())}),
3005 forceCastOperandsToSignature(
3006 rewriter, loc, operands,
3007 {VectorType::get({64}, rewriter.getI8Type()),
3008 VectorType::get({64}, rewriter.getI8Type()),
3009 rewriter.getI32Type()}));
3010 }
else if (resultBitWidth == 16) {
3011 maxOp = xllvm::VectorMaxLt16IntrOp::create(
3013 mlir::LLVM::LLVMStructType::getLiteral(
3014 rewriter.getContext(),
3015 {VectorType::get({32}, rewriter.getI16Type()),
3016 rewriter.getI32Type()}),
3017 forceCastOperandsToSignature(
3018 rewriter, loc, operands,
3019 {VectorType::get({32}, rewriter.getI16Type()),
3020 VectorType::get({32}, rewriter.getI16Type()),
3021 rewriter.getI32Type()}));
3022 }
else if (resultBitWidth == 32) {
3023 maxOp = xllvm::VectorMaxLt32IntrOp::create(
3025 mlir::LLVM::LLVMStructType::getLiteral(
3026 rewriter.getContext(),
3027 {VectorType::get({16}, rewriter.getI32Type()),
3028 rewriter.getI32Type()}),
3029 forceCastOperandsToSignature(
3030 rewriter, loc, operands,
3031 {VectorType::get({16}, rewriter.getI32Type()),
3032 VectorType::get({16}, rewriter.getI32Type()),
3033 rewriter.getI32Type()}));
3036 if (resultBitWidth == 16) {
3037 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
3038 Value lhs = adaptor.getLhs(), rhs = adaptor.getRhs();
3041 if (resultLanes == 16) {
3042 lhs = padVectorWithPoison(rewriter, loc, lhs, 16, 32);
3043 rhs = padVectorWithPoison(rewriter, loc, rhs, 16, 32);
3046 maxOp = xllvm::VectorMaxLtBf16IntrOp::create(
3048 mlir::LLVM::LLVMStructType::getLiteral(
3049 rewriter.getContext(), {v32bf16Ty, rewriter.getI32Type()}),
3050 forceCastOperandsToSignature(rewriter, loc, {lhs, rhs},
3051 {v32bf16Ty, v32bf16Ty}));
3056 op.emitWarning() <<
"aievec.max conversion fails due to unsupported "
3057 "element data type.\n";
3062 Value resultVec = LLVM::ExtractValueOp::create(rewriter, loc, maxOp,
3065 if (resultLanes == 16 && !llvm::isa<IntegerType>(resultScaTy))
3066 resultVec = extractLowerLanes(rewriter, loc, resultVec, 16);
3068 rewriter.replaceOp(op, resultVec);
3076 using ConvertOpToLLVMPattern<aievec::MinOp>::ConvertOpToLLVMPattern;
3080 ConversionPatternRewriter &rewriter)
const override {
3081 Location loc = op.getLoc();
3083 VectorType resultType = cast<VectorType>(op.getResult().getType());
3084 Type resultScaTy = resultType.getElementType();
3085 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
3087 int resultVectorSize = resultBitWidth * resultLanes;
3090 if (resultVectorSize != 512 && resultVectorSize != 256) {
3091 op.emitWarning() <<
"aievec.min conversion with " << resultVectorSize
3092 <<
"-bit result is not supported.\n";
3097 Value minOp =
nullptr;
3098 if (llvm::isa<IntegerType>(resultScaTy)) {
3101 auto cmpCst = LLVM::ConstantOp::create(
3102 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(1));
3103 SmallVector<Value> operands{adaptor.getLhs(), adaptor.getRhs(), cmpCst};
3104 if (resultBitWidth == 8) {
3105 minOp = xllvm::VectorMinGe8IntrOp::create(
3107 mlir::LLVM::LLVMStructType::getLiteral(
3108 rewriter.getContext(),
3109 {VectorType::get({64}, rewriter.getI8Type()),
3110 VectorType::get({2}, rewriter.getI32Type())}),
3111 forceCastOperandsToSignature(
3112 rewriter, loc, operands,
3113 {VectorType::get({64}, rewriter.getI8Type()),
3114 VectorType::get({64}, rewriter.getI8Type()),
3115 rewriter.getI32Type()}));
3116 }
else if (resultBitWidth == 16) {
3117 minOp = xllvm::VectorMinGe16IntrOp::create(
3119 mlir::LLVM::LLVMStructType::getLiteral(
3120 rewriter.getContext(),
3121 {VectorType::get({32}, rewriter.getI16Type()),
3122 rewriter.getI32Type()}),
3123 forceCastOperandsToSignature(
3124 rewriter, loc, operands,
3125 {VectorType::get({32}, rewriter.getI16Type()),
3126 VectorType::get({32}, rewriter.getI16Type()),
3127 rewriter.getI32Type()}));
3128 }
else if (resultBitWidth == 32) {
3129 minOp = xllvm::VectorMinGe32IntrOp::create(
3131 mlir::LLVM::LLVMStructType::getLiteral(
3132 rewriter.getContext(),
3133 {VectorType::get({16}, rewriter.getI32Type()),
3134 rewriter.getI32Type()}),
3135 forceCastOperandsToSignature(
3136 rewriter, loc, operands,
3137 {VectorType::get({16}, rewriter.getI32Type()),
3138 VectorType::get({16}, rewriter.getI32Type()),
3139 rewriter.getI32Type()}));
3142 if (resultBitWidth == 16) {
3143 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
3144 Value lhs = adaptor.getLhs(), rhs = adaptor.getRhs();
3147 if (resultLanes == 16) {
3148 lhs = padVectorWithPoison(rewriter, loc, lhs, 16, 32);
3149 rhs = padVectorWithPoison(rewriter, loc, rhs, 16, 32);
3152 minOp = xllvm::VectorMinGeBf16IntrOp::create(
3154 mlir::LLVM::LLVMStructType::getLiteral(
3155 rewriter.getContext(), {v32bf16Ty, rewriter.getI32Type()}),
3156 forceCastOperandsToSignature(rewriter, loc, {lhs, rhs},
3157 {v32bf16Ty, v32bf16Ty}));
3162 op.emitWarning() <<
"aievec.min conversion fails due to unsupported "
3163 "element data type.\n";
3168 Value resultVec = LLVM::ExtractValueOp::create(rewriter, loc, minOp,
3171 if (resultLanes == 16 && !llvm::isa<IntegerType>(resultScaTy))
3172 resultVec = extractLowerLanes(rewriter, loc, resultVec, 16);
3174 rewriter.replaceOp(op, resultVec);
3182 :
public mlir::ConvertOpToLLVMPattern<aievec::MaxOp> {
3184 using ConvertOpToLLVMPattern<aievec::MaxOp>::ConvertOpToLLVMPattern;
3188 ConversionPatternRewriter &rewriter)
const override {
3189 Location loc = op.getLoc();
3191 VectorType resultType = cast<VectorType>(op.getResult().getType());
3192 Type resultScaTy = resultType.getElementType();
3193 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
3195 int resultVectorSize = resultBitWidth * resultLanes;
3198 if (resultVectorSize != 512 && resultVectorSize != 256) {
3199 op.emitWarning() <<
"aievec.max conversion with " << resultVectorSize
3200 <<
"-bit result is not supported.\n";
3205 Value maxOp =
nullptr;
3206 if (llvm::isa<IntegerType>(resultScaTy)) {
3209 auto cmpCst = LLVM::ConstantOp::create(
3210 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(1));
3211 SmallVector<Value> operands{adaptor.getLhs(), adaptor.getRhs(), cmpCst};
3212 if (resultBitWidth == 8) {
3213 maxOp = xllvm::VectorMaxLt8AIE2pIntrOp::create(
3215 mlir::LLVM::LLVMStructType::getLiteral(
3216 rewriter.getContext(),
3217 {VectorType::get({64}, rewriter.getI8Type()),
3218 VectorType::get({2}, rewriter.getI32Type())}),
3219 forceCastOperandsToSignature(
3220 rewriter, loc, operands,
3221 {VectorType::get({64}, rewriter.getI8Type()),
3222 VectorType::get({64}, rewriter.getI8Type()),
3223 rewriter.getI32Type()}));
3224 }
else if (resultBitWidth == 16) {
3225 maxOp = xllvm::VectorMaxLt16AIE2pIntrOp::create(
3227 mlir::LLVM::LLVMStructType::getLiteral(
3228 rewriter.getContext(),
3229 {VectorType::get({32}, rewriter.getI16Type()),
3230 rewriter.getI32Type()}),
3231 forceCastOperandsToSignature(
3232 rewriter, loc, operands,
3233 {VectorType::get({32}, rewriter.getI16Type()),
3234 VectorType::get({32}, rewriter.getI16Type()),
3235 rewriter.getI32Type()}));
3236 }
else if (resultBitWidth == 32) {
3237 maxOp = xllvm::VectorMaxLt32AIE2pIntrOp::create(
3239 mlir::LLVM::LLVMStructType::getLiteral(
3240 rewriter.getContext(),
3241 {VectorType::get({16}, rewriter.getI32Type()),
3242 rewriter.getI32Type()}),
3243 forceCastOperandsToSignature(
3244 rewriter, loc, operands,
3245 {VectorType::get({16}, rewriter.getI32Type()),
3246 VectorType::get({16}, rewriter.getI32Type()),
3247 rewriter.getI32Type()}));
3250 if (resultBitWidth == 16) {
3251 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
3252 Value lhs = adaptor.getLhs(), rhs = adaptor.getRhs();
3254 if (resultLanes == 16) {
3255 lhs = padVectorWithPoison(rewriter, loc, lhs, 16, 32);
3256 rhs = padVectorWithPoison(rewriter, loc, rhs, 16, 32);
3259 maxOp = xllvm::VectorMaxLtBf16AIE2pIntrOp::create(
3261 mlir::LLVM::LLVMStructType::getLiteral(
3262 rewriter.getContext(), {v32bf16Ty, rewriter.getI32Type()}),
3263 forceCastOperandsToSignature(rewriter, loc, {lhs, rhs},
3264 {v32bf16Ty, v32bf16Ty}));
3269 op.emitWarning() <<
"aievec.max conversion fails due to unsupported "
3270 "element data type.\n";
3274 Value resultVec = LLVM::ExtractValueOp::create(rewriter, loc, maxOp,
3276 if (resultLanes == 16 && !llvm::isa<IntegerType>(resultScaTy))
3277 resultVec = extractLowerLanes(rewriter, loc, resultVec, 16);
3279 rewriter.replaceOp(op, resultVec);
3287 :
public mlir::ConvertOpToLLVMPattern<aievec::MinOp> {
3289 using ConvertOpToLLVMPattern<aievec::MinOp>::ConvertOpToLLVMPattern;
3293 ConversionPatternRewriter &rewriter)
const override {
3294 Location loc = op.getLoc();
3296 VectorType resultType = cast<VectorType>(op.getResult().getType());
3297 Type resultScaTy = resultType.getElementType();
3298 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
3300 int resultVectorSize = resultBitWidth * resultLanes;
3303 if (resultVectorSize != 512 && resultVectorSize != 256) {
3304 op.emitWarning() <<
"aievec.min conversion with " << resultVectorSize
3305 <<
"-bit result is not supported.\n";
3310 Value minOp =
nullptr;
3311 if (llvm::isa<IntegerType>(resultScaTy)) {
3314 auto cmpCst = LLVM::ConstantOp::create(
3315 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(1));
3316 SmallVector<Value> operands{adaptor.getLhs(), adaptor.getRhs(), cmpCst};
3317 if (resultBitWidth == 8) {
3318 minOp = xllvm::VectorMinGe8AIE2pIntrOp::create(
3320 mlir::LLVM::LLVMStructType::getLiteral(
3321 rewriter.getContext(),
3322 {VectorType::get({64}, rewriter.getI8Type()),
3323 VectorType::get({2}, rewriter.getI32Type())}),
3324 forceCastOperandsToSignature(
3325 rewriter, loc, operands,
3326 {VectorType::get({64}, rewriter.getI8Type()),
3327 VectorType::get({64}, rewriter.getI8Type()),
3328 rewriter.getI32Type()}));
3329 }
else if (resultBitWidth == 16) {
3330 minOp = xllvm::VectorMinGe16AIE2pIntrOp::create(
3332 mlir::LLVM::LLVMStructType::getLiteral(
3333 rewriter.getContext(),
3334 {VectorType::get({32}, rewriter.getI16Type()),
3335 rewriter.getI32Type()}),
3336 forceCastOperandsToSignature(
3337 rewriter, loc, operands,
3338 {VectorType::get({32}, rewriter.getI16Type()),
3339 VectorType::get({32}, rewriter.getI16Type()),
3340 rewriter.getI32Type()}));
3341 }
else if (resultBitWidth == 32) {
3342 minOp = xllvm::VectorMinGe32AIE2pIntrOp::create(
3344 mlir::LLVM::LLVMStructType::getLiteral(
3345 rewriter.getContext(),
3346 {VectorType::get({16}, rewriter.getI32Type()),
3347 rewriter.getI32Type()}),
3348 forceCastOperandsToSignature(
3349 rewriter, loc, operands,
3350 {VectorType::get({16}, rewriter.getI32Type()),
3351 VectorType::get({16}, rewriter.getI32Type()),
3352 rewriter.getI32Type()}));
3355 if (resultBitWidth == 16) {
3356 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
3357 Value lhs = adaptor.getLhs(), rhs = adaptor.getRhs();
3359 if (resultLanes == 16) {
3360 lhs = padVectorWithPoison(rewriter, loc, lhs, 16, 32);
3361 rhs = padVectorWithPoison(rewriter, loc, rhs, 16, 32);
3364 minOp = xllvm::VectorMinGeBf16AIE2pIntrOp::create(
3366 mlir::LLVM::LLVMStructType::getLiteral(
3367 rewriter.getContext(), {v32bf16Ty, rewriter.getI32Type()}),
3368 forceCastOperandsToSignature(rewriter, loc, {lhs, rhs},
3369 {v32bf16Ty, v32bf16Ty}));
3374 op.emitWarning() <<
"aievec.min conversion fails due to unsupported "
3375 "element data type.\n";
3379 Value resultVec = LLVM::ExtractValueOp::create(rewriter, loc, minOp,
3381 if (resultLanes == 16 && !llvm::isa<IntegerType>(resultScaTy))
3382 resultVec = extractLowerLanes(rewriter, loc, resultVec, 16);
3384 rewriter.replaceOp(op, resultVec);
3395template <
typename LtOp,
typename GeOp>
3396static Value buildCmpBitmask(OpBuilder &builder, Location loc, StringRef rel,
3397 Value lhs, Value rhs, ValueRange extraOperands) {
3398 auto structTy = LLVM::LLVMStructType::getLiteral(
3399 builder.getContext(), {lhs.getType(), builder.getI32Type()});
3400 auto compare = [&](
bool lessThan, Value a, Value b) -> Value {
3401 SmallVector<Value> operands{a, b};
3402 llvm::append_range(operands, extraOperands);
3403 Value result = lessThan
3404 ? Value(LtOp::create(builder, loc, structTy, operands))
3405 : Value(GeOp::create(builder, loc, structTy, operands));
3406 return LLVM::ExtractValueOp::create(builder, loc, result, 1);
3409 return compare(
true, lhs, rhs);
3411 return compare(
false, lhs, rhs);
3413 return compare(
true, rhs, lhs);
3415 return compare(
false, rhs, lhs);
3416 assert((rel ==
"eq" || rel ==
"ne") &&
"unexpected compare predicate");
3418 bool isEq = rel ==
"eq";
3419 Value forward = compare(!isEq, lhs, rhs);
3420 Value backward = compare(!isEq, rhs, lhs);
3422 return LLVM::AndOp::create(builder, loc, forward, backward);
3423 return LLVM::OrOp::create(builder, loc, forward, backward);
3430template <
typename MaxLtBf16IntrOp,
typename MinGeBf16IntrOp,
3431 typename MaxLt32IntrOp,
typename MinGe32IntrOp,
3432 typename MaxLt16IntrOp,
typename MinGe16IntrOp>
3435 using ConvertOpToLLVMPattern<aievec::CmpOp>::ConvertOpToLLVMPattern;
3439 ConversionPatternRewriter &rewriter)
const override {
3440 Location loc = op.getLoc();
3441 auto vecTy = cast<VectorType>(op.getLhs().getType());
3442 auto elTy = vecTy.getElementType();
3443 unsigned elWidth = elTy.getIntOrFloatBitWidth();
3445 StringRef pred = op.getPred();
3447 bool isUnsigned = llvm::is_contained({
"ult",
"uge",
"ugt",
"ule"}, pred);
3449 !llvm::is_contained({
"slt",
"sge",
"sgt",
"sle",
"eq",
"ne"}, pred))
3452 StringRef rel = pred.size() == 3 ? pred.drop_front() : pred;
3454 Value lhs = adaptor.getLhs(), rhs = adaptor.getRhs();
3456 if (elWidth == 16 && isa<FloatType>(elTy) && (lanes == 16 || lanes == 32)) {
3458 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
3460 lhs = padVectorWithPoison(rewriter, loc, lhs, 16, 32);
3461 rhs = padVectorWithPoison(rewriter, loc, rhs, 16, 32);
3463 SmallVector<Value> operands = forceCastOperandsToSignature(
3464 rewriter, loc, {lhs, rhs}, {v32bf16Ty, v32bf16Ty});
3465 bitmask = buildCmpBitmask<MaxLtBf16IntrOp, MinGeBf16IntrOp>(
3466 rewriter, loc, rel, operands[0], operands[1], {});
3470 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(),
3471 rewriter.getI32IntegerAttr(0xFFFF));
3472 bitmask = LLVM::AndOp::create(rewriter, loc, bitmask, mask);
3474 }
else if (isa<IntegerType>(elTy) && elWidth * lanes == 512 &&
3475 (elWidth == 32 || elWidth == 16)) {
3479 VectorType::get({lanes}, rewriter.getIntegerType(elWidth));
3480 Value sign = LLVM::ConstantOp::create(
3481 rewriter, loc, rewriter.getI32Type(),
3482 rewriter.getI32IntegerAttr(isUnsigned ? 0 : 1));
3483 SmallVector<Value> operands = forceCastOperandsToSignature(
3484 rewriter, loc, {lhs, rhs}, {intVecTy, intVecTy});
3485 bitmask = elWidth == 32
3486 ? buildCmpBitmask<MaxLt32IntrOp, MinGe32IntrOp>(
3487 rewriter, loc, rel, operands[0], operands[1], sign)
3488 : buildCmpBitmask<MaxLt16IntrOp, MinGe16IntrOp>(
3489 rewriter, loc, rel, operands[0], operands[1], sign);
3495 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
3496 op, op.getResult().getType(), bitmask);
3503 xllvm::VectorMinGeBf16IntrOp,
3504 xllvm::VectorMaxLt32IntrOp, xllvm::VectorMinGe32IntrOp,
3505 xllvm::VectorMaxLt16IntrOp, xllvm::VectorMinGe16IntrOp>;
3508 xllvm::VectorMaxLtBf16AIE2pIntrOp, xllvm::VectorMinGeBf16AIE2pIntrOp,
3509 xllvm::VectorMaxLt32AIE2pIntrOp, xllvm::VectorMinGe32AIE2pIntrOp,
3510 xllvm::VectorMaxLt16AIE2pIntrOp, xllvm::VectorMinGe16AIE2pIntrOp>;
3515template <
typename Sel16IntrOp,
typename Sel32IntrOp>
3518 using ConvertOpToLLVMPattern<aievec::SelOp>::ConvertOpToLLVMPattern;
3522 ConversionPatternRewriter &rewriter)
const override {
3523 Location loc = op.getLoc();
3524 auto resultType = cast<VectorType>(op.getResult().getType());
3525 auto elTy = resultType.getElementType();
3526 unsigned elWidth = elTy.getIntOrFloatBitWidth();
3528 auto i32Ty = rewriter.getI32Type();
3531 Value selMask = adaptor.getSel();
3532 if (selMask.getType() != i32Ty)
3534 UnrealizedConversionCastOp::create(rewriter, loc, i32Ty, selMask)
3538 if (elWidth == 16 && isa<FloatType>(elTy)) {
3539 auto v32i16Ty = VectorType::get({32}, rewriter.getI16Type());
3540 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
3542 Value lhs = adaptor.getLhs(), rhs = adaptor.getRhs();
3543 bool needExtract =
false;
3546 lhs = padVectorWithPoison(rewriter, loc, lhs, 16, 32);
3547 rhs = padVectorWithPoison(rewriter, loc, rhs, 16, 32);
3549 }
else if (lanes != 32) {
3554 auto lhsCast = forceCastValueToType(rewriter, loc, lhs, v32bf16Ty);
3555 auto lhsI16 = LLVM::BitcastOp::create(rewriter, loc, v32i16Ty, lhsCast);
3556 auto rhsCast = forceCastValueToType(rewriter, loc, rhs, v32bf16Ty);
3557 auto rhsI16 = LLVM::BitcastOp::create(rewriter, loc, v32i16Ty, rhsCast);
3559 auto selResult = Sel16IntrOp::create(
3560 rewriter, loc, v32i16Ty,
3561 forceCastOperandsToSignature(rewriter, loc, {lhsI16, rhsI16, selMask},
3562 {v32i16Ty, v32i16Ty, i32Ty}));
3566 LLVM::BitcastOp::create(rewriter, loc, v32bf16Ty, selResult);
3569 result = extractLowerLanes(rewriter, loc, result, 16);
3571 rewriter.replaceOp(op, result);
3576 if (elWidth == 32 && isa<IntegerType>(elTy) && lanes == 16) {
3577 auto v16i32Ty = VectorType::get({16}, rewriter.getI32Type());
3578 auto selResult = Sel32IntrOp::create(
3579 rewriter, loc, v16i32Ty,
3580 forceCastOperandsToSignature(
3581 rewriter, loc, {adaptor.getLhs(), adaptor.getRhs(), selMask},
3582 {v16i32Ty, v16i32Ty, i32Ty}));
3583 rewriter.replaceOp(op, selResult->getResult(0));
3588 if (elWidth == 16 && isa<IntegerType>(elTy) && lanes == 32) {
3589 auto v32i16Ty = VectorType::get({32}, rewriter.getI16Type());
3590 auto selResult = Sel16IntrOp::create(
3591 rewriter, loc, v32i16Ty,
3592 forceCastOperandsToSignature(
3593 rewriter, loc, {adaptor.getLhs(), adaptor.getRhs(), selMask},
3594 {v32i16Ty, v32i16Ty, i32Ty}));
3595 rewriter.replaceOp(op, selResult->getResult(0));
3607 xllvm::VectorSel32AIE2pIntrOp>;
3610 :
public mlir::ConvertOpToLLVMPattern<aievec::BroadcastScalarOp> {
3612 using ConvertOpToLLVMPattern<
3613 aievec::BroadcastScalarOp>::ConvertOpToLLVMPattern;
3617 ConversionPatternRewriter &rewriter)
const override {
3618 Location loc = op.getLoc();
3620 Value result = op.getResult();
3621 VectorType resultType = cast<VectorType>(result.getType());
3622 Type resultScaTy = resultType.getElementType();
3623 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
3625 int resultVectorSize = resultBitWidth * resultLanes;
3627 if (resultVectorSize != 512) {
3629 <<
"aievec.broadcast_scalar conversion with result vector size "
3630 << resultVectorSize <<
" is not implemented.\n";
3635 if (llvm::isa<IntegerType>(resultScaTy)) {
3636 Value src = adaptor.getSource();
3637 Type srcType = src.getType();
3638 unsigned srcBitWidth = srcType.getIntOrFloatBitWidth();
3640 if (srcBitWidth < 32) {
3641 src = LLVM::SExtOp::create(rewriter, loc, rewriter.getI32Type(), src);
3644 if (resultBitWidth == 8) {
3645 rewriter.replaceOpWithNewOp<xllvm::VectorBroadcast8I512IntrOp>(
3646 op, VectorType::get({64}, rewriter.getI8Type()), src);
3647 }
else if (resultBitWidth == 16) {
3648 rewriter.replaceOpWithNewOp<xllvm::VectorBroadcast16I512IntrOp>(
3649 op, VectorType::get({32}, rewriter.getI16Type()), src);
3650 }
else if (resultBitWidth == 32) {
3651 rewriter.replaceOpWithNewOp<xllvm::VectorBroadcast32I512IntrOp>(
3652 op, VectorType::get({16}, rewriter.getI32Type()), src);
3655 <<
"aievec.broadcast_scalar conversion with result bitwidth "
3656 << resultBitWidth <<
" is not implemented.\n";
3661 if (resultBitWidth == 16) {
3662 rewriter.replaceOpWithNewOp<xllvm::VectorBroadcast16BF512IntrOp>(
3663 op, VectorType::get({32}, rewriter.getBF16Type()),
3664 adaptor.getSource());
3665 }
else if (resultBitWidth == 32) {
3671 auto srcAsI32 = bitcastValueToType(rewriter, loc, adaptor.getSource(),
3672 rewriter.getI32Type());
3673 auto broadcastI32 = xllvm::VectorBroadcast32I512IntrOp::create(
3674 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
3677 bitcastValueToType(rewriter, loc, broadcastI32,
3678 VectorType::get({16}, rewriter.getF32Type()));
3679 rewriter.replaceOp(op, resultF32);
3682 <<
"aievec.broadcast_scalar conversion with result bitwidth "
3683 << resultBitWidth <<
" is not implemented.\n";
3695 :
public mlir::ConvertOpToLLVMPattern<aievec::BroadcastScalarOp> {
3697 using ConvertOpToLLVMPattern<
3698 aievec::BroadcastScalarOp>::ConvertOpToLLVMPattern;
3702 ConversionPatternRewriter &rewriter)
const override {
3703 Location loc = op.getLoc();
3705 Value result = op.getResult();
3706 VectorType resultType = cast<VectorType>(result.getType());
3707 Type resultScaTy = resultType.getElementType();
3708 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
3710 int resultVectorSize = resultBitWidth * resultLanes;
3713 if (resultVectorSize != 256 && resultVectorSize != 512) {
3715 <<
"aievec.broadcast_scalar conversion with result vector size "
3716 << resultVectorSize <<
" is not implemented for AIE2p.\n";
3720 Value src = adaptor.getSource();
3721 Type srcType = src.getType();
3724 if (llvm::isa<IntegerType>(resultScaTy)) {
3725 unsigned srcBitWidth = srcType.getIntOrFloatBitWidth();
3726 if (srcBitWidth < resultBitWidth) {
3727 src = LLVM::SExtOp::create(rewriter, loc, resultScaTy, src);
3728 }
else if (srcBitWidth > resultBitWidth) {
3729 src = LLVM::TruncOp::create(rewriter, loc, resultScaTy, src);
3734 auto poisonVec = LLVM::PoisonOp::create(rewriter, loc, resultType);
3737 auto idx0 = LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(),
3738 rewriter.getI64IntegerAttr(0));
3739 auto insertedVec = LLVM::InsertElementOp::create(rewriter, loc, resultType,
3740 poisonVec, src, idx0);
3744 SmallVector<int64_t> broadcastMask(resultLanes, 0);
3745 auto broadcastVec = vector::ShuffleOp::create(rewriter, loc, insertedVec,
3746 insertedVec, broadcastMask);
3748 rewriter.replaceOp(op, broadcastVec);
3755 using ConvertOpToLLVMPattern<aievec::ShiftOp>::ConvertOpToLLVMPattern;
3759 ConversionPatternRewriter &rewriter)
const override {
3760 Location loc = op.getLoc();
3762 Value result = op.getResult();
3763 VectorType resultType = cast<VectorType>(result.getType());
3764 Type resultScaTy = resultType.getElementType();
3765 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
3767 int resultVectorSize = resultBitWidth * resultLanes;
3769 if (resultVectorSize != 512) {
3770 op.emitWarning() <<
"aievec.shift conversion with result vector size "
3771 << resultVectorSize <<
" is not implemented.\n";
3776 auto stepCst = LLVM::ConstantOp::create(
3777 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(0));
3780 Value shiftOp =
nullptr;
3781 SmallVector<Value> operands(
3782 {adaptor.getLhs(), adaptor.getRhs(), stepCst, adaptor.getShift()});
3783 if (llvm::isa<IntegerType>(resultScaTy)) {
3785 shiftOp = xllvm::VectorShiftI512I512IntrOp::create(
3786 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
3787 forceCastOperandsToSignature(
3788 rewriter, loc, operands,
3789 {VectorType::get({16}, rewriter.getI32Type()),
3790 VectorType::get({16}, rewriter.getI32Type()),
3791 rewriter.getI32Type(), rewriter.getI32Type()}));
3794 shiftOp = xllvm::VectorShiftBF512BF512IntrOp::create(
3795 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()),
3796 forceCastOperandsToSignature(
3797 rewriter, loc, operands,
3798 {VectorType::get({32}, rewriter.getBF16Type()),
3799 VectorType::get({32}, rewriter.getBF16Type()),
3800 rewriter.getI32Type(), rewriter.getI32Type()}));
3805 forceCastValueToType(rewriter, loc, shiftOp, op.getResult().getType());
3806 rewriter.replaceOp(op, resultVal);
3814 :
public mlir::ConvertOpToLLVMPattern<aievec::ShiftOp> {
3816 using ConvertOpToLLVMPattern<aievec::ShiftOp>::ConvertOpToLLVMPattern;
3820 ConversionPatternRewriter &rewriter)
const override {
3821 Location loc = op.getLoc();
3823 Value result = op.getResult();
3824 VectorType resultType = cast<VectorType>(result.getType());
3825 Type resultScaTy = resultType.getElementType();
3826 unsigned resultBitWidth = resultScaTy.getIntOrFloatBitWidth();
3828 int resultVectorSize = resultBitWidth * resultLanes;
3830 if (resultVectorSize != 512) {
3831 op.emitWarning() <<
"aievec.shift conversion with result vector size "
3832 << resultVectorSize <<
" is not implemented.\n";
3837 auto stepCst = LLVM::ConstantOp::create(
3838 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(0));
3841 Value shiftOp =
nullptr;
3842 SmallVector<Value> operands(
3843 {adaptor.getLhs(), adaptor.getRhs(), stepCst, adaptor.getShift()});
3844 if (llvm::isa<IntegerType>(resultScaTy)) {
3846 shiftOp = xllvm::VectorShiftI512I512AIE2pIntrOp::create(
3847 rewriter, loc, VectorType::get({16}, rewriter.getI32Type()),
3848 forceCastOperandsToSignature(
3849 rewriter, loc, operands,
3850 {VectorType::get({16}, rewriter.getI32Type()),
3851 VectorType::get({16}, rewriter.getI32Type()),
3852 rewriter.getI32Type(), rewriter.getI32Type()}));
3855 shiftOp = xllvm::VectorShiftBF512BF512AIE2pIntrOp::create(
3856 rewriter, loc, VectorType::get({32}, rewriter.getBF16Type()),
3857 forceCastOperandsToSignature(
3858 rewriter, loc, operands,
3859 {VectorType::get({32}, rewriter.getBF16Type()),
3860 VectorType::get({32}, rewriter.getBF16Type()),
3861 rewriter.getI32Type(), rewriter.getI32Type()}));
3866 forceCastValueToType(rewriter, loc, shiftOp, op.getResult().getType());
3867 rewriter.replaceOp(op, resultVal);
3874 :
public mlir::ConvertOpToLLVMPattern<aievec::ExtElemOp> {
3876 using ConvertOpToLLVMPattern<aievec::ExtElemOp>::ConvertOpToLLVMPattern;
3880 ConversionPatternRewriter &rewriter)
const override {
3881 Location loc = op.getLoc();
3883 Type resultType = op.getResult().getType();
3884 unsigned resultBitWidth = resultType.getIntOrFloatBitWidth();
3886 Value src = adaptor.getSource();
3887 VectorType srcType = cast<VectorType>(src.getType());
3888 Type srcScalarType = srcType.getElementType();
3889 unsigned srcBitWidth = srcScalarType.getIntOrFloatBitWidth();
3891 int srcVectorSize = srcBitWidth * srcLanes;
3893 if (srcVectorSize != 512) {
3894 op.emitWarning() <<
"aievec.ext_elem conversion with source vector size "
3895 << srcVectorSize <<
" is not supported.\n";
3900 auto signCst = LLVM::ConstantOp::create(
3901 rewriter, loc, rewriter.getI32Type(), rewriter.getI32IntegerAttr(1));
3904 Value extElemOp =
nullptr;
3905 SmallVector<Value> operands(
3906 {adaptor.getSource(), adaptor.getIndex(), signCst});
3907 if (resultBitWidth == 8) {
3908 extElemOp = xllvm::VectorExtractElem8I512IntrOp::create(
3909 rewriter, loc, rewriter.getI32Type(),
3910 forceCastOperandsToSignature(
3911 rewriter, loc, operands,
3912 {VectorType::get({64}, rewriter.getI8Type()),
3913 rewriter.getI32Type(), rewriter.getI32Type()}));
3914 }
else if (resultBitWidth == 16) {
3915 extElemOp = xllvm::VectorExtractElem16I512IntrOp::create(
3916 rewriter, loc, rewriter.getI32Type(),
3917 forceCastOperandsToSignature(
3918 rewriter, loc, operands,
3919 {VectorType::get({32}, rewriter.getI16Type()),
3920 rewriter.getI32Type(), rewriter.getI32Type()}));
3921 }
else if (resultBitWidth == 32) {
3922 extElemOp = xllvm::VectorExtractElem32I512IntrOp::create(
3923 rewriter, loc, rewriter.getI32Type(),
3924 forceCastOperandsToSignature(
3925 rewriter, loc, operands,
3926 {VectorType::get({16}, rewriter.getI32Type()),
3927 rewriter.getI32Type(), rewriter.getI32Type()}));
3929 op.emitWarning() <<
"aievec.ext_elem conversion with result bit width "
3930 << resultBitWidth <<
" is not implemented.\n";
3935 if (llvm::isa<IntegerType>(resultType)) {
3936 if (resultBitWidth < 32) {
3939 if (resultBitWidth < 16) {
3940 auto i16Ty = rewriter.getI16Type();
3941 auto trunc16 = LLVM::TruncOp::create(rewriter, loc, i16Ty, extElemOp);
3942 rewriter.replaceOpWithNewOp<LLVM::TruncOp>(op, resultType,
3943 trunc16.getResult());
3945 rewriter.replaceOpWithNewOp<LLVM::TruncOp>(op, resultType, extElemOp);
3948 rewriter.replaceOp(op, extElemOp);
3952 if (resultBitWidth == 16) {
3953 extElemOp = LLVM::TruncOp::create(rewriter, loc, rewriter.getI16Type(),
3956 rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, resultType, extElemOp);
3964 :
public mlir::ConvertOpToLLVMPattern<aievec::FMAElemOp> {
3966 using ConvertOpToLLVMPattern<aievec::FMAElemOp>::ConvertOpToLLVMPattern;
3970 ConversionPatternRewriter &rewriter)
const override {
3971 auto loc = fmaOp.getLoc();
3972 auto lhs = adaptor.getLhs();
3973 auto rhs = adaptor.getRhs();
3974 auto acc = adaptor.getAcc();
3975 auto lhsTy = cast<VectorType>(lhs.getType());
3976 auto rhsTy = cast<VectorType>(rhs.getType());
3977 auto accTy = cast<VectorType>(acc.getType());
3983 if (lhsTy != flatLhsTy)
3984 lhs = vector::ShapeCastOp::create(rewriter, loc, flatLhsTy, lhs);
3985 if (rhsTy != flatRhsTy)
3986 rhs = vector::ShapeCastOp::create(rewriter, loc, flatRhsTy, rhs);
3987 if (accTy != flatAccTy)
3988 acc = vector::ShapeCastOp::create(rewriter, loc, flatAccTy, acc);
3991 Type i32ty = rewriter.getI32Type();
3992 auto confCst = LLVM::ConstantOp::create(
3993 rewriter, loc, i32ty,
3994 rewriter.getI32IntegerAttr(aiev2_vmac_compute_control(
4004 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
4005 if (flatLhsTy.getElementType().isBF16() &&
4006 flatLhsTy.getNumElements() < 32) {
4007 auto zero32 = LLVM::ConstantOp::create(rewriter, loc, i32ty,
4008 rewriter.getI32IntegerAttr(0));
4009 auto zeros_i16 = xllvm::VectorBroadcast16I512IntrOp::create(
4010 rewriter, loc, VectorType::get({32}, rewriter.getI16Type()), zero32);
4012 LLVM::BitcastOp::create(rewriter, loc, v32bf16Ty, zeros_i16);
4013 auto zeroVec = xllvm::ExtBF256BF512IntrOp::create(
4014 rewriter, loc, VectorType::get({16}, rewriter.getBF16Type()),
4015 zeros_bf16, zero32);
4017 auto idx1 = LLVM::ConstantOp::create(rewriter, loc, i32ty,
4018 rewriter.getI32IntegerAttr(1));
4020 auto lhsSet = xllvm::VectorSetBF512BF256IntrOp::create(
4021 rewriter, loc, v32bf16Ty, lhs, zero32);
4022 lhs = xllvm::UpdBF512BF256IntrOp::create(rewriter, loc, v32bf16Ty, lhsSet,
4025 auto rhsSet = xllvm::VectorSetBF512BF256IntrOp::create(
4026 rewriter, loc, v32bf16Ty, rhs, zero32);
4027 rhs = xllvm::UpdBF512BF256IntrOp::create(rewriter, loc, v32bf16Ty, rhsSet,
4032 auto v8i64Ty = VectorType::get({8}, rewriter.getI64Type());
4033 auto macIntrOp = xllvm::MacConfBF16IntrOp::create(
4034 rewriter, loc, v8i64Ty,
4035 forceCastOperandsToSignature(rewriter, loc, {lhs, rhs, acc, confCst},
4036 {v32bf16Ty, v32bf16Ty, v8i64Ty, i32ty}));
4040 forceCastValueToType(rewriter, loc, macIntrOp.getResult(), flatAccTy);
4041 if (flatAccTy != accTy)
4042 resVal = vector::ShapeCastOp::create(rewriter, loc, accTy, resVal);
4044 rewriter.replaceOp(fmaOp, resVal);
4052static Value lookThroughShapeCasts(Value v) {
4053 while (
auto castOp = v.getDefiningOp<vector::ShapeCastOp>())
4054 v = castOp.getSource();
4067static std::optional<bool> peelSignednessCast(Value &v) {
4068 auto castOp = v.getDefiningOp<UnrealizedConversionCastOp>();
4070 return std::nullopt;
4071 auto vecTy = dyn_cast<VectorType>(v.getType());
4073 vecTy ? dyn_cast<IntegerType>(vecTy.getElementType()) : IntegerType();
4074 if (!intTy || intTy.isSignless())
4075 return std::nullopt;
4076 v = castOp.getInputs()[0];
4077 return intTy.isSigned();
4110 std::optional<bool> castSigned = peelSignednessCast(v);
4112 Value orig = lookThroughShapeCasts(v);
4113 if (
auto extSIOp = orig.getDefiningOp<arith::ExtSIOp>())
4114 return {lookThroughShapeCasts(extSIOp.getIn()),
true};
4115 if (
auto extUIOp = orig.getDefiningOp<arith::ExtUIOp>())
4116 return {lookThroughShapeCasts(extUIOp.getIn()),
false};
4118 return {v, *castSigned};
4120 auto vecTy = dyn_cast<VectorType>(v.getType());
4122 vecTy ? dyn_cast<IntegerType>(vecTy.getElementType()) : IntegerType();
4123 return {v, !(intTy && intTy.isUnsigned())};
4130static Value transposeTile(OpBuilder &b, Location loc, Value v, int64_t rows,
4131 int64_t cols,
bool aie2p) {
4132 auto vecTy = cast<VectorType>(v.getType());
4133 std::optional<SmallVector<aievec::ShuffleMode, 2>> modes =
4134 aievec::getTileTransposeModes(aie2p, vecTy.getElementType(), rows, cols);
4135 assert(modes &&
"the matmul verifier admits only transposable tiles");
4138 Type i32ty = b.getI32Type();
4139 auto v16i32Ty = VectorType::get({16}, i32ty);
4140 SmallVector<Value, 2> modeCsts;
4141 auto createModeConstants = [&]() {
4142 for (aievec::ShuffleMode mode : *modes)
4143 modeCsts.push_back(LLVM::ConstantOp::create(
4144 b, loc, i32ty, b.getI32IntegerAttr(
static_cast<int32_t
>(mode))));
4146 auto shuffle = [&](Value x, Value y, Value modeCst) -> Value {
4148 return xllvm::VectorShuffleAIE2pIntrOp::create(b, loc, v16i32Ty, x, y,
4150 return xllvm::VectorShuffleIntrOp::create(b, loc, v16i32Ty, x, y, modeCst);
4152 int64_t numElems = vecTy.getNumElements();
4153 if (modes->size() == 2) {
4154 auto halves = forceCastValueToType(b, loc, v, VectorType::get({32}, i32ty));
4155 auto lo = vector::ShuffleOp::create(
4156 b, loc, halves, halves, llvm::to_vector(llvm::seq<int64_t>(0, 16)));
4157 auto hi = vector::ShuffleOp::create(
4158 b, loc, halves, halves, llvm::to_vector(llvm::seq<int64_t>(16, 32)));
4159 createModeConstants();
4160 Value resLo = shuffle(lo, hi, modeCsts[0]);
4161 Value resHi = shuffle(lo, hi, modeCsts[1]);
4162 auto res = vector::ShuffleOp::create(
4163 b, loc, resLo, resHi, llvm::to_vector(llvm::seq<int64_t>(0, 32)));
4164 return forceCastValueToType(b, loc, res, vecTy);
4166 bool half = numElems * vecTy.getElementTypeBitWidth() == 256;
4169 SmallVector<int64_t> padMask =
4170 llvm::to_vector(llvm::seq<int64_t>(0, numElems));
4171 padMask.append(numElems, vector::ShuffleOp::kPoisonIndex);
4172 x = vector::ShuffleOp::create(b, loc, v, v, padMask);
4174 Value xi = forceCastValueToType(b, loc, x, v16i32Ty);
4177 : xllvm::UndefV16I32IntrOp::create(b, loc, v16i32Ty).getResult();
4178 createModeConstants();
4180 forceCastValueToType(b, loc, shuffle(xi, y, modeCsts[0]), x.getType());
4182 res = vector::ShuffleOp::create(
4183 b, loc, res, res, llvm::to_vector(llvm::seq<int64_t>(0, numElems)));
4189static Value materializeOperandLayout(OpBuilder &b, Location loc, Value flat,
4190 VectorType storedTy,
bool stored,
4191 bool native,
bool aie2p) {
4192 if (stored == native)
4194 return transposeTile(b, loc, flat, storedTy.getDimSize(0),
4195 storedTy.getDimSize(1), aie2p);
4199 :
public mlir::ConvertOpToLLVMPattern<aievec::MatMulOp> {
4200 using ConvertOpToLLVMPattern<aievec::MatMulOp>::ConvertOpToLLVMPattern;
4202 enum class Accumulator { I32, I64, BF16 };
4205 static std::tuple<Accumulator, int, int, int>
4209 case K::AIE2_I8xI4_4x16x8:
4210 return {Accumulator::I32, 0, 0, 0};
4211 case K::AIE2_I8xI8_4x8x8:
4212 return {Accumulator::I32, 0, 1, 0};
4213 case K::AIE2_I16xI8_4x4x8:
4214 return {Accumulator::I32, 0, 2, 0};
4215 case K::AIE2_I16xI16_4x2x8:
4216 return {Accumulator::I32, 0, 3, 0};
4217 case K::AIE2_I16xI8_2x8x8_Acc64:
4218 return {Accumulator::I64, 1, 2, 0};
4219 case K::AIE2_I16xI8_4x8x4_Acc64:
4220 return {Accumulator::I64, 1, 2, 1};
4221 case K::AIE2_I16xI16_2x4x8_Acc64:
4222 return {Accumulator::I64, 1, 3, 0};
4223 case K::AIE2_I16xI16_4x4x4_Acc64:
4224 return {Accumulator::I64, 1, 3, 1};
4225 case K::AIE2_I32xI16_4x2x4_Acc64:
4226 return {Accumulator::I64, 1, 0, 0};
4227 case K::AIE2_BF16_4x8x4:
4228 return {Accumulator::BF16, 2, 3, 0};
4230 llvm_unreachable(
"not an AIE2 recipe");
4235 matchAndRewrite(aievec::MatMulOp op, OpAdaptor adaptor,
4236 ConversionPatternRewriter &rewriter)
const override {
4237 auto lhsTy = cast<VectorType>(op.getLhs().getType());
4238 auto rhsTy = cast<VectorType>(op.getRhs().getType());
4239 auto accTy = cast<VectorType>(op.getAcc().getType());
4240 std::optional<aievec::MatMulRecipe> recipe =
4242 op.getLhsTransposed(), op.getRhsTransposed());
4244 return rewriter.notifyMatchFailure(op,
"not an AIE2 matmul tile");
4245 auto [accKind, amode, bmode, variant] = getAccAndModes(recipe->kind);
4250 Value lhs = adaptor.getLhs();
4251 Value rhs = adaptor.getRhs();
4252 int signX = 0, signY = 0;
4253 if (accKind != Accumulator::BF16) {
4254 auto narrow = [](Value v, VectorType ty,
int &sign) {
4257 if (getElementTypeOrSelf(resolved.
narrowed).getIntOrFloatBitWidth() ==
4258 ty.getElementTypeBitWidth())
4262 lhs = narrow(lhs, lhsTy, signX);
4263 rhs = narrow(rhs, rhsTy, signY);
4265 int conf = aiev2_vmac_compute_control(
4266 signX, signY, amode, bmode, variant,
4270 Location loc = op.getLoc();
4273 auto flatten = [&](Value v) -> Value {
4275 if (v.getType() == flatTy)
4277 return vector::ShapeCastOp::create(rewriter, loc, flatTy, v);
4282 Value acc = vector::ShapeCastOp::create(rewriter, loc, accFlattenedVecTy,
4284 lhs = materializeOperandLayout(rewriter, loc, lhs, lhsTy,
4285 op.getLhsTransposed(), recipe->lhsKM,
4287 rhs = materializeOperandLayout(rewriter, loc, rhs, rhsTy,
4288 op.getRhsTransposed(), recipe->rhsNK,
4291 Type i32ty = rewriter.getI32Type();
4292 auto confCst = LLVM::ConstantOp::create(rewriter, loc, i32ty,
4293 rewriter.getI32IntegerAttr(conf));
4294 SmallVector<Value> operands({lhs, rhs, acc, confCst});
4296 if (accKind == Accumulator::BF16)
4298 xllvm::MacConfBF16IntrOp::create(
4299 rewriter, loc, VectorType::get({8}, rewriter.getI64Type()),
4300 forceCastOperandsToSignature(
4301 rewriter, loc, operands,
4302 {VectorType::get({32}, rewriter.getBF16Type()),
4303 VectorType::get({32}, rewriter.getBF16Type()),
4304 VectorType::get({8}, rewriter.getI64Type()), i32ty}))
4307 SmallVector<Type> intrFuncSig(
4308 {VectorType::get({64}, rewriter.getI8Type()),
4309 VectorType::get({16}, i32ty),
4310 VectorType::get({16}, rewriter.getI64Type()), i32ty});
4311 VectorType v16xi64ty = VectorType::get({16}, rewriter.getI64Type());
4312 if (accKind == Accumulator::I32)
4313 matMulResVal = xllvm::MacConfAcc32IntrOp::create(
4314 rewriter, loc, v16xi64ty,
4315 forceCastOperandsToSignature(rewriter, loc, operands,
4319 matMulResVal = xllvm::MacConfAcc64IntrOp::create(
4320 rewriter, loc, v16xi64ty,
4321 forceCastOperandsToSignature(rewriter, loc, operands,
4327 bitcastValueToType(rewriter, loc, matMulResVal, accFlattenedVecTy);
4329 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, op.getType(),
4338static Value widenBF16ToAccFloat(OpBuilder &rewriter, Location loc, Type i32ty,
4340 auto v64bf16Ty = VectorType::get({64}, rewriter.getBF16Type());
4341 auto ones = LLVM::ConstantOp::create(
4342 rewriter, loc, v64bf16Ty,
4343 DenseElementsAttr::get(
4344 v64bf16Ty, rewriter.getFloatAttr(rewriter.getBF16Type(), 1.0)));
4346 auto conf = LLVM::ConstantOp::create(rewriter, loc, i32ty,
4347 rewriter.getI32IntegerAttr(60));
4348 return xllvm::MulConfBF16I1024ACC2048AIE2pIntrOp::create(
4349 rewriter, loc, VectorType::get({64}, rewriter.getF32Type()), v64bf16,
4356static Value perform8x8x4MatMul(OpBuilder &rewriter, Location loc, Type i32ty,
4357 Value lhsKM64bf16, Value rhs32bf16,
4359 auto v32bf16Ty = VectorType::get({32}, rewriter.getBF16Type());
4360 auto v32f32Ty = VectorType::get({32}, rewriter.getF32Type());
4364 vector::ShuffleOp::create(rewriter, loc, lhsKM64bf16, lhsKM64bf16,
4365 llvm::to_vector(llvm::seq<int64_t>(0, 32)));
4367 vector::ShuffleOp::create(rewriter, loc, lhsKM64bf16, lhsKM64bf16,
4368 llvm::to_vector(llvm::seq<int64_t>(32, 64)));
4371 auto extractBroadcastShuffle = [&](Value src,
int idx) -> Value {
4372 SmallVector<int64_t> extractMask;
4373 int startIdx = idx * 8;
4374 for (
int i = 0; i < 8; ++i)
4375 extractMask.push_back(startIdx + i);
4377 for (
int rep = 0; rep < 3; ++rep) {
4378 for (
int i = 0; i < 8; ++i)
4379 extractMask.push_back(startIdx + i);
4382 vector::ShuffleOp::create(rewriter, loc, src, src, extractMask);
4385 auto broadI32 = forceCastValueToType(rewriter, loc, broadcasted,
4386 VectorType::get({16}, i32ty));
4387 auto shuffleMode4x8 = LLVM::ConstantOp::create(
4388 rewriter, loc, i32ty, rewriter.getI32IntegerAttr(29));
4389 auto shuffled = xllvm::VectorShuffleAIE2pIntrOp::create(
4390 rewriter, loc, VectorType::get({16}, i32ty), broadI32, broadI32,
4393 return forceCastValueToType(rewriter, loc, shuffled, v32bf16Ty);
4397 SmallVector<Value> rowVectors;
4398 for (
int i = 0; i < 4; ++i)
4399 rowVectors.push_back(extractBroadcastShuffle(xaBF16, i));
4400 for (
int i = 0; i < 4; ++i)
4401 rowVectors.push_back(extractBroadcastShuffle(xbBF16, i));
4404 auto extractBroadcast4 = [&](Value src,
int idx) -> Value {
4405 SmallVector<int64_t> mask;
4406 int startIdx = idx * 4;
4408 for (
int rep = 0; rep < 8; ++rep) {
4409 for (
int i = 0; i < 4; ++i)
4410 mask.push_back(startIdx + i);
4412 return vector::ShuffleOp::create(rewriter, loc, src, src, mask);
4416 SmallVector<Value> colVectors;
4417 for (
int i = 0; i < 8; ++i)
4418 colVectors.push_back(extractBroadcast4(rhs32bf16, i));
4421 auto conf60 = LLVM::ConstantOp::create(rewriter, loc, i32ty,
4422 rewriter.getI32IntegerAttr(60));
4424 Value acc = acc32f32;
4425 for (
int i = 0; i < 8; ++i) {
4426 acc = xllvm::MacConfBF16I512ACC1024AIE2pIntrOp::create(
4427 rewriter, loc, v32f32Ty, rowVectors[i], colVectors[i], acc, conf60);
4437 :
public mlir::ConvertOpToLLVMPattern<aievec::MatMulOp_AIE2P> {
4438 using ConvertOpToLLVMPattern<aievec::MatMulOp_AIE2P>::ConvertOpToLLVMPattern;
4441 matchAndRewrite(aievec::MatMulOp_AIE2P op, OpAdaptor adaptor,
4442 ConversionPatternRewriter &rewriter)
const override {
4451 :
public mlir::ConvertOpToLLVMPattern<aievec::ToBFP16Op> {
4452 using ConvertOpToLLVMPattern<aievec::ToBFP16Op>::ConvertOpToLLVMPattern;
4455 matchAndRewrite(aievec::ToBFP16Op op, OpAdaptor adaptor,
4456 ConversionPatternRewriter &rewriter)
const override {
4457 Location loc = op.getLoc();
4458 VectorType srcTy = op.getSource().getType();
4459 Value flat = vector::ShapeCastOp::create(
4461 if (op.getTranspose())
4462 flat = transposeTile(rewriter, loc, flat, srcTy.getDimSize(0),
4463 srcTy.getDimSize(1),
true);
4464 Type i32ty = rewriter.getI32Type();
4466 auto v32f32Ty = VectorType::get({32}, rewriter.getF32Type());
4468 f64 = widenBF16ToAccFloat(rewriter, loc, i32ty, flat);
4469 }
else if (srcTy.getNumElements() == 64) {
4471 for (int64_t h = 0; h < 2; ++h) {
4472 Value v = vector::ShuffleOp::create(
4473 rewriter, loc, flat, flat,
4474 llvm::to_vector(llvm::seq<int64_t>(32 * h, 32 * h + 32)));
4475 half[h] = xllvm::Vector32BF16ToV32AccFloatAIE2pIntrOp::create(
4476 rewriter, loc, v32f32Ty, v);
4479 vector::ShuffleOp::create(rewriter, loc, half[0], half[1],
4480 llvm::to_vector(llvm::seq<int64_t>(0, 64)));
4482 SmallVector<int64_t> padMask = llvm::to_vector(llvm::seq<int64_t>(0, 32));
4483 padMask.append(32, vector::ShuffleOp::kPoisonIndex);
4484 Value f32 = xllvm::Vector32BF16ToV32AccFloatAIE2pIntrOp::create(
4485 rewriter, loc, v32f32Ty, flat);
4486 f64 = vector::ShuffleOp::create(rewriter, loc, f32, f32, padMask);
4488 auto bfpStructTy = LLVM::LLVMStructType::getLiteral(
4489 rewriter.getContext(), {VectorType::get({64}, rewriter.getI8Type()),
4490 VectorType::get({8}, rewriter.getI8Type())});
4491 Value bfp = xllvm::Vector64AccFloatToV64BFP16EBS8AIE2pIntrOp::create(
4492 rewriter, loc, bfpStructTy, f64);
4493 rewriter.replaceOp(op,
4494 {LLVM::ExtractValueOp::create(rewriter, loc, bfp, 0),
4495 LLVM::ExtractValueOp::create(rewriter, loc, bfp, 1)});
4503static Value bytePointer(OpBuilder &b, Location loc, Value base, Value offset) {
4504 auto meta = memref::ExtractStridedMetadataOp::create(b, loc, base);
4505 Value start = memref::ExtractAlignedPointerAsIndexOp::create(
4506 b, loc, meta.getBaseBuffer());
4507 Value off = arith::AddIOp::create(b, loc, meta.getOffset(), offset);
4508 Value addr = arith::IndexCastOp::create(
4509 b, loc, b.getI32Type(), arith::AddIOp::create(b, loc, start, off));
4510 return LLVM::IntToPtrOp::create(
4511 b, loc, LLVM::LLVMPointerType::get(b.getContext()), addr);
4516static LLVM::LLVMStructType fifoLdType(MLIRContext *ctx,
bool popped) {
4517 Type i32 = IntegerType::get(ctx, 32), i8 = IntegerType::get(ctx, 8);
4518 SmallVector<Type> fields = {LLVM::LLVMPointerType::get(ctx),
4519 VectorType::get({32}, i32), i32};
4521 fields.push_back(VectorType::get({64}, i8));
4522 fields.push_back(VectorType::get({8}, i8));
4524 return LLVM::LLVMStructType::getLiteral(ctx, fields);
4529static Value fifoLdFill(OpBuilder &b, Location loc, Value ptr) {
4530 Type i32 = b.getI32Type();
4531 auto stateTy = VectorType::get({32}, i32);
4532 Value state = LLVM::UndefOp::create(b, loc, stateTy);
4533 Value count = LLVM::ConstantOp::create(b, loc, i32, b.getI32IntegerAttr(0));
4534 return xllvm::FifoLdFillAIE2pIntrOp::create(
4535 b, loc, fifoLdType(b.getContext(),
false), ptr, state, count);
4540 :
public mlir::ConvertOpToLLVMPattern<aievec::LoadBFP16Op> {
4541 using ConvertOpToLLVMPattern<aievec::LoadBFP16Op>::ConvertOpToLLVMPattern;
4544 matchAndRewrite(aievec::LoadBFP16Op op, OpAdaptor adaptor,
4545 ConversionPatternRewriter &rewriter)
const override {
4546 Location loc = op.getLoc();
4548 fifoLdFill(rewriter, loc,
4549 bytePointer(rewriter, loc, op.getBase(), op.getOffset()));
4550 Value popped = xllvm::FifoLdPopBFP576AIE2pIntrOp::create(
4551 rewriter, loc, fifoLdType(rewriter.getContext(),
true),
4552 LLVM::ExtractValueOp::create(rewriter, loc, filled, 0),
4553 LLVM::ExtractValueOp::create(rewriter, loc, filled, 1),
4554 LLVM::ExtractValueOp::create(rewriter, loc, filled, 2));
4556 op, {LLVM::ExtractValueOp::create(rewriter, loc, popped, 3),
4557 LLVM::ExtractValueOp::create(rewriter, loc, popped, 4)});
4564 :
public mlir::ConvertOpToLLVMPattern<aievec::FifoStartOp> {
4565 using ConvertOpToLLVMPattern<aievec::FifoStartOp>::ConvertOpToLLVMPattern;
4568 matchAndRewrite(aievec::FifoStartOp op, OpAdaptor adaptor,
4569 ConversionPatternRewriter &rewriter)
const override {
4570 Location loc = op.getLoc();
4571 Type i32 = rewriter.getI32Type();
4572 auto stateTy = VectorType::get({32}, i32);
4573 Value next = bytePointer(rewriter, loc, op.getBase(), op.getOffset());
4575 Value state = LLVM::UndefOp::create(rewriter, loc, stateTy);
4576 Value count = LLVM::ConstantOp::create(rewriter, loc, i32,
4577 rewriter.getI32IntegerAttr(0));
4578 rewriter.replaceOp(op, {next, state, count});
4585 :
public mlir::ConvertOpToLLVMPattern<aievec::FifoStPushBFP16Op> {
4586 using ConvertOpToLLVMPattern<
4587 aievec::FifoStPushBFP16Op>::ConvertOpToLLVMPattern;
4590 matchAndRewrite(aievec::FifoStPushBFP16Op op, OpAdaptor adaptor,
4591 ConversionPatternRewriter &rewriter)
const override {
4592 Location loc = op.getLoc();
4593 MLIRContext *ctx = rewriter.getContext();
4594 Value ptr = adaptor.getAddr();
4595 Value pushed = xllvm::FifoStPushBFP576AIE2pIntrOp::create(
4596 rewriter, loc, fifoLdType(ctx,
false), ptr, adaptor.getMantissas(),
4597 adaptor.getExponents(), adaptor.getState(), adaptor.getCount());
4598 Value flushed = xllvm::FifoStFlushAIE2pIntrOp::create(
4599 rewriter, loc, fifoLdType(ctx,
false),
4600 LLVM::ExtractValueOp::create(rewriter, loc, pushed, 0),
4601 LLVM::ExtractValueOp::create(rewriter, loc, pushed, 1),
4602 LLVM::ExtractValueOp::create(rewriter, loc, pushed, 2));
4603 Value next = LLVM::ExtractValueOp::create(rewriter, loc, flushed, 0);
4605 op, {next, LLVM::ExtractValueOp::create(rewriter, loc, flushed, 1),
4606 LLVM::ExtractValueOp::create(rewriter, loc, flushed, 2)});
4614 :
public mlir::ConvertOpToLLVMPattern<aievec::FifoLdPopBFP16Op> {
4615 using ConvertOpToLLVMPattern<
4616 aievec::FifoLdPopBFP16Op>::ConvertOpToLLVMPattern;
4619 matchAndRewrite(aievec::FifoLdPopBFP16Op op, OpAdaptor adaptor,
4620 ConversionPatternRewriter &rewriter)
const override {
4621 Location loc = op.getLoc();
4622 MLIRContext *ctx = rewriter.getContext();
4623 Value ptr = adaptor.getAddr();
4624 Value state = adaptor.getState(), count = adaptor.getCount();
4626 if (!op.getNoFill()) {
4627 Value filled = xllvm::FifoLdFillAIE2pIntrOp::create(
4628 rewriter, loc, fifoLdType(ctx,
false), ptr, state, count);
4629 ptr = LLVM::ExtractValueOp::create(rewriter, loc, filled, 0);
4630 state = LLVM::ExtractValueOp::create(rewriter, loc, filled, 1);
4631 count = LLVM::ExtractValueOp::create(rewriter, loc, filled, 2);
4633 Value popped = xllvm::FifoLdPopBFP576AIE2pIntrOp::create(
4634 rewriter, loc, fifoLdType(ctx,
true), ptr, state, count);
4635 SmallVector<Value> results;
4636 for (int64_t i = 0; i < 5; ++i)
4637 results.push_back(LLVM::ExtractValueOp::create(rewriter, loc, popped, i));
4638 rewriter.replaceOp(op, results);
4646 :
public mlir::ConvertOpToLLVMPattern<aievec::MatMulBFP16Op_AIE2P> {
4647 using ConvertOpToLLVMPattern<
4648 aievec::MatMulBFP16Op_AIE2P>::ConvertOpToLLVMPattern;
4651 matchAndRewrite(aievec::MatMulBFP16Op_AIE2P op, OpAdaptor adaptor,
4652 ConversionPatternRewriter &rewriter)
const override {
4653 Location loc = op.getLoc();
4654 auto accTy = cast<VectorType>(op.getAcc().getType());
4656 Type i32ty = rewriter.getI32Type();
4657 int64_t lanes = accTy.getNumElements();
4658 Value acc = forceCastValueToType(
4660 vector::ShapeCastOp::create(rewriter, loc, accFlatTy, adaptor.getAcc()),
4661 VectorType::get({lanes}, i32ty));
4663 SmallVector<int64_t> padMask = llvm::to_vector(llvm::seq<int64_t>(0, 32));
4664 padMask.append(32, vector::ShuffleOp::kPoisonIndex);
4665 acc = vector::ShuffleOp::create(rewriter, loc, acc, acc, padMask);
4667 auto conf = LLVM::ConstantOp::create(rewriter, loc, i32ty,
4668 rewriter.getI32IntegerAttr(780));
4669 Value res = xllvm::MacConfBFP576ACC2048AIE2pIntrOp::create(
4670 rewriter, loc, VectorType::get({64}, i32ty), adaptor.getLhsMantissas(),
4671 adaptor.getLhsExponents(), adaptor.getRhsMantissas(),
4672 adaptor.getRhsExponents(), acc, conf);
4674 res = vector::ShuffleOp::create(
4675 rewriter, loc, res, res, llvm::to_vector(llvm::seq<int64_t>(0, 32)));
4676 res = forceCastValueToType(rewriter, loc, res, accFlatTy);
4677 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, accTy, res);
4683 :
public mlir::ConvertOpToLLVMPattern<aievec::MatMulOp_AIE2P> {
4684 using ConvertOpToLLVMPattern<aievec::MatMulOp_AIE2P>::ConvertOpToLLVMPattern;
4687 matchAndRewrite(aievec::MatMulOp_AIE2P op, OpAdaptor adaptor,
4688 ConversionPatternRewriter &rewriter)
const override {
4690 auto lhsTy = cast<VectorType>(op.getLhs().getType());
4691 auto rhsTy = cast<VectorType>(op.getRhs().getType());
4692 auto accTy = cast<VectorType>(op.getAcc().getType());
4693 std::optional<aievec::MatMulRecipe> recipe =
4695 op.getLhsTransposed(), op.getRhsTransposed());
4697 return rewriter.notifyMatchFailure(op,
"not an AIE2P matmul tile");
4703 Value lhsOperand = adaptor.getLhs();
4704 Value rhsOperand = adaptor.getRhs();
4705 bool lhsIsSigned = resolveMatMulOperand(lhsOperand).isSigned;
4706 bool rhsIsSigned = resolveMatMulOperand(rhsOperand).isSigned;
4707 peelSignednessCast(lhsOperand);
4708 peelSignednessCast(rhsOperand);
4710 Location loc = op.getLoc();
4713 Value lhsFlat = vector::ShapeCastOp::create(
4717 Value rhsFlat = vector::ShapeCastOp::create(
4722 Value acc = vector::ShapeCastOp::create(rewriter, loc, accFlattenedVecTy,
4724 auto getLhs = [&]() {
4725 return materializeOperandLayout(rewriter, loc, lhsFlat, lhsTy,
4726 op.getLhsTransposed(), recipe->lhsKM,
4729 auto getRhs = [&]() {
4730 return materializeOperandLayout(rewriter, loc, rhsFlat, rhsTy,
4731 op.getRhsTransposed(), recipe->rhsNK,
4734 Type i32ty = rewriter.getI32Type();
4737 switch (recipe->kind) {
4738 case K::AIE2P_I8_8x8x8: {
4745 8 | ((lhsIsSigned ? 1 : 0) << 9) | ((rhsIsSigned ? 1 : 0) << 8);
4746 auto confCst = LLVM::ConstantOp::create(rewriter, loc, i32ty,
4747 rewriter.getI32IntegerAttr(conf));
4749 xllvm::MacConfI512ACC2048AIE2pIntrOp::create(
4750 rewriter, loc, VectorType::get({32}, rewriter.getI64Type()),
4751 forceCastOperandsToSignature(
4752 rewriter, loc, {getLhs(), getRhs(), acc, confCst},
4753 {VectorType::get({16}, rewriter.getI32Type()),
4754 VectorType::get({32}, rewriter.getI16Type()),
4755 VectorType::get({32}, rewriter.getI64Type()), i32ty}))
4759 case K::AIE2P_I16_8x2x8: {
4764 SmallVector<int64_t> padMask = llvm::to_vector(llvm::seq<int64_t>(0, 16));
4765 padMask.append(48, vector::ShuffleOp::kPoisonIndex);
4766 Value lhs = getLhs(), rhs = getRhs();
4768 vector::ShuffleOp::create(rewriter, loc, lhs, lhs, padMask);
4770 vector::ShuffleOp::create(rewriter, loc, rhs, rhs, padMask);
4771 auto confCst = LLVM::ConstantOp::create(rewriter, loc, i32ty,
4772 rewriter.getI32IntegerAttr(24));
4774 xllvm::MacConfI1024ACC2048AIE2pIntrOp::create(
4775 rewriter, loc, VectorType::get({32}, rewriter.getI64Type()),
4776 forceCastOperandsToSignature(
4777 rewriter, loc, {lhsPadded, rhsPadded, acc, confCst},
4778 {VectorType::get({32}, rewriter.getI32Type()),
4779 VectorType::get({64}, rewriter.getI16Type()),
4780 VectorType::get({32}, rewriter.getI64Type()), i32ty}))
4784 case K::AIE2P_BF16Outer_8x1x8: {
4788 auto v64f32Ty = VectorType::get({64}, rewriter.getF32Type());
4791 SmallVector<int64_t> replicateMask;
4792 for (
int rep = 0; rep < 8; ++rep)
4793 for (
int i = 0; i < 8; ++i)
4794 replicateMask.push_back(i);
4795 Value lhs = lhsFlat, rhs = rhsFlat;
4797 vector::ShuffleOp::create(rewriter, loc, lhs, lhs, replicateMask);
4800 SmallVector<int64_t> transposeMask;
4801 for (
int c = 0; c < 8; ++c)
4802 for (
int r = 0; r < 8; ++r)
4803 transposeMask.push_back(r * 8 + c);
4804 auto lhs64bf16Transposed = vector::ShuffleOp::create(
4805 rewriter, loc, lhs64bf16, lhs64bf16, transposeMask);
4807 vector::ShuffleOp::create(rewriter, loc, rhs, rhs, replicateMask);
4810 auto confCst = LLVM::ConstantOp::create(rewriter, loc, i32ty,
4811 rewriter.getI32IntegerAttr(60));
4812 matMulResVal = xllvm::MacConfBF16I1024ACC2048AIE2pIntrOp::create(
4813 rewriter, loc, v64f32Ty, lhs64bf16Transposed, rhs64bf16, acc,
4817 case K::AIE2P_BF16Elem_4x8x4: {
4823 SmallVector<int64_t> lhsPadMask =
4824 llvm::to_vector(llvm::seq<int64_t>(0, 32));
4825 lhsPadMask.append(32, vector::ShuffleOp::kPoisonIndex);
4826 Value lhs = getLhs();
4828 vector::ShuffleOp::create(rewriter, loc, lhs, lhs, lhsPadMask);
4829 lhsPadded = transposeTile(rewriter, loc, lhsPadded, 8, 8,
4832 SmallVector<int64_t> accPadMask =
4833 llvm::to_vector(llvm::seq<int64_t>(0, 16));
4834 accPadMask.append(16, vector::ShuffleOp::kPoisonIndex);
4836 vector::ShuffleOp::create(rewriter, loc, acc, acc, accPadMask);
4838 Value acc32 = perform8x8x4MatMul(rewriter, loc, i32ty, lhsPadded,
4839 getRhs(), accPadded);
4842 vector::ShuffleOp::create(rewriter, loc, acc32, acc32,
4843 llvm::to_vector(llvm::seq<int64_t>(0, 16)));
4846 case K::AIE2P_BF16Elem_8x8x4: {
4848 Value lhs = getLhs();
4849 Value rhs = getRhs();
4850 matMulResVal = perform8x8x4MatMul(rewriter, loc, i32ty, lhs, rhs, acc);
4854 return rewriter.notifyMatchFailure(op,
"not an AIE2P recipe");
4859 forceCastValueToType(rewriter, loc, matMulResVal, accFlattenedVecTy);
4861 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, op.getType(),
4873 using ConvertOpToLLVMPattern<aievec::CastOp>::ConvertOpToLLVMPattern;
4876 static bool isConstantZero(Value val) {
4877 DenseElementsAttr denseAttr;
4880 if (
auto arithConstOp = val.getDefiningOp<arith::ConstantOp>()) {
4881 denseAttr = dyn_cast<DenseElementsAttr>(arithConstOp.getValue());
4882 }
else if (
auto llvmConstOp = val.getDefiningOp<LLVM::ConstantOp>()) {
4883 denseAttr = dyn_cast<DenseElementsAttr>(llvmConstOp.getValue());
4886 if (!denseAttr || !denseAttr.isSplat())
4889 auto splatAttr = denseAttr.getSplatValue<Attribute>();
4890 if (
auto floatAttr = dyn_cast<FloatAttr>(splatAttr))
4891 return floatAttr.getValue().isZero();
4892 if (
auto intAttr = dyn_cast<IntegerAttr>(splatAttr))
4893 return intAttr.getValue().isZero();
4899 matchAndRewrite(aievec::CastOp castOp, OpAdaptor adaptor,
4900 ConversionPatternRewriter &rewriter)
const override {
4904 if (!castOp.getIsResAcc() || !isConstantZero(adaptor.getSource())) {
4906 rewriter.replaceOp(castOp, adaptor.getSource());
4910 Location loc = castOp.getLoc();
4911 auto srcVecType = cast<VectorType>(castOp.getSource().getType());
4912 Type srcElemType = srcVecType.getElementType();
4916 if (srcElemType.isF32() && lanes == 16) {
4918 auto zeroAcc1024 = xllvm::VectorBroadcastZeroAcc1024IntrOp::create(
4919 rewriter, loc, VectorType::get({16}, rewriter.getI64Type()));
4922 SmallVector<int64_t> extractMask = {0, 1, 2, 3, 4, 5, 6, 7};
4923 auto zeroAcc512 = vector::ShuffleOp::create(rewriter, loc, zeroAcc1024,
4924 zeroAcc1024, extractMask);
4927 auto result = LLVM::BitcastOp::create(
4928 rewriter, loc, VectorType::get({16}, rewriter.getF32Type()),
4931 rewriter.replaceOp(castOp, result);
4936 rewriter.replaceOp(castOp, adaptor.getSource());
4949 :
public mlir::ConvertOpToLLVMPattern<aievec::CastOp> {
4950 using ConvertOpToLLVMPattern<aievec::CastOp>::ConvertOpToLLVMPattern;
4953 matchAndRewrite(aievec::CastOp castOp, OpAdaptor adaptor,
4954 ConversionPatternRewriter &rewriter)
const override {
4955 Value src = adaptor.getSource();
4956 Type srcTy = src.getType();
4957 Type dstTy = getTypeConverter()->convertType(castOp.getResult().getType());
4958 if (srcTy == dstTy) {
4959 rewriter.replaceOp(castOp, src);
4962 rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(castOp, dstTy, src);
4968 :
public mlir::ConvertOpToLLVMPattern<aievec::ShuffleOp> {
4969 using ConvertOpToLLVMPattern<aievec::ShuffleOp>::ConvertOpToLLVMPattern;
4972 matchAndRewrite(aievec::ShuffleOp shuffleOp, OpAdaptor adaptor,
4973 ConversionPatternRewriter &rewriter)
const override {
4975 return shuffleOp.emitOpError()
4976 <<
"shuffle mode '" << stringifyShuffleMode(shuffleOp.getMode())
4977 <<
"' is not available on AIE2";
4978 auto loc = shuffleOp.getLoc();
4979 auto lhs = adaptor.getLhs();
4980 auto rhs = adaptor.getRhs();
4981 auto i32ty = rewriter.getI32Type();
4982 auto v16xi32ty = VectorType::get({16}, i32ty);
4984 rhs = xllvm::UndefV16I32IntrOp::create(rewriter, loc, v16xi32ty);
4987 LLVM::ConstantOp::create(rewriter, loc, i32ty,
4988 static_cast<int32_t
>(shuffleOp.getMode()))
4990 auto vShuffleVal = xllvm::VectorShuffleIntrOp::create(
4991 rewriter, loc, v16xi32ty,
4992 forceCastOperandsToSignature(
4994 {lhs, rhs, modeAttrVal},
4995 {v16xi32ty, v16xi32ty, i32ty}))
4998 vShuffleVal = forceCastValueToType(rewriter, loc, vShuffleVal,
4999 shuffleOp.getResult().getType());
5001 rewriter.replaceOp(shuffleOp, vShuffleVal);
5010 :
public mlir::ConvertOpToLLVMPattern<aievec::ShuffleOp> {
5011 using ConvertOpToLLVMPattern<aievec::ShuffleOp>::ConvertOpToLLVMPattern;
5014 matchAndRewrite(aievec::ShuffleOp shuffleOp, OpAdaptor adaptor,
5015 ConversionPatternRewriter &rewriter)
const override {
5016 auto loc = shuffleOp.getLoc();
5017 auto i32ty = rewriter.getI32Type();
5018 auto v16xi32ty = VectorType::get({16}, i32ty);
5019 Value lhs = adaptor.getLhs();
5020 Value rhs = adaptor.getRhs();
5023 auto mode = LLVM::ConstantOp::create(
5024 rewriter, loc, i32ty,
static_cast<int32_t
>(shuffleOp.getMode()));
5025 Value v = xllvm::VectorShuffleAIE2pIntrOp::create(
5026 rewriter, loc, v16xi32ty,
5027 forceCastOperandsToSignature(rewriter, loc, {lhs, rhs, mode},
5028 {v16xi32ty, v16xi32ty, i32ty}))
5030 rewriter.replaceOp(shuffleOp,
5031 forceCastValueToType(rewriter, loc, v,
5032 shuffleOp.getResult().getType()));
5041 :
public mlir::ConvertOpToLLVMPattern<aievec::InvOp> {
5043 using ConvertOpToLLVMPattern<aievec::InvOp>::ConvertOpToLLVMPattern;
5047 ConversionPatternRewriter &rewriter)
const override {
5048 auto loc = invOp.getLoc();
5049 auto operandType = adaptor.getSource().getType();
5052 if (operandType.isF32()) {
5053 auto invResult = xllvm::InvAIE2pIntrOp::create(
5054 rewriter, loc, rewriter.getF32Type(), adaptor.getSource());
5055 rewriter.replaceOp(invOp, invResult);
5060 auto vecType = dyn_cast<VectorType>(operandType);
5061 if (!vecType || !vecType.getElementType().isF32())
5066 Value result = LLVM::PoisonOp::create(rewriter, loc, vecType);
5068 for (
int i = 0; i < numElements; ++i) {
5070 auto indexCst = LLVM::ConstantOp::create(
5071 rewriter, loc, rewriter.getI64Type(), rewriter.getI64IntegerAttr(i));
5072 auto extractedElem = LLVM::ExtractElementOp::create(
5073 rewriter, loc, adaptor.getSource(), indexCst);
5076 auto invResult = xllvm::InvAIE2pIntrOp::create(
5077 rewriter, loc, rewriter.getF32Type(), extractedElem);
5080 result = LLVM::InsertElementOp::create(rewriter, loc, vecType, result,
5081 invResult, indexCst);
5084 rewriter.replaceOp(invOp, result);
5094template <
typename IntrOpTy>
5095static Value applyAccFloatUnaryIntrinsic(ConversionPatternRewriter &rewriter,
5096 Location loc, Value source,
5097 Type resultElemType) {
5098 auto srcType = cast<VectorType>(source.getType());
5099 unsigned laneSize = getVectorLaneSize(srcType);
5100 auto v16bf16Ty = VectorType::get({16}, rewriter.getBF16Type());
5101 auto v16f32Ty = VectorType::get({16}, rewriter.getF32Type());
5102 auto toF32 = [&](Value v16bf16) -> Value {
5103 return xllvm::Vector16BF16ToV16AccFloatAIE2pIntrOp::create(
5104 rewriter, loc, v16f32Ty, v16bf16);
5107 SmallVector<Value, 2> parts;
5108 if (laneSize == 16) {
5109 parts.push_back(source);
5111 for (int64_t half = 0; half < 2; ++half) {
5112 SmallVector<int64_t> mask =
5113 llvm::to_vector(llvm::seq<int64_t>(16 * half, 16 * (half + 1)));
5115 vector::ShuffleOp::create(rewriter, loc, source, source, mask));
5118 if (srcType.getElementType().isBF16())
5119 for (Value &part : parts)
5121 for (Value &part : parts)
5122 part = IntrOpTy::create(rewriter, loc, v16bf16Ty, part);
5123 if (resultElemType.isF32())
5124 for (Value &part : parts)
5127 if (parts.size() == 1)
5128 return parts.front();
5129 SmallVector<int64_t> combineMask = llvm::to_vector(llvm::seq<int64_t>(0, 32));
5130 return vector::ShuffleOp::create(rewriter, loc, parts[0], parts[1],
5138 :
public mlir::ConvertOpToLLVMPattern<aievec::ExpOp> {
5140 using ConvertOpToLLVMPattern<aievec::ExpOp>::ConvertOpToLLVMPattern;
5144 ConversionPatternRewriter &rewriter)
const override {
5145 auto loc = expOp.getLoc();
5146 auto srcType = cast<VectorType>(adaptor.getSource().getType());
5147 auto srcElemType = srcType.getElementType();
5151 if ((laneSize != 16 && laneSize != 32) || !srcElemType.isBF16())
5152 return expOp.emitWarning()
5153 <<
"aievec.exp conversion only supports v16bfloat16 and "
5157 auto log2eBF16Const = LLVM::ConstantOp::create(
5158 rewriter, loc, rewriter.getBF16Type(),
5159 rewriter.getFloatAttr(rewriter.getBF16Type(), 1.442695));
5162 SmallVector<int64_t> broadcastMask;
5163 for (
unsigned i = 0; i < laneSize; ++i)
5164 broadcastMask.push_back(0);
5166 auto v1bf16 = LLVM::UndefOp::create(
5167 rewriter, loc, VectorType::get({1}, rewriter.getBF16Type()));
5168 auto v1bf16Inserted = LLVM::InsertElementOp::create(
5169 rewriter, loc, v1bf16, log2eBF16Const,
5170 LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), 0));
5172 auto log2eVec = vector::ShuffleOp::create(rewriter, loc, v1bf16Inserted,
5173 v1bf16Inserted, broadcastMask);
5179 VectorType::get({(int64_t)laneSize}, rewriter.getF32Type());
5180 auto mulResult = aievec::MulElemOp::create(rewriter, loc, resultF32Ty,
5181 adaptor.getSource(), log2eVec);
5184 Value exp2Result = applyAccFloatUnaryIntrinsic<xllvm::Exp2AIE2pIntrOp>(
5185 rewriter, loc, mulResult, rewriter.getBF16Type());
5187 rewriter.replaceOp(expOp, exp2Result);
5195template <
typename SrcOpTy,
typename IntrOpTy>
5197 :
public mlir::ConvertOpToLLVMPattern<SrcOpTy> {
5199 using ConvertOpToLLVMPattern<SrcOpTy>::ConvertOpToLLVMPattern;
5204 ConversionPatternRewriter &rewriter)
const override {
5205 auto srcType = cast<VectorType>(adaptor.getSource().getType());
5206 Type srcElemType = srcType.getElementType();
5209 if ((laneSize != 16 && laneSize != 32) ||
5210 (!srcElemType.isBF16() && !srcElemType.isF32()))
5211 return op.emitWarning()
5213 <<
" conversion only supports v16 and v32 of bfloat16 or float.\n";
5216 op, applyAccFloatUnaryIntrinsic<IntrOpTy>(
5217 rewriter, op.getLoc(), adaptor.getSource(), srcElemType));
5226 :
public mlir::ConvertOpToLLVMPattern<math::RsqrtOp> {
5228 using ConvertOpToLLVMPattern<math::RsqrtOp>::ConvertOpToLLVMPattern;
5232 ConversionPatternRewriter &rewriter)
const override {
5233 auto loc = rsqrtOp.getLoc();
5234 auto operandType = adaptor.getOperand().getType();
5237 if (operandType.isF32()) {
5238 auto rsqrtResult = xllvm::InvsqrtAIE2pIntrOp::create(
5239 rewriter, loc, rewriter.getF32Type(), adaptor.getOperand());
5240 rewriter.replaceOp(rsqrtOp, rsqrtResult);
5245 auto vecType = dyn_cast<VectorType>(operandType);
5246 if (!vecType || !vecType.getElementType().isF32())
5251 Value result = LLVM::PoisonOp::create(rewriter, loc, vecType);
5253 for (
int i = 0; i < numElements; ++i) {
5255 auto indexCst = LLVM::ConstantOp::create(
5256 rewriter, loc, rewriter.getI64Type(), rewriter.getI64IntegerAttr(i));
5257 auto extractedElem = LLVM::ExtractElementOp::create(
5258 rewriter, loc, adaptor.getOperand(), indexCst);
5261 auto rsqrtResult = xllvm::InvsqrtAIE2pIntrOp::create(
5262 rewriter, loc, rewriter.getF32Type(), extractedElem);
5265 result = LLVM::InsertElementOp::create(rewriter, loc, vecType, result,
5266 rsqrtResult, indexCst);
5269 rewriter.replaceOp(rsqrtOp, result);
5277static bool isOneLaneVectorOfScalar(Type resultType, Type sourceType) {
5278 auto vecType = dyn_cast<VectorType>(resultType);
5279 return vecType && !vecType.isScalable() && vecType.getRank() == 1 &&
5280 vecType.getNumElements() == 1 && !isa<VectorType>(sourceType);
5283static bool isOneLaneBroadcastOfScalar(vector::BroadcastOp op) {
5284 return isOneLaneVectorOfScalar(op.getType(), op.getSourceType());
5287static bool isOneLaneInsertOfScalar(vector::InsertOp op) {
5288 return isOneLaneVectorOfScalar(op.getType(),
5289 op.getValueToStore().getType()) &&
5290 !op.hasDynamicPosition();
5294 :
public mlir::ConvertOpToLLVMPattern<vector::BroadcastOp> {
5296 using ConvertOpToLLVMPattern<vector::BroadcastOp>::ConvertOpToLLVMPattern;
5300 ConversionPatternRewriter &rewriter)
const override {
5301 if (!isOneLaneBroadcastOfScalar(bcastOp))
5303 rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(bcastOp, bcastOp.getType(),
5304 adaptor.getSource());
5310 :
public mlir::ConvertOpToLLVMPattern<vector::InsertOp> {
5312 using ConvertOpToLLVMPattern<vector::InsertOp>::ConvertOpToLLVMPattern;
5316 ConversionPatternRewriter &rewriter)
const override {
5317 if (!isOneLaneInsertOfScalar(insertOp))
5319 rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(insertOp, insertOp.getType(),
5320 adaptor.getValueToStore());
5330 using ConvertOpToLLVMPattern<arith::DivFOp>::ConvertOpToLLVMPattern;
5333 : ConvertOpToLLVMPattern(typeConverter),
deviceName(device.str()) {}
5339 ConversionPatternRewriter &rewriter)
const override {
5340 auto loc = divOp.getLoc();
5341 auto lhsType = adaptor.getLhs().getType();
5345 auto vecType = dyn_cast<VectorType>(lhsType);
5346 if (!vecType || !vecType.getElementType().isF32())
5349 auto rhsType = adaptor.getRhs().getType();
5350 auto rhsVecType = dyn_cast<VectorType>(rhsType);
5351 if (!rhsVecType || rhsVecType != vecType)
5356 auto module = divOp->getParentOfType<ModuleOp>();
5357 auto f32Ty = rewriter.getF32Type();
5362 std::function<void(OpBuilder &, Location, ValueRange)> bodyBuilder;
5364 bodyBuilder = [](OpBuilder &builder, Location loc, ValueRange
args) {
5365 auto invResult = xllvm::InvAIE2pIntrOp::create(
5366 builder, loc, builder.getF32Type(),
args[1]);
5368 arith::MulFOp::create(builder, loc,
args[0], invResult);
5369 LLVM::ReturnOp::create(builder, loc, ValueRange{mulResult});
5372 bodyBuilder = [](OpBuilder &builder, Location loc, ValueRange
args) {
5373 auto divResult = arith::DivFOp::create(builder, loc,
args[0],
args[1]);
5374 LLVM::ReturnOp::create(builder, loc, ValueRange{divResult});
5381 getOrCreateScalarHelperFunc(module, rewriter,
"fdiv",
deviceName,
5383 f32Ty, bodyBuilder);
5387 Value result = LLVM::PoisonOp::create(rewriter, loc, vecType);
5389 for (
int i = 0; i < numElements; ++i) {
5390 auto indexCst = LLVM::ConstantOp::create(
5391 rewriter, loc, rewriter.getI64Type(), rewriter.getI64IntegerAttr(i));
5392 auto lhsElem = LLVM::ExtractElementOp::create(rewriter, loc,
5393 adaptor.getLhs(), indexCst);
5394 auto rhsElem = LLVM::ExtractElementOp::create(rewriter, loc,
5395 adaptor.getRhs(), indexCst);
5397 auto divResult = LLVM::CallOp::create(rewriter, loc, helperFunc,
5398 ValueRange{lhsElem, rhsElem})
5401 result = LLVM::InsertElementOp::create(rewriter, loc, vecType, result,
5402 divResult, indexCst);
5405 rewriter.replaceOp(divOp, result);
5411 mlir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns) {
5429static void populateAIEVecToLLVMAIE2ConversionPatterns(
5430 mlir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns,
5431 Aie2Fp32Emulation aie2Fp32EmulationOption) {
5433 patterns.add<AddElemOpAIE2Conversion, SubElemOpAIE2Conversion>(converter);
5434 patterns.add<NegOpAIE2Conversion>(converter);
5435 patterns.add<MulElemOpConversion>(converter, aie2Fp32EmulationOption);
5436 patterns.add<UPSOpAIE2Conversion, SRSOpAIE2Conversion>(converter);
5437 patterns.add<ShiftOpConversion>(converter);
5438 patterns.add<MaxOpConversion, MinOpConversion>(converter);
5439 patterns.add<CmpOpConversion, SelOpConversion>(converter);
5440 patterns.add<ExtOpConversion>(converter);
5441 patterns.add<ExtractElemOpConversion>(converter);
5442 patterns.add<ConcatOpConversion>(converter);
5443 patterns.add<FMAElemOpConversion>(converter);
5444 patterns.add<FoldAIECastOps>(converter);
5445 patterns.add<FdivOpConversion>(converter,
"aie2");
5446 patterns.add<OneLaneBroadcastOpConversion, OneLaneInsertOpConversion>(
5452 :
public mlir::ConvertOpToLLVMPattern<aievec::ExtElemOp> {
5454 using ConvertOpToLLVMPattern<aievec::ExtElemOp>::ConvertOpToLLVMPattern;
5458 ConversionPatternRewriter &rewriter)
const override {
5459 Location loc = op.getLoc();
5463 Value extracted = LLVM::ExtractElementOp::create(
5464 rewriter, loc, adaptor.getSource(), adaptor.getIndex());
5466 rewriter.replaceOp(op, extracted);
5473 :
public mlir::ConvertOpToLLVMPattern<aievec::ConcatOp> {
5475 using ConvertOpToLLVMPattern<aievec::ConcatOp>::ConvertOpToLLVMPattern;
5482 ConversionPatternRewriter &rewriter)
const override {
5483 Location loc = op.getLoc();
5484 SmallVector<Value> parts(adaptor.getSources());
5485 if (parts.empty()) {
5486 op.emitWarning() <<
"aievec.concat with no sources is not supported.\n";
5489 auto lanesOf = [](Value v) {
5490 return cast<VectorType>(v.getType()).getNumElements();
5492 while (parts.size() > 1) {
5493 SmallVector<Value> next;
5494 for (
size_t i = 0; i + 1 < parts.size(); i += 2) {
5495 Value lhs = parts[i], rhs = parts[i + 1];
5496 int64_t lhsLanes = lanesOf(lhs), rhsLanes = lanesOf(rhs);
5497 if (rhsLanes < lhsLanes) {
5498 SmallVector<int64_t> padMask(lhsLanes, -1);
5499 std::iota(padMask.begin(), padMask.begin() + rhsLanes, 0);
5500 rhs = vector::ShuffleOp::create(rewriter, loc, rhs, rhs, padMask);
5502 SmallVector<int64_t> mask(lhsLanes + rhsLanes);
5503 std::iota(mask.begin(), mask.end(), 0);
5505 vector::ShuffleOp::create(rewriter, loc, lhs, rhs, mask));
5507 if (parts.size() % 2)
5508 next.push_back(parts.back());
5509 parts = std::move(next);
5511 rewriter.replaceOp(op, parts.front());
5517 mlir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns) {
5560 mlir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns,
5570 aie2Fp32EmulationOption);
5574static void configureAIEVecToLLVMLegalizations(LLVMConversionTarget &target) {
5577 target.addDynamicallyLegalOp<arith::DivFOp>([](arith::DivFOp divOp) {
5578 auto resultType = divOp.getType();
5579 if (
auto vecType = dyn_cast<VectorType>(resultType)) {
5581 return !vecType.getElementType().isF32();
5588 target.addDynamicallyLegalOp<vector::BroadcastOp>(
5589 [](vector::BroadcastOp op) {
return !isOneLaneBroadcastOfScalar(op); });
5590 target.addDynamicallyLegalOp<vector::InsertOp>(
5591 [](vector::InsertOp op) {
return !isOneLaneInsertOfScalar(op); });
5598 aieTarget = options.aieTarget;
5599 aie2Fp32Emulation = options.aie2Fp32Emulation;
5603 ModuleOp module = getOperation();
5604 std::optional<AIEVecTargetInfo> targetInfo =
5607 return signalPassFailure();
5611 if (!targetInfo->hasAIEVecLowering()) {
5612 WalkResult result =
module.walk([&](Operation *op) {
5613 if (isa_and_present<AIEVecDialect, aie1::AIEVecAIE1Dialect>(
5614 op->getDialect())) {
5615 op->emitOpError() << "has no LLVM lowering for target '"
5616 << getAIEVecTargetName(*targetInfo) << "'";
5617 return WalkResult::interrupt();
5619 return WalkResult::advance();
5621 if (result.wasInterrupted())
5622 signalPassFailure();
5626 RewritePatternSet patterns(&getContext());
5627 LLVMTypeConverter converter(&getContext());
5631 converter.addConversion(
5632 [&](VectorType type) -> std::optional<Type> {
return type; });
5634 populateAIEVecToLLVMConversionPatterns(converter, patterns,
5635 aie2Fp32Emulation, *targetInfo);
5637 LLVMConversionTarget target(getContext());
5638 target.addIllegalDialect<xilinx::aievec::AIEVecDialect,
5639 xilinx::aievec::aie1::AIEVecAIE1Dialect>();
5640 target.addLegalDialect<arith::ArithDialect, vector::VectorDialect,
5641 memref::MemRefDialect, xilinx::xllvm::XLLVMDialect,
5645 configureAIEVecToLLVMLegalizations(target);
5647 if (failed(applyPartialConversion(getOperation(), target,
5648 std::move(patterns))))
5649 signalPassFailure();
5653std::unique_ptr<mlir::OperationPass<mlir::ModuleOp>>
5655 return std::make_unique<ConvertAIEVecToLLVMPass>();
5658std::unique_ptr<mlir::OperationPass<mlir::ModuleOp>>
5660 const xilinx::ConvertAIEVecToLLVMOptions &options) {
5661 return std::make_unique<ConvertAIEVecToLLVMPass>(options);
typename SrcOpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(SrcOpTy op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
typename SrcOpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(SrcOpTy op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
static DecodedAddElemOp decodeAddElemOp(OpAdaptor op)
LogicalResult matchAndRewrite(aievec::AddElemOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::AddElemOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::aie1::AddOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::BroadcastOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::BroadcastScalarOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::BroadcastScalarOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::CmpOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::ConcatOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::ConcatOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::ExpOp expOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::ExtOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::ExtOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::FMAElemOp fmaOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::FMAElemOp fmaOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::aie1::FMAOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::DivFOp divOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
FdivOpConversion(const LLVMTypeConverter &typeConverter, StringRef device)
LogicalResult matchAndRewrite(aievec::InvOp invOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::MaxOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::MaxOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::MinOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::MinOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
static DecodedMulElemOp decodeMulElemOp(OpAdaptor op)
LogicalResult matchAndRewrite(aievec::MulElemOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult convertToEmulatedFP32MulElem(aievec::MulElemOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const
LogicalResult convertToEmulatedI32MulElem(aievec::MulElemOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const
MulElemOpConversion(const LLVMTypeConverter &typeConverter, Aie2Fp32Emulation aie2Fp32EmulationOption)
Aie2Fp32Emulation aie2Fp32EmulationOption
LogicalResult matchAndRewrite(aievec::MulElemOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
static DecodedMulElemOp decodeMulElemOp(OpAdaptor op)
LogicalResult matchAndRewrite(aievec::aie1::MulOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::NegOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::NegOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::BroadcastOp bcastOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::InsertOp insertOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::PackOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
static std::string getIntrinsicName(aievec::PackOp op)
LogicalResult matchAndRewrite(math::RsqrtOp rsqrtOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::SRSOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::SRSOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::SelOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
static std::string getIntrinsicName(aievec::aie1::SelectOp op)
LogicalResult matchAndRewrite(aievec::aie1::SelectOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::ShiftOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::ShiftOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::SubElemOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
static DecodedSubElemOp decodeSubElemOp(OpAdaptor op)
LogicalResult matchAndRewrite(aievec::aie1::SubOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
static std::string getIntrinsicName(aievec::UPDOp op, int loadSize)
LogicalResult matchAndRewrite(aievec::UPDOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::UPSOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::UPSOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::UnpackOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::UnpackOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
mlir::VectorType getFlattenedVectorType(mlir::VectorType vecTy)
std::optional< MatMulRecipe > checkMatMulTile(bool aie2p, mlir::VectorType lhsTy, mlir::VectorType rhsTy, mlir::VectorType accTy, bool lhsTransposed, bool rhsTransposed, llvm::function_ref< mlir::InFlightDiagnostic()> emitError=nullptr)
int32_t getVectorSizeInBits(mlir::VectorType type)
unsigned getVectorLaneSize(mlir::VectorType type)
bool isAIE2POnlyShuffleMode(ShuffleMode mode)
void populateAIEVecToLLVMCommonConversionPatterns(mlir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns)
void populateAIEVecToLLVMConversionPatterns(mlir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns, Aie2Fp32Emulation aie2Fp32EmulationOption, AIEVecTargetInfo aieTarget)
std::optional< AIEVecTargetInfo > parseAIEVecTarget(llvm::StringRef spelling)
bool isBFP16MatMulRhsOnly(ToBFP16Op op)
SelOpConversionBase< xllvm::VectorSel16AIE2pIntrOp, xllvm::VectorSel32AIE2pIntrOp > SelOpAIE2pConversion
AccFloatAddSubAIE2pConversion< aievec::AddElemOp, xllvm::AddACC2048AccFloatAIE2pIntrOp > AddElemOpAIE2pConversion
mlir::LogicalResult expandBFP16MatMul(mlir::RewriterBase &rewriter, MatMulOp_AIE2P op)
CmpOpConversionBase< xllvm::VectorMaxLtBf16AIE2pIntrOp, xllvm::VectorMinGeBf16AIE2pIntrOp, xllvm::VectorMaxLt32AIE2pIntrOp, xllvm::VectorMinGe32AIE2pIntrOp, xllvm::VectorMaxLt16AIE2pIntrOp, xllvm::VectorMinGe16AIE2pIntrOp > CmpOpAIE2pConversion
void populateAIEVecToLLVMAIE2ConversionPatterns(mlir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns)
void populateAIEVecToLLVMAIE2pConversionPatterns(mlir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns)
SelOpConversionBase< xllvm::VectorSel16IntrOp, xllvm::VectorSel32IntrOp > SelOpConversion
CmpOpConversionBase< xllvm::VectorMaxLtBf16IntrOp, xllvm::VectorMinGeBf16IntrOp, xllvm::VectorMaxLt32IntrOp, xllvm::VectorMinGe32IntrOp, xllvm::VectorMaxLt16IntrOp, xllvm::VectorMinGe16IntrOp > CmpOpConversion
AccFloatAddSubAIE2pConversion< aievec::SubElemOp, xllvm::SubACC2048AccFloatAIE2pIntrOp > SubElemOpAIE2pConversion
std::unique_ptr< mlir::OperationPass< mlir::ModuleOp > > createConvertAIEVecToLLVMPass()
bool hasAIEVecLowering() const
@ FP32_FP32_FP32_16x1x1x1
ConvertAIEVecToLLVMPass(const xilinx::ConvertAIEVecToLLVMOptions &options)
void runOnOperation() override
ConvertAIEVecToLLVMPass()=default
@ BF16_BF16_FP32_32x1x2x1
@ BF16_BF16_FP32_64x1x2x1
@ BF16_BF16_FP32_16x1x1x1
@ FP32_FP32_FP32_16x1x1x1
@ BF16_BF16_FP32_16x1x2x1
@ FP32_FP32_FP32_16x1x1x1