23#include "mlir/Dialect/Affine/IR/AffineOps.h"
24#include "mlir/Dialect/EmitC/IR/EmitC.h"
25#include "mlir/Dialect/Func/IR/FuncOps.h"
26#include "mlir/Dialect/Math/IR/Math.h"
27#include "mlir/Dialect/MemRef/IR/MemRef.h"
28#include "mlir/Dialect/SCF/IR/SCF.h"
29#include "mlir/Dialect/UB/IR/UBOps.h"
30#include "mlir/IR/PatternMatch.h"
31#include "mlir/IR/SymbolTable.h"
32#include "mlir/IR/TypeUtilities.h"
33#include "mlir/Pass/PassManager.h"
34#include "mlir/Transforms/DialectConversion.h"
35#include "mlir/Transforms/Passes.h"
36#include "llvm/ADT/SmallSet.h"
41#define DEBUG_TYPE "lower-vector-to-aievec"
64static std::optional<ScalarSRSChain>
65matchScalarSRSChain(arith::TruncIOp truncOp) {
66 auto dstIntType = dyn_cast<IntegerType>(truncOp.getOut().getType());
69 unsigned dstBits = dstIntType.getWidth();
70 if (dstBits != 8 && dstBits != 16)
72 auto srcIntType = dyn_cast<IntegerType>(truncOp.getIn().getType());
73 if (!srcIntType || srcIntType.getWidth() != 32)
77 Value source = truncOp.getIn();
78 arith::MinSIOp minOp =
nullptr;
79 arith::MaxSIOp maxOp =
nullptr;
80 if (
auto minsiOp = source.getDefiningOp<arith::MinSIOp>()) {
81 if (
auto maxsiOp = minsiOp.getLhs().getDefiningOp<arith::MaxSIOp>()) {
84 source = maxOp.getLhs();
86 }
else if (
auto maxsiOp = source.getDefiningOp<arith::MaxSIOp>()) {
87 if (
auto minsiOp = maxsiOp.getLhs().getDefiningOp<arith::MinSIOp>()) {
90 source = minOp.getLhs();
97 std::optional<int64_t> loVal = getConstantIntValue(maxOp.getRhs());
98 std::optional<int64_t> hiVal = getConstantIntValue(minOp.getRhs());
102 int64_t unsignedHi =
static_cast<int64_t
>((one << dstBits) - 1);
103 int64_t signedLo = -
static_cast<int64_t
>(one << (dstBits - 1));
104 int64_t signedHi =
static_cast<int64_t
>((one << (dstBits - 1)) - 1);
105 if (*loVal == 0 && *hiVal == unsignedHi)
107 if (*loVal == signedLo && *hiVal == signedHi)
120static bool isSRSCompoundCandidate(arith::TruncIOp trunciOp) {
123 if (!isa<VectorType>(trunciOp.getType()) && !matchScalarSRSChain(trunciOp))
126 Value source = trunciOp.getIn();
129 if (source.getDefiningOp<arith::ShRSIOp>())
133 if (
auto minsiOp = source.getDefiningOp<arith::MinSIOp>()) {
134 if (
auto maxsiOp = minsiOp.getLhs().getDefiningOp<arith::MaxSIOp>()) {
135 if (maxsiOp.getLhs().getDefiningOp<arith::ShRSIOp>())
141 if (
auto maxsiOp = source.getDefiningOp<arith::MaxSIOp>()) {
142 if (
auto minsiOp = maxsiOp.getLhs().getDefiningOp<arith::MinSIOp>()) {
143 if (minsiOp.getLhs().getDefiningOp<arith::ShRSIOp>())
150 if (
auto minsiOp = source.getDefiningOp<arith::MinSIOp>()) {
151 if (minsiOp.getLhs().getDefiningOp<arith::MaxSIOp>())
156 if (
auto maxsiOp = source.getDefiningOp<arith::MaxSIOp>()) {
157 if (maxsiOp.getLhs().getDefiningOp<arith::MinSIOp>())
166static bool shrsiUsedByCompoundSRS(arith::ShRSIOp rsOp) {
167 for (Operation *user : rsOp->getUsers()) {
169 if (
auto truncOp = dyn_cast<arith::TruncIOp>(user))
170 if (isSRSCompoundCandidate(truncOp))
174 if (isa<arith::MaxSIOp, arith::MinSIOp>(user)) {
175 for (Operation *user2 : user->getUsers()) {
176 if (
auto truncOp2 = dyn_cast<arith::TruncIOp>(user2))
177 if (isSRSCompoundCandidate(truncOp2))
179 if (isa<arith::MaxSIOp, arith::MinSIOp>(user2)) {
180 for (Operation *user3 : user2->getUsers()) {
181 if (
auto truncOp3 = dyn_cast<arith::TruncIOp>(user3))
182 if (isSRSCompoundCandidate(truncOp3))
195static bool scalarClampInCompoundSRS(Operation *op) {
196 if (!isa<arith::MaxSIOp, arith::MinSIOp>(op))
199 if (isa<VectorType>(op->getResult(0).getType()))
201 for (Operation *user : op->getUsers()) {
202 if (
auto truncOp = dyn_cast<arith::TruncIOp>(user)) {
203 if (isSRSCompoundCandidate(truncOp))
206 if (isa<arith::MaxSIOp, arith::MinSIOp>(user)) {
207 for (Operation *user2 : user->getUsers()) {
208 if (
auto truncOp2 = dyn_cast<arith::TruncIOp>(user2)) {
209 if (isSRSCompoundCandidate(truncOp2))
221static Value skipShapeCastRoundTrip(Value v) {
223 while (
auto castOp = src.getDefiningOp<vector::ShapeCastOp>())
224 src = castOp.getSource();
225 return src.getType() == v.getType() ? src : v;
231static std::optional<Value> getSourceOfWideningOp(Value src) {
232 src = skipShapeCastRoundTrip(src);
233 if (
auto extSIOp = src.getDefiningOp<arith::ExtSIOp>())
234 return extSIOp.getIn();
235 if (
auto extUIOp = src.getDefiningOp<arith::ExtUIOp>())
236 return extUIOp.getIn();
237 if (
auto extFOp = src.getDefiningOp<arith::ExtFOp>())
238 return extFOp.getIn();
239 if (
auto srsOp = src.getDefiningOp<aievec::SRSOp>()) {
243 auto srsSource = srsOp.getSource();
245 if (
auto upsOp = srsSource.getDefiningOp<aievec::UPSOp>())
246 return upsOp.getSource();
248 if (
auto castOp = src.getDefiningOp<aievec::CastOp>()) {
252 auto castSource = castOp.getSource();
254 if (
auto upsOp = castSource.getDefiningOp<aievec::UPSOp>())
255 return upsOp.getSource();
257 return std::optional<Value>();
263static Value getSplatOfWidenedBF16(Value src) {
264 src = skipShapeCastRoundTrip(src);
265 if (
auto concatOp = src.getDefiningOp<aievec::ConcatOp>()) {
266 auto parts = concatOp.getSources();
267 if (parts.empty() || !llvm::all_equal(parts))
272 if (
auto bcastOp = src.getDefiningOp<vector::BroadcastOp>())
273 scalar = bcastOp.getSource();
274 else if (
auto bcastOp = src.getDefiningOp<aievec::BroadcastScalarOp>())
275 scalar = bcastOp.getSource();
276 if (!scalar || isa<VectorType>(scalar.getType()))
278 if (
auto extFOp = scalar.getDefiningOp<arith::ExtFOp>())
279 if (extFOp.getIn().getType().isBF16())
280 return extFOp.getIn();
286static bool holdsBF16(Value src) {
287 if (getSplatOfWidenedBF16(src))
289 if (
auto wideSrc = getSourceOfWideningOp(src))
290 return getElementTypeOrSelf(wideSrc->getType()).isBF16();
291 DenseFPElementsAttr dense;
292 if (!matchPattern(src, m_Constant(&dense)) || !dense.getElementType().isF32())
294 return llvm::all_of(dense.getValues<APFloat>(), [](APFloat v) {
295 bool losesInfo = false;
296 v.convert(APFloat::BFloat(), APFloat::rmNearestTiesToEven, &losesInfo);
304static Value getBF16Source(OpBuilder &builder, Location loc, Value src) {
305 auto bf16Type = VectorType::get(cast<VectorType>(src.getType()).getShape(),
306 builder.getBF16Type());
307 if (Value scalar = getSplatOfWidenedBF16(src))
308 return vector::BroadcastOp::create(builder, loc, bf16Type, scalar);
309 if (
auto wideSrc = getSourceOfWideningOp(src))
312 arith::ConstantOp::create(builder, loc, builder.getI32IntegerAttr(0));
313 return aievec::SRSOp::create(builder, loc, bf16Type, src, shift);
324static std::optional<bool> getSignednessOfWideningOp(Value src) {
327 while (
auto castOp = src.getDefiningOp<vector::ShapeCastOp>())
328 src = castOp.getSource();
329 if (src.getDefiningOp<arith::ExtSIOp>())
331 if (src.getDefiningOp<arith::ExtUIOp>())
336 if (
auto srsOp = src.getDefiningOp<aievec::SRSOp>())
337 return srsOp.getSign() != 0;
343static std::optional<Value> getSourceOfNarrowingOp(Value src) {
344 if (
auto truncFOp = src.getDefiningOp<arith::TruncFOp>())
345 return truncFOp.getIn();
346 if (
auto truncIOp = src.getDefiningOp<arith::TruncIOp>())
347 return truncIOp.getIn();
348 return std::optional<Value>();
357static Value widenValueWithNarrowingCheck(Value val, Type targetType,
359 ConversionPatternRewriter &rewriter) {
361 if (
auto narrowedSrc = getSourceOfNarrowingOp(val)) {
362 if (narrowedSrc->getType() == targetType)
367 if (val.getType() == targetType)
370 return arith::ExtFOp::create(rewriter, loc, targetType, val,
384narrowValueWithWideningCheck(Operation *srcOp, Value val, Type targetType,
386 ConversionPatternRewriter &rewriter) {
393 if (srcOp->hasOneUse()) {
394 Operation *user = *srcOp->getUsers().begin();
395 if (
auto extfOp = dyn_cast<arith::ExtFOp>(user)) {
405 arith::TruncFOp::create(rewriter, loc, targetType, val);
415performBF16BinaryOpInF32(Value lhs, Value rhs, Operation *srcOp, Location loc,
416 ConversionPatternRewriter &rewriter,
417 const std::function<Value(Value, Value)> &opBuilder) {
418 Type f32Type = rewriter.getF32Type();
421 Value lhsF32 = widenValueWithNarrowingCheck(lhs, f32Type, loc, rewriter);
422 Value rhsF32 = widenValueWithNarrowingCheck(rhs, f32Type, loc, rewriter);
425 Value resultF32 = opBuilder(lhsF32, rhsF32);
428 auto narrowResult = narrowValueWithWideningCheck(
429 srcOp, resultF32, lhs.getType(), loc, rewriter);
431 if (narrowResult.skipNarrowing) {
433 rewriter.replaceOp(narrowResult.wideningUser, resultF32);
434 rewriter.eraseOp(srcOp);
436 rewriter.replaceOp(srcOp, narrowResult.narrowedValue);
443static std::optional<std::tuple<Value, Value, Value>>
444extractMACOperandsFromAddOperands(Value addLhs, Value addRhs) {
445 auto *lhsDefOp = addLhs.getDefiningOp();
446 auto *rhsDefOp = addRhs.getDefiningOp();
447 arith::MulIOp mulOp =
nullptr;
450 mulOp = dyn_cast<arith::MulIOp>(lhsDefOp);
453 if (!mulOp && rhsDefOp) {
454 mulOp = dyn_cast<arith::MulIOp>(rhsDefOp);
458 return std::make_tuple(mulOp.getLhs(), mulOp.getRhs(), acc);
461 auto lhsSrsOp = addLhs.getDefiningOp<aievec::SRSOp>();
462 auto rhsSrsOp = addRhs.getDefiningOp<aievec::SRSOp>();
463 aievec::aie1::MulOp aieMulOp =
nullptr;
465 aieMulOp = lhsSrsOp.getSource().getDefiningOp<aievec::aie1::MulOp>();
468 if (!aieMulOp && rhsSrsOp) {
469 aieMulOp = rhsSrsOp.getSource().getDefiningOp<aievec::aie1::MulOp>();
473 return std::make_tuple(aieMulOp.getLhs(), aieMulOp.getRhs(), acc);
480static std::optional<std::tuple<Value, Value, Value>>
481extractFMACOperandsFromAddOperands(Value addLhs, Value addRhs) {
482 auto *lhsDefOp = addLhs.getDefiningOp();
483 auto *rhsDefOp = addRhs.getDefiningOp();
484 arith::MulFOp mulOp =
nullptr;
487 mulOp = dyn_cast<arith::MulFOp>(lhsDefOp);
490 if (!mulOp && rhsDefOp) {
491 mulOp = dyn_cast<arith::MulFOp>(rhsDefOp);
495 return std::make_tuple(mulOp.getLhs(), mulOp.getRhs(), acc);
502static std::optional<Value>
503convertValueToTargetTypeAIE2(ConversionPatternRewriter &rewriter, Location loc,
504 Value inputVal, VectorType tgtType) {
505 auto srcType = cast<VectorType>(inputVal.getType());
506 auto srcElemType = srcType.getElementType();
507 unsigned srcBitWidth = srcElemType.getIntOrFloatBitWidth();
510 auto tgtElemType = tgtType.getElementType();
511 unsigned tgtBitWidth = tgtElemType.getIntOrFloatBitWidth();
514 if (srcType == tgtType)
517 if ((srcElemType == tgtElemType) && (srcLaneSize != tgtLaneSize)) {
519 if ((srcLaneSize == 16 && tgtLaneSize == 32 &&
520 isa<FloatType>(srcElemType)) ||
521 (srcLaneSize == 32 && tgtLaneSize == 64 &&
522 isa<IntegerType>(srcElemType))) {
523 auto zeroConstOp = arith::ConstantOp::create(
524 rewriter, loc, srcType.getElementType(),
525 rewriter.getZeroAttr(srcType.getElementType()));
526 auto broadcastZeroOp = aievec::BroadcastScalarOp::create(
527 rewriter, loc, tgtType, zeroConstOp->getResult(0));
528 auto extOp = aievec::ExtOp::create(rewriter, loc, srcType,
529 broadcastZeroOp->getResult(0), 0);
531 SmallVector<Value> inputSources = {inputVal, extOp->getResult(0)};
533 aievec::ConcatOp::create(rewriter, loc, tgtType, inputSources);
535 return concatOp.getResult();
537 }
else if ((srcElemType != tgtElemType) && (srcLaneSize == tgtLaneSize) &&
538 isa<IntegerType>(srcElemType) && isa<IntegerType>(tgtElemType)) {
539 if (srcBitWidth == 16 && tgtBitWidth == 32 && srcLaneSize == 16) {
543 auto upsOp = aievec::UPSOp::create(rewriter, loc, accType, inputVal);
544 auto castOp = aievec::CastOp::create(
545 rewriter, loc, tgtType, upsOp.getResult(),
false);
546 return castOp.getResult();
549 if (srcBitWidth == 8 && tgtBitWidth == 32 && srcLaneSize == 16) {
554 aievec::ConcatOp::create(rewriter, loc, concatOutType,
555 SmallVector<Value>({inputVal, inputVal}));
558 aievec::UPSOp::create(rewriter, loc, accType, concatOp.getResult());
560 auto castOp = aievec::CastOp::create(
561 rewriter, loc, castType, upsOp.getResult(),
false);
563 aievec::ExtOp::create(rewriter, loc, tgtType, castOp.getResult(), 0);
564 return extOp.getResult();
567 if (srcBitWidth == 8 && tgtBitWidth == 16 && srcLaneSize == 32) {
570 aievec::UnpackOp::create(rewriter, loc, tgtType, inputVal);
571 return unpackOp.getResult();
581static SmallVector<NamedAttribute>
582buildAttributeListForRotationSelectOp(PatternRewriter &rewriter, VectorType vTy,
585 auto elemTy = vTy.getElementType();
586 if (
auto intTy = dyn_cast<IntegerType>(elemTy))
587 width = intTy.getWidth();
588 StringAttr attr0 = rewriter.getStringAttr(
"0");
589 StringAttr attr0x06040200 = rewriter.getStringAttr(
"0x06040200");
590 StringAttr attr0x0e0c0a08 = rewriter.getStringAttr(
"0x0e0c0a08");
591 StringAttr attr0x2103 = rewriter.getStringAttr(
"0x2103");
592 StringAttr attr0x3210 = rewriter.getStringAttr(
"0x3210");
593 StringAttr selectAttrName = rewriter.getStringAttr(
"select");
594 StringAttr xoffsetsAttrName = rewriter.getStringAttr(
"xoffsets");
595 StringAttr xoffsetsHiAttrName = rewriter.getStringAttr(
"xoffsets_hi");
596 StringAttr xsquareAttrName = rewriter.getStringAttr(
"xsquare");
597 StringAttr xstartAttrName = rewriter.getStringAttr(
"xstart");
598 StringAttr yoffsetsAttrName = rewriter.getStringAttr(
"yoffsets");
599 StringAttr yoffsetsHiAttrName = rewriter.getStringAttr(
"yoffsets_hi");
600 StringAttr ysquareAttrName = rewriter.getStringAttr(
"ysquare");
601 StringAttr ystartAttrName = rewriter.getStringAttr(
"ystart");
606 int64_t xstart = rotation + 1;
607 int64_t ystart = rotation - 1;
608 return SmallVector<NamedAttribute, 9>(
609 {{selectAttrName, rewriter.getStringAttr(
"0x11111111")},
610 {xoffsetsAttrName, attr0x06040200},
611 {xoffsetsHiAttrName, attr0x0e0c0a08},
612 {xsquareAttrName, attr0x2103},
613 {xstartAttrName, rewriter.getStringAttr(std::to_string(xstart))},
614 {yoffsetsAttrName, rewriter.getStringAttr(
"0x0503010f")},
615 {yoffsetsHiAttrName, rewriter.getStringAttr(
"0x0d0b0907")},
616 {ysquareAttrName, attr0x2103},
617 {ystartAttrName, rewriter.getStringAttr(std::to_string(ystart))}});
619 return SmallVector<NamedAttribute, 9>(
620 {{selectAttrName, attr0},
621 {xoffsetsAttrName, attr0x06040200},
622 {xoffsetsHiAttrName, attr0x0e0c0a08},
623 {xsquareAttrName, attr0x3210},
624 {xstartAttrName, rewriter.getStringAttr(std::to_string(rotation))},
625 {yoffsetsAttrName, attr0},
626 {yoffsetsHiAttrName, attr0},
627 {ysquareAttrName, attr0},
628 {ystartAttrName, attr0}});
631 return SmallVector<NamedAttribute, 7>(
632 {{selectAttrName, attr0},
633 {xoffsetsAttrName, rewriter.getStringAttr(
"0x76543210")},
634 {xsquareAttrName, attr0x3210},
635 {xstartAttrName, rewriter.getStringAttr(std::to_string(rotation))},
636 {yoffsetsAttrName, attr0},
637 {ysquareAttrName, attr0},
638 {ystartAttrName, attr0}});
640 llvm::report_fatal_error(
"Unexpected width!");
650SmallVector<NamedAttribute>
655 auto elemTy = fmaOp.getLhs().getType().getElementType();
656 if (
auto intTy = dyn_cast<IntegerType>(elemTy))
657 width = intTy.getWidth();
658 auto *ctx = fmaOp.getContext();
678 return SmallVector<NamedAttribute, 11>(
679 {{fmaOp.getXstartAttrName(), StringAttr::get(ctx,
"0")},
680 {fmaOp.getXoffsetsAttrName(), StringAttr::get(ctx,
"0x73727170")},
681 {fmaOp.getXoffsetsHiAttrName(), StringAttr::get(ctx,
"0x77767574")},
682 {fmaOp.getXstepAttrName(), fmaOp.getXstepAttr()},
683 {fmaOp.getXsquareAttrName(), StringAttr::get(ctx,
"0x3120")},
684 {fmaOp.getZstartAttrName(),
685 StringAttr::get(ctx, std::to_string(bcastPos))},
686 {fmaOp.getZoffsetsAttrName(), StringAttr::get(ctx,
"0")},
687 {fmaOp.getZoffsetsHiAttrName(), StringAttr::get(ctx,
"0")},
688 {fmaOp.getZstepAttrName(), StringAttr::get(ctx, std::to_string(step))},
689 {fmaOp.getZsquareAttrName(), fmaOp.getZsquareAttr()},
690 {fmaOp.getFmsubAttrName(), fmaOp.getFmsubAttr()}});
692 return SmallVector<NamedAttribute, 11>(
693 {{fmaOp.getXstartAttrName(), StringAttr::get(ctx,
"0")},
694 {fmaOp.getXoffsetsAttrName(), StringAttr::get(ctx,
"0x76543210")},
695 {fmaOp.getXoffsetsHiAttrName(), fmaOp.getXoffsetsHiAttr()},
696 {fmaOp.getXstepAttrName(), fmaOp.getXstepAttr()},
697 {fmaOp.getXsquareAttrName(), fmaOp.getXsquareAttr()},
698 {fmaOp.getZstartAttrName(),
699 StringAttr::get(ctx, std::to_string(bcastPos))},
700 {fmaOp.getZoffsetsAttrName(), StringAttr::get(ctx,
"0x00000000")},
701 {fmaOp.getZoffsetsHiAttrName(), fmaOp.getZoffsetsHiAttr()},
702 {fmaOp.getZstepAttrName(), fmaOp.getZstepAttr()},
703 {fmaOp.getZsquareAttrName(), fmaOp.getZsquareAttr()},
704 {fmaOp.getFmsubAttrName(), fmaOp.getFmsubAttr()}});
706 llvm::report_fatal_error(
"Unexpected width!");
714template <
typename SrcOpTy,
typename AIEv2ElemOp>
715static LogicalResult genAddElemAIE2(ConversionPatternRewriter &rewriter,
716 Value lval, Value rval, VectorType srcType,
718 auto lCastOp = aievec::CastOp::create(rewriter, srcOp.getLoc(), srcType, lval,
720 auto rCastOp = aievec::CastOp::create(rewriter, srcOp.getLoc(), srcType, rval,
722 auto elemOp = AIEv2ElemOp::create(
723 rewriter, srcOp.getLoc(), lCastOp->getResult(0).getType(),
724 lCastOp->getResult(0), rCastOp->getResult(0));
725 rewriter.replaceOpWithNewOp<aievec::CastOp>(
726 srcOp, srcOp.getType(), elemOp.getResult(),
false);
730static arith::CmpIPredicate
731convertToIntegerPredicate(arith::CmpFPredicate pred) {
733 case CmpFPredicate::UEQ:
734 case CmpFPredicate::OEQ:
735 return CmpIPredicate::eq;
736 case CmpFPredicate::UGT:
737 return CmpIPredicate::ugt;
738 case CmpFPredicate::OGT:
739 return CmpIPredicate::sgt;
740 case CmpFPredicate::UGE:
741 return CmpIPredicate::uge;
742 case CmpFPredicate::OGE:
743 return CmpIPredicate::sge;
744 case CmpFPredicate::ULT:
745 return CmpIPredicate::ult;
746 case CmpFPredicate::OLT:
747 return CmpIPredicate::slt;
748 case CmpFPredicate::ULE:
749 return CmpIPredicate::ule;
750 case CmpFPredicate::OLE:
751 return CmpIPredicate::sle;
752 case CmpFPredicate::UNE:
753 case CmpFPredicate::ONE:
754 return CmpIPredicate::ne;
756 llvm::report_fatal_error(
"Unexpected predicate!");
760static arith::CmpIPredicate
761convertToIntegerPredicate(arith::CmpIPredicate pred) {
765static aievec::CmpOp createCmpOpAIE2(ConversionPatternRewriter &rewriter,
766 CmpIPredicate pred, Location loc,
767 Type type, Value lhs, Value rhs) {
769 case CmpIPredicate::eq:
770 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs,
"eq");
771 case CmpIPredicate::ne:
772 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs,
"ne");
773 case CmpIPredicate::slt:
774 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs,
"slt");
775 case CmpIPredicate::ult:
776 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs,
"ult");
777 case CmpIPredicate::sle:
778 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs,
"sle");
779 case CmpIPredicate::ule:
780 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs,
"ule");
781 case CmpIPredicate::sgt:
782 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs,
"sgt");
783 case CmpIPredicate::ugt:
784 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs,
"ugt");
785 case CmpIPredicate::sge:
786 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs,
"sge");
787 case CmpIPredicate::uge:
788 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs,
"uge");
793template <
typename DstOpTy>
794static aievec::ExtElemOp
795generateAIEVecOpsForReductionOp(ConversionPatternRewriter &rewriter,
796 vector::ReductionOp srcOp,
int shiftIndex,
798 assert(shiftIndex > 0 && (shiftIndex & (shiftIndex - 1)) == 0 &&
799 "shiftIndex must be power of 2");
801 Location loc = srcOp.getLoc();
802 auto vType = cast<VectorType>(curValue.getType());
803 Type scalarType = vType.getElementType();
804 Type vecType = curValue.getType();
805 DstOpTy curOp =
nullptr;
806 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
808 for (
int id = shiftIndex;
id > 0;
id /= 2) {
809 auto constOp = arith::ConstantOp::create(
810 rewriter, loc, rewriter.getI32IntegerAttr(
id * elWidth / 8));
812 auto shiftBytesOp = aievec::ShiftOp::create(
813 rewriter, loc, vecType, curValue, curValue, constOp.getResult());
815 curOp = DstOpTy::create(rewriter, loc, vecType, curValue,
816 shiftBytesOp.getResult());
818 curValue = curOp.getResult();
822 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
823 return aievec::ExtElemOp::create(rewriter, loc, scalarType, curOp,
824 zeroConstOp.getResult());
830static std::pair<Value, unsigned>
831padV16ToV32WithInfinity(ConversionPatternRewriter &rewriter, Location loc,
832 Value inputVec, Type scalarType,
bool negativeInf) {
837 auto infAttr = rewriter.getFloatAttr(
839 APFloat::getInf(cast<FloatType>(scalarType).getFloatSemantics(),
841 auto splatInf = arith::ConstantOp::create(rewriter, loc, infAttr).getResult();
845 aievec::BroadcastScalarOp::create(rewriter, loc, v32bf16Type, splatInf);
847 aievec::ExtOp::create(rewriter, loc, v16bf16Type, infVec, 1);
851 aievec::ConcatOp::create(rewriter, loc, v32bf16Type,
852 ValueRange{inputVec, infUpperHalf.getResult()});
854 return {paddedVec, 32};
859static Value padV16ToV32WithZeros(ConversionPatternRewriter &rewriter,
860 Location loc, Value inputVec,
864 auto zeroAttr = rewriter.getZeroAttr(v16Type);
865 auto zeroVec = arith::ConstantOp::create(rewriter, loc, zeroAttr);
866 return aievec::ConcatOp::create(rewriter, loc, v32Type,
867 ValueRange{inputVec, zeroVec.getResult()});
870static func::FuncOp getOrInsertFuncDecl(ConversionPatternRewriter &rewriter,
871 Operation *parentSymbolTableOp,
872 StringRef funcName, TypeRange inTypes,
873 TypeRange outTypes) {
875 mlir::OpBuilder::InsertionGuard insertGuard(rewriter);
876 rewriter.setInsertionPointToStart(
877 &parentSymbolTableOp->getRegions().front().getBlocks().front());
878 SymbolTable st = SymbolTable(parentSymbolTableOp);
879 func::FuncOp fnOpLookup = st.lookup<func::FuncOp>(funcName);
883 if (fnOpLookup !=
nullptr) {
886 StringAttr t1 = rewriter.getStringAttr(
"sym_visibility");
887 StringAttr t2 = rewriter.getStringAttr(
"private");
888 NamedAttribute funcAccess = NamedAttribute(t1, t2);
889 FunctionType fnType =
890 mlir::FunctionType::get(rewriter.getContext(), inTypes, outTypes);
891 fnOp = func::FuncOp::create(rewriter, parentSymbolTableOp->getLoc(),
892 funcName, fnType, funcAccess);
922template <
typename SrcOpTy,
typename Func>
923static void splitWideVectorOp(SrcOpTy srcOp, ArrayRef<Value> wideInputs,
924 VectorType halfType, VectorType wideType,
925 ConversionPatternRewriter &rewriter,
926 Func &&processHalves) {
928 Location loc = srcOp.getLoc();
931 SmallVector<std::pair<Value, Value>> halfInputs;
932 halfInputs.reserve(wideInputs.size());
933 for (Value wideInput : wideInputs) {
935 aievec::ExtOp::create(rewriter, loc, halfType, wideInput, 0);
937 aievec::ExtOp::create(rewriter, loc, halfType, wideInput, 1);
938 halfInputs.emplace_back(lowerHalf.getResult(), upperHalf.getResult());
942 auto [lowResult, highResult] = processHalves(halfInputs, loc, rewriter);
945 SmallVector<Value> concatSources = {lowResult, highResult};
946 rewriter.replaceOpWithNewOp<aievec::ConcatOp>(srcOp, wideType, concatSources);
950template <
typename SrcOpTy>
951static void splitWideUnaryVectorOp(
952 SrcOpTy srcOp, Value wideInput, VectorType halfType, VectorType wideType,
953 ConversionPatternRewriter &rewriter,
954 std::function<Value(Value, Location, ConversionPatternRewriter &)>
957 splitWideVectorOp<SrcOpTy>(
958 srcOp, {wideInput}, halfType, wideType, rewriter,
959 [&processHalf](ArrayRef<std::pair<Value, Value>> halfInputs, Location loc,
960 ConversionPatternRewriter &rewriter) {
961 auto [lowerHalf, upperHalf] = halfInputs[0];
962 Value lowResult = processHalf(lowerHalf, loc, rewriter);
963 Value highResult = processHalf(upperHalf, loc, rewriter);
964 return std::make_pair(lowResult, highResult);
969template <
typename SrcOpTy>
970static void splitWideBinaryVectorOp(
971 SrcOpTy srcOp, Value lhs, Value rhs, VectorType halfType,
972 VectorType wideType, ConversionPatternRewriter &rewriter,
973 std::function<Value(Value, Value, Location, ConversionPatternRewriter &)>
976 splitWideVectorOp<SrcOpTy>(
977 srcOp, {lhs, rhs}, halfType, wideType, rewriter,
978 [&processHalf](ArrayRef<std::pair<Value, Value>> halfInputs, Location loc,
979 ConversionPatternRewriter &rewriter) {
980 auto [lhsLow, lhsHigh] = halfInputs[0];
981 auto [rhsLow, rhsHigh] = halfInputs[1];
982 Value lowResult = processHalf(lhsLow, rhsLow, loc, rewriter);
983 Value highResult = processHalf(lhsHigh, rhsHigh, loc, rewriter);
984 return std::make_pair(lowResult, highResult);
993static bool matchExpOpForAIE2LUT(math::ExpOp::Adaptor adaptor) {
994 auto srcType = dyn_cast<VectorType>(adaptor.getOperand().getType());
999 Type scalarType = srcType.getElementType();
1000 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
1003 return isa<FloatType>(scalarType) && (laneSize == 16 || laneSize == 32) &&
1008static bool matchExpOpForAIE2P(math::ExpOp::Adaptor adaptor) {
1009 auto srcType = dyn_cast<VectorType>(adaptor.getOperand().getType());
1014 Type scalarType = srcType.getElementType();
1015 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
1018 return scalarType.isBF16() && (laneSize == 16 || laneSize == 32) &&
1025static bool isAIE2PAccFloatUnaryType(VectorType type) {
1026 if (type.isScalable())
1028 Type elemType = type.getElementType();
1030 return (elemType.isBF16() || elemType.isF32()) &&
1031 (laneSize == 16 || laneSize == 32);
1037static bool isAIE2PAccFloatUnaryOrScalarType(Type type) {
1038 auto vecType = dyn_cast<VectorType>(type);
1040 return type.isBF16() || type.isF32();
1041 if (!vecType.isScalable() && vecType.getRank() == 1 &&
1042 vecType.getNumElements() == 1) {
1043 Type elemType = vecType.getElementType();
1044 return elemType.isBF16() || elemType.isF32();
1046 return isAIE2PAccFloatUnaryType(vecType);
1056 : OpConversionPattern<vector::BroadcastOp> {
1057 using OpConversionPattern::OpConversionPattern;
1061 ConversionPatternRewriter &rewriter)
const override {
1063 auto extOp = adaptor.getSource().getDefiningOp<vector::ExtractOp>();
1068 auto src = extOp.getSource();
1069 auto pos = extOp.getStaticPosition();
1070 int64_t posVal = pos[0];
1071 auto srcVecType = cast<VectorType>(src.getType());
1072 auto resultType = cast<VectorType>(bcastOp.getResult().getType());
1073 if (srcVecType != resultType) {
1074 if (srcVecType.getNumElements() != 2 * resultType.getNumElements())
1076 auto half =
static_cast<int8_t
>(posVal / resultType.getNumElements());
1077 posVal -= half * resultType.getNumElements();
1078 src = aievec::ExtOp::create(rewriter, extOp.getLoc(), resultType, src,
1079 rewriter.getI8IntegerAttr(half))
1083 unsigned elWidth = resultType.getElementType().getIntOrFloatBitWidth();
1086 laneSize * elWidth == 512) {
1088 rewriter.replaceOpWithNewOp<aievec::BroadcastOp>(bcastOp, resultType, src,
1090 }
else if (laneSize * elWidth == 256) {
1092 VectorType aievecBcastType =
1095 aievec::ConcatOp::create(rewriter, bcastOp.getLoc(), aievecBcastType,
1096 SmallVector<Value>({src, src}));
1097 auto aieBcastOp = aievec::BroadcastOp::create(
1098 rewriter, bcastOp.getLoc(), aievecBcastType, concatOp.getResult(),
1100 rewriter.replaceOpWithNewOp<aievec::ExtOp>(bcastOp, resultType,
1101 aieBcastOp.getResult(), 0);
1102 }
else if (laneSize * elWidth == 1024) {
1104 VectorType aievecBcastType =
1106 auto half =
static_cast<int8_t
>(posVal / resultType.getNumElements());
1107 posVal -= half * resultType.getNumElements();
1109 aievec::ExtOp::create(rewriter, bcastOp.getLoc(), aievecBcastType,
1110 src, rewriter.getI8IntegerAttr(half));
1111 auto aieBcastOp = aievec::BroadcastOp::create(rewriter, bcastOp.getLoc(),
1113 extOp.getResult(), posVal);
1114 rewriter.replaceOpWithNewOp<aievec::ConcatOp>(
1115 bcastOp, resultType,
1116 SmallVector<Value>({aieBcastOp.getResult(), aieBcastOp.getResult()}));
1126 using OpConversionPattern::OpConversionPattern;
1130 ConversionPatternRewriter &rewriter)
const override {
1132 if (adaptor.getSource().getDefiningOp<vector::ExtractOp>())
1135 auto resultType = cast<VectorType>(bcastOp.getResult().getType());
1137 Type scalarType = resultType.getElementType();
1138 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
1140 auto src = bcastOp.getSource();
1142 if (laneSize * elWidth == 512) {
1143 Value newOp = aievec::BroadcastScalarOp::create(
1144 rewriter, bcastOp.getLoc(), flatResultType, src);
1145 if (resultType != flatResultType)
1146 newOp = vector::ShapeCastOp::create(rewriter, bcastOp.getLoc(),
1148 rewriter.replaceOp(bcastOp, newOp);
1152 if (laneSize * elWidth == 256) {
1154 auto aieBcastOp = aievec::BroadcastScalarOp::create(
1155 rewriter, bcastOp.getLoc(), vecType, src);
1157 aievec::ExtOp::create(rewriter, bcastOp.getLoc(), flatResultType,
1158 aieBcastOp.getResult(), 0);
1159 if (resultType != flatResultType)
1160 newOp = vector::ShapeCastOp::create(rewriter, bcastOp.getLoc(),
1162 rewriter.replaceOp(bcastOp, newOp);
1166 if (laneSize * elWidth == 1024) {
1168 auto aieBcastOp = aievec::BroadcastScalarOp::create(
1169 rewriter, bcastOp.getLoc(), vecType, src);
1170 Value newOp = aievec::ConcatOp::create(
1171 rewriter, bcastOp.getLoc(), flatResultType,
1172 SmallVector<Value>({aieBcastOp.getResult(), aieBcastOp.getResult()}));
1173 if (resultType != flatResultType)
1174 newOp = vector::ShapeCastOp::create(rewriter, bcastOp.getLoc(),
1176 rewriter.replaceOp(bcastOp, newOp);
1187 : OpConversionPattern<arith::AddIOp> {
1188 using OpConversionPattern::OpConversionPattern;
1196 ConversionPatternRewriter &rewriter)
const override {
1198 auto resultType = dyn_cast<VectorType>(addOp.getType());
1204 extractMACOperandsFromAddOperands(adaptor.getLhs(), adaptor.getRhs());
1207 auto [lhs, rhs, acc] = *res;
1210 unsigned resultElWidth =
1211 resultType.getElementType().getIntOrFloatBitWidth();
1214 if ((laneSize != 32 || resultElWidth != 16) &&
1215 (laneSize != 16 || resultElWidth != 32))
1220 auto upsOp = aievec::UPSOp::create(rewriter, addOp.getLoc(), accType, acc,
1222 auto fmaElemOp = aievec::FMAElemOp::create(
1223 rewriter, addOp.getLoc(), accType, lhs, rhs, upsOp.getResult(),
1226 auto shiftParamOp = arith::ConstantOp::create(
1227 rewriter, addOp.getLoc(), rewriter.getI32IntegerAttr(
shiftParam));
1228 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
1229 addOp, resultType, fmaElemOp.getResult(), shiftParamOp.getResult());
1238static Value lowerBF16FMAHalf(Value lhs, Value rhs, Value acc,
1239 unsigned shiftParam, Location loc,
1240 ConversionPatternRewriter &rewriter) {
1241 auto f32AccType = VectorType::get({16}, rewriter.getF32Type());
1243 aievec::UPSOp::create(rewriter, loc, f32AccType, acc, shiftParam);
1244 auto fmaElemOp = aievec::FMAElemOp::create(rewriter, loc, f32AccType, lhs,
1245 rhs, upsOp.getResult(),
1247 auto shiftParamOp = arith::ConstantOp::create(
1248 rewriter, loc, rewriter.getI32IntegerAttr(shiftParam));
1250 aievec::SRSOp::create(rewriter, loc, cast<VectorType>(lhs.getType()),
1251 fmaElemOp.getResult(), shiftParamOp.getResult());
1252 return srsOp.getResult();
1263 : OpConversionPattern<vector::FMAOp> {
1264 using OpConversionPattern::OpConversionPattern;
1272 ConversionPatternRewriter &rewriter)
const override {
1274 auto resVecTy = cast<VectorType>(fmaOp.getType());
1275 auto resElemTy = resVecTy.getElementType();
1279 if ((!resElemTy.isF32() && !resElemTy.isBF16()) ||
1280 (numElems != 16 && numElems != 32))
1281 return rewriter.notifyMatchFailure(
1282 fmaOp,
"Unsupported operand types in vector.fma lowering.");
1284 Value lhs = adaptor.getLhs();
1285 Value rhs = adaptor.getRhs();
1286 Value acc = adaptor.getAcc();
1288 if (resElemTy.isF32()) {
1289 if (!holdsBF16(lhs) || !holdsBF16(rhs))
1290 return rewriter.notifyMatchFailure(
1291 fmaOp,
"vector.fma operands are f32 and do not hold bf16 values; "
1292 "can't lower to aievec.");
1293 Location loc = fmaOp.getLoc();
1294 lhs = getBF16Source(rewriter, loc, lhs);
1295 rhs = getBF16Source(rewriter, loc, rhs);
1296 if (numElems == 16) {
1297 rewriter.replaceOpWithNewOp<aievec::FMAElemOp>(
1298 fmaOp, acc.getType(), lhs, rhs, acc,
false);
1304 SmallVector<Value> halves;
1305 for (
int idx : {0, 1}) {
1307 aievec::ExtOp::create(rewriter, loc, bf16HalfTy, lhs, idx);
1309 aievec::ExtOp::create(rewriter, loc, bf16HalfTy, rhs, idx);
1311 aievec::ExtOp::create(rewriter, loc, f32HalfTy, acc, idx);
1312 halves.push_back(aievec::FMAElemOp::create(rewriter, loc, f32HalfTy,
1313 lhsHalf, rhsHalf, accHalf,
1316 rewriter.replaceOpWithNewOp<aievec::ConcatOp>(fmaOp, resVecTy, halves);
1321 if (numElems == 32 && resElemTy.isBF16()) {
1325 splitWideVectorOp<vector::FMAOp>(
1326 fmaOp, {lhs, rhs, acc}, halfType, resVecTy, rewriter,
1327 [localShiftParam](ArrayRef<std::pair<Value, Value>> halfInputs,
1328 Location loc, ConversionPatternRewriter &rewriter) {
1329 auto [lhsLow, lhsHigh] = halfInputs[0];
1330 auto [rhsLow, rhsHigh] = halfInputs[1];
1331 auto [accLow, accHigh] = halfInputs[2];
1333 Value lowResult = lowerBF16FMAHalf(lhsLow, rhsLow, accLow,
1334 localShiftParam, loc, rewriter);
1335 Value highResult = lowerBF16FMAHalf(lhsHigh, rhsHigh, accHigh,
1336 localShiftParam, loc, rewriter);
1337 return std::make_pair(lowResult, highResult);
1342 acc = aievec::UPSOp::create(rewriter, fmaOp.getLoc(),
1343 VectorType::get({16}, rewriter.getF32Type()),
1346 aievec::FMAElemOp::create(rewriter, fmaOp.getLoc(), acc.getType(), lhs,
1348 auto shiftParamOp = arith::ConstantOp::create(
1349 rewriter, fmaOp.getLoc(), rewriter.getI32IntegerAttr(
shiftParam));
1350 newOp = aievec::SRSOp::create(rewriter, fmaOp.getLoc(), resVecTy, newOp,
1353 rewriter.replaceOp(fmaOp, newOp);
1364 : OpConversionPattern<vector::FMAOp> {
1365 using OpConversionPattern::OpConversionPattern;
1369 ConversionPatternRewriter &rewriter)
const override {
1370 auto resVecTy = cast<VectorType>(fmaOp.getType());
1371 if (!resVecTy.getElementType().isF32() || resVecTy.getRank() != 1 ||
1372 resVecTy.getNumElements() != 32 || !holdsBF16(adaptor.getLhs()) ||
1373 !holdsBF16(adaptor.getRhs()))
1375 Location loc = fmaOp.getLoc();
1376 Value lhs = getBF16Source(rewriter, loc, adaptor.getLhs());
1377 Value rhs = getBF16Source(rewriter, loc, adaptor.getRhs());
1378 rewriter.replaceOpWithNewOp<aievec::FMAElemOp>(
1379 fmaOp, resVecTy, lhs, rhs, adaptor.getAcc(),
false);
1386 : OpConversionPattern<arith::MulFOp> {
1387 using OpConversionPattern::OpConversionPattern;
1391 ConversionPatternRewriter &rewriter)
const override {
1392 auto resVecTy = dyn_cast<VectorType>(mulOp.getType());
1393 if (!resVecTy || !resVecTy.getElementType().isF32() ||
1394 resVecTy.getRank() != 1 ||
1395 (resVecTy.getNumElements() != 32 && resVecTy.getNumElements() != 64) ||
1396 !holdsBF16(adaptor.getLhs()) || !holdsBF16(adaptor.getRhs()))
1398 Location loc = mulOp.getLoc();
1399 Value lhs = getBF16Source(rewriter, loc, adaptor.getLhs());
1400 Value rhs = getBF16Source(rewriter, loc, adaptor.getRhs());
1402 aievec::MulElemOp::create(rewriter, loc, resVecTy, lhs, rhs);
1403 rewriter.replaceOpWithNewOp<aievec::CastOp>(mulOp, resVecTy, mulElemOp,
1412 : OpConversionPattern<arith::AddFOp> {
1413 using OpConversionPattern::OpConversionPattern;
1421 ConversionPatternRewriter &rewriter)
const override {
1423 auto resultType = dyn_cast<VectorType>(addOp.getType());
1428 auto elemType = resultType.getElementType();
1429 if (!elemType.isBF16())
1434 extractFMACOperandsFromAddOperands(adaptor.getLhs(), adaptor.getRhs());
1437 auto [lhs, rhs, acc] = *res;
1442 if (laneSize == 32) {
1446 splitWideVectorOp<arith::AddFOp>(
1447 addOp, {lhs, rhs, acc}, halfType, resultType, rewriter,
1448 [localShiftParam](ArrayRef<std::pair<Value, Value>> halfInputs,
1449 Location loc, ConversionPatternRewriter &rewriter) {
1450 auto [lhsLow, lhsHigh] = halfInputs[0];
1451 auto [rhsLow, rhsHigh] = halfInputs[1];
1452 auto [accLow, accHigh] = halfInputs[2];
1454 Value lowResult = lowerBF16FMAHalf(lhsLow, rhsLow, accLow,
1455 localShiftParam, loc, rewriter);
1456 Value highResult = lowerBF16FMAHalf(lhsHigh, rhsHigh, accHigh,
1457 localShiftParam, loc, rewriter);
1458 return std::make_pair(lowResult, highResult);
1467 auto f32AccType = VectorType::get({16}, rewriter.getF32Type());
1468 auto upsOp = aievec::UPSOp::create(rewriter, addOp.getLoc(), f32AccType,
1470 auto fmaElemOp = aievec::FMAElemOp::create(
1471 rewriter, addOp.getLoc(), f32AccType, lhs, rhs, upsOp.getResult(),
1474 auto shiftParamOp = arith::ConstantOp::create(
1475 rewriter, addOp.getLoc(), rewriter.getI32IntegerAttr(
shiftParam));
1476 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
1477 addOp, resultType, fmaElemOp.getResult(), shiftParamOp.getResult());
1488 : OpConversionPattern<arith::MulFOp> {
1489 using OpConversionPattern::OpConversionPattern;
1497 ConversionPatternRewriter &rewriter)
const override {
1499 auto resultType = dyn_cast<VectorType>(mulOp.getType());
1507 auto isAddOp = [&](Operation *op) {
return isa<arith::AddFOp>(op); };
1508 if (resultType.getElementType().isBF16() && mulOp->hasOneUse() &&
1509 llvm::any_of(mulOp->getUsers(), isAddOp))
1512 unsigned resultElWidth =
1513 resultType.getElementType().getIntOrFloatBitWidth();
1518 if (resultType.getElementType().isF32() &&
1519 (laneSize == 16 || laneSize == 32) && holdsBF16(adaptor.getLhs()) &&
1520 holdsBF16(adaptor.getRhs())) {
1521 Location loc = mulOp.getLoc();
1522 Value lhs = getBF16Source(rewriter, loc, adaptor.getLhs());
1523 Value rhs = getBF16Source(rewriter, loc, adaptor.getRhs());
1526 auto mul16 = [&](Value l, Value r) -> Value {
1528 aievec::MulElemOp::create(rewriter, loc, f32HalfTy, l, r);
1529 return aievec::CastOp::create(rewriter, loc, f32HalfTy, mulElemOp,
1532 if (laneSize == 16) {
1533 rewriter.replaceOp(mulOp, mul16(lhs, rhs));
1536 SmallVector<Value> halves;
1537 for (
int idx : {0, 1})
1539 mul16(aievec::ExtOp::create(rewriter, loc, bf16HalfTy, lhs, idx),
1540 aievec::ExtOp::create(rewriter, loc, bf16HalfTy, rhs, idx)));
1541 rewriter.replaceOpWithNewOp<aievec::ConcatOp>(mulOp, resultType, halves);
1546 if (laneSize == 32 && resultElWidth == 16) {
1550 splitWideBinaryVectorOp<arith::MulFOp>(
1551 mulOp, adaptor.getLhs(), adaptor.getRhs(), halfType, resultType,
1553 [localShiftParam](Value lhsHalf, Value rhsHalf, Location loc,
1554 ConversionPatternRewriter &rewriter) -> Value {
1555 Type accType = getVectorOpDestType(
1556 cast<VectorType>(lhsHalf.getType()), true);
1557 auto mulElemOp = aievec::MulElemOp::create(rewriter, loc, accType,
1559 auto shiftParamOp = arith::ConstantOp::create(
1560 rewriter, loc, rewriter.getI32IntegerAttr(localShiftParam));
1561 auto srsOp = aievec::SRSOp::create(
1562 rewriter, loc, cast<VectorType>(lhsHalf.getType()),
1563 mulElemOp.getResult(), shiftParamOp.getResult());
1564 return srsOp.getResult();
1570 if (laneSize != 16 || (resultElWidth != 16 && resultElWidth != 32))
1574 auto lval = adaptor.getLhs();
1575 auto rval = adaptor.getRhs();
1576 lval = getSourceOfWideningOp(lval).value_or(lval);
1577 rval = getSourceOfWideningOp(rval).value_or(rval);
1578 auto lSrcType = cast<VectorType>(lval.getType());
1579 auto rSrcType = cast<VectorType>(rval.getType());
1580 unsigned lBitWidth = lSrcType.getElementType().getIntOrFloatBitWidth();
1581 unsigned rBitWidth = rSrcType.getElementType().getIntOrFloatBitWidth();
1583 if (rBitWidth > lBitWidth) {
1587 if (lSrcType != rSrcType) {
1592 unsigned bitWidth = (rBitWidth > lBitWidth) ? rBitWidth : lBitWidth;
1593 Type srcElemType = (rBitWidth > lBitWidth) ? rSrcType.getElementType()
1594 : lSrcType.getElementType();
1595 unsigned numLanes = 0;
1598 if (isa<FloatType>(srcElemType) &&
1599 (bitWidth == 16 || bitWidth == 32)) {
1601 }
else if (isa<IntegerType>(srcElemType) &&
1602 (bitWidth == 8 || bitWidth == 16)) {
1604 }
else if (isa<IntegerType>(srcElemType) && (bitWidth == 32)) {
1610 if (targetInputType != lSrcType) {
1611 auto converted = convertValueToTargetTypeAIE2(rewriter, mulOp.getLoc(),
1612 lval, targetInputType);
1617 if (targetInputType != rSrcType) {
1618 auto converted = convertValueToTargetTypeAIE2(rewriter, mulOp.getLoc(),
1619 rval, targetInputType);
1628 auto mulElemOp = aievec::MulElemOp::create(rewriter, mulOp.getLoc(),
1629 accType, lval, rval);
1632 auto mulElemResultType = mulElemOp.getType();
1633 auto mulElemResultElWidth =
1634 mulElemResultType.getElementType().getIntOrFloatBitWidth();
1636 if (mulElemResultElWidth == resultElWidth) {
1637 rewriter.replaceOpWithNewOp<aievec::CastOp>(
1638 mulOp, resultType, mulElemOp.getResult(),
false);
1639 }
else if (mulElemResultElWidth > resultElWidth) {
1640 auto shiftParamOp = arith::ConstantOp::create(
1641 rewriter, mulOp.getLoc(), rewriter.getI32IntegerAttr(
shiftParam));
1642 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
1643 mulOp, resultType, mulElemOp.getResult(), shiftParamOp.getResult());
1657 : OpConversionPattern<arith::MulIOp> {
1658 using OpConversionPattern::OpConversionPattern;
1666 ConversionPatternRewriter &rewriter)
const override {
1668 auto resultType = dyn_cast<VectorType>(mulOp.getType());
1673 auto isAddOp = [&](Operation *op) {
return isa<arith::AddIOp>(op); };
1674 if (mulOp->hasOneUse() && llvm::any_of(mulOp->getUsers(), isAddOp))
1678 unsigned resultElWidth =
1679 resultType.getElementType().getIntOrFloatBitWidth();
1682 if ((laneSize != 32 || (resultElWidth != 16 && resultElWidth != 8)) &&
1683 ((laneSize != 16 && laneSize != 32) || resultElWidth != 32))
1687 auto lval = adaptor.getLhs();
1688 auto rval = adaptor.getRhs();
1690 lval = getSourceOfWideningOp(lval).value_or(lval);
1691 rval = getSourceOfWideningOp(rval).value_or(rval);
1693 auto lSrcType = cast<VectorType>(lval.getType());
1694 auto rSrcType = cast<VectorType>(rval.getType());
1695 unsigned lBitWidth = lSrcType.getElementType().getIntOrFloatBitWidth();
1696 unsigned rBitWidth = rSrcType.getElementType().getIntOrFloatBitWidth();
1698 if (rBitWidth > lBitWidth) {
1703 unsigned bitWidth = (rBitWidth > lBitWidth) ? rBitWidth : lBitWidth;
1704 Type srcElemType = (rBitWidth > lBitWidth) ? rSrcType.getElementType()
1705 : lSrcType.getElementType();
1706 unsigned numLanes = 0;
1709 if (isa<FloatType>(srcElemType) &&
1710 (bitWidth == 16 || bitWidth == 32)) {
1712 }
else if (isa<IntegerType>(srcElemType) &&
1713 (bitWidth == 8 || bitWidth == 16)) {
1715 }
else if (isa<IntegerType>(srcElemType) && (bitWidth == 32)) {
1721 if (targetInputType != lSrcType) {
1722 auto converted = convertValueToTargetTypeAIE2(rewriter, mulOp.getLoc(),
1723 lval, targetInputType);
1728 if (targetInputType != rSrcType) {
1729 auto converted = convertValueToTargetTypeAIE2(rewriter, mulOp.getLoc(),
1730 rval, targetInputType);
1739 auto mulElemOp = aievec::MulElemOp::create(rewriter, mulOp.getLoc(),
1740 accType, lval, rval);
1743 auto mulElemResultType = mulElemOp.getType();
1744 auto mulElemResultElWidth =
1745 mulElemResultType.getElementType().getIntOrFloatBitWidth();
1747 if (mulElemResultElWidth == resultElWidth) {
1748 rewriter.replaceOpWithNewOp<aievec::CastOp>(
1749 mulOp, resultType, mulElemOp.getResult(),
false);
1750 }
else if (mulElemResultElWidth > resultElWidth) {
1751 auto shiftParamOp = arith::ConstantOp::create(
1752 rewriter, mulOp.getLoc(), rewriter.getI32IntegerAttr(
shiftParam));
1753 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
1754 mulOp, resultType, mulElemOp.getResult(), shiftParamOp.getResult());
1768 using OpConversionPattern::OpConversionPattern;
1772 ConversionPatternRewriter &rewriter)
const override {
1774 dyn_cast<aievec::ConcatOp>(adaptor.getLhs().getDefiningOp());
1777 vector::BroadcastOp bcastOp =
nullptr;
1778 auto *concatDefOp = concatOp.getSources()[0].getDefiningOp();
1780 bcastOp = dyn_cast<vector::BroadcastOp>(concatDefOp);
1781 Value lhs = adaptor.getRhs();
1783 bcastOp = dyn_cast<vector::BroadcastOp>(adaptor.getRhs().getDefiningOp());
1786 lhs = concatOp.getSources()[0];
1789 dyn_cast<vector::ExtractOp>(bcastOp.getSource().getDefiningOp());
1793 auto rhs = extOp.getSource();
1794 auto concatVecType = cast<VectorType>(concatOp.getResult().getType());
1796 arith::ConstantOp::create(rewriter, concatOp.getLoc(), lhs.getType(),
1797 rewriter.getZeroAttr(lhs.getType()));
1799 aievec::ConcatOp::create(rewriter, concatOp.getLoc(), concatVecType,
1800 SmallVector<Value, 2>({lhs, zvec}))
1803 auto pos = extOp.getStaticPosition();
1804 int64_t zstart = pos[0];
1806 aievec::aie1::FMAOp::Properties fmaProps;
1807 if (failed(aievec::aie1::FMAOp::setPropertiesFromAttr(
1808 fmaProps, rewriter.getDictionaryAttr(fmaOpAttr),
1809 [&]() { return fmaOp.emitError(); })))
1811 rewriter.replaceOpWithNewOp<aievec::aie1::FMAOp>(
1812 fmaOp, TypeRange({fmaOp.getResult().getType()}),
1813 ValueRange({lhsX2, rhs, adaptor.getAcc()}), fmaProps);
1820 : OpConversionPattern<aievec::aie1::AddOp> {
1821 using OpConversionPattern::OpConversionPattern;
1825 ConversionPatternRewriter &rewriter)
const override {
1826 auto vecType = cast<VectorType>(addOp.getType());
1829 extractMACOperandsFromAddOperands(adaptor.getLhs(), adaptor.getRhs());
1832 auto [lhs, rhs, acc] = *res;
1834 SmallVector<int64_t, 4> concatVecShape(vecType.getShape().begin(),
1835 vecType.getShape().end());
1836 concatVecShape[vecType.getRank() - 1] *= 2;
1837 auto concatVecType =
1838 VectorType::get(concatVecShape, vecType.getElementType());
1842 aievec::ConcatOp::create(rewriter, addOp.getLoc(), concatVecType,
1843 SmallVector<Value, 2>(2, lhs))
1845 auto upsOp = aievec::UPSOp::create(rewriter, addOp.getLoc(), accType, acc);
1846 auto fmaOp = aievec::aie1::FMAOp::create(
1847 rewriter, addOp.getLoc(), accType, lhsX2, rhs, upsOp.getResult(),
1851 auto shiftParamOp = arith::ConstantOp::create(
1852 rewriter, addOp.getLoc(), rewriter.getI32IntegerAttr(0));
1853 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
1854 addOp, vecType, fmaOp.getResult(), shiftParamOp.getResult());
1863 : OpConversionPattern<vector::TransferReadOp> {
1864 using OpConversionPattern::OpConversionPattern;
1875 ConversionPatternRewriter &rewriter)
const override {
1877 if (readOp.getMask())
1878 return readOp.emitError() <<
"AIE doesn't support masked loads.";
1881 AffineMap map = readOp.getPermutationMap();
1882 if (!map.isMinorIdentity())
1886 if (map.isConstant())
1890 auto vType = readOp.getVectorType();
1900 int64_t vSize = vType.getNumElements() * vType.getElementTypeBitWidth();
1909 if ((vSize >
minVectorSize) && std::bitset<8>(multiplicity).count() != 1)
1912 auto updOp = xilinx::aievec::UPDOp::create(
1913 rewriter, readOp.getLoc(), vType, adaptor.getBase(),
1914 adaptor.getIndices(), 0, 0, TypedValue<VectorType>(
nullptr));
1916 updOp = xilinx::aievec::UPDOp::create(
1917 rewriter, readOp.getLoc(), vType, adaptor.getBase(),
1918 adaptor.getIndices(),
maxLoadSize, 1, updOp.getResult());
1920 rewriter.replaceOp(readOp, updOp.getResult());
1930template <
typename SrcOpTy,
typename DstOpTy>
1932 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
1937 ConversionPatternRewriter &rewriter)
const override {
1938 rewriter.replaceOpWithNewOp<DstOpTy>(
1939 srcOp, srcOp.getResult().getType(), adaptor.getLhs(), adaptor.getRhs(),
1947 using OpConversionPattern::OpConversionPattern;
1951 ConversionPatternRewriter &rewriter)
const override {
1952 auto resType = addOp.getType();
1953 if (!isa<VectorType>(resType))
1956 auto lhs = adaptor.getLhs();
1957 auto rhs = adaptor.getRhs();
1958 auto *lhsDefOp = lhs.getDefiningOp();
1959 auto *rhsDefOp = rhs.getDefiningOp();
1960 if ((isa_and_nonnull<arith::MulIOp>(lhsDefOp)) ||
1961 (isa_and_nonnull<arith::MulIOp>(rhsDefOp)))
1964 rewriter.replaceOpWithNewOp<aievec::aie1::AddOp>(
1965 addOp, resType, lhs, rhs,
1982 using OpConversionPattern::OpConversionPattern;
1985 ConversionPatternRewriter &rewriter)
const override {
1986 auto resTy = dyn_cast<VectorType>(mulOp.getType());
1990 auto newMulOp = aievec::aie1::MulOp::create(
1991 rewriter, mulOp.getLoc(), accTy, adaptor.getLhs(), adaptor.getRhs());
1992 auto shiftParamOp = arith::ConstantOp::create(
1993 rewriter, mulOp.getLoc(), rewriter.getI32IntegerAttr(0));
1994 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
1995 mulOp, resTy, newMulOp.getResult(), shiftParamOp.getResult());
2000template <
typename SrcOpTy,
typename DstOpTy>
2002 : OpConversionPattern<SrcOpTy> {
2003 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
2008 ConversionPatternRewriter &rewriter)
const override {
2009 VectorType resultType = dyn_cast<VectorType>(srcOp.getType());
2015 llvm::SmallSet<std::pair<unsigned, signed>, 16> laneSizeElWidthPairSet;
2016 laneSizeElWidthPairSet.insert({64, 8});
2017 laneSizeElWidthPairSet.insert({32, 16});
2018 laneSizeElWidthPairSet.insert({16, 32});
2019 laneSizeElWidthPairSet.insert({32, 32});
2021 auto lhs = adaptor.getLhs();
2022 auto rhs = adaptor.getRhs();
2023 auto lhsDefOp = lhs.getDefiningOp();
2024 auto rhsDefOp = rhs.getDefiningOp();
2029 bool lhsIsMul = lhsDefOp && (isa<arith::MulIOp>(lhsDefOp) ||
2030 isa<arith::MulFOp>(lhsDefOp));
2031 bool rhsIsMul = rhsDefOp && (isa<arith::MulIOp>(rhsDefOp) ||
2032 isa<arith::MulFOp>(rhsDefOp));
2033 bool lhsIsConst = isa_and_nonnull<arith::ConstantOp>(lhsDefOp);
2034 bool rhsIsConst = isa_and_nonnull<arith::ConstantOp>(rhsDefOp);
2039 if (!resultType.getElementType().isF32() &&
2040 ((lhsIsMul && !rhsIsConst) || (rhsIsMul && !lhsIsConst)))
2043 Type scalarType = resultType.getElementType();
2044 unsigned resultElWidth = scalarType.getIntOrFloatBitWidth();
2048 if (isa<IntegerType>(scalarType)) {
2049 if (!laneSizeElWidthPairSet.count(
2050 std::make_pair(laneSize, resultElWidth)))
2056 if (!lhsDefOp && !rhsDefOp) {
2057 if (laneSize * resultElWidth == 512) {
2058 rewriter.replaceOpWithNewOp<DstOpTy>(srcOp, srcOp.getType(), lhs,
2062 return genAddElemAIE2<SrcOpTy, DstOpTy>(rewriter, lhs, rhs, resultType,
2067 if (resultElWidth == 32) {
2068 auto lhsExt = getSourceOfWideningOp(lhs).value_or(
nullptr);
2069 auto rhsExt = getSourceOfWideningOp(rhs).value_or(
nullptr);
2071 if (!lhsExt && !rhsExt) {
2072 if (laneSize * resultElWidth == 512) {
2073 rewriter.replaceOpWithNewOp<DstOpTy>(srcOp, srcOp.getType(), lhs,
2077 return genAddElemAIE2<SrcOpTy, DstOpTy>(rewriter, lhs, rhs,
2081 if (lhsExt && rhsExt) {
2084 VectorType lSrcType = cast<VectorType>(lval.getType());
2088 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, lval);
2090 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, rval);
2091 auto elemOp = DstOpTy::create(
2092 rewriter, srcOp.getLoc(), lUpsOp->getResult(0).getType(),
2093 lUpsOp->getResult(0), rUpsOp->getResult(0));
2094 rewriter.replaceOpWithNewOp<aievec::CastOp>(
2095 srcOp, srcOp.getType(), elemOp.getResult(),
false);
2099 if (!lhsExt || !rhsExt) {
2100 auto lval = lhsExt ? lhsExt : lhs;
2101 auto rval = rhsExt ? rhsExt : rhs;
2102 auto extVal = lhsExt ? lval : rval;
2103 VectorType vType = cast<VectorType>(extVal.getType());
2104 unsigned bitWidth = vType.getElementType().getIntOrFloatBitWidth();
2106 if (bitWidth != 8 && bitWidth != 16) {
2107 return genAddElemAIE2<SrcOpTy, DstOpTy>(rewriter, lhs, rhs,
2111 if (bitWidth * laneSize != 256) {
2112 return genAddElemAIE2<SrcOpTy, DstOpTy>(rewriter, lhs, rhs,
2116 Type accType =
nullptr;
2118 if (bitWidth == 8) {
2120 Value valToUps = lhsExt ? lval : rval;
2121 Value valToCast = lhsExt ? rval : lval;
2122 auto upsOp = aievec::UPSOp::create(rewriter, srcOp.getLoc(),
2125 aievec::CastOp::create(rewriter, srcOp.getLoc(), resultType,
2128 lhsExt ? upsOp->getResult(0) : castOp->getResult(0);
2130 lhsExt ? castOp->getResult(0) : upsOp->getResult(0);
2131 auto elemOp = DstOpTy::create(rewriter, srcOp.getLoc(),
2132 upsOp->getResult(0).getType(),
2133 lhsToElemOp, rhsToElemOp);
2134 rewriter.replaceOpWithNewOp<aievec::CastOp>(
2135 srcOp, srcOp.getType(), elemOp.getResult(),
false);
2139 if (bitWidth == 16) {
2142 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, lval);
2144 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, rval);
2146 auto elemOp = DstOpTy::create(
2147 rewriter, srcOp.getLoc(), lUpsOp->getResult(0).getType(),
2148 lUpsOp->getResult(0), rUpsOp->getResult(0));
2150 auto shiftParamOp = arith::ConstantOp::create(
2151 rewriter, srcOp.getLoc(), rewriter.getI32IntegerAttr(0));
2152 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
2153 srcOp, srcOp.getType(), elemOp.getResult(),
2154 shiftParamOp.getResult());
2159 rewriter.replaceOpWithNewOp<DstOpTy>(srcOp, srcOp.getType(), lhs, rhs);
2166 if (laneSize == 64 && scalarType.isF32())
2167 return genAddElemAIE2<SrcOpTy, DstOpTy>(rewriter, lhs, rhs, resultType,
2169 if (laneSize != 16 && laneSize != 32)
2173 if (laneSize == 32 && resultElWidth == 32) {
2176 splitWideBinaryVectorOp<SrcOpTy>(
2177 srcOp, lhs, rhs, halfType, resultType, rewriter,
2178 [](Value lhsHalf, Value rhsHalf, Location loc,
2179 ConversionPatternRewriter &rewriter) -> Value {
2180 VectorType halfVecType = cast<VectorType>(lhsHalf.getType());
2182 auto lCastOp = aievec::CastOp::create(rewriter, loc, halfVecType,
2184 auto rCastOp = aievec::CastOp::create(rewriter, loc, halfVecType,
2186 auto elemOp = DstOpTy::create(
2187 rewriter, loc, lCastOp->getResult(0).getType(),
2188 lCastOp->getResult(0), rCastOp->getResult(0));
2189 auto resCastOp = aievec::CastOp::create(
2190 rewriter, loc, halfVecType, elemOp.getResult(),
2192 return resCastOp.getResult();
2198 if (laneSize == 32 && resultElWidth == 16) {
2201 splitWideBinaryVectorOp<SrcOpTy>(
2202 srcOp, lhs, rhs, halfType, resultType, rewriter,
2203 [](Value lhsHalf, Value rhsHalf, Location loc,
2204 ConversionPatternRewriter &rewriter) -> Value {
2205 VectorType halfVecType = cast<VectorType>(lhsHalf.getType());
2208 aievec::UPSOp::create(rewriter, loc, accType, lhsHalf);
2210 aievec::UPSOp::create(rewriter, loc, accType, rhsHalf);
2212 DstOpTy::create(rewriter, loc, lUpsOp->getResult(0).getType(),
2213 lUpsOp->getResult(0), rUpsOp->getResult(0));
2214 auto shiftParamOp = arith::ConstantOp::create(
2215 rewriter, loc, rewriter.getI32IntegerAttr(0));
2216 auto srsOp = aievec::SRSOp::create(rewriter, loc, halfVecType,
2218 shiftParamOp.getResult());
2219 return srsOp.getResult();
2226 if (resultElWidth == 32) {
2227 if (!lhsDefOp && !rhsDefOp) {
2228 return genAddElemAIE2<SrcOpTy, DstOpTy>(rewriter, lhs, rhs,
2232 auto lhsExt = getSourceOfWideningOp(lhs).value_or(
nullptr);
2233 auto rhsExt = getSourceOfWideningOp(rhs).value_or(
nullptr);
2235 if (!lhsExt && !rhsExt) {
2236 return genAddElemAIE2<SrcOpTy, DstOpTy>(rewriter, lhs, rhs,
2241 if (lhsExt && rhsExt) {
2244 VectorType vType = cast<VectorType>(lval.getType());
2248 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, lval);
2250 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, rval);
2251 auto elemOp = DstOpTy::create(
2252 rewriter, srcOp.getLoc(), lUpsOp->getResult(0).getType(),
2253 lUpsOp->getResult(0), rUpsOp->getResult(0));
2254 rewriter.replaceOpWithNewOp<aievec::CastOp>(srcOp, srcOp.getType(),
2255 elemOp.getResult());
2260 if (!lhsExt || !rhsExt) {
2261 auto lval = lhsExt ? lhsExt : lhs;
2262 auto rval = rhsExt ? rhsExt : rhs;
2263 auto extVal = lhsExt ? lval : rval;
2264 VectorType vType = cast<VectorType>(extVal.getType());
2267 aievec::UPSOp upsOp;
2268 aievec::CastOp castOp;
2271 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, lval);
2272 castOp = aievec::CastOp::create(rewriter, srcOp.getLoc(),
2277 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, rval);
2278 castOp = aievec::CastOp::create(rewriter, srcOp.getLoc(),
2283 auto elemOp = DstOpTy::create(
2284 rewriter, srcOp.getLoc(), upsOp->getResult(0).getType(),
2285 upsOp->getResult(0), castOp->getResult(0));
2287 rewriter.replaceOpWithNewOp<aievec::CastOp>(
2288 srcOp, srcOp.getType(), elemOp.getResult(),
false);
2297 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, lhs);
2299 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, rhs);
2300 auto elemOp = DstOpTy::create(rewriter, srcOp.getLoc(),
2301 lUpsOp->getResult(0).getType(),
2302 lUpsOp->getResult(0), rUpsOp->getResult(0));
2303 auto shiftParamOp = arith::ConstantOp::create(
2304 rewriter, srcOp.getLoc(), rewriter.getI32IntegerAttr(0));
2305 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
2306 srcOp, srcOp.getType(), elemOp.getResult(), shiftParamOp.getResult());
2328template <
typename SrcOpTy,
typename DstOpTy>
2330 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
2335 ConversionPatternRewriter &rewriter)
const override {
2336 VectorType resultType = dyn_cast<VectorType>(srcOp.getType());
2341 llvm::SmallSet<unsigned, 16> elWidthSet;
2342 elWidthSet.insert(8);
2343 elWidthSet.insert(16);
2344 elWidthSet.insert(32);
2346 Type scalarType = resultType.getElementType();
2347 unsigned resultElWidth = scalarType.getIntOrFloatBitWidth();
2350 unsigned totalBits = laneSize * resultElWidth;
2351 if (!elWidthSet.count(resultElWidth) ||
2352 (totalBits != 512 && (totalBits != 256 || resultElWidth != 16)))
2355 if (totalBits == 256 && resultElWidth == 16) {
2357 Location loc = srcOp.getLoc();
2360 padV16ToV32WithZeros(rewriter, loc, adaptor.getLhs(), scalarType);
2362 padV16ToV32WithZeros(rewriter, loc, adaptor.getRhs(), scalarType);
2363 auto wideOp = DstOpTy::create(rewriter, loc, wideType, lhsPad, rhsPad);
2364 rewriter.replaceOpWithNewOp<aievec::ExtOp>(srcOp, resultType,
2365 wideOp.getResult(), 0);
2369 rewriter.replaceOpWithNewOp<DstOpTy>(srcOp, srcOp.getType(),
2370 adaptor.getLhs(), adaptor.getRhs());
2383template <
typename SrcOpTy, arith::CmpIPredicate Pred>
2385 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
2390 ConversionPatternRewriter &rewriter)
const override {
2391 if (!isa<VectorType>(srcOp.getType()))
2393 Value lhs = adaptor.getLhs();
2394 Value rhs = adaptor.getRhs();
2396 arith::CmpIOp::create(rewriter, srcOp.getLoc(), Pred, lhs, rhs);
2397 rewriter.replaceOpWithNewOp<arith::SelectOp>(srcOp, cmpOp, lhs, rhs);
2409template <
typename SrcOpTy,
typename DstOpTy>
2411 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
2416 ConversionPatternRewriter &rewriter)
const override {
2418 Type resultType = srcOp.getType();
2419 if (isa<VectorType>(resultType))
2422 auto intType = dyn_cast<IntegerType>(resultType);
2426 unsigned elWidth = intType.getWidth();
2427 if (elWidth != 8 && elWidth != 16 && elWidth != 32)
2430 unsigned numLanes = 512 / elWidth;
2432 Location loc = srcOp.getLoc();
2435 auto lhsBcast = aievec::BroadcastScalarOp::create(rewriter, loc, vecType,
2437 auto rhsBcast = aievec::BroadcastScalarOp::create(rewriter, loc, vecType,
2441 auto vecOp = DstOpTy::create(rewriter, loc, vecType, lhsBcast.getResult(),
2442 rhsBcast.getResult());
2446 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
2447 rewriter.replaceOpWithNewOp<aievec::ExtElemOp>(
2448 srcOp, intType, vecOp.getResult(), zeroIdx.getResult());
2465template <
typename SrcOpTy,
typename CmpTy>
2467 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
2472 ConversionPatternRewriter &rewriter)
const override {
2473 VectorType lhsType = dyn_cast<VectorType>(srcOp.getLhs().getType());
2477 llvm::SmallSet<unsigned, 16> elWidthSet;
2478 elWidthSet.insert(8);
2479 elWidthSet.insert(16);
2480 elWidthSet.insert(32);
2482 Type scalarType = lhsType.getElementType();
2483 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
2486 unsigned totalBits = laneSize * elWidth;
2487 if (!elWidthSet.count(elWidth) ||
2488 (totalBits != 512 && (totalBits != 256 || elWidth != 16)))
2491 Location loc = srcOp.getLoc();
2492 Value lhs = srcOp.getLhs();
2493 Value rhs = srcOp.getRhs();
2494 unsigned effectiveLaneSize = laneSize;
2496 if (totalBits == 256 && elWidth == 16) {
2497 lhs = padV16ToV32WithZeros(rewriter, loc, lhs, scalarType);
2498 rhs = padV16ToV32WithZeros(rewriter, loc, rhs, scalarType);
2499 effectiveLaneSize = 32;
2503 Type type = mlir::IntegerType::get(srcOp.getContext(),
2504 effectiveLaneSize <= 32 ? 32 : 64,
2505 mlir::IntegerType::Unsigned);
2507 CmpTy pred = srcOp.getPredicate();
2509 arith::CmpIPredicate ipred = convertToIntegerPredicate(pred);
2511 aievec::CmpOp aieCmpOp =
2512 createCmpOpAIE2(rewriter, ipred, loc, type, lhs, rhs);
2517 VectorType resultType = cast<VectorType>(srcOp.getResult().getType());
2520 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
2521 srcOp, resultType, aieCmpOp.getResult());
2533 using OpConversionPattern::OpConversionPattern;
2537 ConversionPatternRewriter &rewriter)
const override {
2538 auto resultType = dyn_cast<VectorType>(srcOp.getType());
2542 llvm::SmallSet<unsigned, 16> elWidthSet;
2543 elWidthSet.insert(8);
2544 elWidthSet.insert(16);
2545 elWidthSet.insert(32);
2547 Type scalarType = resultType.getElementType();
2548 unsigned resultElWidth = scalarType.getIntOrFloatBitWidth();
2551 unsigned totalBits = laneSize * resultElWidth;
2552 if (!elWidthSet.count(resultElWidth) ||
2553 (totalBits != 512 && (totalBits != 256 || resultElWidth != 16)))
2556 if (totalBits == 256 && resultElWidth == 16) {
2558 Location loc = srcOp.getLoc();
2565 Value falsePad = padV16ToV32WithZeros(rewriter, loc,
2566 srcOp.getFalseValue(), scalarType);
2568 padV16ToV32WithZeros(rewriter, loc, srcOp.getTrueValue(), scalarType);
2572 Type type = mlir::IntegerType::get(srcOp.getContext(), 32,
2573 mlir::IntegerType::Unsigned);
2574 auto convertOp = UnrealizedConversionCastOp::create(
2575 rewriter, loc, type, adaptor.getCondition());
2577 auto wideSelOp = aievec::SelOp::create(rewriter, loc, wideType, falsePad,
2578 truePad, convertOp.getResult(0));
2580 rewriter.replaceOpWithNewOp<aievec::ExtOp>(srcOp, resultType,
2581 wideSelOp.getResult(), 0);
2586 mlir::IntegerType::get(srcOp.getContext(), laneSize <= 32 ? 32 : 64,
2587 mlir::IntegerType::Unsigned);
2589 auto convertOp = UnrealizedConversionCastOp::create(
2590 rewriter, srcOp.getLoc(), type, adaptor.getCondition());
2596 rewriter.replaceOpWithNewOp<aievec::SelOp>(
2597 srcOp, srcOp.getResult().getType(), srcOp.getFalseValue(),
2598 srcOp.getTrueValue(), convertOp.getResult(0));
2605 using OpConversionPattern::OpConversionPattern;
2609 ConversionPatternRewriter &rewriter)
const override {
2610 if (
auto kind = srcOp.getKind(); kind != vector::CombiningKind::MINSI &&
2611 kind != vector::CombiningKind::MINUI &&
2612 kind != vector::CombiningKind::MINIMUMF &&
2613 kind != vector::CombiningKind::MINNUMF)
2616 auto vType = cast<VectorType>(srcOp.getVector().getType());
2617 Type scalarType = vType.getElementType();
2618 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
2620 unsigned vectorSize = laneSize * elWidth;
2623 if (vectorSize != 512 && (vectorSize != 256 || !scalarType.isBF16()))
2626 Location loc = srcOp.getLoc();
2627 Value inputVec = srcOp.getVector();
2630 if (vectorSize == 256) {
2631 std::tie(inputVec, laneSize) = padV16ToV32WithInfinity(
2632 rewriter, loc, srcOp.getVector(), scalarType,
false);
2635 int shiftIndex = laneSize / 2;
2636 auto reduceResultOp = generateAIEVecOpsForReductionOp<aievec::MinOp>(
2637 rewriter, srcOp, shiftIndex, inputVec);
2639 if (srcOp.getAcc()) {
2640 Value reduceResult = reduceResultOp.getResult();
2641 Value acc = srcOp.getAcc();
2644 if (acc.getType().isBF16()) {
2646 auto minOpBuilder = [&](Value lhs, Value rhs) -> Value {
2647 auto cmpOp = arith::CmpFOp::create(
2648 rewriter, srcOp.getLoc(), arith::CmpFPredicate::OLT, lhs, rhs);
2649 return arith::SelectOp::create(rewriter, srcOp.getLoc(), cmpOp, lhs,
2655 performBF16BinaryOpInF32(reduceResult, acc, srcOp, srcOp.getLoc(),
2656 rewriter, minOpBuilder);
2660 arith::CmpFOp::create(rewriter, srcOp.getLoc(),
2661 arith::CmpFPredicate::OLT, reduceResult, acc);
2662 rewriter.replaceOpWithNewOp<arith::SelectOp>(srcOp, cmpOp, reduceResult,
2666 rewriter.replaceOp(srcOp, reduceResultOp);
2673 using OpConversionPattern::OpConversionPattern;
2677 ConversionPatternRewriter &rewriter)
const override {
2678 if (
auto kind = srcOp.getKind(); kind != vector::CombiningKind::MAXSI &&
2679 kind != vector::CombiningKind::MAXUI &&
2680 kind != vector::CombiningKind::MAXIMUMF &&
2681 kind != vector::CombiningKind::MAXNUMF)
2684 auto vType = cast<VectorType>(srcOp.getVector().getType());
2685 Type scalarType = vType.getElementType();
2686 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
2688 unsigned vectorSize = laneSize * elWidth;
2692 if (vectorSize != 512 && (vectorSize != 256 || !scalarType.isBF16()))
2695 Location loc = srcOp.getLoc();
2696 Value inputVec = srcOp.getVector();
2699 if (vectorSize == 256) {
2700 std::tie(inputVec, laneSize) = padV16ToV32WithInfinity(
2701 rewriter, loc, srcOp.getVector(), scalarType,
true);
2704 int shiftIndex = laneSize / 2;
2705 auto reduceResultOp = generateAIEVecOpsForReductionOp<aievec::MaxOp>(
2706 rewriter, srcOp, shiftIndex, inputVec);
2708 if (srcOp.getAcc()) {
2709 Value reduceResult = reduceResultOp.getResult();
2710 Value acc = srcOp.getAcc();
2713 if (acc.getType().isBF16()) {
2715 auto maxOpBuilder = [&](Value lhs, Value rhs) -> Value {
2716 auto cmpOp = arith::CmpFOp::create(
2717 rewriter, srcOp.getLoc(), arith::CmpFPredicate::OGT, lhs, rhs);
2718 return arith::SelectOp::create(rewriter, srcOp.getLoc(), cmpOp, lhs,
2724 performBF16BinaryOpInF32(reduceResult, acc, srcOp, srcOp.getLoc(),
2725 rewriter, maxOpBuilder);
2729 arith::CmpFOp::create(rewriter, srcOp.getLoc(),
2730 arith::CmpFPredicate::OGT, reduceResult, acc);
2731 rewriter.replaceOpWithNewOp<arith::SelectOp>(srcOp, cmpOp, reduceResult,
2735 rewriter.replaceOp(srcOp, reduceResultOp);
2742 using OpConversionPattern::OpConversionPattern;
2746 ConversionPatternRewriter &rewriter)
const override {
2747 if (
auto kind = srcOp.getKind(); kind != vector::CombiningKind::ADD)
2750 auto vType = cast<VectorType>(srcOp.getVector().getType());
2751 Type scalarType = vType.getElementType();
2752 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
2754 llvm::SmallSet<std::pair<unsigned, signed>, 16> laneSizeElWidthPairSet;
2755 laneSizeElWidthPairSet.insert({64, 8});
2756 laneSizeElWidthPairSet.insert({32, 16});
2757 laneSizeElWidthPairSet.insert({32, 32});
2758 laneSizeElWidthPairSet.insert({16, 32});
2760 if (!isa<IntegerType>(scalarType) ||
2761 !laneSizeElWidthPairSet.count(std::make_pair(laneSize, elWidth)))
2764 int shiftIndex = laneSize / 2;
2765 if (laneSize == 32 && elWidth == 32) {
2766 Location loc = srcOp.getLoc();
2770 aievec::ExtOp::create(rewriter, loc, vecType, srcOp.getVector(), 0);
2772 aievec::ExtOp::create(rewriter, loc, vecType, srcOp.getVector(), 1);
2774 aievec::AddElemOp::create(rewriter, loc, lExtOp.getResult().getType(),
2775 lExtOp.getResult(), rExtOp.getResult());
2777 auto reduceResultOp = generateAIEVecOpsForReductionOp<aievec::AddElemOp>(
2778 rewriter, srcOp, shiftIndex, addElemOp.getResult());
2780 rewriter.replaceOpWithNewOp<arith::AddIOp>(
2781 srcOp, reduceResultOp.getResult(), srcOp.getAcc());
2783 rewriter.replaceOp(srcOp, reduceResultOp);
2785 auto reduceResultOp = generateAIEVecOpsForReductionOp<aievec::AddElemOp>(
2786 rewriter, srcOp, shiftIndex, srcOp.getVector());
2788 rewriter.replaceOpWithNewOp<arith::AddIOp>(
2789 srcOp, reduceResultOp.getResult(), srcOp.getAcc());
2791 rewriter.replaceOp(srcOp, reduceResultOp);
2799 : OpConversionPattern<vector::ReductionOp> {
2800 using OpConversionPattern::OpConversionPattern;
2804 ConversionPatternRewriter &rewriter)
const override {
2805 if (
auto kind = srcOp.getKind(); kind != vector::CombiningKind::ADD)
2808 auto vType = cast<VectorType>(srcOp.getVector().getType());
2809 Type scalarType = vType.getElementType();
2810 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
2813 if (!isa<FloatType>(scalarType) || laneSize != 16 || elWidth != 32)
2816 int shiftIndex = laneSize / 2;
2817 assert(shiftIndex > 0 && (shiftIndex & (shiftIndex - 1)) == 0 &&
2818 "shiftIndex must be power of 2");
2820 Location loc = srcOp.getLoc();
2821 Value curValue = srcOp.getVector();
2822 aievec::CastOp curOp =
nullptr;
2824 for (
int id = shiftIndex;
id > 0;
id /= 2) {
2825 auto constOp = arith::ConstantOp::create(
2826 rewriter, loc, rewriter.getI32IntegerAttr(
id * elWidth / 8));
2828 auto shiftBytesOp = aievec::ShiftOp::create(
2829 rewriter, loc, vType, curValue, curValue, constOp.getResult());
2831 auto lCastOp = aievec::CastOp::create(rewriter, loc, vType, curValue,
2834 aievec::CastOp::create(rewriter, loc, vType, shiftBytesOp.getResult(),
2836 auto elemOp = aievec::AddElemOp::create(
2837 rewriter, loc, lCastOp.getResult().getType(), lCastOp.getResult(),
2838 rCastOp.getResult());
2839 curOp = aievec::CastOp::create(rewriter, loc, vType, elemOp.getResult(),
2841 curValue = curOp.getResult();
2845 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
2846 auto reduceResultOp = aievec::ExtElemOp::create(
2847 rewriter, srcOp.getLoc(), scalarType, curOp, zeroConstOp.getResult());
2850 rewriter.replaceOpWithNewOp<arith::AddFOp>(
2851 srcOp, reduceResultOp.getResult(), srcOp.getAcc());
2853 rewriter.replaceOp(srcOp, reduceResultOp);
2861 : OpConversionPattern<vector::ReductionOp> {
2862 using OpConversionPattern::OpConversionPattern;
2866 ConversionPatternRewriter &rewriter)
const override {
2868 if (
auto kind = srcOp.getKind(); kind != vector::CombiningKind::ADD) {
2872 auto vType = cast<VectorType>(srcOp.getVector().getType());
2873 Type scalarType = vType.getElementType();
2874 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
2878 if (!isa<FloatType>(scalarType) || (laneSize != 16 && laneSize != 32) ||
2883 Location loc = srcOp.getLoc();
2884 Value curValue = srcOp.getVector();
2885 VectorType currentVType = vType;
2888 if (laneSize == 32) {
2891 aievec::ExtOp::create(rewriter, loc, halfType, srcOp.getVector(), 0);
2893 aievec::ExtOp::create(rewriter, loc, halfType, srcOp.getVector(), 1);
2897 aievec::UPSOp::create(rewriter, loc, accType, lowerHalf.getResult());
2899 aievec::UPSOp::create(rewriter, loc, accType, upperHalf.getResult());
2900 auto addElemOp = aievec::AddElemOp::create(
2901 rewriter, loc, accType, lUpsOp.getResult(), rUpsOp.getResult());
2902 auto shiftParamOp = arith::ConstantOp::create(
2903 rewriter, loc, rewriter.getI32IntegerAttr(0));
2905 aievec::SRSOp::create(rewriter, loc, halfType, addElemOp.getResult(),
2906 shiftParamOp.getResult());
2907 curValue = srsOp.getResult();
2908 currentVType = halfType;
2915 cast<VectorType>(accType).getElementType().getIntOrFloatBitWidth();
2917 auto upsOp = aievec::UPSOp::create(rewriter, loc, accType, curValue);
2918 curValue = upsOp.getResult();
2920 aievec::AddElemOp curOp =
nullptr;
2922 for (
int id = shiftIndex;
id > 0;
id /= 2) {
2923 auto constOp = arith::ConstantOp::create(
2924 rewriter, loc, rewriter.getI32IntegerAttr(
id * accWidth / 8));
2925 auto shiftBytesOp = aievec::ShiftOp::create(
2926 rewriter, loc, accType, curValue, curValue, constOp,
true);
2927 curOp = aievec::AddElemOp::create(rewriter, loc, accType, curValue,
2928 shiftBytesOp.getResult());
2929 curValue = curOp.getResult();
2932 auto shiftParamOp = arith::ConstantOp::create(
2933 rewriter, srcOp.getLoc(), rewriter.getI32IntegerAttr(0));
2936 aievec::SRSOp::create(rewriter, loc, currentVType, curOp.getResult(),
2937 shiftParamOp.getResult());
2941 SmallVector<Value> concatSources = {srsOp.getResult(), srsOp.getResult()};
2943 aievec::ConcatOp::create(rewriter, loc, vecType, concatSources);
2946 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
2947 auto reduceResultOp =
2948 aievec::ExtElemOp::create(rewriter, srcOp.getLoc(), scalarType,
2949 concatOp, zeroConstOp.getResult());
2952 rewriter.replaceOpWithNewOp<arith::AddFOp>(
2953 srcOp, reduceResultOp.getResult(), srcOp.getAcc());
2955 rewriter.replaceOp(srcOp, reduceResultOp);
2962 : OpConversionPattern<vector::ReductionOp> {
2963 using OpConversionPattern::OpConversionPattern;
2967 ConversionPatternRewriter &rewriter)
const override {
2968 if (
auto kind = srcOp.getKind(); kind != vector::CombiningKind::ADD)
2971 auto vType = cast<VectorType>(srcOp.getVector().getType());
2972 Type scalarType = vType.getElementType();
2973 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
2977 if (!isa<FloatType>(scalarType) || (laneSize != 16 && laneSize != 32) ||
2981 Location loc = srcOp.getLoc();
2982 int shiftIndex = laneSize / 2;
2983 Value inputToReduce = srcOp.getVector();
2986 if (laneSize == 32) {
2991 aievec::ExtOp::create(rewriter, loc, halfType, srcOp.getVector(), 0);
2993 aievec::ExtOp::create(rewriter, loc, halfType, srcOp.getVector(), 1);
2998 aievec::UPSOp::create(rewriter, loc, accType, lowerHalf.getResult());
3000 aievec::UPSOp::create(rewriter, loc, accType, upperHalf.getResult());
3001 auto addElemOp = aievec::AddElemOp::create(
3002 rewriter, loc, accType, lUpsOp.getResult(), rUpsOp.getResult());
3003 auto shiftParamOp = arith::ConstantOp::create(
3004 rewriter, loc, rewriter.getI32IntegerAttr(0));
3006 aievec::SRSOp::create(rewriter, loc, halfType, addElemOp.getResult(),
3007 shiftParamOp.getResult());
3009 inputToReduce = srsOp.getResult();
3015 cast<VectorType>(inputToReduce.getType()),
true);
3017 cast<VectorType>(accType).getElementType().getIntOrFloatBitWidth();
3019 auto upsOp = aievec::UPSOp::create(rewriter, loc, accType, inputToReduce);
3020 Value curValue = upsOp.getResult();
3022 aievec::AddElemOp curOp =
nullptr;
3023 for (
int id = shiftIndex;
id > 0;
id /= 2) {
3024 auto constOp = arith::ConstantOp::create(
3025 rewriter, loc, rewriter.getI32IntegerAttr(
id * accWidth / 8));
3026 auto shiftBytesOp = aievec::ShiftOp::create(
3027 rewriter, loc, accType, curValue, curValue, constOp,
true);
3028 curOp = aievec::AddElemOp::create(rewriter, loc, accType, curValue,
3029 shiftBytesOp.getResult());
3030 curValue = curOp.getResult();
3037 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
3038 auto extractedF32 = aievec::ExtElemOp::create(
3039 rewriter, srcOp.getLoc(), rewriter.getF32Type(), curOp.getResult(),
3040 zeroConstOp.getResult());
3043 auto reduceResultBF16 = arith::TruncFOp::create(
3044 rewriter, srcOp.getLoc(), scalarType, extractedF32.getResult());
3047 rewriter.replaceOpWithNewOp<arith::AddFOp>(srcOp, reduceResultBF16,
3050 rewriter.replaceOp(srcOp, reduceResultBF16);
3058 : OpConversionPattern<vector::ExtractStridedSliceOp> {
3059 using OpConversionPattern::OpConversionPattern;
3063 ConversionPatternRewriter &rewriter)
const override {
3064 auto vType = extractOp.getSourceVectorType();
3065 if (vType.getRank() != 1)
3068 int64_t stride = cast<IntegerAttr>(adaptor.getStrides()[0]).getInt();
3074 return extractOp.emitError()
3075 <<
"AIEv1 doesn't support select ops on int8 types";
3079 int64_t size = cast<IntegerAttr>(adaptor.getSizes()[0]).getInt();
3080 if (vType.getNumElements() != 2 * size)
3083 int64_t offset = cast<IntegerAttr>(adaptor.getOffsets()[0]).getInt();
3084 aievec::aie1::SelectOp::Properties selectProps;
3085 if (failed(aievec::aie1::SelectOp::setPropertiesFromAttr(
3087 rewriter.getDictionaryAttr(
3088 buildAttributeListForRotationSelectOp(rewriter, vType, offset)),
3089 [&]() {
return extractOp.emitError(); })))
3091 auto selectOp = aievec::aie1::SelectOp::create(
3092 rewriter, extractOp.getLoc(), TypeRange({vType}),
3093 ValueRange({adaptor.getSource()}), selectProps);
3094 rewriter.replaceOpWithNewOp<aievec::aie1::ExtOp>(
3095 extractOp, extractOp.getType(), selectOp.getResult(),
3096 rewriter.getI8IntegerAttr(0));
3104 : OpConversionPattern<vector::ExtractStridedSliceOp> {
3105 using OpConversionPattern::OpConversionPattern;
3109 ConversionPatternRewriter &rewriter)
const override {
3110 auto vType = cast<VectorType>(adaptor.getSource().getType());
3111 if (vType.getRank() != 1)
3114 int64_t stride = cast<IntegerAttr>(adaptor.getStrides()[0]).getInt();
3120 int64_t size = cast<IntegerAttr>(adaptor.getSizes()[0]).getInt();
3121 if (vType.getNumElements() != 2 * size)
3124 auto shortVecType = cast<VectorType>(extractOp.getResult().getType());
3125 int64_t offset = cast<IntegerAttr>(adaptor.getOffsets()[0]).getInt();
3127 if (offset == 0 || offset == size) {
3128 rewriter.replaceOpWithNewOp<aievec::ExtOp>(
3129 extractOp, shortVecType, adaptor.getSource(),
3130 rewriter.getI8IntegerAttr(offset == 0 ? 0 : 1));
3134 aievec::ExtOp::create(rewriter, extractOp.getLoc(), shortVecType,
3135 adaptor.getSource(), rewriter.getI8IntegerAttr(0))
3138 aievec::ExtOp::create(rewriter, extractOp.getLoc(), shortVecType,
3139 adaptor.getSource(), rewriter.getI8IntegerAttr(1))
3142 auto shiftBytesConstOp = arith::ConstantOp::create(
3143 rewriter, extractOp.getLoc(), rewriter.getIntegerType(32),
3144 rewriter.getI32IntegerAttr(shiftBytes));
3145 rewriter.replaceOpWithNewOp<aievec::ShiftOp>(
3146 extractOp, shortVecType, bottomHalf, topHalf, shiftBytesConstOp);
3155 using OpConversionPattern::OpConversionPattern;
3158 : OpConversionPattern(context) {}
3162 ConversionPatternRewriter &rewriter)
const override {
3164 if (updOp->hasOneUse() && isa<aievec::ExtOp>(*updOp->getUsers().begin()))
3167 auto vecType = cast<VectorType>(updOp.getType());
3168 SmallVector<int64_t, 4> vecShape(vecType.getShape().begin(),
3169 vecType.getShape().end());
3170 vecShape[vecType.getRank() - 1] *= 2;
3171 auto longVecType = VectorType::get(vecShape, vecType.getElementType());
3172 auto newUpdOp = aievec::UPDOp::create(
3173 rewriter, updOp.getLoc(), longVecType, adaptor.getSource(),
3174 adaptor.getIndices(), adaptor.getOffset(), adaptor.getIndex(),
3175 adaptor.getVector());
3176 rewriter.replaceOpWithNewOp<aievec::ExtOp>(
3177 updOp, vecType, newUpdOp.getResult(), rewriter.getI8IntegerAttr(0));
3186 using OpConversionPattern::OpConversionPattern;
3192 ConversionPatternRewriter &rewriter)
const override {
3194 if (extOp.getIndex() != 0)
3197 auto updOp = dyn_cast<aievec::UPDOp>(extOp.getSource().getDefiningOp());
3202 if (!updOp->hasOneUse())
3205 rewriter.replaceOpWithNewOp<aievec::UPDOp>(
3206 extOp, extOp.getType(), updOp.getSource(), updOp.getIndices(),
3207 updOp.getOffset(), updOp.getIndex(), updOp.getVector());
3216 using OpConversionPattern::OpConversionPattern;
3220 ConversionPatternRewriter &rewriter)
const override {
3221 if (matchExpOpForAIE2P(adaptor)) {
3222 auto srcType = cast<VectorType>(adaptor.getOperand().getType());
3223 rewriter.replaceOpWithNewOp<aievec::ExpOp>(expOp, srcType,
3224 adaptor.getOperand());
3229 auto srcType = dyn_cast<VectorType>(adaptor.getOperand().getType());
3230 if (!srcType || srcType.isScalable() || srcType.getRank() != 1 ||
3231 !srcType.getElementType().isF32() ||
3232 (srcType.getNumElements() != 16 && srcType.getNumElements() != 32))
3235 Location loc = expOp.getLoc();
3236 auto bf16Type = VectorType::get(srcType.getShape(), rewriter.getBF16Type());
3238 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
3239 Value input = aievec::SRSOp::create(rewriter, loc, bf16Type,
3240 adaptor.getOperand(), shift);
3241 Value exp = aievec::ExpOp::create(rewriter, loc, bf16Type, input);
3242 rewriter.replaceOpWithNewOp<aievec::UPSOp>(expOp, srcType, exp,
3250template <
typename MathOpTy,
typename AIEVecOpTy>
3252 : OpConversionPattern<MathOpTy> {
3253 using OpConversionPattern<MathOpTy>::OpConversionPattern;
3258 ConversionPatternRewriter &rewriter)
const override {
3259 auto srcType = dyn_cast<VectorType>(adaptor.getOperand().getType());
3260 if (!srcType || !isAIE2PAccFloatUnaryType(srcType))
3263 rewriter.replaceOpWithNewOp<AIEVecOpTy>(op, srcType, adaptor.getOperand());
3276 using OpConversionPattern::OpConversionPattern;
3280 ConversionPatternRewriter &rewriter)
const override {
3281 auto srcType = dyn_cast<VectorType>(adaptor.getOperand().getType());
3282 if (!srcType || srcType.getRank() != 1 ||
3283 !isAIE2PAccFloatUnaryType(srcType))
3286 constexpr double a = 1.1283791670955126;
3287 constexpr double b = 0.10091092485409293;
3289 Location loc = erfOp.getLoc();
3290 auto bf16Type = VectorType::get(srcType.getShape(), rewriter.getBF16Type());
3291 auto f32Type = VectorType::get(srcType.getShape(), rewriter.getF32Type());
3293 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
3294 auto toBF16 = [&](Value v) -> Value {
3295 return aievec::SRSOp::create(rewriter, loc, bf16Type, v, shift);
3298 auto splatBF16 = [&](
double value) -> std::pair<Value, double> {
3299 auto attr = rewriter.getFloatAttr(rewriter.getBF16Type(), value);
3300 Value splat = arith::ConstantOp::create(
3301 rewriter, loc, DenseElementsAttr::get(bf16Type, attr));
3302 return {splat, attr.getValueAsDouble()};
3305 Value x = adaptor.getOperand();
3306 if (srcType.getElementType().isF32())
3309 auto [aHi, aHiValue] = splatBF16(a);
3310 Value aLo = splatBF16(a - aHiValue).first;
3311 Value bSplat = splatBF16(b).first;
3313 Value x2 = toBF16(aievec::MulElemOp::create(rewriter, loc, f32Type, x, x));
3314 Value x3 = toBF16(aievec::MulElemOp::create(rewriter, loc, f32Type, x2, x));
3315 Value arg = aievec::MulElemOp::create(rewriter, loc, f32Type, x, aHi);
3316 arg = aievec::FMAElemOp::create(rewriter, loc, f32Type, x, aLo, arg,
3318 arg = aievec::FMAElemOp::create(rewriter, loc, f32Type, x3, bSplat, arg,
3320 Value result = aievec::TanhOp::create(rewriter, loc, f32Type, arg);
3321 if (srcType.getElementType().isBF16())
3322 result = toBF16(result);
3324 rewriter.replaceOp(erfOp, result);
3331template <
typename OpTy>
3333 using OpConversionPattern<OpTy>::OpConversionPattern;
3338 ConversionPatternRewriter &rewriter)
const override {
3339 Type type = adaptor.getOperand().getType();
3340 if (!type.isBF16() && !type.isF32())
3343 Location loc = op.getLoc();
3344 auto vecType = VectorType::get({16}, type);
3345 Value splat = aievec::BroadcastScalarOp::create(rewriter, loc, vecType,
3346 adaptor.getOperand());
3347 Value result = OpTy::create(rewriter, loc, splat);
3348 rewriter.replaceOpWithNewOp<vector::ExtractOp>(op, result, 0);
3355template <
typename OpTy>
3357 using OpConversionPattern<OpTy>::OpConversionPattern;
3362 ConversionPatternRewriter &rewriter)
const override {
3363 auto srcType = dyn_cast<VectorType>(adaptor.getOperand().getType());
3364 if (!srcType || srcType.isScalable() || srcType.getRank() != 1 ||
3365 srcType.getNumElements() != 1)
3368 Location loc = op.getLoc();
3370 vector::ExtractOp::create(rewriter, loc, adaptor.getOperand(), 0);
3371 Value result = OpTy::create(rewriter, loc, scalar);
3372 Value poison = ub::PoisonOp::create(rewriter, loc, srcType);
3373 rewriter.replaceOpWithNewOp<vector::InsertOp>(op, result, poison, 0);
3379 using OpConversionPattern::OpConversionPattern;
3383 ConversionPatternRewriter &rewriter)
const override {
3385 if (!matchExpOpForAIE2LUT(adaptor))
3388 auto srcType = dyn_cast<VectorType>(adaptor.getOperand().getType());
3390 Location loc = expOp.getLoc();
3391 StringRef funcName =
"getExpBf16";
3393 VectorType v16bf16Ty = mlir::VectorType::get({16}, rewriter.getBF16Type());
3394 VectorType v8i64Ty = mlir::VectorType::get({8}, rewriter.getI64Type());
3395 func::FuncOp fnOp = getOrInsertFuncDecl(
3396 rewriter, expOp->getParentWithTrait<OpTrait::SymbolTable>(), funcName,
3397 TypeRange{v16bf16Ty}, TypeRange{v8i64Ty});
3400 if (laneSize == 32) {
3401 splitWideUnaryVectorOp<math::ExpOp>(
3402 expOp, adaptor.getOperand(), v16bf16Ty, srcType, rewriter,
3403 [&fnOp](Value halfInput, Location loc,
3404 ConversionPatternRewriter &rewriter) -> Value {
3405 VectorType v16bf16Ty =
3406 mlir::VectorType::get({16}, rewriter.getBF16Type());
3407 auto callOp = func::CallOp::create(rewriter, loc, fnOp,
3408 SmallVector<Value>{halfInput});
3410 auto resCastOp = vector::BitCastOp::create(rewriter, loc, accType,
3411 callOp.getResults());
3412 auto shiftParamOp = arith::ConstantOp::create(
3413 rewriter, loc, rewriter.getI32IntegerAttr(0));
3414 auto srsOp = aievec::SRSOp::create(rewriter, loc, v16bf16Ty,
3415 resCastOp.getResult(),
3416 shiftParamOp.getResult());
3417 return srsOp.getResult();
3423 SmallVector<Value> expOperands = {adaptor.getOperand()};
3426 auto callOp = func::CallOp::create(rewriter, loc, fnOp, expOperands);
3427 auto resCastOp = vector::BitCastOp::create(rewriter, loc, accTypeNative,
3428 callOp.getResults());
3430 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
3431 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
3432 expOp, srcType, resCastOp.getResult(), shiftParamOp.getResult());
3439 using OpConversionPattern::OpConversionPattern;
3443 ConversionPatternRewriter &rewriter)
const override {
3444 if (!matchExpOpForAIE2LUT(adaptor))
3446 auto srcType = dyn_cast<VectorType>(adaptor.getOperand().getType());
3447 StringRef includeName =
"lut_based_ops.h";
3448 auto moduleOp = expOp->getParentOfType<mlir::ModuleOp>();
3449 rewriter.setInsertionPointToStart(
3450 &moduleOp.getRegion().getBlocks().front());
3451 emitc::IncludeOp::create(rewriter, moduleOp.getLoc(), includeName,
false);
3453 rewriter.setInsertionPoint(expOp);
3455 auto v16bf16OpaqueTy =
3456 emitc::OpaqueType::get(rewriter.getContext(),
"v16bfloat16");
3457 auto opaquedOperand =
3458 UnrealizedConversionCastOp::create(
3459 rewriter, expOp.getLoc(), v16bf16OpaqueTy, adaptor.getOperand())
3461 SmallVector<Value> expOperands = {opaquedOperand};
3464 Type v16accf32OpaqueTy =
3465 emitc::OpaqueType::get(rewriter.getContext(),
"v16accfloat");
3466 auto callOp = emitc::CallOpaqueOp::create(
3467 rewriter, expOp.getLoc(), TypeRange{v16accf32OpaqueTy},
"getExpBf16",
3468 nullptr,
nullptr, expOperands);
3469 auto resCastOp = UnrealizedConversionCastOp::create(
3470 rewriter, expOp.getLoc(), accTypeNative, callOp.getResults());
3471 auto shiftParamOp = arith::ConstantOp::create(
3472 rewriter, expOp.getLoc(), rewriter.getI32IntegerAttr(0));
3473 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
3474 expOp, srcType, resCastOp.getResult(0), shiftParamOp.getResult());
3493 using OpConversionPattern::OpConversionPattern;
3497 ConversionPatternRewriter &rewriter)
const override {
3498 Type srcType = adaptor.getLhs().getType();
3501 auto *defOp = divOp.getLhs().getDefiningOp();
3505 auto constOp = dyn_cast<arith::ConstantOp>(defOp);
3510 if (
auto fType = dyn_cast<FloatType>(srcType)) {
3511 if (fType.getWidth() != 32)
3514 auto floatAttr = dyn_cast<FloatAttr>(constOp.getValue());
3515 if (!floatAttr || !floatAttr.getValue().isExactlyValue(1.0))
3518 rewriter.replaceOpWithNewOp<aievec::InvOp>(divOp, srcType,
3524 if (
auto vecType = dyn_cast<VectorType>(srcType)) {
3525 auto elemType = vecType.getElementType();
3526 if (!elemType.isF32())
3531 if (laneSize != 16 && laneSize != 32)
3535 auto denseAttr = dyn_cast<DenseFPElementsAttr>(constOp.getValue());
3536 if (!denseAttr || !denseAttr.isSplat())
3539 if (!denseAttr.getSplatValue<APFloat>().isExactlyValue(1.0))
3542 rewriter.replaceOpWithNewOp<aievec::InvOp>(divOp, vecType,
3553 using OpConversionPattern::OpConversionPattern;
3557 ConversionPatternRewriter &rewriter)
const override {
3558 auto srcType = dyn_cast<VectorType>(tanhOp.getOperand().getType());
3562 Type scalarType = srcType.getElementType();
3563 if (!isa<FloatType>(scalarType))
3567 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
3568 if (elWidth != 16 || laneSize != 16)
3571 StringRef includeName =
"lut_based_ops.h";
3572 auto moduleOp = tanhOp->getParentOfType<mlir::ModuleOp>();
3573 rewriter.setInsertionPointToStart(
3574 &moduleOp.getRegion().getBlocks().front());
3575 emitc::IncludeOp::create(rewriter, moduleOp.getLoc(), includeName,
false);
3577 rewriter.setInsertionPoint(tanhOp);
3578 Type v16bf16OpaqueTy =
3579 emitc::OpaqueType::get(rewriter.getContext(),
"v16bfloat16");
3580 auto opaquedOperand =
3581 UnrealizedConversionCastOp::create(
3582 rewriter, tanhOp.getLoc(), v16bf16OpaqueTy, adaptor.getOperand())
3584 SmallVector<Value> tanhOperands = {opaquedOperand};
3585 auto callOp = emitc::CallOpaqueOp::create(rewriter, tanhOp.getLoc(),
3586 v16bf16OpaqueTy,
"getTanhBf16",
3587 nullptr,
nullptr, tanhOperands);
3588 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
3589 tanhOp, TypeRange{tanhOp.getResult().getType()}, callOp.getResults());
3598 using OpConversionPattern::OpConversionPattern;
3602 ConversionPatternRewriter &rewriter)
const override {
3603 auto srcType = dyn_cast<VectorType>(sqrtOp.getOperand().getType());
3607 Type scalarType = srcType.getElementType();
3608 if (!isa<FloatType>(scalarType))
3612 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
3613 if (elWidth != 16 || (laneSize != 16 && laneSize != 32))
3616 StringRef includeName =
"vec_math.h";
3617 auto moduleOp = sqrtOp->getParentOfType<mlir::ModuleOp>();
3618 rewriter.setInsertionPointToStart(
3619 &moduleOp.getRegion().getBlocks().front());
3620 emitc::IncludeOp::create(rewriter, moduleOp.getLoc(), includeName,
false);
3622 rewriter.setInsertionPoint(sqrtOp);
3623 Type vLNbf16OpaqueTy;
3626 emitc::OpaqueType::get(rewriter.getContext(),
"v16bfloat16");
3629 emitc::OpaqueType::get(rewriter.getContext(),
"v32bfloat16");
3630 auto opaquedOperand =
3631 UnrealizedConversionCastOp::create(
3632 rewriter, sqrtOp.getLoc(), vLNbf16OpaqueTy, adaptor.getOperand())
3634 SmallVector<Value> sqrtOperands = {opaquedOperand};
3635 auto callOp = emitc::CallOpaqueOp::create(
3636 rewriter, sqrtOp.getLoc(), TypeRange{vLNbf16OpaqueTy},
"getSqrtBf16",
3637 nullptr,
nullptr, sqrtOperands);
3638 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
3639 sqrtOp, TypeRange{sqrtOp.getResult().getType()}, callOp.getResults());
3646 using OpConversionPattern::OpConversionPattern;
3650 ConversionPatternRewriter &rewriter)
const override {
3651 auto srcType = dyn_cast<VectorType>(adaptor.getOperand().getType());
3655 Type scalarType = srcType.getElementType();
3656 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
3660 if (!isa<FloatType>(scalarType) || laneSize != 16 || elWidth != 16)
3663 StringRef funcName =
"getRsqrtBf16";
3665 VectorType v16bf16Ty = mlir::VectorType::get({16}, rewriter.getBF16Type());
3666 VectorType v8i64Ty = mlir::VectorType::get({8}, rewriter.getI64Type());
3667 func::FuncOp fnOp = getOrInsertFuncDecl(
3668 rewriter, rsqrtOp->getParentWithTrait<OpTrait::SymbolTable>(), funcName,
3669 TypeRange{v16bf16Ty}, TypeRange{v8i64Ty});
3671 SmallVector<Value> rsqrtOperands = {adaptor.getOperand()};
3675 func::CallOp::create(rewriter, rsqrtOp.getLoc(), fnOp, rsqrtOperands);
3676 auto resCastOp = vector::BitCastOp::create(
3677 rewriter, rsqrtOp.getLoc(), accTypeNative, callOp.getResults());
3678 auto shiftParamOp = arith::ConstantOp::create(
3679 rewriter, rsqrtOp.getLoc(), rewriter.getI32IntegerAttr(0));
3680 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
3681 rsqrtOp, srcType, resCastOp.getResult(), shiftParamOp.getResult());
3690 using OpConversionPattern::OpConversionPattern;
3694 ConversionPatternRewriter &rewriter)
const override {
3695 auto srcType = dyn_cast<VectorType>(rsqrtOp.getOperand().getType());
3699 Type scalarType = srcType.getElementType();
3700 if (!isa<FloatType>(scalarType))
3704 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
3705 if (elWidth != 16 || (laneSize != 16 && laneSize != 32))
3708 StringRef includeName =
"vec_math.h";
3709 auto moduleOp = rsqrtOp->getParentOfType<mlir::ModuleOp>();
3710 rewriter.setInsertionPointToStart(
3711 &moduleOp.getRegion().getBlocks().front());
3712 emitc::IncludeOp::create(rewriter, moduleOp.getLoc(), includeName,
false);
3714 rewriter.setInsertionPoint(rsqrtOp);
3715 Type vLNbf16OpaqueTy;
3718 emitc::OpaqueType::get(rewriter.getContext(),
"v16bfloat16");
3721 emitc::OpaqueType::get(rewriter.getContext(),
"v32bfloat16");
3722 auto opaquedOperand =
3723 UnrealizedConversionCastOp::create(
3724 rewriter, rsqrtOp.getLoc(), vLNbf16OpaqueTy, adaptor.getOperand())
3726 SmallVector<Value> rsqrtOperands = {opaquedOperand};
3727 auto callOp = emitc::CallOpaqueOp::create(
3728 rewriter, rsqrtOp.getLoc(), TypeRange{vLNbf16OpaqueTy},
"getRsqrtBf16",
3729 nullptr,
nullptr, rsqrtOperands);
3730 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
3731 rsqrtOp, TypeRange{rsqrtOp.getResult().getType()}, callOp.getResults());
3740 using OpConversionPattern::OpConversionPattern;
3744 ConversionPatternRewriter &rewriter)
const override {
3745 auto srcType = dyn_cast<VectorType>(erfOp.getOperand().getType());
3749 Type scalarType = srcType.getElementType();
3750 if (!isa<FloatType>(scalarType))
3754 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
3755 if (elWidth != 16 || (laneSize != 16 && laneSize != 32))
3758 StringRef includeName =
"vec_math.h";
3759 auto moduleOp = erfOp->getParentOfType<mlir::ModuleOp>();
3760 rewriter.setInsertionPointToStart(
3761 &moduleOp.getRegion().getBlocks().front());
3762 emitc::IncludeOp::create(rewriter, moduleOp.getLoc(), includeName,
false);
3764 rewriter.setInsertionPoint(erfOp);
3765 Type vLNbf16OpaqueTy;
3768 emitc::OpaqueType::get(rewriter.getContext(),
"v16bfloat16");
3771 emitc::OpaqueType::get(rewriter.getContext(),
"v32bfloat16");
3772 auto opaquedOperand =
3773 UnrealizedConversionCastOp::create(
3774 rewriter, erfOp.getLoc(), vLNbf16OpaqueTy, adaptor.getOperand())
3776 SmallVector<Value> erfOperands = {opaquedOperand};
3777 auto callOp = emitc::CallOpaqueOp::create(
3778 rewriter, erfOp.getLoc(), TypeRange{vLNbf16OpaqueTy},
"getErfBf16",
3779 nullptr,
nullptr, erfOperands);
3780 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
3781 erfOp, TypeRange{erfOp.getResult().getType()}, callOp.getResults());
3789template <
typename SrcOpTy>
3791 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
3796 ConversionPatternRewriter &rewriter)
const override {
3797 auto vecTy = dyn_cast<VectorType>(absOp.getOperand().getType());
3801 Type elemTy = vecTy.getElementType();
3804 unsigned elWidth = elemTy.getIntOrFloatBitWidth();
3806 StringRef includeName =
"vec_math.h";
3807 auto moduleOp = absOp->template getParentOfType<mlir::ModuleOp>();
3808 rewriter.setInsertionPointToStart(
3809 &moduleOp.getRegion().getBlocks().front());
3810 emitc::IncludeOp::create(rewriter, moduleOp.getLoc(), includeName,
false);
3812 rewriter.setInsertionPoint(absOp);
3813 std::ostringstream typeName;
3814 typeName <<
"v" << laneSize;
3815 if (isa<FloatType>(elemTy)) {
3817 typeName <<
"bfloat16";
3819 typeName <<
"float";
3821 typeName <<
"int" << elWidth;
3823 emitc::OpaqueType::get(rewriter.getContext(), typeName.str());
3824 auto opaquedOperand =
3825 UnrealizedConversionCastOp::create(rewriter, absOp.getLoc(),
3826 vecOpaqueTy, adaptor.getOperand())
3828 SmallVector<Value> absOperands = {opaquedOperand};
3829 auto callOp = emitc::CallOpaqueOp::create(rewriter, absOp.getLoc(),
3830 TypeRange{vecOpaqueTy},
"getAbs",
3831 nullptr,
nullptr, absOperands);
3832 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
3833 absOp, TypeRange{absOp.getResult().getType()}, callOp.getResults());
3842template <
typename SrcOpTy>
3844 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
3849 ConversionPatternRewriter &rewriter)
const override {
3850 VectorType srcType = cast<VectorType>(extOp.getIn().getType());
3851 VectorType dstType = cast<VectorType>(extOp.getOut().getType());
3853 Type scalarType = dstType.getElementType();
3854 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
3856 isa<IntegerType>(scalarType) && (elWidth == 32 || elWidth == 64)
3860 aievec::UPSOp::create(rewriter, extOp.getLoc(), accType, extOp.getIn());
3862 if (dstType.getElementType().getIntOrFloatBitWidth() == 16) {
3863 auto shiftParamOp = arith::ConstantOp::create(
3864 rewriter, extOp.getLoc(), rewriter.getI32IntegerAttr(0));
3865 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
3866 extOp, dstType, upsOp.getResult(), shiftParamOp.getResult());
3868 rewriter.replaceOpWithNewOp<aievec::CastOp>(
3869 extOp, dstType, upsOp.getResult(),
false);
3899template <
typename SrcOpTy>
3902 static constexpr bool isSigned = std::is_same_v<SrcOpTy, arith::SIToFPOp>;
3908 PatternBenefit benefit)
3909 : OpConversionPattern<SrcOpTy>(context, benefit),
3915 Value srcI16,
unsigned lanes)
const {
3916 Type i16Ty = rewriter.getI16Type();
3917 Type i32Ty = rewriter.getI32Type();
3918 Type bf16Ty = rewriter.getBF16Type();
3919 Type f32Ty = rewriter.getF32Type();
3920 VectorType vecI16 = VectorType::get({lanes}, i16Ty);
3921 VectorType vecI32 = VectorType::get({lanes}, i32Ty);
3922 VectorType vecBF16 = VectorType::get({lanes}, bf16Ty);
3923 VectorType vecF32 = VectorType::get({lanes}, f32Ty);
3925 DenseElementsAttr magicI16Attr = DenseElementsAttr::get(
3926 vecI16, rewriter.getI16IntegerAttr(
static_cast<int16_t
>(0x4b01)));
3928 arith::ConstantOp::create(rewriter, loc, vecI16, magicI16Attr);
3929 Value magicBF16Vec =
3930 vector::BitCastOp::create(rewriter, loc, vecBF16, magicI16Vec);
3932 aievec::UPSOp::create(rewriter, loc, vecF32, magicBF16Vec);
3933 Value magicAccI = aievec::CastOp::create(rewriter, loc, vecI32, magicAccF,
3936 Value flip = arith::ConstantOp::create(
3937 rewriter, loc, vecI16,
3938 DenseElementsAttr::get(vecI16, rewriter.getI16IntegerAttr(
3939 static_cast<int16_t
>(0x8000))));
3940 srcI16 = arith::XOrIOp::create(rewriter, loc, srcI16, flip);
3942 Value vAccI = aievec::UPSOp::create(rewriter, loc, vecI32, srcI16);
3945 ? aievec::AddElemOp::create(rewriter, loc, vecI32, vAccI, magicAccI)
3947 : arith::AddIOp::create(rewriter, loc, vAccI, magicAccI)
3949 Value vSumAccF = aievec::CastOp::create(rewriter, loc, vecF32, vSumI,
3951 Value subtrahend = magicAccF;
3953 subtrahend = arith::ConstantOp::create(
3954 rewriter, loc, vecF32,
3955 DenseElementsAttr::get(vecF32, rewriter.getF32FloatAttr(8421376.0f)));
3957 aievec::SubElemOp::create(rewriter, loc, vecF32, vSumAccF, subtrahend);
3959 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
3960 return aievec::SRSOp::create(rewriter, loc, vecBF16, vDiffAccF, shiftCst)
3966 ConversionPatternRewriter &rewriter)
const override {
3967 auto srcTy = dyn_cast<VectorType>(adaptor.getIn().getType());
3968 auto dstTy = dyn_cast<VectorType>(op.getType());
3969 if (!srcTy || !dstTy || srcTy.getRank() != 1 || dstTy.getRank() != 1)
3971 if (!srcTy.getElementType().isInteger(16))
3973 if (!dstTy.getElementType().isBF16())
3975 unsigned lanes = srcTy.getNumElements();
3976 if (lanes != 16 && lanes != 32 && lanes != 64)
3979 Location loc = op.getLoc();
3980 if (lanes == 16 || lanes == 32) {
3982 rewriter.replaceOp(op, out);
3988 SmallVector<int64_t> loMask, hiMask, catMask;
3989 for (int64_t i = 0; i < 32; ++i)
3990 loMask.push_back(i);
3991 for (int64_t i = 32; i < 64; ++i)
3992 hiMask.push_back(i);
3993 for (int64_t i = 0; i < 64; ++i)
3994 catMask.push_back(i);
3995 Value lo = vector::ShuffleOp::create(rewriter, loc, adaptor.getIn(),
3996 adaptor.getIn(), loMask)
3998 Value hi = vector::ShuffleOp::create(rewriter, loc, adaptor.getIn(),
3999 adaptor.getIn(), hiMask)
4004 vector::ShuffleOp::create(rewriter, loc, loBF16, hiBF16, catMask)
4006 rewriter.replaceOp(op, cat);
4025 : OpConversionPattern<arith::ExtUIOp> {
4026 using OpConversionPattern<arith::ExtUIOp>::OpConversionPattern;
4030 ConversionPatternRewriter &rewriter)
const override {
4031 auto srcTy = dyn_cast<VectorType>(adaptor.getIn().getType());
4032 auto dstTy = dyn_cast<VectorType>(extOp.getType());
4033 if (!srcTy || !dstTy)
4035 if (srcTy.getRank() != 1 || dstTy.getRank() != 1)
4037 if (!srcTy.getElementType().isInteger(4) ||
4038 !dstTy.getElementType().isInteger(8))
4040 unsigned lanes = srcTy.getNumElements();
4041 if (lanes != dstTy.getNumElements())
4043 if (lanes != 64 && lanes != 128)
4046 auto bitcastOp = adaptor.getIn().getDefiningOp<vector::BitCastOp>();
4049 auto bitcastSrcTy = dyn_cast<VectorType>(bitcastOp.getSource().getType());
4050 if (!bitcastSrcTy || bitcastSrcTy.getRank() != 1 ||
4051 !bitcastSrcTy.getElementType().isInteger(8))
4053 if (bitcastSrcTy.getNumElements() * 2 != lanes)
4056 rewriter.replaceOpWithNewOp<aievec::UnpackOp>(extOp, dstTy,
4057 bitcastOp.getSource());
4069template <
typename SrcOpTy>
4071 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
4076 ConversionPatternRewriter &rewriter)
const override {
4077 auto srcType = dyn_cast<VectorType>(adaptor.getIn().getType());
4078 auto dstType = dyn_cast<VectorType>(op.getType());
4079 if (!srcType || !dstType)
4081 if (srcType.getRank() != 1 || dstType.getRank() != 1)
4083 if (srcType.getNumElements() != dstType.getNumElements())
4086 Location loc = op.getLoc();
4087 Type scalarDstTy = dstType.getElementType();
4088 int64_t lanes = srcType.getNumElements();
4091 Value result = arith::ConstantOp::create(rewriter, loc, dstType,
4092 rewriter.getZeroAttr(dstType));
4094 for (int64_t i = 0; i < lanes; ++i) {
4095 Value scalar = vector::ExtractOp::create(rewriter, loc, adaptor.getIn(),
4096 ArrayRef<int64_t>{i});
4097 Value cvt = SrcOpTy::create(rewriter, loc, scalarDstTy, scalar);
4098 result = vector::InsertOp::create(rewriter, loc, cvt, result,
4099 ArrayRef<int64_t>{i});
4102 rewriter.replaceOp(op, result);
4116template <
typename SrcOpTy>
4118 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
4123 ConversionPatternRewriter &rewriter)
const override {
4124 VectorType srcType = cast<VectorType>(truncOp.getIn().getType());
4125 VectorType dstType = cast<VectorType>(truncOp.getOut().getType());
4126 Type scalarType = srcType.getElementType();
4127 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
4129 isa<IntegerType>(scalarType) && (elWidth == 32 || elWidth == 64)
4133 auto shiftParamOp = arith::ConstantOp::create(
4134 rewriter, truncOp.getLoc(), rewriter.getI32IntegerAttr(0));
4135 if (elWidth == 16) {
4136 auto upsOp = aievec::UPSOp::create(rewriter, truncOp.getLoc(), accType,
4138 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
4139 truncOp, dstType, upsOp.getResult(), shiftParamOp.getResult());
4141 auto castOp = aievec::CastOp::create(rewriter, truncOp.getLoc(), accType,
4142 truncOp.getIn(),
true);
4143 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
4144 truncOp, dstType, castOp.getResult(), shiftParamOp.getResult());
4159static std::optional<Value>
4160getUnOpaquedOperandOfEmitCOpaqueCallOp(Operation *op, StringRef funcName) {
4161 auto uccOp = dyn_cast<UnrealizedConversionCastOp>(op);
4165 auto inVal = uccOp.getInputs()[0];
4166 if (!isa<emitc::OpaqueType>(inVal.getType()))
4169 auto callOp = inVal.getDefiningOp<emitc::CallOpaqueOp>();
4170 if (callOp.getCallee() != funcName)
4173 auto callOperandsUccOp =
4174 callOp.getOperands()[0].getDefiningOp<UnrealizedConversionCastOp>();
4175 if (!callOperandsUccOp)
4178 return callOperandsUccOp.getInputs()[0];
4194template <
typename DivFOpTy>
4195static bool hasSigmoidComputationChain(DivFOpTy divfOp, arith::NegFOp &negOp) {
4196 auto *lhsDefOp = divfOp.getLhs().getDefiningOp();
4199 auto constOp = dyn_cast<arith::ConstantOp>(lhsDefOp);
4203 auto cstDense = dyn_cast<DenseFPElementsAttr>(constOp.getValue());
4207 if (cstDense.template getSplatValue<APFloat>().convertToFloat() != 1.0f)
4210 Operation *addLvalOp;
4211 Operation *addRvalOp;
4217 auto *rhsDefOp = divfOp.getRhs().getDefiningOp();
4220 auto addOp = dyn_cast<arith::AddFOp>(rhsDefOp);
4222 auto srsOp = dyn_cast<aievec::SRSOp>(rhsDefOp);
4227 dyn_cast<aievec::AddElemOp>(srsOp.getSource().getDefiningOp());
4231 auto lUpsOp = dyn_cast<aievec::UPSOp>(addElemOp.getLhs().getDefiningOp());
4232 auto rUpsOp = dyn_cast<aievec::UPSOp>(addElemOp.getRhs().getDefiningOp());
4233 if (!lUpsOp || !rUpsOp)
4236 addLvalOp = lUpsOp.getSource().getDefiningOp();
4237 addRvalOp = rUpsOp.getSource().getDefiningOp();
4240 auto addDefOp = isa<arith::ConstantOp>(addLvalOp)
4241 ? dyn_cast<aievec::SRSOp>(addRvalOp)
4242 : dyn_cast<
aievec::SRSOp>(addLvalOp);
4244 addLvalOp = isa<arith::ConstantOp>(addLvalOp)
4245 ? dyn_cast<math::ExpOp>(addRvalOp)
4246 : dyn_cast<math::ExpOp>(addLvalOp);
4248 addLvalOp = addDefOp.getSource().getDefiningOp();
4250 addRvalOp = isa<arith::ConstantOp>(addLvalOp)
4251 ? lUpsOp.getSource().getDefiningOp()
4252 : rUpsOp.getSource().getDefiningOp();
4254 addLvalOp = addOp.getLhs().getDefiningOp();
4255 addRvalOp = addOp.getRhs().getDefiningOp();
4258 if (!addLvalOp || !addRvalOp)
4261 auto addLvalExpOp = dyn_cast<math::ExpOp>(addLvalOp);
4262 auto addRvalExpOp = dyn_cast<math::ExpOp>(addRvalOp);
4263 auto addLvalExpOpIn =
4264 getUnOpaquedOperandOfEmitCOpaqueCallOp(addLvalOp,
"getExpBf16")
4266 auto addRvalExpOpIn =
4267 getUnOpaquedOperandOfEmitCOpaqueCallOp(addRvalOp,
"getExpBf16")
4269 if (!addLvalExpOpIn && addLvalExpOp)
4270 addLvalExpOpIn = addLvalExpOp.getOperand();
4271 if (!addRvalExpOpIn && addRvalExpOp)
4272 addRvalExpOpIn = addRvalExpOp.getOperand();
4274 if (!((addLvalExpOpIn && isa<arith::ConstantOp>(addRvalOp)) ||
4275 (addRvalExpOpIn && isa<arith::ConstantOp>(addLvalOp))))
4278 constOp = isa<arith::ConstantOp>(addLvalOp)
4279 ? cast<arith::ConstantOp>(addLvalOp)
4280 : cast<
arith::ConstantOp>(addRvalOp);
4282 cstDense = dyn_cast<DenseFPElementsAttr>(constOp.getValue());
4285 if (cstDense.template getSplatValue<APFloat>().convertToFloat() != 1.0f)
4288 auto expOperand = addLvalExpOpIn ? addLvalExpOpIn : addRvalExpOpIn;
4290 negOp = expOperand.getDefiningOp<arith::NegFOp>();
4292 return negOp !=
nullptr;
4309 using OpConversionPattern::OpConversionPattern;
4313 ConversionPatternRewriter &rewriter)
const override {
4314 auto srcType = dyn_cast<VectorType>(adaptor.getLhs().getType());
4318 Type scalarType = srcType.getElementType();
4319 if (!isa<FloatType>(scalarType))
4323 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
4324 if (elWidth != 16 || (laneSize != 16 && laneSize != 32))
4327 arith::NegFOp negOp =
nullptr;
4328 if (!hasSigmoidComputationChain(adaptor, negOp))
4331 StringRef includeName =
"vec_math.h";
4332 auto moduleOp = divfOp->getParentOfType<mlir::ModuleOp>();
4333 rewriter.setInsertionPointToStart(
4334 &moduleOp.getRegion().getBlocks().front());
4335 emitc::IncludeOp::create(rewriter, moduleOp.getLoc(), includeName,
false);
4337 rewriter.setInsertionPoint(divfOp);
4341 emitc::OpaqueType::get(rewriter.getContext(),
"v16bfloat16");
4344 emitc::OpaqueType::get(rewriter.getContext(),
"v32bfloat16");
4345 auto opaquedOperand =
4346 UnrealizedConversionCastOp::create(rewriter, divfOp.getLoc(),
4347 vecOpaqueTy, negOp.getOperand())
4349 SmallVector<Value> sigmoidOperands = {opaquedOperand};
4350 auto callOp = emitc::CallOpaqueOp::create(
4351 rewriter, divfOp.getLoc(), TypeRange{vecOpaqueTy},
"getSigmoidBf16",
4352 nullptr,
nullptr, sigmoidOperands);
4353 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
4354 divfOp, TypeRange{adaptor.getLhs().getType()}, callOp.getResults());
4362 using OpConversionPattern::OpConversionPattern;
4366 ConversionPatternRewriter &rewriter)
const override {
4367 auto srcType = dyn_cast<VectorType>(ceilOp.getOperand().getType());
4371 Type scalarType = srcType.getElementType();
4372 if (!isa<FloatType>(scalarType))
4376 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
4377 if (elWidth != 16 || (laneSize != 16 && laneSize != 32))
4380 StringRef includeName =
"vec_math.h";
4381 auto moduleOp = ceilOp->getParentOfType<mlir::ModuleOp>();
4382 rewriter.setInsertionPointToStart(
4383 &moduleOp.getRegion().getBlocks().front());
4384 emitc::IncludeOp::create(rewriter, moduleOp.getLoc(), includeName,
false);
4386 rewriter.setInsertionPoint(ceilOp);
4390 emitc::OpaqueType::get(rewriter.getContext(),
"v16bfloat16");
4393 emitc::OpaqueType::get(rewriter.getContext(),
"v32bfloat16");
4394 auto opaquedOperand =
4395 UnrealizedConversionCastOp::create(rewriter, ceilOp.getLoc(),
4396 vecOpaqueTy, adaptor.getOperand())
4398 SmallVector<Value> ceilOperands = {opaquedOperand};
4399 auto callOp = emitc::CallOpaqueOp::create(
4400 rewriter, ceilOp.getLoc(), TypeRange{vecOpaqueTy},
"getCeilBf16",
4401 nullptr,
nullptr, ceilOperands);
4402 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
4403 ceilOp, TypeRange{ceilOp.getResult().getType()}, callOp.getResults());
4411 using OpConversionPattern::OpConversionPattern;
4415 ConversionPatternRewriter &rewriter)
const override {
4416 auto srcType = dyn_cast<VectorType>(floorOp.getOperand().getType());
4420 Type scalarType = srcType.getElementType();
4421 if (!isa<FloatType>(scalarType))
4425 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
4426 if (elWidth != 16 || (laneSize != 16 && laneSize != 32))
4429 StringRef includeName =
"vec_math.h";
4430 auto moduleOp = floorOp->getParentOfType<mlir::ModuleOp>();
4431 rewriter.setInsertionPointToStart(
4432 &moduleOp.getRegion().getBlocks().front());
4433 emitc::IncludeOp::create(rewriter, moduleOp.getLoc(), includeName,
false);
4435 rewriter.setInsertionPoint(floorOp);
4439 emitc::OpaqueType::get(rewriter.getContext(),
"v16bfloat16");
4442 emitc::OpaqueType::get(rewriter.getContext(),
"v32bfloat16");
4443 auto opaquedOperand =
4444 UnrealizedConversionCastOp::create(rewriter, floorOp.getLoc(),
4445 vecOpaqueTy, adaptor.getOperand())
4447 SmallVector<Value> floorOperands = {opaquedOperand};
4448 auto callOp = emitc::CallOpaqueOp::create(
4449 rewriter, floorOp.getLoc(), TypeRange{vecOpaqueTy},
"getFloorBf16",
4450 nullptr,
nullptr, floorOperands);
4451 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
4452 floorOp, TypeRange{floorOp.getResult().getType()}, callOp.getResults());
4461 using OpConversionPattern::OpConversionPattern;
4465 ConversionPatternRewriter &rewriter)
const override {
4466 auto srcType = dyn_cast<VectorType>(negOp.getOperand().getType());
4470 Type scalarType = srcType.getElementType();
4471 if (!isa<FloatType>(scalarType))
4475 if (laneSize != 16 && laneSize != 32)
4478 Location loc = negOp.getLoc();
4481 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
4482 if (elWidth == 16) {
4484 aievec::UPSOp::create(rewriter, loc, accType, adaptor.getOperand());
4486 aievec::NegOp::create(rewriter, loc, accType, upsOp.getResult());
4487 auto shiftParamOp = arith::ConstantOp::create(
4488 rewriter, negOp.getLoc(), rewriter.getI32IntegerAttr(0));
4489 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
4490 negOp, srcType, aieNegOp.getResult(), shiftParamOp.getResult());
4492 auto castOp = aievec::CastOp::create(
4493 rewriter, loc, accType, adaptor.getOperand(),
true);
4495 aievec::NegOp::create(rewriter, loc, accType, castOp.getResult());
4496 rewriter.replaceOpWithNewOp<aievec::CastOp>(
4497 negOp, srcType, aieNegOp.getResult(),
false);
4506static bool hasConstNegOneValue(arith::ConstantOp constOp,
unsigned elWidth) {
4510 auto cstDense = dyn_cast<DenseIntElementsAttr>(constOp.getValue());
4515 return cstDense.getSplatValue<int32_t>() == -1;
4517 return cstDense.getSplatValue<int16_t>() == -1;
4519 return cstDense.getSplatValue<int8_t>() == -1;
4526 using OpConversionPattern::OpConversionPattern;
4530 ConversionPatternRewriter &rewriter)
const override {
4531 auto srcType = dyn_cast<VectorType>(xorOp.getLhs().getType());
4535 Type scalarType = srcType.getElementType();
4536 if (!isa<IntegerType>(scalarType))
4540 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
4541 if (laneSize * elWidth != 512)
4544 auto lhsConstOp = xorOp.getLhs().getDefiningOp<arith::ConstantOp>();
4545 auto rhsConstOp = xorOp.getRhs().getDefiningOp<arith::ConstantOp>();
4549 if ((lhsConstOp && hasConstNegOneValue(lhsConstOp, elWidth)) ||
4550 (rhsConstOp && hasConstNegOneValue(rhsConstOp, elWidth))) {
4551 Value val = hasConstNegOneValue(lhsConstOp, elWidth) ? adaptor.getRhs()
4553 rewriter.replaceOpWithNewOp<aievec::BnegOp>(xorOp, srcType, val);
4555 rewriter.replaceOpWithNewOp<aievec::BxorOp>(
4556 xorOp, srcType, adaptor.getLhs(), adaptor.getRhs());
4562template <
typename SrcOpTy,
typename DstOpTy>
4564 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
4569 ConversionPatternRewriter &rewriter)
const override {
4570 VectorType srcType = dyn_cast<VectorType>(srcOp.getLhs().getType());
4574 Type scalarType = srcType.getElementType();
4575 if (!isa<IntegerType>(scalarType))
4579 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
4580 if (laneSize * elWidth != 512)
4583 rewriter.replaceOpWithNewOp<DstOpTy>(srcOp, srcOp.getResult().getType(),
4584 adaptor.getLhs(), adaptor.getRhs());
4599 : OpConversionPattern<arith::ShRSIOp> {
4600 using OpConversionPattern::OpConversionPattern;
4604 ConversionPatternRewriter &rewriter)
const override {
4605 auto srcType = dyn_cast<VectorType>(adaptor.getLhs().getType());
4609 Type scalarType = srcType.getElementType();
4611 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
4612 if (laneSize * elWidth != 512)
4616 dyn_cast<aievec::BroadcastOp>(adaptor.getRhs().getDefiningOp());
4621 arith::ConstantOp::create(rewriter, bcastOp.getLoc(),
4622 rewriter.getI32IntegerAttr(bcastOp.getIdx()));
4623 auto extElemOp = aievec::ExtElemOp::create(
4624 rewriter, bcastOp.getLoc(), scalarType, bcastOp, constOp.getResult());
4625 Location loc = rsOp.getLoc();
4631 auto rsOpLow = aievec::ExtOp::create(rewriter, loc, halfSrcType,
4632 adaptor.getLhs(), 0);
4633 auto rsOpHigh = aievec::ExtOp::create(rewriter, loc, halfSrcType,
4634 adaptor.getLhs(), 1);
4637 aievec::UPSOp::create(rewriter, loc, accType, rsOpLow.getResult());
4639 aievec::SRSOp::create(rewriter, loc, halfSrcType,
4640 upsOpLow.getResult(), extElemOp.getResult());
4642 aievec::UPSOp::create(rewriter, loc, accType, rsOpHigh.getResult());
4644 aievec::SRSOp::create(rewriter, loc, halfSrcType,
4645 upsOpHigh.getResult(), extElemOp.getResult());
4646 SmallVector<Value> inputSources = {srsOpLow.getResult(),
4647 srsOpHigh.getResult()};
4648 rewriter.replaceOpWithNewOp<aievec::ConcatOp>(rsOp, srcType,
4653 aievec::UPSOp::create(rewriter, loc, accType, adaptor.getLhs());
4654 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
4655 rsOp, srcType, upsOp.getResult(), extElemOp.getResult());
4678 using OpConversionPattern::OpConversionPattern;
4681 : OpConversionPattern(context, benefit) {}
4686 auto defOp = val.getDefiningOp<arith::ConstantOp>();
4688 return std::nullopt;
4689 auto denseAttr = dyn_cast<DenseIntElementsAttr>(defOp.getValue());
4690 if (!denseAttr || !denseAttr.isSplat())
4691 return std::nullopt;
4692 return denseAttr.getSplatValue<APInt>().getSExtValue();
4698 static std::optional<Value>
4701 if (
auto constOp = rhs.getDefiningOp<arith::ConstantOp>()) {
4702 auto denseAttr = dyn_cast<DenseIntElementsAttr>(constOp.getValue());
4703 if (denseAttr && denseAttr.isSplat()) {
4704 int64_t shiftVal = denseAttr.getSplatValue<APInt>().getSExtValue();
4705 return arith::ConstantOp::create(rewriter, loc,
4706 rewriter.getI32IntegerAttr(shiftVal))
4711 if (
auto bcastOp = dyn_cast<aievec::BroadcastOp>(rhs.getDefiningOp())) {
4712 auto constOp = arith::ConstantOp::create(
4713 rewriter, bcastOp.getLoc(),
4714 rewriter.getI32IntegerAttr(bcastOp.getIdx()));
4715 return aievec::ExtElemOp::create(rewriter, bcastOp.getLoc(),
4716 rewriter.getI32Type(), bcastOp,
4717 constOp.getResult())
4720 return std::nullopt;
4725 ConversionPatternRewriter &rewriter)
const override {
4726 auto dstType = dyn_cast<VectorType>(truncOp.getOut().getType());
4730 Type dstScalarType = dstType.getElementType();
4731 if (!isa<IntegerType>(dstScalarType))
4735 Value source = adaptor.getIn();
4739 arith::MinSIOp minOp =
nullptr;
4740 arith::MaxSIOp maxOp =
nullptr;
4742 if (
auto minsiOp = source.getDefiningOp<arith::MinSIOp>()) {
4743 if (
auto maxsiOp = minsiOp.getLhs().getDefiningOp<arith::MaxSIOp>()) {
4746 source = maxOp.getLhs();
4748 }
else if (
auto maxsiOp = source.getDefiningOp<arith::MaxSIOp>()) {
4749 if (
auto minsiOp = maxsiOp.getLhs().getDefiningOp<arith::MinSIOp>()) {
4752 source = minOp.getLhs();
4757 if (minOp && maxOp) {
4760 if (!loVal || !hiVal)
4763 unsigned dstBits = dstScalarType.getIntOrFloatBitWidth();
4765 if (dstBits == 0 || dstBits > 63)
4767 uint64_t one = 1ULL;
4768 int64_t unsignedLo = 0;
4769 int64_t unsignedHi =
static_cast<int64_t
>((one << dstBits) - 1);
4770 int64_t signedLo = -
static_cast<int64_t
>(one << (dstBits - 1));
4771 int64_t signedHi =
static_cast<int64_t
>((one << (dstBits - 1)) - 1);
4773 if (*loVal == unsignedLo && *hiVal == unsignedHi) {
4775 }
else if (*loVal == signedLo && *hiVal == signedHi) {
4784 auto shrsiOp = source.getDefiningOp<arith::ShRSIOp>();
4788 auto srcType = dyn_cast<VectorType>(shrsiOp.getLhs().getType());
4792 Type srcScalarType = srcType.getElementType();
4793 if (!isa<IntegerType>(srcScalarType))
4796 unsigned srcElWidth = srcScalarType.getIntOrFloatBitWidth();
4797 unsigned dstElWidth = dstScalarType.getIntOrFloatBitWidth();
4798 if (dstElWidth >= srcElWidth)
4801 Location loc = truncOp.getLoc();
4804 auto shiftVal =
getShiftValue(shrsiOp.getRhs(), rewriter, loc);
4809 Value wideInput = shrsiOp.getLhs();
4812 bool needsPadding = (laneSize % 16 != 0);
4814 VectorType paddedSrcType = srcType;
4815 VectorType paddedDstType = dstType;
4819 unsigned paddedLanes = ((laneSize + 15) / 16) * 16;
4824 auto zeroAttr = rewriter.getZeroAttr(paddedSrcType);
4826 arith::ConstantOp::create(rewriter, loc, zeroAttr).getResult();
4827 SmallVector<int64_t> offsets(1, 0);
4828 SmallVector<int64_t> strides(1, 1);
4829 wideInput = vector::InsertStridedSliceOp::create(
4830 rewriter, loc, wideInput, zeroPad, offsets, strides)
4838 Type accScalarType = paddedSrcType.getElementType();
4839 unsigned accElWidth = accScalarType.getIntOrFloatBitWidth();
4841 if (accElWidth == 16) {
4844 aievec::UPSOp::create(rewriter, loc, accType, wideInput).getResult();
4847 accValue = aievec::CastOp::create(rewriter, loc, paddedSrcType, wideInput,
4851 auto srsOp = aievec::SRSOp::create(rewriter, loc, paddedDstType, accValue,
4854 Value result = srsOp.getResult();
4858 SmallVector<int64_t> offsets(1, 0);
4859 SmallVector<int64_t> sizes = {
static_cast<int64_t
>(laneSize)};
4860 SmallVector<int64_t> strides(1, 1);
4861 result = vector::ExtractStridedSliceOp::create(rewriter, loc, result,
4862 offsets, sizes, strides)
4866 rewriter.replaceOp(truncOp, result);
4871 SmallVector<Operation *, 3> opsToErase;
4872 if (minOp && minOp->use_empty())
4873 opsToErase.push_back(minOp);
4874 if (maxOp && maxOp->use_empty())
4875 opsToErase.push_back(maxOp);
4876 if (shrsiOp->use_empty())
4877 opsToErase.push_back(shrsiOp);
4878 for (Operation *op : opsToErase)
4879 rewriter.eraseOp(op);
4892 using OpConversionPattern::OpConversionPattern;
4896 ConversionPatternRewriter &rewriter)
const override {
4898 if (isa<VectorType>(truncOp.getOut().getType()))
4900 std::optional<ScalarSRSChain> chain = matchScalarSRSChain(truncOp);
4903 Value source = chain->source;
4904 int32_t sign = chain->sign;
4905 arith::MinSIOp minOp = chain->minOp;
4906 arith::MaxSIOp maxOp = chain->maxOp;
4907 auto srcIntType = cast<IntegerType>(truncOp.getIn().getType());
4908 auto dstIntType = cast<IntegerType>(truncOp.getOut().getType());
4909 unsigned dstBits = dstIntType.getWidth();
4912 Location loc = truncOp.getLoc();
4915 arith::ShRSIOp shrsiOp = source.getDefiningOp<arith::ShRSIOp>();
4918 if (!isa<IntegerType>(shrsiOp.getLhs().getType()))
4920 preShiftVal = shrsiOp.getLhs();
4921 shiftVal = shrsiOp.getRhs();
4926 preShiftVal = source;
4927 shiftVal = arith::ConstantOp::create(rewriter, loc,
4928 rewriter.getI32IntegerAttr(0));
4932 unsigned srcLanes = 512 / srcIntType.getWidth();
4936 auto bcast = aievec::BroadcastScalarOp::create(rewriter, loc, bcastVecType,
4940 VectorType srsOutType;
4946 unsigned accLanes = srcLanes * 2;
4947 VectorType accVecType =
4949 auto concatSrc = aievec::ConcatOp::create(
4950 rewriter, loc, accVecType,
4951 SmallVector<Value>({bcast.getResult(), bcast.getResult()}));
4953 aievec::CastOp::create(rewriter, loc, accVecType,
4954 concatSrc.getResult(),
true)
4961 accValue = aievec::CastOp::create(rewriter, loc, bcastVecType,
4962 bcast.getResult(),
true)
4968 auto srsOp = aievec::SRSOp::create(rewriter, loc, srsOutType, accValue,
4972 unsigned extLanes = 512 / dstBits;
4974 auto concatForExt = aievec::ConcatOp::create(
4975 rewriter, loc, extVecType,
4976 SmallVector<Value>({srsOp.getResult(), srsOp.getResult()}));
4980 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
4981 rewriter.replaceOpWithNewOp<aievec::ExtElemOp>(
4982 truncOp, dstIntType, concatForExt.getResult(), zeroIdx.getResult());
4985 SmallVector<Operation *, 3> opsToErase;
4986 if (minOp && minOp->use_empty())
4987 opsToErase.push_back(minOp);
4988 if (maxOp && maxOp->use_empty())
4989 opsToErase.push_back(maxOp);
4990 if (shrsiOp && shrsiOp->use_empty())
4991 opsToErase.push_back(shrsiOp);
4992 for (Operation *op : opsToErase)
4993 rewriter.eraseOp(op);
5003 using OpConversionPattern::OpConversionPattern;
5007 ConversionPatternRewriter &rewriter)
const override {
5009 Type resultType = rsOp.getType();
5010 if (isa<VectorType>(resultType))
5013 auto intType = dyn_cast<IntegerType>(resultType);
5014 if (!intType || intType.getWidth() != 32)
5017 Location loc = rsOp.getLoc();
5021 auto lhsBcast = aievec::BroadcastScalarOp::create(rewriter, loc, vecType,
5027 aievec::UPSOp::create(rewriter, loc, accType, lhsBcast.getResult());
5030 auto srsOp = aievec::SRSOp::create(rewriter, loc, vecType,
5031 upsOp.getResult(), adaptor.getRhs());
5035 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
5036 rewriter.replaceOpWithNewOp<aievec::ExtElemOp>(
5037 rsOp, intType, srsOp.getResult(), zeroIdx.getResult());
5053static LogicalResult analyzeContraction(vector::ContractionOp op,
5056 llvm::raw_string_ostream os(why);
5057 if (op.getKind() != vector::CombiningKind::ADD) {
5058 os <<
"the combining kind is not add";
5061 auto accTy = dyn_cast<VectorType>(op.getAccType());
5063 os <<
"the accumulator is a scalar";
5066 SmallVector<AffineMap, 3> maps = op.getIndexingMapsArray();
5067 if (!llvm::all_of(maps, [](AffineMap map) {
5068 return map.isProjectedPermutation(
false);
5070 os <<
"an indexing map is not a projected permutation";
5073 enum Role { M, N, K, Other };
5074 unsigned numDims = maps[0].getNumDims();
5075 SmallVector<int64_t> dimSize(numDims, 1);
5076 SmallVector<std::array<bool, 3>> indexes(numDims, {
false,
false,
false});
5077 ArrayRef<int64_t> shapes[3] = {op.getLhsType().getShape(),
5078 op.getRhsType().getShape(), accTy.getShape()};
5079 for (
unsigned operand = 0; operand < 3; ++operand)
5080 for (
auto [i, expr] : llvm::enumerate(maps[operand].getResults())) {
5081 unsigned d = cast<AffineDimExpr>(expr).getPosition();
5082 dimSize[d] = shapes[operand][i];
5083 indexes[d][operand] =
true;
5085 SmallVector<vector::IteratorType> iterators = op.getIteratorTypesArray();
5086 std::optional<unsigned> dimOf[3];
5087 for (
unsigned d = 0; d < numDims; ++d) {
5088 if (dimSize[d] == 1)
5090 auto [inLhs, inRhs, inAcc] = indexes[d];
5091 bool reduced = iterators[d] == vector::IteratorType::reduction;
5093 if (inLhs && !inRhs && inAcc && !reduced)
5095 else if (!inLhs && inRhs && inAcc && !reduced)
5097 else if (inLhs && inRhs && !inAcc && reduced)
5099 if (r == Other || dimOf[r]) {
5100 os <<
"d" << d <<
" of size " << dimSize[d]
5101 <<
" is not the only m, n or k dim larger than 1";
5106 auto sizeOf = [&](Role r) -> int64_t {
5107 std::optional<unsigned> d = dimOf[r];
5108 return d ? dimSize[*d] : 1;
5114 auto swapped = [&](AffineMap map, Role first, Role second) {
5115 if (!dimOf[first] || !dimOf[second])
5117 return *map.getResultPosition(
5118 getAffineDimExpr(*dimOf[second], map.getContext())) <
5119 *map.getResultPosition(
5120 getAffineDimExpr(*dimOf[first], map.getContext()));
5135template <
typename MatMulOpTy>
5137 : OpConversionPattern<vector::ContractionOp> {
5138 using OpConversionPattern::OpConversionPattern;
5141 std::is_same_v<MatMulOpTy, aievec::MatMulOp_AIE2P>;
5143 static Value
reshape(OpBuilder &b, Value v, ArrayRef<int64_t> shape) {
5144 auto vecTy = cast<VectorType>(v.getType());
5145 auto newTy = VectorType::get(shape, vecTy.getElementType());
5148 return vector::ShapeCastOp::create(b, v.getLoc(), newTy, v).getResult();
5153 ConversionPatternRewriter &rewriter)
const override {
5154 StringRef target =
isAIE2P ?
"AIE2P" :
"AIE2";
5157 if (failed(analyzeContraction(contractOp, tile, why))) {
5158 contractOp.emitError()
5159 <<
"cannot lower to an " << target <<
" matrix multiply: " << why;
5163 Value lhsIn = adaptor.getLhs(), rhsIn = adaptor.getRhs();
5164 Value lhsOrig = contractOp.getLhs(), rhsOrig = contractOp.getRhs();
5165 int64_t m = tile.
m, n = tile.
n, k = tile.
k;
5168 std::swap(lhsIn, rhsIn);
5169 std::swap(lhsOrig, rhsOrig);
5173 std::tie(lhsT, rhsT) =
5174 std::make_pair(!rhsT && m > 1 && k > 1, !lhsT && n > 1 && k > 1);
5176 SmallVector<int64_t, 2> lhsShape =
5177 lhsT ? SmallVector<int64_t, 2>{k, m} : SmallVector<int64_t, 2>{m, k};
5178 SmallVector<int64_t, 2> rhsShape =
5179 rhsT ? SmallVector<int64_t, 2>{n, k} : SmallVector<int64_t, 2>{k, n};
5181 VectorType::get({m, n}, getElementTypeOrSelf(adaptor.getAcc()));
5186 auto lhsSigned = getSignednessOfWideningOp(lhsIn);
5188 lhsSigned = getSignednessOfWideningOp(lhsOrig);
5189 auto rhsSigned = getSignednessOfWideningOp(rhsIn);
5191 rhsSigned = getSignednessOfWideningOp(rhsOrig);
5197 auto retype = [&](Value v, std::optional<bool> isSigned) -> Value {
5200 auto vecTy = cast<VectorType>(v.getType());
5201 auto intTy = dyn_cast<IntegerType>(vecTy.getElementType());
5202 if (!intTy || !intTy.isSignless())
5204 auto signednessTy = IntegerType::get(intTy.getContext(), intTy.getWidth(),
5205 *isSigned ? IntegerType::Signed
5206 : IntegerType::Unsigned);
5207 return UnrealizedConversionCastOp::create(rewriter, v.getLoc(),
5208 vecTy.clone(signednessTy), v)
5212 auto tileFits = [&](Value lhs, Value rhs,
bool report) {
5213 auto emitError = [&]() {
5214 return contractOp.emitError()
5215 <<
"cannot lower to an " << target <<
" matrix multiply: ";
5218 isAIE2P, VectorType::get(lhsShape, getElementTypeOrSelf(lhs)),
5219 VectorType::get(rhsShape, getElementTypeOrSelf(rhs)), accTy,
5221 report ? llvm::function_ref<InFlightDiagnostic()>(emitError)
5229 if (!tileFits(lhsIn, rhsIn,
false)) {
5230 lhsIn = getSourceOfWideningOp(lhsIn).value_or(lhsIn);
5231 rhsIn = getSourceOfWideningOp(rhsIn).value_or(rhsIn);
5232 if (!tileFits(lhsIn, rhsIn,
true))
5235 Value acc =
reshape(rewriter, adaptor.getAcc(), accTy.getShape());
5236 Value lhs =
reshape(rewriter, lhsIn, lhsShape);
5237 Value rhs =
reshape(rewriter, rhsIn, rhsShape);
5240 MatMulOpTy::create(rewriter, contractOp.getLoc(), acc.getType(),
5241 retype(lhs, lhsSigned), retype(rhs, rhsSigned), acc);
5242 matmulOp.setLhsTransposed(lhsT);
5243 matmulOp.setRhsTransposed(rhsT);
5246 reshape(rewriter, matmulOp.getResult(),
5247 cast<VectorType>(adaptor.getAcc().getType()).getShape()));
5259 : OpConversionPattern<vector::TransposeOp> {
5260 using OpConversionPattern::OpConversionPattern;
5263 ConversionPatternRewriter &rewriter)
const override {
5264 auto resTy = transpOp.getResultVectorType();
5265 auto resShape = resTy.getShape();
5266 auto elemTyBitWidth = resTy.getElementTypeBitWidth();
5267 auto vBitWidth = std::accumulate(resShape.begin(), resShape.end(),
5268 elemTyBitWidth, std::multiplies<>());
5269 if (vBitWidth != 512)
5272 if (elemTyBitWidth != 8 && elemTyBitWidth != 16 && elemTyBitWidth != 32)
5276 for (int64_t i = 0; i < static_cast<int64_t>(resShape.size() - 2); ++i)
5277 if (resShape[i] != 1)
5281 ArrayRef<int64_t> perm = transpOp.getPermutation();
5282 for (int64_t i = 0; i < static_cast<int64_t>(perm.size() - 2); ++i)
5285 if (perm.back() !=
static_cast<int64_t
>(perm.size() - 2))
5288 auto shuffleMode = aievec::ShuffleMode::T32_4X4;
5289 if (elemTyBitWidth == 8) {
5290 switch (resShape.back()) {
5292 shuffleMode = aievec::ShuffleMode::T8_4X16;
5295 shuffleMode = aievec::ShuffleMode::T8_8X8;
5298 shuffleMode = aievec::ShuffleMode::T8_16X4;
5303 }
else if (elemTyBitWidth == 16) {
5304 switch (resShape.back()) {
5306 shuffleMode = aievec::ShuffleMode::T16_2X16;
5309 shuffleMode = aievec::ShuffleMode::T16_4X8;
5312 shuffleMode = aievec::ShuffleMode::T16_8X4;
5315 shuffleMode = aievec::ShuffleMode::T16_16X2;
5320 }
else if (resShape.back() != 4)
5324 VectorType::get({512 / elemTyBitWidth}, resTy.getElementType());
5325 auto loc = transpOp.getLoc();
5326 auto flatInput = vector::ShapeCastOp::create(rewriter, loc, flatVecTy,
5327 adaptor.getVector());
5328 auto shuffOp = aievec::ShuffleOp::create(rewriter, loc, flatVecTy,
5329 flatInput,
nullptr, shuffleMode);
5330 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(transpOp, resTy, shuffOp);
5341populateAIEVecCommonConversionPatterns(RewritePatternSet &patterns) {
5357 patterns.getContext(),
false, 3);
5361static void populateAIEVecV1ConversionPatterns(RewritePatternSet &patterns) {
5379populateAIEVecV2CommonConversionPatterns(RewritePatternSet &patterns) {
5391 >(patterns.getContext());
5437 >(patterns.getContext());
5441static void populateAIEVecV2ConversionPatterns(RewritePatternSet &patterns) {
5442 populateAIEVecV2CommonConversionPatterns(patterns);
5444 patterns.getContext());
5448 patterns.getContext());
5454 : OpConversionPattern<vector::BroadcastOp> {
5455 using OpConversionPattern::OpConversionPattern;
5459 ConversionPatternRewriter &rewriter)
const override {
5461 if (adaptor.getSource().getDefiningOp<vector::ExtractOp>())
5464 auto resultType = cast<VectorType>(bcastOp.getResult().getType());
5466 Type scalarType = resultType.getElementType();
5467 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
5469 auto src = bcastOp.getSource();
5472 if (laneSize * elWidth == 512 || laneSize * elWidth == 256) {
5473 Value newOp = aievec::BroadcastScalarOp::create(
5474 rewriter, bcastOp.getLoc(), flatResultType, src);
5475 if (resultType != flatResultType)
5476 newOp = vector::ShapeCastOp::create(rewriter, bcastOp.getLoc(),
5478 rewriter.replaceOp(bcastOp, newOp);
5482 if (laneSize * elWidth == 1024) {
5484 auto aieBcastOp = aievec::BroadcastScalarOp::create(
5485 rewriter, bcastOp.getLoc(), vecType, src);
5486 Value newOp = aievec::ConcatOp::create(
5487 rewriter, bcastOp.getLoc(), flatResultType,
5488 SmallVector<Value>({aieBcastOp.getResult(), aieBcastOp.getResult()}));
5489 if (resultType != flatResultType)
5490 newOp = vector::ShapeCastOp::create(rewriter, bcastOp.getLoc(),
5492 rewriter.replaceOp(bcastOp, newOp);
5502static bool isFlattenableElementwise(Operation *op) {
5503 if (op->getNumResults() != 1)
5505 auto resultType = dyn_cast<VectorType>(op->getResult(0).getType());
5506 if (!resultType || resultType.getRank() == 1 || resultType.isScalable())
5512 if (isa<arith::ExtFOp>(op) &&
5513 llvm::any_of(op->getUsers(), [](Operation *user) {
5514 return isa<vector::ContractionOp>(user);
5521 return llvm::all_of(op->getOperandTypes(), [&](Type type) {
5522 auto operandType = dyn_cast<VectorType>(type);
5523 return !operandType || operandType.getShape() == resultType.getShape();
5534template <
typename OpTy>
5536 using OpConversionPattern<OpTy>::OpConversionPattern;
5541 ConversionPatternRewriter &rewriter)
const override {
5542 if (!isFlattenableElementwise(op))
5545 auto resultType = cast<VectorType>(op.getType());
5547 Location loc = op.getLoc();
5549 SmallVector<Value> flatOperands;
5550 flatOperands.reserve(adaptor.getOperands().size());
5551 for (Value operand : adaptor.getOperands()) {
5552 auto operandType = dyn_cast<VectorType>(operand.getType());
5554 flatOperands.push_back(operand);
5561 if (
auto cstOp = operand.template getDefiningOp<arith::ConstantOp>()) {
5562 if (
auto dense = dyn_cast<DenseElementsAttr>(cstOp.getValue())) {
5563 if (dense.isSplat()) {
5564 flatOperands.push_back(arith::ConstantOp::create(
5566 DenseElementsAttr::get(flatOperandType,
5567 dense.getSplatValue<Attribute>())));
5572 flatOperands.push_back(
5573 vector::ShapeCastOp::create(rewriter, loc, flatOperandType, operand));
5576 auto flatOp = OpTy::create(rewriter, loc, TypeRange{flatResultType},
5577 flatOperands, op.getProperties(),
5578 op->getDiscardableAttrDictionary().getValue());
5579 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, resultType,
5580 flatOp->getResult(0));
5588static SmallVector<int64_t, 5> getAIE2PNativeLanes(Operation *op) {
5589 Value v = isa<arith::CmpIOp, arith::CmpFOp>(op) ? op->getOperand(0)
5591 auto type = dyn_cast<VectorType>(v.getType());
5592 if (!type || !type.getElementType().isIntOrFloat())
5594 Type elemType = type.getElementType();
5595 unsigned bitWidth = elemType.getIntOrFloatBitWidth();
5596 bool isInt = isa<IntegerType>(elemType) &&
5597 (bitWidth == 8 || bitWidth == 16 || bitWidth == 32);
5602 if (isa<arith::AddFOp, arith::SubFOp>(op)) {
5603 if (elemType.isF32())
5604 return {16, 32, 64};
5605 if (elemType.isBF16())
5612 if (isa<arith::AddIOp, arith::SubIOp>(op)) {
5615 SmallVector<int64_t, 5> lanes;
5616 for (int64_t n = 2; n <= 512 / bitWidth; n *= 2)
5623 if (isa<math::TanhOp, math::ErfOp, math::Exp2Op>(op)) {
5624 if (elemType.isF32() || elemType.isBF16())
5629 if (isa<arith::MaximumFOp, arith::MaxNumFOp, arith::MinimumFOp>(op)) {
5630 if (elemType.isBF16())
5638 if (isa<arith::CmpIOp, arith::CmpFOp, arith::SelectOp>(op)) {
5639 if (
auto cmpOp = dyn_cast<arith::CmpFOp>(op);
5640 cmpOp && !llvm::is_contained({CmpFPredicate::OEQ, CmpFPredicate::ONE,
5641 CmpFPredicate::OLT, CmpFPredicate::OLE,
5642 CmpFPredicate::OGT, CmpFPredicate::OGE},
5643 cmpOp.getPredicate()))
5645 if (elemType.isBF16() || elemType.isInteger(16))
5647 if (elemType.isInteger(32))
5655static bool isAIE2PNativeWidth(Operation *op) {
5656 auto type = dyn_cast<VectorType>(op->getResult(0).getType());
5657 return type && !type.isScalable() &&
5658 llvm::is_contained(getAIE2PNativeLanes(op), type.getNumElements());
5662static Operation *getSelectCompare(arith::SelectOp op) {
5663 Operation *cmp = skipShapeCastRoundTrip(op.getCondition()).getDefiningOp();
5664 return isa_and_nonnull<arith::CmpIOp, arith::CmpFOp>(cmp) ? cmp :
nullptr;
5679static std::optional<int64_t> getAIE2PPartLanes(Operation *op) {
5680 if (op->getNumResults() != 1 || isa<arith::CmpIOp, arith::CmpFOp>(op))
5681 return std::nullopt;
5682 auto type = dyn_cast<VectorType>(op->getResult(0).getType());
5683 if (!type || type.isScalable())
5684 return std::nullopt;
5685 int64_t lanes = type.getNumElements();
5686 if (lanes < 2 || llvm::any_of(
5687 op->getOperandTypes(),
5689 auto vt = dyn_cast<VectorType>(t);
5690 return !vt || vt.getShape() != type.getShape();
5692 return std::nullopt;
5694 SmallVector<int64_t, 5> native = getAIE2PNativeLanes(op);
5695 if (
auto selOp = dyn_cast<arith::SelectOp>(op)) {
5696 Operation *cmp = getSelectCompare(selOp);
5698 return std::nullopt;
5699 SmallVector<int64_t, 5> cmpNative = getAIE2PNativeLanes(cmp);
5701 native, [&](int64_t n) {
return !llvm::is_contained(cmpNative, n); });
5703 if (native.empty() || llvm::is_contained(native, lanes))
5704 return std::nullopt;
5706 if (isa<arith::AddIOp, arith::SubIOp>(op) && lanes < native.back())
5707 return std::nullopt;
5709 for (int64_t part : llvm::reverse(native))
5710 if (part < lanes && lanes % part == 0 && lanes / part <=
kAIE2PMaxParts)
5712 if (isa<arith::SelectOp>(op))
5713 return std::nullopt;
5714 for (int64_t part : native)
5715 if (part > lanes && part % lanes == 0)
5717 return std::nullopt;
5724static Value getAIE2PPart(ConversionPatternRewriter &rewriter, Location loc,
5725 Value v, int64_t partLanes, int64_t index) {
5726 auto type = cast<VectorType>(v.getType());
5727 auto partType = VectorType::get({partLanes}, type.getElementType());
5728 int64_t lanes = type.getNumElements();
5729 if (
auto cstOp = v.getDefiningOp<arith::ConstantOp>())
5730 if (
auto dense = dyn_cast<DenseElementsAttr>(cstOp.getValue());
5731 dense && dense.isSplat())
5732 return arith::ConstantOp::create(
5734 DenseElementsAttr::get(partType, dense.getSplatValue<Attribute>()));
5735 if (
auto bcastOp = v.getDefiningOp<vector::BroadcastOp>())
5736 if (!isa<VectorType>(bcastOp.getSourceType()))
5737 return vector::BroadcastOp::create(rewriter, loc, partType,
5738 bcastOp.getSource());
5739 if (partLanes < lanes) {
5740 if (
auto concatOp = v.getDefiningOp<aievec::ConcatOp>())
5741 if (concatOp.getSources().front().getType() == partType)
5742 return concatOp.getSources()[index];
5743 return aievec::ExtOp::create(rewriter, loc, partType, v, index);
5745 SmallVector<Value> copies(partLanes / lanes, v);
5746 return aievec::ConcatOp::create(rewriter, loc, partType, copies);
5752static Value getAIE2PMaskPart(ConversionPatternRewriter &rewriter, Location loc,
5753 arith::SelectOp selOp, int64_t partLanes,
5755 Operation *cmp = getSelectCompare(selOp);
5756 OperationState state(loc, cmp->getName());
5757 for (Value operand : cmp->getOperands())
5758 state.addOperands(getAIE2PPart(
5759 rewriter, loc, rewriter.getRemappedValue(operand), partLanes, index));
5760 state.addTypes(VectorType::get({partLanes}, rewriter.getI1Type()));
5761 state.addAttributes(cmp->getAttrs());
5762 return rewriter.create(state)->getResult(0);
5770template <
typename OpTy>
5772 using OpConversionPattern<OpTy>::OpConversionPattern;
5777 ConversionPatternRewriter &rewriter)
const override {
5778 auto type = dyn_cast<VectorType>(op->getResult(0).getType());
5779 if (!type || type.getRank() != 1)
5781 std::optional<int64_t> partLanes = getAIE2PPartLanes(op);
5784 int64_t lanes = type.getNumElements();
5785 auto partType = VectorType::get({*partLanes}, type.getElementType());
5786 Location loc = op.getLoc();
5787 auto createPart = [&](int64_t index) -> Value {
5788 SmallVector<Value> operands;
5789 for (Value operand : adaptor.getOperands()) {
5790 if constexpr (std::is_same_v<OpTy, arith::SelectOp>) {
5791 if (operands.empty()) {
5793 getAIE2PMaskPart(rewriter, loc, op, *partLanes, index));
5798 getAIE2PPart(rewriter, loc, operand, *partLanes, index));
5800 return OpTy::create(rewriter, loc, TypeRange{partType}, operands,
5802 op->getDiscardableAttrDictionary().getValue())
5806 if (*partLanes > lanes) {
5807 rewriter.replaceOpWithNewOp<aievec::ExtOp>(op, type, createPart(0), 0);
5810 SmallVector<Value> parts;
5811 for (int64_t i = 0; i < lanes / *partLanes; ++i)
5812 parts.push_back(createPart(i));
5813 rewriter.replaceOpWithNewOp<aievec::ConcatOp>(op, type, parts);
5822template <
typename... OpTys>
5824 static void addPatterns(RewritePatternSet &patterns, PatternBenefit benefit) {
5835 template <
typename OpTy>
5837 std::optional<ConversionTarget::LegalizationAction> action =
5838 target.getOpAction(OperationName(OpTy::getOperationName(), ctx));
5839 if (action == ConversionTarget::LegalizationAction::Legal)
5840 target.addDynamicallyLegalOp<OpTy>([](Operation *op) {
5841 return !isFlattenableElementwise(op) && !getAIE2PPartLanes(op);
5843 else if (action == ConversionTarget::LegalizationAction::Dynamic)
5844 target.addDynamicallyLegalOp<OpTy>(
5845 [](Operation *op) -> std::optional<bool> {
5846 if (isFlattenableElementwise(op) || getAIE2PPartLanes(op))
5848 return std::nullopt;
5853 arith::MulFOp, arith::DivFOp, arith::AddFOp, arith::SubFOp, arith::NegFOp,
5854 arith::TruncFOp, arith::ExtFOp, math::ExpOp, math::TanhOp, math::ErfOp,
5855 math::Exp2Op, arith::MaximumFOp, arith::MaxNumFOp, arith::MinimumFOp,
5856 arith::CmpFOp, arith::CmpIOp, arith::SelectOp, arith::AddIOp, arith::SubIOp,
5857 arith::MulIOp, arith::IndexCastOp>;
5859static void populateAIEVecV2PConversionPatterns(
5860 RewritePatternSet &patterns, std::shared_ptr<AIE2PShufflePlanner> planner) {
5861 populateAIEVecV2CommonConversionPatterns(patterns);
5863 patterns.getContext());
5869 patterns.getContext(),
true, 4);
5878 patterns.getContext());
5903static bool isInSigmoidOperationChain(math::ExpOp expOp) {
5904 if (!expOp.getOperand().getDefiningOp<arith::NegFOp>())
5907 arith::AddFOp addOp =
nullptr;
5908 for (Operation *user : expOp->getUsers()) {
5909 addOp = dyn_cast<arith::AddFOp>(user);
5917 auto *addLvalOp = addOp.getLhs().getDefiningOp();
5918 auto *addRvalOp = addOp.getRhs().getDefiningOp();
5919 if (!((isa<math::ExpOp>(addLvalOp) && isa<arith::ConstantOp>(addRvalOp)) ||
5920 (isa<math::ExpOp>(addRvalOp) && isa<arith::ConstantOp>(addLvalOp))))
5923 auto constOp = isa<arith::ConstantOp>(addLvalOp)
5924 ? cast<arith::ConstantOp>(addLvalOp)
5925 : cast<
arith::ConstantOp>(addRvalOp);
5927 auto cstDense = dyn_cast<DenseFPElementsAttr>(constOp.getValue());
5931 if (cstDense.getSplatValue<APFloat>().convertToFloat() != 1.0f)
5934 arith::DivFOp divOp =
nullptr;
5935 for (Operation *user : addOp->getUsers()) {
5936 divOp = dyn_cast<arith::DivFOp>(user);
5944 constOp = dyn_cast<arith::ConstantOp>(divOp.getLhs().getDefiningOp());
5947 cstDense = dyn_cast<DenseFPElementsAttr>(constOp.getValue());
5950 if (cstDense.getSplatValue<APFloat>().convertToFloat() != 1.0f)
5956static void configureAIEVecCommonLegalizations(ConversionTarget &target) {
5958 .addLegalDialect<xilinx::aievec::aie1::AIEVecAIE1Dialect,
5959 xilinx::aievec::AIEVecDialect, arith::ArithDialect,
5960 ub::UBDialect, emitc::EmitCDialect, func::FuncDialect>();
5961 target.addIllegalOp<vector::ExtractStridedSliceOp>();
5962 target.addLegalOp<vector::BitCastOp>();
5964 target.addDynamicallyLegalOp<arith::ExtFOp>([](arith::ExtFOp extfOp) {
5965 auto srcType = dyn_cast<VectorType>(extfOp.getIn().getType());
5966 auto dstType = dyn_cast<VectorType>(extfOp.getOut().getType());
5967 if (!srcType || !dstType)
5970 Type srcScalarType = srcType.getElementType();
5971 Type dstScalarType = dstType.getElementType();
5972 if (!isa<FloatType>(srcScalarType) || !isa<FloatType>(dstScalarType))
5977 unsigned srcElWidth = srcScalarType.getIntOrFloatBitWidth();
5978 unsigned dstElWidth = dstScalarType.getIntOrFloatBitWidth();
5979 return srcElWidth != 16 || srcLaneSize != 16 || dstElWidth != 32 ||
5983 target.addDynamicallyLegalOp<arith::ExtSIOp>([](arith::ExtSIOp extsiOp) {
5984 auto srcType = dyn_cast<VectorType>(extsiOp.getIn().getType());
5985 auto dstType = dyn_cast<VectorType>(extsiOp.getOut().getType());
5986 if (!srcType || !dstType)
5989 Type srcScalarType = srcType.getElementType();
5990 Type dstScalarType = dstType.getElementType();
5991 if (!isa<IntegerType>(srcScalarType) || !isa<IntegerType>(dstScalarType))
5996 unsigned srcElWidth = srcScalarType.getIntOrFloatBitWidth();
5997 unsigned dstElWidth = dstScalarType.getIntOrFloatBitWidth();
5998 return srcLaneSize != 32 || (dstElWidth <= srcElWidth) ||
5999 (dstLaneSize != srcLaneSize);
6008 target.addDynamicallyLegalOp<arith::ExtUIOp>([](arith::ExtUIOp extuiOp) {
6009 auto srcType = dyn_cast<VectorType>(extuiOp.getIn().getType());
6010 auto dstType = dyn_cast<VectorType>(extuiOp.getOut().getType());
6011 if (!srcType || !dstType)
6013 if (srcType.getRank() != 1 || dstType.getRank() != 1)
6015 Type srcScalarType = srcType.getElementType();
6016 Type dstScalarType = dstType.getElementType();
6017 if (!srcScalarType.isInteger(4) || !dstScalarType.isInteger(8))
6019 unsigned lanes = srcType.getNumElements();
6020 if (lanes != 64 && lanes != 128)
6022 auto bitcastOp = extuiOp.getIn().getDefiningOp<vector::BitCastOp>();
6025 auto bitcastSrcType = dyn_cast<VectorType>(bitcastOp.getSource().getType());
6026 return !bitcastSrcType || bitcastSrcType.getRank() != 1 ||
6027 !bitcastSrcType.getElementType().isInteger(8) ||
6028 bitcastSrcType.getNumElements() * 2 != lanes;
6031 target.addDynamicallyLegalOp<arith::TruncFOp>([](arith::TruncFOp truncfOp) {
6032 auto srcType = dyn_cast<VectorType>(truncfOp.getIn().getType());
6033 auto dstType = dyn_cast<VectorType>(truncfOp.getOut().getType());
6034 if (!srcType || !dstType)
6037 Type srcScalarType = srcType.getElementType();
6038 Type dstScalarType = dstType.getElementType();
6039 if (!isa<FloatType>(srcScalarType) || !isa<FloatType>(dstScalarType))
6044 unsigned srcElWidth = srcScalarType.getIntOrFloatBitWidth();
6045 unsigned dstElWidth = dstScalarType.getIntOrFloatBitWidth();
6046 return srcElWidth != 32 || srcLaneSize != 16 || dstElWidth != 16 ||
6050 target.addDynamicallyLegalOp<arith::TruncIOp>([](arith::TruncIOp trunciOp) {
6051 auto srcType = dyn_cast<VectorType>(trunciOp.getIn().getType());
6052 auto dstType = dyn_cast<VectorType>(trunciOp.getOut().getType());
6053 if (!srcType || !dstType) {
6056 return !(!srcType && !dstType && isSRSCompoundCandidate(trunciOp));
6059 Type srcScalarType = srcType.getElementType();
6060 Type dstScalarType = dstType.getElementType();
6061 if (!isa<IntegerType>(srcScalarType) || !isa<IntegerType>(dstScalarType))
6065 if (isSRSCompoundCandidate(trunciOp))
6070 unsigned srcElWidth = srcScalarType.getIntOrFloatBitWidth();
6071 unsigned dstElWidth = dstScalarType.getIntOrFloatBitWidth();
6073 return srcLaneSize != 32 || (dstElWidth >= srcElWidth) ||
6074 (dstLaneSize != srcLaneSize);
6077 target.addDynamicallyLegalOp<math::TanhOp>([](math::TanhOp tanhOp) {
6078 auto srcType = dyn_cast<VectorType>(tanhOp.getOperand().getType());
6082 Type scalarType = srcType.getElementType();
6083 if (!isa<FloatType>(scalarType))
6087 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6088 return elWidth != 16 || laneSize != 16;
6091 target.addDynamicallyLegalOp<math::SqrtOp>([](math::SqrtOp sqrtOp) {
6092 auto srcType = dyn_cast<VectorType>(sqrtOp.getOperand().getType());
6096 Type scalarType = srcType.getElementType();
6097 if (!isa<FloatType>(scalarType))
6101 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6102 return elWidth != 16 || (laneSize != 16 && laneSize != 32);
6105 target.addDynamicallyLegalOp<math::ErfOp>([](math::ErfOp erfOp) {
6106 auto srcType = dyn_cast<VectorType>(erfOp.getOperand().getType());
6110 Type scalarType = srcType.getElementType();
6111 if (!isa<FloatType>(scalarType))
6115 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6116 return elWidth != 16 || (laneSize != 16 && laneSize != 32);
6119 target.addDynamicallyLegalOp<math::AbsFOp>([](math::AbsFOp absfOp) {
6120 auto srcType = dyn_cast<VectorType>(absfOp.getOperand().getType());
6124 Type scalarType = srcType.getElementType();
6126 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6127 return elWidth * laneSize != 512 && elWidth * laneSize != 256;
6130 target.addDynamicallyLegalOp<math::AbsIOp>([](math::AbsIOp absiOp) {
6131 auto srcType = dyn_cast<VectorType>(absiOp.getOperand().getType());
6135 Type scalarType = srcType.getElementType();
6137 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6138 return elWidth * laneSize != 512 && elWidth * laneSize != 256;
6141 target.addDynamicallyLegalOp<math::CeilOp>([](math::CeilOp ceilOp) {
6142 auto srcType = dyn_cast<VectorType>(ceilOp.getOperand().getType());
6145 Type scalarType = srcType.getElementType();
6146 if (!isa<FloatType>(scalarType))
6150 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6151 return elWidth != 16 || (laneSize != 16 && laneSize != 32);
6154 target.addDynamicallyLegalOp<math::FloorOp>([](math::FloorOp floorOp) {
6155 auto srcType = dyn_cast<VectorType>(floorOp.getOperand().getType());
6158 Type scalarType = srcType.getElementType();
6159 if (!isa<FloatType>(scalarType))
6163 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6164 return elWidth != 16 || (laneSize != 16 && laneSize != 32);
6167 target.addDynamicallyLegalOp<arith::NegFOp>([](arith::NegFOp negOp) {
6168 auto srcType = dyn_cast<VectorType>(negOp.getOperand().getType());
6171 if (Type scalarType = srcType.getElementType(); !isa<FloatType>(scalarType))
6175 return laneSize != 16;
6178 target.addDynamicallyLegalOp<arith::ShRSIOp>([](arith::ShRSIOp rsOp) {
6179 auto srcType = dyn_cast<VectorType>(rsOp.getLhs().getType());
6183 if (
auto intType = dyn_cast<IntegerType>(rsOp.getLhs().getType()))
6184 if (intType.getWidth() == 32) {
6187 return shrsiUsedByCompoundSRS(rsOp);
6194 if (shrsiUsedByCompoundSRS(rsOp))
6197 Type scalarType = srcType.getElementType();
6199 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6201 return laneSize * elWidth != 512;
6204 target.addDynamicallyLegalOp<arith::AddFOp>(
6205 [](arith::AddFOp op) {
return !isa<VectorType>(op.getType()); });
6206 target.addDynamicallyLegalOp<arith::SubFOp>(
6207 [](arith::SubFOp op) {
return !isa<VectorType>(op.getType()); });
6218 auto isRank1Vector = [](Type t) {
6219 auto vt = dyn_cast<VectorType>(t);
6220 return vt && vt.getRank() == 1;
6222 target.addDynamicallyLegalOp<arith::SIToFPOp>(
6223 [=](arith::SIToFPOp op) {
return !isRank1Vector(op.getType()); });
6224 target.addDynamicallyLegalOp<arith::UIToFPOp>(
6225 [=](arith::UIToFPOp op) {
return !isRank1Vector(op.getType()); });
6226 target.addDynamicallyLegalOp<arith::FPToSIOp>(
6227 [=](arith::FPToSIOp op) {
return !isRank1Vector(op.getType()); });
6228 target.addDynamicallyLegalOp<arith::FPToUIOp>(
6229 [=](arith::FPToUIOp op) {
return !isRank1Vector(op.getType()); });
6230 target.addLegalOp<vector::ExtractOp, vector::InsertOp>();
6233static void configureAIEVecV1Legalizations(ConversionTarget &target) {
6234 target.addDynamicallyLegalOp<arith::MulIOp>(
6235 [](arith::MulIOp op) {
return !isa<VectorType>(op.getType()); });
6236 target.addDynamicallyLegalOp<arith::MulFOp>(
6237 [](arith::MulFOp op) {
return !isa<VectorType>(op.getType()); });
6238 target.addDynamicallyLegalOp<aievec::aie1::FMAOp>(
6239 [](xilinx::aievec::aie1::FMAOp op) {
6240 auto *lhsDefOp = op.getLhs().getDefiningOp();
6241 aievec::ConcatOp concatOp =
nullptr;
6243 concatOp = dyn_cast<aievec::ConcatOp>(op.getLhs().getDefiningOp());
6247 vector::BroadcastOp srcBcast =
nullptr;
6248 if (
auto *lhsOp = concatOp.getSources()[0].getDefiningOp())
6249 srcBcast = dyn_cast<vector::BroadcastOp>(lhsOp);
6251 auto *rhsOp = op.getRhs().getDefiningOp();
6254 srcBcast = dyn_cast<vector::BroadcastOp>(rhsOp);
6258 if (
auto *srcOp = srcBcast.getSource().getDefiningOp())
6259 return !isa<vector::ExtractOp>(srcOp);
6264 target.addDynamicallyLegalOp<aievec::aie1::AddOp>([](aievec::aie1::AddOp op) {
6265 auto lSrsOp = op.getLhs().getDefiningOp<aievec::SRSOp>();
6266 auto rSrsOp = op.getRhs().getDefiningOp<aievec::SRSOp>();
6268 !lSrsOp.getSource().getDefiningOp<aievec::aie1::MulOp>()) &&
6270 !rSrsOp.getSource().getDefiningOp<aievec::aie1::MulOp>());
6272 target.addLegalDialect<memref::MemRefDialect>();
6276configureAIEVecV2PLegalizations(ConversionTarget &target,
6277 std::shared_ptr<AIE2PShufflePlanner> planner) {
6281 target.addDynamicallyLegalOp<math::RsqrtOp>([](math::RsqrtOp rsqrtOp) {
6282 auto vecType = dyn_cast<VectorType>(rsqrtOp.getOperand().getType());
6285 return !vecType || !vecType.getElementType().isBF16();
6290 target.addDynamicallyLegalOp<math::ExpOp>([](math::ExpOp expOp) {
6291 auto srcType = dyn_cast<VectorType>(expOp.getOperand().getType());
6295 Type scalarType = srcType.getElementType();
6296 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6299 if (scalarType.isF32())
6300 return srcType.isScalable() || (laneSize != 16 && laneSize != 32);
6302 if (!scalarType.isBF16() || (laneSize != 16 && laneSize != 32) ||
6305 if (expOp->hasOneUse() && isInSigmoidOperationChain(expOp))
6314 target.addDynamicallyLegalOp<math::TanhOp>([](math::TanhOp tanhOp) {
6315 return !isAIE2PAccFloatUnaryOrScalarType(tanhOp.getOperand().getType());
6317 target.addDynamicallyLegalOp<math::ErfOp>([](math::ErfOp erfOp) {
6318 return !isAIE2PAccFloatUnaryOrScalarType(erfOp.getOperand().getType());
6320 target.addDynamicallyLegalOp<math::Exp2Op>([](math::Exp2Op exp2Op) {
6321 return !isAIE2PAccFloatUnaryOrScalarType(exp2Op.getOperand().getType());
6326 target.addDynamicallyLegalOp<arith::DivFOp>([](arith::DivFOp divfOp) {
6327 Type srcType = divfOp.getLhs().getType();
6331 dyn_cast_or_null<arith::ConstantOp>(divfOp.getLhs().getDefiningOp());
6336 if (srcType.isF32()) {
6337 auto floatAttr = dyn_cast<FloatAttr>(constOp.getValue());
6339 return !floatAttr || !floatAttr.getValue().isExactlyValue(1.0);
6343 if (
auto vecType = dyn_cast<VectorType>(srcType)) {
6344 if (vecType.getElementType().isF32()) {
6346 if (laneSize == 16 || laneSize == 32) {
6347 auto denseAttr = dyn_cast<DenseFPElementsAttr>(constOp.getValue());
6348 if (denseAttr && denseAttr.isSplat() &&
6349 denseAttr.getSplatValue<APFloat>().isExactlyValue(1.0))
6359 target.addDynamicallyLegalOp<arith::ExtFOp>([](arith::ExtFOp extfOp) {
6360 auto srcType = dyn_cast<VectorType>(extfOp.getIn().getType());
6361 auto dstType = dyn_cast<VectorType>(extfOp.getOut().getType());
6362 if (!srcType || !dstType)
6365 Type srcScalarType = srcType.getElementType();
6366 Type dstScalarType = dstType.getElementType();
6367 if (!isa<FloatType>(srcScalarType) || !isa<FloatType>(dstScalarType))
6372 return (srcLaneSize % 16 != 0) || (dstLaneSize % 16 != 0);
6376 target.addDynamicallyLegalOp<arith::TruncFOp>([](arith::TruncFOp truncfOp) {
6377 auto srcType = dyn_cast<VectorType>(truncfOp.getIn().getType());
6378 auto dstType = dyn_cast<VectorType>(truncfOp.getOut().getType());
6379 if (!srcType || !dstType)
6382 Type srcScalarType = srcType.getElementType();
6383 Type dstScalarType = dstType.getElementType();
6384 if (!isa<FloatType>(srcScalarType) || !isa<FloatType>(dstScalarType))
6389 return (srcLaneSize % 16 != 0) || (dstLaneSize % 16 != 0);
6393 target.addDynamicallyLegalOp<arith::ExtSIOp>([](arith::ExtSIOp extsiOp) {
6394 auto srcType = dyn_cast<VectorType>(extsiOp.getIn().getType());
6395 auto dstType = dyn_cast<VectorType>(extsiOp.getOut().getType());
6396 if (!srcType || !dstType)
6399 Type srcScalarType = srcType.getElementType();
6400 Type dstScalarType = dstType.getElementType();
6401 if (!isa<IntegerType>(srcScalarType) || !isa<IntegerType>(dstScalarType))
6406 return (srcLaneSize % 16 != 0) || (dstLaneSize % 16 != 0);
6410 target.addDynamicallyLegalOp<arith::TruncIOp>([](arith::TruncIOp trunciOp) {
6411 auto srcType = dyn_cast<VectorType>(trunciOp.getIn().getType());
6412 auto dstType = dyn_cast<VectorType>(trunciOp.getOut().getType());
6413 if (!srcType || !dstType) {
6415 return !(!srcType && !dstType && isSRSCompoundCandidate(trunciOp));
6417 Type srcScalarType = srcType.getElementType();
6418 Type dstScalarType = dstType.getElementType();
6419 if (!isa<IntegerType>(srcScalarType) || !isa<IntegerType>(dstScalarType))
6424 if (isSRSCompoundCandidate(trunciOp))
6429 return (srcLaneSize % 16 != 0) || (dstLaneSize % 16 != 0);
6434 target.addDynamicallyLegalOp<arith::AddFOp>([](arith::AddFOp op) {
6435 auto resultType = dyn_cast<VectorType>(op.getType());
6439 Type scalarType = resultType.getElementType();
6442 if (scalarType.isBF16() || scalarType.isF32())
6443 return !isAIE2PNativeWidth(op);
6446 if (isa<FloatType>(scalarType))
6447 return laneSize != 16 && laneSize != 32;
6450 return laneSize != 16;
6455 target.addDynamicallyLegalOp<arith::SubFOp>([](arith::SubFOp op) {
6456 auto resultType = dyn_cast<VectorType>(op.getType());
6460 Type scalarType = resultType.getElementType();
6463 if (scalarType.isBF16() || scalarType.isF32())
6464 return !isAIE2PNativeWidth(op);
6467 if (isa<FloatType>(scalarType))
6468 return laneSize != 16 && laneSize != 32;
6471 return laneSize != 16;
6476 target.addDynamicallyLegalOp<arith::NegFOp>([](arith::NegFOp negOp) {
6477 auto srcType = dyn_cast<VectorType>(negOp.getOperand().getType());
6481 Type scalarType = srcType.getElementType();
6487 if (!scalarType.isBF16() && !scalarType.isF32())
6498 return laneSize != 32 || srcType.getRank() != 1;
6505 target.addDynamicallyLegalOp<arith::MulFOp>([](arith::MulFOp op) {
6506 auto resultType = dyn_cast<VectorType>(op.getType());
6510 auto isAddOp = [](Operation *user) {
return isa<arith::AddFOp>(user); };
6511 if (resultType.getRank() == 1 && op->hasOneUse() &&
6512 llvm::any_of(op->getUsers(), isAddOp) &&
6513 !(holdsBF16(op.getLhs()) && holdsBF16(op.getRhs())))
6516 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6519 if (laneSize == 16 && (resultElWidth == 16 || resultElWidth == 32))
6521 if (laneSize == 32 && resultElWidth == 16)
6523 if ((laneSize == 32 || laneSize == 64) && resultElWidth == 32 &&
6524 resultType.getRank() == 1 && holdsBF16(op.getLhs()) &&
6525 holdsBF16(op.getRhs()))
6538static void configureAIEVecV2Legalizations(ConversionTarget &target) {
6539 target.addLegalOp<UnrealizedConversionCastOp>();
6540 target.addLegalOp<vector::ShapeCastOp>();
6543 llvm::SmallSet<std::pair<unsigned, unsigned>, 16> laneSizeElWidthPairSet;
6544 laneSizeElWidthPairSet.insert({64, 8});
6545 laneSizeElWidthPairSet.insert({32, 16});
6546 laneSizeElWidthPairSet.insert({16, 32});
6547 laneSizeElWidthPairSet.insert({32, 32});
6550 llvm::SmallSet<unsigned, 16> elWidthSet;
6551 elWidthSet.insert(8);
6552 elWidthSet.insert(16);
6553 elWidthSet.insert(32);
6555 target.addDynamicallyLegalOp<arith::AddFOp>([](arith::AddFOp op) {
6556 auto resultType = dyn_cast<VectorType>(op.getType());
6560 Type scalarType = resultType.getElementType();
6562 unsigned resultElWidth = scalarType.getIntOrFloatBitWidth();
6568 if (laneSize == 32 && resultElWidth == 16)
6571 if (laneSize == 32 && resultElWidth == 32)
6577 target.addDynamicallyLegalOp<arith::SubFOp>([](arith::SubFOp op) {
6578 auto resultType = dyn_cast<VectorType>(op.getType());
6582 Type scalarType = resultType.getElementType();
6584 unsigned resultElWidth = scalarType.getIntOrFloatBitWidth();
6590 if (laneSize == 32 && resultElWidth == 16)
6593 if (laneSize == 32 && resultElWidth == 32)
6599 target.addDynamicallyLegalOp<arith::MulIOp>([](arith::MulIOp op) {
6600 auto resultType = dyn_cast<VectorType>(op.getType());
6603 auto isAddOp = [&](Operation *op) {
return isa<arith::AddIOp>(op); };
6605 if (op->hasOneUse() && llvm::any_of(op->getUsers(), isAddOp))
6608 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6611 return (laneSize != 32 || (resultElWidth != 16 && resultElWidth != 8)) &&
6612 ((laneSize != 16 && laneSize != 32) || resultElWidth != 32);
6615 target.addDynamicallyLegalOp<arith::MulFOp>([](arith::MulFOp op) {
6616 auto resultType = dyn_cast<VectorType>(op.getType());
6620 auto isAddOp = [&](Operation *op) {
return isa<arith::AddFOp>(op); };
6622 if (op->hasOneUse() && llvm::any_of(op->getUsers(), isAddOp) &&
6623 !(holdsBF16(op.getLhs()) && holdsBF16(op.getRhs())))
6626 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6630 if (laneSize == 16 && (resultElWidth == 16 || resultElWidth == 32))
6632 if (laneSize == 32 && resultElWidth == 16)
6634 if (laneSize == 32 && resultElWidth == 32 && holdsBF16(op.getLhs()) &&
6635 holdsBF16(op.getRhs()))
6641 target.addDynamicallyLegalOp<arith::MinSIOp>([=](arith::MinSIOp op) {
6642 auto resultType = dyn_cast<VectorType>(op.getType());
6645 if (
auto intType = dyn_cast<IntegerType>(op.getType())) {
6646 unsigned w = intType.getWidth();
6647 if (w == 8 || w == 16 || w == 32) {
6650 return scalarClampInCompoundSRS(op);
6656 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6659 return !elWidthSet.count(resultElWidth) || laneSize * resultElWidth != 512;
6662 target.addDynamicallyLegalOp<arith::MaxSIOp>([=](arith::MaxSIOp op) {
6663 auto resultType = dyn_cast<VectorType>(op.getType());
6666 if (
auto intType = dyn_cast<IntegerType>(op.getType())) {
6667 unsigned w = intType.getWidth();
6668 if (w == 8 || w == 16 || w == 32) {
6671 return scalarClampInCompoundSRS(op);
6677 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6680 return !elWidthSet.count(resultElWidth) || laneSize * resultElWidth != 512;
6683 target.addDynamicallyLegalOp<arith::MinUIOp>([=](arith::MinUIOp op) {
6684 auto resultType = dyn_cast<VectorType>(op.getType());
6688 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6690 unsigned totalBits = laneSize * resultElWidth;
6692 return !elWidthSet.count(resultElWidth) ||
6693 (totalBits != 512 && (totalBits != 256 || resultElWidth != 16));
6696 target.addDynamicallyLegalOp<arith::MaxUIOp>([=](arith::MaxUIOp op) {
6697 auto resultType = dyn_cast<VectorType>(op.getType());
6701 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6703 unsigned totalBits = laneSize * resultElWidth;
6705 return !elWidthSet.count(resultElWidth) ||
6706 (totalBits != 512 && (totalBits != 256 || resultElWidth != 16));
6709 target.addDynamicallyLegalOp<arith::MinimumFOp>([=](arith::MinimumFOp op) {
6710 auto resultType = dyn_cast<VectorType>(op.getType());
6714 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6716 unsigned totalBits = laneSize * resultElWidth;
6718 return !elWidthSet.count(resultElWidth) ||
6719 (totalBits != 512 && (totalBits != 256 || resultElWidth != 16));
6722 target.addDynamicallyLegalOp<arith::MaximumFOp>([=](arith::MaximumFOp op) {
6723 auto resultType = dyn_cast<VectorType>(op.getType());
6727 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6729 unsigned totalBits = laneSize * resultElWidth;
6731 return !elWidthSet.count(resultElWidth) ||
6732 (totalBits != 512 && (totalBits != 256 || resultElWidth != 16));
6735 target.addDynamicallyLegalOp<arith::MaxNumFOp>([=](arith::MaxNumFOp op) {
6736 auto resultType = dyn_cast<VectorType>(op.getType());
6740 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6742 unsigned totalBits = laneSize * resultElWidth;
6744 return !elWidthSet.count(resultElWidth) ||
6745 (totalBits != 512 && (totalBits != 256 || resultElWidth != 16));
6748 target.addDynamicallyLegalOp<arith::MinNumFOp>([=](arith::MinNumFOp op) {
6749 auto resultType = dyn_cast<VectorType>(op.getType());
6753 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6755 unsigned totalBits = laneSize * resultElWidth;
6757 return !elWidthSet.count(resultElWidth) ||
6758 (totalBits != 512 && (totalBits != 256 || resultElWidth != 16));
6761 target.addDynamicallyLegalOp<arith::CmpIOp>([=](arith::CmpIOp op) {
6762 auto lhsType = dyn_cast<VectorType>(op.getLhs().getType());
6766 auto lhsElWidth = lhsType.getElementType().getIntOrFloatBitWidth();
6768 unsigned totalBits = laneSize * lhsElWidth;
6770 return !elWidthSet.count(lhsElWidth) ||
6771 (totalBits != 512 && (totalBits != 256 || lhsElWidth != 16));
6774 target.addDynamicallyLegalOp<arith::CmpFOp>([=](arith::CmpFOp op) {
6775 auto lhsType = dyn_cast<VectorType>(op.getLhs().getType());
6779 auto lhsElWidth = lhsType.getElementType().getIntOrFloatBitWidth();
6781 unsigned totalBits = laneSize * lhsElWidth;
6783 return !elWidthSet.count(lhsElWidth) ||
6784 (totalBits != 512 && (totalBits != 256 || lhsElWidth != 16));
6787 target.addDynamicallyLegalOp<arith::SelectOp>([=](arith::SelectOp op) {
6788 auto resultType = dyn_cast<VectorType>(op.getType());
6792 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6794 unsigned totalBits = laneSize * resultElWidth;
6796 return !elWidthSet.count(resultElWidth) ||
6797 (totalBits != 512 && (totalBits != 256 || resultElWidth != 16));
6800 target.addDynamicallyLegalOp<vector::ReductionOp>(
6801 [=](vector::ReductionOp op) {
6802 if (
auto kind = op.getKind(); kind != vector::CombiningKind::ADD &&
6803 kind != vector::CombiningKind::MINSI &&
6804 kind != vector::CombiningKind::MINUI &&
6805 kind != vector::CombiningKind::MINIMUMF &&
6806 kind != vector::CombiningKind::MINNUMF &&
6807 kind != vector::CombiningKind::MAXSI &&
6808 kind != vector::CombiningKind::MAXUI &&
6809 kind != vector::CombiningKind::MAXIMUMF &&
6810 kind != vector::CombiningKind::MAXNUMF)
6813 auto vType = dyn_cast<VectorType>(op.getVector().getType());
6817 llvm::SmallSet<std::pair<unsigned, signed>, 16> laneSizeElWidthPairSet;
6818 laneSizeElWidthPairSet.insert({64, 8});
6819 laneSizeElWidthPairSet.insert({32, 16});
6820 laneSizeElWidthPairSet.insert({32, 32});
6821 laneSizeElWidthPairSet.insert({16, 32});
6823 Type scalarType = vType.getElementType();
6824 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6827 if (isa<IntegerType>(scalarType) &&
6828 !laneSizeElWidthPairSet.count(std::make_pair(laneSize, elWidth)))
6831 if (isa<FloatType>(scalarType) && laneSize != 16 && laneSize != 32)
6838 target.addDynamicallyLegalOp<arith::ExtFOp>([](arith::ExtFOp extfOp) {
6839 auto srcType = dyn_cast<VectorType>(extfOp.getIn().getType());
6840 auto dstType = dyn_cast<VectorType>(extfOp.getOut().getType());
6841 if (!srcType || !dstType)
6844 Type srcScalarType = srcType.getElementType();
6845 Type dstScalarType = dstType.getElementType();
6846 if (!isa<FloatType>(srcScalarType) || !isa<FloatType>(dstScalarType))
6851 return (srcLaneSize % 16 != 0) || (dstLaneSize % 16 != 0);
6855 target.addDynamicallyLegalOp<arith::TruncFOp>([](arith::TruncFOp truncfOp) {
6856 auto srcType = dyn_cast<VectorType>(truncfOp.getIn().getType());
6857 auto dstType = dyn_cast<VectorType>(truncfOp.getOut().getType());
6858 if (!srcType || !dstType)
6861 Type srcScalarType = srcType.getElementType();
6862 Type dstScalarType = dstType.getElementType();
6863 if (!isa<FloatType>(srcScalarType) || !isa<FloatType>(dstScalarType))
6868 return (srcLaneSize % 16 != 0) || (dstLaneSize % 16 != 0);
6872 target.addDynamicallyLegalOp<arith::ExtSIOp>([](arith::ExtSIOp extsiOp) {
6873 auto srcType = dyn_cast<VectorType>(extsiOp.getIn().getType());
6874 auto dstType = dyn_cast<VectorType>(extsiOp.getOut().getType());
6875 if (!srcType || !dstType)
6878 Type srcScalarType = srcType.getElementType();
6879 Type dstScalarType = dstType.getElementType();
6880 if (!isa<IntegerType>(srcScalarType) || !isa<IntegerType>(dstScalarType))
6885 return (srcLaneSize % 16 != 0) || (dstLaneSize % 16 != 0);
6889 target.addDynamicallyLegalOp<arith::TruncIOp>([](arith::TruncIOp trunciOp) {
6890 auto srcType = dyn_cast<VectorType>(trunciOp.getIn().getType());
6891 auto dstType = dyn_cast<VectorType>(trunciOp.getOut().getType());
6892 if (!srcType || !dstType) {
6894 return !(!srcType && !dstType && isSRSCompoundCandidate(trunciOp));
6896 Type srcScalarType = srcType.getElementType();
6897 Type dstScalarType = dstType.getElementType();
6898 if (!isa<IntegerType>(srcScalarType) || !isa<IntegerType>(dstScalarType))
6903 if (isSRSCompoundCandidate(trunciOp))
6908 return (srcLaneSize % 16 != 0) || (dstLaneSize % 16 != 0);
6911 target.addIllegalOp<vector::ContractionOp, vector::TransposeOp,
6916 target.addDynamicallyLegalOp<math::ExpOp>([](math::ExpOp expOp) {
6917 auto srcType = dyn_cast<VectorType>(expOp.getOperand().getType());
6921 Type scalarType = srcType.getElementType();
6922 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6925 if (!isa<FloatType>(scalarType) || (laneSize != 16 && laneSize != 32) ||
6928 if (expOp->hasOneUse() && isInSigmoidOperationChain(expOp))
6934 target.addDynamicallyLegalOp<math::RsqrtOp>([](math::RsqrtOp rsqrtOp) {
6935 auto srcType = dyn_cast<VectorType>(rsqrtOp.getOperand().getType());
6939 Type scalarType = srcType.getElementType();
6940 if (!isa<FloatType>(scalarType))
6944 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6945 return elWidth != 16 || (laneSize != 16 && laneSize != 32);
6968 StringRef
getArgument() const final {
return "test-lower-vector-to-aievec"; }
6970 return "Lower vector operations to AIE vector intrinsics";
6974 affine::AffineDialect, xilinx::aievec::aie1::AIEVecAIE1Dialect,
6975 xilinx::aievec::AIEVecDialect, arith::ArithDialect,
6976 memref::MemRefDialect, scf::SCFDialect, ub::UBDialect,
6977 vector::VectorDialect, emitc::EmitCDialect, xilinx::AIE::AIEDialect>();
6981 *
this,
"aie-target",
6983 "Select AIE version: \"aie\", \"aie2\", or \"aie2p\". This will "
6984 "determine the vector size and available operations."),
6985 llvm::cl::init(
"aie")};
6988 auto *op = getOperation();
6989 MLIRContext *context = &getContext();
6990 RewritePatternSet patterns(context);
6991 ConversionTarget target(*context);
6998 target.addLegalOp<xilinx::AIE::RuntimeSequenceOp>();
6999 target.markOpRecursivelyLegal<xilinx::AIE::RuntimeSequenceOp>();
7001 std::optional<AIEVecTargetInfo> targetInfo =
7004 return signalPassFailure();
7006 populateAIEVecCommonConversionPatterns(patterns);
7007 configureAIEVecCommonLegalizations(target);
7008 if (targetInfo->isAIE1()) {
7009 populateAIEVecV1ConversionPatterns(patterns);
7010 configureAIEVecV1Legalizations(target);
7011 }
else if (targetInfo->isAIE2()) {
7012 populateAIEVecV2ConversionPatterns(patterns);
7013 configureAIEVecV2Legalizations(target);
7014 }
else if (targetInfo->isAIE2P()) {
7015 auto planner = std::make_shared<AIE2PShufflePlanner>();
7016 populateAIEVecV2PConversionPatterns(patterns, planner);
7017 configureAIEVecV2Legalizations(target);
7018 configureAIEVecV2PLegalizations(target, planner);
7022 llvm_unreachable(
"AIE version is misconfigured");
7025 if (failed(applyPartialConversion(op, target, std::move(patterns))))
7026 return signalPassFailure();
7030static std::unique_ptr<Pass>
7032 return std::make_unique<LowerVectorToAIEVec>(options);
7046 MLIRContext *context = &getContext();
7047 RewritePatternSet patterns(context);
7048 ConversionTarget target(*context);
7050 target.addLegalDialect<aievec::AIEVecDialect>();
7051 target.addDynamicallyLegalOp<aievec::UPDOp>([](aievec::UPDOp op) {
7052 return op.getVector() ||
7053 (op->hasOneUse() && isa<aievec::UPDOp>(*op->getUsers().begin())) ||
7054 llvm::all_of(op->getUsers(),
7055 [](Operation *op) { return isa<aievec::ExtOp>(op); });
7058 if (
auto *op = getOperation();
7059 failed(applyPartialConversion(op, target, std::move(patterns)))) {
7060 return signalPassFailure();
7073 MLIRContext *context = &getContext();
7074 RewritePatternSet patterns(context);
7075 ConversionTarget target(*context);
7077 target.addLegalDialect<aievec::AIEVecDialect>();
7078 target.addDynamicallyLegalOp<aievec::ExtOp>([](aievec::ExtOp op) {
7079 auto *defOp = op.getSource().getDefiningOp();
7080 return !defOp || !isa<aievec::UPDOp>(defOp) || !defOp->hasOneUse() ||
7084 if (
auto *op = getOperation();
7085 failed(applyPartialConversion(op, target, std::move(patterns)))) {
7086 return signalPassFailure();
7098 pm.addPass(createLowerVectorToAIEVec(options));
7099 pm.addPass(createCanonicalizerPass());
7102 pm.addPass(std::make_unique<ExtendUPDOpsPass>());
7103 pm.addPass(createCSEPass());
7104 pm.addPass(std::make_unique<SimplifyUPDOpsPass>());
7105 pm.addPass(createCanonicalizerPass());
LowerVectorAddOrSubOpToAIEVecAddElemOrSubElemOp< arith::SubFOp, aievec::SubElemOp > LowerVectorSubFOpToAIEVecSubElemOp
LowerVectorMinMaxOpToAIEVecMinMaxOp< arith::MaxNumFOp, aievec::MaxOp > LowerVectorMaxNumFFOpToAIEVecMaxOp
LowerVectorCmpOpToAIEVecCmpOp< arith::CmpIOp, CmpIPredicate > LowerVectorCmpIOpToAIEVecCmpOp
ScalarizeVectorIntFpConversionOpPattern< arith::FPToUIOp > ScalarizeVectorFPToUIOpPattern
AIE2PElementwiseOps< arith::MulFOp, arith::DivFOp, arith::AddFOp, arith::SubFOp, arith::NegFOp, arith::TruncFOp, arith::ExtFOp, math::ExpOp, math::TanhOp, math::ErfOp, math::Exp2Op, arith::MaximumFOp, arith::MaxNumFOp, arith::MinimumFOp, arith::CmpFOp, arith::CmpIOp, arith::SelectOp, arith::AddIOp, arith::SubIOp, arith::MulIOp, arith::IndexCastOp > AIE2PElementwise
OneToOneVectorOpToAIEVecOpPattern< arith::SubIOp, aievec::aie1::SubOp > LowerVectorSubIOpToAIEVecSubOp
ScalarizeVectorIntFpConversionOpPattern< arith::FPToSIOp > ScalarizeVectorFPToSIOpPattern
OneToOneVectorOpToAIEVecOpPattern< arith::SubFOp, aievec::aie1::SubOp > LowerVectorSubFOpToAIEVecSubOp
LowerVectorUMinMaxOpToCmpSelOp< arith::MaxUIOp, arith::CmpIPredicate::ugt > LowerVectorMaxUIOpToCmpSelOp
ScalarizeVectorIntFpConversionOpPattern< arith::UIToFPOp > ScalarizeVectorUIToFPOpPattern
LowerVectorMinMaxOpToAIEVecMinMaxOp< arith::MaxSIOp, aievec::MaxOp > LowerVectorMaxSIOpToAIEVecMaxOp
LowerScalarMinMaxToAIEVecMinMaxOp< arith::MinSIOp, aievec::MinOp > LowerScalarMinSIOpToAIEVecMinOp
LowerVectorContractionOpToAIEVecMatMulPattern< aievec::MatMulOp_AIE2P > LowerVectorContractionOpToAIEVecMatMulOpAIE2P
OneToOneVectorOpToAIEVecOpPattern< arith::AddFOp, aievec::aie1::AddOp > LowerVectorAddFOpToAIEVecAddOp
ComputeAbsOpPattern< math::AbsIOp > ComputeAbsIOpPattern
constexpr int64_t kAIE2PMaxParts
LowerTruncOpPattern< arith::TruncFOp > LowerTruncFOpPattern
LowerVectorIntToFPI16BF16AIE2pPattern< arith::SIToFPOp > LowerVectorSIToFPI16BF16AIE2pPattern
ScalarizeVectorIntFpConversionOpPattern< arith::SIToFPOp > ScalarizeVectorSIToFPOpPattern
ComputeBandAndBorOpPattern< arith::OrIOp, aievec::BorOp > ComputeBorOpPattern
LowerVectorAddOrSubOpToAIEVecAddElemOrSubElemOp< arith::AddFOp, aievec::AddElemOp > LowerVectorAddFOpToAIEVecAddElemOp
LowerExtOpPattern< arith::ExtFOp > LowerExtFOpPattern
OneToOneVectorOpToAIEVecOpPattern< arith::MulFOp, aievec::aie1::MulOp > LowerVectorMulFOpToAIEVecMulOp
ComputeAbsOpPattern< math::AbsFOp > ComputeAbsFOpPattern
LowerVectorAddOrSubOpToAIEVecAddElemOrSubElemOp< arith::SubIOp, aievec::SubElemOp > LowerVectorSubIOpToAIEVecSubElemOp
LowerVectorIntToFPI16BF16AIE2pPattern< arith::UIToFPOp > LowerVectorUIToFPI16BF16AIE2pPattern
LowerVectorUMinMaxOpToCmpSelOp< arith::MinUIOp, arith::CmpIPredicate::ult > LowerVectorMinUIOpToCmpSelOp
ComputeBandAndBorOpPattern< arith::AndIOp, aievec::BandOp > ComputeBandOpPattern
LowerVectorContractionOpToAIEVecMatMulPattern< aievec::MatMulOp > LowerVectorContractionOpToAIEVecMatMulOpAIE2
LowerVectorMinMaxOpToAIEVecMinMaxOp< arith::MinimumFOp, aievec::MinOp > LowerVectorMinimumFOpToAIEVecMinOp
LowerVectorCmpOpToAIEVecCmpOp< arith::CmpFOp, CmpFPredicate > LowerVectorCmpFOpToAIEVecCmpOp
LowerTruncOpPattern< arith::TruncIOp > LowerTruncIOpPattern
LowerExtOpPattern< arith::ExtSIOp > LowerExtSIOpPattern
LowerVectorAddOrSubOpToAIEVecAddElemOrSubElemOp< arith::AddIOp, aievec::AddElemOp > LowerVectorAddIOpToAIEVecAddElemOp
LowerVectorMinMaxOpToAIEVecMinMaxOp< arith::MaximumFOp, aievec::MaxOp > LowerVectorMaximumFOpToAIEVecMaxOp
LowerScalarMinMaxToAIEVecMinMaxOp< arith::MaxSIOp, aievec::MaxOp > LowerScalarMaxSIOpToAIEVecMaxOp
LowerVectorMinMaxOpToAIEVecMinMaxOp< arith::MinSIOp, aievec::MinOp > LowerVectorMinSIOpToAIEVecMinOp
mlir::VectorType getFlattenedVectorType(mlir::VectorType vecTy)
std::optional< MatMulRecipe > checkMatMulTile(bool aie2p, mlir::VectorType lhsTy, mlir::VectorType rhsTy, mlir::VectorType accTy, bool lhsTransposed, bool rhsTransposed, llvm::function_ref< mlir::InFlightDiagnostic()> emitError=nullptr)
unsigned getVectorLaneSize(mlir::VectorType type)
SmallVector< NamedAttribute > buildFMAOpSplatAttrForElemTy(aievec::aie1::FMAOp fmaOp, int64_t bcastPos, int64_t step=1)
std::optional< int64_t > getTransferReadAlignmentOffset(TransferReadLikeOp readOp, mlir::VectorType vType, int64_t alignment)
void populateAIE2PShufflePlannerPatterns(mlir::RewritePatternSet &patterns, std::shared_ptr< AIE2PShufflePlanner > planner)
std::optional< AIEVecTargetInfo > parseAIEVecLoweringTarget(llvm::StringRef spelling, mlir::Operation *op)
mlir::VectorType createVectorType(unsigned lanes, mlir::Type elementType)
int32_t getElementSizeInBits(mlir::VectorType type)
void buildLowerVectorToAIEVec(mlir::OpPassManager &pm, const LowerVectorToAIEVecOptions &options)
mlir::VectorType getVectorOpDestType(mlir::VectorType type, bool AIE2)
void configureAIE2PShufflePlannerLegality(mlir::ConversionTarget &target, std::shared_ptr< AIE2PShufflePlanner > planner)
static void markIllegal(ConversionTarget &target, MLIRContext *ctx)
static void addPatterns(RewritePatternSet &patterns, PatternBenefit benefit)
static void markIllegal(ConversionTarget &target, MLIRContext *ctx)
LogicalResult matchAndRewrite(SrcOpTy absOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
typename SrcOpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
typename SrcOpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(arith::XOrIOp xorOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::CeilOp ceilOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::ErfOp erfOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::ErfOp erfOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::ExpOp expOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::ExpOp expOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::FloorOp floorOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::NegFOp negOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(OpTy op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
typename OpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(math::RsqrtOp rsqrtOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::RsqrtOp rsqrtOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(OpTy op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
typename OpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(arith::DivFOp divfOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::ShRSIOp rsOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::SqrtOp sqrtOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::TanhOp tanhOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::DivFOp divOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::ExpOp expOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
typename MathOpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(MathOpTy op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::AddFOp addOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
ConvertMulAddFToAIEVecFMAElemOpPattern(MLIRContext *context, unsigned shiftParam=0)
LogicalResult matchAndRewrite(arith::AddIOp addOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
ConvertMulAddToAIEVecFMAElemOpPattern(MLIRContext *context, unsigned shiftParam=0)
LogicalResult matchAndRewrite(aievec::aie1::AddOp addOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::MulFOp mulOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::MulFOp mulOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
ConvertMulFToAIEVecMulElemOpPattern(MLIRContext *context, unsigned shiftParam=0)
LogicalResult matchAndRewrite(arith::MulIOp mulOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
ConvertMulIToAIEVecMulElemOpPattern(MLIRContext *context, unsigned shiftParam=0)
LogicalResult matchAndRewrite(vector::BroadcastOp bcastOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::BroadcastOp bcastOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::FMAOp fmaOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::FMAOp fmaOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
ConvertVectorFMAOpToAIEVecFMAElemOpPattern(MLIRContext *context, unsigned shiftParam=0)
ExpandUPDToUPDAndExtPattern(MLIRContext *context)
LogicalResult matchAndRewrite(aievec::UPDOp updOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
void runOnOperation() override
typename OpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(OpTy op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::aie1::FMAOp fmaOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::BroadcastOp bcastOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::ExtOp extOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
FuseExtIntoUPDPattern(MLIRContext *context)
typename SrcOpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(SrcOpTy extOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::ExtUIOp extOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
typename SrcOpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::ShRSIOp rsOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::TruncIOp truncOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(SrcOpTy truncOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
typename SrcOpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(arith::AddIOp addOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
typename SrcOpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
typename SrcOpTy::Adaptor OpAdaptor
static Value reshape(OpBuilder &b, Value v, ArrayRef< int64_t > shape)
LogicalResult matchAndRewrite(vector::ContractionOp contractOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
static constexpr bool isAIE2P
Value emitMagicSiToFp(ConversionPatternRewriter &rewriter, Location loc, Value srcI16, unsigned lanes) const
static constexpr bool isSigned
typename SrcOpTy::Adaptor OpAdaptor
LowerVectorIntToFPI16BF16AIE2pPattern(MLIRContext *context, bool accumulatorAdd, PatternBenefit benefit)
LogicalResult matchAndRewrite(SrcOpTy op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
typename SrcOpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(arith::MulIOp mulOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::ReductionOp srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::ReductionOp srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::ReductionOp srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::ReductionOp srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::ReductionOp srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::ReductionOp srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::SelectOp srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
void getDependentDialects(DialectRegistry ®istry) const override
LowerVectorToAIEVec(const LowerVectorToAIEVecOptions &options)
void runOnOperation() override
StringRef getDescription() const final
Option< std::string > aieTarget
LowerVectorToAIEVec()=default
StringRef getArgument() const final
LogicalResult matchAndRewrite(vector::TransferReadOp readOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LowerVectorTransferReadToAIEUPD(MLIRContext *context, int64_t minVectorSize, int64_t maxVectorSize, int64_t alignment, int64_t maxLoadSize)
LogicalResult matchAndRewrite(vector::TransposeOp transpOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
typename SrcOpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
typename OpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(OpTy op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
typename SrcOpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
typename SrcOpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(SrcOpTy op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
ShiftClampTruncToSRSPattern(MLIRContext *context, PatternBenefit benefit=2)
static std::optional< Value > getShiftValue(Value rhs, ConversionPatternRewriter &rewriter, Location loc)
static std::optional< int64_t > getConstantSplatValue(Value val)
LogicalResult matchAndRewrite(arith::TruncIOp truncOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
void runOnOperation() override
Options for the "lower-vector-to-aievec" pipeline.
PassOptions::Option< std::string > aieTarget