15#include "mlir/Conversion/AffineToStandard/AffineToStandard.h"
16#include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h"
17#include "mlir/Dialect/Affine/IR/AffineOps.h"
18#include "mlir/Dialect/Func/IR/FuncOps.h"
19#include "mlir/Dialect/Math/IR/Math.h"
20#include "mlir/Dialect/MemRef/IR/MemRef.h"
21#include "mlir/Dialect/SCF/IR/SCF.h"
22#include "mlir/Dialect/UB/IR/UBOps.h"
23#include "mlir/Dialect/Utils/ReshapeOpsUtils.h"
24#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
25#include "mlir/IR/Matchers.h"
26#include "mlir/IR/PatternMatch.h"
27#include "mlir/Pass/PassManager.h"
28#include "mlir/Transforms/DialectConversion.h"
29#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
30#include "llvm/ADT/STLExtras.h"
34#define DEBUG_TYPE "aievec-canonicalization"
55 :
public OpConversionPattern<vector::TransferReadOp> {
56 using OpConversionPattern<vector::TransferReadOp>::OpConversionPattern;
60 : OpConversionPattern<
vector::TransferReadOp>(context),
65 ConversionPatternRewriter &rewriter)
const override {
67 if (adaptor.getPermutationMap().isConstant())
71 auto vType = readOp.getVectorType();
79 auto vLen = vType.getShape().back();
80 auto longVecTy = VectorType::get(2 * vLen, vType.getElementType());
88 auto loc = readOp.getLoc();
89 Value oldInnerMostIdx = adaptor.getIndices().back();
90 auto offsetCorrectionMap =
91 AffineMap::get(1, 0, getAffineDimExpr(0, readOp.getContext()) - offset);
92 Value newInnerMostIdx = affine::AffineApplyOp::create(
93 rewriter, readOp.getLoc(), offsetCorrectionMap,
94 SmallVector<Value, 1>({oldInnerMostIdx}))
96 SmallVector<Value, 8> alignedIdx;
97 alignedIdx.append(adaptor.getIndices().begin(), adaptor.getIndices().end());
98 alignedIdx[alignedIdx.size() - 1] = newInnerMostIdx;
102 auto newReadOp = vector::TransferReadOp::create(
103 rewriter, loc, longVecTy, adaptor.getBase(), alignedIdx,
104 adaptor.getPadding());
107 rewriter.replaceOpWithNewOp<vector::ExtractStridedSliceOp>(
108 readOp, newReadOp.getResult(), offset, vLen, 1);
122 :
public OpConversionPattern<vector::TransferReadOp> {
123 using OpConversionPattern<vector::TransferReadOp>::OpConversionPattern;
126 : OpConversionPattern<
vector::TransferReadOp>(context) {}
130 ConversionPatternRewriter &rewriter)
const override {
131 AffineMap map = readOp.getPermutationMap();
132 if (!map.isConstant())
135 Value srcMemRef = adaptor.getBase();
136 SmallVector<Value, 8> indices;
140 if (cast<MemRefType>(srcMemRef.getType()).getRank() == 0) {
141 srcMemRef = memref::ExpandShapeOp::create(
142 rewriter, readOp.getLoc(), SmallVector<int64_t, 1>({1}),
143 srcMemRef, SmallVector<ReassociationIndices, 1>({}))
145 newIdx = arith::ConstantOp::create(rewriter, readOp.getLoc(),
146 rewriter.getIndexAttr(0L));
147 indices.push_back(newIdx);
149 indices.append(adaptor.getIndices().begin(), adaptor.getIndices().end());
150 newIdx = indices[indices.size() - 1];
154 if (
auto applyOp = newIdx.getDefiningOp<affine::AffineApplyOp>())
155 if (applyOp.getAffineMap().getNumDims() == 1) {
156 newIdx = applyOp.getMapOperands()[0];
157 offset = applyOp.getAffineMap().compose(ArrayRef<int64_t>{0})[0];
161 int64_t vlen = readOp.getVector().getType().getShape()[0];
162 if (offset >= vlen) {
165 int64_t numElemsToSkip = vlen * (offset / vlen);
166 offset = offset % vlen;
167 auto newAddrMap = AffineMap::get(
168 1, 0, getAffineDimExpr(0, readOp.getContext()) + numElemsToSkip);
170 affine::AffineApplyOp::create(rewriter, readOp.getLoc(), newAddrMap,
171 SmallVector<Value, 1>({newIdx}))
174 indices[indices.size() - 1] = newIdx;
175 auto newReadOp = vector::TransferReadOp::create(
176 rewriter, readOp.getLoc(), readOp.getVector().getType(), srcMemRef,
177 indices, adaptor.getPadding());
178 auto extractOp = vector::ExtractOp::create(rewriter, readOp.getLoc(),
179 newReadOp.getResult(),
180 ArrayRef<int64_t>{offset});
181 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(
182 readOp, newReadOp.getVector().getType(), extractOp.getResult());
190template <
class UnaryOpA,
class UnaryOpB>
203 PatternRewriter &rewriter)
const override {
205 UnaryOpA::template hasTrait<OpTrait::OneOperand>(),
206 "SwapUnaryOps can only be instantiated for single-operand ops");
208 UnaryOpB::template hasTrait<OpTrait::OneOperand>(),
209 "SwapUnaryOps can only be instantiated for single-operand ops");
210 UnaryOpA aOp = bOp.getOperand().template getDefiningOp<UnaryOpA>();
212 return rewriter.notifyMatchFailure(bOp, UnaryOpB::getOperationName() +
213 " not preceeded by " +
214 UnaryOpA::getOperationName());
219 UnaryOpB::create(rewriter, bOp->getLoc(), SmallVector<Type>({newA2BTy}),
220 aOp->getOperands(), bOp->getAttrs());
221 auto newB = UnaryOpA::create(rewriter, bOp->getLoc(),
222 SmallVector<Type>({bOp.getResult().getType()}),
223 newA->getResults(), aOp->getAttrs());
224 rewriter.replaceOp(bOp, newB.getResult());
229static SmallVector<Value> collapseInnerMostDimIndices(PatternRewriter &b,
230 Location loc,
int numDims,
232 ArrayRef<int64_t> shape,
235 auto newIdxExpr = b.getAffineDimExpr(numDims - 1);
237 for (int64_t dim = numDims - 2; dim >= 0; dim--) {
238 stride *= shape[shape.size() - (numDims - dim - 1)];
239 newIdxExpr = newIdxExpr + b.getAffineDimExpr(dim) * stride;
241 auto newIndexMap = AffineMap::get(numDims, 0, newIdxExpr);
242 Value newInnerMostIdxValue =
243 affine::AffineApplyOp::create(b, loc, newIndexMap,
244 indices.take_back(numDims))
246 SmallVector<Value> newIdxRange;
247 for (
auto idx : indices.drop_back(numDims))
248 newIdxRange.push_back(idx);
249 newIdxRange.push_back(newInnerMostIdxValue);
253static Value collapseInnerMostShapeDims(PatternRewriter &b, Location loc,
254 int numDims, Value val) {
255 auto memRefTy = cast<MemRefType>(val.getType());
256 auto shape = memRefTy.getShape();
257 int64_t newInnerMostDim = std::accumulate(shape.end() - numDims, shape.end(),
258 1, std::multiplies<>());
259 SmallVector<int64_t, 4> newShape{shape.begin(), shape.end() - numDims + 1};
260 newShape[shape.size() - numDims] = newInnerMostDim;
263 auto reassocIndicesOpt = getReassociationIndicesForCollapse(shape, newShape);
264 assert(reassocIndicesOpt &&
"newShape must be a valid collapse of shape");
265 const auto &reassocIndices = *reassocIndicesOpt;
269 memref::CollapseShapeOp::computeCollapsedType(memRefTy, reassocIndices);
270 return memref::CollapseShapeOp::create(b, loc, newMemRefTy, val,
278static bool hasContiguousRowMajorStrides(MemRefType memRefTy) {
279 SmallVector<int64_t> strides;
281 if (failed(memRefTy.getStridesAndOffset(strides, offset)))
283 auto shape = memRefTy.getShape();
284 int64_t expected = 1;
285 for (
int i = shape.size() - 1; i >= 0; --i) {
286 if (strides[i] != ShapedType::kDynamic && strides[i] != expected)
288 if (shape[i] != ShapedType::kDynamic)
289 expected *= shape[i];
291 expected = ShapedType::kDynamic;
300 :
public OpConversionPattern<vector::TransferReadOp> {
301 using OpConversionPattern<vector::TransferReadOp>::OpConversionPattern;
305 ConversionPatternRewriter &rewriter)
const override {
308 if (!adaptor.getPermutationMap().isMinorIdentity() || adaptor.getMask())
310 VectorType vectorTy = readOp.getVector().getType();
311 if (vectorTy.getRank() < 2)
314 MemRefType memRefTy = dyn_cast<MemRefType>(adaptor.getBase().getType());
317 auto memRefShape = memRefTy.getShape();
318 auto vecShape = vectorTy.getShape();
321 VectorType::get({std::accumulate(vecShape.begin(), vecShape.end(), 1,
322 std::multiplies<>())},
323 vectorTy.getElementType());
324 if (!hasContiguousRowMajorStrides(memRefTy))
327 AffineMap layout = memRefTy.getLayout().getAffineMap();
329 collapseInnerMostDimIndices(rewriter, readOp.getLoc(), vecShape.size(),
330 adaptor.getIndices(), memRefShape, layout);
331 auto newSource = collapseInnerMostShapeDims(
332 rewriter, readOp.getLoc(), vecShape.size(), adaptor.getBase());
333 auto newVector = vector::TransferReadOp::create(
334 rewriter, readOp.getLoc(), newVectorTy, newSource, newIndices,
336 arith::getZeroConstant(rewriter, readOp.getLoc(),
337 newVectorTy.getElementType()));
339 auto inBoundsArrayAttrOpt = adaptor.getInBounds();
340 if (inBoundsArrayAttrOpt) {
341 SmallVector<bool> inBounds =
342 llvm::to_vector(inBoundsArrayAttrOpt.getAsValueRange<BoolAttr>());
343 SmallVector<bool> newInBounds({
false});
344 newInBounds[0] = llvm::all_of(inBounds, [](
bool v) {
return v; });
345 newVector.getProperties().setInBounds(
346 rewriter.getBoolArrayAttr(newInBounds));
349 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(readOp, vectorTy,
360 :
public OpConversionPattern<vector::TransferWriteOp> {
361 using OpConversionPattern<vector::TransferWriteOp>::OpConversionPattern;
365 ConversionPatternRewriter &rewriter)
const override {
368 if (!adaptor.getPermutationMap().isMinorIdentity() || adaptor.getMask())
370 VectorType vectorTy = cast<VectorType>(adaptor.getValueToStore().getType());
371 if (vectorTy.getRank() < 2)
374 MemRefType memRefTy = dyn_cast<MemRefType>(adaptor.getBase().getType());
377 auto memRefShape = memRefTy.getShape();
378 auto vecShape = vectorTy.getShape();
380 if (!hasContiguousRowMajorStrides(memRefTy))
384 VectorType::get({std::accumulate(vecShape.begin(), vecShape.end(), 1,
385 std::multiplies<>())},
386 vectorTy.getElementType());
387 AffineMap layout = memRefTy.getLayout().getAffineMap();
389 vector::ShapeCastOp::create(rewriter, writeOp.getLoc(), newVectorTy,
390 adaptor.getValueToStore())
393 collapseInnerMostDimIndices(rewriter, writeOp.getLoc(), vecShape.size(),
394 adaptor.getIndices(), memRefShape, layout);
395 auto newSource = collapseInnerMostShapeDims(
396 rewriter, writeOp.getLoc(), vecShape.size(), adaptor.getBase());
398 auto newOp = rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
399 writeOp, newVector, newSource, newIndices);
401 auto inBoundsArrayAttrOpt = adaptor.getInBounds();
402 if (inBoundsArrayAttrOpt) {
403 SmallVector<bool> inBounds =
404 llvm::to_vector(inBoundsArrayAttrOpt.getAsValueRange<BoolAttr>());
405 SmallVector<bool> newInBounds({
false});
406 newInBounds[0] = llvm::all_of(inBounds, [](
bool v) {
return v; });
407 newOp.getProperties().setInBounds(rewriter.getBoolArrayAttr(newInBounds));
416static LogicalResult isAllZeroOffsetAccess(mlir::OperandRange indices) {
417 if (!llvm::all_of(indices, [](Value val) {
419 if (!matchPattern(val, m_Constant(&attr)))
421 return attr.getInt() == 0;
430static SmallVector<Value> opFoldResultsToValues(PatternRewriter &rewriter,
432 memref::SubViewOp subViewOp) {
433 OpBuilder::InsertionGuard g(rewriter);
434 rewriter.setInsertionPoint(subViewOp);
435 SmallVector<Value> newIndices;
436 for (OpFoldResult offset : subViewOp.getMixedOffsets()) {
438 if (
auto attr = dyn_cast<Attribute>(offset)) {
439 indexVal = arith::ConstantIndexOp::create(
440 rewriter, loc, cast<IntegerAttr>(attr).getInt());
442 indexVal = cast<Value>(offset);
444 newIndices.push_back(indexVal);
469 PatternRewriter &rewriter)
const override {
471 auto subViewOp = dyn_cast_if_present<memref::SubViewOp>(
472 readOp.getBase().getDefiningOp());
477 if (failed(isAllZeroOffsetAccess(readOp.getIndices())))
481 SmallVector<Value> newIndices =
482 opFoldResultsToValues(rewriter, readOp.getLoc(), subViewOp);
485 rewriter.replaceOpWithNewOp<vector::TransferReadOp>(
486 readOp, readOp.getType(), subViewOp.getSource(), newIndices,
487 readOp.getPadding(), readOp.getInBoundsValues());
513 PatternRewriter &rewriter)
const override {
515 auto subViewOp = dyn_cast_if_present<memref::SubViewOp>(
516 writeOp.getBase().getDefiningOp());
521 if (failed(isAllZeroOffsetAccess(writeOp.getIndices())))
525 SmallVector<Value> newIndices =
526 opFoldResultsToValues(rewriter, writeOp.getLoc(), subViewOp);
529 vector::TransferWriteOp::create(rewriter, writeOp.getLoc(),
530 writeOp.getVector(), subViewOp.getSource(),
531 newIndices, writeOp.getInBoundsValues());
534 rewriter.eraseOp(writeOp);
546 using OpRewritePattern::OpRewritePattern;
549 PatternRewriter &rewriter)
const override {
550 for (OpOperand &init : forOp.getInitArgsMutable()) {
551 auto vecType = dyn_cast<VectorType>(init.get().getType());
552 if (!vecType || vecType.getRank() < 2 || vecType.isScalable())
554 Value flat = vector::ShapeCastOp::create(rewriter, forOp.getLoc(),
557 auto shapeCast = [](OpBuilder &builder, Location loc, Type type,
558 Value source) -> Value {
559 return vector::ShapeCastOp::create(builder, loc, type, source);
561 rewriter.replaceOp(forOp, scf::replaceAndCastForOpIterArg(
562 rewriter, forOp, init, flat, shapeCast));
579 PatternRewriter &rewriter)
const override {
580 auto insSrcTy = dyn_cast<VectorType>(insOp.getValueToStoreType());
584 auto srcShape = insSrcTy.getShape();
585 auto dstShape = insOp.getDestVectorType().getShape();
587 unsigned long numLeadUnitDimDst = 0;
588 while (numLeadUnitDimDst < dstShape.size() &&
589 dstShape[numLeadUnitDimDst] == 1)
592 if (!numLeadUnitDimDst)
595 unsigned long numLeadUnitDimSrc = 0;
596 while (numLeadUnitDimSrc < srcShape.size() &&
597 srcShape[numLeadUnitDimSrc] == 1)
600 SmallVector<int64_t> nonLeadUnitDimDstShape(
601 dstShape.begin() + numLeadUnitDimDst, dstShape.end());
602 SmallVector<int64_t> nonLeadUnitDimSrcShape(
603 srcShape.begin() + numLeadUnitDimSrc, srcShape.end());
605 if (nonLeadUnitDimSrcShape != nonLeadUnitDimDstShape)
608 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(
609 insOp, insOp.getDestVectorType(), insOp.getValueToStore());
618configureCommonAIECanonicalizeLegalizations(ConversionTarget &target) {
619 target.addLegalDialect<arith::ArithDialect, affine::AffineDialect,
620 memref::MemRefDialect, vector::VectorDialect,
625populateCommonAIECanonicalizeConversionPatterns(RewritePatternSet &patterns) {
627 patterns.getContext());
634static void configureAIEv1CanonicalizeLegalizations(ConversionTarget &target) {
635 target.addDynamicallyLegalOp<vector::TransferReadOp>(
636 [](vector::TransferReadOp op) {
637 return !op.getPermutationMap().isConstant() &&
644populateAIEv1CanonicalizeConversionPatterns(RewritePatternSet &patterns) {
653static void configureAIE2CanonicalizeLegalizations(ConversionTarget &target) {
654 target.addDynamicallyLegalOp<vector::TransferReadOp>(
655 [](vector::TransferReadOp op) {
656 return !op.getPermutationMap().isConstant() &&
659 op.getVector().getType().getRank() < 2;
661 target.addDynamicallyLegalOp<vector::TransferWriteOp>(
662 [](vector::TransferWriteOp op) {
663 return cast<VectorType>(op.getVector().getType()).getRank() < 2;
668populateAIE2CanonicalizeConversionPatterns(RewritePatternSet &patterns) {
685static Value smartTruncF32ToBF16(PatternRewriter &rewriter, Location loc,
686 Value val, Type bf16Type) {
687 if (
auto extfOp = val.getDefiningOp<arith::ExtFOp>()) {
688 if (extfOp.getIn().getType() == bf16Type)
689 return extfOp.getIn();
691 return arith::TruncFOp::create(rewriter, loc, bf16Type, val);
695static APFloat roundedToBF16(APFloat v) {
696 bool losesInfo =
false;
697 const llvm::fltSemantics &sem = v.getSemantics();
698 v.convert(APFloat::BFloat(), APFloat::rmNearestTiesToEven, &losesInfo);
699 v.convert(sem, APFloat::rmNearestTiesToEven, &losesInfo);
704static bool holdsBF16(Value val) {
706 if (
auto bcastOp = val.getDefiningOp<vector::BroadcastOp>())
707 val = bcastOp.getSource();
708 if (
auto extfOp = val.getDefiningOp<arith::ExtFOp>())
709 return getElementTypeOrSelf(extfOp.getIn().getType()).isBF16();
710 DenseFPElementsAttr dense;
711 return matchPattern(val, m_Constant(&dense)) &&
712 llvm::all_of(dense.getValues<APFloat>(), [](
const APFloat &v) {
713 return roundedToBF16(v).bitwiseIsEqual(v);
718static Value roundToBF16(PatternRewriter &rewriter, Location loc, Value val) {
721 auto vecType = cast<VectorType>(val.getType());
722 DenseFPElementsAttr dense;
723 if (matchPattern(val, m_Constant(&dense))) {
724 SmallVector<APFloat> values;
725 for (
const APFloat &v : dense.getValues<APFloat>())
726 values.push_back(roundedToBF16(v));
727 return arith::ConstantOp::create(rewriter, loc,
728 DenseElementsAttr::get(vecType, values));
731 VectorType::get(vecType.getShape(), rewriter.getBF16Type());
732 Value bf16 = smartTruncF32ToBF16(rewriter, loc, val, bf16VecType);
733 return arith::ExtFOp::create(rewriter, loc, vecType, bf16,
737static bool isF32Vector(Type type) {
738 auto vecType = dyn_cast<VectorType>(type);
739 return vecType && vecType.getElementType().isF32();
750 using OpRewritePattern::OpRewritePattern;
753 PatternRewriter &rewriter)
const override {
754 auto resultType = dyn_cast<VectorType>(op.getType());
755 auto intermediateType = dyn_cast<VectorType>(op.getIn().getType());
756 if (!resultType || !intermediateType ||
757 !resultType.getElementType().isBF16() ||
758 !intermediateType.getElementType().isF32())
761 auto extfOp = op.getIn().getDefiningOp<arith::ExtFOp>();
762 if (!extfOp || extfOp.getIn().getType() != resultType)
765 rewriter.replaceOp(op, extfOp.getIn());
777template <
typename OpTy>
782 PatternRewriter &rewriter)
const override {
783 auto resultType = dyn_cast<VectorType>(op.getType());
784 if (!resultType || !resultType.getElementType().isF32())
787 Location loc = op.getLoc();
789 VectorType::get(resultType.getShape(), rewriter.getBF16Type());
792 smartTruncF32ToBF16(rewriter, loc, op.getLhs(), bf16VecType);
794 smartTruncF32ToBF16(rewriter, loc, op.getRhs(), bf16VecType);
797 OpTy::create(rewriter, loc, bf16VecType, lhsBF16, rhsBF16);
798 auto extOp = arith::ExtFOp::create(rewriter, loc, resultType, newResult,
800 rewriter.replaceOp(op, extOp);
808 using OpRewritePattern::OpRewritePattern;
811 PatternRewriter &rewriter)
const override {
812 auto lhsType = dyn_cast<VectorType>(op.getLhs().getType());
813 if (!lhsType || !lhsType.getElementType().isF32())
816 Location loc = op.getLoc();
818 VectorType::get(lhsType.getShape(), rewriter.getBF16Type());
821 smartTruncF32ToBF16(rewriter, loc, op.getLhs(), bf16VecType);
823 smartTruncF32ToBF16(rewriter, loc, op.getRhs(), bf16VecType);
825 rewriter.replaceOpWithNewOp<arith::CmpFOp>(op, op.getPredicate(), lhsBF16,
835 using OpRewritePattern::OpRewritePattern;
838 PatternRewriter &rewriter)
const override {
839 auto resultType = dyn_cast<VectorType>(op.getType());
840 if (!resultType || !resultType.getElementType().isF32())
843 Location loc = op.getLoc();
845 VectorType::get(resultType.getShape(), rewriter.getBF16Type());
848 smartTruncF32ToBF16(rewriter, loc, op.getTrueValue(), bf16VecType);
850 smartTruncF32ToBF16(rewriter, loc, op.getFalseValue(), bf16VecType);
852 Value newResult = arith::SelectOp::create(rewriter, loc, op.getCondition(),
853 trueValBF16, falseValBF16);
854 auto extOp = arith::ExtFOp::create(rewriter, loc, resultType, newResult,
856 rewriter.replaceOp(op, extOp);
868 using OpRewritePattern::OpRewritePattern;
871 PatternRewriter &rewriter)
const override {
872 if (!isF32Vector(op.getType()) ||
873 (holdsBF16(op.getLhs()) && holdsBF16(op.getRhs())))
875 Location loc = op.getLoc();
876 Value lhs = roundToBF16(rewriter, loc, op.getLhs());
877 Value rhs = roundToBF16(rewriter, loc, op.getRhs());
878 rewriter.replaceOpWithNewOp<arith::MulFOp>(op, lhs, rhs,
879 op.getFastmathAttr());
884static bool contracts(arith::FastMathFlagsAttr flags) {
886 bitEnumContainsAll(flags.getValue(), arith::FastMathFlags::contract);
891static bool fusesIntoFMA(arith::AddFOp op) {
892 auto type = dyn_cast<VectorType>(op.getType());
893 return isF32Vector(op.getType()) && type.getRank() == 1 &&
894 (type.getNumElements() == 16 || type.getNumElements() == 32) &&
895 contracts(op.getFastmathAttr());
902 using OpRewritePattern::OpRewritePattern;
905 PatternRewriter &rewriter)
const override {
906 if (!fusesIntoFMA(op))
908 for (
auto [mulOperand, acc] : {std::pair(op.getLhs(), op.getRhs()),
909 std::pair(op.getRhs(), op.getLhs())}) {
910 auto mulOp = mulOperand.getDefiningOp<arith::MulFOp>();
911 if (!mulOp || !mulOp->hasOneUse() || !contracts(mulOp.getFastmathAttr()))
913 Location loc = op.getLoc();
914 Value lhs = roundToBF16(rewriter, loc, mulOp.getLhs());
915 Value rhs = roundToBF16(rewriter, loc, mulOp.getRhs());
916 rewriter.replaceOpWithNewOp<vector::FMAOp>(op, lhs, rhs, acc);
917 rewriter.eraseOp(mulOp);
928 using OpRewritePattern::OpRewritePattern;
931 PatternRewriter &rewriter)
const override {
932 if (!fusesIntoFMA(op))
934 for (
auto [mulOperand, acc] : {std::pair(op.getLhs(), op.getRhs()),
935 std::pair(op.getRhs(), op.getLhs())}) {
936 auto mulOp = mulOperand.getDefiningOp<arith::MulFOp>();
937 if (!mulOp || !mulOp->hasOneUse() ||
938 !contracts(mulOp.getFastmathAttr()) || !holdsBF16(mulOp.getLhs()) ||
939 !holdsBF16(mulOp.getRhs()))
941 rewriter.replaceOpWithNewOp<vector::FMAOp>(op, mulOp.getLhs(),
942 mulOp.getRhs(), acc);
943 rewriter.eraseOp(mulOp);
953 using OpRewritePattern::OpRewritePattern;
956 PatternRewriter &rewriter)
const override {
957 if (!isF32Vector(op.getType()) ||
958 (holdsBF16(op.getLhs()) && holdsBF16(op.getRhs())))
960 Location loc = op.getLoc();
961 Value lhs = roundToBF16(rewriter, loc, op.getLhs());
962 Value rhs = roundToBF16(rewriter, loc, op.getRhs());
963 rewriter.replaceOpWithNewOp<vector::FMAOp>(op, lhs, rhs, op.getAcc());
969template <
typename OpTy>
974 PatternRewriter &rewriter)
const override {
975 auto resultType = dyn_cast<VectorType>(op.getType());
976 if (!resultType || !resultType.getElementType().isF32())
979 Location loc = op.getLoc();
981 VectorType::get(resultType.getShape(), rewriter.getBF16Type());
984 smartTruncF32ToBF16(rewriter, loc, op->getOperand(0), bf16VecType);
986 Value newResult = OpTy::create(rewriter, loc, bf16VecType, inputBF16);
987 auto extOp = arith::ExtFOp::create(rewriter, loc, resultType, newResult,
989 rewriter.replaceOp(op, extOp);
995 :
public PassWrapper<BF16EmulationPass, OperationPass<>> {
998 auto *op = getOperation();
999 MLIRContext *context = &getContext();
1000 RewritePatternSet patterns(context);
1026 (void)applyPatternsGreedily(op, std::move(patterns));
1031 return std::make_unique<BF16EmulationPass>();
1039 using OpRewritePattern::OpRewritePattern;
1042 PatternRewriter &rewriter)
const override {
1043 auto srcType = dyn_cast<VectorType>(op.getSourceType());
1044 VectorType resType = op.getResultVectorType();
1045 if (!srcType || srcType.getRank() != 1 || resType.getRank() < 2 ||
1046 resType.getShape().back() != srcType.getNumElements() ||
1047 !srcType.getElementType().isIntOrFloat() || resType.isScalable())
1049 int64_t n = srcType.getNumElements();
1050 SmallVector<int64_t> mask;
1051 for (int64_t i = 0; i < resType.getNumElements(); ++i)
1052 mask.push_back(i % n);
1053 Value src = op.getSource();
1055 vector::ShuffleOp::create(rewriter, op.getLoc(), src, src, mask);
1056 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, resType, flat);
1062 :
public PassWrapper<VectorBroadcastLoweringPass, OperationPass<>> {
1066 auto *op = getOperation();
1067 MLIRContext *context = &getContext();
1068 RewritePatternSet patterns(context);
1069 populateVectorBroadcastLoweringPatterns(patterns);
1071 patterns.getContext());
1077 (void)applyPatternsGreedily(op, std::move(patterns));
1084static std::unique_ptr<::mlir::Pass>
1085createVectorBroadcastLoweringPass(
bool aie2p =
false) {
1086 return std::make_unique<VectorBroadcastLoweringPass>(aie2p);
1097 :
public PassWrapper<CanonicalizeVectorForAIEVecPass, OperationPass<>> {
1113 return "test-canonicalize-vector-for-aievec";
1117 return "Canonicalize vector operations for AIEVec conversion";
1122 .insert<arith::ArithDialect, memref::MemRefDialect,
1123 vector::VectorDialect, affine::AffineDialect, ub::UBDialect>();
1127 *
this,
"aie-target",
1129 "Select AIE version: \"aie\", \"aie2\", or \"aie2p\". This will "
1130 "determine the vector size and available operations."),
1131 llvm::cl::init(
"aie")};
1134 auto *op = getOperation();
1135 MLIRContext *context = &getContext();
1136 RewritePatternSet patterns(context);
1137 ConversionTarget target(*context);
1139 std::optional<AIEVecTargetInfo> targetInfo =
1142 signalPassFailure();
1146 populateCommonAIECanonicalizeConversionPatterns(patterns);
1147 configureCommonAIECanonicalizeLegalizations(target);
1148 if (targetInfo->isAIE1()) {
1149 populateAIEv1CanonicalizeConversionPatterns(patterns);
1150 configureAIEv1CanonicalizeLegalizations(target);
1152 populateAIE2CanonicalizeConversionPatterns(patterns);
1153 configureAIE2CanonicalizeLegalizations(target);
1157 RewritePatternSet patterns(context);
1160 if (targetInfo->isAIE2P())
1163 (void)applyPatternsGreedily(op, std::move(patterns));
1166 if (failed(applyPartialConversion(op, target, std::move(patterns)))) {
1167 signalPassFailure();
1172static std::unique_ptr<::mlir::Pass> createCanonicalizeVectorForAIEVecPass(
1174 return std::make_unique<CanonicalizeVectorForAIEVecPass>(options);
1178 :
public PassWrapper<ReorderOperationsPass, OperationPass<>> {
1181 auto *op = getOperation();
1182 MLIRContext *context = &getContext();
1183 RewritePatternSet patterns(context);
1186 patterns.getContext(),
1187 [](arith::ExtSIOp extOp, vector::BroadcastOp bcastOp) -> Type {
1188 Type extInElemTy = extOp.getIn().getType();
1189 auto extInVecTy = dyn_cast<VectorType>(extInElemTy);
1191 extInElemTy = extInVecTy.getElementType();
1192 return VectorType::get(bcastOp.getResultVectorType().getShape(),
1196 (void)applyPatternsGreedily(op, std::move(patterns));
1200static std::unique_ptr<::mlir::Pass> createReorderOperationsPass() {
1201 return std::make_unique<ReorderOperationsPass>();
1218 pm.addPass(createReorderOperationsPass());
1220 std::optional<AIEVecTargetInfo> targetInfo =
1223 createVectorBroadcastLoweringPass(targetInfo && targetInfo->isAIE2P()));
1224 pm.addPass(createCanonicalizeVectorForAIEVecPass(options));
mlir::VectorType getFlattenedVectorType(mlir::VectorType vecTy)
std::unique_ptr<::mlir::Pass > createCopyRemovalPass()
Create a pass that removes unnecessary Copy operations.
std::optional< int64_t > getTransferReadAlignmentOffset(TransferReadLikeOp readOp, mlir::VectorType vType, int64_t alignment)
std::optional< AIEVecTargetInfo > parseAIEVecLoweringTarget(llvm::StringRef spelling, mlir::Operation *op)
std::optional< AIEVecTargetInfo > parseAIEVecTarget(llvm::StringRef spelling)
int32_t getElementSizeInBits(mlir::VectorType type)
std::unique_ptr<::mlir::Pass > createBF16EmulationPass()
Create a pass that runs f32 vector arithmetic on the bf16 datapath: multiplies and transcendentals ta...
void buildCanonicalizeVectorForAIEVec(mlir::OpPassManager &pm, const CanonicalizeVectorForAIEVecOptions &options)
void runOnOperation() override
LogicalResult matchAndRewrite(vector::BroadcastOp op, PatternRewriter &rewriter) const override
Pattern to canonicalize trivial vector.transfer_read operations on subviews.
LogicalResult matchAndRewrite(vector::TransferReadOp readOp, PatternRewriter &rewriter) const override
Pattern to canonicalize trivial vector.transfer_write operations on subviews.
LogicalResult matchAndRewrite(vector::TransferWriteOp writeOp, PatternRewriter &rewriter) const override
StringRef getArgument() const final
void getDependentDialects(DialectRegistry ®istry) const override
StringRef getDescription() const final
Option< std::string > aieTarget
CanonicalizeVectorForAIEVecPass(const CanonicalizeVectorForAIEVecOptions &options)
void runOnOperation() override
CanonicalizeVectorForAIEVecPass()=default
LogicalResult matchAndRewrite(vector::InsertOp insOp, PatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::TransferReadOp readOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
ConvertSplatTransferReadToBroadcastPattern(MLIRContext *context)
Pattern to emulate f32 binary vector arithmetic ops in bf16.
LogicalResult matchAndRewrite(OpTy op, PatternRewriter &rewriter) const override
Pattern to emulate f32 comparison ops in bf16.
LogicalResult matchAndRewrite(arith::CmpFOp op, PatternRewriter &rewriter) const override
Pattern to emulate f32 vector.fma with bf16 inputs.
LogicalResult matchAndRewrite(vector::FMAOp op, PatternRewriter &rewriter) const override
Pattern to emulate an f32 multiply with bf16 inputs.
LogicalResult matchAndRewrite(arith::MulFOp op, PatternRewriter &rewriter) const override
Pattern to emulate f32 select ops in bf16.
LogicalResult matchAndRewrite(arith::SelectOp op, PatternRewriter &rewriter) const override
Pattern to emulate f32 unary vector ops in bf16.
LogicalResult matchAndRewrite(OpTy op, PatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::TransferReadOp readOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::TransferWriteOp writeOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(scf::ForOp forOp, PatternRewriter &rewriter) const override
Pattern to drop bf16 -> f32 -> bf16 round trips.
LogicalResult matchAndRewrite(arith::TruncFOp op, PatternRewriter &rewriter) const override
FuseMulAddFInBF16Pattern for a multiply whose inputs already hold bf16 values, outside bf16 emulation...
LogicalResult matchAndRewrite(arith::AddFOp op, PatternRewriter &rewriter) const override
Pattern to fold an f32 multiply into the add that is its only user, as a vector.fma with bf16 inputs.
LogicalResult matchAndRewrite(arith::AddFOp op, PatternRewriter &rewriter) const override
void runOnOperation() override
SplitUnalignedTransferReadPattern(MLIRContext *context, int64_t maxVectorSize, int64_t alignment)
LogicalResult matchAndRewrite(vector::TransferReadOp readOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
InferTypeB2AFnTy inferTypeB2A
LogicalResult matchAndRewrite(UnaryOpB bOp, PatternRewriter &rewriter) const override
std::function< Type(UnaryOpA aOp, UnaryOpB bOp)> InferTypeB2AFnTy
SwapUnaryOpsPattern(MLIRContext *context, InferTypeB2AFnTy inferType)
void runOnOperation() override
VectorBroadcastLoweringPass(bool aie2p=false)
Options for the "canonicalize-vector-for-aievec" pipeline.
PassOptions::Option< bool > enableBF16Emulation
PassOptions::Option< std::string > aieTarget