13#include "mlir/IR/Dominance.h"
14#include "mlir/Transforms/CSE.h"
24#include "mlir/Dialect/Affine/IR/AffineOps.h"
25#include "mlir/Dialect/Func/IR/FuncOps.h"
26#include "mlir/Dialect/MemRef/IR/MemRef.h"
27#include "mlir/Dialect/SCF/IR/SCF.h"
28#include "mlir/IR/PatternMatch.h"
29#include "mlir/Pass/PassManager.h"
30#include "mlir/Transforms/DialectConversion.h"
31#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
32#include "mlir/Transforms/Passes.h"
34#define DEBUG_TYPE "aievec-optimize"
49SmallVector<NamedAttribute>
56static bool canFoldAIEShiftAndBroadcast(aievec::BroadcastOp op,
57 aievec::ShiftOp &shiftOp,
59 if (!op.getSource().getDefiningOp())
62 shiftOp = dyn_cast<aievec::ShiftOp>(op.getSource().getDefiningOp());
67 VectorType vType = cast<VectorType>(shiftOp->getResult(0).getType());
69 auto constOp = cast<arith::ConstantOp>(shiftOp.getShift().getDefiningOp());
70 int32_t shiftBytes = cast<IntegerAttr>(constOp.getValue()).getInt();
71 idx = shiftBytes * 8 / elemSize + op.getIdx();
76template <
typename AIEv1MACLikeOp,
77 typename = std::enable_if_t<
78 std::is_same_v<AIEv1MACLikeOp, aievec::aie1::FMAOp> ||
79 std::is_same_v<AIEv1MACLikeOp, aievec::aie1::FMAOp::Adaptor>>>
80static bool isSingleColumnInt16VectorTimesScalarMac(AIEv1MACLikeOp fmaOp) {
82 VectorType lhsVTy = cast<VectorType>(fmaOp.getLhs().getType());
83 auto intTy = dyn_cast<IntegerType>(lhsVTy.getElementType());
84 if (!intTy || intTy.getWidth() != 16)
86 if (lhsVTy.getShape()[0] != 32)
89 if (fmaOp.getXoffsets() !=
"0x73727170" ||
90 fmaOp.getXoffsetsHi() !=
"0x77767574" || fmaOp.getXstart() !=
"0" ||
91 fmaOp.getXsquare() !=
"0x3120" || fmaOp.getZoffsets() !=
"0" ||
92 fmaOp.getZoffsetsHi() !=
"0" || fmaOp.getZstep() !=
"1")
95 if (!fmaOp.getLhs().getDefiningOp())
97 aievec::ConcatOp concatOp =
98 dyn_cast<aievec::ConcatOp>(fmaOp.getLhs().getDefiningOp());
101 auto tailVec = concatOp.getSources()[1];
102 if (!tailVec.getDefiningOp())
104 auto constOp = dyn_cast<arith::ConstantOp>(tailVec.getDefiningOp());
107 auto cstDense = dyn_cast<DenseIntElementsAttr>(constOp.getValue());
110 return llvm::all_of(cstDense, [](
const APInt &val) {
return val == 0; });
113static bool singleColumnFMAOpCanFold(aievec::aie1::FMAOp fmaOp) {
114 auto *accProdOp = fmaOp.getAcc().getDefiningOp();
117 auto accFmaOp = dyn_cast<aievec::aie1::FMAOp>(accProdOp);
120 if (!isSingleColumnInt16VectorTimesScalarMac(accFmaOp))
122 return fmaOp.getRhs() == accFmaOp.getRhs() &&
123 !singleColumnFMAOpCanFold(accFmaOp);
130 :
public OpConversionPattern<aievec::aie1::FMAOp> {
131 using OpConversionPattern<aievec::aie1::FMAOp>::OpConversionPattern;
135 ConversionPatternRewriter &rewriter)
const override {
136 if (!isSingleColumnInt16VectorTimesScalarMac(adaptor))
138 auto *accProdOp = adaptor.getAcc().getDefiningOp();
141 auto accFmaOp = dyn_cast<aievec::aie1::FMAOp>(accProdOp);
144 if (!isSingleColumnInt16VectorTimesScalarMac(accFmaOp))
146 if (adaptor.getRhs() != accFmaOp.getRhs())
149 cast<aievec::ConcatOp>(accFmaOp.getLhs().getDefiningOp());
150 auto fmaConcatOp = cast<aievec::ConcatOp>(adaptor.getLhs().getDefiningOp());
151 unsigned fmaZstart, accFmaZstart;
152 if (adaptor.getZstart().getAsInteger(10, fmaZstart) ||
153 accFmaOp.getZstart().getAsInteger(10, accFmaZstart))
155 auto start = std::min(fmaZstart, accFmaZstart);
156 auto step = std::max(fmaZstart, accFmaZstart) - start;
157 auto lowV = accConcatOp.getSources()[0];
158 auto hiV = fmaConcatOp.getSources()[0];
159 if (accFmaZstart > fmaZstart)
160 std::swap(lowV, hiV);
161 auto newConcatOp = aievec::ConcatOp::create(
162 rewriter, fmaOp.getLoc(), adaptor.getLhs().getType(),
163 SmallVector<Value, 2>({lowV, hiV}));
165 aievec::aie1::FMAOp::Properties fmaProps;
166 if (failed(aievec::aie1::FMAOp::setPropertiesFromAttr(
167 fmaProps, rewriter.getDictionaryAttr(newFmaOpAttr),
168 [&]() { return fmaOp.emitError(); })))
170 rewriter.replaceOpWithNewOp<aievec::aie1::FMAOp>(
171 fmaOp, TypeRange({fmaOp.getResult().getType()}),
172 ValueRange({newConcatOp, adaptor.getRhs(), accFmaOp.getAcc()}),
179 :
public OpConversionPattern<aievec::BroadcastOp> {
180 using OpConversionPattern<aievec::BroadcastOp>::OpConversionPattern;
184 ConversionPatternRewriter &rewriter)
const override {
185 aievec::ShiftOp shiftOp =
nullptr;
188 if (!canFoldAIEShiftAndBroadcast(bcastOp, shiftOp, idx)) {
192 VectorType resultType = cast<VectorType>(bcastOp.getResult().getType());
194 rewriter.replaceOpWithNewOp<aievec::BroadcastOp>(bcastOp, resultType,
195 shiftOp.getLhs(), idx);
205populateAIEVecV1TransformationPatterns(RewritePatternSet &patterns) {
210populateAIEVecV2TransformationPatterns(RewritePatternSet &patterns) {
217 :
public OpConversionPattern<aievec::MatMulOp_AIE2P> {
218 using OpConversionPattern::OpConversionPattern;
222 ConversionPatternRewriter &rewriter)
const override {
227static bool isBFP16MatMul(aievec::MatMulOp_AIE2P op) {
229 true, cast<VectorType>(op.getLhs().getType()),
230 cast<VectorType>(op.getRhs().getType()),
231 cast<VectorType>(op.getAcc().getType()), op.getLhsTransposed(),
232 op.getRhsTransposed());
243configureAIEVecV1TransformationLegalizations(ConversionTarget &target) {
244 target.addLegalDialect<aievec::AIEVecDialect,
245 aievec::aie1::AIEVecAIE1Dialect>();
246 target.addDynamicallyLegalOp<aievec::aie1::FMAOp>(
247 [](aievec::aie1::FMAOp fmaOp) {
248 if (isSingleColumnInt16VectorTimesScalarMac(fmaOp))
249 return !singleColumnFMAOpCanFold(fmaOp);
255configureAIEVecV2TransformationLegalizations(ConversionTarget &target) {
256 target.addDynamicallyLegalOp<xilinx::aievec::BroadcastOp>(
257 [](xilinx::aievec::BroadcastOp op) {
258 aievec::ShiftOp shiftOp =
nullptr;
260 return !canFoldAIEShiftAndBroadcast(op, shiftOp, idx);
268 :
public PassWrapper<AIEVecTransformationPass, OperationPass<>> {
282 StringRef
getArgument() const final {
return "test-aievec-optimize"; }
284 return "Optimize groups of simple aievec ops into complex aievec ops.";
288 registry.insert<affine::AffineDialect, xilinx::aievec::AIEVecDialect,
289 aievec::aie1::AIEVecAIE1Dialect, arith::ArithDialect,
290 memref::MemRefDialect, scf::SCFDialect,
291 vector::VectorDialect>();
297 "Select AIE version: \"aie\", \"aie2\", or \"aie2p\". This will "
298 "determine the vector size and available operations."),
299 llvm::cl::init(
"aie")};
302 auto *op = getOperation();
303 MLIRContext *context = &getContext();
304 RewritePatternSet patterns(context);
305 ConversionTarget target(*context);
306 std::optional<AIEVecTargetInfo> targetInfo =
314 if (targetInfo->isAIE1()) {
315 populateAIEVecV1TransformationPatterns(patterns);
316 configureAIEVecV1TransformationLegalizations(target);
318 populateAIEVecV2TransformationPatterns(patterns);
319 configureAIEVecV2TransformationLegalizations(target);
321 if (targetInfo->isAIE2P()) {
323 target.addLegalOp<aievec::ToBFP16Op, aievec::MatMulBFP16Op_AIE2P>();
324 target.addDynamicallyLegalOp<aievec::MatMulOp_AIE2P>(
325 [](aievec::MatMulOp_AIE2P op) {
return !isBFP16MatMul(op); });
328 if (failed(applyPartialConversion(op, target, std::move(patterns)))) {
333 if (targetInfo->isAIE2P()) {
335 IRRewriter rewriter(context);
336 DominanceInfo domInfo(op);
337 eliminateCommonSubExpressions(rewriter, domInfo, op);
340 RewritePatternSet decoded(context);
342 if (failed(applyPatternsGreedily(op, std::move(decoded)))) {
346 RewritePatternSet hoist(context);
348 if (failed(applyPatternsGreedily(op, std::move(hoist)))) {
352 RewritePatternSet streams(context);
354 if (failed(applyPatternsGreedily(op, std::move(streams))))
360static std::unique_ptr<::mlir::Pass>
362 return std::make_unique<AIEVecTransformationPass>(options);
366 :
public PassWrapper<AIEVecConvOpTransformationPass, OperationPass<>> {
382 return "test-aievec-convolution-optimize";
385 return "Optimize chains of macs into AIE2 conv ops.";
389 registry.insert<affine::AffineDialect, xilinx::aievec::AIEVecDialect,
390 aievec::aie1::AIEVecAIE1Dialect, arith::ArithDialect,
391 memref::MemRefDialect, scf::SCFDialect,
392 vector::VectorDialect>();
398 "Select AIE version: \"aie\", \"aie2\", or \"aie2p\". This will "
399 "determine the vector size and available operations."),
400 llvm::cl::init(
"aie")};
404 llvm::cl::desc(
"Shift parameter for rounding and saturation."),
408 auto *op = getOperation();
409 MLIRContext *context = &getContext();
410 RewritePatternSet patterns(context);
411 ConversionTarget target(*context);
412 std::optional<AIEVecTargetInfo> targetInfo =
419 AnalysisManager am = getAnalysisManager();
420 if (targetInfo->isAIE2() || targetInfo->isAIE2P()) {
425 if (failed(applyPartialConversion(op, target, std::move(patterns)))) {
431static std::unique_ptr<::mlir::Pass>
433 return std::make_unique<AIEVecConvOpTransformationPass>(options);
443 pm.addPass(createAIEVecTransformationPass(options));
445 pm.addPass(createCSEPass());
446 pm.addPass(createCanonicalizerPass());
449 std::optional<AIEVecTargetInfo> targetInfo =
451 if (targetInfo && (targetInfo->isAIE2() || targetInfo->isAIE2P())) {
453 pm.addPass(createAIEVecConvOpTransformationPass(options));
457 pm.addPass(createCSEPass());
458 pm.addPass(createCanonicalizerPass());
std::optional< MatMulRecipe > checkMatMulTile(bool aie2p, mlir::VectorType lhsTy, mlir::VectorType rhsTy, mlir::VectorType accTy, bool lhsTransposed, bool rhsTransposed, llvm::function_ref< mlir::InFlightDiagnostic()> emitError=nullptr)
unsigned getVectorLaneSize(mlir::VectorType type)
SmallVector< NamedAttribute > buildFMAOpSplatAttrForElemTy(aievec::aie1::FMAOp fmaOp, int64_t bcastPos, int64_t step=1)
std::unique_ptr< mlir::Pass > createAIEVecConvolutionAnalysisPass()
void populateHoistBFP16ConversionPatterns(mlir::RewritePatternSet &patterns)
std::optional< AIEVecTargetInfo > parseAIEVecLoweringTarget(llvm::StringRef spelling, mlir::Operation *op)
std::optional< AIEVecTargetInfo > parseAIEVecTarget(llvm::StringRef spelling)
int32_t getElementSizeInBits(mlir::VectorType type)
void configureAIEVecConvOpTransformationLegalizations(ConversionTarget &target, AnalysisManager &am)
void populateAIEVecConvOpTransformationPatterns(RewritePatternSet &patterns, AnalysisManager &am, unsigned shiftParam)
mlir::LogicalResult expandBFP16MatMul(mlir::RewriterBase &rewriter, MatMulOp_AIE2P op)
void populateBFP16LoadStreamPatterns(mlir::RewritePatternSet &patterns)
void buildOptimizeAIEVec(mlir::OpPassManager &pm, const OptimizeAIEVecOptions &options)
void populateReadDecodedBFP16Patterns(mlir::RewritePatternSet &patterns)
LogicalResult matchAndRewrite(aievec::MatMulOp_AIE2P op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::BroadcastOp bcastOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::aie1::FMAOp fmaOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
Options for the "optimize-aievec" pipeline.
PassOptions::Option< unsigned > shiftParam
PassOptions::Option< std::string > aieTarget