MLIR-AIE 1.4.6.dev
AIEVecOptimizations.cpp
Go to the documentation of this file.
1//===- AIEVecOptimizations.cpp - Patterns to optimize AIEVec ----*- C++ -*-===//
2//
3// Copyright (C) 2023 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7// This file contains conversions and rewrite that replace common AIEVec ops
8// with more complex, and performant AIEVec ops.
9//===----------------------------------------------------------------------===//
10
11#include "BFP16MatMulOperands.h"
13#include "mlir/IR/Dominance.h"
14#include "mlir/Transforms/CSE.h"
15
23
24#include "mlir/Dialect/Affine/IR/AffineOps.h"
25#include "mlir/Dialect/Func/IR/FuncOps.h"
26#include "mlir/Dialect/MemRef/IR/MemRef.h"
27#include "mlir/Dialect/SCF/IR/SCF.h"
28#include "mlir/IR/PatternMatch.h"
29#include "mlir/Pass/PassManager.h"
30#include "mlir/Transforms/DialectConversion.h"
31#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
32#include "mlir/Transforms/Passes.h"
33
34#define DEBUG_TYPE "aievec-optimize"
35
36using namespace llvm;
37using namespace mlir;
38using namespace arith;
39using namespace vector;
40using namespace xilinx;
41using namespace xilinx::aievec;
42
43//===----------------------------------------------------------------------===//
44// Utility functions
45//===----------------------------------------------------------------------===//
46namespace xilinx {
47namespace aievec {
48
49SmallVector<NamedAttribute>
50buildFMAOpSplatAttrForElemTy(aievec::aie1::FMAOp fmaOp, int64_t bcastPos,
51 int64_t step = 1);
52
53} // namespace aievec
54} // namespace xilinx
55
56static bool canFoldAIEShiftAndBroadcast(aievec::BroadcastOp op,
57 aievec::ShiftOp &shiftOp,
58 int32_t &idx) {
59 if (!op.getSource().getDefiningOp())
60 return false;
61
62 shiftOp = dyn_cast<aievec::ShiftOp>(op.getSource().getDefiningOp());
63
64 if (!shiftOp)
65 return false;
66
67 VectorType vType = cast<VectorType>(shiftOp->getResult(0).getType());
68 int32_t elemSize = getElementSizeInBits(vType);
69 auto constOp = cast<arith::ConstantOp>(shiftOp.getShift().getDefiningOp());
70 int32_t shiftBytes = cast<IntegerAttr>(constOp.getValue()).getInt();
71 idx = shiftBytes * 8 / elemSize + op.getIdx();
72
73 return idx > 0 && idx < (int32_t)getVectorLaneSize(vType);
74}
75
76template <typename AIEv1MACLikeOp,
77 typename = std::enable_if_t<
78 std::is_same_v<AIEv1MACLikeOp, aievec::aie1::FMAOp> ||
79 std::is_same_v<AIEv1MACLikeOp, aievec::aie1::FMAOp::Adaptor>>>
80static bool isSingleColumnInt16VectorTimesScalarMac(AIEv1MACLikeOp fmaOp) {
81 // lhs is a 32xi16 vector
82 VectorType lhsVTy = cast<VectorType>(fmaOp.getLhs().getType());
83 auto intTy = dyn_cast<IntegerType>(lhsVTy.getElementType());
84 if (!intTy || intTy.getWidth() != 16)
85 return false;
86 if (lhsVTy.getShape()[0] != 32)
87 return false;
88 // Attributes match a Vector x Scalar mac
89 if (fmaOp.getXoffsets() != "0x73727170" ||
90 fmaOp.getXoffsetsHi() != "0x77767574" || fmaOp.getXstart() != "0" ||
91 fmaOp.getXsquare() != "0x3120" || fmaOp.getZoffsets() != "0" ||
92 fmaOp.getZoffsetsHi() != "0" || fmaOp.getZstep() != "1")
93 return false;
94 // lhs op is a concat of a vector and a dense<0> constant vector
95 if (!fmaOp.getLhs().getDefiningOp())
96 return false;
97 aievec::ConcatOp concatOp =
98 dyn_cast<aievec::ConcatOp>(fmaOp.getLhs().getDefiningOp());
99 if (!concatOp)
100 return false;
101 auto tailVec = concatOp.getSources()[1];
102 if (!tailVec.getDefiningOp())
103 return false;
104 auto constOp = dyn_cast<arith::ConstantOp>(tailVec.getDefiningOp());
105 if (!constOp)
106 return false;
107 auto cstDense = dyn_cast<DenseIntElementsAttr>(constOp.getValue());
108 if (!cstDense)
109 return false;
110 return llvm::all_of(cstDense, [](const APInt &val) { return val == 0; });
111}
112
113static bool singleColumnFMAOpCanFold(aievec::aie1::FMAOp fmaOp) {
114 auto *accProdOp = fmaOp.getAcc().getDefiningOp();
115 if (!accProdOp)
116 return false;
117 auto accFmaOp = dyn_cast<aievec::aie1::FMAOp>(accProdOp);
118 if (!accFmaOp)
119 return false;
120 if (!isSingleColumnInt16VectorTimesScalarMac(accFmaOp))
121 return false;
122 return fmaOp.getRhs() == accFmaOp.getRhs() &&
123 !singleColumnFMAOpCanFold(accFmaOp);
124}
125
126//===----------------------------------------------------------------------===//
127// Lowering patterns
128//===----------------------------------------------------------------------===//
130 : public OpConversionPattern<aievec::aie1::FMAOp> {
131 using OpConversionPattern<aievec::aie1::FMAOp>::OpConversionPattern;
132
133 LogicalResult
134 matchAndRewrite(aievec::aie1::FMAOp fmaOp, OpAdaptor adaptor,
135 ConversionPatternRewriter &rewriter) const override {
136 if (!isSingleColumnInt16VectorTimesScalarMac(adaptor))
137 return failure();
138 auto *accProdOp = adaptor.getAcc().getDefiningOp();
139 if (!accProdOp)
140 return failure();
141 auto accFmaOp = dyn_cast<aievec::aie1::FMAOp>(accProdOp);
142 if (!accFmaOp)
143 return failure();
144 if (!isSingleColumnInt16VectorTimesScalarMac(accFmaOp))
145 return failure();
146 if (adaptor.getRhs() != accFmaOp.getRhs())
147 return failure();
148 auto accConcatOp =
149 cast<aievec::ConcatOp>(accFmaOp.getLhs().getDefiningOp());
150 auto fmaConcatOp = cast<aievec::ConcatOp>(adaptor.getLhs().getDefiningOp());
151 unsigned fmaZstart, accFmaZstart;
152 if (adaptor.getZstart().getAsInteger(10, fmaZstart) ||
153 accFmaOp.getZstart().getAsInteger(10, accFmaZstart))
154 return failure();
155 auto start = std::min(fmaZstart, accFmaZstart);
156 auto step = std::max(fmaZstart, accFmaZstart) - start;
157 auto lowV = accConcatOp.getSources()[0];
158 auto hiV = fmaConcatOp.getSources()[0];
159 if (accFmaZstart > fmaZstart)
160 std::swap(lowV, hiV);
161 auto newConcatOp = aievec::ConcatOp::create(
162 rewriter, fmaOp.getLoc(), adaptor.getLhs().getType(),
163 SmallVector<Value, 2>({lowV, hiV}));
164 auto newFmaOpAttr = buildFMAOpSplatAttrForElemTy(fmaOp, start, step);
165 aievec::aie1::FMAOp::Properties fmaProps;
166 if (failed(aievec::aie1::FMAOp::setPropertiesFromAttr(
167 fmaProps, rewriter.getDictionaryAttr(newFmaOpAttr),
168 [&]() { return fmaOp.emitError(); })))
169 return failure();
170 rewriter.replaceOpWithNewOp<aievec::aie1::FMAOp>(
171 fmaOp, TypeRange({fmaOp.getResult().getType()}),
172 ValueRange({newConcatOp, adaptor.getRhs(), accFmaOp.getAcc()}),
173 fmaProps);
174 return success();
175 }
176};
177
179 : public OpConversionPattern<aievec::BroadcastOp> {
180 using OpConversionPattern<aievec::BroadcastOp>::OpConversionPattern;
181
182 LogicalResult
183 matchAndRewrite(aievec::BroadcastOp bcastOp, OpAdaptor adaptor,
184 ConversionPatternRewriter &rewriter) const override {
185 aievec::ShiftOp shiftOp = nullptr;
186 int32_t idx = 0;
187
188 if (!canFoldAIEShiftAndBroadcast(bcastOp, shiftOp, idx)) {
189 return failure();
190 }
191
192 VectorType resultType = cast<VectorType>(bcastOp.getResult().getType());
193
194 rewriter.replaceOpWithNewOp<aievec::BroadcastOp>(bcastOp, resultType,
195 shiftOp.getLhs(), idx);
196
197 return success();
198 }
199};
200
201//===----------------------------------------------------------------------===//
202// Pattern collection
203//===----------------------------------------------------------------------===//
204static void
205populateAIEVecV1TransformationPatterns(RewritePatternSet &patterns) {
206 patterns.add<MergeSingleColumnI16FMAOpPattern>(patterns.getContext());
207}
208
209static void
210populateAIEVecV2TransformationPatterns(RewritePatternSet &patterns) {
211 patterns.add<FoldAIEShiftAndBroadcast>(patterns.getContext());
212}
213
214// Expands bf16 matmuls with a bfp16 recipe so that later patterns can
215// rewrite the conversions.
217 : public OpConversionPattern<aievec::MatMulOp_AIE2P> {
218 using OpConversionPattern::OpConversionPattern;
219
220 LogicalResult
221 matchAndRewrite(aievec::MatMulOp_AIE2P op, OpAdaptor adaptor,
222 ConversionPatternRewriter &rewriter) const override {
223 return aievec::expandBFP16MatMul(rewriter, op);
224 }
225};
226
227static bool isBFP16MatMul(aievec::MatMulOp_AIE2P op) {
228 std::optional<aievec::MatMulRecipe> recipe = aievec::checkMatMulTile(
229 /*aie2p=*/true, cast<VectorType>(op.getLhs().getType()),
230 cast<VectorType>(op.getRhs().getType()),
231 cast<VectorType>(op.getAcc().getType()), op.getLhsTransposed(),
232 op.getRhsTransposed());
233 return recipe &&
236}
237
238//===----------------------------------------------------------------------===//
239// Legalizations
240//===----------------------------------------------------------------------===//
241
242static void
243configureAIEVecV1TransformationLegalizations(ConversionTarget &target) {
244 target.addLegalDialect<aievec::AIEVecDialect,
245 aievec::aie1::AIEVecAIE1Dialect>();
246 target.addDynamicallyLegalOp<aievec::aie1::FMAOp>(
247 [](aievec::aie1::FMAOp fmaOp) {
248 if (isSingleColumnInt16VectorTimesScalarMac(fmaOp))
249 return !singleColumnFMAOpCanFold(fmaOp);
250 return true;
251 });
252}
253
254static void
255configureAIEVecV2TransformationLegalizations(ConversionTarget &target) {
256 target.addDynamicallyLegalOp<xilinx::aievec::BroadcastOp>(
257 [](xilinx::aievec::BroadcastOp op) {
258 aievec::ShiftOp shiftOp = nullptr;
259 int32_t idx = 0;
260 return !canFoldAIEShiftAndBroadcast(op, shiftOp, idx);
261 });
262}
263
264//===----------------------------------------------------------------------===//
265// Lowering passes
266//===----------------------------------------------------------------------===//
268 : public PassWrapper<AIEVecTransformationPass, OperationPass<>> {
269 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(AIEVecTransformationPass)
270
274
279
280 // In case we want to register this pass as a standalone pass for test
281 // purposes.
282 StringRef getArgument() const final { return "test-aievec-optimize"; }
283 StringRef getDescription() const final {
284 return "Optimize groups of simple aievec ops into complex aievec ops.";
285 }
286 void getDependentDialects(DialectRegistry &registry) const override {
287 // TODO: Review list of dependent dialects.
288 registry.insert<affine::AffineDialect, xilinx::aievec::AIEVecDialect,
289 aievec::aie1::AIEVecAIE1Dialect, arith::ArithDialect,
290 memref::MemRefDialect, scf::SCFDialect,
291 vector::VectorDialect>();
292 }
293
294 Option<std::string> aieTarget{
295 *this, "aie-target",
296 llvm::cl::desc(
297 "Select AIE version: \"aie\", \"aie2\", or \"aie2p\". This will "
298 "determine the vector size and available operations."),
299 llvm::cl::init("aie")};
300
301 void runOnOperation() override {
302 auto *op = getOperation();
303 MLIRContext *context = &getContext();
304 RewritePatternSet patterns(context);
305 ConversionTarget target(*context);
306 std::optional<AIEVecTargetInfo> targetInfo =
308 if (!targetInfo) {
309 signalPassFailure();
310 return;
311 }
312
313 // AIE2P uses the AIE2 transformations.
314 if (targetInfo->isAIE1()) {
315 populateAIEVecV1TransformationPatterns(patterns);
316 configureAIEVecV1TransformationLegalizations(target);
317 } else {
318 populateAIEVecV2TransformationPatterns(patterns);
319 configureAIEVecV2TransformationLegalizations(target);
320 }
321 if (targetInfo->isAIE2P()) {
322 patterns.add<ExpandBFP16MatMulPattern>(context);
323 target.addLegalOp<aievec::ToBFP16Op, aievec::MatMulBFP16Op_AIE2P>();
324 target.addDynamicallyLegalOp<aievec::MatMulOp_AIE2P>(
325 [](aievec::MatMulOp_AIE2P op) { return !isBFP16MatMul(op); });
326 }
327
328 if (failed(applyPartialConversion(op, target, std::move(patterns)))) {
329 signalPassFailure();
330 return;
331 }
332
333 if (targetInfo->isAIE2P()) {
334 // A tile that feeds several matmuls is converted once.
335 IRRewriter rewriter(context);
336 DominanceInfo domInfo(op);
337 eliminateCommonSubExpressions(rewriter, domInfo, op);
338 // Replace decoded tiles by their blocks before hoisting conversions,
339 // so that the hoisting pattern does not pick up those tiles.
340 RewritePatternSet decoded(context);
342 if (failed(applyPatternsGreedily(op, std::move(decoded)))) {
343 signalPassFailure();
344 return;
345 }
346 RewritePatternSet hoist(context);
348 if (failed(applyPatternsGreedily(op, std::move(hoist)))) {
349 signalPassFailure();
350 return;
351 }
352 RewritePatternSet streams(context);
354 if (failed(applyPatternsGreedily(op, std::move(streams))))
355 signalPassFailure();
356 }
357 }
358};
359
360static std::unique_ptr<::mlir::Pass>
361createAIEVecTransformationPass(const OptimizeAIEVecOptions &options) {
362 return std::make_unique<AIEVecTransformationPass>(options);
363}
364
366 : public PassWrapper<AIEVecConvOpTransformationPass, OperationPass<>> {
367 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(AIEVecConvOpTransformationPass)
368
372
378
379 // In case we want to register this pass as a standalone pass for test
380 // purposes.
381 StringRef getArgument() const final {
382 return "test-aievec-convolution-optimize";
383 }
384 StringRef getDescription() const final {
385 return "Optimize chains of macs into AIE2 conv ops.";
386 }
387 void getDependentDialects(DialectRegistry &registry) const override {
388 // TODO: Review list of dependent dialects.
389 registry.insert<affine::AffineDialect, xilinx::aievec::AIEVecDialect,
390 aievec::aie1::AIEVecAIE1Dialect, arith::ArithDialect,
391 memref::MemRefDialect, scf::SCFDialect,
392 vector::VectorDialect>();
393 }
394
395 Option<std::string> aieTarget{
396 *this, "aie-target",
397 llvm::cl::desc(
398 "Select AIE version: \"aie\", \"aie2\", or \"aie2p\". This will "
399 "determine the vector size and available operations."),
400 llvm::cl::init("aie")};
401
402 Option<unsigned> shiftParam{
403 *this, "shift",
404 llvm::cl::desc("Shift parameter for rounding and saturation."),
405 llvm::cl::init(0)};
406
407 void runOnOperation() override {
408 auto *op = getOperation();
409 MLIRContext *context = &getContext();
410 RewritePatternSet patterns(context);
411 ConversionTarget target(*context);
412 std::optional<AIEVecTargetInfo> targetInfo =
414 if (!targetInfo) {
415 signalPassFailure();
416 return;
417 }
418
419 AnalysisManager am = getAnalysisManager();
420 if (targetInfo->isAIE2() || targetInfo->isAIE2P()) {
423 }
424
425 if (failed(applyPartialConversion(op, target, std::move(patterns)))) {
426 signalPassFailure();
427 }
428 }
429};
430
431static std::unique_ptr<::mlir::Pass>
432createAIEVecConvOpTransformationPass(const OptimizeAIEVecOptions &options) {
433 return std::make_unique<AIEVecConvOpTransformationPass>(options);
434}
435
436//============================================================================//
437//=============== Main AIEVec2AIEVec Pipeline Configuration ==================//
438//============================================================================//
439
440void xilinx::aievec::buildOptimizeAIEVec(OpPassManager &pm,
441 const OptimizeAIEVecOptions &options) {
442 // Add AIEVec transformation pass.
443 pm.addPass(createAIEVecTransformationPass(options));
444
445 pm.addPass(createCSEPass());
446 pm.addPass(createCanonicalizerPass());
447
448 // Add generating aievec convolution ops pass
449 std::optional<AIEVecTargetInfo> targetInfo =
451 if (targetInfo && (targetInfo->isAIE2() || targetInfo->isAIE2P())) {
453 pm.addPass(createAIEVecConvOpTransformationPass(options));
454 }
455
456 // Add post-lowering canonicalization passes.
457 pm.addPass(createCSEPass());
458 pm.addPass(createCanonicalizerPass());
459}
std::optional< MatMulRecipe > checkMatMulTile(bool aie2p, mlir::VectorType lhsTy, mlir::VectorType rhsTy, mlir::VectorType accTy, bool lhsTransposed, bool rhsTransposed, llvm::function_ref< mlir::InFlightDiagnostic()> emitError=nullptr)
unsigned getVectorLaneSize(mlir::VectorType type)
Definition AIEVecUtils.h:54
SmallVector< NamedAttribute > buildFMAOpSplatAttrForElemTy(aievec::aie1::FMAOp fmaOp, int64_t bcastPos, int64_t step=1)
std::unique_ptr< mlir::Pass > createAIEVecConvolutionAnalysisPass()
void populateHoistBFP16ConversionPatterns(mlir::RewritePatternSet &patterns)
std::optional< AIEVecTargetInfo > parseAIEVecLoweringTarget(llvm::StringRef spelling, mlir::Operation *op)
std::optional< AIEVecTargetInfo > parseAIEVecTarget(llvm::StringRef spelling)
int32_t getElementSizeInBits(mlir::VectorType type)
Definition AIEVecUtils.h:48
void configureAIEVecConvOpTransformationLegalizations(ConversionTarget &target, AnalysisManager &am)
void populateAIEVecConvOpTransformationPatterns(RewritePatternSet &patterns, AnalysisManager &am, unsigned shiftParam)
mlir::LogicalResult expandBFP16MatMul(mlir::RewriterBase &rewriter, MatMulOp_AIE2P op)
void populateBFP16LoadStreamPatterns(mlir::RewritePatternSet &patterns)
void buildOptimizeAIEVec(mlir::OpPassManager &pm, const OptimizeAIEVecOptions &options)
void populateReadDecodedBFP16Patterns(mlir::RewritePatternSet &patterns)
AIEVecConvOpTransformationPass(const OptimizeAIEVecOptions &options)
void getDependentDialects(DialectRegistry &registry) const override
StringRef getDescription() const final
AIEVecTransformationPass(const OptimizeAIEVecOptions &options)
AIEVecTransformationPass()=default
void getDependentDialects(DialectRegistry &registry) const override
StringRef getDescription() const final
StringRef getArgument() const final
Option< std::string > aieTarget
LogicalResult matchAndRewrite(aievec::MatMulOp_AIE2P op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::BroadcastOp bcastOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::aie1::FMAOp fmaOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
Options for the "optimize-aievec" pipeline.
Definition Passes.h:53
PassOptions::Option< unsigned > shiftParam
Definition Passes.h:60
PassOptions::Option< std::string > aieTarget
Definition Passes.h:54