MLIR-AIE 1.4.6.dev
AIEVecMatMul.h
Go to the documentation of this file.
1//===- AIEVecMatMul.h - Matmul recipes and tile layouts ---------*- C++ -*-===//
2//
3// Copyright (C) 2026 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7// The tile multiplies that `aievec.matmul` (AIE2) and `aievec.matmul_aie2p`
8// (AIE2P) lower to, and the operand layouts each of them reads. The verifier
9// of the matmul ops, the lowering of `vector.contract` and the lowering to
10// LLVM all choose from this one table.
11//===----------------------------------------------------------------------===//
12
13#ifndef AIE_DIALECT_AIEVEC_IR_AIEVECMATMUL_H
14#define AIE_DIALECT_AIEVEC_IR_AIEVECMATMUL_H
15
17
18#include "mlir/IR/BuiltinTypes.h"
19#include "mlir/IR/Diagnostics.h"
20#include "mlir/IR/PatternMatch.h"
21#include "llvm/ADT/STLFunctionalExtras.h"
22#include "llvm/ADT/SmallVector.h"
23
24#include <optional>
25
26namespace xilinx::aievec {
27
28// One way to multiply an m x k lhs tile by a k x n rhs tile into an m x n
29// accumulator. Names give the operand types and m x k x n.
30enum class MatMulRecipeKind {
31 // AIE2: one `mac.conf` each.
42 // AIE2P integer matrix modes.
45 // AIE2P bf16 through bfp16 (block floating point, one exponent per 8 values
46 // along k), the `8x8_8x8T` matrix mode.
49 // AIE2P bf16 through elementwise multiplies of broadcast rows and columns.
53};
54
57 // The matmul ops take lhs as [m, k] and rhs as [k, n]. These say whether
58 // the recipe reads lhs as [k, m] and rhs as [n, k] instead, so that an
59 // operand stored that way needs no shuffle.
60 bool lhsKM;
61 bool rhsNK;
62};
63
64// The recipe for an m x k x n tile with these element types, if there is one.
65// Integer types are matched by width; their signedness is a separate MAC
66// setting.
67std::optional<MatMulRecipe> getMatMulRecipe(bool aie2p, mlir::Type lhsElemTy,
68 mlir::Type rhsElemTy,
69 mlir::Type accElemTy, int64_t m,
70 int64_t k, int64_t n);
71
72// The shuffle modes that transpose a rows x cols tile of `elemTy` held
73// row-major. Empty when a dim is 1, as the tile then needs no data movement.
74// One mode when the tile fills 512 bits, or 256 bits on AIE2P (the tile then
75// sits in the low half of a 512-bit register). Two modes, the low and high
76// halves of the result, for an 8x8 tile of 16-bit values on AIE2P. Nullopt
77// for any other tile.
78std::optional<llvm::SmallVector<ShuffleMode, 2>>
79getTileTransposeModes(bool aie2p, mlir::Type elemTy, int64_t rows,
80 int64_t cols);
81
82// Checks that lhs times rhs into acc is a tile some recipe multiplies. lhs is
83// stored [m, k], or [k, m] when `lhsTransposed`; rhs is stored [k, n], or
84// [n, k] when `rhsTransposed`; acc is [m, n]. Each stored layout must be the
85// one the recipe reads, or one that getTileTransposeModes turns into it.
86// Returns the recipe. On failure, reports why through `emitError` unless it
87// is null.
88std::optional<MatMulRecipe> checkMatMulTile(
89 bool aie2p, mlir::VectorType lhsTy, mlir::VectorType rhsTy,
90 mlir::VectorType accTy, bool lhsTransposed, bool rhsTransposed,
91 llvm::function_ref<mlir::InFlightDiagnostic()> emitError = nullptr);
92
93// Rewrites the bf16 matmul `op` into an aievec.to_bfp16 of each operand
94// followed by aievec.matmul_aie2p_bfp16. Fails unless `op` uses a bfp16
95// recipe.
96mlir::LogicalResult expandBFP16MatMul(mlir::RewriterBase &rewriter,
97 MatMulOp_AIE2P op);
98
99// Return true if every use of `op` is the lhs (respectively rhs) operand of
100// an aievec.matmul_aie2p_bfp16. The lowering of aievec.to_bfp16 widens the
101// two operands of a matmul differently: an lhs-only tile in two 32-lane
102// halves, an rhs-only tile by a bf16 multiply with 1.0.
103bool isBFP16MatMulLhsOnly(ToBFP16Op op);
104bool isBFP16MatMulRhsOnly(ToBFP16Op op);
105
106} // namespace xilinx::aievec
107
108#endif // AIE_DIALECT_AIEVEC_IR_AIEVECMATMUL_H
std::optional< MatMulRecipe > checkMatMulTile(bool aie2p, mlir::VectorType lhsTy, mlir::VectorType rhsTy, mlir::VectorType accTy, bool lhsTransposed, bool rhsTransposed, llvm::function_ref< mlir::InFlightDiagnostic()> emitError=nullptr)
std::optional< llvm::SmallVector< ShuffleMode, 2 > > getTileTransposeModes(bool aie2p, mlir::Type elemTy, int64_t rows, int64_t cols)
bool isBFP16MatMulRhsOnly(ToBFP16Op op)
mlir::LogicalResult expandBFP16MatMul(mlir::RewriterBase &rewriter, MatMulOp_AIE2P op)
bool isBFP16MatMulLhsOnly(ToBFP16Op op)
std::optional< MatMulRecipe > getMatMulRecipe(bool aie2p, mlir::Type lhsElemTy, mlir::Type rhsElemTy, mlir::Type accElemTy, int64_t m, int64_t k, int64_t n)