MLIR-AIE 1.4.6.dev
VectorToVectorConversions.cpp
Go to the documentation of this file.
1//===-VectorToVectorConversions.cpp - Conversions within Vector -*- C++ -*-===//
2//
3// Copyright (C) 2023-2024 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7// This file contains conversions and rewrites to the Vector dialect to make
8// it compatible with the available vector instructions in AIE architectures
9//===----------------------------------------------------------------------===//
10
15#include "mlir/Conversion/AffineToStandard/AffineToStandard.h"
16#include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h"
17#include "mlir/Dialect/Affine/IR/AffineOps.h"
18#include "mlir/Dialect/Func/IR/FuncOps.h"
19#include "mlir/Dialect/Math/IR/Math.h"
20#include "mlir/Dialect/MemRef/IR/MemRef.h"
21#include "mlir/Dialect/SCF/IR/SCF.h"
22#include "mlir/Dialect/UB/IR/UBOps.h"
23#include "mlir/Dialect/Utils/ReshapeOpsUtils.h"
24#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
25#include "mlir/IR/Matchers.h"
26#include "mlir/IR/PatternMatch.h"
27#include "mlir/Pass/PassManager.h"
28#include "mlir/Transforms/DialectConversion.h"
29#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
30#include "llvm/ADT/STLExtras.h"
31#include <algorithm>
32#include <utility>
33
34#define DEBUG_TYPE "aievec-canonicalization"
35
36using namespace mlir;
37using namespace arith;
38using namespace vector;
39using namespace xilinx;
40using namespace xilinx::aievec;
41
42//============================================================================//
43//================== Common AIE canonicalization analysis ====================//
44//============================================================================//
45
46//============================================================================//
47//============ Common AIE canonicalization conversion patterns ===============//
48//============================================================================//
49
50// This pattern converts a `vector.transfer_read` with an unaligned access
51// into an aligned `vector.transfer_read` twice as long, followed by a
52// `vector.extract_strided_slice` selecting the subvector matching the
53// original `vector.transfer_read`.
55 : public OpConversionPattern<vector::TransferReadOp> {
56 using OpConversionPattern<vector::TransferReadOp>::OpConversionPattern;
57
58 SplitUnalignedTransferReadPattern(MLIRContext *context, int64_t maxVectorSize,
59 int64_t alignment)
60 : OpConversionPattern<vector::TransferReadOp>(context),
62
63 LogicalResult
64 matchAndRewrite(vector::TransferReadOp readOp, OpAdaptor adaptor,
65 ConversionPatternRewriter &rewriter) const override {
66 // Check that it's not a splat transfer read.
67 if (adaptor.getPermutationMap().isConstant())
68 return failure();
69
70 // Check if the transfer is unaligned.
71 auto vType = readOp.getVectorType();
72 int64_t offset =
74 .value_or(0);
75 if (offset == 0)
76 return failure();
77
78 // Verify that we can load a vector 2x as long as the original
79 auto vLen = vType.getShape().back();
80 auto longVecTy = VectorType::get(2 * vLen, vType.getElementType());
81 auto longVecSize = getElementSizeInBits(vType) * 2 * vLen;
82 if (longVecSize > maxVectorSize)
83 return failure();
84
85 // Calculate the aligned indices for the lower and higher parts.
86 // TODO: Add support for cases where the offset is greater than the
87 // TODO: vector length.
88 auto loc = readOp.getLoc();
89 Value oldInnerMostIdx = adaptor.getIndices().back();
90 auto offsetCorrectionMap =
91 AffineMap::get(1, 0, getAffineDimExpr(0, readOp.getContext()) - offset);
92 Value newInnerMostIdx = affine::AffineApplyOp::create(
93 rewriter, readOp.getLoc(), offsetCorrectionMap,
94 SmallVector<Value, 1>({oldInnerMostIdx}))
95 .getResult();
96 SmallVector<Value, 8> alignedIdx;
97 alignedIdx.append(adaptor.getIndices().begin(), adaptor.getIndices().end());
98 alignedIdx[alignedIdx.size() - 1] = newInnerMostIdx;
99
100 // Create the aligned transfer read for a vector 2x as long that covers the
101 // elements of the unaligned vector.
102 auto newReadOp = vector::TransferReadOp::create(
103 rewriter, loc, longVecTy, adaptor.getBase(), alignedIdx,
104 adaptor.getPadding());
105
106 // Create a `vector.extract_strided_slice` to extract the unaligned vector.
107 rewriter.replaceOpWithNewOp<vector::ExtractStridedSliceOp>(
108 readOp, newReadOp.getResult(), offset, vLen, 1);
109
110 return success();
111 }
112
115};
116
117// This pattern converts a `vector.transfer_read` with a splat permutation map
118// into a contiguous `vector.transfer_read` followed by a `vector.extract` to
119// obtain the splat value and a `vector.broadcast` to broadcast it into a
120// vector of the right size.
122 : public OpConversionPattern<vector::TransferReadOp> {
123 using OpConversionPattern<vector::TransferReadOp>::OpConversionPattern;
124
126 : OpConversionPattern<vector::TransferReadOp>(context) {}
127
128 LogicalResult
129 matchAndRewrite(vector::TransferReadOp readOp, OpAdaptor adaptor,
130 ConversionPatternRewriter &rewriter) const override {
131 AffineMap map = readOp.getPermutationMap();
132 if (!map.isConstant())
133 return failure();
134
135 Value srcMemRef = adaptor.getBase();
136 SmallVector<Value, 8> indices;
137 Value newIdx;
138 int64_t offset = 0;
139 // If it's a zero-rank memory access
140 if (cast<MemRefType>(srcMemRef.getType()).getRank() == 0) {
141 srcMemRef = memref::ExpandShapeOp::create(
142 rewriter, readOp.getLoc(), SmallVector<int64_t, 1>({1}),
143 srcMemRef, SmallVector<ReassociationIndices, 1>({}))
144 .getResult();
145 newIdx = arith::ConstantOp::create(rewriter, readOp.getLoc(),
146 rewriter.getIndexAttr(0L));
147 indices.push_back(newIdx);
148 } else {
149 indices.append(adaptor.getIndices().begin(), adaptor.getIndices().end());
150 newIdx = indices[indices.size() - 1];
151 // If the innermost index comes from an `affine.apply` op, take the base
152 // as the new innermost index for the new `vector.transfer_read`, and the
153 // offset as the index for the `aievec.broadcast` op.
154 if (auto applyOp = newIdx.getDefiningOp<affine::AffineApplyOp>())
155 if (applyOp.getAffineMap().getNumDims() == 1) {
156 newIdx = applyOp.getMapOperands()[0];
157 offset = applyOp.getAffineMap().compose(ArrayRef<int64_t>{0})[0];
158 }
159 }
160 // XXX: We assume we are reading 1D vectors
161 int64_t vlen = readOp.getVector().getType().getShape()[0];
162 if (offset >= vlen) {
163 // If the splat element is beyond the first vector, we calculate the
164 // address of the vector containing the element.
165 int64_t numElemsToSkip = vlen * (offset / vlen);
166 offset = offset % vlen;
167 auto newAddrMap = AffineMap::get(
168 1, 0, getAffineDimExpr(0, readOp.getContext()) + numElemsToSkip);
169 newIdx =
170 affine::AffineApplyOp::create(rewriter, readOp.getLoc(), newAddrMap,
171 SmallVector<Value, 1>({newIdx}))
172 .getResult();
173 }
174 indices[indices.size() - 1] = newIdx;
175 auto newReadOp = vector::TransferReadOp::create(
176 rewriter, readOp.getLoc(), readOp.getVector().getType(), srcMemRef,
177 indices, adaptor.getPadding());
178 auto extractOp = vector::ExtractOp::create(rewriter, readOp.getLoc(),
179 newReadOp.getResult(),
180 ArrayRef<int64_t>{offset});
181 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(
182 readOp, newReadOp.getVector().getType(), extractOp.getResult());
183 return success();
184 }
185};
186
187// This pattern swaps a UnaryOpA followed by UnaryOpB. This pattern can be used
188// to improve pattern matching for mixed-type arithmetic ops, by getting sign
189// extension ops closer to the single-type arithmetic operations.
190template <class UnaryOpA, class UnaryOpB>
191struct SwapUnaryOpsPattern : public OpRewritePattern<UnaryOpB> {
193 // This function takes the chain of operations A->B, and returns the new type
194 // between B and A after the swap.
195 using InferTypeB2AFnTy = std::function<Type(UnaryOpA aOp, UnaryOpB bOp)>;
197
198 SwapUnaryOpsPattern(MLIRContext *context, InferTypeB2AFnTy inferType)
199 : OpRewritePattern<UnaryOpB>(context),
200 inferTypeB2A(std::move(inferType)) {}
201
202 LogicalResult matchAndRewrite(UnaryOpB bOp,
203 PatternRewriter &rewriter) const override {
204 static_assert(
205 UnaryOpA::template hasTrait<OpTrait::OneOperand>(),
206 "SwapUnaryOps can only be instantiated for single-operand ops");
207 static_assert(
208 UnaryOpB::template hasTrait<OpTrait::OneOperand>(),
209 "SwapUnaryOps can only be instantiated for single-operand ops");
210 UnaryOpA aOp = bOp.getOperand().template getDefiningOp<UnaryOpA>();
211 if (!aOp)
212 return rewriter.notifyMatchFailure(bOp, UnaryOpB::getOperationName() +
213 " not preceeded by " +
214 UnaryOpA::getOperationName());
215
216 Type newA2BTy = inferTypeB2A(aOp, bOp);
217
218 auto newA =
219 UnaryOpB::create(rewriter, bOp->getLoc(), SmallVector<Type>({newA2BTy}),
220 aOp->getOperands(), bOp->getAttrs());
221 auto newB = UnaryOpA::create(rewriter, bOp->getLoc(),
222 SmallVector<Type>({bOp.getResult().getType()}),
223 newA->getResults(), aOp->getAttrs());
224 rewriter.replaceOp(bOp, newB.getResult());
225 return success();
226 }
227};
228
229static SmallVector<Value> collapseInnerMostDimIndices(PatternRewriter &b,
230 Location loc, int numDims,
231 ValueRange indices,
232 ArrayRef<int64_t> shape,
233 AffineMap layout) {
234 (void)layout; // Layout is verified by callers; index computation uses shape.
235 auto newIdxExpr = b.getAffineDimExpr(numDims - 1);
236 int64_t stride = 1;
237 for (int64_t dim = numDims - 2; dim >= 0; dim--) {
238 stride *= shape[shape.size() - (numDims - dim - 1)];
239 newIdxExpr = newIdxExpr + b.getAffineDimExpr(dim) * stride;
240 }
241 auto newIndexMap = AffineMap::get(numDims, 0, newIdxExpr);
242 Value newInnerMostIdxValue =
243 affine::AffineApplyOp::create(b, loc, newIndexMap,
244 indices.take_back(numDims))
245 .getResult();
246 SmallVector<Value> newIdxRange;
247 for (auto idx : indices.drop_back(numDims))
248 newIdxRange.push_back(idx);
249 newIdxRange.push_back(newInnerMostIdxValue);
250 return newIdxRange;
251}
252
253static Value collapseInnerMostShapeDims(PatternRewriter &b, Location loc,
254 int numDims, Value val) {
255 auto memRefTy = cast<MemRefType>(val.getType());
256 auto shape = memRefTy.getShape();
257 int64_t newInnerMostDim = std::accumulate(shape.end() - numDims, shape.end(),
258 1, std::multiplies<>());
259 SmallVector<int64_t, 4> newShape{shape.begin(), shape.end() - numDims + 1};
260 newShape[shape.size() - numDims] = newInnerMostDim;
261 // newShape is constructed above as exactly shape with its trailing
262 // numDims dims merged into one, so this is always a valid collapse.
263 auto reassocIndicesOpt = getReassociationIndicesForCollapse(shape, newShape);
264 assert(reassocIndicesOpt && "newShape must be a valid collapse of shape");
265 const auto &reassocIndices = *reassocIndicesOpt;
266 // Let CollapseShapeOp::inferResultType compute the correct result type,
267 // which preserves strided layout and dynamic offset from the source.
268 auto newMemRefTy =
269 memref::CollapseShapeOp::computeCollapsedType(memRefTy, reassocIndices);
270 return memref::CollapseShapeOp::create(b, loc, newMemRefTy, val,
271 reassocIndices)
272 .getResult();
273}
274
275/// Check if a memref has contiguous row-major strides (each stride equals the
276/// product of trailing dimensions). Dynamic strides are accepted when the
277/// corresponding dimension size is 1. Memrefs with dynamic offsets are fine.
278static bool hasContiguousRowMajorStrides(MemRefType memRefTy) {
279 SmallVector<int64_t> strides;
280 int64_t offset;
281 if (failed(memRefTy.getStridesAndOffset(strides, offset)))
282 return false;
283 auto shape = memRefTy.getShape();
284 int64_t expected = 1;
285 for (int i = shape.size() - 1; i >= 0; --i) {
286 if (strides[i] != ShapedType::kDynamic && strides[i] != expected)
287 return false;
288 if (shape[i] != ShapedType::kDynamic)
289 expected *= shape[i];
290 else
291 expected = ShapedType::kDynamic;
292 }
293 return true;
294}
295
296// This pattern flatten multidimensional `vector.transfer_read` operations
297// replacing them with a `memref.collapse_shape`, a 1D `vector.transfer_read`,
298// and a `vector.shape_cast`.
300 : public OpConversionPattern<vector::TransferReadOp> {
301 using OpConversionPattern<vector::TransferReadOp>::OpConversionPattern;
302
303 LogicalResult
304 matchAndRewrite(vector::TransferReadOp readOp, OpAdaptor adaptor,
305 ConversionPatternRewriter &rewriter) const override {
306 // We can only deal with unmasked transfer ops with an identity permutation
307 // map.
308 if (!adaptor.getPermutationMap().isMinorIdentity() || adaptor.getMask())
309 return failure();
310 VectorType vectorTy = readOp.getVector().getType();
311 if (vectorTy.getRank() < 2)
312 return failure();
313 // Work only on bufferized reads
314 MemRefType memRefTy = dyn_cast<MemRefType>(adaptor.getBase().getType());
315 if (!memRefTy)
316 return failure();
317 auto memRefShape = memRefTy.getShape();
318 auto vecShape = vectorTy.getShape();
319
320 auto newVectorTy =
321 VectorType::get({std::accumulate(vecShape.begin(), vecShape.end(), 1,
322 std::multiplies<>())},
323 vectorTy.getElementType());
324 if (!hasContiguousRowMajorStrides(memRefTy))
325 return failure();
326
327 AffineMap layout = memRefTy.getLayout().getAffineMap();
328 auto newIndices =
329 collapseInnerMostDimIndices(rewriter, readOp.getLoc(), vecShape.size(),
330 adaptor.getIndices(), memRefShape, layout);
331 auto newSource = collapseInnerMostShapeDims(
332 rewriter, readOp.getLoc(), vecShape.size(), adaptor.getBase());
333 auto newVector = vector::TransferReadOp::create(
334 rewriter, readOp.getLoc(), newVectorTy, newSource, newIndices,
335 /*padding*/
336 arith::getZeroConstant(rewriter, readOp.getLoc(),
337 newVectorTy.getElementType()));
338
339 auto inBoundsArrayAttrOpt = adaptor.getInBounds();
340 if (inBoundsArrayAttrOpt) {
341 SmallVector<bool> inBounds =
342 llvm::to_vector(inBoundsArrayAttrOpt.getAsValueRange<BoolAttr>());
343 SmallVector<bool> newInBounds({false});
344 newInBounds[0] = llvm::all_of(inBounds, [](bool v) { return v; });
345 newVector.getProperties().setInBounds(
346 rewriter.getBoolArrayAttr(newInBounds));
347 }
348
349 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(readOp, vectorTy,
350 newVector);
351
352 return success();
353 }
354};
355
356// This pattern flatten multidimensional `vector.transfer_write` operations
357// replacing them with a `memref.collapse_shape`, a `vector.shape_cast`, and a
358// 1D `vector.transfer_write`,
360 : public OpConversionPattern<vector::TransferWriteOp> {
361 using OpConversionPattern<vector::TransferWriteOp>::OpConversionPattern;
362
363 LogicalResult
364 matchAndRewrite(vector::TransferWriteOp writeOp, OpAdaptor adaptor,
365 ConversionPatternRewriter &rewriter) const override {
366 // We can only deal with unmasked transfer ops with an identity permutation
367 // map.
368 if (!adaptor.getPermutationMap().isMinorIdentity() || adaptor.getMask())
369 return failure();
370 VectorType vectorTy = cast<VectorType>(adaptor.getValueToStore().getType());
371 if (vectorTy.getRank() < 2)
372 return failure();
373 // Work only on bufferized reads
374 MemRefType memRefTy = dyn_cast<MemRefType>(adaptor.getBase().getType());
375 if (!memRefTy)
376 return failure();
377 auto memRefShape = memRefTy.getShape();
378 auto vecShape = vectorTy.getShape();
379
380 if (!hasContiguousRowMajorStrides(memRefTy))
381 return failure();
382
383 auto newVectorTy =
384 VectorType::get({std::accumulate(vecShape.begin(), vecShape.end(), 1,
385 std::multiplies<>())},
386 vectorTy.getElementType());
387 AffineMap layout = memRefTy.getLayout().getAffineMap();
388 auto newVector =
389 vector::ShapeCastOp::create(rewriter, writeOp.getLoc(), newVectorTy,
390 adaptor.getValueToStore())
391 .getResult();
392 auto newIndices =
393 collapseInnerMostDimIndices(rewriter, writeOp.getLoc(), vecShape.size(),
394 adaptor.getIndices(), memRefShape, layout);
395 auto newSource = collapseInnerMostShapeDims(
396 rewriter, writeOp.getLoc(), vecShape.size(), adaptor.getBase());
397
398 auto newOp = rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
399 writeOp, newVector, newSource, newIndices);
400
401 auto inBoundsArrayAttrOpt = adaptor.getInBounds();
402 if (inBoundsArrayAttrOpt) {
403 SmallVector<bool> inBounds =
404 llvm::to_vector(inBoundsArrayAttrOpt.getAsValueRange<BoolAttr>());
405 SmallVector<bool> newInBounds({false});
406 newInBounds[0] = llvm::all_of(inBounds, [](bool v) { return v; });
407 newOp.getProperties().setInBounds(rewriter.getBoolArrayAttr(newInBounds));
408 }
409
410 return success();
411 }
412};
413
414/// Utility function to check if all provided indices are constant zero values.
415/// @return success() if all indices are constant zeros, failure() otherwise
416static LogicalResult isAllZeroOffsetAccess(mlir::OperandRange indices) {
417 if (!llvm::all_of(indices, [](Value val) {
418 IntegerAttr attr;
419 if (!matchPattern(val, m_Constant(&attr)))
420 return false;
421 return attr.getInt() == 0;
422 })) {
423 return failure();
424 }
425 return success();
426}
427
428/// Utility function to convert OpFoldResult offsets from a SubView operation
429/// into a vector of Values.
430static SmallVector<Value> opFoldResultsToValues(PatternRewriter &rewriter,
431 Location loc,
432 memref::SubViewOp subViewOp) {
433 OpBuilder::InsertionGuard g(rewriter);
434 rewriter.setInsertionPoint(subViewOp);
435 SmallVector<Value> newIndices;
436 for (OpFoldResult offset : subViewOp.getMixedOffsets()) {
437 Value indexVal;
438 if (auto attr = dyn_cast<Attribute>(offset)) {
439 indexVal = arith::ConstantIndexOp::create(
440 rewriter, loc, cast<IntegerAttr>(attr).getInt());
441 } else {
442 indexVal = cast<Value>(offset);
443 }
444 newIndices.push_back(indexVal);
445 }
446 return newIndices;
447}
448
449/// Pattern to canonicalize trivial vector.transfer_read operations on subviews.
450///
451/// This pattern eliminates unnecessary memref.subview operations when the
452/// transfer_read accesses the subview with all-zero indices. It transforms:
453///
454/// INPUT:
455/// %subview = memref.subview %memref [offset0, offset1, ...]
456/// %result = vector.transfer_read %subview[0, 0, ...]
457///
458/// OUTPUT:
459/// %result = vector.transfer_read %memref[offset0, offset1, ...]
460///
461/// The pattern only matches when:
462/// - The base of transfer_read is defined by a memref.subview operation
463/// - All indices in the transfer_read are constant zeros
465 : public OpRewritePattern<vector::TransferReadOp> {
466 using OpRewritePattern<vector::TransferReadOp>::OpRewritePattern;
467
468 LogicalResult matchAndRewrite(vector::TransferReadOp readOp,
469 PatternRewriter &rewriter) const override {
470 // Check if the base memref comes from a subview operation
471 auto subViewOp = dyn_cast_if_present<memref::SubViewOp>(
472 readOp.getBase().getDefiningOp());
473 if (!subViewOp)
474 return failure();
475
476 // Verify that all access indices are zero
477 if (failed(isAllZeroOffsetAccess(readOp.getIndices())))
478 return failure();
479
480 // Convert subview offsets to explicit index values
481 SmallVector<Value> newIndices =
482 opFoldResultsToValues(rewriter, readOp.getLoc(), subViewOp);
483
484 // Replace with direct access to the original memref using subview offsets
485 rewriter.replaceOpWithNewOp<vector::TransferReadOp>(
486 readOp, readOp.getType(), subViewOp.getSource(), newIndices,
487 readOp.getPadding(), readOp.getInBoundsValues());
488 return success();
489 }
490};
491
492/// Pattern to canonicalize trivial vector.transfer_write operations on
493/// subviews.
494///
495/// This pattern eliminates unnecessary memref.subview operations when the
496/// transfer_write accesses the subview with all-zero indices. It transforms:
497///
498/// INPUT:
499/// %subview = memref.subview %memref [offset0, offset1, ...]
500/// vector.transfer_write %value, %subview[0, 0, ...]
501///
502/// OUTPUT:
503/// vector.transfer_write %value, %memref[offset0, offset1, ...]
504///
505/// The pattern only matches when:
506/// - The base of transfer_write is defined by a memref.subview operation
507/// - All indices in the transfer_write are constant zeros
509 : public OpRewritePattern<vector::TransferWriteOp> {
510 using OpRewritePattern<vector::TransferWriteOp>::OpRewritePattern;
511
512 LogicalResult matchAndRewrite(vector::TransferWriteOp writeOp,
513 PatternRewriter &rewriter) const override {
514 // Check if the base memref comes from a subview operation
515 auto subViewOp = dyn_cast_if_present<memref::SubViewOp>(
516 writeOp.getBase().getDefiningOp());
517 if (!subViewOp)
518 return failure();
519
520 // Verify that all access indices are zero
521 if (failed(isAllZeroOffsetAccess(writeOp.getIndices())))
522 return failure();
523
524 // Convert subview offsets to explicit index values
525 SmallVector<Value> newIndices =
526 opFoldResultsToValues(rewriter, writeOp.getLoc(), subViewOp);
527
528 // Create new transfer_write with direct access to original memref
529 vector::TransferWriteOp::create(rewriter, writeOp.getLoc(),
530 writeOp.getVector(), subViewOp.getSource(),
531 newIndices, writeOp.getInBoundsValues());
532
533 // Remove the original transfer_write operation
534 rewriter.eraseOp(writeOp);
535 return success();
536 }
537};
538
539// Carry an n-D vector through an scf.for loop as a rank-1 vector. The LLVM
540// lowering turns an n-D vector into an array of rank-1 rows, so a loop that
541// carries one splits it into rows and rebuilds it on every iteration instead
542// of keeping it in registers. AIE2P lowers n-D elementwise ops by flattening
543// them to rank 1, so the shape casts added here at the loop boundary fold
544// against the ones that flattening adds. Each match rewrites one iter_arg.
546 using OpRewritePattern::OpRewritePattern;
547
548 LogicalResult matchAndRewrite(scf::ForOp forOp,
549 PatternRewriter &rewriter) const override {
550 for (OpOperand &init : forOp.getInitArgsMutable()) {
551 auto vecType = dyn_cast<VectorType>(init.get().getType());
552 if (!vecType || vecType.getRank() < 2 || vecType.isScalable())
553 continue;
554 Value flat = vector::ShapeCastOp::create(rewriter, forOp.getLoc(),
555 getFlattenedVectorType(vecType),
556 init.get());
557 auto shapeCast = [](OpBuilder &builder, Location loc, Type type,
558 Value source) -> Value {
559 return vector::ShapeCastOp::create(builder, loc, type, source);
560 };
561 rewriter.replaceOp(forOp, scf::replaceAndCastForOpIterArg(
562 rewriter, forOp, init, flat, shapeCast));
563 return success();
564 }
565 return failure();
566 }
567};
568
569//============================================================================//
570//============ AIE2 canonicalization conversion patterns ===============//
571//============================================================================//
572
574 : public OpRewritePattern<vector::InsertOp> {
575
576 using OpRewritePattern<vector::InsertOp>::OpRewritePattern;
577
578 LogicalResult matchAndRewrite(vector::InsertOp insOp,
579 PatternRewriter &rewriter) const override {
580 auto insSrcTy = dyn_cast<VectorType>(insOp.getValueToStoreType());
581 if (!insSrcTy)
582 return failure();
583
584 auto srcShape = insSrcTy.getShape();
585 auto dstShape = insOp.getDestVectorType().getShape();
586
587 unsigned long numLeadUnitDimDst = 0;
588 while (numLeadUnitDimDst < dstShape.size() &&
589 dstShape[numLeadUnitDimDst] == 1)
590 numLeadUnitDimDst++;
591
592 if (!numLeadUnitDimDst)
593 return failure();
594
595 unsigned long numLeadUnitDimSrc = 0;
596 while (numLeadUnitDimSrc < srcShape.size() &&
597 srcShape[numLeadUnitDimSrc] == 1)
598 numLeadUnitDimSrc++;
599
600 SmallVector<int64_t> nonLeadUnitDimDstShape(
601 dstShape.begin() + numLeadUnitDimDst, dstShape.end());
602 SmallVector<int64_t> nonLeadUnitDimSrcShape(
603 srcShape.begin() + numLeadUnitDimSrc, srcShape.end());
604
605 if (nonLeadUnitDimSrcShape != nonLeadUnitDimDstShape)
606 return failure();
607
608 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(
609 insOp, insOp.getDestVectorType(), insOp.getValueToStore());
610 return success();
611 }
612};
613
614//============================================================================//
615//================ Common AIE canonicalization configuration =================//
616//============================================================================//
617static void
618configureCommonAIECanonicalizeLegalizations(ConversionTarget &target) {
619 target.addLegalDialect<arith::ArithDialect, affine::AffineDialect,
620 memref::MemRefDialect, vector::VectorDialect,
621 ub::UBDialect>();
622}
623
624static void
625populateCommonAIECanonicalizeConversionPatterns(RewritePatternSet &patterns) {
627 patterns.getContext());
628}
629
630//============================================================================//
631//============== AIEv1-specific canonicalization configuration ===============//
632//============================================================================//
633
634static void configureAIEv1CanonicalizeLegalizations(ConversionTarget &target) {
635 target.addDynamicallyLegalOp<vector::TransferReadOp>(
636 [](vector::TransferReadOp op) {
637 return !op.getPermutationMap().isConstant() &&
638 getTransferReadAlignmentOffset(op, op.getVectorType(), 128)
639 .value_or(0) == 0;
640 });
641}
642
643static void
644populateAIEv1CanonicalizeConversionPatterns(RewritePatternSet &patterns) {
645 patterns.add<SplitUnalignedTransferReadPattern>(patterns.getContext(), 512,
646 128);
647}
648
649//============================================================================//
650//============== AIE2-specific canonicalization configuration ===============//
651//============================================================================//
652
653static void configureAIE2CanonicalizeLegalizations(ConversionTarget &target) {
654 target.addDynamicallyLegalOp<vector::TransferReadOp>(
655 [](vector::TransferReadOp op) {
656 return !op.getPermutationMap().isConstant() &&
657 getTransferReadAlignmentOffset(op, op.getVectorType(), 256)
658 .value_or(0) == 0 &&
659 op.getVector().getType().getRank() < 2;
660 });
661 target.addDynamicallyLegalOp<vector::TransferWriteOp>(
662 [](vector::TransferWriteOp op) {
663 return cast<VectorType>(op.getVector().getType()).getRank() < 2;
664 });
665}
666
667static void
668populateAIE2CanonicalizeConversionPatterns(RewritePatternSet &patterns) {
669 patterns.add<SplitUnalignedTransferReadPattern>(patterns.getContext(), 1024,
670 256);
672 FlattenMultDimTransferWritePattern>(patterns.getContext());
673}
674
675//============================================================================//
676//=================== Common AIE Canonicalization Passes =====================//
677//============================================================================//
678
679//===----------------------------------------------------------------------===//
680// BF16 Emulation: Emulate f32 vector arithmetic using bf16 operations.
681//===----------------------------------------------------------------------===//
682
683// Smart truncation helper: if the value was produced by arith.extf from bf16,
684// reuse the bf16 source directly to avoid redundant extf->truncf chains.
685static Value smartTruncF32ToBF16(PatternRewriter &rewriter, Location loc,
686 Value val, Type bf16Type) {
687 if (auto extfOp = val.getDefiningOp<arith::ExtFOp>()) {
688 if (extfOp.getIn().getType() == bf16Type)
689 return extfOp.getIn();
690 }
691 return arith::TruncFOp::create(rewriter, loc, bf16Type, val);
692}
693
694// `v` rounded to the nearest bf16, still in its own semantics.
695static APFloat roundedToBF16(APFloat v) {
696 bool losesInfo = false;
697 const llvm::fltSemantics &sem = v.getSemantics();
698 v.convert(APFloat::BFloat(), APFloat::rmNearestTiesToEven, &losesInfo);
699 v.convert(sem, APFloat::rmNearestTiesToEven, &losesInfo);
700 return v;
701}
702
703// True if `val` is an f32 vector whose elements are all bf16 values.
704static bool holdsBF16(Value val) {
705 // a splat of a scalar widened from bf16
706 if (auto bcastOp = val.getDefiningOp<vector::BroadcastOp>())
707 val = bcastOp.getSource();
708 if (auto extfOp = val.getDefiningOp<arith::ExtFOp>())
709 return getElementTypeOrSelf(extfOp.getIn().getType()).isBF16();
710 DenseFPElementsAttr dense;
711 return matchPattern(val, m_Constant(&dense)) &&
712 llvm::all_of(dense.getValues<APFloat>(), [](const APFloat &v) {
713 return roundedToBF16(v).bitwiseIsEqual(v);
714 });
715}
716
717// `val` rounded to bf16, still as f32. Constants are rounded here.
718static Value roundToBF16(PatternRewriter &rewriter, Location loc, Value val) {
719 if (holdsBF16(val))
720 return val;
721 auto vecType = cast<VectorType>(val.getType());
722 DenseFPElementsAttr dense;
723 if (matchPattern(val, m_Constant(&dense))) {
724 SmallVector<APFloat> values;
725 for (const APFloat &v : dense.getValues<APFloat>())
726 values.push_back(roundedToBF16(v));
727 return arith::ConstantOp::create(rewriter, loc,
728 DenseElementsAttr::get(vecType, values));
729 }
730 auto bf16VecType =
731 VectorType::get(vecType.getShape(), rewriter.getBF16Type());
732 Value bf16 = smartTruncF32ToBF16(rewriter, loc, val, bf16VecType);
733 return arith::ExtFOp::create(rewriter, loc, vecType, bf16,
734 /*fastmath=*/nullptr);
735}
736
737static bool isF32Vector(Type type) {
738 auto vecType = dyn_cast<VectorType>(type);
739 return vecType && vecType.getElementType().isF32();
740}
741
742/// Pattern to drop `bf16 -> f32 -> bf16` round trips.
743///
744/// Arith used to fold these but no longer does: widening a signaling NaN quiets
745/// it, so the original bit pattern cannot be recovered. This pass has always
746/// made that trade anyway -- `smartTruncF32ToBF16` collapses the same pair on
747/// sight -- so fold here to keep the pipeline's pre-existing behavior rather
748/// than leaving the round trips for the backend.
749struct FoldBF16RoundTripPattern : public OpRewritePattern<arith::TruncFOp> {
750 using OpRewritePattern::OpRewritePattern;
751
752 LogicalResult matchAndRewrite(arith::TruncFOp op,
753 PatternRewriter &rewriter) const override {
754 auto resultType = dyn_cast<VectorType>(op.getType());
755 auto intermediateType = dyn_cast<VectorType>(op.getIn().getType());
756 if (!resultType || !intermediateType ||
757 !resultType.getElementType().isBF16() ||
758 !intermediateType.getElementType().isF32())
759 return failure();
760
761 auto extfOp = op.getIn().getDefiningOp<arith::ExtFOp>();
762 if (!extfOp || extfOp.getIn().getType() != resultType)
763 return failure();
764
765 rewriter.replaceOp(op, extfOp.getIn());
766 return success();
767 }
768};
769
770/// Pattern to emulate f32 binary vector arithmetic ops in bf16.
771/// For an op like: %r = arith.addf %a, %b : vector<16xf32>
772/// Produces:
773/// %a_bf16 = arith.truncf %a : vector<16xf32> to vector<16xbf16>
774/// %b_bf16 = arith.truncf %b : vector<16xf32> to vector<16xbf16>
775/// %r_bf16 = arith.addf %a_bf16, %b_bf16 : vector<16xbf16>
776/// %r = arith.extf %r_bf16 : vector<16xbf16> to vector<16xf32>
777template <typename OpTy>
780
781 LogicalResult matchAndRewrite(OpTy op,
782 PatternRewriter &rewriter) const override {
783 auto resultType = dyn_cast<VectorType>(op.getType());
784 if (!resultType || !resultType.getElementType().isF32())
785 return failure();
786
787 Location loc = op.getLoc();
788 auto bf16VecType =
789 VectorType::get(resultType.getShape(), rewriter.getBF16Type());
790
791 Value lhsBF16 =
792 smartTruncF32ToBF16(rewriter, loc, op.getLhs(), bf16VecType);
793 Value rhsBF16 =
794 smartTruncF32ToBF16(rewriter, loc, op.getRhs(), bf16VecType);
795
796 Value newResult =
797 OpTy::create(rewriter, loc, bf16VecType, lhsBF16, rhsBF16);
798 auto extOp = arith::ExtFOp::create(rewriter, loc, resultType, newResult,
799 /*fastmath=*/nullptr);
800 rewriter.replaceOp(op, extOp);
801 return success();
802 }
803};
804
805/// Pattern to emulate f32 comparison ops in bf16.
806/// Result type stays vector<Nxi1>, only operands are truncated.
807struct EmulateCmpFF32InBF16Pattern : public OpRewritePattern<arith::CmpFOp> {
808 using OpRewritePattern::OpRewritePattern;
809
810 LogicalResult matchAndRewrite(arith::CmpFOp op,
811 PatternRewriter &rewriter) const override {
812 auto lhsType = dyn_cast<VectorType>(op.getLhs().getType());
813 if (!lhsType || !lhsType.getElementType().isF32())
814 return failure();
815
816 Location loc = op.getLoc();
817 auto bf16VecType =
818 VectorType::get(lhsType.getShape(), rewriter.getBF16Type());
819
820 Value lhsBF16 =
821 smartTruncF32ToBF16(rewriter, loc, op.getLhs(), bf16VecType);
822 Value rhsBF16 =
823 smartTruncF32ToBF16(rewriter, loc, op.getRhs(), bf16VecType);
824
825 rewriter.replaceOpWithNewOp<arith::CmpFOp>(op, op.getPredicate(), lhsBF16,
826 rhsBF16);
827 return success();
828 }
829};
830
831/// Pattern to emulate f32 select ops in bf16.
832/// Condition stays vector<Nxi1>, true/false values are truncated.
834 : public OpRewritePattern<arith::SelectOp> {
835 using OpRewritePattern::OpRewritePattern;
836
837 LogicalResult matchAndRewrite(arith::SelectOp op,
838 PatternRewriter &rewriter) const override {
839 auto resultType = dyn_cast<VectorType>(op.getType());
840 if (!resultType || !resultType.getElementType().isF32())
841 return failure();
842
843 Location loc = op.getLoc();
844 auto bf16VecType =
845 VectorType::get(resultType.getShape(), rewriter.getBF16Type());
846
847 Value trueValBF16 =
848 smartTruncF32ToBF16(rewriter, loc, op.getTrueValue(), bf16VecType);
849 Value falseValBF16 =
850 smartTruncF32ToBF16(rewriter, loc, op.getFalseValue(), bf16VecType);
851
852 Value newResult = arith::SelectOp::create(rewriter, loc, op.getCondition(),
853 trueValBF16, falseValBF16);
854 auto extOp = arith::ExtFOp::create(rewriter, loc, resultType, newResult,
855 /*fastmath=*/nullptr);
856 rewriter.replaceOp(op, extOp);
857 return success();
858 }
859};
860
861/// Pattern to emulate an f32 multiply with bf16 inputs. The product of two
862/// bf16 values fits in f32 except at the ends of the exponent range, so only
863/// the inputs are rounded:
864/// %r = arith.mulf %a, %b : vector<16xf32>
865/// becomes
866/// %r = arith.mulf (extf (truncf %a)), (extf (truncf %b)) : vector<16xf32>
867struct EmulateMulFInBF16Pattern : public OpRewritePattern<arith::MulFOp> {
868 using OpRewritePattern::OpRewritePattern;
869
870 LogicalResult matchAndRewrite(arith::MulFOp op,
871 PatternRewriter &rewriter) const override {
872 if (!isF32Vector(op.getType()) ||
873 (holdsBF16(op.getLhs()) && holdsBF16(op.getRhs())))
874 return failure();
875 Location loc = op.getLoc();
876 Value lhs = roundToBF16(rewriter, loc, op.getLhs());
877 Value rhs = roundToBF16(rewriter, loc, op.getRhs());
878 rewriter.replaceOpWithNewOp<arith::MulFOp>(op, lhs, rhs,
879 op.getFastmathAttr());
880 return success();
881 }
882};
883
884static bool contracts(arith::FastMathFlagsAttr flags) {
885 return flags &&
886 bitEnumContainsAll(flags.getValue(), arith::FastMathFlags::contract);
887}
888
889/// Whether `op` may take a multiply into a vector.fma: it allows contraction
890/// and has a type the FMA lowerings take, a 1-D f32 vector of 16 or 32 lanes.
891static bool fusesIntoFMA(arith::AddFOp op) {
892 auto type = dyn_cast<VectorType>(op.getType());
893 return isF32Vector(op.getType()) && type.getRank() == 1 &&
894 (type.getNumElements() == 16 || type.getNumElements() == 32) &&
895 contracts(op.getFastmathAttr());
896}
897
898/// Pattern to fold an f32 multiply into the add that is its only user, as a
899/// vector.fma with bf16 inputs. Only when both ops allow contraction: the
900/// fused result can differ where the product underflows or overflows.
901struct FuseMulAddFInBF16Pattern : public OpRewritePattern<arith::AddFOp> {
902 using OpRewritePattern::OpRewritePattern;
903
904 LogicalResult matchAndRewrite(arith::AddFOp op,
905 PatternRewriter &rewriter) const override {
906 if (!fusesIntoFMA(op))
907 return failure();
908 for (auto [mulOperand, acc] : {std::pair(op.getLhs(), op.getRhs()),
909 std::pair(op.getRhs(), op.getLhs())}) {
910 auto mulOp = mulOperand.getDefiningOp<arith::MulFOp>();
911 if (!mulOp || !mulOp->hasOneUse() || !contracts(mulOp.getFastmathAttr()))
912 continue;
913 Location loc = op.getLoc();
914 Value lhs = roundToBF16(rewriter, loc, mulOp.getLhs());
915 Value rhs = roundToBF16(rewriter, loc, mulOp.getRhs());
916 rewriter.replaceOpWithNewOp<vector::FMAOp>(op, lhs, rhs, acc);
917 rewriter.eraseOp(mulOp);
918 return success();
919 }
920 return failure();
921 }
922};
923
924/// FuseMulAddFInBF16Pattern for a multiply whose inputs already hold bf16
925/// values, outside bf16 emulation; likewise only when both ops allow
926/// contraction.
927struct FuseExactMulAddFPattern : public OpRewritePattern<arith::AddFOp> {
928 using OpRewritePattern::OpRewritePattern;
929
930 LogicalResult matchAndRewrite(arith::AddFOp op,
931 PatternRewriter &rewriter) const override {
932 if (!fusesIntoFMA(op))
933 return failure();
934 for (auto [mulOperand, acc] : {std::pair(op.getLhs(), op.getRhs()),
935 std::pair(op.getRhs(), op.getLhs())}) {
936 auto mulOp = mulOperand.getDefiningOp<arith::MulFOp>();
937 if (!mulOp || !mulOp->hasOneUse() ||
938 !contracts(mulOp.getFastmathAttr()) || !holdsBF16(mulOp.getLhs()) ||
939 !holdsBF16(mulOp.getRhs()))
940 continue;
941 rewriter.replaceOpWithNewOp<vector::FMAOp>(op, mulOp.getLhs(),
942 mulOp.getRhs(), acc);
943 rewriter.eraseOp(mulOp);
944 return success();
945 }
946 return failure();
947 }
948};
949
950/// Pattern to emulate f32 vector.fma with bf16 inputs. The accumulator stays
951/// f32.
952struct EmulateFMAF32InBF16Pattern : public OpRewritePattern<vector::FMAOp> {
953 using OpRewritePattern::OpRewritePattern;
954
955 LogicalResult matchAndRewrite(vector::FMAOp op,
956 PatternRewriter &rewriter) const override {
957 if (!isF32Vector(op.getType()) ||
958 (holdsBF16(op.getLhs()) && holdsBF16(op.getRhs())))
959 return failure();
960 Location loc = op.getLoc();
961 Value lhs = roundToBF16(rewriter, loc, op.getLhs());
962 Value rhs = roundToBF16(rewriter, loc, op.getRhs());
963 rewriter.replaceOpWithNewOp<vector::FMAOp>(op, lhs, rhs, op.getAcc());
964 return success();
965 }
966};
967
968/// Pattern to emulate f32 unary vector ops in bf16.
969template <typename OpTy>
972
973 LogicalResult matchAndRewrite(OpTy op,
974 PatternRewriter &rewriter) const override {
975 auto resultType = dyn_cast<VectorType>(op.getType());
976 if (!resultType || !resultType.getElementType().isF32())
977 return failure();
978
979 Location loc = op.getLoc();
980 auto bf16VecType =
981 VectorType::get(resultType.getShape(), rewriter.getBF16Type());
982
983 Value inputBF16 =
984 smartTruncF32ToBF16(rewriter, loc, op->getOperand(0), bf16VecType);
985
986 Value newResult = OpTy::create(rewriter, loc, bf16VecType, inputBF16);
987 auto extOp = arith::ExtFOp::create(rewriter, loc, resultType, newResult,
988 /*fastmath=*/nullptr);
989 rewriter.replaceOp(op, extOp);
990 return success();
991 }
992};
993
995 : public PassWrapper<BF16EmulationPass, OperationPass<>> {
996
997 void runOnOperation() override {
998 auto *op = getOperation();
999 MLIRContext *context = &getContext();
1000 RewritePatternSet patterns(context);
1001
1002 // Multiplies take bf16 inputs; sums stay f32, which the accumulator
1003 // adds natively.
1004 patterns.add<FuseMulAddFInBF16Pattern>(context, /*benefit=*/2);
1008
1009 // Note: arith.divf is NOT demoted because bf16 vector divf is unsupported
1010 // on all AIE targets (Peano does not legalize G_FDIV on <16 x s16>).
1011
1012 // Special-case ops (excluding ReductionOp — its scalar accumulator
1013 // and result lower to fp_to_bf16/bf16_to_fp which older Peano versions
1014 // cannot select on AIE2P; reductions are intentionally left in f32
1015 // to avoid these scalar bf16 conversions)
1017 context);
1018
1019 // Unary ops
1022
1023 // Clean up the round trips the demotion itself introduces.
1024 patterns.add<FoldBF16RoundTripPattern>(context);
1025
1026 (void)applyPatternsGreedily(op, std::move(patterns));
1027 }
1028};
1029
1030std::unique_ptr<::mlir::Pass> xilinx::aievec::createBF16EmulationPass() {
1031 return std::make_unique<BF16EmulationPass>();
1032}
1033
1034// A broadcast of a vector that only adds leading dims, as the vector repeated
1035// by a 1-D shuffle (mask [i % n]). The AIE2P shuffle lowering then sees the
1036// repetition, alone or together with a shuffle that reads it, as one gather.
1038 : public OpRewritePattern<vector::BroadcastOp> {
1039 using OpRewritePattern::OpRewritePattern;
1040
1041 LogicalResult matchAndRewrite(vector::BroadcastOp op,
1042 PatternRewriter &rewriter) const override {
1043 auto srcType = dyn_cast<VectorType>(op.getSourceType());
1044 VectorType resType = op.getResultVectorType();
1045 if (!srcType || srcType.getRank() != 1 || resType.getRank() < 2 ||
1046 resType.getShape().back() != srcType.getNumElements() ||
1047 !srcType.getElementType().isIntOrFloat() || resType.isScalable())
1048 return failure();
1049 int64_t n = srcType.getNumElements();
1050 SmallVector<int64_t> mask;
1051 for (int64_t i = 0; i < resType.getNumElements(); ++i)
1052 mask.push_back(i % n);
1053 Value src = op.getSource();
1054 Value flat =
1055 vector::ShuffleOp::create(rewriter, op.getLoc(), src, src, mask);
1056 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, resType, flat);
1057 return success();
1058 }
1059};
1060
1062 : public PassWrapper<VectorBroadcastLoweringPass, OperationPass<>> {
1063 VectorBroadcastLoweringPass(bool aie2p = false) : aie2p(aie2p) {}
1064
1065 void runOnOperation() override {
1066 auto *op = getOperation();
1067 MLIRContext *context = &getContext();
1068 RewritePatternSet patterns(context);
1069 populateVectorBroadcastLoweringPatterns(patterns);
1071 patterns.getContext());
1072 // Ahead of the generic lowering, which builds the result a row at a time.
1073 if (aie2p)
1074 patterns.add<BroadcastLeadingDimsToRepeatPattern>(patterns.getContext(),
1075 /*benefit=*/2);
1076
1077 (void)applyPatternsGreedily(op, std::move(patterns));
1078 }
1079
1080private:
1081 bool aie2p;
1082};
1083
1084static std::unique_ptr<::mlir::Pass>
1085createVectorBroadcastLoweringPass(bool aie2p = false) {
1086 return std::make_unique<VectorBroadcastLoweringPass>(aie2p);
1087}
1088
1089// This pass converts standard vector ops into a subset of `Vector` ops more
1090// amenable to being converted to `AIEVec`. So far, this process consists of
1091// two steps:
1092// 1) Replace splat transfer reads with contiguous transfer reads followed
1093// by `extract` + `splat` operations.
1094// 2) Split unaligned transfer reads into a wider aligned transfer read
1095// followed by a `vector.extract_strided_slice` operation.
1097 : public PassWrapper<CanonicalizeVectorForAIEVecPass, OperationPass<>> {
1098 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(CanonicalizeVectorForAIEVecPass)
1099
1103
1109
1110 // In case we want to register this pass as a standalone pass for test
1111 // purposes.
1112 StringRef getArgument() const final {
1113 return "test-canonicalize-vector-for-aievec";
1114 }
1115
1116 StringRef getDescription() const final {
1117 return "Canonicalize vector operations for AIEVec conversion";
1118 }
1119
1120 void getDependentDialects(DialectRegistry &registry) const override {
1121 registry
1122 .insert<arith::ArithDialect, memref::MemRefDialect,
1123 vector::VectorDialect, affine::AffineDialect, ub::UBDialect>();
1124 }
1125
1126 Option<std::string> aieTarget{
1127 *this, "aie-target",
1128 llvm::cl::desc(
1129 "Select AIE version: \"aie\", \"aie2\", or \"aie2p\". This will "
1130 "determine the vector size and available operations."),
1131 llvm::cl::init("aie")};
1132
1133 void runOnOperation() override {
1134 auto *op = getOperation();
1135 MLIRContext *context = &getContext();
1136 RewritePatternSet patterns(context);
1137 ConversionTarget target(*context);
1138
1139 std::optional<AIEVecTargetInfo> targetInfo =
1141 if (!targetInfo) {
1142 signalPassFailure();
1143 return;
1144 }
1145
1146 populateCommonAIECanonicalizeConversionPatterns(patterns);
1147 configureCommonAIECanonicalizeLegalizations(target);
1148 if (targetInfo->isAIE1()) {
1149 populateAIEv1CanonicalizeConversionPatterns(patterns);
1150 configureAIEv1CanonicalizeLegalizations(target);
1151 } else {
1152 populateAIE2CanonicalizeConversionPatterns(patterns);
1153 configureAIE2CanonicalizeLegalizations(target);
1154 }
1155
1156 {
1157 RewritePatternSet patterns(context);
1160 if (targetInfo->isAIE2P())
1162 context);
1163 (void)applyPatternsGreedily(op, std::move(patterns));
1164 }
1165
1166 if (failed(applyPartialConversion(op, target, std::move(patterns)))) {
1167 signalPassFailure();
1168 }
1169 }
1170};
1171
1172static std::unique_ptr<::mlir::Pass> createCanonicalizeVectorForAIEVecPass(
1173 const CanonicalizeVectorForAIEVecOptions &options) {
1174 return std::make_unique<CanonicalizeVectorForAIEVecPass>(options);
1175}
1176
1178 : public PassWrapper<ReorderOperationsPass, OperationPass<>> {
1179
1180 void runOnOperation() override {
1181 auto *op = getOperation();
1182 MLIRContext *context = &getContext();
1183 RewritePatternSet patterns(context);
1184
1186 patterns.getContext(),
1187 [](arith::ExtSIOp extOp, vector::BroadcastOp bcastOp) -> Type {
1188 Type extInElemTy = extOp.getIn().getType();
1189 auto extInVecTy = dyn_cast<VectorType>(extInElemTy);
1190 if (extInVecTy)
1191 extInElemTy = extInVecTy.getElementType();
1192 return VectorType::get(bcastOp.getResultVectorType().getShape(),
1193 extInElemTy);
1194 });
1195
1196 (void)applyPatternsGreedily(op, std::move(patterns));
1197 }
1198};
1199
1200static std::unique_ptr<::mlir::Pass> createReorderOperationsPass() {
1201 return std::make_unique<ReorderOperationsPass>();
1202}
1203
1204//============================================================================//
1205//=============== Main Vector2Vector Pipeline Configuration ==================//
1206//============================================================================//
1207
1209 OpPassManager &pm, const CanonicalizeVectorForAIEVecOptions &options) {
1210 // Add `Vector` code canonicalization passes
1211 // TODO: Add passes to unroll vector with unsupported types
1212 // TODO: Add passes to split vectors that won't fit in registers
1213
1214 // If bf16-emulation is enabled, demote f32 vector arithmetic to bf16 first.
1215 if (options.enableBF16Emulation)
1216 pm.addPass(createBF16EmulationPass());
1217
1218 pm.addPass(createReorderOperationsPass());
1219 pm.addPass(createCopyRemovalPass());
1220 std::optional<AIEVecTargetInfo> targetInfo =
1222 pm.addPass(
1223 createVectorBroadcastLoweringPass(targetInfo && targetInfo->isAIE2P()));
1224 pm.addPass(createCanonicalizeVectorForAIEVecPass(options));
1225}
mlir::VectorType getFlattenedVectorType(mlir::VectorType vecTy)
std::unique_ptr<::mlir::Pass > createCopyRemovalPass()
Create a pass that removes unnecessary Copy operations.
std::optional< int64_t > getTransferReadAlignmentOffset(TransferReadLikeOp readOp, mlir::VectorType vType, int64_t alignment)
std::optional< AIEVecTargetInfo > parseAIEVecLoweringTarget(llvm::StringRef spelling, mlir::Operation *op)
std::optional< AIEVecTargetInfo > parseAIEVecTarget(llvm::StringRef spelling)
int32_t getElementSizeInBits(mlir::VectorType type)
Definition AIEVecUtils.h:48
std::unique_ptr<::mlir::Pass > createBF16EmulationPass()
Create a pass that runs f32 vector arithmetic on the bf16 datapath: multiplies and transcendentals ta...
void buildCanonicalizeVectorForAIEVec(mlir::OpPassManager &pm, const CanonicalizeVectorForAIEVecOptions &options)
LogicalResult matchAndRewrite(vector::BroadcastOp op, PatternRewriter &rewriter) const override
Pattern to canonicalize trivial vector.transfer_read operations on subviews.
LogicalResult matchAndRewrite(vector::TransferReadOp readOp, PatternRewriter &rewriter) const override
Pattern to canonicalize trivial vector.transfer_write operations on subviews.
LogicalResult matchAndRewrite(vector::TransferWriteOp writeOp, PatternRewriter &rewriter) const override
void getDependentDialects(DialectRegistry &registry) const override
CanonicalizeVectorForAIEVecPass(const CanonicalizeVectorForAIEVecOptions &options)
LogicalResult matchAndRewrite(vector::InsertOp insOp, PatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::TransferReadOp readOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
Pattern to emulate f32 binary vector arithmetic ops in bf16.
LogicalResult matchAndRewrite(OpTy op, PatternRewriter &rewriter) const override
Pattern to emulate f32 comparison ops in bf16.
LogicalResult matchAndRewrite(arith::CmpFOp op, PatternRewriter &rewriter) const override
Pattern to emulate f32 vector.fma with bf16 inputs.
LogicalResult matchAndRewrite(vector::FMAOp op, PatternRewriter &rewriter) const override
Pattern to emulate an f32 multiply with bf16 inputs.
LogicalResult matchAndRewrite(arith::MulFOp op, PatternRewriter &rewriter) const override
Pattern to emulate f32 select ops in bf16.
LogicalResult matchAndRewrite(arith::SelectOp op, PatternRewriter &rewriter) const override
Pattern to emulate f32 unary vector ops in bf16.
LogicalResult matchAndRewrite(OpTy op, PatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::TransferReadOp readOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::TransferWriteOp writeOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(scf::ForOp forOp, PatternRewriter &rewriter) const override
Pattern to drop bf16 -> f32 -> bf16 round trips.
LogicalResult matchAndRewrite(arith::TruncFOp op, PatternRewriter &rewriter) const override
FuseMulAddFInBF16Pattern for a multiply whose inputs already hold bf16 values, outside bf16 emulation...
LogicalResult matchAndRewrite(arith::AddFOp op, PatternRewriter &rewriter) const override
Pattern to fold an f32 multiply into the add that is its only user, as a vector.fma with bf16 inputs.
LogicalResult matchAndRewrite(arith::AddFOp op, PatternRewriter &rewriter) const override
SplitUnalignedTransferReadPattern(MLIRContext *context, int64_t maxVectorSize, int64_t alignment)
LogicalResult matchAndRewrite(vector::TransferReadOp readOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(UnaryOpB bOp, PatternRewriter &rewriter) const override
std::function< Type(UnaryOpA aOp, UnaryOpB bOp)> InferTypeB2AFnTy
SwapUnaryOpsPattern(MLIRContext *context, InferTypeB2AFnTy inferType)
Options for the "canonicalize-vector-for-aievec" pipeline.
Definition Passes.h:25
PassOptions::Option< bool > enableBF16Emulation
Definition Passes.h:32
PassOptions::Option< std::string > aieTarget
Definition Passes.h:26