MLIR-AIE 1.4.6.dev
SplitVectorLoadUpsChains.cpp
Go to the documentation of this file.
1//===- SplitVectorLoadUpsChains.cpp - Split Load+UPS Chains ----*- C++ -*-===//
2//
3// Copyright (C) 2025 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7// Splits 1024-bit vector loads and stores into 512-bit halves where the
8// value is widened or narrowed one half at a time on AIE2P: vector.load +
9// aievec.ups, aievec.srs + vector.store, and bf16 tile loads that are the lhs
10// of bfp16 matmuls. Each half then maps to its own memory access instead of
11// a shuffle of the full-width value. Then, on request, any other 1024- or
12// 2048-bit load or store into 512-bit ones.
13//===----------------------------------------------------------------------===//
14
18#include "mlir/Dialect/Affine/IR/AffineOps.h"
19#include "mlir/Dialect/Arith/IR/Arith.h"
20#include "mlir/Dialect/MemRef/IR/MemRef.h"
21#include "mlir/Dialect/Vector/IR/VectorOps.h"
22#include "mlir/IR/PatternMatch.h"
23#include "mlir/Pass/Pass.h"
24#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
25#include "llvm/ADT/Sequence.h"
26
27#define DEBUG_TYPE "aievec-split-load-ups-chains"
28
29using namespace mlir;
30using namespace xilinx::aievec;
31
32namespace {
33
34/// Pattern to optimize vector.load + aievec.ups chains by splitting them.
35///
36/// This pattern detects cases where a 1024-bit vector is loaded and then
37/// passed to an aievec.ups operation that produces a 2048-bit result.
38/// Instead of the inefficient approach of:
39/// 1. Load 1024 bits
40/// 2. Shuffle to split into 2×512 bits
41/// 3. Apply 2× UPS operations
42/// 4. Shuffle to concatenate results
43///
44/// It transforms to:
45/// 1. Load 2×512 bits directly
46/// 2. Apply 2× UPS operations immediately
47/// 3. Shuffle once to concatenate results
48///
49/// This reduces shuffle operations from 3 to 1.
50struct SplitVectorLoadUpsChainPattern : public OpRewritePattern<UPSOp> {
52
53 LogicalResult matchAndRewrite(UPSOp upsOp,
54 PatternRewriter &rewriter) const override {
55 // Get source value and its type
56 Value source = upsOp.getSource();
57 auto srcVecTy = dyn_cast<VectorType>(source.getType());
58 if (!srcVecTy)
59 return failure();
60
61 // Get result type
62 auto resultVecTy = dyn_cast<VectorType>(upsOp.getResult().getType());
63 if (!resultVecTy)
64 return failure();
65
66 // Check if this is a 1024-bit -> 2048-bit integer UPS
67 Type srcElemTy = srcVecTy.getElementType();
68 Type resultElemTy = resultVecTy.getElementType();
69
70 if (!srcElemTy.isInteger() || !resultElemTy.isInteger())
71 return failure();
72
73 unsigned srcBitWidth = srcElemTy.getIntOrFloatBitWidth();
74 unsigned resultBitWidth = resultElemTy.getIntOrFloatBitWidth();
75 int64_t srcLanes = srcVecTy.getNumElements();
76 int64_t resultLanes = resultVecTy.getNumElements();
77
78 int64_t srcVectorSize = srcBitWidth * srcLanes;
79 int64_t resultVectorSize = resultBitWidth * resultLanes;
80
81 // Only optimize the 1024-bit -> 2048-bit case
82 // (e.g., v64int16 -> v64acc32)
83 if (srcVectorSize != 1024 || resultVectorSize != 2048)
84 return failure();
85
86 // Check that the UPS result width is 32 and source width is 16
87 if (resultBitWidth != 32 || srcBitWidth != 16)
88 return failure();
89
90 // Check if source is directly from a vector.load
91 auto loadOp = source.getDefiningOp<vector::LoadOp>();
92 if (!loadOp)
93 return failure();
94
95 // Ensure the load is only used by this UPS operation
96 if (!loadOp.getResult().hasOneUse())
97 return failure();
98
99 Location loc = upsOp.getLoc();
100
101 // Get load operation details
102 Value memRef = loadOp.getBase();
103 ValueRange indices = loadOp.getIndices();
104
105 // Create element type for half-sized vector (v32int16)
106 int64_t halfLanes = srcLanes / 2;
107 auto halfSrcVecTy = VectorType::get({halfLanes}, srcElemTy);
108 auto halfResultVecTy = VectorType::get({halfLanes}, resultElemTy);
109
110 // Calculate offset for second half load
111 // For v64int16, we need to offset by 32 elements (64 bytes for i16)
112 int64_t elementOffset = halfLanes;
113
114 // Create indices for first half load (same as original)
115 SmallVector<Value> firstHalfIndices(indices.begin(), indices.end());
116
117 // Create indices for second half load
118 SmallVector<Value> secondHalfIndices(indices.begin(), indices.end());
119
120 // Adjust the last index by the element offset
121 if (!indices.empty()) {
122 Value lastIdx = indices.back();
123 Value offsetVal =
124 arith::ConstantIndexOp::create(rewriter, loc, elementOffset);
125 Value newLastIdx =
126 arith::AddIOp::create(rewriter, loc, lastIdx, offsetVal);
127 secondHalfIndices.back() = newLastIdx;
128 }
129
130 // Create first half load
131 auto loadHalf0 = vector::LoadOp::create(rewriter, loc, halfSrcVecTy, memRef,
132 firstHalfIndices);
133
134 // Create second half load
135 auto loadHalf1 = vector::LoadOp::create(rewriter, loc, halfSrcVecTy, memRef,
136 secondHalfIndices);
137
138 // Create UPS for first half
139 auto upsHalf0 = UPSOp::create(rewriter, loc, halfResultVecTy,
140 loadHalf0.getResult(), upsOp.getShift());
141
142 // Create UPS for second half
143 auto upsHalf1 = UPSOp::create(rewriter, loc, halfResultVecTy,
144 loadHalf1.getResult(), upsOp.getShift());
145
146 // Concatenate the two halves using vector.shuffle
147 // The mask is sequential from 0 to 63 to concatenate [half0; half1]
148 SmallVector<int64_t> concatMask;
149 for (int64_t i = 0; i < resultLanes; ++i) {
150 concatMask.push_back(i);
151 }
152
153 auto concatOp = vector::ShuffleOp::create(
154 rewriter, loc, upsHalf0.getResult(), upsHalf1.getResult(), concatMask);
155
156 // Replace the original UPS operation with the concatenated result
157 rewriter.replaceOp(upsOp, concatOp.getResult());
158
159 // The original load will be removed by dead code elimination
160 // since it no longer has any uses
161
162 return success();
163 }
164};
165
166/// Pattern to optimize aievec.srs + vector.store chains by splitting them.
167///
168/// This pattern detects cases where a 2048-bit vector is passed to an
169/// aievec.srs operation that produces a 1024-bit result, which is then stored.
170/// Instead of the inefficient approach of:
171/// 1. Shuffle to split 2048-bit into 2×1024 bits
172/// 2. Apply 2× SRS operations
173/// 3. Shuffle to concatenate results
174/// 4. Store 1024 bits
175///
176/// It transforms to:
177/// 1. Split source via shuffle into 2×1024 bits (for SRS input)
178/// 2. Apply 2× SRS operations to get 2×512 bits
179/// 3. Store 2×512 bits directly
180///
181/// This reduces shuffle operations from 3 to 1.
182struct SplitVectorSrsStoreChainPattern
183 : public OpRewritePattern<vector::StoreOp> {
184 using OpRewritePattern<vector::StoreOp>::OpRewritePattern;
185
186 LogicalResult matchAndRewrite(vector::StoreOp storeOp,
187 PatternRewriter &rewriter) const override {
188 // Get the value being stored and its type
189 Value valueToStore = storeOp.getValueToStore();
190 auto storeVecTy = dyn_cast<VectorType>(valueToStore.getType());
191 if (!storeVecTy)
192 return failure();
193
194 // Check if the value comes from an aievec.srs operation
195 auto srsOp = valueToStore.getDefiningOp<SRSOp>();
196 if (!srsOp)
197 return failure();
198
199 // Ensure the SRS is only used by this store operation
200 if (!srsOp.getResult().hasOneUse())
201 return failure();
202
203 // Get source and result types of SRS
204 Value srsSource = srsOp.getSource();
205 auto srcVecTy = dyn_cast<VectorType>(srsSource.getType());
206 if (!srcVecTy)
207 return failure();
208
209 Type srcElemTy = srcVecTy.getElementType();
210 Type resultElemTy = storeVecTy.getElementType();
211
212 if (!srcElemTy.isInteger() || !resultElemTy.isInteger())
213 return failure();
214
215 unsigned srcBitWidth = srcElemTy.getIntOrFloatBitWidth();
216 unsigned resultBitWidth = resultElemTy.getIntOrFloatBitWidth();
217 int64_t srcLanes = srcVecTy.getNumElements();
218 int64_t resultLanes = storeVecTy.getNumElements();
219
220 int64_t srcVectorSize = srcBitWidth * srcLanes;
221 int64_t resultVectorSize = resultBitWidth * resultLanes;
222
223 // Only optimize the 2048-bit -> 1024-bit case
224 // (e.g., v64acc32 -> v64int16)
225 if (srcVectorSize != 2048 || resultVectorSize != 1024)
226 return failure();
227
228 // Check that the SRS source width is 32 and result width is 16
229 if (srcBitWidth != 32 || resultBitWidth != 16)
230 return failure();
231
232 Location loc = storeOp.getLoc();
233
234 // Get store operation details
235 Value memRef = storeOp.getBase();
236 ValueRange indices = storeOp.getIndices();
237
238 // Create element types for half-sized vectors
239 int64_t halfSrcLanes = srcLanes / 2;
240 int64_t halfResultLanes = resultLanes / 2;
241 auto halfResultVecTy = VectorType::get({halfResultLanes}, resultElemTy);
242
243 // Split the SRS source into two halves using shuffle
244 SmallVector<int64_t> firstHalfMask, secondHalfMask;
245 for (int64_t i = 0; i < halfSrcLanes; ++i) {
246 firstHalfMask.push_back(i);
247 secondHalfMask.push_back(halfSrcLanes + i);
248 }
249
250 auto srcHalf0 = vector::ShuffleOp::create(rewriter, loc, srsSource,
251 srsSource, firstHalfMask);
252 auto srcHalf1 = vector::ShuffleOp::create(rewriter, loc, srsSource,
253 srsSource, secondHalfMask);
254
255 // Create SRS for first half
256 auto srsHalf0 = SRSOp::create(rewriter, loc, halfResultVecTy,
257 srcHalf0.getResult(), srsOp.getShift());
258
259 // Create SRS for second half
260 auto srsHalf1 = SRSOp::create(rewriter, loc, halfResultVecTy,
261 srcHalf1.getResult(), srsOp.getShift());
262
263 // Calculate offset for second half store
264 int64_t elementOffset = halfResultLanes;
265
266 // Create indices for first half store (same as original)
267 SmallVector<Value> firstHalfIndices(indices.begin(), indices.end());
268
269 // Create indices for second half store
270 SmallVector<Value> secondHalfIndices(indices.begin(), indices.end());
271
272 // Adjust the last index by the element offset
273 if (!indices.empty()) {
274 Value lastIdx = indices.back();
275 Value offsetVal =
276 arith::ConstantIndexOp::create(rewriter, loc, elementOffset);
277 Value newLastIdx =
278 arith::AddIOp::create(rewriter, loc, lastIdx, offsetVal);
279 secondHalfIndices.back() = newLastIdx;
280 }
281
282 // Create first half store
283 vector::StoreOp::create(rewriter, loc, srsHalf0.getResult(), memRef,
284 firstHalfIndices);
285
286 // Create second half store
287 vector::StoreOp::create(rewriter, loc, srsHalf1.getResult(), memRef,
288 secondHalfIndices);
289
290 // Erase the original store operation
291 rewriter.eraseOp(storeOp);
292
293 // The original SRS will be removed by dead code elimination
294 // since it no longer has any uses
295
296 return success();
297 }
298};
299
300/// Splits a 1024-bit bf16 vector.load of an 8x8 tile that is only the lhs of
301/// bfp16 matmuls (converted for them by aievec.to_bfp16, or the lhs of a bf16
302/// aievec.matmul_aie2p not yet expanded) into two 512-bit loads joined by a
303/// vector.shuffle.
304///
305/// The conversion widens such a tile to fp32 one 32-lane half at a time.
306/// With a single 64-lane load, each half is extracted by a shuffle before it is
307/// widened. After this split, and once LLVM folds the halves of the joining
308/// shuffle back to the two loads, each widening reads its own 512-bit load,
309/// which the backend can fuse into a single load-and-convert.
310struct SplitVectorLoadMatMulLhsPattern
311 : public OpRewritePattern<vector::LoadOp> {
312 using OpRewritePattern<vector::LoadOp>::OpRewritePattern;
313
314 LogicalResult matchAndRewrite(vector::LoadOp loadOp,
315 PatternRewriter &rewriter) const override {
316 VectorType loadTy = loadOp.getVectorType();
317 if (loadTy.getRank() != 1 || loadTy.getNumElements() != 64 ||
318 !loadTy.getElementType().isBF16())
319 return failure();
320
321 auto bf16Tile = VectorType::get({8, 8}, rewriter.getBF16Type());
322 auto f32Tile = VectorType::get({8, 8}, rewriter.getF32Type());
323 // The tile is the lhs of bfp16 matmuls: converted for them, or, before
324 // the matmuls are expanded, their bf16 8x8x8 lhs.
325 auto feedsOnlyMatMulLhs = [&](Operation *user) {
326 auto cast = dyn_cast<vector::ShapeCastOp>(user);
327 if (!cast || cast.getResultVectorType() != bf16Tile)
328 return false;
329 return llvm::all_of(cast->getUses(), [&](OpOperand &use) {
330 if (auto conv = dyn_cast<ToBFP16Op>(use.getOwner()))
331 return isBFP16MatMulLhsOnly(conv);
332 auto matmul = dyn_cast<MatMulOp_AIE2P>(use.getOwner());
333 return matmul && use.getOperandNumber() == 0 &&
334 matmul.getRhs().getType() == bf16Tile &&
335 matmul.getAcc().getType() == f32Tile;
336 });
337 };
338 if (loadOp->use_empty() ||
339 !llvm::all_of(loadOp->getUsers(), feedsOnlyMatMulLhs))
340 return failure();
341
342 Location loc = loadOp.getLoc();
343 auto halfTy = VectorType::get({32}, rewriter.getBF16Type());
344 SmallVector<Value> hiIndices(loadOp.getIndices());
345 Value c32 = arith::ConstantIndexOp::create(rewriter, loc, 32);
346 hiIndices.back() =
347 arith::AddIOp::create(rewriter, loc, hiIndices.back(), c32);
348 Value lo = vector::LoadOp::create(rewriter, loc, halfTy, loadOp.getBase(),
349 loadOp.getIndices());
350 Value hi = vector::LoadOp::create(rewriter, loc, halfTy, loadOp.getBase(),
351 hiIndices);
352 SmallVector<int64_t> concatMask =
353 llvm::to_vector(llvm::seq<int64_t>(0, 64));
354 rewriter.replaceOpWithNewOp<vector::ShuffleOp>(loadOp, lo, hi, concatMask);
355 return success();
356 }
357};
358
359// An AIE2P load or store moves at most 512 bits. A 1024- or 2048-bit one is
360// split here into 512-bit ones rather than left to llc, whose own split of
361// such an access is not reliable.
362//
363// The parts of a wide access of vector type `type`: consecutive 512-bit runs
364// of its elements in row-major order, each a box of the vector (1 in the
365// dimensions before `dim`, a slice of `dim`, whole after it). Empty if the
366// access is not split.
367struct WideAccessParts {
368 VectorType partType;
369 SmallVector<SmallVector<int64_t>> offsets;
370};
371
372static std::optional<WideAccessParts> wideAccessParts(VectorType type) {
373 int64_t bits = type.getNumElements() * type.getElementTypeBitWidth();
374 if (bits != 1024 && bits != 2048)
375 return std::nullopt;
376 int64_t partLanes = type.getNumElements() / (bits / 512);
377 ArrayRef<int64_t> shape = type.getShape();
378 for (int64_t dim = 0; dim < type.getRank(); ++dim) {
379 int64_t inner = 1;
380 for (int64_t n : shape.drop_front(dim + 1))
381 inner *= n;
382 if (partLanes % inner || shape[dim] % (partLanes / inner))
383 continue;
384 int64_t slice = partLanes / inner;
385 WideAccessParts parts;
386 SmallVector<int64_t> partShape(shape);
387 for (int64_t k = 0; k < dim; ++k)
388 partShape[k] = 1;
389 partShape[dim] = slice;
390 parts.partType = VectorType::get(partShape, type.getElementType());
391 for (int64_t i = 0; i < bits / 512; ++i) {
392 // Part i: chunk i % (shape[dim] / slice) of `dim`, at the row-major
393 // position i / (shape[dim] / slice) of the dimensions before it.
394 SmallVector<int64_t> at(type.getRank(), 0);
395 int64_t chunks = shape[dim] / slice;
396 at[dim] = (i % chunks) * slice;
397 for (int64_t k = dim - 1, outer = i / chunks; k >= 0; --k) {
398 at[k] = outer % shape[k];
399 outer /= shape[k];
400 }
401 parts.offsets.push_back(at);
402 }
403 return parts;
404 }
405 return std::nullopt;
406}
407
408// The indices of the part at `offsets` of the split access `op`.
409template <typename OpTy>
410static SmallVector<Value> partIndices(PatternRewriter &rewriter, OpTy op,
411 ArrayRef<int64_t> offsets) {
412 SmallVector<Value> indices(op.getIndices());
413 int64_t first = op.getMemRefType().getRank() - op.getVectorType().getRank();
414 for (auto [k, offset] : llvm::enumerate(offsets)) {
415 if (offset == 0)
416 continue;
417 Value step = arith::ConstantIndexOp::create(rewriter, op.getLoc(), offset);
418 indices[first + k] =
419 arith::AddIOp::create(rewriter, op.getLoc(), indices[first + k], step);
420 }
421 return indices;
422}
423
424static SmallVector<int64_t> iota(int64_t from, int64_t count) {
425 return llvm::to_vector(llvm::seq<int64_t>(from, from + count));
426}
427
428// The 1-D vector `v` as consecutive parts of `lanes` lanes: the operands of
429// the concatenations it is made of (through aievec.cast, cast part by part)
430// where they are such parts, otherwise shuffles taking them out.
431static void collectParts(PatternRewriter &rewriter, Location loc, Value v,
432 int64_t lanes, SmallVectorImpl<Value> &parts) {
433 int64_t n = cast<VectorType>(v.getType()).getNumElements();
434 if (n == lanes) {
435 parts.push_back(v);
436 return;
437 }
438 if (auto vcast = v.getDefiningOp<CastOp>()) {
439 SmallVector<Value> inner;
440 collectParts(rewriter, loc, vcast.getSource(), lanes, inner);
441 Type elemType = vcast.getResult().getType().getElementType();
442 for (Value part : inner) {
443 auto partType = VectorType::get(
444 cast<VectorType>(part.getType()).getShape(), elemType);
445 parts.push_back(
446 CastOp::create(rewriter, loc, partType, part, vcast.getIsResAcc()));
447 }
448 return;
449 }
450 if (auto concat = v.getDefiningOp<vector::ShuffleOp>()) {
451 auto half = cast<VectorType>(concat.getV1().getType()).getNumElements();
452 if (concat.getV1().getType() == concat.getV2().getType() &&
453 half % lanes == 0 && llvm::equal(concat.getMask(), iota(0, 2 * half))) {
454 collectParts(rewriter, loc, concat.getV1(), lanes, parts);
455 collectParts(rewriter, loc, concat.getV2(), lanes, parts);
456 return;
457 }
458 }
459 for (int64_t i = 0; i < n; i += lanes)
460 parts.push_back(
461 vector::ShuffleOp::create(rewriter, loc, v, v, iota(i, lanes)));
462}
463
464struct SplitWideLoadPattern : public OpRewritePattern<vector::LoadOp> {
465 using OpRewritePattern::OpRewritePattern;
466
467 LogicalResult matchAndRewrite(vector::LoadOp op,
468 PatternRewriter &rewriter) const override {
469 VectorType type = op.getVectorType();
470 auto split = wideAccessParts(type);
471 if (!split)
472 return failure();
473 Location loc = op.getLoc();
474 SmallVector<Value> parts;
475 for (ArrayRef<int64_t> at : split->offsets)
476 parts.push_back(vector::LoadOp::create(rewriter, loc, split->partType,
477 op.getBase(),
478 partIndices(rewriter, op, at)));
479 if (type.getRank() == 1) {
480 // Concatenate pairwise until one vector is left.
481 while (parts.size() > 1) {
482 SmallVector<Value> next;
483 for (size_t i = 0; i < parts.size(); i += 2) {
484 int64_t m = cast<VectorType>(parts[i].getType()).getNumElements();
485 next.push_back(vector::ShuffleOp::create(
486 rewriter, loc, parts[i], parts[i + 1], iota(0, 2 * m)));
487 }
488 parts = std::move(next);
489 }
490 rewriter.replaceOp(op, parts.front());
491 return success();
492 }
493 Value whole = arith::ConstantOp::create(rewriter, loc, type,
494 rewriter.getZeroAttr(type));
495 SmallVector<int64_t> strides(type.getRank(), 1);
496 for (auto [part, at] : llvm::zip(parts, split->offsets))
497 whole = vector::InsertStridedSliceOp::create(rewriter, loc, part, whole,
498 at, strides);
499 rewriter.replaceOp(op, whole);
500 return success();
501 }
502};
503
504struct SplitWideStorePattern : public OpRewritePattern<vector::StoreOp> {
505 using OpRewritePattern::OpRewritePattern;
506
507 LogicalResult matchAndRewrite(vector::StoreOp op,
508 PatternRewriter &rewriter) const override {
509 VectorType type = op.getVectorType();
510 auto split = wideAccessParts(type);
511 if (!split)
512 return failure();
513 Location loc = op.getLoc();
514 SmallVector<Value> parts;
515 if (type.getRank() == 1) {
516 collectParts(rewriter, loc, op.getValueToStore(),
517 split->partType.getNumElements(), parts);
518 } else {
519 SmallVector<int64_t> strides(type.getRank(), 1);
520 for (ArrayRef<int64_t> at : split->offsets)
521 parts.push_back(vector::ExtractStridedSliceOp::create(
522 rewriter, loc, op.getValueToStore(), at, split->partType.getShape(),
523 strides));
524 }
525 for (auto [part, at] : llvm::zip(parts, split->offsets))
526 vector::StoreOp::create(rewriter, loc, part, op.getBase(),
527 partIndices(rewriter, op, at));
528 rewriter.eraseOp(op);
529 return success();
530 }
531};
532
533/// Pass to split vector.load + aievec.ups chains for better performance
534struct SplitVectorLoadUpsChainsPass
535 : public PassWrapper<SplitVectorLoadUpsChainsPass, OperationPass<>> {
536 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(SplitVectorLoadUpsChainsPass)
537
538 SplitVectorLoadUpsChainsPass() = default;
539 SplitVectorLoadUpsChainsPass(const SplitVectorLoadUpsChainsPass &other)
540 : PassWrapper(other) {}
541 SplitVectorLoadUpsChainsPass(bool chains, bool wideAccesses) {
542 this->chains = chains;
543 this->wideAccesses = wideAccesses;
544 }
545
546 StringRef getArgument() const final { return "aievec-split-load-ups-chains"; }
547
548 StringRef getDescription() const final {
549 return "Split 1024-bit vector loads and stores whose halves are widened "
550 "or narrowed separately";
551 }
552
553 void getDependentDialects(DialectRegistry &registry) const override {
554 registry.insert<vector::VectorDialect, arith::ArithDialect,
555 memref::MemRefDialect, affine::AffineDialect,
556 xilinx::aievec::AIEVecDialect>();
557 }
558
559 void runOnOperation() override {
560 Operation *op = getOperation();
561 MLIRContext *context = &getContext();
562
563 if (chains) {
564 RewritePatternSet patterns(context);
565 patterns
566 .add<SplitVectorLoadUpsChainPattern, SplitVectorSrsStoreChainPattern,
567 SplitVectorLoadMatMulLhsPattern>(context);
568 if (failed(applyPatternsGreedily(op, std::move(patterns))))
569 return signalPassFailure();
570 }
571
572 if (wideAccesses) {
573 // Loads first, so that a store sees the parts a value was joined from.
574 RewritePatternSet loads(context);
575 loads.add<SplitWideLoadPattern>(context);
576 if (failed(applyPatternsGreedily(op, std::move(loads))))
577 return signalPassFailure();
578 RewritePatternSet stores(context);
579 stores.add<SplitWideStorePattern>(context);
580 if (failed(applyPatternsGreedily(op, std::move(stores))))
581 signalPassFailure();
582 }
583 }
584
585 Option<bool> chains{*this, "chains",
586 llvm::cl::desc("Split the loads and stores of ups, srs "
587 "and bfp16 matmul lhs chains"),
588 llvm::cl::init(true)};
589 Option<bool> wideAccesses{
590 *this, "wide-accesses",
591 llvm::cl::desc("Split every other 1024- and 2048-bit load and store "
592 "into 512-bit ones (AIE2P)"),
593 llvm::cl::init(false)};
594};
595
596} // namespace
597
598namespace xilinx {
599namespace aievec {
600
601std::unique_ptr<::mlir::Pass> createSplitVectorLoadUpsChainsPass() {
602 return std::make_unique<SplitVectorLoadUpsChainsPass>();
603}
604
605std::unique_ptr<::mlir::Pass>
606createSplitVectorLoadUpsChainsPass(bool chains, bool wideAccesses) {
607 return std::make_unique<SplitVectorLoadUpsChainsPass>(chains, wideAccesses);
608}
609
610} // namespace aievec
611} // namespace xilinx
std::unique_ptr<::mlir::Pass > createSplitVectorLoadUpsChainsPass()
Create a pass that splits vector.load + aievec.ups chains to reduce shuffle operations for AIE2p targ...