18#include "mlir/Dialect/Affine/IR/AffineOps.h"
19#include "mlir/Dialect/Arith/IR/Arith.h"
20#include "mlir/Dialect/MemRef/IR/MemRef.h"
21#include "mlir/Dialect/Vector/IR/VectorOps.h"
22#include "mlir/IR/PatternMatch.h"
23#include "mlir/Pass/Pass.h"
24#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
25#include "llvm/ADT/Sequence.h"
27#define DEBUG_TYPE "aievec-split-load-ups-chains"
53 LogicalResult matchAndRewrite(UPSOp upsOp,
54 PatternRewriter &rewriter)
const override {
56 Value source = upsOp.getSource();
57 auto srcVecTy = dyn_cast<VectorType>(source.getType());
62 auto resultVecTy = dyn_cast<VectorType>(upsOp.getResult().getType());
67 Type srcElemTy = srcVecTy.getElementType();
68 Type resultElemTy = resultVecTy.getElementType();
70 if (!srcElemTy.isInteger() || !resultElemTy.isInteger())
73 unsigned srcBitWidth = srcElemTy.getIntOrFloatBitWidth();
74 unsigned resultBitWidth = resultElemTy.getIntOrFloatBitWidth();
75 int64_t srcLanes = srcVecTy.getNumElements();
76 int64_t resultLanes = resultVecTy.getNumElements();
78 int64_t srcVectorSize = srcBitWidth * srcLanes;
79 int64_t resultVectorSize = resultBitWidth * resultLanes;
83 if (srcVectorSize != 1024 || resultVectorSize != 2048)
87 if (resultBitWidth != 32 || srcBitWidth != 16)
91 auto loadOp = source.getDefiningOp<vector::LoadOp>();
96 if (!loadOp.getResult().hasOneUse())
99 Location loc = upsOp.getLoc();
102 Value memRef = loadOp.getBase();
103 ValueRange indices = loadOp.getIndices();
106 int64_t halfLanes = srcLanes / 2;
107 auto halfSrcVecTy = VectorType::get({halfLanes}, srcElemTy);
108 auto halfResultVecTy = VectorType::get({halfLanes}, resultElemTy);
112 int64_t elementOffset = halfLanes;
115 SmallVector<Value> firstHalfIndices(indices.begin(), indices.end());
118 SmallVector<Value> secondHalfIndices(indices.begin(), indices.end());
121 if (!indices.empty()) {
122 Value lastIdx = indices.back();
124 arith::ConstantIndexOp::create(rewriter, loc, elementOffset);
126 arith::AddIOp::create(rewriter, loc, lastIdx, offsetVal);
127 secondHalfIndices.back() = newLastIdx;
131 auto loadHalf0 = vector::LoadOp::create(rewriter, loc, halfSrcVecTy, memRef,
135 auto loadHalf1 = vector::LoadOp::create(rewriter, loc, halfSrcVecTy, memRef,
139 auto upsHalf0 = UPSOp::create(rewriter, loc, halfResultVecTy,
140 loadHalf0.getResult(), upsOp.getShift());
143 auto upsHalf1 = UPSOp::create(rewriter, loc, halfResultVecTy,
144 loadHalf1.getResult(), upsOp.getShift());
148 SmallVector<int64_t> concatMask;
149 for (int64_t i = 0; i < resultLanes; ++i) {
150 concatMask.push_back(i);
153 auto concatOp = vector::ShuffleOp::create(
154 rewriter, loc, upsHalf0.getResult(), upsHalf1.getResult(), concatMask);
157 rewriter.replaceOp(upsOp, concatOp.getResult());
182struct SplitVectorSrsStoreChainPattern
186 LogicalResult matchAndRewrite(vector::StoreOp storeOp,
187 PatternRewriter &rewriter)
const override {
189 Value valueToStore = storeOp.getValueToStore();
190 auto storeVecTy = dyn_cast<VectorType>(valueToStore.getType());
195 auto srsOp = valueToStore.getDefiningOp<SRSOp>();
200 if (!srsOp.getResult().hasOneUse())
204 Value srsSource = srsOp.getSource();
205 auto srcVecTy = dyn_cast<VectorType>(srsSource.getType());
209 Type srcElemTy = srcVecTy.getElementType();
210 Type resultElemTy = storeVecTy.getElementType();
212 if (!srcElemTy.isInteger() || !resultElemTy.isInteger())
215 unsigned srcBitWidth = srcElemTy.getIntOrFloatBitWidth();
216 unsigned resultBitWidth = resultElemTy.getIntOrFloatBitWidth();
217 int64_t srcLanes = srcVecTy.getNumElements();
218 int64_t resultLanes = storeVecTy.getNumElements();
220 int64_t srcVectorSize = srcBitWidth * srcLanes;
221 int64_t resultVectorSize = resultBitWidth * resultLanes;
225 if (srcVectorSize != 2048 || resultVectorSize != 1024)
229 if (srcBitWidth != 32 || resultBitWidth != 16)
232 Location loc = storeOp.getLoc();
235 Value memRef = storeOp.getBase();
236 ValueRange indices = storeOp.getIndices();
239 int64_t halfSrcLanes = srcLanes / 2;
240 int64_t halfResultLanes = resultLanes / 2;
241 auto halfResultVecTy = VectorType::get({halfResultLanes}, resultElemTy);
244 SmallVector<int64_t> firstHalfMask, secondHalfMask;
245 for (int64_t i = 0; i < halfSrcLanes; ++i) {
246 firstHalfMask.push_back(i);
247 secondHalfMask.push_back(halfSrcLanes + i);
250 auto srcHalf0 = vector::ShuffleOp::create(rewriter, loc, srsSource,
251 srsSource, firstHalfMask);
252 auto srcHalf1 = vector::ShuffleOp::create(rewriter, loc, srsSource,
253 srsSource, secondHalfMask);
256 auto srsHalf0 = SRSOp::create(rewriter, loc, halfResultVecTy,
257 srcHalf0.getResult(), srsOp.getShift());
260 auto srsHalf1 = SRSOp::create(rewriter, loc, halfResultVecTy,
261 srcHalf1.getResult(), srsOp.getShift());
264 int64_t elementOffset = halfResultLanes;
267 SmallVector<Value> firstHalfIndices(indices.begin(), indices.end());
270 SmallVector<Value> secondHalfIndices(indices.begin(), indices.end());
273 if (!indices.empty()) {
274 Value lastIdx = indices.back();
276 arith::ConstantIndexOp::create(rewriter, loc, elementOffset);
278 arith::AddIOp::create(rewriter, loc, lastIdx, offsetVal);
279 secondHalfIndices.back() = newLastIdx;
283 vector::StoreOp::create(rewriter, loc, srsHalf0.getResult(), memRef,
287 vector::StoreOp::create(rewriter, loc, srsHalf1.getResult(), memRef,
291 rewriter.eraseOp(storeOp);
310struct SplitVectorLoadMatMulLhsPattern
314 LogicalResult matchAndRewrite(vector::LoadOp loadOp,
315 PatternRewriter &rewriter)
const override {
316 VectorType loadTy = loadOp.getVectorType();
317 if (loadTy.getRank() != 1 || loadTy.getNumElements() != 64 ||
318 !loadTy.getElementType().isBF16())
321 auto bf16Tile = VectorType::get({8, 8}, rewriter.getBF16Type());
322 auto f32Tile = VectorType::get({8, 8}, rewriter.getF32Type());
325 auto feedsOnlyMatMulLhs = [&](Operation *user) {
326 auto cast = dyn_cast<vector::ShapeCastOp>(user);
327 if (!cast || cast.getResultVectorType() != bf16Tile)
329 return llvm::all_of(cast->getUses(), [&](OpOperand &use) {
330 if (auto conv = dyn_cast<ToBFP16Op>(use.getOwner()))
331 return isBFP16MatMulLhsOnly(conv);
332 auto matmul = dyn_cast<MatMulOp_AIE2P>(use.getOwner());
333 return matmul && use.getOperandNumber() == 0 &&
334 matmul.getRhs().getType() == bf16Tile &&
335 matmul.getAcc().getType() == f32Tile;
338 if (loadOp->use_empty() ||
339 !llvm::all_of(loadOp->getUsers(), feedsOnlyMatMulLhs))
342 Location loc = loadOp.getLoc();
343 auto halfTy = VectorType::get({32}, rewriter.getBF16Type());
344 SmallVector<Value> hiIndices(loadOp.getIndices());
345 Value c32 = arith::ConstantIndexOp::create(rewriter, loc, 32);
347 arith::AddIOp::create(rewriter, loc, hiIndices.back(), c32);
348 Value lo = vector::LoadOp::create(rewriter, loc, halfTy, loadOp.getBase(),
349 loadOp.getIndices());
350 Value hi = vector::LoadOp::create(rewriter, loc, halfTy, loadOp.getBase(),
352 SmallVector<int64_t> concatMask =
353 llvm::to_vector(llvm::seq<int64_t>(0, 64));
354 rewriter.replaceOpWithNewOp<vector::ShuffleOp>(loadOp, lo, hi, concatMask);
367struct WideAccessParts {
369 SmallVector<SmallVector<int64_t>> offsets;
372static std::optional<WideAccessParts> wideAccessParts(VectorType type) {
373 int64_t bits = type.getNumElements() * type.getElementTypeBitWidth();
374 if (bits != 1024 && bits != 2048)
376 int64_t partLanes = type.getNumElements() / (bits / 512);
377 ArrayRef<int64_t> shape = type.getShape();
378 for (int64_t dim = 0; dim < type.getRank(); ++dim) {
380 for (int64_t n : shape.drop_front(dim + 1))
382 if (partLanes % inner || shape[dim] % (partLanes / inner))
384 int64_t slice = partLanes / inner;
385 WideAccessParts parts;
386 SmallVector<int64_t> partShape(shape);
387 for (int64_t k = 0; k < dim; ++k)
389 partShape[dim] = slice;
390 parts.partType = VectorType::get(partShape, type.getElementType());
391 for (int64_t i = 0; i < bits / 512; ++i) {
394 SmallVector<int64_t> at(type.getRank(), 0);
395 int64_t chunks = shape[dim] / slice;
396 at[dim] = (i % chunks) * slice;
397 for (int64_t k = dim - 1, outer = i / chunks; k >= 0; --k) {
398 at[k] = outer % shape[k];
401 parts.offsets.push_back(at);
409template <
typename OpTy>
410static SmallVector<Value> partIndices(PatternRewriter &rewriter, OpTy op,
411 ArrayRef<int64_t> offsets) {
412 SmallVector<Value> indices(op.getIndices());
413 int64_t first = op.getMemRefType().getRank() - op.getVectorType().getRank();
414 for (
auto [k, offset] : llvm::enumerate(offsets)) {
417 Value step = arith::ConstantIndexOp::create(rewriter, op.getLoc(), offset);
419 arith::AddIOp::create(rewriter, op.getLoc(), indices[first + k], step);
424static SmallVector<int64_t> iota(int64_t from, int64_t count) {
425 return llvm::to_vector(llvm::seq<int64_t>(from, from + count));
431static void collectParts(PatternRewriter &rewriter, Location loc, Value v,
432 int64_t lanes, SmallVectorImpl<Value> &parts) {
433 int64_t n = cast<VectorType>(v.getType()).getNumElements();
438 if (
auto vcast = v.getDefiningOp<CastOp>()) {
439 SmallVector<Value> inner;
440 collectParts(rewriter, loc, vcast.getSource(), lanes, inner);
441 Type elemType = vcast.getResult().getType().getElementType();
442 for (Value part : inner) {
443 auto partType = VectorType::get(
444 cast<VectorType>(part.getType()).getShape(), elemType);
446 CastOp::create(rewriter, loc, partType, part, vcast.getIsResAcc()));
450 if (
auto concat = v.getDefiningOp<vector::ShuffleOp>()) {
451 auto half = cast<VectorType>(concat.getV1().getType()).getNumElements();
452 if (concat.getV1().getType() == concat.getV2().getType() &&
453 half % lanes == 0 && llvm::equal(concat.getMask(), iota(0, 2 * half))) {
454 collectParts(rewriter, loc, concat.getV1(), lanes, parts);
455 collectParts(rewriter, loc, concat.getV2(), lanes, parts);
459 for (int64_t i = 0; i < n; i += lanes)
461 vector::ShuffleOp::create(rewriter, loc, v, v, iota(i, lanes)));
465 using OpRewritePattern::OpRewritePattern;
467 LogicalResult matchAndRewrite(vector::LoadOp op,
468 PatternRewriter &rewriter)
const override {
469 VectorType type = op.getVectorType();
470 auto split = wideAccessParts(type);
473 Location loc = op.getLoc();
474 SmallVector<Value> parts;
475 for (ArrayRef<int64_t> at : split->offsets)
476 parts.push_back(vector::LoadOp::create(rewriter, loc, split->partType,
478 partIndices(rewriter, op, at)));
479 if (type.getRank() == 1) {
481 while (parts.size() > 1) {
482 SmallVector<Value> next;
483 for (
size_t i = 0; i < parts.size(); i += 2) {
484 int64_t m = cast<VectorType>(parts[i].getType()).getNumElements();
485 next.push_back(vector::ShuffleOp::create(
486 rewriter, loc, parts[i], parts[i + 1], iota(0, 2 * m)));
488 parts = std::move(next);
490 rewriter.replaceOp(op, parts.front());
493 Value whole = arith::ConstantOp::create(rewriter, loc, type,
494 rewriter.getZeroAttr(type));
495 SmallVector<int64_t> strides(type.getRank(), 1);
496 for (
auto [part, at] : llvm::zip(parts, split->offsets))
497 whole = vector::InsertStridedSliceOp::create(rewriter, loc, part, whole,
499 rewriter.replaceOp(op, whole);
505 using OpRewritePattern::OpRewritePattern;
507 LogicalResult matchAndRewrite(vector::StoreOp op,
508 PatternRewriter &rewriter)
const override {
509 VectorType type = op.getVectorType();
510 auto split = wideAccessParts(type);
513 Location loc = op.getLoc();
514 SmallVector<Value> parts;
515 if (type.getRank() == 1) {
516 collectParts(rewriter, loc, op.getValueToStore(),
517 split->partType.getNumElements(), parts);
519 SmallVector<int64_t> strides(type.getRank(), 1);
520 for (ArrayRef<int64_t> at : split->offsets)
521 parts.push_back(vector::ExtractStridedSliceOp::create(
522 rewriter, loc, op.getValueToStore(), at, split->partType.getShape(),
525 for (
auto [part, at] : llvm::zip(parts, split->offsets))
526 vector::StoreOp::create(rewriter, loc, part, op.getBase(),
527 partIndices(rewriter, op, at));
528 rewriter.eraseOp(op);
534struct SplitVectorLoadUpsChainsPass
535 :
public PassWrapper<SplitVectorLoadUpsChainsPass, OperationPass<>> {
536 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(SplitVectorLoadUpsChainsPass)
538 SplitVectorLoadUpsChainsPass() =
default;
539 SplitVectorLoadUpsChainsPass(
const SplitVectorLoadUpsChainsPass &other)
541 SplitVectorLoadUpsChainsPass(
bool chains,
bool wideAccesses) {
542 this->chains = chains;
543 this->wideAccesses = wideAccesses;
546 StringRef getArgument()
const final {
return "aievec-split-load-ups-chains"; }
548 StringRef getDescription()
const final {
549 return "Split 1024-bit vector loads and stores whose halves are widened "
550 "or narrowed separately";
553 void getDependentDialects(DialectRegistry ®istry)
const override {
554 registry.insert<vector::VectorDialect, arith::ArithDialect,
555 memref::MemRefDialect, affine::AffineDialect,
556 xilinx::aievec::AIEVecDialect>();
559 void runOnOperation()
override {
560 Operation *op = getOperation();
561 MLIRContext *context = &getContext();
564 RewritePatternSet patterns(context);
566 .add<SplitVectorLoadUpsChainPattern, SplitVectorSrsStoreChainPattern,
567 SplitVectorLoadMatMulLhsPattern>(context);
568 if (failed(applyPatternsGreedily(op, std::move(patterns))))
569 return signalPassFailure();
574 RewritePatternSet loads(context);
575 loads.add<SplitWideLoadPattern>(context);
576 if (failed(applyPatternsGreedily(op, std::move(loads))))
577 return signalPassFailure();
578 RewritePatternSet stores(context);
579 stores.add<SplitWideStorePattern>(context);
580 if (failed(applyPatternsGreedily(op, std::move(stores))))
585 Option<bool> chains{*
this,
"chains",
586 llvm::cl::desc(
"Split the loads and stores of ups, srs "
587 "and bfp16 matmul lhs chains"),
588 llvm::cl::init(
true)};
589 Option<bool> wideAccesses{
590 *
this,
"wide-accesses",
591 llvm::cl::desc(
"Split every other 1024- and 2048-bit load and store "
592 "into 512-bit ones (AIE2P)"),
593 llvm::cl::init(
false)};
602 return std::make_unique<SplitVectorLoadUpsChainsPass>();
605std::unique_ptr<::mlir::Pass>
607 return std::make_unique<SplitVectorLoadUpsChainsPass>(chains, wideAccesses);
std::unique_ptr<::mlir::Pass > createSplitVectorLoadUpsChainsPass()
Create a pass that splits vector.load + aievec.ups chains to reduce shuffle operations for AIE2p targ...