MLIR-AIE 1.4.6.dev
AIEHoistVectorTransferPointers.cpp
Go to the documentation of this file.
1//===- AIEHoistVectorTransferPointers.cpp -----------------------*- C++ -*-===//
2//
3// Copyright (C) 2025 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7//
8// This pass hoists vector transfer operations with IV-dependent pointers
9// out of scf.for loops by using iter_args to track pointer updates. This
10// optimization reduces address computation overhead in loops by maintaining
11// a running pointer offset rather than recomputing addresses each iteration.
12//
13//===----------------------------------------------------------------------===//
14
17
18#include "mlir/Dialect/Affine/IR/AffineOps.h"
19#include "mlir/Dialect/Arith/IR/Arith.h"
20#include "mlir/Dialect/Func/IR/FuncOps.h"
21#include "mlir/Dialect/MemRef/IR/MemRef.h"
22#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
23#include "mlir/Dialect/SCF/IR/SCF.h"
24#include "mlir/Dialect/SCF/Utils/Utils.h"
25#include "mlir/Dialect/Utils/StaticValueUtils.h"
26#include "mlir/Dialect/Vector/IR/VectorOps.h"
27#include "mlir/IR/IRMapping.h"
28#include "mlir/IR/PatternMatch.h"
29#include "mlir/Interfaces/LoopLikeInterface.h"
30#include "mlir/Interfaces/SideEffectInterfaces.h"
31#include "mlir/Pass/Pass.h"
32#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
33#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
34
35namespace xilinx::AIE {
36#define GEN_PASS_DEF_AIEHOISTVECTORTRANSFERPOINTERS
37#include "aie/Dialect/AIE/Transforms/AIEPasses.h.inc"
38} // namespace xilinx::AIE
39
40#define DEBUG_TYPE "aie-hoist-vector-transfer-pointers"
41
42using namespace mlir;
43using namespace xilinx;
44using namespace xilinx::AIE;
45
46namespace {
47
48//===----------------------------------------------------------------------===//
49// Helper Functions
50//===----------------------------------------------------------------------===//
51
52/// How much `expr` changes each iteration, given how much each of the map's
53/// operands does (dims first, then symbols). nullopt if that is not the same
54/// every iteration.
55static std::optional<int64_t> getExprCoefficient(AffineExpr expr,
56 ArrayRef<int64_t> operandCoefs,
57 unsigned numDims) {
58 if (auto dim = dyn_cast<AffineDimExpr>(expr))
59 return operandCoefs[dim.getPosition()];
60 if (auto sym = dyn_cast<AffineSymbolExpr>(expr))
61 return operandCoefs[numDims + sym.getPosition()];
62 if (isa<AffineConstantExpr>(expr))
63 return 0;
64 auto bin = dyn_cast<AffineBinaryOpExpr>(expr);
65 if (!bin)
66 return std::nullopt;
67 std::optional<int64_t> lhs =
68 getExprCoefficient(bin.getLHS(), operandCoefs, numDims);
69 std::optional<int64_t> rhs =
70 getExprCoefficient(bin.getRHS(), operandCoefs, numDims);
71 if (!lhs || !rhs)
72 return std::nullopt;
73 switch (bin.getKind()) {
74 case AffineExprKind::Add:
75 return *lhs + *rhs;
76 case AffineExprKind::Mul:
77 // In an affine expression one side of a multiplication is a constant.
78 if (auto c = dyn_cast<AffineConstantExpr>(bin.getRHS()))
79 return *lhs * c.getValue();
80 if (auto c = dyn_cast<AffineConstantExpr>(bin.getLHS()))
81 return *rhs * c.getValue();
82 return (*lhs == 0 && *rhs == 0) ? std::optional<int64_t>(0) : std::nullopt;
83 default: {
84 // x floordiv c (ceildiv c) changes by d / c where x changes by d, a
85 // multiple of c; x mod c then not at all.
86 auto c = dyn_cast<AffineConstantExpr>(bin.getRHS());
87 if (*rhs == 0 && c && c.getValue() > 0 && *lhs % c.getValue() == 0)
88 return bin.getKind() == AffineExprKind::Mod ? 0 : *lhs / c.getValue();
89 return (*lhs == 0 && *rhs == 0) ? std::optional<int64_t>(0) : std::nullopt;
90 }
91 }
92}
93
94/// `map`'s result with `iv floordiv c` and `iv mod c` (iv `forOp`'s IV, an
95/// operand of the map) replaced by 0 and iv where the loop keeps iv in
96/// [0, c).
97static AffineExpr withinIVRange(AffineMap map, ValueRange operands,
98 scf::ForOp forOp) {
99 std::optional<int64_t> lb = getConstantIntValue(forOp.getLowerBound());
100 std::optional<int64_t> ub = getConstantIntValue(forOp.getUpperBound());
101 AffineExpr e = map.getResult(0);
102 // (an unsigned loop's bounds are not their signed values)
103 if (forOp.getUnsignedCmp() || !lb || !ub || *lb < 0)
104 return e;
105 std::function<AffineExpr(AffineExpr)> rw = [&](AffineExpr x) -> AffineExpr {
106 auto bin = dyn_cast<AffineBinaryOpExpr>(x);
107 if (!bin)
108 return x;
109 AffineExpr l = rw(bin.getLHS()), r = rw(bin.getRHS());
110 switch (bin.getKind()) {
111 case AffineExprKind::Add:
112 return l + r;
113 case AffineExprKind::Mul:
114 return l * r;
115 case AffineExprKind::CeilDiv:
116 return l.ceilDiv(r);
117 default:
118 break;
119 }
120 // iv floordiv c, iv mod c with iv in [0, c).
121 auto c = dyn_cast<AffineConstantExpr>(r);
122 std::optional<unsigned> pos;
123 if (auto d = dyn_cast<AffineDimExpr>(l))
124 pos = d.getPosition();
125 else if (auto sy = dyn_cast<AffineSymbolExpr>(l))
126 pos = map.getNumDims() + sy.getPosition();
127 bool inRange = c && pos && operands[*pos] == forOp.getInductionVar() &&
128 *ub <= c.getValue();
129 if (bin.getKind() == AffineExprKind::FloorDiv)
130 return inRange ? getAffineConstantExpr(0, x.getContext()) : l.floorDiv(r);
131 return inRange ? l : l % r;
132 };
133 return rw(e);
134}
135
136/// How much `v` changes each iteration of `forOp` (`coef * step` for `v` of
137/// the form `coef * iv + inv`, where `inv` does not change across iterations
138/// and can be recomputed before the loop). nullopt when that is not the same
139/// every iteration: `v` reads an iter_arg, loads from memory, or depends on
140/// the IV non-linearly, or the step is not a constant.
141static std::optional<int64_t>
142getIVCoefficient(Value v, scf::ForOp forOp,
143 DenseMap<Value, std::optional<int64_t>> &cache) {
144 if (v == forOp.getInductionVar()) {
145 if (auto st = forOp.getConstantStep())
146 return st->getSExtValue();
147 return std::nullopt;
148 }
149 if (forOp.isDefinedOutsideOfLoop(v))
150 return 0;
151 auto it = cache.find(v);
152 if (it != cache.end())
153 return it->second;
154
155 std::optional<int64_t> result;
156 Operation *def = v.getDefiningOp();
157 // A block argument other than the IV (an iter_arg) changes every iteration.
158 if (def && isPure(def)) {
159 SmallVector<int64_t> coefs;
160 bool linearOperands = true;
161 for (Value operand : def->getOperands()) {
162 std::optional<int64_t> c = getIVCoefficient(operand, forOp, cache);
163 if (!c) {
164 linearOperands = false;
165 break;
166 }
167 coefs.push_back(*c);
168 }
169 if (linearOperands) {
170 bool invariant = llvm::all_of(coefs, [](int64_t c) { return c == 0; });
171 if (isa<arith::AddIOp>(def)) {
172 result = coefs[0] + coefs[1];
173 } else if (isa<arith::SubIOp>(def)) {
174 result = coefs[0] - coefs[1];
175 } else if (auto mul = dyn_cast<arith::MulIOp>(def)) {
176 if (std::optional<int64_t> c = getConstantIntValue(mul.getRhs()))
177 result = coefs[0] * *c;
178 else if (std::optional<int64_t> c = getConstantIntValue(mul.getLhs()))
179 result = coefs[1] * *c;
180 else if (invariant)
181 result = 0;
182 } else if (auto apply = dyn_cast<affine::AffineApplyOp>(def)) {
183 AffineMap map = apply.getAffineMap();
184 result = getExprCoefficient(
185 withinIVRange(map, apply.getMapOperands(), forOp), coefs,
186 map.getNumDims());
187 } else if (invariant) {
188 // Any other pure op over loop-invariant operands is loop-invariant.
189 result = 0;
190 }
191 }
192 }
193 cache[v] = result;
194 return result;
195}
196
197/// Recomputes `v` (of the form getIVCoefficient accepts) before `forOp` with
198/// the IV replaced by the loop's lower bound.
199static Value cloneAtLowerBound(Value v, scf::ForOp forOp, OpBuilder &builder,
200 IRMapping &mapping) {
201 if (v == forOp.getInductionVar())
202 return forOp.getLowerBound();
203 if (forOp.isDefinedOutsideOfLoop(v))
204 return v;
205 if (Value mapped = mapping.lookupOrNull(v))
206 return mapped;
207 Operation *def = v.getDefiningOp();
208 for (Value operand : def->getOperands())
209 mapping.map(operand, cloneAtLowerBound(operand, forOp, builder, mapping));
210 builder.clone(*def, mapping);
211 return mapping.lookup(v);
212}
213
214/// Get the total number of elements in a vector type
215static int64_t getVectorNumElements(VectorType vectorType) {
216 int64_t numElements = 1;
217 for (int64_t dim : vectorType.getShape()) {
218 numElements *= dim;
219 }
220 return numElements;
221}
222
223//===----------------------------------------------------------------------===//
224// HoistVectorTransferPointers Pattern
225//===----------------------------------------------------------------------===//
226
227/// Information about a vector transfer operation
228struct TransferOpInfo {
229 Operation *op;
230 Value base;
231 MemRefType memrefType;
232 VectorType vectorType;
233 SmallVector<Value> indices;
234 int64_t constantStride; // Total constant stride per iteration
235 bool hasIVDependentIndices;
236};
237
238/// Pattern to hoist vector transfer operations with IV-dependent pointers
239/// out of scf.for loops by using iter_args to track pointer updates
240struct HoistVectorTransferPointersPattern
241 : public OpRewritePattern<scf::ForOp> {
242 using OpRewritePattern<scf::ForOp>::OpRewritePattern;
243
244 LogicalResult matchAndRewrite(scf::ForOp forOp,
245 PatternRewriter &rewriter) const override {
246 Location loc = forOp.getLoc();
247
248 // Collect all vector transfer operations with IV-dependent indices
249 SmallVector<TransferOpInfo> transferOps;
250 DenseMap<Value, std::optional<int64_t>> ivCoefs;
251
252 for (Operation &op : forOp.getBody()->without_terminator()) {
253 Value base;
254 VectorType vectorType;
255 SmallVector<Value> indices;
256
257 if (auto readOp = dyn_cast<vector::TransferReadOp>(&op)) {
258 base = readOp.getBase();
259 vectorType = readOp.getVectorType();
260 indices.assign(readOp.getIndices().begin(), readOp.getIndices().end());
261 } else if (auto writeOp = dyn_cast<vector::TransferWriteOp>(&op)) {
262 base = writeOp.getBase();
263 vectorType = writeOp.getVectorType();
264 indices.assign(writeOp.getIndices().begin(),
265 writeOp.getIndices().end());
266 } else {
267 continue;
268 }
269
270 auto memrefType = dyn_cast<MemRefType>(base.getType());
271 if (!memrefType)
272 continue;
273
274 // The rewrite accesses the vector as one contiguous, unmasked, in-bounds
275 // run of the flattened buffer. Skip transfers that are not of that form.
276 auto xfer = cast<VectorTransferOpInterface>(op);
277 if (xfer.getMask() || !xfer.getPermutationMap().isMinorIdentity() ||
278 !llvm::all_of(xfer.getInBoundsValues(), [](bool b) { return b; }))
279 continue;
280 if (!memref::isStaticShapeAndContiguousRowMajor(memrefType))
281 continue;
282 // A vector<4x16> row block is contiguous only if each vector dim inside
283 // its outermost non-unit one spans its whole memref dim (so
284 // vector<1x1x8x8> of memref<4x8x8x8> is, vector<4x16> of memref<64x64>
285 // is not).
286 int64_t vecRank = vectorType.getRank();
287 int64_t memRank = memrefType.getRank();
288 if (vecRank > memRank)
289 continue;
290 int64_t outer = 0;
291 while (outer < vecRank - 1 && vectorType.getDimSize(outer) == 1)
292 ++outer;
293 bool contiguous = true;
294 for (int64_t i = outer + 1; i < vecRank; ++i)
295 if (vectorType.getDimSize(i) !=
296 memrefType.getDimSize(memRank - vecRank + i))
297 contiguous = false;
298 if (!contiguous)
299 continue;
300 // The flattened view is created before the loop.
301 if (!forOp.isDefinedOutsideOfLoop(base))
302 continue;
303
304 // Each index must change by the same amount every iteration; the
305 // pointer then advances by sum(change * dimStride) elements.
306 std::optional<int64_t> step =
307 forOp.getConstantStep()
308 ? std::optional<int64_t>(forOp.getConstantStep()->getSExtValue())
309 : std::nullopt;
310 bool analyzable = true;
311 int64_t elementsPerIteration = 0;
312 int64_t dimStride = 1;
313 for (int64_t d = memRank - 1; d >= 0; --d) {
314 std::optional<int64_t> coef =
315 getIVCoefficient(indices[d], forOp, ivCoefs);
316 if (!coef) {
317 analyzable = false;
318 break;
319 }
320 elementsPerIteration += *coef * dimStride;
321 dimStride *= memrefType.getDimSize(d);
322 }
323 if (!analyzable)
324 continue;
325 bool hasIVDependentIndices = elementsPerIteration != 0;
326 if (hasIVDependentIndices && !step)
327 continue;
328 int64_t constantStride = hasIVDependentIndices ? elementsPerIteration : 0;
329
330 transferOps.push_back({&op, base, memrefType, vectorType, indices,
331 constantStride, hasIVDependentIndices});
332 }
333
334 // If there are no transfer ops, don't modify
335 if (transferOps.empty())
336 return failure();
337
338 // Prepare to add iter_args for each transfer operation with IV-dependent
339 // indices
340 SmallVector<Value> newInitArgs;
341 SmallVector<Value> flatMemrefs;
342
343 for (const auto &info : transferOps) {
344 if (!info.hasIVDependentIndices)
345 continue;
346
347 // Flatten the memref if needed
348 rewriter.setInsertionPoint(forOp);
349 Value flatMemref = info.base;
350 if (info.memrefType.getRank() > 1) {
351 int64_t totalSize = 1;
352 for (int64_t dim : info.memrefType.getShape()) {
353 if (dim == ShapedType::kDynamic)
354 return failure(); // Dynamic memref shapes not supported
355 totalSize *= dim;
356 }
357
358 // Preserve strided layout if present
359 MemRefType flatMemrefType;
360 if (auto stridedLayout = dyn_cast_or_null<StridedLayoutAttr>(
361 info.memrefType.getLayout())) {
362 // The collapsed stride is the innermost stride (last element)
363 int64_t collapsedStride = stridedLayout.getStrides().back();
364 int64_t offset = stridedLayout.getOffset();
365
366 auto newLayout = StridedLayoutAttr::get(rewriter.getContext(), offset,
367 {collapsedStride});
368 flatMemrefType =
369 MemRefType::get({totalSize}, info.memrefType.getElementType(),
370 newLayout, info.memrefType.getMemorySpace());
371 } else {
372 flatMemrefType =
373 MemRefType::get({totalSize}, info.memrefType.getElementType(),
374 AffineMap(), info.memrefType.getMemorySpace());
375 }
376
377 SmallVector<ReassociationIndices> reassociation;
378 ReassociationIndices allDims;
379 for (size_t i = 0; i < static_cast<size_t>(info.memrefType.getRank());
380 ++i) {
381 allDims.push_back(i);
382 }
383 reassociation.push_back(allDims);
384
385 flatMemref = memref::CollapseShapeOp::create(
386 rewriter, loc, flatMemrefType, info.base, reassociation);
387 }
388 flatMemrefs.push_back(flatMemref);
389
390 // Compute base pointer (with zeros for IV-dependent parts)
391 int64_t rank = info.memrefType.getRank();
392 AffineExpr linearExpr = rewriter.getAffineConstantExpr(0);
393 int64_t stride = 1;
394 for (int64_t i = rank - 1; i >= 0; --i) {
395 linearExpr = linearExpr + rewriter.getAffineDimExpr(i) * stride;
396 if (i > 0)
397 stride *= info.memrefType.getShape()[i];
398 }
399 auto linearMap = AffineMap::get(rank, 0, linearExpr);
400
401 // Initial pointer: every index evaluated at the lower bound.
402 SmallVector<Value> evaluatedIndices;
403 IRMapping indexMapping;
404 for (Value idx : info.indices)
405 evaluatedIndices.push_back(
406 cloneAtLowerBound(idx, forOp, rewriter, indexMapping));
407
408 Value basePointer = affine::AffineApplyOp::create(
409 rewriter, loc, linearMap, evaluatedIndices);
410
411 newInitArgs.push_back(basePointer);
412 }
413
414 // If there are no IV-dependent transfers, just process them to flatten
415 // vectors
416 if (newInitArgs.empty()) {
417 // Check if any transfer needs flattening (avoid infinite rewrites)
418 bool needsFlattening = false;
419 bool hasProcessableTransfers = false;
420 for (const auto &info : transferOps) {
421 // Skip if base is defined inside the loop (e.g., a subview)
422 // We can't hoist these
423 if (info.base.getDefiningOp() &&
424 forOp->isProperAncestor(info.base.getDefiningOp()))
425 continue;
426
427 hasProcessableTransfers = true;
428
429 // Check if this transfer has already been flattened
430 // (flattened transfers use 1D identity map)
431 if (auto readOp = dyn_cast<vector::TransferReadOp>(info.op)) {
432 if (readOp.getPermutationMap().getNumDims() != 1)
433 needsFlattening = true;
434 } else if (auto writeOp = dyn_cast<vector::TransferWriteOp>(info.op)) {
435 if (writeOp.getPermutationMap().getNumDims() != 1)
436 needsFlattening = true;
437 }
438 }
439
440 // If there are no processable transfers (all bases defined in loop)
441 // or nothing needs flattening, bail out
442 if (!hasProcessableTransfers || !needsFlattening)
443 return failure();
444
445 // First, create flattened memrefs outside the loop for bases not defined
446 // inside
447 DenseMap<Value, Value> baseFlatMemrefs;
448 rewriter.setInsertionPoint(forOp);
449 for (const auto &info : transferOps) {
450 if (baseFlatMemrefs.count(info.base))
451 continue;
452
453 // Skip if base is defined inside the loop (e.g., a subview)
454 if (info.base.getDefiningOp() &&
455 forOp->isProperAncestor(info.base.getDefiningOp()))
456 continue;
457
458 Value flatMemref = info.base;
459 if (info.memrefType.getRank() > 1) {
460 int64_t totalSize = 1;
461 for (int64_t dim : info.memrefType.getShape()) {
462 totalSize *= dim;
463 }
464
465 // Preserve strided layout if present
466 MemRefType flatMemrefType;
467 if (auto stridedLayout = dyn_cast_or_null<StridedLayoutAttr>(
468 info.memrefType.getLayout())) {
469 int64_t collapsedStride = stridedLayout.getStrides().back();
470 int64_t offset = stridedLayout.getOffset();
471
472 auto newLayout = StridedLayoutAttr::get(rewriter.getContext(),
473 offset, {collapsedStride});
474 flatMemrefType =
475 MemRefType::get({totalSize}, info.memrefType.getElementType(),
476 newLayout, info.memrefType.getMemorySpace());
477 } else {
478 flatMemrefType =
479 MemRefType::get({totalSize}, info.memrefType.getElementType(),
480 AffineMap(), info.memrefType.getMemorySpace());
481 }
482
483 SmallVector<ReassociationIndices> reassociation;
484 ReassociationIndices allDims;
485 for (size_t i = 0; i < static_cast<size_t>(info.memrefType.getRank());
486 ++i) {
487 allDims.push_back(i);
488 }
489 reassociation.push_back(allDims);
490 flatMemref = memref::CollapseShapeOp::create(
491 rewriter, loc, flatMemrefType, info.base, reassociation);
492 }
493 baseFlatMemrefs[info.base] = flatMemref;
494 }
495
496 // Process all transfers without using iter_args
497 bool madeChanges = false;
498 for (const auto &info : transferOps) {
499 // Skip if base is defined inside the loop
500 if (info.base.getDefiningOp() &&
501 forOp->isProperAncestor(info.base.getDefiningOp()))
502 continue;
503
504 // Skip if we don't have a flattened version
505 if (!baseFlatMemrefs.count(info.base))
506 continue;
507
508 rewriter.setInsertionPoint(info.op);
509
510 // Flatten vector type
511 int64_t numElements = getVectorNumElements(info.vectorType);
512 VectorType flatVectorType =
513 VectorType::get({numElements}, info.vectorType.getElementType());
514
515 // Get the flattened memref
516 Value flatMemref = baseFlatMemrefs[info.base];
517
518 // Compute pointer from indices
519 int64_t rank = info.memrefType.getRank();
520 AffineExpr linearExpr = rewriter.getAffineConstantExpr(0);
521 int64_t stride = 1;
522 for (int64_t i = rank - 1; i >= 0; --i) {
523 linearExpr = linearExpr + rewriter.getAffineDimExpr(i) * stride;
524 if (i > 0)
525 stride *= info.memrefType.getShape()[i];
526 }
527 auto linearMap = AffineMap::get(rank, 0, linearExpr);
528
529 Value currentPointer = affine::AffineApplyOp::create(
530 rewriter, loc, linearMap, info.indices);
531
532 // Transform the transfer operation
533 AffineMap identityMap1D = AffineMap::get(
534 1, 0, rewriter.getAffineDimExpr(0), rewriter.getContext());
535 auto inBoundsAttr = rewriter.getBoolArrayAttr({true});
536
537 if (auto readOp = dyn_cast<vector::TransferReadOp>(info.op)) {
538 Value flatRead = vector::TransferReadOp::create(
539 rewriter, loc, flatVectorType, flatMemref,
540 ValueRange{currentPointer}, AffineMapAttr::get(identityMap1D),
541 readOp.getPadding(),
542 /*mask=*/Value(), inBoundsAttr);
543 Value shapedRead = vector::ShapeCastOp::create(
544 rewriter, loc, info.vectorType, flatRead);
545 rewriter.replaceOp(readOp, shapedRead);
546 madeChanges = true;
547 } else if (auto writeOp = dyn_cast<vector::TransferWriteOp>(info.op)) {
548 Value flatValue = vector::ShapeCastOp::create(
549 rewriter, loc, flatVectorType, writeOp.getVector());
550 rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
551 writeOp, flatValue, flatMemref, ValueRange{currentPointer},
552 AffineMapAttr::get(identityMap1D), /*mask=*/Value(),
553 inBoundsAttr);
554 madeChanges = true;
555 }
556 }
557 return madeChanges ? success() : failure();
558 }
559
560 // Use replaceWithAdditionalYields to add pointer iter_args
561 auto yieldValuesFn =
562 [&](OpBuilder &b, Location yieldLoc,
563 ArrayRef<BlockArgument> newBbArgs) -> SmallVector<Value> {
564 SmallVector<Value> yieldValues;
565
566 // Process each transfer operation with IV-dependent indices
567 size_t iterArgIdx = 0;
568 for (const auto &info : transferOps) {
569 if (!info.hasIVDependentIndices)
570 continue;
571
572 BlockArgument ptrIterArg =
573 newBbArgs[newBbArgs.size() - newInitArgs.size() + iterArgIdx];
574 Value flatMemref = flatMemrefs[iterArgIdx];
575
576 // Flatten vector type
577 int64_t numElements = getVectorNumElements(info.vectorType);
578 VectorType flatVectorType =
579 VectorType::get({numElements}, info.vectorType.getElementType());
580
581 // Transform the transfer operation to use the iter_arg pointer
582 b.setInsertionPoint(info.op);
583
584 AffineMap identityMap1D =
585 AffineMap::get(1, 0, b.getAffineDimExpr(0), b.getContext());
586 auto inBoundsAttr = b.getBoolArrayAttr({true});
587
588 if (auto readOp = dyn_cast<vector::TransferReadOp>(info.op)) {
589 Value flatRead = vector::TransferReadOp::create(
590 b, loc, flatVectorType, flatMemref, ValueRange{ptrIterArg},
591 AffineMapAttr::get(identityMap1D), readOp.getPadding(),
592 /*mask=*/Value(), inBoundsAttr);
593 Value shapedRead =
594 vector::ShapeCastOp::create(b, loc, info.vectorType, flatRead);
595 rewriter.replaceOp(readOp, shapedRead);
596 } else if (auto writeOp = dyn_cast<vector::TransferWriteOp>(info.op)) {
597 Value flatValue = vector::ShapeCastOp::create(b, loc, flatVectorType,
598 writeOp.getVector());
599 rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
600 writeOp, flatValue, flatMemref, ValueRange{ptrIterArg},
601 AffineMapAttr::get(identityMap1D), /*mask=*/Value(),
602 inBoundsAttr);
603 }
604
605 // Compute next pointer value: current_ptr + constant_stride
606 Value strideConst =
607 arith::ConstantIndexOp::create(b, yieldLoc, info.constantStride);
608 Value nextPtr =
609 arith::AddIOp::create(b, yieldLoc, ptrIterArg, strideConst);
610 yieldValues.push_back(nextPtr);
611
612 iterArgIdx++;
613 }
614
615 return yieldValues;
616 };
617
618 // Create new loop with additional iter_args for pointers
619 FailureOr<LoopLikeOpInterface> newLoopResult =
620 cast<LoopLikeOpInterface>(forOp.getOperation())
621 .replaceWithAdditionalYields(
622 rewriter, newInitArgs, // new init operands (base pointers)
623 true, // replace uses in loop
624 yieldValuesFn);
625
626 if (failed(newLoopResult))
627 return failure();
628
629 return success();
630 }
631};
632
633//===----------------------------------------------------------------------===//
634// AIEHoistVectorTransferPointersPass
635//===----------------------------------------------------------------------===//
636
637struct AIEHoistVectorTransferPointersPass
638 : xilinx::AIE::impl::AIEHoistVectorTransferPointersBase<
639 AIEHoistVectorTransferPointersPass> {
640 void getDependentDialects(DialectRegistry &registry) const override {
641 registry.insert<affine::AffineDialect, arith::ArithDialect,
642 memref::MemRefDialect, scf::SCFDialect,
643 vector::VectorDialect>();
644 }
645
646 void runOnOperation() override {
647 ModuleOp moduleOp = getOperation();
648 MLIRContext *context = &getContext();
649
650 // A transfer's buffer viewed inside a loop it does not depend on (a
651 // reshape made next to the transfer): the view out of the loop first, so
652 // that the transfer's base is defined outside it.
653 moduleOp.walk([&](scf::ForOp forOp) {
654 moveLoopInvariantCode(
655 forOp.getLoopRegions(),
656 [&](Value v, Region *) { return forOp.isDefinedOutsideOfLoop(v); },
657 [&](Operation *op, Region *) {
658 return isa<memref::CollapseShapeOp, memref::ExpandShapeOp,
659 memref::CastOp, memref::ReinterpretCastOp>(op);
660 },
661 [&](Operation *op, Region *) { forOp.moveOutOfLoop(op); });
662 });
663
664 RewritePatternSet patterns(context);
665 patterns.add<HoistVectorTransferPointersPattern>(context);
666
667 // Apply patterns to the entire module - the pattern will only match scf.for
668 // ops within aie.core regions
669 if (failed(applyPatternsGreedily(moduleOp, std::move(patterns))))
670 signalPassFailure();
671 }
672};
673
674} // namespace
675
676std::unique_ptr<OperationPass<ModuleOp>>
678 return std::make_unique<AIEHoistVectorTransferPointersPass>();
679}
std::unique_ptr< mlir::OperationPass< mlir::ModuleOp > > createAIEHoistVectorTransferPointersPass()