18#include "mlir/Dialect/Affine/IR/AffineOps.h"
19#include "mlir/Dialect/Arith/IR/Arith.h"
20#include "mlir/Dialect/Func/IR/FuncOps.h"
21#include "mlir/Dialect/MemRef/IR/MemRef.h"
22#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
23#include "mlir/Dialect/SCF/IR/SCF.h"
24#include "mlir/Dialect/SCF/Utils/Utils.h"
25#include "mlir/Dialect/Utils/StaticValueUtils.h"
26#include "mlir/Dialect/Vector/IR/VectorOps.h"
27#include "mlir/IR/IRMapping.h"
28#include "mlir/IR/PatternMatch.h"
29#include "mlir/Interfaces/LoopLikeInterface.h"
30#include "mlir/Interfaces/SideEffectInterfaces.h"
31#include "mlir/Pass/Pass.h"
32#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
33#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
36#define GEN_PASS_DEF_AIEHOISTVECTORTRANSFERPOINTERS
37#include "aie/Dialect/AIE/Transforms/AIEPasses.h.inc"
40#define DEBUG_TYPE "aie-hoist-vector-transfer-pointers"
55static std::optional<int64_t> getExprCoefficient(AffineExpr expr,
56 ArrayRef<int64_t> operandCoefs,
58 if (
auto dim = dyn_cast<AffineDimExpr>(expr))
59 return operandCoefs[dim.getPosition()];
60 if (
auto sym = dyn_cast<AffineSymbolExpr>(expr))
61 return operandCoefs[numDims + sym.getPosition()];
62 if (isa<AffineConstantExpr>(expr))
64 auto bin = dyn_cast<AffineBinaryOpExpr>(expr);
67 std::optional<int64_t> lhs =
68 getExprCoefficient(bin.getLHS(), operandCoefs, numDims);
69 std::optional<int64_t> rhs =
70 getExprCoefficient(bin.getRHS(), operandCoefs, numDims);
73 switch (bin.getKind()) {
74 case AffineExprKind::Add:
76 case AffineExprKind::Mul:
78 if (
auto c = dyn_cast<AffineConstantExpr>(bin.getRHS()))
79 return *lhs * c.getValue();
80 if (
auto c = dyn_cast<AffineConstantExpr>(bin.getLHS()))
81 return *rhs * c.getValue();
82 return (*lhs == 0 && *rhs == 0) ? std::optional<int64_t>(0) : std::nullopt;
86 auto c = dyn_cast<AffineConstantExpr>(bin.getRHS());
87 if (*rhs == 0 && c && c.getValue() > 0 && *lhs % c.getValue() == 0)
88 return bin.getKind() == AffineExprKind::Mod ? 0 : *lhs / c.getValue();
89 return (*lhs == 0 && *rhs == 0) ? std::optional<int64_t>(0) : std::nullopt;
97static AffineExpr withinIVRange(AffineMap map, ValueRange operands,
99 std::optional<int64_t> lb = getConstantIntValue(forOp.getLowerBound());
100 std::optional<int64_t> ub = getConstantIntValue(forOp.getUpperBound());
101 AffineExpr e = map.getResult(0);
103 if (forOp.getUnsignedCmp() || !lb || !ub || *lb < 0)
105 std::function<AffineExpr(AffineExpr)> rw = [&](AffineExpr x) -> AffineExpr {
106 auto bin = dyn_cast<AffineBinaryOpExpr>(x);
109 AffineExpr l = rw(bin.getLHS()), r = rw(bin.getRHS());
110 switch (bin.getKind()) {
111 case AffineExprKind::Add:
113 case AffineExprKind::Mul:
115 case AffineExprKind::CeilDiv:
121 auto c = dyn_cast<AffineConstantExpr>(r);
122 std::optional<unsigned> pos;
123 if (
auto d = dyn_cast<AffineDimExpr>(l))
124 pos = d.getPosition();
125 else if (
auto sy = dyn_cast<AffineSymbolExpr>(l))
126 pos = map.getNumDims() + sy.getPosition();
127 bool inRange = c && pos && operands[*pos] == forOp.getInductionVar() &&
129 if (bin.getKind() == AffineExprKind::FloorDiv)
130 return inRange ? getAffineConstantExpr(0, x.getContext()) : l.floorDiv(r);
131 return inRange ? l : l % r;
141static std::optional<int64_t>
142getIVCoefficient(Value v, scf::ForOp forOp,
143 DenseMap<Value, std::optional<int64_t>> &cache) {
144 if (v == forOp.getInductionVar()) {
145 if (
auto st = forOp.getConstantStep())
146 return st->getSExtValue();
149 if (forOp.isDefinedOutsideOfLoop(v))
151 auto it = cache.find(v);
152 if (it != cache.end())
155 std::optional<int64_t> result;
156 Operation *def = v.getDefiningOp();
158 if (def && isPure(def)) {
159 SmallVector<int64_t> coefs;
160 bool linearOperands =
true;
161 for (Value operand : def->getOperands()) {
162 std::optional<int64_t> c = getIVCoefficient(operand, forOp, cache);
164 linearOperands =
false;
169 if (linearOperands) {
170 bool invariant = llvm::all_of(coefs, [](int64_t c) {
return c == 0; });
171 if (isa<arith::AddIOp>(def)) {
172 result = coefs[0] + coefs[1];
173 }
else if (isa<arith::SubIOp>(def)) {
174 result = coefs[0] - coefs[1];
175 }
else if (
auto mul = dyn_cast<arith::MulIOp>(def)) {
176 if (std::optional<int64_t> c = getConstantIntValue(mul.getRhs()))
177 result = coefs[0] * *c;
178 else if (std::optional<int64_t> c = getConstantIntValue(mul.getLhs()))
179 result = coefs[1] * *c;
182 }
else if (
auto apply = dyn_cast<affine::AffineApplyOp>(def)) {
183 AffineMap map = apply.getAffineMap();
184 result = getExprCoefficient(
185 withinIVRange(map, apply.getMapOperands(), forOp), coefs,
187 }
else if (invariant) {
199static Value cloneAtLowerBound(Value v, scf::ForOp forOp, OpBuilder &builder,
200 IRMapping &mapping) {
201 if (v == forOp.getInductionVar())
202 return forOp.getLowerBound();
203 if (forOp.isDefinedOutsideOfLoop(v))
205 if (Value mapped = mapping.lookupOrNull(v))
207 Operation *def = v.getDefiningOp();
208 for (Value operand : def->getOperands())
209 mapping.map(operand, cloneAtLowerBound(operand, forOp, builder, mapping));
210 builder.clone(*def, mapping);
211 return mapping.lookup(v);
215static int64_t getVectorNumElements(VectorType vectorType) {
216 int64_t numElements = 1;
217 for (int64_t dim : vectorType.getShape()) {
228struct TransferOpInfo {
231 MemRefType memrefType;
232 VectorType vectorType;
233 SmallVector<Value> indices;
234 int64_t constantStride;
235 bool hasIVDependentIndices;
240struct HoistVectorTransferPointersPattern
244 LogicalResult matchAndRewrite(scf::ForOp forOp,
245 PatternRewriter &rewriter)
const override {
246 Location loc = forOp.getLoc();
249 SmallVector<TransferOpInfo> transferOps;
250 DenseMap<Value, std::optional<int64_t>> ivCoefs;
252 for (Operation &op : forOp.getBody()->without_terminator()) {
254 VectorType vectorType;
255 SmallVector<Value> indices;
257 if (
auto readOp = dyn_cast<vector::TransferReadOp>(&op)) {
258 base = readOp.getBase();
259 vectorType = readOp.getVectorType();
260 indices.assign(readOp.getIndices().begin(), readOp.getIndices().end());
261 }
else if (
auto writeOp = dyn_cast<vector::TransferWriteOp>(&op)) {
262 base = writeOp.getBase();
263 vectorType = writeOp.getVectorType();
264 indices.assign(writeOp.getIndices().begin(),
265 writeOp.getIndices().end());
270 auto memrefType = dyn_cast<MemRefType>(base.getType());
276 auto xfer = cast<VectorTransferOpInterface>(op);
277 if (xfer.getMask() || !xfer.getPermutationMap().isMinorIdentity() ||
278 !llvm::all_of(xfer.getInBoundsValues(), [](
bool b) { return b; }))
280 if (!memref::isStaticShapeAndContiguousRowMajor(memrefType))
286 int64_t vecRank = vectorType.getRank();
287 int64_t memRank = memrefType.getRank();
288 if (vecRank > memRank)
291 while (outer < vecRank - 1 && vectorType.getDimSize(outer) == 1)
293 bool contiguous =
true;
294 for (int64_t i = outer + 1; i < vecRank; ++i)
295 if (vectorType.getDimSize(i) !=
296 memrefType.getDimSize(memRank - vecRank + i))
301 if (!forOp.isDefinedOutsideOfLoop(base))
306 std::optional<int64_t> step =
307 forOp.getConstantStep()
308 ? std::optional<int64_t>(forOp.getConstantStep()->getSExtValue())
310 bool analyzable =
true;
311 int64_t elementsPerIteration = 0;
312 int64_t dimStride = 1;
313 for (int64_t d = memRank - 1; d >= 0; --d) {
314 std::optional<int64_t> coef =
315 getIVCoefficient(indices[d], forOp, ivCoefs);
320 elementsPerIteration += *coef * dimStride;
321 dimStride *= memrefType.getDimSize(d);
325 bool hasIVDependentIndices = elementsPerIteration != 0;
326 if (hasIVDependentIndices && !step)
328 int64_t constantStride = hasIVDependentIndices ? elementsPerIteration : 0;
330 transferOps.push_back({&op, base, memrefType, vectorType, indices,
331 constantStride, hasIVDependentIndices});
335 if (transferOps.empty())
340 SmallVector<Value> newInitArgs;
341 SmallVector<Value> flatMemrefs;
343 for (
const auto &info : transferOps) {
344 if (!info.hasIVDependentIndices)
348 rewriter.setInsertionPoint(forOp);
349 Value flatMemref = info.base;
350 if (info.memrefType.getRank() > 1) {
351 int64_t totalSize = 1;
352 for (int64_t dim : info.memrefType.getShape()) {
353 if (dim == ShapedType::kDynamic)
359 MemRefType flatMemrefType;
360 if (
auto stridedLayout = dyn_cast_or_null<StridedLayoutAttr>(
361 info.memrefType.getLayout())) {
363 int64_t collapsedStride = stridedLayout.getStrides().back();
364 int64_t offset = stridedLayout.getOffset();
366 auto newLayout = StridedLayoutAttr::get(rewriter.getContext(), offset,
369 MemRefType::get({totalSize}, info.memrefType.getElementType(),
370 newLayout, info.memrefType.getMemorySpace());
373 MemRefType::get({totalSize}, info.memrefType.getElementType(),
374 AffineMap(), info.memrefType.getMemorySpace());
377 SmallVector<ReassociationIndices> reassociation;
378 ReassociationIndices allDims;
379 for (
size_t i = 0; i < static_cast<size_t>(info.memrefType.getRank());
381 allDims.push_back(i);
383 reassociation.push_back(allDims);
385 flatMemref = memref::CollapseShapeOp::create(
386 rewriter, loc, flatMemrefType, info.base, reassociation);
388 flatMemrefs.push_back(flatMemref);
391 int64_t rank = info.memrefType.getRank();
392 AffineExpr linearExpr = rewriter.getAffineConstantExpr(0);
394 for (int64_t i = rank - 1; i >= 0; --i) {
395 linearExpr = linearExpr + rewriter.getAffineDimExpr(i) * stride;
397 stride *= info.memrefType.getShape()[i];
399 auto linearMap = AffineMap::get(rank, 0, linearExpr);
402 SmallVector<Value> evaluatedIndices;
403 IRMapping indexMapping;
404 for (Value idx : info.indices)
405 evaluatedIndices.push_back(
406 cloneAtLowerBound(idx, forOp, rewriter, indexMapping));
408 Value basePointer = affine::AffineApplyOp::create(
409 rewriter, loc, linearMap, evaluatedIndices);
411 newInitArgs.push_back(basePointer);
416 if (newInitArgs.empty()) {
418 bool needsFlattening =
false;
419 bool hasProcessableTransfers =
false;
420 for (
const auto &info : transferOps) {
423 if (info.base.getDefiningOp() &&
424 forOp->isProperAncestor(info.base.getDefiningOp()))
427 hasProcessableTransfers =
true;
431 if (
auto readOp = dyn_cast<vector::TransferReadOp>(info.op)) {
432 if (readOp.getPermutationMap().getNumDims() != 1)
433 needsFlattening =
true;
434 }
else if (
auto writeOp = dyn_cast<vector::TransferWriteOp>(info.op)) {
435 if (writeOp.getPermutationMap().getNumDims() != 1)
436 needsFlattening =
true;
442 if (!hasProcessableTransfers || !needsFlattening)
447 DenseMap<Value, Value> baseFlatMemrefs;
448 rewriter.setInsertionPoint(forOp);
449 for (
const auto &info : transferOps) {
450 if (baseFlatMemrefs.count(info.base))
454 if (info.base.getDefiningOp() &&
455 forOp->isProperAncestor(info.base.getDefiningOp()))
458 Value flatMemref = info.base;
459 if (info.memrefType.getRank() > 1) {
460 int64_t totalSize = 1;
461 for (int64_t dim : info.memrefType.getShape()) {
466 MemRefType flatMemrefType;
467 if (
auto stridedLayout = dyn_cast_or_null<StridedLayoutAttr>(
468 info.memrefType.getLayout())) {
469 int64_t collapsedStride = stridedLayout.getStrides().back();
470 int64_t offset = stridedLayout.getOffset();
472 auto newLayout = StridedLayoutAttr::get(rewriter.getContext(),
473 offset, {collapsedStride});
475 MemRefType::get({totalSize}, info.memrefType.getElementType(),
476 newLayout, info.memrefType.getMemorySpace());
479 MemRefType::get({totalSize}, info.memrefType.getElementType(),
480 AffineMap(), info.memrefType.getMemorySpace());
483 SmallVector<ReassociationIndices> reassociation;
484 ReassociationIndices allDims;
485 for (
size_t i = 0; i < static_cast<size_t>(info.memrefType.getRank());
487 allDims.push_back(i);
489 reassociation.push_back(allDims);
490 flatMemref = memref::CollapseShapeOp::create(
491 rewriter, loc, flatMemrefType, info.base, reassociation);
493 baseFlatMemrefs[info.base] = flatMemref;
497 bool madeChanges =
false;
498 for (
const auto &info : transferOps) {
500 if (info.base.getDefiningOp() &&
501 forOp->isProperAncestor(info.base.getDefiningOp()))
505 if (!baseFlatMemrefs.count(info.base))
508 rewriter.setInsertionPoint(info.op);
511 int64_t numElements = getVectorNumElements(info.vectorType);
512 VectorType flatVectorType =
513 VectorType::get({numElements}, info.vectorType.getElementType());
516 Value flatMemref = baseFlatMemrefs[info.base];
519 int64_t rank = info.memrefType.getRank();
520 AffineExpr linearExpr = rewriter.getAffineConstantExpr(0);
522 for (int64_t i = rank - 1; i >= 0; --i) {
523 linearExpr = linearExpr + rewriter.getAffineDimExpr(i) * stride;
525 stride *= info.memrefType.getShape()[i];
527 auto linearMap = AffineMap::get(rank, 0, linearExpr);
529 Value currentPointer = affine::AffineApplyOp::create(
530 rewriter, loc, linearMap, info.indices);
533 AffineMap identityMap1D = AffineMap::get(
534 1, 0, rewriter.getAffineDimExpr(0), rewriter.getContext());
535 auto inBoundsAttr = rewriter.getBoolArrayAttr({
true});
537 if (
auto readOp = dyn_cast<vector::TransferReadOp>(info.op)) {
538 Value flatRead = vector::TransferReadOp::create(
539 rewriter, loc, flatVectorType, flatMemref,
540 ValueRange{currentPointer}, AffineMapAttr::get(identityMap1D),
542 Value(), inBoundsAttr);
543 Value shapedRead = vector::ShapeCastOp::create(
544 rewriter, loc, info.vectorType, flatRead);
545 rewriter.replaceOp(readOp, shapedRead);
547 }
else if (
auto writeOp = dyn_cast<vector::TransferWriteOp>(info.op)) {
548 Value flatValue = vector::ShapeCastOp::create(
549 rewriter, loc, flatVectorType, writeOp.getVector());
550 rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
551 writeOp, flatValue, flatMemref, ValueRange{currentPointer},
552 AffineMapAttr::get(identityMap1D), Value(),
557 return madeChanges ? success() : failure();
562 [&](OpBuilder &b, Location yieldLoc,
563 ArrayRef<BlockArgument> newBbArgs) -> SmallVector<Value> {
564 SmallVector<Value> yieldValues;
567 size_t iterArgIdx = 0;
568 for (
const auto &info : transferOps) {
569 if (!info.hasIVDependentIndices)
572 BlockArgument ptrIterArg =
573 newBbArgs[newBbArgs.size() - newInitArgs.size() + iterArgIdx];
574 Value flatMemref = flatMemrefs[iterArgIdx];
577 int64_t numElements = getVectorNumElements(info.vectorType);
578 VectorType flatVectorType =
579 VectorType::get({numElements}, info.vectorType.getElementType());
582 b.setInsertionPoint(info.op);
584 AffineMap identityMap1D =
585 AffineMap::get(1, 0, b.getAffineDimExpr(0), b.getContext());
586 auto inBoundsAttr = b.getBoolArrayAttr({
true});
588 if (
auto readOp = dyn_cast<vector::TransferReadOp>(info.op)) {
589 Value flatRead = vector::TransferReadOp::create(
590 b, loc, flatVectorType, flatMemref, ValueRange{ptrIterArg},
591 AffineMapAttr::get(identityMap1D), readOp.getPadding(),
592 Value(), inBoundsAttr);
594 vector::ShapeCastOp::create(b, loc, info.vectorType, flatRead);
595 rewriter.replaceOp(readOp, shapedRead);
596 }
else if (
auto writeOp = dyn_cast<vector::TransferWriteOp>(info.op)) {
597 Value flatValue = vector::ShapeCastOp::create(b, loc, flatVectorType,
598 writeOp.getVector());
599 rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
600 writeOp, flatValue, flatMemref, ValueRange{ptrIterArg},
601 AffineMapAttr::get(identityMap1D), Value(),
607 arith::ConstantIndexOp::create(b, yieldLoc, info.constantStride);
609 arith::AddIOp::create(b, yieldLoc, ptrIterArg, strideConst);
610 yieldValues.push_back(nextPtr);
619 FailureOr<LoopLikeOpInterface> newLoopResult =
620 cast<LoopLikeOpInterface>(forOp.getOperation())
621 .replaceWithAdditionalYields(
622 rewriter, newInitArgs,
626 if (failed(newLoopResult))
637struct AIEHoistVectorTransferPointersPass
638 : xilinx::AIE::impl::AIEHoistVectorTransferPointersBase<
639 AIEHoistVectorTransferPointersPass> {
640 void getDependentDialects(DialectRegistry ®istry)
const override {
641 registry.insert<affine::AffineDialect, arith::ArithDialect,
642 memref::MemRefDialect, scf::SCFDialect,
643 vector::VectorDialect>();
646 void runOnOperation()
override {
647 ModuleOp moduleOp = getOperation();
648 MLIRContext *context = &getContext();
653 moduleOp.walk([&](scf::ForOp forOp) {
654 moveLoopInvariantCode(
655 forOp.getLoopRegions(),
656 [&](Value v, Region *) { return forOp.isDefinedOutsideOfLoop(v); },
657 [&](Operation *op, Region *) {
658 return isa<memref::CollapseShapeOp, memref::ExpandShapeOp,
659 memref::CastOp, memref::ReinterpretCastOp>(op);
661 [&](Operation *op, Region *) { forOp.moveOutOfLoop(op); });
664 RewritePatternSet patterns(context);
665 patterns.add<HoistVectorTransferPointersPattern>(context);
669 if (failed(applyPatternsGreedily(moduleOp, std::move(patterns))))
676std::unique_ptr<OperationPass<ModuleOp>>
678 return std::make_unique<AIEHoistVectorTransferPointersPass>();
std::unique_ptr< mlir::OperationPass< mlir::ModuleOp > > createAIEHoistVectorTransferPointersPass()