18#include "mlir/Analysis/AliasAnalysis/LocalAliasAnalysis.h"
19#include "mlir/Dialect/Affine/IR/AffineOps.h"
20#include "mlir/Dialect/Arith/IR/Arith.h"
21#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
22#include "mlir/Dialect/MemRef/IR/MemRef.h"
23#include "mlir/Dialect/SCF/IR/SCF.h"
24#include "mlir/Dialect/Utils/StaticValueUtils.h"
25#include "mlir/Dialect/Vector/IR/VectorOps.h"
26#include "mlir/IR/IRMapping.h"
27#include "mlir/IR/PatternMatch.h"
28#include "mlir/Interfaces/SideEffectInterfaces.h"
40Type fifoAddrType(MLIRContext *ctx) {
return LLVM::LLVMPointerType::get(ctx); }
42constexpr int64_t kTile = 64;
43constexpr int64_t kExpBytes = 8;
45constexpr int64_t kMaxDiv = 1 << 30;
49static AIE::BufferOp rootBuffer(Value v) {
50 while (Operation *d = v.getDefiningOp()) {
51 if (
auto b = dyn_cast<AIE::BufferOp>(d))
53 if (!isa<memref::CollapseShapeOp, memref::ExpandShapeOp, memref::CastOp>(d))
60static int64_t mulCapped(int64_t a, int64_t b) {
61 return (a >= kMaxDiv / std::max<int64_t>(b, 1)) ? kMaxDiv : a * b;
64static int64_t knownDivisor(Value v,
int depth = 0);
68static int64_t exprDivisor(AffineExpr e, ValueRange operands,
unsigned nDims,
70 if (
auto c = dyn_cast<AffineConstantExpr>(e))
71 return c.getValue() ? std::abs(c.getValue()) : kMaxDiv;
72 if (
auto d = dyn_cast<AffineDimExpr>(e))
73 return knownDivisor(operands[d.getPosition()], depth + 1);
74 if (
auto s = dyn_cast<AffineSymbolExpr>(e))
75 return knownDivisor(operands[nDims + s.getPosition()], depth + 1);
76 auto bin = cast<AffineBinaryOpExpr>(e);
77 int64_t l = exprDivisor(bin.getLHS(), operands, nDims, depth);
78 int64_t r = exprDivisor(bin.getRHS(), operands, nDims, depth);
79 switch (bin.getKind()) {
80 case AffineExprKind::Add:
81 return std::gcd(l, r);
82 case AffineExprKind::Mul:
83 return mulCapped(l, r);
84 case AffineExprKind::Mod:
86 if (
auto c = dyn_cast<AffineConstantExpr>(bin.getRHS()))
87 return std::gcd(l, std::abs(c.getValue()));
96static int64_t knownDivisor(Value v,
int depth) {
99 if (std::optional<int64_t> c = getConstantIntValue(v))
100 return *c ? std::abs(*c) : kMaxDiv;
101 if (
auto arg = dyn_cast<BlockArgument>(v)) {
102 auto loop = dyn_cast<scf::ForOp>(arg.getOwner()->getParentOp());
105 if (arg == loop.getInductionVar())
106 return std::gcd(knownDivisor(loop.getLowerBound(), depth + 1),
107 knownDivisor(loop.getStep(), depth + 1));
110 OpOperand *init = loop.getTiedLoopInit(arg);
111 Value next = loop.getTiedLoopYieldedValue(arg)->get();
112 auto add = next.getDefiningOp<arith::AddIOp>();
113 if (!init || !add || (add.getLhs() != arg && add.getRhs() != arg))
115 Value step = add.getLhs() == arg ? add.getRhs() : add.getLhs();
116 return std::gcd(knownDivisor(init->get(), depth + 1),
117 knownDivisor(step, depth + 1));
119 Operation *d = v.getDefiningOp();
120 if (isa<arith::AddIOp, arith::SubIOp>(d))
121 return std::gcd(knownDivisor(d->getOperand(0), depth + 1),
122 knownDivisor(d->getOperand(1), depth + 1));
123 if (isa<arith::MulIOp>(d))
124 return mulCapped(knownDivisor(d->getOperand(0), depth + 1),
125 knownDivisor(d->getOperand(1), depth + 1));
126 if (isa<arith::IndexCastOp, arith::IndexCastUIOp>(d))
127 return knownDivisor(d->getOperand(0), depth + 1);
128 if (
auto ap = dyn_cast<affine::AffineApplyOp>(d))
129 return exprDivisor(ap.getAffineMap().getResult(0), ap.getMapOperands(),
130 ap.getAffineMap().getNumDims(), depth);
135static std::optional<int64_t> staticTrips(scf::ForOp loop) {
136 std::optional<APInt> n = loop.getStaticTripCount();
137 return n ? std::optional<int64_t>(n->getSExtValue()) : std::nullopt;
142static int64_t tripsOrTwo(scf::ForOp loop) {
143 return staticTrips(loop).value_or(2);
148 vector::TransferReadOp read;
154static std::optional<TileRead> getTileRead(ToBFP16Op conv) {
155 VectorType srcTy = conv.getSource().getType();
156 if (srcTy.getNumElements() != kTile)
158 Value src = conv.getSource();
159 while (
auto sc = src.getDefiningOp<vector::ShapeCastOp>())
160 src = sc.getSource();
161 auto read = src.getDefiningOp<vector::TransferReadOp>();
162 if (!read || read.getMask() || !read.getPermutationMap().isMinorIdentity())
164 if (llvm::any_of(read.getInBoundsValues(), [](
bool b) { return !b; }))
166 auto memTy = dyn_cast<MemRefType>(read.getBase().getType());
167 VectorType vecTy = read.getVectorType();
168 if (!memTy || !memTy.hasStaticShape() || !memTy.getLayout().isIdentity() ||
169 vecTy.getNumElements() != kTile)
173 int64_t vr = vecTy.getRank(), mr = memTy.getRank();
176 for (int64_t i = 1; i < vr; ++i)
177 if (vecTy.getDimSize(i) != memTy.getDimSize(mr - vr + i))
179 AIE::BufferOp buf = rootBuffer(read.getBase());
182 auto bufTy = cast<MemRefType>(buf.getType());
183 if (!bufTy.getLayout().isIdentity() || !bufTy.getElementType().isBF16() ||
184 bufTy.getNumElements() % kTile)
187 ArrayRef<int64_t> shape = memTy.getShape();
188 int64_t stride = 1, div = kMaxDiv;
189 for (int64_t i = mr - 1; i >= 0; --i) {
190 div = std::gcd(div, mulCapped(knownDivisor(read.getIndices()[i]), stride));
195 return TileRead{read, buf};
199static Value linearOffset(OpBuilder &b, Location loc,
200 vector::TransferReadOp read) {
201 auto memTy = cast<MemRefType>(read.getBase().getType());
202 Value off = arith::ConstantIndexOp::create(b, loc, 0);
204 for (int64_t i = memTy.getRank() - 1; i >= 0; --i) {
205 Value s = arith::ConstantIndexOp::create(b, loc, stride);
206 off = arith::AddIOp::create(
208 arith::MulIOp::create(b, loc, read.getIndices()[i], s).getResult());
209 stride *= memTy.getDimSize(i);
219 DenseSet<Value> locks;
222 bool contains(Value lock)
const {
return any || locks.contains(lock); }
225static BufferLocks bufferLocks(AIE::BufferOp buf, Operation *core) {
227 SmallVector<Value> work = {buf.getBuffer()};
228 while (!work.empty()) {
229 Value v = work.pop_back_val();
230 for (Operation *user : v.getUsers()) {
231 if (isa<ViewLikeOpInterface>(user))
232 work.push_back(user->getResult(0));
233 else if (
auto c = user->getParentOfType<AIE::CoreOp>(); c && c != core)
237 auto dev = buf->getParentOfType<AIE::DeviceOp>();
241 auto next = [](Block *b) -> Block * {
242 auto n = dyn_cast<AIE::NextBDOp>(b->getTerminator());
243 return n ? n.getDest() :
nullptr;
245 auto prev = [&](Block *b) -> Block * {
246 for (Block *p : b->getPredecessors())
253 auto scan = [&](AIE::DMABDOp bd,
bool forward,
254 function_ref<bool(AIE::UseLockOp)> want) -> Value {
255 Block *blk = bd->getBlock();
256 Operation *op = forward ? bd->getNextNode() : bd->getPrevNode();
257 for (
unsigned n = 0; blk && n < 64; ++n) {
258 for (; op; op = forward ? op->getNextNode() : op->getPrevNode())
259 if (
auto ul = dyn_cast<AIE::UseLockOp>(op); ul && want(ul))
261 blk = forward ? next(blk) : prev(blk);
263 op = forward ? &blk->front() : &blk->back();
267 dev.walk([&](AIE::DMABDOp bd) {
268 if (rootBuffer(bd.getBuffer()) != buf)
270 for (Value lock : {scan(bd,
false,
271 [](AIE::UseLockOp ul) {
272 return ul.acquire() || ul.acquireGE();
275 [](AIE::UseLockOp ul) {
return ul.release(); })})
277 bl.locks.insert(lock);
285class BufferAliasAnalysis :
public LocalAliasAnalysis {
287 AliasResult aliasImpl(Value lhs, Value rhs)
override {
288 AliasResult r = LocalAliasAnalysis::aliasImpl(lhs, rhs);
291 auto root = [](Value v) {
292 while (
auto view = v.getDefiningOp<ViewLikeOpInterface>())
293 v = view.getViewSource();
296 Operation *a = root(lhs).getDefiningOp(), *b = root(rhs).getDefiningOp();
297 auto isBuffer = [](Operation *op) {
298 return isa_and_nonnull<AIE::BufferOp>(op);
300 auto isMemory = [](Operation *op) {
301 return isa_and_nonnull<AIE::BufferOp, memref::AllocOp, memref::AllocaOp,
302 memref::GetGlobalOp>(op);
304 if (a != b && (isBuffer(a) || isBuffer(b)) && isMemory(a) && isMemory(b))
305 return AliasResult::NoAlias;
312static bool mayWrite(Operation *op, Value mem) {
313 if (op->hasTrait<OpTrait::HasRecursiveMemoryEffects>())
315 return BufferAliasAnalysis().getModRef(op, mem).isMod();
320static bool mayChange(Operation *op, Value mem,
const BufferLocks &locks) {
321 if (
auto ul = dyn_cast<AIE::UseLockOp>(op))
322 return (ul.acquire() || ul.acquireGE()) && locks.contains(ul.getLock());
323 return mayWrite(op, mem);
326static bool loopMayChange(scf::ForOp loop, Value mem,
327 const BufferLocks &locks) {
328 return loop.getBody()
329 ->walk([&](Operation *op) {
330 return mayChange(op, mem, locks) ? WalkResult::interrupt()
331 : WalkResult::advance();
337static int64_t runsPerExecution(Operation *op, scf::ForOp loop) {
339 for (
auto l = op->getParentOfType<scf::ForOp>(); l;
340 l = l->getParentOfType<scf::ForOp>()) {
341 n = mulCapped(n, tripsOrTwo(l));
350static bool fitsInCoreMemory(AIE::BufferOp buf, int64_t extra) {
351 auto dev = buf->getParentOfType<AIE::DeviceOp>();
352 auto tile = buf.getTile().getDefiningOp<AIE::TileOp>();
355 int64_t bytes = extra;
356 dev.walk([&](AIE::BufferOp b) {
357 if (b.getTile() == buf.getTile())
358 bytes += b.getAllocationSize();
360 dev.walk([&](AIE::CoreOp core) {
361 if (core.getTile() == buf.getTile())
362 bytes += core.getEffectiveStackSize();
364 return bytes <= int64_t(dev.getTargetModel().getLocalMemorySize());
369static AffineExpr toExpr(Value v, SmallVectorImpl<Value> &syms,
int depth = 0) {
370 MLIRContext *ctx = v.getContext();
371 if (std::optional<int64_t> c = getConstantIntValue(v))
372 return getAffineConstantExpr(*c, ctx);
374 if (
auto ap = v.getDefiningOp<affine::AffineApplyOp>()) {
375 AffineMap m = ap.getAffineMap();
376 ValueRange ops = ap.getMapOperands();
377 SmallVector<AffineExpr> dims, symbols;
378 for (
unsigned i = 0; i < m.getNumDims(); ++i)
379 dims.push_back(toExpr(ops[i], syms, depth + 1));
380 for (
unsigned i = 0; i < m.getNumSymbols(); ++i)
381 symbols.push_back(toExpr(ops[m.getNumDims() + i], syms, depth + 1));
382 return m.getResult(0).replaceDimsAndSymbols(dims, symbols);
384 if (
auto add = v.getDefiningOp<arith::AddIOp>())
385 return toExpr(add.getLhs(), syms, depth + 1) +
386 toExpr(add.getRhs(), syms, depth + 1);
387 if (
auto mul = v.getDefiningOp<arith::MulIOp>()) {
388 if (std::optional<int64_t> c = getConstantIntValue(mul.getRhs()))
389 return toExpr(mul.getLhs(), syms, depth + 1) * *c;
390 if (std::optional<int64_t> c = getConstantIntValue(mul.getLhs()))
391 return toExpr(mul.getRhs(), syms, depth + 1) * *c;
394 auto it = llvm::find(syms, v);
395 if (it == syms.end()) {
397 it = std::prev(syms.end());
399 return getAffineSymbolExpr(it - syms.begin(), ctx);
403static std::pair<Value, int64_t> splitConstant(Value v) {
405 while (
auto add = v.getDefiningOp<arith::AddIOp>()) {
406 if (std::optional<int64_t> x = getConstantIntValue(add.getRhs())) {
409 }
else if (std::optional<int64_t> x = getConstantIntValue(add.getLhs())) {
422static std::optional<SmallVector<SmallVector<int64_t>>>
423valuesOver(ArrayRef<AffineExpr> exprs, ArrayRef<Value> syms) {
424 SmallVector<SmallVector<int64_t>> ranges;
426 for (Value v : syms) {
427 auto arg = dyn_cast<BlockArgument>(v);
429 arg ? dyn_cast<scf::ForOp>(arg.getOwner()->getParentOp()) : nullptr;
430 if (!loop || loop.getInductionVar() != arg)
432 std::optional<int64_t> lb = getConstantIntValue(loop.getLowerBound());
433 std::optional<int64_t> st = getConstantIntValue(loop.getStep());
434 std::optional<int64_t> n = staticTrips(loop);
435 if (!lb || !st || !n || *n <= 0 || *n > 4096 / total)
438 SmallVector<int64_t> r;
439 for (int64_t i = 0; i < *n; ++i)
440 r.push_back(*lb + i * *st);
441 ranges.push_back(std::move(r));
443 SmallVector<SmallVector<int64_t>> out;
444 SmallVector<size_t> at(syms.size(), 0);
445 for (int64_t n = 0; n < total; ++n) {
446 SmallVector<AffineExpr> vals;
447 for (
size_t i = 0; i < syms.size(); ++i)
449 getAffineConstantExpr(ranges[i][at[i]], syms[i].getContext()));
450 SmallVector<int64_t> row;
451 for (AffineExpr e : exprs) {
452 auto c = dyn_cast<AffineConstantExpr>(
453 simplifyAffineExpr(e.replaceSymbols(vals), 0, 0));
456 row.push_back(c.getValue());
458 out.push_back(std::move(row));
459 for (
size_t i = 0; i < syms.size(); ++i) {
460 if (++at[i] < ranges[i].size())
473 int64_t s = 0, K = 1, d = 1, r = 1, T = 0;
475 bool operator==(
const Layout &o)
const {
476 return s == o.s && K == o.K && d == o.d && r == o.r && T == o.T;
478 bool identity()
const {
return s == 0; }
482 std::pair<std::pair<int64_t, int64_t>, std::pair<int64_t, int64_t>>
484 std::pair<int64_t, int64_t> k = {s, K}, i = {d, r};
485 return s < d ? std::pair(k, i) : std::pair(i, k);
488 static std::optional<Layout> make(int64_t s, int64_t K, int64_t d, int64_t r,
490 Layout l{s, K, d, r, T};
491 if (s <= 0 || K <= 0 || d <= 0 || r <= 0 || T <= 0)
493 auto [lo, hi] = l.digits();
494 if (hi.first % (lo.first * lo.second) || T % (hi.first * hi.second))
500 Value block(OpBuilder &b, Location loc, Value t)
const {
503 auto c = [&](int64_t v) -> Value {
504 return arith::ConstantIndexOp::create(b, loc, v);
506 auto div = [&](Value x, int64_t v) -> Value {
507 return v == 1 ? x : arith::DivUIOp::create(b, loc, x, c(v)).getResult();
509 auto rem = [&](Value x, int64_t v) -> Value {
510 return arith::RemUIOp::create(b, loc, x, c(v));
512 auto mul = [&](Value x, int64_t v) -> Value {
513 return v == 1 ? x : arith::MulIOp::create(b, loc, x, c(v)).getResult();
515 auto add = [&](Value x, Value y) -> Value {
516 return arith::AddIOp::create(b, loc, x, y);
518 auto [lo, hi] = digits();
519 int64_t m = hi.first / (lo.first * lo.second);
520 Value k = rem(div(t, s), K), i = rem(div(t, d), r);
522 add(rem(t, lo.first), mul(add(rem(div(t, lo.first * lo.second), m),
523 mul(div(t, hi.first * hi.second), m)),
525 return add(mul(rest, r * K), add(mul(k, r), i));
538static std::optional<SteppedRead> steppedRead(vector::TransferReadOp rd) {
539 auto loop = dyn_cast<scf::ForOp>(rd->getParentOp());
540 if (!loop || rd.getIndices().size() != 1)
542 auto arg = dyn_cast<BlockArgument>(rd.getIndices()[0]);
543 if (!arg || arg.getOwner() != loop.getBody() || arg.getArgNumber() == 0)
545 auto [base, c] = splitConstant(loop.getTiedLoopYieldedValue(arg)->get());
546 std::optional<int64_t> trips = staticTrips(loop);
547 if (base != arg || c <= 0 || c % kTile || !trips || *trips <= 0)
549 return SteppedRead{loop, loop.getTiedLoopInit(arg)->get(), c / kTile, *trips};
556 SmallVector<std::pair<vector::TransferReadOp, int64_t>> reads;
563static std::pair<Layout, SmallVector<Stream>>
564chooseLayout(ArrayRef<vector::TransferReadOp> reads, int64_t T) {
565 llvm::MapVector<Operation *, SmallVector<vector::TransferReadOp>> byLoop;
566 for (vector::TransferReadOp rd : reads)
567 if (std::optional<SteppedRead> sr = steppedRead(rd))
568 byLoop[sr->loop].push_back(rd);
569 std::optional<Layout> chosen;
570 SmallVector<Stream> streams;
571 for (
auto &[loopOp, rds] : byLoop) {
572 SteppedRead first = *steppedRead(rds.front());
573 SmallVector<Value> syms;
574 SmallVector<AffineExpr> exprs;
576 for (vector::TransferReadOp rd : rds) {
577 SteppedRead sr = *steppedRead(rd);
578 ok &= sr.step == first.step;
579 exprs.push_back(toExpr(sr.init, syms));
583 std::optional<SmallVector<SmallVector<int64_t>>> vals =
584 valuesOver(exprs, syms);
588 SmallVector<int64_t> off;
589 for (
size_t j = 0; j < rds.size(); ++j) {
590 int64_t o = (*vals)[0][j] - (*vals)[0][0];
591 for (
auto &row : *vals)
592 ok &= row[j] - row[0] == o && row[j] % kTile == 0;
593 off.push_back(o / kTile);
595 int64_t lo = *llvm::min_element(off);
596 SmallVector<int64_t> distinct;
597 for (int64_t &o : off) {
599 if (!llvm::is_contained(distinct, o))
600 distinct.push_back(o);
602 llvm::sort(distinct);
603 int64_t r = distinct.size(), d = r > 1 ? distinct[1] : 1;
604 for (int64_t j = 0; j < r; ++j)
605 ok &= distinct[j] == j * d;
608 std::optional<Layout> layout =
609 Layout::make(first.step, first.trips, d, r, T);
610 if (!layout || (chosen && !(*chosen == *layout)))
614 for (
size_t j = 0; j < rds.size(); ++j)
617 for (
auto &row : *vals) {
618 int64_t t = row[lowest] / kTile;
619 ok &= t >= 0 && t < T && (t / layout->s) % layout->K == 0 &&
620 (t / layout->d) % layout->r == 0;
625 Stream st{cast<scf::ForOp>(loopOp), {}, steppedRead(rds[lowest])->init};
626 for (
size_t j = 0; j < rds.size(); ++j)
627 st.reads.push_back({rds[j], off[j] / d});
628 streams.push_back(std::move(st));
631 return {Layout{0, 1, 1, 1, T}, {}};
632 return {*chosen, std::move(streams)};
636static int64_t loadStreams(scf::ForOp loop) {
638 for (Operation &op : *loop.getBody())
639 if (
auto pop = dyn_cast<FifoLdPopBFP16Op>(op))
640 if (
auto arg = dyn_cast<BlockArgument>(pop.getAddr()))
641 n += arg.getOwner() == loop.getBody();
646 using OpRewritePattern::OpRewritePattern;
648 LogicalResult matchAndRewrite(ToBFP16Op conv,
649 PatternRewriter &rewriter)
const override {
650 std::optional<TileRead> tr = getTileRead(conv);
652 return rewriter.notifyMatchFailure(conv,
"not a tile read of a buffer");
653 AIE::BufferOp buf = tr->buf;
654 auto core = conv->getParentOfType<AIE::CoreOp>();
656 return rewriter.notifyMatchFailure(conv,
"not in a core");
660 BufferLocks locks = bufferLocks(buf, core);
662 for (
auto l = conv->getParentOfType<scf::ForOp>();
663 l && core->isProperAncestor(l); l = l->getParentOfType<scf::ForOp>()) {
664 if (loopMayChange(l, buf.getBuffer(), locks))
669 return rewriter.notifyMatchFailure(conv,
"no loop keeps the buffer");
674 auto partnered = [](ToBFP16Op c) {
675 return !c->use_empty() && llvm::all_of(c->getUses(), [&](OpOperand &u) {
676 auto mm = dyn_cast<MatMulBFP16Op_AIE2P>(u.getOwner());
679 Value other = u.getOperandNumber() < 2 ? mm.getRhsMantissas()
680 : mm.getLhsMantissas();
681 return isa_and_nonnull<LoadBFP16Op, FifoLdPopBFP16Op>(
682 other.getDefiningOp());
685 if (!partnered(conv))
686 return rewriter.notifyMatchFailure(conv,
"the other operand converted");
687 SmallVector<std::pair<ToBFP16Op, vector::TransferReadOp>> group;
689 loop.getBody()->walk([&](ToBFP16Op c) {
690 if (c.getTranspose() != conv.getTranspose() || !partnered(c))
692 std::optional<TileRead> t = getTileRead(c);
693 if (!t || t->buf != buf || !loop->isAncestor(t->read))
695 group.push_back({c, t->read});
696 runs = std::min(kMaxDiv, runs + runsPerExecution(c, loop));
701 auto bufTy = cast<MemRefType>(buf.getType());
702 int64_t tiles = bufTy.getNumElements() / kTile;
703 if (runs < 2 * tiles)
704 return rewriter.notifyMatchFailure(conv,
"too few conversions per tile");
705 if (!fitsInCoreMemory(buf, tiles * (kTile + kExpBytes)))
706 return rewriter.notifyMatchFailure(conv,
"no room for the bfp16 copy");
710 Location loc = conv.getLoc();
711 Attribute space = bufTy.getMemorySpace();
712 Type i8 = rewriter.getI8Type(), i32 = rewriter.getI32Type();
713 Type addrTy = fifoAddrType(rewriter.getContext());
714 auto stateTy = VectorType::get({32}, i32);
715 auto mantTy = VectorType::get({kTile}, i8);
716 auto expTy = VectorType::get({kExpBytes}, i8);
719 OpBuilder::InsertionGuard g(rewriter);
720 rewriter.setInsertionPointAfter(buf);
722 (buf.getSymName() ? buf.getSymName()->str() : std::string(
"buf")) +
724 DenseSet<StringRef> taken;
725 if (
auto dev = buf->getParentOfType<AIE::DeviceOp>())
726 dev.walk([&](Operation *op) {
727 if (
auto sym = op->getAttrOfType<StringAttr>(
728 SymbolTable::getSymbolAttrName()))
729 taken.insert(sym.getValue());
731 std::string name = base;
732 for (
int i = 0; taken.contains(name); ++i)
733 name = base +
"_" + std::to_string(i);
734 copy = AIE::BufferOp::create(
735 rewriter, buf.getLoc(),
736 MemRefType::get({tiles * (kTile + kExpBytes)}, i8,
nullptr, space),
737 buf.getTile(), rewriter.getStringAttr(name));
739 SmallVector<vector::TransferReadOp> reads;
740 for (
auto &[c, rd] : group)
741 if (!llvm::is_contained(reads, rd))
743 auto [layout, streams] = chooseLayout(reads, tiles);
744 auto cIdx = [&](OpBuilder &b, Location l, int64_t v) -> Value {
745 return arith::ConstantIndexOp::create(b, l, v);
747 auto blockOffset = [&](OpBuilder &b, Location l, Value t) -> Value {
748 return arith::MulIOp::create(b, l, layout.block(b, l, t),
749 cIdx(b, l, kTile + kExpBytes));
755 rewriter.setInsertionPoint(loop);
756 SmallVector<ReassociationIndices> all(1);
757 for (int64_t i = 0; i < bufTy.getRank(); ++i)
759 Value flat = bufTy.getRank() == 1 ? Value(buf.getBuffer())
760 : memref::CollapseShapeOp::create(
761 rewriter, loc, buf.getBuffer(), all)
763 SmallVector<std::pair<int64_t, int64_t>> nest;
764 if (layout.identity()) {
765 nest.push_back({tiles, 1});
767 auto [lo, hi] = layout.digits();
768 int64_t m = hi.first / (lo.first * lo.second);
769 nest.push_back({tiles / (hi.first * hi.second), hi.first * hi.second});
770 nest.push_back({m, lo.first * lo.second});
771 nest.push_back({lo.first, 1});
772 nest.push_back({layout.K, layout.s});
773 nest.push_back({layout.r, layout.d});
777 while (nest.size() > 1 && nest.back().first == 1)
779 int64_t inner = nest.back().first;
780 int64_t U = inner % 4 == 0 ? 4 : inner % 2 == 0 ? 2 : 1;
781 Value bfZero = arith::ConstantOp::create(
782 rewriter, loc, rewriter.getZeroAttr(rewriter.getBF16Type()));
783 auto tileTy = VectorType::get({kTile}, rewriter.getBF16Type());
784 VectorType convTy = conv.getSource().getType();
785 UnitAttr transpose = conv.getTransposeAttr();
786 std::function<SmallVector<Value>(OpBuilder &, Location,
unsigned, Value,
788 build = [&](OpBuilder &b, Location l,
unsigned level, Value t,
789 ValueRange st) -> SmallVector<Value> {
790 if (level == nest.size()) {
791 int64_t stride = nest.back().second;
792 SmallVector<Value> tileVals;
793 for (int64_t u = 0; u < U; ++u) {
794 Value tu = arith::AddIOp::create(b, l, t, cIdx(b, l, u * stride));
795 Value at = arith::MulIOp::create(b, l, tu, cIdx(b, l, kTile));
797 vector::TransferReadOp::create(b, l, tileTy, flat, ValueRange{at},
798 bfZero, ArrayRef<bool>{true}));
800 SmallVector<Value> cur(st.begin(), st.end());
801 for (Value v : tileVals) {
802 auto c = ToBFP16Op::create(
803 b, l, mantTy, expTy, vector::ShapeCastOp::create(b, l, convTy, v),
805 auto push = FifoStPushBFP16Op::create(
806 b, l, addrTy, stateTy, i32, cur[0], cur[1], cur[2],
807 c.getMantissas(), c.getExponents());
808 cur = SmallVector<Value>(push->getResults());
812 auto [extent, stride] = nest[level];
813 bool last = level + 1 == nest.size();
814 int64_t trips = last ? extent / U : extent;
815 int64_t step = last ? stride * U : stride;
817 return build(b, l, level + 1, t, st);
818 auto nloop = scf::ForOp::create(
819 b, l, cIdx(b, l, 0), cIdx(b, l, trips), cIdx(b, l, 1), st,
820 [&](OpBuilder &ib, Location il, Value iv, ValueRange ist) {
821 Value ti = arith::AddIOp::create(
823 arith::MulIOp::create(ib, il, iv, cIdx(ib, il, step)));
824 scf::YieldOp::create(ib, il, build(ib, il, level + 1, ti, ist));
826 return SmallVector<Value>(nloop.getResults());
828 auto start = FifoStartOp::create(rewriter, loc, addrTy, stateTy, i32,
829 copy.getBuffer(), cIdx(rewriter, loc, 0));
830 (void)build(rewriter, loc, 0, cIdx(rewriter, loc, 0),
831 ValueRange(start->getResults()));
834 auto setBFP16 = [&](vector::TransferReadOp rd, Value m, Value e) {
835 for (
auto &[c, r] : group)
837 c.getMantissas().replaceAllUsesWith(m);
838 c.getExponents().replaceAllUsesWith(e);
844 SmallPtrSet<Operation *, 16> streamed;
845 for (Stream &st : streams) {
846 if (loadStreams(st.loop) >= 2)
848 rewriter.setInsertionPoint(st.loop);
849 Location l = st.loop.getLoc();
850 Value t0 = arith::DivUIOp::create(rewriter, l, st.init,
851 cIdx(rewriter, l, kTile));
853 FifoStartOp::create(rewriter, l, addrTy, stateTy, i32,
854 copy.getBuffer(), blockOffset(rewriter, l, t0));
855 int64_t r = layout.r;
856 (void)st.loop.replaceWithAdditionalYields(
857 rewriter, begin->getResults(),
false,
858 [&](OpBuilder &b, Location, ArrayRef<BlockArgument>
args) {
859 Operation *first = st.reads.front().first;
860 for (auto &[rd, _] : st.reads)
861 if (rd->isBeforeInBlock(first))
863 OpBuilder::InsertionGuard g(b);
864 b.setInsertionPoint(first);
865 Value addr = args[0], state = args[1], count = args[2];
866 for (int64_t i = 0; i < r; ++i) {
867 auto pop = FifoLdPopBFP16Op::create(
868 b, first->getLoc(), addrTy, stateTy, i32, mantTy, expTy, addr,
870 r % 8 == 0 && i % 8 ? b.getUnitAttr() : UnitAttr());
871 for (auto &[rd, place] : st.reads)
873 setBFP16(rd, pop.getMantissas(), pop.getExponents());
874 addr = pop.getNext();
875 state = pop.getStateOut();
876 count = pop.getCountOut();
878 return SmallVector<Value>{addr, state, count};
880 for (
auto &[rd, _] : st.reads)
884 for (vector::TransferReadOp rd : reads) {
885 if (streamed.contains(rd))
887 rewriter.setInsertionPoint(rd);
888 Location l = rd.getLoc();
889 Value t = arith::DivUIOp::create(
890 rewriter, l, linearOffset(rewriter, l, rd), cIdx(rewriter, l, kTile));
892 LoadBFP16Op::create(rewriter, l, mantTy, expTy, copy.getBuffer(),
893 blockOffset(rewriter, l, t));
894 setBFP16(rd, ld.getMantissas(), ld.getExponents());
896 for (
auto &[c, rd] : group)
904static scf::ForOp loopOver(Value v, int64_t n) {
905 auto arg = dyn_cast<BlockArgument>(v);
907 arg ? dyn_cast<scf::ForOp>(arg.getOwner()->getParentOp()) : nullptr;
908 if (!loop || loop.getInductionVar() != arg ||
909 getConstantIntValue(loop.getLowerBound()) != 0 ||
910 getConstantIntValue(loop.getStep()) != 1 ||
911 getConstantIntValue(loop.getUpperBound()) != n)
924 AIE::BufferOp bytes, tiles;
926 SmallVector<unsigned> perm;
927 SmallVector<int64_t> blocks;
930static std::optional<Decode> matchDecode(memref::StoreOp st) {
931 auto trunc = st.getValueToStore().getDefiningOp<arith::TruncFOp>();
932 auto mul = trunc ? trunc.getIn().getDefiningOp<arith::MulFOp>() : nullptr;
933 if (!mul || !trunc.getType().isBF16())
936 arith::BitcastOp scale;
937 for (
auto [x, y] : {std::pair(mul.getLhs(), mul.getRhs()),
938 std::pair(mul.getRhs(), mul.getLhs())})
939 if (x.getDefiningOp<arith::SIToFPOp>() &&
940 y.getDefiningOp<arith::BitcastOp>()) {
941 man = x.getDefiningOp<arith::SIToFPOp>();
942 scale = y.getDefiningOp<arith::BitcastOp>();
946 auto shl = scale.getIn().getDefiningOp<arith::ShLIOp>();
947 auto sub = shl ? shl.getLhs().getDefiningOp<arith::SubIOp>() : nullptr;
948 auto ext = sub ? sub.getLhs().getDefiningOp<arith::ExtUIOp>() : nullptr;
949 auto expLd = ext ? ext.getIn().getDefiningOp<memref::LoadOp>() : nullptr;
950 auto manLd = man.getIn().getDefiningOp<memref::LoadOp>();
951 if (!expLd || !manLd || getConstantIntValue(shl.getRhs()) != 23 ||
952 getConstantIntValue(sub.getRhs()) != 6 || !scale.getType().isF32() ||
953 !man.getType().isF32() || expLd.getMemref() != manLd.getMemref())
955 auto bBuf = expLd.getMemref().getDefiningOp<AIE::BufferOp>();
956 auto xBuf = st.getMemref().getDefiningOp<AIE::BufferOp>();
959 auto bt = cast<MemRefType>(bBuf.getType());
960 auto xt = cast<MemRefType>(xBuf.getType());
961 int64_t n = xt.getRank(), nb = bt.getRank();
962 if (!bt.hasStaticShape() || !xt.hasStaticShape() ||
963 !bt.getLayout().isIdentity() || !xt.getLayout().isIdentity() || nb < n ||
964 n < 3 || !bt.getElementType().isInteger(8) ||
965 !xt.getElementType().isBF16() || bt.getDimSize(nb - 2) != 8 ||
966 bt.getDimSize(nb - 1) != 9 || xt.getDimSize(n - 2) != 8 ||
967 xt.getDimSize(n - 1) != 8)
971 SmallVector<scf::ForOp> loops;
972 for (int64_t i = 0; i < n; ++i) {
973 scf::ForOp l = loopOver(st.getIndices()[i], xt.getDimSize(i));
974 if (!l || llvm::is_contained(loops, l))
978 scf::ForOp nest = loops.front();
979 for (scf::ForOp l : loops)
980 if (l->isProperAncestor(nest))
983 if (!nest->use_empty())
987 Decode d{bBuf, xBuf, nest, {}, {}};
988 ValueRange tileIdx = st.getIndices().take_front(n - 2);
989 for (int64_t j = 0; j < nb - 2; ++j) {
990 Value v = manLd.getIndices()[j];
991 if (expLd.getIndices()[j] != v)
993 if (bt.getDimSize(j) == 1 && getConstantIntValue(v) == 0)
995 auto it = llvm::find(tileIdx, v);
996 if (it == tileIdx.end() ||
997 llvm::is_contained(d.perm,
unsigned(it - tileIdx.begin())))
999 d.perm.push_back(it - tileIdx.begin());
1000 d.blocks.push_back(bt.getDimSize(j));
1002 if (d.perm.size() !=
size_t(n - 2) ||
1003 manLd.getIndices()[nb - 2] != st.getIndices()[n - 2] ||
1004 expLd.getIndices()[nb - 2] != st.getIndices()[n - 2] ||
1005 getConstantIntValue(expLd.getIndices()[nb - 1]) != 0)
1006 return std::nullopt;
1007 SmallVector<Value> syms;
1008 AffineExpr off = toExpr(manLd.getIndices()[nb - 1], syms) -
1009 toExpr(st.getIndices()[n - 1], syms);
1011 dyn_cast<AffineConstantExpr>(simplifyAffineExpr(off, 0, syms.size()));
1012 if (!c || c.getValue() != 1)
1013 return std::nullopt;
1015 WalkResult clean = nest->walk([&](Operation *op) {
1016 if (op == st || op == manLd || op == expLd || isPure(op) ||
1017 isa<scf::ForOp, scf::YieldOp>(op))
1018 return WalkResult::advance();
1019 return WalkResult::interrupt();
1021 if (clean.wasInterrupted())
1022 return std::nullopt;
1027static Value blockOf(OpBuilder &b, Location loc, Decode &d, Value t) {
1028 auto xt = cast<MemRefType>(d.tiles.getType());
1029 auto c = [&](int64_t v) -> Value {
1030 return arith::ConstantIndexOp::create(b, loc, v);
1032 int64_t n = xt.getRank() - 2;
1033 SmallVector<Value> dig(n);
1034 for (int64_t i = n - 1; i >= 0; --i) {
1036 i == 0 ? t : arith::RemUIOp::create(b, loc, t, c(xt.getDimSize(i)));
1038 t = arith::DivUIOp::create(b, loc, t, c(xt.getDimSize(i)));
1041 for (
size_t j = 0; j < d.blocks.size(); ++j)
1042 blk = blk ? arith::AddIOp::create(
1043 b, loc, arith::MulIOp::create(b, loc, blk, c(d.blocks[j])),
1055 using OpRewritePattern::OpRewritePattern;
1057 LogicalResult matchAndRewrite(memref::StoreOp st,
1058 PatternRewriter &rewriter)
const override {
1059 std::optional<Decode> d = matchDecode(st);
1061 return rewriter.notifyMatchFailure(st,
"not a bfp16 decode");
1062 auto core = st->getParentOfType<AIE::CoreOp>();
1063 BufferLocks bytesLocks = bufferLocks(d->bytes, core);
1064 BufferLocks tileLocks = bufferLocks(d->tiles, core);
1065 if (bytesLocks.any || bytesLocks.locks.empty() || tileLocks.any ||
1066 !tileLocks.locks.empty() || d->tiles.getInitialValue())
1067 return rewriter.notifyMatchFailure(st,
"not DMA bytes into core tiles");
1071 SmallVector<Operation *> views;
1072 SmallVector<std::pair<ToBFP16Op, vector::TransferReadOp>> convs;
1073 SmallVector<Value> work = {d->tiles.getBuffer()};
1074 Block *blk = d->nest->getBlock();
1075 Operation *last = d->nest;
1076 while (!work.empty()) {
1077 Value v = work.pop_back_val();
1078 for (Operation *user : v.getUsers()) {
1079 if (user == st.getOperation())
1081 if (isa<memref::CollapseShapeOp, memref::ExpandShapeOp, memref::CastOp>(
1083 views.push_back(user);
1084 work.push_back(user->getResult(0));
1087 auto rd = dyn_cast<vector::TransferReadOp>(user);
1088 Operation *anc = rd ? blk->findAncestorOpInBlock(*rd) :
nullptr;
1089 if (!anc || !d->nest->isBeforeInBlock(anc))
1090 return rewriter.notifyMatchFailure(st,
"another use of the tiles");
1091 for (Operation *u : rd->getUsers()) {
1092 while (isa<vector::ShapeCastOp>(u) && u->hasOneUse())
1093 u = *u->getUsers().begin();
1094 auto conv = dyn_cast<ToBFP16Op>(u);
1095 std::optional<TileRead> tr = conv ? getTileRead(conv) : std::nullopt;
1096 if (!conv || conv.getTranspose() || !tr || tr->read != rd)
1097 return rewriter.notifyMatchFailure(st,
"a tile read not converted");
1098 convs.push_back({conv, rd});
1100 if (last->isBeforeInBlock(anc))
1105 return rewriter.notifyMatchFailure(st,
"no conversions");
1109 AIE::UseLockOp release;
1110 for (Operation *op = d->nest->getNextNode(); op; op = op->getNextNode()) {
1111 bool usesLock = op->walk([&](AIE::UseLockOp ul) {
1112 return bytesLocks.contains(ul.getLock())
1113 ? WalkResult::interrupt()
1114 : WalkResult::advance();
1115 }).wasInterrupted();
1116 auto ul = dyn_cast<AIE::UseLockOp>(op);
1117 if (ul && ul.release() && bytesLocks.contains(ul.getLock()) && !release)
1120 return rewriter.notifyMatchFailure(st,
"the bytes' lock in between");
1121 bool writes = op->walk([&](Operation *o) {
1122 return mayChange(o, d->bytes.getBuffer(), {})
1123 ? WalkResult::interrupt()
1124 : WalkResult::advance();
1125 }).wasInterrupted();
1127 return rewriter.notifyMatchFailure(st,
"the bytes may change");
1133 rewriter.setInsertionPoint(d->nest);
1134 Location loc = st.getLoc();
1135 auto bt = cast<MemRefType>(d->bytes.getType());
1136 SmallVector<ReassociationIndices> all(1);
1137 for (int64_t i = 0; i < bt.getRank(); ++i)
1138 all[0].push_back(i);
1139 Value flat = memref::CollapseShapeOp::create(rewriter, loc,
1140 d->bytes.getBuffer(), all);
1143 DenseMap<Operation *, LoadBFP16Op> loads;
1144 for (
auto [conv, rd] : convs) {
1145 LoadBFP16Op &ld = loads[rd];
1147 rewriter.setInsertionPointAfter(rd);
1148 Location l = rd.getLoc();
1149 Value t = arith::DivUIOp::create(
1150 rewriter, l, linearOffset(rewriter, l, rd),
1151 arith::ConstantIndexOp::create(rewriter, l, kTile));
1152 Value off = arith::MulIOp::create(
1153 rewriter, l, blockOf(rewriter, l, *d, t),
1154 arith::ConstantIndexOp::create(rewriter, l, kTile + kExpBytes));
1155 ld = LoadBFP16Op::create(rewriter, l, conv.getMantissas().getType(),
1156 conv.getExponents().getType(), flat, off);
1158 rewriter.replaceOp(conv, ld.getResults());
1160 if (release && release->isBeforeInBlock(last))
1161 rewriter.moveOpAfter(release, last);
1162 rewriter.eraseOp(d->nest);
1165 for (
auto &[rd, ld] : loads) {
1166 for (Operation *u : llvm::make_early_inc_range(rd->getUsers()))
1168 rewriter.eraseOp(u);
1169 if (rd->use_empty())
1170 rewriter.eraseOp(rd);
1172 for (Operation *v : llvm::reverse(views))
1174 rewriter.eraseOp(v);
1175 if (d->tiles->use_empty())
1176 rewriter.eraseOp(d->tiles);
1184struct TripEvaluator {
1186 DenseMap<Value, int64_t> env;
1188 std::optional<int64_t> eval(Value v,
int depth = 0) {
1189 if (
auto it = env.find(v); it != env.end())
1191 if (std::optional<int64_t> c = getConstantIntValue(v))
1193 Operation *d = v.getDefiningOp();
1194 if (!d || depth > 32 || !scope->isAncestor(d))
1195 return std::nullopt;
1196 auto bin = [&](
auto f) -> std::optional<int64_t> {
1197 std::optional<int64_t> a = eval(d->getOperand(0), depth + 1);
1198 std::optional<int64_t> b = eval(d->getOperand(1), depth + 1);
1200 return std::nullopt;
1203 if (isa<arith::AddIOp>(d))
1205 [](int64_t a, int64_t b) -> std::optional<int64_t> {
return a + b; });
1206 if (isa<arith::SubIOp>(d))
1208 [](int64_t a, int64_t b) -> std::optional<int64_t> {
return a - b; });
1209 if (isa<arith::MulIOp>(d))
1211 [](int64_t a, int64_t b) -> std::optional<int64_t> {
return a * b; });
1212 if (isa<arith::DivUIOp, arith::DivSIOp>(d))
1213 return bin([](int64_t a, int64_t b) -> std::optional<int64_t> {
1214 return b > 0 && a >= 0 ? std::optional<int64_t>(a / b) : std::nullopt;
1216 if (isa<arith::RemUIOp, arith::RemSIOp>(d))
1217 return bin([](int64_t a, int64_t b) -> std::optional<int64_t> {
1218 return b > 0 && a >= 0 ? std::optional<int64_t>(a % b) : std::nullopt;
1220 if (isa<arith::IndexCastOp, arith::IndexCastUIOp>(d))
1221 return eval(d->getOperand(0), depth + 1);
1222 if (
auto ap = dyn_cast<affine::AffineApplyOp>(d)) {
1223 SmallVector<int64_t> vals;
1224 for (Value o : ap.getMapOperands()) {
1225 std::optional<int64_t> x = eval(o, depth + 1);
1227 return std::nullopt;
1230 AffineMap m = ap.getAffineMap();
1231 SmallVector<AffineExpr> dims, syms;
1232 MLIRContext *ctx = ap.getContext();
1233 for (
unsigned i = 0; i < m.getNumDims(); ++i)
1234 dims.push_back(getAffineConstantExpr(vals[i], ctx));
1235 for (
unsigned i = 0; i < m.getNumSymbols(); ++i)
1236 syms.push_back(getAffineConstantExpr(vals[m.getNumDims() + i], ctx));
1237 auto c = dyn_cast<AffineConstantExpr>(
1238 m.getResult(0).replaceDimsAndSymbols(dims, syms));
1239 return c ? std::optional<int64_t>(c.getValue()) : std::nullopt;
1241 return std::nullopt;
1247static SmallVector<Value> valuesFromOutside(scf::ForOp loop, ValueRange vals) {
1248 auto yield = cast<scf::YieldOp>(loop.getBody()->getTerminator());
1249 SmallVector<Value> outside, work(vals.begin(), vals.end());
1250 DenseSet<Value> seen;
1251 while (!work.empty()) {
1252 Value v = work.pop_back_val();
1253 if (!seen.insert(v).second)
1255 auto arg = dyn_cast<BlockArgument>(v);
1256 if (arg && arg.getOwner() == loop.getBody()) {
1257 if (OpOperand *init = loop.getTiedLoopInit(arg)) {
1258 outside.push_back(init->get());
1260 yield.getOperand(arg.getArgNumber() - loop.getNumInductionVars()));
1262 }
else if (Operation *d = v.getDefiningOp(); d && loop->isAncestor(d)) {
1263 work.append(d->getOperands().begin(), d->getOperands().end());
1265 outside.push_back(v);
1273static std::optional<SmallVector<Value>> outerIVs(scf::ForOp loop,
1274 ArrayRef<Value> vals) {
1275 SmallVector<Value> ivs, work(vals.begin(), vals.end());
1276 DenseSet<Value> seen;
1277 while (!work.empty()) {
1278 Value v = work.pop_back_val();
1279 if (!seen.insert(v).second)
1281 if (
auto arg = dyn_cast<BlockArgument>(v)) {
1282 auto l = dyn_cast<scf::ForOp>(arg.getOwner()->getParentOp());
1283 if (!l || l.getInductionVar() != arg || !l->isProperAncestor(loop))
1284 return std::nullopt;
1286 }
else if (Operation *d = v.getDefiningOp()) {
1287 work.append(d->getOperands().begin(), d->getOperands().end());
1299 using OpRewritePattern::OpRewritePattern;
1301 LogicalResult matchAndRewrite(scf::ForOp loop,
1302 PatternRewriter &rewriter)
const override {
1303 std::optional<int64_t> lb = getConstantIntValue(loop.getLowerBound());
1304 std::optional<int64_t> st = getConstantIntValue(loop.getStep());
1305 std::optional<int64_t> n = staticTrips(loop);
1306 if (!lb || !st || !n || *n <= 0)
1307 return rewriter.notifyMatchFailure(loop,
"trips not known");
1311 llvm::MapVector<Value, SmallVector<LoadBFP16Op>> byBase;
1312 for (Operation &op : *loop.getBody())
1313 if (
auto ld = dyn_cast<LoadBFP16Op>(op))
1314 byBase[ld.getBase()].push_back(ld);
1315 bool nested =
false;
1316 loop.getBody()->walk([&](Operation *op) {
1317 if (op->getParentOp() != loop &&
1318 isa<LoadBFP16Op, FifoStartOp, FifoLdPopBFP16Op>(op))
1321 if (byBase.empty() || nested)
1322 return rewriter.notifyMatchFailure(loop,
"no loads to stream");
1326 Operation *scope = loop->getParentOfType<AIE::CoreOp>();
1328 scope = loop->getParentOp();
1329 SmallVector<Value> offsets;
1330 for (
auto &[base, loads] : byBase)
1331 for (LoadBFP16Op ld : loads)
1332 offsets.push_back(ld.getOffset());
1333 std::optional<SmallVector<Value>> ivs =
1334 outerIVs(loop, valuesFromOutside(loop, offsets));
1335 std::optional<SmallVector<SmallVector<int64_t>>> combos;
1337 SmallVector<AffineExpr> syms;
1338 for (
unsigned i = 0; i < ivs->size(); ++i)
1339 syms.push_back(getAffineSymbolExpr(i, loop.getContext()));
1340 combos = valuesOver(syms, *ivs);
1343 constexpr int64_t kMaxEvals = 1 << 16;
1344 if (combos && (int64_t(offsets.size()) * trips > kMaxEvals ||
1345 int64_t(combos->size()) >
1346 kMaxEvals / (int64_t(offsets.size()) * trips)))
1348 SmallVector<std::pair<Value, SmallVector<LoadBFP16Op>>> streams;
1349 for (
auto &[base, loads] : byBase) {
1352 AIE::BufferOp root = rootBuffer(base);
1356 locks = bufferLocks(root, loop->getParentOfType<AIE::CoreOp>());
1359 ->walk([&](Operation *op) {
1360 auto ul = dyn_cast<AIE::UseLockOp>(op);
1361 bool c = ul ? (ul.acquire() || ul.acquireGE()) &&
1362 locks.contains(ul.getLock())
1363 : mayWrite(op, base);
1364 return c ? WalkResult::interrupt() : WalkResult::advance();
1367 if (!loop.isDefinedOutsideOfLoop(base) || !combos || changes)
1369 int64_t r = loads.size();
1371 for (ArrayRef<int64_t> combo : *combos) {
1372 DenseMap<Value, int64_t> outer;
1373 for (
auto [iv, x] : llvm::zip(*ivs, combo))
1375 TripEvaluator ev{scope, outer};
1377 SmallVector<std::optional<int64_t>> carried;
1378 for (Value init : loop.getInitArgs())
1379 carried.push_back(ev.eval(init));
1380 std::optional<int64_t> start;
1381 for (int64_t k = 0; k < trips && ok; ++k) {
1383 ev.env[loop.getInductionVar()] = *lb + k * *st;
1384 for (
auto [arg, val] : llvm::zip(loop.getRegionIterArgs(), carried))
1387 for (int64_t j = 0; j < r && ok; ++j) {
1388 std::optional<int64_t> off = ev.eval(loads[j].getOffset());
1389 if (!off || *off % (kTile + kExpBytes)) {
1395 ok = *off == *start + (kTile + kExpBytes) * (r * k + j);
1397 auto yield = cast<scf::YieldOp>(loop.getBody()->getTerminator());
1398 for (
auto [i, y] : llvm::enumerate(yield.getOperands()))
1399 carried[i] = ev.eval(y);
1401 if (!ok || !start) {
1407 streams.push_back({base, loads});
1410 if (streams.empty() || int64_t(streams.size()) + loadStreams(loop) > 2)
1411 return rewriter.notifyMatchFailure(loop,
"no streams, or too many");
1413 Location loc = loop.getLoc();
1414 rewriter.setInsertionPoint(loop);
1415 SmallVector<Value> inits;
1416 for (
auto &[base, loads] : streams) {
1419 first.map(loop.getInductionVar(), loop.getLowerBound());
1420 first.map(loop.getRegionIterArgs(), loop.getInitArgs());
1421 std::function<Value(Value)> before = [&](Value v) -> Value {
1422 if (Value m = first.lookupOrNull(v))
1424 Operation *d = v.getDefiningOp();
1425 if (!d || !loop->isAncestor(d))
1427 for (Value o : d->getOperands())
1429 return rewriter.clone(*d, first)->getResult(
1430 cast<OpResult>(v).getResultNumber());
1432 auto begin = FifoStartOp::create(
1433 rewriter, loc, fifoAddrType(rewriter.getContext()),
1434 VectorType::get({32}, rewriter.getI32Type()), rewriter.getI32Type(),
1435 base, before(loads.front().getOffset()));
1436 inits.append(begin->getResults().begin(), begin->getResults().end());
1438 (void)loop.replaceWithAdditionalYields(
1439 rewriter, inits,
false,
1440 [&](OpBuilder &b, Location l, ArrayRef<BlockArgument> args) {
1441 SmallVector<Value> yields;
1442 for (auto [s, entry] : llvm::enumerate(streams)) {
1443 Value addr = args[3 * s], state = args[3 * s + 1],
1444 count = args[3 * s + 2];
1449 int64_t r = entry.second.size();
1450 Operation *first = entry.second.front();
1451 for (LoadBFP16Op ld : entry.second)
1452 if (ld->isBeforeInBlock(first))
1454 OpBuilder::InsertionGuard g(b);
1455 b.setInsertionPoint(first);
1456 for (auto [j, ld] : llvm::enumerate(entry.second)) {
1457 auto pop = FifoLdPopBFP16Op::create(
1458 b, ld.getLoc(), fifoAddrType(b.getContext()), state.getType(),
1459 b.getI32Type(), ld.getMantissas().getType(),
1460 ld.getExponents().getType(), addr, state, count,
1461 r % 8 == 0 && j % 8 ? b.getUnitAttr() : UnitAttr());
1462 addr = pop.getNext();
1463 state = pop.getStateOut();
1464 count = pop.getCountOut();
1465 ld.getMantissas().replaceAllUsesWith(pop.getMantissas());
1466 ld.getExponents().replaceAllUsesWith(pop.getExponents());
1468 yields.append({addr, state, count});
1472 for (
auto &[base, loads] : streams)
1473 for (LoadBFP16Op ld : loads)
1474 rewriter.eraseOp(ld);
1482 RewritePatternSet &patterns) {
1483 patterns.add<ReadDecodedBFP16Pattern>(patterns.getContext());
1487 RewritePatternSet &patterns) {
1488 patterns.add<HoistBFP16ConversionPattern>(patterns.getContext());
1492 RewritePatternSet &patterns) {
1493 patterns.add<BFP16LoadStreamPattern>(patterns.getContext());
void populateHoistBFP16ConversionPatterns(mlir::RewritePatternSet &patterns)
void populateBFP16LoadStreamPatterns(mlir::RewritePatternSet &patterns)
void populateReadDecodedBFP16Patterns(mlir::RewritePatternSet &patterns)