MLIR-AIE 1.4.6.dev
VectorToAIEVecConversions.cpp
Go to the documentation of this file.
1//===-VectorToAIEVecConversions.cpp - Vector to AIEVec convs. ---*- C++ -*-===//
2//
3// Copyright (C) 2023 Advanced Micro Devices, Inc.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7// This file contains conversions from the Vector dialect into the AIEVec
8// dialect. Conversions assume that the Vector dialect has been rectricted
9// to ops that can be translated to a sequence of valid AIEVec ops.
10//===----------------------------------------------------------------------===//
11
12#include "AIE2PShufflePlanner.h"
13
21
23#include "mlir/Dialect/Affine/IR/AffineOps.h"
24#include "mlir/Dialect/EmitC/IR/EmitC.h"
25#include "mlir/Dialect/Func/IR/FuncOps.h"
26#include "mlir/Dialect/Math/IR/Math.h"
27#include "mlir/Dialect/MemRef/IR/MemRef.h"
28#include "mlir/Dialect/SCF/IR/SCF.h"
29#include "mlir/Dialect/UB/IR/UBOps.h"
30#include "mlir/IR/PatternMatch.h"
31#include "mlir/IR/SymbolTable.h"
32#include "mlir/IR/TypeUtilities.h"
33#include "mlir/Pass/PassManager.h"
34#include "mlir/Transforms/DialectConversion.h"
35#include "mlir/Transforms/Passes.h"
36#include "llvm/ADT/SmallSet.h"
37#include <bitset>
38#include <optional>
39#include <tuple>
40
41#define DEBUG_TYPE "lower-vector-to-aievec"
42
43using namespace llvm;
44using namespace mlir;
45using namespace arith;
46using namespace vector;
47using namespace xilinx;
48using namespace xilinx::aievec;
49
50//===----------------------------------------------------------------------===//
51// Utility functions
52//===----------------------------------------------------------------------===//
53
54// A scalar i32 -> i8/i16 trunci's [clamp] chain as one SRS: the value
55// before the clamp, the sign of the saturation (1 signed, 0 unsigned) and the
56// clamp's ops, if any. nullopt if the clamp is not the destination type's
57// full range.
59 Value source;
60 int32_t sign;
61 arith::MinSIOp minOp;
62 arith::MaxSIOp maxOp;
63};
64static std::optional<ScalarSRSChain>
65matchScalarSRSChain(arith::TruncIOp truncOp) {
66 auto dstIntType = dyn_cast<IntegerType>(truncOp.getOut().getType());
67 if (!dstIntType)
68 return std::nullopt;
69 unsigned dstBits = dstIntType.getWidth();
70 if (dstBits != 8 && dstBits != 16)
71 return std::nullopt;
72 auto srcIntType = dyn_cast<IntegerType>(truncOp.getIn().getType());
73 if (!srcIntType || srcIntType.getWidth() != 32)
74 return std::nullopt;
75
76 // Walk backward through optional clamp chain
77 Value source = truncOp.getIn();
78 arith::MinSIOp minOp = nullptr;
79 arith::MaxSIOp maxOp = nullptr;
80 if (auto minsiOp = source.getDefiningOp<arith::MinSIOp>()) {
81 if (auto maxsiOp = minsiOp.getLhs().getDefiningOp<arith::MaxSIOp>()) {
82 minOp = minsiOp;
83 maxOp = maxsiOp;
84 source = maxOp.getLhs();
85 }
86 } else if (auto maxsiOp = source.getDefiningOp<arith::MaxSIOp>()) {
87 if (auto minsiOp = maxsiOp.getLhs().getDefiningOp<arith::MinSIOp>()) {
88 maxOp = maxsiOp;
89 minOp = minsiOp;
90 source = minOp.getLhs();
91 }
92 }
93 if (!minOp || !maxOp)
94 return ScalarSRSChain{source, 1, nullptr, nullptr};
95
96 // A clamp must be a valid saturation range
97 std::optional<int64_t> loVal = getConstantIntValue(maxOp.getRhs());
98 std::optional<int64_t> hiVal = getConstantIntValue(minOp.getRhs());
99 if (!loVal || !hiVal)
100 return std::nullopt;
101 uint64_t one = 1ULL;
102 int64_t unsignedHi = static_cast<int64_t>((one << dstBits) - 1);
103 int64_t signedLo = -static_cast<int64_t>(one << (dstBits - 1));
104 int64_t signedHi = static_cast<int64_t>((one << (dstBits - 1)) - 1);
105 if (*loVal == 0 && *hiVal == unsignedHi)
106 return ScalarSRSChain{source, 0, minOp, maxOp};
107 if (*loVal == signedLo && *hiVal == signedHi)
108 return ScalarSRSChain{source, 1, minOp, maxOp};
109 return std::nullopt;
110}
111
112// Check if a TruncIOp is part of a shrsi+[clamp]+trunc chain that can be
113// lowered to a compound SRS pattern. Only returns true for chains that the
114// ShiftClampTruncToSRSPattern rewrite will actually match:
115// - shrsi → trunci (no clamp)
116// - shrsi → maxsi → minsi → trunci (full clamp pair)
117// - shrsi → minsi → maxsi → trunci (reversed clamp pair)
118// Does NOT return true for a single min or max without a matching pair,
119// to avoid marking trunci illegal when no rewrite can handle it.
120static bool isSRSCompoundCandidate(arith::TruncIOp trunciOp) {
121 // LowerScalarShiftClampTruncToSRS converts fewer scalar chains than match
122 // the shapes below.
123 if (!isa<VectorType>(trunciOp.getType()) && !matchScalarSRSChain(trunciOp))
124 return false;
125
126 Value source = trunciOp.getIn();
127
128 // Case 1: direct shrsi → trunci
129 if (source.getDefiningOp<arith::ShRSIOp>())
130 return true;
131
132 // Case 2: minsi(maxsi(shrsi(...))) → trunci
133 if (auto minsiOp = source.getDefiningOp<arith::MinSIOp>()) {
134 if (auto maxsiOp = minsiOp.getLhs().getDefiningOp<arith::MaxSIOp>()) {
135 if (maxsiOp.getLhs().getDefiningOp<arith::ShRSIOp>())
136 return true;
137 }
138 }
139
140 // Case 3: maxsi(minsi(shrsi(...))) → trunci (reversed order)
141 if (auto maxsiOp = source.getDefiningOp<arith::MaxSIOp>()) {
142 if (auto minsiOp = maxsiOp.getLhs().getDefiningOp<arith::MinSIOp>()) {
143 if (minsiOp.getLhs().getDefiningOp<arith::ShRSIOp>())
144 return true;
145 }
146 }
147
148 // Case 4: minsi(maxsi(x)) → trunci (clamp-only, no shrsi)
149 // Used for saturating clamp after skip-add (e.g., skip_scale=0).
150 if (auto minsiOp = source.getDefiningOp<arith::MinSIOp>()) {
151 if (minsiOp.getLhs().getDefiningOp<arith::MaxSIOp>())
152 return true;
153 }
154
155 // Case 5: maxsi(minsi(x)) → trunci (reversed clamp-only, no shrsi)
156 if (auto maxsiOp = source.getDefiningOp<arith::MaxSIOp>()) {
157 if (maxsiOp.getLhs().getDefiningOp<arith::MinSIOp>())
158 return true;
159 }
160
161 return false;
162}
163
164// Check if a ShRSIOp's result feeds into a trunci (possibly through
165// clamp ops), meaning the compound pattern will consume it.
166static bool shrsiUsedByCompoundSRS(arith::ShRSIOp rsOp) {
167 for (Operation *user : rsOp->getUsers()) {
168 // Direct: shrsi → trunci (validate full chain via isSRSCompoundCandidate)
169 if (auto truncOp = dyn_cast<arith::TruncIOp>(user))
170 if (isSRSCompoundCandidate(truncOp))
171 return true;
172 // Through clamp: shrsi → maxsi → minsi → trunci
173 // or: shrsi → minsi → maxsi → trunci
174 if (isa<arith::MaxSIOp, arith::MinSIOp>(user)) {
175 for (Operation *user2 : user->getUsers()) {
176 if (auto truncOp2 = dyn_cast<arith::TruncIOp>(user2))
177 if (isSRSCompoundCandidate(truncOp2))
178 return true;
179 if (isa<arith::MaxSIOp, arith::MinSIOp>(user2)) {
180 for (Operation *user3 : user2->getUsers()) {
181 if (auto truncOp3 = dyn_cast<arith::TruncIOp>(user3))
182 if (isSRSCompoundCandidate(truncOp3))
183 return true;
184 }
185 }
186 }
187 }
188 }
189 return false;
190}
191
192// Check if a scalar maxsi/minsi is sandwiched in a compound SRS chain
193// (between shrsi and trunci). Used to keep the op legal so the scalar
194// compound SRS pattern can consume the entire chain.
195static bool scalarClampInCompoundSRS(Operation *op) {
196 if (!isa<arith::MaxSIOp, arith::MinSIOp>(op))
197 return false;
198 // Only apply to scalar types
199 if (isa<VectorType>(op->getResult(0).getType()))
200 return false;
201 for (Operation *user : op->getUsers()) {
202 if (auto truncOp = dyn_cast<arith::TruncIOp>(user)) {
203 if (isSRSCompoundCandidate(truncOp))
204 return true;
205 }
206 if (isa<arith::MaxSIOp, arith::MinSIOp>(user)) {
207 for (Operation *user2 : user->getUsers()) {
208 if (auto truncOp2 = dyn_cast<arith::TruncIOp>(user2)) {
209 if (isSRSCompoundCandidate(truncOp2))
210 return true;
211 }
212 }
213 }
214 }
215 return false;
216}
217
218// The value at the bottom of a chain of vector.shape_casts that ends at the
219// type it started from, or `v` itself. Flattening an op to rank 1 leaves such
220// round trips between it and the ops that use it.
221static Value skipShapeCastRoundTrip(Value v) {
222 Value src = v;
223 while (auto castOp = src.getDefiningOp<vector::ShapeCastOp>())
224 src = castOp.getSource();
225 return src.getType() == v.getType() ? src : v;
226}
227
228// Given a Value, if it is defined by a widening op (arith:ExtSIOp,
229// arith::ExtUIOp, arith::ExtFOp, aievec::UPSOp + aievec::SRSOp,
230// aievec::UPSOp + aievec::CastOp), return the source of the widening op.
231static std::optional<Value> getSourceOfWideningOp(Value src) {
232 src = skipShapeCastRoundTrip(src);
233 if (auto extSIOp = src.getDefiningOp<arith::ExtSIOp>())
234 return extSIOp.getIn();
235 if (auto extUIOp = src.getDefiningOp<arith::ExtUIOp>())
236 return extUIOp.getIn();
237 if (auto extFOp = src.getDefiningOp<arith::ExtFOp>())
238 return extFOp.getIn();
239 if (auto srsOp = src.getDefiningOp<aievec::SRSOp>()) {
240 // Conversion through AIE intrinsics takes two steps:
241 // 1) Load to accumulator: aievec.ups
242 // 2) Move from accumulator: aievec.srs
243 auto srsSource = srsOp.getSource();
244 if (srsSource)
245 if (auto upsOp = srsSource.getDefiningOp<aievec::UPSOp>())
246 return upsOp.getSource();
247 }
248 if (auto castOp = src.getDefiningOp<aievec::CastOp>()) {
249 // Conversion through AIE intrinsics can also take the following two steps:
250 // 1) Load to accumulator: aievec.ups
251 // 2) Move from accumulator: aievec.cast
252 auto castSource = castOp.getSource();
253 if (castSource)
254 if (auto upsOp = castSource.getDefiningOp<aievec::UPSOp>())
255 return upsOp.getSource();
256 }
257 return std::optional<Value>();
258}
259
260// The bf16 scalar that the f32 vector `src` splats, where `src` is a splat
261// of a scalar widened from bf16, possibly already lowered to aievec
262// broadcasts; else null.
263static Value getSplatOfWidenedBF16(Value src) {
264 src = skipShapeCastRoundTrip(src);
265 if (auto concatOp = src.getDefiningOp<aievec::ConcatOp>()) {
266 auto parts = concatOp.getSources();
267 if (parts.empty() || !llvm::all_equal(parts))
268 return {};
269 src = parts.front();
270 }
271 Value scalar;
272 if (auto bcastOp = src.getDefiningOp<vector::BroadcastOp>())
273 scalar = bcastOp.getSource();
274 else if (auto bcastOp = src.getDefiningOp<aievec::BroadcastScalarOp>())
275 scalar = bcastOp.getSource();
276 if (!scalar || isa<VectorType>(scalar.getType()))
277 return {};
278 if (auto extFOp = scalar.getDefiningOp<arith::ExtFOp>())
279 if (extFOp.getIn().getType().isBF16())
280 return extFOp.getIn();
281 return {};
282}
283
284// Whether the f32 vector `src` holds bf16 values: it is widened from bf16, a
285// splat of a scalar widened from bf16, or a constant that is exact in bf16.
286static bool holdsBF16(Value src) {
287 if (getSplatOfWidenedBF16(src))
288 return true;
289 if (auto wideSrc = getSourceOfWideningOp(src))
290 return getElementTypeOrSelf(wideSrc->getType()).isBF16();
291 DenseFPElementsAttr dense;
292 if (!matchPattern(src, m_Constant(&dense)) || !dense.getElementType().isF32())
293 return false;
294 return llvm::all_of(dense.getValues<APFloat>(), [](APFloat v) {
295 bool losesInfo = false;
296 v.convert(APFloat::BFloat(), APFloat::rmNearestTiesToEven, &losesInfo);
297 return !losesInfo;
298 });
299}
300
301// The bf16 values held by `src`, which must satisfy holdsBF16. A constant is
302// narrowed where it is used rather than folded to a bf16 constant, so it does
303// not occupy a vector register outside the code that uses it.
304static Value getBF16Source(OpBuilder &builder, Location loc, Value src) {
305 auto bf16Type = VectorType::get(cast<VectorType>(src.getType()).getShape(),
306 builder.getBF16Type());
307 if (Value scalar = getSplatOfWidenedBF16(src))
308 return vector::BroadcastOp::create(builder, loc, bf16Type, scalar);
309 if (auto wideSrc = getSourceOfWideningOp(src))
310 return *wideSrc;
311 auto shift =
312 arith::ConstantOp::create(builder, loc, builder.getI32IntegerAttr(0));
313 return aievec::SRSOp::create(builder, loc, bf16Type, src, shift);
314}
315
316// Given a Value, if it is defined by an integer widening op, return whether
317// that widening is signed: arith.extsi -> signed, arith.extui -> unsigned, and
318// an aievec.srs (from an already-rewritten extsi/extui) -> its own `sign`
319// attribute. Returns std::nullopt when the defining op is not a recognized
320// integer widening op, or carries no signedness: a floating-point arith.extf,
321// or an aievec.ups/aievec.cast (which do not encode signedness). Callers record
322// this on the created matmul op, since peeling the widening op would otherwise
323// discard it (MLIR integers are signless).
324static std::optional<bool> getSignednessOfWideningOp(Value src) {
325 // Look through shape casts, which may sit between the extension op and the
326 // consumer.
327 while (auto castOp = src.getDefiningOp<vector::ShapeCastOp>())
328 src = castOp.getSource();
329 if (src.getDefiningOp<arith::ExtSIOp>())
330 return true;
331 if (src.getDefiningOp<arith::ExtUIOp>())
332 return false;
333 // An arith.extsi/extui may already have been rewritten into an
334 // aievec.ups + aievec.srs pair by another pattern in this same conversion.
335 // Only aievec.srs records a signedness; aievec.ups and aievec.cast do not.
336 if (auto srsOp = src.getDefiningOp<aievec::SRSOp>())
337 return srsOp.getSign() != 0;
338 return std::nullopt;
339}
340
341// Given a Value, if it is defined by a narrowing op (arith::TruncFOp,
342// arith::TruncIOp), return the source of the narrowing op.
343static std::optional<Value> getSourceOfNarrowingOp(Value src) {
344 if (auto truncFOp = src.getDefiningOp<arith::TruncFOp>())
345 return truncFOp.getIn();
346 if (auto truncIOp = src.getDefiningOp<arith::TruncIOp>())
347 return truncIOp.getIn();
348 return std::optional<Value>();
349}
350
351//===----------------------------------------------------------------------===//
352// Type conversion utilities with narrowing/widening optimization awareness
353//===----------------------------------------------------------------------===//
354
355// Smart widen a value to target type. If the value was previously narrowed
356// from the target type, reuse the original source to avoid truncf->extf chains.
357static Value widenValueWithNarrowingCheck(Value val, Type targetType,
358 Location loc,
359 ConversionPatternRewriter &rewriter) {
360 // Check if this value was narrowed from the target type
361 if (auto narrowedSrc = getSourceOfNarrowingOp(val)) {
362 if (narrowedSrc->getType() == targetType)
363 return *narrowedSrc; // Reuse the original value (skip truncf->extf)
364 }
365
366 // Otherwise, create the widening op if needed
367 if (val.getType() == targetType)
368 return val;
369
370 return arith::ExtFOp::create(rewriter, loc, targetType, val,
371 /*fastmath=*/nullptr);
372}
373
374// Result structure for smart narrowing operation
376 Value narrowedValue; // The narrowed value (or original if optimized)
377 bool skipNarrowing; // True if we should skip creating truncf
378 Operation *wideningUser; // The widening op to replace (if skipNarrowing)
379};
380
381// Smart narrow a value to target type. If the result will be immediately
382// widened back, skip both truncf and extf operations.
383static NarrowingResult
384narrowValueWithWideningCheck(Operation *srcOp, Value val, Type targetType,
385 Location loc,
386 ConversionPatternRewriter &rewriter) {
387 NarrowingResult result;
388 result.narrowedValue = val;
389 result.skipNarrowing = false;
390 result.wideningUser = nullptr;
391
392 // Check if srcOp will be immediately widened back
393 if (srcOp->hasOneUse()) {
394 Operation *user = *srcOp->getUsers().begin();
395 if (auto extfOp = dyn_cast<arith::ExtFOp>(user)) {
396 // The result will be widened - skip both truncf and extf
397 result.skipNarrowing = true;
398 result.wideningUser = extfOp;
399 return result;
400 }
401 }
402
403 // Normal case: create the narrowing op
404 result.narrowedValue =
405 arith::TruncFOp::create(rewriter, loc, targetType, val);
406 return result;
407}
408
409// High-level helper to perform a binary operation on bf16 values in f32
410// precision. This function handles:
411// 1. Smart widening of operands (reuses f32 source if narrowed from f32)
412// 2. Executing the operation in f32
413// 3. Smart narrowing back to bf16 (skips truncf->extf if result is widened)
414static void
415performBF16BinaryOpInF32(Value lhs, Value rhs, Operation *srcOp, Location loc,
416 ConversionPatternRewriter &rewriter,
417 const std::function<Value(Value, Value)> &opBuilder) {
418 Type f32Type = rewriter.getF32Type();
419
420 // Smart widen both operands (reuse f32 source if narrowed from f32)
421 Value lhsF32 = widenValueWithNarrowingCheck(lhs, f32Type, loc, rewriter);
422 Value rhsF32 = widenValueWithNarrowingCheck(rhs, f32Type, loc, rewriter);
423
424 // Perform operation in f32
425 Value resultF32 = opBuilder(lhsF32, rhsF32);
426
427 // Smart narrow back to bf16 (skip if result will be widened)
428 auto narrowResult = narrowValueWithWideningCheck(
429 srcOp, resultF32, lhs.getType(), loc, rewriter);
430
431 if (narrowResult.skipNarrowing) {
432 // Replace the widening user directly with f32 result
433 rewriter.replaceOp(narrowResult.wideningUser, resultF32);
434 rewriter.eraseOp(srcOp);
435 } else {
436 rewriter.replaceOp(srcOp, narrowResult.narrowedValue);
437 }
438}
439
440// Given the LHS and RHS of an `arith::AddIOp`, if one of them is defined by an
441// `arith::MulIOp`, return a tuple with the `lhs`, `rhs`, and `acc` of the MAC
442// operation that can replace them.
443static std::optional<std::tuple<Value, Value, Value>>
444extractMACOperandsFromAddOperands(Value addLhs, Value addRhs) {
445 auto *lhsDefOp = addLhs.getDefiningOp();
446 auto *rhsDefOp = addRhs.getDefiningOp();
447 arith::MulIOp mulOp = nullptr;
448 Value acc;
449 if (lhsDefOp) {
450 mulOp = dyn_cast<arith::MulIOp>(lhsDefOp);
451 acc = addRhs;
452 }
453 if (!mulOp && rhsDefOp) {
454 mulOp = dyn_cast<arith::MulIOp>(rhsDefOp);
455 acc = addLhs;
456 }
457 if (mulOp)
458 return std::make_tuple(mulOp.getLhs(), mulOp.getRhs(), acc);
459
460 // If the MulIOp has been already translated to aievec::aie1::MulOp:
461 auto lhsSrsOp = addLhs.getDefiningOp<aievec::SRSOp>();
462 auto rhsSrsOp = addRhs.getDefiningOp<aievec::SRSOp>();
463 aievec::aie1::MulOp aieMulOp = nullptr;
464 if (lhsSrsOp) {
465 aieMulOp = lhsSrsOp.getSource().getDefiningOp<aievec::aie1::MulOp>();
466 acc = addRhs;
467 }
468 if (!aieMulOp && rhsSrsOp) {
469 aieMulOp = rhsSrsOp.getSource().getDefiningOp<aievec::aie1::MulOp>();
470 acc = addLhs;
471 }
472 if (aieMulOp)
473 return std::make_tuple(aieMulOp.getLhs(), aieMulOp.getRhs(), acc);
474 return {};
475}
476
477// Given the LHS and RHS of an `arith::AddFOp`, if one of them is defined by an
478// `arith::MulFOp`, return a tuple with the `lhs`, `rhs`, and `acc` of the FMA
479// operation that can replace them.
480static std::optional<std::tuple<Value, Value, Value>>
481extractFMACOperandsFromAddOperands(Value addLhs, Value addRhs) {
482 auto *lhsDefOp = addLhs.getDefiningOp();
483 auto *rhsDefOp = addRhs.getDefiningOp();
484 arith::MulFOp mulOp = nullptr;
485 Value acc;
486 if (lhsDefOp) {
487 mulOp = dyn_cast<arith::MulFOp>(lhsDefOp);
488 acc = addRhs;
489 }
490 if (!mulOp && rhsDefOp) {
491 mulOp = dyn_cast<arith::MulFOp>(rhsDefOp);
492 acc = addLhs;
493 }
494 if (mulOp)
495 return std::make_tuple(mulOp.getLhs(), mulOp.getRhs(), acc);
496 return {};
497}
498
499// Convert a input value to a target vector type. This function can insert
500// multiple aievec ops depending on the combination of input and output vector
501// types.
502static std::optional<Value>
503convertValueToTargetTypeAIE2(ConversionPatternRewriter &rewriter, Location loc,
504 Value inputVal, VectorType tgtType) {
505 auto srcType = cast<VectorType>(inputVal.getType());
506 auto srcElemType = srcType.getElementType();
507 unsigned srcBitWidth = srcElemType.getIntOrFloatBitWidth();
508 unsigned srcLaneSize = getVectorLaneSize(srcType);
509
510 auto tgtElemType = tgtType.getElementType();
511 unsigned tgtBitWidth = tgtElemType.getIntOrFloatBitWidth();
512 unsigned tgtLaneSize = getVectorLaneSize(tgtType);
513
514 if (srcType == tgtType)
515 return inputVal;
516
517 if ((srcElemType == tgtElemType) && (srcLaneSize != tgtLaneSize)) {
518 // TODO: relax the condition below?
519 if ((srcLaneSize == 16 && tgtLaneSize == 32 &&
520 isa<FloatType>(srcElemType)) ||
521 (srcLaneSize == 32 && tgtLaneSize == 64 &&
522 isa<IntegerType>(srcElemType))) {
523 auto zeroConstOp = arith::ConstantOp::create(
524 rewriter, loc, srcType.getElementType(),
525 rewriter.getZeroAttr(srcType.getElementType()));
526 auto broadcastZeroOp = aievec::BroadcastScalarOp::create(
527 rewriter, loc, tgtType, zeroConstOp->getResult(0));
528 auto extOp = aievec::ExtOp::create(rewriter, loc, srcType,
529 broadcastZeroOp->getResult(0), 0);
530
531 SmallVector<Value> inputSources = {inputVal, extOp->getResult(0)};
532 auto concatOp =
533 aievec::ConcatOp::create(rewriter, loc, tgtType, inputSources);
534
535 return concatOp.getResult();
536 }
537 } else if ((srcElemType != tgtElemType) && (srcLaneSize == tgtLaneSize) &&
538 isa<IntegerType>(srcElemType) && isa<IntegerType>(tgtElemType)) {
539 if (srcBitWidth == 16 && tgtBitWidth == 32 && srcLaneSize == 16) {
540 // Case 1: vector<16xi16> to vector<16xi32> conversion by aievec.ups +
541 // aievec.cast
542 auto accType = getVectorOpDestType(srcType, /*AIE2 =*/true);
543 auto upsOp = aievec::UPSOp::create(rewriter, loc, accType, inputVal);
544 auto castOp = aievec::CastOp::create(
545 rewriter, loc, tgtType, upsOp.getResult(), /*isResAcc*/ false);
546 return castOp.getResult();
547 }
548
549 if (srcBitWidth == 8 && tgtBitWidth == 32 && srcLaneSize == 16) {
550 // Case 2: vector<16xi8> to vector<16xi32> conversion by aievec.concat +
551 // aievec.ups + aievec.cast + aievec.ext
552 auto concatOutType = createVectorType(32, srcElemType);
553 auto concatOp =
554 aievec::ConcatOp::create(rewriter, loc, concatOutType,
555 SmallVector<Value>({inputVal, inputVal}));
556 auto accType = getVectorOpDestType(concatOutType, /*AIE2 =*/true);
557 auto upsOp =
558 aievec::UPSOp::create(rewriter, loc, accType, concatOp.getResult());
559 auto castType = createVectorType(32, tgtElemType);
560 auto castOp = aievec::CastOp::create(
561 rewriter, loc, castType, upsOp.getResult(), /*isResAcc*/ false);
562 auto extOp =
563 aievec::ExtOp::create(rewriter, loc, tgtType, castOp.getResult(), 0);
564 return extOp.getResult();
565 }
566
567 if (srcBitWidth == 8 && tgtBitWidth == 16 && srcLaneSize == 32) {
568 // Case 3: vector<32xi8> to vector<32xi16> conversion by aievec.unpack
569 auto unpackOp =
570 aievec::UnpackOp::create(rewriter, loc, tgtType, inputVal);
571 return unpackOp.getResult();
572 }
573 }
574
575 return std::nullopt;
576}
577
578// Return the list of attributes that configure an `aievec.select` op to
579// perform a rotation of the input vector by `rotation` number of elements.
580// The attribute values depend on the vector type of the select operation.
581static SmallVector<NamedAttribute>
582buildAttributeListForRotationSelectOp(PatternRewriter &rewriter, VectorType vTy,
583 int64_t rotation) {
584 unsigned width = 0;
585 auto elemTy = vTy.getElementType();
586 if (auto intTy = dyn_cast<IntegerType>(elemTy))
587 width = intTy.getWidth();
588 StringAttr attr0 = rewriter.getStringAttr("0");
589 StringAttr attr0x06040200 = rewriter.getStringAttr("0x06040200");
590 StringAttr attr0x0e0c0a08 = rewriter.getStringAttr("0x0e0c0a08");
591 StringAttr attr0x2103 = rewriter.getStringAttr("0x2103");
592 StringAttr attr0x3210 = rewriter.getStringAttr("0x3210");
593 StringAttr selectAttrName = rewriter.getStringAttr("select");
594 StringAttr xoffsetsAttrName = rewriter.getStringAttr("xoffsets");
595 StringAttr xoffsetsHiAttrName = rewriter.getStringAttr("xoffsets_hi");
596 StringAttr xsquareAttrName = rewriter.getStringAttr("xsquare");
597 StringAttr xstartAttrName = rewriter.getStringAttr("xstart");
598 StringAttr yoffsetsAttrName = rewriter.getStringAttr("yoffsets");
599 StringAttr yoffsetsHiAttrName = rewriter.getStringAttr("yoffsets_hi");
600 StringAttr ysquareAttrName = rewriter.getStringAttr("ysquare");
601 StringAttr ystartAttrName = rewriter.getStringAttr("ystart");
602
603 switch (width) {
604 case 16: {
605 if (rotation % 2) {
606 int64_t xstart = rotation + 1;
607 int64_t ystart = rotation - 1;
608 return SmallVector<NamedAttribute, 9>(
609 {{selectAttrName, rewriter.getStringAttr("0x11111111")},
610 {xoffsetsAttrName, attr0x06040200},
611 {xoffsetsHiAttrName, attr0x0e0c0a08},
612 {xsquareAttrName, attr0x2103},
613 {xstartAttrName, rewriter.getStringAttr(std::to_string(xstart))},
614 {yoffsetsAttrName, rewriter.getStringAttr("0x0503010f")},
615 {yoffsetsHiAttrName, rewriter.getStringAttr("0x0d0b0907")},
616 {ysquareAttrName, attr0x2103},
617 {ystartAttrName, rewriter.getStringAttr(std::to_string(ystart))}});
618 }
619 return SmallVector<NamedAttribute, 9>(
620 {{selectAttrName, attr0},
621 {xoffsetsAttrName, attr0x06040200},
622 {xoffsetsHiAttrName, attr0x0e0c0a08},
623 {xsquareAttrName, attr0x3210},
624 {xstartAttrName, rewriter.getStringAttr(std::to_string(rotation))},
625 {yoffsetsAttrName, attr0},
626 {yoffsetsHiAttrName, attr0},
627 {ysquareAttrName, attr0},
628 {ystartAttrName, attr0}});
629 }
630 case 32:
631 return SmallVector<NamedAttribute, 7>(
632 {{selectAttrName, attr0},
633 {xoffsetsAttrName, rewriter.getStringAttr("0x76543210")},
634 {xsquareAttrName, attr0x3210},
635 {xstartAttrName, rewriter.getStringAttr(std::to_string(rotation))},
636 {yoffsetsAttrName, attr0},
637 {ysquareAttrName, attr0},
638 {ystartAttrName, attr0}});
639 default:
640 llvm::report_fatal_error("Unexpected width!");
641 }
642
643 return {};
644}
645
646namespace xilinx::aievec {
647
648// Also called from AIEVecOptimizations.cpp via a forward declaration there;
649// not static.
650SmallVector<NamedAttribute>
651// NOLINTNEXTLINE(misc-use-internal-linkage)
652buildFMAOpSplatAttrForElemTy(aievec::aie1::FMAOp fmaOp, int64_t bcastPos,
653 int64_t step = 1) {
654 unsigned width = 0;
655 auto elemTy = fmaOp.getLhs().getType().getElementType();
656 if (auto intTy = dyn_cast<IntegerType>(elemTy))
657 width = intTy.getWidth();
658 auto *ctx = fmaOp.getContext();
659 switch (width) {
660 case 16:
661 // NOTE: The pattern is:
662 // acc[0] = x[0] * z[bcastPos] + x[16] * z[bcastPos+step]
663 // acc[1] = x[1] * z[bcastPos] + x[17] * z[bcastPos+step]
664 // acc[2] = x[2] * z[bcastPos] + x[18] * z[bcastPos+step]
665 // acc[3] = x[3] * z[bcastPos] + x[19] * z[bcastPos+step]
666 // acc[4] = x[4] * z[bcastPos] + x[20] * z[bcastPos+step]
667 // acc[5] = x[5] * z[bcastPos] + x[21] * z[bcastPos+step]
668 // acc[6] = x[6] * z[bcastPos] + x[22] * z[bcastPos+step]
669 // acc[7] = x[7] * z[bcastPos] + x[23] * z[bcastPos+step]
670 // acc[8] = x[8] * z[bcastPos] + x[24] * z[bcastPos+step]
671 // acc[9] = x[9] * z[bcastPos] + x[25] * z[bcastPos+step]
672 // acc[10] = x[10] * z[bcastPos] + x[26] * z[bcastPos+step]
673 // acc[11] = x[11] * z[bcastPos] + x[27] * z[bcastPos+step]
674 // acc[12] = x[12] * z[bcastPos] + x[28] * z[bcastPos+step]
675 // acc[13] = x[13] * z[bcastPos] + x[29] * z[bcastPos+step]
676 // acc[14] = x[14] * z[bcastPos] + x[30] * z[bcastPos+step]
677 // acc[15] = x[15] * z[bcastPos] + x[31] * z[bcastPos+step]
678 return SmallVector<NamedAttribute, 11>(
679 {{fmaOp.getXstartAttrName(), StringAttr::get(ctx, "0")},
680 {fmaOp.getXoffsetsAttrName(), StringAttr::get(ctx, "0x73727170")},
681 {fmaOp.getXoffsetsHiAttrName(), StringAttr::get(ctx, "0x77767574")},
682 {fmaOp.getXstepAttrName(), fmaOp.getXstepAttr()},
683 {fmaOp.getXsquareAttrName(), StringAttr::get(ctx, "0x3120")},
684 {fmaOp.getZstartAttrName(),
685 StringAttr::get(ctx, std::to_string(bcastPos))},
686 {fmaOp.getZoffsetsAttrName(), StringAttr::get(ctx, "0")},
687 {fmaOp.getZoffsetsHiAttrName(), StringAttr::get(ctx, "0")},
688 {fmaOp.getZstepAttrName(), StringAttr::get(ctx, std::to_string(step))},
689 {fmaOp.getZsquareAttrName(), fmaOp.getZsquareAttr()},
690 {fmaOp.getFmsubAttrName(), fmaOp.getFmsubAttr()}});
691 case 32:
692 return SmallVector<NamedAttribute, 11>(
693 {{fmaOp.getXstartAttrName(), StringAttr::get(ctx, "0")},
694 {fmaOp.getXoffsetsAttrName(), StringAttr::get(ctx, "0x76543210")},
695 {fmaOp.getXoffsetsHiAttrName(), fmaOp.getXoffsetsHiAttr()},
696 {fmaOp.getXstepAttrName(), fmaOp.getXstepAttr()},
697 {fmaOp.getXsquareAttrName(), fmaOp.getXsquareAttr()},
698 {fmaOp.getZstartAttrName(),
699 StringAttr::get(ctx, std::to_string(bcastPos))},
700 {fmaOp.getZoffsetsAttrName(), StringAttr::get(ctx, "0x00000000")},
701 {fmaOp.getZoffsetsHiAttrName(), fmaOp.getZoffsetsHiAttr()},
702 {fmaOp.getZstepAttrName(), fmaOp.getZstepAttr()},
703 {fmaOp.getZsquareAttrName(), fmaOp.getZsquareAttr()},
704 {fmaOp.getFmsubAttrName(), fmaOp.getFmsubAttr()}});
705 default:
706 llvm::report_fatal_error("Unexpected width!");
707 }
708
709 return {};
710}
711
712} // namespace xilinx::aievec
713
714template <typename SrcOpTy, typename AIEv2ElemOp>
715static LogicalResult genAddElemAIE2(ConversionPatternRewriter &rewriter,
716 Value lval, Value rval, VectorType srcType,
717 SrcOpTy srcOp) {
718 auto lCastOp = aievec::CastOp::create(rewriter, srcOp.getLoc(), srcType, lval,
719 /*isResAcc*/ true);
720 auto rCastOp = aievec::CastOp::create(rewriter, srcOp.getLoc(), srcType, rval,
721 /*isResAcc*/ true);
722 auto elemOp = AIEv2ElemOp::create(
723 rewriter, srcOp.getLoc(), lCastOp->getResult(0).getType(),
724 lCastOp->getResult(0), rCastOp->getResult(0));
725 rewriter.replaceOpWithNewOp<aievec::CastOp>(
726 srcOp, srcOp.getType(), elemOp.getResult(), /*isResAcc*/ false);
727 return success();
728}
729
730static arith::CmpIPredicate
731convertToIntegerPredicate(arith::CmpFPredicate pred) {
732 switch (pred) {
733 case CmpFPredicate::UEQ:
734 case CmpFPredicate::OEQ:
735 return CmpIPredicate::eq;
736 case CmpFPredicate::UGT:
737 return CmpIPredicate::ugt;
738 case CmpFPredicate::OGT:
739 return CmpIPredicate::sgt;
740 case CmpFPredicate::UGE:
741 return CmpIPredicate::uge;
742 case CmpFPredicate::OGE:
743 return CmpIPredicate::sge;
744 case CmpFPredicate::ULT:
745 return CmpIPredicate::ult;
746 case CmpFPredicate::OLT:
747 return CmpIPredicate::slt;
748 case CmpFPredicate::ULE:
749 return CmpIPredicate::ule;
750 case CmpFPredicate::OLE:
751 return CmpIPredicate::sle;
752 case CmpFPredicate::UNE:
753 case CmpFPredicate::ONE:
754 return CmpIPredicate::ne;
755 default:
756 llvm::report_fatal_error("Unexpected predicate!");
757 }
758}
759
760static arith::CmpIPredicate
761convertToIntegerPredicate(arith::CmpIPredicate pred) {
762 return pred;
763}
764
765static aievec::CmpOp createCmpOpAIE2(ConversionPatternRewriter &rewriter,
766 CmpIPredicate pred, Location loc,
767 Type type, Value lhs, Value rhs) {
768 switch (pred) {
769 case CmpIPredicate::eq:
770 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs, "eq");
771 case CmpIPredicate::ne:
772 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs, "ne");
773 case CmpIPredicate::slt:
774 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs, "slt");
775 case CmpIPredicate::ult:
776 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs, "ult");
777 case CmpIPredicate::sle:
778 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs, "sle");
779 case CmpIPredicate::ule:
780 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs, "ule");
781 case CmpIPredicate::sgt:
782 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs, "sgt");
783 case CmpIPredicate::ugt:
784 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs, "ugt");
785 case CmpIPredicate::sge:
786 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs, "sge");
787 case CmpIPredicate::uge:
788 return aievec::CmpOp::create(rewriter, loc, type, lhs, rhs, "uge");
789 }
790 return nullptr;
791}
792
793template <typename DstOpTy>
794static aievec::ExtElemOp
795generateAIEVecOpsForReductionOp(ConversionPatternRewriter &rewriter,
796 vector::ReductionOp srcOp, int shiftIndex,
797 Value curValue) {
798 assert(shiftIndex > 0 && (shiftIndex & (shiftIndex - 1)) == 0 &&
799 "shiftIndex must be power of 2");
800
801 Location loc = srcOp.getLoc();
802 auto vType = cast<VectorType>(curValue.getType());
803 Type scalarType = vType.getElementType();
804 Type vecType = curValue.getType();
805 DstOpTy curOp = nullptr;
806 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
807
808 for (int id = shiftIndex; id > 0; id /= 2) {
809 auto constOp = arith::ConstantOp::create(
810 rewriter, loc, rewriter.getI32IntegerAttr(id * elWidth / 8));
811
812 auto shiftBytesOp = aievec::ShiftOp::create(
813 rewriter, loc, vecType, curValue, curValue, constOp.getResult());
814
815 curOp = DstOpTy::create(rewriter, loc, vecType, curValue,
816 shiftBytesOp.getResult());
817
818 curValue = curOp.getResult();
819 }
820
821 auto zeroConstOp =
822 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
823 return aievec::ExtElemOp::create(rewriter, loc, scalarType, curOp,
824 zeroConstOp.getResult());
825}
826
827// Helper to pad a v16bf16 vector to v32bf16 by concatenating with a splat of
828// the given infinity value. Used by min/max reduction patterns.
829// Returns {paddedVector, newLaneSize}.
830static std::pair<Value, unsigned>
831padV16ToV32WithInfinity(ConversionPatternRewriter &rewriter, Location loc,
832 Value inputVec, Type scalarType, bool negativeInf) {
833 VectorType v32bf16Type = createVectorType(32, scalarType);
834 VectorType v16bf16Type = createVectorType(16, scalarType);
835
836 // Create a scalar infinity constant
837 auto infAttr = rewriter.getFloatAttr(
838 scalarType,
839 APFloat::getInf(cast<FloatType>(scalarType).getFloatSemantics(),
840 negativeInf));
841 auto splatInf = arith::ConstantOp::create(rewriter, loc, infAttr).getResult();
842
843 // Broadcast to v32bf16, then extract upper half (which is also infinity)
844 auto infVec =
845 aievec::BroadcastScalarOp::create(rewriter, loc, v32bf16Type, splatInf);
846 auto infUpperHalf =
847 aievec::ExtOp::create(rewriter, loc, v16bf16Type, infVec, 1);
848
849 // Concatenate input with infinity padding
850 Value paddedVec =
851 aievec::ConcatOp::create(rewriter, loc, v32bf16Type,
852 ValueRange{inputVec, infUpperHalf.getResult()});
853
854 return {paddedVec, 32};
855}
856
857// Helper to pad a v16bf16 vector to v32bf16 by concatenating with zeros.
858// Used by elementwise min/max/cmp/sel patterns for 256-bit bf16 support.
859static Value padV16ToV32WithZeros(ConversionPatternRewriter &rewriter,
860 Location loc, Value inputVec,
861 Type scalarType) {
862 VectorType v32Type = createVectorType(32, scalarType);
863 VectorType v16Type = createVectorType(16, scalarType);
864 auto zeroAttr = rewriter.getZeroAttr(v16Type);
865 auto zeroVec = arith::ConstantOp::create(rewriter, loc, zeroAttr);
866 return aievec::ConcatOp::create(rewriter, loc, v32Type,
867 ValueRange{inputVec, zeroVec.getResult()});
868}
869
870static func::FuncOp getOrInsertFuncDecl(ConversionPatternRewriter &rewriter,
871 Operation *parentSymbolTableOp,
872 StringRef funcName, TypeRange inTypes,
873 TypeRange outTypes) {
874
875 mlir::OpBuilder::InsertionGuard insertGuard(rewriter);
876 rewriter.setInsertionPointToStart(
877 &parentSymbolTableOp->getRegions().front().getBlocks().front());
878 SymbolTable st = SymbolTable(parentSymbolTableOp);
879 func::FuncOp fnOpLookup = st.lookup<func::FuncOp>(funcName);
880 func::FuncOp fnOp;
881 // if the function is already declared, use the existing function, don't
882 // declare multiple times
883 if (fnOpLookup != nullptr) {
884 fnOp = fnOpLookup;
885 } else {
886 StringAttr t1 = rewriter.getStringAttr("sym_visibility");
887 StringAttr t2 = rewriter.getStringAttr("private");
888 NamedAttribute funcAccess = NamedAttribute(t1, t2);
889 FunctionType fnType =
890 mlir::FunctionType::get(rewriter.getContext(), inTypes, outTypes);
891 fnOp = func::FuncOp::create(rewriter, parentSymbolTableOp->getLoc(),
892 funcName, fnType, funcAccess);
893 }
894 return fnOp;
895}
896
897//===----------------------------------------------------------------------===//
898// Wide vector splitting utility
899//===----------------------------------------------------------------------===//
900
901// Utility function to split a wide vector operation (e.g., v32bf16) into two
902// half-width operations (e.g., v16bf16) and concatenate the results.
903// This pattern is common for AIE2 when hardware only supports 16-lane
904// operations but we need to process 32-lane vectors.
905//
906// Template parameters:
907// SrcOpTy - The source operation type (e.g., arith::MulFOp, math::ExpOp)
908//
909// Parameters:
910// srcOp - The source operation to replace
911// wideInputs - The wide input values (e.g., lhs and rhs for binary ops)
912// halfType - The half-width vector type (e.g., vector<16xbf16>)
913// wideType - The wide vector type (e.g., vector<32xbf16>)
914// rewriter - The pattern rewriter
915// processHalves - Callback that processes the lower and upper halves
916// and returns a pair of half-width results to be concatenated
917//
918// The callback signature is:
919// std::pair<Value, Value>(ArrayRef<std::pair<Value, Value>> halfInputs,
920// Location loc, ConversionPatternRewriter &rewriter)
921// where halfInputs[i] is {lowerHalf, upperHalf} for each wideInput
922template <typename SrcOpTy, typename Func>
923static void splitWideVectorOp(SrcOpTy srcOp, ArrayRef<Value> wideInputs,
924 VectorType halfType, VectorType wideType,
925 ConversionPatternRewriter &rewriter,
926 Func &&processHalves) {
927
928 Location loc = srcOp.getLoc();
929
930 // Extract lower and upper halves for each wide input
931 SmallVector<std::pair<Value, Value>> halfInputs;
932 halfInputs.reserve(wideInputs.size());
933 for (Value wideInput : wideInputs) {
934 auto lowerHalf =
935 aievec::ExtOp::create(rewriter, loc, halfType, wideInput, 0);
936 auto upperHalf =
937 aievec::ExtOp::create(rewriter, loc, halfType, wideInput, 1);
938 halfInputs.emplace_back(lowerHalf.getResult(), upperHalf.getResult());
939 }
940
941 // Process halves using the callback
942 auto [lowResult, highResult] = processHalves(halfInputs, loc, rewriter);
943
944 // Concatenate results
945 SmallVector<Value> concatSources = {lowResult, highResult};
946 rewriter.replaceOpWithNewOp<aievec::ConcatOp>(srcOp, wideType, concatSources);
947}
948
949// Simplified version for unary operations
950template <typename SrcOpTy>
951static void splitWideUnaryVectorOp(
952 SrcOpTy srcOp, Value wideInput, VectorType halfType, VectorType wideType,
953 ConversionPatternRewriter &rewriter,
954 std::function<Value(Value, Location, ConversionPatternRewriter &)>
955 processHalf) {
956
957 splitWideVectorOp<SrcOpTy>(
958 srcOp, {wideInput}, halfType, wideType, rewriter,
959 [&processHalf](ArrayRef<std::pair<Value, Value>> halfInputs, Location loc,
960 ConversionPatternRewriter &rewriter) {
961 auto [lowerHalf, upperHalf] = halfInputs[0];
962 Value lowResult = processHalf(lowerHalf, loc, rewriter);
963 Value highResult = processHalf(upperHalf, loc, rewriter);
964 return std::make_pair(lowResult, highResult);
965 });
966}
967
968// Simplified version for binary operations
969template <typename SrcOpTy>
970static void splitWideBinaryVectorOp(
971 SrcOpTy srcOp, Value lhs, Value rhs, VectorType halfType,
972 VectorType wideType, ConversionPatternRewriter &rewriter,
973 std::function<Value(Value, Value, Location, ConversionPatternRewriter &)>
974 processHalf) {
975
976 splitWideVectorOp<SrcOpTy>(
977 srcOp, {lhs, rhs}, halfType, wideType, rewriter,
978 [&processHalf](ArrayRef<std::pair<Value, Value>> halfInputs, Location loc,
979 ConversionPatternRewriter &rewriter) {
980 auto [lhsLow, lhsHigh] = halfInputs[0];
981 auto [rhsLow, rhsHigh] = halfInputs[1];
982 Value lowResult = processHalf(lhsLow, rhsLow, loc, rewriter);
983 Value highResult = processHalf(lhsHigh, rhsHigh, loc, rewriter);
984 return std::make_pair(lowResult, highResult);
985 });
986}
987
988//===----------------------------------------------------------------------===//
989// Math operation matching utilities
990//===----------------------------------------------------------------------===//
991
992// Check if math.exp op matches AIE2 LUT-based exp constraints
993static bool matchExpOpForAIE2LUT(math::ExpOp::Adaptor adaptor) {
994 auto srcType = dyn_cast<VectorType>(adaptor.getOperand().getType());
995
996 if (!srcType)
997 return false;
998
999 Type scalarType = srcType.getElementType();
1000 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
1001 unsigned laneSize = getVectorLaneSize(srcType);
1002 // AIE2 LUT-based exp: supports v16bf16 and v32bf16
1003 return isa<FloatType>(scalarType) && (laneSize == 16 || laneSize == 32) &&
1004 elWidth == 16;
1005}
1006
1007// Check if math.exp op matches AIE2P exp constraints
1008static bool matchExpOpForAIE2P(math::ExpOp::Adaptor adaptor) {
1009 auto srcType = dyn_cast<VectorType>(adaptor.getOperand().getType());
1010
1011 if (!srcType)
1012 return false;
1013
1014 Type scalarType = srcType.getElementType();
1015 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
1016 unsigned laneSize = getVectorLaneSize(srcType);
1017 // AIE2P exp: supports v16bf16 and v32bf16
1018 return scalarType.isBF16() && (laneSize == 16 || laneSize == 32) &&
1019 elWidth == 16;
1020}
1021
1022// Check if a vector type matches the AIE2P tanh and exp2 constraints. These
1023// intrinsics take an accfloat operand, so f32 vectors map onto them as well as
1024// bf16.
1025static bool isAIE2PAccFloatUnaryType(VectorType type) {
1026 if (type.isScalable())
1027 return false;
1028 Type elemType = type.getElementType();
1029 unsigned laneSize = getVectorLaneSize(type);
1030 return (elemType.isBF16() || elemType.isF32()) &&
1031 (laneSize == 16 || laneSize == 32);
1032}
1033
1034// Check if math.tanh, math.erf or math.exp2 of `type` is lowered for AIE2P: a
1035// bf16 or f32 scalar, one-lane vector, or vector matching
1036// isAIE2PAccFloatUnaryType.
1037static bool isAIE2PAccFloatUnaryOrScalarType(Type type) {
1038 auto vecType = dyn_cast<VectorType>(type);
1039 if (!vecType)
1040 return type.isBF16() || type.isF32();
1041 if (!vecType.isScalable() && vecType.getRank() == 1 &&
1042 vecType.getNumElements() == 1) {
1043 Type elemType = vecType.getElementType();
1044 return elemType.isBF16() || elemType.isF32();
1045 }
1046 return isAIE2PAccFloatUnaryType(vecType);
1047}
1048
1049//===----------------------------------------------------------------------===//
1050// Rewrite patterns
1051//===----------------------------------------------------------------------===//
1052
1053// This pattern fold `vector.extract` and `vector.broadcast` into
1054// `aievec.broadcast` for AIE2
1056 : OpConversionPattern<vector::BroadcastOp> {
1057 using OpConversionPattern::OpConversionPattern;
1058
1059 LogicalResult
1060 matchAndRewrite(vector::BroadcastOp bcastOp, OpAdaptor adaptor,
1061 ConversionPatternRewriter &rewriter) const override {
1062
1063 auto extOp = adaptor.getSource().getDefiningOp<vector::ExtractOp>();
1064
1065 if (!extOp)
1066 return failure();
1067
1068 auto src = extOp.getSource();
1069 auto pos = extOp.getStaticPosition();
1070 int64_t posVal = pos[0];
1071 auto srcVecType = cast<VectorType>(src.getType());
1072 auto resultType = cast<VectorType>(bcastOp.getResult().getType());
1073 if (srcVecType != resultType) {
1074 if (srcVecType.getNumElements() != 2 * resultType.getNumElements())
1075 return failure();
1076 auto half = static_cast<int8_t>(posVal / resultType.getNumElements());
1077 posVal -= half * resultType.getNumElements();
1078 src = aievec::ExtOp::create(rewriter, extOp.getLoc(), resultType, src,
1079 rewriter.getI8IntegerAttr(half))
1080 .getResult();
1081 }
1082
1083 unsigned elWidth = resultType.getElementType().getIntOrFloatBitWidth();
1084
1085 if (unsigned laneSize = getVectorLaneSize(resultType);
1086 laneSize * elWidth == 512) {
1087 // Common use case for the broadcast_elem intrinsic
1088 rewriter.replaceOpWithNewOp<aievec::BroadcastOp>(bcastOp, resultType, src,
1089 posVal);
1090 } else if (laneSize * elWidth == 256) {
1091 // e.g. need v16bf16 due to the subsequent v16accfloat operation
1092 VectorType aievecBcastType =
1093 createVectorType(512 / elWidth, resultType.getElementType());
1094 auto concatOp =
1095 aievec::ConcatOp::create(rewriter, bcastOp.getLoc(), aievecBcastType,
1096 SmallVector<Value>({src, src}));
1097 auto aieBcastOp = aievec::BroadcastOp::create(
1098 rewriter, bcastOp.getLoc(), aievecBcastType, concatOp.getResult(),
1099 posVal);
1100 rewriter.replaceOpWithNewOp<aievec::ExtOp>(bcastOp, resultType,
1101 aieBcastOp.getResult(), 0);
1102 } else if (laneSize * elWidth == 1024) {
1103 // e.g. need v32int32 due to the subsequent v32acc32 operation
1104 VectorType aievecBcastType =
1105 createVectorType(512 / elWidth, resultType.getElementType());
1106 auto half = static_cast<int8_t>(posVal / resultType.getNumElements());
1107 posVal -= half * resultType.getNumElements();
1108 auto extOp =
1109 aievec::ExtOp::create(rewriter, bcastOp.getLoc(), aievecBcastType,
1110 src, rewriter.getI8IntegerAttr(half));
1111 auto aieBcastOp = aievec::BroadcastOp::create(rewriter, bcastOp.getLoc(),
1112 aievecBcastType,
1113 extOp.getResult(), posVal);
1114 rewriter.replaceOpWithNewOp<aievec::ConcatOp>(
1115 bcastOp, resultType,
1116 SmallVector<Value>({aieBcastOp.getResult(), aieBcastOp.getResult()}));
1117 } else {
1118 return failure();
1119 }
1120
1121 return success();
1122 }
1123};
1124
1125struct ConvertSplatToAIEBroadcast : OpConversionPattern<vector::BroadcastOp> {
1126 using OpConversionPattern::OpConversionPattern;
1127
1128 LogicalResult
1129 matchAndRewrite(vector::BroadcastOp bcastOp, OpAdaptor adaptor,
1130 ConversionPatternRewriter &rewriter) const override {
1131
1132 if (adaptor.getSource().getDefiningOp<vector::ExtractOp>())
1133 return failure();
1134
1135 auto resultType = cast<VectorType>(bcastOp.getResult().getType());
1136 auto flatResultType = getFlattenedVectorType(resultType);
1137 Type scalarType = resultType.getElementType();
1138 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
1139 unsigned laneSize = getVectorLaneSize(resultType);
1140 auto src = bcastOp.getSource();
1141
1142 if (laneSize * elWidth == 512) {
1143 Value newOp = aievec::BroadcastScalarOp::create(
1144 rewriter, bcastOp.getLoc(), flatResultType, src);
1145 if (resultType != flatResultType)
1146 newOp = vector::ShapeCastOp::create(rewriter, bcastOp.getLoc(),
1147 resultType, newOp);
1148 rewriter.replaceOp(bcastOp, newOp);
1149 return success();
1150 }
1151
1152 if (laneSize * elWidth == 256) {
1153 VectorType vecType = createVectorType(512 / elWidth, scalarType);
1154 auto aieBcastOp = aievec::BroadcastScalarOp::create(
1155 rewriter, bcastOp.getLoc(), vecType, src);
1156 Value newOp =
1157 aievec::ExtOp::create(rewriter, bcastOp.getLoc(), flatResultType,
1158 aieBcastOp.getResult(), 0);
1159 if (resultType != flatResultType)
1160 newOp = vector::ShapeCastOp::create(rewriter, bcastOp.getLoc(),
1161 resultType, newOp);
1162 rewriter.replaceOp(bcastOp, newOp);
1163 return success();
1164 }
1165
1166 if (laneSize * elWidth == 1024) {
1167 VectorType vecType = createVectorType(512 / elWidth, scalarType);
1168 auto aieBcastOp = aievec::BroadcastScalarOp::create(
1169 rewriter, bcastOp.getLoc(), vecType, src);
1170 Value newOp = aievec::ConcatOp::create(
1171 rewriter, bcastOp.getLoc(), flatResultType,
1172 SmallVector<Value>({aieBcastOp.getResult(), aieBcastOp.getResult()}));
1173 if (resultType != flatResultType)
1174 newOp = vector::ShapeCastOp::create(rewriter, bcastOp.getLoc(),
1175 resultType, newOp);
1176 rewriter.replaceOp(bcastOp, newOp);
1177 return success();
1178 }
1179
1180 return failure();
1181 }
1182};
1183
1184// This pattern replaces `arith.muli`+`arith.addi` on vectors with
1185// `aievec.mac_elem`. This pattern works for AIE2.
1187 : OpConversionPattern<arith::AddIOp> {
1188 using OpConversionPattern::OpConversionPattern;
1189
1191 unsigned shiftParam = 0)
1192 : OpConversionPattern(context), shiftParam(shiftParam) {}
1193
1194 LogicalResult
1195 matchAndRewrite(arith::AddIOp addOp, OpAdaptor adaptor,
1196 ConversionPatternRewriter &rewriter) const override {
1197 // Verify it's a vector operation
1198 auto resultType = dyn_cast<VectorType>(addOp.getType());
1199 if (!resultType)
1200 return failure();
1201
1202 // Verify it can be replaced by a MAC
1203 auto res =
1204 extractMACOperandsFromAddOperands(adaptor.getLhs(), adaptor.getRhs());
1205 if (!res)
1206 return failure();
1207 auto [lhs, rhs, acc] = *res;
1208
1209 // Verify the vector type is supported by AIE2
1210 unsigned resultElWidth =
1211 resultType.getElementType().getIntOrFloatBitWidth();
1212 unsigned laneSize = getVectorLaneSize(resultType);
1213
1214 if ((laneSize != 32 || resultElWidth != 16) &&
1215 (laneSize != 16 || resultElWidth != 32))
1216 return failure();
1217
1218 Type accType = getVectorOpDestType(cast<VectorType>(acc.getType()),
1219 /*AIE2 =*/true);
1220 auto upsOp = aievec::UPSOp::create(rewriter, addOp.getLoc(), accType, acc,
1221 shiftParam);
1222 auto fmaElemOp = aievec::FMAElemOp::create(
1223 rewriter, addOp.getLoc(), accType, lhs, rhs, upsOp.getResult(),
1224 /*fmsub=*/false);
1225
1226 auto shiftParamOp = arith::ConstantOp::create(
1227 rewriter, addOp.getLoc(), rewriter.getI32IntegerAttr(shiftParam));
1228 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
1229 addOp, resultType, fmaElemOp.getResult(), shiftParamOp.getResult());
1230
1231 return success();
1232 }
1233
1234 unsigned shiftParam;
1235};
1236
1237// Lower a single 16-lane bf16 FMA half: UPS -> FMAElem -> SRS.
1238static Value lowerBF16FMAHalf(Value lhs, Value rhs, Value acc,
1239 unsigned shiftParam, Location loc,
1240 ConversionPatternRewriter &rewriter) {
1241 auto f32AccType = VectorType::get({16}, rewriter.getF32Type());
1242 auto upsOp =
1243 aievec::UPSOp::create(rewriter, loc, f32AccType, acc, shiftParam);
1244 auto fmaElemOp = aievec::FMAElemOp::create(rewriter, loc, f32AccType, lhs,
1245 rhs, upsOp.getResult(),
1246 /*fmsub=*/false);
1247 auto shiftParamOp = arith::ConstantOp::create(
1248 rewriter, loc, rewriter.getI32IntegerAttr(shiftParam));
1249 auto srsOp =
1250 aievec::SRSOp::create(rewriter, loc, cast<VectorType>(lhs.getType()),
1251 fmaElemOp.getResult(), shiftParamOp.getResult());
1252 return srsOp.getResult();
1253}
1254
1255// Convert `vector.fma` to `aievec.mac_elem`. Supported operand types:
1256// `vector<16xf32>`, `vector<16xbf16>`, and `vector<32xbf16>` (split into two
1257// 16-lane FMAs). In the case of vectors with
1258// `f32` elemental type, this pattern will try to match `bf16` to `f32`
1259// widening ops in the `lhs` and `rhs` operands, or fail otherwise.
1260// TODO: When sign extensions are not found, a conversion from `f32` to `bf16`
1261// TODO: can be inserted to emulate `f32` fma with `bf16` logic.
1263 : OpConversionPattern<vector::FMAOp> {
1264 using OpConversionPattern::OpConversionPattern;
1265
1267 unsigned shiftParam = 0)
1268 : OpConversionPattern(context), shiftParam(shiftParam) {}
1269
1270 LogicalResult
1271 matchAndRewrite(vector::FMAOp fmaOp, OpAdaptor adaptor,
1272 ConversionPatternRewriter &rewriter) const override {
1273 // Verify the vector type is supported by AIE2
1274 auto resVecTy = cast<VectorType>(fmaOp.getType());
1275 auto resElemTy = resVecTy.getElementType();
1276 unsigned numElems = getVectorLaneSize(resVecTy);
1277
1278 // Only support f32 and bf16 with 16 or 32 lanes.
1279 if ((!resElemTy.isF32() && !resElemTy.isBF16()) ||
1280 (numElems != 16 && numElems != 32))
1281 return rewriter.notifyMatchFailure(
1282 fmaOp, "Unsupported operand types in vector.fma lowering.");
1283
1284 Value lhs = adaptor.getLhs();
1285 Value rhs = adaptor.getRhs();
1286 Value acc = adaptor.getAcc();
1287
1288 if (resElemTy.isF32()) {
1289 if (!holdsBF16(lhs) || !holdsBF16(rhs))
1290 return rewriter.notifyMatchFailure(
1291 fmaOp, "vector.fma operands are f32 and do not hold bf16 values; "
1292 "can't lower to aievec.");
1293 Location loc = fmaOp.getLoc();
1294 lhs = getBF16Source(rewriter, loc, lhs);
1295 rhs = getBF16Source(rewriter, loc, rhs);
1296 if (numElems == 16) {
1297 rewriter.replaceOpWithNewOp<aievec::FMAElemOp>(
1298 fmaOp, acc.getType(), lhs, rhs, acc, /*fmsub=*/false);
1299 return success();
1300 }
1301 // 32 lanes: two 16-lane halves.
1302 auto bf16HalfTy = createVectorType(16, rewriter.getBF16Type());
1303 auto f32HalfTy = createVectorType(16, rewriter.getF32Type());
1304 SmallVector<Value> halves;
1305 for (int idx : {0, 1}) {
1306 Value lhsHalf =
1307 aievec::ExtOp::create(rewriter, loc, bf16HalfTy, lhs, idx);
1308 Value rhsHalf =
1309 aievec::ExtOp::create(rewriter, loc, bf16HalfTy, rhs, idx);
1310 Value accHalf =
1311 aievec::ExtOp::create(rewriter, loc, f32HalfTy, acc, idx);
1312 halves.push_back(aievec::FMAElemOp::create(rewriter, loc, f32HalfTy,
1313 lhsHalf, rhsHalf, accHalf,
1314 /*fmsub=*/false));
1315 }
1316 rewriter.replaceOpWithNewOp<aievec::ConcatOp>(fmaOp, resVecTy, halves);
1317 return success();
1318 }
1319
1320 // Handle vector<32xbf16> by splitting into two vector<16xbf16> FMAs
1321 if (numElems == 32 && resElemTy.isBF16()) {
1322 VectorType halfType = createVectorType(16, resElemTy);
1323 unsigned localShiftParam = shiftParam;
1324
1325 splitWideVectorOp<vector::FMAOp>(
1326 fmaOp, {lhs, rhs, acc}, halfType, resVecTy, rewriter,
1327 [localShiftParam](ArrayRef<std::pair<Value, Value>> halfInputs,
1328 Location loc, ConversionPatternRewriter &rewriter) {
1329 auto [lhsLow, lhsHigh] = halfInputs[0];
1330 auto [rhsLow, rhsHigh] = halfInputs[1];
1331 auto [accLow, accHigh] = halfInputs[2];
1332
1333 Value lowResult = lowerBF16FMAHalf(lhsLow, rhsLow, accLow,
1334 localShiftParam, loc, rewriter);
1335 Value highResult = lowerBF16FMAHalf(lhsHigh, rhsHigh, accHigh,
1336 localShiftParam, loc, rewriter);
1337 return std::make_pair(lowResult, highResult);
1338 });
1339 return success();
1340 }
1341
1342 acc = aievec::UPSOp::create(rewriter, fmaOp.getLoc(),
1343 VectorType::get({16}, rewriter.getF32Type()),
1344 acc, shiftParam);
1345 Value newOp =
1346 aievec::FMAElemOp::create(rewriter, fmaOp.getLoc(), acc.getType(), lhs,
1347 rhs, acc, /*fmsub=*/false);
1348 auto shiftParamOp = arith::ConstantOp::create(
1349 rewriter, fmaOp.getLoc(), rewriter.getI32IntegerAttr(shiftParam));
1350 newOp = aievec::SRSOp::create(rewriter, fmaOp.getLoc(), resVecTy, newOp,
1351 shiftParamOp);
1352
1353 rewriter.replaceOp(fmaOp, newOp);
1354
1355 return success();
1356 }
1357
1358 unsigned shiftParam;
1359};
1360
1361// AIE2P multiplies 32 bf16 lanes into an f32 accumulator in one op. Lower a
1362// 32-lane f32 `vector.fma` with bf16 inputs to it.
1364 : OpConversionPattern<vector::FMAOp> {
1365 using OpConversionPattern::OpConversionPattern;
1366
1367 LogicalResult
1368 matchAndRewrite(vector::FMAOp fmaOp, OpAdaptor adaptor,
1369 ConversionPatternRewriter &rewriter) const override {
1370 auto resVecTy = cast<VectorType>(fmaOp.getType());
1371 if (!resVecTy.getElementType().isF32() || resVecTy.getRank() != 1 ||
1372 resVecTy.getNumElements() != 32 || !holdsBF16(adaptor.getLhs()) ||
1373 !holdsBF16(adaptor.getRhs()))
1374 return failure();
1375 Location loc = fmaOp.getLoc();
1376 Value lhs = getBF16Source(rewriter, loc, adaptor.getLhs());
1377 Value rhs = getBF16Source(rewriter, loc, adaptor.getRhs());
1378 rewriter.replaceOpWithNewOp<aievec::FMAElemOp>(
1379 fmaOp, resVecTy, lhs, rhs, adaptor.getAcc(), /*fmsub=*/false);
1380 return success();
1381 }
1382};
1383
1384// The multiply-only form of the pattern above, for 32 or 64 lanes.
1386 : OpConversionPattern<arith::MulFOp> {
1387 using OpConversionPattern::OpConversionPattern;
1388
1389 LogicalResult
1390 matchAndRewrite(arith::MulFOp mulOp, OpAdaptor adaptor,
1391 ConversionPatternRewriter &rewriter) const override {
1392 auto resVecTy = dyn_cast<VectorType>(mulOp.getType());
1393 if (!resVecTy || !resVecTy.getElementType().isF32() ||
1394 resVecTy.getRank() != 1 ||
1395 (resVecTy.getNumElements() != 32 && resVecTy.getNumElements() != 64) ||
1396 !holdsBF16(adaptor.getLhs()) || !holdsBF16(adaptor.getRhs()))
1397 return failure();
1398 Location loc = mulOp.getLoc();
1399 Value lhs = getBF16Source(rewriter, loc, adaptor.getLhs());
1400 Value rhs = getBF16Source(rewriter, loc, adaptor.getRhs());
1401 auto mulElemOp =
1402 aievec::MulElemOp::create(rewriter, loc, resVecTy, lhs, rhs);
1403 rewriter.replaceOpWithNewOp<aievec::CastOp>(mulOp, resVecTy, mulElemOp,
1404 /*isResAcc=*/false);
1405 return success();
1406 }
1407};
1408
1409// This pattern fuses `arith.mulf` + `arith.addf` on bf16 vectors into
1410// `aievec.mac_elem` (float FMA). This pattern works for AIE2.
1412 : OpConversionPattern<arith::AddFOp> {
1413 using OpConversionPattern::OpConversionPattern;
1414
1416 unsigned shiftParam = 0)
1417 : OpConversionPattern(context), shiftParam(shiftParam) {}
1418
1419 LogicalResult
1420 matchAndRewrite(arith::AddFOp addOp, OpAdaptor adaptor,
1421 ConversionPatternRewriter &rewriter) const override {
1422 // Verify it's a vector operation
1423 auto resultType = dyn_cast<VectorType>(addOp.getType());
1424 if (!resultType)
1425 return failure();
1426
1427 // Only handle bf16 element type
1428 auto elemType = resultType.getElementType();
1429 if (!elemType.isBF16())
1430 return failure();
1431
1432 // Verify it can be replaced by an FMA
1433 auto res =
1434 extractFMACOperandsFromAddOperands(adaptor.getLhs(), adaptor.getRhs());
1435 if (!res)
1436 return failure();
1437 auto [lhs, rhs, acc] = *res;
1438
1439 unsigned laneSize = getVectorLaneSize(resultType);
1440
1441 // Handle vector<32xbf16> by splitting into two vector<16xbf16> FMAs
1442 if (laneSize == 32) {
1443 VectorType halfType = createVectorType(16, elemType);
1444 unsigned localShiftParam = shiftParam;
1445
1446 splitWideVectorOp<arith::AddFOp>(
1447 addOp, {lhs, rhs, acc}, halfType, resultType, rewriter,
1448 [localShiftParam](ArrayRef<std::pair<Value, Value>> halfInputs,
1449 Location loc, ConversionPatternRewriter &rewriter) {
1450 auto [lhsLow, lhsHigh] = halfInputs[0];
1451 auto [rhsLow, rhsHigh] = halfInputs[1];
1452 auto [accLow, accHigh] = halfInputs[2];
1453
1454 Value lowResult = lowerBF16FMAHalf(lhsLow, rhsLow, accLow,
1455 localShiftParam, loc, rewriter);
1456 Value highResult = lowerBF16FMAHalf(lhsHigh, rhsHigh, accHigh,
1457 localShiftParam, loc, rewriter);
1458 return std::make_pair(lowResult, highResult);
1459 });
1460 return success();
1461 }
1462
1463 // Handle vector<16xbf16>
1464 if (laneSize != 16)
1465 return failure();
1466
1467 auto f32AccType = VectorType::get({16}, rewriter.getF32Type());
1468 auto upsOp = aievec::UPSOp::create(rewriter, addOp.getLoc(), f32AccType,
1469 acc, shiftParam);
1470 auto fmaElemOp = aievec::FMAElemOp::create(
1471 rewriter, addOp.getLoc(), f32AccType, lhs, rhs, upsOp.getResult(),
1472 /*fmsub=*/false);
1473
1474 auto shiftParamOp = arith::ConstantOp::create(
1475 rewriter, addOp.getLoc(), rewriter.getI32IntegerAttr(shiftParam));
1476 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
1477 addOp, resultType, fmaElemOp.getResult(), shiftParamOp.getResult());
1478
1479 return success();
1480 }
1481
1482 unsigned shiftParam;
1483};
1484
1485// This pattern replaces `arith.mulf` on vectors with
1486// `aievec.mul_elem`. This pattern works for AIE2.
1488 : OpConversionPattern<arith::MulFOp> {
1489 using OpConversionPattern::OpConversionPattern;
1490
1492 unsigned shiftParam = 0)
1493 : OpConversionPattern(context), shiftParam(shiftParam) {}
1494
1495 LogicalResult
1496 matchAndRewrite(arith::MulFOp mulOp, OpAdaptor adaptor,
1497 ConversionPatternRewriter &rewriter) const override {
1498 // Verify it's a vector operation
1499 auto resultType = dyn_cast<VectorType>(mulOp.getType());
1500 if (!resultType)
1501 return failure();
1502
1503 // Skip standalone mul conversion when this MulFOp feeds into an AddFOp as
1504 // its sole user. ConvertMulAddFToAIEVecFMAElemOpPattern will fuse the
1505 // multiply and add into a single aievec.mac_elem.
1506 // Only defer to FMA for bf16 where the FMA pattern is registered.
1507 auto isAddOp = [&](Operation *op) { return isa<arith::AddFOp>(op); };
1508 if (resultType.getElementType().isBF16() && mulOp->hasOneUse() &&
1509 llvm::any_of(mulOp->getUsers(), isAddOp))
1510 return failure();
1511
1512 unsigned resultElWidth =
1513 resultType.getElementType().getIntOrFloatBitWidth();
1514
1515 unsigned laneSize = getVectorLaneSize(resultType);
1516
1517 // f32 product of bf16 inputs, in 16-lane pieces.
1518 if (resultType.getElementType().isF32() &&
1519 (laneSize == 16 || laneSize == 32) && holdsBF16(adaptor.getLhs()) &&
1520 holdsBF16(adaptor.getRhs())) {
1521 Location loc = mulOp.getLoc();
1522 Value lhs = getBF16Source(rewriter, loc, adaptor.getLhs());
1523 Value rhs = getBF16Source(rewriter, loc, adaptor.getRhs());
1524 auto bf16HalfTy = createVectorType(16, rewriter.getBF16Type());
1525 auto f32HalfTy = createVectorType(16, rewriter.getF32Type());
1526 auto mul16 = [&](Value l, Value r) -> Value {
1527 auto mulElemOp =
1528 aievec::MulElemOp::create(rewriter, loc, f32HalfTy, l, r);
1529 return aievec::CastOp::create(rewriter, loc, f32HalfTy, mulElemOp,
1530 /*isResAcc=*/false);
1531 };
1532 if (laneSize == 16) {
1533 rewriter.replaceOp(mulOp, mul16(lhs, rhs));
1534 return success();
1535 }
1536 SmallVector<Value> halves;
1537 for (int idx : {0, 1})
1538 halves.push_back(
1539 mul16(aievec::ExtOp::create(rewriter, loc, bf16HalfTy, lhs, idx),
1540 aievec::ExtOp::create(rewriter, loc, bf16HalfTy, rhs, idx)));
1541 rewriter.replaceOpWithNewOp<aievec::ConcatOp>(mulOp, resultType, halves);
1542 return success();
1543 }
1544
1545 // Handle vector<32xbf16> by splitting into two vector<16xbf16> operations
1546 if (laneSize == 32 && resultElWidth == 16) {
1547 VectorType halfType = createVectorType(16, resultType.getElementType());
1548 unsigned localShiftParam = shiftParam;
1549
1550 splitWideBinaryVectorOp<arith::MulFOp>(
1551 mulOp, adaptor.getLhs(), adaptor.getRhs(), halfType, resultType,
1552 rewriter,
1553 [localShiftParam](Value lhsHalf, Value rhsHalf, Location loc,
1554 ConversionPatternRewriter &rewriter) -> Value {
1555 Type accType = getVectorOpDestType(
1556 cast<VectorType>(lhsHalf.getType()), /*AIE2 =*/true);
1557 auto mulElemOp = aievec::MulElemOp::create(rewriter, loc, accType,
1558 lhsHalf, rhsHalf);
1559 auto shiftParamOp = arith::ConstantOp::create(
1560 rewriter, loc, rewriter.getI32IntegerAttr(localShiftParam));
1561 auto srsOp = aievec::SRSOp::create(
1562 rewriter, loc, cast<VectorType>(lhsHalf.getType()),
1563 mulElemOp.getResult(), shiftParamOp.getResult());
1564 return srsOp.getResult();
1565 });
1566 return success();
1567 }
1568
1569 // bfloat16 and float type (laneSize == 16)
1570 if (laneSize != 16 || (resultElWidth != 16 && resultElWidth != 32))
1571 return failure();
1572
1573 // Decide the accType for aievec.mul_elem based on mulOp's lhs & rhs
1574 auto lval = adaptor.getLhs();
1575 auto rval = adaptor.getRhs();
1576 lval = getSourceOfWideningOp(lval).value_or(lval);
1577 rval = getSourceOfWideningOp(rval).value_or(rval);
1578 auto lSrcType = cast<VectorType>(lval.getType());
1579 auto rSrcType = cast<VectorType>(rval.getType());
1580 unsigned lBitWidth = lSrcType.getElementType().getIntOrFloatBitWidth();
1581 unsigned rBitWidth = rSrcType.getElementType().getIntOrFloatBitWidth();
1582 Type accType = getVectorOpDestType(lSrcType, /*AIE2 =*/true);
1583 if (rBitWidth > lBitWidth) {
1584 accType = getVectorOpDestType(rSrcType, /*AIE2 =*/true);
1585 }
1586 // Only support the same lhs/rhs type at the moment
1587 if (lSrcType != rSrcType) {
1588 return failure();
1589 }
1590
1591 // Prepare lhr/rhs for the aievec.mul_elem op
1592 unsigned bitWidth = (rBitWidth > lBitWidth) ? rBitWidth : lBitWidth;
1593 Type srcElemType = (rBitWidth > lBitWidth) ? rSrcType.getElementType()
1594 : lSrcType.getElementType();
1595 unsigned numLanes = 0;
1596 // float16/32 and int32 both land on 16 lanes -- genuinely distinct
1597 // source types happen to share a lane count, not a copy-paste clone.
1598 if (isa<FloatType>(srcElemType) &&
1599 (bitWidth == 16 || bitWidth == 32)) { // NOLINT(bugprone-branch-clone)
1600 numLanes = 16;
1601 } else if (isa<IntegerType>(srcElemType) &&
1602 (bitWidth == 8 || bitWidth == 16)) {
1603 numLanes = 32;
1604 } else if (isa<IntegerType>(srcElemType) && (bitWidth == 32)) {
1605 numLanes = 16;
1606 } else {
1607 return failure();
1608 }
1609 VectorType targetInputType = createVectorType(numLanes, srcElemType);
1610 if (targetInputType != lSrcType) {
1611 auto converted = convertValueToTargetTypeAIE2(rewriter, mulOp.getLoc(),
1612 lval, targetInputType);
1613 if (!converted)
1614 return failure();
1615 lval = *converted;
1616 }
1617 if (targetInputType != rSrcType) {
1618 auto converted = convertValueToTargetTypeAIE2(rewriter, mulOp.getLoc(),
1619 rval, targetInputType);
1620 if (!converted)
1621 return failure();
1622 rval = *converted;
1623 }
1624 if (!lval || !rval)
1625 return failure();
1626
1627 // Create an aievec.mul_elem op
1628 auto mulElemOp = aievec::MulElemOp::create(rewriter, mulOp.getLoc(),
1629 accType, lval, rval);
1630
1631 // Create an aievec.cast or an aievec.srs op
1632 auto mulElemResultType = mulElemOp.getType();
1633 auto mulElemResultElWidth =
1634 mulElemResultType.getElementType().getIntOrFloatBitWidth();
1635
1636 if (mulElemResultElWidth == resultElWidth) {
1637 rewriter.replaceOpWithNewOp<aievec::CastOp>(
1638 mulOp, resultType, mulElemOp.getResult(), /*isResAcc*/ false);
1639 } else if (mulElemResultElWidth > resultElWidth) {
1640 auto shiftParamOp = arith::ConstantOp::create(
1641 rewriter, mulOp.getLoc(), rewriter.getI32IntegerAttr(shiftParam));
1642 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
1643 mulOp, resultType, mulElemOp.getResult(), shiftParamOp.getResult());
1644 } else {
1645 return failure();
1646 }
1647
1648 return success();
1649 }
1650
1651 unsigned shiftParam;
1652};
1653
1654// This pattern replaces `arith.muli` on vectors with
1655// `aievec.mul_elem`. This pattern works for AIE2.
1657 : OpConversionPattern<arith::MulIOp> {
1658 using OpConversionPattern::OpConversionPattern;
1659
1661 unsigned shiftParam = 0)
1662 : OpConversionPattern(context), shiftParam(shiftParam) {}
1663
1664 LogicalResult
1665 matchAndRewrite(arith::MulIOp mulOp, OpAdaptor adaptor,
1666 ConversionPatternRewriter &rewriter) const override {
1667 // Verify it's a vector operation
1668 auto resultType = dyn_cast<VectorType>(mulOp.getType());
1669 if (!resultType)
1670 return failure();
1671
1672 // FIXME: Verify it is not a part of MAC
1673 auto isAddOp = [&](Operation *op) { return isa<arith::AddIOp>(op); };
1674 if (mulOp->hasOneUse() && llvm::any_of(mulOp->getUsers(), isAddOp))
1675 return failure();
1676
1677 // Verify the vector type is supported by AIE2
1678 unsigned resultElWidth =
1679 resultType.getElementType().getIntOrFloatBitWidth();
1680 unsigned laneSize = getVectorLaneSize(resultType);
1681
1682 if ((laneSize != 32 || (resultElWidth != 16 && resultElWidth != 8)) &&
1683 ((laneSize != 16 && laneSize != 32) || resultElWidth != 32))
1684 return failure();
1685
1686 // Decide the accType for aievec.mul_elem based on mulOp's lhs & rhs
1687 auto lval = adaptor.getLhs();
1688 auto rval = adaptor.getRhs();
1689
1690 lval = getSourceOfWideningOp(lval).value_or(lval);
1691 rval = getSourceOfWideningOp(rval).value_or(rval);
1692
1693 auto lSrcType = cast<VectorType>(lval.getType());
1694 auto rSrcType = cast<VectorType>(rval.getType());
1695 unsigned lBitWidth = lSrcType.getElementType().getIntOrFloatBitWidth();
1696 unsigned rBitWidth = rSrcType.getElementType().getIntOrFloatBitWidth();
1697 Type accType = getVectorOpDestType(lSrcType, /*AIE2 =*/true);
1698 if (rBitWidth > lBitWidth) {
1699 accType = getVectorOpDestType(rSrcType, /*AIE2 =*/true);
1700 }
1701
1702 // Prepare lhr/rhs for the aievec.mul_elem op
1703 unsigned bitWidth = (rBitWidth > lBitWidth) ? rBitWidth : lBitWidth;
1704 Type srcElemType = (rBitWidth > lBitWidth) ? rSrcType.getElementType()
1705 : lSrcType.getElementType();
1706 unsigned numLanes = 0;
1707 // float16/32 and int32 both land on 16 lanes -- genuinely distinct
1708 // source types happen to share a lane count, not a copy-paste clone.
1709 if (isa<FloatType>(srcElemType) &&
1710 (bitWidth == 16 || bitWidth == 32)) { // NOLINT(bugprone-branch-clone)
1711 numLanes = 16;
1712 } else if (isa<IntegerType>(srcElemType) &&
1713 (bitWidth == 8 || bitWidth == 16)) {
1714 numLanes = 32;
1715 } else if (isa<IntegerType>(srcElemType) && (bitWidth == 32)) {
1716 numLanes = 16;
1717 } else {
1718 return failure();
1719 }
1720 VectorType targetInputType = createVectorType(numLanes, srcElemType);
1721 if (targetInputType != lSrcType) {
1722 auto converted = convertValueToTargetTypeAIE2(rewriter, mulOp.getLoc(),
1723 lval, targetInputType);
1724 if (!converted)
1725 return failure();
1726 lval = *converted;
1727 }
1728 if (targetInputType != rSrcType) {
1729 auto converted = convertValueToTargetTypeAIE2(rewriter, mulOp.getLoc(),
1730 rval, targetInputType);
1731 if (!converted)
1732 return failure();
1733 rval = *converted;
1734 }
1735 if (!lval || !rval)
1736 return failure();
1737
1738 // Create an aievec.mul_elem op
1739 auto mulElemOp = aievec::MulElemOp::create(rewriter, mulOp.getLoc(),
1740 accType, lval, rval);
1741
1742 // Create an aievec.cast or an aievec.srs op
1743 auto mulElemResultType = mulElemOp.getType();
1744 auto mulElemResultElWidth =
1745 mulElemResultType.getElementType().getIntOrFloatBitWidth();
1746
1747 if (mulElemResultElWidth == resultElWidth) {
1748 rewriter.replaceOpWithNewOp<aievec::CastOp>(
1749 mulOp, resultType, mulElemOp.getResult(), /*isResAcc*/ false);
1750 } else if (mulElemResultElWidth > resultElWidth) {
1751 auto shiftParamOp = arith::ConstantOp::create(
1752 rewriter, mulOp.getLoc(), rewriter.getI32IntegerAttr(shiftParam));
1753 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
1754 mulOp, resultType, mulElemOp.getResult(), shiftParamOp.getResult());
1755 } else {
1756 return failure();
1757 }
1758
1759 return success();
1760 }
1761
1762 unsigned shiftParam;
1763};
1764
1765// This pattern folds an extract + broadcast feeding into an
1766// `aievec::aie1::FMAOp` into the op, using the shuffle attributes.
1767struct FoldSplatToFMAOp : OpConversionPattern<aievec::aie1::FMAOp> {
1768 using OpConversionPattern::OpConversionPattern;
1769
1770 LogicalResult
1771 matchAndRewrite(aievec::aie1::FMAOp fmaOp, OpAdaptor adaptor,
1772 ConversionPatternRewriter &rewriter) const override {
1773 auto concatOp =
1774 dyn_cast<aievec::ConcatOp>(adaptor.getLhs().getDefiningOp());
1775 if (!concatOp)
1776 return failure();
1777 vector::BroadcastOp bcastOp = nullptr;
1778 auto *concatDefOp = concatOp.getSources()[0].getDefiningOp();
1779 if (concatDefOp)
1780 bcastOp = dyn_cast<vector::BroadcastOp>(concatDefOp);
1781 Value lhs = adaptor.getRhs();
1782 if (!bcastOp) {
1783 bcastOp = dyn_cast<vector::BroadcastOp>(adaptor.getRhs().getDefiningOp());
1784 if (!bcastOp)
1785 return failure();
1786 lhs = concatOp.getSources()[0];
1787 }
1788 auto extOp =
1789 dyn_cast<vector::ExtractOp>(bcastOp.getSource().getDefiningOp());
1790 if (!extOp)
1791 return failure();
1792
1793 auto rhs = extOp.getSource();
1794 auto concatVecType = cast<VectorType>(concatOp.getResult().getType());
1795 auto zvec =
1796 arith::ConstantOp::create(rewriter, concatOp.getLoc(), lhs.getType(),
1797 rewriter.getZeroAttr(lhs.getType()));
1798 auto lhsX2 =
1799 aievec::ConcatOp::create(rewriter, concatOp.getLoc(), concatVecType,
1800 SmallVector<Value, 2>({lhs, zvec}))
1801 .getResult();
1802 // XXX: We assume a 1D vector
1803 auto pos = extOp.getStaticPosition();
1804 int64_t zstart = pos[0];
1805 auto fmaOpAttr = buildFMAOpSplatAttrForElemTy(fmaOp, zstart);
1806 aievec::aie1::FMAOp::Properties fmaProps;
1807 if (failed(aievec::aie1::FMAOp::setPropertiesFromAttr(
1808 fmaProps, rewriter.getDictionaryAttr(fmaOpAttr),
1809 [&]() { return fmaOp.emitError(); })))
1810 return failure();
1811 rewriter.replaceOpWithNewOp<aievec::aie1::FMAOp>(
1812 fmaOp, TypeRange({fmaOp.getResult().getType()}),
1813 ValueRange({lhsX2, rhs, adaptor.getAcc()}), fmaProps);
1814
1815 return success();
1816 }
1817};
1818
1820 : OpConversionPattern<aievec::aie1::AddOp> {
1821 using OpConversionPattern::OpConversionPattern;
1822
1823 LogicalResult
1824 matchAndRewrite(aievec::aie1::AddOp addOp, OpAdaptor adaptor,
1825 ConversionPatternRewriter &rewriter) const override {
1826 auto vecType = cast<VectorType>(addOp.getType());
1827
1828 auto res =
1829 extractMACOperandsFromAddOperands(adaptor.getLhs(), adaptor.getRhs());
1830 if (!res)
1831 return failure();
1832 auto [lhs, rhs, acc] = *res;
1833
1834 SmallVector<int64_t, 4> concatVecShape(vecType.getShape().begin(),
1835 vecType.getShape().end());
1836 concatVecShape[vecType.getRank() - 1] *= 2;
1837 auto concatVecType =
1838 VectorType::get(concatVecShape, vecType.getElementType());
1839 Type accType = getVectorOpDestType(cast<VectorType>(acc.getType()),
1840 /*AIE2 =*/false);
1841 auto lhsX2 =
1842 aievec::ConcatOp::create(rewriter, addOp.getLoc(), concatVecType,
1843 SmallVector<Value, 2>(2, lhs))
1844 .getResult();
1845 auto upsOp = aievec::UPSOp::create(rewriter, addOp.getLoc(), accType, acc);
1846 auto fmaOp = aievec::aie1::FMAOp::create(
1847 rewriter, addOp.getLoc(), accType, lhsX2, rhs, upsOp.getResult(),
1848 /*xstart=*/"", /*xoffsets=*/"", /*xoffsets_hi=*/"", /*xstep=*/"",
1849 /*xsquare=*/"", /*zstart=*/"", /*zoffsets=*/"", /*zoffsets_hi=*/"",
1850 /*zstep=*/"", /*zsquare=*/"", /*fmsub=*/false);
1851 auto shiftParamOp = arith::ConstantOp::create(
1852 rewriter, addOp.getLoc(), rewriter.getI32IntegerAttr(0));
1853 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
1854 addOp, vecType, fmaOp.getResult(), shiftParamOp.getResult());
1855 return success();
1856 }
1857};
1858
1859// This pattern replaces `vector.transfer_read` with `aievec.upd`. Right now,
1860// it performs a naïve direct translation. This needs to be expanded to
1861// support more complex scenarios.
1863 : OpConversionPattern<vector::TransferReadOp> {
1864 using OpConversionPattern::OpConversionPattern;
1865
1866 LowerVectorTransferReadToAIEUPD(MLIRContext *context, int64_t minVectorSize,
1867 int64_t maxVectorSize, int64_t alignment,
1868 int64_t maxLoadSize)
1869 : OpConversionPattern(context), minVectorSize(minVectorSize),
1872
1873 LogicalResult
1874 matchAndRewrite(vector::TransferReadOp readOp, OpAdaptor adaptor,
1875 ConversionPatternRewriter &rewriter) const override {
1876 // Masked loads
1877 if (readOp.getMask())
1878 return readOp.emitError() << "AIE doesn't support masked loads.";
1879
1880 // Non-contiguous loads
1881 AffineMap map = readOp.getPermutationMap();
1882 if (!map.isMinorIdentity())
1883 return failure();
1884
1885 // Splats
1886 if (map.isConstant())
1887 return failure();
1888
1889 // Misaligned accesses
1890 auto vType = readOp.getVectorType();
1892 .value_or(0) != 0)
1893 return failure();
1894
1895 // Invalid vector size.
1896 // We can handle cases where the vector size is:
1897 // 1) the minimum vector size
1898 // 2) a square multiple of the alignment size and up to the maximum
1899 // vector size.
1900 int64_t vSize = vType.getNumElements() * vType.getElementTypeBitWidth();
1901 if (vSize > maxVectorSize ||
1902 (vSize % vectorAlignment && vSize != minVectorSize))
1903 return failure();
1904 // We can deal with linked update instructions when the vector size is
1905 // exactly twice the load size. This could change in future architectures
1906 if (vSize > maxLoadSize && vSize != maxLoadSize * 2)
1907 return failure();
1908 int64_t multiplicity = vSize / vectorAlignment;
1909 if ((vSize > minVectorSize) && std::bitset<8>(multiplicity).count() != 1)
1910 return failure();
1911
1912 auto updOp = xilinx::aievec::UPDOp::create(
1913 rewriter, readOp.getLoc(), vType, adaptor.getBase(),
1914 adaptor.getIndices(), 0, 0, TypedValue<VectorType>(nullptr));
1915 if (vSize > maxLoadSize) {
1916 updOp = xilinx::aievec::UPDOp::create(
1917 rewriter, readOp.getLoc(), vType, adaptor.getBase(),
1918 adaptor.getIndices(), maxLoadSize, 1, updOp.getResult());
1919 }
1920 rewriter.replaceOp(readOp, updOp.getResult());
1921
1922 return success();
1923 }
1924
1926};
1927
1928// XXX: Notice that this template doesn't verify that the vector element type
1929// XXX: is supported by the target architecture.
1930template <typename SrcOpTy, typename DstOpTy>
1931struct OneToOneVectorOpToAIEVecOpPattern : OpConversionPattern<SrcOpTy> {
1932 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
1933 using OpAdaptor = typename SrcOpTy::Adaptor;
1934
1935 LogicalResult
1936 matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor,
1937 ConversionPatternRewriter &rewriter) const override {
1938 rewriter.replaceOpWithNewOp<DstOpTy>(
1939 srcOp, srcOp.getResult().getType(), adaptor.getLhs(), adaptor.getRhs(),
1940 /*xstart=*/"", /*xoffsets=*/"", /*xoffsets_hi=*/"", /*xsquare=*/"",
1941 /*zstart=*/"", /*zoffsets=*/"", /*zoffsets_hi=*/"", /*zsquare=*/"");
1942 return success();
1943 }
1944};
1945
1946struct LowerVectorAddIOpToAIEVecAddOp : OpConversionPattern<arith::AddIOp> {
1947 using OpConversionPattern::OpConversionPattern;
1948
1949 LogicalResult
1950 matchAndRewrite(arith::AddIOp addOp, OpAdaptor adaptor,
1951 ConversionPatternRewriter &rewriter) const override {
1952 auto resType = addOp.getType();
1953 if (!isa<VectorType>(resType))
1954 return failure();
1955
1956 auto lhs = adaptor.getLhs();
1957 auto rhs = adaptor.getRhs();
1958 auto *lhsDefOp = lhs.getDefiningOp();
1959 auto *rhsDefOp = rhs.getDefiningOp();
1960 if ((isa_and_nonnull<arith::MulIOp>(lhsDefOp)) ||
1961 (isa_and_nonnull<arith::MulIOp>(rhsDefOp)))
1962 return failure();
1963
1964 rewriter.replaceOpWithNewOp<aievec::aie1::AddOp>(
1965 addOp, resType, lhs, rhs,
1966 /*xstart=*/"", /*xoffsets=*/"", /*xoffsets_hi=*/"", /*xsquare=*/"",
1967 /*zstart=*/"", /*zoffsets=*/"", /*zoffsets_hi=*/"", /*zsquare=*/"");
1968 return success();
1969 }
1970};
1971
1980
1981struct LowerVectorMulIOpToAIEVecMulOp : OpConversionPattern<arith::MulIOp> {
1982 using OpConversionPattern::OpConversionPattern;
1983 LogicalResult
1984 matchAndRewrite(arith::MulIOp mulOp, OpAdaptor adaptor,
1985 ConversionPatternRewriter &rewriter) const override {
1986 auto resTy = dyn_cast<VectorType>(mulOp.getType());
1987 if (!resTy)
1988 return failure();
1989 auto accTy = getVectorOpDestType(resTy, /*AIE2 =*/false);
1990 auto newMulOp = aievec::aie1::MulOp::create(
1991 rewriter, mulOp.getLoc(), accTy, adaptor.getLhs(), adaptor.getRhs());
1992 auto shiftParamOp = arith::ConstantOp::create(
1993 rewriter, mulOp.getLoc(), rewriter.getI32IntegerAttr(0));
1994 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
1995 mulOp, resTy, newMulOp.getResult(), shiftParamOp.getResult());
1996 return success();
1997 }
1998};
1999
2000template <typename SrcOpTy, typename DstOpTy>
2002 : OpConversionPattern<SrcOpTy> {
2003 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
2004 using OpAdaptor = typename SrcOpTy::Adaptor;
2005
2006 LogicalResult
2007 matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor,
2008 ConversionPatternRewriter &rewriter) const override {
2009 VectorType resultType = dyn_cast<VectorType>(srcOp.getType());
2010 if (!resultType)
2011 return failure();
2012
2013 // A set recording the vector lane size and element width we are supporting
2014 // for AIE2.
2015 llvm::SmallSet<std::pair<unsigned, signed>, 16> laneSizeElWidthPairSet;
2016 laneSizeElWidthPairSet.insert({64, 8});
2017 laneSizeElWidthPairSet.insert({32, 16});
2018 laneSizeElWidthPairSet.insert({16, 32});
2019 laneSizeElWidthPairSet.insert({32, 32});
2020
2021 auto lhs = adaptor.getLhs();
2022 auto rhs = adaptor.getRhs();
2023 auto lhsDefOp = lhs.getDefiningOp();
2024 auto rhsDefOp = rhs.getDefiningOp();
2025 // Check if this is part of a MAC/FMA pattern (mul + add).
2026 // We only skip conversion if BOTH operands could potentially be part of an
2027 // FMA pattern (i.e., neither is a constant). Constants can never be the
2028 // multiply result in an FMA, so we should allow conversion in those cases.
2029 bool lhsIsMul = lhsDefOp && (isa<arith::MulIOp>(lhsDefOp) ||
2030 isa<arith::MulFOp>(lhsDefOp));
2031 bool rhsIsMul = rhsDefOp && (isa<arith::MulIOp>(rhsDefOp) ||
2032 isa<arith::MulFOp>(rhsDefOp));
2033 bool lhsIsConst = isa_and_nonnull<arith::ConstantOp>(lhsDefOp);
2034 bool rhsIsConst = isa_and_nonnull<arith::ConstantOp>(rhsDefOp);
2035
2036 // Defer to FMA/MAC patterns when a multiply feeds into add, UNLESS the
2037 // element type is f32 (where no FMA pattern exists). For bf16 and integer
2038 // types, FMA/MAC patterns handle the fusion.
2039 if (!resultType.getElementType().isF32() &&
2040 ((lhsIsMul && !rhsIsConst) || (rhsIsMul && !lhsIsConst)))
2041 return failure();
2042
2043 Type scalarType = resultType.getElementType();
2044 unsigned resultElWidth = scalarType.getIntOrFloatBitWidth();
2045 unsigned laneSize = getVectorLaneSize(resultType);
2046
2047 // Integer cases
2048 if (isa<IntegerType>(scalarType)) {
2049 if (!laneSizeElWidthPairSet.count(
2050 std::make_pair(laneSize, resultElWidth)))
2051 return failure();
2052
2053 // If the ops are defined without extension ops and with supported data
2054 // type, the arith::AddI or arith::SubI can be directly replaced with
2055 // aievec::AddElem or aievec::SubElem.
2056 if (!lhsDefOp && !rhsDefOp) {
2057 if (laneSize * resultElWidth == 512) {
2058 rewriter.replaceOpWithNewOp<DstOpTy>(srcOp, srcOp.getType(), lhs,
2059 rhs);
2060 return success();
2061 }
2062 return genAddElemAIE2<SrcOpTy, DstOpTy>(rewriter, lhs, rhs, resultType,
2063 srcOp);
2064 }
2065
2066 // If element width is 32, we need to consider sign extension cases
2067 if (resultElWidth == 32) {
2068 auto lhsExt = getSourceOfWideningOp(lhs).value_or(nullptr);
2069 auto rhsExt = getSourceOfWideningOp(rhs).value_or(nullptr);
2070
2071 if (!lhsExt && !rhsExt) {
2072 if (laneSize * resultElWidth == 512) {
2073 rewriter.replaceOpWithNewOp<DstOpTy>(srcOp, srcOp.getType(), lhs,
2074 rhs);
2075 return success();
2076 }
2077 return genAddElemAIE2<SrcOpTy, DstOpTy>(rewriter, lhs, rhs,
2078 resultType, srcOp);
2079 }
2080
2081 if (lhsExt && rhsExt) {
2082 auto lval = lhsExt;
2083 auto rval = rhsExt;
2084 VectorType lSrcType = cast<VectorType>(lval.getType());
2085
2086 Type accType = getVectorOpDestType(lSrcType, /*AIE2 =*/true);
2087 auto lUpsOp =
2088 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, lval);
2089 auto rUpsOp =
2090 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, rval);
2091 auto elemOp = DstOpTy::create(
2092 rewriter, srcOp.getLoc(), lUpsOp->getResult(0).getType(),
2093 lUpsOp->getResult(0), rUpsOp->getResult(0));
2094 rewriter.replaceOpWithNewOp<aievec::CastOp>(
2095 srcOp, srcOp.getType(), elemOp.getResult(), /*isResAcc*/ false);
2096 return success();
2097 }
2098
2099 if (!lhsExt || !rhsExt) {
2100 auto lval = lhsExt ? lhsExt : lhs;
2101 auto rval = rhsExt ? rhsExt : rhs;
2102 auto extVal = lhsExt ? lval : rval;
2103 VectorType vType = cast<VectorType>(extVal.getType());
2104 unsigned bitWidth = vType.getElementType().getIntOrFloatBitWidth();
2105
2106 if (bitWidth != 8 && bitWidth != 16) {
2107 return genAddElemAIE2<SrcOpTy, DstOpTy>(rewriter, lhs, rhs,
2108 resultType, srcOp);
2109 }
2110
2111 if (bitWidth * laneSize != 256) {
2112 return genAddElemAIE2<SrcOpTy, DstOpTy>(rewriter, lhs, rhs,
2113 resultType, srcOp);
2114 }
2115
2116 Type accType = nullptr;
2117
2118 if (bitWidth == 8) {
2119 accType = getVectorOpDestType(vType, /*AIE2 =*/true);
2120 Value valToUps = lhsExt ? lval : rval;
2121 Value valToCast = lhsExt ? rval : lval;
2122 auto upsOp = aievec::UPSOp::create(rewriter, srcOp.getLoc(),
2123 accType, valToUps);
2124 auto castOp =
2125 aievec::CastOp::create(rewriter, srcOp.getLoc(), resultType,
2126 valToCast, /*isResAcc*/ true);
2127 Value lhsToElemOp =
2128 lhsExt ? upsOp->getResult(0) : castOp->getResult(0);
2129 Value rhsToElemOp =
2130 lhsExt ? castOp->getResult(0) : upsOp->getResult(0);
2131 auto elemOp = DstOpTy::create(rewriter, srcOp.getLoc(),
2132 upsOp->getResult(0).getType(),
2133 lhsToElemOp, rhsToElemOp);
2134 rewriter.replaceOpWithNewOp<aievec::CastOp>(
2135 srcOp, srcOp.getType(), elemOp.getResult(), /*isResAcc*/ false);
2136 return success();
2137 }
2138
2139 if (bitWidth == 16) {
2140 accType = getVectorOpDestType(resultType, /*AIE2 =*/true);
2141 auto lUpsOp =
2142 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, lval);
2143 auto rUpsOp =
2144 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, rval);
2145
2146 auto elemOp = DstOpTy::create(
2147 rewriter, srcOp.getLoc(), lUpsOp->getResult(0).getType(),
2148 lUpsOp->getResult(0), rUpsOp->getResult(0));
2149
2150 auto shiftParamOp = arith::ConstantOp::create(
2151 rewriter, srcOp.getLoc(), rewriter.getI32IntegerAttr(0));
2152 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
2153 srcOp, srcOp.getType(), elemOp.getResult(),
2154 shiftParamOp.getResult());
2155 return success();
2156 }
2157 }
2158 } else {
2159 rewriter.replaceOpWithNewOp<DstOpTy>(srcOp, srcOp.getType(), lhs, rhs);
2160 return success();
2161 }
2162 }
2163 // Float types
2164 else {
2165 // f32 at 64 lanes, the width of the accumulator, as one op.
2166 if (laneSize == 64 && scalarType.isF32())
2167 return genAddElemAIE2<SrcOpTy, DstOpTy>(rewriter, lhs, rhs, resultType,
2168 srcOp);
2169 if (laneSize != 16 && laneSize != 32)
2170 return failure();
2171
2172 // v32f32: split into two v16f32 ops
2173 if (laneSize == 32 && resultElWidth == 32) {
2174 VectorType halfType = createVectorType(16, scalarType);
2175
2176 splitWideBinaryVectorOp<SrcOpTy>(
2177 srcOp, lhs, rhs, halfType, resultType, rewriter,
2178 [](Value lhsHalf, Value rhsHalf, Location loc,
2179 ConversionPatternRewriter &rewriter) -> Value {
2180 VectorType halfVecType = cast<VectorType>(lhsHalf.getType());
2181 // For f32, use cast to acc, add_elem/sub_elem, cast back
2182 auto lCastOp = aievec::CastOp::create(rewriter, loc, halfVecType,
2183 lhsHalf, /*isResAcc*/ true);
2184 auto rCastOp = aievec::CastOp::create(rewriter, loc, halfVecType,
2185 rhsHalf, /*isResAcc*/ true);
2186 auto elemOp = DstOpTy::create(
2187 rewriter, loc, lCastOp->getResult(0).getType(),
2188 lCastOp->getResult(0), rCastOp->getResult(0));
2189 auto resCastOp = aievec::CastOp::create(
2190 rewriter, loc, halfVecType, elemOp.getResult(),
2191 /*isResAcc*/ false);
2192 return resCastOp.getResult();
2193 });
2194 return success();
2195 }
2196
2197 // v32bf16: split into two v16bf16 ops
2198 if (laneSize == 32 && resultElWidth == 16) {
2199 VectorType halfType = createVectorType(16, scalarType);
2200
2201 splitWideBinaryVectorOp<SrcOpTy>(
2202 srcOp, lhs, rhs, halfType, resultType, rewriter,
2203 [](Value lhsHalf, Value rhsHalf, Location loc,
2204 ConversionPatternRewriter &rewriter) -> Value {
2205 VectorType halfVecType = cast<VectorType>(lhsHalf.getType());
2206 Type accType = getVectorOpDestType(halfVecType, /*AIE2 =*/true);
2207 auto lUpsOp =
2208 aievec::UPSOp::create(rewriter, loc, accType, lhsHalf);
2209 auto rUpsOp =
2210 aievec::UPSOp::create(rewriter, loc, accType, rhsHalf);
2211 auto elemOp =
2212 DstOpTy::create(rewriter, loc, lUpsOp->getResult(0).getType(),
2213 lUpsOp->getResult(0), rUpsOp->getResult(0));
2214 auto shiftParamOp = arith::ConstantOp::create(
2215 rewriter, loc, rewriter.getI32IntegerAttr(0));
2216 auto srsOp = aievec::SRSOp::create(rewriter, loc, halfVecType,
2217 elemOp.getResult(),
2218 shiftParamOp.getResult());
2219 return srsOp.getResult();
2220 });
2221 return success();
2222 }
2223
2224 // Now we know laneSize == 16 for remaining float cases
2225 // v16float or v16bf16 with extension op case
2226 if (resultElWidth == 32) {
2227 if (!lhsDefOp && !rhsDefOp) {
2228 return genAddElemAIE2<SrcOpTy, DstOpTy>(rewriter, lhs, rhs,
2229 resultType, srcOp);
2230 }
2231
2232 auto lhsExt = getSourceOfWideningOp(lhs).value_or(nullptr);
2233 auto rhsExt = getSourceOfWideningOp(rhs).value_or(nullptr);
2234 // v16float
2235 if (!lhsExt && !rhsExt) {
2236 return genAddElemAIE2<SrcOpTy, DstOpTy>(rewriter, lhs, rhs,
2237 resultType, srcOp);
2238 }
2239
2240 // v16bf16 with two extension ops
2241 if (lhsExt && rhsExt) {
2242 auto lval = lhsExt;
2243 auto rval = rhsExt;
2244 VectorType vType = cast<VectorType>(lval.getType());
2245
2246 Type accType = getVectorOpDestType(vType, /*AIE2 =*/true);
2247 auto lUpsOp =
2248 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, lval);
2249 auto rUpsOp =
2250 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, rval);
2251 auto elemOp = DstOpTy::create(
2252 rewriter, srcOp.getLoc(), lUpsOp->getResult(0).getType(),
2253 lUpsOp->getResult(0), rUpsOp->getResult(0));
2254 rewriter.replaceOpWithNewOp<aievec::CastOp>(srcOp, srcOp.getType(),
2255 elemOp.getResult());
2256 return success();
2257 }
2258
2259 // v16bf16 with one extension op
2260 if (!lhsExt || !rhsExt) {
2261 auto lval = lhsExt ? lhsExt : lhs;
2262 auto rval = rhsExt ? rhsExt : rhs;
2263 auto extVal = lhsExt ? lval : rval;
2264 VectorType vType = cast<VectorType>(extVal.getType());
2265 Type accType = getVectorOpDestType(vType, /*AIE2 =*/true);
2266
2267 aievec::UPSOp upsOp;
2268 aievec::CastOp castOp;
2269 if (lhsExt) {
2270 upsOp =
2271 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, lval);
2272 castOp = aievec::CastOp::create(rewriter, srcOp.getLoc(),
2273 resultType, rval,
2274 /*isResAcc*/ true);
2275 } else {
2276 upsOp =
2277 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, rval);
2278 castOp = aievec::CastOp::create(rewriter, srcOp.getLoc(),
2279 resultType, lval,
2280 /*isResAcc*/ true);
2281 }
2282
2283 auto elemOp = DstOpTy::create(
2284 rewriter, srcOp.getLoc(), upsOp->getResult(0).getType(),
2285 upsOp->getResult(0), castOp->getResult(0));
2286
2287 rewriter.replaceOpWithNewOp<aievec::CastOp>(
2288 srcOp, srcOp.getType(), elemOp.getResult(), /*isResAcc*/ false);
2289
2290 return success();
2291 }
2292 }
2293
2294 // v16bfloat16
2295 Type accType = getVectorOpDestType(resultType, /*AIE2 =*/true);
2296 auto lUpsOp =
2297 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, lhs);
2298 auto rUpsOp =
2299 aievec::UPSOp::create(rewriter, srcOp.getLoc(), accType, rhs);
2300 auto elemOp = DstOpTy::create(rewriter, srcOp.getLoc(),
2301 lUpsOp->getResult(0).getType(),
2302 lUpsOp->getResult(0), rUpsOp->getResult(0));
2303 auto shiftParamOp = arith::ConstantOp::create(
2304 rewriter, srcOp.getLoc(), rewriter.getI32IntegerAttr(0));
2305 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
2306 srcOp, srcOp.getType(), elemOp.getResult(), shiftParamOp.getResult());
2307
2308 return success();
2309 }
2310
2311 return failure();
2312 }
2313};
2314
2317 aievec::AddElemOp>;
2320 aievec::SubElemOp>;
2323 aievec::AddElemOp>;
2326 aievec::SubElemOp>;
2327
2328template <typename SrcOpTy, typename DstOpTy>
2329struct LowerVectorMinMaxOpToAIEVecMinMaxOp : OpConversionPattern<SrcOpTy> {
2330 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
2331 using OpAdaptor = typename SrcOpTy::Adaptor;
2332
2333 LogicalResult
2334 matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor,
2335 ConversionPatternRewriter &rewriter) const override {
2336 VectorType resultType = dyn_cast<VectorType>(srcOp.getType());
2337 if (!resultType)
2338 return failure();
2339
2340 // A set recording the element width we are supporting for AIE2.
2341 llvm::SmallSet<unsigned, 16> elWidthSet;
2342 elWidthSet.insert(8);
2343 elWidthSet.insert(16);
2344 elWidthSet.insert(32);
2345
2346 Type scalarType = resultType.getElementType();
2347 unsigned resultElWidth = scalarType.getIntOrFloatBitWidth();
2348 unsigned laneSize = getVectorLaneSize(resultType);
2349
2350 unsigned totalBits = laneSize * resultElWidth;
2351 if (!elWidthSet.count(resultElWidth) ||
2352 (totalBits != 512 && (totalBits != 256 || resultElWidth != 16)))
2353 return failure();
2354
2355 if (totalBits == 256 && resultElWidth == 16) {
2356 // Pad v16bf16 to v32bf16, apply max/min, extract lower half
2357 Location loc = srcOp.getLoc();
2358 VectorType wideType = createVectorType(32, scalarType);
2359 Value lhsPad =
2360 padV16ToV32WithZeros(rewriter, loc, adaptor.getLhs(), scalarType);
2361 Value rhsPad =
2362 padV16ToV32WithZeros(rewriter, loc, adaptor.getRhs(), scalarType);
2363 auto wideOp = DstOpTy::create(rewriter, loc, wideType, lhsPad, rhsPad);
2364 rewriter.replaceOpWithNewOp<aievec::ExtOp>(srcOp, resultType,
2365 wideOp.getResult(), 0);
2366 return success();
2367 }
2368
2369 rewriter.replaceOpWithNewOp<DstOpTy>(srcOp, srcOp.getType(),
2370 adaptor.getLhs(), adaptor.getRhs());
2371 return success();
2372 }
2373};
2374
2379
2380// aievec.min/aievec.max are signed-only, so unsigned integer min/max have no
2381// direct AIE intrinsic. Re-expand them into cmp+select, which the aievec
2382// cmp/sel lowerings turn into aievec.cmp/aievec.sel.
2383template <typename SrcOpTy, arith::CmpIPredicate Pred>
2384struct LowerVectorUMinMaxOpToCmpSelOp : OpConversionPattern<SrcOpTy> {
2385 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
2386 using OpAdaptor = typename SrcOpTy::Adaptor;
2387
2388 LogicalResult
2389 matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor,
2390 ConversionPatternRewriter &rewriter) const override {
2391 if (!isa<VectorType>(srcOp.getType()))
2392 return failure();
2393 Value lhs = adaptor.getLhs();
2394 Value rhs = adaptor.getRhs();
2395 auto cmpOp =
2396 arith::CmpIOp::create(rewriter, srcOp.getLoc(), Pred, lhs, rhs);
2397 rewriter.replaceOpWithNewOp<arith::SelectOp>(srcOp, cmpOp, lhs, rhs);
2398 return success();
2399 }
2400};
2401
2406
2407// Promote scalar arith.maxsi/arith.minsi to vector aievec.max/aievec.min
2408// to avoid the AIE2 G_SELECT legalizer crash on scalar i32 select.
2409template <typename SrcOpTy, typename DstOpTy>
2410struct LowerScalarMinMaxToAIEVecMinMaxOp : OpConversionPattern<SrcOpTy> {
2411 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
2412 using OpAdaptor = typename SrcOpTy::Adaptor;
2413
2414 LogicalResult
2415 matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor,
2416 ConversionPatternRewriter &rewriter) const override {
2417 // Only match scalar integer types (reject vectors)
2418 Type resultType = srcOp.getType();
2419 if (isa<VectorType>(resultType))
2420 return failure();
2421
2422 auto intType = dyn_cast<IntegerType>(resultType);
2423 if (!intType)
2424 return failure();
2425
2426 unsigned elWidth = intType.getWidth();
2427 if (elWidth != 8 && elWidth != 16 && elWidth != 32)
2428 return failure();
2429
2430 unsigned numLanes = 512 / elWidth;
2431 VectorType vecType = createVectorType(numLanes, intType);
2432 Location loc = srcOp.getLoc();
2433
2434 // Broadcast both scalars to 512-bit vectors
2435 auto lhsBcast = aievec::BroadcastScalarOp::create(rewriter, loc, vecType,
2436 adaptor.getLhs());
2437 auto rhsBcast = aievec::BroadcastScalarOp::create(rewriter, loc, vecType,
2438 adaptor.getRhs());
2439
2440 // Apply vector min/max
2441 auto vecOp = DstOpTy::create(rewriter, loc, vecType, lhsBcast.getResult(),
2442 rhsBcast.getResult());
2443
2444 // Extract element 0 back to scalar
2445 auto zeroIdx =
2446 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
2447 rewriter.replaceOpWithNewOp<aievec::ExtElemOp>(
2448 srcOp, intType, vecOp.getResult(), zeroIdx.getResult());
2449 return success();
2450 }
2451};
2452
2457
2464
2465template <typename SrcOpTy, typename CmpTy>
2466struct LowerVectorCmpOpToAIEVecCmpOp : OpConversionPattern<SrcOpTy> {
2467 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
2468 using OpAdaptor = typename SrcOpTy::Adaptor;
2469
2470 LogicalResult
2471 matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor,
2472 ConversionPatternRewriter &rewriter) const override {
2473 VectorType lhsType = dyn_cast<VectorType>(srcOp.getLhs().getType());
2474 if (!lhsType)
2475 return failure();
2476
2477 llvm::SmallSet<unsigned, 16> elWidthSet;
2478 elWidthSet.insert(8);
2479 elWidthSet.insert(16);
2480 elWidthSet.insert(32);
2481
2482 Type scalarType = lhsType.getElementType();
2483 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
2484 unsigned laneSize = getVectorLaneSize(lhsType);
2485
2486 unsigned totalBits = laneSize * elWidth;
2487 if (!elWidthSet.count(elWidth) ||
2488 (totalBits != 512 && (totalBits != 256 || elWidth != 16)))
2489 return failure();
2490
2491 Location loc = srcOp.getLoc();
2492 Value lhs = srcOp.getLhs();
2493 Value rhs = srcOp.getRhs();
2494 unsigned effectiveLaneSize = laneSize;
2495
2496 if (totalBits == 256 && elWidth == 16) {
2497 lhs = padV16ToV32WithZeros(rewriter, loc, lhs, scalarType);
2498 rhs = padV16ToV32WithZeros(rewriter, loc, rhs, scalarType);
2499 effectiveLaneSize = 32;
2500 }
2501
2502 // Unsigned int and unsigned long long are acceptable type.
2503 Type type = mlir::IntegerType::get(srcOp.getContext(),
2504 effectiveLaneSize <= 32 ? 32 : 64,
2505 mlir::IntegerType::Unsigned);
2506
2507 CmpTy pred = srcOp.getPredicate();
2508
2509 arith::CmpIPredicate ipred = convertToIntegerPredicate(pred);
2510
2511 aievec::CmpOp aieCmpOp =
2512 createCmpOpAIE2(rewriter, ipred, loc, type, lhs, rhs);
2513
2514 if (!aieCmpOp)
2515 return failure();
2516
2517 VectorType resultType = cast<VectorType>(srcOp.getResult().getType());
2518 // Convert vector i1 type to unsigned interger type by built-in unrealized
2519 // conversion cast op.
2520 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
2521 srcOp, resultType, aieCmpOp.getResult());
2522
2523 return success();
2524 }
2525};
2526
2531
2532struct LowerVectorSelectOpToAIEVecSelOp : OpConversionPattern<arith::SelectOp> {
2533 using OpConversionPattern::OpConversionPattern;
2534
2535 LogicalResult
2536 matchAndRewrite(arith::SelectOp srcOp, OpAdaptor adaptor,
2537 ConversionPatternRewriter &rewriter) const override {
2538 auto resultType = dyn_cast<VectorType>(srcOp.getType());
2539 if (!resultType)
2540 return failure();
2541
2542 llvm::SmallSet<unsigned, 16> elWidthSet;
2543 elWidthSet.insert(8);
2544 elWidthSet.insert(16);
2545 elWidthSet.insert(32);
2546
2547 Type scalarType = resultType.getElementType();
2548 unsigned resultElWidth = scalarType.getIntOrFloatBitWidth();
2549 unsigned laneSize = getVectorLaneSize(resultType);
2550
2551 unsigned totalBits = laneSize * resultElWidth;
2552 if (!elWidthSet.count(resultElWidth) ||
2553 (totalBits != 512 && (totalBits != 256 || resultElWidth != 16)))
2554 return failure();
2555
2556 if (totalBits == 256 && resultElWidth == 16) {
2557 // Pad trueValue and falseValue to v32, do sel, extract lower half
2558 Location loc = srcOp.getLoc();
2559 VectorType wideType = createVectorType(32, scalarType);
2560
2561 // aievec.sel: bit=0 selects lhs, bit=1 selects rhs.
2562 // aievec.cmp: bit=1 where predicate is true.
2563 // arith.select: condition=1 returns true_value.
2564 // So false_value goes to lhs (bit=0) and true_value goes to rhs (bit=1).
2565 Value falsePad = padV16ToV32WithZeros(rewriter, loc,
2566 srcOp.getFalseValue(), scalarType);
2567 Value truePad =
2568 padV16ToV32WithZeros(rewriter, loc, srcOp.getTrueValue(), scalarType);
2569
2570 // The condition bitmask from cmp was produced at 32 lanes (since cmp
2571 // was also padded). Use 32-lane integer type for the cast.
2572 Type type = mlir::IntegerType::get(srcOp.getContext(), 32,
2573 mlir::IntegerType::Unsigned);
2574 auto convertOp = UnrealizedConversionCastOp::create(
2575 rewriter, loc, type, adaptor.getCondition());
2576
2577 auto wideSelOp = aievec::SelOp::create(rewriter, loc, wideType, falsePad,
2578 truePad, convertOp.getResult(0));
2579
2580 rewriter.replaceOpWithNewOp<aievec::ExtOp>(srcOp, resultType,
2581 wideSelOp.getResult(), 0);
2582 return success();
2583 }
2584
2585 Type type =
2586 mlir::IntegerType::get(srcOp.getContext(), laneSize <= 32 ? 32 : 64,
2587 mlir::IntegerType::Unsigned);
2588
2589 auto convertOp = UnrealizedConversionCastOp::create(
2590 rewriter, srcOp.getLoc(), type, adaptor.getCondition());
2591
2592 // aievec.sel: bit=0 selects lhs, bit=1 selects rhs.
2593 // aievec.cmp: bit=1 where predicate is true.
2594 // arith.select: condition=1 returns true_value.
2595 // So false_value goes to lhs (bit=0) and true_value goes to rhs (bit=1).
2596 rewriter.replaceOpWithNewOp<aievec::SelOp>(
2597 srcOp, srcOp.getResult().getType(), srcOp.getFalseValue(),
2598 srcOp.getTrueValue(), convertOp.getResult(0));
2599
2600 return success();
2601 }
2602};
2603
2604struct LowerVectorReductionMinOp : OpConversionPattern<vector::ReductionOp> {
2605 using OpConversionPattern::OpConversionPattern;
2606
2607 LogicalResult
2608 matchAndRewrite(vector::ReductionOp srcOp, OpAdaptor adaptor,
2609 ConversionPatternRewriter &rewriter) const override {
2610 if (auto kind = srcOp.getKind(); kind != vector::CombiningKind::MINSI &&
2611 kind != vector::CombiningKind::MINUI &&
2612 kind != vector::CombiningKind::MINIMUMF &&
2613 kind != vector::CombiningKind::MINNUMF)
2614 return failure();
2615
2616 auto vType = cast<VectorType>(srcOp.getVector().getType());
2617 Type scalarType = vType.getElementType();
2618 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
2619 unsigned laneSize = getVectorLaneSize(vType);
2620 unsigned vectorSize = laneSize * elWidth;
2621
2622 // Support 512-bit vectors directly, and 256-bit bf16 vectors by padding
2623 if (vectorSize != 512 && (vectorSize != 256 || !scalarType.isBF16()))
2624 return failure();
2625
2626 Location loc = srcOp.getLoc();
2627 Value inputVec = srcOp.getVector();
2628
2629 // For 256-bit bf16 (v16bf16), pad to 512-bit (v32bf16) with +inf
2630 if (vectorSize == 256) {
2631 std::tie(inputVec, laneSize) = padV16ToV32WithInfinity(
2632 rewriter, loc, srcOp.getVector(), scalarType, /*negativeInf=*/false);
2633 }
2634
2635 int shiftIndex = laneSize / 2;
2636 auto reduceResultOp = generateAIEVecOpsForReductionOp<aievec::MinOp>(
2637 rewriter, srcOp, shiftIndex, inputVec);
2638
2639 if (srcOp.getAcc()) {
2640 Value reduceResult = reduceResultOp.getResult();
2641 Value acc = srcOp.getAcc();
2642
2643 // If accumulator is bf16, use the high-level helper for bf16->f32->bf16
2644 if (acc.getType().isBF16()) {
2645 // Define the min operation to be performed in f32
2646 auto minOpBuilder = [&](Value lhs, Value rhs) -> Value {
2647 auto cmpOp = arith::CmpFOp::create(
2648 rewriter, srcOp.getLoc(), arith::CmpFPredicate::OLT, lhs, rhs);
2649 return arith::SelectOp::create(rewriter, srcOp.getLoc(), cmpOp, lhs,
2650 rhs);
2651 };
2652
2653 // Use helper to handle bf16->f32 conversion, perform min, and convert
2654 // back
2655 performBF16BinaryOpInF32(reduceResult, acc, srcOp, srcOp.getLoc(),
2656 rewriter, minOpBuilder);
2657 } else {
2658 // Non-bf16 path: perform min using cmpf and select
2659 auto cmpOp =
2660 arith::CmpFOp::create(rewriter, srcOp.getLoc(),
2661 arith::CmpFPredicate::OLT, reduceResult, acc);
2662 rewriter.replaceOpWithNewOp<arith::SelectOp>(srcOp, cmpOp, reduceResult,
2663 acc);
2664 }
2665 } else {
2666 rewriter.replaceOp(srcOp, reduceResultOp);
2667 }
2668 return success();
2669 }
2670};
2671
2672struct LowerVectorReductionMaxOp : OpConversionPattern<vector::ReductionOp> {
2673 using OpConversionPattern::OpConversionPattern;
2674
2675 LogicalResult
2676 matchAndRewrite(vector::ReductionOp srcOp, OpAdaptor adaptor,
2677 ConversionPatternRewriter &rewriter) const override {
2678 if (auto kind = srcOp.getKind(); kind != vector::CombiningKind::MAXSI &&
2679 kind != vector::CombiningKind::MAXUI &&
2680 kind != vector::CombiningKind::MAXIMUMF &&
2681 kind != vector::CombiningKind::MAXNUMF)
2682 return failure();
2683
2684 auto vType = cast<VectorType>(srcOp.getVector().getType());
2685 Type scalarType = vType.getElementType();
2686 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
2687 unsigned laneSize = getVectorLaneSize(vType);
2688 unsigned vectorSize = laneSize * elWidth;
2689
2690 // Support 512-bit vectors directly, and 256-bit bf16 vectors by padding
2691 // Only bf16 is supported for the 256-bit padding path (not f16)
2692 if (vectorSize != 512 && (vectorSize != 256 || !scalarType.isBF16()))
2693 return failure();
2694
2695 Location loc = srcOp.getLoc();
2696 Value inputVec = srcOp.getVector();
2697
2698 // For 256-bit bf16 (v16bf16), pad to 512-bit (v32bf16) with -inf
2699 if (vectorSize == 256) {
2700 std::tie(inputVec, laneSize) = padV16ToV32WithInfinity(
2701 rewriter, loc, srcOp.getVector(), scalarType, /*negativeInf=*/true);
2702 }
2703
2704 int shiftIndex = laneSize / 2;
2705 auto reduceResultOp = generateAIEVecOpsForReductionOp<aievec::MaxOp>(
2706 rewriter, srcOp, shiftIndex, inputVec);
2707
2708 if (srcOp.getAcc()) {
2709 Value reduceResult = reduceResultOp.getResult();
2710 Value acc = srcOp.getAcc();
2711
2712 // If accumulator is bf16, use the high-level helper for bf16->f32->bf16
2713 if (acc.getType().isBF16()) {
2714 // Define the max operation to be performed in f32
2715 auto maxOpBuilder = [&](Value lhs, Value rhs) -> Value {
2716 auto cmpOp = arith::CmpFOp::create(
2717 rewriter, srcOp.getLoc(), arith::CmpFPredicate::OGT, lhs, rhs);
2718 return arith::SelectOp::create(rewriter, srcOp.getLoc(), cmpOp, lhs,
2719 rhs);
2720 };
2721
2722 // Use helper to handle bf16->f32 conversion, perform max, and convert
2723 // back
2724 performBF16BinaryOpInF32(reduceResult, acc, srcOp, srcOp.getLoc(),
2725 rewriter, maxOpBuilder);
2726 } else {
2727 // Non-bf16 path: perform max directly
2728 auto cmpOp =
2729 arith::CmpFOp::create(rewriter, srcOp.getLoc(),
2730 arith::CmpFPredicate::OGT, reduceResult, acc);
2731 rewriter.replaceOpWithNewOp<arith::SelectOp>(srcOp, cmpOp, reduceResult,
2732 acc);
2733 }
2734 } else {
2735 rewriter.replaceOp(srcOp, reduceResultOp);
2736 }
2737 return success();
2738 }
2739};
2740
2741struct LowerVectorReductionAddIntOp : OpConversionPattern<vector::ReductionOp> {
2742 using OpConversionPattern::OpConversionPattern;
2743
2744 LogicalResult
2745 matchAndRewrite(vector::ReductionOp srcOp, OpAdaptor adaptor,
2746 ConversionPatternRewriter &rewriter) const override {
2747 if (auto kind = srcOp.getKind(); kind != vector::CombiningKind::ADD)
2748 return failure();
2749
2750 auto vType = cast<VectorType>(srcOp.getVector().getType());
2751 Type scalarType = vType.getElementType();
2752 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
2753 unsigned laneSize = getVectorLaneSize(vType);
2754 llvm::SmallSet<std::pair<unsigned, signed>, 16> laneSizeElWidthPairSet;
2755 laneSizeElWidthPairSet.insert({64, 8});
2756 laneSizeElWidthPairSet.insert({32, 16});
2757 laneSizeElWidthPairSet.insert({32, 32});
2758 laneSizeElWidthPairSet.insert({16, 32});
2759
2760 if (!isa<IntegerType>(scalarType) ||
2761 !laneSizeElWidthPairSet.count(std::make_pair(laneSize, elWidth)))
2762 return failure();
2763
2764 int shiftIndex = laneSize / 2;
2765 if (laneSize == 32 && elWidth == 32) {
2766 Location loc = srcOp.getLoc();
2767 VectorType vecType = createVectorType(laneSize / 2, scalarType);
2768
2769 auto lExtOp =
2770 aievec::ExtOp::create(rewriter, loc, vecType, srcOp.getVector(), 0);
2771 auto rExtOp =
2772 aievec::ExtOp::create(rewriter, loc, vecType, srcOp.getVector(), 1);
2773 auto addElemOp =
2774 aievec::AddElemOp::create(rewriter, loc, lExtOp.getResult().getType(),
2775 lExtOp.getResult(), rExtOp.getResult());
2776 shiftIndex /= 2;
2777 auto reduceResultOp = generateAIEVecOpsForReductionOp<aievec::AddElemOp>(
2778 rewriter, srcOp, shiftIndex, addElemOp.getResult());
2779 if (srcOp.getAcc())
2780 rewriter.replaceOpWithNewOp<arith::AddIOp>(
2781 srcOp, reduceResultOp.getResult(), srcOp.getAcc());
2782 else
2783 rewriter.replaceOp(srcOp, reduceResultOp);
2784 } else {
2785 auto reduceResultOp = generateAIEVecOpsForReductionOp<aievec::AddElemOp>(
2786 rewriter, srcOp, shiftIndex, srcOp.getVector());
2787 if (srcOp.getAcc())
2788 rewriter.replaceOpWithNewOp<arith::AddIOp>(
2789 srcOp, reduceResultOp.getResult(), srcOp.getAcc());
2790 else
2791 rewriter.replaceOp(srcOp, reduceResultOp);
2792 }
2793
2794 return success();
2795 }
2796};
2797
2799 : OpConversionPattern<vector::ReductionOp> {
2800 using OpConversionPattern::OpConversionPattern;
2801
2802 LogicalResult
2803 matchAndRewrite(vector::ReductionOp srcOp, OpAdaptor adaptor,
2804 ConversionPatternRewriter &rewriter) const override {
2805 if (auto kind = srcOp.getKind(); kind != vector::CombiningKind::ADD)
2806 return failure();
2807
2808 auto vType = cast<VectorType>(srcOp.getVector().getType());
2809 Type scalarType = vType.getElementType();
2810 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
2811 unsigned laneSize = getVectorLaneSize(vType);
2812
2813 if (!isa<FloatType>(scalarType) || laneSize != 16 || elWidth != 32)
2814 return failure();
2815
2816 int shiftIndex = laneSize / 2;
2817 assert(shiftIndex > 0 && (shiftIndex & (shiftIndex - 1)) == 0 &&
2818 "shiftIndex must be power of 2");
2819
2820 Location loc = srcOp.getLoc();
2821 Value curValue = srcOp.getVector();
2822 aievec::CastOp curOp = nullptr;
2823
2824 for (int id = shiftIndex; id > 0; id /= 2) {
2825 auto constOp = arith::ConstantOp::create(
2826 rewriter, loc, rewriter.getI32IntegerAttr(id * elWidth / 8));
2827
2828 auto shiftBytesOp = aievec::ShiftOp::create(
2829 rewriter, loc, vType, curValue, curValue, constOp.getResult());
2830
2831 auto lCastOp = aievec::CastOp::create(rewriter, loc, vType, curValue,
2832 /*isResAcc*/ true);
2833 auto rCastOp =
2834 aievec::CastOp::create(rewriter, loc, vType, shiftBytesOp.getResult(),
2835 /*isResAcc*/ true);
2836 auto elemOp = aievec::AddElemOp::create(
2837 rewriter, loc, lCastOp.getResult().getType(), lCastOp.getResult(),
2838 rCastOp.getResult());
2839 curOp = aievec::CastOp::create(rewriter, loc, vType, elemOp.getResult(),
2840 /*isResAcc*/ false);
2841 curValue = curOp.getResult();
2842 }
2843
2844 auto zeroConstOp =
2845 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
2846 auto reduceResultOp = aievec::ExtElemOp::create(
2847 rewriter, srcOp.getLoc(), scalarType, curOp, zeroConstOp.getResult());
2848
2849 if (srcOp.getAcc())
2850 rewriter.replaceOpWithNewOp<arith::AddFOp>(
2851 srcOp, reduceResultOp.getResult(), srcOp.getAcc());
2852 else
2853 rewriter.replaceOp(srcOp, reduceResultOp);
2854 return success();
2855 }
2856};
2857
2858// AIE2-specific bf16 ADD reduction - requires concat to v32bf16 before ext_elem
2859// due to aie2 ext_elem limitation
2861 : OpConversionPattern<vector::ReductionOp> {
2862 using OpConversionPattern::OpConversionPattern;
2863
2864 LogicalResult
2865 matchAndRewrite(vector::ReductionOp srcOp, OpAdaptor adaptor,
2866 ConversionPatternRewriter &rewriter) const override {
2867
2868 if (auto kind = srcOp.getKind(); kind != vector::CombiningKind::ADD) {
2869 return failure();
2870 }
2871
2872 auto vType = cast<VectorType>(srcOp.getVector().getType());
2873 Type scalarType = vType.getElementType();
2874 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
2875 unsigned laneSize = getVectorLaneSize(vType);
2876
2877 // Support both lane=16 and lane=32 for bf16
2878 if (!isa<FloatType>(scalarType) || (laneSize != 16 && laneSize != 32) ||
2879 elWidth != 16) {
2880 return failure();
2881 }
2882
2883 Location loc = srcOp.getLoc();
2884 Value curValue = srcOp.getVector();
2885 VectorType currentVType = vType; // Track current working vector type
2886
2887 // For lane=32, split into two v16bf16 halves and add them
2888 if (laneSize == 32) {
2889 VectorType halfType = createVectorType(laneSize / 2, scalarType);
2890 auto lowerHalf =
2891 aievec::ExtOp::create(rewriter, loc, halfType, srcOp.getVector(), 0);
2892 auto upperHalf =
2893 aievec::ExtOp::create(rewriter, loc, halfType, srcOp.getVector(), 1);
2894
2895 Type accType = getVectorOpDestType(halfType, /*AIE2 =*/true);
2896 auto lUpsOp =
2897 aievec::UPSOp::create(rewriter, loc, accType, lowerHalf.getResult());
2898 auto rUpsOp =
2899 aievec::UPSOp::create(rewriter, loc, accType, upperHalf.getResult());
2900 auto addElemOp = aievec::AddElemOp::create(
2901 rewriter, loc, accType, lUpsOp.getResult(), rUpsOp.getResult());
2902 auto shiftParamOp = arith::ConstantOp::create(
2903 rewriter, loc, rewriter.getI32IntegerAttr(0));
2904 auto srsOp =
2905 aievec::SRSOp::create(rewriter, loc, halfType, addElemOp.getResult(),
2906 shiftParamOp.getResult());
2907 curValue = srsOp.getResult();
2908 currentVType = halfType; // Update to v16bf16 after split
2909 }
2910
2911 int shiftIndex = 8; // Always 8 since we work with v16bf16
2912 Type accType = getVectorOpDestType(cast<VectorType>(curValue.getType()),
2913 /*AIE2 =*/true);
2914 unsigned accWidth =
2915 cast<VectorType>(accType).getElementType().getIntOrFloatBitWidth();
2916
2917 auto upsOp = aievec::UPSOp::create(rewriter, loc, accType, curValue);
2918 curValue = upsOp.getResult();
2919
2920 aievec::AddElemOp curOp = nullptr;
2921
2922 for (int id = shiftIndex; id > 0; id /= 2) {
2923 auto constOp = arith::ConstantOp::create(
2924 rewriter, loc, rewriter.getI32IntegerAttr(id * accWidth / 8));
2925 auto shiftBytesOp = aievec::ShiftOp::create(
2926 rewriter, loc, accType, curValue, curValue, constOp, true);
2927 curOp = aievec::AddElemOp::create(rewriter, loc, accType, curValue,
2928 shiftBytesOp.getResult());
2929 curValue = curOp.getResult();
2930 }
2931
2932 auto shiftParamOp = arith::ConstantOp::create(
2933 rewriter, srcOp.getLoc(), rewriter.getI32IntegerAttr(0));
2934 // Use currentVType instead of vType to ensure lane count matches
2935 auto srsOp =
2936 aievec::SRSOp::create(rewriter, loc, currentVType, curOp.getResult(),
2937 shiftParamOp.getResult());
2938
2939 // AIE2 ext_elem requires v32bf16, so concat v16bf16 to v32bf16
2940 VectorType vecType = createVectorType(32, scalarType);
2941 SmallVector<Value> concatSources = {srsOp.getResult(), srsOp.getResult()};
2942 auto concatOp =
2943 aievec::ConcatOp::create(rewriter, loc, vecType, concatSources);
2944
2945 auto zeroConstOp =
2946 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
2947 auto reduceResultOp =
2948 aievec::ExtElemOp::create(rewriter, srcOp.getLoc(), scalarType,
2949 concatOp, zeroConstOp.getResult());
2950
2951 if (srcOp.getAcc())
2952 rewriter.replaceOpWithNewOp<arith::AddFOp>(
2953 srcOp, reduceResultOp.getResult(), srcOp.getAcc());
2954 else
2955 rewriter.replaceOp(srcOp, reduceResultOp);
2956 return success();
2957 }
2958};
2959
2960// AIE2P-specific bf16 ADD reduction - can extract directly from v16bf16
2962 : OpConversionPattern<vector::ReductionOp> {
2963 using OpConversionPattern::OpConversionPattern;
2964
2965 LogicalResult
2966 matchAndRewrite(vector::ReductionOp srcOp, OpAdaptor adaptor,
2967 ConversionPatternRewriter &rewriter) const override {
2968 if (auto kind = srcOp.getKind(); kind != vector::CombiningKind::ADD)
2969 return failure();
2970
2971 auto vType = cast<VectorType>(srcOp.getVector().getType());
2972 Type scalarType = vType.getElementType();
2973 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
2974 unsigned laneSize = getVectorLaneSize(vType);
2975
2976 // Support both lane=16 and lane=32 for bf16
2977 if (!isa<FloatType>(scalarType) || (laneSize != 16 && laneSize != 32) ||
2978 elWidth != 16)
2979 return failure();
2980
2981 Location loc = srcOp.getLoc();
2982 int shiftIndex = laneSize / 2;
2983 Value inputToReduce = srcOp.getVector();
2984
2985 // For lane=32, split into two v16bf16 halves, add them, then reduce
2986 if (laneSize == 32) {
2987 VectorType halfType = createVectorType(laneSize / 2, scalarType);
2988
2989 // Extract lower and upper halves
2990 auto lowerHalf =
2991 aievec::ExtOp::create(rewriter, loc, halfType, srcOp.getVector(), 0);
2992 auto upperHalf =
2993 aievec::ExtOp::create(rewriter, loc, halfType, srcOp.getVector(), 1);
2994
2995 // Add the two halves together
2996 Type accType = getVectorOpDestType(halfType, /*AIE2 =*/true);
2997 auto lUpsOp =
2998 aievec::UPSOp::create(rewriter, loc, accType, lowerHalf.getResult());
2999 auto rUpsOp =
3000 aievec::UPSOp::create(rewriter, loc, accType, upperHalf.getResult());
3001 auto addElemOp = aievec::AddElemOp::create(
3002 rewriter, loc, accType, lUpsOp.getResult(), rUpsOp.getResult());
3003 auto shiftParamOp = arith::ConstantOp::create(
3004 rewriter, loc, rewriter.getI32IntegerAttr(0));
3005 auto srsOp =
3006 aievec::SRSOp::create(rewriter, loc, halfType, addElemOp.getResult(),
3007 shiftParamOp.getResult());
3008
3009 inputToReduce = srsOp.getResult();
3010 shiftIndex = 8;
3011 }
3012
3013 // Perform reduction using utility
3014 Type accType = getVectorOpDestType(
3015 cast<VectorType>(inputToReduce.getType()), /*AIE2 =*/true);
3016 unsigned accWidth =
3017 cast<VectorType>(accType).getElementType().getIntOrFloatBitWidth();
3018
3019 auto upsOp = aievec::UPSOp::create(rewriter, loc, accType, inputToReduce);
3020 Value curValue = upsOp.getResult();
3021
3022 aievec::AddElemOp curOp = nullptr;
3023 for (int id = shiftIndex; id > 0; id /= 2) {
3024 auto constOp = arith::ConstantOp::create(
3025 rewriter, loc, rewriter.getI32IntegerAttr(id * accWidth / 8));
3026 auto shiftBytesOp = aievec::ShiftOp::create(
3027 rewriter, loc, accType, curValue, curValue, constOp, true);
3028 curOp = aievec::AddElemOp::create(rewriter, loc, accType, curValue,
3029 shiftBytesOp.getResult());
3030 curValue = curOp.getResult();
3031 }
3032
3033 // Extract element 0 from the f32 accumulator
3034 // The loop has already fully reduced the vector to a single value in
3035 // element 0
3036 auto zeroConstOp =
3037 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
3038 auto extractedF32 = aievec::ExtElemOp::create(
3039 rewriter, srcOp.getLoc(), rewriter.getF32Type(), curOp.getResult(),
3040 zeroConstOp.getResult());
3041
3042 // Convert extracted f32 to bf16
3043 auto reduceResultBF16 = arith::TruncFOp::create(
3044 rewriter, srcOp.getLoc(), scalarType, extractedF32.getResult());
3045
3046 if (srcOp.getAcc())
3047 rewriter.replaceOpWithNewOp<arith::AddFOp>(srcOp, reduceResultBF16,
3048 srcOp.getAcc());
3049 else
3050 rewriter.replaceOp(srcOp, reduceResultBF16);
3051 return success();
3052 }
3053};
3054
3055// Convert a `vector.extract_strided_slice` op on 1D vectors into an
3056// `aievec.select` + `aievec.ext` op.
3058 : OpConversionPattern<vector::ExtractStridedSliceOp> {
3059 using OpConversionPattern::OpConversionPattern;
3060
3061 LogicalResult
3062 matchAndRewrite(vector::ExtractStridedSliceOp extractOp, OpAdaptor adaptor,
3063 ConversionPatternRewriter &rewriter) const override {
3064 auto vType = extractOp.getSourceVectorType();
3065 if (vType.getRank() != 1)
3066 return failure();
3067
3068 int64_t stride = cast<IntegerAttr>(adaptor.getStrides()[0]).getInt();
3069 if (stride != 1)
3070 return failure();
3071
3072 // AIE doesn't support select operations on i8
3073 if (getElementSizeInBits(vType) == 8)
3074 return extractOp.emitError()
3075 << "AIEv1 doesn't support select ops on int8 types";
3076
3077 // We only accept the case where we are extracting a slice half the size of
3078 // the input vector.
3079 int64_t size = cast<IntegerAttr>(adaptor.getSizes()[0]).getInt();
3080 if (vType.getNumElements() != 2 * size)
3081 return failure();
3082
3083 int64_t offset = cast<IntegerAttr>(adaptor.getOffsets()[0]).getInt();
3084 aievec::aie1::SelectOp::Properties selectProps;
3085 if (failed(aievec::aie1::SelectOp::setPropertiesFromAttr(
3086 selectProps,
3087 rewriter.getDictionaryAttr(
3088 buildAttributeListForRotationSelectOp(rewriter, vType, offset)),
3089 [&]() { return extractOp.emitError(); })))
3090 return failure();
3091 auto selectOp = aievec::aie1::SelectOp::create(
3092 rewriter, extractOp.getLoc(), TypeRange({vType}),
3093 ValueRange({adaptor.getSource()}), selectProps);
3094 rewriter.replaceOpWithNewOp<aievec::aie1::ExtOp>(
3095 extractOp, extractOp.getType(), selectOp.getResult(),
3096 rewriter.getI8IntegerAttr(0));
3097 return success();
3098 }
3099};
3100
3101// Convert a `vector.extract_strided_slice` op on 1D vectors into an
3102// `aievec.shift` op (an `aievec.ext` alone for the bottom or top half).
3104 : OpConversionPattern<vector::ExtractStridedSliceOp> {
3105 using OpConversionPattern::OpConversionPattern;
3106
3107 LogicalResult
3108 matchAndRewrite(vector::ExtractStridedSliceOp extractOp, OpAdaptor adaptor,
3109 ConversionPatternRewriter &rewriter) const override {
3110 auto vType = cast<VectorType>(adaptor.getSource().getType());
3111 if (vType.getRank() != 1)
3112 return failure();
3113
3114 int64_t stride = cast<IntegerAttr>(adaptor.getStrides()[0]).getInt();
3115 if (stride != 1)
3116 return failure();
3117
3118 // We only accept the case where we are extracting a slice half the size of
3119 // the input vector.
3120 int64_t size = cast<IntegerAttr>(adaptor.getSizes()[0]).getInt();
3121 if (vType.getNumElements() != 2 * size)
3122 return failure();
3123
3124 auto shortVecType = cast<VectorType>(extractOp.getResult().getType());
3125 int64_t offset = cast<IntegerAttr>(adaptor.getOffsets()[0]).getInt();
3126 // The bottom or the top half itself: no shift.
3127 if (offset == 0 || offset == size) {
3128 rewriter.replaceOpWithNewOp<aievec::ExtOp>(
3129 extractOp, shortVecType, adaptor.getSource(),
3130 rewriter.getI8IntegerAttr(offset == 0 ? 0 : 1));
3131 return success();
3132 }
3133 auto bottomHalf =
3134 aievec::ExtOp::create(rewriter, extractOp.getLoc(), shortVecType,
3135 adaptor.getSource(), rewriter.getI8IntegerAttr(0))
3136 .getResult();
3137 auto topHalf =
3138 aievec::ExtOp::create(rewriter, extractOp.getLoc(), shortVecType,
3139 adaptor.getSource(), rewriter.getI8IntegerAttr(1))
3140 .getResult();
3141 int32_t shiftBytes = offset * getElementSizeInBits(vType) / 8;
3142 auto shiftBytesConstOp = arith::ConstantOp::create(
3143 rewriter, extractOp.getLoc(), rewriter.getIntegerType(32),
3144 rewriter.getI32IntegerAttr(shiftBytes));
3145 rewriter.replaceOpWithNewOp<aievec::ShiftOp>(
3146 extractOp, shortVecType, bottomHalf, topHalf, shiftBytesConstOp);
3147
3148 return success();
3149 }
3150};
3151
3152// Replaces a short UPD op with a wide one followed by an ext op of the bottom
3153// half.
3154struct ExpandUPDToUPDAndExtPattern : OpConversionPattern<aievec::UPDOp> {
3155 using OpConversionPattern::OpConversionPattern;
3156
3157 ExpandUPDToUPDAndExtPattern(MLIRContext *context)
3158 : OpConversionPattern(context) {}
3159
3160 LogicalResult
3161 matchAndRewrite(aievec::UPDOp updOp, OpAdaptor adaptor,
3162 ConversionPatternRewriter &rewriter) const override {
3163 // Verify that we haven't already expanded this one
3164 if (updOp->hasOneUse() && isa<aievec::ExtOp>(*updOp->getUsers().begin()))
3165 return failure();
3166
3167 auto vecType = cast<VectorType>(updOp.getType());
3168 SmallVector<int64_t, 4> vecShape(vecType.getShape().begin(),
3169 vecType.getShape().end());
3170 vecShape[vecType.getRank() - 1] *= 2;
3171 auto longVecType = VectorType::get(vecShape, vecType.getElementType());
3172 auto newUpdOp = aievec::UPDOp::create(
3173 rewriter, updOp.getLoc(), longVecType, adaptor.getSource(),
3174 adaptor.getIndices(), adaptor.getOffset(), adaptor.getIndex(),
3175 adaptor.getVector());
3176 rewriter.replaceOpWithNewOp<aievec::ExtOp>(
3177 updOp, vecType, newUpdOp.getResult(), rewriter.getI8IntegerAttr(0));
3178
3179 return success();
3180 }
3181};
3182
3183// Replaces a wide UPD op followed by an ext op of the bottom half with a short
3184// UPD op.
3185struct FuseExtIntoUPDPattern : OpConversionPattern<aievec::ExtOp> {
3186 using OpConversionPattern::OpConversionPattern;
3187
3188 FuseExtIntoUPDPattern(MLIRContext *context) : OpConversionPattern(context) {}
3189
3190 LogicalResult
3191 matchAndRewrite(aievec::ExtOp extOp, OpAdaptor adaptor,
3192 ConversionPatternRewriter &rewriter) const override {
3193 // Verify we are extracting the lower half...
3194 if (extOp.getIndex() != 0)
3195 return failure();
3196 // ...of a UPDOp
3197 auto updOp = dyn_cast<aievec::UPDOp>(extOp.getSource().getDefiningOp());
3198 if (!updOp)
3199 return failure();
3200
3201 // Verify that this is a direct upd -> ext pattern
3202 if (!updOp->hasOneUse())
3203 return failure();
3204
3205 rewriter.replaceOpWithNewOp<aievec::UPDOp>(
3206 extOp, extOp.getType(), updOp.getSource(), updOp.getIndices(),
3207 updOp.getOffset(), updOp.getIndex(), updOp.getVector());
3208
3209 return success();
3210 }
3211};
3212
3213// Convert math.exp to aievec.exp for AIE2P (will be further lowered to exp2
3214// intrinsic)
3215struct ConvertMathExpToAIEVecExpOpPattern : OpConversionPattern<math::ExpOp> {
3216 using OpConversionPattern::OpConversionPattern;
3217
3218 LogicalResult
3219 matchAndRewrite(math::ExpOp expOp, OpAdaptor adaptor,
3220 ConversionPatternRewriter &rewriter) const override {
3221 if (matchExpOpForAIE2P(adaptor)) {
3222 auto srcType = cast<VectorType>(adaptor.getOperand().getType());
3223 rewriter.replaceOpWithNewOp<aievec::ExpOp>(expOp, srcType,
3224 adaptor.getOperand());
3225 return success();
3226 }
3227
3228 // aievec.exp takes bf16, so narrow an f32 operand and widen the result.
3229 auto srcType = dyn_cast<VectorType>(adaptor.getOperand().getType());
3230 if (!srcType || srcType.isScalable() || srcType.getRank() != 1 ||
3231 !srcType.getElementType().isF32() ||
3232 (srcType.getNumElements() != 16 && srcType.getNumElements() != 32))
3233 return failure();
3234
3235 Location loc = expOp.getLoc();
3236 auto bf16Type = VectorType::get(srcType.getShape(), rewriter.getBF16Type());
3237 auto shift =
3238 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
3239 Value input = aievec::SRSOp::create(rewriter, loc, bf16Type,
3240 adaptor.getOperand(), shift);
3241 Value exp = aievec::ExpOp::create(rewriter, loc, bf16Type, input);
3242 rewriter.replaceOpWithNewOp<aievec::UPSOp>(expOp, srcType, exp,
3243 /*shift=*/0);
3244 return success();
3245 }
3246};
3247
3248// Convert math.tanh to aievec.tanh, or math.exp2 to aievec.exp2, for AIE2P.
3249// Both are further lowered to the hardware intrinsic of the same name.
3250template <typename MathOpTy, typename AIEVecOpTy>
3252 : OpConversionPattern<MathOpTy> {
3253 using OpConversionPattern<MathOpTy>::OpConversionPattern;
3254 using OpAdaptor = typename MathOpTy::Adaptor;
3255
3256 LogicalResult
3257 matchAndRewrite(MathOpTy op, OpAdaptor adaptor,
3258 ConversionPatternRewriter &rewriter) const override {
3259 auto srcType = dyn_cast<VectorType>(adaptor.getOperand().getType());
3260 if (!srcType || !isAIE2PAccFloatUnaryType(srcType))
3261 return failure();
3262
3263 rewriter.replaceOpWithNewOp<AIEVecOpTy>(op, srcType, adaptor.getOperand());
3264 return success();
3265 }
3266};
3267
3268// Convert math.erf to aievec.tanh for AIE2P, using
3269//
3270// erf(x) ~= tanh(a * x + b * x^3), a = 2/sqrt(pi), b = 2 * a * 0.044715
3271//
3272// (the constants of the tanh form of GELU). The tanh argument is accumulated
3273// in f32 from bf16 multiplies. `a` is split into bf16 high and low parts so
3274// the linear term keeps f32 precision; x^2, x^3 and `b` are rounded to bf16.
3275struct ComputeErfOpAIE2PPattern : OpConversionPattern<math::ErfOp> {
3276 using OpConversionPattern::OpConversionPattern;
3277
3278 LogicalResult
3279 matchAndRewrite(math::ErfOp erfOp, OpAdaptor adaptor,
3280 ConversionPatternRewriter &rewriter) const override {
3281 auto srcType = dyn_cast<VectorType>(adaptor.getOperand().getType());
3282 if (!srcType || srcType.getRank() != 1 ||
3283 !isAIE2PAccFloatUnaryType(srcType))
3284 return failure();
3285
3286 constexpr double a = 1.1283791670955126; // 2/sqrt(pi)
3287 constexpr double b = 0.10091092485409293; // 2 * a * 0.044715
3288
3289 Location loc = erfOp.getLoc();
3290 auto bf16Type = VectorType::get(srcType.getShape(), rewriter.getBF16Type());
3291 auto f32Type = VectorType::get(srcType.getShape(), rewriter.getF32Type());
3292 auto shift =
3293 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
3294 auto toBF16 = [&](Value v) -> Value {
3295 return aievec::SRSOp::create(rewriter, loc, bf16Type, v, shift);
3296 };
3297 // A bf16 splat of `value`, and the value it was rounded to.
3298 auto splatBF16 = [&](double value) -> std::pair<Value, double> {
3299 auto attr = rewriter.getFloatAttr(rewriter.getBF16Type(), value);
3300 Value splat = arith::ConstantOp::create(
3301 rewriter, loc, DenseElementsAttr::get(bf16Type, attr));
3302 return {splat, attr.getValueAsDouble()};
3303 };
3304
3305 Value x = adaptor.getOperand();
3306 if (srcType.getElementType().isF32())
3307 x = toBF16(x);
3308
3309 auto [aHi, aHiValue] = splatBF16(a);
3310 Value aLo = splatBF16(a - aHiValue).first;
3311 Value bSplat = splatBF16(b).first;
3312
3313 Value x2 = toBF16(aievec::MulElemOp::create(rewriter, loc, f32Type, x, x));
3314 Value x3 = toBF16(aievec::MulElemOp::create(rewriter, loc, f32Type, x2, x));
3315 Value arg = aievec::MulElemOp::create(rewriter, loc, f32Type, x, aHi);
3316 arg = aievec::FMAElemOp::create(rewriter, loc, f32Type, x, aLo, arg,
3317 /*fmsub=*/false);
3318 arg = aievec::FMAElemOp::create(rewriter, loc, f32Type, x3, bSplat, arg,
3319 /*fmsub=*/false);
3320 Value result = aievec::TanhOp::create(rewriter, loc, f32Type, arg);
3321 if (srcType.getElementType().isBF16())
3322 result = toBF16(result);
3323
3324 rewriter.replaceOp(erfOp, result);
3325 return success();
3326 }
3327};
3328
3329// Convert a scalar math.tanh, math.erf or math.exp2 for AIE2P by broadcasting
3330// it to a 16-lane vector, applying the vector op and extracting lane 0.
3331template <typename OpTy>
3332struct ComputeScalarMathOpByVectorAIE2PPattern : OpConversionPattern<OpTy> {
3333 using OpConversionPattern<OpTy>::OpConversionPattern;
3334 using OpAdaptor = typename OpTy::Adaptor;
3335
3336 LogicalResult
3337 matchAndRewrite(OpTy op, OpAdaptor adaptor,
3338 ConversionPatternRewriter &rewriter) const override {
3339 Type type = adaptor.getOperand().getType();
3340 if (!type.isBF16() && !type.isF32())
3341 return failure();
3342
3343 Location loc = op.getLoc();
3344 auto vecType = VectorType::get({16}, type);
3345 Value splat = aievec::BroadcastScalarOp::create(rewriter, loc, vecType,
3346 adaptor.getOperand());
3347 Value result = OpTy::create(rewriter, loc, splat);
3348 rewriter.replaceOpWithNewOp<vector::ExtractOp>(op, result, 0);
3349 return success();
3350 }
3351};
3352
3353// Convert a one-lane math.tanh, math.erf or math.exp2 for AIE2P to the scalar
3354// op, which ComputeScalarMathOpByVectorAIE2PPattern lowers.
3355template <typename OpTy>
3356struct ComputeOneLaneMathOpAIE2PPattern : OpConversionPattern<OpTy> {
3357 using OpConversionPattern<OpTy>::OpConversionPattern;
3358 using OpAdaptor = typename OpTy::Adaptor;
3359
3360 LogicalResult
3361 matchAndRewrite(OpTy op, OpAdaptor adaptor,
3362 ConversionPatternRewriter &rewriter) const override {
3363 auto srcType = dyn_cast<VectorType>(adaptor.getOperand().getType());
3364 if (!srcType || srcType.isScalable() || srcType.getRank() != 1 ||
3365 srcType.getNumElements() != 1)
3366 return failure();
3367
3368 Location loc = op.getLoc();
3369 Value scalar =
3370 vector::ExtractOp::create(rewriter, loc, adaptor.getOperand(), 0);
3371 Value result = OpTy::create(rewriter, loc, scalar);
3372 Value poison = ub::PoisonOp::create(rewriter, loc, srcType);
3373 rewriter.replaceOpWithNewOp<vector::InsertOp>(op, result, poison, 0);
3374 return success();
3375 }
3376};
3377
3378struct ComputeExpOpByLUTLLVMPattern : OpConversionPattern<math::ExpOp> {
3379 using OpConversionPattern::OpConversionPattern;
3380
3381 LogicalResult
3382 matchAndRewrite(math::ExpOp expOp, OpAdaptor adaptor,
3383 ConversionPatternRewriter &rewriter) const override {
3384
3385 if (!matchExpOpForAIE2LUT(adaptor))
3386 return failure();
3387
3388 auto srcType = dyn_cast<VectorType>(adaptor.getOperand().getType());
3389 unsigned laneSize = getVectorLaneSize(srcType);
3390 Location loc = expOp.getLoc();
3391 StringRef funcName = "getExpBf16";
3392
3393 VectorType v16bf16Ty = mlir::VectorType::get({16}, rewriter.getBF16Type());
3394 VectorType v8i64Ty = mlir::VectorType::get({8}, rewriter.getI64Type());
3395 func::FuncOp fnOp = getOrInsertFuncDecl(
3396 rewriter, expOp->getParentWithTrait<OpTrait::SymbolTable>(), funcName,
3397 TypeRange{v16bf16Ty}, TypeRange{v8i64Ty});
3398
3399 // Handle v32bf16 by splitting into two v16bf16 operations
3400 if (laneSize == 32) {
3401 splitWideUnaryVectorOp<math::ExpOp>(
3402 expOp, adaptor.getOperand(), v16bf16Ty, srcType, rewriter,
3403 [&fnOp](Value halfInput, Location loc,
3404 ConversionPatternRewriter &rewriter) -> Value {
3405 VectorType v16bf16Ty =
3406 mlir::VectorType::get({16}, rewriter.getBF16Type());
3407 auto callOp = func::CallOp::create(rewriter, loc, fnOp,
3408 SmallVector<Value>{halfInput});
3409 Type accType = getVectorOpDestType(v16bf16Ty, /*AIE2 =*/true);
3410 auto resCastOp = vector::BitCastOp::create(rewriter, loc, accType,
3411 callOp.getResults());
3412 auto shiftParamOp = arith::ConstantOp::create(
3413 rewriter, loc, rewriter.getI32IntegerAttr(0));
3414 auto srsOp = aievec::SRSOp::create(rewriter, loc, v16bf16Ty,
3415 resCastOp.getResult(),
3416 shiftParamOp.getResult());
3417 return srsOp.getResult();
3418 });
3419 return success();
3420 }
3421
3422 // Handle v16bf16 directly
3423 SmallVector<Value> expOperands = {adaptor.getOperand()};
3424
3425 Type accTypeNative = getVectorOpDestType(srcType, /*AIE2 =*/true);
3426 auto callOp = func::CallOp::create(rewriter, loc, fnOp, expOperands);
3427 auto resCastOp = vector::BitCastOp::create(rewriter, loc, accTypeNative,
3428 callOp.getResults());
3429 auto shiftParamOp =
3430 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
3431 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
3432 expOp, srcType, resCastOp.getResult(), shiftParamOp.getResult());
3433
3434 return success();
3435 }
3436};
3437// Lower ExpOp to function call
3438struct ComputeExpOpByLUTPattern : OpConversionPattern<math::ExpOp> {
3439 using OpConversionPattern::OpConversionPattern;
3440
3441 LogicalResult
3442 matchAndRewrite(math::ExpOp expOp, OpAdaptor adaptor,
3443 ConversionPatternRewriter &rewriter) const override {
3444 if (!matchExpOpForAIE2LUT(adaptor))
3445 return failure();
3446 auto srcType = dyn_cast<VectorType>(adaptor.getOperand().getType());
3447 StringRef includeName = "lut_based_ops.h";
3448 auto moduleOp = expOp->getParentOfType<mlir::ModuleOp>();
3449 rewriter.setInsertionPointToStart(
3450 &moduleOp.getRegion().getBlocks().front());
3451 emitc::IncludeOp::create(rewriter, moduleOp.getLoc(), includeName, false);
3452
3453 rewriter.setInsertionPoint(expOp);
3454
3455 auto v16bf16OpaqueTy =
3456 emitc::OpaqueType::get(rewriter.getContext(), "v16bfloat16");
3457 auto opaquedOperand =
3458 UnrealizedConversionCastOp::create(
3459 rewriter, expOp.getLoc(), v16bf16OpaqueTy, adaptor.getOperand())
3460 .getResult(0);
3461 SmallVector<Value> expOperands = {opaquedOperand};
3462
3463 Type accTypeNative = getVectorOpDestType(srcType, /*AIE2 =*/true);
3464 Type v16accf32OpaqueTy =
3465 emitc::OpaqueType::get(rewriter.getContext(), "v16accfloat");
3466 auto callOp = emitc::CallOpaqueOp::create(
3467 rewriter, expOp.getLoc(), TypeRange{v16accf32OpaqueTy}, "getExpBf16",
3468 nullptr, nullptr, expOperands);
3469 auto resCastOp = UnrealizedConversionCastOp::create(
3470 rewriter, expOp.getLoc(), accTypeNative, callOp.getResults());
3471 auto shiftParamOp = arith::ConstantOp::create(
3472 rewriter, expOp.getLoc(), rewriter.getI32IntegerAttr(0));
3473 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
3474 expOp, srcType, resCastOp.getResult(0), shiftParamOp.getResult());
3475
3476 return success();
3477 }
3478};
3479
3480// Lower the inverse of a float to aievec.inv (LLVMIR backend for AIE2P)
3481// Supports both scalar f32 and vector<Nxf32> types.
3482// Convert the pattern-
3483// %cst = arith.constant 1.000000e+00 : f32
3484// %0 = arith.divf %cst, %arg1 : f32
3485// to -
3486// %0 = aievec.inv %arg1 : f32
3487// Also supports:
3488// %cst = arith.constant dense<1.0> : vector<16xf32>
3489// %0 = arith.divf %cst, %arg1 : vector<16xf32>
3490// to -
3491// %0 = aievec.inv %arg1 : vector<16xf32>
3492struct ConvertDivFToAIEVecInvOpPattern : OpConversionPattern<arith::DivFOp> {
3493 using OpConversionPattern::OpConversionPattern;
3494
3495 LogicalResult
3496 matchAndRewrite(arith::DivFOp divOp, OpAdaptor adaptor,
3497 ConversionPatternRewriter &rewriter) const override {
3498 Type srcType = adaptor.getLhs().getType();
3499
3500 // Check if LHS is defined by an operation
3501 auto *defOp = divOp.getLhs().getDefiningOp();
3502 if (!defOp)
3503 return failure();
3504
3505 auto constOp = dyn_cast<arith::ConstantOp>(defOp);
3506 if (!constOp)
3507 return failure();
3508
3509 // Handle scalar f32 case
3510 if (auto fType = dyn_cast<FloatType>(srcType)) {
3511 if (fType.getWidth() != 32)
3512 return failure();
3513
3514 auto floatAttr = dyn_cast<FloatAttr>(constOp.getValue());
3515 if (!floatAttr || !floatAttr.getValue().isExactlyValue(1.0))
3516 return failure();
3517
3518 rewriter.replaceOpWithNewOp<aievec::InvOp>(divOp, srcType,
3519 adaptor.getRhs());
3520 return success();
3521 }
3522
3523 // Handle vector f32 case
3524 if (auto vecType = dyn_cast<VectorType>(srcType)) {
3525 auto elemType = vecType.getElementType();
3526 if (!elemType.isF32())
3527 return failure();
3528
3529 // Check for supported vector sizes (16 or 32 lanes)
3530 unsigned laneSize = getVectorLaneSize(vecType);
3531 if (laneSize != 16 && laneSize != 32)
3532 return failure();
3533
3534 // Check if it's a splat of 1.0
3535 auto denseAttr = dyn_cast<DenseFPElementsAttr>(constOp.getValue());
3536 if (!denseAttr || !denseAttr.isSplat())
3537 return failure();
3538
3539 if (!denseAttr.getSplatValue<APFloat>().isExactlyValue(1.0))
3540 return failure();
3541
3542 rewriter.replaceOpWithNewOp<aievec::InvOp>(divOp, vecType,
3543 adaptor.getRhs());
3544 return success();
3545 }
3546
3547 return failure();
3548 }
3549};
3550
3551// Convert math.tanh to a function call to compute tanh(x) by look up tables
3552struct ComputeTanhOpByLUTPattern : OpConversionPattern<math::TanhOp> {
3553 using OpConversionPattern::OpConversionPattern;
3554
3555 LogicalResult
3556 matchAndRewrite(math::TanhOp tanhOp, OpAdaptor adaptor,
3557 ConversionPatternRewriter &rewriter) const override {
3558 auto srcType = dyn_cast<VectorType>(tanhOp.getOperand().getType());
3559 if (!srcType)
3560 return failure();
3561
3562 Type scalarType = srcType.getElementType();
3563 if (!isa<FloatType>(scalarType))
3564 return failure();
3565
3566 unsigned laneSize = getVectorLaneSize(srcType);
3567 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
3568 if (elWidth != 16 || laneSize != 16)
3569 return failure();
3570
3571 StringRef includeName = "lut_based_ops.h";
3572 auto moduleOp = tanhOp->getParentOfType<mlir::ModuleOp>();
3573 rewriter.setInsertionPointToStart(
3574 &moduleOp.getRegion().getBlocks().front());
3575 emitc::IncludeOp::create(rewriter, moduleOp.getLoc(), includeName, false);
3576
3577 rewriter.setInsertionPoint(tanhOp);
3578 Type v16bf16OpaqueTy =
3579 emitc::OpaqueType::get(rewriter.getContext(), "v16bfloat16");
3580 auto opaquedOperand =
3581 UnrealizedConversionCastOp::create(
3582 rewriter, tanhOp.getLoc(), v16bf16OpaqueTy, adaptor.getOperand())
3583 .getResult(0);
3584 SmallVector<Value> tanhOperands = {opaquedOperand};
3585 auto callOp = emitc::CallOpaqueOp::create(rewriter, tanhOp.getLoc(),
3586 v16bf16OpaqueTy, "getTanhBf16",
3587 nullptr, nullptr, tanhOperands);
3588 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
3589 tanhOp, TypeRange{tanhOp.getResult().getType()}, callOp.getResults());
3590
3591 return success();
3592 }
3593};
3594
3595// Convert math.sqrt to a function call to compute sqrt(x) for v16bfloat16 and
3596// v32bfloat16 types
3597struct ComputeSqrtOpAIE2Pattern : OpConversionPattern<math::SqrtOp> {
3598 using OpConversionPattern::OpConversionPattern;
3599
3600 LogicalResult
3601 matchAndRewrite(math::SqrtOp sqrtOp, OpAdaptor adaptor,
3602 ConversionPatternRewriter &rewriter) const override {
3603 auto srcType = dyn_cast<VectorType>(sqrtOp.getOperand().getType());
3604 if (!srcType)
3605 return failure();
3606
3607 Type scalarType = srcType.getElementType();
3608 if (!isa<FloatType>(scalarType))
3609 return failure();
3610
3611 unsigned laneSize = getVectorLaneSize(srcType);
3612 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
3613 if (elWidth != 16 || (laneSize != 16 && laneSize != 32))
3614 return failure();
3615
3616 StringRef includeName = "vec_math.h";
3617 auto moduleOp = sqrtOp->getParentOfType<mlir::ModuleOp>();
3618 rewriter.setInsertionPointToStart(
3619 &moduleOp.getRegion().getBlocks().front());
3620 emitc::IncludeOp::create(rewriter, moduleOp.getLoc(), includeName, false);
3621
3622 rewriter.setInsertionPoint(sqrtOp);
3623 Type vLNbf16OpaqueTy;
3624 if (laneSize == 16)
3625 vLNbf16OpaqueTy =
3626 emitc::OpaqueType::get(rewriter.getContext(), "v16bfloat16");
3627 else
3628 vLNbf16OpaqueTy =
3629 emitc::OpaqueType::get(rewriter.getContext(), "v32bfloat16");
3630 auto opaquedOperand =
3631 UnrealizedConversionCastOp::create(
3632 rewriter, sqrtOp.getLoc(), vLNbf16OpaqueTy, adaptor.getOperand())
3633 .getResult(0);
3634 SmallVector<Value> sqrtOperands = {opaquedOperand};
3635 auto callOp = emitc::CallOpaqueOp::create(
3636 rewriter, sqrtOp.getLoc(), TypeRange{vLNbf16OpaqueTy}, "getSqrtBf16",
3637 nullptr, nullptr, sqrtOperands);
3638 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
3639 sqrtOp, TypeRange{sqrtOp.getResult().getType()}, callOp.getResults());
3640
3641 return success();
3642 }
3643};
3644
3645struct ComputeRsqrtOpLLVMAIE2Pattern : OpConversionPattern<math::RsqrtOp> {
3646 using OpConversionPattern::OpConversionPattern;
3647
3648 LogicalResult
3649 matchAndRewrite(math::RsqrtOp rsqrtOp, OpAdaptor adaptor,
3650 ConversionPatternRewriter &rewriter) const override {
3651 auto srcType = dyn_cast<VectorType>(adaptor.getOperand().getType());
3652 if (!srcType)
3653 return failure();
3654
3655 Type scalarType = srcType.getElementType();
3656 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
3657 unsigned laneSize = getVectorLaneSize(srcType);
3658
3659 // Only support v16bf16 for LLVM backend
3660 if (!isa<FloatType>(scalarType) || laneSize != 16 || elWidth != 16)
3661 return failure();
3662
3663 StringRef funcName = "getRsqrtBf16";
3664
3665 VectorType v16bf16Ty = mlir::VectorType::get({16}, rewriter.getBF16Type());
3666 VectorType v8i64Ty = mlir::VectorType::get({8}, rewriter.getI64Type());
3667 func::FuncOp fnOp = getOrInsertFuncDecl(
3668 rewriter, rsqrtOp->getParentWithTrait<OpTrait::SymbolTable>(), funcName,
3669 TypeRange{v16bf16Ty}, TypeRange{v8i64Ty});
3670
3671 SmallVector<Value> rsqrtOperands = {adaptor.getOperand()};
3672
3673 Type accTypeNative = getVectorOpDestType(srcType, /*AIE2 =*/true);
3674 auto callOp =
3675 func::CallOp::create(rewriter, rsqrtOp.getLoc(), fnOp, rsqrtOperands);
3676 auto resCastOp = vector::BitCastOp::create(
3677 rewriter, rsqrtOp.getLoc(), accTypeNative, callOp.getResults());
3678 auto shiftParamOp = arith::ConstantOp::create(
3679 rewriter, rsqrtOp.getLoc(), rewriter.getI32IntegerAttr(0));
3680 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
3681 rsqrtOp, srcType, resCastOp.getResult(), shiftParamOp.getResult());
3682
3683 return success();
3684 }
3685};
3686
3687// Convert math.rsqrt to a function call to compute 1.0f / sqrt(x) for
3688// v16bfloat16 and v32bfloat16 types
3689struct ComputeRsqrtOpPattern : OpConversionPattern<math::RsqrtOp> {
3690 using OpConversionPattern::OpConversionPattern;
3691
3692 LogicalResult
3693 matchAndRewrite(math::RsqrtOp rsqrtOp, OpAdaptor adaptor,
3694 ConversionPatternRewriter &rewriter) const override {
3695 auto srcType = dyn_cast<VectorType>(rsqrtOp.getOperand().getType());
3696 if (!srcType)
3697 return failure();
3698
3699 Type scalarType = srcType.getElementType();
3700 if (!isa<FloatType>(scalarType))
3701 return failure();
3702
3703 unsigned laneSize = getVectorLaneSize(srcType);
3704 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
3705 if (elWidth != 16 || (laneSize != 16 && laneSize != 32))
3706 return failure();
3707
3708 StringRef includeName = "vec_math.h";
3709 auto moduleOp = rsqrtOp->getParentOfType<mlir::ModuleOp>();
3710 rewriter.setInsertionPointToStart(
3711 &moduleOp.getRegion().getBlocks().front());
3712 emitc::IncludeOp::create(rewriter, moduleOp.getLoc(), includeName, false);
3713
3714 rewriter.setInsertionPoint(rsqrtOp);
3715 Type vLNbf16OpaqueTy;
3716 if (laneSize == 16)
3717 vLNbf16OpaqueTy =
3718 emitc::OpaqueType::get(rewriter.getContext(), "v16bfloat16");
3719 else
3720 vLNbf16OpaqueTy =
3721 emitc::OpaqueType::get(rewriter.getContext(), "v32bfloat16");
3722 auto opaquedOperand =
3723 UnrealizedConversionCastOp::create(
3724 rewriter, rsqrtOp.getLoc(), vLNbf16OpaqueTy, adaptor.getOperand())
3725 .getResult(0);
3726 SmallVector<Value> rsqrtOperands = {opaquedOperand};
3727 auto callOp = emitc::CallOpaqueOp::create(
3728 rewriter, rsqrtOp.getLoc(), TypeRange{vLNbf16OpaqueTy}, "getRsqrtBf16",
3729 nullptr, nullptr, rsqrtOperands);
3730 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
3731 rsqrtOp, TypeRange{rsqrtOp.getResult().getType()}, callOp.getResults());
3732
3733 return success();
3734 }
3735};
3736
3737// Convert math.erf to a function call to compute erf(x) for v16bfloat16 and
3738// v32bfloat16 types
3739struct ComputeErfOpPattern : OpConversionPattern<math::ErfOp> {
3740 using OpConversionPattern::OpConversionPattern;
3741
3742 LogicalResult
3743 matchAndRewrite(math::ErfOp erfOp, OpAdaptor adaptor,
3744 ConversionPatternRewriter &rewriter) const override {
3745 auto srcType = dyn_cast<VectorType>(erfOp.getOperand().getType());
3746 if (!srcType)
3747 return failure();
3748
3749 Type scalarType = srcType.getElementType();
3750 if (!isa<FloatType>(scalarType))
3751 return failure();
3752
3753 unsigned laneSize = getVectorLaneSize(srcType);
3754 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
3755 if (elWidth != 16 || (laneSize != 16 && laneSize != 32))
3756 return failure();
3757
3758 StringRef includeName = "vec_math.h";
3759 auto moduleOp = erfOp->getParentOfType<mlir::ModuleOp>();
3760 rewriter.setInsertionPointToStart(
3761 &moduleOp.getRegion().getBlocks().front());
3762 emitc::IncludeOp::create(rewriter, moduleOp.getLoc(), includeName, false);
3763
3764 rewriter.setInsertionPoint(erfOp);
3765 Type vLNbf16OpaqueTy;
3766 if (laneSize == 16)
3767 vLNbf16OpaqueTy =
3768 emitc::OpaqueType::get(rewriter.getContext(), "v16bfloat16");
3769 else
3770 vLNbf16OpaqueTy =
3771 emitc::OpaqueType::get(rewriter.getContext(), "v32bfloat16");
3772 auto opaquedOperand =
3773 UnrealizedConversionCastOp::create(
3774 rewriter, erfOp.getLoc(), vLNbf16OpaqueTy, adaptor.getOperand())
3775 .getResult(0);
3776 SmallVector<Value> erfOperands = {opaquedOperand};
3777 auto callOp = emitc::CallOpaqueOp::create(
3778 rewriter, erfOp.getLoc(), TypeRange{vLNbf16OpaqueTy}, "getErfBf16",
3779 nullptr, nullptr, erfOperands);
3780 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
3781 erfOp, TypeRange{erfOp.getResult().getType()}, callOp.getResults());
3782
3783 return success();
3784 }
3785};
3786
3787// Convert math.absf and math.absi to a function call to compute abs(x) for
3788// v16bfloat16, v32bfloat16, v16float, v16int32, v32int16 and v64int8 types
3789template <typename SrcOpTy>
3790struct ComputeAbsOpPattern : OpConversionPattern<SrcOpTy> {
3791 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
3792 using OpAdaptor = typename SrcOpTy::Adaptor;
3793
3794 LogicalResult
3795 matchAndRewrite(SrcOpTy absOp, OpAdaptor adaptor,
3796 ConversionPatternRewriter &rewriter) const override {
3797 auto vecTy = dyn_cast<VectorType>(absOp.getOperand().getType());
3798 if (!vecTy)
3799 return failure();
3800
3801 Type elemTy = vecTy.getElementType();
3802
3803 unsigned laneSize = getVectorLaneSize(vecTy);
3804 unsigned elWidth = elemTy.getIntOrFloatBitWidth();
3805
3806 StringRef includeName = "vec_math.h";
3807 auto moduleOp = absOp->template getParentOfType<mlir::ModuleOp>();
3808 rewriter.setInsertionPointToStart(
3809 &moduleOp.getRegion().getBlocks().front());
3810 emitc::IncludeOp::create(rewriter, moduleOp.getLoc(), includeName, false);
3811
3812 rewriter.setInsertionPoint(absOp);
3813 std::ostringstream typeName;
3814 typeName << "v" << laneSize;
3815 if (isa<FloatType>(elemTy)) {
3816 if (elWidth == 16)
3817 typeName << "bfloat16";
3818 else
3819 typeName << "float";
3820 } else
3821 typeName << "int" << elWidth;
3822 Type vecOpaqueTy =
3823 emitc::OpaqueType::get(rewriter.getContext(), typeName.str());
3824 auto opaquedOperand =
3825 UnrealizedConversionCastOp::create(rewriter, absOp.getLoc(),
3826 vecOpaqueTy, adaptor.getOperand())
3827 .getResult(0);
3828 SmallVector<Value> absOperands = {opaquedOperand};
3829 auto callOp = emitc::CallOpaqueOp::create(rewriter, absOp.getLoc(),
3830 TypeRange{vecOpaqueTy}, "getAbs",
3831 nullptr, nullptr, absOperands);
3832 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
3833 absOp, TypeRange{absOp.getResult().getType()}, callOp.getResults());
3834
3835 return success();
3836 }
3837};
3838
3841
3842template <typename SrcOpTy>
3843struct LowerExtOpPattern : OpConversionPattern<SrcOpTy> {
3844 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
3845 using OpAdaptor = typename SrcOpTy::Adaptor;
3846
3847 LogicalResult
3848 matchAndRewrite(SrcOpTy extOp, OpAdaptor adaptor,
3849 ConversionPatternRewriter &rewriter) const override {
3850 VectorType srcType = cast<VectorType>(extOp.getIn().getType());
3851 VectorType dstType = cast<VectorType>(extOp.getOut().getType());
3852
3853 Type scalarType = dstType.getElementType();
3854 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
3855 auto accType =
3856 isa<IntegerType>(scalarType) && (elWidth == 32 || elWidth == 64)
3857 ? dstType
3858 : getVectorOpDestType(srcType, /*AIE2 =*/true);
3859 auto upsOp =
3860 aievec::UPSOp::create(rewriter, extOp.getLoc(), accType, extOp.getIn());
3861
3862 if (dstType.getElementType().getIntOrFloatBitWidth() == 16) {
3863 auto shiftParamOp = arith::ConstantOp::create(
3864 rewriter, extOp.getLoc(), rewriter.getI32IntegerAttr(0));
3865 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
3866 extOp, dstType, upsOp.getResult(), shiftParamOp.getResult());
3867 } else
3868 rewriter.replaceOpWithNewOp<aievec::CastOp>(
3869 extOp, dstType, upsOp.getResult(), /*isResAcc*/ false);
3870
3871 return success();
3872 }
3873};
3874
3877
3878// Vector arith.sitofp i16 -> bf16 on AIE2P via the classic "magic number"
3879// trick that aie_api uses in detail/aie2p/elementary.hpp:
3880//
3881// 1. UPS the int16 input to acc32 (32-bit integer accumulator lanes).
3882// 2. Add a per-lane "magic_l" constant to the acc32 vector. The constant
3883// is the bit pattern of bf16(0x4b01) (= 2^23 + 128) broadcast over the
3884// lanes and reinterpreted as i32 -- chosen so that the resulting acc32
3885// vector, when bit-reinterpreted as f32 and subtracted from the same
3886// magic constant in accfloat space, yields the float representation of
3887// the original signed int16 input.
3888// 3. aievec.cast acc32 -> accfloat is a register bit-alias on AIE2P
3889// (the same physical accumulator register).
3890// 4. accfloat subtract of the magic constant.
3891// 5. SRS accfloat -> bf16.
3892//
3893// Lane width 16 or 32 maps directly to the native AIE2P 256/1024-bit
3894// accumulator. 64 splits into two 32-wide halves and concats the results.
3895//
3896// arith.uitofp takes u ^ 0x8000, which as a signed value is u - 32768,
3897// through the same sequence, and subtracts a magic constant 32768 smaller,
3898// 2^23 + 32768, as an f32 vector (bf16 has no such value).
3899template <typename SrcOpTy>
3900struct LowerVectorIntToFPI16BF16AIE2pPattern : OpConversionPattern<SrcOpTy> {
3901 using OpAdaptor = typename SrcOpTy::Adaptor;
3902 static constexpr bool isSigned = std::is_same_v<SrcOpTy, arith::SIToFPOp>;
3903
3904 // `accumulatorAdd`: add the magic constant on the accumulator
3905 // (aievec.add_elem, AIE2P only) rather than as an arith.addi.
3907 bool accumulatorAdd,
3908 PatternBenefit benefit)
3909 : OpConversionPattern<SrcOpTy>(context, benefit),
3911
3913
3914 Value emitMagicSiToFp(ConversionPatternRewriter &rewriter, Location loc,
3915 Value srcI16, unsigned lanes) const {
3916 Type i16Ty = rewriter.getI16Type();
3917 Type i32Ty = rewriter.getI32Type();
3918 Type bf16Ty = rewriter.getBF16Type();
3919 Type f32Ty = rewriter.getF32Type();
3920 VectorType vecI16 = VectorType::get({lanes}, i16Ty);
3921 VectorType vecI32 = VectorType::get({lanes}, i32Ty);
3922 VectorType vecBF16 = VectorType::get({lanes}, bf16Ty);
3923 VectorType vecF32 = VectorType::get({lanes}, f32Ty);
3924
3925 DenseElementsAttr magicI16Attr = DenseElementsAttr::get(
3926 vecI16, rewriter.getI16IntegerAttr(static_cast<int16_t>(0x4b01)));
3927 Value magicI16Vec =
3928 arith::ConstantOp::create(rewriter, loc, vecI16, magicI16Attr);
3929 Value magicBF16Vec =
3930 vector::BitCastOp::create(rewriter, loc, vecBF16, magicI16Vec);
3931 Value magicAccF =
3932 aievec::UPSOp::create(rewriter, loc, vecF32, magicBF16Vec);
3933 Value magicAccI = aievec::CastOp::create(rewriter, loc, vecI32, magicAccF,
3934 /*isResAcc=*/true);
3935 if (!isSigned) {
3936 Value flip = arith::ConstantOp::create(
3937 rewriter, loc, vecI16,
3938 DenseElementsAttr::get(vecI16, rewriter.getI16IntegerAttr(
3939 static_cast<int16_t>(0x8000))));
3940 srcI16 = arith::XOrIOp::create(rewriter, loc, srcI16, flip);
3941 }
3942 Value vAccI = aievec::UPSOp::create(rewriter, loc, vecI32, srcI16);
3943 Value vSumI =
3945 ? aievec::AddElemOp::create(rewriter, loc, vecI32, vAccI, magicAccI)
3946 .getResult()
3947 : arith::AddIOp::create(rewriter, loc, vAccI, magicAccI)
3948 .getResult();
3949 Value vSumAccF = aievec::CastOp::create(rewriter, loc, vecF32, vSumI,
3950 /*isResAcc=*/true);
3951 Value subtrahend = magicAccF;
3952 if (!isSigned)
3953 subtrahend = arith::ConstantOp::create(
3954 rewriter, loc, vecF32,
3955 DenseElementsAttr::get(vecF32, rewriter.getF32FloatAttr(8421376.0f)));
3956 Value vDiffAccF =
3957 aievec::SubElemOp::create(rewriter, loc, vecF32, vSumAccF, subtrahend);
3958 Value shiftCst =
3959 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
3960 return aievec::SRSOp::create(rewriter, loc, vecBF16, vDiffAccF, shiftCst)
3961 .getResult();
3962 }
3963
3964 LogicalResult
3965 matchAndRewrite(SrcOpTy op, OpAdaptor adaptor,
3966 ConversionPatternRewriter &rewriter) const override {
3967 auto srcTy = dyn_cast<VectorType>(adaptor.getIn().getType());
3968 auto dstTy = dyn_cast<VectorType>(op.getType());
3969 if (!srcTy || !dstTy || srcTy.getRank() != 1 || dstTy.getRank() != 1)
3970 return failure();
3971 if (!srcTy.getElementType().isInteger(16))
3972 return failure();
3973 if (!dstTy.getElementType().isBF16())
3974 return failure();
3975 unsigned lanes = srcTy.getNumElements();
3976 if (lanes != 16 && lanes != 32 && lanes != 64)
3977 return failure();
3978
3979 Location loc = op.getLoc();
3980 if (lanes == 16 || lanes == 32) {
3981 Value out = emitMagicSiToFp(rewriter, loc, adaptor.getIn(), lanes);
3982 rewriter.replaceOp(op, out);
3983 return success();
3984 }
3985
3986 // 64-wide: split into two 32-wide halves via vector.shuffle
3987 // (vector.extract_strided_slice is illegal in this conversion target).
3988 SmallVector<int64_t> loMask, hiMask, catMask;
3989 for (int64_t i = 0; i < 32; ++i)
3990 loMask.push_back(i);
3991 for (int64_t i = 32; i < 64; ++i)
3992 hiMask.push_back(i);
3993 for (int64_t i = 0; i < 64; ++i)
3994 catMask.push_back(i);
3995 Value lo = vector::ShuffleOp::create(rewriter, loc, adaptor.getIn(),
3996 adaptor.getIn(), loMask)
3997 .getResult();
3998 Value hi = vector::ShuffleOp::create(rewriter, loc, adaptor.getIn(),
3999 adaptor.getIn(), hiMask)
4000 .getResult();
4001 Value loBF16 = emitMagicSiToFp(rewriter, loc, lo, 32);
4002 Value hiBF16 = emitMagicSiToFp(rewriter, loc, hi, 32);
4003 Value cat =
4004 vector::ShuffleOp::create(rewriter, loc, loBF16, hiBF16, catMask)
4005 .getResult();
4006 rewriter.replaceOp(op, cat);
4007 return success();
4008 }
4009};
4014
4015// Match `arith.extui (vector.bitcast %x : vector<Mxi8> to vector<2Mxi4>)
4016// to vector<2Mxi8>` and replace with `aievec.unpack %x` directly on the
4017// byte-packed source. The vector<...xi4> intermediate never reaches Peano
4018// llc, which cannot legalize llvm.bitcast on i4 vectors.
4019//
4020// This recognises the canonical standard-MLIR encoding of an AWQ int4
4021// unpack: byte load of packed nibbles -> bitcast to i4 vector -> extui to
4022// i8 vector. The aievec.unpack op then lowers to the AIE2P intrinsic
4023// llvm.aie2p.unpack.I512.I8.I4 (or the I1024 variant) directly.
4025 : OpConversionPattern<arith::ExtUIOp> {
4026 using OpConversionPattern<arith::ExtUIOp>::OpConversionPattern;
4027
4028 LogicalResult
4029 matchAndRewrite(arith::ExtUIOp extOp, OpAdaptor adaptor,
4030 ConversionPatternRewriter &rewriter) const override {
4031 auto srcTy = dyn_cast<VectorType>(adaptor.getIn().getType());
4032 auto dstTy = dyn_cast<VectorType>(extOp.getType());
4033 if (!srcTy || !dstTy)
4034 return failure();
4035 if (srcTy.getRank() != 1 || dstTy.getRank() != 1)
4036 return failure();
4037 if (!srcTy.getElementType().isInteger(4) ||
4038 !dstTy.getElementType().isInteger(8))
4039 return failure();
4040 unsigned lanes = srcTy.getNumElements();
4041 if (lanes != dstTy.getNumElements())
4042 return failure();
4043 if (lanes != 64 && lanes != 128)
4044 return failure();
4045
4046 auto bitcastOp = adaptor.getIn().getDefiningOp<vector::BitCastOp>();
4047 if (!bitcastOp)
4048 return failure();
4049 auto bitcastSrcTy = dyn_cast<VectorType>(bitcastOp.getSource().getType());
4050 if (!bitcastSrcTy || bitcastSrcTy.getRank() != 1 ||
4051 !bitcastSrcTy.getElementType().isInteger(8))
4052 return failure();
4053 if (bitcastSrcTy.getNumElements() * 2 != lanes)
4054 return failure();
4055
4056 rewriter.replaceOpWithNewOp<aievec::UnpackOp>(extOp, dstTy,
4057 bitcastOp.getSource());
4058 return success();
4059 }
4060};
4061
4062// Scalarize a vector arith int<->fp conversion (sitofp / uitofp / fptosi /
4063// fptoui). The AIE2/AIE2P GISel legalizers only handle the scalar form of
4064// these conversions (libcalls); there is no vector int<->fp instruction.
4065// Vector forms reach the legalizer as G_SITOFP/G_FPTOSI on a vector type
4066// and abort with "unable to legalize". Lower the vector form here into
4067// per-lane extract -> scalar conversion -> insert so each lane goes through
4068// the supported scalar path.
4069template <typename SrcOpTy>
4070struct ScalarizeVectorIntFpConversionOpPattern : OpConversionPattern<SrcOpTy> {
4071 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
4072 using OpAdaptor = typename SrcOpTy::Adaptor;
4073
4074 LogicalResult
4075 matchAndRewrite(SrcOpTy op, OpAdaptor adaptor,
4076 ConversionPatternRewriter &rewriter) const override {
4077 auto srcType = dyn_cast<VectorType>(adaptor.getIn().getType());
4078 auto dstType = dyn_cast<VectorType>(op.getType());
4079 if (!srcType || !dstType)
4080 return failure(); // already scalar — let the standard path handle it
4081 if (srcType.getRank() != 1 || dstType.getRank() != 1)
4082 return failure(); // conservative: 1-D vectors only
4083 if (srcType.getNumElements() != dstType.getNumElements())
4084 return failure();
4085
4086 Location loc = op.getLoc();
4087 Type scalarDstTy = dstType.getElementType();
4088 int64_t lanes = srcType.getNumElements();
4089
4090 // Seed result with a zero vector of the destination type.
4091 Value result = arith::ConstantOp::create(rewriter, loc, dstType,
4092 rewriter.getZeroAttr(dstType));
4093
4094 for (int64_t i = 0; i < lanes; ++i) {
4095 Value scalar = vector::ExtractOp::create(rewriter, loc, adaptor.getIn(),
4096 ArrayRef<int64_t>{i});
4097 Value cvt = SrcOpTy::create(rewriter, loc, scalarDstTy, scalar);
4098 result = vector::InsertOp::create(rewriter, loc, cvt, result,
4099 ArrayRef<int64_t>{i});
4100 }
4101
4102 rewriter.replaceOp(op, result);
4103 return success();
4104 }
4105};
4106
4115
4116template <typename SrcOpTy>
4117struct LowerTruncOpPattern : OpConversionPattern<SrcOpTy> {
4118 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
4119 using OpAdaptor = typename SrcOpTy::Adaptor;
4120
4121 LogicalResult
4122 matchAndRewrite(SrcOpTy truncOp, OpAdaptor adaptor,
4123 ConversionPatternRewriter &rewriter) const override {
4124 VectorType srcType = cast<VectorType>(truncOp.getIn().getType());
4125 VectorType dstType = cast<VectorType>(truncOp.getOut().getType());
4126 Type scalarType = srcType.getElementType();
4127 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
4128 auto accType =
4129 isa<IntegerType>(scalarType) && (elWidth == 32 || elWidth == 64)
4130 ? srcType
4131 : getVectorOpDestType(srcType, /*AIE2 =*/true);
4132
4133 auto shiftParamOp = arith::ConstantOp::create(
4134 rewriter, truncOp.getLoc(), rewriter.getI32IntegerAttr(0));
4135 if (elWidth == 16) {
4136 auto upsOp = aievec::UPSOp::create(rewriter, truncOp.getLoc(), accType,
4137 truncOp.getIn());
4138 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
4139 truncOp, dstType, upsOp.getResult(), shiftParamOp.getResult());
4140 } else {
4141 auto castOp = aievec::CastOp::create(rewriter, truncOp.getLoc(), accType,
4142 truncOp.getIn(), true);
4143 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
4144 truncOp, dstType, castOp.getResult(), shiftParamOp.getResult());
4145 }
4146
4147 return success();
4148 }
4149};
4150
4153
4154// If `op` is the last operation in the sequence:
4155// %0 = unrealized_conversion_cast <%IN> : <native type>, !emitc.opaque_type
4156// %1 = emitc.call_opaque <funcName>, %0...
4157// %2 = unrealized_conversion_cast %1 : !emitc.opaque_type, <native type>
4158// return the value <%IN>.
4159static std::optional<Value>
4160getUnOpaquedOperandOfEmitCOpaqueCallOp(Operation *op, StringRef funcName) {
4161 auto uccOp = dyn_cast<UnrealizedConversionCastOp>(op);
4162 if (!uccOp)
4163 return {};
4164
4165 auto inVal = uccOp.getInputs()[0];
4166 if (!isa<emitc::OpaqueType>(inVal.getType()))
4167 return {};
4168
4169 auto callOp = inVal.getDefiningOp<emitc::CallOpaqueOp>();
4170 if (callOp.getCallee() != funcName)
4171 return {};
4172
4173 auto callOperandsUccOp =
4174 callOp.getOperands()[0].getDefiningOp<UnrealizedConversionCastOp>();
4175 if (!callOperandsUccOp)
4176 return {};
4177
4178 return callOperandsUccOp.getInputs()[0];
4179}
4180
4181// Check there is an operation chain like-
4182//
4183// %cst_0 = arith.constant dense<1.000000e+00> : vector<16xbf16>
4184// %cst_1 = arith.constant 0.000000e+00 : bf16
4185// %0 = vector.transfer_read %arg0[%arg2], %cst_1 : memref<1024xbf16>,
4186// vector<16xbf16>
4187// %1 = arith.negf %0 : vector<16xbf16>
4188// %2 = math.exp %1 : vector<16xbf16>
4189// %3 = arith.addf %2, %cst_0 : vector<16xbf16>
4190// %4 = arith.divf %cst_0, %3 : vector<16xbf16>
4191//
4192// so that this operation chain can be converted to a function call to compute
4193// sigmoid value for v16bfloat16 and v32bfloat16 types
4194template <typename DivFOpTy>
4195static bool hasSigmoidComputationChain(DivFOpTy divfOp, arith::NegFOp &negOp) {
4196 auto *lhsDefOp = divfOp.getLhs().getDefiningOp();
4197 if (!lhsDefOp)
4198 return false;
4199 auto constOp = dyn_cast<arith::ConstantOp>(lhsDefOp);
4200 if (!constOp)
4201 return false;
4202
4203 auto cstDense = dyn_cast<DenseFPElementsAttr>(constOp.getValue());
4204 if (!cstDense)
4205 return false;
4206
4207 if (cstDense.template getSplatValue<APFloat>().convertToFloat() != 1.0f)
4208 return false;
4209
4210 Operation *addLvalOp;
4211 Operation *addRvalOp;
4212 // divfOp's rval could be an arith::AddFOp or the pattern like-
4213 // %1 = aievec.ups %a
4214 // %2 = aievec.ups %b;
4215 // %3 = aievec.add_elem %1, %2
4216 // %4 = aievec.srs %3;
4217 auto *rhsDefOp = divfOp.getRhs().getDefiningOp();
4218 if (!rhsDefOp)
4219 return false;
4220 auto addOp = dyn_cast<arith::AddFOp>(rhsDefOp);
4221 if (!addOp) {
4222 auto srsOp = dyn_cast<aievec::SRSOp>(rhsDefOp);
4223 if (!srsOp)
4224 return false;
4225
4226 auto addElemOp =
4227 dyn_cast<aievec::AddElemOp>(srsOp.getSource().getDefiningOp());
4228 if (!addElemOp)
4229 return false;
4230
4231 auto lUpsOp = dyn_cast<aievec::UPSOp>(addElemOp.getLhs().getDefiningOp());
4232 auto rUpsOp = dyn_cast<aievec::UPSOp>(addElemOp.getRhs().getDefiningOp());
4233 if (!lUpsOp || !rUpsOp)
4234 return false;
4235
4236 addLvalOp = lUpsOp.getSource().getDefiningOp();
4237 addRvalOp = rUpsOp.getSource().getDefiningOp();
4238 // One of add operation's operand is a constant op and another operand could
4239 // be arith::ExpOp or the combination of emitc.call and aievec.srs
4240 auto addDefOp = isa<arith::ConstantOp>(addLvalOp)
4241 ? dyn_cast<aievec::SRSOp>(addRvalOp)
4242 : dyn_cast<aievec::SRSOp>(addLvalOp);
4243 if (!addDefOp)
4244 addLvalOp = isa<arith::ConstantOp>(addLvalOp)
4245 ? dyn_cast<math::ExpOp>(addRvalOp)
4246 : dyn_cast<math::ExpOp>(addLvalOp);
4247 else
4248 addLvalOp = addDefOp.getSource().getDefiningOp();
4249
4250 addRvalOp = isa<arith::ConstantOp>(addLvalOp)
4251 ? lUpsOp.getSource().getDefiningOp()
4252 : rUpsOp.getSource().getDefiningOp();
4253 } else {
4254 addLvalOp = addOp.getLhs().getDefiningOp();
4255 addRvalOp = addOp.getRhs().getDefiningOp();
4256 }
4257
4258 if (!addLvalOp || !addRvalOp)
4259 return false;
4260
4261 auto addLvalExpOp = dyn_cast<math::ExpOp>(addLvalOp);
4262 auto addRvalExpOp = dyn_cast<math::ExpOp>(addRvalOp);
4263 auto addLvalExpOpIn =
4264 getUnOpaquedOperandOfEmitCOpaqueCallOp(addLvalOp, "getExpBf16")
4265 .value_or(nullptr);
4266 auto addRvalExpOpIn =
4267 getUnOpaquedOperandOfEmitCOpaqueCallOp(addRvalOp, "getExpBf16")
4268 .value_or(nullptr);
4269 if (!addLvalExpOpIn && addLvalExpOp)
4270 addLvalExpOpIn = addLvalExpOp.getOperand();
4271 if (!addRvalExpOpIn && addRvalExpOp)
4272 addRvalExpOpIn = addRvalExpOp.getOperand();
4273
4274 if (!((addLvalExpOpIn && isa<arith::ConstantOp>(addRvalOp)) ||
4275 (addRvalExpOpIn && isa<arith::ConstantOp>(addLvalOp))))
4276 return false;
4277
4278 constOp = isa<arith::ConstantOp>(addLvalOp)
4279 ? cast<arith::ConstantOp>(addLvalOp)
4280 : cast<arith::ConstantOp>(addRvalOp);
4281
4282 cstDense = dyn_cast<DenseFPElementsAttr>(constOp.getValue());
4283 if (!cstDense)
4284 return false;
4285 if (cstDense.template getSplatValue<APFloat>().convertToFloat() != 1.0f)
4286 return false;
4287
4288 auto expOperand = addLvalExpOpIn ? addLvalExpOpIn : addRvalExpOpIn;
4289
4290 negOp = expOperand.getDefiningOp<arith::NegFOp>();
4291
4292 return negOp != nullptr;
4293}
4294
4295// Convert the operation chain like-
4296//
4297// %cst_0 = arith.constant dense<1.000000e+00> : vector<16xbf16>
4298// %cst_1 = arith.constant 0.000000e+00 : bf16
4299// %0 = vector.transfer_read %arg0[%arg2], %cst_1 : memref<1024xbf16>,
4300// vector<16xbf16>
4301// %1 = arith.negf %0 : vector<16xbf16>
4302// %2 = math.exp %1 :vector<16xbf16>
4303// %3 = arith.addf %2, %cst_0 : vector<16xbf16>
4304// %4 = arith.divf %cst_0, %3 : vector<16xbf16>
4305//
4306// to a function call to compute sigmoid value for v16bfloat16 and
4307// v32bfloat16 types
4308struct ComputeSigmoidOpPattern : OpConversionPattern<arith::DivFOp> {
4309 using OpConversionPattern::OpConversionPattern;
4310
4311 LogicalResult
4312 matchAndRewrite(arith::DivFOp divfOp, OpAdaptor adaptor,
4313 ConversionPatternRewriter &rewriter) const override {
4314 auto srcType = dyn_cast<VectorType>(adaptor.getLhs().getType());
4315 if (!srcType)
4316 return failure();
4317
4318 Type scalarType = srcType.getElementType();
4319 if (!isa<FloatType>(scalarType))
4320 return failure();
4321
4322 unsigned laneSize = getVectorLaneSize(srcType);
4323 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
4324 if (elWidth != 16 || (laneSize != 16 && laneSize != 32))
4325 return failure();
4326
4327 arith::NegFOp negOp = nullptr;
4328 if (!hasSigmoidComputationChain(adaptor, negOp))
4329 return failure();
4330
4331 StringRef includeName = "vec_math.h";
4332 auto moduleOp = divfOp->getParentOfType<mlir::ModuleOp>();
4333 rewriter.setInsertionPointToStart(
4334 &moduleOp.getRegion().getBlocks().front());
4335 emitc::IncludeOp::create(rewriter, moduleOp.getLoc(), includeName, false);
4336
4337 rewriter.setInsertionPoint(divfOp);
4338 Type vecOpaqueTy;
4339 if (laneSize == 16)
4340 vecOpaqueTy =
4341 emitc::OpaqueType::get(rewriter.getContext(), "v16bfloat16");
4342 else
4343 vecOpaqueTy =
4344 emitc::OpaqueType::get(rewriter.getContext(), "v32bfloat16");
4345 auto opaquedOperand =
4346 UnrealizedConversionCastOp::create(rewriter, divfOp.getLoc(),
4347 vecOpaqueTy, negOp.getOperand())
4348 .getResult(0);
4349 SmallVector<Value> sigmoidOperands = {opaquedOperand};
4350 auto callOp = emitc::CallOpaqueOp::create(
4351 rewriter, divfOp.getLoc(), TypeRange{vecOpaqueTy}, "getSigmoidBf16",
4352 nullptr, nullptr, sigmoidOperands);
4353 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
4354 divfOp, TypeRange{adaptor.getLhs().getType()}, callOp.getResults());
4355
4356 return success();
4357 }
4358};
4359
4360// Convert math.ceil to a function call to compute ceil(x) for v16bfloat16
4361struct ComputeCeilOpPattern : OpConversionPattern<math::CeilOp> {
4362 using OpConversionPattern::OpConversionPattern;
4363
4364 LogicalResult
4365 matchAndRewrite(math::CeilOp ceilOp, OpAdaptor adaptor,
4366 ConversionPatternRewriter &rewriter) const override {
4367 auto srcType = dyn_cast<VectorType>(ceilOp.getOperand().getType());
4368 if (!srcType)
4369 return failure();
4370
4371 Type scalarType = srcType.getElementType();
4372 if (!isa<FloatType>(scalarType))
4373 return failure();
4374
4375 unsigned laneSize = getVectorLaneSize(srcType);
4376 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
4377 if (elWidth != 16 || (laneSize != 16 && laneSize != 32))
4378 return failure();
4379
4380 StringRef includeName = "vec_math.h";
4381 auto moduleOp = ceilOp->getParentOfType<mlir::ModuleOp>();
4382 rewriter.setInsertionPointToStart(
4383 &moduleOp.getRegion().getBlocks().front());
4384 emitc::IncludeOp::create(rewriter, moduleOp.getLoc(), includeName, false);
4385
4386 rewriter.setInsertionPoint(ceilOp);
4387 Type vecOpaqueTy;
4388 if (laneSize == 16)
4389 vecOpaqueTy =
4390 emitc::OpaqueType::get(rewriter.getContext(), "v16bfloat16");
4391 else
4392 vecOpaqueTy =
4393 emitc::OpaqueType::get(rewriter.getContext(), "v32bfloat16");
4394 auto opaquedOperand =
4395 UnrealizedConversionCastOp::create(rewriter, ceilOp.getLoc(),
4396 vecOpaqueTy, adaptor.getOperand())
4397 .getResult(0);
4398 SmallVector<Value> ceilOperands = {opaquedOperand};
4399 auto callOp = emitc::CallOpaqueOp::create(
4400 rewriter, ceilOp.getLoc(), TypeRange{vecOpaqueTy}, "getCeilBf16",
4401 nullptr, nullptr, ceilOperands);
4402 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
4403 ceilOp, TypeRange{ceilOp.getResult().getType()}, callOp.getResults());
4404
4405 return success();
4406 }
4407};
4408
4409// Convert math.floor to a function call to compute floor(x) for v16bfloat16
4410struct ComputeFloorOpPattern : OpConversionPattern<math::FloorOp> {
4411 using OpConversionPattern::OpConversionPattern;
4412
4413 LogicalResult
4414 matchAndRewrite(math::FloorOp floorOp, OpAdaptor adaptor,
4415 ConversionPatternRewriter &rewriter) const override {
4416 auto srcType = dyn_cast<VectorType>(floorOp.getOperand().getType());
4417 if (!srcType)
4418 return failure();
4419
4420 Type scalarType = srcType.getElementType();
4421 if (!isa<FloatType>(scalarType))
4422 return failure();
4423
4424 unsigned laneSize = getVectorLaneSize(srcType);
4425 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
4426 if (elWidth != 16 || (laneSize != 16 && laneSize != 32))
4427 return failure();
4428
4429 StringRef includeName = "vec_math.h";
4430 auto moduleOp = floorOp->getParentOfType<mlir::ModuleOp>();
4431 rewriter.setInsertionPointToStart(
4432 &moduleOp.getRegion().getBlocks().front());
4433 emitc::IncludeOp::create(rewriter, moduleOp.getLoc(), includeName, false);
4434
4435 rewriter.setInsertionPoint(floorOp);
4436 Type vecOpaqueTy;
4437 if (laneSize == 16)
4438 vecOpaqueTy =
4439 emitc::OpaqueType::get(rewriter.getContext(), "v16bfloat16");
4440 else
4441 vecOpaqueTy =
4442 emitc::OpaqueType::get(rewriter.getContext(), "v32bfloat16");
4443 auto opaquedOperand =
4444 UnrealizedConversionCastOp::create(rewriter, floorOp.getLoc(),
4445 vecOpaqueTy, adaptor.getOperand())
4446 .getResult(0);
4447 SmallVector<Value> floorOperands = {opaquedOperand};
4448 auto callOp = emitc::CallOpaqueOp::create(
4449 rewriter, floorOp.getLoc(), TypeRange{vecOpaqueTy}, "getFloorBf16",
4450 nullptr, nullptr, floorOperands);
4451 rewriter.replaceOpWithNewOp<UnrealizedConversionCastOp>(
4452 floorOp, TypeRange{floorOp.getResult().getType()}, callOp.getResults());
4453
4454 return success();
4455 }
4456};
4457
4458// Convert arith.negf to aievec.neg to negate the vector for v16bfloat16 and
4459// v16float types, and for v32 of either on AIE2P.
4460struct ComputeNegOpPattern : OpConversionPattern<arith::NegFOp> {
4461 using OpConversionPattern::OpConversionPattern;
4462
4463 LogicalResult
4464 matchAndRewrite(arith::NegFOp negOp, OpAdaptor adaptor,
4465 ConversionPatternRewriter &rewriter) const override {
4466 auto srcType = dyn_cast<VectorType>(negOp.getOperand().getType());
4467 if (!srcType)
4468 return failure();
4469
4470 Type scalarType = srcType.getElementType();
4471 if (!isa<FloatType>(scalarType))
4472 return failure();
4473
4474 unsigned laneSize = getVectorLaneSize(srcType);
4475 if (laneSize != 16 && laneSize != 32)
4476 return failure();
4477
4478 Location loc = negOp.getLoc();
4479 auto accType = getVectorOpDestType(srcType, /*AIE2 =*/true);
4480
4481 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
4482 if (elWidth == 16) {
4483 auto upsOp =
4484 aievec::UPSOp::create(rewriter, loc, accType, adaptor.getOperand());
4485 auto aieNegOp =
4486 aievec::NegOp::create(rewriter, loc, accType, upsOp.getResult());
4487 auto shiftParamOp = arith::ConstantOp::create(
4488 rewriter, negOp.getLoc(), rewriter.getI32IntegerAttr(0));
4489 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
4490 negOp, srcType, aieNegOp.getResult(), shiftParamOp.getResult());
4491 } else {
4492 auto castOp = aievec::CastOp::create(
4493 rewriter, loc, accType, adaptor.getOperand(), /*isResAcc*/ true);
4494 auto aieNegOp =
4495 aievec::NegOp::create(rewriter, loc, accType, castOp.getResult());
4496 rewriter.replaceOpWithNewOp<aievec::CastOp>(
4497 negOp, srcType, aieNegOp.getResult(), /*isResAcc*/ false);
4498 }
4499
4500 return success();
4501 }
4502};
4503
4504// Check whether the value of constant operation is int type and the dense value
4505// is -1.
4506static bool hasConstNegOneValue(arith::ConstantOp constOp, unsigned elWidth) {
4507 if (!constOp)
4508 return false;
4509
4510 auto cstDense = dyn_cast<DenseIntElementsAttr>(constOp.getValue());
4511 if (!cstDense)
4512 return false;
4513
4514 if (elWidth == 32)
4515 return cstDense.getSplatValue<int32_t>() == -1;
4516 if (elWidth == 16)
4517 return cstDense.getSplatValue<int16_t>() == -1;
4518 if (elWidth == 8)
4519 return cstDense.getSplatValue<int8_t>() == -1;
4520 return false;
4521}
4522
4523// Convert arith.xori to aievec.bxor to compute bitwise xor of two vectors for
4524// integer types
4525struct ComputeBxorAndBnegOpPattern : OpConversionPattern<arith::XOrIOp> {
4526 using OpConversionPattern::OpConversionPattern;
4527
4528 LogicalResult
4529 matchAndRewrite(arith::XOrIOp xorOp, OpAdaptor adaptor,
4530 ConversionPatternRewriter &rewriter) const override {
4531 auto srcType = dyn_cast<VectorType>(xorOp.getLhs().getType());
4532 if (!srcType)
4533 return failure();
4534
4535 Type scalarType = srcType.getElementType();
4536 if (!isa<IntegerType>(scalarType))
4537 return failure();
4538
4539 unsigned laneSize = getVectorLaneSize(srcType);
4540 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
4541 if (laneSize * elWidth != 512)
4542 return failure();
4543
4544 auto lhsConstOp = xorOp.getLhs().getDefiningOp<arith::ConstantOp>();
4545 auto rhsConstOp = xorOp.getRhs().getDefiningOp<arith::ConstantOp>();
4546
4547 // If one of operands in xorOp is a constant -1, xorOp will be replaced with
4548 // aievec::BnegOp.
4549 if ((lhsConstOp && hasConstNegOneValue(lhsConstOp, elWidth)) ||
4550 (rhsConstOp && hasConstNegOneValue(rhsConstOp, elWidth))) {
4551 Value val = hasConstNegOneValue(lhsConstOp, elWidth) ? adaptor.getRhs()
4552 : adaptor.getLhs();
4553 rewriter.replaceOpWithNewOp<aievec::BnegOp>(xorOp, srcType, val);
4554 } else
4555 rewriter.replaceOpWithNewOp<aievec::BxorOp>(
4556 xorOp, srcType, adaptor.getLhs(), adaptor.getRhs());
4557
4558 return success();
4559 }
4560};
4561
4562template <typename SrcOpTy, typename DstOpTy>
4563struct ComputeBandAndBorOpPattern : OpConversionPattern<SrcOpTy> {
4564 using OpConversionPattern<SrcOpTy>::OpConversionPattern;
4565 using OpAdaptor = typename SrcOpTy::Adaptor;
4566
4567 LogicalResult
4568 matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor,
4569 ConversionPatternRewriter &rewriter) const override {
4570 VectorType srcType = dyn_cast<VectorType>(srcOp.getLhs().getType());
4571 if (!srcType)
4572 return failure();
4573
4574 Type scalarType = srcType.getElementType();
4575 if (!isa<IntegerType>(scalarType))
4576 return failure();
4577
4578 unsigned laneSize = getVectorLaneSize(srcType);
4579 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
4580 if (laneSize * elWidth != 512)
4581 return failure();
4582
4583 rewriter.replaceOpWithNewOp<DstOpTy>(srcOp, srcOp.getResult().getType(),
4584 adaptor.getLhs(), adaptor.getRhs());
4585
4586 return success();
4587 }
4588};
4589
4594
4595// Convert arith.shrsi to a combination of aievec.ups and aievec.srs to compute
4596// arithmetic right shift for integer types. Currently, only support the shift
4597// value with a broadcast vector.
4599 : OpConversionPattern<arith::ShRSIOp> {
4600 using OpConversionPattern::OpConversionPattern;
4601
4602 LogicalResult
4603 matchAndRewrite(arith::ShRSIOp rsOp, OpAdaptor adaptor,
4604 ConversionPatternRewriter &rewriter) const override {
4605 auto srcType = dyn_cast<VectorType>(adaptor.getLhs().getType());
4606 if (!srcType)
4607 return failure();
4608
4609 Type scalarType = srcType.getElementType();
4610 unsigned laneSize = getVectorLaneSize(srcType);
4611 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
4612 if (laneSize * elWidth != 512)
4613 return failure();
4614
4615 auto bcastOp =
4616 dyn_cast<aievec::BroadcastOp>(adaptor.getRhs().getDefiningOp());
4617 if (!bcastOp)
4618 return failure();
4619
4620 auto constOp =
4621 arith::ConstantOp::create(rewriter, bcastOp.getLoc(),
4622 rewriter.getI32IntegerAttr(bcastOp.getIdx()));
4623 auto extElemOp = aievec::ExtElemOp::create(
4624 rewriter, bcastOp.getLoc(), scalarType, bcastOp, constOp.getResult());
4625 Location loc = rsOp.getLoc();
4626
4627 // The vector with v64int8 type can be divided into two v32int8 vectors and
4628 // be processed individually and be concatenated at the end.
4629 if (elWidth == 8) {
4630 VectorType halfSrcType = createVectorType(laneSize / 2, scalarType);
4631 auto rsOpLow = aievec::ExtOp::create(rewriter, loc, halfSrcType,
4632 adaptor.getLhs(), 0);
4633 auto rsOpHigh = aievec::ExtOp::create(rewriter, loc, halfSrcType,
4634 adaptor.getLhs(), 1);
4635 Type accType = getVectorOpDestType(halfSrcType, /*AIE2 =*/true);
4636 auto upsOpLow =
4637 aievec::UPSOp::create(rewriter, loc, accType, rsOpLow.getResult());
4638 auto srsOpLow =
4639 aievec::SRSOp::create(rewriter, loc, halfSrcType,
4640 upsOpLow.getResult(), extElemOp.getResult());
4641 auto upsOpHigh =
4642 aievec::UPSOp::create(rewriter, loc, accType, rsOpHigh.getResult());
4643 auto srsOpHigh =
4644 aievec::SRSOp::create(rewriter, loc, halfSrcType,
4645 upsOpHigh.getResult(), extElemOp.getResult());
4646 SmallVector<Value> inputSources = {srsOpLow.getResult(),
4647 srsOpHigh.getResult()};
4648 rewriter.replaceOpWithNewOp<aievec::ConcatOp>(rsOp, srcType,
4649 inputSources);
4650 } else {
4651 Type accType = getVectorOpDestType(srcType, /*AIE2 =*/true);
4652 auto upsOp =
4653 aievec::UPSOp::create(rewriter, loc, accType, adaptor.getLhs());
4654 rewriter.replaceOpWithNewOp<aievec::SRSOp>(
4655 rsOp, srcType, upsOp.getResult(), extElemOp.getResult());
4656 }
4657
4658 return success();
4659 }
4660};
4661
4662// Recognize the compound shift+clamp+truncate pattern and lower it to
4663// aievec.ups + aievec.srs. This maps the standard quantized neural network
4664// "shift-round-saturate + clamp" idiom to the AIE2 SRS hardware unit.
4665//
4666// Pattern A (with clamp):
4667// %shifted = arith.shrsi %wide, %shift_splat
4668// %clamped0 = arith.maxsi %shifted, %c_lo
4669// %clamped = arith.minsi %clamped0, %c_hi
4670// %result = arith.trunci %clamped
4671//
4672// Pattern B (without clamp):
4673// %shifted = arith.shrsi %wide, %shift_splat
4674// %result = arith.trunci %shifted
4675//
4676// Both lower to: aievec.ups + aievec.srs
4677struct ShiftClampTruncToSRSPattern : OpConversionPattern<arith::TruncIOp> {
4678 using OpConversionPattern::OpConversionPattern;
4679
4680 ShiftClampTruncToSRSPattern(MLIRContext *context, PatternBenefit benefit = 2)
4681 : OpConversionPattern(context, benefit) {}
4682
4683 // Try to extract a scalar integer splat value from a Value.
4684 // Returns std::nullopt if the value is not a constant splat.
4685 static std::optional<int64_t> getConstantSplatValue(Value val) {
4686 auto defOp = val.getDefiningOp<arith::ConstantOp>();
4687 if (!defOp)
4688 return std::nullopt;
4689 auto denseAttr = dyn_cast<DenseIntElementsAttr>(defOp.getValue());
4690 if (!denseAttr || !denseAttr.isSplat())
4691 return std::nullopt;
4692 return denseAttr.getSplatValue<APInt>().getSExtValue();
4693 }
4694
4695 // Try to extract the shift amount from the right operand of shrsi.
4696 // Accepts either a constant splat vector or an aievec.broadcast of a
4697 // constant.
4698 static std::optional<Value>
4699 getShiftValue(Value rhs, ConversionPatternRewriter &rewriter, Location loc) {
4700 // Case 1: constant splat vector
4701 if (auto constOp = rhs.getDefiningOp<arith::ConstantOp>()) {
4702 auto denseAttr = dyn_cast<DenseIntElementsAttr>(constOp.getValue());
4703 if (denseAttr && denseAttr.isSplat()) {
4704 int64_t shiftVal = denseAttr.getSplatValue<APInt>().getSExtValue();
4705 return arith::ConstantOp::create(rewriter, loc,
4706 rewriter.getI32IntegerAttr(shiftVal))
4707 .getResult();
4708 }
4709 }
4710 // Case 2: aievec.broadcast
4711 if (auto bcastOp = dyn_cast<aievec::BroadcastOp>(rhs.getDefiningOp())) {
4712 auto constOp = arith::ConstantOp::create(
4713 rewriter, bcastOp.getLoc(),
4714 rewriter.getI32IntegerAttr(bcastOp.getIdx()));
4715 return aievec::ExtElemOp::create(rewriter, bcastOp.getLoc(),
4716 rewriter.getI32Type(), bcastOp,
4717 constOp.getResult())
4718 .getResult();
4719 }
4720 return std::nullopt;
4721 }
4722
4723 LogicalResult
4724 matchAndRewrite(arith::TruncIOp truncOp, OpAdaptor adaptor,
4725 ConversionPatternRewriter &rewriter) const override {
4726 auto dstType = dyn_cast<VectorType>(truncOp.getOut().getType());
4727 if (!dstType)
4728 return failure();
4729
4730 Type dstScalarType = dstType.getElementType();
4731 if (!isa<IntegerType>(dstScalarType))
4732 return failure();
4733
4734 // Walk backward through optional clamp chain
4735 Value source = adaptor.getIn();
4736 int32_t sign = 1; // default: signed
4737
4738 // Check for minsi(maxsi(...), hi) or maxsi(minsi(...), lo) clamp pattern
4739 arith::MinSIOp minOp = nullptr;
4740 arith::MaxSIOp maxOp = nullptr;
4741
4742 if (auto minsiOp = source.getDefiningOp<arith::MinSIOp>()) {
4743 if (auto maxsiOp = minsiOp.getLhs().getDefiningOp<arith::MaxSIOp>()) {
4744 minOp = minsiOp;
4745 maxOp = maxsiOp;
4746 source = maxOp.getLhs();
4747 }
4748 } else if (auto maxsiOp = source.getDefiningOp<arith::MaxSIOp>()) {
4749 if (auto minsiOp = maxsiOp.getLhs().getDefiningOp<arith::MinSIOp>()) {
4750 maxOp = maxsiOp;
4751 minOp = minsiOp;
4752 source = minOp.getLhs();
4753 }
4754 }
4755
4756 // If we found a clamp, verify it's a valid saturation range
4757 if (minOp && maxOp) {
4758 auto loVal = getConstantSplatValue(maxOp.getRhs());
4759 auto hiVal = getConstantSplatValue(minOp.getRhs());
4760 if (!loVal || !hiVal)
4761 return failure();
4762
4763 unsigned dstBits = dstScalarType.getIntOrFloatBitWidth();
4764 // Guard against UB from shifting into or past the sign bit.
4765 if (dstBits == 0 || dstBits > 63)
4766 return failure();
4767 uint64_t one = 1ULL;
4768 int64_t unsignedLo = 0;
4769 int64_t unsignedHi = static_cast<int64_t>((one << dstBits) - 1);
4770 int64_t signedLo = -static_cast<int64_t>(one << (dstBits - 1));
4771 int64_t signedHi = static_cast<int64_t>((one << (dstBits - 1)) - 1);
4772
4773 if (*loVal == unsignedLo && *hiVal == unsignedHi) {
4774 sign = 0; // unsigned saturation
4775 } else if (*loVal == signedLo && *hiVal == signedHi) {
4776 sign = 1; // signed saturation
4777 } else {
4778 // Clamp range doesn't match standard saturation — don't match
4779 return failure();
4780 }
4781 }
4782
4783 // Now source should be the shrsi result
4784 auto shrsiOp = source.getDefiningOp<arith::ShRSIOp>();
4785 if (!shrsiOp)
4786 return failure();
4787
4788 auto srcType = dyn_cast<VectorType>(shrsiOp.getLhs().getType());
4789 if (!srcType)
4790 return failure();
4791
4792 Type srcScalarType = srcType.getElementType();
4793 if (!isa<IntegerType>(srcScalarType))
4794 return failure();
4795
4796 unsigned srcElWidth = srcScalarType.getIntOrFloatBitWidth();
4797 unsigned dstElWidth = dstScalarType.getIntOrFloatBitWidth();
4798 if (dstElWidth >= srcElWidth)
4799 return failure();
4800
4801 Location loc = truncOp.getLoc();
4802
4803 // Extract the shift amount
4804 auto shiftVal = getShiftValue(shrsiOp.getRhs(), rewriter, loc);
4805 if (!shiftVal)
4806 return failure();
4807
4808 // Get the wide input (pre-shift)
4809 Value wideInput = shrsiOp.getLhs();
4810
4811 unsigned laneSize = getVectorLaneSize(srcType);
4812 bool needsPadding = (laneSize % 16 != 0);
4813
4814 VectorType paddedSrcType = srcType;
4815 VectorType paddedDstType = dstType;
4816
4817 if (needsPadding) {
4818 // Round up to nearest multiple of 16
4819 unsigned paddedLanes = ((laneSize + 15) / 16) * 16;
4820 paddedSrcType = createVectorType(paddedLanes, srcScalarType);
4821 paddedDstType = createVectorType(paddedLanes, dstScalarType);
4822
4823 // Zero-pad the input using insert_strided_slice
4824 auto zeroAttr = rewriter.getZeroAttr(paddedSrcType);
4825 auto zeroPad =
4826 arith::ConstantOp::create(rewriter, loc, zeroAttr).getResult();
4827 SmallVector<int64_t> offsets(1, 0);
4828 SmallVector<int64_t> strides(1, 1);
4829 wideInput = vector::InsertStridedSliceOp::create(
4830 rewriter, loc, wideInput, zeroPad, offsets, strides)
4831 .getResult();
4832 }
4833
4834 // Determine accumulator type and create the accumulator value.
4835 // For i16 source: use UPS to widen to accumulator type.
4836 // For i32/i64 source: use CastOp (marks as accumulator without widening),
4837 // matching the approach in LowerTruncOpPattern.
4838 Type accScalarType = paddedSrcType.getElementType();
4839 unsigned accElWidth = accScalarType.getIntOrFloatBitWidth();
4840 Value accValue;
4841 if (accElWidth == 16) {
4842 Type accType = getVectorOpDestType(paddedSrcType, /*AIE2=*/true);
4843 accValue =
4844 aievec::UPSOp::create(rewriter, loc, accType, wideInput).getResult();
4845 } else {
4846 // For i32/i64: CastOp with isResAcc=true marks as accumulator
4847 accValue = aievec::CastOp::create(rewriter, loc, paddedSrcType, wideInput,
4848 /*isResAcc=*/true)
4849 .getResult();
4850 }
4851 auto srsOp = aievec::SRSOp::create(rewriter, loc, paddedDstType, accValue,
4852 *shiftVal, sign);
4853
4854 Value result = srsOp.getResult();
4855
4856 if (needsPadding) {
4857 // Extract original lanes from the padded result
4858 SmallVector<int64_t> offsets(1, 0);
4859 SmallVector<int64_t> sizes = {static_cast<int64_t>(laneSize)};
4860 SmallVector<int64_t> strides(1, 1);
4861 result = vector::ExtractStridedSliceOp::create(rewriter, loc, result,
4862 offsets, sizes, strides)
4863 .getResult();
4864 }
4865
4866 rewriter.replaceOp(truncOp, result);
4867
4868 // Erase the intermediate clamp/shift ops if they have no other uses.
4869 // These must be cleaned up because shrsi on 512-bit vectors is marked
4870 // illegal and would cause conversion failure if left as dead ops.
4871 SmallVector<Operation *, 3> opsToErase;
4872 if (minOp && minOp->use_empty())
4873 opsToErase.push_back(minOp);
4874 if (maxOp && maxOp->use_empty())
4875 opsToErase.push_back(maxOp);
4876 if (shrsiOp->use_empty())
4877 opsToErase.push_back(shrsiOp);
4878 for (Operation *op : opsToErase)
4879 rewriter.eraseOp(op);
4880
4881 return success();
4882 }
4883};
4884
4885// Promote scalar shrsi + [clamp] + trunci chain to a vectorized SRS sequence.
4886// Anchored on scalar arith::TruncIOp. Fuses the entire chain into:
4887// broadcast_scalar -> cast(isResAcc) -> srs(narrowed, shift, sign) ->
4888// ext_elem
4889// This prevents scalar arith.trunci i32->i8 from reaching the AIE2 backend
4890// where it crashes (the backend only supports 32->16/20/32 truncations).
4891struct LowerScalarShiftClampTruncToSRS : OpConversionPattern<arith::TruncIOp> {
4892 using OpConversionPattern::OpConversionPattern;
4893
4894 LogicalResult
4895 matchAndRewrite(arith::TruncIOp truncOp, OpAdaptor adaptor,
4896 ConversionPatternRewriter &rewriter) const override {
4897 // Only match scalar types (vector compound pattern handles vectors)
4898 if (isa<VectorType>(truncOp.getOut().getType()))
4899 return failure();
4900 std::optional<ScalarSRSChain> chain = matchScalarSRSChain(truncOp);
4901 if (!chain)
4902 return failure();
4903 Value source = chain->source;
4904 int32_t sign = chain->sign;
4905 arith::MinSIOp minOp = chain->minOp;
4906 arith::MaxSIOp maxOp = chain->maxOp;
4907 auto srcIntType = cast<IntegerType>(truncOp.getIn().getType());
4908 auto dstIntType = cast<IntegerType>(truncOp.getOut().getType());
4909 unsigned dstBits = dstIntType.getWidth();
4910
4911 // Now source should be the shrsi result, or any i32 value for clamp-only
4912 Location loc = truncOp.getLoc();
4913 Value preShiftVal;
4914 Value shiftVal;
4915 arith::ShRSIOp shrsiOp = source.getDefiningOp<arith::ShRSIOp>();
4916 if (shrsiOp) {
4917 // Verify shrsi operand types
4918 if (!isa<IntegerType>(shrsiOp.getLhs().getType()))
4919 return failure();
4920 preShiftVal = shrsiOp.getLhs();
4921 shiftVal = shrsiOp.getRhs();
4922 } else {
4923 // No shrsi found: treat as identity shift (shift=0).
4924 // This handles clamp+trunci patterns without a preceding shift,
4925 // e.g., after skip-add with skip_scale=0.
4926 preShiftVal = source;
4927 shiftVal = arith::ConstantOp::create(rewriter, loc,
4928 rewriter.getI32IntegerAttr(0));
4929 }
4930
4931 // Create 512-bit vector type: vector<16xi32>
4932 unsigned srcLanes = 512 / srcIntType.getWidth(); // 16 for i32
4933 VectorType bcastVecType = createVectorType(srcLanes, srcIntType);
4934
4935 // Broadcast pre-shift value to 512-bit vector
4936 auto bcast = aievec::BroadcastScalarOp::create(rewriter, loc, bcastVecType,
4937 preShiftVal);
4938
4939 Value accValue;
4940 VectorType srsOutType;
4941
4942 if (dstBits == 8) {
4943 // i32→i8: The SRS intrinsic I256V32Acc32Srs needs 1024-bit source
4944 // (vector<32xi32>, cast to vector<16xi64> internally) and produces
4945 // vector<32xi8> (256-bit). Concat two broadcast copies to get 1024 bits.
4946 unsigned accLanes = srcLanes * 2; // 32
4947 VectorType accVecType =
4948 createVectorType(accLanes, srcIntType); // vector<32xi32>
4949 auto concatSrc = aievec::ConcatOp::create(
4950 rewriter, loc, accVecType,
4951 SmallVector<Value>({bcast.getResult(), bcast.getResult()}));
4952 accValue =
4953 aievec::CastOp::create(rewriter, loc, accVecType,
4954 concatSrc.getResult(), /*isResAcc=*/true)
4955 .getResult();
4956 srsOutType = createVectorType(accLanes, dstIntType); // vector<32xi8>
4957 } else {
4958 // i32→i16: The SRS intrinsic I256V16Acc32Srs needs 512-bit source
4959 // (vector<16xi32>, cast to vector<8xi64> internally) and produces
4960 // vector<16xi16> (256-bit). 512-bit broadcast works directly.
4961 accValue = aievec::CastOp::create(rewriter, loc, bcastVecType,
4962 bcast.getResult(), /*isResAcc=*/true)
4963 .getResult();
4964 srsOutType = createVectorType(srcLanes, dstIntType); // vector<16xi16>
4965 }
4966
4967 // SRS: accumulator → narrowed output with shift and sign
4968 auto srsOp = aievec::SRSOp::create(rewriter, loc, srsOutType, accValue,
4969 shiftVal, sign);
4970
4971 // ExtElem needs 512-bit source. SRS output is 256-bit, so concat to 512.
4972 unsigned extLanes = 512 / dstBits; // 64 for i8, 32 for i16
4973 VectorType extVecType = createVectorType(extLanes, dstIntType);
4974 auto concatForExt = aievec::ConcatOp::create(
4975 rewriter, loc, extVecType,
4976 SmallVector<Value>({srsOp.getResult(), srsOp.getResult()}));
4977
4978 // Extract element 0 back to scalar
4979 auto zeroIdx =
4980 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
4981 rewriter.replaceOpWithNewOp<aievec::ExtElemOp>(
4982 truncOp, dstIntType, concatForExt.getResult(), zeroIdx.getResult());
4983
4984 // Erase dead intermediate ops
4985 SmallVector<Operation *, 3> opsToErase;
4986 if (minOp && minOp->use_empty())
4987 opsToErase.push_back(minOp);
4988 if (maxOp && maxOp->use_empty())
4989 opsToErase.push_back(maxOp);
4990 if (shrsiOp && shrsiOp->use_empty())
4991 opsToErase.push_back(shrsiOp);
4992 for (Operation *op : opsToErase)
4993 rewriter.eraseOp(op);
4994
4995 return success();
4996 }
4997};
4998
4999// Promote scalar arith.shrsi to vector aievec.ups + aievec.srs to prevent
5000// LLVM's SLP vectorizer from creating sub-512-bit vector shifts that the
5001// AIE2 backend cannot legalize (G_LSHR on <4 x s32>).
5002struct LowerScalarShRSIToAIEVecUPSSRS : OpConversionPattern<arith::ShRSIOp> {
5003 using OpConversionPattern::OpConversionPattern;
5004
5005 LogicalResult
5006 matchAndRewrite(arith::ShRSIOp rsOp, OpAdaptor adaptor,
5007 ConversionPatternRewriter &rewriter) const override {
5008 // Only match scalar i32
5009 Type resultType = rsOp.getType();
5010 if (isa<VectorType>(resultType))
5011 return failure();
5012
5013 auto intType = dyn_cast<IntegerType>(resultType);
5014 if (!intType || intType.getWidth() != 32)
5015 return failure();
5016
5017 Location loc = rsOp.getLoc();
5018 VectorType vecType = createVectorType(16, intType); // vector<16xi32>
5019
5020 // Broadcast scalar value to 512-bit vector
5021 auto lhsBcast = aievec::BroadcastScalarOp::create(rewriter, loc, vecType,
5022 adaptor.getLhs());
5023
5024 // UPS: vector<16xi32> -> accumulator type (vector<16xi64>)
5025 Type accType = getVectorOpDestType(vecType, /*AIE2=*/true);
5026 auto upsOp =
5027 aievec::UPSOp::create(rewriter, loc, accType, lhsBcast.getResult());
5028
5029 // SRS: accumulator + i32 shift -> vector<16xi32>
5030 auto srsOp = aievec::SRSOp::create(rewriter, loc, vecType,
5031 upsOp.getResult(), adaptor.getRhs());
5032
5033 // Extract element 0 back to scalar
5034 auto zeroIdx =
5035 arith::ConstantOp::create(rewriter, loc, rewriter.getI32IntegerAttr(0));
5036 rewriter.replaceOpWithNewOp<aievec::ExtElemOp>(
5037 rsOp, intType, srsOp.getResult(), zeroIdx.getResult());
5038 return success();
5039 }
5040};
5041
5042// A `vector.contract` read as one matmul tile: the sizes of its m, n and k
5043// dims, and the order each operand stores them in once its unit dims are
5044// dropped. An m (n) dim indexes lhs (rhs) and acc, a k dim indexes lhs and
5045// rhs and is reduced. Unit dims may sit anywhere and play any role.
5047 int64_t m = 1, n = 1, k = 1;
5048 // lhs stored [k, m], rhs stored [n, k], acc stored [n, m]. Only set when
5049 // both dims of the operand are larger than 1.
5050 bool lhsTransposed = false, rhsTransposed = false, accTransposed = false;
5051};
5052
5053static LogicalResult analyzeContraction(vector::ContractionOp op,
5054 ContractionTile &tile,
5055 std::string &why) {
5056 llvm::raw_string_ostream os(why);
5057 if (op.getKind() != vector::CombiningKind::ADD) {
5058 os << "the combining kind is not add";
5059 return failure();
5060 }
5061 auto accTy = dyn_cast<VectorType>(op.getAccType());
5062 if (!accTy) {
5063 os << "the accumulator is a scalar";
5064 return failure();
5065 }
5066 SmallVector<AffineMap, 3> maps = op.getIndexingMapsArray();
5067 if (!llvm::all_of(maps, [](AffineMap map) {
5068 return map.isProjectedPermutation(/*allowZeroInResults=*/false);
5069 })) {
5070 os << "an indexing map is not a projected permutation";
5071 return failure();
5072 }
5073 enum Role { M, N, K, Other };
5074 unsigned numDims = maps[0].getNumDims();
5075 SmallVector<int64_t> dimSize(numDims, 1);
5076 SmallVector<std::array<bool, 3>> indexes(numDims, {false, false, false});
5077 ArrayRef<int64_t> shapes[3] = {op.getLhsType().getShape(),
5078 op.getRhsType().getShape(), accTy.getShape()};
5079 for (unsigned operand = 0; operand < 3; ++operand)
5080 for (auto [i, expr] : llvm::enumerate(maps[operand].getResults())) {
5081 unsigned d = cast<AffineDimExpr>(expr).getPosition();
5082 dimSize[d] = shapes[operand][i];
5083 indexes[d][operand] = true;
5084 }
5085 SmallVector<vector::IteratorType> iterators = op.getIteratorTypesArray();
5086 std::optional<unsigned> dimOf[3];
5087 for (unsigned d = 0; d < numDims; ++d) {
5088 if (dimSize[d] == 1)
5089 continue;
5090 auto [inLhs, inRhs, inAcc] = indexes[d];
5091 bool reduced = iterators[d] == vector::IteratorType::reduction;
5092 Role r = Other;
5093 if (inLhs && !inRhs && inAcc && !reduced)
5094 r = M;
5095 else if (!inLhs && inRhs && inAcc && !reduced)
5096 r = N;
5097 else if (inLhs && inRhs && !inAcc && reduced)
5098 r = K;
5099 if (r == Other || dimOf[r]) {
5100 os << "d" << d << " of size " << dimSize[d]
5101 << " is not the only m, n or k dim larger than 1";
5102 return failure();
5103 }
5104 dimOf[r] = d;
5105 }
5106 auto sizeOf = [&](Role r) -> int64_t {
5107 std::optional<unsigned> d = dimOf[r];
5108 return d ? dimSize[*d] : 1;
5109 };
5110 tile.m = sizeOf(M);
5111 tile.n = sizeOf(N);
5112 tile.k = sizeOf(K);
5113 // Whether `second` comes before `first` among the results of `map`.
5114 auto swapped = [&](AffineMap map, Role first, Role second) {
5115 if (!dimOf[first] || !dimOf[second])
5116 return false;
5117 return *map.getResultPosition(
5118 getAffineDimExpr(*dimOf[second], map.getContext())) <
5119 *map.getResultPosition(
5120 getAffineDimExpr(*dimOf[first], map.getContext()));
5121 };
5122 tile.lhsTransposed = swapped(maps[0], M, K);
5123 tile.rhsTransposed = swapped(maps[1], K, N);
5124 tile.accTransposed = swapped(maps[2], M, N);
5125 return success();
5126}
5127
5128// Convert a `vector.contract` op to an `aievec.matmul` op for AIE2 or
5129// `aievec.matmul_aie2p` for AIE2P. The operands keep the layout the
5130// contraction stores them in, recorded by the lhs_transposed and
5131// rhs_transposed flags; the lowering of the matmul op moves data only where
5132// the multiply reads an operand the other way. An accumulator stored [n, m]
5133// is handled by multiplying rhs^T by lhs^T, which swaps the operands and
5134// flips both of their flags.
5135template <typename MatMulOpTy>
5137 : OpConversionPattern<vector::ContractionOp> {
5138 using OpConversionPattern::OpConversionPattern;
5139
5140 static constexpr bool isAIE2P =
5141 std::is_same_v<MatMulOpTy, aievec::MatMulOp_AIE2P>;
5142
5143 static Value reshape(OpBuilder &b, Value v, ArrayRef<int64_t> shape) {
5144 auto vecTy = cast<VectorType>(v.getType());
5145 auto newTy = VectorType::get(shape, vecTy.getElementType());
5146 if (newTy == vecTy)
5147 return v;
5148 return vector::ShapeCastOp::create(b, v.getLoc(), newTy, v).getResult();
5149 }
5150
5151 LogicalResult
5152 matchAndRewrite(vector::ContractionOp contractOp, OpAdaptor adaptor,
5153 ConversionPatternRewriter &rewriter) const override {
5154 StringRef target = isAIE2P ? "AIE2P" : "AIE2";
5155 std::string why;
5156 ContractionTile tile;
5157 if (failed(analyzeContraction(contractOp, tile, why))) {
5158 contractOp.emitError()
5159 << "cannot lower to an " << target << " matrix multiply: " << why;
5160 return failure();
5161 }
5162
5163 Value lhsIn = adaptor.getLhs(), rhsIn = adaptor.getRhs();
5164 Value lhsOrig = contractOp.getLhs(), rhsOrig = contractOp.getRhs();
5165 int64_t m = tile.m, n = tile.n, k = tile.k;
5166 bool lhsT = tile.lhsTransposed, rhsT = tile.rhsTransposed;
5167 if (tile.accTransposed) {
5168 std::swap(lhsIn, rhsIn);
5169 std::swap(lhsOrig, rhsOrig);
5170 std::swap(m, n);
5171 // A flag only matters, and is only set, when both dims are larger
5172 // than 1.
5173 std::tie(lhsT, rhsT) =
5174 std::make_pair(!rhsT && m > 1 && k > 1, !lhsT && n > 1 && k > 1);
5175 }
5176 SmallVector<int64_t, 2> lhsShape =
5177 lhsT ? SmallVector<int64_t, 2>{k, m} : SmallVector<int64_t, 2>{m, k};
5178 SmallVector<int64_t, 2> rhsShape =
5179 rhsT ? SmallVector<int64_t, 2>{n, k} : SmallVector<int64_t, 2>{k, n};
5180 auto accTy =
5181 VectorType::get({m, n}, getElementTypeOrSelf(adaptor.getAcc()));
5182
5183 // Recover operand signedness from the widening op before it is peeled.
5184 // Consult both the (possibly already-rewritten) adaptor operands and the
5185 // original pre-conversion contraction operands.
5186 auto lhsSigned = getSignednessOfWideningOp(lhsIn);
5187 if (!lhsSigned)
5188 lhsSigned = getSignednessOfWideningOp(lhsOrig);
5189 auto rhsSigned = getSignednessOfWideningOp(rhsIn);
5190 if (!rhsSigned)
5191 rhsSigned = getSignednessOfWideningOp(rhsOrig);
5192
5193 // Carry the signedness in the matmul operand element type (si8/ui8). MLIR
5194 // has no signless->signed/unsigned cast, so materialize an
5195 // unrealized_conversion_cast; it folds to a no-op once both sides lower to
5196 // signless LLVM integers. bf16/unknown-signedness operands are unchanged.
5197 auto retype = [&](Value v, std::optional<bool> isSigned) -> Value {
5198 if (!isSigned)
5199 return v;
5200 auto vecTy = cast<VectorType>(v.getType());
5201 auto intTy = dyn_cast<IntegerType>(vecTy.getElementType());
5202 if (!intTy || !intTy.isSignless())
5203 return v;
5204 auto signednessTy = IntegerType::get(intTy.getContext(), intTy.getWidth(),
5205 *isSigned ? IntegerType::Signed
5206 : IntegerType::Unsigned);
5207 return UnrealizedConversionCastOp::create(rewriter, v.getLoc(),
5208 vecTy.clone(signednessTy), v)
5209 .getResult(0);
5210 };
5211
5212 auto tileFits = [&](Value lhs, Value rhs, bool report) {
5213 auto emitError = [&]() {
5214 return contractOp.emitError()
5215 << "cannot lower to an " << target << " matrix multiply: ";
5216 };
5217 return checkMatMulTile(
5218 isAIE2P, VectorType::get(lhsShape, getElementTypeOrSelf(lhs)),
5219 VectorType::get(rhsShape, getElementTypeOrSelf(rhs)), accTy,
5220 lhsT, rhsT,
5221 report ? llvm::function_ref<InFlightDiagnostic()>(emitError)
5222 : nullptr)
5223 .has_value();
5224 };
5225 // Multiply the operands as they are if they form a tile. Otherwise they
5226 // may be widened from a type a multiply takes, as when a linalg op is
5227 // converted to contractions with its operands cast to the accumulator
5228 // type outside the contraction.
5229 if (!tileFits(lhsIn, rhsIn, /*report=*/false)) {
5230 lhsIn = getSourceOfWideningOp(lhsIn).value_or(lhsIn);
5231 rhsIn = getSourceOfWideningOp(rhsIn).value_or(rhsIn);
5232 if (!tileFits(lhsIn, rhsIn, /*report=*/true))
5233 return failure();
5234 }
5235 Value acc = reshape(rewriter, adaptor.getAcc(), accTy.getShape());
5236 Value lhs = reshape(rewriter, lhsIn, lhsShape);
5237 Value rhs = reshape(rewriter, rhsIn, rhsShape);
5238
5239 auto matmulOp =
5240 MatMulOpTy::create(rewriter, contractOp.getLoc(), acc.getType(),
5241 retype(lhs, lhsSigned), retype(rhs, rhsSigned), acc);
5242 matmulOp.setLhsTransposed(lhsT);
5243 matmulOp.setRhsTransposed(rhsT);
5244 rewriter.replaceOp(
5245 contractOp,
5246 reshape(rewriter, matmulOp.getResult(),
5247 cast<VectorType>(adaptor.getAcc().getType()).getShape()));
5248 return success();
5249 }
5250};
5251
5256
5257// Convert a `vector.transpose` op to an `aievec.shuffle` op for AIE2.
5259 : OpConversionPattern<vector::TransposeOp> {
5260 using OpConversionPattern::OpConversionPattern;
5261 LogicalResult
5262 matchAndRewrite(vector::TransposeOp transpOp, OpAdaptor adaptor,
5263 ConversionPatternRewriter &rewriter) const override {
5264 auto resTy = transpOp.getResultVectorType();
5265 auto resShape = resTy.getShape();
5266 auto elemTyBitWidth = resTy.getElementTypeBitWidth();
5267 auto vBitWidth = std::accumulate(resShape.begin(), resShape.end(),
5268 elemTyBitWidth, std::multiplies<>());
5269 if (vBitWidth != 512)
5270 return failure();
5271
5272 if (elemTyBitWidth != 8 && elemTyBitWidth != 16 && elemTyBitWidth != 32)
5273 return failure();
5274
5275 // Verify leading dimensions are all 1.
5276 for (int64_t i = 0; i < static_cast<int64_t>(resShape.size() - 2); ++i)
5277 if (resShape[i] != 1)
5278 return failure();
5279
5280 // Only permutation of the 2 innermost dimensions are supported.
5281 ArrayRef<int64_t> perm = transpOp.getPermutation();
5282 for (int64_t i = 0; i < static_cast<int64_t>(perm.size() - 2); ++i)
5283 if (perm[i] != i)
5284 return failure();
5285 if (perm.back() != static_cast<int64_t>(perm.size() - 2))
5286 return failure();
5287
5288 auto shuffleMode = aievec::ShuffleMode::T32_4X4;
5289 if (elemTyBitWidth == 8) {
5290 switch (resShape.back()) {
5291 case 4:
5292 shuffleMode = aievec::ShuffleMode::T8_4X16;
5293 break;
5294 case 8:
5295 shuffleMode = aievec::ShuffleMode::T8_8X8;
5296 break;
5297 case 16:
5298 shuffleMode = aievec::ShuffleMode::T8_16X4;
5299 break;
5300 default:
5301 return failure();
5302 }
5303 } else if (elemTyBitWidth == 16) {
5304 switch (resShape.back()) {
5305 case 2:
5306 shuffleMode = aievec::ShuffleMode::T16_2X16;
5307 break;
5308 case 4:
5309 shuffleMode = aievec::ShuffleMode::T16_4X8;
5310 break;
5311 case 8:
5312 shuffleMode = aievec::ShuffleMode::T16_8X4;
5313 break;
5314 case 16:
5315 shuffleMode = aievec::ShuffleMode::T16_16X2;
5316 break;
5317 default:
5318 return failure();
5319 }
5320 } else if (resShape.back() != 4)
5321 return failure();
5322
5323 auto flatVecTy =
5324 VectorType::get({512 / elemTyBitWidth}, resTy.getElementType());
5325 auto loc = transpOp.getLoc();
5326 auto flatInput = vector::ShapeCastOp::create(rewriter, loc, flatVecTy,
5327 adaptor.getVector());
5328 auto shuffOp = aievec::ShuffleOp::create(rewriter, loc, flatVecTy,
5329 flatInput, nullptr, shuffleMode);
5330 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(transpOp, resTy, shuffOp);
5331
5332 return success();
5333 }
5334};
5335
5336//===----------------------------------------------------------------------===//
5337// Pattern collection
5338//===----------------------------------------------------------------------===//
5339
5340static void
5341populateAIEVecCommonConversionPatterns(RewritePatternSet &patterns) {
5342 // clang-format off
5343 patterns.add<LowerExtFOpPattern,
5346 LowerTruncIOpPattern>(patterns.getContext());
5347 // Match canonical int4 unpack (vector.bitcast + arith.extui) ahead of the
5348 // generic ExtUI/ExtSI handling; runs on both AIE2 and AIE2P call sites
5349 // but only triggers for the i4-packed shape, so it is benign for AIE2.
5350 patterns.add<LowerExtUIOfBitcastI4ToUnpackPattern>(patterns.getContext(),
5351 /*benefit=*/2);
5352 // AIE2P magic-number lowering for arith.sitofp v16i16/v32i16 -> v16/v32
5353 // bf16. Higher benefit than the generic per-lane scalarise pattern; runs
5354 // on AIE2 too but only fires when the legality predicate marks the op
5355 // illegal (currently AIE2P only).
5357 patterns.getContext(), /*accumulatorAdd=*/false, /*benefit=*/3);
5358 // clang-format on
5359}
5360
5361static void populateAIEVecV1ConversionPatterns(RewritePatternSet &patterns) {
5362 patterns.add<LowerVectorTransferReadToAIEUPD>(patterns.getContext(), 128, 512,
5363 128, 256);
5364 // clang-format off
5365 patterns.add<LowerVectorAddIOpToAIEVecAddOp,
5373 LowerVectorExtractStridedSliceOpAIEv1Pattern>(patterns.getContext());
5374 // clang-format on
5375}
5376
5377// Populate common conversion patterns for AIE2 and AIE2P
5378static void
5379populateAIEVecV2CommonConversionPatterns(RewritePatternSet &patterns) {
5380 // clang-format off
5381 // TODO: Reorder these alphabetically
5382 patterns.add<
5385 // AIE2/AIE2P have no vector int<->fp instruction; scalarize so each
5386 // lane reaches the supported scalar libcall path.
5391 >(patterns.getContext());
5392
5393 // Add the compound shift+clamp+trunc→SRS pattern with higher benefit
5394 // so it takes priority over the individual shrsi and trunci patterns.
5395 patterns.add<ShiftClampTruncToSRSPattern>(patterns.getContext(),
5396 /*benefit=*/2);
5397 // Scalar version of compound SRS with even higher benefit.
5398 patterns.add<LowerScalarShiftClampTruncToSRS>(patterns.getContext(),
5399 /*benefit=*/3);
5400 patterns.add<
5437 >(patterns.getContext());
5438 // clang-format on
5439}
5440
5441static void populateAIEVecV2ConversionPatterns(RewritePatternSet &patterns) {
5442 populateAIEVecV2CommonConversionPatterns(patterns);
5444 patterns.getContext());
5445 patterns.add<LowerVectorReductionAddBfloat16OpAIE2>(patterns.getContext());
5446 // For AIE2 with LLVMIR backend, use LUT-based exp and rsqrt
5448 patterns.getContext());
5449}
5450
5451// AIE2p-specific version of ConvertSplatToAIEBroadcast that supports direct
5452// 256-bit broadcasts without extract
5454 : OpConversionPattern<vector::BroadcastOp> {
5455 using OpConversionPattern::OpConversionPattern;
5456
5457 LogicalResult
5458 matchAndRewrite(vector::BroadcastOp bcastOp, OpAdaptor adaptor,
5459 ConversionPatternRewriter &rewriter) const override {
5460
5461 if (adaptor.getSource().getDefiningOp<vector::ExtractOp>())
5462 return failure();
5463
5464 auto resultType = cast<VectorType>(bcastOp.getResult().getType());
5465 auto flatResultType = getFlattenedVectorType(resultType);
5466 Type scalarType = resultType.getElementType();
5467 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
5468 unsigned laneSize = getVectorLaneSize(resultType);
5469 auto src = bcastOp.getSource();
5470
5471 // AIE2p supports both 256-bit and 512-bit broadcast directly
5472 if (laneSize * elWidth == 512 || laneSize * elWidth == 256) {
5473 Value newOp = aievec::BroadcastScalarOp::create(
5474 rewriter, bcastOp.getLoc(), flatResultType, src);
5475 if (resultType != flatResultType)
5476 newOp = vector::ShapeCastOp::create(rewriter, bcastOp.getLoc(),
5477 resultType, newOp);
5478 rewriter.replaceOp(bcastOp, newOp);
5479 return success();
5480 }
5481
5482 if (laneSize * elWidth == 1024) {
5483 VectorType vecType = createVectorType(512 / elWidth, scalarType);
5484 auto aieBcastOp = aievec::BroadcastScalarOp::create(
5485 rewriter, bcastOp.getLoc(), vecType, src);
5486 Value newOp = aievec::ConcatOp::create(
5487 rewriter, bcastOp.getLoc(), flatResultType,
5488 SmallVector<Value>({aieBcastOp.getResult(), aieBcastOp.getResult()}));
5489 if (resultType != flatResultType)
5490 newOp = vector::ShapeCastOp::create(rewriter, bcastOp.getLoc(),
5491 resultType, newOp);
5492 rewriter.replaceOp(bcastOp, newOp);
5493 return success();
5494 }
5495
5496 return failure();
5497 }
5498};
5499
5500// Whether FlattenElementwiseVectorToRank1Pattern rewrites `op`: an n-D
5501// elementwise op whose vector operands all have the shape of its result.
5502static bool isFlattenableElementwise(Operation *op) {
5503 if (op->getNumResults() != 1)
5504 return false;
5505 auto resultType = dyn_cast<VectorType>(op->getResult(0).getType());
5506 if (!resultType || resultType.getRank() == 1 || resultType.isScalable())
5507 return false;
5508
5509 // The contraction lowering folds an extf on its operand into the matmul.
5510 // It looks for the extf directly, so a flattened extf hidden behind a
5511 // shape_cast would leave the contraction with no lowering.
5512 if (isa<arith::ExtFOp>(op) &&
5513 llvm::any_of(op->getUsers(), [](Operation *user) {
5514 return isa<vector::ContractionOp>(user);
5515 }))
5516 return false;
5517
5518 // Only shapes that agree with the result are elementwise; anything else (a
5519 // broadcast of a differently shaped vector, say) is left to the patterns
5520 // that understand it.
5521 return llvm::all_of(op->getOperandTypes(), [&](Type type) {
5522 auto operandType = dyn_cast<VectorType>(type);
5523 return !operandType || operandType.getShape() == resultType.getShape();
5524 });
5525}
5526
5527// Rewrite an elementwise op on an n-D vector into the same op on the rank-1
5528// vector with the same lane count, between a pair of vector.shape_casts.
5529//
5530// The legality predicates count lanes with getVectorLaneSize, the product of
5531// all dimensions, while the patterns they select match rank-1 operands.
5532// Flattening first makes the two agree; the widths are left as they were.
5533// Rank 1 is excluded so this cannot re-match its own output.
5534template <typename OpTy>
5535struct FlattenElementwiseVectorToRank1Pattern : OpConversionPattern<OpTy> {
5536 using OpConversionPattern<OpTy>::OpConversionPattern;
5537 using OpAdaptor = typename OpTy::Adaptor;
5538
5539 LogicalResult
5540 matchAndRewrite(OpTy op, OpAdaptor adaptor,
5541 ConversionPatternRewriter &rewriter) const override {
5542 if (!isFlattenableElementwise(op))
5543 return failure();
5544
5545 auto resultType = cast<VectorType>(op.getType());
5546 auto flatResultType = getFlattenedVectorType(resultType);
5547 Location loc = op.getLoc();
5548
5549 SmallVector<Value> flatOperands;
5550 flatOperands.reserve(adaptor.getOperands().size());
5551 for (Value operand : adaptor.getOperands()) {
5552 auto operandType = dyn_cast<VectorType>(operand.getType());
5553 if (!operandType) {
5554 flatOperands.push_back(operand);
5555 continue;
5556 }
5557 auto flatOperandType = getFlattenedVectorType(operandType);
5558 // Rematerialise a splat at the flat type instead of shape_casting it:
5559 // patterns that match on a constant operand (the reciprocal lowering
5560 // looks for a dividend of 1.0) cannot see through a shape_cast.
5561 if (auto cstOp = operand.template getDefiningOp<arith::ConstantOp>()) {
5562 if (auto dense = dyn_cast<DenseElementsAttr>(cstOp.getValue())) {
5563 if (dense.isSplat()) {
5564 flatOperands.push_back(arith::ConstantOp::create(
5565 rewriter, loc,
5566 DenseElementsAttr::get(flatOperandType,
5567 dense.getSplatValue<Attribute>())));
5568 continue;
5569 }
5570 }
5571 }
5572 flatOperands.push_back(
5573 vector::ShapeCastOp::create(rewriter, loc, flatOperandType, operand));
5574 }
5575
5576 auto flatOp = OpTy::create(rewriter, loc, TypeRange{flatResultType},
5577 flatOperands, op.getProperties(),
5578 op->getDiscardableAttrDictionary().getValue());
5579 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, resultType,
5580 flatOp->getResult(0));
5581 return success();
5582 }
5583};
5584
5585// The lane counts at which the AIE2P lowering of `op` takes one op, smallest
5586// first, or none if it has no such lowering. For a compare they are those of
5587// its operands.
5588static SmallVector<int64_t, 5> getAIE2PNativeLanes(Operation *op) {
5589 Value v = isa<arith::CmpIOp, arith::CmpFOp>(op) ? op->getOperand(0)
5590 : op->getResult(0);
5591 auto type = dyn_cast<VectorType>(v.getType());
5592 if (!type || !type.getElementType().isIntOrFloat())
5593 return {};
5594 Type elemType = type.getElementType();
5595 unsigned bitWidth = elemType.getIntOrFloatBitWidth();
5596 bool isInt = isa<IntegerType>(elemType) &&
5597 (bitWidth == 8 || bitWidth == 16 || bitWidth == 32);
5598
5599 // f32 adds and subtracts run on the 2048-bit accumulator, which holds 64
5600 // lanes; the LLVM lowering pads 16 or 32 lanes to it. bf16 ones are moved
5601 // to the accumulator and back at 16 lanes, and 32 lanes as two halves.
5602 if (isa<arith::AddFOp, arith::SubFOp>(op)) {
5603 if (elemType.isF32())
5604 return {16, 32, 64};
5605 if (elemType.isBF16())
5606 return {16, 32};
5607 return {};
5608 }
5609 // Integer adds and subtracts are left to LLVM, which takes one 512-bit
5610 // register or less. Splitting wider ones here rather than in LLVM lets the
5611 // parts feed the parts of a split compare directly.
5612 if (isa<arith::AddIOp, arith::SubIOp>(op)) {
5613 if (!isInt)
5614 return {};
5615 SmallVector<int64_t, 5> lanes;
5616 for (int64_t n = 2; n <= 512 / bitWidth; n *= 2)
5617 lanes.push_back(n);
5618 return lanes;
5619 }
5620 // The tanh and exp2 intrinsics take 16 lanes of accfloat, which f32 maps
5621 // onto as well as bf16; 32 lanes are taken as two halves. erf is computed
5622 // with tanh.
5623 if (isa<math::TanhOp, math::ErfOp, math::Exp2Op>(op)) {
5624 if (elemType.isF32() || elemType.isBF16())
5625 return {16, 32};
5626 return {};
5627 }
5628 // Float max and min take one 512-bit register of bf16.
5629 if (isa<arith::MaximumFOp, arith::MaxNumFOp, arith::MinimumFOp>(op)) {
5630 if (elemType.isBF16())
5631 return {32};
5632 return {};
5633 }
5634 // Compare and select take 32 lanes of bf16 or i16, 16 of which are padded
5635 // to 32, and 16 lanes of i32. The LLVM lowering compares bf16 as ordered
5636 // (false if either is NaN); a compare it does not honour is not given a
5637 // native width, so it is not split or repeated.
5638 if (isa<arith::CmpIOp, arith::CmpFOp, arith::SelectOp>(op)) {
5639 if (auto cmpOp = dyn_cast<arith::CmpFOp>(op);
5640 cmpOp && !llvm::is_contained({CmpFPredicate::OEQ, CmpFPredicate::ONE,
5641 CmpFPredicate::OLT, CmpFPredicate::OLE,
5642 CmpFPredicate::OGT, CmpFPredicate::OGE},
5643 cmpOp.getPredicate()))
5644 return {};
5645 if (elemType.isBF16() || elemType.isInteger(16))
5646 return {16, 32};
5647 if (elemType.isInteger(32))
5648 return {16};
5649 return {};
5650 }
5651 return {};
5652}
5653
5654// Whether the AIE2P lowering of `op` takes it at its lane count.
5655static bool isAIE2PNativeWidth(Operation *op) {
5656 auto type = dyn_cast<VectorType>(op->getResult(0).getType());
5657 return type && !type.isScalable() &&
5658 llvm::is_contained(getAIE2PNativeLanes(op), type.getNumElements());
5659}
5660
5661// The compare that `op`, a select, selects on.
5662static Operation *getSelectCompare(arith::SelectOp op) {
5663 Operation *cmp = skipShapeCastRoundTrip(op.getCondition()).getDefiningOp();
5664 return isa_and_nonnull<arith::CmpIOp, arith::CmpFOp>(cmp) ? cmp : nullptr;
5665}
5666
5667// aievec.ext numbers the parts of its source with an i8.
5668constexpr int64_t kAIE2PMaxParts = 128;
5669
5670// The lanes of each part an elementwise op is rewritten into when the AIE2P
5671// lowering does not take its lane count, or none. Lanes are counted over all
5672// dimensions. A vector is split into parts of the widest native width that
5673// divides it or, if none does, repeated to the narrowest native width that
5674// holds a whole number of copies.
5675//
5676// A compare is split together with the select that uses it, as the parts of
5677// its mask cannot be put back together; the parts are as wide as both the
5678// compare and the select take.
5679static std::optional<int64_t> getAIE2PPartLanes(Operation *op) {
5680 if (op->getNumResults() != 1 || isa<arith::CmpIOp, arith::CmpFOp>(op))
5681 return std::nullopt;
5682 auto type = dyn_cast<VectorType>(op->getResult(0).getType());
5683 if (!type || type.isScalable())
5684 return std::nullopt;
5685 int64_t lanes = type.getNumElements();
5686 if (lanes < 2 || llvm::any_of(
5687 op->getOperandTypes(),
5688 [&](Type t) {
5689 auto vt = dyn_cast<VectorType>(t);
5690 return !vt || vt.getShape() != type.getShape();
5691 }))
5692 return std::nullopt;
5693
5694 SmallVector<int64_t, 5> native = getAIE2PNativeLanes(op);
5695 if (auto selOp = dyn_cast<arith::SelectOp>(op)) {
5696 Operation *cmp = getSelectCompare(selOp);
5697 if (!cmp)
5698 return std::nullopt;
5699 SmallVector<int64_t, 5> cmpNative = getAIE2PNativeLanes(cmp);
5700 llvm::erase_if(
5701 native, [&](int64_t n) { return !llvm::is_contained(cmpNative, n); });
5702 }
5703 if (native.empty() || llvm::is_contained(native, lanes))
5704 return std::nullopt;
5705 // LLVM takes an integer add of one register whatever its lane count.
5706 if (isa<arith::AddIOp, arith::SubIOp>(op) && lanes < native.back())
5707 return std::nullopt;
5708
5709 for (int64_t part : llvm::reverse(native))
5710 if (part < lanes && lanes % part == 0 && lanes / part <= kAIE2PMaxParts)
5711 return part;
5712 if (isa<arith::SelectOp>(op))
5713 return std::nullopt;
5714 for (int64_t part : native)
5715 if (part > lanes && part % lanes == 0)
5716 return part;
5717 return std::nullopt;
5718}
5719
5720// Part `index` of `partLanes` lanes of `v`, or `v` repeated to `partLanes`
5721// lanes when it has fewer. A splat is built again at the width of the part,
5722// which keeps the parts of a splat equal, and the part of a concatenation of
5723// such parts is taken from it directly.
5724static Value getAIE2PPart(ConversionPatternRewriter &rewriter, Location loc,
5725 Value v, int64_t partLanes, int64_t index) {
5726 auto type = cast<VectorType>(v.getType());
5727 auto partType = VectorType::get({partLanes}, type.getElementType());
5728 int64_t lanes = type.getNumElements();
5729 if (auto cstOp = v.getDefiningOp<arith::ConstantOp>())
5730 if (auto dense = dyn_cast<DenseElementsAttr>(cstOp.getValue());
5731 dense && dense.isSplat())
5732 return arith::ConstantOp::create(
5733 rewriter, loc,
5734 DenseElementsAttr::get(partType, dense.getSplatValue<Attribute>()));
5735 if (auto bcastOp = v.getDefiningOp<vector::BroadcastOp>())
5736 if (!isa<VectorType>(bcastOp.getSourceType()))
5737 return vector::BroadcastOp::create(rewriter, loc, partType,
5738 bcastOp.getSource());
5739 if (partLanes < lanes) {
5740 if (auto concatOp = v.getDefiningOp<aievec::ConcatOp>())
5741 if (concatOp.getSources().front().getType() == partType)
5742 return concatOp.getSources()[index];
5743 return aievec::ExtOp::create(rewriter, loc, partType, v, index);
5744 }
5745 SmallVector<Value> copies(partLanes / lanes, v);
5746 return aievec::ConcatOp::create(rewriter, loc, partType, copies);
5747}
5748
5749// Part `index` of the mask of `selOp`. The mask is not cut apart: its compare
5750// is built again on the parts of the compare's operands. The compare is taken
5751// from `selOp` itself, as the mask it is converted to may not be a vector.
5752static Value getAIE2PMaskPart(ConversionPatternRewriter &rewriter, Location loc,
5753 arith::SelectOp selOp, int64_t partLanes,
5754 int64_t index) {
5755 Operation *cmp = getSelectCompare(selOp);
5756 OperationState state(loc, cmp->getName());
5757 for (Value operand : cmp->getOperands())
5758 state.addOperands(getAIE2PPart(
5759 rewriter, loc, rewriter.getRemappedValue(operand), partLanes, index));
5760 state.addTypes(VectorType::get({partLanes}, rewriter.getI1Type()));
5761 state.addAttributes(cmp->getAttrs());
5762 return rewriter.create(state)->getResult(0);
5763}
5764
5765// Rewrite a rank-1 elementwise op whose lane count the AIE2P lowering does
5766// not take into the same op on parts of a lane count it does take (see
5767// getAIE2PPartLanes): ops on the parts joined by aievec.concat, or one op on
5768// the repeated operands followed by aievec.ext of the lanes wanted. A select
5769// takes its compare along.
5770template <typename OpTy>
5771struct NormalizeLanesAIE2PPattern : OpConversionPattern<OpTy> {
5772 using OpConversionPattern<OpTy>::OpConversionPattern;
5773 using OpAdaptor = typename OpTy::Adaptor;
5774
5775 LogicalResult
5776 matchAndRewrite(OpTy op, OpAdaptor adaptor,
5777 ConversionPatternRewriter &rewriter) const override {
5778 auto type = dyn_cast<VectorType>(op->getResult(0).getType());
5779 if (!type || type.getRank() != 1)
5780 return failure();
5781 std::optional<int64_t> partLanes = getAIE2PPartLanes(op);
5782 if (!partLanes)
5783 return failure();
5784 int64_t lanes = type.getNumElements();
5785 auto partType = VectorType::get({*partLanes}, type.getElementType());
5786 Location loc = op.getLoc();
5787 auto createPart = [&](int64_t index) -> Value {
5788 SmallVector<Value> operands;
5789 for (Value operand : adaptor.getOperands()) {
5790 if constexpr (std::is_same_v<OpTy, arith::SelectOp>) {
5791 if (operands.empty()) {
5792 operands.push_back(
5793 getAIE2PMaskPart(rewriter, loc, op, *partLanes, index));
5794 continue;
5795 }
5796 }
5797 operands.push_back(
5798 getAIE2PPart(rewriter, loc, operand, *partLanes, index));
5799 }
5800 return OpTy::create(rewriter, loc, TypeRange{partType}, operands,
5801 op.getProperties(),
5802 op->getDiscardableAttrDictionary().getValue())
5803 ->getResult(0);
5804 };
5805
5806 if (*partLanes > lanes) {
5807 rewriter.replaceOpWithNewOp<aievec::ExtOp>(op, type, createPart(0), 0);
5808 return success();
5809 }
5810 SmallVector<Value> parts;
5811 for (int64_t i = 0; i < lanes / *partLanes; ++i)
5812 parts.push_back(createPart(i));
5813 rewriter.replaceOpWithNewOp<aievec::ConcatOp>(op, type, parts);
5814 return success();
5815 }
5816};
5817
5818// The elementwise ops AIE2P flattens to rank 1 and, for those with an entry
5819// in getAIE2PNativeLanes, brings to a lane count its lowering takes. An n-D
5820// op the rank-1 patterns would leave alone is flattened all the same: the
5821// LLVM lowering cannot take an n-D vector either.
5822template <typename... OpTys>
5824 static void addPatterns(RewritePatternSet &patterns, PatternBenefit benefit) {
5826 NormalizeLanesAIE2PPattern<OpTys>...>(patterns.getContext(),
5827 benefit);
5828 }
5829 static void markIllegal(ConversionTarget &target, MLIRContext *ctx) {
5830 (markIllegal<OpTys>(target, ctx), ...);
5831 }
5832 // An n-D op to flatten, or one to bring to another lane count, is illegal.
5833 // An op that is already illegal stays so; a dynamically legal one keeps its
5834 // predicate for the rest.
5835 template <typename OpTy>
5836 static void markIllegal(ConversionTarget &target, MLIRContext *ctx) {
5837 std::optional<ConversionTarget::LegalizationAction> action =
5838 target.getOpAction(OperationName(OpTy::getOperationName(), ctx));
5839 if (action == ConversionTarget::LegalizationAction::Legal)
5840 target.addDynamicallyLegalOp<OpTy>([](Operation *op) {
5841 return !isFlattenableElementwise(op) && !getAIE2PPartLanes(op);
5842 });
5843 else if (action == ConversionTarget::LegalizationAction::Dynamic)
5844 target.addDynamicallyLegalOp<OpTy>(
5845 [](Operation *op) -> std::optional<bool> {
5846 if (isFlattenableElementwise(op) || getAIE2PPartLanes(op))
5847 return false;
5848 return std::nullopt;
5849 });
5850 }
5851};
5853 arith::MulFOp, arith::DivFOp, arith::AddFOp, arith::SubFOp, arith::NegFOp,
5854 arith::TruncFOp, arith::ExtFOp, math::ExpOp, math::TanhOp, math::ErfOp,
5855 math::Exp2Op, arith::MaximumFOp, arith::MaxNumFOp, arith::MinimumFOp,
5856 arith::CmpFOp, arith::CmpIOp, arith::SelectOp, arith::AddIOp, arith::SubIOp,
5857 arith::MulIOp, arith::IndexCastOp>;
5858
5859static void populateAIEVecV2PConversionPatterns(
5860 RewritePatternSet &patterns, std::shared_ptr<AIE2PShufflePlanner> planner) {
5861 populateAIEVecV2CommonConversionPatterns(patterns);
5863 patterns.getContext());
5865 ConvertMulFToAIEVecMulElemOpAIE2PPattern>(patterns.getContext(),
5866 /*benefit=*/2);
5869 patterns.getContext(), /*accumulatorAdd=*/true, /*benefit=*/4);
5870 // AIE2p-specific broadcast pattern that handles 256-bit directly
5871 patterns.add<ConvertSplatToAIEBroadcastAIE2p>(patterns.getContext());
5872 populateAIE2PShufflePlannerPatterns(patterns, std::move(planner));
5873 patterns.add<LowerVectorReductionAddBfloat16OpAIE2P>(patterns.getContext());
5874 // For AIE2P with LLVMIR backend, use aievec.exp and aievec.inv
5875 // math.rsqrt is kept legal and will be lowered in AIEVecToLLVM pass
5876 patterns
5878 patterns.getContext());
5879 // Higher benefit than the AIE2P-specific patterns below, which would
5880 // otherwise take an n-D op before it has been flattened.
5881 AIE2PElementwise::addPatterns(patterns, /*benefit=*/3);
5882 // Higher benefit to take priority over the AIE2 LUT-based tanh and the
5883 // emitc-based erf patterns registered in the common patterns.
5884 patterns.add<
5894 /*benefit=*/2);
5895}
5896
5897//===----------------------------------------------------------------------===//
5898// Legalizations
5899//===----------------------------------------------------------------------===//
5900
5901// TODO: Review the validity of these legalizations beyond basic cases.
5902
5903static bool isInSigmoidOperationChain(math::ExpOp expOp) {
5904 if (!expOp.getOperand().getDefiningOp<arith::NegFOp>())
5905 return false;
5906
5907 arith::AddFOp addOp = nullptr;
5908 for (Operation *user : expOp->getUsers()) {
5909 addOp = dyn_cast<arith::AddFOp>(user);
5910 if (addOp)
5911 break;
5912 }
5913
5914 if (!addOp)
5915 return false;
5916
5917 auto *addLvalOp = addOp.getLhs().getDefiningOp();
5918 auto *addRvalOp = addOp.getRhs().getDefiningOp();
5919 if (!((isa<math::ExpOp>(addLvalOp) && isa<arith::ConstantOp>(addRvalOp)) ||
5920 (isa<math::ExpOp>(addRvalOp) && isa<arith::ConstantOp>(addLvalOp))))
5921 return false;
5922
5923 auto constOp = isa<arith::ConstantOp>(addLvalOp)
5924 ? cast<arith::ConstantOp>(addLvalOp)
5925 : cast<arith::ConstantOp>(addRvalOp);
5926
5927 auto cstDense = dyn_cast<DenseFPElementsAttr>(constOp.getValue());
5928 if (!cstDense)
5929 return false;
5930
5931 if (cstDense.getSplatValue<APFloat>().convertToFloat() != 1.0f)
5932 return false;
5933
5934 arith::DivFOp divOp = nullptr;
5935 for (Operation *user : addOp->getUsers()) {
5936 divOp = dyn_cast<arith::DivFOp>(user);
5937 if (divOp)
5938 break;
5939 }
5940
5941 if (!divOp)
5942 return false;
5943
5944 constOp = dyn_cast<arith::ConstantOp>(divOp.getLhs().getDefiningOp());
5945 if (!constOp)
5946 return false;
5947 cstDense = dyn_cast<DenseFPElementsAttr>(constOp.getValue());
5948 if (!cstDense)
5949 return false;
5950 if (cstDense.getSplatValue<APFloat>().convertToFloat() != 1.0f)
5951 return false;
5952
5953 return true;
5954}
5955
5956static void configureAIEVecCommonLegalizations(ConversionTarget &target) {
5957 target
5958 .addLegalDialect<xilinx::aievec::aie1::AIEVecAIE1Dialect,
5959 xilinx::aievec::AIEVecDialect, arith::ArithDialect,
5960 ub::UBDialect, emitc::EmitCDialect, func::FuncDialect>();
5961 target.addIllegalOp<vector::ExtractStridedSliceOp>();
5962 target.addLegalOp<vector::BitCastOp>();
5963
5964 target.addDynamicallyLegalOp<arith::ExtFOp>([](arith::ExtFOp extfOp) {
5965 auto srcType = dyn_cast<VectorType>(extfOp.getIn().getType());
5966 auto dstType = dyn_cast<VectorType>(extfOp.getOut().getType());
5967 if (!srcType || !dstType)
5968 return true;
5969
5970 Type srcScalarType = srcType.getElementType();
5971 Type dstScalarType = dstType.getElementType();
5972 if (!isa<FloatType>(srcScalarType) || !isa<FloatType>(dstScalarType))
5973 return true;
5974
5975 unsigned srcLaneSize = getVectorLaneSize(srcType);
5976 unsigned dstLaneSize = getVectorLaneSize(dstType);
5977 unsigned srcElWidth = srcScalarType.getIntOrFloatBitWidth();
5978 unsigned dstElWidth = dstScalarType.getIntOrFloatBitWidth();
5979 return srcElWidth != 16 || srcLaneSize != 16 || dstElWidth != 32 ||
5980 dstLaneSize != 16;
5981 });
5982
5983 target.addDynamicallyLegalOp<arith::ExtSIOp>([](arith::ExtSIOp extsiOp) {
5984 auto srcType = dyn_cast<VectorType>(extsiOp.getIn().getType());
5985 auto dstType = dyn_cast<VectorType>(extsiOp.getOut().getType());
5986 if (!srcType || !dstType)
5987 return true;
5988
5989 Type srcScalarType = srcType.getElementType();
5990 Type dstScalarType = dstType.getElementType();
5991 if (!isa<IntegerType>(srcScalarType) || !isa<IntegerType>(dstScalarType))
5992 return true;
5993
5994 unsigned srcLaneSize = getVectorLaneSize(srcType);
5995 unsigned dstLaneSize = getVectorLaneSize(dstType);
5996 unsigned srcElWidth = srcScalarType.getIntOrFloatBitWidth();
5997 unsigned dstElWidth = dstScalarType.getIntOrFloatBitWidth();
5998 return srcLaneSize != 32 || (dstElWidth <= srcElWidth) ||
5999 (dstLaneSize != srcLaneSize);
6000 });
6001
6002 // Mark `arith.extui vector<Nxi4> to vector<Nxi8>` illegal precisely when
6003 // the producer is a `vector.bitcast vector<N/2xi8> to vector<Nxi4>` of a
6004 // supported lane count (64 or 128 nibbles). That is the only shape
6005 // LowerExtUIOfBitcastI4ToUnpackPattern can rewrite; stray ExtUI ops
6006 // (including i4 ones not fed by a bitcast) stay legal and pass through
6007 // to the standard arith-to-llvm path.
6008 target.addDynamicallyLegalOp<arith::ExtUIOp>([](arith::ExtUIOp extuiOp) {
6009 auto srcType = dyn_cast<VectorType>(extuiOp.getIn().getType());
6010 auto dstType = dyn_cast<VectorType>(extuiOp.getOut().getType());
6011 if (!srcType || !dstType)
6012 return true;
6013 if (srcType.getRank() != 1 || dstType.getRank() != 1)
6014 return true;
6015 Type srcScalarType = srcType.getElementType();
6016 Type dstScalarType = dstType.getElementType();
6017 if (!srcScalarType.isInteger(4) || !dstScalarType.isInteger(8))
6018 return true;
6019 unsigned lanes = srcType.getNumElements();
6020 if (lanes != 64 && lanes != 128)
6021 return true;
6022 auto bitcastOp = extuiOp.getIn().getDefiningOp<vector::BitCastOp>();
6023 if (!bitcastOp)
6024 return true;
6025 auto bitcastSrcType = dyn_cast<VectorType>(bitcastOp.getSource().getType());
6026 return !bitcastSrcType || bitcastSrcType.getRank() != 1 ||
6027 !bitcastSrcType.getElementType().isInteger(8) ||
6028 bitcastSrcType.getNumElements() * 2 != lanes;
6029 });
6030
6031 target.addDynamicallyLegalOp<arith::TruncFOp>([](arith::TruncFOp truncfOp) {
6032 auto srcType = dyn_cast<VectorType>(truncfOp.getIn().getType());
6033 auto dstType = dyn_cast<VectorType>(truncfOp.getOut().getType());
6034 if (!srcType || !dstType)
6035 return true;
6036
6037 Type srcScalarType = srcType.getElementType();
6038 Type dstScalarType = dstType.getElementType();
6039 if (!isa<FloatType>(srcScalarType) || !isa<FloatType>(dstScalarType))
6040 return true;
6041
6042 unsigned srcLaneSize = getVectorLaneSize(srcType);
6043 unsigned dstLaneSize = getVectorLaneSize(dstType);
6044 unsigned srcElWidth = srcScalarType.getIntOrFloatBitWidth();
6045 unsigned dstElWidth = dstScalarType.getIntOrFloatBitWidth();
6046 return srcElWidth != 32 || srcLaneSize != 16 || dstElWidth != 16 ||
6047 dstLaneSize != 16;
6048 });
6049
6050 target.addDynamicallyLegalOp<arith::TruncIOp>([](arith::TruncIOp trunciOp) {
6051 auto srcType = dyn_cast<VectorType>(trunciOp.getIn().getType());
6052 auto dstType = dyn_cast<VectorType>(trunciOp.getOut().getType());
6053 if (!srcType || !dstType) {
6054 // Scalar trunci: mark illegal if part of compound SRS chain
6055 // so the LowerScalarShiftClampTruncToSRS pattern can convert it.
6056 return !(!srcType && !dstType && isSRSCompoundCandidate(trunciOp));
6057 }
6058
6059 Type srcScalarType = srcType.getElementType();
6060 Type dstScalarType = dstType.getElementType();
6061 if (!isa<IntegerType>(srcScalarType) || !isa<IntegerType>(dstScalarType))
6062 return true;
6063
6064 // Also mark vector trunci as illegal if it's part of a compound SRS chain
6065 if (isSRSCompoundCandidate(trunciOp))
6066 return false;
6067
6068 unsigned srcLaneSize = getVectorLaneSize(srcType);
6069 unsigned dstLaneSize = getVectorLaneSize(dstType);
6070 unsigned srcElWidth = srcScalarType.getIntOrFloatBitWidth();
6071 unsigned dstElWidth = dstScalarType.getIntOrFloatBitWidth();
6072
6073 return srcLaneSize != 32 || (dstElWidth >= srcElWidth) ||
6074 (dstLaneSize != srcLaneSize);
6075 });
6076
6077 target.addDynamicallyLegalOp<math::TanhOp>([](math::TanhOp tanhOp) {
6078 auto srcType = dyn_cast<VectorType>(tanhOp.getOperand().getType());
6079 if (!srcType)
6080 return true;
6081
6082 Type scalarType = srcType.getElementType();
6083 if (!isa<FloatType>(scalarType))
6084 return true;
6085
6086 unsigned laneSize = getVectorLaneSize(srcType);
6087 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6088 return elWidth != 16 || laneSize != 16;
6089 });
6090
6091 target.addDynamicallyLegalOp<math::SqrtOp>([](math::SqrtOp sqrtOp) {
6092 auto srcType = dyn_cast<VectorType>(sqrtOp.getOperand().getType());
6093 if (!srcType)
6094 return true;
6095
6096 Type scalarType = srcType.getElementType();
6097 if (!isa<FloatType>(scalarType))
6098 return true;
6099
6100 unsigned laneSize = getVectorLaneSize(srcType);
6101 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6102 return elWidth != 16 || (laneSize != 16 && laneSize != 32);
6103 });
6104
6105 target.addDynamicallyLegalOp<math::ErfOp>([](math::ErfOp erfOp) {
6106 auto srcType = dyn_cast<VectorType>(erfOp.getOperand().getType());
6107 if (!srcType)
6108 return true;
6109
6110 Type scalarType = srcType.getElementType();
6111 if (!isa<FloatType>(scalarType))
6112 return true;
6113
6114 unsigned laneSize = getVectorLaneSize(srcType);
6115 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6116 return elWidth != 16 || (laneSize != 16 && laneSize != 32);
6117 });
6118
6119 target.addDynamicallyLegalOp<math::AbsFOp>([](math::AbsFOp absfOp) {
6120 auto srcType = dyn_cast<VectorType>(absfOp.getOperand().getType());
6121 if (!srcType)
6122 return true;
6123
6124 Type scalarType = srcType.getElementType();
6125 unsigned laneSize = getVectorLaneSize(srcType);
6126 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6127 return elWidth * laneSize != 512 && elWidth * laneSize != 256;
6128 });
6129
6130 target.addDynamicallyLegalOp<math::AbsIOp>([](math::AbsIOp absiOp) {
6131 auto srcType = dyn_cast<VectorType>(absiOp.getOperand().getType());
6132 if (!srcType)
6133 return true;
6134
6135 Type scalarType = srcType.getElementType();
6136 unsigned laneSize = getVectorLaneSize(srcType);
6137 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6138 return elWidth * laneSize != 512 && elWidth * laneSize != 256;
6139 });
6140
6141 target.addDynamicallyLegalOp<math::CeilOp>([](math::CeilOp ceilOp) {
6142 auto srcType = dyn_cast<VectorType>(ceilOp.getOperand().getType());
6143 if (!srcType)
6144 return true;
6145 Type scalarType = srcType.getElementType();
6146 if (!isa<FloatType>(scalarType))
6147 return true;
6148
6149 unsigned laneSize = getVectorLaneSize(srcType);
6150 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6151 return elWidth != 16 || (laneSize != 16 && laneSize != 32);
6152 });
6153
6154 target.addDynamicallyLegalOp<math::FloorOp>([](math::FloorOp floorOp) {
6155 auto srcType = dyn_cast<VectorType>(floorOp.getOperand().getType());
6156 if (!srcType)
6157 return true;
6158 Type scalarType = srcType.getElementType();
6159 if (!isa<FloatType>(scalarType))
6160 return true;
6161
6162 unsigned laneSize = getVectorLaneSize(srcType);
6163 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6164 return elWidth != 16 || (laneSize != 16 && laneSize != 32);
6165 });
6166
6167 target.addDynamicallyLegalOp<arith::NegFOp>([](arith::NegFOp negOp) {
6168 auto srcType = dyn_cast<VectorType>(negOp.getOperand().getType());
6169 if (!srcType)
6170 return true;
6171 if (Type scalarType = srcType.getElementType(); !isa<FloatType>(scalarType))
6172 return true;
6173
6174 unsigned laneSize = getVectorLaneSize(srcType);
6175 return laneSize != 16;
6176 });
6177
6178 target.addDynamicallyLegalOp<arith::ShRSIOp>([](arith::ShRSIOp rsOp) {
6179 auto srcType = dyn_cast<VectorType>(rsOp.getLhs().getType());
6180 if (!srcType) {
6181 // Scalar i32 shrsi: illegal unless it feeds into a compound SRS chain
6182 // (the compound pattern consumes it via the trunci anchor)
6183 if (auto intType = dyn_cast<IntegerType>(rsOp.getLhs().getType()))
6184 if (intType.getWidth() == 32) {
6185 // legal (compound pattern will handle) iff feeding a compound SRS;
6186 // otherwise illegal so the individual pattern promotes it
6187 return shrsiUsedByCompoundSRS(rsOp);
6188 }
6189 return true;
6190 }
6191
6192 // If the shrsi feeds into a compound SRS pattern (shrsi+clamp+trunc),
6193 // keep it legal — the compound pattern will consume it via the trunci.
6194 if (shrsiUsedByCompoundSRS(rsOp))
6195 return true;
6196
6197 Type scalarType = srcType.getElementType();
6198 unsigned laneSize = getVectorLaneSize(srcType);
6199 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6200
6201 return laneSize * elWidth != 512;
6202 });
6203
6204 target.addDynamicallyLegalOp<arith::AddFOp>(
6205 [](arith::AddFOp op) { return !isa<VectorType>(op.getType()); });
6206 target.addDynamicallyLegalOp<arith::SubFOp>(
6207 [](arith::SubFOp op) { return !isa<VectorType>(op.getType()); });
6208
6209 // AIE2/AIE2P GISel legalize only the scalar form of int<->fp conversions
6210 // (libcalls). Vector forms have no legalizer rule and abort with
6211 // "unable to legalize G_SITOFP <N x sX>". Mark the vector forms that
6212 // ScalarizeVectorIntFpConversionOpPattern can rewrite (1-D vectors) as
6213 // illegal so the pattern fires. Scalar and >1-D vector forms stay legal:
6214 // scalar reaches the supported libcall path directly, and >1-D forms
6215 // pass through to surface a clearer error from a later stage instead of
6216 // aborting this conversion. The scalarize pattern emits vector.extract /
6217 // vector.insert / scalar conv sequences, so those ops must be legal too.
6218 auto isRank1Vector = [](Type t) {
6219 auto vt = dyn_cast<VectorType>(t);
6220 return vt && vt.getRank() == 1;
6221 };
6222 target.addDynamicallyLegalOp<arith::SIToFPOp>(
6223 [=](arith::SIToFPOp op) { return !isRank1Vector(op.getType()); });
6224 target.addDynamicallyLegalOp<arith::UIToFPOp>(
6225 [=](arith::UIToFPOp op) { return !isRank1Vector(op.getType()); });
6226 target.addDynamicallyLegalOp<arith::FPToSIOp>(
6227 [=](arith::FPToSIOp op) { return !isRank1Vector(op.getType()); });
6228 target.addDynamicallyLegalOp<arith::FPToUIOp>(
6229 [=](arith::FPToUIOp op) { return !isRank1Vector(op.getType()); });
6230 target.addLegalOp<vector::ExtractOp, vector::InsertOp>();
6231}
6232
6233static void configureAIEVecV1Legalizations(ConversionTarget &target) {
6234 target.addDynamicallyLegalOp<arith::MulIOp>(
6235 [](arith::MulIOp op) { return !isa<VectorType>(op.getType()); });
6236 target.addDynamicallyLegalOp<arith::MulFOp>(
6237 [](arith::MulFOp op) { return !isa<VectorType>(op.getType()); });
6238 target.addDynamicallyLegalOp<aievec::aie1::FMAOp>(
6239 [](xilinx::aievec::aie1::FMAOp op) {
6240 auto *lhsDefOp = op.getLhs().getDefiningOp();
6241 aievec::ConcatOp concatOp = nullptr;
6242 if (lhsDefOp)
6243 concatOp = dyn_cast<aievec::ConcatOp>(op.getLhs().getDefiningOp());
6244 if (!concatOp)
6245 return true;
6246
6247 vector::BroadcastOp srcBcast = nullptr;
6248 if (auto *lhsOp = concatOp.getSources()[0].getDefiningOp())
6249 srcBcast = dyn_cast<vector::BroadcastOp>(lhsOp);
6250 if (!srcBcast) {
6251 auto *rhsOp = op.getRhs().getDefiningOp();
6252 if (!rhsOp)
6253 return true;
6254 srcBcast = dyn_cast<vector::BroadcastOp>(rhsOp);
6255 }
6256
6257 if (srcBcast)
6258 if (auto *srcOp = srcBcast.getSource().getDefiningOp())
6259 return !isa<vector::ExtractOp>(srcOp);
6260
6261 return true;
6262 });
6263
6264 target.addDynamicallyLegalOp<aievec::aie1::AddOp>([](aievec::aie1::AddOp op) {
6265 auto lSrsOp = op.getLhs().getDefiningOp<aievec::SRSOp>();
6266 auto rSrsOp = op.getRhs().getDefiningOp<aievec::SRSOp>();
6267 return (!lSrsOp ||
6268 !lSrsOp.getSource().getDefiningOp<aievec::aie1::MulOp>()) &&
6269 (!rSrsOp ||
6270 !rSrsOp.getSource().getDefiningOp<aievec::aie1::MulOp>());
6271 });
6272 target.addLegalDialect<memref::MemRefDialect>();
6273}
6274
6275static void
6276configureAIEVecV2PLegalizations(ConversionTarget &target,
6277 std::shared_ptr<AIE2PShufflePlanner> planner) {
6278 // AIE2P-specific legalization for rsqrt with LLVMIR backend
6279 // Vector bf16 rsqrt is illegal (no hardware support)
6280 // Scalar f32 and vector f32 rsqrt are legal (lowered in AIEVecToLLVM pass)
6281 target.addDynamicallyLegalOp<math::RsqrtOp>([](math::RsqrtOp rsqrtOp) {
6282 auto vecType = dyn_cast<VectorType>(rsqrtOp.getOperand().getType());
6283 // Vector bf16 rsqrt is illegal; everything else is legal (scalar f32,
6284 // vector f32)
6285 return !vecType || !vecType.getElementType().isBF16();
6286 });
6287
6288 // AIE2P-specific legalization for exp with LLVMIR backend
6289 // v16bf16 and v32bf16 exp are illegal (uses hardware intrinsic)
6290 target.addDynamicallyLegalOp<math::ExpOp>([](math::ExpOp expOp) {
6291 auto srcType = dyn_cast<VectorType>(expOp.getOperand().getType());
6292 if (!srcType)
6293 return true;
6294
6295 Type scalarType = srcType.getElementType();
6296 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6297 unsigned laneSize = getVectorLaneSize(srcType);
6298 // AIE2P LLVMIR: v16f32 and v32f32 are illegal (uses aievec.exp on bf16)
6299 if (scalarType.isF32())
6300 return srcType.isScalable() || (laneSize != 16 && laneSize != 32);
6301 // AIE2P LLVMIR: v16bf16 and v32bf16 are illegal (uses aievec.exp)
6302 if (!scalarType.isBF16() || (laneSize != 16 && laneSize != 32) ||
6303 elWidth != 16)
6304 return true;
6305 if (expOp->hasOneUse() && isInSigmoidOperationChain(expOp))
6306 return true;
6307
6308 return false;
6309 });
6310
6311 // AIE2P-specific legalization for tanh, erf and exp2 with LLVMIR backend
6312 // bf16 and f32 scalars, and one-, 16- and 32-lane vectors are illegal (uses
6313 // aievec.tanh or aievec.exp2)
6314 target.addDynamicallyLegalOp<math::TanhOp>([](math::TanhOp tanhOp) {
6315 return !isAIE2PAccFloatUnaryOrScalarType(tanhOp.getOperand().getType());
6316 });
6317 target.addDynamicallyLegalOp<math::ErfOp>([](math::ErfOp erfOp) {
6318 return !isAIE2PAccFloatUnaryOrScalarType(erfOp.getOperand().getType());
6319 });
6320 target.addDynamicallyLegalOp<math::Exp2Op>([](math::Exp2Op exp2Op) {
6321 return !isAIE2PAccFloatUnaryOrScalarType(exp2Op.getOperand().getType());
6322 });
6323
6324 // AIE2P-specific legalization for divf 1.0/x pattern with LLVMIR backend
6325 // Scalar f32 or vector<Nxf32> divf with constant 1.0 LHS is illegal
6326 target.addDynamicallyLegalOp<arith::DivFOp>([](arith::DivFOp divfOp) {
6327 Type srcType = divfOp.getLhs().getType();
6328
6329 // Check if LHS is defined by a constant operation
6330 auto constOp =
6331 dyn_cast_or_null<arith::ConstantOp>(divfOp.getLhs().getDefiningOp());
6332 if (!constOp)
6333 return true;
6334
6335 // Scalar f32 case - check for exactly 1.0
6336 if (srcType.isF32()) {
6337 auto floatAttr = dyn_cast<FloatAttr>(constOp.getValue());
6338 // illegal (will be converted to aievec.inv) iff exactly 1.0
6339 return !floatAttr || !floatAttr.getValue().isExactlyValue(1.0);
6340 }
6341
6342 // Vector f32 case - check for splat of exactly 1.0
6343 if (auto vecType = dyn_cast<VectorType>(srcType)) {
6344 if (vecType.getElementType().isF32()) {
6345 unsigned laneSize = getVectorLaneSize(vecType);
6346 if (laneSize == 16 || laneSize == 32) {
6347 auto denseAttr = dyn_cast<DenseFPElementsAttr>(constOp.getValue());
6348 if (denseAttr && denseAttr.isSplat() &&
6349 denseAttr.getSplatValue<APFloat>().isExactlyValue(1.0))
6350 return false; // illegal - will be converted to aievec.inv
6351 }
6352 }
6353 }
6354
6355 return true;
6356 });
6357
6358 // AIE2P-specific legalization: ExtFOp on vector is always illegal
6359 target.addDynamicallyLegalOp<arith::ExtFOp>([](arith::ExtFOp extfOp) {
6360 auto srcType = dyn_cast<VectorType>(extfOp.getIn().getType());
6361 auto dstType = dyn_cast<VectorType>(extfOp.getOut().getType());
6362 if (!srcType || !dstType)
6363 return true;
6364
6365 Type srcScalarType = srcType.getElementType();
6366 Type dstScalarType = dstType.getElementType();
6367 if (!isa<FloatType>(srcScalarType) || !isa<FloatType>(dstScalarType))
6368 return true;
6369
6370 unsigned srcLaneSize = getVectorLaneSize(srcType);
6371 unsigned dstLaneSize = getVectorLaneSize(dstType);
6372 return (srcLaneSize % 16 != 0) || (dstLaneSize % 16 != 0);
6373 });
6374
6375 // AIE2P-specific legalization: TruncFOp on vector is always illegal
6376 target.addDynamicallyLegalOp<arith::TruncFOp>([](arith::TruncFOp truncfOp) {
6377 auto srcType = dyn_cast<VectorType>(truncfOp.getIn().getType());
6378 auto dstType = dyn_cast<VectorType>(truncfOp.getOut().getType());
6379 if (!srcType || !dstType)
6380 return true;
6381
6382 Type srcScalarType = srcType.getElementType();
6383 Type dstScalarType = dstType.getElementType();
6384 if (!isa<FloatType>(srcScalarType) || !isa<FloatType>(dstScalarType))
6385 return true;
6386
6387 unsigned srcLaneSize = getVectorLaneSize(srcType);
6388 unsigned dstLaneSize = getVectorLaneSize(dstType);
6389 return (srcLaneSize % 16 != 0) || (dstLaneSize % 16 != 0);
6390 });
6391
6392 // AIE2P-specific legalization: ExtSIOp on vector is always illegal
6393 target.addDynamicallyLegalOp<arith::ExtSIOp>([](arith::ExtSIOp extsiOp) {
6394 auto srcType = dyn_cast<VectorType>(extsiOp.getIn().getType());
6395 auto dstType = dyn_cast<VectorType>(extsiOp.getOut().getType());
6396 if (!srcType || !dstType)
6397 return true;
6398
6399 Type srcScalarType = srcType.getElementType();
6400 Type dstScalarType = dstType.getElementType();
6401 if (!isa<IntegerType>(srcScalarType) || !isa<IntegerType>(dstScalarType))
6402 return true;
6403
6404 unsigned srcLaneSize = getVectorLaneSize(srcType);
6405 unsigned dstLaneSize = getVectorLaneSize(dstType);
6406 return (srcLaneSize % 16 != 0) || (dstLaneSize % 16 != 0);
6407 });
6408
6409 // AIE2P-specific legalization: TruncIOp on vector is always illegal
6410 target.addDynamicallyLegalOp<arith::TruncIOp>([](arith::TruncIOp trunciOp) {
6411 auto srcType = dyn_cast<VectorType>(trunciOp.getIn().getType());
6412 auto dstType = dyn_cast<VectorType>(trunciOp.getOut().getType());
6413 if (!srcType || !dstType) {
6414 // Scalar trunci: mark illegal if part of compound SRS chain
6415 return !(!srcType && !dstType && isSRSCompoundCandidate(trunciOp));
6416 }
6417 Type srcScalarType = srcType.getElementType();
6418 Type dstScalarType = dstType.getElementType();
6419 if (!isa<IntegerType>(srcScalarType) || !isa<IntegerType>(dstScalarType))
6420 return true;
6421
6422 // Also mark as illegal if it's part of a shrsi+clamp+trunc SRS pattern,
6423 // even for sub-AIE-width vectors
6424 if (isSRSCompoundCandidate(trunciOp))
6425 return false;
6426
6427 unsigned srcLaneSize = getVectorLaneSize(srcType);
6428 unsigned dstLaneSize = getVectorLaneSize(dstType);
6429 return (srcLaneSize % 16 != 0) || (dstLaneSize % 16 != 0);
6430 });
6431
6432 // AIE2P-specific legalization: Override AddFOp to support laneSize==32 for
6433 // float types, and bf16 and f32 at the widths getAIE2PNativeLanes gives.
6434 target.addDynamicallyLegalOp<arith::AddFOp>([](arith::AddFOp op) {
6435 auto resultType = dyn_cast<VectorType>(op.getType());
6436 if (!resultType)
6437 return true;
6438
6439 Type scalarType = resultType.getElementType();
6440 unsigned laneSize = getVectorLaneSize(resultType);
6441
6442 if (scalarType.isBF16() || scalarType.isF32())
6443 return !isAIE2PNativeWidth(op);
6444
6445 // For other float types, support both laneSize==16 and laneSize==32
6446 if (isa<FloatType>(scalarType))
6447 return laneSize != 16 && laneSize != 32;
6448
6449 // For other types, only laneSize==16 (same as AIE2)
6450 return laneSize != 16;
6451 });
6452
6453 // AIE2P-specific legalization: Override SubFOp to support laneSize==32 for
6454 // float types, and bf16 and f32 at the widths getAIE2PNativeLanes gives.
6455 target.addDynamicallyLegalOp<arith::SubFOp>([](arith::SubFOp op) {
6456 auto resultType = dyn_cast<VectorType>(op.getType());
6457 if (!resultType)
6458 return true;
6459
6460 Type scalarType = resultType.getElementType();
6461 unsigned laneSize = getVectorLaneSize(resultType);
6462
6463 if (scalarType.isBF16() || scalarType.isF32())
6464 return !isAIE2PNativeWidth(op);
6465
6466 // For other float types, support both laneSize==16 and laneSize==32
6467 if (isa<FloatType>(scalarType))
6468 return laneSize != 16 && laneSize != 32;
6469
6470 // For other types, only laneSize==16 (same as AIE2)
6471 return laneSize != 16;
6472 });
6473
6474 // AIE2P-specific legalization: Override NegFOp to support laneSize==32 for
6475 // bf16 and f32.
6476 target.addDynamicallyLegalOp<arith::NegFOp>([](arith::NegFOp negOp) {
6477 auto srcType = dyn_cast<VectorType>(negOp.getOperand().getType());
6478 if (!srcType)
6479 return true;
6480
6481 Type scalarType = srcType.getElementType();
6482 unsigned laneSize = getVectorLaneSize(srcType);
6483
6484 // bf16 and f32 only: the aievec lowering needs an f32 accumulator, and
6485 // other 16-bit floats (e.g. f16) would take the bf16 UPS path and be
6486 // reinterpreted.
6487 if (!scalarType.isBF16() && !scalarType.isF32())
6488 return true;
6489
6490 // 16 lanes as before, at whatever rank the common config already took.
6491 if (laneSize == 16)
6492 return false;
6493
6494 // 32 only at rank 1. `getVectorLaneSize` is the product of every
6495 // dimension, so vector<2x16xf32> counts 32 as well; the widening is
6496 // scoped to the rank the aievec patterns match, and leaves n-D negates
6497 // as they were before this branch.
6498 return laneSize != 32 || srcType.getRank() != 1;
6499 });
6500
6501 // AIE2P-specific legalization: Override MulFOp to narrow the mul+add FMA
6502 // exemption to rank 1. The pattern that fuses the pair matches a rank-1
6503 // multiply, so above rank 1 the multiply has to stay illegal and be
6504 // flattened first.
6505 target.addDynamicallyLegalOp<arith::MulFOp>([](arith::MulFOp op) {
6506 auto resultType = dyn_cast<VectorType>(op.getType());
6507 if (!resultType)
6508 return true;
6509
6510 auto isAddOp = [](Operation *user) { return isa<arith::AddFOp>(user); };
6511 if (resultType.getRank() == 1 && op->hasOneUse() &&
6512 llvm::any_of(op->getUsers(), isAddOp) &&
6513 !(holdsBF16(op.getLhs()) && holdsBF16(op.getRhs())))
6514 return true;
6515
6516 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6517 unsigned laneSize = getVectorLaneSize(resultType);
6518
6519 if (laneSize == 16 && (resultElWidth == 16 || resultElWidth == 32))
6520 return false; // illegal - will be converted
6521 if (laneSize == 32 && resultElWidth == 16)
6522 return false; // illegal - will be split into two v16bf16 ops
6523 if ((laneSize == 32 || laneSize == 64) && resultElWidth == 32 &&
6524 resultType.getRank() == 1 && holdsBF16(op.getLhs()) &&
6525 holdsBF16(op.getRhs()))
6526 return false; // illegal - bf16 inputs, f32 product
6527
6528 return true; // legal - not supported
6529 });
6530
6531 // A vector.shuffle stays legal for LLVM unless the shuffle planner has a
6532 // cheaper lowering. Moves of whole register halves, such as the splitting
6533 // and joining shuffles LowerVectorSIToFPI16BF16AIE2pPattern emits, always
6534 // stay.
6535 configureAIE2PShufflePlannerLegality(target, std::move(planner));
6536}
6537
6538static void configureAIEVecV2Legalizations(ConversionTarget &target) {
6539 target.addLegalOp<UnrealizedConversionCastOp>();
6540 target.addLegalOp<vector::ShapeCastOp>();
6541
6542 // A set recording the vector lane size and element width supported
6543 llvm::SmallSet<std::pair<unsigned, unsigned>, 16> laneSizeElWidthPairSet;
6544 laneSizeElWidthPairSet.insert({64, 8});
6545 laneSizeElWidthPairSet.insert({32, 16});
6546 laneSizeElWidthPairSet.insert({16, 32});
6547 laneSizeElWidthPairSet.insert({32, 32});
6548
6549 // A set recording the element width supported
6550 llvm::SmallSet<unsigned, 16> elWidthSet;
6551 elWidthSet.insert(8);
6552 elWidthSet.insert(16);
6553 elWidthSet.insert(32);
6554
6555 target.addDynamicallyLegalOp<arith::AddFOp>([](arith::AddFOp op) {
6556 auto resultType = dyn_cast<VectorType>(op.getType());
6557 if (!resultType)
6558 return true;
6559
6560 Type scalarType = resultType.getElementType();
6561 unsigned laneSize = getVectorLaneSize(resultType);
6562 unsigned resultElWidth = scalarType.getIntOrFloatBitWidth();
6563
6564 // Support laneSize == 16 for f32/bf16
6565 if (laneSize == 16)
6566 return false; // illegal - will be converted
6567 // Support laneSize == 32 for bf16 (split into two v16bf16 ops)
6568 if (laneSize == 32 && resultElWidth == 16)
6569 return false; // illegal - will be split
6570 // Support laneSize == 32 for f32 (split into two v16f32 ops)
6571 if (laneSize == 32 && resultElWidth == 32)
6572 return false; // illegal - will be split into two v16f32 ops
6573
6574 return true; // legal - not supported
6575 });
6576
6577 target.addDynamicallyLegalOp<arith::SubFOp>([](arith::SubFOp op) {
6578 auto resultType = dyn_cast<VectorType>(op.getType());
6579 if (!resultType)
6580 return true;
6581
6582 Type scalarType = resultType.getElementType();
6583 unsigned laneSize = getVectorLaneSize(resultType);
6584 unsigned resultElWidth = scalarType.getIntOrFloatBitWidth();
6585
6586 // Support laneSize == 16 for f32/bf16
6587 if (laneSize == 16)
6588 return false; // illegal - will be converted
6589 // Support laneSize == 32 for bf16 (split into two v16bf16 ops)
6590 if (laneSize == 32 && resultElWidth == 16)
6591 return false; // illegal - will be split
6592 // Support laneSize == 32 for f32 (split into two v16f32 ops)
6593 if (laneSize == 32 && resultElWidth == 32)
6594 return false; // illegal - will be split into two v16f32 ops
6595
6596 return true; // legal - not supported
6597 });
6598
6599 target.addDynamicallyLegalOp<arith::MulIOp>([](arith::MulIOp op) {
6600 auto resultType = dyn_cast<VectorType>(op.getType());
6601 if (!resultType)
6602 return true;
6603 auto isAddOp = [&](Operation *op) { return isa<arith::AddIOp>(op); };
6604 // Verify it is not a part of MAC
6605 if (op->hasOneUse() && llvm::any_of(op->getUsers(), isAddOp))
6606 return true;
6607
6608 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6609 unsigned laneSize = getVectorLaneSize(resultType);
6610
6611 return (laneSize != 32 || (resultElWidth != 16 && resultElWidth != 8)) &&
6612 ((laneSize != 16 && laneSize != 32) || resultElWidth != 32);
6613 });
6614
6615 target.addDynamicallyLegalOp<arith::MulFOp>([](arith::MulFOp op) {
6616 auto resultType = dyn_cast<VectorType>(op.getType());
6617 if (!resultType)
6618 return true;
6619
6620 auto isAddOp = [&](Operation *op) { return isa<arith::AddFOp>(op); };
6621 // Verify it is not a part of FMA
6622 if (op->hasOneUse() && llvm::any_of(op->getUsers(), isAddOp) &&
6623 !(holdsBF16(op.getLhs()) && holdsBF16(op.getRhs())))
6624 return true;
6625
6626 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6627 unsigned laneSize = getVectorLaneSize(resultType);
6628
6629 // Support laneSize == 16 for bf16/f32, and laneSize == 32 for bf16 (split)
6630 if (laneSize == 16 && (resultElWidth == 16 || resultElWidth == 32))
6631 return false; // illegal - will be converted
6632 if (laneSize == 32 && resultElWidth == 16)
6633 return false; // illegal - will be split into two v16bf16 ops
6634 if (laneSize == 32 && resultElWidth == 32 && holdsBF16(op.getLhs()) &&
6635 holdsBF16(op.getRhs()))
6636 return false; // illegal - bf16 inputs, f32 product
6637
6638 return true; // legal - not supported
6639 });
6640
6641 target.addDynamicallyLegalOp<arith::MinSIOp>([=](arith::MinSIOp op) {
6642 auto resultType = dyn_cast<VectorType>(op.getType());
6643 if (!resultType) {
6644 // Scalar i8/i16/i32 minsi: illegal unless in compound SRS chain
6645 if (auto intType = dyn_cast<IntegerType>(op.getType())) {
6646 unsigned w = intType.getWidth();
6647 if (w == 8 || w == 16 || w == 32) {
6648 // legal (compound pattern consumes) iff feeding a compound SRS;
6649 // otherwise illegal so the individual pattern promotes it
6650 return scalarClampInCompoundSRS(op);
6651 }
6652 }
6653 return true;
6654 }
6655
6656 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6657 unsigned laneSize = getVectorLaneSize(resultType);
6658
6659 return !elWidthSet.count(resultElWidth) || laneSize * resultElWidth != 512;
6660 });
6661
6662 target.addDynamicallyLegalOp<arith::MaxSIOp>([=](arith::MaxSIOp op) {
6663 auto resultType = dyn_cast<VectorType>(op.getType());
6664 if (!resultType) {
6665 // Scalar i8/i16/i32 maxsi: illegal unless in compound SRS chain
6666 if (auto intType = dyn_cast<IntegerType>(op.getType())) {
6667 unsigned w = intType.getWidth();
6668 if (w == 8 || w == 16 || w == 32) {
6669 // legal (compound pattern consumes) iff feeding a compound SRS;
6670 // otherwise illegal so the individual pattern promotes it
6671 return scalarClampInCompoundSRS(op);
6672 }
6673 }
6674 return true;
6675 }
6676
6677 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6678 unsigned laneSize = getVectorLaneSize(resultType);
6679
6680 return !elWidthSet.count(resultElWidth) || laneSize * resultElWidth != 512;
6681 });
6682
6683 target.addDynamicallyLegalOp<arith::MinUIOp>([=](arith::MinUIOp op) {
6684 auto resultType = dyn_cast<VectorType>(op.getType());
6685 if (!resultType)
6686 return true;
6687
6688 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6689 unsigned laneSize = getVectorLaneSize(resultType);
6690 unsigned totalBits = laneSize * resultElWidth;
6691
6692 return !elWidthSet.count(resultElWidth) ||
6693 (totalBits != 512 && (totalBits != 256 || resultElWidth != 16));
6694 });
6695
6696 target.addDynamicallyLegalOp<arith::MaxUIOp>([=](arith::MaxUIOp op) {
6697 auto resultType = dyn_cast<VectorType>(op.getType());
6698 if (!resultType)
6699 return true;
6700
6701 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6702 unsigned laneSize = getVectorLaneSize(resultType);
6703 unsigned totalBits = laneSize * resultElWidth;
6704
6705 return !elWidthSet.count(resultElWidth) ||
6706 (totalBits != 512 && (totalBits != 256 || resultElWidth != 16));
6707 });
6708
6709 target.addDynamicallyLegalOp<arith::MinimumFOp>([=](arith::MinimumFOp op) {
6710 auto resultType = dyn_cast<VectorType>(op.getType());
6711 if (!resultType)
6712 return true;
6713
6714 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6715 unsigned laneSize = getVectorLaneSize(resultType);
6716 unsigned totalBits = laneSize * resultElWidth;
6717
6718 return !elWidthSet.count(resultElWidth) ||
6719 (totalBits != 512 && (totalBits != 256 || resultElWidth != 16));
6720 });
6721
6722 target.addDynamicallyLegalOp<arith::MaximumFOp>([=](arith::MaximumFOp op) {
6723 auto resultType = dyn_cast<VectorType>(op.getType());
6724 if (!resultType)
6725 return true;
6726
6727 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6728 unsigned laneSize = getVectorLaneSize(resultType);
6729 unsigned totalBits = laneSize * resultElWidth;
6730
6731 return !elWidthSet.count(resultElWidth) ||
6732 (totalBits != 512 && (totalBits != 256 || resultElWidth != 16));
6733 });
6734
6735 target.addDynamicallyLegalOp<arith::MaxNumFOp>([=](arith::MaxNumFOp op) {
6736 auto resultType = dyn_cast<VectorType>(op.getType());
6737 if (!resultType)
6738 return true;
6739
6740 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6741 unsigned laneSize = getVectorLaneSize(resultType);
6742 unsigned totalBits = laneSize * resultElWidth;
6743
6744 return !elWidthSet.count(resultElWidth) ||
6745 (totalBits != 512 && (totalBits != 256 || resultElWidth != 16));
6746 });
6747
6748 target.addDynamicallyLegalOp<arith::MinNumFOp>([=](arith::MinNumFOp op) {
6749 auto resultType = dyn_cast<VectorType>(op.getType());
6750 if (!resultType)
6751 return true;
6752
6753 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6754 unsigned laneSize = getVectorLaneSize(resultType);
6755 unsigned totalBits = laneSize * resultElWidth;
6756
6757 return !elWidthSet.count(resultElWidth) ||
6758 (totalBits != 512 && (totalBits != 256 || resultElWidth != 16));
6759 });
6760
6761 target.addDynamicallyLegalOp<arith::CmpIOp>([=](arith::CmpIOp op) {
6762 auto lhsType = dyn_cast<VectorType>(op.getLhs().getType());
6763 if (!lhsType)
6764 return true;
6765
6766 auto lhsElWidth = lhsType.getElementType().getIntOrFloatBitWidth();
6767 unsigned laneSize = getVectorLaneSize(lhsType);
6768 unsigned totalBits = laneSize * lhsElWidth;
6769
6770 return !elWidthSet.count(lhsElWidth) ||
6771 (totalBits != 512 && (totalBits != 256 || lhsElWidth != 16));
6772 });
6773
6774 target.addDynamicallyLegalOp<arith::CmpFOp>([=](arith::CmpFOp op) {
6775 auto lhsType = dyn_cast<VectorType>(op.getLhs().getType());
6776 if (!lhsType)
6777 return true;
6778
6779 auto lhsElWidth = lhsType.getElementType().getIntOrFloatBitWidth();
6780 unsigned laneSize = getVectorLaneSize(lhsType);
6781 unsigned totalBits = laneSize * lhsElWidth;
6782
6783 return !elWidthSet.count(lhsElWidth) ||
6784 (totalBits != 512 && (totalBits != 256 || lhsElWidth != 16));
6785 });
6786
6787 target.addDynamicallyLegalOp<arith::SelectOp>([=](arith::SelectOp op) {
6788 auto resultType = dyn_cast<VectorType>(op.getType());
6789 if (!resultType)
6790 return true;
6791
6792 auto resultElWidth = resultType.getElementType().getIntOrFloatBitWidth();
6793 unsigned laneSize = getVectorLaneSize(resultType);
6794 unsigned totalBits = laneSize * resultElWidth;
6795
6796 return !elWidthSet.count(resultElWidth) ||
6797 (totalBits != 512 && (totalBits != 256 || resultElWidth != 16));
6798 });
6799
6800 target.addDynamicallyLegalOp<vector::ReductionOp>(
6801 [=](vector::ReductionOp op) {
6802 if (auto kind = op.getKind(); kind != vector::CombiningKind::ADD &&
6803 kind != vector::CombiningKind::MINSI &&
6804 kind != vector::CombiningKind::MINUI &&
6805 kind != vector::CombiningKind::MINIMUMF &&
6806 kind != vector::CombiningKind::MINNUMF &&
6807 kind != vector::CombiningKind::MAXSI &&
6808 kind != vector::CombiningKind::MAXUI &&
6809 kind != vector::CombiningKind::MAXIMUMF &&
6810 kind != vector::CombiningKind::MAXNUMF)
6811 return true;
6812
6813 auto vType = dyn_cast<VectorType>(op.getVector().getType());
6814 if (!vType)
6815 return true;
6816
6817 llvm::SmallSet<std::pair<unsigned, signed>, 16> laneSizeElWidthPairSet;
6818 laneSizeElWidthPairSet.insert({64, 8});
6819 laneSizeElWidthPairSet.insert({32, 16});
6820 laneSizeElWidthPairSet.insert({32, 32});
6821 laneSizeElWidthPairSet.insert({16, 32});
6822
6823 Type scalarType = vType.getElementType();
6824 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6825 unsigned laneSize = getVectorLaneSize(vType);
6826
6827 if (isa<IntegerType>(scalarType) &&
6828 !laneSizeElWidthPairSet.count(std::make_pair(laneSize, elWidth)))
6829 return true;
6830
6831 if (isa<FloatType>(scalarType) && laneSize != 16 && laneSize != 32)
6832 return true;
6833
6834 return false;
6835 });
6836
6837 // AIE2-specific legalization: ExtFOp on vector is always illegal
6838 target.addDynamicallyLegalOp<arith::ExtFOp>([](arith::ExtFOp extfOp) {
6839 auto srcType = dyn_cast<VectorType>(extfOp.getIn().getType());
6840 auto dstType = dyn_cast<VectorType>(extfOp.getOut().getType());
6841 if (!srcType || !dstType)
6842 return true;
6843
6844 Type srcScalarType = srcType.getElementType();
6845 Type dstScalarType = dstType.getElementType();
6846 if (!isa<FloatType>(srcScalarType) || !isa<FloatType>(dstScalarType))
6847 return true;
6848
6849 unsigned srcLaneSize = getVectorLaneSize(srcType);
6850 unsigned dstLaneSize = getVectorLaneSize(dstType);
6851 return (srcLaneSize % 16 != 0) || (dstLaneSize % 16 != 0);
6852 });
6853
6854 // AIE2-specific legalization: TruncFOp on vector is always illegal
6855 target.addDynamicallyLegalOp<arith::TruncFOp>([](arith::TruncFOp truncfOp) {
6856 auto srcType = dyn_cast<VectorType>(truncfOp.getIn().getType());
6857 auto dstType = dyn_cast<VectorType>(truncfOp.getOut().getType());
6858 if (!srcType || !dstType)
6859 return true;
6860
6861 Type srcScalarType = srcType.getElementType();
6862 Type dstScalarType = dstType.getElementType();
6863 if (!isa<FloatType>(srcScalarType) || !isa<FloatType>(dstScalarType))
6864 return true;
6865
6866 unsigned srcLaneSize = getVectorLaneSize(srcType);
6867 unsigned dstLaneSize = getVectorLaneSize(dstType);
6868 return (srcLaneSize % 16 != 0) || (dstLaneSize % 16 != 0);
6869 });
6870
6871 // AIE2-specific legalization: ExtSIOp on vector is always illegal
6872 target.addDynamicallyLegalOp<arith::ExtSIOp>([](arith::ExtSIOp extsiOp) {
6873 auto srcType = dyn_cast<VectorType>(extsiOp.getIn().getType());
6874 auto dstType = dyn_cast<VectorType>(extsiOp.getOut().getType());
6875 if (!srcType || !dstType)
6876 return true;
6877
6878 Type srcScalarType = srcType.getElementType();
6879 Type dstScalarType = dstType.getElementType();
6880 if (!isa<IntegerType>(srcScalarType) || !isa<IntegerType>(dstScalarType))
6881 return true;
6882
6883 unsigned srcLaneSize = getVectorLaneSize(srcType);
6884 unsigned dstLaneSize = getVectorLaneSize(dstType);
6885 return (srcLaneSize % 16 != 0) || (dstLaneSize % 16 != 0);
6886 });
6887
6888 // AIE2-specific legalization: TruncIOp on vector is always illegal
6889 target.addDynamicallyLegalOp<arith::TruncIOp>([](arith::TruncIOp trunciOp) {
6890 auto srcType = dyn_cast<VectorType>(trunciOp.getIn().getType());
6891 auto dstType = dyn_cast<VectorType>(trunciOp.getOut().getType());
6892 if (!srcType || !dstType) {
6893 // Scalar trunci: mark illegal if part of compound SRS chain
6894 return !(!srcType && !dstType && isSRSCompoundCandidate(trunciOp));
6895 }
6896 Type srcScalarType = srcType.getElementType();
6897 Type dstScalarType = dstType.getElementType();
6898 if (!isa<IntegerType>(srcScalarType) || !isa<IntegerType>(dstScalarType))
6899 return true;
6900
6901 // Also mark as illegal if it's part of a shrsi+clamp+trunc SRS pattern,
6902 // even for sub-AIE-width vectors
6903 if (isSRSCompoundCandidate(trunciOp))
6904 return false;
6905
6906 unsigned srcLaneSize = getVectorLaneSize(srcType);
6907 unsigned dstLaneSize = getVectorLaneSize(dstType);
6908 return (srcLaneSize % 16 != 0) || (dstLaneSize % 16 != 0);
6909 });
6910
6911 target.addIllegalOp<vector::ContractionOp, vector::TransposeOp,
6912 vector::FMAOp>();
6913
6914 // AIE2-specific legalization: math.exp for v16bf16 and v32bf16 is illegal
6915 // (uses LUT)
6916 target.addDynamicallyLegalOp<math::ExpOp>([](math::ExpOp expOp) {
6917 auto srcType = dyn_cast<VectorType>(expOp.getOperand().getType());
6918 if (!srcType)
6919 return true;
6920
6921 Type scalarType = srcType.getElementType();
6922 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6923 unsigned laneSize = getVectorLaneSize(srcType);
6924 // AIE2: v16bf16 and v32bf16 are illegal (uses LUT-based lowering)
6925 if (!isa<FloatType>(scalarType) || (laneSize != 16 && laneSize != 32) ||
6926 elWidth != 16)
6927 return true;
6928 if (expOp->hasOneUse() && isInSigmoidOperationChain(expOp))
6929 return true;
6930
6931 return false;
6932 });
6933
6934 target.addDynamicallyLegalOp<math::RsqrtOp>([](math::RsqrtOp rsqrtOp) {
6935 auto srcType = dyn_cast<VectorType>(rsqrtOp.getOperand().getType());
6936 if (!srcType)
6937 return true;
6938
6939 Type scalarType = srcType.getElementType();
6940 if (!isa<FloatType>(scalarType))
6941 return true;
6942
6943 unsigned laneSize = getVectorLaneSize(srcType);
6944 unsigned elWidth = scalarType.getIntOrFloatBitWidth();
6945 return elWidth != 16 || (laneSize != 16 && laneSize != 32);
6946 });
6947}
6948
6949//===----------------------------------------------------------------------===//
6950// Lowering passes
6951//===----------------------------------------------------------------------===//
6952
6953/// Lower incoming vector operations into their corresponding AIE vector
6954/// intrinsics.
6955struct LowerVectorToAIEVec : PassWrapper<LowerVectorToAIEVec, OperationPass<>> {
6956 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerVectorToAIEVec)
6957
6960
6965
6966 // In case we want to register this pass as a standalone pass for test
6967 // purposes.
6968 StringRef getArgument() const final { return "test-lower-vector-to-aievec"; }
6969 StringRef getDescription() const final {
6970 return "Lower vector operations to AIE vector intrinsics";
6971 }
6972 void getDependentDialects(DialectRegistry &registry) const override {
6973 registry.insert<
6974 affine::AffineDialect, xilinx::aievec::aie1::AIEVecAIE1Dialect,
6975 xilinx::aievec::AIEVecDialect, arith::ArithDialect,
6976 memref::MemRefDialect, scf::SCFDialect, ub::UBDialect,
6977 vector::VectorDialect, emitc::EmitCDialect, xilinx::AIE::AIEDialect>();
6978 }
6979
6980 Option<std::string> aieTarget{
6981 *this, "aie-target",
6982 llvm::cl::desc(
6983 "Select AIE version: \"aie\", \"aie2\", or \"aie2p\". This will "
6984 "determine the vector size and available operations."),
6985 llvm::cl::init("aie")};
6986
6987 void runOnOperation() override {
6988 auto *op = getOperation();
6989 MLIRContext *context = &getContext();
6990 RewritePatternSet patterns(context);
6991 ConversionTarget target(*context);
6992
6993 // `aie.runtime_sequence` bodies are host-side control code (either NPU
6994 // instruction streams or C++ TXN builders), not AIE-core compute. Vector
6995 // and AIEVec ops are meaningless there, so leave anything nested inside a
6996 // runtime sequence untouched regardless of the legalizations configured
6997 // below.
6998 target.addLegalOp<xilinx::AIE::RuntimeSequenceOp>();
6999 target.markOpRecursivelyLegal<xilinx::AIE::RuntimeSequenceOp>();
7000
7001 std::optional<AIEVecTargetInfo> targetInfo =
7003 if (!targetInfo)
7004 return signalPassFailure();
7005
7006 populateAIEVecCommonConversionPatterns(patterns);
7007 configureAIEVecCommonLegalizations(target);
7008 if (targetInfo->isAIE1()) {
7009 populateAIEVecV1ConversionPatterns(patterns);
7010 configureAIEVecV1Legalizations(target);
7011 } else if (targetInfo->isAIE2()) {
7012 populateAIEVecV2ConversionPatterns(patterns);
7013 configureAIEVecV2Legalizations(target);
7014 } else if (targetInfo->isAIE2P()) {
7015 auto planner = std::make_shared<AIE2PShufflePlanner>();
7016 populateAIEVecV2PConversionPatterns(patterns, planner);
7017 configureAIEVecV2Legalizations(target);
7018 configureAIEVecV2PLegalizations(target, planner);
7019 // Last, as it overrides the predicates above for the ops it rewrites.
7020 AIE2PElementwise::markIllegal(target, &getContext());
7021 } else {
7022 llvm_unreachable("AIE version is misconfigured");
7023 }
7024
7025 if (failed(applyPartialConversion(op, target, std::move(patterns))))
7026 return signalPassFailure();
7027 }
7028};
7029
7030static std::unique_ptr<Pass>
7031createLowerVectorToAIEVec(const LowerVectorToAIEVecOptions &options) {
7032 return std::make_unique<LowerVectorToAIEVec>(options);
7033}
7034
7035//===---------------------------------------------------------------------------
7036// Custom canonicalization passes
7037//===---------------------------------------------------------------------------
7038
7039// This pass widens UPD ops to twice the width followed by an ext op of the
7040// bottom half. This can be used together with SimplifyUPDOpsPass to find
7041// additional common subexpressions with UPDs generated from unaligned
7042// `transfer_read` ops.
7043struct ExtendUPDOpsPass : PassWrapper<ExtendUPDOpsPass, OperationPass<>> {
7044
7045 void runOnOperation() override {
7046 MLIRContext *context = &getContext();
7047 RewritePatternSet patterns(context);
7048 ConversionTarget target(*context);
7049 patterns.add<ExpandUPDToUPDAndExtPattern>(patterns.getContext());
7050 target.addLegalDialect<aievec::AIEVecDialect>();
7051 target.addDynamicallyLegalOp<aievec::UPDOp>([](aievec::UPDOp op) {
7052 return op.getVector() ||
7053 (op->hasOneUse() && isa<aievec::UPDOp>(*op->getUsers().begin())) ||
7054 llvm::all_of(op->getUsers(),
7055 [](Operation *op) { return isa<aievec::ExtOp>(op); });
7056 });
7057
7058 if (auto *op = getOperation();
7059 failed(applyPartialConversion(op, target, std::move(patterns)))) {
7060 return signalPassFailure();
7061 }
7062 }
7063};
7064
7065// This pass replaces wide UPD ops that are only used by a single ext op of the
7066// bottom half. This pass undos the work of ExtendUPDOpsPass.
7067// TODO: This pass can be extended to work with wide UPD ops that are used by
7068// TODO: a single ext op of the top half, which might be a good opportunity to
7069// TODO: further optimize wide UPDs.
7070struct SimplifyUPDOpsPass : PassWrapper<SimplifyUPDOpsPass, OperationPass<>> {
7071
7072 void runOnOperation() override {
7073 MLIRContext *context = &getContext();
7074 RewritePatternSet patterns(context);
7075 ConversionTarget target(*context);
7076 patterns.add<FuseExtIntoUPDPattern>(patterns.getContext());
7077 target.addLegalDialect<aievec::AIEVecDialect>();
7078 target.addDynamicallyLegalOp<aievec::ExtOp>([](aievec::ExtOp op) {
7079 auto *defOp = op.getSource().getDefiningOp();
7080 return !defOp || !isa<aievec::UPDOp>(defOp) || !defOp->hasOneUse() ||
7081 op.getIndex() != 0;
7082 });
7083
7084 if (auto *op = getOperation();
7085 failed(applyPartialConversion(op, target, std::move(patterns)))) {
7086 return signalPassFailure();
7087 }
7088 }
7089};
7090
7091//============================================================================//
7092//=============== Main Vector2AIEVec Pipeline Configuration ==================//
7093//============================================================================//
7094
7096 OpPassManager &pm, const LowerVectorToAIEVecOptions &options) {
7097 // Add lowering from `Vector` to `AIEVec`
7098 pm.addPass(createLowerVectorToAIEVec(options));
7099 pm.addPass(createCanonicalizerPass());
7100
7101 // Simplify UPD ops
7102 pm.addPass(std::make_unique<ExtendUPDOpsPass>());
7103 pm.addPass(createCSEPass());
7104 pm.addPass(std::make_unique<SimplifyUPDOpsPass>());
7105 pm.addPass(createCanonicalizerPass());
7106}
LowerVectorAddOrSubOpToAIEVecAddElemOrSubElemOp< arith::SubFOp, aievec::SubElemOp > LowerVectorSubFOpToAIEVecSubElemOp
LowerVectorMinMaxOpToAIEVecMinMaxOp< arith::MaxNumFOp, aievec::MaxOp > LowerVectorMaxNumFFOpToAIEVecMaxOp
LowerVectorCmpOpToAIEVecCmpOp< arith::CmpIOp, CmpIPredicate > LowerVectorCmpIOpToAIEVecCmpOp
ScalarizeVectorIntFpConversionOpPattern< arith::FPToUIOp > ScalarizeVectorFPToUIOpPattern
AIE2PElementwiseOps< arith::MulFOp, arith::DivFOp, arith::AddFOp, arith::SubFOp, arith::NegFOp, arith::TruncFOp, arith::ExtFOp, math::ExpOp, math::TanhOp, math::ErfOp, math::Exp2Op, arith::MaximumFOp, arith::MaxNumFOp, arith::MinimumFOp, arith::CmpFOp, arith::CmpIOp, arith::SelectOp, arith::AddIOp, arith::SubIOp, arith::MulIOp, arith::IndexCastOp > AIE2PElementwise
OneToOneVectorOpToAIEVecOpPattern< arith::SubIOp, aievec::aie1::SubOp > LowerVectorSubIOpToAIEVecSubOp
ScalarizeVectorIntFpConversionOpPattern< arith::FPToSIOp > ScalarizeVectorFPToSIOpPattern
OneToOneVectorOpToAIEVecOpPattern< arith::SubFOp, aievec::aie1::SubOp > LowerVectorSubFOpToAIEVecSubOp
LowerVectorUMinMaxOpToCmpSelOp< arith::MaxUIOp, arith::CmpIPredicate::ugt > LowerVectorMaxUIOpToCmpSelOp
ScalarizeVectorIntFpConversionOpPattern< arith::UIToFPOp > ScalarizeVectorUIToFPOpPattern
LowerVectorMinMaxOpToAIEVecMinMaxOp< arith::MaxSIOp, aievec::MaxOp > LowerVectorMaxSIOpToAIEVecMaxOp
LowerScalarMinMaxToAIEVecMinMaxOp< arith::MinSIOp, aievec::MinOp > LowerScalarMinSIOpToAIEVecMinOp
LowerVectorContractionOpToAIEVecMatMulPattern< aievec::MatMulOp_AIE2P > LowerVectorContractionOpToAIEVecMatMulOpAIE2P
OneToOneVectorOpToAIEVecOpPattern< arith::AddFOp, aievec::aie1::AddOp > LowerVectorAddFOpToAIEVecAddOp
ComputeAbsOpPattern< math::AbsIOp > ComputeAbsIOpPattern
constexpr int64_t kAIE2PMaxParts
LowerTruncOpPattern< arith::TruncFOp > LowerTruncFOpPattern
LowerVectorIntToFPI16BF16AIE2pPattern< arith::SIToFPOp > LowerVectorSIToFPI16BF16AIE2pPattern
ScalarizeVectorIntFpConversionOpPattern< arith::SIToFPOp > ScalarizeVectorSIToFPOpPattern
ComputeBandAndBorOpPattern< arith::OrIOp, aievec::BorOp > ComputeBorOpPattern
LowerVectorAddOrSubOpToAIEVecAddElemOrSubElemOp< arith::AddFOp, aievec::AddElemOp > LowerVectorAddFOpToAIEVecAddElemOp
LowerExtOpPattern< arith::ExtFOp > LowerExtFOpPattern
OneToOneVectorOpToAIEVecOpPattern< arith::MulFOp, aievec::aie1::MulOp > LowerVectorMulFOpToAIEVecMulOp
ComputeAbsOpPattern< math::AbsFOp > ComputeAbsFOpPattern
LowerVectorAddOrSubOpToAIEVecAddElemOrSubElemOp< arith::SubIOp, aievec::SubElemOp > LowerVectorSubIOpToAIEVecSubElemOp
LowerVectorIntToFPI16BF16AIE2pPattern< arith::UIToFPOp > LowerVectorUIToFPI16BF16AIE2pPattern
LowerVectorUMinMaxOpToCmpSelOp< arith::MinUIOp, arith::CmpIPredicate::ult > LowerVectorMinUIOpToCmpSelOp
ComputeBandAndBorOpPattern< arith::AndIOp, aievec::BandOp > ComputeBandOpPattern
LowerVectorContractionOpToAIEVecMatMulPattern< aievec::MatMulOp > LowerVectorContractionOpToAIEVecMatMulOpAIE2
LowerVectorMinMaxOpToAIEVecMinMaxOp< arith::MinimumFOp, aievec::MinOp > LowerVectorMinimumFOpToAIEVecMinOp
LowerVectorCmpOpToAIEVecCmpOp< arith::CmpFOp, CmpFPredicate > LowerVectorCmpFOpToAIEVecCmpOp
LowerTruncOpPattern< arith::TruncIOp > LowerTruncIOpPattern
LowerExtOpPattern< arith::ExtSIOp > LowerExtSIOpPattern
LowerVectorAddOrSubOpToAIEVecAddElemOrSubElemOp< arith::AddIOp, aievec::AddElemOp > LowerVectorAddIOpToAIEVecAddElemOp
LowerVectorMinMaxOpToAIEVecMinMaxOp< arith::MaximumFOp, aievec::MaxOp > LowerVectorMaximumFOpToAIEVecMaxOp
LowerScalarMinMaxToAIEVecMinMaxOp< arith::MaxSIOp, aievec::MaxOp > LowerScalarMaxSIOpToAIEVecMaxOp
LowerVectorMinMaxOpToAIEVecMinMaxOp< arith::MinSIOp, aievec::MinOp > LowerVectorMinSIOpToAIEVecMinOp
mlir::VectorType getFlattenedVectorType(mlir::VectorType vecTy)
std::optional< MatMulRecipe > checkMatMulTile(bool aie2p, mlir::VectorType lhsTy, mlir::VectorType rhsTy, mlir::VectorType accTy, bool lhsTransposed, bool rhsTransposed, llvm::function_ref< mlir::InFlightDiagnostic()> emitError=nullptr)
unsigned getVectorLaneSize(mlir::VectorType type)
Definition AIEVecUtils.h:54
SmallVector< NamedAttribute > buildFMAOpSplatAttrForElemTy(aievec::aie1::FMAOp fmaOp, int64_t bcastPos, int64_t step=1)
std::optional< int64_t > getTransferReadAlignmentOffset(TransferReadLikeOp readOp, mlir::VectorType vType, int64_t alignment)
void populateAIE2PShufflePlannerPatterns(mlir::RewritePatternSet &patterns, std::shared_ptr< AIE2PShufflePlanner > planner)
std::optional< AIEVecTargetInfo > parseAIEVecLoweringTarget(llvm::StringRef spelling, mlir::Operation *op)
mlir::VectorType createVectorType(unsigned lanes, mlir::Type elementType)
Definition AIEVecUtils.h:41
int32_t getElementSizeInBits(mlir::VectorType type)
Definition AIEVecUtils.h:48
void buildLowerVectorToAIEVec(mlir::OpPassManager &pm, const LowerVectorToAIEVecOptions &options)
mlir::VectorType getVectorOpDestType(mlir::VectorType type, bool AIE2)
Definition AIEVecUtils.h:79
void configureAIE2PShufflePlannerLegality(mlir::ConversionTarget &target, std::shared_ptr< AIE2PShufflePlanner > planner)
static void markIllegal(ConversionTarget &target, MLIRContext *ctx)
static void addPatterns(RewritePatternSet &patterns, PatternBenefit benefit)
static void markIllegal(ConversionTarget &target, MLIRContext *ctx)
LogicalResult matchAndRewrite(SrcOpTy absOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
typename SrcOpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::XOrIOp xorOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::CeilOp ceilOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::ErfOp erfOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::ErfOp erfOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::ExpOp expOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::ExpOp expOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::FloorOp floorOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::NegFOp negOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(OpTy op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::RsqrtOp rsqrtOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::RsqrtOp rsqrtOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(OpTy op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::DivFOp divfOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::ShRSIOp rsOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::SqrtOp sqrtOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::TanhOp tanhOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::DivFOp divOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(math::ExpOp expOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(MathOpTy op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::AddFOp addOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
ConvertMulAddFToAIEVecFMAElemOpPattern(MLIRContext *context, unsigned shiftParam=0)
LogicalResult matchAndRewrite(arith::AddIOp addOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
ConvertMulAddToAIEVecFMAElemOpPattern(MLIRContext *context, unsigned shiftParam=0)
LogicalResult matchAndRewrite(aievec::aie1::AddOp addOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::MulFOp mulOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::MulFOp mulOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
ConvertMulFToAIEVecMulElemOpPattern(MLIRContext *context, unsigned shiftParam=0)
LogicalResult matchAndRewrite(arith::MulIOp mulOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
ConvertMulIToAIEVecMulElemOpPattern(MLIRContext *context, unsigned shiftParam=0)
LogicalResult matchAndRewrite(vector::BroadcastOp bcastOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::BroadcastOp bcastOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::FMAOp fmaOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::FMAOp fmaOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
ConvertVectorFMAOpToAIEVecFMAElemOpPattern(MLIRContext *context, unsigned shiftParam=0)
ExpandUPDToUPDAndExtPattern(MLIRContext *context)
LogicalResult matchAndRewrite(aievec::UPDOp updOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(OpTy op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::aie1::FMAOp fmaOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::BroadcastOp bcastOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(aievec::ExtOp extOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
FuseExtIntoUPDPattern(MLIRContext *context)
typename SrcOpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(SrcOpTy extOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::ExtUIOp extOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::ShRSIOp rsOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::TruncIOp truncOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(SrcOpTy truncOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
typename SrcOpTy::Adaptor OpAdaptor
LogicalResult matchAndRewrite(arith::AddIOp addOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
static Value reshape(OpBuilder &b, Value v, ArrayRef< int64_t > shape)
LogicalResult matchAndRewrite(vector::ContractionOp contractOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::ExtractStridedSliceOp extractOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::ExtractStridedSliceOp extractOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
Value emitMagicSiToFp(ConversionPatternRewriter &rewriter, Location loc, Value srcI16, unsigned lanes) const
LowerVectorIntToFPI16BF16AIE2pPattern(MLIRContext *context, bool accumulatorAdd, PatternBenefit benefit)
LogicalResult matchAndRewrite(SrcOpTy op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::MulIOp mulOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::ReductionOp srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::ReductionOp srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::ReductionOp srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::ReductionOp srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::ReductionOp srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(vector::ReductionOp srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(arith::SelectOp srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
void getDependentDialects(DialectRegistry &registry) const override
LowerVectorToAIEVec(const LowerVectorToAIEVecOptions &options)
StringRef getDescription() const final
LowerVectorToAIEVec()=default
StringRef getArgument() const final
LogicalResult matchAndRewrite(vector::TransferReadOp readOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LowerVectorTransferReadToAIEUPD(MLIRContext *context, int64_t minVectorSize, int64_t maxVectorSize, int64_t alignment, int64_t maxLoadSize)
LogicalResult matchAndRewrite(vector::TransposeOp transpOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(OpTy op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(SrcOpTy srcOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(SrcOpTy op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
ShiftClampTruncToSRSPattern(MLIRContext *context, PatternBenefit benefit=2)
static std::optional< Value > getShiftValue(Value rhs, ConversionPatternRewriter &rewriter, Location loc)
static std::optional< int64_t > getConstantSplatValue(Value val)
LogicalResult matchAndRewrite(arith::TruncIOp truncOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
Options for the "lower-vector-to-aievec" pipeline.
Definition Passes.h:42
PassOptions::Option< std::string > aieTarget
Definition Passes.h:43