MLIR-AIE 1.4.6.dev
AIEDialect.cpp
Go to the documentation of this file.
1//===- AIEDialect.cpp -------------------------------------------*- C++ -*-===//
2//
3// Copyright (C) 2019-2022 Xilinx, Inc.
4// Copyright (C) 2022-2026 Advanced Micro Devices, Inc.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
10
11#include "mlir/Dialect/Arith/IR/Arith.h"
12#include "mlir/Dialect/Func/IR/FuncOps.h"
13#include "mlir/Dialect/MemRef/IR/MemRef.h"
14#include "mlir/Dialect/Utils/StaticValueUtils.h"
15#include "mlir/IR/DialectImplementation.h"
16#include "mlir/IR/Matchers.h"
17#include "mlir/IR/OpDefinition.h"
18#include "mlir/IR/SymbolTable.h"
19#include "mlir/Interfaces/FoldInterfaces.h"
20#include "mlir/Interfaces/ViewLikeInterface.h"
21#include "mlir/Transforms/InliningUtils.h"
22#include "llvm/ADT/STLExtras.h"
23
24#include "llvm/ADT/SmallPtrSet.h"
25#include "llvm/ADT/SmallSet.h"
26#include "llvm/ADT/TypeSwitch.h"
27#include "llvm/Support/CheckedArithmetic.h"
28#include "llvm/Support/MathExtras.h"
29
30#include <limits>
31#include <map>
32#include <numeric>
33#include <optional>
34#include <string>
35#include <tuple>
36#include <utility>
37#include <vector>
38
39using namespace mlir;
40using namespace xilinx::AIE;
41
42// Add TableGen'erated dialect definitions (including constructor)
43// We implement the initialize() function further below
44#include "aie/Dialect/AIE/IR/AIEDialect.cpp.inc"
45
46#define GET_TYPEDEF_CLASSES
47#include "aie/Dialect/AIE/IR/AIETypes.cpp.inc"
48
49namespace {
50
51struct AIEInlinerInterface : DialectInlinerInterface {
52 using DialectInlinerInterface::DialectInlinerInterface;
53 // We don't have any special restrictions on what can be inlined into
54 // destination regions. Always allow it.
55 bool isLegalToInline(Region *dest, Region *src, bool wouldBeCloned,
56 IRMapping &valueMapping) const final {
57 return true;
58 }
59
60 // Operations in aie dialect are always legal to inline since they are
61 // pure.
62 bool isLegalToInline(Operation *op, Region *, bool wouldBeCloned,
63 IRMapping &) const final {
64 return true;
65 }
66
67 // Handle the given inlined terminator by replacing it with a new operation
68 // as necessary. Required when the inlined region has more than one block.
69 void handleTerminator(Operation *op, Block *newDest) const final {}
70
71 // Handle the given inlined terminator by replacing it with a new operation
72 // as necessary. Required when the region has only one block.
73 void handleTerminator(Operation *op, ValueRange valuesToRepl) const final {}
74};
75
76struct AIEDialectFoldInterface : DialectFoldInterface {
77 using DialectFoldInterface::DialectFoldInterface;
78
79 /// Registered hook to check if the given region, which is attached to an
80 /// operation that is *not* isolated from above, should be used when
81 /// materializing constants.
82 bool shouldMaterializeInto(Region *region) const final {
83 // Materialize constants into the op that "owns" them rather than letting
84 // them hoist up to the enclosing IsolatedFromAbove aie.device:
85 // - aie.core bodies are outlined into standalone funcs, so their
86 // constants must stay local for the func to be self-contained.
87 // - aie.runtime_sequence bodies carry constant operands (e.g. the scalar
88 // fields of npu.* ops). Hoisting them to the device body lets CSE merge
89 // them with a core's identical constants, which would leave the core
90 // referencing a device-level value that is erased when the core is
91 // outlined.
92 // - Make sure SSA values for aie.use_lock operands in
93 // aie.mem/aie.memtile_dma/aie.shim_dma bodies do not get
94 // hoisted.
95 return isa<CoreOp, RuntimeSequenceOp, MemOp, MemTileDMAOp, ShimDMAOp>(
96 region->getParentOp());
97 }
98};
99
100} // end anonymous namespace
101
102void AIEDialect::initialize() {
103 addTypes<
104#define GET_TYPEDEF_LIST
105#include "aie/Dialect/AIE/IR/AIETypes.cpp.inc"
106 >();
107 addAttributes<
108#define GET_ATTRDEF_LIST
109#include "aie/Dialect/AIE/IR/AIEAttrs.cpp.inc"
110 >();
111 addOperations<
112#define GET_OP_LIST
113#include "aie/Dialect/AIE/IR/AIEOps.cpp.inc"
114 >();
115 addInterfaces<AIEInlinerInterface, AIEDialectFoldInterface>();
116}
117
118// Helper methods to retrieve the encoding associated to a burst length,
119// or to find the highest available burst length if the requested one is 0
120// (default value).
121
122static std::pair<uint32_t, uint32_t>
123getShimBurstLength(const xilinx::AIE::AIETargetModel &tm,
124 uint32_t burstLength) {
125
126 std::vector<std::pair<uint32_t, uint32_t>> bel =
128
129 // If we have the default burst length (no burst length was specified),
130 // use the highest one available on our target model
131 if (burstLength == 0) {
132 return *llvm::max_element(bel, [](auto pair1, auto pair2) {
133 return pair1.second < pair2.second;
134 });
135 }
136
137 // Note that if we are given a burst size, we are checking its existence in
138 // the pass verification already, so we can safely assume it exists.
139 return *llvm::find_if(bel, [=](auto p) { return p.second == burstLength; });
140}
141
143 uint32_t burstLength) {
144
145 return getShimBurstLength(tm, burstLength).second;
146}
147
149 uint32_t burstLength) {
150
151 return getShimBurstLength(tm, burstLength).first;
152}
153
154Operation *xilinx::AIE::lookupNamedOpIn(Operation *symbolTableOp,
155 StringAttr name) {
156 if (!symbolTableOp->hasTrait<mlir::OpTrait::SymbolTable>() ||
157 symbolTableOp->getRegion(0).empty()) {
158 return nullptr;
159 }
160 // One walk finds both kinds: a symbol op's name is its `sym_name`, and
161 // `getAttr` reads it whether it is inherent or discardable.
162 for (Operation &op : symbolTableOp->getRegion(0).front()) {
163 if (op.getAttrOfType<StringAttr>(mlir::SymbolTable::getSymbolAttrName()) ==
164 name) {
165 return &op;
166 }
167 }
168 return nullptr;
169}
170
172 : context(symbolTableOp->getContext()) {
173 if (!symbolTableOp->hasTrait<mlir::OpTrait::SymbolTable>() ||
174 symbolTableOp->getRegion(0).empty()) {
175 return;
176 }
177 for (Operation &op : symbolTableOp->getRegion(0).front()) {
178 if (auto name = op.getAttrOfType<StringAttr>(
179 mlir::SymbolTable::getSymbolAttrName())) {
180 ops.try_emplace(name, &op);
181 }
182 }
183}
184
185Operation *xilinx::AIE::lookupNamedOpIn(Operation *symbolTableOp,
186 StringRef name) {
187 return lookupNamedOpIn(symbolTableOp,
188 StringAttr::get(symbolTableOp->getContext(), name));
189}
190
191Operation *xilinx::AIE::lookupNamedOp(Operation *from, StringAttr name) {
192 Operation *symbolTableOp = mlir::SymbolTable::getNearestSymbolTable(from);
193 return symbolTableOp ? lookupNamedOpIn(symbolTableOp, name) : nullptr;
194}
195
196Operation *xilinx::AIE::lookupNamedOp(Operation *from, StringRef name) {
197 return lookupNamedOp(from, StringAttr::get(from->getContext(), name));
198}
199
200std::string xilinx::AIE::NamedOpTable::uniqueName(llvm::StringRef prefix,
201 unsigned &counter) const {
202 std::string name;
203 do {
204 name = (prefix + llvm::Twine(counter++)).str();
205 } while (ops.count(StringAttr::get(context, name)));
206 return name;
207}
208
209LogicalResult
210xilinx::AIE::myVerifyOffsetSizeAndStrideOp(OffsetSizeAndStrideOpInterface op) {
211 std::array<unsigned, 3> maxRanks = op.getArrayAttrMaxRanks();
212 // Count checks FIRST, matching mlir::detail::verifyOffsetSizeAndStrideOp.
213 // getMixedOffsets()/getMixedSizes() assert that the number of dynamic
214 // operands equals the number of kDynamic sentinels in the static list, so
215 // running the rank comparison before these validates nothing and aborts on
216 // malformed input instead of diagnosing it.
217 if (failed(verifyListOfOperandsOrIntegers(
218 op, "offset", maxRanks[0], op.getStaticOffsets(), op.getOffsets())))
219 return failure();
220 if (failed(verifyListOfOperandsOrIntegers(
221 op, "size", maxRanks[1], op.getStaticSizes(), op.getSizes())))
222 return failure();
223 if (failed(verifyListOfOperandsOrIntegers(
224 op, "stride", maxRanks[2], op.getStaticStrides(), op.getStrides())))
225 return failure();
226 // Each getMixed* call rebuilds its list, so read them once.
227 const size_t numMixedOffsets = op.getMixedOffsets().size();
228 const size_t numMixedSizes = op.getMixedSizes().size();
229 if (!(numMixedOffsets == 1 && maxRanks[0] == 1) && // NOLINT
230 numMixedOffsets != numMixedSizes)
231 return op->emitError(
232 "expected mixed offsets rank to match mixed sizes rank (")
233 << numMixedOffsets << " vs " << numMixedSizes
234 << ") so the rank of the result type is well-formed.";
235 for (int64_t offset : op.getStaticOffsets())
236 if (offset < 0 && !ShapedType::isDynamic(offset))
237 return op->emitError("expected offsets to be non-negative, but got ")
238 << offset;
239 for (int64_t size : op.getStaticSizes())
240 if (size < 0 && !ShapedType::isDynamic(size))
241 return op->emitError("expected sizes to be non-negative, but got ")
242 << size;
243
244 return success();
245}
246
247static VC1902TargetModel VC1902model;
248static VE2302TargetModel VE2302model;
249static VE2802TargetModel VE2802model;
250static VE3858TargetModel VE3858model;
251static VirtualizedNPU1TargetModel NPUmodel1col(1);
252static VirtualizedNPU1TargetModel NPUmodel2col(2);
253static VirtualizedNPU1TargetModel NPUmodel3col(3);
254static VirtualizedNPU1TargetModel NPUmodel4col(4);
255static NPU2TargetModel NPU2model;
256static VirtualizedNPU2TargetModel NPU2model1col(1);
257static VirtualizedNPU2TargetModel NPU2model2col(2);
258static VirtualizedNPU2TargetModel NPU2model3col(3);
259static VirtualizedNPU2TargetModel NPU2model4col(4);
260static VirtualizedNPU2TargetModel NPU2model5col(5);
261static VirtualizedNPU2TargetModel NPU2model6col(6);
262static VirtualizedNPU2TargetModel NPU2model7col(7);
263
264const AIETargetModel &xilinx::AIE::getTargetModel(Operation *op) {
265 if (auto t = dyn_cast<AIETarget>(op))
266 return t.getTargetModel();
267 if (auto t = op->getParentOfType<AIETarget>())
268 return t.getTargetModel();
269
270 // For backward compatibility, return a basic device model compatible with
271 // the VCK190
272 return VC1902model;
273}
274
276 return getTargetModel(a.getOperation())
277 .getSharedMemory({a.getCol(), a.getRow()}, {b.getCol(), b.getRow()});
278}
279
281 switch (device) {
282 case AIEDevice::xcvc1902:
283 return VC1902model;
284 case AIEDevice::xcve2302:
285 return VE2302model;
286 case AIEDevice::xcve2802:
287 return VE2802model;
288 case AIEDevice::npu1:
289 return NPUmodel4col;
290 case AIEDevice::npu1_1col:
291 return NPUmodel1col;
292 case AIEDevice::npu1_2col:
293 return NPUmodel2col;
294 case AIEDevice::npu1_3col:
295 return NPUmodel3col;
296 case AIEDevice::npu2:
297 return NPU2model;
298 case AIEDevice::npu2_1col:
299 return NPU2model1col;
300 case AIEDevice::npu2_2col:
301 return NPU2model2col;
302 case AIEDevice::npu2_3col:
303 return NPU2model3col;
304 case AIEDevice::npu2_4col:
305 return NPU2model4col;
306 case AIEDevice::npu2_5col:
307 return NPU2model5col;
308 case AIEDevice::npu2_6col:
309 return NPU2model6col;
310 case AIEDevice::npu2_7col:
311 return NPU2model7col;
312 case AIEDevice::xcve3858:
313 return VE3858model;
314 }
315 // No default: label above, so -Wswitch still reports a newly added device.
316 // This handles values that are not enumerators at all, which
317 // aieGetTargetModel admits by casting an unchecked uint32_t.
318 llvm::report_fatal_error("getTargetModel: unknown AIEDevice value " +
319 llvm::Twine(static_cast<uint32_t>(device)));
320}
321
322// Walk the operation hierarchy until we find a containing TileElement.
323// If no parent is a TileElement, then return null.
324static TileElement getParentTileElement(Operation *op) {
325 auto *parent = op->getParentOp();
326 while (!llvm::isa_and_nonnull<DeviceOp, ModuleOp>(parent)) {
327 if (auto element = llvm::dyn_cast<TileElement>(parent))
328 return element;
329 parent = parent->getParentOp();
330 }
331 return llvm::dyn_cast<TileElement>(parent);
332}
333
334// Returns the maximum index described by the input dimensions.
335static int64_t getDimsMaxIdx(ArrayRef<BDDimLayoutAttr> dims) {
336 int64_t maxIdx = 0;
337 for (BDDimLayoutAttr dim : dims) {
338 maxIdx += dim.getStride() * (dim.getSize() - 1);
339 }
340 return maxIdx;
341}
342
344 int64_t elements = 1;
345 for (BDDimLayoutAttr dim : inner)
346 elements *= dim.getSize();
347 return elements;
348}
349
351 ArrayRef<BDDimLayoutAttr> dims, ArrayRef<BDPadLayoutAttr> pads,
352 int64_t objectSize, function_ref<InFlightDiagnostic()> emitError,
353 StreamWindow &window) {
354 if (dims.size() < 2)
355 return emitError() << "a walk over a stream needs an outer dimension of "
356 "outputs and at least one inner dimension";
357 StreamWindow w;
358 w.objectSize = objectSize;
359 w.outputs = dims.front().getSize();
360 int64_t outerStride = dims.front().getStride();
361 if (outerStride % objectSize)
362 return emitError() << "a walk over a stream steps whole objects between "
363 "outputs, but its outer stride "
364 << outerStride << " is not a multiple of the "
365 << objectSize << "-element object";
366 w.advance = outerStride / objectSize;
367 w.inner.assign(dims.begin() + 1, dims.end());
368 w.rowDim = -1;
369 int64_t reach = 0;
370 for (auto [index, dim] : llvm::enumerate(w.inner)) {
371 if (dim.getSize() > 1 && dim.getStride() >= objectSize) {
372 if (w.rowDim >= 0 || dim.getStride() != objectSize)
373 return emitError() << "a walk over a stream may step one inner "
374 "dimension by exactly one "
375 << objectSize << "-element object";
376 w.rowDim = index;
377 continue;
378 }
379 reach += dim.getStride() * (dim.getSize() - 1);
380 }
381 if (reach >= objectSize)
382 return emitError() << "a walk over a stream reads " << reach + 1
383 << " elements of each object, past the " << objectSize
384 << "-element object";
385 w.span = w.rowDim >= 0 ? w.inner[w.rowDim].getSize() : 1;
386 w.top = w.bottom = 0;
387 for (auto [index, pad] : llvm::enumerate(pads)) {
388 if (index == 0) {
389 w.top = pad.getConstPadBefore();
390 w.bottom = pad.getConstPadAfter();
391 } else if (pad.getConstPadBefore() || pad.getConstPadAfter()) {
392 return emitError() << "a walk over a stream pads only its outer "
393 "dimension, with the objects before and after "
394 "the stream";
395 }
396 }
397 w.objects = (w.outputs - 1) * w.advance + w.span - w.top - w.bottom;
398 if (w.objects < w.span || w.top >= w.span || w.bottom >= w.span)
399 return emitError() << "a walk over a stream of " << w.objects
400 << " objects with " << w.top << " and " << w.bottom
401 << " padding objects leaves an output without data";
402 window = w;
403 return success();
404}
405
406SmallVector<StreamWindowStep>
408 SmallVector<StreamWindowStep> steps;
409 int64_t acquired = 0, released = 0;
410 for (int64_t j = 0; j < w.outputs; j++) {
411 int64_t start = j * w.advance - w.top;
412 int64_t newest = std::max(
413 acquired, start + std::clamp<int64_t>(w.objects - start, 0, w.span));
414 int64_t next =
415 j + 1 < w.outputs
416 ? std::clamp<int64_t>((j + 1) * w.advance - w.top, 0, newest)
417 : w.objects;
418 next = std::max(released, next);
419 steps.push_back({j * w.advance, newest - acquired, next - released});
420 acquired = newest;
421 released = next;
422 }
423 return steps;
424}
425
426namespace {
427
428struct UsesAreAccessible {
429 static LogicalResult verifyTrait(Operation *op) {
430 auto thisElement = cast<TileElement>(op);
431
432 // Skip accessibility checks for logical tiles as we cannot tell until tile
433 // is placed
434 if (!isa<TileOp>(thisElement.getTile().getDefiningOp()))
435 return success();
436
437 auto thisID = thisElement.getTileID();
438 auto users = op->getResult(0).getUsers();
439 const auto &targetModel = getTargetModel(op);
440 for (auto *user : users) {
441 // AIE.useLock may be used in a device to set the lock's default value
442 // Allow in a toplevel module for backward compatibility
443 if (llvm::isa_and_nonnull<DeviceOp, ModuleOp>(user->getParentOp())) {
444 continue;
445 }
446 // If any parent or the user itself prescribe that accessibility checks be
447 // skipped, skip the check for that user.
448 if (user->getParentWithTrait<SkipAccessibilityCheckTrait>() ||
449 user->hasTrait<SkipAccessibilityCheckTrait>()) {
450 continue;
451 }
452 TileElement element = llvm::dyn_cast<TileElement>(user);
453 if (!element) {
454 element = getParentTileElement(user);
455 }
456 if (!element) {
457 // This should probably be caught elsewhere as well.
458 return op->emitOpError("is accessed outside of a tile")
459 .attachNote(user->getLoc())
460 << "user";
461 }
462 auto tileID = element.getTileID();
463 if (!targetModel.isLegalMemAffinity(tileID.col, tileID.row, thisID.col,
464 thisID.row)) {
465 return (op->emitOpError("in Column ")
466 << thisID.col << " and Row " << thisID.row
467 << " is accessed from an unreachable tile in Column "
468 << tileID.col << " and Row " << tileID.row)
469 .attachNote(user->getLoc())
470 << "user";
471 }
472 }
473 return success();
474 }
475};
476
477} // namespace
478
479// Check that the operation only contains terminators in
480// TerminatorOpTypes.
481template <typename... TerminatorOpTypes>
483 static LogicalResult verifyTrait(Operation *op) {
484 for (auto &region : op->getRegions()) {
485 for (auto &block : region) {
486 if (!block.empty()) {
487 if (Operation *operation = &block.back();
488 !llvm::isa_and_nonnull<TerminatorOpTypes...>(operation))
489 return operation->emitOpError("is not an allowed terminator")
490 .attachNote(op->getLoc())
491 .append("in this context: ");
492 }
493 }
494 }
495 return success();
496 }
497};
498
499// Check that the given DMA-like op (e.g. MemOp, ShimDMAOp)
500// has valid BDs.
501template <typename ConcreteType>
502LogicalResult HasValidBDs<ConcreteType>::verifyTrait(Operation *op) {
503 auto element = cast<ConcreteType>(op);
504 const auto &targetModel = getTargetModel(op);
505
506 TileLike tile = element.getTileLike();
507 if (!tile)
508 return op->emitOpError("tile must implement TileLike interface");
509
510 int bdMax = targetModel.getNumBDs(tile.getTileType());
511
512 int bdNum = 0;
513 for (auto &block : element.getBody()) {
514 auto bdOps = llvm::to_vector_of<DMABDOp>(block.template getOps<DMABDOp>());
515
516 // Skip entry/end block
517 if (bdOps.empty())
518 continue;
519
520 // Check BD count limit
521 if (bdNum >= bdMax) {
522 return (op->emitOpError("has more than ") << bdMax << " blocks")
523 .attachNote(bdOps.front().getLoc())
524 .append("no space for this BD");
525 }
526 bdNum++;
527
528 // Check exactly 1 DMABDOp per BD block
529 if (bdOps.size() != 1) {
530 return (op->emitOpError("BD block must have exactly one DMABDOp, found ")
531 << bdOps.size())
532 .attachNote(block.front().getLoc())
533 .append("in this BD block");
534 }
535
536 // Check at most 2 UseLockOps per BD block (1 acquire, 1 release)
537 auto useLockOps =
538 llvm::to_vector_of<UseLockOp>(block.template getOps<UseLockOp>());
539 int acquireCount = 0;
540 int releaseCount = 0;
541 for (auto useLock : useLockOps) {
542 if (useLock.acquire() || useLock.acquireGE())
543 acquireCount++;
544 else if (useLock.release())
545 releaseCount++;
546 }
547
548 if (acquireCount > 1) {
549 return (op->emitOpError(
550 "BD block must have at most one acquire UseLockOp, found ")
551 << acquireCount)
552 .attachNote(block.front().getLoc())
553 .append("in this BD block");
554 }
555 if (releaseCount > 1) {
556 return (op->emitOpError(
557 "BD block must have at most one release UseLockOp, found ")
558 << releaseCount)
559 .attachNote(block.front().getLoc())
560 .append("in this BD block");
561 }
562 }
563 return success();
564}
565
566// Check that the given DMA-like op (e.g. MemOp, ShimDMAOp)
567// has valid DMA channels.
568template <typename ConcreteType>
570 DenseSet<DMAChannel> inputChannels;
571 DenseSet<DMAChannel> outputChannels;
572 DenseSet<std::pair<DMAChannelDir, StringAttr>> namedChannels;
573 auto element = cast<ConcreteType>(op);
574 Region &body = element.getBody();
575 if (body.empty())
576 return op->emitOpError("should have non-empty body");
577 for (auto &bodyOp : body.getOps()) {
578 // check for duplicate DMA channels within the same ShimDMAOp
579 if (auto dmaStart = dyn_cast<DMAStartOp>(bodyOp)) {
580 if (FlatSymbolRefAttr endpoint = dmaStart.getEndpoint()) {
581 namedChannels.insert({dmaStart.getChannelDir(), endpoint.getAttr()});
582 continue;
583 }
584 DMAChannel dmaChan = {dmaStart.getChannelDir(),
585 dmaStart.getChannelIndex()};
586 // check if number of input and output channels is more than available
587 // hardware
588 if (dmaChan.direction == DMAChannelDir::S2MM)
589 inputChannels.insert(dmaChan);
590 else
591 outputChannels.insert(dmaChan);
592 }
593 }
594
595 TileLike tile = element.getTileLike();
596 if (!tile)
597 return op->emitOpError("tile must implement TileLike interface");
598
599 auto named = [&](DMAChannelDir dir) -> size_t {
600 return llvm::count_if(namedChannels,
601 [&](auto channel) { return channel.first == dir; });
602 };
603 if (inputChannels.size() + named(DMAChannelDir::S2MM) >
604 tile.getNumSourceConnections(WireBundle::DMA))
605 return op->emitOpError(
606 "uses more input channels than available on this tile");
607
608 if (outputChannels.size() + named(DMAChannelDir::MM2S) >
609 tile.getNumDestConnections(WireBundle::DMA))
610 return op->emitOpError(
611 "uses more output channels than available on this tile");
612 return success();
613}
614
615LogicalResult
616LockAcquireAttr::verify(function_ref<InFlightDiagnostic()> emitError,
617 int32_t column, int32_t row, int32_t lock_id,
618 LockAction action, int32_t value) {
619 if (column < 0 || row < 0 || lock_id < 0)
620 return emitError() << "lock column, row, and ID must be non-negative";
621 if (action != LockAction::Acquire &&
622 action != LockAction::AcquireGreaterEqual)
623 return emitError() << "expected Acquire or AcquireGreaterEqual, got "
624 << stringifyLockAction(action);
625 return success();
626}
627
628//===----------------------------------------------------------------------===//
629// ObjectFifoTransportAttr
630//===----------------------------------------------------------------------===//
631
632LogicalResult
633ObjectFifoTransportAttr::verify(function_ref<InFlightDiagnostic()> emitError,
634 ObjectFifoTransportMode mode,
635 PacketInfoAttr packet, bool out_of_order) {
636 // A packet header rides a stream-switch connection the DMAs drive, so only
637 // the paths that may use the DMAs can honour one.
638 if (packet && mode == ObjectFifoTransportMode::SharedMem)
639 return emitError() << "`packet` belongs to a dma or auto transport, not "
640 << stringifyObjectFifoTransportMode(mode);
641 if (out_of_order && !packet)
642 return emitError() << "`out_of_order` requires a packet transport";
643 return success();
644}
645
646//===----------------------------------------------------------------------===//
647// ConsumedByAttr
648//===----------------------------------------------------------------------===//
649
650LogicalResult
651ConsumedByAttr::verify(function_ref<InFlightDiagnostic()> emitError,
652 FlatSymbolRefAttr output, int64_t input_bytes,
653 int64_t output_bytes, int64_t granule_bytes,
654 FlatSymbolRefAttr device) {
655 if (input_bytes <= 0 || output_bytes <= 0)
656 return emitError() << "consumed_by needs positive byte counts, got "
657 << input_bytes << " and " << output_bytes;
658 if (granule_bytes < 0)
659 return emitError() << "consumed_by needs a granule of 0 or more bytes, got "
660 << granule_bytes;
661 return success();
662}
663
664//===----------------------------------------------------------------------===//
665// ObjectFifoEndPortAttr
666//===----------------------------------------------------------------------===//
667
668LogicalResult
669ObjectFifoEndPortAttr::verify(function_ref<InFlightDiagnostic()> emitError,
670 WireBundle bundle,
671 std::optional<uint32_t> channel) {
672 if (bundle != WireBundle::DMA && bundle != WireBundle::Core)
673 return emitError() << "an objectFifo end is driven by a DMA channel or a "
674 "Core stream port, not "
675 << stringifyWireBundle(bundle);
676 if (bundle == WireBundle::Core) {
677 if (!channel)
678 return emitError() << "a Core end needs its stream port";
679 if (*channel > 1)
680 return emitError() << "a Core end's stream port is 0 or 1, not "
681 << *channel;
682 }
683 return success();
684}
685
686//===----------------------------------------------------------------------===//
687// ObjectFifoCreateOp
688//===----------------------------------------------------------------------===//
689
690// Each initial value is one object's dense contents, in the object's shape
691// and element type: that is all a buffer can be loaded with.
692static LogicalResult verifyInitValues(Operation *op, ArrayAttr initValues,
693 MemRefType object) {
694 for (Attribute value : initValues) {
695 auto dense = dyn_cast<DenseElementsAttr>(value);
696 if (!dense)
697 return op->emitOpError("initial value ")
698 << value << " is not a dense elements attribute";
699 if (dense.getType().getShape() != object.getShape() ||
700 dense.getElementType() != object.getElementType())
701 return op->emitOpError("initial value of type ")
702 << dense.getType() << " does not match the object type " << object;
703 }
704 return success();
705}
706
707LogicalResult ObjectFifoCreateOp::verify() {
708 for (Type fifoType : {Type(getElemType()), getConsumerElemTypeOrDefault()})
709 if (MemRefType memref = cast<AIEObjectFifoType>(fifoType).getElementType();
710 !memref.hasStaticShape())
711 return emitOpError("type should be static shaped memref, but got ")
712 << memref;
713
714 // MLIR carries an attribute it does not know about rather than rejecting it,
715 // so IR written against the switches these replaced would keep parsing and
716 // quietly take the default path instead. Name them to fail that loudly.
717 static constexpr std::pair<llvm::StringLiteral, llvm::StringLiteral>
718 replaced[] = {
719 {"out_of_order", "transport = #aie.transport<dma, packet = "
720 "#aie.packet_info<>, out_of_order = true>"},
721 {"via_DMA", "transport = #aie.transport<dma>"},
722 {"packet", "transport = #aie.transport<dma, packet = "
723 "#aie.packet_info<>>"},
724 {"packet_id", "transport = #aie.transport<dma, packet = "
725 "#aie.packet_info<pkt_id = 3>>"},
726 {"aie_stream", "prod_port = #aie.end_port<Core : 0> or cons_ports "
727 "= [#aie.end_port<Core : 0>]"},
728 {"aie_stream_port", "prod_port = #aie.end_port<Core : 0> or "
729 "cons_ports = [#aie.end_port<Core : 0>]"},
730 {"prod_dma_channel", "prod_port = #aie.end_port<DMA : 1>"},
731 {"cons_dma_channels", "cons_ports = [#aie.end_port<DMA : 1>]"}};
732 for (auto [gone, replacement] : replaced)
733 if ((*this)->hasAttr(gone))
734 return emitOpError("`")
735 << gone << "` has been replaced; write " << replacement;
736
737 if (isa<ArrayAttr>(getElemNumber())) {
738 if (size_t numDepths = dyn_cast<ArrayAttr>(getElemNumber()).size();
739 numDepths != getConsumerTiles().size() + 1) // +1 for producer depth
740 return emitOpError("does not have enough depths specified for producer "
741 "and for each consumer.");
742 }
743
744 if (ArrayAttr consPorts = getConsPortsAttr();
745 consPorts && consPorts.size() != getConsumerTiles().size())
746 return emitOpError("`cons_ports` has ")
747 << consPorts.size() << " entries for " << getConsumerTiles().size()
748 << " consumers";
749
750 // Helper to get tile interface from Value
751 auto getTileLikeFromValue = [](Value v) -> TileLike {
752 return llvm::dyn_cast<TileLike>(v.getDefiningOp());
753 };
754
755 TileLike producerTile = getTileLikeFromValue(getProducerTile());
756 if (!producerTile)
757 return emitError("producer tile must implement TileLike interface");
758
759 // data layout transformations on shim tiles are handled by runtime operations
760 if (producerTile.isShimTile() && !getDimensionsToStream().empty()) {
761 return emitError(
762 "`dimensionsToStream` data layout transformations are not supported "
763 "on shim tile producers");
764 }
765 for (auto consTileVal : getConsumerTiles()) {
766 TileLike consTile = getTileLikeFromValue(consTileVal);
767 if (!consTile)
768 return emitError("consumer tile must implement TileLike interface");
769 if (consTile.isShimTile() &&
770 !getDimensionsFromStream(consTileVal).empty()) {
771 return emitError(
772 "`dimensionsFromStreamPerConsumer` data layout transformations are "
773 "not supported on shim tile consumers");
774 }
775 }
776
777 // A core takes its consumer accesses from the first of its tile's ends
778 // a link does not already claim; a link on the tile reads one end each
779 // (getConsumerLink), in the order its own aie.objectfifo.link ops appear.
780 for (auto [index, consumer] : llvm::enumerate(getConsumerTiles())) {
781 size_t first =
782 llvm::find(getConsumerTiles(), consumer) - getConsumerTiles().begin();
783 TileLike tile = getTileLikeFromValue(consumer);
784 if (first == index || tile.getTileType() != AIETileType::CoreTile ||
785 getConsumerLink(index))
786 continue;
787 InFlightDiagnostic diag = emitOpError("lists core tile ");
788 if (std::optional<int> col = tile.tryGetCol(), row = tile.tryGetRow();
789 col && row)
790 diag << "(" << *col << ", " << *row << ") ";
791 diag << "as consumers " << first << " and " << index
792 << "; its core reads only the first of those ends, so the other "
793 "would never be drained";
794 diag.attachNote(consumer.getLoc()) << "the tile";
795 return diag;
796 }
797
798 if (getRepeatCount().has_value()) {
799 if (producerTile.isShimTile())
800 return emitError("`repeat_count` unavailable for shim tiles");
801 }
802
803 if (getPadValue() != 0) {
804 if (!getPadDimensions().has_value())
805 return emitError("`padValue` requires `padDimensions`");
806 if (!getTargetModel(getOperation()).isMemTilePadValueSupported())
807 return emitError("`padValue` requires the CONSTANT_PAD_VALUE register, "
808 "unavailable on this target");
809 }
810
811 if (usesStream()) {
812 if (getConsumerTiles().size() > 1)
813 return emitError("a Core stream port end can only be used in 1-to-1 "
814 "object FIFOs");
815
816 if (forcesSharedMemory())
817 return emitError("a Core stream port end sends over the stream, not "
818 "through shared memory");
819
820 if (streamsFromProducer()) {
821 if (producerTile.isShimTile() || producerTile.isMemTile())
822 return emitError(
823 "a Core stream port end is not available for shim and mem tiles");
824
825 if (getRepeatCount().has_value())
826 return emitError("`repeat_count` unavailable on stream end");
827
828 if (getInitValues().has_value())
829 return emitError("`init_values` unavailable on stream end");
830
831 if (getIterCount().has_value())
832 return emitError("`iter_count` unavailable on stream end");
833
834 if (!getDimensionsToStream().empty())
835 return emitError("`dimensionsToStream` data layout transformations are "
836 "unavailable on stream end");
837 }
838
839 if (streamsToConsumer(0)) {
840 TileLike consTile = getTileLikeFromValue(getConsumerTiles()[0]);
841 if (consTile && (consTile.isShimTile() || consTile.isMemTile()))
842 return emitError(
843 "a Core stream port end is not available for shim and mem tiles");
844 }
845
846 if (!getDimensionsFromStreamPerConsumer()[0].empty())
847 return emitError("`dimensionsFromStreamPerConsumer` data layout "
848 "transformations are unavailable on stream end");
849 }
850
851 if (getInitValues().has_value()) {
852 if (producerTile.isShimTile())
853 return emitError("`init_values` unavailable for shim tiles");
854 }
855
856 if (auto initValues = getInitValues()) {
857 if ((int)initValues->size() != size())
858 return emitError("`init_values` does not initialize all objects");
859 if (failed(verifyInitValues(
860 *this, *initValues,
861 cast<AIEObjectFifoType>(getElemType()).getElementType())))
862 return failure();
863 }
864
865 if (auto iterCountAttr = getIterCount()) {
866 int iterCount = *iterCountAttr;
867 if (iterCount < 1 || iterCount > 256)
868 return emitError("`iter_count` must be between 1 and 256");
869 }
870
871 if (auto consumerElemType = getConsumerElemType()) {
872 auto consType = llvm::dyn_cast<AIEObjectFifoType>(*consumerElemType);
873 if (!consType)
874 return emitError("consumer element type must be an "
875 "!aie.objectfifo<memref<...>> type");
876 auto prodType = llvm::cast<AIEObjectFifoType>(getElemType());
877 auto prodMemref = prodType.getElementType();
878 auto consMemref = consType.getElementType();
879 if (prodMemref.getElementType() != consMemref.getElementType())
880 return emitError("producer and consumer must have the same scalar "
881 "element type, but got ")
882 << prodMemref.getElementType() << " vs "
883 << consMemref.getElementType();
884 int64_t prodSize = prodMemref.getNumElements();
885 int64_t consSize = consMemref.getNumElements();
886 if (consSize <= 0)
887 return emitError("consumer element count must be positive");
888 if (prodSize % consSize != 0)
889 return emitError("producer element size (")
890 << prodSize << ") must be an integer multiple of consumer "
891 << "element size (" << consSize << ")";
892 }
893
894 return success();
895}
896
897TileOp ObjectFifoCreateOp::getProducerTileOp() {
898 return cast<TileOp>(getProducerTile().getDefiningOp());
899}
900
901//===----------------------------------------------------------------------===//
902// ObjectFifoPoolOp
903//===----------------------------------------------------------------------===//
904
905namespace {
906/// Resolve a list of symbol names to the ops they name, skipping any that are
907/// not yet declared.
908template <typename OpTy>
909SmallVector<OpTy> lookupAll(Operation *from, std::optional<ArrayAttr> names) {
910 SmallVector<OpTy> ops;
911 if (names) {
912 for (auto name : names->getAsRange<FlatSymbolRefAttr>()) {
913 if (auto op =
914 dyn_cast_or_null<OpTy>(lookupNamedOp(from, name.getAttr()))) {
915 ops.push_back(op);
916 }
917 }
918 }
919 return ops;
920}
921
922} // namespace
923
924TileLike ObjectFifoPoolOp::getTileLike() {
925 return dyn_cast<TileLike>(getTile().getDefiningOp());
926}
927
928int64_t ObjectFifoPoolOp::getObjectSize() {
929 return llvm::cast<MemRefType>(getElemType()).getNumElements();
930}
931
932int64_t ObjectFifoPoolOp::getObjectsPerBuffer() {
933 if (!getContiguous())
934 return 1;
935 int64_t objects = getDepth();
936 if (auto zeros = getZeroObjects())
937 objects += (*zeros)[0] + (*zeros)[1];
938 return objects;
939}
940
941LogicalResult ObjectFifoPoolOp::verify() {
942 auto &target = (*this)->getParentOfType<DeviceOp>().getTargetModel();
943 bool semaphoreLocks = target.hasProperty(AIETargetModel::UsesSemaphoreLocks);
944
945 // A binary lock travels with the buffer it guards; a semaphore lock counts
946 // objects for a whole segment.
947 if (auto locks = getLocks()) {
948 if (semaphoreLocks) {
949 return emitOpError("'locks' names binary locks, which this device has "
950 "no use for");
951 }
952 if (static_cast<int64_t>(locks->size()) != getDepth()) {
953 return emitOpError("expects one lock per buffer");
954 }
955 }
956
957 if (auto zeros = getZeroObjects()) {
958 if (!getContiguous() || zeros->size() != 2 ||
959 llvm::any_of(*zeros, [](int32_t count) { return count < 0; }))
960 return emitOpError("'zeroObjects' expects [before, after] counts on a "
961 "contiguous pool");
962 }
963
964 if (auto buffers = getBuffers()) {
965 if (getContiguous() && buffers->size() != 1) {
966 return emitOpError("expects one buffer holding its objects side by side");
967 }
968 if (!getContiguous() &&
969 static_cast<int64_t>(buffers->size()) != getDepth()) {
970 return emitOpError("expects 'depth' buffers");
971 }
972 }
973
974 if (auto initValues = getInitValues()) {
975 if (static_cast<int64_t>(initValues->size()) > getDepth()) {
976 return emitOpError("has more 'init_values' than 'depth' objects");
977 }
978 if (failed(verifyInitValues(*this, *initValues, getElemType())))
979 return failure();
980 }
981
982 std::vector<ObjectFifoSegmentOp> segments = getSegmentOps();
983 if (segments.empty()) {
984 return emitOpError("expects at least one segment");
985 }
986
987 int64_t previous = -1;
988 int64_t covered = 0;
989 for (ObjectFifoSegmentOp segment : segments) {
990 if (static_cast<int64_t>(segment.getOffset()) <= previous) {
991 return emitOpError("segments must be ordered by increasing offset");
992 }
993 if (static_cast<int64_t>(segment.getOffset()) != covered) {
994 return segment.emitOpError("leaves a gap or overlaps the segment before "
995 "it; segments must tile the object");
996 }
997 previous = segment.getOffset();
998 covered = segment.getOffset() + segment.getSize();
999 if (!semaphoreLocks &&
1000 (segment.getProduceLock() || segment.getConsumeLock())) {
1001 return emitOpError("segment locks are counting locks, which this "
1002 "device has no use for");
1003 }
1004 }
1005
1006 if (covered != getObjectSize()) {
1007 return emitOpError("segments cover ")
1008 << covered << " of " << getObjectSize() << " elements";
1009 }
1010
1011 if (segments.size() > 1 && !semaphoreLocks) {
1012 return emitOpError(
1013 "multi-segment pools are unsupported on binary lock architectures");
1014 }
1015
1016 return success();
1017}
1018
1019int64_t xilinx::AIE::getMemRefSizeInBytes(MemRefType type,
1020 const DataLayout &layout) {
1021 return type.getNumElements() * layout.getTypeSize(type.getElementType());
1022}
1023
1024int64_t ObjectFifoPoolOp::getObjectSizeInBytes() {
1025 return getMemRefSizeInBytes(getElemType(), DataLayout::closest(*this));
1026}
1027
1028// Whether one iterating descriptor could walk a pool of `depth` objects on a
1029// tile: a mem tile's pool, deeper than one object and no deeper than a lock
1030// counts (ObjectFifoPoolOp::canWalkWhole, ObjectFifoCreateOp::plannedDmaEnd).
1031static bool walksWholeDepth(bool isMemTile, int64_t depth,
1032 int64_t maxLockValue) {
1033 return isMemTile && depth > 1 && depth <= maxLockValue;
1034}
1035
1036// Whether `fifo`'s producer DMA end pushes a task per object: it asks for
1037// `push_queue`, or sharesSendChannel().
1038static bool usesPushQueue(ObjectFifoCreateOp fifo) {
1039 return fifo.getReleaseStrategy() == ObjectFifoRelease::PushQueue ||
1040 fifo.sharesSendChannel();
1041}
1042
1043std::vector<ObjectFifoSegmentOp> ObjectFifoPoolOp::getSegmentOps() {
1044 auto ops = getSegments().getOps<ObjectFifoSegmentOp>();
1045 return {ops.begin(), ops.end()};
1046}
1047
1048Value BufferOp::getBufferTile() { return getTile(); }
1049
1050SmallVector<BufferLike> ObjectFifoPoolOp::getBufferOps() {
1051 // Buffers and locks live beside the pool, not in the symbol table it opens
1052 // for its segments.
1053 return lookupAll<BufferLike>((*this)->getParentOfType<DeviceOp>(),
1054 getBuffers());
1055}
1056
1057SmallVector<LockOp> ObjectFifoPoolOp::getLockOps() {
1058 auto device = (*this)->getParentOfType<DeviceOp>();
1059 SmallVector<LockOp> locks = lookupAll<LockOp>(device, getLocks());
1060 for (ObjectFifoSegmentOp segment : getSegmentOps()) {
1061 for (std::optional<FlatSymbolRefAttr> name :
1062 {segment.getProduceLockAttr(), segment.getConsumeLockAttr()}) {
1063 if (name && *name) {
1064 if (auto lock = lookupNamedOp<LockOp>(device, name->getAttr())) {
1065 locks.push_back(lock);
1066 }
1067 }
1068 }
1069 }
1070 return locks;
1071}
1072
1073StringRef ObjectFifoPoolOp::getBaseName() {
1074 StringRef name = getSymName();
1075 return name.consume_back("__pool") ? name : getSymName();
1076}
1077
1078bool ObjectFifoPoolOp::canWalkWhole() {
1079 auto device = (*this)->getParentOfType<DeviceOp>();
1080 return walksWholeDepth(
1081 getTileLike().isMemTile(), getDepth(),
1082 static_cast<int64_t>(device.getTargetModel().getMaxLockValue())) &&
1083 !getContiguous() && getSegmentOps().size() == 1 && !getRepeatCount() &&
1084 !getBuffers() &&
1085 llvm::all_of(device.getOps<ObjectFifoDmaEndpointOp>(),
1086 [&](ObjectFifoDmaEndpointOp end) {
1087 auto dims = end.getDimensions();
1088 return end.getPool() != getSymName() ||
1089 (!end.getDispatch() && !end.getOutOfOrder() &&
1090 end.getReleaseStrategy() !=
1091 ObjectFifoRelease::PushQueue &&
1092 (!end.drains() || !dims || dims->empty() ||
1093 (*dims)[0].getValue().empty()));
1094 });
1095}
1096
1097//===----------------------------------------------------------------------===//
1098// ObjectFifo endpoints
1099//===----------------------------------------------------------------------===//
1100
1101namespace {
1102
1103LogicalResult verifyEndpoint(Operation *op, ObjectFifoPoolOp pool,
1104 std::optional<ArrayAttr> segments) {
1105 std::vector<ObjectFifoSegmentOp> all = pool.getSegmentOps();
1106 if (!segments) {
1107 if (all.size() > 1) {
1108 return op->emitOpError(
1109 "must list segments explicitly for a multi-segment pool");
1110 }
1111 return success();
1112 }
1113 if (segments->empty()) {
1114 return op->emitOpError("expects at least one segment");
1115 }
1116
1117 int64_t previous = -1;
1118 for (auto name : segments->getAsRange<FlatSymbolRefAttr>()) {
1119 auto segment = SymbolTable::lookupNearestSymbolFrom<ObjectFifoSegmentOp>(
1120 pool, name.getAttr());
1121 if (!segment) {
1122 return op->emitOpError("references undefined segment '")
1123 << name.getValue() << "' in pool '" << pool.getSymName() << "'";
1124 }
1125 // The endpoint's memref is one run of the object, and `dimensions` pairs
1126 // with this list positionally.
1127 if (static_cast<int64_t>(segment.getOffset()) <= previous) {
1128 return op->emitOpError("segments must be named in increasing offset "
1129 "order");
1130 }
1131 previous = segment.getOffset();
1132 }
1133 return success();
1134}
1135
1136ObjectFifoPoolOp lookupPool(Operation *op, StringRef name) {
1137 auto device = op->getParentOfType<DeviceOp>();
1138 return SymbolTable::lookupNearestSymbolFrom<ObjectFifoPoolOp>(
1139 device, StringAttr::get(op->getContext(), name));
1140}
1141
1142} // namespace
1143
1144/// The subset of `pool`'s segments `selected` names. Omission selects the
1145/// pool's only segment.
1146static std::vector<ObjectFifoSegmentOp>
1147selectSegments(ObjectFifoPoolOp pool, std::optional<ArrayAttr> selected) {
1148 if (!pool) {
1149 return {};
1150 }
1151 std::vector<ObjectFifoSegmentOp> all = pool.getSegmentOps();
1152 if (!selected) {
1153 return all.empty() ? std::vector<ObjectFifoSegmentOp>{}
1154 : std::vector<ObjectFifoSegmentOp>{all.front()};
1155 }
1156 std::vector<ObjectFifoSegmentOp> chosen;
1157 for (auto name : selected->getAsRange<FlatSymbolRefAttr>()) {
1158 if (auto segment =
1159 SymbolTable::lookupNearestSymbolFrom<ObjectFifoSegmentOp>(
1160 pool, name.getAttr())) {
1161 chosen.push_back(segment);
1162 }
1163 }
1164 return chosen;
1165}
1166
1167TileLike ObjectFifoCoreEndpointOp::getTileLike() {
1168 return dyn_cast<TileLike>(getTile().getDefiningOp());
1169}
1170
1171ObjectFifoPoolOp ObjectFifoCoreEndpointOp::getPoolOp() {
1172 return lookupPool(*this, getPool());
1173}
1174
1175std::vector<ObjectFifoSegmentOp>
1176ObjectFifoCoreEndpointOp::getSelectedSegments() {
1177 return selectSegments(getPoolOp(), getSegments());
1178}
1179
1180LogicalResult ObjectFifoCoreEndpointOp::verify() {
1181 ObjectFifoPoolOp pool = getPoolOp();
1182 if (!pool) {
1183 return emitOpError("references undefined pool '") << getPool() << "'";
1184 }
1185 if (failed(verifyEndpoint(*this, pool, getSegments()))) {
1186 return failure();
1187 }
1188
1189 if (auto push = getCorePushAttr()) {
1190 if (drains())
1191 return emitOpError("core_push requires a filling core endpoint");
1192 if (auto values = pool.getInitValues(); values && !values->empty())
1193 return emitOpError("core_push does not support initialized objects");
1194 if (push.getBdTags().size() != static_cast<size_t>(pool.getDepth()))
1195 return emitOpError("core_push requires one BD tag per pool buffer");
1196 if (llvm::any_of(push.getBdTags(), [](int32_t tag) { return tag < 0; }))
1197 return emitOpError("core_push requires nonnegative BD tags");
1198 }
1199
1200 // The core sees one memref, so the segments it selects have to be a single
1201 // run of the object.
1202 int64_t next = -1;
1203 for (ObjectFifoSegmentOp segment : getSelectedSegments()) {
1204 if (next >= 0 && segment.getOffset() != next) {
1205 return emitOpError("a core endpoint's segments must be contiguous");
1206 }
1207 next = segment.getOffset() + segment.getSize();
1208 }
1209 return success();
1210}
1211
1212std::pair<int64_t, int64_t> ObjectFifoCoreEndpointOp::getExtent() {
1213 std::vector<ObjectFifoSegmentOp> selected = getSelectedSegments();
1214 if (selected.empty()) {
1215 return {0, 0};
1216 }
1217 int64_t offset = selected.front().getOffset();
1218 int64_t end = selected.back().getOffset() + selected.back().getSize();
1219 return {offset, end - offset};
1220}
1221
1222MemRefType ObjectFifoCoreEndpointOp::getAccessType() {
1223 MemRefType elemType = getPoolOp().getElemType();
1224 auto [offset, size] = getExtent();
1225 if (offset == 0 && size == elemType.getNumElements()) {
1226 return elemType;
1227 }
1228 return cast<MemRefType>(memref::SubViewOp::inferRankReducedResultType(
1229 {size}, elemType, {offset}, {size}, {1}));
1230}
1231
1232TileLike ObjectFifoDmaEndpointOp::getTileLike() {
1233 return dyn_cast<TileLike>(getTile().getDefiningOp());
1234}
1235
1236ObjectFifoPoolOp ObjectFifoDmaEndpointOp::getPoolOp() {
1237 return lookupPool(*this, getPool());
1238}
1239
1240std::vector<ObjectFifoSegmentOp>
1241ObjectFifoDmaEndpointOp::getSelectedSegments() {
1242 return selectSegments(getPoolOp(), getSegments());
1243}
1244
1245int64_t ObjectFifoDmaEndpointOp::getNumDescriptors() {
1246 ObjectFifoPoolOp pool = getPoolOp();
1247 if (!pool)
1248 return 0;
1249 if (pool.getContiguous()) {
1250 // Taking k turns a pass, k descriptors walk the objects, each k apart.
1251 if (!walksStream()) {
1252 SmallVector<ObjectFifoDmaEndpointOp> ends = turnEndpoints(*this);
1253 if (ends.empty())
1254 return 1;
1255 ArrayRef<Attribute> turns = getTurnsAttr().getValue();
1256 SmallVector<std::pair<int64_t, int64_t>> counts;
1257 for (ObjectFifoDmaEndpointOp end : ends)
1258 counts.push_back(
1259 {llvm::count(turns, FlatSymbolRefAttr::get(end.getSymNameAttr())),
1260 end.getPoolOp().getContiguous() ? 0 : end.getNumDescriptors()});
1261 return llvm::count(turns, FlatSymbolRefAttr::get(getSymNameAttr())) *
1262 turnPasses(counts);
1263 }
1264 StreamWindow window;
1265 return succeeded(getStreamWindow(window)) ? window.outputs : 0;
1266 }
1267 return pool.getDepth() * getSelectedSegments().size();
1268}
1269
1270bool ObjectFifoDmaEndpointOp::walksStream() {
1271 ObjectFifoPoolOp pool = getPoolOp();
1272 auto dims = getDimensions();
1273 return pool && pool.getContiguous() && drains() && dims && !dims->empty() &&
1274 !(*dims)[0].getValue().empty();
1275}
1276
1277LogicalResult ObjectFifoDmaEndpointOp::getStreamWindow(StreamWindow &window) {
1278 auto dims = getDimensions();
1279 if (!dims || dims->empty())
1280 return emitOpError("walks no stream: it has no dimensions");
1281 ArrayRef<BDPadLayoutAttr> pads;
1282 if (auto padding = getPadDimensions(); padding && !padding->empty())
1283 pads = (*padding)[0].getValue();
1284 return inferStreamWindow((*dims)[0].getValue(), pads,
1285 getPoolOp().getObjectSize(),
1286 [&] { return emitOpError(); }, window);
1287}
1288
1289int64_t ObjectFifoDmaEndpointOp::getRepeat() {
1290 ObjectFifoPoolOp pool = getPoolOp();
1291 // The filling end covers the batch in one acquire instead.
1292 return pool && drains() ? pool.getRepeatCount().value_or(1) : 1;
1293}
1294
1295bool ObjectFifoDmaEndpointOp::repeatsInHardware() {
1296 return getRepeat() > 1 &&
1297 (getReleaseStrategy() == ObjectFifoRelease::PushQueue ||
1298 (getNumDescriptors() == 1 && !getIterCount()));
1299}
1300
1301int64_t ObjectFifoDmaEndpointOp::getDispatchPasses() {
1302 ObjectFifoPoolOp pool = getPoolOp();
1303 std::optional<ArrayAttr> turns = getDispatch();
1304 if (!pool || !turns || turns->empty())
1305 return 1;
1306 int64_t depth = std::max<int64_t>(pool.getDepth(), 1);
1307 return std::lcm(depth, static_cast<int64_t>(turns->size())) / depth;
1308}
1309
1310SmallVector<ObjectFifoDmaEndpointOp>
1311xilinx::AIE::turnEndpoints(ObjectFifoDmaEndpointOp endpoint) {
1312 SmallVector<ObjectFifoDmaEndpointOp> ends;
1313 if (std::optional<ArrayAttr> turns = endpoint.getTurns())
1314 for (auto name : turns->getAsRange<FlatSymbolRefAttr>())
1315 if (auto end =
1316 SymbolTable::lookupNearestSymbolFrom<ObjectFifoDmaEndpointOp>(
1317 endpoint, name);
1318 end && !llvm::is_contained(ends, end))
1319 ends.push_back(end);
1320 return ends;
1321}
1322
1323int64_t xilinx::AIE::turnPasses(ArrayRef<std::pair<int64_t, int64_t>> ends) {
1324 int64_t passes = 1;
1325 for (auto [appearances, count] : ends)
1326 if (count > 0)
1327 passes = std::lcm(passes, count / std::gcd(count, appearances));
1328 return passes;
1329}
1330
1331int64_t ObjectFifoDmaEndpointOp::getNumBDs() {
1332 SmallVector<ObjectFifoDmaEndpointOp> ends = turnEndpoints(*this);
1333 if (ends.empty())
1334 return getNumDescriptors() * (repeatsInHardware() ? 1 : getRepeat()) *
1335 getDispatchPasses();
1336 if (ends.front() != *this)
1337 return 0;
1338 ArrayRef<Attribute> turns = getTurnsAttr().getValue();
1339 SmallVector<std::pair<int64_t, int64_t>> counts;
1340 for (ObjectFifoDmaEndpointOp end : ends)
1341 counts.push_back(
1342 {llvm::count(turns, FlatSymbolRefAttr::get(end.getSymNameAttr())),
1343 end.getPoolOp().getContiguous() ? 0 : end.getNumDescriptors()});
1344 return turnPasses(counts) * static_cast<int64_t>(turns.size());
1345}
1346
1347DMAChannelDir ObjectFifoDmaEndpointOp::getRouteDirection() {
1348 return drains() ? DMAChannelDir::MM2S : DMAChannelDir::S2MM;
1349}
1350
1351WireBundle ObjectFifoDmaEndpointOp::getRouteBundle() { return WireBundle::DMA; }
1352
1353std::optional<int> ObjectFifoDmaEndpointOp::getRouteChannel() {
1354 return getChannelIndex();
1355}
1356
1357void ObjectFifoDmaEndpointOp::setRouteChannel(int channel) {
1358 setChannelIndex(channel);
1359}
1360
1361LogicalResult ObjectFifoDmaEndpointOp::verify() {
1362 ObjectFifoPoolOp pool = getPoolOp();
1363 if (!pool) {
1364 return emitOpError("references undefined pool '") << getPool() << "'";
1365 }
1366
1367 if (failed(verifyEndpoint(*this, pool, getSegments()))) {
1368 return failure();
1369 }
1370 if (failed(verifyAssignedPacket(*this, getPacketAttr()))) {
1371 return failure();
1372 }
1373
1374 if (getReleaseStrategy() == ObjectFifoRelease::PushQueue) {
1375 if (!drains() || !getTileLike().isCoreTile() ||
1377 return emitOpError("push_queue requires a sending DMA on a core tile "
1378 "with core-issued task support");
1379 if (auto values = pool.getInitValues(); values && !values->empty())
1380 return emitOpError("push_queue does not support initialized objects");
1381 if (getRepeat() - 1 > getTargetModel(*this).getMaxRepeatCount())
1382 return emitOpError(
1383 "push_queue repeat_count exceeds the hardware task limit");
1384 if (getOutOfOrder() && getRepeat() > 1)
1385 return emitOpError(
1386 "push_queue cannot repeat out-of-order packets: "
1387 "each repetition may need a different destination BD ID");
1388 }
1389
1390 if (std::optional<ArrayAttr> turns = getDispatch()) {
1391 if (!drains())
1392 return emitOpError("only a draining endpoint can dispatch");
1393 if (turns->empty())
1394 return emitOpError("dispatch names no destination");
1395 if (getPacketAttr())
1396 return emitOpError("a dispatching endpoint carries a header per turn in "
1397 "`dispatchPackets`, not one `packet`");
1398 if (pool.getRepeatCount().value_or(1) > 1)
1399 return emitOpError("cannot dispatch a pool that repeats its objects");
1400 if (std::optional<int32_t> iterCount = getIterCount();
1401 iterCount && *iterCount % std::max<int64_t>(getDispatchPasses(), 1))
1402 return emitOpError("iterCount ")
1403 << *iterCount << " is not a whole number of dispatch rounds ("
1404 << getDispatchPasses() << " passes over the pool)";
1405 }
1406 if (std::optional<ArrayAttr> headers = getDispatchPackets()) {
1407 std::optional<ArrayAttr> turns = getDispatch();
1408 if (!turns || turns->size() != headers->size())
1409 return emitOpError("dispatchPackets needs one header per dispatch turn");
1410 for (auto header : headers->getAsRange<PacketInfoAttr>())
1411 if (failed(verifyAssignedPacket(*this, header, "dispatchPackets")))
1412 return failure();
1413 }
1414 if (std::optional<ArrayAttr> turns = getTurns()) {
1415 if (!llvm::is_contained(*turns, FlatSymbolRefAttr::get(getSymNameAttr())))
1416 return emitOpError("takes no turn of its own in `turns`");
1417 for (auto name : turns->getAsRange<FlatSymbolRefAttr>())
1418 if (!SymbolTable::lookupNearestSymbolFrom<ObjectFifoDmaEndpointOp>(*this,
1419 name))
1420 return emitOpError("turns names ")
1421 << name << ", which is no objectfifo.dma_endpoint";
1422 if (getDispatch() || getPacketAttr() || getIterCount() || getRepeat() > 1)
1423 return emitOpError("taking turns on a channel, it cannot also dispatch, "
1424 "carry a header, stop after iterCount or repeat");
1425 for (ObjectFifoDmaEndpointOp end : turnEndpoints(*this))
1426 if (end.getTile() != getTile() || end.getRole() != getRole() ||
1427 end.getTurnsAttr() != getTurnsAttr() ||
1428 end.getChannelIndexAttr() != getChannelIndexAttr())
1429 return emitOpError("takes turns with ")
1430 << end.getSymNameAttr()
1431 << ", which is not on this tile and channel with the same turns";
1432 if (pool.getContiguous() && !walksStream() &&
1433 pool.getDepth() % getNumDescriptors())
1434 return emitOpError("walks a contiguous pool of ")
1435 << pool.getDepth() << " objects in " << getNumDescriptors()
1436 << " turns a pass, which do not divide them";
1437 }
1438
1439 std::optional<ArrayAttr> segmentNames = getSegments();
1440 size_t selectedCount = segmentNames ? segmentNames->size() : 1;
1441 auto dimensions = getDimensions();
1442 if (dimensions && dimensions->size() != selectedCount) {
1443 return emitOpError("dimensions has ")
1444 << dimensions->size() << " entries for " << selectedCount
1445 << " selected segments";
1446 }
1447 auto padding = getPadDimensions();
1448 if (padding && padding->size() != selectedCount) {
1449 return emitOpError("padDimensions has ")
1450 << padding->size() << " entries for " << selectedCount
1451 << " selected segments";
1452 }
1453 if (padding && !drains()) {
1454 return emitOpError("padDimensions is only valid on a draining endpoint");
1455 }
1456 if (getPadValue() != 0 &&
1457 (!padding || llvm::none_of(*padding, [](BDPadLayoutArrayAttr entry) {
1458 return !entry.empty();
1459 }))) {
1460 return emitOpError("nonzero padValue requires a non-empty padDimensions "
1461 "entry");
1462 }
1463
1464 // A walk over the stream reads across the pool's objects, outputs apart.
1465 StreamWindow window;
1466 if (walksStream())
1467 return getStreamWindow(window);
1468
1469 std::vector<ObjectFifoSegmentOp> selected = getSelectedSegments();
1470 for (size_t position = 0; position < selectedCount; ++position) {
1471 BDDimLayoutArrayAttr dims =
1472 dimensions ? (*dimensions)[position] : BDDimLayoutArrayAttr();
1473 BDPadLayoutArrayAttr pads =
1474 padding ? (*padding)[position] : BDPadLayoutArrayAttr();
1475 if (pads && !pads.empty() && (!dims || dims.empty())) {
1476 return emitOpError("padDimensions entry ")
1477 << position << " requires dimensions for the same segment";
1478 }
1479 if (dims && pads && !pads.empty() && dims.size() != pads.size()) {
1480 return emitOpError("dimensions and padDimensions entry ")
1481 << position << " have different ranks";
1482 }
1483 if (dims && !dims.empty() &&
1484 getDimsMaxIdx(dims) >= selected[position].getSize()) {
1485 return emitOpError("dimensions entry ")
1486 << position << " exceeds selected segment '"
1487 << selected[position].getSymName() << "' of size "
1488 << selected[position].getSize();
1489 }
1490 }
1491 return success();
1492}
1493
1494TileLike RouteEndpointOp::getTileLike() {
1495 return dyn_cast<TileLike>(getTile().getDefiningOp());
1496}
1497
1498DMAChannelDir RouteEndpointOp::getRouteDirection() {
1499 // A flow runs from its source, so an end named there sends and every other
1500 // end receives.
1501 auto device = getOperation()->getParentOfType<DeviceOp>();
1502 StringRef name = getSymName();
1503 bool named = false;
1504 for (auto flow : device.getOps<RouteOp>()) {
1505 if (flow.namesAsSource(name)) {
1506 return DMAChannelDir::MM2S;
1507 }
1508 named |= flow.namesAsDestination(name);
1509 }
1510 assert(named && "endpoint is named by no flow, so it has no direction to "
1511 "read; --aie-objectfifo-verify rejects this");
1512 (void)named;
1513 return DMAChannelDir::S2MM;
1514}
1515
1516WireBundle RouteEndpointOp::getRouteBundle() { return getBundle(); }
1517
1518std::optional<int> RouteEndpointOp::getRouteChannel() {
1519 return getChannelIndex();
1520}
1521
1522void RouteEndpointOp::setRouteChannel(int channel) { setChannelIndex(channel); }
1523
1524// consumed_by sits on a shim input (MM2S) and each of its proofs names a shim
1525// output (S2MM), the `outputs` kind of op that `namesShimOutput` recognises. A
1526// route end's direction is its flows', which allocation removes, so there only
1527// the shim is checked.
1528static LogicalResult
1529verifyConsumedBy(Operation *op, ArrayAttr proofs, bool shimInput,
1530 StringRef outputs,
1531 function_ref<bool(FlatSymbolRefAttr)> namesShimOutput) {
1532 if (!shimInput)
1533 return op->emitOpError("consumed_by belongs on a shim input (MM2S)");
1534 for (auto proof : proofs.getAsRange<ConsumedByAttr>())
1535 if (!namesShimOutput(proof.getOutput()))
1536 return op->emitOpError("consumed_by output ")
1537 << proof.getOutput() << " names no shim output (S2MM " << outputs
1538 << ")";
1539 return success();
1540}
1541
1542LogicalResult RouteEndpointOp::verify() {
1543 TileLike tile = getTileLike();
1544 if (!tile) {
1545 return emitOpError("tile operand is not an aie.tile or aie.logical_tile");
1546 }
1547 if (failed(verifyAssignedPacket(*this, getPacketAttr()))) {
1548 return failure();
1549 }
1550 // getRouteDirection() reads this end's direction off the flows naming it, so
1551 // they must all name it on the same side. Packet flows can share the channel;
1552 // a circuit owns it.
1553 StringRef name = getSymName();
1554 SmallVector<RouteOp> sources, dests;
1555 for (auto flow : (*this)->getParentOfType<DeviceOp>().getOps<RouteOp>()) {
1556 if (flow.namesAsSource(name))
1557 sources.push_back(flow);
1558 if (flow.namesAsDestination(name))
1559 dests.push_back(flow);
1560 }
1561 if (!sources.empty() && !dests.empty()) {
1562 return emitOpError("is named as both a source and a destination, so its "
1563 "direction is ambiguous");
1564 }
1565 ArrayRef<RouteOp> sharers = sources.empty() ? dests : sources;
1566 if (sharers.size() > 1) {
1567 if (llvm::any_of(sharers, [](RouteOp flow) { return !flow.getPacket(); }))
1568 return emitOpError("drives one channel, which a circuit flow owns, but "
1569 "it is named ")
1570 << sharers.size() << " times";
1571 llvm::SmallDenseSet<int> ids;
1572 for (RouteOp flow : sources) {
1573 std::optional<uint16_t> id = flow.getPacketAttr().getPktId();
1574 if (!id)
1575 return emitOpError("sends several packet flows, so each must pin its "
1576 "pkt_id for the BDs to state it");
1577 if (!ids.insert(*id).second)
1578 return emitOpError("sends several packet flows, two of which pin the "
1579 "same pkt_id ")
1580 << static_cast<int>(*id) << "; each needs its own";
1581 }
1582 // The port keeps or drops every header it receives.
1583 if (llvm::any_of(dests, [&](RouteOp flow) {
1584 return flow.getKeepPktHeader() != dests[0].getKeepPktHeader();
1585 }))
1586 return emitOpError("receives several packet flows, so they must agree "
1587 "on keep_pkt_header");
1588 }
1589 if (ArrayAttr proofs = getConsumedByAttr()) {
1590 auto device = (*this)->getParentOfType<DeviceOp>();
1591 if (failed(verifyConsumedBy(
1592 *this, proofs, tile.isShimTile(),
1593 "aie.route_endpoint or aie.shim_dma_allocation",
1594 [&](FlatSymbolRefAttr symbol) {
1595 Operation *output = device.lookupSymbol(symbol.getAttr());
1596 if (auto alloc = dyn_cast_or_null<ShimDMAAllocationOp>(output))
1597 return alloc.getChannelDir() == DMAChannelDir::S2MM;
1598 auto end = dyn_cast_or_null<RouteEndpointOp>(output);
1599 return end && end.getTileLike().isShimTile();
1600 })))
1601 return failure();
1602 }
1603 switch (getBundle()) {
1604 case WireBundle::DMA:
1605 // The runtime drives a shim's DMA, and a DMA program or the runtime
1606 // sequence any other tile's; either names this op for the channel.
1607 return success();
1608 case WireBundle::PLIO:
1609 if (!tile.isShimTile()) {
1610 return emitOpError("a PLIO end is on a shim");
1611 }
1612 return success();
1613 case WireBundle::Core:
1614 return success();
1615 default:
1616 return emitOpError("bundle must be DMA, PLIO or Core");
1617 }
1618}
1619
1620//===----------------------------------------------------------------------===//
1621// RouteOp
1622//===----------------------------------------------------------------------===//
1623
1624// A route's sources print bare when there is one, bracketed when there are
1625// several; both forms parse.
1626static ParseResult parseRouteSources(OpAsmParser &parser, ArrayAttr &sources) {
1627 SmallVector<Attribute> names;
1628 auto parseOne = [&]() -> ParseResult {
1629 FlatSymbolRefAttr name;
1630 if (parser.parseAttribute(name))
1631 return failure();
1632 names.push_back(name);
1633 return success();
1634 };
1635 if (succeeded(parser.parseOptionalLSquare())) {
1636 if (failed(parser.parseOptionalRSquare()) &&
1637 (parser.parseCommaSeparatedList(parseOne) || parser.parseRSquare()))
1638 return failure();
1639 } else if (parseOne()) {
1640 return failure();
1641 }
1642 sources = parser.getBuilder().getArrayAttr(names);
1643 return success();
1644}
1645
1646static void printRouteSources(OpAsmPrinter &p, Operation *, ArrayAttr sources) {
1647 if (sources.size() == 1) {
1648 p.printAttribute(sources[0]);
1649 return;
1650 }
1651 p << '[';
1652 llvm::interleaveComma(sources, p,
1653 [&](Attribute name) { p.printAttribute(name); });
1654 p << ']';
1655}
1656
1657LogicalResult RouteOp::verify() {
1658 if (getSources().empty()) {
1659 return emitOpError("expects at least one source");
1660 }
1661 if (getDestinations().empty()) {
1662 return emitOpError("expects at least one destination");
1663 }
1664
1665 if (getKeepPktHeader() && !getPacket()) {
1666 return emitOpError("keep_pkt_header is only meaningful on a packet flow");
1667 }
1668
1669 auto device = (*this)->getParentOfType<DeviceOp>();
1670 auto lookup = [&](StringRef name) {
1671 return dyn_cast_or_null<RouteEndpoint>(SymbolTable::lookupNearestSymbolFrom(
1672 device, StringAttr::get(getContext(), name)));
1673 };
1674
1675 llvm::SmallDenseSet<StringRef> seen;
1676 for (StringRef source : getSourceNames()) {
1677 if (!lookup(source)) {
1678 return emitOpError("source '") << source << "' is not an endpoint";
1679 }
1680 if (!seen.insert(source).second) {
1681 return emitOpError("names '@") << source << "' as a source twice";
1682 }
1683 }
1684 for (StringRef destination : getDestinationNames()) {
1685 if (!lookup(destination)) {
1686 return emitOpError("destination '")
1687 << destination << "' is not an endpoint";
1688 }
1689 if (!seen.insert(destination).second) {
1690 return emitOpError("names '@")
1691 << destination
1692 << "' more than once; an end is a source or a "
1693 "destination, and only one of each";
1694 }
1695 }
1696
1697 // A circuit joins one slave port to its master ports, so only packets can
1698 // share a destination channel between sources.
1699 if (isFanIn() && !getPacket()) {
1700 return emitOpError("has several sources, so it needs a `packet` header; "
1701 "a circuit cannot merge streams");
1702 }
1703 return success();
1704}
1705
1706BDDimLayoutArrayAttr
1707ObjectFifoCreateOp::getDimensionsFromStream(Value consumerTile) {
1708 int dimsIndex = 0;
1709 for (auto cons : getConsumerTiles()) {
1710 if (cons == consumerTile)
1711 break;
1712 dimsIndex++;
1713 }
1714 return getDimensionsFromStreamPerConsumer()[dimsIndex];
1715}
1716
1718 OpAsmParser &parser, OpAsmParser::UnresolvedOperand &operand,
1719 BDDimLayoutArrayAttr &dimensions) {
1720 std::vector<BDDimLayoutAttr> emptyDims = {};
1721 if (parser.parseOperand(operand))
1722 return failure();
1723 if (succeeded(parser.parseOptionalKeyword("dimensionsToStream"))) {
1724 if (parser.parseCustomAttributeWithFallback<BDDimLayoutArrayAttr>(
1725 dimensions)) {
1726 return failure();
1727 }
1728 } else {
1729 dimensions =
1730 BDDimLayoutArrayAttr::get(parser.getContext(), ArrayRef(emptyDims));
1731 }
1732 return success();
1733}
1734
1735void xilinx::AIE::printObjectFifoProducerTile(OpAsmPrinter &printer,
1736 Operation *op, Value operand,
1737 BDDimLayoutArrayAttr dimensions) {
1738 printer << operand;
1739 if (!dimensions.empty()) {
1740 printer << " dimensionsToStream ";
1741 printer.printStrippedAttrOrType(dimensions);
1742 }
1743}
1744
1745ParseResult xilinx::AIE::parseDMAStartChannel(OpAsmParser &parser,
1746 Attribute &channel) {
1747 StringAttr name;
1748 if (succeeded(parser.parseOptionalSymbolName(name))) {
1749 channel = FlatSymbolRefAttr::get(name);
1750 return success();
1751 }
1752 int32_t index;
1753 if (parser.parseInteger(index))
1754 return failure();
1755 channel = parser.getBuilder().getI32IntegerAttr(index);
1756 return success();
1757}
1758
1759LogicalResult xilinx::AIE::verifyDMAChannelsResolved(DeviceOp device) {
1760 bool resolved = true;
1761 device.walkOutsideRuntimeSequences<DMAStartOp>([&](DMAStartOp start) {
1762 if (FlatSymbolRefAttr endpoint = start.getEndpoint()) {
1763 start.emitOpError() << "names route endpoint " << endpoint
1764 << " in place of a channel index; run "
1765 "--aie-objectfifo-allocate to assign one";
1766 resolved = false;
1767 }
1768 });
1769 return success(resolved);
1770}
1771
1772void xilinx::AIE::printDMAStartChannel(OpAsmPrinter &printer, Operation *op,
1773 Attribute channel) {
1774 if (auto index = dyn_cast<IntegerAttr>(channel))
1775 printer << index.getInt();
1776 else
1777 printer.printAttributeWithoutType(channel);
1778}
1779
1780ParseResult
1782 ObjectFifoPortAttr &port,
1783 SmallVectorImpl<Type> &objects) {
1784 if (parser.parseLParen())
1785 return failure();
1786
1787 StringRef portName;
1788 SMLoc portLoc = parser.getCurrentLocation();
1789 if (succeeded(parser.parseOptionalKeyword(&portName))) {
1790 std::optional<ObjectFifoPort> parsed = symbolizeObjectFifoPort(portName);
1791 if (!parsed)
1792 return parser.emitError(portLoc, "invalid objectFifo port: ") << portName;
1793 port = ObjectFifoPortAttr::get(parser.getContext(), *parsed);
1794 if (parser.parseComma())
1795 return failure();
1796 }
1797
1798 int64_t count;
1799 SMLoc countLoc = parser.getCurrentLocation();
1800 if (parser.parseInteger(count) || parser.parseRParen() ||
1801 parser.parseColon() || parser.parseTypeList(objects))
1802 return failure();
1803
1804 if (count != static_cast<int64_t>(objects.size()))
1805 return parser.emitError(countLoc, "acquires ")
1806 << count << " objects but names " << objects.size() << " of them";
1807 return success();
1808}
1809
1810void xilinx::AIE::printObjectFifoAcquireObjects(OpAsmPrinter &printer,
1811 Operation *op,
1812 ObjectFifoPortAttr port,
1813 TypeRange objects) {
1814 printer << "(";
1815 if (port)
1816 printer << stringifyObjectFifoPort(port.getValue()) << ", ";
1817 printer << objects.size() << ") : ";
1818 llvm::interleaveComma(objects, printer);
1819}
1820
1821ParseResult xilinx::AIE::parseObjectFifoReleaseCount(OpAsmParser &parser,
1822 ObjectFifoPortAttr &port,
1823 IntegerAttr &size) {
1824 if (parser.parseLParen())
1825 return failure();
1826
1827 StringRef portName;
1828 SMLoc portLoc = parser.getCurrentLocation();
1829 if (succeeded(parser.parseOptionalKeyword(&portName))) {
1830 std::optional<ObjectFifoPort> parsed = symbolizeObjectFifoPort(portName);
1831 if (!parsed)
1832 return parser.emitError(portLoc, "invalid objectFifo port: ") << portName;
1833 port = ObjectFifoPortAttr::get(parser.getContext(), *parsed);
1834 if (parser.parseComma())
1835 return failure();
1836 }
1837
1838 int64_t count;
1839 if (parser.parseInteger(count) || parser.parseRParen())
1840 return failure();
1841 size = parser.getBuilder().getI32IntegerAttr(count);
1842 return success();
1843}
1844
1845void xilinx::AIE::printObjectFifoReleaseCount(OpAsmPrinter &printer,
1846 Operation *op,
1847 ObjectFifoPortAttr port,
1848 IntegerAttr size) {
1849 printer << "(";
1850 if (port)
1851 printer << stringifyObjectFifoPort(port.getValue()) << ", ";
1852 printer << size.getInt() << ")";
1853}
1854
1856 OpAsmParser &parser, SmallVectorImpl<OpAsmParser::UnresolvedOperand> &tiles,
1857 BDDimLayoutArrayArrayAttr &dimensions) {
1858 // parseCommaSeparatedList doesn't handle the missing case for "none",
1859 // so we handle it custom here.
1860 std::vector<BDDimLayoutArrayAttr> tileDims = {};
1861
1862 auto parseOneOperand = [&]() -> ParseResult {
1863 if (parser.parseOperand(tiles.emplace_back(), true)) {
1864 return failure();
1865 }
1866 // By default, create empty dimensions array for each consumer; this way,
1867 // we can be certain to have as many entries in the dimensions array as
1868 // there are customer
1869 BDDimLayoutArrayAttr dimAttr =
1870 BDDimLayoutArrayAttr::get(parser.getContext(), {});
1871
1872 if (succeeded(parser.parseOptionalKeyword("dimensionsFromStream"))) {
1873 // If specified, parse actual data layout transform dimensions
1874 if (parser.parseCustomAttributeWithFallback<BDDimLayoutArrayAttr>(
1875 dimAttr)) {
1876 return failure();
1877 }
1878 }
1879 tileDims.emplace_back(dimAttr);
1880 return success();
1881 };
1882
1883 if (parser.parseCommaSeparatedList(AsmParser::Delimiter::None,
1884 parseOneOperand, " in operand list"))
1885 return failure();
1886
1887 dimensions = BDDimLayoutArrayArrayAttr::get(parser.getContext(), tileDims);
1888 return success();
1889}
1890
1892 OpAsmPrinter &printer, Operation *op, OperandRange tiles,
1893 BDDimLayoutArrayArrayAttr dimsPerTileAttr) {
1894 size_t tileIdx = 0;
1895 for (auto tile : tiles) {
1896 printer << tile;
1897 if (dimsPerTileAttr && tileIdx < dimsPerTileAttr.size() &&
1898 dimsPerTileAttr[tileIdx] && !dimsPerTileAttr[tileIdx].empty()) {
1899 printer << " dimensionsFromStream ";
1900 printer.printStrippedAttrOrType(dimsPerTileAttr[tileIdx]);
1901 }
1902 if (tileIdx < tiles.size() - 1) {
1903 printer << ", ";
1904 }
1905 tileIdx++;
1906 }
1907}
1908
1909static void printObjectFifoConsumerElemType(OpAsmPrinter &p,
1910 ObjectFifoCreateOp op,
1911 TypeAttr consumerElemType) {
1912 if (consumerElemType)
1913 p << " -> " << consumerElemType;
1914}
1915
1916ParseResult xilinx::AIE::parseFlowVias(
1917 OpAsmParser &parser, SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vias,
1918 DenseI32ArrayAttr &ingressBundles, DenseI32ArrayAttr &ingressChannels,
1919 DenseI32ArrayAttr &egressBundles, DenseI32ArrayAttr &egressChannels) {
1920 SmallVector<int32_t> inBundles, inChannels, egBundles, egChannels;
1921 auto parseBundle = [&](SmallVectorImpl<int32_t> &out) -> ParseResult {
1922 StringRef kw;
1923 if (parser.parseKeyword(&kw)) {
1924 return failure();
1925 }
1926 auto bundle = symbolizeWireBundle(kw);
1927 if (!bundle) {
1928 return parser.emitError(parser.getCurrentLocation(),
1929 "invalid wire bundle '" + kw + "'");
1930 }
1931 out.push_back(static_cast<int32_t>(*bundle));
1932 return success();
1933 };
1934 // via ( %tile : <ingress bundle> : <ch> -> <egress bundle> : <ch>, ... )
1935 if (succeeded(parser.parseOptionalKeyword("via"))) {
1936 auto parseOne = [&]() -> ParseResult {
1937 int32_t inChan, egChan;
1938 if (parser.parseOperand(vias.emplace_back()) || parser.parseColon() ||
1939 parseBundle(inBundles) || parser.parseColon() ||
1940 parser.parseInteger(inChan) || parser.parseArrow() ||
1941 parseBundle(egBundles) || parser.parseColon() ||
1942 parser.parseInteger(egChan)) {
1943 return failure();
1944 }
1945 inChannels.push_back(inChan);
1946 egChannels.push_back(egChan);
1947 return success();
1948 };
1949 if (parser.parseCommaSeparatedList(AsmParser::Delimiter::Paren, parseOne)) {
1950 return failure();
1951 }
1952 }
1953 if (!vias.empty()) {
1954 MLIRContext *ctx = parser.getContext();
1955 ingressBundles = DenseI32ArrayAttr::get(ctx, inBundles);
1956 ingressChannels = DenseI32ArrayAttr::get(ctx, inChannels);
1957 egressBundles = DenseI32ArrayAttr::get(ctx, egBundles);
1958 egressChannels = DenseI32ArrayAttr::get(ctx, egChannels);
1959 }
1960 return success();
1961}
1962
1963void xilinx::AIE::printFlowVias(OpAsmPrinter &printer, Operation *op,
1964 OperandRange vias,
1965 DenseI32ArrayAttr ingressBundles,
1966 DenseI32ArrayAttr ingressChannels,
1967 DenseI32ArrayAttr egressBundles,
1968 DenseI32ArrayAttr egressChannels) {
1969 if (vias.empty()) {
1970 return;
1971 }
1972 printer << "via (";
1973 for (size_t i = 0; i < vias.size(); i++) {
1974 if (i) {
1975 printer << ", ";
1976 }
1977 printer << vias[i] << " : "
1978 << stringifyWireBundle(static_cast<WireBundle>(ingressBundles[i]))
1979 << " : " << ingressChannels[i] << " -> "
1980 << stringifyWireBundle(static_cast<WireBundle>(egressBundles[i]))
1981 << " : " << egressChannels[i];
1982 }
1983 printer << ")";
1984}
1985
1986static ParseResult parseObjectFifoConsumerElemType(OpAsmParser &parser,
1987 TypeAttr &consumerElemType) {
1988 if (failed(parser.parseOptionalArrow()))
1989 return success(); // no consumer type
1990 Type type;
1991 if (parser.parseType(type))
1992 return failure();
1993 consumerElemType = TypeAttr::get(type);
1994 return success();
1995}
1996
1997static void printObjectFifoInitValues(OpAsmPrinter &p, ObjectFifoCreateOp op,
1998 Attribute numElem, TypeAttr type,
1999 Attribute initValues) {
2000 if (op.getInitValues()) {
2001 p << "= [";
2002 // `numElem` may be an IntegerAttr (scalar depth) or an ArrayAttr of
2003 // per-endpoint depths; initValues populates the producer side, which
2004 // is the first entry of the ArrayAttr.
2005 int depth;
2006 if (isa<ArrayAttr>(numElem)) {
2007 depth =
2008 llvm::cast<mlir::IntegerAttr>(llvm::cast<mlir::ArrayAttr>(numElem)[0])
2009 .getInt();
2010 } else {
2011 depth = llvm::cast<mlir::IntegerAttr>(numElem).getInt();
2012 }
2013 for (int i = 0; i < depth; i++) {
2014 p.printStrippedAttrOrType(llvm::cast<mlir::ArrayAttr>(initValues)[i]);
2015 if (i < depth - 1) {
2016 p << ", ";
2017 }
2018 }
2019 p << "]";
2020 }
2021}
2022
2023static ParseResult parseObjectFifoInitValues(OpAsmParser &parser,
2024 Attribute numElem, TypeAttr type,
2025 Attribute &initValues) {
2026 int depth;
2027 if (isa<ArrayAttr>(numElem)) {
2028 depth =
2029 llvm::cast<mlir::IntegerAttr>(llvm::cast<mlir::ArrayAttr>(numElem)[0])
2030 .getInt();
2031 } else {
2032 depth = llvm::cast<mlir::IntegerAttr>(numElem).getInt();
2033 }
2034 auto objfifoType = llvm::cast<AIEObjectFifoType>(type.getValue());
2035 auto memrefType = llvm::cast<MemRefType>(objfifoType.getElementType());
2036
2037 if (!memrefType.hasStaticShape())
2038 return parser.emitError(parser.getNameLoc())
2039 << "type should be static shaped memref, but got " << memrefType;
2040
2041 if (parser.parseOptionalEqual())
2042 return success();
2043
2044 Type tensorType = mlir::memref::getTensorTypeFromMemRefType(memrefType);
2045 if (parser.parseAttribute(initValues, tensorType))
2046 return failure();
2047 for (int i = 0; i < depth; i++) {
2048 auto initialValues = llvm::dyn_cast<mlir::ArrayAttr>(initValues);
2049 if ((int)initialValues.size() != depth)
2050 return parser.emitError(parser.getNameLoc())
2051 << "initial values should initialize all objects";
2052 if (!llvm::isa<ElementsAttr>(initialValues[i]))
2053 return parser.emitError(parser.getNameLoc())
2054 << "initial value should be an elements attribute";
2055 }
2056
2057 return success();
2058}
2059
2060//===----------------------------------------------------------------------===//
2061// ObjectFifoCreateOp DMA decision
2062//===----------------------------------------------------------------------===//
2063
2064std::optional<ObjectFifoLinkOp> ObjectFifoCreateOp::getLink() {
2065 auto self = FlatSymbolRefAttr::get(getSymNameAttr());
2066 for (ObjectFifoLinkOp link :
2067 (*this)->getParentOfType<DeviceOp>().getOps<ObjectFifoLinkOp>())
2068 if (llvm::is_contained(link.getFifoIns(), self) ||
2069 llvm::is_contained(link.getFifoOuts(), self))
2070 return link;
2071 return std::nullopt;
2072}
2073
2074std::optional<ObjectFifoAllocateOp> ObjectFifoCreateOp::getAllocate() {
2075 for (ObjectFifoAllocateOp alloc :
2076 (*this)->getParentOfType<DeviceOp>().getOps<ObjectFifoAllocateOp>())
2077 if (alloc.getObjFifoName() == name().getValue())
2078 return alloc;
2079 return std::nullopt;
2080}
2081
2082bool ObjectFifoCreateOp::delegateReaches(const AIETargetModel &target,
2083 TileID delegate, TileID producer,
2084 TileID consumer) {
2085 using SharedMemory = AIETargetModel::SharedMemory;
2086 return llvm::all_of(std::array{producer, consumer}, [&](TileID end) {
2087 SharedMemory s = target.getSharedMemory(delegate, end);
2088 return s == SharedMemory::First || s == SharedMemory::Either;
2089 });
2090}
2091
2092bool ObjectFifoCreateOp::allocatedHome(const AIETargetModel &target,
2093 TileID producer, TileID consumer,
2094 TileID delegate,
2095 std::optional<TileID> linkTile) {
2096 TileID home = linkTile.value_or(delegate);
2097 return delegateReaches(target, delegate, producer, consumer) &&
2098 delegateReaches(target, home, producer, consumer);
2099}
2100
2101bool ObjectFifoCreateOp::allocateFallsBackToDMA() {
2102 auto self = FlatSymbolRefAttr::get(getSymNameAttr());
2103 bool linked = false;
2104 for (ObjectFifoLinkOp link :
2105 (*this)->getParentOfType<DeviceOp>().getOps<ObjectFifoLinkOp>()) {
2106 if (!llvm::is_contained(link.getFifoIns(), self) &&
2107 !llvm::is_contained(link.getFifoOuts(), self))
2108 continue;
2109 if (link.getOptionalSharedTile() && link.getPoolOwner() == *this)
2110 return false;
2111 linked = true;
2112 }
2113 return linked;
2114}
2115
2116LinkPoolMisfit ObjectFifoCreateOp::linkPoolMisfit() {
2117 ObjectFifoLinkOp producerLink = getProducerLink();
2118 ObjectFifoLinkOp consumerLink =
2119 getConsumerTiles().empty() ? ObjectFifoLinkOp() : getConsumerLink(0);
2120 if (producerLink && consumerLink)
2122 if (consumerLink && consumerLink.isMerge())
2124 if (producerLink && producerLink.isDispatch())
2126 ObjectFifoLinkOp link = producerLink ? producerLink : consumerLink;
2127 if (!link)
2128 return LinkPoolMisfit::None;
2129 ObjectFifoCreateOp owner = link.getPoolOwner();
2130 if (owner == *this)
2131 return LinkPoolMisfit::None;
2132 if (owner.getElemType() != getElemType())
2134 return getInitValues() ? LinkPoolMisfit::InitialValues : LinkPoolMisfit::None;
2135}
2136
2137std::optional<StringRef> ObjectFifoCreateOp::dmaReason() {
2138 if (forcesDMA())
2139 return "it asks for a dma transport";
2140 if (getRepeatCount())
2141 return "it repeats objects (`repeat_count`)";
2142 if (usesStream())
2143 return "an end is a Core stream port";
2144 if (getConsumerElemType())
2145 return "its consumer takes a different element type";
2146 if (getConsumerTiles().size() > 1)
2147 return "it has more than one consumer";
2148 if (!getDimensionsToStream().empty())
2149 return "its producer reshapes the data (`dimensionsToStream`)";
2150 for (BDDimLayoutArrayAttr dims : getDimensionsFromStreamPerConsumer())
2151 if (!dims.empty())
2152 return "a consumer reshapes the data (`dimensionsFromStream`)";
2153 // A link passes objects between two fifos through a tile's DMAs, unless an
2154 // aie.objectfifo.allocate names a delegate whose memory both ends reach.
2155 if (getLink() && !getAllocate())
2156 return "it is linked through a tile's DMAs";
2157 return std::nullopt;
2158}
2159
2160AIETargetModel::SharedMemory ObjectFifoCreateOp::sharedMemoryOf(
2161 const AIETargetModel &target, TileID producer, TileID consumer,
2162 std::optional<TileID> delegate, std::optional<TileID> linkTile) {
2163 using SharedMemory = AIETargetModel::SharedMemory;
2164 // A delegate both ends reach holds the objects even where the ends share no
2165 // memory of their own; a link's objects need one.
2166 if (delegate &&
2167 allocatedHome(target, producer, consumer, *delegate, linkTile)) {
2168 SharedMemory own = target.getSharedMemory(producer, consumer);
2169 return own == SharedMemory::None ? SharedMemory::Either : own;
2170 }
2171 if (linkTile)
2172 return SharedMemory::None;
2173 return target.getSharedMemory(producer, consumer);
2174}
2175
2176static std::optional<TileID> placedAt(Value tile) {
2177 auto tileLike = dyn_cast_or_null<TileLike>(tile.getDefiningOp());
2178 if (!tileLike)
2179 return std::nullopt;
2180 std::optional<int> col = tileLike.tryGetCol(), row = tileLike.tryGetRow();
2181 if (!col || !row)
2182 return std::nullopt;
2183 return TileID{*col, *row};
2184}
2185
2186std::optional<StringRef>
2187ObjectFifoCreateOp::placedDmaReason(AIETargetModel::SharedMemory &shared) {
2188 if (std::optional<StringRef> reason = dmaReason())
2189 return reason;
2190 std::optional<ObjectFifoAllocateOp> alloc = getAllocate();
2191 std::optional<TileID> producer = placedAt(getProducerTile());
2192 std::optional<TileID> consumer = placedAt(getConsumerTiles()[0]);
2193 std::optional<TileID> delegate =
2194 alloc ? placedAt(alloc->getDelegateTile()) : std::nullopt;
2195 std::optional<ObjectFifoLinkOp> link = getLink();
2196 std::optional<Value> linkPoint =
2197 link ? link->getOptionalSharedTile() : std::nullopt;
2198 std::optional<TileID> linkTile =
2199 linkPoint ? placedAt(*linkPoint) : std::nullopt;
2200 if (!producer || !consumer || (alloc && !delegate) || (link && !linkTile))
2201 return std::nullopt;
2202 shared = sharedMemoryOf(getTargetModel(*this), *producer, *consumer, delegate,
2203 linkTile);
2205 return link ? "it is linked through a tile's DMAs"
2206 : "its ends share no memory module";
2207 return std::nullopt;
2208}
2209
2210std::optional<StringRef> ObjectFifoCreateOp::pushQueueRefusal() {
2211 if (auto values = getInitValues(); values && !values->empty())
2212 return StringRef("push_queue does not support initialized objects");
2213 auto tile = cast<TileLike>(getProducerTile().getDefiningOp());
2214 auto device = (*this)->getParentOfType<DeviceOp>();
2215 if (!tile.isCoreTile() || streamsFromProducer() ||
2216 device.getTargetModel().getDmaTaskQueueDepth() == 0)
2217 return StringRef("push_queue requires a sending DMA on a core tile with "
2218 "core-issued task support");
2219 bool hasRelease = false;
2220 for (Operation *user : getProducerTile().getUsers())
2221 if (auto core = dyn_cast<CoreOp>(user))
2222 core.walk([&](ObjectFifoReleaseOp release) {
2223 hasRelease |= release.getObjectFifo() == *this &&
2224 release.getPort() == ObjectFifoPort::Produce;
2225 });
2226 if (!hasRelease)
2227 return StringRef("push_queue requires producer releases in a local core "
2228 "to submit send tasks");
2229 return std::nullopt;
2230}
2231
2232std::optional<StringRef>
2233ObjectFifoCreateOp::sharedReceiveRefusal(bool shim, bool outOfOrder) {
2234 if (shim)
2235 return StringRef("a shim's receiving channel writes into the runtime's "
2236 "tasks, which no object header picks");
2237 if (!outOfOrder)
2238 return StringRef("a channel its fifos receive in turn holds their order; "
2239 "out_of_order puts that order in each object's header");
2240 return std::nullopt;
2241}
2242
2243bool ObjectFifoCreateOp::sharesSendChannel() {
2244 auto pinned = [](ObjectFifoCreateOp fifo) -> std::optional<int> {
2245 ObjectFifoEndPortAttr port = fifo.getProdPortAttr();
2246 return port && !port.isCore() ? port.channelIndex() : std::nullopt;
2247 };
2248 auto byDMA = [](ObjectFifoCreateOp fifo) {
2250 return fifo.placedDmaReason(shared).has_value();
2251 };
2252 std::optional<int> channel = pinned(*this);
2253 Value tile = getProducerTile();
2254 if (!channel || cast<TileLike>(tile.getDefiningOp()).isShimTile() ||
2255 !byDMA(*this))
2256 return false;
2257 std::optional<TileID> at = placedAt(tile);
2258 for (ObjectFifoCreateOp other :
2259 (*this)->getParentOfType<DeviceOp>().getOps<ObjectFifoCreateOp>()) {
2260 Value otherTile = other.getProducerTile();
2261 if (other != *this && pinned(other) == channel &&
2262 (otherTile == tile || (at && placedAt(otherTile) == at)) &&
2263 byDMA(other))
2264 return true;
2265 }
2266 return false;
2267}
2268
2269int ObjectFifoCreateOp::dmaDepth(std::optional<unsigned> consumer) {
2270 if (consumer ? streamsToConsumer(*consumer) : streamsFromProducer())
2271 return 0;
2272 if (!consumer && getInitValues())
2273 return size();
2274 if (consumer && isa<ArrayAttr>(getElemNumber()))
2275 return size(*consumer + 1);
2276 if (size() == 0)
2277 return 0;
2278 // Enough objects for the largest acquire the end's core makes, plus one so
2279 // it can hold an object while the next arrives. A core on the producer's
2280 // tile may also consume, through another end.
2281 Value tile = consumer ? getConsumerTiles()[*consumer] : getProducerTile();
2282 ObjectFifoPort port =
2283 consumer ? ObjectFifoPort::Consume : ObjectFifoPort::Produce;
2284 int maxAcquire = 0;
2285 for (Operation *user : tile.getUsers())
2286 if (auto core = dyn_cast<CoreOp>(user))
2287 core.walk([&](ObjectFifoAcquireOp acquire) {
2288 if (acquire.getObjectFifo() == *this && acquire.getPort() == port)
2289 maxAcquire = std::max(maxAcquire, acquire.acqNumber());
2290 });
2291 if (maxAcquire == 0)
2292 return size();
2293 if (maxAcquire == 1 && size() == 1)
2294 return 1;
2295 return maxAcquire + 1;
2296}
2297
2298ObjectFifoLinkOp ObjectFifoCreateOp::getProducerLink() {
2299 auto self = FlatSymbolRefAttr::get(getSymNameAttr());
2300 for (ObjectFifoLinkOp link :
2301 (*this)->getParentOfType<DeviceOp>().getOps<ObjectFifoLinkOp>())
2302 if (llvm::is_contained(link.getFifoOuts(), self) &&
2303 link.getOptionalSharedTile() == getProducerTile())
2304 return link;
2305 return {};
2306}
2307
2308ObjectFifoLinkOp ObjectFifoCreateOp::getConsumerLink(unsigned consumer) {
2309 auto consumers = getConsumerTiles();
2310 Value tile = consumers[consumer];
2311 size_t earlier = llvm::count(consumers.take_front(consumer), tile);
2312 auto self = FlatSymbolRefAttr::get(getSymNameAttr());
2313 for (ObjectFifoLinkOp link :
2314 (*this)->getParentOfType<DeviceOp>().getOps<ObjectFifoLinkOp>())
2315 if (llvm::is_contained(link.getFifoIns(), self) &&
2316 link.getOptionalSharedTile() == tile && earlier-- == 0)
2317 return link;
2318 return {};
2319}
2320
2321std::optional<PlannedDmaEnd>
2322ObjectFifoCreateOp::plannedDmaEnd(std::optional<unsigned> consumer) {
2323 return plannedDmaEnd(consumer, placedAt);
2324}
2325
2326std::optional<PlannedDmaEnd>
2327ObjectFifoCreateOp::plannedDmaEnd(std::optional<unsigned> consumer,
2328 TilePositions positions) {
2329 bool drains = !consumer;
2330 Value tile = drains ? getProducerTile() : getConsumerTiles()[*consumer];
2331 auto tileLike = cast<TileLike>(tile.getDefiningOp());
2332 if (tileLike.isShimTile() ||
2333 (drains ? streamsFromProducer() : streamsToConsumer(*consumer)))
2334 return std::nullopt;
2335 auto device = (*this)->getParentOfType<DeviceOp>();
2336 int64_t maxLock = device.getTargetModel().getMaxLockValue();
2337 ObjectFifoEndPortAttr port =
2338 drains ? getProdPortAttr() : getConsumerPort(*consumer);
2339 std::optional<int> channel = port ? port.channelIndex() : std::nullopt;
2340 auto walkable = [&](int64_t depth) {
2341 return walksWholeDepth(tileLike.isMemTile(), depth, maxLock);
2342 };
2343
2344 ObjectFifoLinkOp link =
2345 drains ? getProducerLink() : getConsumerLink(*consumer);
2346 DataLayout layout = DataLayout::closest(*this);
2347 auto objectBytes = [&](Type fifoType) -> int64_t {
2348 MemRefType type = cast<AIEObjectFifoType>(fifoType).getElementType();
2349 return getMemRefSizeInBytes(type, layout);
2350 };
2351 auto ordered = [](ObjectFifoCreateOp fifo, bool drains) {
2352 return !fifo.getOutOfOrder() && (!drains || !usesPushQueue(fifo));
2353 };
2354 if (!link) {
2355 int depth = dmaDepth(consumer);
2356 if (!drains)
2357 return PlannedDmaEnd{depth,
2358 *this,
2359 static_cast<int>(*consumer),
2360 depth,
2361 walkable(depth) && ordered(*this, false),
2362 objectBytes(getConsumerElemTypeOrDefault()),
2363 0,
2364 channel};
2365 int64_t repeat = getRepeatCount().value_or(1);
2366 bool inHardware =
2367 repeat > 1 && (usesPushQueue(*this) || (depth == 1 && !getIterCount()));
2368 return PlannedDmaEnd{depth * (inHardware ? 1 : repeat),
2369 *this,
2370 -1,
2371 depth,
2372 walkable(depth) && ordered(*this, true) &&
2373 !getRepeatCount() &&
2374 getDimensionsToStream().empty(),
2375 objectBytes(getElemType()),
2376 0,
2377 channel};
2378 }
2379
2380 // A merge's participants fill one end, a dispatch's drain one, made by the
2381 // first of them the split reaches.
2382 if (drains ? link.isDispatch() : link.isMerge()) {
2383 std::vector<ObjectFifoCreateOp> sharing =
2384 drains ? link.getOutputObjectFifos() : link.getInputObjectFifos();
2385 for (ObjectFifoCreateOp fifo : device.getOps<ObjectFifoCreateOp>())
2386 if (llvm::is_contained(sharing, fifo)) {
2387 if (fifo != *this)
2388 return std::nullopt;
2389 break;
2390 }
2391 }
2392 int64_t depth = link.getPoolDepth(positions);
2393 int64_t segments = link.getPoolSegments().size();
2394 int64_t selected =
2395 (drains ? link.isDistribute() : link.isJoin()) ? 1 : segments;
2396 int64_t descriptors = depth * selected;
2397 int64_t contiguousObjects = 0;
2398 if (link.walksStream()) {
2399 StreamWindow window;
2400 bool inferred = succeeded(link.getStreamWindow(window));
2401 descriptors = !drains ? 1 : inferred ? window.outputs : 0;
2402 if (inferred)
2403 contiguousObjects = depth + window.top + window.bottom;
2404 }
2405 int64_t repeat = drains ? link.getRepeatCount().value_or(1) : 1;
2406 bool inHardware = repeat > 1 && ((drains && usesPushQueue(*this)) ||
2407 (descriptors == 1 && !getIterCount()));
2408 int64_t passes = 1;
2409 if (drains && link.isDispatch()) {
2410 int64_t objects = std::max<int64_t>(depth, 1);
2411 passes =
2412 std::lcm(objects, static_cast<int64_t>(link.getFifoOuts().size())) /
2413 objects;
2414 }
2415 bool drainLaysOut =
2416 llvm::any_of(link.getOutputObjectFifos(), [](ObjectFifoCreateOp out) {
2417 return !out.getDimensionsToStream().empty();
2418 });
2419 // A merge's one fill end is made for its first output.
2420 bool linkOrdered =
2421 llvm::all_of(
2422 link.getOutputObjectFifos(),
2423 [&](ObjectFifoCreateOp out) { return ordered(out, true); }) &&
2424 (link.isMerge() ||
2425 llvm::all_of(link.getInputObjectFifos(),
2426 [&](ObjectFifoCreateOp in) { return ordered(in, false); }));
2427 return PlannedDmaEnd{descriptors * (inHardware ? 1 : repeat) * passes,
2428 link,
2429 -1,
2430 depth,
2431 walkable(depth) && segments == 1 &&
2432 !link.getRepeatCount() && !link.walksStream() &&
2433 !link.isDispatch() && !drainLaysOut && linkOrdered,
2434 objectBytes(link.getPoolOwner().getElemType()),
2435 contiguousObjects,
2436 channel};
2437}
2438
2439bool ObjectFifoCreateOp::canTakeTurns() {
2440 if (getConsumerTiles().size() != 1)
2441 return false;
2442 auto pinned = [](ObjectFifoEndPortAttr port) {
2443 return port && (port.isCore() || port.channelIndex());
2444 };
2445 bool dispatch = false, walksStream = false;
2446 int64_t repeat = getRepeatCount().value_or(1);
2447 for (ObjectFifoLinkOp link :
2448 (*this)->getParentOfType<DeviceOp>().getOps<ObjectFifoLinkOp>()) {
2449 if (llvm::is_contained(link.getOutputObjectFifos(), *this)) {
2450 dispatch |= link.isDispatch();
2451 walksStream |= link.walksStream();
2452 repeat = std::max<int64_t>(repeat, link.getRepeatCount().value_or(1));
2453 }
2454 if (link.isMerge() && llvm::is_contained(link.getInputObjectFifos(), *this))
2455 return false;
2456 }
2457 return ObjectFifoDmaEndpointOp::canTakeTurns(
2458 usesPushQueue(*this),
2459 pinned(getProdPortAttr()) || pinned(getConsumerPort(0)), dispatch,
2460 packetHeader() != nullptr, getIterCount().has_value(), repeat,
2461 walksStream);
2462}
2463
2464bool ObjectFifoDmaEndpointOp::canTakeTurns(bool pushQueue, bool pinned,
2465 bool dispatch, bool packet,
2466 bool iterates, int64_t repeat,
2467 bool walksStream) {
2468 return !pushQueue && !pinned && !dispatch && !packet && !iterates &&
2469 repeat == 1 && !walksStream;
2470}
2471
2472int ObjectFifoCreateOp::sharedDepth() {
2473 int depth = size();
2474 if (auto depths = dyn_cast<ArrayAttr>(getElemNumber()))
2475 for (int index = 1, e = depths.size(); index < e; ++index)
2476 depth = std::max(depth, size(index));
2477 return depth;
2478}
2479
2480//===----------------------------------------------------------------------===//
2481// ObjectFifoAllocateOp
2482//===----------------------------------------------------------------------===//
2483
2484LogicalResult ObjectFifoAllocateOp::verify() {
2485 ObjectFifoCreateOp objFifo = getObjectFifo();
2486 if (!objFifo)
2487 return emitError("cannot retrieve associated object FIFO");
2488 if (!isa_and_nonnull<TileLike>(getDelegateTile().getDefiningOp()))
2489 return emitOpError("delegate must be a tile");
2490 if (objFifo.getConsumerTiles().size() != 1)
2491 return emitError("can only be used in 1-to-1 object FIFOs");
2492 // A linked fifo that shares memory keeps its objects in the link's one
2493 // pool, which a core fills or drains one pool object per acquire. Whether
2494 // it shares depends on where its tiles are placed.
2495 std::optional<ObjectFifoLinkOp> linked = objFifo.getLink();
2496 if (std::optional<StringRef> reason = objFifo.dmaReason())
2497 return emitError("cannot allocate a shared memory module to an objectfifo "
2498 "that moves by DMA: ")
2499 << *reason;
2500 if (LinkPoolMisfit misfit =
2501 linked ? objFifo.linkPoolMisfit() : LinkPoolMisfit::None;
2502 linked && misfit != LinkPoolMisfit::None) {
2503 ObjectFifoLinkOp link = *linked;
2504 auto producer = dyn_cast<TileOp>(objFifo.getProducerTile().getDefiningOp());
2505 auto consumer =
2506 dyn_cast<TileOp>(objFifo.getConsumerTiles()[0].getDefiningOp());
2507 auto delegate = dyn_cast<TileOp>(getDelegateTile().getDefiningOp());
2508 // A link without a link point is the link verifier's to report.
2509 std::optional<Value> linkPoint = link.getOptionalSharedTile();
2510 auto linkTile =
2511 linkPoint ? dyn_cast<TileOp>(linkPoint->getDefiningOp()) : TileOp();
2512 bool shared = producer && consumer && delegate && linkTile &&
2513 ObjectFifoCreateOp::allocatedHome(
2514 (*this)->getParentOfType<DeviceOp>().getTargetModel(),
2515 producer.getTileID(), consumer.getTileID(),
2516 delegate.getTileID(), linkTile.getTileID());
2517 StringRef name = objFifo.name().getValue();
2518 if (shared) {
2519 InFlightDiagnostic diag = emitOpError("places @")
2520 << name << "'s objects in the link's pool";
2521 switch (misfit) {
2523 ObjectFifoCreateOp owner = link.getPoolOwner();
2524 diag << ", whose objects are @" << owner.name().getValue() << "'s "
2525 << owner.getElemType().getElementType()
2526 << "; a fifo that shares memory with a link must have the pool's "
2527 "object type (@"
2528 << name << "'s is " << objFifo.getElemType().getElementType()
2529 << ")";
2530 break;
2531 }
2533 diag << ", which starts with @" << link.getPoolOwner().name().getValue()
2534 << "'s initial objects, "
2535 << "but @" << name << " has initial objects of its own";
2536 break;
2538 diag << ", but @" << name
2539 << " is a merge's input, and a merge's inputs take turns filling "
2540 "the pool through one DMA end";
2541 break;
2543 diag << ", but @" << name
2544 << " is a dispatch's output, and a dispatch's outputs take turns "
2545 "draining the pool through one DMA end";
2546 break;
2548 diag << ", but @" << name
2549 << " is one link's output and another link's input, so its "
2550 "objects would be both links' pools";
2551 break;
2553 break;
2554 }
2555 return diag;
2556 }
2557 }
2558 for (Operation *op = (*this)->getPrevNode(); op; op = op->getPrevNode())
2559 if (auto other = dyn_cast<ObjectFifoAllocateOp>(op);
2560 other && other.getObjFifoName() == getObjFifoName())
2561 return emitOpError("objectfifo ")
2562 << getObjFifoNameAttr() << " already has an allocate operation";
2563 return success();
2564}
2565
2566TileOp ObjectFifoAllocateOp::getDelegateTileOp() {
2567 return cast<TileOp>(getDelegateTile().getDefiningOp());
2568}
2569
2570ObjectFifoCreateOp ObjectFifoAllocateOp::getObjectFifo() {
2571 Operation *parent = getOperation();
2572 while ((parent = parent->getParentOp())) {
2573 if (parent->hasTrait<OpTrait::SymbolTable>()) {
2574 if (auto *st = SymbolTable::lookupSymbolIn(parent, getObjFifoName());
2575 isa_and_nonnull<ObjectFifoCreateOp>(st))
2576 return cast<ObjectFifoCreateOp>(st);
2577 }
2578 }
2579 return {};
2580}
2581
2582//===----------------------------------------------------------------------===//
2583// ObjectFifoLinkOp
2584//===----------------------------------------------------------------------===//
2585
2586LogicalResult ObjectFifoLinkOp::verify() {
2587 for (Attribute name : llvm::concat<const Attribute>(
2588 getFifoIns().getValue(), getFifoOuts().getValue())) {
2589 auto flat = dyn_cast<FlatSymbolRefAttr>(name);
2590 if (!flat || !isa_and_nonnull<ObjectFifoCreateOp>(
2591 SymbolTable::lookupNearestSymbolFrom(*this, flat)))
2592 return emitOpError("names no objectfifo ") << name;
2593 }
2594
2595 if (isJoin() && isDistribute())
2596 return emitError("ObjectFifoLinkOp does not support 'join' and "
2597 "'distribute' at the same time");
2598
2599 if (isTime()) {
2600 if (isMerge() && isDispatch())
2601 return emitOpError("a time link merges or dispatches, not both");
2602 if (!isMerge() && !isDispatch())
2603 return emitOpError("a time link takes turns, so it needs several inputs "
2604 "or several outputs");
2605 if (!getSrcOffsets().empty() || !getDstOffsets().empty())
2606 return emitOpError("a time link moves whole objects and takes no "
2607 "offsets");
2608 std::vector<ObjectFifoCreateOp> fifos = getInputObjectFifos();
2609 std::vector<ObjectFifoCreateOp> outs = getOutputObjectFifos();
2610 fifos.insert(fifos.end(), outs.begin(), outs.end());
2611 for (ObjectFifoCreateOp fifo : fifos) {
2612 if (fifo.getElemType() != fifos.front().getElemType())
2613 return emitOpError("a time link's participants carry one object "
2614 "type, but '")
2615 << fifo.name().getValue() << "' carries " << fifo.getElemType()
2616 << " and '" << fifos.front().name().getValue() << "' carries "
2617 << fifos.front().getElemType();
2618 }
2619 if (isDispatch()) {
2620 for (ObjectFifoCreateOp out : outs)
2621 if (out.getConsumerTiles().size() != 1)
2622 return emitOpError("each turn of a dispatch reaches one consumer, "
2623 "but '")
2624 << out.name().getValue() << "' has "
2625 << out.getConsumerTiles().size();
2626 }
2627 if (isMerge()) {
2628 PacketInfoAttr pinned;
2629 for (ObjectFifoCreateOp in : getInputObjectFifos()) {
2630 PacketInfoAttr header = in.packetHeader();
2631 if (!header || !header.isAssigned())
2632 continue;
2633 if (pinned && pinned != header)
2634 return emitOpError("a merge is one route with one header, but its "
2635 "inputs pin different ones");
2636 pinned = header;
2637 }
2638 }
2639 }
2640
2641 auto sharedTile = getOptionalSharedTile();
2642 if (!sharedTile)
2643 return emitError("ObjectFifoLinkOp must have a link point, i.e., a "
2644 "shared tile between objectFifos");
2645 // getOptionalSharedTile reads only the first of a time link's inputs and
2646 // outputs.
2647 if (isTime()) {
2648 for (ObjectFifoCreateOp out : getOutputObjectFifos())
2649 if (out.getProducerTile() != *sharedTile)
2650 return emitOpError("a time link's participants meet on its tile, but '")
2651 << out.name().getValue() << "' is produced on another";
2652 for (ObjectFifoCreateOp in : getInputObjectFifos())
2653 if (!llvm::is_contained(in.getConsumerTiles(), *sharedTile))
2654 return emitOpError("a time link's participants meet on its tile, but '")
2655 << in.name().getValue() << "' is consumed on others";
2656 }
2657
2658 // A fifo has one producer, so one link at most fills it; each link reading
2659 // it takes one of its consumer ends on the link's tile. A fifo out of one
2660 // link may go into the next, as a chain of mem tile holds does.
2661 DenseMap<Operation *, size_t> reads;
2662 for (auto other :
2663 (*this)->getParentOfType<DeviceOp>().getOps<ObjectFifoLinkOp>()) {
2664 if (other == *this) {
2665 break;
2666 }
2667 for (ObjectFifoCreateOp fifo : getOutputObjectFifos())
2668 if (llvm::is_contained(other.getOutputObjectFifos(), fifo))
2669 return fifo.emitOpError(
2670 "objectfifo cannot be filled by more than one ObjectFifoLinkOp");
2671 for (ObjectFifoCreateOp fifo : getInputObjectFifos())
2672 if (llvm::is_contained(other.getInputObjectFifos(), fifo) &&
2673 other.getOptionalSharedTile() == sharedTile)
2674 reads[fifo]++;
2675 }
2676 for (ObjectFifoCreateOp fifo : getInputObjectFifos())
2677 if (reads.lookup(fifo) >=
2678 static_cast<size_t>(llvm::count(fifo.getConsumerTiles(), *sharedTile)))
2679 return fifo.emitOpError("objectfifo is read by more ObjectFifoLinkOps "
2680 "on a tile than it has consumer ends there");
2681
2682 TileLike tile = llvm::dyn_cast<TileLike>(sharedTile.value().getDefiningOp());
2683 if (!tile)
2684 return emitError("shared tile must implement TileLike interface");
2685 // Each participant occupies its own slice of the shared object, so the link
2686 // point needs a memory module to hold that object and one counting lock pair
2687 // per slice.
2688 if (isJoin() || isDistribute()) {
2689 if (tile.isShimTile())
2690 return emitError("ObjectFifoLinkOp join and distribute are "
2691 "unavailable on shim tiles");
2692 if (!getTargetModel(getOperation())
2694 return emitError("ObjectFifoLinkOp join and distribute require "
2695 "semaphore locks, which this device lacks");
2696 }
2697 if (isTime() && tile.isShimTile())
2698 return emitOpError("a time link holds its objects on the link point, which "
2699 "a shim tile has no memory for");
2700
2701 // A join or distribute slot must hold a whole number of its participant's
2702 // objects, or one straddles two shared objects and the DMA stalls.
2703 auto elementsOf = [](Type objectFifoType) {
2704 return cast<MemRefType>(
2705 cast<AIEObjectFifoType>(objectFifoType).getElementType())
2706 .getNumElements();
2707 };
2708 // The data layout sizes block-float elements too, which have no bit width.
2709 DataLayout dataLayout = DataLayout::closest(getOperation());
2710 auto bytesOf = [&](Type objectFifoType, int64_t elements) {
2711 return elements *
2712 static_cast<int64_t>(dataLayout.getTypeSize(
2713 cast<MemRefType>(
2714 cast<AIEObjectFifoType>(objectFifoType).getElementType())
2715 .getElementType()));
2716 };
2717 auto checkSlot = [&](StringRef role, ObjectFifoCreateOp participant,
2718 int64_t object, ObjectFifoCreateOp shared,
2719 int64_t slotBytes, ArrayAttr offsets) -> LogicalResult {
2720 if (object <= 0)
2721 return emitOpError() << role << " @" << participant.name().getValue()
2722 << " has empty objects";
2723 if (slotBytes % object == 0)
2724 return success();
2725 std::string offsetText;
2726 llvm::raw_string_ostream os(offsetText);
2727 llvm::interleaveComma(offsets, os, [&](Attribute offset) {
2728 os << *getConstantIntValue(offset);
2729 });
2730 return emitOpError() << role << " @" << participant.name().getValue()
2731 << " has " << object << " B objects, but its "
2732 << slotBytes << " B slot in @"
2733 << shared.name().getValue() << " from offsets ["
2734 << offsetText << "] is not a whole number of them";
2735 };
2736
2737 if (isJoin()) {
2738 if (getFifoIns().size() != getSrcOffsets().size())
2739 return emitOpError("number of provided src offsets must be equal "
2740 "to the number of input objectFifos");
2741
2742 if (!getDstOffsets().empty())
2743 return emitOpError("dst offsets should be empty for join");
2744
2745 // Each input's transfer runs to the next input's offset, so the offsets
2746 // must increase; otherwise the lengths come out wrong and the link hangs.
2747 for (auto [idx, len] : llvm::enumerate(getJoinTransferLengths()))
2748 if (len <= 0)
2749 return emitOpError("src offsets must increase: input objectFifo ")
2750 << idx << " would transfer " << len << " elements";
2751
2752 ObjectFifoCreateOp fifoOut = getOutputObjectFifos()[0];
2753 for (auto [fifoIn, slot] :
2754 llvm::zip(getInputObjectFifos(), getJoinTransferLengths())) {
2755 int64_t object =
2756 fifoIn.getConsumerElemType()
2757 ? bytesOf(fifoIn.getConsumerElemTypeOrDefault(),
2758 elementsOf(fifoIn.getConsumerElemTypeOrDefault()))
2759 : bytesOf(fifoIn.getElemType(),
2760 paddedSize(fifoIn.getDimensionsToStream(),
2761 fifoIn.getPadDimensions().value_or(
2762 ArrayRef<BDPadLayoutAttr>{}),
2763 elementsOf(fifoIn.getElemType())));
2764 if (failed(checkSlot("input", fifoIn, object, fifoOut,
2765 bytesOf(fifoOut.getElemType(), slot),
2766 getSrcOffsets())))
2767 return failure();
2768 }
2769 if (!fifoOut.getDimensionsToStream().empty()) {
2770 int64_t maxIdx = getDimsMaxIdx(fifoOut.getDimensionsToStream());
2771 int64_t minInputBufferSize = -1;
2772 for (auto lenIn : getJoinTransferLengths()) {
2773 if (lenIn <= minInputBufferSize || minInputBufferSize < 0)
2774 minInputBufferSize = lenIn;
2775 }
2776 if (minInputBufferSize <= maxIdx) {
2777 return emitOpError()
2778 << "specified output stride(s) and size(s) result in out "
2779 "of bounds access in join input, for index "
2780 << std::to_string(maxIdx) << " in transfer of length "
2781 << std::to_string(minInputBufferSize) << ".";
2782 }
2783 }
2784
2785 } else if (isDistribute()) {
2786 if (getFifoOuts().size() != getDstOffsets().size())
2787 return emitOpError("number of provided dst offsets must be equal "
2788 "to the number of output objectFifos");
2789
2790 if (!getSrcOffsets().empty())
2791 return emitOpError("src offsets should be empty for distribute");
2792
2793 for (auto [idx, len] : llvm::enumerate(getDistributeTransferLengths()))
2794 if (len <= 0)
2795 return emitOpError("dst offsets must increase: output objectFifo ")
2796 << idx << " would transfer " << len << " elements";
2797
2798 ObjectFifoCreateOp fifoIn = getInputObjectFifos()[0];
2799 for (auto [fifoOut, slot] :
2800 llvm::zip(getOutputObjectFifos(), getDistributeTransferLengths())) {
2801 // The memtile sends the slot, padded if the output pads.
2802 if (failed(checkSlot(
2803 "output", fifoOut,
2804 bytesOf(fifoOut.getElemType(), elementsOf(fifoOut.getElemType())),
2805 fifoIn,
2806 bytesOf(fifoIn.getElemType(),
2807 paddedSize(fifoOut.getDimensionsToStream(),
2808 fifoOut.getPadDimensions().value_or(
2809 ArrayRef<BDPadLayoutAttr>{}),
2810 slot)),
2811 getDstOffsets())))
2812 return failure();
2813 }
2814 // The end this link reads: the next on its tile after earlier links'.
2815 size_t end = 0;
2816 for (size_t seen = 0; end < fifoIn.getConsumerTiles().size(); end++)
2817 if (fifoIn.getConsumerTiles()[end] == *sharedTile &&
2818 seen++ == reads.lookup(fifoIn))
2819 break;
2820 BDDimLayoutArrayAttr dims =
2821 fifoIn.getDimensionsFromStreamPerConsumer()[end];
2822 if (!dims.empty()) {
2823 int64_t maxIdx = getDimsMaxIdx(dims);
2824 int64_t minOutputBufferSize = -1;
2825 for (auto lenOut : getDistributeTransferLengths()) {
2826 if (lenOut <= minOutputBufferSize || minOutputBufferSize < 0)
2827 minOutputBufferSize = lenOut;
2828 }
2829 if (minOutputBufferSize <= maxIdx) {
2830 return emitOpError()
2831 << "specified input stride(s) and size(s) result in out "
2832 "of bounds access in distribute output, for index "
2833 << std::to_string(maxIdx) << " in transfer of length "
2834 << std::to_string(minOutputBufferSize) << ".";
2835 }
2836 }
2837 } else {
2838 if (!getSrcOffsets().empty() && !getDstOffsets().empty())
2839 return emitOpError("all offsets should be empty if there is no "
2840 "join or distribute");
2841 // Each pool object leaves whole in one descriptor, so a walk that covers
2842 // less would run its outer dimension on past the walk.
2843 std::vector<ObjectFifoCreateOp> outs = getOutputObjectFifos();
2844 if (!isTime() && !walksStream() && !outs.empty()) {
2845 ArrayRef<BDDimLayoutAttr> dims = outs[0].getDimensionsToStream();
2846 std::optional<ArrayRef<BDPadLayoutAttr>> pads =
2847 outs[0].getPadDimensions();
2848 int64_t walked = 1;
2849 for (BDDimLayoutAttr dim : dims)
2850 walked *= dim.getSize();
2851 int64_t object = elementsOf(getPoolOwner().getElemType());
2852 if (!dims.empty() && (!pads || pads->empty()) && walked != object)
2853 return emitOpError()
2854 << "@" << outs[0].name().getValue() << "'s walk covers "
2855 << walked << " elements, but each transfer sends a whole "
2856 << object
2857 << "-element object; to send part of each input object, walk "
2858 "the stream (an outer dimension stepping whole input "
2859 "objects)";
2860 }
2861 }
2862
2863 if (walksStream()) {
2864 StreamWindow window;
2865 if (failed(getStreamWindow(window)))
2866 return failure();
2867 if (!tile.isMemTile())
2868 return emitOpError("a walk over a stream needs a MemTile link point");
2869 ObjectFifoCreateOp in = getInputObjectFifos()[0];
2870 ObjectFifoCreateOp out = getOutputObjectFifos()[0];
2871 if (in.getRepeatCount() || out.getRepeatCount())
2872 return emitOpError("a walk over a stream does not repeat objects");
2873 if (in.getIterCount() || out.getIterCount())
2874 return emitOpError("a walk over a stream runs without an iter_count");
2875 const AIETargetModel &target = getTargetModel(getOperation());
2876 // Its pool counts the stream's objects on one lock pair, and each output
2877 // is a descriptor of its own on a MemTile channel.
2878 if (window.objects > static_cast<int64_t>(target.getMaxLockValue()))
2879 return emitOpError() << "a walk over a stream of " << window.objects
2880 << " objects needs a lock counting that many, "
2881 "but locks count to "
2882 << target.getMaxLockValue();
2883 int64_t bds = target.getNumBDs(AIETileType::MemTile) / 2;
2884 if (window.outputs > bds)
2885 return emitOpError() << "a walk over a stream lowers to one descriptor "
2886 "per output, "
2887 << window.outputs << ", but a MemTile channel has "
2888 << bds;
2889 }
2890
2891 // Distribute outputs drain one shared pool with a single repeat count.
2892 // Join inputs repeat at independent producer pools.
2893 if (isDistribute()) {
2894 auto fifos = getOutputObjectFifos();
2895 for (size_t i = 1; i < fifos.size(); ++i) {
2896 ObjectFifoCreateOp first = fifos[0], fifo = fifos[i];
2897 int repeat = first.getRepeatCount().value_or(1);
2898 int otherRepeat = fifo.getRepeatCount().value_or(1);
2899 if (repeat != otherRepeat)
2900 return emitOpError()
2901 << "distribute outputs must have equal effective repeat counts: "
2902 "@"
2903 << first.name().getValue() << " has " << repeat
2904 << (first.getRepeatCount() ? "" : " (default)") << ", @"
2905 << fifo.name().getValue() << " has " << otherRepeat
2906 << (fifo.getRepeatCount() ? "" : " (default)");
2907 }
2908 }
2909
2910 return success();
2911}
2912
2913std::optional<Value> ObjectFifoLinkOp::getOptionalSharedTile() {
2914 std::vector<ObjectFifoCreateOp> fifoIn = getInputObjectFifos();
2915 std::vector<ObjectFifoCreateOp> fifoOut = getOutputObjectFifos();
2916 // A name that resolves to no fifo is the link verifier's to report.
2917 if (fifoIn.size() < getFifoIns().size() ||
2918 fifoOut.size() < getFifoOuts().size() || fifoIn.empty() ||
2919 fifoOut.empty())
2920 return {};
2921
2922 // An input read by several links meets each at its consumer end on that
2923 // link's tile, whichever of its ends that is.
2924 if (isJoin()) {
2925 Value tile = fifoOut[0].getProducerTile();
2926 for (auto in : fifoIn)
2927 if (!llvm::is_contained(in.getConsumerTiles(), tile))
2928 return {};
2929 return {tile};
2930 }
2931
2932 if (isDistribute()) {
2933 Value tile = fifoOut[0].getProducerTile();
2934 for (auto out : fifoOut)
2935 if (out.getProducerTile() != tile)
2936 return {};
2937 if (!llvm::is_contained(fifoIn[0].getConsumerTiles(), tile))
2938 return {};
2939 return {tile};
2940 }
2941
2942 for (auto consumerIn : fifoIn[0].getConsumerTiles())
2943 if (consumerIn == fifoOut[0].getProducerTile())
2944 return {fifoOut[0].getProducerTile()};
2945 return {};
2946}
2947
2948bool ObjectFifoLinkOp::walksStream() {
2949 if (isJoin() || isDistribute())
2950 return false;
2951 std::vector<ObjectFifoCreateOp> outs = getOutputObjectFifos();
2952 if (outs.empty())
2953 return false;
2954 ArrayRef<BDDimLayoutAttr> dims = outs[0].getDimensionsToStream();
2955 if (dims.size() < 2 || dims.front().getSize() < 2)
2956 return false;
2957 int64_t inner = 1;
2958 for (BDDimLayoutAttr dim : dims.drop_front())
2959 inner *= dim.getSize();
2960 int64_t objectSize =
2961 cast<AIEObjectFifoType>(getInputObjectFifos()[0].getElemType())
2962 .getElementType()
2963 .getNumElements();
2964 // A walk within one input object (rate conversion) is no window.
2965 return getDimsMaxIdx(dims) >= objectSize &&
2966 inner == cast<AIEObjectFifoType>(outs[0].getElemType())
2967 .getElementType()
2968 .getNumElements();
2969}
2970
2971LogicalResult ObjectFifoLinkOp::getStreamWindow(StreamWindow &window) {
2972 ObjectFifoCreateOp in = getInputObjectFifos()[0];
2973 ObjectFifoCreateOp out = getOutputObjectFifos()[0];
2974 ArrayRef<BDPadLayoutAttr> pads;
2975 if (auto padding = out.getPadDimensions())
2976 pads = *padding;
2977 return inferStreamWindow(
2978 out.getDimensionsToStream(), pads,
2979 cast<AIEObjectFifoType>(in.getElemType())
2980 .getElementType()
2981 .getNumElements(),
2982 [&] { return emitOpError(); }, window);
2983}
2984
2985std::optional<unsigned> ObjectFifoLinkOp::getOwnerConsumerIndex() {
2986 ObjectFifoCreateOp owner = getPoolOwner();
2987 if (!getOptionalSharedTile() ||
2988 llvm::is_contained(getOutputObjectFifos(), owner))
2989 return std::nullopt;
2990 for (unsigned index = 0, e = owner.getConsumerTiles().size(); index < e;
2991 ++index)
2992 if (owner.getConsumerLink(index) == *this)
2993 return index;
2994 return std::nullopt;
2995}
2996
2997int64_t ObjectFifoLinkOp::getPoolDepth() { return getPoolDepth(placedAt); }
2998
2999int64_t ObjectFifoLinkOp::getPoolDepth(TilePositions positions) {
3000 return getPoolPlan().depthAt(getTargetModel(*this), positions);
3001}
3002
3003LinkPoolPlan ObjectFifoLinkOp::getPoolPlan() {
3004 LinkPoolPlan plan{getOptionalSharedTile().value_or(Value()), 0, {}};
3005 StreamWindow window;
3006 if (walksStream()) {
3007 plan.depth = succeeded(getStreamWindow(window)) ? window.objects : 0;
3008 return plan;
3009 }
3010 ObjectFifoCreateOp owner = getPoolOwner();
3011 if (!owner.getAllocate()) {
3012 std::optional<unsigned> index = getOwnerConsumerIndex();
3013 plan.depth = index ? owner.dmaDepth(index) : owner.size();
3014 }
3015 // The owner's allocate puts both its ends in the pool; another
3016 // participant's puts there only its end off the link's tile, the one its
3017 // no-allocate twin keeps in a pool of its own (no core acquires on the
3018 // link's tile).
3019 std::vector<ObjectFifoCreateOp> ins = getInputObjectFifos();
3020 std::vector<ObjectFifoCreateOp> outs = getOutputObjectFifos();
3021 for (ObjectFifoCreateOp fifo : llvm::concat<ObjectFifoCreateOp>(ins, outs)) {
3022 std::optional<ObjectFifoAllocateOp> alloc = fifo.getAllocate();
3023 if (!alloc)
3024 continue;
3025 if (fifo == owner)
3026 plan.depth = std::max<int64_t>(
3027 {plan.depth, fifo.dmaDepth(std::nullopt), fifo.dmaDepth(0)});
3028 else if (plan.linkTile && fifo.getConsumerTiles().size() == 1 &&
3029 fifo.linkPoolMisfit() == LinkPoolMisfit::None)
3030 plan.sharers.push_back(
3031 {fifo.getProducerTile(), fifo.getConsumerTiles()[0],
3032 alloc->getDelegateTile(),
3033 llvm::is_contained(outs, fifo) ? fifo.dmaDepth(0)
3034 : fifo.dmaDepth(std::nullopt)});
3035 }
3036 return plan;
3037}
3038
3040 TilePositions positions) const {
3041 int64_t result = depth;
3042 std::optional<TileID> link =
3043 sharers.empty() ? std::nullopt : positions(linkTile);
3044 for (const LinkPoolSharer &sharer : sharers) {
3045 std::optional<TileID> producer = positions(sharer.producer);
3046 std::optional<TileID> consumer = positions(sharer.consumer);
3047 std::optional<TileID> delegate = positions(sharer.delegate);
3048 if (link && producer && consumer && delegate &&
3049 ObjectFifoCreateOp::allocatedHome(target, *producer, *consumer,
3050 *delegate, link))
3051 result = std::max(result, sharer.depth);
3052 }
3053 return result;
3054}
3055
3056ObjectFifoCreateOp ObjectFifoLinkOp::getPoolOwner() {
3057 std::vector<ObjectFifoCreateOp> ins = getInputObjectFifos();
3058 std::vector<ObjectFifoCreateOp> outs = getOutputObjectFifos();
3059 if (isJoin() || isMerge())
3060 return outs[0];
3061 if (isDistribute() || isDispatch())
3062 return ins[0];
3063 // A window's pool holds the stream it walks, object by object.
3064 if (walksStream())
3065 return ins[0];
3066 auto count = [](ObjectFifoCreateOp fifo) {
3067 return cast<AIEObjectFifoType>(fifo.getElemType())
3068 .getElementType()
3069 .getNumElements();
3070 };
3071 // Padding is applied as the objects leave, so the pool holds what arrives.
3072 if (outs[0].getInitValues() ||
3073 (count(outs[0]) > count(ins[0]) && !outs[0].getPadDimensions()))
3074 return outs[0];
3075 return ins[0];
3076}
3077
3078SmallVector<std::pair<int64_t, int64_t>> ObjectFifoLinkOp::getPoolSegments() {
3079 auto count = [](ObjectFifoCreateOp fifo) {
3080 return cast<AIEObjectFifoType>(fifo.getElemType())
3081 .getElementType()
3082 .getNumElements();
3083 };
3084 int64_t whole = count(getPoolOwner());
3085 if (!isJoin() && !isDistribute())
3086 return {{0, whole}};
3087 ArrayAttr offsets = isJoin() ? getSrcOffsets() : getDstOffsets();
3088 std::vector<ObjectFifoCreateOp> side =
3089 isJoin() ? getInputObjectFifos() : getOutputObjectFifos();
3090 SmallVector<std::pair<int64_t, int64_t>> segments;
3091 for (auto [index, participant] : llvm::enumerate(side)) {
3092 int64_t offset = cast<IntegerAttr>(offsets[index]).getInt();
3093 int64_t next = index + 1 < side.size()
3094 ? cast<IntegerAttr>(offsets[index + 1]).getInt()
3095 : whole;
3096 int64_t size = next - offset;
3097 if (size <= 0)
3098 size = count(participant);
3099 segments.emplace_back(offset, size);
3100 }
3101 return segments;
3102}
3103
3104std::vector<ObjectFifoCreateOp> ObjectFifoLinkOp::getInputObjectFifos() {
3105 std::vector<ObjectFifoCreateOp> inputObjFifos;
3106 Operation *parent = getOperation();
3107 while ((parent = parent->getParentOp())) {
3108 if (parent->hasTrait<OpTrait::SymbolTable>()) {
3109 for (auto sym : getFifoIns()) {
3110 auto name = dyn_cast<FlatSymbolRefAttr>(sym);
3111 if (!name)
3112 continue;
3113 if (auto *st = SymbolTable::lookupSymbolIn(parent, name);
3114 isa_and_nonnull<ObjectFifoCreateOp>(st))
3115 inputObjFifos.push_back(cast<ObjectFifoCreateOp>(st));
3116 }
3117 }
3118 }
3119 return inputObjFifos;
3120}
3121
3122std::vector<ObjectFifoCreateOp> ObjectFifoLinkOp::getOutputObjectFifos() {
3123 std::vector<ObjectFifoCreateOp> outputObjFifos;
3124 Operation *parent = getOperation();
3125 while ((parent = parent->getParentOp())) {
3126 if (parent->hasTrait<OpTrait::SymbolTable>()) {
3127 for (auto sym : getFifoOuts()) {
3128 auto name = dyn_cast<FlatSymbolRefAttr>(sym);
3129 if (!name)
3130 continue;
3131 if (auto *st = mlir::SymbolTable::lookupSymbolIn(parent, name);
3132 isa_and_nonnull<ObjectFifoCreateOp>(st))
3133 outputObjFifos.push_back(cast<ObjectFifoCreateOp>(st));
3134 }
3135 }
3136 }
3137 return outputObjFifos;
3138}
3139
3140std::vector<int> ObjectFifoLinkOp::getJoinTransferLengths() {
3141 std::vector<int> lengths;
3142 if (isJoin()) {
3143 auto fifoOut =
3144 llvm::cast<AIEObjectFifoType>(getOutputObjectFifos()[0].getElemType());
3145 auto elemTypeOut = llvm::cast<MemRefType>(fifoOut.getElementType());
3146 int lenOut = elemTypeOut.getNumElements();
3147 // src_offsets is an I64ArrayAttr, so every element is an IntegerAttr and
3148 // getConstantIntValue always yields a value; assert to keep this fail-fast.
3149 auto srcOffset = [&](size_t idx) -> int {
3150 std::optional<int64_t> v = getConstantIntValue(getSrcOffsets()[idx]);
3151 assert(v && "src_offsets element must be a constant integer");
3152 return static_cast<int>(*v);
3153 };
3154 for (size_t i = 0; i < getFifoIns().size(); i++) {
3155 int len = 0;
3156 int offset = srcOffset(i);
3157 if (i == getFifoIns().size() - 1)
3158 len = lenOut - offset;
3159 else
3160 len = srcOffset(i + 1) - offset;
3161 lengths.push_back(len);
3162 }
3163 }
3164 return lengths;
3165}
3166
3167std::vector<int> ObjectFifoLinkOp::getDistributeTransferLengths() {
3168 std::vector<int> lengths;
3169 if (isDistribute()) {
3170 auto fifoIn =
3171 llvm::cast<AIEObjectFifoType>(getInputObjectFifos()[0].getElemType());
3172 auto elemTypeIn = llvm::cast<MemRefType>(fifoIn.getElementType());
3173 int lenIn = elemTypeIn.getNumElements();
3174 // dst_offsets is an I64ArrayAttr, so every element is an IntegerAttr and
3175 // getConstantIntValue always yields a value; assert to keep this fail-fast.
3176 auto dstOffset = [&](size_t idx) -> int {
3177 std::optional<int64_t> v = getConstantIntValue(getDstOffsets()[idx]);
3178 assert(v && "dst_offsets element must be a constant integer");
3179 return static_cast<int>(*v);
3180 };
3181 for (size_t i = 0; i < getFifoOuts().size(); i++) {
3182 int offset = dstOffset(i);
3183 int len = 0;
3184 if (i == getFifoOuts().size() - 1)
3185 len = lenIn - offset;
3186 else
3187 len = dstOffset(i + 1) - offset;
3188 lengths.push_back(len);
3189 }
3190 }
3191 return lengths;
3192}
3193
3194std::optional<int> ObjectFifoLinkOp::getRepeatCount() {
3195 for (auto fifoOut : getOutputObjectFifos())
3196 if (fifoOut.getRepeatCount().has_value())
3197 return {fifoOut.getRepeatCount()};
3198 return {};
3199}
3200
3201//===----------------------------------------------------------------------===//
3202// ObjectFifoRegisterExternalBuffersOp
3203//===----------------------------------------------------------------------===//
3204
3205TileOp ObjectFifoRegisterExternalBuffersOp::getTileOp() {
3206 return cast<TileOp>(getTile().getDefiningOp());
3207}
3208
3209ObjectFifoCreateOp ObjectFifoRegisterExternalBuffersOp::getObjectFifo() {
3210 Operation *parent = getOperation();
3211 while ((parent = parent->getParentOp())) {
3212 if (parent->hasTrait<OpTrait::SymbolTable>()) {
3213 if (auto *st = SymbolTable::lookupSymbolIn(parent, getObjFifoName());
3214 isa_and_nonnull<ObjectFifoCreateOp>(st))
3215 return cast<ObjectFifoCreateOp>(st);
3216 }
3217 }
3218 return {};
3219}
3220
3221//===----------------------------------------------------------------------===//
3222// ObjectFifoAcquireOp
3223//===----------------------------------------------------------------------===//
3224
3225namespace {
3226
3227/// Symbol an objectFifo access names: the fifo it belongs to, or the core
3228/// endpoint it works through.
3229Operation *lookupAccessTarget(Operation *op, StringRef name) {
3230 for (Operation *parent = op->getParentOp(); parent;
3231 parent = parent->getParentOp()) {
3232 if (parent->hasTrait<OpTrait::SymbolTable>()) {
3233 if (Operation *target = SymbolTable::lookupSymbolIn(parent, name)) {
3234 return target;
3235 }
3236 }
3237 }
3238 return nullptr;
3239}
3240
3241/// Element type an access sees, or a null type if the symbol names neither a
3242/// fifo nor an endpoint.
3243Type accessElementType(Operation *target) {
3244 if (auto fifo = dyn_cast_or_null<ObjectFifoCreateOp>(target)) {
3245 return llvm::cast<AIEObjectFifoType>(fifo.getElemType()).getElementType();
3246 }
3247 if (auto endpoint = dyn_cast_or_null<ObjectFifoCoreEndpointOp>(target)) {
3248 if (endpoint.getPoolOp()) {
3249 return endpoint.getAccessType();
3250 }
3251 }
3252 return {};
3253}
3254
3255/// A core may only reach a fifo end that is placed on its own tile.
3256LogicalResult verifyAccessPlacement(Operation *op, Operation *target,
3257 std::optional<ObjectFifoPort> port) {
3258 auto core = op->getParentOfType<CoreOp>();
3259 Value coreTile = core.getTile();
3260
3261 if (auto endpoint = dyn_cast_or_null<ObjectFifoCoreEndpointOp>(target)) {
3262 if (port) {
3263 return op->emitOpError("port is implied by the endpoint's role");
3264 }
3265 if (endpoint.getTile() != coreTile) {
3266 return core.emitOpError(
3267 "objectFifo endpoint accessed by core running on another tile");
3268 }
3269 return success();
3270 }
3271
3272 auto fifo = dyn_cast_or_null<ObjectFifoCreateOp>(target);
3273 if (!fifo) {
3274 return op->emitError("cannot retrieve associated object FIFO");
3275 }
3276 if (!port) {
3277 return op->emitOpError("port is required when accessing an objectFifo");
3278 }
3279
3280 if (*port == ObjectFifoPort::Produce) {
3281 if (coreTile != fifo.getProducerTile()) {
3282 return core.emitOpError(
3283 "producer port of objectFifo accessed by core running "
3284 "on non-producer tile");
3285 }
3286 } else {
3287 if (!llvm::is_contained(fifo.getConsumerTiles(), coreTile)) {
3288 return core.emitOpError(
3289 "consumer port of objectFifo accessed by core running "
3290 "on non-consumer tile");
3291 }
3292 }
3293 return success();
3294}
3295
3296} // namespace
3297
3298LogicalResult ObjectFifoAcquireOp::verify() {
3299 auto parent = getOperation()->getParentOfType<CoreOp>();
3300 if (parent == nullptr) {
3301 return emitOpError("must be called from inside a CoreOp");
3302 }
3303
3304 Operation *target = lookupAccessTarget(*this, getObjFifoName());
3305 if (failed(verifyAccessPlacement(*this, target, getPort()))) {
3306 return failure();
3307 }
3308
3309 if (getObjects().empty()) {
3310 return emitOpError("must acquire at least one object");
3311 }
3312
3313 Type elem = accessElementType(target);
3314 if (!elem) {
3315 return success();
3316 }
3317 auto fifo = dyn_cast<ObjectFifoCreateOp>(target);
3318 if (fifo) {
3319 // A fifo may state a depth per endpoint, the producer first.
3320 int index = 0;
3321 if (getPort() == ObjectFifoPort::Consume) {
3322 for (auto [position, consumer] :
3323 llvm::enumerate(fifo.getConsumerTiles())) {
3324 if (consumer == parent.getTile()) {
3325 index = position + 1;
3326 break;
3327 }
3328 }
3329 }
3330 if (static_cast<int>(getObjects().size()) > fifo.size(index)) {
3331 return emitOpError("acquires ")
3332 << getObjects().size() << " objects from an objectFifo that holds "
3333 << fifo.size(index);
3334 }
3335 }
3336 // An asymmetric fifo hands the consumer a different element type.
3337 auto consType = fifo ? llvm::dyn_cast<AIEObjectFifoType>(
3338 fifo.getConsumerElemTypeOrDefault())
3339 : nullptr;
3340 if (fifo && !consType) {
3341 return emitOpError("ObjectFifo consumer element type must be an "
3342 "!aie.objectfifo<memref<...>> type");
3343 }
3344
3345 for (Type object : getObjects().getTypes()) {
3346 if (object != elem && (!consType || consType.getElementType() != object)) {
3347 return emitOpError("acquired object type ")
3348 << object << " does not match the objectFifo's " << elem;
3349 }
3350 }
3351
3352 return success();
3353}
3354
3355ObjectFifoCreateOp ObjectFifoAcquireOp::getObjectFifo() {
3356 Operation *parent = getOperation();
3357 while ((parent = parent->getParentOp())) {
3358 if (parent->hasTrait<OpTrait::SymbolTable>()) {
3359 if (auto *st = SymbolTable::lookupSymbolIn(parent, getObjFifoName());
3360 isa_and_nonnull<ObjectFifoCreateOp>(st))
3361 return cast<ObjectFifoCreateOp>(st);
3362 }
3363 }
3364 return {};
3365}
3366
3367//===----------------------------------------------------------------------===//
3368// ObjectFifoReleaseOp
3369//===----------------------------------------------------------------------===//
3370
3371LogicalResult ObjectFifoReleaseOp::verify() {
3372 auto parent = getOperation()->getParentOfType<CoreOp>();
3373 if (parent == nullptr)
3374 return emitOpError("must be called from inside a CoreOp");
3375
3376 Operation *target = lookupAccessTarget(*this, getObjFifoName());
3377 return verifyAccessPlacement(*this, target, getPort());
3378}
3379
3380ObjectFifoCreateOp ObjectFifoReleaseOp::getObjectFifo() {
3381 Operation *parent = getOperation();
3382 while ((parent = parent->getParentOp())) {
3383 if (parent->hasTrait<OpTrait::SymbolTable>()) {
3384 if (auto *st = SymbolTable::lookupSymbolIn(parent, getObjFifoName());
3385 isa_and_nonnull<ObjectFifoCreateOp>(st))
3386 return cast<ObjectFifoCreateOp>(st);
3387 }
3388 }
3389 return {};
3390}
3391
3392//===----------------------------------------------------------------------===//
3393// CascadeFlowOp
3394//===----------------------------------------------------------------------===//
3395
3396LogicalResult CascadeFlowOp::verify() {
3397 TileLike src = getSourceTileLike();
3398 TileLike dst = getDestTileLike();
3399
3400 if (!src || !dst)
3401 return emitOpError("source and dest must be tile-like operations");
3402
3403 if (src.isShimTile() || dst.isShimTile())
3404 return emitOpError("shimTile row has no cascade stream interface");
3405 if (src.isMemTile() || dst.isMemTile())
3406 return emitOpError("memTile row has no cascade stream interface");
3407
3408 const auto &t = getTargetModel(*this);
3409 // Which ways a cascade may run is the target's; an op built outside any
3410 // device is checked for adjacency alone.
3411 bool onTarget = (*this)->getParentOfType<AIETarget>() != nullptr;
3412 if (onTarget && t.getCascadeSteps().empty())
3413 return emitOpError("cascade not supported in ")
3414 << stringifyAIEArch(t.getTargetArch());
3415
3416 std::optional<int> srcCol = src.tryGetCol();
3417 std::optional<int> srcRow = src.tryGetRow();
3418 std::optional<int> dstCol = dst.tryGetCol();
3419 std::optional<int> dstRow = dst.tryGetRow();
3420
3421 if (srcCol && srcRow && dstCol && dstRow) {
3422 if (!t.isSouth(*srcCol, *srcRow, *dstCol, *dstRow) &&
3423 !t.isWest(*srcCol, *srcRow, *dstCol, *dstRow) &&
3424 !t.isNorth(*srcCol, *srcRow, *dstCol, *dstRow) &&
3425 !t.isEast(*srcCol, *srcRow, *dstCol, *dstRow)) {
3426 return emitOpError("tiles must be adjacent");
3427 }
3428 if (onTarget && !t.isLegalCascade(*srcCol, *srcRow, *dstCol, *dstRow))
3429 return emitOpError("source tile must be to the North or West of the "
3430 "destination tile");
3431 }
3432
3433 return success();
3434}
3435
3436TileLike CascadeFlowOp::getSourceTileLike() {
3437 return dyn_cast<TileLike>(getSourceTile().getDefiningOp());
3438}
3439
3440TileLike CascadeFlowOp::getDestTileLike() {
3441 return dyn_cast<TileLike>(getDestTile().getDefiningOp());
3442}
3443
3444TileOp CascadeFlowOp::getSourceTileOp() {
3445 if (auto tileOp = dyn_cast_or_null<TileOp>(getSourceTile().getDefiningOp()))
3446 return tileOp;
3447 llvm::report_fatal_error("Calling getSourceTileOp requires TileOp.");
3448}
3449
3450TileOp CascadeFlowOp::getDestTileOp() {
3451 if (auto tileOp = dyn_cast_or_null<TileOp>(getDestTile().getDefiningOp()))
3452 return tileOp;
3453 llvm::report_fatal_error("Calling getDestTileOp requires TileOp.");
3454}
3455
3456//===----------------------------------------------------------------------===//
3457// ConfigureCascadeOp
3458//===----------------------------------------------------------------------===//
3459
3460// The direction a cascade step (dstCol - srcCol, dstRow - srcRow) leaves the
3461// source tile on; getCascadeSteps() only ever returns axis-aligned unit steps.
3462static CascadeDir cascadeStepOutputDir(std::pair<int, int> step) {
3463 if (step.first == 1)
3464 return CascadeDir::East;
3465 if (step.first == -1)
3466 return CascadeDir::West;
3467 return step.second == 1 ? CascadeDir::North : CascadeDir::South;
3468}
3469
3470static CascadeDir oppositeCascadeDir(CascadeDir dir) {
3471 switch (dir) {
3472 case CascadeDir::North:
3473 return CascadeDir::South;
3474 case CascadeDir::South:
3475 return CascadeDir::North;
3476 case CascadeDir::East:
3477 return CascadeDir::West;
3478 case CascadeDir::West:
3479 return CascadeDir::East;
3480 }
3481 llvm_unreachable("unhandled CascadeDir");
3482}
3483
3484static std::string describeCascadeDirs(ArrayRef<CascadeDir> dirs) {
3485 std::string s;
3486 llvm::raw_string_ostream os(s);
3487 llvm::interleave(
3488 dirs, os, [&](CascadeDir d) { os << stringifyCascadeDir(d); }, " or ");
3489 return s;
3490}
3491
3492LogicalResult ConfigureCascadeOp::verify() {
3493 if (!isa<TileOp>(getTile().getDefiningOp()))
3494 return emitOpError("requires a placed tile (aie.tile), not a logical tile");
3495
3496 const auto &t = getTargetModel(*this);
3497 TileOp tile = cast<TileOp>(getTile().getDefiningOp());
3498 CascadeDir inputDir = getInputDir();
3499 CascadeDir outputDir = getOutputDir();
3500
3501 if (tile.isShimTile())
3502 return emitOpError("shimTile row has no cascade stream interface");
3503 if (tile.isMemTile())
3504 return emitOpError("memTile row has no cascade stream interface");
3505
3506 llvm::ArrayRef<std::pair<int, int>> steps = t.getCascadeSteps();
3507 if (!steps.empty()) {
3508 SmallVector<CascadeDir> legalOutputs, legalInputs;
3509 for (std::pair<int, int> step : steps) {
3510 legalOutputs.push_back(cascadeStepOutputDir(step));
3511 legalInputs.push_back(oppositeCascadeDir(legalOutputs.back()));
3512 }
3513 if (!llvm::is_contained(legalInputs, inputDir))
3514 return emitOpError("input direction of cascade must be ")
3515 << describeCascadeDirs(legalInputs) << " on "
3516 << stringifyAIEArch(t.getTargetArch());
3517 if (!llvm::is_contained(legalOutputs, outputDir))
3518 return emitOpError("output direction of cascade must be ")
3519 << describeCascadeDirs(legalOutputs) << " on "
3520 << stringifyAIEArch(t.getTargetArch());
3521 } else {
3522 return emitOpError("cascade not supported in ")
3523 << stringifyAIEArch(t.getTargetArch());
3524 }
3525 return success();
3526}
3527
3528//===----------------------------------------------------------------------===//
3529// PutCascadeOp
3530//===----------------------------------------------------------------------===//
3531
3532LogicalResult PutCascadeOp::verify() {
3533 const auto &targetModel = getTargetModel(*this);
3534 Type type = getCascadeValue().getType();
3535 DataLayout dataLayout = DataLayout::closest(*this);
3536 auto bits = dataLayout.getTypeSizeInBits(type);
3537 auto archbits = targetModel.getAccumulatorCascadeSize();
3538 if (bits != archbits)
3539 return emitOpError("type must match architecture cascade width (")
3540 << archbits << " bits in "
3541 << stringifyAIEArch(targetModel.getTargetArch()) << ")";
3542 return success();
3543}
3544
3545//===----------------------------------------------------------------------===//
3546// GetCascadeOp
3547//===----------------------------------------------------------------------===//
3548
3549LogicalResult GetCascadeOp::verify() {
3550 const auto &targetModel = getTargetModel(*this);
3551 Type type = getCascadeValue().getType();
3552 DataLayout dataLayout = DataLayout::closest(*this);
3553 auto bits = dataLayout.getTypeSizeInBits(type);
3554 if (isa<AIE1TargetModel>(targetModel)) {
3555 if (bits != 384)
3556 return emitOpError("must be a 384-bit type");
3557 } else if (isa<AIE2TargetModel>(targetModel)) {
3558 if (bits != 512)
3559 return emitOpError("must be a 512-bit type");
3560 } else
3561 return emitOpError("cascade not supported in ")
3562 << stringifyAIEArch(targetModel.getTargetArch());
3563 return success();
3564}
3565
3566//===----------------------------------------------------------------------===//
3567// DeviceOp
3568//===----------------------------------------------------------------------===//
3569
3570namespace {
3571
3572// A flow endpoint reduced to a comparable key: resolved tile coordinates plus
3573// the stream-switch port, as (col, row, bundle, channel). Only used for
3574// equality and ordering, so the raw WireBundle enum value is a fine component.
3575using PortKey = std::tuple<int, int, int, int>;
3576
3577// Returns the key for the port (`tileValue`, `bundle`, `channel`), or nullopt
3578// when the tile's coordinates are not known yet -- an aie.logical_tile that
3579// --aie-place-tiles has not placed. Two such endpoints cannot be compared
3580// until placement runs, at which point the caller's check runs again on the
3581// resulting aie.tile coordinates.
3582std::optional<PortKey> tryGetPortKey(Value tileValue, WireBundle bundle,
3583 int channel) {
3584 auto tile = llvm::dyn_cast_or_null<TileLike>(tileValue.getDefiningOp());
3585 if (!tile)
3586 return std::nullopt;
3587 std::optional<int> col = tile.tryGetCol();
3588 std::optional<int> row = tile.tryGetRow();
3589 if (!col || !row)
3590 return std::nullopt;
3591 return PortKey{*col, *row, static_cast<int>(bundle), channel};
3592}
3593
3594std::string to_string(const PortKey &key) {
3595 auto [col, row, bundle, channel] = key;
3596 return "(" + std::to_string(col) + ", " + std::to_string(row) + ") " +
3597 stringifyWireBundle(static_cast<WireBundle>(bundle)).str() + " : " +
3598 std::to_string(channel);
3599}
3600
3601// Rejects two aie.flow ops that declare the same source and destination port.
3602// A repeated flow is not a harmless redundancy: Pathfinder::addFlow folds a
3603// flow into any existing flow with the same source, so the duplicate lands in
3604// that flow's destination list a second time and the router is asked to
3605// broadcast to the same endpoint twice.
3606LogicalResult verifyNoDuplicateFlows(DeviceOp device) {
3607 std::map<std::pair<PortKey, PortKey>, FlowOp> flowSeen;
3608 for (FlowOp flow : device.getOps<FlowOp>()) {
3609 std::optional<PortKey> src = tryGetPortKey(
3610 flow.getSource(), flow.getSourceBundle(), flow.getSourceChannel());
3611 std::optional<PortKey> dst = tryGetPortKey(
3612 flow.getDest(), flow.getDestBundle(), flow.getDestChannel());
3613 if (!src || !dst)
3614 continue;
3615 auto [it, inserted] = flowSeen.try_emplace({*src, *dst}, flow);
3616 if (!inserted) {
3617 InFlightDiagnostic diag = flow.emitOpError()
3618 << "duplicates an earlier flow; "
3619 << to_string(*src) << " -> " << to_string(*dst)
3620 << " is already declared";
3621 diag.attachNote(it->second.getLoc()) << "the other flow is here";
3622 return failure();
3623 }
3624 }
3625 return success();
3626}
3627
3628// Rejects two aie.packet_flow ops that carry the same ID between the same set
3629// of sources and destinations under the same mask. Declaring the flow twice
3630// makes the routing problem look more congested than it is and gives the two
3631// copies conflicting keep_pkt_header/priority_route settings when their
3632// attributes disagree.
3633LogicalResult verifyNoDuplicatePacketFlows(DeviceOp device) {
3634 using PacketFlowKey =
3635 std::tuple<int, int, std::vector<PortKey>, std::vector<PortKey>>;
3636 std::map<PacketFlowKey, PacketFlowOp> packetFlowSeen;
3637 for (PacketFlowOp packetFlow : device.getOps<PacketFlowOp>()) {
3638 Region &body = packetFlow.getPorts();
3639 if (body.empty())
3640 continue;
3641 // Sources and destinations are unordered within a packet flow, so sort
3642 // both to give permutations of the same flow the same key.
3643 std::vector<PortKey> sources, dests;
3644 bool comparable = true;
3645 for (Operation &op : body.front()) {
3646 std::vector<PortKey> *endpoints = nullptr;
3647 std::optional<PortKey> port;
3648 if (auto source = dyn_cast<PacketSourceOp>(op)) {
3649 endpoints = &sources;
3650 port = tryGetPortKey(source.getTile(), source.getBundle(),
3651 source.getChannel());
3652 } else if (auto dest = dyn_cast<PacketDestOp>(op)) {
3653 endpoints = &dests;
3654 port =
3655 tryGetPortKey(dest.getTile(), dest.getBundle(), dest.getChannel());
3656 } else {
3657 continue;
3658 }
3659 // One unplaced endpoint makes the whole flow incomparable.
3660 if (!port) {
3661 comparable = false;
3662 break;
3663 }
3664 endpoints->push_back(*port);
3665 }
3666 if (!comparable || sources.empty() || dests.empty())
3667 continue;
3668 llvm::sort(sources);
3669 llvm::sort(dests);
3670 int mask = packetFlow.getMask().value_or(
3671 device.getTargetModel().getPacketIdMask());
3672 auto [it, inserted] = packetFlowSeen.try_emplace(
3673 {packetFlow.IDInt(), mask, std::move(sources), std::move(dests)},
3674 packetFlow);
3675 if (!inserted) {
3676 InFlightDiagnostic diag =
3677 packetFlow.emitOpError()
3678 << "duplicates an earlier packet flow; ID " << packetFlow.IDInt()
3679 << " under mask 0x" << llvm::utohexstr(mask)
3680 << " is already declared between the same sources and destinations";
3681 diag.attachNote(it->second.getLoc()) << "the other packet flow is here";
3682 return failure();
3683 }
3684 }
3685 return success();
3686}
3687
3688// Rejects two DMA programs on one tile starting the same channel: each would
3689// queue its own chain on it, and the later never runs behind an endless one.
3690static LogicalResult verifyNoSharedProgramChannels(DeviceOp device) {
3691 std::map<std::tuple<Operation *, int, int, DMAChannelDir, int>,
3692 std::pair<DMAStartOp, Operation *>>
3693 starts;
3694 for (DmaBody program : device.getOps<DmaBody>()) {
3695 Operation *tile = cast<TileElement>(*program).getTile().getDefiningOp();
3696 auto like = dyn_cast_or_null<TileLike>(tile);
3697 std::optional<int> col = like ? like.tryGetCol() : std::nullopt;
3698 std::optional<int> row = like ? like.tryGetRow() : std::nullopt;
3699 Operation *key = col && row ? nullptr : tile;
3700 for (DMAStartOp start : program.getDmaBody().getOps<DMAStartOp>()) {
3701 if (start.getEndpoint())
3702 continue;
3703 auto [it, added] =
3704 starts.try_emplace({key, col.value_or(-1), row.value_or(-1),
3705 start.getChannelDir(), start.getChannelIndex()},
3706 start, program);
3707 if (added || it->second.second == program)
3708 continue;
3709 InFlightDiagnostic diag =
3710 start.emitOpError()
3711 << stringifyDMAChannelDir(start.getChannelDir()) << " channel "
3712 << start.getChannelIndex()
3713 << " is started by another DMA program on this tile too";
3714 diag.attachNote(it->second.first.getLoc()) << "the other start is here";
3715 return failure();
3716 }
3717 }
3718 return success();
3719}
3720
3721// Rejects two aie.flows into one port and a packet flow sharing an endpoint
3722// with an aie.flow: a circuit sends every word from its source, packets
3723// included, and its destination takes words from nothing else.
3724LogicalResult verifyNoSharedCircuitPorts(DeviceOp device) {
3725 std::map<PortKey, FlowOp> circuitSources, circuitDests;
3726 for (FlowOp flow : device.getOps<FlowOp>()) {
3727 std::optional<PortKey> src = tryGetPortKey(
3728 flow.getSource(), flow.getSourceBundle(), flow.getSourceChannel());
3729 std::optional<PortKey> dst = tryGetPortKey(
3730 flow.getDest(), flow.getDestBundle(), flow.getDestChannel());
3731 if (src)
3732 circuitSources.try_emplace(*src, flow);
3733 if (!dst)
3734 continue;
3735 auto [it, inserted] = circuitDests.try_emplace(*dst, flow);
3736 if (inserted)
3737 continue;
3738 InFlightDiagnostic diag =
3739 flow.emitOpError()
3740 << "ends at " << to_string(*dst)
3741 << ", where another circuit flow ends; a port takes one circuit";
3742 diag.attachNote(it->second.getLoc()) << "the other circuit flow is here";
3743 return failure();
3744 }
3745 for (PacketFlowOp packetFlow : device.getOps<PacketFlowOp>()) {
3746 Region &body = packetFlow.getPorts();
3747 if (body.empty())
3748 continue;
3749 for (Operation &op : body.front()) {
3750 std::optional<PortKey> port;
3751 std::map<PortKey, FlowOp> *circuits = nullptr;
3752 StringRef role;
3753 if (auto source = dyn_cast<PacketSourceOp>(op)) {
3754 port = tryGetPortKey(source.getTile(), source.getBundle(),
3755 source.getChannel());
3756 circuits = &circuitSources;
3757 role = "starts";
3758 } else if (auto dest = dyn_cast<PacketDestOp>(op)) {
3759 port =
3760 tryGetPortKey(dest.getTile(), dest.getBundle(), dest.getChannel());
3761 circuits = &circuitDests;
3762 role = "ends";
3763 }
3764 if (!port)
3765 continue;
3766 auto it = circuits->find(*port);
3767 if (it == circuits->end())
3768 continue;
3769 InFlightDiagnostic diag =
3770 packetFlow.emitOpError()
3771 << role << " at " << to_string(*port) << ", where a circuit flow "
3772 << role << "; a port carries either one circuit or packets";
3773 diag.attachNote(it->second.getLoc()) << "the circuit flow is here";
3774 return failure();
3775 }
3776 }
3777 return success();
3778}
3779
3780LogicalResult verifyNoSharedCircuitViaEgresses(DeviceOp device) {
3781 enum class ClaimKind { FlowDest, PacketDest, Connect, MasterSet, Via };
3782 struct Claim {
3783 Operation *op;
3784 ClaimKind kind;
3785 size_t viaIndex = 0;
3786 };
3787 std::map<PortKey, Claim> claimedEgresses;
3788
3789 auto addInterconnectClaims = [&](auto interconnect) {
3790 Value tile = interconnect.getTile();
3791 for (ConnectOp connect :
3792 interconnect.getConnections().template getOps<ConnectOp>()) {
3793 Port dest = connect.destPort();
3794 if (std::optional<PortKey> port =
3795 tryGetPortKey(tile, dest.bundle, dest.channel))
3796 claimedEgresses.try_emplace(*port, Claim{connect, ClaimKind::Connect});
3797 }
3798 for (MasterSetOp masterSet :
3799 interconnect.getConnections().template getOps<MasterSetOp>()) {
3800 Port dest = masterSet.destPort();
3801 if (std::optional<PortKey> port =
3802 tryGetPortKey(tile, dest.bundle, dest.channel))
3803 claimedEgresses.try_emplace(*port,
3804 Claim{masterSet, ClaimKind::MasterSet});
3805 }
3806 };
3807 for (SwitchboxOp switchbox : device.getOps<SwitchboxOp>())
3808 addInterconnectClaims(switchbox);
3809 for (ShimMuxOp shimMux : device.getOps<ShimMuxOp>())
3810 addInterconnectClaims(shimMux);
3811 for (FlowOp flow : device.getOps<FlowOp>()) {
3812 if (std::optional<PortKey> dest = tryGetPortKey(
3813 flow.getDest(), flow.getDestBundle(), flow.getDestChannel()))
3814 claimedEgresses.try_emplace(*dest, Claim{flow, ClaimKind::FlowDest});
3815 }
3816 for (PacketFlowOp packetFlow : device.getOps<PacketFlowOp>()) {
3817 for (PacketDestOp dest : packetFlow.getPorts().getOps<PacketDestOp>()) {
3818 if (std::optional<PortKey> port = tryGetPortKey(
3819 dest.getTile(), dest.getBundle(), dest.getChannel()))
3820 claimedEgresses.try_emplace(*port, Claim{dest, ClaimKind::PacketDest});
3821 }
3822 }
3823
3824 WalkResult result = device.walk([&](FlowOp flow) {
3825 ArrayRef<int32_t> bundles =
3826 flow.getViaEgressBundlesAttr()
3827 ? flow.getViaEgressBundlesAttr().asArrayRef()
3828 : ArrayRef<int32_t>();
3829 ArrayRef<int32_t> channels =
3830 flow.getViaEgressChannelsAttr()
3831 ? flow.getViaEgressChannelsAttr().asArrayRef()
3832 : ArrayRef<int32_t>();
3833 if (bundles.size() != flow.getVias().size() ||
3834 channels.size() != flow.getVias().size()) {
3835 return WalkResult::advance();
3836 }
3837 for (auto [index, via] : llvm::enumerate(flow.getVias())) {
3838 std::optional<PortKey> egress = tryGetPortKey(
3839 via, static_cast<WireBundle>(bundles[index]), channels[index]);
3840 if (!egress) {
3841 continue;
3842 }
3843 Claim viaClaim{flow, ClaimKind::Via, index};
3844 auto [it, inserted] = claimedEgresses.try_emplace(*egress, viaClaim);
3845 if (inserted)
3846 continue;
3847 if (index + 1 == flow.getVias().size() && it->second.op == flow &&
3848 it->second.kind == ClaimKind::FlowDest) {
3849 it->second = viaClaim;
3850 continue;
3851 }
3852 InFlightDiagnostic diag = flow.emitOpError()
3853 << "via " << index << " claims circuit egress "
3854 << to_string(*egress)
3855 << ", which another operation already claims";
3856 Diagnostic &note = diag.attachNote(it->second.op->getLoc());
3857 switch (it->second.kind) {
3858 case ClaimKind::FlowDest:
3859 note << "a circuit flow ends at the same port";
3860 break;
3861 case ClaimKind::PacketDest:
3862 note << "a packet flow ends at the same port";
3863 break;
3864 case ClaimKind::Connect:
3865 note << "a fixed connect drives the same port";
3866 break;
3867 case ClaimKind::MasterSet:
3868 note << "a fixed master set drives the same port";
3869 break;
3870 case ClaimKind::Via:
3871 note << "via " << it->second.viaIndex
3872 << " claims the same circuit egress";
3873 break;
3874 }
3875 return WalkResult::interrupt();
3876 }
3877 return WalkResult::advance();
3878 });
3879 return failure(result.wasInterrupted());
3880}
3881
3882// `mlir::detail::verifySymbolTable` compares names carried by `Symbol` ops.
3883// `aie.buffer`, `aie.external_buffer`, `aie.lock` and `aie.dma` name themselves
3884// with `sym_name` but define an SSA value, so they are not `Symbol` ops and
3885// their names escape that check. `lookupNamedOpIn` resolves them by name, so a
3886// repeated name would bind every reference to whichever op comes first.
3887LogicalResult verifyNoDuplicateNames(Operation *symbolTableOp) {
3888 if (symbolTableOp->getNumRegions() == 0 ||
3889 symbolTableOp->getRegion(0).empty()) {
3890 return success();
3891 }
3892 DenseMap<StringAttr, Operation *> nameSeen;
3893 for (Operation &op : symbolTableOp->getRegion(0).front()) {
3894 auto name = op.getAttrOfType<StringAttr>(SymbolTable::getSymbolAttrName());
3895 if (!name) {
3896 continue;
3897 }
3898 auto [it, inserted] = nameSeen.try_emplace(name, &op);
3899 if (inserted) {
3900 continue;
3901 }
3902 // Leave a pair of `Symbol` ops to `verifySymbolTable`, which runs after the
3903 // body and so lets the body ops report about themselves first.
3904 if (isa<SymbolOpInterface>(op) && isa<SymbolOpInterface>(it->second)) {
3905 continue;
3906 }
3907 InFlightDiagnostic diag = op.emitError() << "redefinition of symbol named '"
3908 << name.getValue() << "'";
3909 diag.attachNote(it->second->getLoc())
3910 << "see existing symbol definition here";
3911 return failure();
3912 }
3913 return success();
3914}
3915
3916} // namespace
3917
3918LogicalResult DeviceOp::verify() {
3919 // Cores and flows are the device's own ops, as the router and the CDO read
3920 // them; a walk would also visit every op of the runtime sequences.
3921 //
3922 // A compute tile has exactly one core in hardware, so at most one aie.core
3923 // may resolve to any given (col, row). Cores whose tile is a logical_tile
3924 // with unspecified coordinates are skipped here — they cannot collide until
3925 // --aie-place-tiles assigns them a position, at which point this same check
3926 // runs again on the resulting aie.tile coordinates.
3927 DenseMap<TileID, CoreOp> coreAtTile;
3928 for (CoreOp core : getOps<CoreOp>()) {
3929 auto tile =
3930 llvm::dyn_cast_or_null<TileLike>(core.getTile().getDefiningOp());
3931 if (!tile)
3932 continue;
3933 std::optional<int> col = tile.tryGetCol();
3934 std::optional<int> row = tile.tryGetRow();
3935 if (!col || !row)
3936 continue;
3937 TileID id{*col, *row};
3938 auto [it, inserted] = coreAtTile.try_emplace(id, core);
3939 if (!inserted) {
3940 InFlightDiagnostic diag =
3941 core.emitOpError()
3942 << "tile (" << *col << ", " << *row
3943 << ") already has a core; each compute tile can host only one core";
3944 diag.attachNote(it->second.getLoc()) << "the other core is here";
3945 return failure();
3946 }
3947 }
3948
3949 // A flow declared twice is always a mistake, and one that is otherwise only
3950 // noticed (if at all) as unexplained routing pressure much later on.
3951 if (failed(verifyNoDuplicateFlows(*this)))
3952 return failure();
3953 if (failed(verifyNoDuplicatePacketFlows(*this)))
3954 return failure();
3955 if (failed(verifyNoSharedCircuitPorts(*this)))
3956 return failure();
3957 if (failed(verifyNoSharedProgramChannels(*this)))
3958 return failure();
3959 if (failed(verifyNoSharedCircuitViaEgresses(*this))) {
3960 return failure();
3961 }
3962
3963 if (failed(verifyNoDuplicateNames(*this))) {
3964 return failure();
3965 }
3966
3967 return success();
3968}
3969
3970LogicalResult DeviceOp::verifyRegions() {
3971 // Region verification follows all child invariants, including the joined
3972 // verification of isolated children. Named uses may safely inspect forward
3973 // definitions here. The collection belongs only to this verification sweep.
3975 WalkResult result = getOperation()->walk<WalkOrder::PreOrder>(
3976 [&](Operation *op) -> WalkResult {
3977 if (op != getOperation() && isa<DeviceOp>(op))
3978 return WalkResult::skip();
3979 if (auto user = dyn_cast<NamedOpUserOpInterface>(op))
3980 if (failed(user.verifyNamedOpUses(names)))
3981 return WalkResult::interrupt();
3982 return WalkResult::advance();
3983 });
3984 return failure(result.wasInterrupted());
3985}
3986
3987const AIETargetModel &DeviceOp::getTargetModel() {
3988 return xilinx::AIE::getTargetModel(getDevice());
3989}
3990
3991xilinx::AIE::DeviceOp DeviceOp::getForSymbolInModule(mlir::ModuleOp module,
3992 llvm::StringRef symbol) {
3993 DeviceOp deviceOp;
3994 if (symbol.empty()) {
3995 // If no device name is given, assume 'main'
3996 symbol = "main";
3997 }
3998 Operation *maybeDeviceOp = mlir::SymbolTable::lookupSymbolIn(module, symbol);
3999 if (!maybeDeviceOp) {
4000 return nullptr;
4001 }
4002 deviceOp = llvm::dyn_cast<DeviceOp>(maybeDeviceOp);
4003 return deviceOp;
4004}
4005
4006xilinx::AIE::DeviceOp
4007DeviceOp::getForSymbolInModuleOrError(mlir::ModuleOp module,
4008 llvm::StringRef symbol) {
4009 DeviceOp deviceOp = getForSymbolInModule(module, symbol);
4010 if (!deviceOp) {
4011 if (!symbol.empty()) {
4012 module.emitError("No such device: ") << symbol;
4013 } else {
4014 module.emitError("No 'main' device in module");
4015 }
4016 }
4017 return deviceOp;
4018}
4019
4020//===----------------------------------------------------------------------===//
4021// TileElement
4022//===----------------------------------------------------------------------===//
4023
4024TileOp TileElement::tryGetTileOp() {
4025 auto element = cast<TileElement>(this->getOperation());
4026 return dyn_cast_or_null<TileOp>(element.getTile().getDefiningOp());
4027}
4028
4029TileOp TileElement::getTileOp() {
4030 if (auto tileOp = tryGetTileOp())
4031 return tileOp;
4032 llvm::report_fatal_error("Calling getTileOp requires TileOp.");
4033}
4034
4035//===----------------------------------------------------------------------===//
4036// LogicalTileOp
4037//===----------------------------------------------------------------------===//
4038
4039LogicalResult LogicalTileOp::verify() {
4040 if (failed(verifyAssignedPacket(
4041 *this, (*this)->getAttrOfType<PacketInfoAttr>("controller_id"),
4042 "controller_id")))
4043 return failure();
4044 const auto &targetModel = getTargetModel(*this);
4045 int columns = targetModel.columns();
4046 int rows = targetModel.rows();
4047
4048 // Only verify col/row bounds if they are specified
4049 if (auto col = getCol()) {
4050 if (*col >= columns)
4051 return emitOpError("column index (")
4052 << *col
4053 << ") must be less than the number of columns in the device ("
4054 << columns << ")";
4055 }
4056 if (auto row = getRow()) {
4057 if (*row >= rows)
4058 return emitOpError("row index (")
4059 << *row << ") must be less than the number of rows in the device ("
4060 << rows << ")";
4061 }
4062
4063 // Check that the specified tile type exists on the target device
4064 AIETileType tileType = getTileType();
4065 bool tileTypeExists = false;
4066 for (int col = 0; col < columns && !tileTypeExists; col++) {
4067 for (int row = 0; row < rows && !tileTypeExists; row++) {
4068 if (targetModel.getTileType(col, row) == tileType)
4069 tileTypeExists = true;
4070 }
4071 }
4072 if (!tileTypeExists) {
4073 return emitOpError("tile type '")
4074 << stringifyAIETileType(tileType)
4075 << "' does not exist on the target device";
4076 }
4077
4078 // Check logical tile type matches coordinates on device
4079 // Only validate when both col and row are specified
4080 if (auto col = tryGetCol()) {
4081 if (auto row = tryGetRow()) {
4082 if (targetModel.getTileType(*col, *row) != tileType) {
4083 return emitOpError("declared logical tile type does not match "
4084 "the tile type at coordinates (")
4085 << *col << ", " << *row << ")";
4086 }
4087 }
4088 }
4089
4090 return success();
4091}
4092
4093TileID LogicalTileOp::getCanonicalTileID() {
4094 const auto &targetModel = getTargetModel(*this);
4095
4096 // If col and row are both specified, use them directly
4097 std::optional<int32_t> col = getCol();
4098 std::optional<int32_t> row = getRow();
4099 if (col.has_value() && row.has_value())
4100 return {*col, *row};
4101
4102 // Otherwise, find a representative tile of the given type
4103 AIETileType tileType = getTileType();
4104 for (int col = 0; col < targetModel.columns(); col++) {
4105 for (int row = 0; row < targetModel.rows(); row++) {
4106 if (targetModel.getTileType(col, row) == tileType) {
4107 return {col, row};
4108 }
4109 }
4110 }
4111 llvm_unreachable("No tile of matching tile type found in AIE device");
4112}
4113
4114size_t LogicalTileOp::getNumSourceConnections(WireBundle bundle) {
4115 const auto &targetModel = getTargetModel(*this);
4116 TileID tile = getCanonicalTileID();
4117
4118 if (bundle == WireBundle::Core || bundle == WireBundle::DMA) {
4119 // Note dest is correct here, since direction is reversed.
4120 if (isShimNOCorPLTile())
4121 return targetModel.getNumDestShimMuxConnections(tile.col, tile.row,
4122 bundle);
4123 return targetModel.getNumDestSwitchboxConnections(tile.col, tile.row,
4124 bundle);
4125 }
4126 return 0;
4127}
4128
4129size_t LogicalTileOp::getNumDestConnections(WireBundle bundle) {
4130 const auto &targetModel = getTargetModel(*this);
4131 TileID tile = getCanonicalTileID();
4132
4133 if (bundle == WireBundle::Core || bundle == WireBundle::DMA) {
4134 // Note source is correct here, since direction is reversed.
4135 if (isShimNOCorPLTile())
4136 return targetModel.getNumSourceShimMuxConnections(tile.col, tile.row,
4137 bundle);
4138 return targetModel.getNumSourceSwitchboxConnections(tile.col, tile.row,
4139 bundle);
4140 }
4141 return 0;
4142}
4143
4144std::optional<int> LogicalTileOp::tryGetCol() {
4145 if (auto col = getCol())
4146 return col;
4147 return std::nullopt;
4148}
4149
4150std::optional<int> LogicalTileOp::tryGetRow() {
4151 if (auto row = getRow())
4152 return row;
4153 return std::nullopt;
4154}
4155
4156//===----------------------------------------------------------------------===//
4157// Custom Printer and Parser for LogicalTileOp
4158//===----------------------------------------------------------------------===//
4159
4160ParseResult LogicalTileOp::parse(OpAsmParser &parser, OperationState &result) {
4161 AIETileType tileType;
4162 if (parser.parseLess())
4163 return failure();
4164
4165 StringRef tileTypeStr;
4166 if (parser.parseKeyword(&tileTypeStr))
4167 return failure();
4168
4169 auto tileTypeOpt = symbolizeAIETileType(tileTypeStr);
4170 if (!tileTypeOpt)
4171 return parser.emitError(parser.getCurrentLocation(),
4172 "unknown logical tile type: ")
4173 << tileTypeStr;
4174 tileType = *tileTypeOpt;
4175
4176 if (parser.parseGreater())
4177 return failure();
4178
4179 if (parser.parseLParen())
4180 return failure();
4181
4182 std::optional<int32_t> col;
4183 if (succeeded(parser.parseOptionalQuestion())) {
4184 // col is unspecified
4185 } else {
4186 int32_t colVal;
4187 if (parser.parseInteger(colVal))
4188 return failure();
4189 col = colVal;
4190 }
4191
4192 if (parser.parseComma())
4193 return failure();
4194
4195 std::optional<int32_t> row;
4196 if (succeeded(parser.parseOptionalQuestion())) {
4197 // row is unspecified
4198 } else {
4199 int32_t rowVal;
4200 if (parser.parseInteger(rowVal))
4201 return failure();
4202 row = rowVal;
4203 }
4204
4205 if (parser.parseRParen())
4206 return failure();
4207
4208 // Parse optional attributes
4209 if (parser.parseOptionalAttrDict(result.attributes))
4210 return failure();
4211
4212 // Add the parsed attributes to the result
4213 result.getOrAddProperties<LogicalTileOp::Properties>().tile_type =
4214 AIETileTypeAttr::get(parser.getContext(), tileType);
4215 if (col)
4216 result.getOrAddProperties<LogicalTileOp::Properties>().col =
4217 parser.getBuilder().getI32IntegerAttr(*col);
4218 if (row)
4219 result.getOrAddProperties<LogicalTileOp::Properties>().row =
4220 parser.getBuilder().getI32IntegerAttr(*row);
4221
4222 // Add result type (index)
4223 result.addTypes(parser.getBuilder().getIndexType());
4224
4225 return success();
4226}
4227
4228void LogicalTileOp::print(OpAsmPrinter &printer) {
4229 printer << "<" << stringifyAIETileType(getTileType()) << ">";
4230
4231 printer << "(";
4232 if (auto col = getCol())
4233 printer << *col;
4234 else
4235 printer << "?";
4236 printer << ", ";
4237 if (auto row = getRow())
4238 printer << *row;
4239 else
4240 printer << "?";
4241 printer << ")";
4242
4243 SmallVector<StringRef, 3> elidedAttrs = {"tile_type", "col", "row"};
4244 printer.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
4245}
4246
4247//===----------------------------------------------------------------------===//
4248// TileOp
4249//===----------------------------------------------------------------------===//
4250
4251LogicalResult TileOp::verify() {
4252 if (failed(verifyAssignedPacket(
4253 *this, (*this)->getAttrOfType<PacketInfoAttr>("controller_id"),
4254 "controller_id")))
4255 return failure();
4256 const auto &targetModel = getTargetModel(*this);
4257 int columns = targetModel.columns();
4258 int rows = targetModel.rows();
4259 if (colIndex() >= columns)
4260 return emitOpError("column index (")
4261 << colIndex()
4262 << ") must be less than the number of columns in the device ("
4263 << columns << ")";
4264 if (rowIndex() >= rows)
4265 return emitOpError("row index (")
4266 << rowIndex()
4267 << ") must be less than the number of rows in the device (" << rows
4268 << ")";
4269
4270 auto users = getResult().getUsers();
4271 bool found = false;
4272 for (auto *user : users) {
4273 if (llvm::isa<SwitchboxOp>(*user)) {
4274 if (found)
4275 return emitOpError("can only have one switchbox");
4276 found = true;
4277 }
4278 }
4279
4280 return success();
4281}
4282
4283size_t TileOp::getNumSourceConnections(WireBundle bundle) {
4284 const auto &targetModel = getTargetModel(*this);
4285 if (bundle == WireBundle::Core || bundle == WireBundle::DMA)
4286 // Note dest is correct here, since direction is reversed.
4287 {
4288 // Note dest is correct here, since direction is reversed.
4289 if (isShimNOCorPLTile())
4290 return targetModel.getNumDestShimMuxConnections(getCol(), getRow(),
4291 bundle);
4292 return targetModel.getNumDestSwitchboxConnections(getCol(), getRow(),
4293 bundle);
4294 }
4295 return 0;
4296}
4297
4298size_t TileOp::getNumDestConnections(WireBundle bundle) {
4299 const auto &targetModel = getTargetModel(*this);
4300 if (bundle == WireBundle::Core || bundle == WireBundle::DMA)
4301 // Note source is correct here, since direction is reversed.
4302 {
4303 // Note source is correct here, since direction is reversed.
4304 if (isShimNOCorPLTile())
4305 return targetModel.getNumSourceShimMuxConnections(getCol(), getRow(),
4306 bundle);
4307 return targetModel.getNumSourceSwitchboxConnections(getCol(), getRow(),
4308 bundle);
4309 }
4310 return 0;
4311}
4312
4313std::optional<int> TileOp::tryGetCol() { return getCol(); }
4314std::optional<int> TileOp::tryGetRow() { return getRow(); }
4315
4316AIETileType TileOp::getTileType() {
4317 const auto &targetModel = getTargetModel(*this);
4318 return targetModel.getTileType(getCol(), getRow());
4319}
4320
4321static bool isLegalTileConnection(TileOp tile,
4322 const AIETargetModel &targetModel,
4323 MasterSetOp masterOp, PacketRulesOp slaveOp) {
4324 auto srcBundle = slaveOp.sourcePort().bundle;
4325 auto srcChan = slaveOp.sourcePort().channel;
4326 auto dstBundle = masterOp.destPort().bundle;
4327 auto dstChan = masterOp.destPort().channel;
4328 return targetModel.isLegalTileConnection(
4329 tile.colIndex(), tile.rowIndex(), srcBundle, srcChan, dstBundle, dstChan);
4330}
4331
4332static bool isLegalTileConnection(TileOp tile,
4333 const AIETargetModel &targetModel,
4334 ConnectOp connectOp) {
4335 auto srcBundle = connectOp.getSourceBundle();
4336 auto srcChan = connectOp.getSourceChannel();
4337 auto dstBundle = connectOp.getDestBundle();
4338 auto dstChan = connectOp.getDestChannel();
4339 return targetModel.isLegalTileConnection(
4340 tile.colIndex(), tile.rowIndex(), srcBundle, srcChan, dstBundle, dstChan);
4341}
4342
4343TileOp TileOp::getOrCreate(mlir::OpBuilder builder, DeviceOp device, int col,
4344 int row, std::optional<mlir::Location> loc) {
4345 TileOp tile = nullptr;
4346 // Find matching predefined tile at device top level, ...
4347 for (auto t : device.getOps<AIE::TileOp>()) {
4348 if (t.getRow() == row && t.getCol() == col) {
4349 tile = t;
4350 break;
4351 }
4352 }
4353 // ... or if undefined, create a new tile op
4354 if (!tile) {
4355 OpBuilder::InsertionGuard guard(builder);
4356 mlir::Block &device_start_block = *device.getBodyRegion().begin();
4357 builder.setInsertionPointToStart(&device_start_block);
4358 tile = TileOp::create(builder, loc.value_or(device.getLoc()),
4359 builder.getIndexType(), col, row);
4360 }
4361 return tile;
4362}
4363
4364//===----------------------------------------------------------------------===//
4365// ShimMuxOp
4366//===----------------------------------------------------------------------===//
4367
4368LogicalResult ShimMuxOp::verify() {
4369 // The port/connection checks below are keyed off the target model for a
4370 // placed tile. Before --aie-place-tiles the tile is still an
4371 // aie.logical_tile, so defer those checks to the post-placement re-verify --
4372 // mirroring SwitchboxOp::verify and the UsesAreAccessible trait.
4373 if (!isa<TileOp>(getTile().getDefiningOp()))
4374 return success();
4375
4376 Region &body = getConnections();
4377 DenseSet<Port> destset;
4378 if (body.empty())
4379 return emitOpError("should have non-empty body");
4380
4381 for (auto &ops : body.front()) {
4382 if (auto connectOp = dyn_cast<ConnectOp>(ops)) {
4383 Port dest = {connectOp.getDestBundle(), connectOp.destIndex()};
4384 if (destset.count(dest))
4385 return connectOp.emitOpError("targets same destination ")
4386 << stringifyWireBundle(dest.bundle) << ": " << dest.channel
4387 << " as another connect operation";
4388 destset.insert(dest);
4389 } else if (isa<EndOp>(ops)) {
4390 // continue;
4391 } else {
4392 return ops.emitOpError("cannot be contained in a Switchbox op");
4393 }
4394 }
4395 return success();
4396}
4397
4398size_t ShimMuxOp::getNumSourceConnections(WireBundle bundle) {
4399 auto tile = getTileOp();
4400 const auto &targetModel = getTargetModel(*this);
4401 return targetModel.getNumSourceShimMuxConnections(tile.getCol(),
4402 tile.getRow(), bundle);
4403}
4404
4405size_t ShimMuxOp::getNumDestConnections(WireBundle bundle) {
4406 auto tile = getTileOp();
4407 const auto &targetModel = getTargetModel(*this);
4408 return targetModel.getNumDestShimMuxConnections(tile.getCol(), tile.getRow(),
4409 bundle);
4410}
4411
4412//===----------------------------------------------------------------------===//
4413// ShimDMAOp
4414//===----------------------------------------------------------------------===//
4415
4416LogicalResult ShimDMAOp::verify() {
4418 .failed())
4419 return failure();
4420 return success();
4421}
4422
4423TileOp ShimDMAOp::getTileOp() {
4424 return cast<TileElement>(this->getOperation()).getTileOp();
4425}
4426
4427LogicalResult PacketRulesOp::verify() {
4428 if (Region &body = getRules(); body.empty())
4429 return emitOpError("should have non-empty body");
4430 return success();
4431}
4432
4433LogicalResult MasterSetOp::verify() {
4434 int arbiter = -1;
4435 for (auto val : getAmsels()) {
4436 auto amsel = dyn_cast_if_present<AMSelOp>(val.getDefiningOp());
4437 if (!amsel)
4438 return emitOpError("amsel operand must be produced by an 'aie.amsel' op");
4439 if (arbiter != -1 && arbiter != amsel.arbiterIndex())
4440 return emitOpError("a master port can only be tied to one arbiter");
4441 arbiter = amsel.arbiterIndex();
4442 }
4443 return success();
4444}
4445
4446bool MasterSetOp::keepsPktHeader() {
4447 return AIE::keepsPktHeader(
4448 (*this)->getParentOfType<SwitchboxOp>().getTileOp().getTileID(),
4449 destPort(), getKeepPktHeader());
4450}
4451
4453 std::optional<bool> keep) {
4454 if (keep)
4455 return *keep;
4456 return master.bundle != WireBundle::DMA &&
4457 (tile.row != 0 || master.bundle != WireBundle::South);
4458}
4459
4460// A Core or DMA port a flow starts or ends at on a placed tile must exist
4461// there. The placer checks logical tiles' channel budgets.
4462static LogicalResult verifyFlowEndpoint(Operation *op, Value tile, Port port,
4463 bool isSource) {
4464 if (port.bundle != WireBundle::Core && port.bundle != WireBundle::DMA)
4465 return success();
4466 auto tileOp = dyn_cast_or_null<TileOp>(tile.getDefiningOp());
4467 if (!tileOp)
4468 return success();
4469 size_t channels = isSource ? tileOp.getNumDestConnections(port.bundle)
4470 : tileOp.getNumSourceConnections(port.bundle);
4471 if (static_cast<size_t>(port.channel) < channels)
4472 return success();
4473 return op->emitOpError() << (isSource ? "source " : "destination ")
4474 << stringifyWireBundle(port.bundle) << ":"
4475 << port.channel << " does not exist: tile ("
4476 << tileOp.getCol() << ", " << tileOp.getRow()
4477 << ") has " << channels << " "
4478 << stringifyWireBundle(port.bundle) << " port"
4479 << (channels == 1 ? "" : "s")
4480 << (isSource ? " into" : " out of")
4481 << " its stream switch";
4482}
4483
4484static LogicalResult verifyFlowVias(Operation *op, OperandRange vias,
4485 DenseI32ArrayAttr ingressBundles,
4486 DenseI32ArrayAttr ingressChannels,
4487 DenseI32ArrayAttr egressBundles,
4488 DenseI32ArrayAttr egressChannels) {
4489 size_t numVias = vias.size();
4490 DenseI32ArrayAttr arrays[] = {ingressBundles, ingressChannels, egressBundles,
4491 egressChannels};
4492 for (DenseI32ArrayAttr array : arrays) {
4493 size_t size = array ? array.size() : 0;
4494 if (size != numVias) {
4495 return op->emitOpError("has ")
4496 << numVias << " via tile(s) but a via port array of size " << size;
4497 }
4498 }
4499 if (numVias == 0) {
4500 return success();
4501 }
4502
4503 for (auto [index, via] : llvm::enumerate(vias)) {
4504 if (!isa_and_nonnull<TileLike>(via.getDefiningOp())) {
4505 return op->emitOpError("has via operand at index ")
4506 << index << " that is not defined by a tile-like operation";
4507 }
4508 }
4509
4510 auto verifyBundles = [&](DenseI32ArrayAttr bundles,
4511 StringRef direction) -> LogicalResult {
4512 for (auto [index, bundle] : llvm::enumerate(bundles.asArrayRef())) {
4513 if (bundle < 0 ||
4514 bundle > static_cast<int32_t>(getMaxEnumValForWireBundle())) {
4515 return op->emitOpError("has invalid via ")
4516 << direction << " bundle " << bundle << " at index " << index;
4517 }
4518 }
4519 return success();
4520 };
4521 auto verifyChannels = [&](DenseI32ArrayAttr channels,
4522 StringRef direction) -> LogicalResult {
4523 for (auto [index, channel] : llvm::enumerate(channels.asArrayRef())) {
4524 if (channel < 0) {
4525 return op->emitOpError("has negative via ")
4526 << direction << " channel " << channel << " at index " << index;
4527 }
4528 }
4529 return success();
4530 };
4531 if (failed(verifyBundles(ingressBundles, "ingress")) ||
4532 failed(verifyChannels(ingressChannels, "ingress")) ||
4533 failed(verifyBundles(egressBundles, "egress")) ||
4534 failed(verifyChannels(egressChannels, "egress"))) {
4535 return failure();
4536 }
4537 return success();
4538}
4539
4540LogicalResult FlowOp::verify() {
4541 if (failed(verifyFlowEndpoint(*this, getSource(),
4542 {getSourceBundle(), sourceIndex()}, true)))
4543 return failure();
4544 if (failed(verifyFlowEndpoint(*this, getDest(),
4545 {getDestBundle(), destIndex()}, false))) {
4546 return failure();
4547 }
4548
4549 return verifyFlowVias(*this, getVias(), getViaIngressBundlesAttr(),
4550 getViaIngressChannelsAttr(), getViaEgressBundlesAttr(),
4551 getViaEgressChannelsAttr());
4552}
4553
4554LogicalResult PacketSourceOp::verify() {
4555 return verifyFlowEndpoint(*this, getTile(), port(), true);
4556}
4557
4558LogicalResult PacketDestOp::verify() {
4559 return verifyFlowEndpoint(*this, getTile(), port(), false);
4560}
4561
4562LogicalResult PacketFlowOp::verify() {
4563 Region &body = getPorts();
4564 if (body.empty())
4565 return emitOpError("should have non-empty body");
4566
4567 int numSources = 0, numDests = 0;
4568 for (auto &ops : body.front()) {
4569 if (!isa<PacketSourceOp, PacketDestOp, EndOp>(ops))
4570 return ops.emitOpError("cannot be contained in a PacketFlow op");
4571 if (isa<PacketSourceOp>(ops))
4572 ++numSources;
4573 if (isa<PacketDestOp>(ops))
4574 ++numDests;
4575 }
4576
4577 if (numSources < 1)
4578 return emitOpError("must have at least one aie.packet_source");
4579 if (numDests < 1)
4580 return emitOpError("must have at least one aie.packet_dest");
4581 if (!getVias().empty() && numSources != 1) {
4582 return emitOpError(
4583 "with via waypoints must have exactly one aie.packet_source");
4584 }
4585 if (!getVias().empty() && numDests != 1) {
4586 return emitOpError(
4587 "with via waypoints must have exactly one aie.packet_dest");
4588 }
4589
4590 uint8_t id = getID();
4591 if (uint32_t maxId = getTargetModel(getOperation()).getMaxPacketId();
4592 id > maxId)
4593 return emitOpError("packet id ")
4594 << static_cast<int>(id) << " exceeds the maximum of " << maxId;
4595
4596 // A slave port accepts a packet when `incoming & mask == ID`, so a bit set
4597 // in ID and clear in the mask rejects every packet.
4598 if (std::optional<uint8_t> mask = getMask()) {
4599 if ((id & *mask) != id) {
4600 return emitOpError("has ID 0x")
4601 << llvm::utohexstr(id) << " outside mask 0x"
4602 << llvm::utohexstr(*mask) << ", which no packet can match";
4603 }
4604 }
4605
4606 return verifyFlowVias(*this, getVias(), getViaIngressBundlesAttr(),
4607 getViaIngressChannelsAttr(), getViaEgressBundlesAttr(),
4608 getViaEgressChannelsAttr());
4609}
4610
4611//===----------------------------------------------------------------------===//
4612// CoreOp
4613//===----------------------------------------------------------------------===//
4614
4615/// The channels a core's own tile has in `direction`.
4616static uint32_t channelsOfOwnTile(TileOp tile, DMAChannelDir direction,
4617 const AIETargetModel &targetModel) {
4618 int col = tile.getCol(), row = tile.getRow();
4619 if (direction == DMAChannelDir::MM2S)
4620 return targetModel.getNumSourceSwitchboxConnections(col, row,
4621 WireBundle::DMA);
4622 return targetModel.getNumDestSwitchboxConnections(col, row, WireBundle::DMA);
4623}
4624
4625LogicalResult DmaPushQueueOp::verify() {
4626 if (getOutOfOrder() && getDirection() != DMAChannelDir::S2MM)
4627 return emitOpError("out_of_order is only supported on S2MM channels");
4628 if (getBdId().has_value() == getBdTag().has_value())
4629 return emitOpError("names the descriptor it pushes either by bd_id or, "
4630 "before aie-assign-bd-ids runs, by bd_tag");
4631 auto core = (*this)->getParentOfType<CoreOp>();
4632 const AIETargetModel &targetModel = getTargetModel(*this);
4633 if (auto tile = core.getTile().getDefiningOp<TileOp>()) {
4634 uint32_t channels = channelsOfOwnTile(tile, getDirection(), targetModel);
4635 if (getChannel() >= static_cast<int32_t>(channels))
4636 return emitOpError("pushes onto ")
4637 << stringifyDMAChannelDir(getDirection()) << " channel "
4638 << getChannel() << ", but tile (" << tile.getCol() << ", "
4639 << tile.getRow() << ") has " << channels;
4640 }
4641 uint32_t maxRepeat = targetModel.getMaxRepeatCount();
4642 if (getRepeatCount() < 0 ||
4643 static_cast<uint32_t>(getRepeatCount()) > maxRepeat)
4644 return emitOpError("repeat_count must be in [0, ") << maxRepeat << "]";
4645 return success();
4646}
4647
4648LogicalResult DmaPollForQueueSpaceOp::verify() {
4649 auto core = (*this)->getParentOfType<CoreOp>();
4650 const AIETargetModel &targetModel = getTargetModel(*this);
4651 if (auto tile = core.getTile().getDefiningOp<TileOp>()) {
4652 uint32_t channels = channelsOfOwnTile(tile, getDirection(), targetModel);
4653 if (getChannel() >= static_cast<int32_t>(channels))
4654 return emitOpError("waits on ")
4655 << stringifyDMAChannelDir(getDirection()) << " channel "
4656 << getChannel() << ", but tile (" << tile.getCol() << ", "
4657 << tile.getRow() << ") has " << channels;
4658 }
4659 if (targetModel.getDmaTaskQueueDepth() == 0)
4660 return emitOpError("waits on a task queue, which this device has none of");
4661 return success();
4662}
4663
4664LogicalResult CoreOp::verify() {
4665 if (getBody().empty())
4666 return emitOpError("should have non-empty body");
4667 if (getElfFile()) {
4668 // If an ELF file is specified, no MLIR body is allowed (to remove
4669 // ambiguity); the ELF file will fully dictate what runs on the
4670 // core and any MLIR would be ignored.
4671 if (!isEmpty()) {
4672 return emitOpError(
4673 "When `elf_file` attribute is specified, core body must be empty "
4674 "(consist of exactly one `aie.end` op).");
4675 }
4676 }
4677 if (getLinkWith() && getLinkFiles())
4678 return emitOpError(
4679 "cannot specify both 'link_with' (deprecated) and 'link_files' "
4680 "on the same core; run aie-assign-core-link-files to migrate");
4681 if (getLinkWith() && getLinkMergeFiles())
4682 return emitOpError(
4683 "cannot specify both 'link_with' (deprecated) and 'link_merge_files' "
4684 "on the same core; run aie-assign-core-link-files to migrate");
4685 // An artifact is either merged into the core's LLVM module or handed to the
4686 // final link, never both: doing both would define its symbols twice.
4687 if (auto linkFiles = getLinkFiles())
4688 if (auto mergeFiles = getLinkMergeFiles()) {
4689 llvm::SmallSet<StringRef, 8> linked;
4690 for (auto f : linkFiles->getAsRange<StringAttr>())
4691 linked.insert(f.getValue());
4692 for (auto f : mergeFiles->getAsRange<StringAttr>())
4693 if (linked.count(f.getValue()))
4694 return emitOpError("artifact '")
4695 << f.getValue()
4696 << "' appears in both 'link_files' and 'link_merge_files'; an "
4697 "artifact must be either merged or linked, not both";
4698 }
4699 // The core's own sections live in a `core_data` aie.buffer, so the buffer
4700 // verifier covers their placement. Size and measured alignment belong here.
4701 if (auto measured = getMeasuredDataSize())
4702 if (auto declared = getDataSize(); declared && *declared < *measured)
4703 return emitOpError("data_size ")
4704 << *declared << " is smaller than the " << *measured
4705 << " bytes this core's linked sections occupy";
4706 // Where the stack sits. A pin the allocator could never honor is a user
4707 // constraint, so it is rejected here rather than at placement.
4708 const auto &targetModel = getTargetModel(*this);
4709 if (auto summary = getLockAcquiresAttr()) {
4710 for (LockAcquireAttr acquire : summary.getAcquires()) {
4711 int32_t column = acquire.getColumn();
4712 int32_t row = acquire.getRow();
4713 if (column >= targetModel.columns() || row >= targetModel.rows())
4714 return emitOpError(
4715 "lock_acquires references a tile outside the device: ")
4716 << "(" << column << ", " << row << ")";
4717 if (static_cast<uint32_t>(acquire.getLockId()) >=
4718 targetModel.getNumLocks(column, row))
4719 return emitOpError("lock_acquires references an invalid lock ID ")
4720 << acquire.getLockId() << " on tile (" << column << ", " << row
4721 << ")";
4722 if (acquire.getAction() == LockAction::AcquireGreaterEqual &&
4723 !targetModel.hasProperty(AIETargetModel::UsesSemaphoreLocks))
4724 return emitOpError(
4725 "lock_acquires uses AcquireGreaterEqual on a target without "
4726 "semaphore locks");
4727 }
4728 }
4729 int64_t localMem = targetModel.getLocalMemorySize();
4730 auto validAlignment = [localMem](int64_t alignment) {
4731 return alignment > 0 && alignment <= localMem &&
4732 llvm::isPowerOf2_64(alignment);
4733 };
4734 if (auto alignment = getMeasuredDataAlignment();
4735 alignment && !validAlignment(*alignment))
4736 return emitOpError("measured_data_alignment must be a power of two between "
4737 "1 and ")
4738 << localMem << " bytes";
4739 if (auto alignments = getMeasuredBankAlignments())
4740 if (!llvm::all_of(*alignments, validAlignment))
4741 return emitOpError("measured_bank_alignments must contain only powers of "
4742 "two between 1 and ")
4743 << localMem << " bytes";
4744 MemoryRun stackRun = getStackRun();
4745 auto tile = dyn_cast_if_present<TileOp>(getTile().getDefiningOp());
4746 int64_t numBanks =
4747 tile ? targetModel.getNumBanks(tile.getCol(), tile.getRow()) : 0;
4748 int64_t bankSize =
4749 numBanks > 0 ? targetModel.getLocalMemorySize() / numBanks : 0;
4750 if (auto bank = getStackBank()) {
4751 if (tile && *bank >= numBanks)
4752 return emitOpError("stack_bank ")
4753 << *bank << " does not exist; this tile has " << numBanks
4754 << " banks";
4755 // Bank-specific stack accesses must stay inside the selected bank.
4756 if (bankSize > 0 && stackRun.size > bankSize)
4757 return emitOpError("stack_bank pins a ")
4758 << stackRun.size << "-byte stack to bank " << *bank
4759 << ", which holds " << bankSize
4760 << " bytes; omit stack_bank and stack_address for legacy "
4761 "placement";
4762 if (getStackAddress() && bankSize > 0 && stackRun.start / bankSize != *bank)
4763 return emitOpError("stack_address 0x")
4764 << llvm::utohexstr(stackRun.start) << " lies in bank "
4765 << stackRun.start / bankSize << ", but stack_bank requests bank "
4766 << *bank;
4767 if (getStackAddress() && bankSize > 0 &&
4768 stackRun.end() > (*bank + 1) * bankSize)
4769 return emitOpError("a ")
4770 << stackRun.size << "-byte stack at 0x"
4771 << llvm::utohexstr(stackRun.start) << " runs past stack_bank "
4772 << *bank << " (ending at 0x"
4773 << llvm::utohexstr((*bank + 1) * bankSize) << ")";
4774 }
4775 // Checked last so they do not pre-empt the diagnostics above on an op with
4776 // more than one defect. Size and placement are separate faults: a stack can
4777 // fit the tile yet be placed so it runs off the end.
4778 if (stackRun.size >= localMem)
4779 return emitOpError(getStackSize() ? "stack_size "
4780 : "measured stack requirement ")
4781 << stackRun.size
4782 << " leaves no local memory for this tile's buffers (" << localMem
4783 << " bytes total)";
4784 if (stackRun.end() > localMem)
4785 return emitOpError("a ") << stackRun.size << "-byte stack at 0x"
4786 << llvm::utohexstr(stackRun.start)
4787 << " runs past this tile's local memory ("
4788 << localMem << " bytes total)";
4789 if (getStackAddress() &&
4790 stackRun.start % targetModel.getCoreStackAlignment() != 0)
4791 return emitOpError("stack_address must be aligned to ")
4792 << targetModel.getCoreStackAlignment()
4793 << " bytes for this target's stack ABI";
4794 return success();
4795}
4796
4797bool CoreOp::isEmpty() {
4798 Region &body = getBody();
4799 // Return iff. core body contains exactly one block with exactly one AIE.EndOp
4800 return (body.hasOneBlock() && body.front().getOperations().size() == 1 &&
4801 llvm::isa<AIE::EndOp>(body.front().front()));
4802}
4803
4804TileOp CoreOp::getTileOp() {
4805 return cast<TileElement>(this->getOperation()).getTileOp();
4806}
4807
4808uint32_t CoreOp::getEffectiveStackSize() {
4809 if (auto declared = getStackSize())
4810 return *declared;
4811 const auto &targetModel = getTargetModel(*this);
4812 uint32_t fallback = targetModel.getDefaultCoreStackSize();
4813 if (auto measured = getMeasuredStackSize())
4814 return std::max<uint32_t>(
4815 fallback,
4816 llvm::alignTo(*measured, targetModel.getCoreStackAlignment()));
4817 return fallback;
4818}
4819
4820MemoryRun CoreOp::getStackRun() {
4821 return {getStackAddress().value_or(0), getEffectiveStackSize()};
4822}
4823
4824int64_t CoreOp::getSectionAlignment(std::optional<int32_t> bank) {
4825 if (!bank)
4826 return getMeasuredDataAlignment().value_or(1);
4827 auto alignments = getMeasuredBankAlignments();
4828 if (alignments && *bank >= 0 &&
4829 static_cast<size_t>(*bank) < alignments->size())
4830 return std::max<int64_t>((*alignments)[*bank], 1);
4831 return 1;
4832}
4833
4834SmallVector<BufferOp> CoreOp::getSectionReservationBuffers(
4835 ArrayRef<CoreSectionReservation> reservations) {
4836 using Kind = CoreSectionReservation::Kind;
4837 SmallVector<BufferOp> onTile;
4838 (*this)->getParentOp()->walk([&](BufferOp buffer) {
4839 if (buffer.getTile() == getTile())
4840 onTile.push_back(buffer);
4841 });
4842 SmallVector<BufferOp> holders;
4843 DenseSet<Operation *> claimed;
4844 for (const CoreSectionReservation &held : reservations) {
4845 auto holds = [&](BufferOp buffer) {
4846 if (claimed.contains(buffer) || buffer.getAllocationSize() != held.bytes)
4847 return false;
4848 if (held.kind == Kind::DataRegion)
4849 return static_cast<bool>(buffer.getCoreData());
4850 return buffer.getBankReserved() &&
4851 buffer.getAligned() == held.aligned() &&
4852 (held.kind == Kind::Bank ? buffer.getMemBank() == held.at
4853 : buffer.getAddress() == held.at);
4854 };
4855 auto *it = llvm::find_if(onTile, holds);
4856 holders.push_back(it == onTile.end() ? BufferOp() : *it);
4857 if (it != onTile.end())
4858 claimed.insert(*it);
4859 }
4860 return holders;
4861}
4862
4863SmallVector<CoreSectionReservation> CoreOp::getSectionReservations() {
4864 using Kind = CoreSectionReservation::Kind;
4865 SmallVector<CoreSectionReservation> held;
4866 // A declared data_size wins over the measurement: it is a reservation the
4867 // design asked for, and may deliberately exceed what this build measures.
4868 int64_t data = getDataSize().value_or(getMeasuredDataSize().value_or(0));
4869 if (data > 0)
4870 held.push_back(
4871 {Kind::DataRegion, data, 0, getSectionAlignment(std::nullopt)});
4872 if (auto sizes = getMeasuredBankSizes())
4873 for (auto [index, size] : llvm::enumerate(*sizes)) {
4874 auto bank = static_cast<int32_t>(index);
4875 if (size > 0)
4876 held.push_back({Kind::Bank, size, bank, getSectionAlignment(bank)});
4877 }
4878 if (auto ranges = getMeasuredDataRanges())
4879 for (size_t i = 0; i + 1 < ranges->size(); i += 2)
4880 if ((*ranges)[i + 1] > 0)
4881 held.push_back({Kind::Prebaked, (*ranges)[i + 1], (*ranges)[i]});
4882 return held;
4883}
4884
4885//===----------------------------------------------------------------------===//
4886// BufferOp
4887//===----------------------------------------------------------------------===//
4888
4889int64_t BufferOp::getSectionAlignment() {
4890 if (!getCoreData() && !getBankReserved())
4891 return 1;
4892 for (Operation *user : getTile().getUsers())
4893 if (auto core = dyn_cast<CoreOp>(user)) {
4894 if (getCoreData())
4895 return core.getSectionAlignment(std::nullopt);
4896 std::optional<int32_t> bank = getMemBank();
4897 return bank ? core.getSectionAlignment(bank) : 1;
4898 }
4899 return 1;
4900}
4901
4902int64_t BufferOp::getAllocationSize() {
4903 return getMemRefSizeInBytes(llvm::cast<MemRefType>(getType()),
4904 DataLayout::closest(getOperation()));
4905}
4906
4907LogicalResult BufferOp::verify() {
4908 if (UsesAreAccessible::verifyTrait(*this).failed()) {
4909 return failure();
4910 }
4911
4912 if (auto init = getInitialValue()) {
4913 auto initType = cast<ShapedType>(init->getType());
4914 MemRefType type = getType();
4915 if (initType.getShape() != type.getShape() ||
4916 initType.getElementType() != type.getElementType())
4917 return emitOpError("initial value of type ")
4918 << initType << " does not cover the buffer's " << type;
4919 }
4920
4921 if (FlatSymbolRefAttr resident = getResidentAttr()) {
4922 auto module = (*this)->getParentOfType<mlir::ModuleOp>();
4923 auto global = dyn_cast_or_null<memref::GlobalOp>(
4924 module ? SymbolTable::lookupSymbolIn(module, resident.getValue())
4925 : nullptr);
4926 if (!global)
4927 return emitOpError("resident names no memref.global ")
4928 << resident << " at module scope";
4929 if (global.getType() != getType())
4930 return emitOpError("resident ")
4931 << resident << "'s type " << global.getType()
4932 << " does not match this buffer's " << getType();
4933 }
4934
4935 // Each marks space held for a different reservation of the core's sections.
4936 if (getCoreData() && getBankReserved()) {
4937 return emitOpError("cannot be both core_data and bank_reserved; a buffer "
4938 "holds one reservation of a core's sections");
4939 }
4940
4941 // A bank pin no placement can satisfy: one its address space excludes, or
4942 // one past the banks of its tile type.
4943 if (std::optional<int32_t> memBank = getMemBank()) {
4944 SmallVector<int, 2> banks = banksFromMemorySpace(getType());
4945 if (!banks.empty() && !llvm::is_contained(banks, *memBank)) {
4946 auto diag = emitOpError("has address space for bank ") << banks.front();
4947 if (banks.size() > 1)
4948 diag << " or " << banks.back();
4949 return diag << " but is pinned to bank " << *memBank
4950 << ". These constraints disagree";
4951 }
4952 if (auto tileLike =
4953 dyn_cast_if_present<TileLike>(getTile().getDefiningOp());
4954 tileLike && (tileLike.isCoreTile() || tileLike.isMemTile())) {
4955 const AIETargetModel &targetModel = getTargetModel(*this);
4956 std::optional<int> col = tileLike.tryGetCol();
4957 std::optional<int> row = tileLike.tryGetRow();
4958 for (int c = 0; !(col && row) && c < targetModel.columns(); ++c)
4959 for (int r = 0; !(col && row) && r < targetModel.rows(); ++r)
4960 if (targetModel.getTileType(c, r) == tileLike.getTileType()) {
4961 col = c;
4962 row = r;
4963 }
4964 if (col && row &&
4965 (*memBank < 0 ||
4966 *memBank >=
4967 static_cast<int32_t>(targetModel.getNumBanks(*col, *row))))
4968 return emitOpError("mem_bank attribute value is out of range");
4969 }
4970 }
4971
4972 // A logical tile stands for a placement the compiler has yet to choose, so
4973 // neither check below has a tile to reason about.
4974 auto tile = dyn_cast_if_present<TileOp>(getTile().getDefiningOp());
4975 if (!tile) {
4976 return success();
4977 }
4978
4979 // A tile has one data region, so the code that looks the buffer up takes the
4980 // first match.
4981 if (getCoreData()) {
4982 for (Operation *user : tile.getResult().getUsers()) {
4983 auto other = dyn_cast<BufferOp>(user);
4984 if (other && other != *this && other.getCoreData()) {
4985 return emitOpError("is a second core_data buffer on tile (")
4986 << tile.getCol() << ", " << tile.getRow()
4987 << "); the core's data region is one extent";
4988 }
4989 }
4990 }
4991
4992 // A pin the allocator cannot honor: the address is outside the bank, so one
4993 // of the two is stale. A zero-sized buffer covers no byte, and one at the top
4994 // of the tile holds the one-past-the-end address, which is in no bank.
4995 std::optional<int32_t> address = getAddress();
4996 std::optional<int32_t> memBank = getMemBank();
4997 if (address && memBank && getAllocationSize() > 0) {
4998 const auto &targetModel = getTargetModel(*this);
4999 int64_t numBanks = targetModel.getNumBanks(tile.getCol(), tile.getRow());
5000 int64_t memSize = tile.isMemTile() ? targetModel.getMemTileSize()
5001 : targetModel.getLocalMemorySize();
5002 if (numBanks > 0) {
5003 int64_t bankSize = memSize / numBanks;
5004 int64_t bankOfAddress = *address / bankSize;
5005 if (bankOfAddress != *memBank) {
5006 return emitOpError("address 0x")
5007 << llvm::utohexstr(*address) << " lies in bank " << bankOfAddress
5008 << ", but mem_bank requests bank " << *memBank;
5009 }
5010 }
5011 }
5012 return success();
5013}
5014
5015// Peano models a bank as an address space (`aiebase_resources.h`: a..d are
5016// 5..8, 9..14 are pairs) and schedules loads on the strength of the qualifier,
5017// while nothing placed the buffer to match. Reading it here makes that
5018// assumption true rather than merely asserted.
5019SmallVector<int, 2> xilinx::AIE::banksFromMemorySpace(Type type) {
5020 auto memref = dyn_cast<MemRefType>(type);
5021 if (!memref) {
5022 return {};
5023 }
5024 auto space = dyn_cast_or_null<IntegerAttr>(memref.getMemorySpace());
5025 if (!space) {
5026 return {};
5027 }
5028 int64_t value = space.getInt();
5029 if (value >= 5 && value <= 8) {
5030 return {static_cast<int>(value - 5)};
5031 }
5032 switch (value) {
5033 case 9:
5034 return {0, 1};
5035 case 10:
5036 return {0, 2};
5037 case 11:
5038 return {0, 3};
5039 case 12:
5040 return {1, 2};
5041 case 13:
5042 return {1, 3};
5043 case 14:
5044 return {2, 3};
5045 default:
5046 return {};
5047 }
5048}
5049
5050// FIXME: make address assignment for buffers explicit and move this function to
5051// an interface
5052int32_t xilinx::AIE::getBufferBaseAddress(Operation *bufOp) {
5053 if (auto buf = dyn_cast<BufferOp>(bufOp)) {
5054 std::optional<int32_t> address = buf.getAddress();
5055 assert(address.has_value() && "buffer must have address assigned");
5056 return *address;
5057 }
5058 if (isa_and_nonnull<ExternalBufferOp>(bufOp))
5059 llvm::report_fatal_error(
5060 "External buffer addresses are assigned at runtime.");
5061 llvm::report_fatal_error("unknown buffer type");
5062}
5063
5064void xilinx::AIE::collectTiles(DeviceOp &device,
5065 DenseMap<TileID, Operation *> &tiles) {
5066 for (auto tile : device.getOps<TileOp>()) {
5067 tiles[tile.getTileID()] = tile;
5068 }
5069}
5070
5072 DeviceOp &device,
5073 DenseMap<Operation *, SmallVector<BufferOp, 4>> &buffers) {
5074 for (BufferOp buffer : device.getOps<BufferOp>()) {
5075 Operation *tileOp = buffer.getTile().getDefiningOp();
5076 buffers[tileOp].push_back(buffer);
5077 }
5078}
5079
5080static void printBufferInitialValue(OpAsmPrinter &p, BufferOp op, Type type,
5081 Attribute initialValue) {
5082 if (op.getInitialValue()) {
5083 p << "= ";
5084 p.printAttributeWithoutType(initialValue);
5085 }
5086}
5087
5088static ParseResult parseBufferInitialValue(OpAsmParser &parser, Type &type,
5089 Attribute &initialValue) {
5090 auto memrefType = llvm::cast<MemRefType>(type);
5091 if (!memrefType.hasStaticShape())
5092 return parser.emitError(parser.getNameLoc())
5093 << "type should be static shaped memref, but got " << type;
5094
5095 if (parser.parseOptionalEqual())
5096 return success();
5097
5098 Type tensorType = mlir::memref::getTensorTypeFromMemRefType(memrefType);
5099 if (parser.parseAttribute(initialValue, tensorType))
5100 return failure();
5101 if (!llvm::isa<ElementsAttr>(initialValue))
5102 return parser.emitError(parser.getNameLoc())
5103 << "initial value should be an elements attribute";
5104 return success();
5105}
5106
5107TileOp BufferOp::getTileOp() {
5108 return cast<TileElement>(this->getOperation()).getTileOp();
5109}
5110
5111//===----------------------------------------------------------------------===//
5112// MemOp
5113//===----------------------------------------------------------------------===//
5114
5115LogicalResult MemOp::verify() {
5116 Region &body = getBody();
5118 .failed())
5119 return failure();
5120
5121 for (auto &bodyOp : body.getOps()) {
5122 if (auto allocOp = dyn_cast<memref::AllocOp>(bodyOp))
5123 if (!allocOp->getAttr("id"))
5124 return allocOp.emitOpError()
5125 << "allocOp in MemOp region should have an id attribute";
5126 }
5127 return success();
5128}
5129
5130TileOp MemOp::getTileOp() {
5131 return cast<TileElement>(this->getOperation()).getTileOp();
5132}
5133
5134//===----------------------------------------------------------------------===//
5135// MemTileDMAOp
5136//===----------------------------------------------------------------------===//
5137
5138LogicalResult MemTileDMAOp::verify() {
5139 assert(getOperation()->getNumRegions() == 1 &&
5140 "MemTileDMAOp has zero region!");
5141
5143 .failed())
5144 return failure();
5145
5146 for (auto &bodyOp : getBody().getOps()) {
5147 if (auto allocOp = dyn_cast<memref::AllocOp>(bodyOp)) {
5148 if (!allocOp->getAttr("id"))
5149 return allocOp.emitOpError()
5150 << "allocOp in MemTileDMAOp region should have an id attribute";
5151 }
5152 if (auto startOp = dyn_cast<DMAStartOp>(bodyOp)) {
5153 // An endpoint's channel is not known yet; allocation keeps one whose
5154 // BDs reach another tile's memory off the local-only channels.
5155 if (!startOp.getEndpoint() && startOp.getChannelIndex() > 3) {
5156 // Channels 4 and 5 in a memtile are restricted to only access local
5157 // buffers and locks.
5158
5159 // TODO: Move this code to the dialect
5160 // Set of blocks found to be reachable within a given region.
5161 llvm::SmallSet<Block *, 16> reachable;
5162 SmallVector<Block *, 16> worklist;
5163 Block *firstBD = startOp.getSuccessor(0);
5164 reachable.insert(firstBD);
5165 worklist.push_back(firstBD);
5166 while (!worklist.empty()) {
5167 Block *block = worklist.pop_back_val();
5168 if (block->empty())
5169 continue;
5170 auto successors = block->getTerminator()->getSuccessors();
5171 for (auto *i : successors) {
5172 if (!reachable.contains(i)) {
5173 reachable.insert(i);
5174 worklist.push_back(i);
5175 }
5176 }
5177 }
5178 for (Block *b : reachable) {
5179 for (DMABDOp bd : b->getOps<DMABDOp>()) {
5180 if (auto bufferOp = bd.getBufferOp();
5181 bufferOp.getTile() != getTile()) {
5182 InFlightDiagnostic err =
5183 bd.emitOpError()
5184 << "is reachable from DMA channel "
5185 << startOp.getChannelIndex()
5186 << " and attempts to access a non-local buffer\n";
5187 err.attachNote(startOp->getLoc()) << "channel";
5188 err.attachNote(bufferOp->getLoc()) << "buffer";
5189 return err;
5190 }
5191 }
5192 for (auto useLock : b->getOps<UseLockOp>()) {
5193 if (auto lockOp = useLock.getLockOp();
5194 lockOp.getTile() != getTile()) {
5195 InFlightDiagnostic err =
5196 useLock.emitOpError()
5197 << "is reachable from DMA channel "
5198 << startOp.getChannelIndex()
5199 << " and attempts to access a non-local lock\n";
5200 err.attachNote(startOp->getLoc()) << "channel";
5201 err.attachNote(lockOp->getLoc()) << "lock";
5202 return err;
5203 }
5204 }
5205 }
5206 }
5207 }
5208 }
5209
5210 return success();
5211}
5212
5213TileOp MemTileDMAOp::getTileOp() {
5214 return cast<TileElement>(this->getOperation()).getTileOp();
5215}
5216
5217//===----------------------------------------------------------------------===//
5218// DMAOp
5219//===----------------------------------------------------------------------===//
5220
5221static bool isBdPacketEnabled(DMABDOp bd) {
5222 if (bd.getPacket().has_value())
5223 return true;
5224 if (Block *blk = bd->getBlock())
5225 return !blk->getOps<DMABDPACKETOp>().empty();
5226 return false;
5227}
5228
5229LogicalResult xilinx::AIE::verifyDMAChannelIndex(Operation *op, TileLike tile,
5230 DMAChannelDir dir,
5231 int64_t channel) {
5232 int64_t channels = dir == DMAChannelDir::MM2S
5233 ? tile.getNumDestConnections(WireBundle::DMA)
5234 : tile.getNumSourceConnections(WireBundle::DMA);
5235 if (channel >= 0 && channel < channels)
5236 return success();
5237 return op->emitOpError() << stringifyDMAChannelDir(dir) << " channel "
5238 << channel << " is out of range: the tile has "
5239 << channels << " " << stringifyDMAChannelDir(dir)
5240 << " channels";
5241}
5242
5243LogicalResult xilinx::AIE::verifyAssignedPacket(Operation *op,
5244 PacketInfoAttr packet,
5245 StringRef what) {
5246 if (packet && !packet.isAssigned())
5247 return op->emitOpError()
5248 << what
5249 << " has no pkt_id; allocation assigns one, so a header at "
5250 "this level must carry it";
5251 return success();
5252}
5253
5254LogicalResult
5255xilinx::AIE::verifyDMABDOutOfOrderId(DMABDOp bd, bool packetEnabledByContext) {
5256 std::optional<int32_t> oooId = bd.getOutOfOrderId();
5257 if (!oooId.has_value())
5258 return success();
5259 const AIETargetModel &targetModel = getTargetModel(bd.getOperation());
5261 return bd.emitOpError("out_of_order_id is not supported on this device");
5262 if (!packetEnabledByContext && !isBdPacketEnabled(bd))
5263 return bd.emitOpError("out_of_order_id requires a packet-enabled BD");
5264 uint32_t maxOooId = targetModel.getMaxOutOfOrderId();
5265 if (*oooId < 0 || static_cast<uint32_t>(*oooId) > maxOooId)
5266 return bd.emitOpError("out_of_order_id must be in [0, ") << maxOooId << "]";
5267 return success();
5268}
5269
5270// BD iteration bounds. Values are true/element (aie-codegen encodes value-1);
5271// size <= 1 disables iteration (stride ignored). The stride is checked in
5272// whole 32-bit words against the tile-specific step field. aiex.npu.writebd
5273// checks the same tile-correct step limit (getDmaBdStepBits) inline in its own
5274// raw-register terms.
5275LogicalResult xilinx::AIE::verifyDMABDIteration(DMABDOp bd,
5276 AIETileType tileType) {
5277 std::optional<BDIterationAttr> iter = bd.getIteration();
5278 if (!iter)
5279 return success();
5280 const AIETargetModel &targetModel = getTargetModel(bd.getOperation());
5282 return bd.emitOpError("BD iteration is not supported on this target");
5283 uint32_t size = iter->getSize(), current = iter->getCurrent();
5284 uint32_t maxSize = 1u << targetModel.getDmaBdIterBits(tileType);
5285 if (size < 1 || size > maxSize)
5286 return bd.emitOpError("BD iteration size must be in [1, ")
5287 << maxSize << "]";
5288 if (size > 1) {
5289 int64_t strideInBytes = static_cast<int64_t>(iter->getStride()) *
5290 bd.getBufferElementTypeWidthInBytes();
5291 if (strideInBytes % 4)
5292 return bd.emitOpError(
5293 "BD iteration stride must be aligned to 32-bit words");
5294 int64_t stepInWords = strideInBytes / 4;
5295 int64_t maxStep = 1LL << targetModel.getDmaBdStepBits(tileType);
5296 if (stepInWords < 1 || stepInWords > maxStep)
5297 return bd.emitOpError() << "BD iteration stride must be in [1, "
5298 << maxStep << "] 32-bit words";
5299 }
5300 if (current >= size)
5301 return bd.emitOpError("BD iteration current must be in [0, size)");
5302 return success();
5303}
5304
5305static LogicalResult verifyDMARepeatCount(Operation *op, int32_t repeatCount) {
5306 uint32_t maxRepeat = getTargetModel(op).getMaxRepeatCount();
5307 if (maxRepeat == 0) {
5308 if (repeatCount != 0)
5309 return op->emitOpError("repeat_count is not supported on this target");
5310 return success();
5311 }
5312 if (repeatCount < 0 || static_cast<uint32_t>(repeatCount) > maxRepeat)
5313 return op->emitOpError("repeat_count ")
5314 << repeatCount << " is out of range [0, " << maxRepeat
5315 << "] for this target";
5316 return success();
5317}
5318
5319LogicalResult
5320xilinx::AIE::verifyOutOfOrderChannel(Operation *op, DMAChannelDir dir,
5321 bool outOfOrder, ArrayRef<DMABDOp> bds,
5322 bool packetEnabledByContext) {
5323 if (!outOfOrder)
5324 return success();
5325 if (dir != DMAChannelDir::S2MM)
5326 return op->emitOpError("out_of_order is only valid on an S2MM channel");
5328 return op->emitOpError(
5329 "out-of-order S2MM DMA is not supported on this device");
5330 if (bds.empty())
5331 return op->emitOpError("out-of-order S2MM channel must have at least one "
5332 "receive buffer descriptor"); // else stall
5333 for (DMABDOp bd : bds) {
5334 if (!packetEnabledByContext && !isBdPacketEnabled(bd))
5335 return bd.emitOpError(
5336 "out-of-order S2MM receive buffer descriptor must be packet-enabled");
5337 if (bd.getOutOfOrderId().has_value())
5338 return bd.emitOpError(
5339 "out_of_order_id belongs on the sender buffer descriptor, not an "
5340 "out-of-order S2MM receive buffer descriptor");
5341 }
5342 // an inter-BD lock dependency can deadlock because arrival order is unknown
5343 DenseMap<Operation *, Operation *> releasedByRecvBd;
5344 for (DMABDOp bd : bds)
5345 for (auto useLock : bd->getBlock()->getOps<UseLockOp>())
5346 if (useLock.release())
5347 if (Operation *lockDef = useLock.getLock().getDefiningOp())
5348 releasedByRecvBd.try_emplace(lockDef, bd.getOperation());
5349 for (DMABDOp bd : bds)
5350 for (auto useLock : bd->getBlock()->getOps<UseLockOp>())
5351 if (useLock.acquire() || useLock.acquireGE())
5352 if (Operation *lockDef = useLock.getLock().getDefiningOp()) {
5353 auto it = releasedByRecvBd.find(lockDef);
5354 if (it != releasedByRecvBd.end() && it->second != bd.getOperation())
5355 return bd.emitOpError(
5356 "out-of-order S2MM prohibits inter-BD lock dependencies; "
5357 "can deadlock");
5358 }
5359 return success();
5360}
5361
5362LogicalResult xilinx::AIE::verifyBdLockPair(Block &block, bool outOfOrder,
5363 UseLockOp &acquire,
5364 UseLockOp &release) {
5365 acquire = nullptr;
5366 release = nullptr;
5367 auto reject = [&](UseLockOp at) {
5368 auto diag = at.emitOpError(
5369 "does not fit its buffer descriptor, which has one lock-acquire field "
5370 "and one lock-release field; a buffer descriptor block uses either no "
5371 "lock or one use_lock(acquire) and one use_lock(release)");
5372 if (outOfOrder)
5373 diag << ", or a use_lock(release) alone when out-of-order";
5374 return diag;
5375 };
5376 for (UseLockOp op : block.getOps<UseLockOp>()) {
5377 UseLockOp &slot = op.release() ? release : acquire;
5378 if (slot)
5379 return reject(op);
5380 slot = op;
5381 }
5382 if (!acquire && !release)
5383 return success();
5384 if (release && (acquire || outOfOrder))
5385 return success();
5386 return reject(acquire ? acquire : release);
5387}
5388
5389LogicalResult DMAOp::verify() {
5390 auto *parentOp = getOperation()->getParentOp();
5391 if (parentOp->getRegion(0).getBlocks().size() > 1)
5392 return emitOpError("DMAOp can only appear in single block region");
5393 if (!parentOp->getRegion(0).getOps<DMAStartOp>().empty())
5394 return emitOpError("DMAOp is not compatible with DMAStart ops");
5395 auto bdRegions = getBds();
5396 for (auto &bdRegion : bdRegions) {
5397 if (!bdRegion.hasOneBlock())
5398 return emitOpError("DMAOp regions must have only one block");
5399 auto bds = llvm::to_vector_of<DMABDOp>(bdRegion.front().getOps<DMABDOp>());
5400 if (bds.size() != 1)
5401 return emitOpError("DMAOp regions/blocks must have exactly one DMABDOp");
5402 auto useLocks =
5403 llvm::to_vector_of<UseLockOp>(bdRegion.front().getOps<UseLockOp>());
5404 if (useLocks.size() != 2)
5405 return emitOpError(
5406 "DMAOp regions/blocks must have exactly two UseLock ops");
5407 }
5408 if (getPadValue() != 0) {
5409 if (!isa<MemTileDMAOp>(parentOp))
5410 return emitOpError("pad_value is only supported on memtile DMA channels");
5411 if (getChannelDir() != DMAChannelDir::MM2S)
5412 return emitOpError("pad_value is only supported on MM2S DMA channels");
5413 if (!getTargetModel(getOperation()).isMemTilePadValueSupported())
5414 return emitOpError("pad_value requires the CONSTANT_PAD_VALUE register, "
5415 "unavailable on this target");
5416 }
5417 if (failed(verifyDMARepeatCount(getOperation(), getRepeatCount())))
5418 return failure();
5419 SmallVector<DMABDOp> bds;
5420 for (auto &bdRegion : getBds())
5421 llvm::append_range(bds, bdRegion.front().getOps<DMABDOp>());
5422 return verifyOutOfOrderChannel(getOperation(), getChannelDir(),
5423 getOutOfOrder(), bds);
5424}
5425
5426//===----------------------------------------------------------------------===//
5427// DMABDOp
5428//===----------------------------------------------------------------------===//
5429
5430BufferOp DMABDOp::getBufferOp() {
5431 return cast<BufferOp>(getBuffer().getDefiningOp());
5432}
5433
5434// The optional `: type` after an SSA operand, defaulting to `defaultType`, so
5435// the common width keeps its untyped spelling.
5436static ParseResult parseOptionalOperandType(OpAsmParser &parser, Type &type,
5437 Type defaultType) {
5438 type = defaultType;
5439 if (succeeded(parser.parseOptionalColon()))
5440 return parser.parseType(type);
5441 return success();
5442}
5443
5444static void printOptionallyTypedOperand(OpAsmPrinter &printer, Value operand,
5445 Type defaultType) {
5446 printer << operand;
5447 if (operand.getType() != defaultType)
5448 printer << " : " << operand.getType();
5449}
5450
5451// Parse/print hooks for the custom<DynamicScalar>($operand, $static_attr,
5452// type($operand)) directive: a single scalar that is either an SSA value
5453// (runtime, `%v` for i32 or `%v : type`) or a compile-time integer constant
5454// (folded into the attribute), so a constant never materializes an operand.
5455// The scalar analog of custom<TypedDynamicIndexList>.
5456static ParseResult
5457parseDynamicScalar(OpAsmParser &parser,
5458 std::optional<OpAsmParser::UnresolvedOperand> &operand,
5459 IntegerAttr &staticAttr, Type &type) {
5460 int64_t intValue;
5461 OptionalParseResult intResult = parser.parseOptionalInteger(intValue);
5462 if (intResult.has_value()) {
5463 if (failed(*intResult))
5464 return failure();
5465 staticAttr =
5466 parser.getBuilder().getI32IntegerAttr(static_cast<int32_t>(intValue));
5467 return success();
5468 }
5469 OpAsmParser::UnresolvedOperand op;
5470 if (parser.parseOperand(op) ||
5471 parseOptionalOperandType(parser, type, parser.getBuilder().getI32Type()))
5472 return failure();
5473 operand = op;
5474 return success();
5475}
5476
5477static void printDynamicScalar(OpAsmPrinter &printer, Operation *,
5478 Value operand, IntegerAttr staticAttr, Type) {
5479 if (operand)
5480 printOptionallyTypedOperand(printer, operand,
5481 IntegerType::get(operand.getContext(), 32));
5482 else
5483 printer << staticAttr.getInt();
5484}
5485
5487 OpAsmParser &parser,
5488 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &values,
5489 DenseI64ArrayAttr &integers, SmallVectorImpl<Type> &types) {
5490 SmallVector<int64_t> ints;
5491 auto parseEntry = [&]() -> ParseResult {
5492 OpAsmParser::UnresolvedOperand operand;
5493 OptionalParseResult isOperand = parser.parseOptionalOperand(operand);
5494 if (!isOperand.has_value())
5495 return parser.parseInteger(ints.emplace_back());
5496 Type type;
5497 if (failed(*isOperand) ||
5498 parseOptionalOperandType(parser, type,
5499 parser.getBuilder().getI64Type()))
5500 return failure();
5501 values.push_back(operand);
5502 types.push_back(type);
5503 ints.push_back(ShapedType::kDynamic);
5504 return success();
5505 };
5506 if (parser.parseCommaSeparatedList(AsmParser::Delimiter::Square, parseEntry))
5507 return failure();
5508 integers = parser.getBuilder().getDenseI64ArrayAttr(ints);
5509 return success();
5510}
5511
5512void xilinx::AIE::printTypedDynamicIndexList(OpAsmPrinter &printer, Operation *,
5513 OperandRange values,
5514 ArrayRef<int64_t> integers,
5515 TypeRange) {
5516 printer << '[';
5517 auto value = values.begin();
5518 llvm::interleaveComma(integers, printer, [&](int64_t i) {
5519 if (!ShapedType::isDynamic(i)) {
5520 printer << i;
5521 return;
5522 }
5523 Value v = *value++;
5524 printOptionallyTypedOperand(printer, v,
5525 IntegerType::get(v.getContext(), 64));
5526 });
5527 printer << ']';
5528}
5529
5530// Split a scalar OpFoldResult into an operand (runtime value) or an i32
5531// attribute (compile-time constant). A null OpFoldResult leaves both unset.
5532static void splitScalarOfr(mlir::OpBuilder &builder, mlir::OpFoldResult ofr,
5533 mlir::Value &operand, mlir::IntegerAttr &attr) {
5534 operand = nullptr;
5535 attr = nullptr;
5536 if (!ofr)
5537 return;
5538 if (auto v = llvm::dyn_cast_if_present<mlir::Value>(ofr)) {
5539 operand = v;
5540 return;
5541 }
5542 auto intAttr =
5543 llvm::cast<mlir::IntegerAttr>(llvm::cast<mlir::Attribute>(ofr));
5544 attr = builder.getI32IntegerAttr(static_cast<int32_t>(intAttr.getInt()));
5545}
5546
5547void DMABDOp::buildMixed(mlir::OpBuilder &builder, mlir::OperationState &state,
5548 mlir::Value buffer, mlir::OpFoldResult offset,
5549 mlir::OpFoldResult len,
5550 llvm::ArrayRef<mlir::OpFoldResult> sizes,
5551 llvm::ArrayRef<mlir::OpFoldResult> strides,
5552 BDPadLayoutArrayAttr padDims, PacketInfoAttr packet) {
5553 // Split each mixed list into its dynamic operands + static array (with the
5554 // ShapedType::kDynamic sentinel for the runtime entries).
5555 llvm::SmallVector<int64_t> staticSizes, staticStrides;
5556 llvm::SmallVector<mlir::Value> dynSizes, dynStrides;
5557 mlir::dispatchIndexOpFoldResults(sizes, dynSizes, staticSizes);
5558 mlir::dispatchIndexOpFoldResults(strides, dynStrides, staticStrides);
5559
5560 // Leave the static arrays unset when there is no ND layout so they elide
5561 // from the printed form.
5562 DenseI64ArrayAttr staticSizesAttr =
5563 staticSizes.empty() ? DenseI64ArrayAttr{}
5564 : builder.getDenseI64ArrayAttr(staticSizes);
5565 DenseI64ArrayAttr staticStridesAttr =
5566 staticStrides.empty() ? DenseI64ArrayAttr{}
5567 : builder.getDenseI64ArrayAttr(staticStrides);
5568
5569 // A constant offset/len lands in the static_* attribute; a runtime value
5570 // becomes the operand (same operand-vs-attr split as sizes/strides).
5571 mlir::Value offsetVal, lenVal;
5572 mlir::IntegerAttr offsetAttr, lenAttr;
5573 splitScalarOfr(builder, offset, offsetVal, offsetAttr);
5574 splitScalarOfr(builder, len, lenVal, lenAttr);
5575
5576 build(builder, state, buffer, /*offset=*/offsetVal, /*len=*/lenVal,
5577 /*static_offset=*/offsetAttr, /*static_len=*/lenAttr,
5578 /*sizes=*/dynSizes, /*strides=*/dynStrides,
5579 /*static_sizes=*/staticSizesAttr,
5580 /*static_strides=*/staticStridesAttr,
5581 /*pad_dimensions=*/padDims,
5582 /*bd_id_val=*/nullptr,
5583 /*bd_id=*/nullptr,
5584 /*packet=*/packet,
5585 /*out_of_order_id=*/nullptr,
5586 /*bd_tag=*/nullptr,
5587 /*out_of_order_tag=*/nullptr,
5588 /*burst_length=*/nullptr,
5589 /*axcache=*/nullptr,
5590 /*iteration=*/nullptr,
5591 /*offset_parameter=*/nullptr,
5592 /*offset_state_table_idx=*/nullptr,
5593 /*length_parameter=*/nullptr,
5594 /*length_unit=*/nullptr,
5595 /*length_state_table_idx=*/nullptr,
5596 /*length_core_encoded=*/nullptr,
5597 /*next_bd_id=*/nullptr);
5598}
5599
5600void DMABDOp::buildWithConstants(mlir::OpBuilder &builder,
5601 mlir::OperationState &state,
5602 mlir::Value buffer, int32_t offset,
5603 int32_t len, BDDimLayoutArrayAttr dims,
5604 BDPadLayoutArrayAttr padDims,
5605 PacketInfoAttr packet) {
5606 // Constant offset/len flow to the static_offset/static_len attributes (no
5607 // arith.constant materialized). i32 matches AIEI32Attr, keeping the cast
5608 // sign-exact for negative offsets.
5609 mlir::OpFoldResult offsetOfr = builder.getI32IntegerAttr(offset);
5610 mlir::OpFoldResult lenOfr = builder.getI32IntegerAttr(len);
5611
5612 // Turn the outermost-first BDDimLayoutArrayAttr into all-constant
5613 // OpFoldResults for buildMixed.
5614 llvm::SmallVector<mlir::OpFoldResult> sizes, strides;
5615 if (dims) {
5616 for (BDDimLayoutAttr d : dims) {
5617 sizes.push_back(builder.getI64IntegerAttr(d.getSize()));
5618 strides.push_back(builder.getI64IntegerAttr(d.getStride()));
5619 }
5620 }
5621 buildMixed(builder, state, buffer, offsetOfr, lenOfr, sizes, strides, padDims,
5622 packet);
5623}
5624
5625llvm::SmallVector<mlir::OpFoldResult> DMABDOp::getMixedSizes() {
5626 return ::mlir::getMixedValues(
5627 getStaticSizes().value_or(llvm::ArrayRef<int64_t>{}), getSizes(),
5628 getContext());
5629}
5630
5631llvm::SmallVector<mlir::OpFoldResult> DMABDOp::getMixedStrides() {
5632 return ::mlir::getMixedValues(
5633 getStaticStrides().value_or(llvm::ArrayRef<int64_t>{}), getStrides(),
5634 getContext());
5635}
5636
5637LogicalResult DMABDOp::verifyMixedSizesAndStrides() {
5638 llvm::ArrayRef<int64_t> staticSizes =
5639 getStaticSizes().value_or(llvm::ArrayRef<int64_t>{});
5640 llvm::ArrayRef<int64_t> staticStrides =
5641 getStaticStrides().value_or(llvm::ArrayRef<int64_t>{});
5642 if (failed(mlir::verifyListOfOperandsOrIntegers(
5643 *this, "sizes", staticSizes.size(), staticSizes, getSizes())))
5644 return failure();
5645 if (failed(mlir::verifyListOfOperandsOrIntegers(
5646 *this, "strides", staticStrides.size(), staticStrides, getStrides())))
5647 return failure();
5648 if (staticSizes.size() != staticStrides.size())
5649 return emitOpError("expected the same number of sizes (")
5650 << staticSizes.size() << ") and strides (" << staticStrides.size()
5651 << ")";
5652 return success();
5653}
5654
5655std::optional<llvm::SmallVector<BDDimLayoutAttr>> DMABDOp::getFoldedDimensions(
5656 llvm::function_ref<mlir::InFlightDiagnostic()> emitError) {
5657 llvm::ArrayRef<int64_t> sizes =
5658 getStaticSizes().value_or(llvm::ArrayRef<int64_t>{});
5659 llvm::ArrayRef<int64_t> strides =
5660 getStaticStrides().value_or(llvm::ArrayRef<int64_t>{});
5661 if (sizes.size() != strides.size()) {
5662 emitError() << "expected the same number of sizes (" << sizes.size()
5663 << ") and strides (" << strides.size() << ")";
5664 return std::nullopt;
5665 }
5666 mlir::OperandRange dynSizes = getSizes(), dynStrides = getStrides();
5667 unsigned nextSize = 0, nextStride = 0;
5668 llvm::SmallVector<BDDimLayoutAttr> dims;
5669 dims.reserve(sizes.size());
5670 for (auto [s, t] : llvm::zip(sizes, strides)) {
5671 std::optional<int64_t> sc =
5672 ShapedType::isDynamic(s)
5673 ? mlir::getConstantIntValue(dynSizes[nextSize++])
5674 : s;
5675 std::optional<int64_t> tc =
5676 ShapedType::isDynamic(t)
5677 ? mlir::getConstantIntValue(dynStrides[nextStride++])
5678 : t;
5679 if (!sc || !tc) {
5680 emitError() << "buffer descriptor size/stride is a runtime value; a "
5681 "compile-time constant is required on this path";
5682 return std::nullopt;
5683 }
5684 dims.push_back(BDDimLayoutAttr::get(
5685 getContext(), static_cast<uint32_t>(*sc), static_cast<uint32_t>(*tc)));
5686 }
5687 return dims;
5688}
5689
5690std::optional<llvm::SmallVector<BDDimLayoutAttr>>
5691DMABDOp::getConstantDimensions() {
5692 return getFoldedDimensions([&]() { return emitOpError(); });
5693}
5694
5695// With dims the furthest index bounds the access (a stride-0 dim may transfer
5696// more elements than the buffer holds); without them the length does. A
5697// runtime-sequence argument's type does not bound the host buffer behind it.
5698LogicalResult DMABDOp::verifyTransferInBounds() {
5699 auto buffer = llvm::dyn_cast<MemRefType>(getBuffer().getType());
5700 if (!buffer || !buffer.hasStaticShape() ||
5701 llvm::isa<BlockArgument>(getBuffer()))
5702 return success();
5703 if ((getOffsetParameterAttr() || getOffsetStateTableIdxAttr()) && !hasLen())
5704 return emitOpError("offset_parameter requires an explicit `len` on a local "
5705 "buffer; the default whole-buffer length runs past the "
5706 "end for any nonzero runtime offset");
5707 std::optional<int32_t> offset = hasOffset() ? getConstantOffset() : 0;
5708 if (!offset || *offset < 0)
5709 return success();
5710 int64_t numElements = buffer.getNumElements();
5711 SmallVector<OpFoldResult> sizes = getMixedSizes();
5712 SmallVector<OpFoldResult> strides = getMixedStrides();
5713 if (sizes.empty()) {
5714 std::optional<int64_t> len =
5715 hasLen() ? std::optional<int64_t>(getConstantLen()) : numElements;
5716 if (!len || *len <= 0 || *offset + *len <= numElements)
5717 return success();
5718 InFlightDiagnostic err = emitOpError("transfer of ")
5719 << *len << " elements at offset " << *offset
5720 << " runs past the end of the " << numElements
5721 << "-element buffer";
5722 if (!hasLen())
5723 err.attachNote() << "an omitted `len` is the whole buffer; give the "
5724 "length to transfer from this offset";
5725 return err;
5726 }
5727 int64_t maxIdx = *offset;
5728 for (auto [size, stride] : llvm::zip(sizes, strides)) {
5729 std::optional<int64_t> s = getConstantIntValue(size);
5730 std::optional<int64_t> t = getConstantIntValue(stride);
5731 if (!s || !t)
5732 return success();
5733 maxIdx += *t * (*s - 1);
5734 }
5735 if (maxIdx < numElements)
5736 return success();
5737 return emitOpError() << "Specified stride(s) and size(s) result in out of "
5738 "bounds access in buffer, for index "
5739 << maxIdx << " in memref of length " << numElements
5740 << ".";
5741}
5742
5743// A BDDimLayoutAttr array (outermost-first) describes a contiguous row-major
5744// scan when the innermost stride is 1 and each outer stride equals the product
5745// of all inner sizes. Used by both DMABDOp verification and canonicalization.
5746int64_t xilinx::AIE::paddedSize(llvm::ArrayRef<BDDimLayoutAttr> dims,
5747 llvm::ArrayRef<BDPadLayoutAttr> pads,
5748 int64_t fallback) {
5749 if (dims.empty() || pads.empty())
5750 return fallback;
5751 int64_t size = 1;
5752 for (auto [dim, pad] : llvm::zip(dims, pads))
5753 size *= dim.getSize() + pad.getConstPadBefore() + pad.getConstPadAfter();
5754 return size;
5755}
5756
5757bool xilinx::AIE::isContiguousBDTransfer(llvm::ArrayRef<BDDimLayoutAttr> dims) {
5758 if (dims.empty())
5759 return true; // no ND layout = trivially contiguous
5760 // Innermost (last) stride must be 1.
5761 if (dims.back().getStride() != 1)
5762 return false;
5763 // Each outer stride must equal the product of all inner sizes.
5764 // Use uint64_t throughout to match the unsigned getSize()/getStride() types.
5765 uint64_t product = 1;
5766 for (int i = static_cast<int>(dims.size()) - 1; i >= 1; --i) {
5767 product *= dims[i].getSize();
5768 // A size-1 dimension's stride is irrelevant (it is never stepped).
5769 if (dims[i - 1].getSize() > 1 && dims[i - 1].getStride() != product)
5770 return false;
5771 }
5772 return true;
5773}
5774
5775std::optional<SmallVector<BDDimLayoutAttr>>
5776xilinx::AIE::normalizedBDDims(ArrayRef<BDDimLayoutAttr> dims, int64_t len) {
5777 SmallVector<BDDimLayoutAttr> kept;
5778 int64_t product = 1;
5779 for (BDDimLayoutAttr dim : dims) {
5780 product *= dim.getSize();
5781 if (dim.getSize() != 1)
5782 kept.push_back(dim);
5783 }
5784 if (kept.size() == dims.size() || len != product)
5785 return std::nullopt;
5786 if (isContiguousBDTransfer(kept))
5787 return SmallVector<BDDimLayoutAttr>{};
5788 return kept;
5789}
5790
5791// The firmware's UPDATE_REG reads the 32-bit length word with its low two bits
5792// cleared, adds the delta, and writes the sum back with them cleared again.
5793// The length word counts 32-bit words, so both the static length and every
5794// added unit must be multiples of four words (16 bytes) to survive.
5795FailureOr<int64_t> xilinx::AIE::getLengthUnitBytes(Operation *op,
5796 int64_t lengthUnit,
5797 BaseMemRefType buffer) {
5798 uint64_t elemBitWidth = buffer.getElementTypeBitWidth();
5799 if (elemBitWidth == 0 || (elemBitWidth % 8) != 0)
5800 return op->emitOpError("length_parameter requires a whole-byte element "
5801 "type");
5802 std::optional<int64_t> unitBytes =
5803 llvm::checkedMul(lengthUnit, static_cast<int64_t>(elemBitWidth / 8));
5804 if (!unitBytes || *unitBytes / 4 > std::numeric_limits<uint32_t>::max())
5805 return op->emitOpError("length_unit (")
5806 << lengthUnit
5807 << " elements) exceeds the 32-bit word count of a BD length";
5808 if (*unitBytes % 16 != 0)
5809 return op->emitOpError("length_unit must be a multiple of 16 bytes, got ")
5810 << *unitBytes << " bytes";
5811 return *unitBytes;
5812}
5813
5815 Operation *op, std::optional<int64_t> lengthUnit, BaseMemRefType buffer,
5816 std::optional<int64_t> lenElems, bool contiguous,
5817 llvm::ArrayRef<int64_t> innerSizes) {
5818 if (!lengthUnit)
5819 return op->emitOpError("length_parameter requires length_unit");
5820 if (failed(getLengthUnitBytes(op, *lengthUnit, buffer)))
5821 return failure();
5822 int64_t elemBytes = buffer.getElementTypeBitWidth() / 8;
5823 if (!lenElems)
5824 return op->emitOpError("length_parameter requires a constant length");
5825 if ((*lenElems * elemBytes) % 16 != 0)
5826 return op->emitOpError("length_parameter requires a static length that is "
5827 "a multiple of 16 bytes, got ")
5828 << *lenElems * elemBytes << " bytes";
5829 if (contiguous)
5830 return success();
5831 llvm::SmallVector<int64_t, 3> sizes(innerSizes);
5832 llvm::SmallVector<int64_t, 3> strides(3, 0);
5833 placeRuntimeLengthDimension(sizes, strides);
5834 if (sizes[2] <= 1)
5835 return op->emitOpError("length_parameter on a non-contiguous pattern "
5836 "requires its second or third dimension to have a "
5837 "size above one: the added length continues it");
5838 int64_t rowElems = sizes[0] * sizes[1];
5839 if (*lengthUnit % rowElems != 0)
5840 return op->emitOpError("length_unit (")
5841 << *lengthUnit << " elements) must be a multiple of the " << rowElems
5842 << " elements moved per step of the dimension the added length "
5843 "continues";
5844 return success();
5845}
5846
5848 llvm::MutableArrayRef<int64_t> sizes,
5849 llvm::MutableArrayRef<int64_t> strides) {
5850 if (sizes[2] > 1 || sizes[1] <= 1)
5851 return;
5852 sizes[2] = sizes[1];
5853 strides[2] = strides[1];
5854 sizes[1] = 1;
5855 strides[1] = 0;
5856}
5857
5859 Operation *op, const AIETargetModel &targetModel, int col, int row) {
5860 AIEArch arch = targetModel.getTargetArch();
5861 if (arch != AIEArch::AIE2 && arch != AIEArch::AIE2p)
5862 return op->emitOpError("length_parameter is only supported on AIE2 and "
5863 "AIE2P devices");
5864 if (!targetModel.isShimNOCTile(col, row))
5865 return op->emitOpError("length_parameter is only supported on shim NOC "
5866 "tiles, got tile (")
5867 << col << ", " << row << ")";
5868 const DmaBdLayout *layout = targetModel.getDmaBdLayout(col, row);
5869 if (!layout || layout->bufferLength.shift != 0 ||
5870 layout->bufferLength.width != 32)
5871 return op->emitOpError("length_parameter requires a BD whose buffer "
5872 "length fills a 32-bit register");
5873 return success();
5874}
5875
5876llvm::SmallVector<uint32_t> xilinx::AIE::getAssignedBdIds(DmaBody program) {
5877 llvm::SmallVector<uint32_t> ids;
5878 program.getDmaBody().walk([&](DMABDOp bd) {
5879 if (auto id = bd.getBdId())
5880 ids.push_back(*id);
5881 });
5882 return ids;
5883}
5884
5885LogicalResult DMABDOp::verify() {
5886 if (getOffsetParameterAttr() || getOffsetStateTableIdxAttr()) {
5887 uint64_t elemBitWidth = llvm::cast<BaseMemRefType>(getBuffer().getType())
5888 .getElementTypeBitWidth();
5889 if (elemBitWidth == 0 || (elemBitWidth % 8) != 0)
5890 return emitOpError("offset_parameter requires a whole-byte element type");
5891 }
5892
5893 if (getLengthParameterAttr() || getLengthStateTableIdxAttr()) {
5894 // The length is patched by a runtime-sequence instruction, so only a BD
5895 // the runtime sequence configures can have one.
5896 if (llvm::isa<MemOp, MemTileDMAOp, ShimDMAOp, DMAOp>(
5897 (*this)->getParentOp()))
5898 return emitOpError("length_parameter is only supported on a BD in a "
5899 "runtime sequence");
5900 std::optional<llvm::SmallVector<BDDimLayoutAttr>> dims =
5901 getFoldedDimensions([&]() {
5902 return emitOpError("length_parameter requires a static pattern: ");
5903 });
5904 if (!dims)
5905 return failure();
5906 // aie-decompose-large-dma-bd leaves a runtime-length BD whole.
5907 if (dims->size() > 4)
5908 return emitOpError("length_parameter requires at most 4 dimensions, got ")
5909 << dims->size();
5910 // The outermost of four dimensions is the iteration, which repeats the
5911 // whole length.
5912 llvm::ArrayRef<BDDimLayoutAttr> inner(*dims);
5913 if (inner.size() == 4)
5914 inner = inner.drop_front();
5915 llvm::SmallVector<int64_t, 3> innerSizes(3, 1);
5916 for (auto [i, dim] : llvm::enumerate(llvm::reverse(inner)))
5917 innerSizes[i] = dim.getSize();
5918 auto buffer = llvm::cast<BaseMemRefType>(getBuffer().getType());
5919 std::optional<int64_t> lenElems;
5920 if (std::optional<int32_t> len = getConstantLen())
5921 lenElems = *len;
5922 else if (!hasLen() && buffer.hasStaticShape())
5923 lenElems = buffer.getNumElements();
5924 if (failed(verifyLengthParameter(*this, getLengthUnit(), buffer, lenElems,
5926 innerSizes)))
5927 return failure();
5928 }
5929
5930 // Skip verification of the BDOp outside of mem operations.
5931 // BDOps may appear elsewhere and subsequent lowerings will place them in the
5932 // correct mem ops.
5933 Operation *p = (*this)->getParentOp();
5934 if (!llvm::isa<MemOp, MemTileDMAOp, ShimDMAOp, DMAOp>(*p)) {
5935 return success();
5936 }
5937
5938 // Check if buffer is an unranked memref (e.g., from function argument)
5939 bool isUnrankedMemRef = llvm::isa<UnrankedMemRefType>(getBuffer().getType());
5940
5941 // For unranked memrefs, we can't verify as strictly since we don't know
5942 // the buffer's defining op or its full type at compile time
5943 if (!isUnrankedMemRef) {
5944 if (!isa<BufferOp, ExternalBufferOp>(getBuffer().getDefiningOp()))
5945 return emitOpError(
5946 "BDs only support BufferOp or ExternalBufferOp operands.");
5947 }
5948
5949 if (getLenInBytes() % 4)
5950 return emitOpError("transfer length must be multiple of 4 (i.e., represent "
5951 "4 byte aligned address)");
5952
5953 TileElement parentTileElement = getParentTileElement(getOperation());
5954 TileLike parentTile = parentTileElement.getTileLike();
5955 if (!parentTile)
5956 return emitOpError("parent tile must implement TileLike interface");
5957
5958 if (!isUnrankedMemRef && getOperation()->getParentOfType<MemOp>() &&
5959 getBufferOp().getTile() != parentTileElement.getTile())
5960 return emitOpError(
5961 "Core tile DMAs can only access a buffer in the same tile.");
5962
5963 const AIETargetModel &targetModel = getTargetModel(getOperation());
5964
5965 uint32_t maxBds = targetModel.getNumBDs(parentTile.getTileType());
5966 if (std::optional<int32_t> bdId = getBdId();
5967 bdId.has_value() && static_cast<uint32_t>(*bdId) >= maxBds)
5968 return emitOpError("bdId attribute exceeds max: ") << maxBds - 1;
5969 if (std::optional<int32_t> nextBdId = getNextBdId();
5970 nextBdId.has_value() && static_cast<uint32_t>(*nextBdId) >= maxBds)
5971 return emitOpError("nextBdId attribute exceeds max: ") << maxBds - 1;
5972
5973 if (getBdIdVal() && getBdId().has_value())
5974 return emitOpError("bd_id and bd_id_val are mutually exclusive");
5975
5976 // Issue #1097: the buffer_length field of a DMA buffer descriptor has a
5977 // tile-type-specific bit width (e.g. on AIE2: 32-bit shim, 17-bit mem tile,
5978 // 14-bit core tile). The hardware field counts address-generation granules
5979 // (32-bit words), so convert the byte length accordingly and reject
5980 // transfers that would overflow the field (they would otherwise be silently
5981 // truncated during lowering). Only checkable when the length is statically
5982 // known: either an explicit constant `len`, or (when no `len` is given) a
5983 // statically-shaped buffer whose full size is used.
5984 {
5985 bool lenStaticallyKnown = getConstantLen().has_value();
5986 if (!lenStaticallyKnown && !hasLen()) {
5987 if (auto shaped = llvm::dyn_cast<mlir::ShapedType>(getBuffer().getType()))
5988 lenStaticallyKnown = shaped.hasStaticShape();
5989 }
5990 // Skip validation for non-positive constant lengths: negative values
5991 // indicate a separate issue (e.g. distribute-link offset inference) and
5992 // would wrap to huge values via getLenInBytes().
5993 if (lenStaticallyKnown) {
5994 if (auto constLen = getConstantLen();
5995 constLen.has_value() && *constLen <= 0) {
5996 // non-positive explicit length — skip overflow check
5997 } else {
5998 uint64_t lenInBytes = getLenInBytes();
5999 uint32_t granularity = targetModel.getAddressGenGranularity();
6000 if (granularity != 0) {
6001 uint64_t lenInWords = lenInBytes * 8 / granularity;
6002 uint64_t maxLen =
6003 targetModel.getDmaBdMaxLen(parentTile.getTileType());
6004 if (lenInWords > maxLen)
6005 return emitOpError()
6006 << "buffer descriptor length (" << lenInWords
6007 << " 32-bit words) exceeds the maximum of " << maxLen
6008 << " words supported by this tile type";
6009 }
6010 }
6011 }
6012 }
6013
6014 if (failed(verifyMixedSizesAndStrides()) || failed(verifyTransferInBounds()))
6015 return failure();
6016
6017 // Fold the mixed sizes/strides to a constant BDDimLayoutAttr list for
6018 // verification; a runtime size/stride yields nullopt plus a diagnostic.
6019 std::optional<llvm::SmallVector<BDDimLayoutAttr>> dims;
6020 if (!getMixedSizes().empty()) {
6021 dims = getConstantDimensions();
6022 if (!dims.has_value())
6023 return failure();
6024 }
6025 if (dims.has_value()) {
6026 // The per-BD ND access-pattern limit is a hardware property of the tile;
6027 // query it from the target model by tile type rather than inferring it
6028 // from the parent op type.
6029 size_t maxNDims = targetModel.getBDMaxDims(parentTile.getTileType());
6030 if (dims->size() > maxNDims)
6031 return emitOpError() << "Cannot give more than "
6032 << std::to_string(maxNDims)
6033 << " dimensions for step sizes and wraps on this "
6034 "tile (got "
6035 << std::to_string(dims->size()) << " dimensions).";
6036
6037 auto buffer = llvm::dyn_cast<MemRefType>(getBuffer().getType());
6038 if (!buffer)
6039 return emitOpError() << "dimensions attribute cannot be used with "
6040 "unranked memref buffer type.";
6041 // A contiguous row-major access on a shim tile is lowered to linear mode
6042 // by aie-dma-tasks-to-npu / aie-dma-to-npu, using the wide buffer_length
6043 // register which is exempt from the 10-bit ND wrap-size limit.
6044 // Skip the per-dimension size check when the BD is on a shim tile and the
6045 // access is contiguous, so the natural ND form can be written without
6046 // triggering a spurious verifier error before lowering.
6047 //
6048 // Note: the verifier early-exit above means we only reach this code when
6049 // the parent op is MemOp, MemTileDMAOp, ShimDMAOp, or DMAOp -- all of
6050 // which are TileElements, so parentTile is always non-null here.
6051 bool skipSizeCheck =
6052 parentTile.isShimTile() && xilinx::AIE::isContiguousBDTransfer(*dims);
6053
6054 // The wrap (size) and step (stride) field widths are tile-type specific
6055 // (e.g. on AIE2: core tiles have an 8-bit wrap, mem/shim tiles 10-bit).
6056 // Wrap fields are unbiased, so a W-bit wrap field admits sizes up to
6057 // 2^W - 1. Step fields are hardware-encoded as actual-1, so an S-bit
6058 // step field admits actual (unbiased, as declared here) strides up to
6059 // 2^S. Deriving the limits from the target model (instead of a single
6060 // hardcoded shim-sized constant) keeps this check correct for all tile
6061 // types, and computing the message from the same value it checks
6062 // against means the two can never disagree again.
6063 uint32_t wrapBits = targetModel.getDmaBdWrapBits(parentTile.getTileType());
6064 uint32_t stepBits = targetModel.getDmaBdStepBits(parentTile.getTileType());
6065 uint64_t maxSize = wrapBits > 0 ? (1ULL << wrapBits) - 1 : 0;
6066 uint64_t maxStride = stepBits > 0 ? (1ULL << stepBits) : 0;
6067
6068 for (BDDimLayoutAttr dim : *dims) {
6069 if (0 == dim.getStride() && dim.getSize() != 1)
6070 return emitOpError()
6071 << "Invalid step size; must be a positive integer.";
6072 if (dim.getStride() > buffer.getNumElements())
6073 return emitOpError() << "Step size " << std::to_string(dim.getStride())
6074 << " exceeds memref size "
6075 << std::to_string(buffer.getNumElements());
6076 if (!skipSizeCheck && dim.getSize() > maxSize)
6077 return emitOpError() << "Size may not exceed " << maxSize << ".";
6078 if (dim.getStride() > maxStride)
6079 return emitOpError() << "Stride may not exceed " << maxStride << ".";
6080 }
6081
6082 // Since streams read 32b words, there's no way to read eg 16b with stride
6083 // of 2 (ie lower halfs of each 32b). So force it to be 1 (and then in
6084 // CDODirect/XAIEV2 scale the size by 4/getBufferElementTypeWidthInBytes).
6085 if (getBufferElementTypeWidthInBytes() < 4 && dims->back().getStride() != 1)
6086 return emitOpError(
6087 "For <32b width datatypes, inner-most dim stride must be 1");
6088
6089 if (getBufferElementTypeWidthInBytes() > 4 && dims->back().getStride() != 1)
6090 return emitOpError(
6091 "For >32b width datatypes, inner-most dim stride must be 1");
6092
6093 // The hardware stepsize/wrap registers are denominated in 32-bit words.
6094 // lib/Targets/AIERT.cpp's static/CDO lowering converts element-
6095 // granularity strides/sizes to words by scaling by
6096 // elementWidthInBytes/4.0 and truncating via an unguarded static_cast;
6097 // when that scale factor isn't an integer, a declared value that isn't
6098 // itself a whole number of words is silently replaced by a different,
6099 // hardware-expressible value instead of being rejected. Reject those
6100 // values here instead, since they are genuinely inexpressible in the
6101 // hardware's word-granularity registers. The trigger condition is
6102 // "element width not a multiple of 4 bytes" rather than "< 4 bytes" so
6103 // that this also covers the `bfp` block-floating-point types, whose
6104 // widths (9 and 17 bytes) are >4 bytes but still not word multiples.
6105 // Dimensions with size == 1 are skipped: the loop runs once so stride is
6106 // never stepped and that dimension's size/stride do not affect hardware
6107 // address generation; word alignment is not required for them.
6108 //
6109 // Which dim is "innermost" is determined by the last dim with size > 1,
6110 // not by array position: `dims` is outermost-first, and trailing dims
6111 // with size == 1 (which never step) do not change which dim actually
6112 // performs the finest-granularity access. If every dim has size == 1,
6113 // there is no innermost dim to check and the whole block is skipped.
6114 int32_t elementWidthInBytes = getBufferElementTypeWidthInBytes();
6115 if (elementWidthInBytes % 4 != 0) {
6116 std::optional<size_t> effectiveInnermost;
6117 for (size_t i = 0; i < dims->size(); i++)
6118 if ((*dims)[i].getSize() > 1)
6119 effectiveInnermost = i;
6120 if (effectiveInnermost.has_value()) {
6121 for (size_t i = 0; i < dims->size(); i++) {
6122 BDDimLayoutAttr dim = (*dims)[i];
6123 if (dim.getSize() <= 1)
6124 continue;
6125 if (i == *effectiveInnermost) {
6126 // Effective innermost dim: the size (element count transferred
6127 // at this level) must itself be a whole number of 32-bit words.
6128 if ((dim.getSize() * elementWidthInBytes) % 4)
6129 return emitOpError()
6130 << "Innermost dim size (" << dim.getSize() << ") * "
6131 << elementWidthInBytes << "-byte element width = "
6132 << (dim.getSize() * elementWidthInBytes)
6133 << " bytes: innermost dim size must be a multiple of 4 "
6134 "bytes for sub-32b element types (the hardware size "
6135 "register is 32-bit-word granularity).";
6136 } else {
6137 // Dim outside the effective innermost dim: the stride (address
6138 // step between elements at this level) must itself be a whole
6139 // number of 32-bit words.
6140 if ((dim.getStride() * elementWidthInBytes) % 4)
6141 return emitOpError()
6142 << "Dim " << i << " stride (" << dim.getStride() << ") * "
6143 << elementWidthInBytes << "-byte element width = "
6144 << (dim.getStride() * elementWidthInBytes)
6145 << " bytes: non-innermost dim stride must be a multiple "
6146 "of 4 bytes for sub-32b element types (the hardware "
6147 "stepsize register is 32-bit-word granularity).";
6148 }
6149 }
6150 }
6151 }
6152 }
6153 if (auto paddims = getPadDimensions(); paddims.has_value()) {
6154 if (!dims.has_value())
6155 return emitOpError() << "Padding requires n-d data layouts expressed as"
6156 << " wrap(s) and stride(s).";
6157 const DmaBdLayout *layout =
6158 targetModel.getDmaBdLayout(parentTile.getTileType());
6159 llvm::SmallVector<std::pair<DmaBdField, DmaBdField>, 3> padFields;
6160 if (layout)
6161 padFields = layout->padFields();
6162 if (padFields.empty())
6163 return emitOpError() << "Padding is only supported by memtile dma bds.";
6164 if (dims->size() != paddims->size())
6165 return emitOpError() << "Mismatch number of dimensions between padding(s)"
6166 << " and wrap(s) and stride(s).";
6167 size_t numPadded = padFields.size();
6168 for (auto [d, pad] : llvm::enumerate(llvm::reverse(*paddims)))
6169 if (d >= numPadded && (pad.getConstPadBefore() || pad.getConstPadAfter()))
6170 return emitOpError()
6171 << "a " << paddims->size() << "-dimensional walk pads only its "
6172 << numPadded << " innermost dimensions";
6173 int actuallen = 1;
6174 for (unsigned i = 0; i < paddims->size(); i++) {
6175 auto dim = (*dims)[i];
6176 auto paddim = (*paddims)[i];
6177 actuallen *= paddim.getConstPadBefore() + paddim.getConstPadAfter() +
6178 dim.getSize();
6179 if (std::optional<int32_t> len = getConstantLen(); len.has_value()) {
6180 if (actuallen > *len)
6181 return emitOpError() << "Data exceeds len after padding.";
6182 } else if (getLen()) {
6183 return emitOpError()
6184 << "Padding with a runtime len operand is not yet supported; "
6185 "use a compile-time constant len with padded BDs.";
6186 }
6187 }
6188 if ((paddims->back().getConstPadBefore() *
6189 getBufferElementTypeWidthInBytes()) %
6190 4)
6191 return emitOpError() << "Inner-most padding-before count must result in"
6192 << " padding in 32-bit words.";
6193 if ((paddims->back().getConstPadAfter() *
6194 getBufferElementTypeWidthInBytes()) %
6195 4)
6196 return emitOpError() << "Inner-most padding-after count must result in"
6197 << " padding in 32-bit words.";
6198 for (auto [d, pad] : llvm::enumerate(llvm::reverse(*paddims))) {
6199 if (d >= numPadded)
6200 break;
6201 int64_t before = pad.getConstPadBefore();
6202 int64_t after = pad.getConstPadAfter();
6203 if (d == 0) {
6204 before = before * getBufferElementTypeWidthInBytes() / 4;
6205 after = after * getBufferElementTypeWidthInBytes() / 4;
6206 }
6207 uint32_t maxBefore = padFields[d].first.mask();
6208 uint32_t maxAfter = padFields[d].second.mask();
6209 if (before > maxBefore || after > maxAfter)
6210 return emitOpError()
6211 << "D" << d << " pad_before/pad_after exceeds the [0:"
6212 << (before > maxBefore ? maxBefore : maxAfter) << "] range"
6213 << (d == 0 ? " of 32-bit words." : ".");
6214 }
6215 }
6216 if (!isUnrankedMemRef &&
6217 (parentTile.isMemTile() || parentTile.isCoreTile())) {
6218 if (auto baseAddr = getBufferOp().getAddress(); baseAddr.has_value()) {
6219 int64_t offsetInBytes = *baseAddr + getOffsetInBytes();
6220 if (offsetInBytes % 4)
6221 return emitOpError("bd address must be 4 byte (32b) aligned; got "
6222 "base+offset: ")
6223 << offsetInBytes << " (bytes)";
6224 }
6225 }
6226 if (auto packetInfo = getPacket()) {
6227 if (failed(verifyAssignedPacket(*this, *packetInfo)))
6228 return failure();
6229 if (packetInfo->getPktType() > 7)
6230 return emitOpError("Packet type field can only hold 3 bits.");
6231 if (packetInfo->assignedId() >
6232 getTargetModel(getOperation()).getMaxPacketId())
6233 return emitOpError("Packet ID field can only hold 5 bits.");
6234 }
6235
6236 if (failed(verifyDMABDOutOfOrderId(*this)))
6237 return failure();
6238
6239 // A runtime len operand or the static_len attribute both count as having a
6240 // length here.
6241 if (!hasLen() && !getBuffer().getType().hasStaticShape())
6242 return emitOpError() << "buffer with dynamic shape requires static length.";
6243
6244 if (getBurstLength() != 0 && !parentTile.isShimNOCTile())
6245 return emitOpError("Burst length is only supported in Shim NOC tiles that "
6246 "are connected to the memory-mapped NOC.");
6247
6248 if (auto axcache = getAxcache()) {
6249 if (!parentTile.isShimNOCTile()) {
6250 return emitOpError("AxCACHE is only supported in Shim NOC tiles "
6251 "that are connected to the memory-mapped NOC.");
6252 }
6253 if (axcache > 0xF) {
6254 return emitOpError("AxCache value out of 4-bit range.");
6255 }
6256 }
6257
6258 return verifyDMABDIteration(*this, parentTile.getTileType());
6259}
6260
6261LogicalResult DMABDPACKETOp::verify() {
6262 // Both fields read back signed (AIEI32Attr), so the range is two-sided even
6263 // though the hardware fields are unsigned: aie.dma_bd_packet(-1, -1) parses.
6264 if (getPacketType() < 0 || getPacketType() > 7)
6265 return emitOpError("Packet type field can only hold 3 bits.");
6266 if (getPacketID() < 0 ||
6267 getPacketID() >
6268 static_cast<int32_t>(getTargetModel(getOperation()).getMaxPacketId()))
6269 return emitOpError("Packet ID field can only hold 5 bits.");
6270 return success();
6271}
6272
6273std::optional<int64_t> DMABDOp::getStaticLenInBytes() {
6274 int64_t elems;
6275 if (std::optional<int32_t> len = getConstantLen()) {
6276 elems = static_cast<uint32_t>(*len);
6277 } else {
6278 // With runtime sizes/strides, an omitted len is their inner extent
6279 // (AIEDMATasksToNPU), not the whole buffer.
6280 if (hasLen() || !getSizes().empty() || !getStrides().empty())
6281 return std::nullopt;
6282 auto buffer = dyn_cast<MemRefType>(getBuffer().getType());
6283 if (!buffer || !buffer.hasStaticShape())
6284 return std::nullopt;
6285 elems = buffer.getNumElements();
6286 }
6287 return llvm::checkedMul(
6288 elems, static_cast<int64_t>(getBufferElementTypeWidthInBytes()));
6289}
6290
6291uint32_t DMABDOp::getAxcacheOrDefault() {
6292 return getAxcache().value_or(
6293 getTargetModel(getOperation()).getDefaultAxCache());
6294}
6295
6296//===----------------------------------------------------------------------===//
6297// DMAStartOp
6298//===----------------------------------------------------------------------===//
6299
6300static LogicalResult FoldDMAStartOp(DMAStartOp op, PatternRewriter &rewriter) {
6301
6302 llvm::SetVector<Block *> reachable;
6303 SmallVector<Block *, 16> worklist;
6304 Block *firstBD = op.getSuccessor(0);
6305 reachable.insert(firstBD);
6306 worklist.push_back(firstBD);
6307 while (!worklist.empty()) {
6308 Block *block = worklist.pop_back_val();
6309 if (block->empty())
6310 continue;
6311 auto successors = block->getTerminator()->getSuccessors();
6312 for (auto *i : successors) {
6313 if (!reachable.contains(i)) {
6314 reachable.insert(i);
6315 worklist.push_back(i);
6316 }
6317 }
6318 }
6319
6320 // BD chain ends with an EndOp, indicating non-repeating pattern: BD chain
6321 // folding not applicable.
6322 if (isa<EndOp>((reachable.back())->getTerminator()))
6323 return failure();
6324
6325 // Check for identical bds.
6326 auto areEquivalentBDs = [](Block *b1, Block *b2) {
6327 auto b1OpRange = b1->without_terminator();
6328 auto b2OpRange = b2->without_terminator();
6329 if (llvm::range_size(b1OpRange) != llvm::range_size(b2OpRange))
6330 return false;
6331 if (!llvm::all_of(llvm::zip_equal(b1OpRange, b2OpRange),
6332 [](std::tuple<Operation &, Operation &> pair) {
6333 return OperationEquivalence::isEquivalentTo(
6334 &std::get<0>(pair), &std::get<1>(pair),
6335 OperationEquivalence::IgnoreLocations);
6336 }))
6337 return false;
6338 return true;
6339 };
6340
6341 // Get a vector of unique BDs.
6342 SmallVector<Block *> uniquePattern;
6343 const auto *patternIt = reachable.begin();
6344 while (patternIt != reachable.end() &&
6345 llvm::none_of(uniquePattern, [patternIt, areEquivalentBDs](Block *b1) {
6346 return areEquivalentBDs(*patternIt, b1);
6347 })) {
6348 uniquePattern.push_back(*patternIt);
6349 patternIt++;
6350 }
6351
6352 // No block repeats, so there is no sub-chain to collapse.
6353 if (patternIt == reachable.end())
6354 return failure();
6355
6356 unsigned idx = 0;
6357 while (patternIt != reachable.end()) {
6358 // BD repetition found. Check if repeating pattern.
6359 if (!areEquivalentBDs(*patternIt, uniquePattern[idx]))
6360 return failure();
6361 patternIt++;
6362 idx = (idx + 1) % uniquePattern.size();
6363 }
6364
6365 // Repeating BD chains detected. Erasing repetitions.
6366 auto lastBDTerm = cast<NextBDOp>(reachable.back()->getTerminator());
6367 auto lastUniqueBDTerm = cast<NextBDOp>(uniquePattern.back()->getTerminator());
6368 rewriter.modifyOpInPlace(lastUniqueBDTerm, [&] {
6369 lastUniqueBDTerm.setSuccessor(lastBDTerm.getSuccessor());
6370 });
6371
6372 return success();
6373}
6374
6375// Canonicalization pattern for DMABDOp: on shim tiles, fold a contiguous
6376// row-major ND access pattern into canonical linear form (no dimensions
6377// attribute), so that the hardware uses the wide buffer_length register.
6378namespace {
6379struct LinearizeContiguousBDTransfer : public mlir::OpRewritePattern<DMABDOp> {
6380 using OpRewritePattern::OpRewritePattern;
6381
6382 mlir::LogicalResult
6383 matchAndRewrite(DMABDOp op, mlir::PatternRewriter &rewriter) const override {
6384 if (op.getStaticSizes().value_or(llvm::ArrayRef<int64_t>{}).empty())
6385 return mlir::failure();
6386
6387 // Only fire for shim DMA BDs: ExternalBufferOp buffer
6388 // (dma_configure_task_for stage), parent TileElement is shim
6389 // (dma_configure_task stage), or inside ShimDMAOp (after full lowering).
6390 bool bufferIsExternal =
6391 isa_and_nonnull<ExternalBufferOp>(op.getBuffer().getDefiningOp());
6392 TileElement parentElem = getParentTileElement(op.getOperation());
6393 TileLike parentTileLike =
6394 parentElem ? parentElem.getTileLike() : TileLike{};
6395 bool parentIsShim = parentTileLike && parentTileLike.isShimTile();
6396 bool inShimDMA = (bool)op->getParentOfType<ShimDMAOp>();
6397 if (!bufferIsExternal && !parentIsShim && !inShimDMA)
6398 return mlir::failure();
6399
6400 // Only ND dimensions that are all-constant can be linearized; decline
6401 // silently on runtime-valued sizes/strides so valid dynamic IR doesn't get
6402 // a spurious diagnostic.
6403 auto isConstant = [](mlir::Value v) {
6404 return mlir::getConstantIntValue(v).has_value();
6405 };
6406 if (!llvm::all_of(op.getSizes(), isConstant) ||
6407 !llvm::all_of(op.getStrides(), isConstant))
6408 return mlir::failure();
6409 std::optional<llvm::SmallVector<BDDimLayoutAttr>> dims =
6410 op.getFoldedDimensions([&]() { return op.emitError(); });
6411 if (!dims.has_value() || dims->empty())
6412 return mlir::failure();
6414 return mlir::failure();
6415 // Already linear (single dimension with stride 1)?
6416 if (dims->size() == 1 && dims->front().getStride() == 1)
6417 return mlir::failure();
6418
6419 int64_t product = 1;
6420 for (BDDimLayoutAttr dim : *dims)
6421 product *= dim.getSize();
6422
6423 // len < product: the outermost dim describes a hardware BD iteration
6424 // (preserved downstream as iteration_size/stride). Don't fold it away.
6425 // A runtime len can't be compared, so bail. Also bail if len is absent but
6426 // the buffer is unranked (can't safely infer product == full transfer).
6427 if (!op.hasLen() && !op.getBuffer().getType().hasStaticShape())
6428 return mlir::failure();
6429 std::optional<int32_t> lenVal = op.getConstantLen();
6430 if (lenVal.has_value() && static_cast<int64_t>(*lenVal) != product)
6431 return mlir::failure();
6432 int32_t len = static_cast<int32_t>(product);
6433
6434 // Rewrite to a linear BD: len as the static_len attribute, sizes/strides
6435 // cleared. Other attributes are preserved.
6436 rewriter.modifyOpInPlace(op, [&]() {
6437 op.getLenMutable().clear();
6438 op.setStaticLen(len);
6439 op.getSizesMutable().clear();
6440 op.getStridesMutable().clear();
6441 op.setStaticSizes(std::nullopt);
6442 op.setStaticStrides(std::nullopt);
6443 });
6444 return mlir::success();
6445 }
6446};
6447} // namespace
6448
6449// Canonicalization pattern for DMABDOp: fold a constant offset/len/size/stride
6450// operand back into the corresponding static_* attribute, removing the operand.
6451namespace {
6452struct FoldConstantBDDimList : public mlir::OpRewritePattern<DMABDOp> {
6453 using OpRewritePattern::OpRewritePattern;
6454
6455 mlir::LogicalResult
6456 matchAndRewrite(DMABDOp op, mlir::PatternRewriter &rewriter) const override {
6457 if (op.getSizes().empty() && op.getStrides().empty() && !op.getOffset() &&
6458 !op.getLen())
6459 return mlir::failure();
6460 llvm::SmallVector<mlir::OpFoldResult> sizes = op.getMixedSizes();
6461 llvm::SmallVector<mlir::OpFoldResult> strides = op.getMixedStrides();
6462 // foldDynamicIndexList returns success() only if it replaced a dynamic
6463 // entry with a constant. Bitwise (not logical) OR is deliberate: both
6464 // calls must run unconditionally, so strides still folds even once
6465 // sizes already succeeded.
6466 // NOLINTNEXTLINE(clang-diagnostic-bitwise-instead-of-logical)
6467 bool changed = succeeded(mlir::foldDynamicIndexList(sizes)) |
6468 succeeded(mlir::foldDynamicIndexList(strides));
6469
6470 // Fold a constant offset/len operand into its static_* attribute.
6471 std::optional<int32_t> foldOffset, foldLen;
6472 if (op.getOffset())
6473 if (auto c = mlir::getConstantIntValue(op.getOffset()))
6474 foldOffset = static_cast<int32_t>(*c);
6475 if (op.getLen())
6476 if (auto c = mlir::getConstantIntValue(op.getLen()))
6477 foldLen = static_cast<int32_t>(*c);
6478 changed |= foldOffset.has_value() || foldLen.has_value();
6479
6480 if (!changed)
6481 return mlir::failure();
6482
6483 llvm::SmallVector<int64_t> staticSizes, staticStrides;
6484 llvm::SmallVector<mlir::Value> dynSizes, dynStrides;
6485 mlir::dispatchIndexOpFoldResults(sizes, dynSizes, staticSizes);
6486 mlir::dispatchIndexOpFoldResults(strides, dynStrides, staticStrides);
6487
6488 // Without `len`, a BD with runtime dims transfers their d0*d1*d2 extent,
6489 // but a fully static one transfers its whole buffer. Pin the extent
6490 // before the last runtime dim folds away so the transfer keeps its size.
6491 bool wasDynamic = !op.getSizes().empty() || !op.getStrides().empty();
6492 if (!op.hasLen() && wasDynamic && dynSizes.empty() && dynStrides.empty() &&
6493 !staticSizes.empty()) {
6494 int64_t extent = 1;
6495 for (int64_t s : llvm::ArrayRef(staticSizes).take_back(3))
6496 extent *= s;
6497 foldLen = static_cast<int32_t>(extent);
6498 }
6499
6500 rewriter.modifyOpInPlace(op, [&]() {
6501 op.getSizesMutable().assign(dynSizes);
6502 op.getStridesMutable().assign(dynStrides);
6503 // Leave the static arrays unset when there is no ND layout so they elide.
6504 if (staticSizes.empty())
6505 op.removeStaticSizesAttr();
6506 else
6507 op.setStaticSizes(staticSizes);
6508 if (staticStrides.empty())
6509 op.removeStaticStridesAttr();
6510 else
6511 op.setStaticStrides(staticStrides);
6512 if (foldOffset) {
6513 op.getOffsetMutable().clear();
6514 op.setStaticOffset(foldOffset);
6515 }
6516 if (foldLen) {
6517 op.getLenMutable().clear();
6518 op.setStaticLen(foldLen);
6519 }
6520 });
6521 return mlir::success();
6522 }
6523};
6524} // namespace
6525
6526void DMABDOp::getCanonicalizationPatterns(RewritePatternSet &results,
6527 MLIRContext *context) {
6528 results.add<FoldConstantBDDimList, LinearizeContiguousBDTransfer>(context);
6529}
6530
6531void DMAStartOp::getCanonicalizationPatterns(RewritePatternSet &results,
6532 MLIRContext *context) {
6533 results.add(FoldDMAStartOp);
6534}
6535
6536RouteEndpointOp DMAStartOp::getEndpointOp() {
6537 FlatSymbolRefAttr name = getEndpoint();
6538 if (!name)
6539 return nullptr;
6540 return SymbolTable::lookupNearestSymbolFrom<RouteEndpointOp>(*this, name);
6541}
6542
6543LogicalResult DMAStartOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
6544 FlatSymbolRefAttr name = getEndpoint();
6545 if (!name)
6546 return success();
6547 auto program = dyn_cast<DmaBody>((*this)->getParentOp());
6548 auto device = (*this)->getParentOfType<DeviceOp>();
6549 if (!program || !device)
6550 return emitOpError("only a tile's DMA program may name its channel by a "
6551 "route endpoint");
6552 auto endpoint = symbolTable.lookupSymbolIn<RouteEndpointOp>(device, name);
6553 if (!endpoint)
6554 return emitOpError() << name << " is not an aie.route_endpoint";
6555 if (endpoint.getBundle() != WireBundle::DMA)
6556 return emitOpError() << name << " names a "
6557 << stringifyWireBundle(endpoint.getBundle())
6558 << " port, not a DMA channel";
6559 auto here = cast<TileLike>(program.getTile().getDefiningOp());
6560 TileLike there = endpoint.getTileLike();
6561 bool sameTile = endpoint.getTile() == program.getTile() ||
6562 (here.tryGetCol() && here.tryGetRow() &&
6563 here.tryGetCol() == there.tryGetCol() &&
6564 here.tryGetRow() == there.tryGetRow());
6565 if (!sameTile)
6566 return emitOpError() << name
6567 << " is on a different tile than this DMA program";
6568 return success();
6569}
6570
6571LogicalResult DMAStartOp::verify() {
6572 if (getPadValue() != 0) {
6573 if (!isa<MemTileDMAOp>(getOperation()->getParentOp()))
6574 return emitOpError("pad_value is only supported on memtile DMA channels");
6575 if (getChannelDir() != DMAChannelDir::MM2S)
6576 return emitOpError("pad_value is only supported on MM2S DMA channels");
6577 if (!getTargetModel(getOperation()).isMemTilePadValueSupported())
6578 return emitOpError("pad_value requires the CONSTANT_PAD_VALUE register, "
6579 "unavailable on this target");
6580 }
6581 if (failed(verifyDMARepeatCount(getOperation(), getRepeatCount())))
6582 return failure();
6583 if (auto program = dyn_cast<DmaBody>((*this)->getParentOp());
6584 program && !getEndpoint() &&
6585 failed(verifyDMAChannelIndex(
6586 getOperation(), cast<TileLike>(program.getTile().getDefiningOp()),
6587 getChannelDir(), getChannelIndex())))
6588 return failure();
6589 SmallVector<DMABDOp> bds;
6590 if (getOutOfOrder()) {
6591 llvm::SmallPtrSet<Block *, 8> visited;
6592 for (Block *b = getDest(); b && visited.insert(b).second;
6593 b = b->getNumSuccessors() > 0 ? b->getSuccessor(0) : nullptr)
6594 llvm::append_range(bds, b->getOps<DMABDOp>());
6595 }
6596 return verifyOutOfOrderChannel(getOperation(), getChannelDir(),
6597 getOutOfOrder(), bds);
6598}
6599
6600//===----------------------------------------------------------------------===//
6601// SwitchboxOp
6602//===----------------------------------------------------------------------===//
6603
6604LogicalResult SwitchboxOp::verify() {
6605 // The remaining checks (port bounds, legal-connection rules) are all keyed
6606 // off the target model for a placed tile. Before --aie-place-tiles runs the
6607 // tile is still an aie.logical_tile, so defer those checks to the
6608 // post-placement re-verify -- mirroring how UsesAreAccessible and FlowOp
6609 // skip target-model checks on logical tiles.
6610 if (!isa<TileOp>(getTile().getDefiningOp()))
6611 return success();
6612
6613 Region &body = getConnections();
6614 DenseSet<Port> sourceset;
6615 DenseSet<Port> destset;
6616 auto tile = getTileOp();
6617 const auto &targetModel = getTargetModel(tile);
6618 if (body.empty())
6619 return emitOpError("should have non-empty body");
6620 // A circuit connection claims its source port wherever it appears.
6621 for (auto connectOp : body.front().getOps<ConnectOp>())
6622 sourceset.insert({connectOp.getSourceBundle(), connectOp.sourceIndex()});
6623 for (auto &ops : body.front()) {
6624 // Would be simpler if this could be templatized.
6625 auto checkBound = [&ops](StringRef dir, WireBundle bundle, int index,
6626 int bound) -> LogicalResult {
6627 if (index >= bound) {
6628 if (bound > 0)
6629 return ops.emitOpError("index ")
6630 << index << " for " << dir << " bundle "
6631 << stringifyWireBundle(bundle) << " must be less than "
6632 << bound;
6633 return ops.emitOpError()
6634 << dir << " bundle " << stringifyWireBundle(bundle)
6635 << " not supported; index: " << index << ", bound: " << bound;
6636 }
6637 return success();
6638 };
6639
6640 if (auto connectOp = dyn_cast<ConnectOp>(ops)) {
6641 Port source = {connectOp.getSourceBundle(), connectOp.sourceIndex()};
6642
6643 Port dest = {connectOp.getDestBundle(), connectOp.destIndex()};
6644 if (destset.count(dest)) {
6645 return connectOp.emitOpError()
6646 << "; connecting " << to_string(source) << " to "
6647 << to_string(dest) << " on "
6648 << to_string(this->getTileOp().getTileID())
6649 << " targets same dst as another connect op; existing "
6650 "destinations: "
6651 << llvm::join(llvm::map_range(
6652 destset, [](auto &p) { return to_string(p); }),
6653 ", ");
6654 }
6655 destset.insert(dest);
6656
6657 if (connectOp.sourceIndex() < 0)
6658 return connectOp.emitOpError("source index cannot be less than zero");
6659
6660 if (checkBound("source", connectOp.getSourceBundle(),
6661 connectOp.sourceIndex(),
6662 getNumSourceConnections(connectOp.getSourceBundle()))
6663 .failed())
6664 return failure();
6665
6666 if (connectOp.destIndex() < 0)
6667 return connectOp.emitOpError("dest index cannot be less than zero");
6668
6669 if (checkBound("dest", connectOp.getDestBundle(), connectOp.destIndex(),
6670 getNumDestConnections(connectOp.getDestBundle()))
6671 .failed())
6672 return failure();
6673
6674 // Stream switch connection constraints
6675 if (!isLegalTileConnection(tile, targetModel, connectOp))
6676 return connectOp.emitOpError("illegal stream switch connection");
6677
6678 } else if (auto connectOp = dyn_cast<MasterSetOp>(ops)) {
6679 Port dest = {connectOp.getDestBundle(), connectOp.destIndex()};
6680 if (destset.count(dest))
6681 return connectOp.emitOpError("targets same destination ")
6682 << stringifyWireBundle(dest.bundle) << ": " << dest.channel
6683 << " as another connect or masterset operation";
6684 destset.insert(dest);
6685
6686 if (connectOp.destIndex() < 0)
6687 return connectOp.emitOpError("dest index cannot be less than zero");
6688
6689 if (checkBound("dest", connectOp.getDestBundle(), connectOp.destIndex(),
6690 getNumDestConnections(connectOp.getDestBundle()))
6691 .failed())
6692 return failure();
6693
6694 // The single-arbiter invariant is checked by MasterSetOp::verify.
6695 } else if (auto connectOp = dyn_cast<PacketRulesOp>(ops)) {
6696 Port source = {connectOp.getSourceBundle(), connectOp.sourceIndex()};
6697 if (sourceset.count(source))
6698 return connectOp.emitOpError("packet switched source ")
6699 << stringifyWireBundle(source.bundle) << source.channel
6700 << " cannot match another connect or masterset operation";
6701 sourceset.insert(source);
6702
6703 } else if (auto amselOp = dyn_cast<AMSelOp>(ops)) {
6704 std::vector<MasterSetOp> mstrs;
6705 std::vector<PacketRulesOp> slvs;
6706 for (auto *user : amselOp.getResult().getUsers()) {
6707 if (auto s = dyn_cast<PacketRuleOp>(user)) {
6708 auto pktRules = cast<PacketRulesOp>(s->getParentOp());
6709 slvs.push_back(pktRules);
6710 } else if (auto m = dyn_cast<MasterSetOp>(user))
6711 mstrs.push_back(m);
6712 }
6713 for (auto m : mstrs) {
6714 for (auto s : slvs) {
6715 // Stream switch connection constraints
6716 if (!isLegalTileConnection(tile, targetModel, m, s)) {
6717 return amselOp->emitOpError("illegal stream switch connection");
6718 }
6719 }
6720 }
6721 } else if (isa<EndOp>(ops)) {
6722 // continue;
6723 } else {
6724 return ops.emitOpError("cannot be contained in a Switchbox op");
6725 }
6726 }
6727
6728 return success();
6729}
6730
6731template <typename... ParentOpTypes>
6733 static LogicalResult verifyTrait(Operation *op) {
6734 Operation *operation = op->getParentOp();
6735 while (operation) {
6736 if (llvm::isa_and_nonnull<ParentOpTypes...>(operation))
6737 return success();
6738 operation = operation->getParentOp();
6739 }
6740 return failure();
6741 }
6742};
6743
6744TileOp LockOp::getTileOp() {
6745 return cast<TileElement>(this->getOperation()).getTileOp();
6746}
6747
6748LogicalResult LockOp::verify() {
6749 if (auto result = UsesAreAccessible::verifyTrait(*this); result.failed())
6750 return result;
6751
6752 if (auto lockID = getLockID()) {
6753 TileLike tileLike = getTileLike();
6754 if (!tileLike)
6755 return emitOpError("tile operand must implement TileLike interface");
6756 const auto &targetModel = getTargetModel(*this);
6757 auto tileType = tileLike.getTileType();
6758 if (int numLocks = targetModel.getNumLocks(tileType); *lockID >= numLocks)
6759 return emitOpError("lock assigned invalid id (maximum is ")
6760 << numLocks - 1 << ")";
6761 }
6762
6763 if (std::optional<int32_t> init = getInit()) {
6764 int64_t max = getTargetModel(*this).getMaxLockValue();
6765 if (*init < 0 || *init > max)
6766 return emitOpError("init value ")
6767 << *init << " is outside the lock's range [0, " << max << "]";
6768 }
6769
6770 return success();
6771}
6772
6773// Look up for compile-time constant lock values, if any.
6774// Returns std::nullopt if lock value does not reference an `arith.constant`.
6775static std::optional<int32_t> getConstantLockValue(UseLockOp op) {
6776 if (auto constant = op.getValue().getDefiningOp<arith::ConstantOp>())
6777 if (auto intAttr = llvm::dyn_cast<IntegerAttr>(constant.getValue()))
6778 return (int32_t)intAttr.getInt();
6779 return std::nullopt;
6780}
6781
6783 static LogicalResult verifyTrait(Operation *op) {
6784 auto *block = op->getBlock();
6785 int lockID = -1;
6786 for (auto op : block->getOps<UseLockOp>()) {
6787 if (auto lock = dyn_cast<LockOp>(op.getLock().getDefiningOp());
6788 lock.getLockID().has_value()) {
6789 if (lockID != -1 && lockID != lock.getLockIDValue())
6790 return failure();
6791 lockID = lock.getLockIDValue();
6792 }
6793 }
6794 return success();
6795 }
6796};
6797
6799 static LogicalResult verifyTrait(Operation *op) {
6800 auto *block = op->getBlock();
6801 int acqValue = -1, relValue = -1;
6802 for (auto op : block->getOps<UseLockOp>()) {
6803 // Non-constant lock values cannot be compared here; the passes that
6804 // require a constant enforce that separately via getConstantValue().
6805 auto value = getConstantLockValue(op);
6806 if (!value)
6807 continue;
6808 if (op.acquire() || op.acquireGE()) {
6809 if (acqValue != -1 && acqValue != *value) {
6810 return failure();
6811 }
6812 acqValue = *value;
6813 } else if (op.release()) {
6814 if (relValue != -1 && relValue != *value) {
6815 return failure();
6816 }
6817 relValue = *value;
6818 }
6819 }
6820 return success();
6821 }
6822};
6823
6825 static LogicalResult verifyTrait(Operation *op) {
6826 if (auto memOp = op->getParentOfType<MemOp>()) {
6827 auto useLock = cast<UseLockOp>(op);
6828 if (auto lock = useLock.getLockOp(); lock.getTile() != memOp.getTile())
6829 return failure();
6830 }
6831 return success();
6832 }
6833};
6834
6835LogicalResult UseLockOp::verify() {
6836 // AIE.useLock cannot be used at the top level
6837 if (llvm::isa_and_nonnull<DeviceOp, ModuleOp>((*this)->getParentOp()))
6838 return (*this)->emitOpError("must be used in a core or memory operation.");
6839
6840 const auto &targetModel = getTargetModel(*this);
6841 if (targetModel.getTargetArch() == AIEArch::AIE1 && acquireGE())
6842 return (*this)->emitOpError(
6843 "AcquireGreaterEqual is not supported in AIE1.");
6844
6845 // Locks used inside a DMA/BD block are configured via static register writes
6846 // and therefore require a compile-time constant value.
6848 .succeeded() &&
6849 !getConstantLockValue(*this))
6850 return (*this)->emitOpError(
6851 "lock value in a DMA/BD block must be a compile-time constant "
6852 "(defined by an arith.constant).");
6853
6854 // Otherwise, AIE.useLock should be inside MemOp, MemTileDMAOp, or
6855 // ShimDMAOp,
6857 .succeeded()) {
6858 if (!(*this)->getBlock())
6859 return (*this)->emitOpError("is not in a block.");
6860
6861 if (targetModel.getTargetArch() == AIEArch::AIE1 &&
6862 UsesOneLockInDMABlock::verifyTrait(*this).failed())
6863 return (*this)->emitOpError(
6864 "used in a DMA block that have multiple locks.");
6865
6867 return (*this)->emitOpError("acquires/releases the lock in a DMA block "
6868 "from/to multiple states.");
6869
6870 if (HasSomeParent<MemOp>::verifyTrait(*this).succeeded() &&
6871 AccessesLocalLocks::verifyTrait(*this).failed())
6872 return (*this)->emitOpError("can only access a lock in the same tile");
6873 return success();
6874
6875 // Or it can be in a CoreOp, or some FuncOp called from a CoreOp
6876 }
6877 if (HasSomeParent<CoreOp, func::FuncOp>::verifyTrait(*this).succeeded()) {
6878 return success();
6879 }
6880 // Or it can be in a DMAConfigureTaskOp or DMAConfigureTaskForOp (for runtime
6881 // DMA configuration). Check by operation name to avoid circular dependency
6882 // with AIEX dialect
6883 {
6884 Operation *operation = (*this)->getParentOp();
6885 while (operation) {
6886 StringRef name = operation->getName().getStringRef();
6887 if (name == "aiex.dma_configure_task" ||
6888 name == "aiex.dma_configure_task_for")
6889 return success();
6890 operation = operation->getParentOp();
6891 }
6892 }
6893 return (*this)->emitOpError()
6894 << "expects some parent op to be one of "
6895 << "AIE::device, AIE::core, func::func, AIE::mem, AIE::shimDMA, or "
6896 "AIEX::dma_configure_task";
6897}
6898
6899#include "aie/Dialect/AIE/IR/AIEEnums.cpp.inc"
6900#include "aie/Dialect/AIE/IR/AIEInterfaces.cpp.inc"
6901
6902//===----------------------------------------------------------------------===//
6903// TraceEventAttr
6904//===----------------------------------------------------------------------===//
6905
6906// Custom parser for TraceEventAttr value (uses shared helper)
6907static ParseResult parseTraceEventValue(AsmParser &parser, Attribute &value) {
6908 return xilinx::AIE::parseTraceEvent(parser, value);
6909}
6910
6911// Custom printer for TraceEventAttr value (uses shared helper)
6912static void printTraceEventValue(AsmPrinter &printer, Attribute value) {
6913 xilinx::AIE::printTraceEventEnum(printer, value);
6914}
6915
6916// Custom parser for TraceEventAttr value (uses shared helper)
6917static ParseResult parseTraceRegValue(OpAsmParser &parser, Attribute &value) {
6918
6919 // Try to parse as number
6920 int64_t intValue;
6921 OptionalParseResult parseResult = parser.parseOptionalInteger(intValue);
6922 if (parseResult.has_value() && succeeded(parseResult.value())) {
6923 MLIRContext *ctx = parser.getContext();
6924 value = IntegerAttr::get(IntegerType::get(ctx, 32), intValue);
6925 return success();
6926 }
6927 return xilinx::AIE::parseTraceEvent(parser, value);
6928}
6929
6930// Custom printer for TraceEventAttr value (uses shared helper)
6931static void printTraceRegValue(OpAsmPrinter &printer, Operation *op,
6932 Attribute value) {
6933 // if it's an intattr
6934 if (auto intAttr = llvm::dyn_cast<IntegerAttr>(value)) {
6935 printer << intAttr.getInt();
6936 return;
6937 }
6938 xilinx::AIE::printTraceEventEnum(printer, value);
6939}
6940
6941// Helper to parse a LockBlocking enum keyword.
6942#define GET_OP_CLASSES
6943#include "aie/Dialect/AIE/IR/AIEOps.cpp.inc"
6944
6945FailureOr<int32_t> UseLockOp::getConstantValue() {
6946 if (auto value = getConstantLockValue(*this))
6947 return *value;
6948 return emitOpError("expected the lock value to be a compile-time constant "
6949 "(defined by an arith.constant).");
6950}
6951
6952LogicalResult UseLockOp::canonicalize(UseLockOp op, PatternRewriter &rewriter) {
6953 // An AcquireGreaterEqual by a compile-time-constant 0 is a no-op: the lock is
6954 // already >= 0 (semaphore values are non-negative) and it decrements the lock
6955 // by 0. A buffer descriptor's acquire is kept: its release needs the pair.
6956 if (op.acquireGE() && op->getBlock()->getOps<DMABDOp>().empty()) {
6957 if (auto value = getConstantLockValue(op); value && *value == 0) {
6958 rewriter.eraseOp(op);
6959 return success();
6960 }
6961 }
6962 return failure();
6963}
6964
6965TileOp SwitchboxOp::getTileOp() {
6966 return cast<TileElement>(this->getOperation()).getTileOp();
6967}
6968
6969size_t SwitchboxOp::getNumSourceConnections(WireBundle bundle) {
6970 auto tile = getTileOp();
6971 const auto &targetModel = getTargetModel(*this);
6972 return targetModel.getNumSourceSwitchboxConnections(tile.getCol(),
6973 tile.getRow(), bundle);
6974}
6975
6976size_t SwitchboxOp::getNumDestConnections(WireBundle bundle) {
6977 auto tile = getTileOp();
6978 const auto &targetModel = getTargetModel(*this);
6979 return targetModel.getNumDestSwitchboxConnections(tile.getCol(),
6980 tile.getRow(), bundle);
6981}
6982
6983TileOp ShimMuxOp::getTileOp() {
6984 return cast<TileElement>(this->getOperation()).getTileOp();
6985}
6986
6987WireBundle xilinx::AIE::getConnectingBundle(WireBundle dir) {
6988 switch (dir) {
6989 case WireBundle::North:
6990 return WireBundle::South;
6991 case WireBundle::South:
6992 return WireBundle::North;
6993 case WireBundle::East:
6994 return WireBundle::West;
6995 case WireBundle::West:
6996 return WireBundle::East;
6997 default:
6998 return dir;
6999 }
7000}
7001
7002std::optional<std::pair<TileID, Port>>
7004 TileID tile, Port port) {
7005 switch (port.bundle) {
7006 case WireBundle::North:
7007 tile.row++;
7008 break;
7009 case WireBundle::South:
7010 tile.row--;
7011 break;
7012 case WireBundle::East:
7013 tile.col++;
7014 break;
7015 case WireBundle::West:
7016 tile.col--;
7017 break;
7018 default:
7019 return std::nullopt;
7020 }
7021 if (!targetModel.isValidTile(tile))
7022 return std::nullopt;
7023 return std::pair(tile, Port{getConnectingBundle(port.bundle), port.channel});
7024}
7025
7026//===----------------------------------------------------------------------===//
7027// BDChainOp
7028//===----------------------------------------------------------------------===//
7029
7030ParseResult BDChainOp::parse(OpAsmParser &parser, OperationState &result) {
7031 SmallVector<OpAsmParser::Argument> entryArgs;
7032
7033 // Symbol name, e.g. @my_chain
7034 StringAttr symNameAttr;
7035 if (parser.parseSymbolName(symNameAttr, SymbolTable::getSymbolAttrName(),
7036 result.attributes)) {
7037 return failure();
7038 }
7039
7040 // Entry arguments (placeholders), e.g. (%addr: memref<1xi32>)
7041 ParseResult argParseResult = parser.parseCommaSeparatedList(
7042 OpAsmParser::Delimiter::Paren, [&]() -> ParseResult {
7043 OpAsmParser::Argument argument;
7044 if (parser.parseArgument(argument, true, true)) {
7045 return failure();
7046 }
7047 entryArgs.push_back(argument);
7048 return success();
7049 });
7050 if (argParseResult) {
7051 return argParseResult;
7052 }
7053
7054 // BD Chain Body
7055 auto *body = result.addRegion();
7056 ParseResult bodyParseResult = parser.parseRegion(*body, entryArgs, false);
7057 if (bodyParseResult) {
7058 return bodyParseResult;
7059 }
7060
7061 return success();
7062}
7063
7064void BDChainOp::print(OpAsmPrinter &printer) {
7065 auto taskName =
7066 (*this)
7067 ->getAttrOfType<StringAttr>(SymbolTable::getSymbolAttrName())
7068 .getValue();
7069 printer << ' ';
7070 printer.printSymbolName(taskName);
7071
7072 Region &body = getRegion();
7073 auto argsIter = body.getArguments();
7074 printer << '(';
7075 for (auto *it = argsIter.begin(); it != argsIter.end(); ++it) {
7076 if (it != argsIter.begin()) {
7077 printer << ", ";
7078 }
7079 printer.printRegionArgument(*it);
7080 }
7081 printer << ')';
7082
7083 printer << ' ';
7084 printer.printRegion(body, false, true);
7085}
7086
7087//===----------------------------------------------------------------------===//
7088// ShimDMAAllocationOp
7089//===----------------------------------------------------------------------===//
7090
7091LogicalResult ShimDMAAllocationOp::verify() {
7092 TileLike tileLike = llvm::dyn_cast<TileLike>(getTile().getDefiningOp());
7093 if (!tileLike) {
7094 return emitOpError("tile operand must implement TileLike interface");
7095 }
7096
7097 if (!tileLike.isShimNOCorPLTile()) {
7098 // if placed, provide detailed error message
7099 auto col = tileLike.tryGetCol();
7100 auto row = tileLike.tryGetRow();
7101 if (col && row) {
7102 return emitOpError("tile must be a shim tile, but got tile(")
7103 << *col << ", " << *row << ")";
7104 }
7105 return emitOpError("tile must be a shim tile");
7106 }
7107
7108 if (ArrayAttr proofs = getConsumedByAttr()) {
7109 auto device = (*this)->getParentOfType<DeviceOp>();
7110 if (failed(verifyConsumedBy(
7111 *this, proofs, getChannelDir() == DMAChannelDir::MM2S,
7112 "aie.shim_dma_allocation", [&](FlatSymbolRefAttr symbol) {
7113 auto output = getForSymbol(device, symbol.getValue());
7114 return output && output.getChannelDir() == DMAChannelDir::S2MM;
7115 })))
7116 return failure();
7117 }
7118
7119 return verifyAssignedPacket(*this, getPacketAttr());
7120}
7121
7122TileOp ShimDMAAllocationOp::getTileOp() {
7123 return dyn_cast_or_null<TileOp>(getTile().getDefiningOp());
7124}
7125
7126ShimDMAAllocationOp ShimDMAAllocationOp::getForSymbol(DeviceOp device,
7127 llvm::StringRef symbol) {
7128 Operation *maybeOp = device.lookupSymbol(symbol);
7129 if (maybeOp) {
7130 if (ShimDMAAllocationOp op = dyn_cast<ShimDMAAllocationOp>(maybeOp)) {
7131 return op;
7132 }
7133 }
7134 return nullptr;
7135}
7136
7137//===----------------------------------------------------------------------===//
7138// ObjectFifoRearmBindingOp
7139//===----------------------------------------------------------------------===//
7140
7141LogicalResult ObjectFifoRearmBindingOp::verify() {
7142 if (getChannelDirs().size() != getChannelTiles().size())
7143 return emitOpError("expected one channel_dirs entry per channel tile (")
7144 << getChannelTiles().size() << " tiles, " << getChannelDirs().size()
7145 << " dirs)";
7146 if (getChannelIndices().size() != getChannelTiles().size())
7147 return emitOpError("expected one channel_indices entry per channel tile (")
7148 << getChannelTiles().size() << " tiles, "
7149 << getChannelIndices().size() << " indices)";
7150 if (getLockInits().size() != getLocks().size())
7151 return emitOpError("expected one lock_inits entry per lock (")
7152 << getLocks().size() << " locks, " << getLockInits().size()
7153 << " inits)";
7154 // head_bd_ids / repeat_counts are optional (populated by
7155 // --aie-assign-bd-ids), but if present they are one-per-channel and travel as
7156 // a pair: the re-push needs both, so a binding carrying one without the other
7157 // is malformed.
7158 if (getHeadBdIds().has_value() != getRepeatCounts().has_value())
7159 return emitOpError("head_bd_ids and repeat_counts must both be set or both "
7160 "be absent");
7161 if (auto headBdIds = getHeadBdIds();
7162 headBdIds.has_value() && headBdIds->size() != getChannelTiles().size())
7163 return emitOpError("expected one head_bd_ids entry per channel tile (")
7164 << getChannelTiles().size() << " tiles, " << headBdIds->size()
7165 << " ids)";
7166 if (auto repeatCounts = getRepeatCounts();
7167 repeatCounts.has_value() &&
7168 repeatCounts->size() != getChannelTiles().size())
7169 return emitOpError("expected one repeat_counts entry per channel tile (")
7170 << getChannelTiles().size() << " tiles, " << repeatCounts->size()
7171 << " counts)";
7172 for (int32_t dir : getChannelDirs())
7173 if (dir != 0 && dir != 1)
7174 return emitOpError("channel_dirs entries must be 0 (S2MM) or 1 (MM2S), "
7175 "got ")
7176 << dir;
7177 // The lowering resolves each operand to its aie.tile / aie.lock, so reject a
7178 // binding whose operands are not those (otherwise the lowering would cast a
7179 // non-lock/non-tile operand and abort).
7180 for (Value tile : getChannelTiles()) {
7181 auto tileOp = tile.getDefiningOp<TileOp>();
7182 if (!tileOp)
7183 return emitOpError("channel_tiles operands must be aie.tile values");
7184 // A shim DMA endpoint is host-managed: the host re-pushes its BD program
7185 // and re-arms its locks. Re-arming it here would fight the host, so a
7186 // re-arm binding only ever records non-shim channels -- reject a shim tile
7187 // even in a hand-authored binding.
7188 if (tileOp.isShimTile())
7189 return emitOpError("channel_tiles must be non-shim tiles; a shim DMA "
7190 "endpoint is host-managed and is not re-armed here");
7191 }
7192 for (Value lock : getLocks())
7193 if (!lock.getDefiningOp<LockOp>())
7194 return emitOpError("locks operands must be aie.lock values");
7195 // The re-push starts each channel on its head BD, which must be in the
7196 // channel's BD pool.
7197 if (std::optional<ArrayRef<int32_t>> headBdIds = getHeadBdIds()) {
7198 const auto &targetModel = getTargetModel(getOperation());
7199 for (auto [tile, channel, bdId] :
7200 llvm::zip(getChannelTiles(), getChannelIndices(), *headBdIds)) {
7201 auto tileOp = tile.getDefiningOp<TileOp>();
7202 if (bdId < 0 || !targetModel.isBdChannelAccessible(
7203 tileOp.getCol(), tileOp.getRow(), bdId, channel))
7204 return emitOpError("head BD ")
7205 << bdId << " is not accessible from channel " << channel
7206 << " of tile (" << tileOp.getCol() << ", " << tileOp.getRow()
7207 << ")";
7208 }
7209 }
7210 return success();
7211}
7212
7213//===----------------------------------------------------------------------===//
7214// RuntimeSequenceOp
7215//===----------------------------------------------------------------------===//
7216
7217ParseResult RuntimeSequenceOp::parse(OpAsmParser &parser,
7218 OperationState &result) {
7219
7220 // Name of this runtime sequence
7221 StringAttr nameAttr;
7222 (void)parser.parseOptionalSymbolName(
7223 nameAttr, mlir::SymbolTable::getSymbolAttrName(), result.attributes);
7224
7225 SmallVector<OpAsmParser::Argument> entryArgs;
7226
7227 // Entry arguments, e.g. (%addr: memref<1xi32>)
7228 ParseResult argParseResult = parser.parseCommaSeparatedList(
7229 OpAsmParser::Delimiter::Paren, [&]() -> ParseResult {
7230 OpAsmParser::Argument argument;
7231 if (parser.parseArgument(argument, true, true)) {
7232 return failure();
7233 }
7234 entryArgs.push_back(argument);
7235 return success();
7236 });
7237 if (argParseResult) {
7238 return argParseResult;
7239 }
7240
7241 // Optional `attributes { ... }` clause (before the body so it does not
7242 // conflict with the `{` that opens the region).
7243 if (parser.parseOptionalAttrDictWithKeyword(result.attributes))
7244 return failure();
7245
7246 // Body
7247 auto *body = result.addRegion();
7248 ParseResult bodyParseResult = parser.parseRegion(*body, entryArgs, false);
7249 if (bodyParseResult) {
7250 return bodyParseResult;
7251 }
7252
7253 return success();
7254}
7255
7256void RuntimeSequenceOp::print(OpAsmPrinter &printer) {
7257 Region &body = getRegion();
7258
7259 auto nameAttr = (*this)->getAttrOfType<StringAttr>(
7260 mlir::SymbolTable::getSymbolAttrName());
7261 if (nameAttr &&
7262 nameAttr != ::mlir::OpBuilder((*this)->getContext())
7263 .getStringAttr(getDefaultRuntimeSequenceName())) {
7264 printer << ' ';
7265 printer.printSymbolName(nameAttr);
7266 }
7267
7268 printer << '(';
7269 for (unsigned i = 0, n = body.getNumArguments(); i < n; i++) {
7270 if (i > 0) {
7271 printer << ", ";
7272 }
7273 printer.printRegionArgument(body.getArgument(i));
7274 }
7275 printer << ')';
7276
7277 printer.printOptionalAttrDictWithKeyword(
7278 (*this)->getAttrs(),
7279 /*elidedAttrs=*/{mlir::SymbolTable::getSymbolAttrName()});
7280
7281 printer << ' ';
7282 printer.printRegion(body, false, true);
7283}
7284
7285LogicalResult RuntimeSequenceOp::verify() {
7286 DeviceOp device = (*this)->getParentOfType<DeviceOp>();
7287 if (!device) {
7288 // this check is redudnant with the HasParent trait, but can't hurt
7289 (*this)->emitOpError() << "must be inside AIE device operation.";
7290 return failure();
7291 }
7292 return success();
7293}
7294
7295RuntimeSequenceOp
7296RuntimeSequenceOp::getForSymbolInDevice(DeviceOp deviceOp,
7297 llvm::StringRef symbol) {
7298 RuntimeSequenceOp runtimeSequenceOp;
7299 if (symbol.empty()) {
7300 auto range = deviceOp.getOps<RuntimeSequenceOp>();
7301 if (range.begin() == range.end()) {
7302 // No runtime sequence in the device; let the caller emit a diagnostic
7303 // rather than dereferencing an end iterator (which crashes).
7304 return nullptr;
7305 }
7306 runtimeSequenceOp = *range.begin();
7307 } else {
7308 Operation *maybeRuntimeSequenceOp =
7309 mlir::SymbolTable::lookupSymbolIn(deviceOp, symbol);
7310 if (!maybeRuntimeSequenceOp) {
7311 return nullptr;
7312 }
7313 runtimeSequenceOp =
7314 llvm::dyn_cast<RuntimeSequenceOp>(maybeRuntimeSequenceOp);
7315 }
7316 return runtimeSequenceOp;
7317}
7318
7319RuntimeSequenceOp
7320RuntimeSequenceOp::getForSymbolInDeviceOrError(DeviceOp deviceOp,
7321 llvm::StringRef symbol) {
7322 RuntimeSequenceOp runtimeSequenceOp = getForSymbolInDevice(deviceOp, symbol);
7323 if (!runtimeSequenceOp) {
7324 if (!symbol.empty()) {
7325 deviceOp.emitError("No such runtime sequence: ") << symbol;
7326 } else {
7327 deviceOp.emitError("No runtime sequence in device");
7328 }
7329 }
7330 return runtimeSequenceOp;
7331}
7332
7333LogicalResult RuntimeSequenceOp::verifyBeforeMaterialization() {
7334 // Check that all symbol references within the runtime sequence
7335 // are either to ShimDMAAllocationOp, DeviceOp or another RuntimeSequenceOp;
7336 // these are the only symbols that can be lowered with the NPU passes
7337 auto result = (*this)->walk([&](Operation *op) {
7338 for (NamedAttribute namedAttr : op->getAttrs()) {
7339 Attribute attr = namedAttr.getValue();
7340 auto walkResult = attr.walk([&](SymbolRefAttr symbolRef) {
7341 Operation *symbolDefOp =
7342 SymbolTable::lookupNearestSymbolFrom(*this, symbolRef);
7343 if (!symbolDefOp) {
7344 if (auto flat = dyn_cast<FlatSymbolRefAttr>(symbolRef)) {
7345 symbolDefOp = lookupNamedOp(*this, flat.getAttr());
7346 }
7347 }
7348 if (symbolDefOp) {
7349 if (!llvm::isa<ShimDMAAllocationOp>(symbolDefOp) &&
7350 !llvm::isa<DeviceOp>(symbolDefOp) &&
7351 !llvm::isa<RuntimeSequenceOp>(symbolDefOp) &&
7352 !llvm::isa<BufferOp>(symbolDefOp) &&
7353 !llvm::isa<ObjectFifoRearmBindingOp>(symbolDefOp) &&
7354 !llvm::isa<memref::GlobalOp>(symbolDefOp)) {
7355 op->emitOpError()
7356 << "references symbol '"
7357 << symbolRef.getRootReference().getValue()
7358 << "' which must be either a ShimDMAAllocationOp, DeviceOp, "
7359 "RuntimeSequenceOp, BufferOp, ObjectFifoRearmBindingOp, or "
7360 "GlobalOp, but got: "
7361 << symbolDefOp->getName().getStringRef();
7362 return WalkResult::interrupt();
7363 }
7364 if (BufferOp bufferOp = llvm::dyn_cast<BufferOp>(symbolDefOp)) {
7365 if (!bufferOp.getAddress()) {
7366 op->emitOpError()
7367 << "Unallocated buffer; fixed addresses are required before "
7368 "runtime sequence materialization.";
7369 return WalkResult::interrupt();
7370 }
7371 }
7372 }
7373 return WalkResult::advance();
7374 });
7375 if (walkResult.wasInterrupted()) {
7376 return WalkResult::interrupt();
7377 }
7378 }
7379 return WalkResult::advance();
7380 });
7381
7382 if (result.wasInterrupted()) {
7383 return failure();
7384 }
7385
7386 return success();
7387}
7388
7389// Include implementations for custom attributes
7390#define GET_ATTRDEF_CLASSES
7391#include "aie/Dialect/AIE/IR/AIEAttrs.cpp.inc"
7392
7393bool xilinx::AIE::hasCoreAccess(DeviceOp device, Value tile,
7394 ObjectFifoCreateOp objFifo,
7395 ObjectFifoPort port) {
7396 for (auto coreOp : device.getOps<CoreOp>()) {
7397 if (coreOp.getTile() != tile) {
7398 continue;
7399 }
7400 bool found = false;
7401 coreOp.walk([&](Operation *op) {
7402 if (auto acq = dyn_cast<ObjectFifoAcquireOp>(op)) {
7403 found |= acq.getObjectFifo() == objFifo && acq.getPort() == port;
7404 } else if (auto rel = dyn_cast<ObjectFifoReleaseOp>(op)) {
7405 found |= rel.getObjectFifo() == objFifo && rel.getPort() == port;
7406 }
7407 });
7408 if (found) {
7409 return true;
7410 }
7411 }
7412 return false;
7413}
virtual uint64_t getDmaBdMaxLen(AIETileType tileType) const =0
Return the maximum transfer length, measured in address-generation granules (see getAddressGenGranula...
virtual const DmaBdLayout * getDmaBdLayout(AIETileType tileType) const
Return the register layout of a buffer descriptor on the given tile type, or null if the target has n...
virtual AIEArch getTargetArch() const =0
Return the target architecture.
virtual uint32_t getNumSourceShimMuxConnections(int col, int row, WireBundle bundle) const =0
Return the number of sources of connections inside a shimmux.
virtual uint32_t getDmaBdWrapBits(AIETileType tileType) const =0
Return the bit width of the wrap (size) field in a DMA buffer descriptor for the given tile type.
virtual uint32_t getLocalMemorySize() const =0
Return the size (in bytes) of the local data memory of a core.
virtual uint32_t getNumLocks(AIETileType tileType) const =0
Return the number of lock objects for a given tile type.
virtual uint32_t getDmaBdIterBits(AIETileType tileType) const =0
Return the bit width of the iteration (repeat) wrap/stride fields in a DMA buffer descriptor for the ...
virtual uint32_t getBDMaxDims(AIETileType tileType) const =0
Return the maximum number of addressing dimensions (wraps/strides) a single buffer descriptor support...
virtual std::vector< std::pair< uint32_t, uint32_t > > getShimBurstEncodingsAndLengths() const =0
virtual bool isLegalTileConnection(int col, int row, WireBundle srcBundle, int srcChan, WireBundle dstBundle, int dstChan) const =0
virtual uint32_t getNumBanks(int col, int row) const =0
Return the number of memory banks of a given tile.
virtual uint32_t getDmaTaskQueueDepth() const =0
Return how many DMA tasks a single channel's task queue holds (unsupported = 0).
virtual uint32_t getDmaBdStepBits(AIETileType tileType) const =0
Return the bit width of the step (stride) field in a DMA buffer descriptor for the given tile type.
virtual bool isBdChannelAccessible(int col, int row, uint32_t bd_id, int channel) const =0
Return true iff buffer descriptor bd_id on tile (col, row) can be submitted on channel channel.
virtual AIETileType getTileType(int col, int row) const =0
Return the tile type for the given tile coordinates.
virtual bool isValidTile(TileID src) const
Return true if the given tile ID is valid.
virtual uint32_t getMaxRepeatCount() const =0
Return the largest DMA task repeat count (unsupported = 0).
virtual int rows() const =0
Return the number of rows in the device.
bool isShimNOCTile(int col, int row) const
Return true if the given tile is a ShimNOC tile.
virtual uint32_t getNumBDs(AIETileType tileType) const =0
Return the number of buffer descriptors for a given tile type.
virtual uint32_t getMaxLockValue() const =0
Return the maximum value that can be stored in a lock register.
virtual uint32_t getMaxOutOfOrderId() const =0
Return the largest out-of-order BD id (unsupported = 0).
bool hasProperty(ModelProperty Prop) const
virtual int columns() const =0
Return the number of columns in the device.
virtual uint32_t getNumDestShimMuxConnections(int col, int row, WireBundle bundle) const =0
Return the number of destinations of connections inside a shimmux.
virtual uint32_t getNumDestSwitchboxConnections(int col, int row, WireBundle bundle) const =0
Return the number of destinations of connections inside a switchbox.
SharedMemory
Which of two tiles' memory modules both of them can address.
@ None
Neither tile can address the other's memory module.
SharedMemory getSharedMemory(TileID a, TileID b) const
Return the memory module a and b can both reach, if any.
virtual uint32_t getAddressGenGranularity() const =0
Return the data bus width of the device.
virtual uint32_t getMemTileSize() const =0
Return the size (in bytes) of a MemTile.
virtual uint32_t getNumSourceSwitchboxConnections(int col, int row, WireBundle bundle) const =0
Return the number of sources of connections inside a switchbox.
NamedOpTable(mlir::Operation *symbolTableOp)
std::string uniqueName(llvm::StringRef prefix, unsigned &counter) const
bool isContiguousBDTransfer(llvm::ArrayRef< BDDimLayoutAttr > dims)
mlir::ParseResult parseFlowVias(mlir::OpAsmParser &parser, llvm::SmallVectorImpl< mlir::OpAsmParser::UnresolvedOperand > &vias, mlir::DenseI32ArrayAttr &ingressBundles, mlir::DenseI32ArrayAttr &ingressChannels, mlir::DenseI32ArrayAttr &egressBundles, mlir::DenseI32ArrayAttr &egressChannels)
void printObjectFifoProducerTile(mlir::OpAsmPrinter &printer, mlir::Operation *op, mlir::Value operand, BDDimLayoutArrayAttr dimensions)
void printTypedDynamicIndexList(mlir::OpAsmPrinter &printer, mlir::Operation *op, mlir::OperandRange values, llvm::ArrayRef< int64_t > integers, mlir::TypeRange types)
llvm::SmallVector< ObjectFifoDmaEndpointOp > turnEndpoints(ObjectFifoDmaEndpointOp end)
llvm::function_ref< std::optional< TileID >(mlir::Value)> TilePositions
Definition AIEDialect.h:422
std::optional< llvm::SmallVector< BDDimLayoutAttr > > normalizedBDDims(llvm::ArrayRef< BDDimLayoutAttr > dims, int64_t len)
bool hasCoreAccess(DeviceOp device, mlir::Value tile, ObjectFifoCreateOp fifo, ObjectFifoPort port)
mlir::LogicalResult verifyLengthParameterTile(mlir::Operation *op, const AIETargetModel &targetModel, int col, int row)
uint32_t getShimBurstLengthBytes(const AIE::AIETargetModel &tm, uint32_t burstLength)
void collectTiles(DeviceOp &device, llvm::DenseMap< TileID, mlir::Operation * > &tiles)
int64_t getMemRefSizeInBytes(mlir::MemRefType type, const mlir::DataLayout &layout)
llvm::SmallVector< int, 2 > banksFromMemorySpace(mlir::Type type)
void collectBuffers(DeviceOp &device, llvm::DenseMap< mlir::Operation *, llvm::SmallVector< BufferOp, 4 > > &buffers)
mlir::ParseResult parseTraceEvent(mlir::AsmParser &parser, mlir::Attribute &result)
void printObjectFifoAcquireObjects(mlir::OpAsmPrinter &printer, mlir::Operation *op, ObjectFifoPortAttr port, mlir::TypeRange objects)
mlir::LogicalResult verifyBdLockPair(mlir::Block &block, bool outOfOrder, UseLockOp &acquire, UseLockOp &release)
mlir::LogicalResult verifyAssignedPacket(mlir::Operation *op, PacketInfoAttr packet, llvm::StringRef what="packet")
mlir::ParseResult parseTypedDynamicIndexList(mlir::OpAsmParser &parser, llvm::SmallVectorImpl< mlir::OpAsmParser::UnresolvedOperand > &values, mlir::DenseI64ArrayAttr &integers, llvm::SmallVectorImpl< mlir::Type > &types)
void printObjectFifoConsumerTiles(mlir::OpAsmPrinter &printer, mlir::Operation *op, mlir::OperandRange tiles, BDDimLayoutArrayArrayAttr dimsPerTileAttr)
mlir::Operation * lookupNamedOpIn(mlir::Operation *symbolTableOp, mlir::StringAttr name)
uint32_t getShimBurstLengthEncoding(const AIE::AIETargetModel &tm, uint32_t burstLength)
mlir::LogicalResult verifyDMABDOutOfOrderId(DMABDOp bd, bool packetEnabledByContext=false)
mlir::ParseResult parseObjectFifoAcquireObjects(mlir::OpAsmParser &parser, ObjectFifoPortAttr &port, llvm::SmallVectorImpl< mlir::Type > &objects)
mlir::LogicalResult verifyOutOfOrderChannel(mlir::Operation *op, DMAChannelDir dir, bool outOfOrder, llvm::ArrayRef< DMABDOp > bds, bool packetEnabledByContext=false)
struct Connect { Port src; Port dst; bool operator==(const Connect &rhs) const { return std::tie(src, dst)==std::tie(rhs.src, rhs.dst); } bool operator!=(const Connect &rhs) const { return !(*this==rhs);} bool operator<(const Connect &rhs) const { return std::tie(src, dst)< std::tie(rhs.src, rhs.dst); } } Connect
Definition AIEDialect.h:299
void printDMAStartChannel(mlir::OpAsmPrinter &printer, mlir::Operation *op, mlir::Attribute channel)
mlir::LogicalResult verifyLengthParameter(mlir::Operation *op, std::optional< int64_t > lengthUnit, mlir::BaseMemRefType buffer, std::optional< int64_t > lenElems, bool contiguous, llvm::ArrayRef< int64_t > innerSizes)
mlir::LogicalResult verifyDMAChannelIndex(mlir::Operation *op, TileLike tile, DMAChannelDir dir, int64_t channel)
struct TileID { friend std::ostream &operator<<(std::ostream &os, const TileID &s) { os<< "TileID("<< s.col<< ", "<< s.row<< ")"; return os; } friend std::string to_string(const TileID &s) { std::ostringstream ss; ss<< s; return ss.str(); } friend llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const TileID &s) { os<< to_string(s); return os; } inline bool operator<(const TileID &rhs) const { return std::tie(col, row)< std::tie(rhs.col, rhs.row); } bool operator==(const TileID &rhs) const { return std::tie(col, row)==std::tie(rhs.col, rhs.row); } bool operator!=(const TileID &rhs) const { return !(*this==rhs);} int col, row;} TileID
int32_t getBufferBaseAddress(mlir::Operation *bufOp)
mlir::ParseResult parseDMAStartChannel(mlir::OpAsmParser &parser, mlir::Attribute &channel)
llvm::SmallVector< uint32_t > getAssignedBdIds(DmaBody program)
llvm::SmallVector< StreamWindowStep > streamWindowSteps(const StreamWindow &w)
int64_t turnPasses(llvm::ArrayRef< std::pair< int64_t, int64_t > > ends)
const AIETargetModel & getTargetModel(mlir::Operation *op)
bool keepsPktHeader(TileID tile, Port master, std::optional< bool > keep)
struct DMAChannel { DMAChannelDir direction; int channel; bool operator==(const DMAChannel &rhs) const { return std::tie(direction, channel)==std::tie(rhs.direction, rhs.channel); } } DMAChannel
Definition AIEDialect.h:314
mlir::LogicalResult verifyDMAChannelsResolved(DeviceOp device)
mlir::Operation * lookupNamedOp(mlir::Operation *from, mlir::StringAttr name)
mlir::ParseResult parseObjectFifoProducerTile(mlir::OpAsmParser &parser, mlir::OpAsmParser::UnresolvedOperand &operand, BDDimLayoutArrayAttr &dimensions)
void printTraceEventEnum(mlir::AsmPrinter &printer, mlir::Attribute attr)
void placeRuntimeLengthDimension(llvm::MutableArrayRef< int64_t > sizes, llvm::MutableArrayRef< int64_t > strides)
mlir::LogicalResult myVerifyOffsetSizeAndStrideOp(mlir::OffsetSizeAndStrideOpInterface op)
mlir::LogicalResult inferStreamWindow(llvm::ArrayRef< BDDimLayoutAttr > dims, llvm::ArrayRef< BDPadLayoutAttr > pads, int64_t objectSize, llvm::function_ref< mlir::InFlightDiagnostic()> emitError, StreamWindow &window)
mlir::LogicalResult verifyDMABDIteration(DMABDOp bd, AIETileType tileType)
void printFlowVias(mlir::OpAsmPrinter &printer, mlir::Operation *op, mlir::OperandRange vias, mlir::DenseI32ArrayAttr ingressBundles, mlir::DenseI32ArrayAttr ingressChannels, mlir::DenseI32ArrayAttr egressBundles, mlir::DenseI32ArrayAttr egressChannels)
struct Port { WireBundle bundle; int channel; bool operator==(const Port &rhs) const { return std::tie(bundle, channel)==std::tie(rhs.bundle, rhs.channel); } bool operator!=(const Port &rhs) const { return !(*this==rhs);} bool operator<(const Port &rhs) const { return std::tie(bundle, channel)< std::tie(rhs.bundle, rhs.channel); } friend std::ostream &operator<<(std::ostream &os, const Port &port) { os<< "("; os<< stringifyWireBundle(port.bundle).str(); os<< ": "<< std::to_string(port.channel)<< ")"; return os; } friend llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const Port &port) { os<< to_string(port); return os; } } Port
Definition AIEDialect.h:265
mlir::ParseResult parseObjectFifoReleaseCount(mlir::OpAsmParser &parser, ObjectFifoPortAttr &port, mlir::IntegerAttr &size)
AIETargetModel::SharedMemory sharedMemory(TileOp a, TileOp b)
Which of a's and b's memory modules both tiles can address.
std::optional< std::pair< TileID, Port > > getConnectingWireEndpoint(const AIETargetModel &targetModel, TileID tile, Port port)
mlir::FailureOr< int64_t > getLengthUnitBytes(mlir::Operation *op, int64_t lengthUnit, mlir::BaseMemRefType buffer)
int64_t paddedSize(llvm::ArrayRef< BDDimLayoutAttr > dims, llvm::ArrayRef< BDPadLayoutAttr > pads, int64_t fallback)
mlir::ParseResult parseObjectFifoConsumerTiles(mlir::OpAsmParser &parser, llvm::SmallVectorImpl< mlir::OpAsmParser::UnresolvedOperand > &tiles, BDDimLayoutArrayArrayAttr &dimensions)
void printObjectFifoReleaseCount(mlir::OpAsmPrinter &printer, mlir::Operation *op, ObjectFifoPortAttr port, mlir::IntegerAttr size)
WireBundle getConnectingBundle(WireBundle dir)
static LogicalResult verifyTrait(Operation *op)
static LogicalResult verifyTrait(Operation *op)
static LogicalResult verifyTrait(Operation *op)
static LogicalResult verifyTrait(Operation *op)
static LogicalResult verifyTrait(Operation *op)
The register layout of one tile type's DMA buffer descriptor.
llvm::SmallVector< std::pair< DmaBdField, DmaBdField >, 3 > padFields() const
The (before, after) zero-padding fields, innermost dimension first.
static mlir::LogicalResult verifyTrait(mlir::Operation *op)
static mlir::LogicalResult verifyTrait(mlir::Operation *op)
int64_t depthAt(const AIETargetModel &target, TilePositions positions) const
llvm::SmallVector< LinkPoolSharer > sharers
Definition AIEDialect.h:439
int64_t end() const
llvm::SmallVector< BDDimLayoutAttr > inner
Definition AIEDialect.h:414