11#include "mlir/Dialect/Arith/IR/Arith.h"
12#include "mlir/Dialect/Func/IR/FuncOps.h"
13#include "mlir/Dialect/MemRef/IR/MemRef.h"
14#include "mlir/Dialect/Utils/StaticValueUtils.h"
15#include "mlir/IR/DialectImplementation.h"
16#include "mlir/IR/Matchers.h"
17#include "mlir/IR/OpDefinition.h"
18#include "mlir/IR/SymbolTable.h"
19#include "mlir/Interfaces/FoldInterfaces.h"
20#include "mlir/Interfaces/ViewLikeInterface.h"
21#include "mlir/Transforms/InliningUtils.h"
22#include "llvm/ADT/STLExtras.h"
24#include "llvm/ADT/SmallPtrSet.h"
25#include "llvm/ADT/SmallSet.h"
26#include "llvm/ADT/TypeSwitch.h"
27#include "llvm/Support/CheckedArithmetic.h"
28#include "llvm/Support/MathExtras.h"
44#include "aie/Dialect/AIE/IR/AIEDialect.cpp.inc"
46#define GET_TYPEDEF_CLASSES
47#include "aie/Dialect/AIE/IR/AIETypes.cpp.inc"
51struct AIEInlinerInterface : DialectInlinerInterface {
52 using DialectInlinerInterface::DialectInlinerInterface;
55 bool isLegalToInline(Region *dest, Region *src,
bool wouldBeCloned,
56 IRMapping &valueMapping)
const final {
62 bool isLegalToInline(Operation *op, Region *,
bool wouldBeCloned,
63 IRMapping &)
const final {
69 void handleTerminator(Operation *op, Block *newDest)
const final {}
73 void handleTerminator(Operation *op, ValueRange valuesToRepl)
const final {}
76struct AIEDialectFoldInterface : DialectFoldInterface {
77 using DialectFoldInterface::DialectFoldInterface;
82 bool shouldMaterializeInto(Region *region)
const final {
95 return isa<CoreOp, RuntimeSequenceOp, MemOp, MemTileDMAOp, ShimDMAOp>(
96 region->getParentOp());
102void AIEDialect::initialize() {
104#define GET_TYPEDEF_LIST
105#include "aie/Dialect/AIE/IR/AIETypes.cpp.inc"
108#define GET_ATTRDEF_LIST
109#include "aie/Dialect/AIE/IR/AIEAttrs.cpp.inc"
113#include "aie/Dialect/AIE/IR/AIEOps.cpp.inc"
115 addInterfaces<AIEInlinerInterface, AIEDialectFoldInterface>();
122static std::pair<uint32_t, uint32_t>
124 uint32_t burstLength) {
126 std::vector<std::pair<uint32_t, uint32_t>> bel =
131 if (burstLength == 0) {
132 return *llvm::max_element(bel, [](
auto pair1,
auto pair2) {
133 return pair1.second < pair2.second;
139 return *llvm::find_if(bel, [=](
auto p) {
return p.second == burstLength; });
143 uint32_t burstLength) {
145 return getShimBurstLength(tm, burstLength).second;
149 uint32_t burstLength) {
151 return getShimBurstLength(tm, burstLength).first;
156 if (!symbolTableOp->hasTrait<mlir::OpTrait::SymbolTable>() ||
157 symbolTableOp->getRegion(0).empty()) {
162 for (Operation &op : symbolTableOp->getRegion(0).front()) {
163 if (op.getAttrOfType<StringAttr>(mlir::SymbolTable::getSymbolAttrName()) ==
172 : context(symbolTableOp->getContext()) {
173 if (!symbolTableOp->hasTrait<mlir::OpTrait::SymbolTable>() ||
174 symbolTableOp->getRegion(0).empty()) {
177 for (Operation &op : symbolTableOp->getRegion(0).front()) {
178 if (auto name = op.getAttrOfType<StringAttr>(
179 mlir::SymbolTable::getSymbolAttrName())) {
180 ops.try_emplace(name, &op);
188 StringAttr::get(symbolTableOp->getContext(), name));
192 Operation *symbolTableOp = mlir::SymbolTable::getNearestSymbolTable(from);
197 return lookupNamedOp(from, StringAttr::get(from->getContext(), name));
201 unsigned &counter)
const {
204 name = (prefix + llvm::Twine(counter++)).str();
205 }
while (ops.count(StringAttr::get(context, name)));
211 std::array<unsigned, 3> maxRanks = op.getArrayAttrMaxRanks();
217 if (failed(verifyListOfOperandsOrIntegers(
218 op,
"offset", maxRanks[0], op.getStaticOffsets(), op.getOffsets())))
220 if (failed(verifyListOfOperandsOrIntegers(
221 op,
"size", maxRanks[1], op.getStaticSizes(), op.getSizes())))
223 if (failed(verifyListOfOperandsOrIntegers(
224 op,
"stride", maxRanks[2], op.getStaticStrides(), op.getStrides())))
227 const size_t numMixedOffsets = op.getMixedOffsets().size();
228 const size_t numMixedSizes = op.getMixedSizes().size();
229 if (!(numMixedOffsets == 1 && maxRanks[0] == 1) &&
230 numMixedOffsets != numMixedSizes)
231 return op->emitError(
232 "expected mixed offsets rank to match mixed sizes rank (")
233 << numMixedOffsets <<
" vs " << numMixedSizes
234 <<
") so the rank of the result type is well-formed.";
235 for (int64_t offset : op.getStaticOffsets())
236 if (offset < 0 && !ShapedType::isDynamic(offset))
237 return op->emitError(
"expected offsets to be non-negative, but got ")
239 for (int64_t size : op.getStaticSizes())
240 if (size < 0 && !ShapedType::isDynamic(size))
241 return op->emitError(
"expected sizes to be non-negative, but got ")
265 if (
auto t = dyn_cast<AIETarget>(op))
266 return t.getTargetModel();
267 if (
auto t = op->getParentOfType<AIETarget>())
268 return t.getTargetModel();
282 case AIEDevice::xcvc1902:
284 case AIEDevice::xcve2302:
286 case AIEDevice::xcve2802:
288 case AIEDevice::npu1:
290 case AIEDevice::npu1_1col:
292 case AIEDevice::npu1_2col:
294 case AIEDevice::npu1_3col:
296 case AIEDevice::npu2:
298 case AIEDevice::npu2_1col:
299 return NPU2model1col;
300 case AIEDevice::npu2_2col:
301 return NPU2model2col;
302 case AIEDevice::npu2_3col:
303 return NPU2model3col;
304 case AIEDevice::npu2_4col:
305 return NPU2model4col;
306 case AIEDevice::npu2_5col:
307 return NPU2model5col;
308 case AIEDevice::npu2_6col:
309 return NPU2model6col;
310 case AIEDevice::npu2_7col:
311 return NPU2model7col;
312 case AIEDevice::xcve3858:
318 llvm::report_fatal_error(
"getTargetModel: unknown AIEDevice value " +
319 llvm::Twine(
static_cast<uint32_t
>(device)));
324static TileElement getParentTileElement(Operation *op) {
325 auto *parent = op->getParentOp();
326 while (!llvm::isa_and_nonnull<DeviceOp, ModuleOp>(parent)) {
327 if (
auto element = llvm::dyn_cast<TileElement>(parent))
329 parent = parent->getParentOp();
331 return llvm::dyn_cast<TileElement>(parent);
335static int64_t getDimsMaxIdx(ArrayRef<BDDimLayoutAttr> dims) {
337 for (BDDimLayoutAttr dim : dims) {
338 maxIdx += dim.getStride() * (dim.getSize() - 1);
344 int64_t elements = 1;
345 for (BDDimLayoutAttr dim :
inner)
346 elements *= dim.getSize();
351 ArrayRef<BDDimLayoutAttr> dims, ArrayRef<BDPadLayoutAttr> pads,
352 int64_t objectSize, function_ref<InFlightDiagnostic()> emitError,
355 return emitError() <<
"a walk over a stream needs an outer dimension of "
356 "outputs and at least one inner dimension";
359 w.
outputs = dims.front().getSize();
360 int64_t outerStride = dims.front().getStride();
361 if (outerStride % objectSize)
362 return emitError() <<
"a walk over a stream steps whole objects between "
363 "outputs, but its outer stride "
364 << outerStride <<
" is not a multiple of the "
365 << objectSize <<
"-element object";
366 w.
advance = outerStride / objectSize;
367 w.
inner.assign(dims.begin() + 1, dims.end());
370 for (
auto [index, dim] : llvm::enumerate(w.
inner)) {
371 if (dim.getSize() > 1 && dim.getStride() >= objectSize) {
372 if (w.
rowDim >= 0 || dim.getStride() != objectSize)
373 return emitError() <<
"a walk over a stream may step one inner "
374 "dimension by exactly one "
375 << objectSize <<
"-element object";
379 reach += dim.getStride() * (dim.getSize() - 1);
381 if (reach >= objectSize)
382 return emitError() <<
"a walk over a stream reads " << reach + 1
383 <<
" elements of each object, past the " << objectSize
384 <<
"-element object";
387 for (
auto [index, pad] : llvm::enumerate(pads)) {
389 w.
top = pad.getConstPadBefore();
390 w.
bottom = pad.getConstPadAfter();
391 }
else if (pad.getConstPadBefore() || pad.getConstPadAfter()) {
392 return emitError() <<
"a walk over a stream pads only its outer "
393 "dimension, with the objects before and after "
399 return emitError() <<
"a walk over a stream of " << w.
objects
400 <<
" objects with " << w.
top <<
" and " << w.
bottom
401 <<
" padding objects leaves an output without data";
406SmallVector<StreamWindowStep>
408 SmallVector<StreamWindowStep> steps;
409 int64_t acquired = 0, released = 0;
410 for (int64_t j = 0; j < w.
outputs; j++) {
412 int64_t newest = std::max(
413 acquired, start + std::clamp<int64_t>(w.
objects - start, 0, w.
span));
416 ? std::clamp<int64_t>((j + 1) * w.
advance - w.
top, 0, newest)
418 next = std::max(released, next);
419 steps.push_back({j * w.
advance, newest - acquired, next - released});
428struct UsesAreAccessible {
429 static LogicalResult verifyTrait(Operation *op) {
430 auto thisElement = cast<TileElement>(op);
434 if (!isa<TileOp>(thisElement.getTile().getDefiningOp()))
437 auto thisID = thisElement.getTileID();
438 auto users = op->getResult(0).getUsers();
440 for (
auto *user : users) {
443 if (llvm::isa_and_nonnull<DeviceOp, ModuleOp>(user->getParentOp())) {
452 TileElement element = llvm::dyn_cast<TileElement>(user);
454 element = getParentTileElement(user);
458 return op->emitOpError(
"is accessed outside of a tile")
459 .attachNote(user->getLoc())
462 auto tileID = element.getTileID();
463 if (!targetModel.isLegalMemAffinity(tileID.col, tileID.row, thisID.col,
465 return (op->emitOpError(
"in Column ")
466 << thisID.col <<
" and Row " << thisID.row
467 <<
" is accessed from an unreachable tile in Column "
468 << tileID.col <<
" and Row " << tileID.row)
469 .attachNote(user->getLoc())
481template <
typename... TerminatorOpTypes>
484 for (
auto ®ion : op->getRegions()) {
485 for (
auto &block : region) {
486 if (!block.empty()) {
487 if (Operation *operation = &block.back();
488 !llvm::isa_and_nonnull<TerminatorOpTypes...>(operation))
489 return operation->emitOpError(
"is not an allowed terminator")
490 .attachNote(op->getLoc())
491 .append(
"in this context: ");
501template <
typename ConcreteType>
503 auto element = cast<ConcreteType>(op);
506 TileLike tile = element.getTileLike();
508 return op->emitOpError(
"tile must implement TileLike interface");
510 int bdMax = targetModel.getNumBDs(tile.getTileType());
513 for (
auto &block : element.getBody()) {
514 auto bdOps = llvm::to_vector_of<DMABDOp>(block.template getOps<DMABDOp>());
521 if (bdNum >= bdMax) {
522 return (op->emitOpError(
"has more than ") << bdMax <<
" blocks")
523 .attachNote(bdOps.front().getLoc())
524 .append(
"no space for this BD");
529 if (bdOps.size() != 1) {
530 return (op->emitOpError(
"BD block must have exactly one DMABDOp, found ")
532 .attachNote(block.front().getLoc())
533 .append(
"in this BD block");
538 llvm::to_vector_of<UseLockOp>(block.template getOps<UseLockOp>());
539 int acquireCount = 0;
540 int releaseCount = 0;
541 for (
auto useLock : useLockOps) {
542 if (useLock.acquire() || useLock.acquireGE())
544 else if (useLock.release())
548 if (acquireCount > 1) {
549 return (op->emitOpError(
550 "BD block must have at most one acquire UseLockOp, found ")
552 .attachNote(block.front().getLoc())
553 .append(
"in this BD block");
555 if (releaseCount > 1) {
556 return (op->emitOpError(
557 "BD block must have at most one release UseLockOp, found ")
559 .attachNote(block.front().getLoc())
560 .append(
"in this BD block");
568template <
typename ConcreteType>
570 DenseSet<DMAChannel> inputChannels;
571 DenseSet<DMAChannel> outputChannels;
572 DenseSet<std::pair<DMAChannelDir, StringAttr>> namedChannels;
573 auto element = cast<ConcreteType>(op);
574 Region &body = element.getBody();
576 return op->emitOpError(
"should have non-empty body");
577 for (
auto &bodyOp : body.getOps()) {
579 if (
auto dmaStart = dyn_cast<DMAStartOp>(bodyOp)) {
580 if (FlatSymbolRefAttr endpoint = dmaStart.getEndpoint()) {
581 namedChannels.insert({dmaStart.getChannelDir(), endpoint.getAttr()});
584 DMAChannel dmaChan = {dmaStart.getChannelDir(),
585 dmaStart.getChannelIndex()};
588 if (dmaChan.direction == DMAChannelDir::S2MM)
589 inputChannels.insert(dmaChan);
591 outputChannels.insert(dmaChan);
595 TileLike tile = element.getTileLike();
597 return op->emitOpError(
"tile must implement TileLike interface");
599 auto named = [&](DMAChannelDir dir) ->
size_t {
600 return llvm::count_if(namedChannels,
601 [&](
auto channel) {
return channel.first == dir; });
603 if (inputChannels.size() + named(DMAChannelDir::S2MM) >
604 tile.getNumSourceConnections(WireBundle::DMA))
605 return op->emitOpError(
606 "uses more input channels than available on this tile");
608 if (outputChannels.size() + named(DMAChannelDir::MM2S) >
609 tile.getNumDestConnections(WireBundle::DMA))
610 return op->emitOpError(
611 "uses more output channels than available on this tile");
616LockAcquireAttr::verify(function_ref<InFlightDiagnostic()> emitError,
617 int32_t column, int32_t row, int32_t lock_id,
618 LockAction action, int32_t value) {
619 if (column < 0 || row < 0 || lock_id < 0)
620 return emitError() <<
"lock column, row, and ID must be non-negative";
621 if (action != LockAction::Acquire &&
622 action != LockAction::AcquireGreaterEqual)
623 return emitError() <<
"expected Acquire or AcquireGreaterEqual, got "
624 << stringifyLockAction(action);
633ObjectFifoTransportAttr::verify(function_ref<InFlightDiagnostic()> emitError,
634 ObjectFifoTransportMode mode,
635 PacketInfoAttr packet,
bool out_of_order) {
638 if (packet && mode == ObjectFifoTransportMode::SharedMem)
639 return emitError() <<
"`packet` belongs to a dma or auto transport, not "
640 << stringifyObjectFifoTransportMode(mode);
641 if (out_of_order && !packet)
642 return emitError() <<
"`out_of_order` requires a packet transport";
651ConsumedByAttr::verify(function_ref<InFlightDiagnostic()> emitError,
652 FlatSymbolRefAttr output, int64_t input_bytes,
653 int64_t output_bytes, int64_t granule_bytes,
654 FlatSymbolRefAttr device) {
655 if (input_bytes <= 0 || output_bytes <= 0)
656 return emitError() <<
"consumed_by needs positive byte counts, got "
657 << input_bytes <<
" and " << output_bytes;
658 if (granule_bytes < 0)
659 return emitError() <<
"consumed_by needs a granule of 0 or more bytes, got "
669ObjectFifoEndPortAttr::verify(function_ref<InFlightDiagnostic()> emitError,
671 std::optional<uint32_t> channel) {
672 if (bundle != WireBundle::DMA && bundle != WireBundle::Core)
673 return emitError() <<
"an objectFifo end is driven by a DMA channel or a "
674 "Core stream port, not "
675 << stringifyWireBundle(bundle);
676 if (bundle == WireBundle::Core) {
678 return emitError() <<
"a Core end needs its stream port";
680 return emitError() <<
"a Core end's stream port is 0 or 1, not "
692static LogicalResult verifyInitValues(Operation *op, ArrayAttr initValues,
694 for (Attribute value : initValues) {
695 auto dense = dyn_cast<DenseElementsAttr>(value);
697 return op->emitOpError(
"initial value ")
698 << value <<
" is not a dense elements attribute";
699 if (dense.getType().getShape() !=
object.getShape() ||
700 dense.getElementType() !=
object.getElementType())
701 return op->emitOpError(
"initial value of type ")
702 << dense.getType() <<
" does not match the object type " << object;
707LogicalResult ObjectFifoCreateOp::verify() {
708 for (Type fifoType : {Type(getElemType()), getConsumerElemTypeOrDefault()})
709 if (MemRefType memref = cast<AIEObjectFifoType>(fifoType).getElementType();
710 !memref.hasStaticShape())
711 return emitOpError(
"type should be static shaped memref, but got ")
717 static constexpr std::pair<llvm::StringLiteral, llvm::StringLiteral>
719 {
"out_of_order",
"transport = #aie.transport<dma, packet = "
720 "#aie.packet_info<>, out_of_order = true>"},
721 {
"via_DMA",
"transport = #aie.transport<dma>"},
722 {
"packet",
"transport = #aie.transport<dma, packet = "
723 "#aie.packet_info<>>"},
724 {
"packet_id",
"transport = #aie.transport<dma, packet = "
725 "#aie.packet_info<pkt_id = 3>>"},
726 {
"aie_stream",
"prod_port = #aie.end_port<Core : 0> or cons_ports "
727 "= [#aie.end_port<Core : 0>]"},
728 {
"aie_stream_port",
"prod_port = #aie.end_port<Core : 0> or "
729 "cons_ports = [#aie.end_port<Core : 0>]"},
730 {
"prod_dma_channel",
"prod_port = #aie.end_port<DMA : 1>"},
731 {
"cons_dma_channels",
"cons_ports = [#aie.end_port<DMA : 1>]"}};
732 for (
auto [gone, replacement] : replaced)
733 if ((*this)->hasAttr(gone))
734 return emitOpError(
"`")
735 << gone <<
"` has been replaced; write " << replacement;
737 if (isa<ArrayAttr>(getElemNumber())) {
738 if (
size_t numDepths = dyn_cast<ArrayAttr>(getElemNumber()).size();
739 numDepths != getConsumerTiles().size() + 1)
740 return emitOpError(
"does not have enough depths specified for producer "
741 "and for each consumer.");
744 if (ArrayAttr consPorts = getConsPortsAttr();
745 consPorts && consPorts.size() != getConsumerTiles().size())
746 return emitOpError(
"`cons_ports` has ")
747 << consPorts.size() <<
" entries for " << getConsumerTiles().size()
751 auto getTileLikeFromValue = [](Value v) -> TileLike {
752 return llvm::dyn_cast<TileLike>(v.getDefiningOp());
755 TileLike producerTile = getTileLikeFromValue(getProducerTile());
757 return emitError(
"producer tile must implement TileLike interface");
760 if (producerTile.isShimTile() && !getDimensionsToStream().empty()) {
762 "`dimensionsToStream` data layout transformations are not supported "
763 "on shim tile producers");
765 for (
auto consTileVal : getConsumerTiles()) {
766 TileLike consTile = getTileLikeFromValue(consTileVal);
768 return emitError(
"consumer tile must implement TileLike interface");
769 if (consTile.isShimTile() &&
770 !getDimensionsFromStream(consTileVal).empty()) {
772 "`dimensionsFromStreamPerConsumer` data layout transformations are "
773 "not supported on shim tile consumers");
780 for (
auto [index, consumer] : llvm::enumerate(getConsumerTiles())) {
782 llvm::find(getConsumerTiles(), consumer) - getConsumerTiles().begin();
783 TileLike tile = getTileLikeFromValue(consumer);
784 if (first == index || tile.getTileType() != AIETileType::CoreTile ||
785 getConsumerLink(index))
787 InFlightDiagnostic diag = emitOpError(
"lists core tile ");
788 if (std::optional<int> col = tile.tryGetCol(), row = tile.tryGetRow();
790 diag <<
"(" << *col <<
", " << *row <<
") ";
791 diag <<
"as consumers " << first <<
" and " << index
792 <<
"; its core reads only the first of those ends, so the other "
793 "would never be drained";
794 diag.attachNote(consumer.getLoc()) <<
"the tile";
798 if (getRepeatCount().has_value()) {
799 if (producerTile.isShimTile())
800 return emitError(
"`repeat_count` unavailable for shim tiles");
803 if (getPadValue() != 0) {
804 if (!getPadDimensions().has_value())
805 return emitError(
"`padValue` requires `padDimensions`");
807 return emitError(
"`padValue` requires the CONSTANT_PAD_VALUE register, "
808 "unavailable on this target");
812 if (getConsumerTiles().size() > 1)
813 return emitError(
"a Core stream port end can only be used in 1-to-1 "
816 if (forcesSharedMemory())
817 return emitError(
"a Core stream port end sends over the stream, not "
818 "through shared memory");
820 if (streamsFromProducer()) {
821 if (producerTile.isShimTile() || producerTile.isMemTile())
823 "a Core stream port end is not available for shim and mem tiles");
825 if (getRepeatCount().has_value())
826 return emitError(
"`repeat_count` unavailable on stream end");
828 if (getInitValues().has_value())
829 return emitError(
"`init_values` unavailable on stream end");
831 if (getIterCount().has_value())
832 return emitError(
"`iter_count` unavailable on stream end");
834 if (!getDimensionsToStream().empty())
835 return emitError(
"`dimensionsToStream` data layout transformations are "
836 "unavailable on stream end");
839 if (streamsToConsumer(0)) {
840 TileLike consTile = getTileLikeFromValue(getConsumerTiles()[0]);
841 if (consTile && (consTile.isShimTile() || consTile.isMemTile()))
843 "a Core stream port end is not available for shim and mem tiles");
846 if (!getDimensionsFromStreamPerConsumer()[0].empty())
847 return emitError(
"`dimensionsFromStreamPerConsumer` data layout "
848 "transformations are unavailable on stream end");
851 if (getInitValues().has_value()) {
852 if (producerTile.isShimTile())
853 return emitError(
"`init_values` unavailable for shim tiles");
856 if (
auto initValues = getInitValues()) {
857 if ((
int)initValues->size() != size())
858 return emitError(
"`init_values` does not initialize all objects");
859 if (failed(verifyInitValues(
861 cast<AIEObjectFifoType>(getElemType()).getElementType())))
865 if (
auto iterCountAttr = getIterCount()) {
866 int iterCount = *iterCountAttr;
867 if (iterCount < 1 || iterCount > 256)
868 return emitError(
"`iter_count` must be between 1 and 256");
871 if (
auto consumerElemType = getConsumerElemType()) {
872 auto consType = llvm::dyn_cast<AIEObjectFifoType>(*consumerElemType);
874 return emitError(
"consumer element type must be an "
875 "!aie.objectfifo<memref<...>> type");
876 auto prodType = llvm::cast<AIEObjectFifoType>(getElemType());
877 auto prodMemref = prodType.getElementType();
878 auto consMemref = consType.getElementType();
879 if (prodMemref.getElementType() != consMemref.getElementType())
880 return emitError(
"producer and consumer must have the same scalar "
881 "element type, but got ")
882 << prodMemref.getElementType() <<
" vs "
883 << consMemref.getElementType();
884 int64_t prodSize = prodMemref.getNumElements();
885 int64_t consSize = consMemref.getNumElements();
887 return emitError(
"consumer element count must be positive");
888 if (prodSize % consSize != 0)
889 return emitError(
"producer element size (")
890 << prodSize <<
") must be an integer multiple of consumer "
891 <<
"element size (" << consSize <<
")";
897TileOp ObjectFifoCreateOp::getProducerTileOp() {
898 return cast<TileOp>(getProducerTile().getDefiningOp());
908template <
typename OpTy>
909SmallVector<OpTy> lookupAll(Operation *from, std::optional<ArrayAttr> names) {
910 SmallVector<OpTy> ops;
912 for (
auto name : names->getAsRange<FlatSymbolRefAttr>()) {
914 dyn_cast_or_null<OpTy>(
lookupNamedOp(from, name.getAttr()))) {
924TileLike ObjectFifoPoolOp::getTileLike() {
925 return dyn_cast<TileLike>(getTile().getDefiningOp());
928int64_t ObjectFifoPoolOp::getObjectSize() {
929 return llvm::cast<MemRefType>(getElemType()).getNumElements();
932int64_t ObjectFifoPoolOp::getObjectsPerBuffer() {
933 if (!getContiguous())
935 int64_t objects = getDepth();
936 if (
auto zeros = getZeroObjects())
937 objects += (*zeros)[0] + (*zeros)[1];
941LogicalResult ObjectFifoPoolOp::verify() {
942 auto &target = (*this)->getParentOfType<DeviceOp>().
getTargetModel();
947 if (
auto locks = getLocks()) {
948 if (semaphoreLocks) {
949 return emitOpError(
"'locks' names binary locks, which this device has "
952 if (
static_cast<int64_t
>(locks->size()) != getDepth()) {
953 return emitOpError(
"expects one lock per buffer");
957 if (
auto zeros = getZeroObjects()) {
958 if (!getContiguous() || zeros->size() != 2 ||
959 llvm::any_of(*zeros, [](int32_t count) { return count < 0; }))
960 return emitOpError(
"'zeroObjects' expects [before, after] counts on a "
964 if (
auto buffers = getBuffers()) {
965 if (getContiguous() && buffers->size() != 1) {
966 return emitOpError(
"expects one buffer holding its objects side by side");
968 if (!getContiguous() &&
969 static_cast<int64_t
>(buffers->size()) != getDepth()) {
970 return emitOpError(
"expects 'depth' buffers");
974 if (
auto initValues = getInitValues()) {
975 if (
static_cast<int64_t
>(initValues->size()) > getDepth()) {
976 return emitOpError(
"has more 'init_values' than 'depth' objects");
978 if (failed(verifyInitValues(*
this, *initValues, getElemType())))
982 std::vector<ObjectFifoSegmentOp> segments = getSegmentOps();
983 if (segments.empty()) {
984 return emitOpError(
"expects at least one segment");
987 int64_t previous = -1;
989 for (ObjectFifoSegmentOp segment : segments) {
990 if (
static_cast<int64_t
>(segment.getOffset()) <= previous) {
991 return emitOpError(
"segments must be ordered by increasing offset");
993 if (
static_cast<int64_t
>(segment.getOffset()) != covered) {
994 return segment.emitOpError(
"leaves a gap or overlaps the segment before "
995 "it; segments must tile the object");
997 previous = segment.getOffset();
998 covered = segment.getOffset() + segment.getSize();
999 if (!semaphoreLocks &&
1000 (segment.getProduceLock() || segment.getConsumeLock())) {
1001 return emitOpError(
"segment locks are counting locks, which this "
1002 "device has no use for");
1006 if (covered != getObjectSize()) {
1007 return emitOpError(
"segments cover ")
1008 << covered <<
" of " << getObjectSize() <<
" elements";
1011 if (segments.size() > 1 && !semaphoreLocks) {
1013 "multi-segment pools are unsupported on binary lock architectures");
1020 const DataLayout &layout) {
1021 return type.getNumElements() * layout.getTypeSize(type.getElementType());
1024int64_t ObjectFifoPoolOp::getObjectSizeInBytes() {
1031static bool walksWholeDepth(
bool isMemTile, int64_t depth,
1032 int64_t maxLockValue) {
1033 return isMemTile && depth > 1 && depth <= maxLockValue;
1038static bool usesPushQueue(ObjectFifoCreateOp fifo) {
1039 return fifo.getReleaseStrategy() == ObjectFifoRelease::PushQueue ||
1040 fifo.sharesSendChannel();
1043std::vector<ObjectFifoSegmentOp> ObjectFifoPoolOp::getSegmentOps() {
1044 auto ops = getSegments().getOps<ObjectFifoSegmentOp>();
1045 return {ops.begin(), ops.end()};
1048Value BufferOp::getBufferTile() {
return getTile(); }
1050SmallVector<BufferLike> ObjectFifoPoolOp::getBufferOps() {
1053 return lookupAll<BufferLike>((*this)->getParentOfType<DeviceOp>(),
1057SmallVector<LockOp> ObjectFifoPoolOp::getLockOps() {
1058 auto device = (*this)->getParentOfType<DeviceOp>();
1059 SmallVector<LockOp> locks = lookupAll<LockOp>(device, getLocks());
1060 for (ObjectFifoSegmentOp segment : getSegmentOps()) {
1061 for (std::optional<FlatSymbolRefAttr> name :
1062 {segment.getProduceLockAttr(), segment.getConsumeLockAttr()}) {
1063 if (name && *name) {
1065 locks.push_back(lock);
1073StringRef ObjectFifoPoolOp::getBaseName() {
1074 StringRef name = getSymName();
1075 return name.consume_back(
"__pool") ? name : getSymName();
1078bool ObjectFifoPoolOp::canWalkWhole() {
1079 auto device = (*this)->getParentOfType<DeviceOp>();
1080 return walksWholeDepth(
1081 getTileLike().isMemTile(), getDepth(),
1082 static_cast<int64_t
>(device.getTargetModel().getMaxLockValue())) &&
1083 !getContiguous() && getSegmentOps().size() == 1 && !getRepeatCount() &&
1085 llvm::all_of(device.getOps<ObjectFifoDmaEndpointOp>(),
1086 [&](ObjectFifoDmaEndpointOp end) {
1087 auto dims = end.getDimensions();
1088 return end.getPool() != getSymName() ||
1089 (!end.getDispatch() && !end.getOutOfOrder() &&
1090 end.getReleaseStrategy() !=
1091 ObjectFifoRelease::PushQueue &&
1092 (!end.drains() || !dims || dims->empty() ||
1093 (*dims)[0].getValue().empty()));
1103LogicalResult verifyEndpoint(Operation *op, ObjectFifoPoolOp pool,
1104 std::optional<ArrayAttr> segments) {
1105 std::vector<ObjectFifoSegmentOp> all = pool.getSegmentOps();
1107 if (all.size() > 1) {
1108 return op->emitOpError(
1109 "must list segments explicitly for a multi-segment pool");
1113 if (segments->empty()) {
1114 return op->emitOpError(
"expects at least one segment");
1117 int64_t previous = -1;
1118 for (
auto name : segments->getAsRange<FlatSymbolRefAttr>()) {
1119 auto segment = SymbolTable::lookupNearestSymbolFrom<ObjectFifoSegmentOp>(
1120 pool, name.getAttr());
1122 return op->emitOpError(
"references undefined segment '")
1123 << name.getValue() <<
"' in pool '" << pool.getSymName() <<
"'";
1127 if (
static_cast<int64_t
>(segment.getOffset()) <= previous) {
1128 return op->emitOpError(
"segments must be named in increasing offset "
1131 previous = segment.getOffset();
1136ObjectFifoPoolOp lookupPool(Operation *op, StringRef name) {
1137 auto device = op->getParentOfType<DeviceOp>();
1138 return SymbolTable::lookupNearestSymbolFrom<ObjectFifoPoolOp>(
1139 device, StringAttr::get(op->getContext(), name));
1146static std::vector<ObjectFifoSegmentOp>
1147selectSegments(ObjectFifoPoolOp pool, std::optional<ArrayAttr> selected) {
1151 std::vector<ObjectFifoSegmentOp> all = pool.getSegmentOps();
1153 return all.empty() ? std::vector<ObjectFifoSegmentOp>{}
1154 : std::vector<ObjectFifoSegmentOp>{all.front()};
1156 std::vector<ObjectFifoSegmentOp> chosen;
1157 for (
auto name : selected->getAsRange<FlatSymbolRefAttr>()) {
1159 SymbolTable::lookupNearestSymbolFrom<ObjectFifoSegmentOp>(
1160 pool, name.getAttr())) {
1161 chosen.push_back(segment);
1167TileLike ObjectFifoCoreEndpointOp::getTileLike() {
1168 return dyn_cast<TileLike>(getTile().getDefiningOp());
1171ObjectFifoPoolOp ObjectFifoCoreEndpointOp::getPoolOp() {
1172 return lookupPool(*
this, getPool());
1175std::vector<ObjectFifoSegmentOp>
1176ObjectFifoCoreEndpointOp::getSelectedSegments() {
1177 return selectSegments(getPoolOp(), getSegments());
1180LogicalResult ObjectFifoCoreEndpointOp::verify() {
1181 ObjectFifoPoolOp pool = getPoolOp();
1183 return emitOpError(
"references undefined pool '") << getPool() <<
"'";
1185 if (failed(verifyEndpoint(*
this, pool, getSegments()))) {
1189 if (
auto push = getCorePushAttr()) {
1191 return emitOpError(
"core_push requires a filling core endpoint");
1192 if (
auto values = pool.getInitValues(); values && !values->empty())
1193 return emitOpError(
"core_push does not support initialized objects");
1194 if (push.getBdTags().size() !=
static_cast<size_t>(pool.getDepth()))
1195 return emitOpError(
"core_push requires one BD tag per pool buffer");
1196 if (llvm::any_of(push.getBdTags(), [](int32_t tag) { return tag < 0; }))
1197 return emitOpError(
"core_push requires nonnegative BD tags");
1203 for (ObjectFifoSegmentOp segment : getSelectedSegments()) {
1204 if (next >= 0 && segment.getOffset() != next) {
1205 return emitOpError(
"a core endpoint's segments must be contiguous");
1207 next = segment.getOffset() + segment.getSize();
1212std::pair<int64_t, int64_t> ObjectFifoCoreEndpointOp::getExtent() {
1213 std::vector<ObjectFifoSegmentOp> selected = getSelectedSegments();
1214 if (selected.empty()) {
1217 int64_t offset = selected.front().getOffset();
1218 int64_t end = selected.back().getOffset() + selected.back().getSize();
1219 return {offset, end - offset};
1222MemRefType ObjectFifoCoreEndpointOp::getAccessType() {
1223 MemRefType elemType = getPoolOp().getElemType();
1224 auto [offset, size] = getExtent();
1225 if (offset == 0 && size == elemType.getNumElements()) {
1228 return cast<MemRefType>(memref::SubViewOp::inferRankReducedResultType(
1229 {size}, elemType, {offset}, {size}, {1}));
1232TileLike ObjectFifoDmaEndpointOp::getTileLike() {
1233 return dyn_cast<TileLike>(getTile().getDefiningOp());
1236ObjectFifoPoolOp ObjectFifoDmaEndpointOp::getPoolOp() {
1237 return lookupPool(*
this, getPool());
1240std::vector<ObjectFifoSegmentOp>
1241ObjectFifoDmaEndpointOp::getSelectedSegments() {
1242 return selectSegments(getPoolOp(), getSegments());
1245int64_t ObjectFifoDmaEndpointOp::getNumDescriptors() {
1246 ObjectFifoPoolOp pool = getPoolOp();
1249 if (pool.getContiguous()) {
1251 if (!walksStream()) {
1252 SmallVector<ObjectFifoDmaEndpointOp> ends =
turnEndpoints(*
this);
1255 ArrayRef<Attribute> turns = getTurnsAttr().getValue();
1256 SmallVector<std::pair<int64_t, int64_t>> counts;
1257 for (ObjectFifoDmaEndpointOp end : ends)
1259 {llvm::count(turns, FlatSymbolRefAttr::get(end.getSymNameAttr())),
1260 end.getPoolOp().getContiguous() ? 0 : end.getNumDescriptors()});
1261 return llvm::count(turns, FlatSymbolRefAttr::get(getSymNameAttr())) *
1265 return succeeded(getStreamWindow(window)) ? window.
outputs : 0;
1267 return pool.getDepth() * getSelectedSegments().size();
1270bool ObjectFifoDmaEndpointOp::walksStream() {
1271 ObjectFifoPoolOp pool = getPoolOp();
1272 auto dims = getDimensions();
1273 return pool && pool.getContiguous() && drains() && dims && !dims->empty() &&
1274 !(*dims)[0].getValue().empty();
1277LogicalResult ObjectFifoDmaEndpointOp::getStreamWindow(
StreamWindow &window) {
1278 auto dims = getDimensions();
1279 if (!dims || dims->empty())
1280 return emitOpError(
"walks no stream: it has no dimensions");
1281 ArrayRef<BDPadLayoutAttr> pads;
1282 if (
auto padding = getPadDimensions(); padding && !padding->empty())
1283 pads = (*padding)[0].getValue();
1285 getPoolOp().getObjectSize(),
1286 [&] {
return emitOpError(); }, window);
1289int64_t ObjectFifoDmaEndpointOp::getRepeat() {
1290 ObjectFifoPoolOp pool = getPoolOp();
1292 return pool && drains() ? pool.getRepeatCount().value_or(1) : 1;
1295bool ObjectFifoDmaEndpointOp::repeatsInHardware() {
1296 return getRepeat() > 1 &&
1297 (getReleaseStrategy() == ObjectFifoRelease::PushQueue ||
1298 (getNumDescriptors() == 1 && !getIterCount()));
1301int64_t ObjectFifoDmaEndpointOp::getDispatchPasses() {
1302 ObjectFifoPoolOp pool = getPoolOp();
1303 std::optional<ArrayAttr> turns = getDispatch();
1304 if (!pool || !turns || turns->empty())
1306 int64_t depth = std::max<int64_t>(pool.getDepth(), 1);
1307 return std::lcm(depth,
static_cast<int64_t
>(turns->size())) / depth;
1310SmallVector<ObjectFifoDmaEndpointOp>
1312 SmallVector<ObjectFifoDmaEndpointOp> ends;
1313 if (std::optional<ArrayAttr> turns = endpoint.getTurns())
1314 for (
auto name : turns->getAsRange<FlatSymbolRefAttr>())
1316 SymbolTable::lookupNearestSymbolFrom<ObjectFifoDmaEndpointOp>(
1318 end && !llvm::is_contained(ends, end))
1319 ends.push_back(end);
1325 for (
auto [appearances, count] : ends)
1327 passes = std::lcm(passes, count / std::gcd(count, appearances));
1331int64_t ObjectFifoDmaEndpointOp::getNumBDs() {
1332 SmallVector<ObjectFifoDmaEndpointOp> ends =
turnEndpoints(*
this);
1334 return getNumDescriptors() * (repeatsInHardware() ? 1 : getRepeat()) *
1335 getDispatchPasses();
1336 if (ends.front() != *
this)
1338 ArrayRef<Attribute> turns = getTurnsAttr().getValue();
1339 SmallVector<std::pair<int64_t, int64_t>> counts;
1340 for (ObjectFifoDmaEndpointOp end : ends)
1342 {llvm::count(turns, FlatSymbolRefAttr::get(end.getSymNameAttr())),
1343 end.getPoolOp().getContiguous() ? 0 : end.getNumDescriptors()});
1344 return turnPasses(counts) *
static_cast<int64_t
>(turns.size());
1347DMAChannelDir ObjectFifoDmaEndpointOp::getRouteDirection() {
1348 return drains() ? DMAChannelDir::MM2S : DMAChannelDir::S2MM;
1351WireBundle ObjectFifoDmaEndpointOp::getRouteBundle() {
return WireBundle::DMA; }
1353std::optional<int> ObjectFifoDmaEndpointOp::getRouteChannel() {
1354 return getChannelIndex();
1357void ObjectFifoDmaEndpointOp::setRouteChannel(
int channel) {
1358 setChannelIndex(channel);
1361LogicalResult ObjectFifoDmaEndpointOp::verify() {
1362 ObjectFifoPoolOp pool = getPoolOp();
1364 return emitOpError(
"references undefined pool '") << getPool() <<
"'";
1367 if (failed(verifyEndpoint(*
this, pool, getSegments()))) {
1374 if (getReleaseStrategy() == ObjectFifoRelease::PushQueue) {
1375 if (!drains() || !getTileLike().isCoreTile() ||
1377 return emitOpError(
"push_queue requires a sending DMA on a core tile "
1378 "with core-issued task support");
1379 if (
auto values = pool.getInitValues(); values && !values->empty())
1380 return emitOpError(
"push_queue does not support initialized objects");
1383 "push_queue repeat_count exceeds the hardware task limit");
1384 if (getOutOfOrder() && getRepeat() > 1)
1386 "push_queue cannot repeat out-of-order packets: "
1387 "each repetition may need a different destination BD ID");
1390 if (std::optional<ArrayAttr> turns = getDispatch()) {
1392 return emitOpError(
"only a draining endpoint can dispatch");
1394 return emitOpError(
"dispatch names no destination");
1395 if (getPacketAttr())
1396 return emitOpError(
"a dispatching endpoint carries a header per turn in "
1397 "`dispatchPackets`, not one `packet`");
1398 if (pool.getRepeatCount().value_or(1) > 1)
1399 return emitOpError(
"cannot dispatch a pool that repeats its objects");
1400 if (std::optional<int32_t> iterCount = getIterCount();
1401 iterCount && *iterCount % std::max<int64_t>(getDispatchPasses(), 1))
1402 return emitOpError(
"iterCount ")
1403 << *iterCount <<
" is not a whole number of dispatch rounds ("
1404 << getDispatchPasses() <<
" passes over the pool)";
1406 if (std::optional<ArrayAttr> headers = getDispatchPackets()) {
1407 std::optional<ArrayAttr> turns = getDispatch();
1408 if (!turns || turns->size() != headers->size())
1409 return emitOpError(
"dispatchPackets needs one header per dispatch turn");
1410 for (
auto header : headers->getAsRange<PacketInfoAttr>())
1414 if (std::optional<ArrayAttr> turns = getTurns()) {
1415 if (!llvm::is_contained(*turns, FlatSymbolRefAttr::get(getSymNameAttr())))
1416 return emitOpError(
"takes no turn of its own in `turns`");
1417 for (
auto name : turns->getAsRange<FlatSymbolRefAttr>())
1418 if (!SymbolTable::lookupNearestSymbolFrom<ObjectFifoDmaEndpointOp>(*
this,
1420 return emitOpError(
"turns names ")
1421 << name <<
", which is no objectfifo.dma_endpoint";
1422 if (getDispatch() || getPacketAttr() || getIterCount() || getRepeat() > 1)
1423 return emitOpError(
"taking turns on a channel, it cannot also dispatch, "
1424 "carry a header, stop after iterCount or repeat");
1426 if (end.getTile() != getTile() || end.getRole() != getRole() ||
1427 end.getTurnsAttr() != getTurnsAttr() ||
1428 end.getChannelIndexAttr() != getChannelIndexAttr())
1429 return emitOpError(
"takes turns with ")
1430 << end.getSymNameAttr()
1431 <<
", which is not on this tile and channel with the same turns";
1432 if (pool.getContiguous() && !walksStream() &&
1433 pool.getDepth() % getNumDescriptors())
1434 return emitOpError(
"walks a contiguous pool of ")
1435 << pool.getDepth() <<
" objects in " << getNumDescriptors()
1436 <<
" turns a pass, which do not divide them";
1439 std::optional<ArrayAttr> segmentNames = getSegments();
1440 size_t selectedCount = segmentNames ? segmentNames->size() : 1;
1441 auto dimensions = getDimensions();
1442 if (dimensions && dimensions->size() != selectedCount) {
1443 return emitOpError(
"dimensions has ")
1444 << dimensions->size() <<
" entries for " << selectedCount
1445 <<
" selected segments";
1447 auto padding = getPadDimensions();
1448 if (padding && padding->size() != selectedCount) {
1449 return emitOpError(
"padDimensions has ")
1450 << padding->size() <<
" entries for " << selectedCount
1451 <<
" selected segments";
1453 if (padding && !drains()) {
1454 return emitOpError(
"padDimensions is only valid on a draining endpoint");
1456 if (getPadValue() != 0 &&
1457 (!padding || llvm::none_of(*padding, [](BDPadLayoutArrayAttr entry) {
1458 return !entry.empty();
1460 return emitOpError(
"nonzero padValue requires a non-empty padDimensions "
1467 return getStreamWindow(window);
1469 std::vector<ObjectFifoSegmentOp> selected = getSelectedSegments();
1470 for (
size_t position = 0; position < selectedCount; ++position) {
1471 BDDimLayoutArrayAttr dims =
1472 dimensions ? (*dimensions)[position] : BDDimLayoutArrayAttr();
1473 BDPadLayoutArrayAttr pads =
1474 padding ? (*padding)[position] : BDPadLayoutArrayAttr();
1475 if (pads && !pads.empty() && (!dims || dims.empty())) {
1476 return emitOpError(
"padDimensions entry ")
1477 << position <<
" requires dimensions for the same segment";
1479 if (dims && pads && !pads.empty() && dims.size() != pads.size()) {
1480 return emitOpError(
"dimensions and padDimensions entry ")
1481 << position <<
" have different ranks";
1483 if (dims && !dims.empty() &&
1484 getDimsMaxIdx(dims) >= selected[position].getSize()) {
1485 return emitOpError(
"dimensions entry ")
1486 << position <<
" exceeds selected segment '"
1487 << selected[position].getSymName() <<
"' of size "
1488 << selected[position].getSize();
1494TileLike RouteEndpointOp::getTileLike() {
1495 return dyn_cast<TileLike>(getTile().getDefiningOp());
1498DMAChannelDir RouteEndpointOp::getRouteDirection() {
1501 auto device = getOperation()->getParentOfType<DeviceOp>();
1502 StringRef name = getSymName();
1504 for (
auto flow : device.getOps<RouteOp>()) {
1505 if (flow.namesAsSource(name)) {
1506 return DMAChannelDir::MM2S;
1508 named |= flow.namesAsDestination(name);
1510 assert(named &&
"endpoint is named by no flow, so it has no direction to "
1511 "read; --aie-objectfifo-verify rejects this");
1513 return DMAChannelDir::S2MM;
1516WireBundle RouteEndpointOp::getRouteBundle() {
return getBundle(); }
1518std::optional<int> RouteEndpointOp::getRouteChannel() {
1519 return getChannelIndex();
1522void RouteEndpointOp::setRouteChannel(
int channel) { setChannelIndex(channel); }
1529verifyConsumedBy(Operation *op, ArrayAttr proofs,
bool shimInput,
1531 function_ref<
bool(FlatSymbolRefAttr)> namesShimOutput) {
1533 return op->emitOpError(
"consumed_by belongs on a shim input (MM2S)");
1534 for (
auto proof : proofs.getAsRange<ConsumedByAttr>())
1535 if (!namesShimOutput(proof.getOutput()))
1536 return op->emitOpError(
"consumed_by output ")
1537 << proof.getOutput() <<
" names no shim output (S2MM " << outputs
1542LogicalResult RouteEndpointOp::verify() {
1543 TileLike tile = getTileLike();
1545 return emitOpError(
"tile operand is not an aie.tile or aie.logical_tile");
1553 StringRef name = getSymName();
1554 SmallVector<RouteOp> sources, dests;
1555 for (
auto flow : (*this)->getParentOfType<DeviceOp>().getOps<RouteOp>()) {
1556 if (flow.namesAsSource(name))
1557 sources.push_back(flow);
1558 if (flow.namesAsDestination(name))
1559 dests.push_back(flow);
1561 if (!sources.empty() && !dests.empty()) {
1562 return emitOpError(
"is named as both a source and a destination, so its "
1563 "direction is ambiguous");
1565 ArrayRef<RouteOp> sharers = sources.empty() ? dests : sources;
1566 if (sharers.size() > 1) {
1567 if (llvm::any_of(sharers, [](RouteOp flow) {
return !flow.getPacket(); }))
1568 return emitOpError(
"drives one channel, which a circuit flow owns, but "
1570 << sharers.size() <<
" times";
1571 llvm::SmallDenseSet<int> ids;
1572 for (RouteOp flow : sources) {
1573 std::optional<uint16_t>
id = flow.getPacketAttr().getPktId();
1575 return emitOpError(
"sends several packet flows, so each must pin its "
1576 "pkt_id for the BDs to state it");
1577 if (!ids.insert(*id).second)
1578 return emitOpError(
"sends several packet flows, two of which pin the "
1580 <<
static_cast<int>(*id) <<
"; each needs its own";
1583 if (llvm::any_of(dests, [&](RouteOp flow) {
1584 return flow.getKeepPktHeader() != dests[0].getKeepPktHeader();
1586 return emitOpError(
"receives several packet flows, so they must agree "
1587 "on keep_pkt_header");
1589 if (ArrayAttr proofs = getConsumedByAttr()) {
1590 auto device = (*this)->getParentOfType<DeviceOp>();
1591 if (failed(verifyConsumedBy(
1592 *
this, proofs, tile.isShimTile(),
1593 "aie.route_endpoint or aie.shim_dma_allocation",
1594 [&](FlatSymbolRefAttr symbol) {
1595 Operation *output = device.lookupSymbol(symbol.getAttr());
1596 if (auto alloc = dyn_cast_or_null<ShimDMAAllocationOp>(output))
1597 return alloc.getChannelDir() == DMAChannelDir::S2MM;
1598 auto end = dyn_cast_or_null<RouteEndpointOp>(output);
1599 return end && end.getTileLike().isShimTile();
1603 switch (getBundle()) {
1604 case WireBundle::DMA:
1608 case WireBundle::PLIO:
1609 if (!tile.isShimTile()) {
1610 return emitOpError(
"a PLIO end is on a shim");
1613 case WireBundle::Core:
1616 return emitOpError(
"bundle must be DMA, PLIO or Core");
1626static ParseResult parseRouteSources(OpAsmParser &parser, ArrayAttr &sources) {
1627 SmallVector<Attribute> names;
1628 auto parseOne = [&]() -> ParseResult {
1629 FlatSymbolRefAttr name;
1630 if (parser.parseAttribute(name))
1632 names.push_back(name);
1635 if (succeeded(parser.parseOptionalLSquare())) {
1636 if (failed(parser.parseOptionalRSquare()) &&
1637 (parser.parseCommaSeparatedList(parseOne) || parser.parseRSquare()))
1639 }
else if (parseOne()) {
1642 sources = parser.getBuilder().getArrayAttr(names);
1646static void printRouteSources(OpAsmPrinter &p, Operation *, ArrayAttr sources) {
1647 if (sources.size() == 1) {
1648 p.printAttribute(sources[0]);
1652 llvm::interleaveComma(sources, p,
1653 [&](Attribute name) { p.printAttribute(name); });
1657LogicalResult RouteOp::verify() {
1658 if (getSources().empty()) {
1659 return emitOpError(
"expects at least one source");
1661 if (getDestinations().empty()) {
1662 return emitOpError(
"expects at least one destination");
1665 if (getKeepPktHeader() && !getPacket()) {
1666 return emitOpError(
"keep_pkt_header is only meaningful on a packet flow");
1669 auto device = (*this)->getParentOfType<DeviceOp>();
1670 auto lookup = [&](StringRef name) {
1671 return dyn_cast_or_null<RouteEndpoint>(SymbolTable::lookupNearestSymbolFrom(
1672 device, StringAttr::get(getContext(), name)));
1675 llvm::SmallDenseSet<StringRef> seen;
1676 for (StringRef source : getSourceNames()) {
1677 if (!lookup(source)) {
1678 return emitOpError(
"source '") << source <<
"' is not an endpoint";
1680 if (!seen.insert(source).second) {
1681 return emitOpError(
"names '@") << source <<
"' as a source twice";
1684 for (StringRef destination : getDestinationNames()) {
1685 if (!lookup(destination)) {
1686 return emitOpError(
"destination '")
1687 << destination <<
"' is not an endpoint";
1689 if (!seen.insert(destination).second) {
1690 return emitOpError(
"names '@")
1692 <<
"' more than once; an end is a source or a "
1693 "destination, and only one of each";
1699 if (isFanIn() && !getPacket()) {
1700 return emitOpError(
"has several sources, so it needs a `packet` header; "
1701 "a circuit cannot merge streams");
1707ObjectFifoCreateOp::getDimensionsFromStream(Value consumerTile) {
1709 for (
auto cons : getConsumerTiles()) {
1710 if (cons == consumerTile)
1714 return getDimensionsFromStreamPerConsumer()[dimsIndex];
1718 OpAsmParser &parser, OpAsmParser::UnresolvedOperand &operand,
1719 BDDimLayoutArrayAttr &dimensions) {
1720 std::vector<BDDimLayoutAttr> emptyDims = {};
1721 if (parser.parseOperand(operand))
1723 if (succeeded(parser.parseOptionalKeyword(
"dimensionsToStream"))) {
1724 if (parser.parseCustomAttributeWithFallback<BDDimLayoutArrayAttr>(
1730 BDDimLayoutArrayAttr::get(parser.getContext(), ArrayRef(emptyDims));
1736 Operation *op, Value operand,
1737 BDDimLayoutArrayAttr dimensions) {
1739 if (!dimensions.empty()) {
1740 printer <<
" dimensionsToStream ";
1741 printer.printStrippedAttrOrType(dimensions);
1746 Attribute &channel) {
1748 if (succeeded(parser.parseOptionalSymbolName(name))) {
1749 channel = FlatSymbolRefAttr::get(name);
1753 if (parser.parseInteger(index))
1755 channel = parser.getBuilder().getI32IntegerAttr(index);
1760 bool resolved =
true;
1761 device.walkOutsideRuntimeSequences<DMAStartOp>([&](DMAStartOp start) {
1762 if (FlatSymbolRefAttr endpoint = start.getEndpoint()) {
1763 start.emitOpError() <<
"names route endpoint " << endpoint
1764 <<
" in place of a channel index; run "
1765 "--aie-objectfifo-allocate to assign one";
1769 return success(resolved);
1773 Attribute channel) {
1774 if (
auto index = dyn_cast<IntegerAttr>(channel))
1775 printer << index.getInt();
1777 printer.printAttributeWithoutType(channel);
1782 ObjectFifoPortAttr &port,
1783 SmallVectorImpl<Type> &objects) {
1784 if (parser.parseLParen())
1788 SMLoc portLoc = parser.getCurrentLocation();
1789 if (succeeded(parser.parseOptionalKeyword(&portName))) {
1790 std::optional<ObjectFifoPort> parsed = symbolizeObjectFifoPort(portName);
1792 return parser.emitError(portLoc,
"invalid objectFifo port: ") << portName;
1793 port = ObjectFifoPortAttr::get(parser.getContext(), *parsed);
1794 if (parser.parseComma())
1799 SMLoc countLoc = parser.getCurrentLocation();
1800 if (parser.parseInteger(count) || parser.parseRParen() ||
1801 parser.parseColon() || parser.parseTypeList(objects))
1804 if (count !=
static_cast<int64_t
>(objects.size()))
1805 return parser.emitError(countLoc,
"acquires ")
1806 << count <<
" objects but names " << objects.size() <<
" of them";
1812 ObjectFifoPortAttr port,
1813 TypeRange objects) {
1816 printer << stringifyObjectFifoPort(port.getValue()) <<
", ";
1817 printer << objects.size() <<
") : ";
1818 llvm::interleaveComma(objects, printer);
1822 ObjectFifoPortAttr &port,
1823 IntegerAttr &size) {
1824 if (parser.parseLParen())
1828 SMLoc portLoc = parser.getCurrentLocation();
1829 if (succeeded(parser.parseOptionalKeyword(&portName))) {
1830 std::optional<ObjectFifoPort> parsed = symbolizeObjectFifoPort(portName);
1832 return parser.emitError(portLoc,
"invalid objectFifo port: ") << portName;
1833 port = ObjectFifoPortAttr::get(parser.getContext(), *parsed);
1834 if (parser.parseComma())
1839 if (parser.parseInteger(count) || parser.parseRParen())
1841 size = parser.getBuilder().getI32IntegerAttr(count);
1847 ObjectFifoPortAttr port,
1851 printer << stringifyObjectFifoPort(port.getValue()) <<
", ";
1852 printer << size.getInt() <<
")";
1856 OpAsmParser &parser, SmallVectorImpl<OpAsmParser::UnresolvedOperand> &tiles,
1857 BDDimLayoutArrayArrayAttr &dimensions) {
1860 std::vector<BDDimLayoutArrayAttr> tileDims = {};
1862 auto parseOneOperand = [&]() -> ParseResult {
1863 if (parser.parseOperand(tiles.emplace_back(),
true)) {
1869 BDDimLayoutArrayAttr dimAttr =
1870 BDDimLayoutArrayAttr::get(parser.getContext(), {});
1872 if (succeeded(parser.parseOptionalKeyword(
"dimensionsFromStream"))) {
1874 if (parser.parseCustomAttributeWithFallback<BDDimLayoutArrayAttr>(
1879 tileDims.emplace_back(dimAttr);
1883 if (parser.parseCommaSeparatedList(AsmParser::Delimiter::None,
1884 parseOneOperand,
" in operand list"))
1887 dimensions = BDDimLayoutArrayArrayAttr::get(parser.getContext(), tileDims);
1892 OpAsmPrinter &printer, Operation *op, OperandRange tiles,
1893 BDDimLayoutArrayArrayAttr dimsPerTileAttr) {
1895 for (
auto tile : tiles) {
1897 if (dimsPerTileAttr && tileIdx < dimsPerTileAttr.size() &&
1898 dimsPerTileAttr[tileIdx] && !dimsPerTileAttr[tileIdx].empty()) {
1899 printer <<
" dimensionsFromStream ";
1900 printer.printStrippedAttrOrType(dimsPerTileAttr[tileIdx]);
1902 if (tileIdx < tiles.size() - 1) {
1909static void printObjectFifoConsumerElemType(OpAsmPrinter &p,
1910 ObjectFifoCreateOp op,
1911 TypeAttr consumerElemType) {
1912 if (consumerElemType)
1913 p <<
" -> " << consumerElemType;
1917 OpAsmParser &parser, SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vias,
1918 DenseI32ArrayAttr &ingressBundles, DenseI32ArrayAttr &ingressChannels,
1919 DenseI32ArrayAttr &egressBundles, DenseI32ArrayAttr &egressChannels) {
1920 SmallVector<int32_t> inBundles, inChannels, egBundles, egChannels;
1921 auto parseBundle = [&](SmallVectorImpl<int32_t> &out) -> ParseResult {
1923 if (parser.parseKeyword(&kw)) {
1926 auto bundle = symbolizeWireBundle(kw);
1928 return parser.emitError(parser.getCurrentLocation(),
1929 "invalid wire bundle '" + kw +
"'");
1931 out.push_back(
static_cast<int32_t
>(*bundle));
1935 if (succeeded(parser.parseOptionalKeyword(
"via"))) {
1936 auto parseOne = [&]() -> ParseResult {
1937 int32_t inChan, egChan;
1938 if (parser.parseOperand(vias.emplace_back()) || parser.parseColon() ||
1939 parseBundle(inBundles) || parser.parseColon() ||
1940 parser.parseInteger(inChan) || parser.parseArrow() ||
1941 parseBundle(egBundles) || parser.parseColon() ||
1942 parser.parseInteger(egChan)) {
1945 inChannels.push_back(inChan);
1946 egChannels.push_back(egChan);
1949 if (parser.parseCommaSeparatedList(AsmParser::Delimiter::Paren, parseOne)) {
1953 if (!vias.empty()) {
1954 MLIRContext *ctx = parser.getContext();
1955 ingressBundles = DenseI32ArrayAttr::get(ctx, inBundles);
1956 ingressChannels = DenseI32ArrayAttr::get(ctx, inChannels);
1957 egressBundles = DenseI32ArrayAttr::get(ctx, egBundles);
1958 egressChannels = DenseI32ArrayAttr::get(ctx, egChannels);
1965 DenseI32ArrayAttr ingressBundles,
1966 DenseI32ArrayAttr ingressChannels,
1967 DenseI32ArrayAttr egressBundles,
1968 DenseI32ArrayAttr egressChannels) {
1973 for (
size_t i = 0; i < vias.size(); i++) {
1977 printer << vias[i] <<
" : "
1978 << stringifyWireBundle(
static_cast<WireBundle
>(ingressBundles[i]))
1979 <<
" : " << ingressChannels[i] <<
" -> "
1980 << stringifyWireBundle(
static_cast<WireBundle
>(egressBundles[i]))
1981 <<
" : " << egressChannels[i];
1986static ParseResult parseObjectFifoConsumerElemType(OpAsmParser &parser,
1987 TypeAttr &consumerElemType) {
1988 if (failed(parser.parseOptionalArrow()))
1991 if (parser.parseType(type))
1993 consumerElemType = TypeAttr::get(type);
1997static void printObjectFifoInitValues(OpAsmPrinter &p, ObjectFifoCreateOp op,
1998 Attribute numElem, TypeAttr type,
1999 Attribute initValues) {
2000 if (op.getInitValues()) {
2006 if (isa<ArrayAttr>(numElem)) {
2008 llvm::cast<mlir::IntegerAttr>(llvm::cast<mlir::ArrayAttr>(numElem)[0])
2011 depth = llvm::cast<mlir::IntegerAttr>(numElem).getInt();
2013 for (
int i = 0; i < depth; i++) {
2014 p.printStrippedAttrOrType(llvm::cast<mlir::ArrayAttr>(initValues)[i]);
2015 if (i < depth - 1) {
2023static ParseResult parseObjectFifoInitValues(OpAsmParser &parser,
2024 Attribute numElem, TypeAttr type,
2025 Attribute &initValues) {
2027 if (isa<ArrayAttr>(numElem)) {
2029 llvm::cast<mlir::IntegerAttr>(llvm::cast<mlir::ArrayAttr>(numElem)[0])
2032 depth = llvm::cast<mlir::IntegerAttr>(numElem).getInt();
2034 auto objfifoType = llvm::cast<AIEObjectFifoType>(type.getValue());
2035 auto memrefType = llvm::cast<MemRefType>(objfifoType.getElementType());
2037 if (!memrefType.hasStaticShape())
2038 return parser.emitError(parser.getNameLoc())
2039 <<
"type should be static shaped memref, but got " << memrefType;
2041 if (parser.parseOptionalEqual())
2044 Type tensorType = mlir::memref::getTensorTypeFromMemRefType(memrefType);
2045 if (parser.parseAttribute(initValues, tensorType))
2047 for (
int i = 0; i < depth; i++) {
2048 auto initialValues = llvm::dyn_cast<mlir::ArrayAttr>(initValues);
2049 if ((
int)initialValues.size() != depth)
2050 return parser.emitError(parser.getNameLoc())
2051 <<
"initial values should initialize all objects";
2052 if (!llvm::isa<ElementsAttr>(initialValues[i]))
2053 return parser.emitError(parser.getNameLoc())
2054 <<
"initial value should be an elements attribute";
2064std::optional<ObjectFifoLinkOp> ObjectFifoCreateOp::getLink() {
2065 auto self = FlatSymbolRefAttr::get(getSymNameAttr());
2066 for (ObjectFifoLinkOp link :
2067 (*this)->getParentOfType<DeviceOp>().getOps<ObjectFifoLinkOp>())
2068 if (llvm::is_contained(link.getFifoIns(), self) ||
2069 llvm::is_contained(link.getFifoOuts(), self))
2071 return std::nullopt;
2074std::optional<ObjectFifoAllocateOp> ObjectFifoCreateOp::getAllocate() {
2075 for (ObjectFifoAllocateOp alloc :
2076 (*this)->getParentOfType<DeviceOp>().getOps<ObjectFifoAllocateOp>())
2077 if (alloc.getObjFifoName() == name().getValue())
2079 return std::nullopt;
2082bool ObjectFifoCreateOp::delegateReaches(
const AIETargetModel &target,
2086 return llvm::all_of(std::array{producer, consumer}, [&](
TileID end) {
2088 return s == SharedMemory::First || s == SharedMemory::Either;
2092bool ObjectFifoCreateOp::allocatedHome(
const AIETargetModel &target,
2095 std::optional<TileID> linkTile) {
2096 TileID home = linkTile.value_or(delegate);
2097 return delegateReaches(target, delegate, producer, consumer) &&
2098 delegateReaches(target, home, producer, consumer);
2101bool ObjectFifoCreateOp::allocateFallsBackToDMA() {
2102 auto self = FlatSymbolRefAttr::get(getSymNameAttr());
2103 bool linked =
false;
2104 for (ObjectFifoLinkOp link :
2105 (*this)->getParentOfType<DeviceOp>().getOps<ObjectFifoLinkOp>()) {
2106 if (!llvm::is_contained(link.getFifoIns(), self) &&
2107 !llvm::is_contained(link.getFifoOuts(), self))
2109 if (link.getOptionalSharedTile() && link.getPoolOwner() == *
this)
2117 ObjectFifoLinkOp producerLink = getProducerLink();
2118 ObjectFifoLinkOp consumerLink =
2119 getConsumerTiles().empty() ? ObjectFifoLinkOp() : getConsumerLink(0);
2120 if (producerLink && consumerLink)
2122 if (consumerLink && consumerLink.isMerge())
2124 if (producerLink && producerLink.isDispatch())
2126 ObjectFifoLinkOp link = producerLink ? producerLink : consumerLink;
2129 ObjectFifoCreateOp owner = link.getPoolOwner();
2132 if (owner.getElemType() != getElemType())
2137std::optional<StringRef> ObjectFifoCreateOp::dmaReason() {
2139 return "it asks for a dma transport";
2140 if (getRepeatCount())
2141 return "it repeats objects (`repeat_count`)";
2143 return "an end is a Core stream port";
2144 if (getConsumerElemType())
2145 return "its consumer takes a different element type";
2146 if (getConsumerTiles().size() > 1)
2147 return "it has more than one consumer";
2148 if (!getDimensionsToStream().empty())
2149 return "its producer reshapes the data (`dimensionsToStream`)";
2150 for (BDDimLayoutArrayAttr dims : getDimensionsFromStreamPerConsumer())
2152 return "a consumer reshapes the data (`dimensionsFromStream`)";
2155 if (getLink() && !getAllocate())
2156 return "it is linked through a tile's DMAs";
2157 return std::nullopt;
2162 std::optional<TileID> delegate, std::optional<TileID> linkTile) {
2167 allocatedHome(target, producer, consumer, *delegate, linkTile)) {
2169 return own == SharedMemory::None ? SharedMemory::Either : own;
2172 return SharedMemory::None;
2176static std::optional<TileID> placedAt(Value tile) {
2177 auto tileLike = dyn_cast_or_null<TileLike>(tile.getDefiningOp());
2179 return std::nullopt;
2180 std::optional<int> col = tileLike.tryGetCol(), row = tileLike.tryGetRow();
2182 return std::nullopt;
2183 return TileID{*col, *row};
2186std::optional<StringRef>
2188 if (std::optional<StringRef> reason = dmaReason())
2190 std::optional<ObjectFifoAllocateOp> alloc = getAllocate();
2191 std::optional<TileID> producer = placedAt(getProducerTile());
2192 std::optional<TileID> consumer = placedAt(getConsumerTiles()[0]);
2193 std::optional<TileID> delegate =
2194 alloc ? placedAt(alloc->getDelegateTile()) : std::nullopt;
2195 std::optional<ObjectFifoLinkOp> link = getLink();
2196 std::optional<Value> linkPoint =
2197 link ? link->getOptionalSharedTile() : std::nullopt;
2198 std::optional<TileID> linkTile =
2199 linkPoint ? placedAt(*linkPoint) : std::nullopt;
2200 if (!producer || !consumer || (alloc && !delegate) || (link && !linkTile))
2201 return std::nullopt;
2202 shared = sharedMemoryOf(
getTargetModel(*
this), *producer, *consumer, delegate,
2205 return link ?
"it is linked through a tile's DMAs"
2206 :
"its ends share no memory module";
2207 return std::nullopt;
2210std::optional<StringRef> ObjectFifoCreateOp::pushQueueRefusal() {
2211 if (
auto values = getInitValues(); values && !values->empty())
2212 return StringRef(
"push_queue does not support initialized objects");
2213 auto tile = cast<TileLike>(getProducerTile().getDefiningOp());
2214 auto device = (*this)->getParentOfType<DeviceOp>();
2215 if (!tile.isCoreTile() || streamsFromProducer() ||
2216 device.getTargetModel().getDmaTaskQueueDepth() == 0)
2217 return StringRef(
"push_queue requires a sending DMA on a core tile with "
2218 "core-issued task support");
2219 bool hasRelease =
false;
2220 for (Operation *user : getProducerTile().getUsers())
2221 if (
auto core = dyn_cast<CoreOp>(user))
2222 core.walk([&](ObjectFifoReleaseOp release) {
2223 hasRelease |= release.getObjectFifo() == *
this &&
2224 release.getPort() == ObjectFifoPort::Produce;
2227 return StringRef(
"push_queue requires producer releases in a local core "
2228 "to submit send tasks");
2229 return std::nullopt;
2232std::optional<StringRef>
2233ObjectFifoCreateOp::sharedReceiveRefusal(
bool shim,
bool outOfOrder) {
2235 return StringRef(
"a shim's receiving channel writes into the runtime's "
2236 "tasks, which no object header picks");
2238 return StringRef(
"a channel its fifos receive in turn holds their order; "
2239 "out_of_order puts that order in each object's header");
2240 return std::nullopt;
2243bool ObjectFifoCreateOp::sharesSendChannel() {
2244 auto pinned = [](ObjectFifoCreateOp fifo) -> std::optional<int> {
2245 ObjectFifoEndPortAttr port = fifo.getProdPortAttr();
2246 return port && !port.isCore() ? port.channelIndex() : std::nullopt;
2248 auto byDMA = [](ObjectFifoCreateOp fifo) {
2250 return fifo.placedDmaReason(shared).has_value();
2252 std::optional<int> channel = pinned(*
this);
2253 Value tile = getProducerTile();
2254 if (!channel || cast<TileLike>(tile.getDefiningOp()).isShimTile() ||
2257 std::optional<TileID> at = placedAt(tile);
2258 for (ObjectFifoCreateOp other :
2259 (*this)->getParentOfType<DeviceOp>().getOps<ObjectFifoCreateOp>()) {
2260 Value otherTile = other.getProducerTile();
2261 if (other != *
this && pinned(other) == channel &&
2262 (otherTile == tile || (at && placedAt(otherTile) == at)) &&
2269int ObjectFifoCreateOp::dmaDepth(std::optional<unsigned> consumer) {
2270 if (consumer ? streamsToConsumer(*consumer) : streamsFromProducer())
2272 if (!consumer && getInitValues())
2274 if (consumer && isa<ArrayAttr>(getElemNumber()))
2275 return size(*consumer + 1);
2281 Value tile = consumer ? getConsumerTiles()[*consumer] : getProducerTile();
2282 ObjectFifoPort port =
2283 consumer ? ObjectFifoPort::Consume : ObjectFifoPort::Produce;
2285 for (Operation *user : tile.getUsers())
2286 if (
auto core = dyn_cast<CoreOp>(user))
2287 core.walk([&](ObjectFifoAcquireOp acquire) {
2288 if (acquire.getObjectFifo() == *
this && acquire.getPort() == port)
2289 maxAcquire = std::max(maxAcquire, acquire.acqNumber());
2291 if (maxAcquire == 0)
2293 if (maxAcquire == 1 && size() == 1)
2295 return maxAcquire + 1;
2298ObjectFifoLinkOp ObjectFifoCreateOp::getProducerLink() {
2299 auto self = FlatSymbolRefAttr::get(getSymNameAttr());
2300 for (ObjectFifoLinkOp link :
2301 (*this)->getParentOfType<DeviceOp>().getOps<ObjectFifoLinkOp>())
2302 if (llvm::is_contained(link.getFifoOuts(), self) &&
2303 link.getOptionalSharedTile() == getProducerTile())
2308ObjectFifoLinkOp ObjectFifoCreateOp::getConsumerLink(
unsigned consumer) {
2309 auto consumers = getConsumerTiles();
2310 Value tile = consumers[consumer];
2311 size_t earlier = llvm::count(consumers.take_front(consumer), tile);
2312 auto self = FlatSymbolRefAttr::get(getSymNameAttr());
2313 for (ObjectFifoLinkOp link :
2314 (*this)->getParentOfType<DeviceOp>().getOps<ObjectFifoLinkOp>())
2315 if (llvm::is_contained(link.getFifoIns(), self) &&
2316 link.getOptionalSharedTile() == tile && earlier-- == 0)
2321std::optional<PlannedDmaEnd>
2322ObjectFifoCreateOp::plannedDmaEnd(std::optional<unsigned> consumer) {
2323 return plannedDmaEnd(consumer, placedAt);
2326std::optional<PlannedDmaEnd>
2327ObjectFifoCreateOp::plannedDmaEnd(std::optional<unsigned> consumer,
2329 bool drains = !consumer;
2330 Value tile = drains ? getProducerTile() : getConsumerTiles()[*consumer];
2331 auto tileLike = cast<TileLike>(tile.getDefiningOp());
2332 if (tileLike.isShimTile() ||
2333 (drains ? streamsFromProducer() : streamsToConsumer(*consumer)))
2334 return std::nullopt;
2335 auto device = (*this)->getParentOfType<DeviceOp>();
2336 int64_t maxLock = device.getTargetModel().getMaxLockValue();
2337 ObjectFifoEndPortAttr port =
2338 drains ? getProdPortAttr() : getConsumerPort(*consumer);
2339 std::optional<int> channel = port ? port.channelIndex() : std::nullopt;
2340 auto walkable = [&](int64_t depth) {
2341 return walksWholeDepth(tileLike.isMemTile(), depth, maxLock);
2344 ObjectFifoLinkOp link =
2345 drains ? getProducerLink() : getConsumerLink(*consumer);
2346 DataLayout layout = DataLayout::closest(*
this);
2347 auto objectBytes = [&](Type fifoType) -> int64_t {
2348 MemRefType type = cast<AIEObjectFifoType>(fifoType).getElementType();
2351 auto ordered = [](ObjectFifoCreateOp fifo,
bool drains) {
2352 return !fifo.getOutOfOrder() && (!drains || !usesPushQueue(fifo));
2355 int depth = dmaDepth(consumer);
2359 static_cast<int>(*consumer),
2361 walkable(depth) && ordered(*
this,
false),
2362 objectBytes(getConsumerElemTypeOrDefault()),
2365 int64_t repeat = getRepeatCount().value_or(1);
2367 repeat > 1 && (usesPushQueue(*
this) || (depth == 1 && !getIterCount()));
2372 walkable(depth) && ordered(*
this,
true) &&
2373 !getRepeatCount() &&
2374 getDimensionsToStream().empty(),
2375 objectBytes(getElemType()),
2382 if (drains ? link.isDispatch() : link.isMerge()) {
2383 std::vector<ObjectFifoCreateOp> sharing =
2384 drains ? link.getOutputObjectFifos() : link.getInputObjectFifos();
2385 for (ObjectFifoCreateOp fifo : device.getOps<ObjectFifoCreateOp>())
2386 if (llvm::is_contained(sharing, fifo)) {
2388 return std::nullopt;
2392 int64_t depth = link.getPoolDepth(positions);
2393 int64_t segments = link.getPoolSegments().size();
2395 (drains ? link.isDistribute() : link.isJoin()) ? 1 : segments;
2396 int64_t descriptors = depth * selected;
2397 int64_t contiguousObjects = 0;
2398 if (link.walksStream()) {
2400 bool inferred = succeeded(link.getStreamWindow(window));
2401 descriptors = !drains ? 1 : inferred ? window.
outputs : 0;
2403 contiguousObjects = depth + window.
top + window.
bottom;
2405 int64_t repeat = drains ? link.getRepeatCount().value_or(1) : 1;
2406 bool inHardware = repeat > 1 && ((drains && usesPushQueue(*
this)) ||
2407 (descriptors == 1 && !getIterCount()));
2409 if (drains && link.isDispatch()) {
2410 int64_t objects = std::max<int64_t>(depth, 1);
2412 std::lcm(objects,
static_cast<int64_t
>(link.getFifoOuts().size())) /
2416 llvm::any_of(link.getOutputObjectFifos(), [](ObjectFifoCreateOp out) {
2417 return !out.getDimensionsToStream().empty();
2422 link.getOutputObjectFifos(),
2423 [&](ObjectFifoCreateOp out) { return ordered(out, true); }) &&
2425 llvm::all_of(link.getInputObjectFifos(),
2426 [&](ObjectFifoCreateOp in) { return ordered(in, false); }));
2427 return PlannedDmaEnd{descriptors * (inHardware ? 1 : repeat) * passes,
2431 walkable(depth) && segments == 1 &&
2432 !link.getRepeatCount() && !link.walksStream() &&
2433 !link.isDispatch() && !drainLaysOut && linkOrdered,
2434 objectBytes(link.getPoolOwner().getElemType()),
2439bool ObjectFifoCreateOp::canTakeTurns() {
2440 if (getConsumerTiles().size() != 1)
2442 auto pinned = [](ObjectFifoEndPortAttr port) {
2443 return port && (port.isCore() || port.channelIndex());
2445 bool dispatch =
false, walksStream =
false;
2446 int64_t repeat = getRepeatCount().value_or(1);
2447 for (ObjectFifoLinkOp link :
2448 (*this)->getParentOfType<DeviceOp>().getOps<ObjectFifoLinkOp>()) {
2449 if (llvm::is_contained(link.getOutputObjectFifos(), *
this)) {
2450 dispatch |= link.isDispatch();
2451 walksStream |= link.walksStream();
2452 repeat = std::max<int64_t>(repeat, link.getRepeatCount().value_or(1));
2454 if (link.isMerge() && llvm::is_contained(link.getInputObjectFifos(), *
this))
2457 return ObjectFifoDmaEndpointOp::canTakeTurns(
2458 usesPushQueue(*
this),
2459 pinned(getProdPortAttr()) || pinned(getConsumerPort(0)), dispatch,
2460 packetHeader() !=
nullptr, getIterCount().has_value(), repeat,
2464bool ObjectFifoDmaEndpointOp::canTakeTurns(
bool pushQueue,
bool pinned,
2465 bool dispatch,
bool packet,
2466 bool iterates, int64_t repeat,
2468 return !pushQueue && !pinned && !dispatch && !packet && !iterates &&
2469 repeat == 1 && !walksStream;
2472int ObjectFifoCreateOp::sharedDepth() {
2474 if (
auto depths = dyn_cast<ArrayAttr>(getElemNumber()))
2475 for (
int index = 1, e = depths.size(); index < e; ++index)
2476 depth = std::max(depth, size(index));
2484LogicalResult ObjectFifoAllocateOp::verify() {
2485 ObjectFifoCreateOp objFifo = getObjectFifo();
2487 return emitError(
"cannot retrieve associated object FIFO");
2488 if (!isa_and_nonnull<TileLike>(getDelegateTile().getDefiningOp()))
2489 return emitOpError(
"delegate must be a tile");
2490 if (objFifo.getConsumerTiles().size() != 1)
2491 return emitError(
"can only be used in 1-to-1 object FIFOs");
2495 std::optional<ObjectFifoLinkOp> linked = objFifo.getLink();
2496 if (std::optional<StringRef> reason = objFifo.dmaReason())
2497 return emitError(
"cannot allocate a shared memory module to an objectfifo "
2498 "that moves by DMA: ")
2503 ObjectFifoLinkOp link = *linked;
2504 auto producer = dyn_cast<TileOp>(objFifo.getProducerTile().getDefiningOp());
2506 dyn_cast<TileOp>(objFifo.getConsumerTiles()[0].getDefiningOp());
2507 auto delegate = dyn_cast<TileOp>(getDelegateTile().getDefiningOp());
2509 std::optional<Value> linkPoint = link.getOptionalSharedTile();
2511 linkPoint ? dyn_cast<TileOp>(linkPoint->getDefiningOp()) : TileOp();
2512 bool shared = producer && consumer && delegate && linkTile &&
2513 ObjectFifoCreateOp::allocatedHome(
2514 (*this)->getParentOfType<DeviceOp>().getTargetModel(),
2515 producer.getTileID(), consumer.getTileID(),
2516 delegate.getTileID(), linkTile.getTileID());
2517 StringRef name = objFifo.name().getValue();
2519 InFlightDiagnostic diag = emitOpError(
"places @")
2520 << name <<
"'s objects in the link's pool";
2523 ObjectFifoCreateOp owner = link.getPoolOwner();
2524 diag <<
", whose objects are @" << owner.name().getValue() <<
"'s "
2525 << owner.getElemType().getElementType()
2526 <<
"; a fifo that shares memory with a link must have the pool's "
2528 << name <<
"'s is " << objFifo.getElemType().getElementType()
2533 diag <<
", which starts with @" << link.getPoolOwner().name().getValue()
2534 <<
"'s initial objects, "
2535 <<
"but @" << name <<
" has initial objects of its own";
2538 diag <<
", but @" << name
2539 <<
" is a merge's input, and a merge's inputs take turns filling "
2540 "the pool through one DMA end";
2543 diag <<
", but @" << name
2544 <<
" is a dispatch's output, and a dispatch's outputs take turns "
2545 "draining the pool through one DMA end";
2548 diag <<
", but @" << name
2549 <<
" is one link's output and another link's input, so its "
2550 "objects would be both links' pools";
2558 for (Operation *op = (*this)->getPrevNode(); op; op = op->getPrevNode())
2559 if (
auto other = dyn_cast<ObjectFifoAllocateOp>(op);
2560 other && other.getObjFifoName() == getObjFifoName())
2561 return emitOpError(
"objectfifo ")
2562 << getObjFifoNameAttr() <<
" already has an allocate operation";
2566TileOp ObjectFifoAllocateOp::getDelegateTileOp() {
2567 return cast<TileOp>(getDelegateTile().getDefiningOp());
2570ObjectFifoCreateOp ObjectFifoAllocateOp::getObjectFifo() {
2571 Operation *parent = getOperation();
2572 while ((parent = parent->getParentOp())) {
2573 if (parent->hasTrait<OpTrait::SymbolTable>()) {
2574 if (
auto *st = SymbolTable::lookupSymbolIn(parent, getObjFifoName());
2575 isa_and_nonnull<ObjectFifoCreateOp>(st))
2576 return cast<ObjectFifoCreateOp>(st);
2586LogicalResult ObjectFifoLinkOp::verify() {
2587 for (Attribute name : llvm::concat<const Attribute>(
2588 getFifoIns().getValue(), getFifoOuts().getValue())) {
2589 auto flat = dyn_cast<FlatSymbolRefAttr>(name);
2590 if (!flat || !isa_and_nonnull<ObjectFifoCreateOp>(
2591 SymbolTable::lookupNearestSymbolFrom(*
this, flat)))
2592 return emitOpError(
"names no objectfifo ") << name;
2595 if (isJoin() && isDistribute())
2596 return emitError(
"ObjectFifoLinkOp does not support 'join' and "
2597 "'distribute' at the same time");
2600 if (isMerge() && isDispatch())
2601 return emitOpError(
"a time link merges or dispatches, not both");
2602 if (!isMerge() && !isDispatch())
2603 return emitOpError(
"a time link takes turns, so it needs several inputs "
2604 "or several outputs");
2605 if (!getSrcOffsets().empty() || !getDstOffsets().empty())
2606 return emitOpError(
"a time link moves whole objects and takes no "
2608 std::vector<ObjectFifoCreateOp> fifos = getInputObjectFifos();
2609 std::vector<ObjectFifoCreateOp> outs = getOutputObjectFifos();
2610 fifos.insert(fifos.end(), outs.begin(), outs.end());
2611 for (ObjectFifoCreateOp fifo : fifos) {
2612 if (fifo.getElemType() != fifos.front().getElemType())
2613 return emitOpError(
"a time link's participants carry one object "
2615 << fifo.name().getValue() <<
"' carries " << fifo.getElemType()
2616 <<
" and '" << fifos.front().name().getValue() <<
"' carries "
2617 << fifos.front().getElemType();
2620 for (ObjectFifoCreateOp out : outs)
2621 if (out.getConsumerTiles().size() != 1)
2622 return emitOpError(
"each turn of a dispatch reaches one consumer, "
2624 << out.name().getValue() <<
"' has "
2625 << out.getConsumerTiles().size();
2628 PacketInfoAttr pinned;
2629 for (ObjectFifoCreateOp in : getInputObjectFifos()) {
2630 PacketInfoAttr header = in.packetHeader();
2631 if (!header || !header.isAssigned())
2633 if (pinned && pinned != header)
2634 return emitOpError(
"a merge is one route with one header, but its "
2635 "inputs pin different ones");
2641 auto sharedTile = getOptionalSharedTile();
2643 return emitError(
"ObjectFifoLinkOp must have a link point, i.e., a "
2644 "shared tile between objectFifos");
2648 for (ObjectFifoCreateOp out : getOutputObjectFifos())
2649 if (out.getProducerTile() != *sharedTile)
2650 return emitOpError(
"a time link's participants meet on its tile, but '")
2651 << out.name().getValue() <<
"' is produced on another";
2652 for (ObjectFifoCreateOp in : getInputObjectFifos())
2653 if (!llvm::is_contained(in.getConsumerTiles(), *sharedTile))
2654 return emitOpError(
"a time link's participants meet on its tile, but '")
2655 << in.name().getValue() <<
"' is consumed on others";
2661 DenseMap<Operation *, size_t> reads;
2663 (*this)->getParentOfType<DeviceOp>().getOps<ObjectFifoLinkOp>()) {
2664 if (other == *
this) {
2667 for (ObjectFifoCreateOp fifo : getOutputObjectFifos())
2668 if (llvm::is_contained(other.getOutputObjectFifos(), fifo))
2669 return fifo.emitOpError(
2670 "objectfifo cannot be filled by more than one ObjectFifoLinkOp");
2671 for (ObjectFifoCreateOp fifo : getInputObjectFifos())
2672 if (llvm::is_contained(other.getInputObjectFifos(), fifo) &&
2673 other.getOptionalSharedTile() == sharedTile)
2676 for (ObjectFifoCreateOp fifo : getInputObjectFifos())
2677 if (reads.lookup(fifo) >=
2678 static_cast<size_t>(llvm::count(fifo.getConsumerTiles(), *sharedTile)))
2679 return fifo.emitOpError(
"objectfifo is read by more ObjectFifoLinkOps "
2680 "on a tile than it has consumer ends there");
2682 TileLike tile = llvm::dyn_cast<TileLike>(sharedTile.value().getDefiningOp());
2684 return emitError(
"shared tile must implement TileLike interface");
2688 if (isJoin() || isDistribute()) {
2689 if (tile.isShimTile())
2690 return emitError(
"ObjectFifoLinkOp join and distribute are "
2691 "unavailable on shim tiles");
2694 return emitError(
"ObjectFifoLinkOp join and distribute require "
2695 "semaphore locks, which this device lacks");
2697 if (isTime() && tile.isShimTile())
2698 return emitOpError(
"a time link holds its objects on the link point, which "
2699 "a shim tile has no memory for");
2703 auto elementsOf = [](Type objectFifoType) {
2704 return cast<MemRefType>(
2705 cast<AIEObjectFifoType>(objectFifoType).getElementType())
2709 DataLayout dataLayout = DataLayout::closest(getOperation());
2710 auto bytesOf = [&](Type objectFifoType, int64_t elements) {
2712 static_cast<int64_t
>(dataLayout.getTypeSize(
2714 cast<AIEObjectFifoType>(objectFifoType).getElementType())
2715 .getElementType()));
2717 auto checkSlot = [&](StringRef role, ObjectFifoCreateOp participant,
2718 int64_t object, ObjectFifoCreateOp shared,
2719 int64_t slotBytes, ArrayAttr offsets) -> LogicalResult {
2721 return emitOpError() << role <<
" @" << participant.name().getValue()
2722 <<
" has empty objects";
2723 if (slotBytes %
object == 0)
2725 std::string offsetText;
2726 llvm::raw_string_ostream os(offsetText);
2727 llvm::interleaveComma(offsets, os, [&](Attribute offset) {
2728 os << *getConstantIntValue(offset);
2730 return emitOpError() << role <<
" @" << participant.name().getValue()
2731 <<
" has " <<
object <<
" B objects, but its "
2732 << slotBytes <<
" B slot in @"
2733 << shared.name().getValue() <<
" from offsets ["
2734 << offsetText <<
"] is not a whole number of them";
2738 if (getFifoIns().size() != getSrcOffsets().size())
2739 return emitOpError(
"number of provided src offsets must be equal "
2740 "to the number of input objectFifos");
2742 if (!getDstOffsets().empty())
2743 return emitOpError(
"dst offsets should be empty for join");
2747 for (
auto [idx, len] : llvm::enumerate(getJoinTransferLengths()))
2749 return emitOpError(
"src offsets must increase: input objectFifo ")
2750 << idx <<
" would transfer " << len <<
" elements";
2752 ObjectFifoCreateOp fifoOut = getOutputObjectFifos()[0];
2753 for (
auto [fifoIn, slot] :
2754 llvm::zip(getInputObjectFifos(), getJoinTransferLengths())) {
2756 fifoIn.getConsumerElemType()
2757 ? bytesOf(fifoIn.getConsumerElemTypeOrDefault(),
2758 elementsOf(fifoIn.getConsumerElemTypeOrDefault()))
2759 : bytesOf(fifoIn.getElemType(),
2761 fifoIn.getPadDimensions().value_or(
2762 ArrayRef<BDPadLayoutAttr>{}),
2763 elementsOf(fifoIn.getElemType())));
2764 if (failed(checkSlot(
"input", fifoIn,
object, fifoOut,
2765 bytesOf(fifoOut.getElemType(), slot),
2769 if (!fifoOut.getDimensionsToStream().empty()) {
2770 int64_t maxIdx = getDimsMaxIdx(fifoOut.getDimensionsToStream());
2771 int64_t minInputBufferSize = -1;
2772 for (
auto lenIn : getJoinTransferLengths()) {
2773 if (lenIn <= minInputBufferSize || minInputBufferSize < 0)
2774 minInputBufferSize = lenIn;
2776 if (minInputBufferSize <= maxIdx) {
2777 return emitOpError()
2778 <<
"specified output stride(s) and size(s) result in out "
2779 "of bounds access in join input, for index "
2780 << std::to_string(maxIdx) <<
" in transfer of length "
2781 << std::to_string(minInputBufferSize) <<
".";
2785 }
else if (isDistribute()) {
2786 if (getFifoOuts().size() != getDstOffsets().size())
2787 return emitOpError(
"number of provided dst offsets must be equal "
2788 "to the number of output objectFifos");
2790 if (!getSrcOffsets().empty())
2791 return emitOpError(
"src offsets should be empty for distribute");
2793 for (
auto [idx, len] : llvm::enumerate(getDistributeTransferLengths()))
2795 return emitOpError(
"dst offsets must increase: output objectFifo ")
2796 << idx <<
" would transfer " << len <<
" elements";
2798 ObjectFifoCreateOp fifoIn = getInputObjectFifos()[0];
2799 for (
auto [fifoOut, slot] :
2800 llvm::zip(getOutputObjectFifos(), getDistributeTransferLengths())) {
2802 if (failed(checkSlot(
2804 bytesOf(fifoOut.getElemType(), elementsOf(fifoOut.getElemType())),
2806 bytesOf(fifoIn.getElemType(),
2808 fifoOut.getPadDimensions().value_or(
2809 ArrayRef<BDPadLayoutAttr>{}),
2816 for (
size_t seen = 0; end < fifoIn.getConsumerTiles().size(); end++)
2817 if (fifoIn.getConsumerTiles()[end] == *sharedTile &&
2818 seen++ == reads.lookup(fifoIn))
2820 BDDimLayoutArrayAttr dims =
2821 fifoIn.getDimensionsFromStreamPerConsumer()[end];
2822 if (!dims.empty()) {
2823 int64_t maxIdx = getDimsMaxIdx(dims);
2824 int64_t minOutputBufferSize = -1;
2825 for (
auto lenOut : getDistributeTransferLengths()) {
2826 if (lenOut <= minOutputBufferSize || minOutputBufferSize < 0)
2827 minOutputBufferSize = lenOut;
2829 if (minOutputBufferSize <= maxIdx) {
2830 return emitOpError()
2831 <<
"specified input stride(s) and size(s) result in out "
2832 "of bounds access in distribute output, for index "
2833 << std::to_string(maxIdx) <<
" in transfer of length "
2834 << std::to_string(minOutputBufferSize) <<
".";
2838 if (!getSrcOffsets().empty() && !getDstOffsets().empty())
2839 return emitOpError(
"all offsets should be empty if there is no "
2840 "join or distribute");
2843 std::vector<ObjectFifoCreateOp> outs = getOutputObjectFifos();
2844 if (!isTime() && !walksStream() && !outs.empty()) {
2845 ArrayRef<BDDimLayoutAttr> dims = outs[0].getDimensionsToStream();
2846 std::optional<ArrayRef<BDPadLayoutAttr>> pads =
2847 outs[0].getPadDimensions();
2849 for (BDDimLayoutAttr dim : dims)
2850 walked *= dim.getSize();
2851 int64_t
object = elementsOf(getPoolOwner().getElemType());
2852 if (!dims.empty() && (!pads || pads->empty()) && walked !=
object)
2853 return emitOpError()
2854 <<
"@" << outs[0].name().getValue() <<
"'s walk covers "
2855 << walked <<
" elements, but each transfer sends a whole "
2857 <<
"-element object; to send part of each input object, walk "
2858 "the stream (an outer dimension stepping whole input "
2863 if (walksStream()) {
2865 if (failed(getStreamWindow(window)))
2867 if (!tile.isMemTile())
2868 return emitOpError(
"a walk over a stream needs a MemTile link point");
2869 ObjectFifoCreateOp in = getInputObjectFifos()[0];
2870 ObjectFifoCreateOp out = getOutputObjectFifos()[0];
2871 if (in.getRepeatCount() || out.getRepeatCount())
2872 return emitOpError(
"a walk over a stream does not repeat objects");
2873 if (in.getIterCount() || out.getIterCount())
2874 return emitOpError(
"a walk over a stream runs without an iter_count");
2879 return emitOpError() <<
"a walk over a stream of " << window.
objects
2880 <<
" objects needs a lock counting that many, "
2881 "but locks count to "
2883 int64_t bds = target.
getNumBDs(AIETileType::MemTile) / 2;
2885 return emitOpError() <<
"a walk over a stream lowers to one descriptor "
2887 << window.
outputs <<
", but a MemTile channel has "
2893 if (isDistribute()) {
2894 auto fifos = getOutputObjectFifos();
2895 for (
size_t i = 1; i < fifos.size(); ++i) {
2896 ObjectFifoCreateOp first = fifos[0], fifo = fifos[i];
2897 int repeat = first.getRepeatCount().value_or(1);
2898 int otherRepeat = fifo.getRepeatCount().value_or(1);
2899 if (repeat != otherRepeat)
2900 return emitOpError()
2901 <<
"distribute outputs must have equal effective repeat counts: "
2903 << first.name().getValue() <<
" has " << repeat
2904 << (first.getRepeatCount() ?
"" :
" (default)") <<
", @"
2905 << fifo.name().getValue() <<
" has " << otherRepeat
2906 << (fifo.getRepeatCount() ?
"" :
" (default)");
2913std::optional<Value> ObjectFifoLinkOp::getOptionalSharedTile() {
2914 std::vector<ObjectFifoCreateOp> fifoIn = getInputObjectFifos();
2915 std::vector<ObjectFifoCreateOp> fifoOut = getOutputObjectFifos();
2917 if (fifoIn.size() < getFifoIns().size() ||
2918 fifoOut.size() < getFifoOuts().size() || fifoIn.empty() ||
2925 Value tile = fifoOut[0].getProducerTile();
2926 for (
auto in : fifoIn)
2927 if (!llvm::is_contained(in.getConsumerTiles(), tile))
2932 if (isDistribute()) {
2933 Value tile = fifoOut[0].getProducerTile();
2934 for (
auto out : fifoOut)
2935 if (out.getProducerTile() != tile)
2937 if (!llvm::is_contained(fifoIn[0].getConsumerTiles(), tile))
2942 for (
auto consumerIn : fifoIn[0].getConsumerTiles())
2943 if (consumerIn == fifoOut[0].getProducerTile())
2944 return {fifoOut[0].getProducerTile()};
2948bool ObjectFifoLinkOp::walksStream() {
2949 if (isJoin() || isDistribute())
2951 std::vector<ObjectFifoCreateOp> outs = getOutputObjectFifos();
2954 ArrayRef<BDDimLayoutAttr> dims = outs[0].getDimensionsToStream();
2955 if (dims.size() < 2 || dims.front().getSize() < 2)
2958 for (BDDimLayoutAttr dim : dims.drop_front())
2959 inner *= dim.getSize();
2960 int64_t objectSize =
2961 cast<AIEObjectFifoType>(getInputObjectFifos()[0].getElemType())
2965 return getDimsMaxIdx(dims) >= objectSize &&
2966 inner == cast<AIEObjectFifoType>(outs[0].getElemType())
2971LogicalResult ObjectFifoLinkOp::getStreamWindow(
StreamWindow &window) {
2972 ObjectFifoCreateOp in = getInputObjectFifos()[0];
2973 ObjectFifoCreateOp out = getOutputObjectFifos()[0];
2974 ArrayRef<BDPadLayoutAttr> pads;
2975 if (
auto padding = out.getPadDimensions())
2978 out.getDimensionsToStream(), pads,
2979 cast<AIEObjectFifoType>(in.getElemType())
2982 [&] { return emitOpError(); }, window);
2985std::optional<unsigned> ObjectFifoLinkOp::getOwnerConsumerIndex() {
2986 ObjectFifoCreateOp owner = getPoolOwner();
2987 if (!getOptionalSharedTile() ||
2988 llvm::is_contained(getOutputObjectFifos(), owner))
2989 return std::nullopt;
2990 for (
unsigned index = 0, e = owner.getConsumerTiles().size(); index < e;
2992 if (owner.getConsumerLink(index) == *
this)
2994 return std::nullopt;
2997int64_t ObjectFifoLinkOp::getPoolDepth() {
return getPoolDepth(placedAt); }
2999int64_t ObjectFifoLinkOp::getPoolDepth(
TilePositions positions) {
3004 LinkPoolPlan plan{getOptionalSharedTile().value_or(Value()), 0, {}};
3006 if (walksStream()) {
3007 plan.
depth = succeeded(getStreamWindow(window)) ? window.
objects : 0;
3010 ObjectFifoCreateOp owner = getPoolOwner();
3011 if (!owner.getAllocate()) {
3012 std::optional<unsigned> index = getOwnerConsumerIndex();
3013 plan.
depth = index ? owner.dmaDepth(index) : owner.size();
3019 std::vector<ObjectFifoCreateOp> ins = getInputObjectFifos();
3020 std::vector<ObjectFifoCreateOp> outs = getOutputObjectFifos();
3021 for (ObjectFifoCreateOp fifo : llvm::concat<ObjectFifoCreateOp>(ins, outs)) {
3022 std::optional<ObjectFifoAllocateOp> alloc = fifo.getAllocate();
3026 plan.
depth = std::max<int64_t>(
3027 {plan.
depth, fifo.dmaDepth(std::nullopt), fifo.dmaDepth(0)});
3028 else if (plan.
linkTile && fifo.getConsumerTiles().size() == 1 &&
3031 {fifo.getProducerTile(), fifo.getConsumerTiles()[0],
3032 alloc->getDelegateTile(),
3033 llvm::is_contained(outs, fifo) ? fifo.dmaDepth(0)
3034 : fifo.dmaDepth(std::nullopt)});
3041 int64_t result =
depth;
3042 std::optional<TileID> link =
3045 std::optional<TileID> producer = positions(sharer.producer);
3046 std::optional<TileID> consumer = positions(sharer.consumer);
3047 std::optional<TileID> delegate = positions(sharer.delegate);
3048 if (link && producer && consumer && delegate &&
3049 ObjectFifoCreateOp::allocatedHome(target, *producer, *consumer,
3051 result = std::max(result, sharer.depth);
3056ObjectFifoCreateOp ObjectFifoLinkOp::getPoolOwner() {
3057 std::vector<ObjectFifoCreateOp> ins = getInputObjectFifos();
3058 std::vector<ObjectFifoCreateOp> outs = getOutputObjectFifos();
3059 if (isJoin() || isMerge())
3061 if (isDistribute() || isDispatch())
3066 auto count = [](ObjectFifoCreateOp fifo) {
3067 return cast<AIEObjectFifoType>(fifo.getElemType())
3072 if (outs[0].getInitValues() ||
3073 (count(outs[0]) > count(ins[0]) && !outs[0].getPadDimensions()))
3078SmallVector<std::pair<int64_t, int64_t>> ObjectFifoLinkOp::getPoolSegments() {
3079 auto count = [](ObjectFifoCreateOp fifo) {
3080 return cast<AIEObjectFifoType>(fifo.getElemType())
3084 int64_t whole = count(getPoolOwner());
3085 if (!isJoin() && !isDistribute())
3086 return {{0, whole}};
3087 ArrayAttr offsets = isJoin() ? getSrcOffsets() : getDstOffsets();
3088 std::vector<ObjectFifoCreateOp> side =
3089 isJoin() ? getInputObjectFifos() : getOutputObjectFifos();
3090 SmallVector<std::pair<int64_t, int64_t>> segments;
3091 for (
auto [index, participant] : llvm::enumerate(side)) {
3092 int64_t offset = cast<IntegerAttr>(offsets[index]).getInt();
3093 int64_t next = index + 1 < side.size()
3094 ? cast<IntegerAttr>(offsets[index + 1]).getInt()
3096 int64_t size = next - offset;
3098 size = count(participant);
3099 segments.emplace_back(offset, size);
3104std::vector<ObjectFifoCreateOp> ObjectFifoLinkOp::getInputObjectFifos() {
3105 std::vector<ObjectFifoCreateOp> inputObjFifos;
3106 Operation *parent = getOperation();
3107 while ((parent = parent->getParentOp())) {
3108 if (parent->hasTrait<OpTrait::SymbolTable>()) {
3109 for (
auto sym : getFifoIns()) {
3110 auto name = dyn_cast<FlatSymbolRefAttr>(sym);
3113 if (
auto *st = SymbolTable::lookupSymbolIn(parent, name);
3114 isa_and_nonnull<ObjectFifoCreateOp>(st))
3115 inputObjFifos.push_back(cast<ObjectFifoCreateOp>(st));
3119 return inputObjFifos;
3122std::vector<ObjectFifoCreateOp> ObjectFifoLinkOp::getOutputObjectFifos() {
3123 std::vector<ObjectFifoCreateOp> outputObjFifos;
3124 Operation *parent = getOperation();
3125 while ((parent = parent->getParentOp())) {
3126 if (parent->hasTrait<OpTrait::SymbolTable>()) {
3127 for (
auto sym : getFifoOuts()) {
3128 auto name = dyn_cast<FlatSymbolRefAttr>(sym);
3131 if (
auto *st = mlir::SymbolTable::lookupSymbolIn(parent, name);
3132 isa_and_nonnull<ObjectFifoCreateOp>(st))
3133 outputObjFifos.push_back(cast<ObjectFifoCreateOp>(st));
3137 return outputObjFifos;
3140std::vector<int> ObjectFifoLinkOp::getJoinTransferLengths() {
3141 std::vector<int> lengths;
3144 llvm::cast<AIEObjectFifoType>(getOutputObjectFifos()[0].getElemType());
3145 auto elemTypeOut = llvm::cast<MemRefType>(fifoOut.getElementType());
3146 int lenOut = elemTypeOut.getNumElements();
3149 auto srcOffset = [&](
size_t idx) ->
int {
3150 std::optional<int64_t> v = getConstantIntValue(getSrcOffsets()[idx]);
3151 assert(v &&
"src_offsets element must be a constant integer");
3152 return static_cast<int>(*v);
3154 for (
size_t i = 0; i < getFifoIns().size(); i++) {
3156 int offset = srcOffset(i);
3157 if (i == getFifoIns().size() - 1)
3158 len = lenOut - offset;
3160 len = srcOffset(i + 1) - offset;
3161 lengths.push_back(len);
3167std::vector<int> ObjectFifoLinkOp::getDistributeTransferLengths() {
3168 std::vector<int> lengths;
3169 if (isDistribute()) {
3171 llvm::cast<AIEObjectFifoType>(getInputObjectFifos()[0].getElemType());
3172 auto elemTypeIn = llvm::cast<MemRefType>(fifoIn.getElementType());
3173 int lenIn = elemTypeIn.getNumElements();
3176 auto dstOffset = [&](
size_t idx) ->
int {
3177 std::optional<int64_t> v = getConstantIntValue(getDstOffsets()[idx]);
3178 assert(v &&
"dst_offsets element must be a constant integer");
3179 return static_cast<int>(*v);
3181 for (
size_t i = 0; i < getFifoOuts().size(); i++) {
3182 int offset = dstOffset(i);
3184 if (i == getFifoOuts().size() - 1)
3185 len = lenIn - offset;
3187 len = dstOffset(i + 1) - offset;
3188 lengths.push_back(len);
3194std::optional<int> ObjectFifoLinkOp::getRepeatCount() {
3195 for (
auto fifoOut : getOutputObjectFifos())
3196 if (fifoOut.getRepeatCount().has_value())
3197 return {fifoOut.getRepeatCount()};
3205TileOp ObjectFifoRegisterExternalBuffersOp::getTileOp() {
3206 return cast<TileOp>(getTile().getDefiningOp());
3209ObjectFifoCreateOp ObjectFifoRegisterExternalBuffersOp::getObjectFifo() {
3210 Operation *parent = getOperation();
3211 while ((parent = parent->getParentOp())) {
3212 if (parent->hasTrait<OpTrait::SymbolTable>()) {
3213 if (
auto *st = SymbolTable::lookupSymbolIn(parent, getObjFifoName());
3214 isa_and_nonnull<ObjectFifoCreateOp>(st))
3215 return cast<ObjectFifoCreateOp>(st);
3229Operation *lookupAccessTarget(Operation *op, StringRef name) {
3230 for (Operation *parent = op->getParentOp(); parent;
3231 parent = parent->getParentOp()) {
3232 if (parent->hasTrait<OpTrait::SymbolTable>()) {
3233 if (Operation *target = SymbolTable::lookupSymbolIn(parent, name)) {
3243Type accessElementType(Operation *target) {
3244 if (
auto fifo = dyn_cast_or_null<ObjectFifoCreateOp>(target)) {
3245 return llvm::cast<AIEObjectFifoType>(fifo.getElemType()).getElementType();
3247 if (
auto endpoint = dyn_cast_or_null<ObjectFifoCoreEndpointOp>(target)) {
3248 if (endpoint.getPoolOp()) {
3249 return endpoint.getAccessType();
3256LogicalResult verifyAccessPlacement(Operation *op, Operation *target,
3257 std::optional<ObjectFifoPort> port) {
3258 auto core = op->getParentOfType<CoreOp>();
3259 Value coreTile = core.getTile();
3261 if (
auto endpoint = dyn_cast_or_null<ObjectFifoCoreEndpointOp>(target)) {
3263 return op->emitOpError(
"port is implied by the endpoint's role");
3265 if (endpoint.getTile() != coreTile) {
3266 return core.emitOpError(
3267 "objectFifo endpoint accessed by core running on another tile");
3272 auto fifo = dyn_cast_or_null<ObjectFifoCreateOp>(target);
3274 return op->emitError(
"cannot retrieve associated object FIFO");
3277 return op->emitOpError(
"port is required when accessing an objectFifo");
3280 if (*port == ObjectFifoPort::Produce) {
3281 if (coreTile != fifo.getProducerTile()) {
3282 return core.emitOpError(
3283 "producer port of objectFifo accessed by core running "
3284 "on non-producer tile");
3287 if (!llvm::is_contained(fifo.getConsumerTiles(), coreTile)) {
3288 return core.emitOpError(
3289 "consumer port of objectFifo accessed by core running "
3290 "on non-consumer tile");
3298LogicalResult ObjectFifoAcquireOp::verify() {
3299 auto parent = getOperation()->getParentOfType<CoreOp>();
3300 if (parent ==
nullptr) {
3301 return emitOpError(
"must be called from inside a CoreOp");
3304 Operation *target = lookupAccessTarget(*
this, getObjFifoName());
3305 if (failed(verifyAccessPlacement(*
this, target, getPort()))) {
3309 if (getObjects().empty()) {
3310 return emitOpError(
"must acquire at least one object");
3313 Type elem = accessElementType(target);
3317 auto fifo = dyn_cast<ObjectFifoCreateOp>(target);
3321 if (getPort() == ObjectFifoPort::Consume) {
3322 for (
auto [position, consumer] :
3323 llvm::enumerate(fifo.getConsumerTiles())) {
3324 if (consumer == parent.getTile()) {
3325 index = position + 1;
3330 if (
static_cast<int>(getObjects().size()) > fifo.size(index)) {
3331 return emitOpError(
"acquires ")
3332 << getObjects().size() <<
" objects from an objectFifo that holds "
3333 << fifo.size(index);
3337 auto consType = fifo ? llvm::dyn_cast<AIEObjectFifoType>(
3338 fifo.getConsumerElemTypeOrDefault())
3340 if (fifo && !consType) {
3341 return emitOpError(
"ObjectFifo consumer element type must be an "
3342 "!aie.objectfifo<memref<...>> type");
3345 for (Type
object : getObjects().getTypes()) {
3346 if (
object != elem && (!consType || consType.getElementType() !=
object)) {
3347 return emitOpError(
"acquired object type ")
3348 <<
object <<
" does not match the objectFifo's " << elem;
3355ObjectFifoCreateOp ObjectFifoAcquireOp::getObjectFifo() {
3356 Operation *parent = getOperation();
3357 while ((parent = parent->getParentOp())) {
3358 if (parent->hasTrait<OpTrait::SymbolTable>()) {
3359 if (
auto *st = SymbolTable::lookupSymbolIn(parent, getObjFifoName());
3360 isa_and_nonnull<ObjectFifoCreateOp>(st))
3361 return cast<ObjectFifoCreateOp>(st);
3371LogicalResult ObjectFifoReleaseOp::verify() {
3372 auto parent = getOperation()->getParentOfType<CoreOp>();
3373 if (parent ==
nullptr)
3374 return emitOpError(
"must be called from inside a CoreOp");
3376 Operation *target = lookupAccessTarget(*
this, getObjFifoName());
3377 return verifyAccessPlacement(*
this, target, getPort());
3380ObjectFifoCreateOp ObjectFifoReleaseOp::getObjectFifo() {
3381 Operation *parent = getOperation();
3382 while ((parent = parent->getParentOp())) {
3383 if (parent->hasTrait<OpTrait::SymbolTable>()) {
3384 if (
auto *st = SymbolTable::lookupSymbolIn(parent, getObjFifoName());
3385 isa_and_nonnull<ObjectFifoCreateOp>(st))
3386 return cast<ObjectFifoCreateOp>(st);
3396LogicalResult CascadeFlowOp::verify() {
3397 TileLike src = getSourceTileLike();
3398 TileLike dst = getDestTileLike();
3401 return emitOpError(
"source and dest must be tile-like operations");
3403 if (src.isShimTile() || dst.isShimTile())
3404 return emitOpError(
"shimTile row has no cascade stream interface");
3405 if (src.isMemTile() || dst.isMemTile())
3406 return emitOpError(
"memTile row has no cascade stream interface");
3411 bool onTarget = (*this)->getParentOfType<AIETarget>() !=
nullptr;
3412 if (onTarget && t.getCascadeSteps().empty())
3413 return emitOpError(
"cascade not supported in ")
3414 << stringifyAIEArch(t.getTargetArch());
3416 std::optional<int> srcCol = src.tryGetCol();
3417 std::optional<int> srcRow = src.tryGetRow();
3418 std::optional<int> dstCol = dst.tryGetCol();
3419 std::optional<int> dstRow = dst.tryGetRow();
3421 if (srcCol && srcRow && dstCol && dstRow) {
3422 if (!t.isSouth(*srcCol, *srcRow, *dstCol, *dstRow) &&
3423 !t.isWest(*srcCol, *srcRow, *dstCol, *dstRow) &&
3424 !t.isNorth(*srcCol, *srcRow, *dstCol, *dstRow) &&
3425 !t.isEast(*srcCol, *srcRow, *dstCol, *dstRow)) {
3426 return emitOpError(
"tiles must be adjacent");
3428 if (onTarget && !t.isLegalCascade(*srcCol, *srcRow, *dstCol, *dstRow))
3429 return emitOpError(
"source tile must be to the North or West of the "
3430 "destination tile");
3436TileLike CascadeFlowOp::getSourceTileLike() {
3437 return dyn_cast<TileLike>(getSourceTile().getDefiningOp());
3440TileLike CascadeFlowOp::getDestTileLike() {
3441 return dyn_cast<TileLike>(getDestTile().getDefiningOp());
3444TileOp CascadeFlowOp::getSourceTileOp() {
3445 if (
auto tileOp = dyn_cast_or_null<TileOp>(getSourceTile().getDefiningOp()))
3447 llvm::report_fatal_error(
"Calling getSourceTileOp requires TileOp.");
3450TileOp CascadeFlowOp::getDestTileOp() {
3451 if (
auto tileOp = dyn_cast_or_null<TileOp>(getDestTile().getDefiningOp()))
3453 llvm::report_fatal_error(
"Calling getDestTileOp requires TileOp.");
3462static CascadeDir cascadeStepOutputDir(std::pair<int, int> step) {
3463 if (step.first == 1)
3464 return CascadeDir::East;
3465 if (step.first == -1)
3466 return CascadeDir::West;
3467 return step.second == 1 ? CascadeDir::North : CascadeDir::South;
3470static CascadeDir oppositeCascadeDir(CascadeDir dir) {
3472 case CascadeDir::North:
3473 return CascadeDir::South;
3474 case CascadeDir::South:
3475 return CascadeDir::North;
3476 case CascadeDir::East:
3477 return CascadeDir::West;
3478 case CascadeDir::West:
3479 return CascadeDir::East;
3481 llvm_unreachable(
"unhandled CascadeDir");
3484static std::string describeCascadeDirs(ArrayRef<CascadeDir> dirs) {
3486 llvm::raw_string_ostream os(s);
3488 dirs, os, [&](CascadeDir d) { os << stringifyCascadeDir(d); },
" or ");
3492LogicalResult ConfigureCascadeOp::verify() {
3493 if (!isa<TileOp>(getTile().getDefiningOp()))
3494 return emitOpError(
"requires a placed tile (aie.tile), not a logical tile");
3497 TileOp tile = cast<TileOp>(getTile().getDefiningOp());
3498 CascadeDir inputDir = getInputDir();
3499 CascadeDir outputDir = getOutputDir();
3501 if (tile.isShimTile())
3502 return emitOpError(
"shimTile row has no cascade stream interface");
3503 if (tile.isMemTile())
3504 return emitOpError(
"memTile row has no cascade stream interface");
3506 llvm::ArrayRef<std::pair<int, int>> steps = t.getCascadeSteps();
3507 if (!steps.empty()) {
3508 SmallVector<CascadeDir> legalOutputs, legalInputs;
3509 for (std::pair<int, int> step : steps) {
3510 legalOutputs.push_back(cascadeStepOutputDir(step));
3511 legalInputs.push_back(oppositeCascadeDir(legalOutputs.back()));
3513 if (!llvm::is_contained(legalInputs, inputDir))
3514 return emitOpError(
"input direction of cascade must be ")
3515 << describeCascadeDirs(legalInputs) <<
" on "
3516 << stringifyAIEArch(t.getTargetArch());
3517 if (!llvm::is_contained(legalOutputs, outputDir))
3518 return emitOpError(
"output direction of cascade must be ")
3519 << describeCascadeDirs(legalOutputs) <<
" on "
3520 << stringifyAIEArch(t.getTargetArch());
3522 return emitOpError(
"cascade not supported in ")
3523 << stringifyAIEArch(t.getTargetArch());
3532LogicalResult PutCascadeOp::verify() {
3534 Type type = getCascadeValue().getType();
3535 DataLayout dataLayout = DataLayout::closest(*
this);
3536 auto bits = dataLayout.getTypeSizeInBits(type);
3537 auto archbits = targetModel.getAccumulatorCascadeSize();
3538 if (bits != archbits)
3539 return emitOpError(
"type must match architecture cascade width (")
3540 << archbits <<
" bits in "
3541 << stringifyAIEArch(targetModel.getTargetArch()) <<
")";
3549LogicalResult GetCascadeOp::verify() {
3551 Type type = getCascadeValue().getType();
3552 DataLayout dataLayout = DataLayout::closest(*
this);
3553 auto bits = dataLayout.getTypeSizeInBits(type);
3554 if (isa<AIE1TargetModel>(targetModel)) {
3556 return emitOpError(
"must be a 384-bit type");
3557 }
else if (isa<AIE2TargetModel>(targetModel)) {
3559 return emitOpError(
"must be a 512-bit type");
3561 return emitOpError(
"cascade not supported in ")
3562 << stringifyAIEArch(targetModel.getTargetArch());
3575using PortKey = std::tuple<int, int, int, int>;
3582std::optional<PortKey> tryGetPortKey(Value tileValue, WireBundle bundle,
3584 auto tile = llvm::dyn_cast_or_null<TileLike>(tileValue.getDefiningOp());
3586 return std::nullopt;
3587 std::optional<int> col = tile.tryGetCol();
3588 std::optional<int> row = tile.tryGetRow();
3590 return std::nullopt;
3591 return PortKey{*col, *row,
static_cast<int>(bundle), channel};
3594std::string to_string(
const PortKey &key) {
3595 auto [col, row, bundle, channel] = key;
3596 return "(" + std::to_string(col) +
", " + std::to_string(row) +
") " +
3597 stringifyWireBundle(
static_cast<WireBundle
>(bundle)).str() +
" : " +
3598 std::to_string(channel);
3606LogicalResult verifyNoDuplicateFlows(DeviceOp device) {
3607 std::map<std::pair<PortKey, PortKey>, FlowOp> flowSeen;
3608 for (FlowOp flow : device.getOps<FlowOp>()) {
3609 std::optional<PortKey> src = tryGetPortKey(
3610 flow.getSource(), flow.getSourceBundle(), flow.getSourceChannel());
3611 std::optional<PortKey> dst = tryGetPortKey(
3612 flow.getDest(), flow.getDestBundle(), flow.getDestChannel());
3615 auto [it, inserted] = flowSeen.try_emplace({*src, *dst}, flow);
3617 InFlightDiagnostic diag = flow.emitOpError()
3618 <<
"duplicates an earlier flow; "
3619 << to_string(*src) <<
" -> " << to_string(*dst)
3620 <<
" is already declared";
3621 diag.attachNote(it->second.getLoc()) <<
"the other flow is here";
3633LogicalResult verifyNoDuplicatePacketFlows(DeviceOp device) {
3634 using PacketFlowKey =
3635 std::tuple<int, int, std::vector<PortKey>, std::vector<PortKey>>;
3636 std::map<PacketFlowKey, PacketFlowOp> packetFlowSeen;
3637 for (PacketFlowOp packetFlow : device.getOps<PacketFlowOp>()) {
3638 Region &body = packetFlow.getPorts();
3643 std::vector<PortKey> sources, dests;
3644 bool comparable =
true;
3645 for (Operation &op : body.front()) {
3646 std::vector<PortKey> *endpoints =
nullptr;
3647 std::optional<PortKey> port;
3648 if (
auto source = dyn_cast<PacketSourceOp>(op)) {
3649 endpoints = &sources;
3650 port = tryGetPortKey(source.getTile(), source.getBundle(),
3651 source.getChannel());
3652 }
else if (
auto dest = dyn_cast<PacketDestOp>(op)) {
3655 tryGetPortKey(dest.getTile(), dest.getBundle(), dest.getChannel());
3664 endpoints->push_back(*port);
3666 if (!comparable || sources.empty() || dests.empty())
3668 llvm::sort(sources);
3670 int mask = packetFlow.getMask().value_or(
3671 device.getTargetModel().getPacketIdMask());
3672 auto [it, inserted] = packetFlowSeen.try_emplace(
3673 {packetFlow.IDInt(), mask, std::move(sources), std::move(dests)},
3676 InFlightDiagnostic diag =
3677 packetFlow.emitOpError()
3678 <<
"duplicates an earlier packet flow; ID " << packetFlow.IDInt()
3679 <<
" under mask 0x" << llvm::utohexstr(mask)
3680 <<
" is already declared between the same sources and destinations";
3681 diag.attachNote(it->second.getLoc()) <<
"the other packet flow is here";
3690static LogicalResult verifyNoSharedProgramChannels(DeviceOp device) {
3691 std::map<std::tuple<Operation *, int, int, DMAChannelDir, int>,
3692 std::pair<DMAStartOp, Operation *>>
3694 for (DmaBody program : device.getOps<DmaBody>()) {
3695 Operation *tile = cast<TileElement>(*program).getTile().getDefiningOp();
3696 auto like = dyn_cast_or_null<TileLike>(tile);
3697 std::optional<int> col = like ? like.tryGetCol() : std::nullopt;
3698 std::optional<int> row = like ? like.tryGetRow() : std::nullopt;
3699 Operation *key = col && row ? nullptr : tile;
3700 for (DMAStartOp start : program.getDmaBody().getOps<DMAStartOp>()) {
3701 if (start.getEndpoint())
3704 starts.try_emplace({key, col.value_or(-1), row.value_or(-1),
3705 start.getChannelDir(), start.getChannelIndex()},
3707 if (added || it->second.second == program)
3709 InFlightDiagnostic diag =
3711 << stringifyDMAChannelDir(start.getChannelDir()) <<
" channel "
3712 << start.getChannelIndex()
3713 <<
" is started by another DMA program on this tile too";
3714 diag.attachNote(it->second.first.getLoc()) <<
"the other start is here";
3724LogicalResult verifyNoSharedCircuitPorts(DeviceOp device) {
3725 std::map<PortKey, FlowOp> circuitSources, circuitDests;
3726 for (FlowOp flow : device.getOps<FlowOp>()) {
3727 std::optional<PortKey> src = tryGetPortKey(
3728 flow.getSource(), flow.getSourceBundle(), flow.getSourceChannel());
3729 std::optional<PortKey> dst = tryGetPortKey(
3730 flow.getDest(), flow.getDestBundle(), flow.getDestChannel());
3732 circuitSources.try_emplace(*src, flow);
3735 auto [it, inserted] = circuitDests.try_emplace(*dst, flow);
3738 InFlightDiagnostic diag =
3740 <<
"ends at " << to_string(*dst)
3741 <<
", where another circuit flow ends; a port takes one circuit";
3742 diag.attachNote(it->second.getLoc()) <<
"the other circuit flow is here";
3745 for (PacketFlowOp packetFlow : device.getOps<PacketFlowOp>()) {
3746 Region &body = packetFlow.getPorts();
3749 for (Operation &op : body.front()) {
3750 std::optional<PortKey> port;
3751 std::map<PortKey, FlowOp> *circuits =
nullptr;
3753 if (
auto source = dyn_cast<PacketSourceOp>(op)) {
3754 port = tryGetPortKey(source.getTile(), source.getBundle(),
3755 source.getChannel());
3756 circuits = &circuitSources;
3758 }
else if (
auto dest = dyn_cast<PacketDestOp>(op)) {
3760 tryGetPortKey(dest.getTile(), dest.getBundle(), dest.getChannel());
3761 circuits = &circuitDests;
3766 auto it = circuits->find(*port);
3767 if (it == circuits->end())
3769 InFlightDiagnostic diag =
3770 packetFlow.emitOpError()
3771 << role <<
" at " << to_string(*port) <<
", where a circuit flow "
3772 << role <<
"; a port carries either one circuit or packets";
3773 diag.attachNote(it->second.getLoc()) <<
"the circuit flow is here";
3780LogicalResult verifyNoSharedCircuitViaEgresses(DeviceOp device) {
3781 enum class ClaimKind { FlowDest, PacketDest,
Connect, MasterSet, Via };
3785 size_t viaIndex = 0;
3787 std::map<PortKey, Claim> claimedEgresses;
3789 auto addInterconnectClaims = [&](
auto interconnect) {
3790 Value tile = interconnect.getTile();
3791 for (ConnectOp connect :
3792 interconnect.getConnections().template getOps<ConnectOp>()) {
3793 Port dest = connect.destPort();
3794 if (std::optional<PortKey> port =
3795 tryGetPortKey(tile, dest.bundle, dest.channel))
3796 claimedEgresses.try_emplace(*port, Claim{connect, ClaimKind::Connect});
3798 for (MasterSetOp masterSet :
3799 interconnect.getConnections().template getOps<MasterSetOp>()) {
3800 Port dest = masterSet.destPort();
3801 if (std::optional<PortKey> port =
3802 tryGetPortKey(tile, dest.bundle, dest.channel))
3803 claimedEgresses.try_emplace(*port,
3804 Claim{masterSet, ClaimKind::MasterSet});
3807 for (SwitchboxOp switchbox : device.getOps<SwitchboxOp>())
3808 addInterconnectClaims(switchbox);
3809 for (ShimMuxOp shimMux : device.getOps<ShimMuxOp>())
3810 addInterconnectClaims(shimMux);
3811 for (FlowOp flow : device.getOps<FlowOp>()) {
3812 if (std::optional<PortKey> dest = tryGetPortKey(
3813 flow.getDest(), flow.getDestBundle(), flow.getDestChannel()))
3814 claimedEgresses.try_emplace(*dest, Claim{flow, ClaimKind::FlowDest});
3816 for (PacketFlowOp packetFlow : device.getOps<PacketFlowOp>()) {
3817 for (PacketDestOp dest : packetFlow.getPorts().getOps<PacketDestOp>()) {
3818 if (std::optional<PortKey> port = tryGetPortKey(
3819 dest.getTile(), dest.getBundle(), dest.getChannel()))
3820 claimedEgresses.try_emplace(*port, Claim{dest, ClaimKind::PacketDest});
3824 WalkResult result = device.walk([&](FlowOp flow) {
3825 ArrayRef<int32_t> bundles =
3826 flow.getViaEgressBundlesAttr()
3827 ? flow.getViaEgressBundlesAttr().asArrayRef()
3828 : ArrayRef<int32_t>();
3829 ArrayRef<int32_t> channels =
3830 flow.getViaEgressChannelsAttr()
3831 ? flow.getViaEgressChannelsAttr().asArrayRef()
3832 : ArrayRef<int32_t>();
3833 if (bundles.size() != flow.getVias().size() ||
3834 channels.size() != flow.getVias().size()) {
3835 return WalkResult::advance();
3837 for (
auto [index, via] : llvm::enumerate(flow.getVias())) {
3838 std::optional<PortKey> egress = tryGetPortKey(
3839 via,
static_cast<WireBundle
>(bundles[index]), channels[index]);
3843 Claim viaClaim{flow, ClaimKind::Via, index};
3844 auto [it, inserted] = claimedEgresses.try_emplace(*egress, viaClaim);
3847 if (index + 1 == flow.getVias().size() && it->second.op == flow &&
3848 it->second.kind == ClaimKind::FlowDest) {
3849 it->second = viaClaim;
3852 InFlightDiagnostic diag = flow.emitOpError()
3853 <<
"via " << index <<
" claims circuit egress "
3854 << to_string(*egress)
3855 <<
", which another operation already claims";
3856 Diagnostic ¬e = diag.attachNote(it->second.op->getLoc());
3857 switch (it->second.kind) {
3858 case ClaimKind::FlowDest:
3859 note <<
"a circuit flow ends at the same port";
3861 case ClaimKind::PacketDest:
3862 note <<
"a packet flow ends at the same port";
3864 case ClaimKind::Connect:
3865 note <<
"a fixed connect drives the same port";
3867 case ClaimKind::MasterSet:
3868 note <<
"a fixed master set drives the same port";
3870 case ClaimKind::Via:
3871 note <<
"via " << it->second.viaIndex
3872 <<
" claims the same circuit egress";
3875 return WalkResult::interrupt();
3877 return WalkResult::advance();
3879 return failure(result.wasInterrupted());
3887LogicalResult verifyNoDuplicateNames(Operation *symbolTableOp) {
3888 if (symbolTableOp->getNumRegions() == 0 ||
3889 symbolTableOp->getRegion(0).empty()) {
3892 DenseMap<StringAttr, Operation *> nameSeen;
3893 for (Operation &op : symbolTableOp->getRegion(0).front()) {
3894 auto name = op.getAttrOfType<StringAttr>(SymbolTable::getSymbolAttrName());
3898 auto [it, inserted] = nameSeen.try_emplace(name, &op);
3904 if (isa<SymbolOpInterface>(op) && isa<SymbolOpInterface>(it->second)) {
3907 InFlightDiagnostic diag = op.emitError() <<
"redefinition of symbol named '"
3908 << name.getValue() <<
"'";
3909 diag.attachNote(it->second->getLoc())
3910 <<
"see existing symbol definition here";
3918LogicalResult DeviceOp::verify() {
3927 DenseMap<TileID, CoreOp> coreAtTile;
3928 for (CoreOp core : getOps<CoreOp>()) {
3930 llvm::dyn_cast_or_null<TileLike>(core.getTile().getDefiningOp());
3933 std::optional<int> col = tile.tryGetCol();
3934 std::optional<int> row = tile.tryGetRow();
3938 auto [it, inserted] = coreAtTile.try_emplace(
id, core);
3940 InFlightDiagnostic diag =
3942 <<
"tile (" << *col <<
", " << *row
3943 <<
") already has a core; each compute tile can host only one core";
3944 diag.attachNote(it->second.getLoc()) <<
"the other core is here";
3951 if (failed(verifyNoDuplicateFlows(*
this)))
3953 if (failed(verifyNoDuplicatePacketFlows(*
this)))
3955 if (failed(verifyNoSharedCircuitPorts(*
this)))
3957 if (failed(verifyNoSharedProgramChannels(*
this)))
3959 if (failed(verifyNoSharedCircuitViaEgresses(*
this))) {
3963 if (failed(verifyNoDuplicateNames(*
this))) {
3970LogicalResult DeviceOp::verifyRegions() {
3975 WalkResult result = getOperation()->walk<WalkOrder::PreOrder>(
3976 [&](Operation *op) -> WalkResult {
3977 if (op != getOperation() && isa<DeviceOp>(op))
3978 return WalkResult::skip();
3979 if (
auto user = dyn_cast<NamedOpUserOpInterface>(op))
3980 if (failed(user.verifyNamedOpUses(names)))
3981 return WalkResult::interrupt();
3982 return WalkResult::advance();
3984 return failure(result.wasInterrupted());
3991xilinx::AIE::DeviceOp DeviceOp::getForSymbolInModule(mlir::ModuleOp module,
3992 llvm::StringRef symbol) {
3994 if (symbol.empty()) {
3998 Operation *maybeDeviceOp = mlir::SymbolTable::lookupSymbolIn(module, symbol);
3999 if (!maybeDeviceOp) {
4002 deviceOp = llvm::dyn_cast<DeviceOp>(maybeDeviceOp);
4006xilinx::AIE::DeviceOp
4007DeviceOp::getForSymbolInModuleOrError(mlir::ModuleOp module,
4008 llvm::StringRef symbol) {
4009 DeviceOp deviceOp = getForSymbolInModule(module, symbol);
4011 if (!symbol.empty()) {
4012 module.emitError("No such device: ") << symbol;
4014 module.emitError("No 'main' device in module");
4024TileOp TileElement::tryGetTileOp() {
4025 auto element = cast<TileElement>(this->getOperation());
4026 return dyn_cast_or_null<TileOp>(element.getTile().getDefiningOp());
4029TileOp TileElement::getTileOp() {
4030 if (
auto tileOp = tryGetTileOp())
4032 llvm::report_fatal_error(
"Calling getTileOp requires TileOp.");
4039LogicalResult LogicalTileOp::verify() {
4041 *
this, (*this)->getAttrOfType<PacketInfoAttr>(
"controller_id"),
4045 int columns = targetModel.columns();
4046 int rows = targetModel.rows();
4049 if (
auto col = getCol()) {
4050 if (*col >= columns)
4051 return emitOpError(
"column index (")
4053 <<
") must be less than the number of columns in the device ("
4056 if (
auto row = getRow()) {
4058 return emitOpError(
"row index (")
4059 << *row <<
") must be less than the number of rows in the device ("
4064 AIETileType tileType = getTileType();
4065 bool tileTypeExists =
false;
4066 for (
int col = 0; col < columns && !tileTypeExists; col++) {
4067 for (
int row = 0; row < rows && !tileTypeExists; row++) {
4068 if (targetModel.getTileType(col, row) == tileType)
4069 tileTypeExists =
true;
4072 if (!tileTypeExists) {
4073 return emitOpError(
"tile type '")
4074 << stringifyAIETileType(tileType)
4075 <<
"' does not exist on the target device";
4080 if (
auto col = tryGetCol()) {
4081 if (
auto row = tryGetRow()) {
4082 if (targetModel.getTileType(*col, *row) != tileType) {
4083 return emitOpError(
"declared logical tile type does not match "
4084 "the tile type at coordinates (")
4085 << *col <<
", " << *row <<
")";
4093TileID LogicalTileOp::getCanonicalTileID() {
4097 std::optional<int32_t> col = getCol();
4098 std::optional<int32_t> row = getRow();
4099 if (col.has_value() && row.has_value())
4100 return {*col, *row};
4103 AIETileType tileType = getTileType();
4104 for (
int col = 0; col < targetModel.columns(); col++) {
4105 for (
int row = 0; row < targetModel.rows(); row++) {
4106 if (targetModel.getTileType(col, row) == tileType) {
4111 llvm_unreachable(
"No tile of matching tile type found in AIE device");
4114size_t LogicalTileOp::getNumSourceConnections(WireBundle bundle) {
4116 TileID tile = getCanonicalTileID();
4118 if (bundle == WireBundle::Core || bundle == WireBundle::DMA) {
4120 if (isShimNOCorPLTile())
4121 return targetModel.getNumDestShimMuxConnections(tile.col, tile.row,
4123 return targetModel.getNumDestSwitchboxConnections(tile.col, tile.row,
4129size_t LogicalTileOp::getNumDestConnections(WireBundle bundle) {
4131 TileID tile = getCanonicalTileID();
4133 if (bundle == WireBundle::Core || bundle == WireBundle::DMA) {
4135 if (isShimNOCorPLTile())
4136 return targetModel.getNumSourceShimMuxConnections(tile.col, tile.row,
4138 return targetModel.getNumSourceSwitchboxConnections(tile.col, tile.row,
4144std::optional<int> LogicalTileOp::tryGetCol() {
4145 if (
auto col = getCol())
4147 return std::nullopt;
4150std::optional<int> LogicalTileOp::tryGetRow() {
4151 if (
auto row = getRow())
4153 return std::nullopt;
4160ParseResult LogicalTileOp::parse(OpAsmParser &parser, OperationState &result) {
4161 AIETileType tileType;
4162 if (parser.parseLess())
4165 StringRef tileTypeStr;
4166 if (parser.parseKeyword(&tileTypeStr))
4169 auto tileTypeOpt = symbolizeAIETileType(tileTypeStr);
4171 return parser.emitError(parser.getCurrentLocation(),
4172 "unknown logical tile type: ")
4174 tileType = *tileTypeOpt;
4176 if (parser.parseGreater())
4179 if (parser.parseLParen())
4182 std::optional<int32_t> col;
4183 if (succeeded(parser.parseOptionalQuestion())) {
4187 if (parser.parseInteger(colVal))
4192 if (parser.parseComma())
4195 std::optional<int32_t> row;
4196 if (succeeded(parser.parseOptionalQuestion())) {
4200 if (parser.parseInteger(rowVal))
4205 if (parser.parseRParen())
4209 if (parser.parseOptionalAttrDict(result.attributes))
4213 result.getOrAddProperties<LogicalTileOp::Properties>().tile_type =
4214 AIETileTypeAttr::get(parser.getContext(), tileType);
4216 result.getOrAddProperties<LogicalTileOp::Properties>().col =
4217 parser.getBuilder().getI32IntegerAttr(*col);
4219 result.getOrAddProperties<LogicalTileOp::Properties>().row =
4220 parser.getBuilder().getI32IntegerAttr(*row);
4223 result.addTypes(parser.getBuilder().getIndexType());
4228void LogicalTileOp::print(OpAsmPrinter &printer) {
4229 printer <<
"<" << stringifyAIETileType(getTileType()) <<
">";
4232 if (
auto col = getCol())
4237 if (
auto row = getRow())
4243 SmallVector<StringRef, 3> elidedAttrs = {
"tile_type",
"col",
"row"};
4244 printer.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
4251LogicalResult TileOp::verify() {
4253 *
this, (*this)->getAttrOfType<PacketInfoAttr>(
"controller_id"),
4257 int columns = targetModel.columns();
4258 int rows = targetModel.rows();
4259 if (colIndex() >= columns)
4260 return emitOpError(
"column index (")
4262 <<
") must be less than the number of columns in the device ("
4264 if (rowIndex() >= rows)
4265 return emitOpError(
"row index (")
4267 <<
") must be less than the number of rows in the device (" << rows
4270 auto users = getResult().getUsers();
4272 for (
auto *user : users) {
4273 if (llvm::isa<SwitchboxOp>(*user)) {
4275 return emitOpError(
"can only have one switchbox");
4283size_t TileOp::getNumSourceConnections(WireBundle bundle) {
4285 if (bundle == WireBundle::Core || bundle == WireBundle::DMA)
4289 if (isShimNOCorPLTile())
4290 return targetModel.getNumDestShimMuxConnections(getCol(), getRow(),
4292 return targetModel.getNumDestSwitchboxConnections(getCol(), getRow(),
4298size_t TileOp::getNumDestConnections(WireBundle bundle) {
4300 if (bundle == WireBundle::Core || bundle == WireBundle::DMA)
4304 if (isShimNOCorPLTile())
4305 return targetModel.getNumSourceShimMuxConnections(getCol(), getRow(),
4307 return targetModel.getNumSourceSwitchboxConnections(getCol(), getRow(),
4313std::optional<int> TileOp::tryGetCol() {
return getCol(); }
4314std::optional<int> TileOp::tryGetRow() {
return getRow(); }
4316AIETileType TileOp::getTileType() {
4318 return targetModel.getTileType(getCol(), getRow());
4321static bool isLegalTileConnection(TileOp tile,
4323 MasterSetOp masterOp, PacketRulesOp slaveOp) {
4324 auto srcBundle = slaveOp.sourcePort().bundle;
4325 auto srcChan = slaveOp.sourcePort().channel;
4326 auto dstBundle = masterOp.destPort().bundle;
4327 auto dstChan = masterOp.destPort().channel;
4329 tile.colIndex(), tile.rowIndex(), srcBundle, srcChan, dstBundle, dstChan);
4332static bool isLegalTileConnection(TileOp tile,
4334 ConnectOp connectOp) {
4335 auto srcBundle = connectOp.getSourceBundle();
4336 auto srcChan = connectOp.getSourceChannel();
4337 auto dstBundle = connectOp.getDestBundle();
4338 auto dstChan = connectOp.getDestChannel();
4340 tile.colIndex(), tile.rowIndex(), srcBundle, srcChan, dstBundle, dstChan);
4343TileOp TileOp::getOrCreate(mlir::OpBuilder builder, DeviceOp device,
int col,
4344 int row, std::optional<mlir::Location> loc) {
4345 TileOp tile =
nullptr;
4347 for (
auto t : device.getOps<AIE::TileOp>()) {
4348 if (t.getRow() == row && t.getCol() == col) {
4355 OpBuilder::InsertionGuard guard(builder);
4356 mlir::Block &device_start_block = *device.getBodyRegion().begin();
4357 builder.setInsertionPointToStart(&device_start_block);
4358 tile = TileOp::create(builder, loc.value_or(device.getLoc()),
4359 builder.getIndexType(), col, row);
4368LogicalResult ShimMuxOp::verify() {
4373 if (!isa<TileOp>(getTile().getDefiningOp()))
4376 Region &body = getConnections();
4377 DenseSet<Port> destset;
4379 return emitOpError(
"should have non-empty body");
4381 for (
auto &ops : body.front()) {
4382 if (
auto connectOp = dyn_cast<ConnectOp>(ops)) {
4383 Port dest = {connectOp.getDestBundle(), connectOp.destIndex()};
4384 if (destset.count(dest))
4385 return connectOp.emitOpError(
"targets same destination ")
4386 << stringifyWireBundle(dest.bundle) <<
": " << dest.channel
4387 <<
" as another connect operation";
4388 destset.insert(dest);
4389 }
else if (isa<EndOp>(ops)) {
4392 return ops.emitOpError(
"cannot be contained in a Switchbox op");
4398size_t ShimMuxOp::getNumSourceConnections(WireBundle bundle) {
4399 auto tile = getTileOp();
4402 tile.getRow(), bundle);
4405size_t ShimMuxOp::getNumDestConnections(WireBundle bundle) {
4406 auto tile = getTileOp();
4416LogicalResult ShimDMAOp::verify() {
4423TileOp ShimDMAOp::getTileOp() {
4424 return cast<TileElement>(this->getOperation()).getTileOp();
4427LogicalResult PacketRulesOp::verify() {
4428 if (Region &body = getRules(); body.empty())
4429 return emitOpError(
"should have non-empty body");
4433LogicalResult MasterSetOp::verify() {
4435 for (
auto val : getAmsels()) {
4436 auto amsel = dyn_cast_if_present<AMSelOp>(val.getDefiningOp());
4438 return emitOpError(
"amsel operand must be produced by an 'aie.amsel' op");
4439 if (arbiter != -1 && arbiter != amsel.arbiterIndex())
4440 return emitOpError(
"a master port can only be tied to one arbiter");
4441 arbiter = amsel.arbiterIndex();
4446bool MasterSetOp::keepsPktHeader() {
4448 (*this)->getParentOfType<SwitchboxOp>().getTileOp().getTileID(),
4449 destPort(), getKeepPktHeader());
4453 std::optional<bool> keep) {
4456 return master.bundle != WireBundle::DMA &&
4457 (tile.row != 0 || master.bundle != WireBundle::South);
4462static LogicalResult verifyFlowEndpoint(Operation *op, Value tile,
Port port,
4464 if (port.bundle != WireBundle::Core && port.bundle != WireBundle::DMA)
4466 auto tileOp = dyn_cast_or_null<TileOp>(tile.getDefiningOp());
4469 size_t channels = isSource ? tileOp.getNumDestConnections(port.bundle)
4470 : tileOp.getNumSourceConnections(port.bundle);
4471 if (
static_cast<size_t>(port.channel) < channels)
4473 return op->emitOpError() << (isSource ?
"source " :
"destination ")
4474 << stringifyWireBundle(port.bundle) <<
":"
4475 << port.channel <<
" does not exist: tile ("
4476 << tileOp.getCol() <<
", " << tileOp.getRow()
4477 <<
") has " << channels <<
" "
4478 << stringifyWireBundle(port.bundle) <<
" port"
4479 << (channels == 1 ?
"" :
"s")
4480 << (isSource ?
" into" :
" out of")
4481 <<
" its stream switch";
4484static LogicalResult verifyFlowVias(Operation *op, OperandRange vias,
4485 DenseI32ArrayAttr ingressBundles,
4486 DenseI32ArrayAttr ingressChannels,
4487 DenseI32ArrayAttr egressBundles,
4488 DenseI32ArrayAttr egressChannels) {
4489 size_t numVias = vias.size();
4490 DenseI32ArrayAttr arrays[] = {ingressBundles, ingressChannels, egressBundles,
4492 for (DenseI32ArrayAttr array : arrays) {
4493 size_t size = array ? array.size() : 0;
4494 if (size != numVias) {
4495 return op->emitOpError(
"has ")
4496 << numVias <<
" via tile(s) but a via port array of size " << size;
4503 for (
auto [index, via] : llvm::enumerate(vias)) {
4504 if (!isa_and_nonnull<TileLike>(via.getDefiningOp())) {
4505 return op->emitOpError(
"has via operand at index ")
4506 << index <<
" that is not defined by a tile-like operation";
4510 auto verifyBundles = [&](DenseI32ArrayAttr bundles,
4511 StringRef direction) -> LogicalResult {
4512 for (
auto [index, bundle] : llvm::enumerate(bundles.asArrayRef())) {
4514 bundle >
static_cast<int32_t
>(getMaxEnumValForWireBundle())) {
4515 return op->emitOpError(
"has invalid via ")
4516 << direction <<
" bundle " << bundle <<
" at index " << index;
4521 auto verifyChannels = [&](DenseI32ArrayAttr channels,
4522 StringRef direction) -> LogicalResult {
4523 for (
auto [index, channel] : llvm::enumerate(channels.asArrayRef())) {
4525 return op->emitOpError(
"has negative via ")
4526 << direction <<
" channel " << channel <<
" at index " << index;
4531 if (failed(verifyBundles(ingressBundles,
"ingress")) ||
4532 failed(verifyChannels(ingressChannels,
"ingress")) ||
4533 failed(verifyBundles(egressBundles,
"egress")) ||
4534 failed(verifyChannels(egressChannels,
"egress"))) {
4540LogicalResult FlowOp::verify() {
4541 if (failed(verifyFlowEndpoint(*
this, getSource(),
4542 {getSourceBundle(), sourceIndex()},
true)))
4544 if (failed(verifyFlowEndpoint(*
this, getDest(),
4545 {getDestBundle(), destIndex()},
false))) {
4549 return verifyFlowVias(*
this, getVias(), getViaIngressBundlesAttr(),
4550 getViaIngressChannelsAttr(), getViaEgressBundlesAttr(),
4551 getViaEgressChannelsAttr());
4554LogicalResult PacketSourceOp::verify() {
4555 return verifyFlowEndpoint(*
this, getTile(), port(),
true);
4558LogicalResult PacketDestOp::verify() {
4559 return verifyFlowEndpoint(*
this, getTile(), port(),
false);
4562LogicalResult PacketFlowOp::verify() {
4563 Region &body = getPorts();
4565 return emitOpError(
"should have non-empty body");
4567 int numSources = 0, numDests = 0;
4568 for (
auto &ops : body.front()) {
4569 if (!isa<PacketSourceOp, PacketDestOp, EndOp>(ops))
4570 return ops.emitOpError(
"cannot be contained in a PacketFlow op");
4571 if (isa<PacketSourceOp>(ops))
4573 if (isa<PacketDestOp>(ops))
4578 return emitOpError(
"must have at least one aie.packet_source");
4580 return emitOpError(
"must have at least one aie.packet_dest");
4581 if (!getVias().empty() && numSources != 1) {
4583 "with via waypoints must have exactly one aie.packet_source");
4585 if (!getVias().empty() && numDests != 1) {
4587 "with via waypoints must have exactly one aie.packet_dest");
4590 uint8_t
id = getID();
4591 if (uint32_t maxId =
getTargetModel(getOperation()).getMaxPacketId();
4593 return emitOpError(
"packet id ")
4594 <<
static_cast<int>(id) <<
" exceeds the maximum of " << maxId;
4598 if (std::optional<uint8_t> mask = getMask()) {
4599 if ((
id & *mask) !=
id) {
4600 return emitOpError(
"has ID 0x")
4601 << llvm::utohexstr(
id) <<
" outside mask 0x"
4602 << llvm::utohexstr(*mask) <<
", which no packet can match";
4606 return verifyFlowVias(*
this, getVias(), getViaIngressBundlesAttr(),
4607 getViaIngressChannelsAttr(), getViaEgressBundlesAttr(),
4608 getViaEgressChannelsAttr());
4616static uint32_t channelsOfOwnTile(TileOp tile, DMAChannelDir direction,
4618 int col = tile.getCol(), row = tile.getRow();
4619 if (direction == DMAChannelDir::MM2S)
4625LogicalResult DmaPushQueueOp::verify() {
4626 if (getOutOfOrder() && getDirection() != DMAChannelDir::S2MM)
4627 return emitOpError(
"out_of_order is only supported on S2MM channels");
4628 if (getBdId().has_value() == getBdTag().has_value())
4629 return emitOpError(
"names the descriptor it pushes either by bd_id or, "
4630 "before aie-assign-bd-ids runs, by bd_tag");
4631 auto core = (*this)->getParentOfType<CoreOp>();
4633 if (
auto tile = core.getTile().getDefiningOp<TileOp>()) {
4634 uint32_t channels = channelsOfOwnTile(tile, getDirection(), targetModel);
4635 if (getChannel() >=
static_cast<int32_t
>(channels))
4636 return emitOpError(
"pushes onto ")
4637 << stringifyDMAChannelDir(getDirection()) <<
" channel "
4638 << getChannel() <<
", but tile (" << tile.getCol() <<
", "
4639 << tile.getRow() <<
") has " << channels;
4641 uint32_t maxRepeat = targetModel.getMaxRepeatCount();
4642 if (getRepeatCount() < 0 ||
4643 static_cast<uint32_t
>(getRepeatCount()) > maxRepeat)
4644 return emitOpError(
"repeat_count must be in [0, ") << maxRepeat <<
"]";
4648LogicalResult DmaPollForQueueSpaceOp::verify() {
4649 auto core = (*this)->getParentOfType<CoreOp>();
4651 if (
auto tile = core.getTile().getDefiningOp<TileOp>()) {
4652 uint32_t channels = channelsOfOwnTile(tile, getDirection(), targetModel);
4653 if (getChannel() >=
static_cast<int32_t
>(channels))
4654 return emitOpError(
"waits on ")
4655 << stringifyDMAChannelDir(getDirection()) <<
" channel "
4656 << getChannel() <<
", but tile (" << tile.getCol() <<
", "
4657 << tile.getRow() <<
") has " << channels;
4659 if (targetModel.getDmaTaskQueueDepth() == 0)
4660 return emitOpError(
"waits on a task queue, which this device has none of");
4664LogicalResult CoreOp::verify() {
4665 if (getBody().empty())
4666 return emitOpError(
"should have non-empty body");
4673 "When `elf_file` attribute is specified, core body must be empty "
4674 "(consist of exactly one `aie.end` op).");
4677 if (getLinkWith() && getLinkFiles())
4679 "cannot specify both 'link_with' (deprecated) and 'link_files' "
4680 "on the same core; run aie-assign-core-link-files to migrate");
4681 if (getLinkWith() && getLinkMergeFiles())
4683 "cannot specify both 'link_with' (deprecated) and 'link_merge_files' "
4684 "on the same core; run aie-assign-core-link-files to migrate");
4687 if (
auto linkFiles = getLinkFiles())
4688 if (
auto mergeFiles = getLinkMergeFiles()) {
4689 llvm::SmallSet<StringRef, 8> linked;
4690 for (
auto f : linkFiles->getAsRange<StringAttr>())
4691 linked.insert(f.getValue());
4692 for (
auto f : mergeFiles->getAsRange<StringAttr>())
4693 if (linked.count(f.getValue()))
4694 return emitOpError(
"artifact '")
4696 <<
"' appears in both 'link_files' and 'link_merge_files'; an "
4697 "artifact must be either merged or linked, not both";
4701 if (
auto measured = getMeasuredDataSize())
4702 if (
auto declared = getDataSize(); declared && *declared < *measured)
4703 return emitOpError(
"data_size ")
4704 << *declared <<
" is smaller than the " << *measured
4705 <<
" bytes this core's linked sections occupy";
4709 if (
auto summary = getLockAcquiresAttr()) {
4710 for (LockAcquireAttr acquire : summary.getAcquires()) {
4711 int32_t column = acquire.getColumn();
4712 int32_t row = acquire.getRow();
4713 if (column >= targetModel.columns() || row >= targetModel.rows())
4715 "lock_acquires references a tile outside the device: ")
4716 <<
"(" << column <<
", " << row <<
")";
4717 if (
static_cast<uint32_t
>(acquire.getLockId()) >=
4718 targetModel.getNumLocks(column, row))
4719 return emitOpError(
"lock_acquires references an invalid lock ID ")
4720 << acquire.getLockId() <<
" on tile (" << column <<
", " << row
4722 if (acquire.getAction() == LockAction::AcquireGreaterEqual &&
4725 "lock_acquires uses AcquireGreaterEqual on a target without "
4729 int64_t localMem = targetModel.getLocalMemorySize();
4730 auto validAlignment = [localMem](int64_t alignment) {
4731 return alignment > 0 && alignment <= localMem &&
4732 llvm::isPowerOf2_64(alignment);
4734 if (
auto alignment = getMeasuredDataAlignment();
4735 alignment && !validAlignment(*alignment))
4736 return emitOpError(
"measured_data_alignment must be a power of two between "
4738 << localMem <<
" bytes";
4739 if (
auto alignments = getMeasuredBankAlignments())
4740 if (!llvm::all_of(*alignments, validAlignment))
4741 return emitOpError(
"measured_bank_alignments must contain only powers of "
4742 "two between 1 and ")
4743 << localMem <<
" bytes";
4745 auto tile = dyn_cast_if_present<TileOp>(getTile().getDefiningOp());
4747 tile ? targetModel.getNumBanks(tile.getCol(), tile.getRow()) : 0;
4749 numBanks > 0 ? targetModel.getLocalMemorySize() / numBanks : 0;
4750 if (
auto bank = getStackBank()) {
4751 if (tile && *bank >= numBanks)
4752 return emitOpError(
"stack_bank ")
4753 << *bank <<
" does not exist; this tile has " << numBanks
4756 if (bankSize > 0 && stackRun.
size > bankSize)
4757 return emitOpError(
"stack_bank pins a ")
4758 << stackRun.
size <<
"-byte stack to bank " << *bank
4759 <<
", which holds " << bankSize
4760 <<
" bytes; omit stack_bank and stack_address for legacy "
4762 if (getStackAddress() && bankSize > 0 && stackRun.
start / bankSize != *bank)
4763 return emitOpError(
"stack_address 0x")
4764 << llvm::utohexstr(stackRun.
start) <<
" lies in bank "
4765 << stackRun.
start / bankSize <<
", but stack_bank requests bank "
4767 if (getStackAddress() && bankSize > 0 &&
4768 stackRun.
end() > (*bank + 1) * bankSize)
4769 return emitOpError(
"a ")
4770 << stackRun.
size <<
"-byte stack at 0x"
4771 << llvm::utohexstr(stackRun.
start) <<
" runs past stack_bank "
4772 << *bank <<
" (ending at 0x"
4773 << llvm::utohexstr((*bank + 1) * bankSize) <<
")";
4778 if (stackRun.
size >= localMem)
4779 return emitOpError(getStackSize() ?
"stack_size "
4780 :
"measured stack requirement ")
4782 <<
" leaves no local memory for this tile's buffers (" << localMem
4784 if (stackRun.
end() > localMem)
4785 return emitOpError(
"a ") << stackRun.
size <<
"-byte stack at 0x"
4786 << llvm::utohexstr(stackRun.
start)
4787 <<
" runs past this tile's local memory ("
4788 << localMem <<
" bytes total)";
4789 if (getStackAddress() &&
4790 stackRun.
start % targetModel.getCoreStackAlignment() != 0)
4791 return emitOpError(
"stack_address must be aligned to ")
4792 << targetModel.getCoreStackAlignment()
4793 <<
" bytes for this target's stack ABI";
4797bool CoreOp::isEmpty() {
4798 Region &body = getBody();
4800 return (body.hasOneBlock() && body.front().getOperations().size() == 1 &&
4801 llvm::isa<AIE::EndOp>(body.front().front()));
4804TileOp CoreOp::getTileOp() {
4805 return cast<TileElement>(this->getOperation()).getTileOp();
4808uint32_t CoreOp::getEffectiveStackSize() {
4809 if (
auto declared = getStackSize())
4812 uint32_t fallback = targetModel.getDefaultCoreStackSize();
4813 if (
auto measured = getMeasuredStackSize())
4814 return std::max<uint32_t>(
4816 llvm::alignTo(*measured, targetModel.getCoreStackAlignment()));
4821 return {getStackAddress().value_or(0), getEffectiveStackSize()};
4824int64_t CoreOp::getSectionAlignment(std::optional<int32_t> bank) {
4826 return getMeasuredDataAlignment().value_or(1);
4827 auto alignments = getMeasuredBankAlignments();
4828 if (alignments && *bank >= 0 &&
4829 static_cast<size_t>(*bank) < alignments->size())
4830 return std::max<int64_t>((*alignments)[*bank], 1);
4834SmallVector<BufferOp> CoreOp::getSectionReservationBuffers(
4835 ArrayRef<CoreSectionReservation> reservations) {
4837 SmallVector<BufferOp> onTile;
4838 (*this)->getParentOp()->walk([&](BufferOp buffer) {
4839 if (buffer.getTile() == getTile())
4840 onTile.push_back(buffer);
4842 SmallVector<BufferOp> holders;
4843 DenseSet<Operation *> claimed;
4845 auto holds = [&](BufferOp buffer) {
4846 if (claimed.contains(buffer) || buffer.getAllocationSize() != held.bytes)
4848 if (held.kind == Kind::DataRegion)
4849 return static_cast<bool>(buffer.getCoreData());
4850 return buffer.getBankReserved() &&
4851 buffer.getAligned() == held.aligned() &&
4852 (held.kind == Kind::Bank ? buffer.getMemBank() == held.at
4853 : buffer.getAddress() == held.at);
4855 auto *it = llvm::find_if(onTile, holds);
4856 holders.push_back(it == onTile.end() ? BufferOp() : *it);
4857 if (it != onTile.end())
4858 claimed.insert(*it);
4863SmallVector<CoreSectionReservation> CoreOp::getSectionReservations() {
4865 SmallVector<CoreSectionReservation> held;
4868 int64_t data = getDataSize().value_or(getMeasuredDataSize().value_or(0));
4871 {Kind::DataRegion, data, 0, getSectionAlignment(std::nullopt)});
4872 if (
auto sizes = getMeasuredBankSizes())
4873 for (
auto [index, size] : llvm::enumerate(*sizes)) {
4874 auto bank =
static_cast<int32_t
>(index);
4876 held.push_back({Kind::Bank, size, bank, getSectionAlignment(bank)});
4878 if (
auto ranges = getMeasuredDataRanges())
4879 for (
size_t i = 0; i + 1 < ranges->size(); i += 2)
4880 if ((*ranges)[i + 1] > 0)
4881 held.push_back({Kind::Prebaked, (*ranges)[i + 1], (*ranges)[i]});
4889int64_t BufferOp::getSectionAlignment() {
4890 if (!getCoreData() && !getBankReserved())
4892 for (Operation *user : getTile().getUsers())
4893 if (
auto core = dyn_cast<CoreOp>(user)) {
4895 return core.getSectionAlignment(std::nullopt);
4896 std::optional<int32_t> bank = getMemBank();
4897 return bank ? core.getSectionAlignment(bank) : 1;
4902int64_t BufferOp::getAllocationSize() {
4904 DataLayout::closest(getOperation()));
4907LogicalResult BufferOp::verify() {
4908 if (UsesAreAccessible::verifyTrait(*this).failed()) {
4912 if (
auto init = getInitialValue()) {
4913 auto initType = cast<ShapedType>(init->getType());
4914 MemRefType type = getType();
4915 if (initType.getShape() != type.getShape() ||
4916 initType.getElementType() != type.getElementType())
4917 return emitOpError(
"initial value of type ")
4918 << initType <<
" does not cover the buffer's " << type;
4921 if (FlatSymbolRefAttr resident = getResidentAttr()) {
4922 auto module = (*this)->getParentOfType<mlir::ModuleOp>();
4923 auto global = dyn_cast_or_null<memref::GlobalOp>(
4924 module ? SymbolTable::lookupSymbolIn(module, resident.getValue())
4927 return emitOpError(
"resident names no memref.global ")
4928 << resident <<
" at module scope";
4929 if (global.getType() != getType())
4930 return emitOpError(
"resident ")
4931 << resident <<
"'s type " << global.getType()
4932 <<
" does not match this buffer's " << getType();
4936 if (getCoreData() && getBankReserved()) {
4937 return emitOpError(
"cannot be both core_data and bank_reserved; a buffer "
4938 "holds one reservation of a core's sections");
4943 if (std::optional<int32_t> memBank = getMemBank()) {
4945 if (!banks.empty() && !llvm::is_contained(banks, *memBank)) {
4946 auto diag = emitOpError(
"has address space for bank ") << banks.front();
4947 if (banks.size() > 1)
4948 diag <<
" or " << banks.back();
4949 return diag <<
" but is pinned to bank " << *memBank
4950 <<
". These constraints disagree";
4953 dyn_cast_if_present<TileLike>(getTile().getDefiningOp());
4954 tileLike && (tileLike.isCoreTile() || tileLike.isMemTile())) {
4956 std::optional<int> col = tileLike.tryGetCol();
4957 std::optional<int> row = tileLike.tryGetRow();
4958 for (
int c = 0; !(col && row) && c < targetModel.
columns(); ++c)
4959 for (
int r = 0; !(col && row) && r < targetModel.
rows(); ++r)
4960 if (targetModel.
getTileType(c, r) == tileLike.getTileType()) {
4967 static_cast<int32_t
>(targetModel.
getNumBanks(*col, *row))))
4968 return emitOpError(
"mem_bank attribute value is out of range");
4974 auto tile = dyn_cast_if_present<TileOp>(getTile().getDefiningOp());
4981 if (getCoreData()) {
4982 for (Operation *user : tile.getResult().getUsers()) {
4983 auto other = dyn_cast<BufferOp>(user);
4984 if (other && other != *
this && other.getCoreData()) {
4985 return emitOpError(
"is a second core_data buffer on tile (")
4986 << tile.getCol() <<
", " << tile.getRow()
4987 <<
"); the core's data region is one extent";
4995 std::optional<int32_t> address = getAddress();
4996 std::optional<int32_t> memBank = getMemBank();
4997 if (address && memBank && getAllocationSize() > 0) {
4999 int64_t numBanks = targetModel.
getNumBanks(tile.getCol(), tile.getRow());
5000 int64_t memSize = tile.isMemTile() ? targetModel.
getMemTileSize()
5003 int64_t bankSize = memSize / numBanks;
5004 int64_t bankOfAddress = *address / bankSize;
5005 if (bankOfAddress != *memBank) {
5006 return emitOpError(
"address 0x")
5007 << llvm::utohexstr(*address) <<
" lies in bank " << bankOfAddress
5008 <<
", but mem_bank requests bank " << *memBank;
5020 auto memref = dyn_cast<MemRefType>(type);
5024 auto space = dyn_cast_or_null<IntegerAttr>(
memref.getMemorySpace());
5028 int64_t value = space.getInt();
5029 if (value >= 5 && value <= 8) {
5030 return {
static_cast<int>(value - 5)};
5053 if (
auto buf = dyn_cast<BufferOp>(bufOp)) {
5054 std::optional<int32_t> address = buf.getAddress();
5055 assert(address.has_value() &&
"buffer must have address assigned");
5058 if (isa_and_nonnull<ExternalBufferOp>(bufOp))
5059 llvm::report_fatal_error(
5060 "External buffer addresses are assigned at runtime.");
5061 llvm::report_fatal_error(
"unknown buffer type");
5065 DenseMap<TileID, Operation *> &tiles) {
5066 for (
auto tile : device.getOps<TileOp>()) {
5067 tiles[tile.getTileID()] = tile;
5073 DenseMap<Operation *, SmallVector<BufferOp, 4>> &buffers) {
5074 for (BufferOp buffer : device.getOps<BufferOp>()) {
5075 Operation *tileOp = buffer.getTile().getDefiningOp();
5076 buffers[tileOp].push_back(buffer);
5080static void printBufferInitialValue(OpAsmPrinter &p, BufferOp op, Type type,
5081 Attribute initialValue) {
5082 if (op.getInitialValue()) {
5084 p.printAttributeWithoutType(initialValue);
5088static ParseResult parseBufferInitialValue(OpAsmParser &parser, Type &type,
5089 Attribute &initialValue) {
5090 auto memrefType = llvm::cast<MemRefType>(type);
5091 if (!memrefType.hasStaticShape())
5092 return parser.emitError(parser.getNameLoc())
5093 <<
"type should be static shaped memref, but got " << type;
5095 if (parser.parseOptionalEqual())
5098 Type tensorType = mlir::memref::getTensorTypeFromMemRefType(memrefType);
5099 if (parser.parseAttribute(initialValue, tensorType))
5101 if (!llvm::isa<ElementsAttr>(initialValue))
5102 return parser.emitError(parser.getNameLoc())
5103 <<
"initial value should be an elements attribute";
5107TileOp BufferOp::getTileOp() {
5108 return cast<TileElement>(this->getOperation()).getTileOp();
5115LogicalResult MemOp::verify() {
5116 Region &body = getBody();
5121 for (
auto &bodyOp : body.getOps()) {
5122 if (
auto allocOp = dyn_cast<memref::AllocOp>(bodyOp))
5123 if (!allocOp->getAttr(
"id"))
5124 return allocOp.emitOpError()
5125 <<
"allocOp in MemOp region should have an id attribute";
5130TileOp MemOp::getTileOp() {
5131 return cast<TileElement>(this->getOperation()).getTileOp();
5138LogicalResult MemTileDMAOp::verify() {
5139 assert(getOperation()->getNumRegions() == 1 &&
5140 "MemTileDMAOp has zero region!");
5146 for (
auto &bodyOp : getBody().getOps()) {
5147 if (
auto allocOp = dyn_cast<memref::AllocOp>(bodyOp)) {
5148 if (!allocOp->getAttr(
"id"))
5149 return allocOp.emitOpError()
5150 <<
"allocOp in MemTileDMAOp region should have an id attribute";
5152 if (
auto startOp = dyn_cast<DMAStartOp>(bodyOp)) {
5155 if (!startOp.getEndpoint() && startOp.getChannelIndex() > 3) {
5161 llvm::SmallSet<Block *, 16> reachable;
5162 SmallVector<Block *, 16> worklist;
5163 Block *firstBD = startOp.getSuccessor(0);
5164 reachable.insert(firstBD);
5165 worklist.push_back(firstBD);
5166 while (!worklist.empty()) {
5167 Block *block = worklist.pop_back_val();
5170 auto successors = block->getTerminator()->getSuccessors();
5171 for (
auto *i : successors) {
5172 if (!reachable.contains(i)) {
5173 reachable.insert(i);
5174 worklist.push_back(i);
5178 for (Block *b : reachable) {
5179 for (DMABDOp bd : b->getOps<DMABDOp>()) {
5180 if (
auto bufferOp = bd.getBufferOp();
5181 bufferOp.getTile() != getTile()) {
5182 InFlightDiagnostic err =
5184 <<
"is reachable from DMA channel "
5185 << startOp.getChannelIndex()
5186 <<
" and attempts to access a non-local buffer\n";
5187 err.attachNote(startOp->getLoc()) <<
"channel";
5188 err.attachNote(bufferOp->getLoc()) <<
"buffer";
5192 for (
auto useLock : b->getOps<UseLockOp>()) {
5193 if (
auto lockOp = useLock.getLockOp();
5194 lockOp.getTile() != getTile()) {
5195 InFlightDiagnostic err =
5196 useLock.emitOpError()
5197 <<
"is reachable from DMA channel "
5198 << startOp.getChannelIndex()
5199 <<
" and attempts to access a non-local lock\n";
5200 err.attachNote(startOp->getLoc()) <<
"channel";
5201 err.attachNote(lockOp->getLoc()) <<
"lock";
5213TileOp MemTileDMAOp::getTileOp() {
5214 return cast<TileElement>(this->getOperation()).getTileOp();
5221static bool isBdPacketEnabled(DMABDOp bd) {
5222 if (bd.getPacket().has_value())
5224 if (Block *blk = bd->getBlock())
5225 return !blk->getOps<DMABDPACKETOp>().empty();
5232 int64_t channels = dir == DMAChannelDir::MM2S
5233 ? tile.getNumDestConnections(WireBundle::DMA)
5234 : tile.getNumSourceConnections(WireBundle::DMA);
5235 if (channel >= 0 && channel < channels)
5237 return op->emitOpError() << stringifyDMAChannelDir(dir) <<
" channel "
5238 << channel <<
" is out of range: the tile has "
5239 << channels <<
" " << stringifyDMAChannelDir(dir)
5244 PacketInfoAttr packet,
5246 if (packet && !packet.isAssigned())
5247 return op->emitOpError()
5249 <<
" has no pkt_id; allocation assigns one, so a header at "
5250 "this level must carry it";
5256 std::optional<int32_t> oooId = bd.getOutOfOrderId();
5257 if (!oooId.has_value())
5261 return bd.emitOpError(
"out_of_order_id is not supported on this device");
5262 if (!packetEnabledByContext && !isBdPacketEnabled(bd))
5263 return bd.emitOpError(
"out_of_order_id requires a packet-enabled BD");
5265 if (*oooId < 0 ||
static_cast<uint32_t
>(*oooId) > maxOooId)
5266 return bd.emitOpError(
"out_of_order_id must be in [0, ") << maxOooId <<
"]";
5276 AIETileType tileType) {
5277 std::optional<BDIterationAttr> iter = bd.getIteration();
5282 return bd.emitOpError(
"BD iteration is not supported on this target");
5283 uint32_t size = iter->getSize(), current = iter->getCurrent();
5285 if (size < 1 || size > maxSize)
5286 return bd.emitOpError(
"BD iteration size must be in [1, ")
5289 int64_t strideInBytes =
static_cast<int64_t
>(iter->getStride()) *
5290 bd.getBufferElementTypeWidthInBytes();
5291 if (strideInBytes % 4)
5292 return bd.emitOpError(
5293 "BD iteration stride must be aligned to 32-bit words");
5294 int64_t stepInWords = strideInBytes / 4;
5296 if (stepInWords < 1 || stepInWords > maxStep)
5297 return bd.emitOpError() <<
"BD iteration stride must be in [1, "
5298 << maxStep <<
"] 32-bit words";
5300 if (current >= size)
5301 return bd.emitOpError(
"BD iteration current must be in [0, size)");
5305static LogicalResult verifyDMARepeatCount(Operation *op, int32_t repeatCount) {
5307 if (maxRepeat == 0) {
5308 if (repeatCount != 0)
5309 return op->emitOpError(
"repeat_count is not supported on this target");
5312 if (repeatCount < 0 ||
static_cast<uint32_t
>(repeatCount) > maxRepeat)
5313 return op->emitOpError(
"repeat_count ")
5314 << repeatCount <<
" is out of range [0, " << maxRepeat
5315 <<
"] for this target";
5321 bool outOfOrder, ArrayRef<DMABDOp> bds,
5322 bool packetEnabledByContext) {
5325 if (dir != DMAChannelDir::S2MM)
5326 return op->emitOpError(
"out_of_order is only valid on an S2MM channel");
5328 return op->emitOpError(
5329 "out-of-order S2MM DMA is not supported on this device");
5331 return op->emitOpError(
"out-of-order S2MM channel must have at least one "
5332 "receive buffer descriptor");
5333 for (DMABDOp bd : bds) {
5334 if (!packetEnabledByContext && !isBdPacketEnabled(bd))
5335 return bd.emitOpError(
5336 "out-of-order S2MM receive buffer descriptor must be packet-enabled");
5337 if (bd.getOutOfOrderId().has_value())
5338 return bd.emitOpError(
5339 "out_of_order_id belongs on the sender buffer descriptor, not an "
5340 "out-of-order S2MM receive buffer descriptor");
5343 DenseMap<Operation *, Operation *> releasedByRecvBd;
5344 for (DMABDOp bd : bds)
5345 for (
auto useLock : bd->getBlock()->getOps<UseLockOp>())
5346 if (useLock.release())
5347 if (Operation *lockDef = useLock.getLock().getDefiningOp())
5348 releasedByRecvBd.try_emplace(lockDef, bd.getOperation());
5349 for (DMABDOp bd : bds)
5350 for (
auto useLock : bd->getBlock()->getOps<UseLockOp>())
5351 if (useLock.acquire() || useLock.acquireGE())
5352 if (Operation *lockDef = useLock.getLock().getDefiningOp()) {
5353 auto it = releasedByRecvBd.find(lockDef);
5354 if (it != releasedByRecvBd.end() && it->second != bd.getOperation())
5355 return bd.emitOpError(
5356 "out-of-order S2MM prohibits inter-BD lock dependencies; "
5364 UseLockOp &release) {
5367 auto reject = [&](UseLockOp at) {
5368 auto diag = at.emitOpError(
5369 "does not fit its buffer descriptor, which has one lock-acquire field "
5370 "and one lock-release field; a buffer descriptor block uses either no "
5371 "lock or one use_lock(acquire) and one use_lock(release)");
5373 diag <<
", or a use_lock(release) alone when out-of-order";
5376 for (UseLockOp op : block.getOps<UseLockOp>()) {
5377 UseLockOp &slot = op.release() ? release : acquire;
5382 if (!acquire && !release)
5384 if (release && (acquire || outOfOrder))
5386 return reject(acquire ? acquire : release);
5389LogicalResult DMAOp::verify() {
5390 auto *parentOp = getOperation()->getParentOp();
5391 if (parentOp->getRegion(0).getBlocks().size() > 1)
5392 return emitOpError(
"DMAOp can only appear in single block region");
5393 if (!parentOp->getRegion(0).getOps<DMAStartOp>().empty())
5394 return emitOpError(
"DMAOp is not compatible with DMAStart ops");
5395 auto bdRegions = getBds();
5396 for (
auto &bdRegion : bdRegions) {
5397 if (!bdRegion.hasOneBlock())
5398 return emitOpError(
"DMAOp regions must have only one block");
5399 auto bds = llvm::to_vector_of<DMABDOp>(bdRegion.front().getOps<DMABDOp>());
5400 if (bds.size() != 1)
5401 return emitOpError(
"DMAOp regions/blocks must have exactly one DMABDOp");
5403 llvm::to_vector_of<UseLockOp>(bdRegion.front().getOps<UseLockOp>());
5404 if (useLocks.size() != 2)
5406 "DMAOp regions/blocks must have exactly two UseLock ops");
5408 if (getPadValue() != 0) {
5409 if (!isa<MemTileDMAOp>(parentOp))
5410 return emitOpError(
"pad_value is only supported on memtile DMA channels");
5411 if (getChannelDir() != DMAChannelDir::MM2S)
5412 return emitOpError(
"pad_value is only supported on MM2S DMA channels");
5413 if (!
getTargetModel(getOperation()).isMemTilePadValueSupported())
5414 return emitOpError(
"pad_value requires the CONSTANT_PAD_VALUE register, "
5415 "unavailable on this target");
5417 if (failed(verifyDMARepeatCount(getOperation(), getRepeatCount())))
5419 SmallVector<DMABDOp> bds;
5420 for (
auto &bdRegion : getBds())
5421 llvm::append_range(bds, bdRegion.front().getOps<DMABDOp>());
5423 getOutOfOrder(), bds);
5430BufferOp DMABDOp::getBufferOp() {
5431 return cast<BufferOp>(getBuffer().getDefiningOp());
5436static ParseResult parseOptionalOperandType(OpAsmParser &parser, Type &type,
5439 if (succeeded(parser.parseOptionalColon()))
5440 return parser.parseType(type);
5444static void printOptionallyTypedOperand(OpAsmPrinter &printer, Value operand,
5447 if (operand.getType() != defaultType)
5448 printer <<
" : " << operand.getType();
5457parseDynamicScalar(OpAsmParser &parser,
5458 std::optional<OpAsmParser::UnresolvedOperand> &operand,
5459 IntegerAttr &staticAttr, Type &type) {
5461 OptionalParseResult intResult = parser.parseOptionalInteger(intValue);
5462 if (intResult.has_value()) {
5463 if (failed(*intResult))
5466 parser.getBuilder().getI32IntegerAttr(
static_cast<int32_t
>(intValue));
5469 OpAsmParser::UnresolvedOperand op;
5470 if (parser.parseOperand(op) ||
5471 parseOptionalOperandType(parser, type, parser.getBuilder().getI32Type()))
5477static void printDynamicScalar(OpAsmPrinter &printer, Operation *,
5478 Value operand, IntegerAttr staticAttr, Type) {
5480 printOptionallyTypedOperand(printer, operand,
5481 IntegerType::get(operand.getContext(), 32));
5483 printer << staticAttr.getInt();
5487 OpAsmParser &parser,
5488 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &values,
5489 DenseI64ArrayAttr &integers, SmallVectorImpl<Type> &types) {
5490 SmallVector<int64_t> ints;
5491 auto parseEntry = [&]() -> ParseResult {
5492 OpAsmParser::UnresolvedOperand operand;
5493 OptionalParseResult isOperand = parser.parseOptionalOperand(operand);
5494 if (!isOperand.has_value())
5495 return parser.parseInteger(ints.emplace_back());
5497 if (failed(*isOperand) ||
5498 parseOptionalOperandType(parser, type,
5499 parser.getBuilder().getI64Type()))
5501 values.push_back(operand);
5502 types.push_back(type);
5503 ints.push_back(ShapedType::kDynamic);
5506 if (parser.parseCommaSeparatedList(AsmParser::Delimiter::Square, parseEntry))
5508 integers = parser.getBuilder().getDenseI64ArrayAttr(ints);
5513 OperandRange values,
5514 ArrayRef<int64_t> integers,
5517 auto value = values.begin();
5518 llvm::interleaveComma(integers, printer, [&](int64_t i) {
5519 if (!ShapedType::isDynamic(i)) {
5524 printOptionallyTypedOperand(printer, v,
5525 IntegerType::get(v.getContext(), 64));
5532static void splitScalarOfr(mlir::OpBuilder &builder, mlir::OpFoldResult ofr,
5533 mlir::Value &operand, mlir::IntegerAttr &attr) {
5538 if (
auto v = llvm::dyn_cast_if_present<mlir::Value>(ofr)) {
5543 llvm::cast<mlir::IntegerAttr>(llvm::cast<mlir::Attribute>(ofr));
5544 attr = builder.getI32IntegerAttr(
static_cast<int32_t
>(intAttr.getInt()));
5547void DMABDOp::buildMixed(mlir::OpBuilder &builder, mlir::OperationState &state,
5548 mlir::Value buffer, mlir::OpFoldResult offset,
5549 mlir::OpFoldResult len,
5550 llvm::ArrayRef<mlir::OpFoldResult> sizes,
5551 llvm::ArrayRef<mlir::OpFoldResult> strides,
5552 BDPadLayoutArrayAttr padDims, PacketInfoAttr packet) {
5555 llvm::SmallVector<int64_t> staticSizes, staticStrides;
5556 llvm::SmallVector<mlir::Value> dynSizes, dynStrides;
5557 mlir::dispatchIndexOpFoldResults(sizes, dynSizes, staticSizes);
5558 mlir::dispatchIndexOpFoldResults(strides, dynStrides, staticStrides);
5562 DenseI64ArrayAttr staticSizesAttr =
5563 staticSizes.empty() ? DenseI64ArrayAttr{}
5564 : builder.getDenseI64ArrayAttr(staticSizes);
5565 DenseI64ArrayAttr staticStridesAttr =
5566 staticStrides.empty() ? DenseI64ArrayAttr{}
5567 : builder.getDenseI64ArrayAttr(staticStrides);
5571 mlir::Value offsetVal, lenVal;
5572 mlir::IntegerAttr offsetAttr, lenAttr;
5573 splitScalarOfr(builder, offset, offsetVal, offsetAttr);
5574 splitScalarOfr(builder, len, lenVal, lenAttr);
5576 build(builder, state, buffer, offsetVal, lenVal,
5577 offsetAttr, lenAttr,
5578 dynSizes, dynStrides,
5600void DMABDOp::buildWithConstants(mlir::OpBuilder &builder,
5601 mlir::OperationState &state,
5602 mlir::Value buffer, int32_t offset,
5603 int32_t len, BDDimLayoutArrayAttr dims,
5604 BDPadLayoutArrayAttr padDims,
5605 PacketInfoAttr packet) {
5609 mlir::OpFoldResult offsetOfr = builder.getI32IntegerAttr(offset);
5610 mlir::OpFoldResult lenOfr = builder.getI32IntegerAttr(len);
5614 llvm::SmallVector<mlir::OpFoldResult> sizes, strides;
5616 for (BDDimLayoutAttr d : dims) {
5617 sizes.push_back(builder.getI64IntegerAttr(d.getSize()));
5618 strides.push_back(builder.getI64IntegerAttr(d.getStride()));
5621 buildMixed(builder, state, buffer, offsetOfr, lenOfr, sizes, strides, padDims,
5625llvm::SmallVector<mlir::OpFoldResult> DMABDOp::getMixedSizes() {
5626 return ::mlir::getMixedValues(
5627 getStaticSizes().value_or(llvm::ArrayRef<int64_t>{}), getSizes(),
5631llvm::SmallVector<mlir::OpFoldResult> DMABDOp::getMixedStrides() {
5632 return ::mlir::getMixedValues(
5633 getStaticStrides().value_or(llvm::ArrayRef<int64_t>{}), getStrides(),
5637LogicalResult DMABDOp::verifyMixedSizesAndStrides() {
5638 llvm::ArrayRef<int64_t> staticSizes =
5639 getStaticSizes().value_or(llvm::ArrayRef<int64_t>{});
5640 llvm::ArrayRef<int64_t> staticStrides =
5641 getStaticStrides().value_or(llvm::ArrayRef<int64_t>{});
5642 if (failed(mlir::verifyListOfOperandsOrIntegers(
5643 *
this,
"sizes", staticSizes.size(), staticSizes, getSizes())))
5645 if (failed(mlir::verifyListOfOperandsOrIntegers(
5646 *
this,
"strides", staticStrides.size(), staticStrides, getStrides())))
5648 if (staticSizes.size() != staticStrides.size())
5649 return emitOpError(
"expected the same number of sizes (")
5650 << staticSizes.size() <<
") and strides (" << staticStrides.size()
5655std::optional<llvm::SmallVector<BDDimLayoutAttr>> DMABDOp::getFoldedDimensions(
5656 llvm::function_ref<mlir::InFlightDiagnostic()> emitError) {
5657 llvm::ArrayRef<int64_t> sizes =
5658 getStaticSizes().value_or(llvm::ArrayRef<int64_t>{});
5659 llvm::ArrayRef<int64_t> strides =
5660 getStaticStrides().value_or(llvm::ArrayRef<int64_t>{});
5661 if (sizes.size() != strides.size()) {
5662 emitError() <<
"expected the same number of sizes (" << sizes.size()
5663 <<
") and strides (" << strides.size() <<
")";
5664 return std::nullopt;
5666 mlir::OperandRange dynSizes = getSizes(), dynStrides = getStrides();
5667 unsigned nextSize = 0, nextStride = 0;
5668 llvm::SmallVector<BDDimLayoutAttr> dims;
5669 dims.reserve(sizes.size());
5670 for (
auto [s, t] : llvm::zip(sizes, strides)) {
5671 std::optional<int64_t> sc =
5672 ShapedType::isDynamic(s)
5673 ? mlir::getConstantIntValue(dynSizes[nextSize++])
5675 std::optional<int64_t> tc =
5676 ShapedType::isDynamic(t)
5677 ? mlir::getConstantIntValue(dynStrides[nextStride++])
5680 emitError() <<
"buffer descriptor size/stride is a runtime value; a "
5681 "compile-time constant is required on this path";
5682 return std::nullopt;
5684 dims.push_back(BDDimLayoutAttr::get(
5685 getContext(),
static_cast<uint32_t
>(*sc),
static_cast<uint32_t
>(*tc)));
5690std::optional<llvm::SmallVector<BDDimLayoutAttr>>
5691DMABDOp::getConstantDimensions() {
5692 return getFoldedDimensions([&]() {
return emitOpError(); });
5698LogicalResult DMABDOp::verifyTransferInBounds() {
5699 auto buffer = llvm::dyn_cast<MemRefType>(getBuffer().getType());
5700 if (!buffer || !buffer.hasStaticShape() ||
5701 llvm::isa<BlockArgument>(getBuffer()))
5703 if ((getOffsetParameterAttr() || getOffsetStateTableIdxAttr()) && !hasLen())
5704 return emitOpError(
"offset_parameter requires an explicit `len` on a local "
5705 "buffer; the default whole-buffer length runs past the "
5706 "end for any nonzero runtime offset");
5707 std::optional<int32_t> offset = hasOffset() ? getConstantOffset() : 0;
5708 if (!offset || *offset < 0)
5710 int64_t numElements = buffer.getNumElements();
5711 SmallVector<OpFoldResult> sizes = getMixedSizes();
5712 SmallVector<OpFoldResult> strides = getMixedStrides();
5713 if (sizes.empty()) {
5714 std::optional<int64_t> len =
5715 hasLen() ? std::optional<int64_t>(getConstantLen()) : numElements;
5716 if (!len || *len <= 0 || *offset + *len <= numElements)
5718 InFlightDiagnostic err = emitOpError(
"transfer of ")
5719 << *len <<
" elements at offset " << *offset
5720 <<
" runs past the end of the " << numElements
5721 <<
"-element buffer";
5723 err.attachNote() <<
"an omitted `len` is the whole buffer; give the "
5724 "length to transfer from this offset";
5727 int64_t maxIdx = *offset;
5728 for (
auto [size, stride] : llvm::zip(sizes, strides)) {
5729 std::optional<int64_t> s = getConstantIntValue(size);
5730 std::optional<int64_t> t = getConstantIntValue(stride);
5733 maxIdx += *t * (*s - 1);
5735 if (maxIdx < numElements)
5737 return emitOpError() <<
"Specified stride(s) and size(s) result in out of "
5738 "bounds access in buffer, for index "
5739 << maxIdx <<
" in memref of length " << numElements
5747 llvm::ArrayRef<BDPadLayoutAttr> pads,
5749 if (dims.empty() || pads.empty())
5752 for (
auto [dim, pad] : llvm::zip(dims, pads))
5753 size *= dim.getSize() + pad.getConstPadBefore() + pad.getConstPadAfter();
5761 if (dims.back().getStride() != 1)
5765 uint64_t product = 1;
5766 for (
int i =
static_cast<int>(dims.size()) - 1; i >= 1; --i) {
5767 product *= dims[i].getSize();
5769 if (dims[i - 1].getSize() > 1 && dims[i - 1].getStride() != product)
5775std::optional<SmallVector<BDDimLayoutAttr>>
5777 SmallVector<BDDimLayoutAttr> kept;
5778 int64_t product = 1;
5779 for (BDDimLayoutAttr dim : dims) {
5780 product *= dim.getSize();
5781 if (dim.getSize() != 1)
5782 kept.push_back(dim);
5784 if (kept.size() == dims.size() || len != product)
5785 return std::nullopt;
5787 return SmallVector<BDDimLayoutAttr>{};
5797 BaseMemRefType buffer) {
5798 uint64_t elemBitWidth = buffer.getElementTypeBitWidth();
5799 if (elemBitWidth == 0 || (elemBitWidth % 8) != 0)
5800 return op->emitOpError(
"length_parameter requires a whole-byte element "
5802 std::optional<int64_t> unitBytes =
5803 llvm::checkedMul(lengthUnit,
static_cast<int64_t
>(elemBitWidth / 8));
5804 if (!unitBytes || *unitBytes / 4 > std::numeric_limits<uint32_t>::max())
5805 return op->emitOpError(
"length_unit (")
5807 <<
" elements) exceeds the 32-bit word count of a BD length";
5808 if (*unitBytes % 16 != 0)
5809 return op->emitOpError(
"length_unit must be a multiple of 16 bytes, got ")
5810 << *unitBytes <<
" bytes";
5815 Operation *op, std::optional<int64_t> lengthUnit, BaseMemRefType buffer,
5816 std::optional<int64_t> lenElems,
bool contiguous,
5817 llvm::ArrayRef<int64_t> innerSizes) {
5819 return op->emitOpError(
"length_parameter requires length_unit");
5822 int64_t elemBytes = buffer.getElementTypeBitWidth() / 8;
5824 return op->emitOpError(
"length_parameter requires a constant length");
5825 if ((*lenElems * elemBytes) % 16 != 0)
5826 return op->emitOpError(
"length_parameter requires a static length that is "
5827 "a multiple of 16 bytes, got ")
5828 << *lenElems * elemBytes <<
" bytes";
5831 llvm::SmallVector<int64_t, 3> sizes(innerSizes);
5832 llvm::SmallVector<int64_t, 3> strides(3, 0);
5835 return op->emitOpError(
"length_parameter on a non-contiguous pattern "
5836 "requires its second or third dimension to have a "
5837 "size above one: the added length continues it");
5838 int64_t rowElems = sizes[0] * sizes[1];
5839 if (*lengthUnit % rowElems != 0)
5840 return op->emitOpError(
"length_unit (")
5841 << *lengthUnit <<
" elements) must be a multiple of the " << rowElems
5842 <<
" elements moved per step of the dimension the added length "
5848 llvm::MutableArrayRef<int64_t> sizes,
5849 llvm::MutableArrayRef<int64_t> strides) {
5850 if (sizes[2] > 1 || sizes[1] <= 1)
5852 sizes[2] = sizes[1];
5853 strides[2] = strides[1];
5859 Operation *op,
const AIETargetModel &targetModel,
int col,
int row) {
5861 if (arch != AIEArch::AIE2 && arch != AIEArch::AIE2p)
5862 return op->emitOpError(
"length_parameter is only supported on AIE2 and "
5865 return op->emitOpError(
"length_parameter is only supported on shim NOC "
5866 "tiles, got tile (")
5867 << col <<
", " << row <<
")";
5871 return op->emitOpError(
"length_parameter requires a BD whose buffer "
5872 "length fills a 32-bit register");
5877 llvm::SmallVector<uint32_t> ids;
5878 program.getDmaBody().walk([&](DMABDOp bd) {
5879 if (
auto id = bd.getBdId())
5885LogicalResult DMABDOp::verify() {
5886 if (getOffsetParameterAttr() || getOffsetStateTableIdxAttr()) {
5887 uint64_t elemBitWidth = llvm::cast<BaseMemRefType>(getBuffer().getType())
5888 .getElementTypeBitWidth();
5889 if (elemBitWidth == 0 || (elemBitWidth % 8) != 0)
5890 return emitOpError(
"offset_parameter requires a whole-byte element type");
5893 if (getLengthParameterAttr() || getLengthStateTableIdxAttr()) {
5896 if (llvm::isa<MemOp, MemTileDMAOp, ShimDMAOp, DMAOp>(
5897 (*this)->getParentOp()))
5898 return emitOpError(
"length_parameter is only supported on a BD in a "
5899 "runtime sequence");
5900 std::optional<llvm::SmallVector<BDDimLayoutAttr>> dims =
5901 getFoldedDimensions([&]() {
5902 return emitOpError(
"length_parameter requires a static pattern: ");
5907 if (dims->size() > 4)
5908 return emitOpError(
"length_parameter requires at most 4 dimensions, got ")
5912 llvm::ArrayRef<BDDimLayoutAttr> inner(*dims);
5913 if (inner.size() == 4)
5914 inner = inner.drop_front();
5915 llvm::SmallVector<int64_t, 3> innerSizes(3, 1);
5916 for (
auto [i, dim] : llvm::enumerate(llvm::reverse(inner)))
5917 innerSizes[i] = dim.getSize();
5918 auto buffer = llvm::cast<BaseMemRefType>(getBuffer().getType());
5919 std::optional<int64_t> lenElems;
5920 if (std::optional<int32_t> len = getConstantLen())
5922 else if (!hasLen() && buffer.hasStaticShape())
5923 lenElems = buffer.getNumElements();
5933 Operation *p = (*this)->getParentOp();
5934 if (!llvm::isa<MemOp, MemTileDMAOp, ShimDMAOp, DMAOp>(*p)) {
5939 bool isUnrankedMemRef = llvm::isa<UnrankedMemRefType>(getBuffer().getType());
5943 if (!isUnrankedMemRef) {
5944 if (!isa<BufferOp, ExternalBufferOp>(getBuffer().getDefiningOp()))
5946 "BDs only support BufferOp or ExternalBufferOp operands.");
5949 if (getLenInBytes() % 4)
5950 return emitOpError(
"transfer length must be multiple of 4 (i.e., represent "
5951 "4 byte aligned address)");
5953 TileElement parentTileElement = getParentTileElement(getOperation());
5954 TileLike parentTile = parentTileElement.getTileLike();
5956 return emitOpError(
"parent tile must implement TileLike interface");
5958 if (!isUnrankedMemRef && getOperation()->getParentOfType<MemOp>() &&
5959 getBufferOp().getTile() != parentTileElement.getTile())
5961 "Core tile DMAs can only access a buffer in the same tile.");
5965 uint32_t maxBds = targetModel.
getNumBDs(parentTile.getTileType());
5966 if (std::optional<int32_t> bdId = getBdId();
5967 bdId.has_value() &&
static_cast<uint32_t
>(*bdId) >= maxBds)
5968 return emitOpError(
"bdId attribute exceeds max: ") << maxBds - 1;
5969 if (std::optional<int32_t> nextBdId = getNextBdId();
5970 nextBdId.has_value() &&
static_cast<uint32_t
>(*nextBdId) >= maxBds)
5971 return emitOpError(
"nextBdId attribute exceeds max: ") << maxBds - 1;
5973 if (getBdIdVal() && getBdId().has_value())
5974 return emitOpError(
"bd_id and bd_id_val are mutually exclusive");
5985 bool lenStaticallyKnown = getConstantLen().has_value();
5986 if (!lenStaticallyKnown && !hasLen()) {
5987 if (
auto shaped = llvm::dyn_cast<mlir::ShapedType>(getBuffer().getType()))
5988 lenStaticallyKnown = shaped.hasStaticShape();
5993 if (lenStaticallyKnown) {
5994 if (
auto constLen = getConstantLen();
5995 constLen.has_value() && *constLen <= 0) {
5998 uint64_t lenInBytes = getLenInBytes();
6000 if (granularity != 0) {
6001 uint64_t lenInWords = lenInBytes * 8 / granularity;
6004 if (lenInWords > maxLen)
6005 return emitOpError()
6006 <<
"buffer descriptor length (" << lenInWords
6007 <<
" 32-bit words) exceeds the maximum of " << maxLen
6008 <<
" words supported by this tile type";
6014 if (failed(verifyMixedSizesAndStrides()) || failed(verifyTransferInBounds()))
6019 std::optional<llvm::SmallVector<BDDimLayoutAttr>> dims;
6020 if (!getMixedSizes().empty()) {
6021 dims = getConstantDimensions();
6022 if (!dims.has_value())
6025 if (dims.has_value()) {
6029 size_t maxNDims = targetModel.
getBDMaxDims(parentTile.getTileType());
6030 if (dims->size() > maxNDims)
6031 return emitOpError() <<
"Cannot give more than "
6032 << std::to_string(maxNDims)
6033 <<
" dimensions for step sizes and wraps on this "
6035 << std::to_string(dims->size()) <<
" dimensions).";
6037 auto buffer = llvm::dyn_cast<MemRefType>(getBuffer().getType());
6039 return emitOpError() <<
"dimensions attribute cannot be used with "
6040 "unranked memref buffer type.";
6051 bool skipSizeCheck =
6063 uint32_t wrapBits = targetModel.
getDmaBdWrapBits(parentTile.getTileType());
6064 uint32_t stepBits = targetModel.
getDmaBdStepBits(parentTile.getTileType());
6065 uint64_t maxSize = wrapBits > 0 ? (1ULL << wrapBits) - 1 : 0;
6066 uint64_t maxStride = stepBits > 0 ? (1ULL << stepBits) : 0;
6068 for (BDDimLayoutAttr dim : *dims) {
6069 if (0 == dim.getStride() && dim.getSize() != 1)
6070 return emitOpError()
6071 <<
"Invalid step size; must be a positive integer.";
6072 if (dim.getStride() > buffer.getNumElements())
6073 return emitOpError() <<
"Step size " << std::to_string(dim.getStride())
6074 <<
" exceeds memref size "
6075 << std::to_string(buffer.getNumElements());
6076 if (!skipSizeCheck && dim.getSize() > maxSize)
6077 return emitOpError() <<
"Size may not exceed " << maxSize <<
".";
6078 if (dim.getStride() > maxStride)
6079 return emitOpError() <<
"Stride may not exceed " << maxStride <<
".";
6085 if (getBufferElementTypeWidthInBytes() < 4 && dims->back().getStride() != 1)
6087 "For <32b width datatypes, inner-most dim stride must be 1");
6089 if (getBufferElementTypeWidthInBytes() > 4 && dims->back().getStride() != 1)
6091 "For >32b width datatypes, inner-most dim stride must be 1");
6114 int32_t elementWidthInBytes = getBufferElementTypeWidthInBytes();
6115 if (elementWidthInBytes % 4 != 0) {
6116 std::optional<size_t> effectiveInnermost;
6117 for (
size_t i = 0; i < dims->size(); i++)
6118 if ((*dims)[i].getSize() > 1)
6119 effectiveInnermost = i;
6120 if (effectiveInnermost.has_value()) {
6121 for (
size_t i = 0; i < dims->size(); i++) {
6122 BDDimLayoutAttr dim = (*dims)[i];
6123 if (dim.getSize() <= 1)
6125 if (i == *effectiveInnermost) {
6128 if ((dim.getSize() * elementWidthInBytes) % 4)
6129 return emitOpError()
6130 <<
"Innermost dim size (" << dim.getSize() <<
") * "
6131 << elementWidthInBytes <<
"-byte element width = "
6132 << (dim.getSize() * elementWidthInBytes)
6133 <<
" bytes: innermost dim size must be a multiple of 4 "
6134 "bytes for sub-32b element types (the hardware size "
6135 "register is 32-bit-word granularity).";
6140 if ((dim.getStride() * elementWidthInBytes) % 4)
6141 return emitOpError()
6142 <<
"Dim " << i <<
" stride (" << dim.getStride() <<
") * "
6143 << elementWidthInBytes <<
"-byte element width = "
6144 << (dim.getStride() * elementWidthInBytes)
6145 <<
" bytes: non-innermost dim stride must be a multiple "
6146 "of 4 bytes for sub-32b element types (the hardware "
6147 "stepsize register is 32-bit-word granularity).";
6153 if (
auto paddims = getPadDimensions(); paddims.has_value()) {
6154 if (!dims.has_value())
6155 return emitOpError() <<
"Padding requires n-d data layouts expressed as"
6156 <<
" wrap(s) and stride(s).";
6159 llvm::SmallVector<std::pair<DmaBdField, DmaBdField>, 3> padFields;
6162 if (padFields.empty())
6163 return emitOpError() <<
"Padding is only supported by memtile dma bds.";
6164 if (dims->size() != paddims->size())
6165 return emitOpError() <<
"Mismatch number of dimensions between padding(s)"
6166 <<
" and wrap(s) and stride(s).";
6167 size_t numPadded = padFields.size();
6168 for (
auto [d, pad] : llvm::enumerate(llvm::reverse(*paddims)))
6169 if (d >= numPadded && (pad.getConstPadBefore() || pad.getConstPadAfter()))
6170 return emitOpError()
6171 <<
"a " << paddims->size() <<
"-dimensional walk pads only its "
6172 << numPadded <<
" innermost dimensions";
6174 for (
unsigned i = 0; i < paddims->size(); i++) {
6175 auto dim = (*dims)[i];
6176 auto paddim = (*paddims)[i];
6177 actuallen *= paddim.getConstPadBefore() + paddim.getConstPadAfter() +
6179 if (std::optional<int32_t> len = getConstantLen(); len.has_value()) {
6180 if (actuallen > *len)
6181 return emitOpError() <<
"Data exceeds len after padding.";
6182 }
else if (getLen()) {
6183 return emitOpError()
6184 <<
"Padding with a runtime len operand is not yet supported; "
6185 "use a compile-time constant len with padded BDs.";
6188 if ((paddims->back().getConstPadBefore() *
6189 getBufferElementTypeWidthInBytes()) %
6191 return emitOpError() <<
"Inner-most padding-before count must result in"
6192 <<
" padding in 32-bit words.";
6193 if ((paddims->back().getConstPadAfter() *
6194 getBufferElementTypeWidthInBytes()) %
6196 return emitOpError() <<
"Inner-most padding-after count must result in"
6197 <<
" padding in 32-bit words.";
6198 for (
auto [d, pad] : llvm::enumerate(llvm::reverse(*paddims))) {
6201 int64_t before = pad.getConstPadBefore();
6202 int64_t after = pad.getConstPadAfter();
6204 before = before * getBufferElementTypeWidthInBytes() / 4;
6205 after = after * getBufferElementTypeWidthInBytes() / 4;
6207 uint32_t maxBefore = padFields[d].first.mask();
6208 uint32_t maxAfter = padFields[d].second.mask();
6209 if (before > maxBefore || after > maxAfter)
6210 return emitOpError()
6211 <<
"D" << d <<
" pad_before/pad_after exceeds the [0:"
6212 << (before > maxBefore ? maxBefore : maxAfter) <<
"] range"
6213 << (d == 0 ?
" of 32-bit words." :
".");
6216 if (!isUnrankedMemRef &&
6217 (parentTile.isMemTile() || parentTile.isCoreTile())) {
6218 if (
auto baseAddr = getBufferOp().getAddress(); baseAddr.has_value()) {
6219 int64_t offsetInBytes = *baseAddr + getOffsetInBytes();
6220 if (offsetInBytes % 4)
6221 return emitOpError(
"bd address must be 4 byte (32b) aligned; got "
6223 << offsetInBytes <<
" (bytes)";
6226 if (
auto packetInfo = getPacket()) {
6229 if (packetInfo->getPktType() > 7)
6230 return emitOpError(
"Packet type field can only hold 3 bits.");
6231 if (packetInfo->assignedId() >
6233 return emitOpError(
"Packet ID field can only hold 5 bits.");
6241 if (!hasLen() && !getBuffer().getType().hasStaticShape())
6242 return emitOpError() <<
"buffer with dynamic shape requires static length.";
6244 if (getBurstLength() != 0 && !parentTile.isShimNOCTile())
6245 return emitOpError(
"Burst length is only supported in Shim NOC tiles that "
6246 "are connected to the memory-mapped NOC.");
6248 if (
auto axcache = getAxcache()) {
6249 if (!parentTile.isShimNOCTile()) {
6250 return emitOpError(
"AxCACHE is only supported in Shim NOC tiles "
6251 "that are connected to the memory-mapped NOC.");
6253 if (axcache > 0xF) {
6254 return emitOpError(
"AxCache value out of 4-bit range.");
6261LogicalResult DMABDPACKETOp::verify() {
6264 if (getPacketType() < 0 || getPacketType() > 7)
6265 return emitOpError(
"Packet type field can only hold 3 bits.");
6266 if (getPacketID() < 0 ||
6268 static_cast<int32_t
>(
getTargetModel(getOperation()).getMaxPacketId()))
6269 return emitOpError(
"Packet ID field can only hold 5 bits.");
6273std::optional<int64_t> DMABDOp::getStaticLenInBytes() {
6275 if (std::optional<int32_t> len = getConstantLen()) {
6276 elems =
static_cast<uint32_t
>(*len);
6280 if (hasLen() || !getSizes().empty() || !getStrides().empty())
6281 return std::nullopt;
6282 auto buffer = dyn_cast<MemRefType>(getBuffer().getType());
6283 if (!buffer || !buffer.hasStaticShape())
6284 return std::nullopt;
6285 elems = buffer.getNumElements();
6287 return llvm::checkedMul(
6288 elems,
static_cast<int64_t
>(getBufferElementTypeWidthInBytes()));
6291uint32_t DMABDOp::getAxcacheOrDefault() {
6292 return getAxcache().value_or(
6300static LogicalResult FoldDMAStartOp(DMAStartOp op, PatternRewriter &rewriter) {
6302 llvm::SetVector<Block *> reachable;
6303 SmallVector<Block *, 16> worklist;
6304 Block *firstBD = op.getSuccessor(0);
6305 reachable.insert(firstBD);
6306 worklist.push_back(firstBD);
6307 while (!worklist.empty()) {
6308 Block *block = worklist.pop_back_val();
6311 auto successors = block->getTerminator()->getSuccessors();
6312 for (
auto *i : successors) {
6313 if (!reachable.contains(i)) {
6314 reachable.insert(i);
6315 worklist.push_back(i);
6322 if (isa<EndOp>((reachable.back())->getTerminator()))
6326 auto areEquivalentBDs = [](Block *b1, Block *b2) {
6327 auto b1OpRange = b1->without_terminator();
6328 auto b2OpRange = b2->without_terminator();
6329 if (llvm::range_size(b1OpRange) != llvm::range_size(b2OpRange))
6331 if (!llvm::all_of(llvm::zip_equal(b1OpRange, b2OpRange),
6332 [](std::tuple<Operation &, Operation &> pair) {
6333 return OperationEquivalence::isEquivalentTo(
6334 &std::get<0>(pair), &std::get<1>(pair),
6335 OperationEquivalence::IgnoreLocations);
6342 SmallVector<Block *> uniquePattern;
6343 const auto *patternIt = reachable.begin();
6344 while (patternIt != reachable.end() &&
6345 llvm::none_of(uniquePattern, [patternIt, areEquivalentBDs](Block *b1) {
6346 return areEquivalentBDs(*patternIt, b1);
6348 uniquePattern.push_back(*patternIt);
6353 if (patternIt == reachable.end())
6357 while (patternIt != reachable.end()) {
6359 if (!areEquivalentBDs(*patternIt, uniquePattern[idx]))
6362 idx = (idx + 1) % uniquePattern.size();
6366 auto lastBDTerm = cast<NextBDOp>(reachable.back()->getTerminator());
6367 auto lastUniqueBDTerm = cast<NextBDOp>(uniquePattern.back()->getTerminator());
6368 rewriter.modifyOpInPlace(lastUniqueBDTerm, [&] {
6369 lastUniqueBDTerm.setSuccessor(lastBDTerm.getSuccessor());
6379struct LinearizeContiguousBDTransfer :
public mlir::OpRewritePattern<DMABDOp> {
6380 using OpRewritePattern::OpRewritePattern;
6383 matchAndRewrite(DMABDOp op, mlir::PatternRewriter &rewriter)
const override {
6384 if (op.getStaticSizes().value_or(llvm::ArrayRef<int64_t>{}).empty())
6385 return mlir::failure();
6390 bool bufferIsExternal =
6391 isa_and_nonnull<ExternalBufferOp>(op.getBuffer().getDefiningOp());
6392 TileElement parentElem = getParentTileElement(op.getOperation());
6393 TileLike parentTileLike =
6394 parentElem ? parentElem.getTileLike() : TileLike{};
6395 bool parentIsShim = parentTileLike && parentTileLike.isShimTile();
6396 bool inShimDMA = (bool)op->getParentOfType<ShimDMAOp>();
6397 if (!bufferIsExternal && !parentIsShim && !inShimDMA)
6398 return mlir::failure();
6403 auto isConstant = [](mlir::Value v) {
6404 return mlir::getConstantIntValue(v).has_value();
6406 if (!llvm::all_of(op.getSizes(), isConstant) ||
6407 !llvm::all_of(op.getStrides(), isConstant))
6408 return mlir::failure();
6409 std::optional<llvm::SmallVector<BDDimLayoutAttr>> dims =
6410 op.getFoldedDimensions([&]() {
return op.emitError(); });
6411 if (!dims.has_value() || dims->empty())
6412 return mlir::failure();
6414 return mlir::failure();
6416 if (dims->size() == 1 && dims->front().getStride() == 1)
6417 return mlir::failure();
6419 int64_t product = 1;
6420 for (BDDimLayoutAttr dim : *dims)
6421 product *= dim.getSize();
6427 if (!op.hasLen() && !op.getBuffer().getType().hasStaticShape())
6428 return mlir::failure();
6429 std::optional<int32_t> lenVal = op.getConstantLen();
6430 if (lenVal.has_value() &&
static_cast<int64_t
>(*lenVal) != product)
6431 return mlir::failure();
6432 int32_t len =
static_cast<int32_t
>(product);
6436 rewriter.modifyOpInPlace(op, [&]() {
6437 op.getLenMutable().clear();
6438 op.setStaticLen(len);
6439 op.getSizesMutable().clear();
6440 op.getStridesMutable().clear();
6441 op.setStaticSizes(std::nullopt);
6442 op.setStaticStrides(std::nullopt);
6444 return mlir::success();
6452struct FoldConstantBDDimList :
public mlir::OpRewritePattern<DMABDOp> {
6453 using OpRewritePattern::OpRewritePattern;
6456 matchAndRewrite(DMABDOp op, mlir::PatternRewriter &rewriter)
const override {
6457 if (op.getSizes().empty() && op.getStrides().empty() && !op.getOffset() &&
6459 return mlir::failure();
6460 llvm::SmallVector<mlir::OpFoldResult> sizes = op.getMixedSizes();
6461 llvm::SmallVector<mlir::OpFoldResult> strides = op.getMixedStrides();
6467 bool changed = succeeded(mlir::foldDynamicIndexList(sizes)) |
6468 succeeded(mlir::foldDynamicIndexList(strides));
6471 std::optional<int32_t> foldOffset, foldLen;
6473 if (
auto c = mlir::getConstantIntValue(op.getOffset()))
6474 foldOffset =
static_cast<int32_t
>(*c);
6476 if (
auto c = mlir::getConstantIntValue(op.getLen()))
6477 foldLen =
static_cast<int32_t
>(*c);
6478 changed |= foldOffset.has_value() || foldLen.has_value();
6481 return mlir::failure();
6483 llvm::SmallVector<int64_t> staticSizes, staticStrides;
6484 llvm::SmallVector<mlir::Value> dynSizes, dynStrides;
6485 mlir::dispatchIndexOpFoldResults(sizes, dynSizes, staticSizes);
6486 mlir::dispatchIndexOpFoldResults(strides, dynStrides, staticStrides);
6491 bool wasDynamic = !op.getSizes().empty() || !op.getStrides().empty();
6492 if (!op.hasLen() && wasDynamic && dynSizes.empty() && dynStrides.empty() &&
6493 !staticSizes.empty()) {
6495 for (int64_t s : llvm::ArrayRef(staticSizes).take_back(3))
6497 foldLen =
static_cast<int32_t
>(extent);
6500 rewriter.modifyOpInPlace(op, [&]() {
6501 op.getSizesMutable().assign(dynSizes);
6502 op.getStridesMutable().assign(dynStrides);
6504 if (staticSizes.empty())
6505 op.removeStaticSizesAttr();
6507 op.setStaticSizes(staticSizes);
6508 if (staticStrides.empty())
6509 op.removeStaticStridesAttr();
6511 op.setStaticStrides(staticStrides);
6513 op.getOffsetMutable().clear();
6514 op.setStaticOffset(foldOffset);
6517 op.getLenMutable().clear();
6518 op.setStaticLen(foldLen);
6521 return mlir::success();
6526void DMABDOp::getCanonicalizationPatterns(RewritePatternSet &results,
6527 MLIRContext *context) {
6528 results.add<FoldConstantBDDimList, LinearizeContiguousBDTransfer>(context);
6531void DMAStartOp::getCanonicalizationPatterns(RewritePatternSet &results,
6532 MLIRContext *context) {
6533 results.add(FoldDMAStartOp);
6536RouteEndpointOp DMAStartOp::getEndpointOp() {
6537 FlatSymbolRefAttr name = getEndpoint();
6540 return SymbolTable::lookupNearestSymbolFrom<RouteEndpointOp>(*
this, name);
6543LogicalResult DMAStartOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
6544 FlatSymbolRefAttr name = getEndpoint();
6547 auto program = dyn_cast<DmaBody>((*this)->getParentOp());
6548 auto device = (*this)->getParentOfType<DeviceOp>();
6549 if (!program || !device)
6550 return emitOpError(
"only a tile's DMA program may name its channel by a "
6552 auto endpoint = symbolTable.lookupSymbolIn<RouteEndpointOp>(device, name);
6554 return emitOpError() << name <<
" is not an aie.route_endpoint";
6555 if (endpoint.getBundle() != WireBundle::DMA)
6556 return emitOpError() << name <<
" names a "
6557 << stringifyWireBundle(endpoint.getBundle())
6558 <<
" port, not a DMA channel";
6559 auto here = cast<TileLike>(program.getTile().getDefiningOp());
6560 TileLike there = endpoint.getTileLike();
6561 bool sameTile = endpoint.getTile() == program.getTile() ||
6562 (here.tryGetCol() && here.tryGetRow() &&
6563 here.tryGetCol() == there.tryGetCol() &&
6564 here.tryGetRow() == there.tryGetRow());
6566 return emitOpError() << name
6567 <<
" is on a different tile than this DMA program";
6571LogicalResult DMAStartOp::verify() {
6572 if (getPadValue() != 0) {
6573 if (!isa<MemTileDMAOp>(getOperation()->getParentOp()))
6574 return emitOpError(
"pad_value is only supported on memtile DMA channels");
6575 if (getChannelDir() != DMAChannelDir::MM2S)
6576 return emitOpError(
"pad_value is only supported on MM2S DMA channels");
6577 if (!
getTargetModel(getOperation()).isMemTilePadValueSupported())
6578 return emitOpError(
"pad_value requires the CONSTANT_PAD_VALUE register, "
6579 "unavailable on this target");
6581 if (failed(verifyDMARepeatCount(getOperation(), getRepeatCount())))
6583 if (
auto program = dyn_cast<DmaBody>((*this)->getParentOp());
6584 program && !getEndpoint() &&
6586 getOperation(), cast<TileLike>(program.getTile().getDefiningOp()),
6587 getChannelDir(), getChannelIndex())))
6589 SmallVector<DMABDOp> bds;
6590 if (getOutOfOrder()) {
6591 llvm::SmallPtrSet<Block *, 8> visited;
6592 for (Block *b = getDest(); b && visited.insert(b).second;
6593 b = b->getNumSuccessors() > 0 ? b->getSuccessor(0) :
nullptr)
6594 llvm::append_range(bds, b->getOps<DMABDOp>());
6597 getOutOfOrder(), bds);
6604LogicalResult SwitchboxOp::verify() {
6610 if (!isa<TileOp>(getTile().getDefiningOp()))
6613 Region &body = getConnections();
6614 DenseSet<Port> sourceset;
6615 DenseSet<Port> destset;
6616 auto tile = getTileOp();
6619 return emitOpError(
"should have non-empty body");
6621 for (
auto connectOp : body.front().getOps<ConnectOp>())
6622 sourceset.insert({connectOp.getSourceBundle(), connectOp.sourceIndex()});
6623 for (
auto &ops : body.front()) {
6625 auto checkBound = [&ops](StringRef dir, WireBundle bundle,
int index,
6626 int bound) -> LogicalResult {
6627 if (index >= bound) {
6629 return ops.emitOpError(
"index ")
6630 << index <<
" for " << dir <<
" bundle "
6631 << stringifyWireBundle(bundle) <<
" must be less than "
6633 return ops.emitOpError()
6634 << dir <<
" bundle " << stringifyWireBundle(bundle)
6635 <<
" not supported; index: " << index <<
", bound: " << bound;
6640 if (
auto connectOp = dyn_cast<ConnectOp>(ops)) {
6641 Port source = {connectOp.getSourceBundle(), connectOp.sourceIndex()};
6643 Port dest = {connectOp.getDestBundle(), connectOp.destIndex()};
6644 if (destset.count(dest)) {
6645 return connectOp.emitOpError()
6646 <<
"; connecting " << to_string(source) <<
" to "
6647 << to_string(dest) <<
" on "
6648 << to_string(this->getTileOp().getTileID())
6649 <<
" targets same dst as another connect op; existing "
6651 << llvm::join(llvm::map_range(
6652 destset, [](
auto &p) {
return to_string(p); }),
6655 destset.insert(dest);
6657 if (connectOp.sourceIndex() < 0)
6658 return connectOp.emitOpError(
"source index cannot be less than zero");
6660 if (checkBound(
"source", connectOp.getSourceBundle(),
6661 connectOp.sourceIndex(),
6662 getNumSourceConnections(connectOp.getSourceBundle()))
6666 if (connectOp.destIndex() < 0)
6667 return connectOp.emitOpError(
"dest index cannot be less than zero");
6669 if (checkBound(
"dest", connectOp.getDestBundle(), connectOp.destIndex(),
6670 getNumDestConnections(connectOp.getDestBundle()))
6675 if (!isLegalTileConnection(tile, targetModel, connectOp))
6676 return connectOp.emitOpError(
"illegal stream switch connection");
6678 }
else if (
auto connectOp = dyn_cast<MasterSetOp>(ops)) {
6679 Port dest = {connectOp.getDestBundle(), connectOp.destIndex()};
6680 if (destset.count(dest))
6681 return connectOp.emitOpError(
"targets same destination ")
6682 << stringifyWireBundle(dest.bundle) <<
": " << dest.channel
6683 <<
" as another connect or masterset operation";
6684 destset.insert(dest);
6686 if (connectOp.destIndex() < 0)
6687 return connectOp.emitOpError(
"dest index cannot be less than zero");
6689 if (checkBound(
"dest", connectOp.getDestBundle(), connectOp.destIndex(),
6690 getNumDestConnections(connectOp.getDestBundle()))
6695 }
else if (
auto connectOp = dyn_cast<PacketRulesOp>(ops)) {
6696 Port source = {connectOp.getSourceBundle(), connectOp.sourceIndex()};
6697 if (sourceset.count(source))
6698 return connectOp.emitOpError(
"packet switched source ")
6699 << stringifyWireBundle(source.bundle) << source.channel
6700 <<
" cannot match another connect or masterset operation";
6701 sourceset.insert(source);
6703 }
else if (
auto amselOp = dyn_cast<AMSelOp>(ops)) {
6704 std::vector<MasterSetOp> mstrs;
6705 std::vector<PacketRulesOp> slvs;
6706 for (
auto *user : amselOp.getResult().getUsers()) {
6707 if (
auto s = dyn_cast<PacketRuleOp>(user)) {
6708 auto pktRules = cast<PacketRulesOp>(s->getParentOp());
6709 slvs.push_back(pktRules);
6710 }
else if (
auto m = dyn_cast<MasterSetOp>(user))
6713 for (
auto m : mstrs) {
6714 for (
auto s : slvs) {
6716 if (!isLegalTileConnection(tile, targetModel, m, s)) {
6717 return amselOp->emitOpError(
"illegal stream switch connection");
6721 }
else if (isa<EndOp>(ops)) {
6724 return ops.emitOpError(
"cannot be contained in a Switchbox op");
6731template <
typename... ParentOpTypes>
6734 Operation *operation = op->getParentOp();
6736 if (llvm::isa_and_nonnull<ParentOpTypes...>(operation))
6738 operation = operation->getParentOp();
6744TileOp LockOp::getTileOp() {
6745 return cast<TileElement>(this->getOperation()).getTileOp();
6748LogicalResult LockOp::verify() {
6749 if (
auto result = UsesAreAccessible::verifyTrait(*
this); result.failed())
6752 if (
auto lockID = getLockID()) {
6753 TileLike tileLike = getTileLike();
6755 return emitOpError(
"tile operand must implement TileLike interface");
6757 auto tileType = tileLike.getTileType();
6758 if (
int numLocks = targetModel.
getNumLocks(tileType); *lockID >= numLocks)
6759 return emitOpError(
"lock assigned invalid id (maximum is ")
6760 << numLocks - 1 <<
")";
6763 if (std::optional<int32_t> init = getInit()) {
6765 if (*init < 0 || *init > max)
6766 return emitOpError(
"init value ")
6767 << *init <<
" is outside the lock's range [0, " << max <<
"]";
6775static std::optional<int32_t> getConstantLockValue(UseLockOp op) {
6776 if (
auto constant = op.getValue().getDefiningOp<arith::ConstantOp>())
6777 if (
auto intAttr = llvm::dyn_cast<IntegerAttr>(constant.getValue()))
6778 return (int32_t)intAttr.getInt();
6779 return std::nullopt;
6784 auto *block = op->getBlock();
6786 for (
auto op : block->getOps<UseLockOp>()) {
6787 if (
auto lock = dyn_cast<LockOp>(op.getLock().getDefiningOp());
6788 lock.getLockID().has_value()) {
6789 if (lockID != -1 && lockID != lock.getLockIDValue())
6791 lockID = lock.getLockIDValue();
6800 auto *block = op->getBlock();
6801 int acqValue = -1, relValue = -1;
6802 for (
auto op : block->getOps<UseLockOp>()) {
6805 auto value = getConstantLockValue(op);
6808 if (op.acquire() || op.acquireGE()) {
6809 if (acqValue != -1 && acqValue != *value) {
6813 }
else if (op.release()) {
6814 if (relValue != -1 && relValue != *value) {
6826 if (
auto memOp = op->getParentOfType<MemOp>()) {
6827 auto useLock = cast<UseLockOp>(op);
6828 if (
auto lock = useLock.getLockOp(); lock.getTile() != memOp.getTile())
6835LogicalResult UseLockOp::verify() {
6837 if (llvm::isa_and_nonnull<DeviceOp, ModuleOp>((*this)->getParentOp()))
6838 return (*this)->emitOpError(
"must be used in a core or memory operation.");
6841 if (targetModel.
getTargetArch() == AIEArch::AIE1 && acquireGE())
6842 return (*this)->emitOpError(
6843 "AcquireGreaterEqual is not supported in AIE1.");
6849 !getConstantLockValue(*
this))
6850 return (*this)->emitOpError(
6851 "lock value in a DMA/BD block must be a compile-time constant "
6852 "(defined by an arith.constant).");
6858 if (!(*this)->getBlock())
6859 return (*this)->emitOpError(
"is not in a block.");
6863 return (*this)->emitOpError(
6864 "used in a DMA block that have multiple locks.");
6867 return (*this)->emitOpError(
"acquires/releases the lock in a DMA block "
6868 "from/to multiple states.");
6872 return (*this)->emitOpError(
"can only access a lock in the same tile");
6884 Operation *operation = (*this)->getParentOp();
6886 StringRef name = operation->getName().getStringRef();
6887 if (name ==
"aiex.dma_configure_task" ||
6888 name ==
"aiex.dma_configure_task_for")
6890 operation = operation->getParentOp();
6893 return (*this)->emitOpError()
6894 <<
"expects some parent op to be one of "
6895 <<
"AIE::device, AIE::core, func::func, AIE::mem, AIE::shimDMA, or "
6896 "AIEX::dma_configure_task";
6899#include "aie/Dialect/AIE/IR/AIEEnums.cpp.inc"
6900#include "aie/Dialect/AIE/IR/AIEInterfaces.cpp.inc"
6907static ParseResult parseTraceEventValue(AsmParser &parser, Attribute &value) {
6912static void printTraceEventValue(AsmPrinter &printer, Attribute value) {
6917static ParseResult parseTraceRegValue(OpAsmParser &parser, Attribute &value) {
6921 OptionalParseResult parseResult = parser.parseOptionalInteger(intValue);
6922 if (parseResult.has_value() && succeeded(parseResult.value())) {
6923 MLIRContext *ctx = parser.getContext();
6924 value = IntegerAttr::get(IntegerType::get(ctx, 32), intValue);
6931static void printTraceRegValue(OpAsmPrinter &printer, Operation *op,
6934 if (
auto intAttr = llvm::dyn_cast<IntegerAttr>(value)) {
6935 printer << intAttr.getInt();
6942#define GET_OP_CLASSES
6943#include "aie/Dialect/AIE/IR/AIEOps.cpp.inc"
6945FailureOr<int32_t> UseLockOp::getConstantValue() {
6946 if (
auto value = getConstantLockValue(*
this))
6948 return emitOpError(
"expected the lock value to be a compile-time constant "
6949 "(defined by an arith.constant).");
6952LogicalResult UseLockOp::canonicalize(UseLockOp op, PatternRewriter &rewriter) {
6956 if (op.acquireGE() && op->getBlock()->getOps<DMABDOp>().empty()) {
6957 if (
auto value = getConstantLockValue(op); value && *value == 0) {
6958 rewriter.eraseOp(op);
6965TileOp SwitchboxOp::getTileOp() {
6966 return cast<TileElement>(this->getOperation()).getTileOp();
6969size_t SwitchboxOp::getNumSourceConnections(WireBundle bundle) {
6970 auto tile = getTileOp();
6973 tile.getRow(), bundle);
6976size_t SwitchboxOp::getNumDestConnections(WireBundle bundle) {
6977 auto tile = getTileOp();
6980 tile.getRow(), bundle);
6983TileOp ShimMuxOp::getTileOp() {
6984 return cast<TileElement>(this->getOperation()).getTileOp();
6989 case WireBundle::North:
6990 return WireBundle::South;
6991 case WireBundle::South:
6992 return WireBundle::North;
6993 case WireBundle::East:
6994 return WireBundle::West;
6995 case WireBundle::West:
6996 return WireBundle::East;
7002std::optional<std::pair<TileID, Port>>
7005 switch (port.bundle) {
7006 case WireBundle::North:
7009 case WireBundle::South:
7012 case WireBundle::East:
7015 case WireBundle::West:
7019 return std::nullopt;
7022 return std::nullopt;
7030ParseResult BDChainOp::parse(OpAsmParser &parser, OperationState &result) {
7031 SmallVector<OpAsmParser::Argument> entryArgs;
7034 StringAttr symNameAttr;
7035 if (parser.parseSymbolName(symNameAttr, SymbolTable::getSymbolAttrName(),
7036 result.attributes)) {
7041 ParseResult argParseResult = parser.parseCommaSeparatedList(
7042 OpAsmParser::Delimiter::Paren, [&]() -> ParseResult {
7043 OpAsmParser::Argument argument;
7044 if (parser.parseArgument(argument,
true,
true)) {
7047 entryArgs.push_back(argument);
7050 if (argParseResult) {
7051 return argParseResult;
7055 auto *body = result.addRegion();
7056 ParseResult bodyParseResult = parser.parseRegion(*body, entryArgs,
false);
7057 if (bodyParseResult) {
7058 return bodyParseResult;
7064void BDChainOp::print(OpAsmPrinter &printer) {
7067 ->getAttrOfType<StringAttr>(SymbolTable::getSymbolAttrName())
7070 printer.printSymbolName(taskName);
7072 Region &body = getRegion();
7073 auto argsIter = body.getArguments();
7075 for (
auto *it = argsIter.begin(); it != argsIter.end(); ++it) {
7076 if (it != argsIter.begin()) {
7079 printer.printRegionArgument(*it);
7084 printer.printRegion(body,
false,
true);
7091LogicalResult ShimDMAAllocationOp::verify() {
7092 TileLike tileLike = llvm::dyn_cast<TileLike>(getTile().getDefiningOp());
7094 return emitOpError(
"tile operand must implement TileLike interface");
7097 if (!tileLike.isShimNOCorPLTile()) {
7099 auto col = tileLike.tryGetCol();
7100 auto row = tileLike.tryGetRow();
7102 return emitOpError(
"tile must be a shim tile, but got tile(")
7103 << *col <<
", " << *row <<
")";
7105 return emitOpError(
"tile must be a shim tile");
7108 if (ArrayAttr proofs = getConsumedByAttr()) {
7109 auto device = (*this)->getParentOfType<DeviceOp>();
7110 if (failed(verifyConsumedBy(
7111 *
this, proofs, getChannelDir() == DMAChannelDir::MM2S,
7112 "aie.shim_dma_allocation", [&](FlatSymbolRefAttr symbol) {
7113 auto output = getForSymbol(device, symbol.getValue());
7114 return output && output.getChannelDir() == DMAChannelDir::S2MM;
7122TileOp ShimDMAAllocationOp::getTileOp() {
7123 return dyn_cast_or_null<TileOp>(getTile().getDefiningOp());
7126ShimDMAAllocationOp ShimDMAAllocationOp::getForSymbol(DeviceOp device,
7127 llvm::StringRef symbol) {
7128 Operation *maybeOp = device.lookupSymbol(symbol);
7130 if (ShimDMAAllocationOp op = dyn_cast<ShimDMAAllocationOp>(maybeOp)) {
7141LogicalResult ObjectFifoRearmBindingOp::verify() {
7142 if (getChannelDirs().size() != getChannelTiles().size())
7143 return emitOpError(
"expected one channel_dirs entry per channel tile (")
7144 << getChannelTiles().size() <<
" tiles, " << getChannelDirs().size()
7146 if (getChannelIndices().size() != getChannelTiles().size())
7147 return emitOpError(
"expected one channel_indices entry per channel tile (")
7148 << getChannelTiles().size() <<
" tiles, "
7149 << getChannelIndices().size() <<
" indices)";
7150 if (getLockInits().size() != getLocks().size())
7151 return emitOpError(
"expected one lock_inits entry per lock (")
7152 << getLocks().size() <<
" locks, " << getLockInits().size()
7158 if (getHeadBdIds().has_value() != getRepeatCounts().has_value())
7159 return emitOpError(
"head_bd_ids and repeat_counts must both be set or both "
7161 if (
auto headBdIds = getHeadBdIds();
7162 headBdIds.has_value() && headBdIds->size() != getChannelTiles().size())
7163 return emitOpError(
"expected one head_bd_ids entry per channel tile (")
7164 << getChannelTiles().size() <<
" tiles, " << headBdIds->size()
7166 if (
auto repeatCounts = getRepeatCounts();
7167 repeatCounts.has_value() &&
7168 repeatCounts->size() != getChannelTiles().size())
7169 return emitOpError(
"expected one repeat_counts entry per channel tile (")
7170 << getChannelTiles().size() <<
" tiles, " << repeatCounts->size()
7172 for (int32_t dir : getChannelDirs())
7173 if (dir != 0 && dir != 1)
7174 return emitOpError(
"channel_dirs entries must be 0 (S2MM) or 1 (MM2S), "
7180 for (Value tile : getChannelTiles()) {
7181 auto tileOp = tile.getDefiningOp<TileOp>();
7183 return emitOpError(
"channel_tiles operands must be aie.tile values");
7188 if (tileOp.isShimTile())
7189 return emitOpError(
"channel_tiles must be non-shim tiles; a shim DMA "
7190 "endpoint is host-managed and is not re-armed here");
7192 for (Value lock : getLocks())
7193 if (!lock.getDefiningOp<LockOp>())
7194 return emitOpError(
"locks operands must be aie.lock values");
7197 if (std::optional<ArrayRef<int32_t>> headBdIds = getHeadBdIds()) {
7199 for (
auto [tile, channel, bdId] :
7200 llvm::zip(getChannelTiles(), getChannelIndices(), *headBdIds)) {
7201 auto tileOp = tile.getDefiningOp<TileOp>();
7203 tileOp.getCol(), tileOp.getRow(), bdId, channel))
7204 return emitOpError(
"head BD ")
7205 << bdId <<
" is not accessible from channel " << channel
7206 <<
" of tile (" << tileOp.getCol() <<
", " << tileOp.getRow()
7217ParseResult RuntimeSequenceOp::parse(OpAsmParser &parser,
7218 OperationState &result) {
7221 StringAttr nameAttr;
7222 (void)parser.parseOptionalSymbolName(
7223 nameAttr, mlir::SymbolTable::getSymbolAttrName(), result.attributes);
7225 SmallVector<OpAsmParser::Argument> entryArgs;
7228 ParseResult argParseResult = parser.parseCommaSeparatedList(
7229 OpAsmParser::Delimiter::Paren, [&]() -> ParseResult {
7230 OpAsmParser::Argument argument;
7231 if (parser.parseArgument(argument,
true,
true)) {
7234 entryArgs.push_back(argument);
7237 if (argParseResult) {
7238 return argParseResult;
7243 if (parser.parseOptionalAttrDictWithKeyword(result.attributes))
7247 auto *body = result.addRegion();
7248 ParseResult bodyParseResult = parser.parseRegion(*body, entryArgs,
false);
7249 if (bodyParseResult) {
7250 return bodyParseResult;
7256void RuntimeSequenceOp::print(OpAsmPrinter &printer) {
7257 Region &body = getRegion();
7259 auto nameAttr = (*this)->getAttrOfType<StringAttr>(
7260 mlir::SymbolTable::getSymbolAttrName());
7262 nameAttr != ::mlir::OpBuilder((*this)->getContext())
7263 .getStringAttr(getDefaultRuntimeSequenceName())) {
7265 printer.printSymbolName(nameAttr);
7269 for (
unsigned i = 0, n = body.getNumArguments(); i < n; i++) {
7273 printer.printRegionArgument(body.getArgument(i));
7277 printer.printOptionalAttrDictWithKeyword(
7278 (*this)->getAttrs(),
7279 {mlir::SymbolTable::getSymbolAttrName()});
7282 printer.printRegion(body,
false,
true);
7285LogicalResult RuntimeSequenceOp::verify() {
7286 DeviceOp device = (*this)->getParentOfType<DeviceOp>();
7289 (*this)->emitOpError() <<
"must be inside AIE device operation.";
7296RuntimeSequenceOp::getForSymbolInDevice(DeviceOp deviceOp,
7297 llvm::StringRef symbol) {
7298 RuntimeSequenceOp runtimeSequenceOp;
7299 if (symbol.empty()) {
7300 auto range = deviceOp.getOps<RuntimeSequenceOp>();
7301 if (range.begin() == range.end()) {
7306 runtimeSequenceOp = *range.begin();
7308 Operation *maybeRuntimeSequenceOp =
7309 mlir::SymbolTable::lookupSymbolIn(deviceOp, symbol);
7310 if (!maybeRuntimeSequenceOp) {
7314 llvm::dyn_cast<RuntimeSequenceOp>(maybeRuntimeSequenceOp);
7316 return runtimeSequenceOp;
7320RuntimeSequenceOp::getForSymbolInDeviceOrError(DeviceOp deviceOp,
7321 llvm::StringRef symbol) {
7322 RuntimeSequenceOp runtimeSequenceOp = getForSymbolInDevice(deviceOp, symbol);
7323 if (!runtimeSequenceOp) {
7324 if (!symbol.empty()) {
7325 deviceOp.emitError(
"No such runtime sequence: ") << symbol;
7327 deviceOp.emitError(
"No runtime sequence in device");
7330 return runtimeSequenceOp;
7333LogicalResult RuntimeSequenceOp::verifyBeforeMaterialization() {
7337 auto result = (*this)->walk([&](Operation *op) {
7338 for (NamedAttribute namedAttr : op->getAttrs()) {
7339 Attribute attr = namedAttr.getValue();
7340 auto walkResult = attr.walk([&](SymbolRefAttr symbolRef) {
7341 Operation *symbolDefOp =
7342 SymbolTable::lookupNearestSymbolFrom(*
this, symbolRef);
7344 if (
auto flat = dyn_cast<FlatSymbolRefAttr>(symbolRef)) {
7349 if (!llvm::isa<ShimDMAAllocationOp>(symbolDefOp) &&
7350 !llvm::isa<DeviceOp>(symbolDefOp) &&
7351 !llvm::isa<RuntimeSequenceOp>(symbolDefOp) &&
7352 !llvm::isa<BufferOp>(symbolDefOp) &&
7353 !llvm::isa<ObjectFifoRearmBindingOp>(symbolDefOp) &&
7354 !llvm::isa<memref::GlobalOp>(symbolDefOp)) {
7356 <<
"references symbol '"
7357 << symbolRef.getRootReference().getValue()
7358 <<
"' which must be either a ShimDMAAllocationOp, DeviceOp, "
7359 "RuntimeSequenceOp, BufferOp, ObjectFifoRearmBindingOp, or "
7360 "GlobalOp, but got: "
7361 << symbolDefOp->getName().getStringRef();
7362 return WalkResult::interrupt();
7364 if (BufferOp bufferOp = llvm::dyn_cast<BufferOp>(symbolDefOp)) {
7365 if (!bufferOp.getAddress()) {
7367 <<
"Unallocated buffer; fixed addresses are required before "
7368 "runtime sequence materialization.";
7369 return WalkResult::interrupt();
7373 return WalkResult::advance();
7375 if (walkResult.wasInterrupted()) {
7376 return WalkResult::interrupt();
7379 return WalkResult::advance();
7382 if (result.wasInterrupted()) {
7390#define GET_ATTRDEF_CLASSES
7391#include "aie/Dialect/AIE/IR/AIEAttrs.cpp.inc"
7394 ObjectFifoCreateOp objFifo,
7395 ObjectFifoPort port) {
7396 for (
auto coreOp : device.getOps<CoreOp>()) {
7397 if (coreOp.getTile() != tile) {
7401 coreOp.walk([&](Operation *op) {
7402 if (
auto acq = dyn_cast<ObjectFifoAcquireOp>(op)) {
7403 found |= acq.getObjectFifo() == objFifo && acq.getPort() == port;
7404 }
else if (
auto rel = dyn_cast<ObjectFifoReleaseOp>(op)) {
7405 found |= rel.getObjectFifo() == objFifo && rel.getPort() == port;
virtual uint64_t getDmaBdMaxLen(AIETileType tileType) const =0
Return the maximum transfer length, measured in address-generation granules (see getAddressGenGranula...
virtual const DmaBdLayout * getDmaBdLayout(AIETileType tileType) const
Return the register layout of a buffer descriptor on the given tile type, or null if the target has n...
virtual AIEArch getTargetArch() const =0
Return the target architecture.
virtual uint32_t getNumSourceShimMuxConnections(int col, int row, WireBundle bundle) const =0
Return the number of sources of connections inside a shimmux.
virtual uint32_t getDmaBdWrapBits(AIETileType tileType) const =0
Return the bit width of the wrap (size) field in a DMA buffer descriptor for the given tile type.
virtual uint32_t getLocalMemorySize() const =0
Return the size (in bytes) of the local data memory of a core.
virtual uint32_t getNumLocks(AIETileType tileType) const =0
Return the number of lock objects for a given tile type.
virtual uint32_t getDmaBdIterBits(AIETileType tileType) const =0
Return the bit width of the iteration (repeat) wrap/stride fields in a DMA buffer descriptor for the ...
virtual uint32_t getBDMaxDims(AIETileType tileType) const =0
Return the maximum number of addressing dimensions (wraps/strides) a single buffer descriptor support...
virtual std::vector< std::pair< uint32_t, uint32_t > > getShimBurstEncodingsAndLengths() const =0
virtual bool isLegalTileConnection(int col, int row, WireBundle srcBundle, int srcChan, WireBundle dstBundle, int dstChan) const =0
virtual uint32_t getNumBanks(int col, int row) const =0
Return the number of memory banks of a given tile.
virtual uint32_t getDmaTaskQueueDepth() const =0
Return how many DMA tasks a single channel's task queue holds (unsupported = 0).
virtual uint32_t getDmaBdStepBits(AIETileType tileType) const =0
Return the bit width of the step (stride) field in a DMA buffer descriptor for the given tile type.
virtual bool isBdChannelAccessible(int col, int row, uint32_t bd_id, int channel) const =0
Return true iff buffer descriptor bd_id on tile (col, row) can be submitted on channel channel.
virtual AIETileType getTileType(int col, int row) const =0
Return the tile type for the given tile coordinates.
virtual bool isValidTile(TileID src) const
Return true if the given tile ID is valid.
virtual uint32_t getMaxRepeatCount() const =0
Return the largest DMA task repeat count (unsupported = 0).
virtual int rows() const =0
Return the number of rows in the device.
bool isShimNOCTile(int col, int row) const
Return true if the given tile is a ShimNOC tile.
virtual uint32_t getNumBDs(AIETileType tileType) const =0
Return the number of buffer descriptors for a given tile type.
virtual uint32_t getMaxLockValue() const =0
Return the maximum value that can be stored in a lock register.
virtual uint32_t getMaxOutOfOrderId() const =0
Return the largest out-of-order BD id (unsupported = 0).
bool hasProperty(ModelProperty Prop) const
virtual int columns() const =0
Return the number of columns in the device.
virtual uint32_t getNumDestShimMuxConnections(int col, int row, WireBundle bundle) const =0
Return the number of destinations of connections inside a shimmux.
virtual uint32_t getNumDestSwitchboxConnections(int col, int row, WireBundle bundle) const =0
Return the number of destinations of connections inside a switchbox.
SharedMemory
Which of two tiles' memory modules both of them can address.
@ None
Neither tile can address the other's memory module.
SharedMemory getSharedMemory(TileID a, TileID b) const
Return the memory module a and b can both reach, if any.
virtual uint32_t getAddressGenGranularity() const =0
Return the data bus width of the device.
virtual uint32_t getMemTileSize() const =0
Return the size (in bytes) of a MemTile.
virtual uint32_t getNumSourceSwitchboxConnections(int col, int row, WireBundle bundle) const =0
Return the number of sources of connections inside a switchbox.
NamedOpTable(mlir::Operation *symbolTableOp)
std::string uniqueName(llvm::StringRef prefix, unsigned &counter) const
bool isContiguousBDTransfer(llvm::ArrayRef< BDDimLayoutAttr > dims)
mlir::ParseResult parseFlowVias(mlir::OpAsmParser &parser, llvm::SmallVectorImpl< mlir::OpAsmParser::UnresolvedOperand > &vias, mlir::DenseI32ArrayAttr &ingressBundles, mlir::DenseI32ArrayAttr &ingressChannels, mlir::DenseI32ArrayAttr &egressBundles, mlir::DenseI32ArrayAttr &egressChannels)
void printObjectFifoProducerTile(mlir::OpAsmPrinter &printer, mlir::Operation *op, mlir::Value operand, BDDimLayoutArrayAttr dimensions)
void printTypedDynamicIndexList(mlir::OpAsmPrinter &printer, mlir::Operation *op, mlir::OperandRange values, llvm::ArrayRef< int64_t > integers, mlir::TypeRange types)
llvm::SmallVector< ObjectFifoDmaEndpointOp > turnEndpoints(ObjectFifoDmaEndpointOp end)
llvm::function_ref< std::optional< TileID >(mlir::Value)> TilePositions
std::optional< llvm::SmallVector< BDDimLayoutAttr > > normalizedBDDims(llvm::ArrayRef< BDDimLayoutAttr > dims, int64_t len)
bool hasCoreAccess(DeviceOp device, mlir::Value tile, ObjectFifoCreateOp fifo, ObjectFifoPort port)
mlir::LogicalResult verifyLengthParameterTile(mlir::Operation *op, const AIETargetModel &targetModel, int col, int row)
uint32_t getShimBurstLengthBytes(const AIE::AIETargetModel &tm, uint32_t burstLength)
void collectTiles(DeviceOp &device, llvm::DenseMap< TileID, mlir::Operation * > &tiles)
int64_t getMemRefSizeInBytes(mlir::MemRefType type, const mlir::DataLayout &layout)
llvm::SmallVector< int, 2 > banksFromMemorySpace(mlir::Type type)
void collectBuffers(DeviceOp &device, llvm::DenseMap< mlir::Operation *, llvm::SmallVector< BufferOp, 4 > > &buffers)
mlir::ParseResult parseTraceEvent(mlir::AsmParser &parser, mlir::Attribute &result)
void printObjectFifoAcquireObjects(mlir::OpAsmPrinter &printer, mlir::Operation *op, ObjectFifoPortAttr port, mlir::TypeRange objects)
mlir::LogicalResult verifyBdLockPair(mlir::Block &block, bool outOfOrder, UseLockOp &acquire, UseLockOp &release)
mlir::LogicalResult verifyAssignedPacket(mlir::Operation *op, PacketInfoAttr packet, llvm::StringRef what="packet")
mlir::ParseResult parseTypedDynamicIndexList(mlir::OpAsmParser &parser, llvm::SmallVectorImpl< mlir::OpAsmParser::UnresolvedOperand > &values, mlir::DenseI64ArrayAttr &integers, llvm::SmallVectorImpl< mlir::Type > &types)
void printObjectFifoConsumerTiles(mlir::OpAsmPrinter &printer, mlir::Operation *op, mlir::OperandRange tiles, BDDimLayoutArrayArrayAttr dimsPerTileAttr)
mlir::Operation * lookupNamedOpIn(mlir::Operation *symbolTableOp, mlir::StringAttr name)
uint32_t getShimBurstLengthEncoding(const AIE::AIETargetModel &tm, uint32_t burstLength)
mlir::LogicalResult verifyDMABDOutOfOrderId(DMABDOp bd, bool packetEnabledByContext=false)
mlir::ParseResult parseObjectFifoAcquireObjects(mlir::OpAsmParser &parser, ObjectFifoPortAttr &port, llvm::SmallVectorImpl< mlir::Type > &objects)
mlir::LogicalResult verifyOutOfOrderChannel(mlir::Operation *op, DMAChannelDir dir, bool outOfOrder, llvm::ArrayRef< DMABDOp > bds, bool packetEnabledByContext=false)
struct Connect { Port src; Port dst; bool operator==(const Connect &rhs) const { return std::tie(src, dst)==std::tie(rhs.src, rhs.dst); } bool operator!=(const Connect &rhs) const { return !(*this==rhs);} bool operator<(const Connect &rhs) const { return std::tie(src, dst)< std::tie(rhs.src, rhs.dst); } } Connect
void printDMAStartChannel(mlir::OpAsmPrinter &printer, mlir::Operation *op, mlir::Attribute channel)
mlir::LogicalResult verifyLengthParameter(mlir::Operation *op, std::optional< int64_t > lengthUnit, mlir::BaseMemRefType buffer, std::optional< int64_t > lenElems, bool contiguous, llvm::ArrayRef< int64_t > innerSizes)
mlir::LogicalResult verifyDMAChannelIndex(mlir::Operation *op, TileLike tile, DMAChannelDir dir, int64_t channel)
struct TileID { friend std::ostream &operator<<(std::ostream &os, const TileID &s) { os<< "TileID("<< s.col<< ", "<< s.row<< ")"; return os; } friend std::string to_string(const TileID &s) { std::ostringstream ss; ss<< s; return ss.str(); } friend llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const TileID &s) { os<< to_string(s); return os; } inline bool operator<(const TileID &rhs) const { return std::tie(col, row)< std::tie(rhs.col, rhs.row); } bool operator==(const TileID &rhs) const { return std::tie(col, row)==std::tie(rhs.col, rhs.row); } bool operator!=(const TileID &rhs) const { return !(*this==rhs);} int col, row;} TileID
int32_t getBufferBaseAddress(mlir::Operation *bufOp)
mlir::ParseResult parseDMAStartChannel(mlir::OpAsmParser &parser, mlir::Attribute &channel)
llvm::SmallVector< uint32_t > getAssignedBdIds(DmaBody program)
llvm::SmallVector< StreamWindowStep > streamWindowSteps(const StreamWindow &w)
int64_t turnPasses(llvm::ArrayRef< std::pair< int64_t, int64_t > > ends)
const AIETargetModel & getTargetModel(mlir::Operation *op)
bool keepsPktHeader(TileID tile, Port master, std::optional< bool > keep)
struct DMAChannel { DMAChannelDir direction; int channel; bool operator==(const DMAChannel &rhs) const { return std::tie(direction, channel)==std::tie(rhs.direction, rhs.channel); } } DMAChannel
mlir::LogicalResult verifyDMAChannelsResolved(DeviceOp device)
mlir::Operation * lookupNamedOp(mlir::Operation *from, mlir::StringAttr name)
mlir::ParseResult parseObjectFifoProducerTile(mlir::OpAsmParser &parser, mlir::OpAsmParser::UnresolvedOperand &operand, BDDimLayoutArrayAttr &dimensions)
void printTraceEventEnum(mlir::AsmPrinter &printer, mlir::Attribute attr)
void placeRuntimeLengthDimension(llvm::MutableArrayRef< int64_t > sizes, llvm::MutableArrayRef< int64_t > strides)
mlir::LogicalResult myVerifyOffsetSizeAndStrideOp(mlir::OffsetSizeAndStrideOpInterface op)
mlir::LogicalResult inferStreamWindow(llvm::ArrayRef< BDDimLayoutAttr > dims, llvm::ArrayRef< BDPadLayoutAttr > pads, int64_t objectSize, llvm::function_ref< mlir::InFlightDiagnostic()> emitError, StreamWindow &window)
mlir::LogicalResult verifyDMABDIteration(DMABDOp bd, AIETileType tileType)
void printFlowVias(mlir::OpAsmPrinter &printer, mlir::Operation *op, mlir::OperandRange vias, mlir::DenseI32ArrayAttr ingressBundles, mlir::DenseI32ArrayAttr ingressChannels, mlir::DenseI32ArrayAttr egressBundles, mlir::DenseI32ArrayAttr egressChannels)
struct Port { WireBundle bundle; int channel; bool operator==(const Port &rhs) const { return std::tie(bundle, channel)==std::tie(rhs.bundle, rhs.channel); } bool operator!=(const Port &rhs) const { return !(*this==rhs);} bool operator<(const Port &rhs) const { return std::tie(bundle, channel)< std::tie(rhs.bundle, rhs.channel); } friend std::ostream &operator<<(std::ostream &os, const Port &port) { os<< "("; os<< stringifyWireBundle(port.bundle).str(); os<< ": "<< std::to_string(port.channel)<< ")"; return os; } friend llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const Port &port) { os<< to_string(port); return os; } } Port
mlir::ParseResult parseObjectFifoReleaseCount(mlir::OpAsmParser &parser, ObjectFifoPortAttr &port, mlir::IntegerAttr &size)
AIETargetModel::SharedMemory sharedMemory(TileOp a, TileOp b)
Which of a's and b's memory modules both tiles can address.
std::optional< std::pair< TileID, Port > > getConnectingWireEndpoint(const AIETargetModel &targetModel, TileID tile, Port port)
mlir::FailureOr< int64_t > getLengthUnitBytes(mlir::Operation *op, int64_t lengthUnit, mlir::BaseMemRefType buffer)
int64_t paddedSize(llvm::ArrayRef< BDDimLayoutAttr > dims, llvm::ArrayRef< BDPadLayoutAttr > pads, int64_t fallback)
mlir::ParseResult parseObjectFifoConsumerTiles(mlir::OpAsmParser &parser, llvm::SmallVectorImpl< mlir::OpAsmParser::UnresolvedOperand > &tiles, BDDimLayoutArrayArrayAttr &dimensions)
void printObjectFifoReleaseCount(mlir::OpAsmPrinter &printer, mlir::Operation *op, ObjectFifoPortAttr port, mlir::IntegerAttr size)
WireBundle getConnectingBundle(WireBundle dir)
static LogicalResult verifyTrait(Operation *op)
static LogicalResult verifyTrait(Operation *op)
static LogicalResult verifyTrait(Operation *op)
static LogicalResult verifyTrait(Operation *op)
static LogicalResult verifyTrait(Operation *op)
The register layout of one tile type's DMA buffer descriptor.
llvm::SmallVector< std::pair< DmaBdField, DmaBdField >, 3 > padFields() const
The (before, after) zero-padding fields, innermost dimension first.
static mlir::LogicalResult verifyTrait(mlir::Operation *op)
static mlir::LogicalResult verifyTrait(mlir::Operation *op)
int64_t depthAt(const AIETargetModel &target, TilePositions positions) const
llvm::SmallVector< LinkPoolSharer > sharers
llvm::SmallVector< BDDimLayoutAttr > inner