290 AIE::TileOp &tile, OpFoldResult bdId,
291 OpFoldResult lengthField) {
293 auto buf = bd_op.getBuffer();
294 auto col = tile.getCol();
295 auto row = tile.getRow();
297 auto constBdId = bdId ? getConstantIntValue(bdId) : std::nullopt;
298 uint32_t bd_id = constBdId ? *constBdId : bd_op.getBdId().value_or(0);
299 uint64_t register_addr = target_model.
getDmaBdAddress(col, row, bd_id) +
301 std::optional<uint64_t> staticByteAddress;
306 Value runtimeRegisterAddr;
307 if (bdId && !getConstantIntValue(bdId)) {
310 runtimeRegisterAddr = arith::AddIOp::create(
311 builder, bd_op.getLoc(), base,
320 if (bd_op.getOffsetStateTableIdxAttr() && runtimeRegisterAddr)
321 return bd_op->emitOpError(
322 "offset_state_table_idx is not supported with a runtime bd_id; the "
323 "scratchpad update targets a compile-time BD register address.");
329 mlir::BlockArgument buf_arg =
nullptr;
332 if (
auto directArg = llvm::dyn_cast<mlir::BlockArgument>(buf)) {
336 buf_arg = traceResult->rootArg;
337 offset = traceResult->offsetInBytes;
341 if (!target_model.
isShimNOCTile(tile.getCol(), tile.getRow())) {
342 return bd_op->emitOpError(
"DDR memory (runtime input arguments) can "
343 "only be referred to on shim tiles.");
348 return bd_op->emitOpError(
349 "buffer must resolve to a memref argument of the runtime sequence");
350 unsigned arg_idx = *hostIdx;
355 OpFoldResult offsetOfr =
356 bd_op.getOffset() ? OpFoldResult(bd_op.getOffset())
357 : OpFoldResult(builder.getI32IntegerAttr(
358 bd_op.getConstantOffset().value_or(0)));
359 OpFoldResult oneStride = builder.getI32IntegerAttr(1);
362 bd_op.getBufferElementTypeWidthInBytes(), offset,
366 NpuAddressPatchOp::create(builder, bd_op.getLoc(),
370 }
else if (AIE::BufferOp buffer =
371 llvm::dyn_cast<AIE::BufferOp>(buf.getDefiningOp())) {
372 std::optional<uint32_t> bufferAddr = buffer.getAddress();
373 if (!bufferAddr.has_value()) {
374 return bd_op->emitOpError(
375 "Cannot lower buffer without associated address. Run pass "
376 "--aie-assign-buffer-addresses first or manually assign an "
379 uint64_t buf_addr = *bufferAddr;
385 col, row, buffer.getTileOp().getCol(), buffer.getTileOp().getRow());
387 buf_addr += addrOffset.value();
392 return bd_op->emitOpError(
393 "has no buffer descriptor layout on this tile");
396 uint64_t maxByteAddr = (uint64_t(addrBits.
mask()) + 1) * divisor - 1;
397 if (buf_addr > maxByteAddr)
398 return bd_op->emitOpError(
"buffer address exceeds the tile's DMA "
403 bool constOffset = !bd_op.getOffset() || bd_op.getConstantOffset();
404 Location loc = bd_op.getLoc();
407 buf_addr += bd_op.getOffsetInBytes();
408 if (buf_addr > maxByteAddr)
409 return bd_op->emitOpError(
"buffer address exceeds the tile's DMA "
411 staticByteAddress = buf_addr;
417 {OpFoldResult(builder.getI32IntegerAttr(1))},
418 bd_op.getBufferElementTypeWidthInBytes(),
419 (int64_t)buf_addr, divisor);
422 Type i64 = builder.getI64Type();
423 Value fits = builder.createOrFold<arith::CmpIOp>(
424 loc, arith::CmpIPredicate::ule, byteAddr,
425 arith::ConstantOp::create(builder, loc,
426 IntegerAttr::get(i64, maxByteAddr)));
428 "a runtime DMA offset puts the buffer "
429 "address past the tile's DMA address "
433 byteAddr = arith::DivUIOp::create(
434 builder, loc, byteAddr,
435 arith::ConstantOp::create(builder, loc,
436 IntegerAttr::get(i64, divisor)));
437 addrField = arith::TruncIOp::create(builder, loc, builder.getI32Type(),
439 if (addrBits.
shift != 0)
440 addrField = arith::ShLIOp::create(
441 builder, loc, addrField,
448 ? runtimeRegisterAddr
450 static_cast<uint32_t
>(register_addr));
451 uint32_t addrMask = addrBits.
place(0xFFFFFFFF);
452 if (addrMask == 0xFFFFFFFF)
453 NpuWrite32Op::create(builder, loc, regAddrVal, addrField,
nullptr,
456 NpuMaskWrite32Op::create(builder, loc, regAddrVal, addrField,
458 nullptr,
nullptr,
nullptr);
460 return bd_op->emitOpError(
461 "Buffer argument must be a constant aie.buffer, a runtime sequence "
462 "input argument, or a supported chain of memref.subview, "
463 "memref.view, memref.cast, or memref.reinterpret_cast operations "
464 "rooted at a block argument. Subviews must be static and contiguous; "
465 "views must have a constant byte shift and no dynamic result sizes.");
474 if (bd_op.getOffsetStateTableIdxAttr()) {
475 auto bufType = llvm::cast<BaseMemRefType>(bd_op.getBuffer().getType());
477 builder, bd_op, bufType, register_addr, target_model, col, row,
478 staticByteAddress, lengthField)))
491 std::optional<xilinx::AIE::PacketInfoAttr> packet,
492 bool outOfOrder =
false) {
497 if (std::optional<uint32_t> nextBdId = bd_op.getNextBdId()) {
506 auto info = bd_op.getPacket().value_or(packet.value_or(
nullptr));
508 if (!info.isAssigned()) {
509 bd_op.emitOpError(
"packet has no pkt_id; allocation assigns one, so a "
510 "header at this level must carry it");
518 if (std::optional<int32_t> oooId = bd_op.getOutOfOrderId())
521 if (std::optional<AIE::BDIterationAttr> iter = bd_op.getIteration())
526 auto lockValue = [&](AIE::UseLockOp op, int32_t &constOut,
527 Value &valueOut) -> LogicalResult {
528 if (std::optional<int64_t> c = getConstantIntValue(op.getValue())) {
529 constOut = op.acquireGE() ? -
static_cast<int32_t
>(*c)
530 :
static_cast<int32_t
>(*c);
533 Location loc = op.getLoc();
534 int32_t minValue = op.acquireGE() ? 1 : 0;
536 Value fromMin = builder.createOrFold<arith::SubIOp>(
538 Value inRange = builder.createOrFold<arith::CmpIOp>(
539 loc, arith::CmpIPredicate::ule, fromMin,
542 "a runtime lock value must be in [" +
543 Twine(minValue) +
":" + Twine(maxValue) +
546 valueOut = op.getValue();
548 valueOut = arith::SubIOp::create(
553 AIE::UseLockOp acquire_op, release_op;
559 auto lockIdFor = [&](AIE::LockOp lock, int32_t id) {
560 if (target_model.
isMemTile(tile.getCol(), tile.getRow()))
562 tile.getCol(), tile.getRow(), lock.colIndex(), lock.rowIndex()))
567 AIE::LockOp rel_lock = release_op.getLockOp();
571 AIE::LockOp acq_lock = acquire_op.getLockOp();
572 if (std::optional<int32_t> acqLockId = acq_lock.getLockID()) {
580 if (std::optional<int32_t> relLockId = rel_lock.getLockID()) {
599 std::optional<xilinx::AIE::PacketInfoAttr> packet,
600 Value runtimeBdId =
nullptr,
bool outOfOrder =
false) {
602 Location loc = bd_op.getLoc();
603 int col = tile.getCol();
604 int row = tile.getRow();
607 target_model, packet, outOfOrder);
608 if (failed(fieldsOr))
618 OpFoldResult bdIdOfr;
620 bdIdOfr = runtimeBdId;
622 std::optional<uint32_t> staticBdId = bd_op.getBdId();
623 assert(staticBdId &&
"static bd_id must be assigned (checked by "
624 "verifyBdInBlock before lowering)");
625 bdIdOfr = builder.getI32IntegerAttr(*staticBdId);
634 SmallVector<OpFoldResult> sizes(bd_op.getMixedSizes());
635 SmallVector<OpFoldResult> strides(bd_op.getMixedStrides());
636 if (sizes.size() > 4 || strides.size() > 4)
637 return bd_op->emitOpError(
638 "At most four data layout transformation dimensions may be "
639 "provided; aie-decompose-large-dma-bd splits a longer one.");
640 OpFoldResult one = builder.getI64IntegerAttr(1);
641 OpFoldResult zeroOfr = builder.getI64IntegerAttr(0);
642 SmallVector<OpFoldResult, 4> sizes4(4, one), strides4(4, zeroOfr);
644 strides4.back() = one;
645 for (
size_t i = 0; i < sizes.size(); i++) {
646 sizes4[4 - sizes.size() + i] = sizes[i];
647 strides4[4 - strides.size() + i] = strides[i];
651 if (std::optional<AIE::BDIterationAttr> iter = bd_op.getIteration()) {
652 sizes4.front() = builder.getI64IntegerAttr(iter->getSize());
653 strides4.front() = builder.getI64IntegerAttr(iter->getStride());
661 if (Value lenOperand = bd_op.getLen())
663 else if (std::optional<int32_t> constLen = bd_op.getConstantLen())
664 lenOfr = builder.getI32IntegerAttr(*constLen);
665 else if (bd_op.getSizes().empty() && bd_op.getStrides().empty()) {
666 auto bufTy = cast<BaseMemRefType>(bd_op.getBuffer().getType());
667 if (!bufTy.hasStaticShape() ||
668 (bd_op.getOffset() && !bd_op.getConstantOffset()))
669 return bd_op->emitOpError(
670 "runtime-valued BD requires an explicit transfer length; provide a "
671 "`len` for this buffer descriptor.");
672 lenOfr = builder.getI64IntegerAttr(bufTy.getNumElements());
677 std::optional<int64_t> constLen =
678 lenOfr ? getConstantIntValue(lenOfr) : std::nullopt;
679 std::optional<int64_t> dimsLen = 1;
680 for (OpFoldResult s : llvm::drop_begin(sizes4)) {
681 std::optional<int64_t> c = getConstantIntValue(s);
689 static_cast<uint64_t
>(bd_op.getBufferElementTypeWidthInBytes()) * 8;
690 if (!sizes.empty() && constLen && dimsLen && *dimsLen != *constLen) {
691 auto err = bd_op->emitOpError(
692 "Buffer descriptor length does not match length of "
693 "transfer expressed by lowest three dimensions of data "
694 "layout transformation strides/wraps. ")
695 <<
"BD length is " << (*constLen * elemWidth / 8)
696 <<
" bytes. Lowest three dimensions of data layout "
697 "transformation would result in transfer of "
698 << (*dimsLen * elemWidth / 8) <<
" bytes. ";
699 err.attachNote() <<
"Do not include the highest dimension size in "
700 "transfer length, as this is the BD repeat count.";
711 (lenOfr && isConstantIntValue(strides4[3], 1) &&
712 llvm::all_of(ArrayRef(sizes4).drop_back(),
713 [](OpFoldResult v) {
return isConstantIntValue(v, 1); }));
722 SmallVector<Value> bdWords;
725 builder, loc, target_model, col, row, f, sizes4, strides4,
726 elemWidth, bd_op.getBurstLength(), bd_op.getAxcacheOrDefault(),
727 linear ? OpFoldResult() : lenOfr, bdRepeatCount, bdWords)))
732 OpFoldResult offsetOfr =
733 bd_op.getOffset() ? OpFoldResult(bd_op.getOffset())
734 : OpFoldResult(builder.getI32IntegerAttr(
735 bd_op.getConstantOffset().value_or(0)));
737 builder, loc, cast<BaseMemRefType>(trace->rootArg.getType()),
738 trace->offsetInBytes, elemWidth / 8, {offsetOfr}, {one}, sizes4,
746 NpuBlockWriteValuesOp::create(builder, loc, bdBase, bdWords);
749 builder, bd_op, tile, bdIdOfr,
756 AIE::DMAChannelDir channelDir,
757 std::optional<xilinx::AIE::PacketInfoAttr> packet,
758 bool outOfOrder =
false) {
760 Value runtimeBdId = bd_op.getBdIdVal();
762 auto buffer_type = llvm::cast<BaseMemRefType>(bd_op.getBuffer().getType());
763 uint32_t addr_granularity = target_model.getAddressGenGranularity();
773 bool runtimeLen = bd_op.getLen() && !bd_op.getConstantLen();
774 bool runtimeOffset = bd_op.getOffset() && !bd_op.getConstantOffset();
776 llvm::any_of(bd_op.getMixedSizes(),
777 [](OpFoldResult s) { return !getConstantIntValue(s); }) ||
778 llvm::any_of(bd_op.getMixedStrides(),
779 [](OpFoldResult s) { return !getConstantIntValue(s); });
782 llvm::any_of(block.getOps<AIE::UseLockOp>(), [](AIE::UseLockOp op) {
783 return !getConstantIntValue(op.getValue());
785 if (runtimeLen || runtimeDims || runtimeOffset || runtimeBdId ||
786 runtimeNextBd || runtimeLock) {
790 if (bd_op.getLengthStateTableIdxAttr())
791 return bd_op->emitOpError(
"length_parameter requires a constant "
792 "offset, bd_id, next_bd and lock value");
793 int col = tile.getCol(), row = tile.getRow();
794 if (!target_model.isShimNOCTile(col, row) &&
795 !target_model.isMemTile(col, row) &&
796 !target_model.isCoreTile(col, row))
797 return bd_op->emitOpError(
798 "runtime-valued BD size/stride/len/bd_id/next_bd/lock value is "
799 "only supported on tiles with a DMA buffer descriptor layout (shim "
801 "core tiles); use compile-time constants here.");
802 if (bd_op.getPadDimensions().has_value())
803 return bd_op->emitOpError(
804 "zero padding is not supported with runtime sizes/strides/len.");
809 static_cast<uint64_t
>(bd_op.getBufferElementTypeWidthInBytes()) * 8;
810 SmallVector<OpFoldResult, 4> sizesRev(
811 llvm::reverse(bd_op.getMixedSizes()));
812 SmallVector<OpFoldResult, 4> stridesRev(
813 llvm::reverse(bd_op.getMixedStrides()));
815 bd_op, sizesRev, stridesRev, elemWidth,
816 target_model.getAddressGenGranularity())))
819 runtimeBdId, outOfOrder);
827 std::optional<uint32_t> bdIdAttr = bd_op.getBdId();
828 assert(bdIdAttr &&
"static bd_id must be assigned (checked by "
829 "verifyBdInBlock before lowering)");
830 uint32_t bd_id = *bdIdAttr;
831 int64_t offset = bd_op.getOffsetInBytes();
832 uint64_t len = bd_op.getLenInBytes();
833 uint64_t len_addr_granularity = len * 8 / addr_granularity;
835 if (offset * 8 % addr_granularity != 0) {
836 return bd_op->emitOpError(
"Offset must be aligned to ")
837 << (addr_granularity / 8) <<
" byte boundary.";
841 if (len < addr_granularity / 8 && !bd_op.getLengthStateTableIdxAttr()) {
842 return bd_op->emitOpError(
"Transfer size of ")
843 << len <<
" bytes falls below minimum hardware transfer unit of "
844 << (addr_granularity / 8) <<
" bytes.";
847 std::optional<llvm::SmallVector<AIE::BDDimLayoutAttr>> dimsStorage =
848 bd_op.getConstantDimensions();
850 return bd_op->emitOpError(
"internal error folding BD dimensions");
853 std::optional<AIE::BDIterationAttr> iter = bd_op.getIteration();
854 if (iter && dimsStorage->empty())
855 dimsStorage->push_back(AIE::BDDimLayoutAttr::get(
856 builder.getContext(), len / bd_op.getBufferElementTypeWidthInBytes(),
858 std::optional<llvm::ArrayRef<AIE::BDDimLayoutAttr>> dims;
859 if (!dimsStorage->empty())
860 dims = llvm::ArrayRef<AIE::BDDimLayoutAttr>(*dimsStorage);
861 llvm::SmallVector<int64_t, 4> sizes = llvm::SmallVector<int64_t, 4>(4, 0);
862 llvm::SmallVector<int64_t, 4> strides = llvm::SmallVector<int64_t, 4>(4, 0);
865 std::optional<llvm::ArrayRef<AIE::BDPadLayoutAttr>> padDims =
866 bd_op.getPadDimensions();
867 llvm::SmallVector<int64_t, 4> padBefore =
868 llvm::SmallVector<int64_t, 4>(4, 0);
869 llvm::SmallVector<int64_t, 4> padAfter =
870 llvm::SmallVector<int64_t, 4>(4, 0);
871 llvm::fill(padBefore, 0);
872 llvm::fill(padAfter, 0);
880 auto iteration_size = 0;
881 auto iteration_stride = 0;
883 if (dims && !dims->empty()) {
884 llvm::SmallVector<int64_t, 4> input_sizes =
885 llvm::SmallVector<int64_t, 4>(4, 1);
886 llvm::SmallVector<int64_t, 4> input_strides =
887 llvm::SmallVector<int64_t, 4>(4, 0);
888 if (dims->size() > 4) {
889 return bd_op->emitOpError(
890 "At most four data layout transformation dimensions may be "
891 "provided; aie-decompose-large-dma-bd splits a longer one.");
894 for (
size_t i = 0; i < dims->size(); i++) {
898 int j = dims->size() - i - 1;
899 input_sizes[i] = (*dims)[j].getSize();
900 input_strides[i] = (*dims)[j].getStride();
903 input_sizes[3] = iter->getSize();
904 input_strides[3] = iter->getStride();
914 !padDims.has_value() &&
916 input_sizes, input_strides);
917 if (!treatAsLinear && bd_op.getLengthStateTableIdxAttr())
920 if (padDims.has_value()) {
922 target_model.getDmaBdLayout(tile.getCol(), tile.getRow());
923 size_t numPadFields = layout ? layout->
padFields().size() : 0;
924 if (numPadFields == 0)
925 return bd_op->emitOpError()
926 <<
"Padding is only supported by memtile dma bds.";
927 if (padDims->size() > dims->size())
928 return bd_op->emitOpError()
929 <<
"Mismatch number of dimensions between padding(s)"
930 <<
" and wrap(s) and stride(s).";
931 for (
auto [d, pad] : llvm::enumerate(llvm::reverse(*padDims)))
932 if (d >= numPadFields &&
933 (pad.getConstPadBefore() || pad.getConstPadAfter()))
934 return bd_op->emitOpError()
936 <<
" zero padding is not supported on this tile.";
937 if (channelDir == AIE::DMAChannelDir::MM2S) {
938 for (
size_t i = 0; i < padDims->size(); i++) {
939 int j = padDims->size() - i - 1;
940 padBefore[i] = (*padDims)[j].getConstPadBefore();
941 padAfter[i] = (*padDims)[j].getConstPadAfter();
943 for (
size_t i = padDims->size(); i < dims->size(); i++) {
947 int64_t elementBytes = bd_op.getBufferElementTypeWidthInBytes();
948 if ((padBefore[0] * elementBytes) % 4 ||
949 (padAfter[0] * elementBytes) % 4)
950 return bd_op->emitOpError()
951 <<
"Inner-most padding count must result in padding in "
953 padBefore[0] = padBefore[0] * elementBytes / 4;
954 padAfter[0] = padAfter[0] * elementBytes / 4;
956 return bd_op->emitOpError()
957 <<
"supports padding only for MM2S direction on MemTiles.";
960 input_strides, sizes, strides);
963 bd_op, [&] {
return bd_op->emitOpError(); }, buffer_type,
964 tile.getCol(), tile.getRow(), input_sizes, input_strides, sizes,
965 strides, treatAsLinear))) {
969 iteration_size = sizes[3];
970 iteration_stride = strides[3];
972 if (!treatAsLinear) {
975 d0stride = strides[0];
979 d1stride = strides[1];
982 d2stride = strides[2];
987 d2size = (target_model.isMemTile(tile.getCol(), tile.getRow()))
991 if (input_sizes[3] > 1 && input_strides[3] == 0) {
997 iteration_stride = 0;
1004 uint64_t len_dims_addr_granularity = 1;
1005 for (
size_t i = 0; i < 3; i++) {
1006 len_dims_addr_granularity *= sizes[i] + padBefore[i] + padAfter[i];
1009 bool runtimeLengthFromZero =
1010 len_addr_granularity == 0 && bd_op.getLengthStateTableIdxAttr();
1011 if (len_dims_addr_granularity != len_addr_granularity &&
1012 !runtimeLengthFromZero) {
1015 "Buffer descriptor length does not match length of transfer "
1016 "expressed by lowest three dimensions of data layout "
1017 "transformation strides/wraps. ")
1018 <<
"BD length is " << (len_addr_granularity * addr_granularity / 8)
1020 <<
"Lowest three dimensions of data layout transformation would "
1021 "result in transfer of "
1022 << (len_dims_addr_granularity * addr_granularity / 8) <<
" bytes. ";
1023 err.attachNote() <<
"Do not include the highest dimension size in "
1024 "transfer length, as this is the BD repeat count.";
1028 if (padDims && target_model.isMemTile(tile.getCol(), tile.getRow()) &&
1029 channelDir == AIE::DMAChannelDir::MM2S) {
1030 return bd_op->emitOpError()
1031 <<
"Padding requires n-d data layouts expressed as "
1032 <<
"wrap(s) and stride(s).";
1035 return bd_op->emitOpError()
1036 <<
" Padding is supported only on MemTiles.";
1040 target_model, packet, outOfOrder);
1041 if (failed(fieldsOr))
1046 NpuWriteBdOp::create(
1047 builder, bd_op.getLoc(), tile.getCol(), bd_id, len_addr_granularity,
1067 padBefore[1], padBefore[2],
1068 padAfter[0], padAfter[1],
1070 bd_op.getBurstLength(),
1072 target_model.isShimNOCTile(tile.getCol(), tile.getRow())
1073 ? builder.getI32IntegerAttr(bd_op.getAxcacheOrDefault())
1076 builder, bd_op, tile, {},
1077 builder.getI64IntegerAttr(len_addr_granularity))))
1083 auto lengthUnit = bd_op.getLengthUnit();
1084 if (bd_op.getLengthStateTableIdxAttr() && lengthUnit)
1086 builder, bd_op, buffer_type, *lengthUnit, target_model, tile.getCol(),
1087 tile.getRow(), bd_id);