Migration Guide: QDMA to MDB5 DMA

arrow_back Software Docs

7. Code Migration Reference

This section provides side-by-side code comparisons between QDMA and MDB5 DMA implementations for key operations.

ℹ Note

The code snippets shown below highlight the core functionality to illustrate the similarities and differences between QDMA and MDB5 DMA implementations.

7.1 Application Code

7.1.1 Register Access (dma-ctl)

Both use mmap on /sys/bus/pci/devices/.../resource<BAR>. QDMA falls back to netlink if mmap fails.

QDMA (apps/dma-utils/dmactl_reg.c) MDB5 DMA (client-apps/dma-utils/dma_common_utils.c) 
static int32_t reg_read_mmap(
    struct xnl_dev_info *dev_info,
    unsigned char barno,
    struct xcmd_info *xcmd)
{
    uint32_t *bar;
    char fname[256];
    int rv = 0;
    uint32_t attrs[XNL_ATTR_MAX] = {0};

    get_syspath_bar_mmap(fname,
        dev_info->pci_bus,
        dev_info->pci_dev,
        dev_info->dev_func, barno);

    bar = mmap_bar(fname,
        xcmd->req.reg.reg + 4, PROT_READ);
    if (!bar) {
        rv = xnl_common_msg_send(xcmd, attrs);
        return rv;
    }

    xcmd->req.reg.val =
        le32toh(bar[xcmd->req.reg.reg / 4]);
    munmap(bar, xcmd->req.reg.reg + 4);
    return rv;
}
 
int reg_read_mmap(struct mdb5_dma_reg *reg)
{
    uint32_t offset = reg->offset;
    uint32_t *bar;
    uint8_t bar_num = PCI_BAR0;
    char fname[256];
    char *bdf = reg->pci_bdf;
    int ret = 0;

    get_syspath_bar_mmap(fname,
        PCI_DOMAIN, bdf, bar_num);

    if (!is_file_available(fname)) {
        ret = -ENOENT;
        goto error;
    }

    bar = mmap_bar(fname,
        offset + REGISTER_SIZE, PROT_READ);
    if (!bar) {
        ret = -1;
        goto error;
    }

    reg->data = bar[offset / REGISTER_SIZE];
    munmap(bar, offset + REGISTER_SIZE);
error:
    return ret;
}

7.1.2 Queue/Channel Operations (dma-ctl)

QDMA uses netlink for queue management (add, start, stop, delete queues). In contrast, MDB5 DMA channels are always available and do not require explicit creation or destruction. The dma-ctl utility in MDB5 DMA provides various channel management capabilities including: configuring transfer modes (Simple or Scatter-Gather), setting aperture size for SG mode, displaying channel statistics, and reading/writing registers for debugging purposes.

QDMA (apps/dma-utils/dmactl.c) MDB5 DMA (client-apps/dma-ctl/dma_ctl.c) 
int qdma_q_add(struct xcmd_info *cmd)
{
    uint32_t attrs[XNL_ATTR_MAX] = {0};
    return xnl_common_msg_send(cmd, attrs);
}

int qdma_q_start(struct xcmd_info *cmd)
{
    uint32_t attrs[XNL_ATTR_MAX] = {0};
    return xnl_common_msg_send(cmd, attrs);
}

int qdma_q_stop(struct xcmd_info *cmd)
{
    uint32_t attrs[XNL_ATTR_MAX] = {0};
    return xnl_common_msg_send(cmd, attrs);
}

int qdma_q_del(struct xcmd_info *cmd)
{
    uint32_t attrs[XNL_ATTR_MAX] = {0};
    return xnl_common_msg_send(cmd, attrs);
}
 
static int32_t (*process_fn[CMD_MAX + 1])(void *arg) = {
    mdb5_dma_reg_read,           /* CMD_REG_READ */
    mdb5_dma_reg_write,          /* CMD_REG_WRITE */
    mdb5_dma_stats_read,         /* CMD_STATS_READ */
    mdb5_dma_transfer_mode,      /* CMD_SET_TRANSFER_MODE */
    mdb5_dma_transfer_mode,      /* CMD_GET_TRANSFER_MODE */
    mdb5_dma_aperture_sz,        /* CMD_SET_APERTURE_MODE */
    mdb5_dma_aperture_sz,        /* CMD_GET_APERTURE_MODE */
    NULL
};

static int process_cmd(struct mdb5_dma_common *h)
{
    struct mdb5_dma_ioctl *hioc = &h->hioc;
    struct mdb5_dma_reg *reg = &h->reg;
    struct mdb5_dma_channel *chan = h->chan;
    void *arg;

    switch (h->op) {
    case CMD_REG_READ:
    case CMD_REG_WRITE:
        arg = (void *)reg;
        break;
    case CMD_STATS:
        hioc->h = h;
        memcpy(&hioc->cstats.name, &chan->name,
               MDB5_NODE_CHAN_SZ);
        arg = (void *)hioc;
        break;
    case CMD_SET_TRANSFER_MODE:
    case CMD_GET_TRANSFER_MODE:
        hioc->h = h;
        memcpy(&hioc->cmode.name, &chan->name,
               MDB5_NODE_CHAN_SZ);
        arg = (void *)hioc;
        break;
    case CMD_SET_APERTURE_MODE:
    case CMD_GET_APERTURE_MODE:
        hioc->h = h;
        memcpy(&hioc->caperture.name, &chan->name,
               MDB5_NODE_CHAN_SZ);
        arg = (void *)hioc;
        break;
    default:
        usage(stderr);
    }

    if (process_fn[h->op])
        return process_fn[h->op](arg);

    return -EOPNOTSUPP;
}

7.1.3 Channel/Queue Information Flow

Understanding how channel and queue information is passed through the application is crucial for migration:

QDMA Approach:

MDB5 DMA Approach:

QDMA (apps/dma-xfer/dmaxfer.c) MDB5 DMA (client-apps/dma-xfer/dma_xfer.c) 
/* QDMA: Explicit queue management */

// 1. Add queue via netlink
qdma_prepare_q_add(&xcmd, dir, qid, pf);
qdma_q_add(&xcmd);

// 2. Start queue
qdma_q_start(&xcmd);

// 3. Open device node
sprintf(dev_path, "/dev/qdma%02x%03x-MM-%d",
        bus, dev_func, qid);
fd = open(dev_path, O_RDWR);

// 4. Perform transfers using fd
// ... (transfer operations)

// 5. Cleanup: stop and delete queue
qdma_q_stop(&xcmd);
qdma_q_del(&xcmd);
close(fd);
 
/* MDB5: Direct channel access */

// 1. Open pre-created channel device node
sprintf(dev_path, "/dev/mdb5_%s%02d",
        dir == TO_DEV ? "write" : "read",
        channel_id);
fd = open(dev_path, O_RDWR);

// 2. Optionally configure transfer mode
ctrl_fd = open("/dev/mdb5_ctrl", O_RDWR);
cmode.channel = channel_id;
cmode.mode = MDB5_MODE_SG; // or MDB5_MODE_SIMPLE
ioctl(ctrl_fd, IOCTL_MDB5_SET_TRANSFER_MODE, &cmode);
close(ctrl_fd);

// 3. Perform transfers using fd
// ... (transfer operations)

// 4. Cleanup
close(fd);

7.1.4 Data Transfer (dma-to-device / dma-from-device)

The core data transfer functions in both drivers use standard POSIX I/O syscalls. The channel/queue information is implicitly passed through the file descriptor obtained by opening the appropriate device node (e.g., /dev/qdma... or /dev/mdb5_write00). The offset parameter specifies the target address on the device.

QDMA uses lseek() followed by write(), while MDB5 uses the combined pwrite() call.

H2C Transfer (write_from_buffer)
QDMA (apps/dma-utils/dma_xfer_utils.c) MDB5 DMA (client-apps/dma-utils/dma_common_utils.c) 
ssize_t write_from_buffer(char *fname,
    int fd, char *buffer,
    uint64_t size, uint64_t base)
{
    ssize_t rc;
    uint64_t count = 0;
    char *buf = buffer;
    off_t offset = base;

    do {
        uint64_t bytes = size - count;
        if (bytes > RW_MAX_SIZE)
            bytes = RW_MAX_SIZE;

        if (offset) {
            rc = lseek(fd, offset, SEEK_SET);
            if (rc != offset)
                return -EIO;
        }

        rc = write(fd, buf, bytes);
        if (rc < 0)
            return -EIO;

        count += bytes;
        buf += bytes;
        offset += bytes;
    } while (count < size);

    return count;
}
 
ssize_t write_from_buffer(char *fname,
    int fd, char *buffer,
    size_t size, size_t base)
{
    ssize_t rc;
    size_t count = 0;
    char *buf = buffer;
    off_t offset = (off_t) base;
    int ret;

    do {
        size_t bytes = size - count;
        if (bytes > RW_MAX_SIZE)
            bytes = RW_MAX_SIZE;

        rc = pwrite(fd, buf, bytes, offset);
        if (rc < 0) {
            ret = -errno;
            goto error;
        }

        count += bytes;
        buf += bytes;
        offset += bytes;
    } while (count < size);

    return count;
error:
    return ret;
}
C2H Transfer (read_to_buffer)

Similarly, for reading data from device to host, both use POSIX read syscalls:

QDMA (apps/dma-utils/dma_xfer_utils.c) MDB5 DMA (client-apps/dma-utils/dma_common_utils.c) 
ssize_t read_to_buffer(char *fname,
    int fd, char *buffer,
    uint64_t size, uint64_t base)
{
    ssize_t rc;
    uint64_t count = 0;
    char *buf = buffer;
    off_t offset = base;

    do {
        uint64_t bytes = size - count;
        if (bytes > RW_MAX_SIZE)
            bytes = RW_MAX_SIZE;

        if (offset) {
            rc = lseek(fd, offset, SEEK_SET);
            if (rc != offset)
                return -EIO;
        }

        rc = read(fd, buf, bytes);
        if (rc < 0)
            return -EIO;

        count += bytes;
        buf += bytes;
        offset += bytes;
    } while (count < size);

    return count;
}
 
ssize_t read_to_buffer(char *fname,
    int fd, char *buffer,
    size_t size, size_t base)
{
    ssize_t rc;
    size_t count = 0;
    char *buf = buffer;
    off_t offset = (off_t) base;
    int ret;

    do {
        size_t bytes = size - count;
        if (bytes > RW_MAX_SIZE)
            bytes = RW_MAX_SIZE;

        rc = pread(fd, buf, bytes, offset);
        if (rc < 0) {
            ret = -errno;
            goto error;
        }

        count += bytes;
        buf += bytes;
        offset += bytes;
    } while (count < size);

    return count;
error:
    return ret;
}

7.1.5 dma-xfer Tool

Both dma-xfer implementations share the same configuration file format and overall structure: parsing key=value pairs, validating parameters, setting up channels/queues, and executing transfers. The main difference lies in resource lifecycle management.

QDMA Approach:

MDB5 DMA Approach:

Queue/Channel Setup and Transfer Execution
QDMA (apps/dma-xfer/dmaxfer.c) MDB5 DMA (client-apps/dma-xfer/dma_xfer.c) 
int main(int argc, char *argv[])
{
    struct queue_info *q_info;
    int q_count;

    /* 1. Parse and validate config */
    parse_config_file(cfg_fname);
    qdma_validate_qrange();

    /* 2. Create queues via netlink */
    q_count = qdma_setup_queues(&q_info);
    // Internally calls:
    //   qdma_prepare_q_add() -> qdma_q_add()
    //   qdma_prepare_q_start() -> qdma_q_start()

    /* 3. Register cleanup handler */
    atexit(qdma_env_cleanup);
    // Will call qdma_q_stop() and qdma_q_del()
    // for each queue on exit

    /* 4. Perform transfers */
    qdmautils_xfer(q_info, q_count, io_type);
    // Opens /dev/qdma<BDF>-MM-<qid>
    // Calls write_from_buffer() or read_to_buffer()

    return 0;
}

static int qdma_setup_queues(
    struct queue_info **pq_info)
{
    struct queue_info *q_info;
    struct xcmd_info xcmd;

    q_info = calloc(q_count,
                    sizeof(struct queue_info));

    for (qid = 0; qid < num_q; qid++) {
        /* Prepare queue via netlink */
        qdma_prepare_q_add(&xcmd, dir, qid, pf);
        qdma_q_add(&xcmd);

        qdma_prepare_q_start(&xcmd, dir, qid, pf);
        qdma_q_start(&xcmd);

        /* Store queue info */
        qdma_q_prep_name(q_info, qid, pf);
        q_info->qid = qid;
        q_info->dir = dir;
    }
    return q_count;
}
 
int main(int argc, char *argv[])
{
    struct mdb5_dma_common h;
    struct mdb5_dma_channel chan[max_chan];
    int ch_count = 0;

    /* 1. Parse and validate config */
    parse_config_file(cfg_fname, &h);
    validate_input_params(&h, cfg_fname);

    /* 2. Configure channels (no creation needed) */
    ch_count = mdb5_dma_config_channels(&h, chan);
    // Opens existing /dev/mdb5_write<NN> or
    // /dev/mdb5_read<NN> nodes
    // Sets transfer mode via ioctl

    /* 3. Perform transfers */
    mdb5_dmautils_xfer(&h, ch_count);
    // Calls mdb5_dmautils_write() or
    // mdb5_dmautils_read()
    // Using write_from_buffer() or read_to_buffer()

    /* 4. No cleanup needed - channels persist */
    return 0;
}

static int mdb5_dma_config_channels(
    struct mdb5_dma_common *h,
    struct mdb5_dma_channel *chan)
{
    struct mdb5_dma_ioctl *hioc = &h->hioc;

    /* Open control device */
    prepare_node_name(h->ctrl_name, "ctrl", 0);

    /* Configure write channels */
    for (i = 0; i < num_wr_chan; i++) {
        chan[i].ch_id = i;
        chan[i].dir = MDB5_CHAN_DIR_TO_DEV;

        /* Open existing device node */
        prepare_node_name(chan[i].name, "write", i);

        /* Set transfer mode via ioctl */
        memcpy(&hioc->cmode.name, &chan[i].name,
               MDB5_NODE_CHAN_SZ);
        mdb5_dma_transfer_mode((void *)hioc);
    }

    /* Configure read channels similarly */
    for (i = 0; i < num_rd_chan; i++) {
        /* Same pattern for read channels */
    }
    return ch_count;
}