XOCL (PCIe User Physical Function) Driver Interfaces¶
A GEM style driver for Xilinx PCIe based accelerators
This file defines ioctl command codes and associated structures for interacting with xocl PCI driver for Xilinx FPGA platforms.
Device memory allocation is modeled as buffer objects (bo). For each bo driver tracks the host pointer backed by scatter gather list – which provides backing storage on host – and the corresponding device side allocation of contiguous buffer in one of the memory mapped DDRs/BRAMs, etc.
Exection model is asynchronous where execute commands are submitted using command buffers and POSIX poll is used to wait for finished commands. Commands for a compute unit can only be submitted after an explicit context has been opened by the client.
xocl driver functionality is described in the following table. All the APIs are multi-threading and multi-process safe.
# | Functionality | ioctl request code | data format |
---|---|---|---|
1 | Allocate buffer on device | DRM_IOCTL_XOCL_CREATE_BO | drm_xocl_create_bo |
2 | Allocate buffer on device with userptr | DRM_IOCTL_XOCL_USERPTR_BO | drm_xocl_userptr_bo |
3 | Prepare bo for mapping into user’s address space | DRM_IOCTL_XOCL_MAP_BO | drm_xocl_map_bo |
4 | Synchronize (DMA) buffer contents in requested direction | DRM_IOCTL_XOCL_SYNC_BO | drm_xocl_sync_bo |
5 | Obtain information about buffer object | DRM_IOCTL_XOCL_INFO_BO | drm_xocl_info_bo |
6 | Update bo backing storage with user’s data | DRM_IOCTL_XOCL_PWRITE_BO | drm_xocl_pwrite_bo |
7 | Read back data in bo backing storage | DRM_IOCTL_XOCL_PREAD_BO | drm_xocl_pread_bo |
8 | Open/close a context on a compute unit on the device | DRM_XOCL_CTX | drm_xocl_ctx |
9 | Unprotected write to device memory | DRM_IOCTL_XOCL_PWRITE_UNMGD | drm_xocl_pwrite_unmgd |
10 | Unprotected read from device memory | DRM_IOCTL_XOCL_PREAD_UNMGD | drm_xocl_pread_unmgd |
11 | Send an execute job to a compute unit | DRM_IOCTL_XOCL_EXECBUF | drm_xocl_execbuf |
12 | Register eventfd handle for MSIX interrupt | DRM_IOCTL_XOCL_USER_INTR | drm_xocl_user_intr |
13 | Update device view with a specific xclbin image | DRM_XOCL_READ_AXLF | drm_xocl_axlf |
14 | Write buffer from device to peer FPGA buffer | DRM_IOCTL_XOCL_COPY_BO | drm_xocl_copy_bo |
-
struct
drm_xocl_create_bo
¶ Create buffer object used with DRM_IOCTL_XOCL_CREATE_BO ioctl
Definition
struct drm_xocl_create_bo {
uint64_t size;
uint32_t handle;
uint32_t flags;
uint32_t type;
};
Members
size
- Requested size of the buffer object
handle
- bo handle returned by the driver
flags
- DRM_XOCL_BO_XXX flags
type
- The type of bo
-
struct
drm_xocl_userptr_bo
¶ Create buffer object with user’s pointer used with DRM_IOCTL_XOCL_USERPTR_BO ioctl
Definition
struct drm_xocl_userptr_bo {
uint64_t addr;
uint64_t size;
uint32_t handle;
uint32_t flags;
uint32_t type;
};
Members
addr
- Address of buffer allocated by user
size
- Requested size of the buffer object
handle
- bo handle returned by the driver
flags
- DRM_XOCL_BO_XXX flags
type
- The type of bo
-
struct
drm_xocl_map_bo
¶ Prepare a buffer object for mmap used with DRM_IOCTL_XOCL_MAP_BO ioctl
Definition
struct drm_xocl_map_bo {
uint32_t handle;
uint32_t pad;
uint64_t offset;
};
Members
handle
- bo handle
pad
- Unused
offset
- ‘Fake’ offset returned by the driver which can be used with POSIX mmap
-
struct
drm_xocl_sync_bo
¶ Synchronize the buffer in the requested direction between device and host used with DRM_IOCTL_XOCL_SYNC_BO ioctl
Definition
struct drm_xocl_sync_bo {
uint32_t handle;
uint32_t flags;
uint64_t size;
uint64_t offset;
enum drm_xocl_sync_bo_dir dir;
};
Members
handle
- bo handle
flags
- Unused
size
- Number of bytes to synchronize
offset
- Offset into the object to synchronize
dir
- DRM_XOCL_SYNC_DIR_XXX
-
struct
drm_xocl_info_bo
¶ Obtain information about an allocated buffer obbject used with DRM_IOCTL_XOCL_INFO_BO IOCTL
Definition
struct drm_xocl_info_bo {
uint32_t handle;
uint32_t flags;
uint64_t size;
uint64_t paddr;
};
Members
handle
- bo handle
flags
- Unused
size
- Size of buffer object (out)
paddr
- Physical address (out)
-
struct
drm_xocl_copy_bo
¶ copy source buffer to destination buffer between device and device used with DRM_IOCTL_XOCL_COPY_BO ioctl
Definition
struct drm_xocl_copy_bo {
uint32_t dst_handle;
uint32_t src_handle;
uint32_t flags;
uint64_t size;
uint64_t dst_offset;
uint64_t src_offset;
};
Members
dst_handle
- destination bo handle
src_handle
- source bo handle
flags
- Unused
size
- Number of bytes to synchronize
dst_offset
- Offset into the object to destination buffer to synchronize
src_offset
- Offset into the object to source buffer to synchronize
-
struct
drm_xocl_axlf
¶ load xclbin (AXLF) device image used with DRM_IOCTL_XOCL_READ_AXLF ioctl
Definition
struct drm_xocl_axlf {
struct axlf * xclbin;
};
Members
xclbin
- Pointer to user’s xclbin structure in memory
NOTE
This ioctl will be removed in next release
-
struct
drm_xocl_pwrite_bo
¶ Update bo with user’s data used with DRM_IOCTL_XOCL_PWRITE_BO ioctl
Definition
struct drm_xocl_pwrite_bo {
uint32_t handle;
uint32_t pad;
uint64_t offset;
uint64_t size;
uint64_t data_ptr;
};
Members
handle
- bo handle
pad
- Unused
offset
- Offset into the buffer object to write to
size
- Length of data to write
data_ptr
- User’s pointer to read the data from
-
struct
drm_xocl_pread_bo
¶ Read data from bo used with DRM_IOCTL_XOCL_PREAD_BO ioctl
Definition
struct drm_xocl_pread_bo {
uint32_t handle;
uint32_t pad;
uint64_t offset;
uint64_t size;
uint64_t data_ptr;
};
Members
handle
- bo handle
pad
- Unused
offset
- Offset into the buffer object to read from
size
- Length of data to read
data_ptr
- User’s pointer to write the data into
-
struct
drm_xocl_ctx
¶ Open or close a context on a compute unit on device used with DRM_XOCL_CTX ioctl
Definition
struct drm_xocl_ctx {
enum drm_xocl_ctx_code op;
xuid_t xclbin_id;
uint32_t cu_index;
uint32_t flags;
uint32_t handle;
};
Members
op
- Alloc or free a context (XOCL_CTX_OP_ALLOC_CTX/XOCL_CTX_OP_FREE_CTX)
xclbin_id
- UUID of the device image (xclbin)
cu_index
- Index of the compute unit in the device inage for which the request is being made
flags
- Shared or exclusive context (XOCL_CTX_SHARED/XOCL_CTX_EXCLUSIVE)
handle
- Unused
-
struct
drm_xocl_pwrite_unmgd
¶ unprotected write to device memory used with DRM_IOCTL_XOCL_PWRITE_UNMGD ioctl
Definition
struct drm_xocl_pwrite_unmgd {
uint32_t address_space;
uint32_t pad;
uint64_t paddr;
uint64_t size;
uint64_t data_ptr;
};
Members
address_space
- Address space in the DSA; currently only 0 is suported
pad
- Unused
paddr
- Physical address in the specified address space
size
- Length of data to write
data_ptr
- User’s pointer to read the data from
-
struct
drm_xocl_pread_unmgd
¶ unprotected read from device memory used with DRM_IOCTL_XOCL_PREAD_UNMGD ioctl
Definition
struct drm_xocl_pread_unmgd {
uint32_t address_space;
uint32_t pad;
uint64_t paddr;
uint64_t size;
uint64_t data_ptr;
};
Members
address_space
- Address space in the DSA; currently only 0 is valid
pad
- Unused
paddr
- Physical address in the specified address space
size
- Length of data to write
data_ptr
- User’s pointer to write the data to
-
struct
drm_xocl_usage_stat
¶ obtain device memory usage and DMA statistics used with DRM_IOCTL_XOCL_USAGE_STAT ioctl
Definition
struct drm_xocl_usage_stat {
unsigned dma_channel_count;
unsigned mm_channel_count;
uint64_t h2c;
uint64_t c2h;
struct drm_xocl_mm_stat mm;
};
Members
dma_channel_count
- How many DMA channels are present
mm_channel_count
- How many storage banks (DDR) are present
h2c
- Total data transferred from host to device by a DMA channel
c2h
- Total data transferred from device to host by a DMA channel
mm
- BO statistics for a storage bank (DDR)
-
struct
drm_xocl_execbuf
¶ Submit a command buffer for execution on a compute unit used with DRM_IOCTL_XOCL_EXECBUF ioctl
Definition
struct drm_xocl_execbuf {
uint32_t ctx_id;
uint32_t exec_bo_handle;
uint32_t deps;
};
Members
ctx_id
- Pass 0
exec_bo_handle
- BO handle of command buffer formatted as ERT command
deps
- Upto 8 dependency command BO handles this command is dependent on for automatic event dependency handling by ERT
-
struct
drm_xocl_user_intr
¶ Register user’s eventfd for MSIX interrupt used with DRM_IOCTL_XOCL_USER_INTR ioctl
Definition
struct drm_xocl_user_intr {
uint32_t ctx_id;
int fd;
int msix;
};
Members
ctx_id
- Pass 0
fd
- File descriptor created with eventfd system call
msix
- User interrupt number (0 to 15)
-
struct
drm_xocl_p2p_enable
¶ enable/disable P2P
Definition
struct drm_xocl_p2p_enable {
int enable;
};
Members
enable
- 0: disable, 1: enable