Re: [Qemu-devel] [PATCH V10 5/9] hw/rdma: Definitions for rdma device and rdma resource manager

[Yanjun Zhu <yanjun.zhu@oracle.com>]

On 2018/2/13 2:08, Marcel Apfelbaum wrote:
> From: Yuval Shaia <yuval.shaia@oracle.com>
>
> Definition of various structures and constants used in backend and
> resource manager modules.
>
> Reviewed-by: Dotan Barak <dotanb@mellanox.com>
> Signed-off-by: Yuval Shaia <yuval.shaia@oracle.com>
> Signed-off-by: Marcel Apfelbaum <marcel@redhat.com>
Reviewed-by: Zhu Yanjun <yanjun.zhu@oracle.com>

Zhu Yanjun
> ---
>   hw/rdma/rdma_backend_defs.h |  62 ++++++++++++++++++++++++++
>   hw/rdma/rdma_rm_defs.h      | 104 ++++++++++++++++++++++++++++++++++++++++++++
>   2 files changed, 166 insertions(+)
>   create mode 100644 hw/rdma/rdma_backend_defs.h
>   create mode 100644 hw/rdma/rdma_rm_defs.h
>
> diff --git a/hw/rdma/rdma_backend_defs.h b/hw/rdma/rdma_backend_defs.h
> new file mode 100644
> index 0000000000..837e32419c
> --- /dev/null
> +++ b/hw/rdma/rdma_backend_defs.h
> @@ -0,0 +1,62 @@
> +/*
> + *  RDMA device: Definitions of Backend Device structures
> + *
> + * Copyright (C) 2018 Oracle
> + * Copyright (C) 2018 Red Hat Inc
> + *
> + * Authors:
> + *     Yuval Shaia <yuval.shaia@oracle.com>
> + *     Marcel Apfelbaum <marcel@redhat.com>
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2 or later.
> + * See the COPYING file in the top-level directory.
> + *
> + */
> +
> +#ifndef RDMA_BACKEND_DEFS_H
> +#define RDMA_BACKEND_DEFS_H
> +
> +#include <infiniband/verbs.h>
> +#include <qemu/thread.h>
> +
> +typedef struct RdmaDeviceResources RdmaDeviceResources;
> +
> +typedef struct RdmaBackendThread {
> +    QemuThread thread;
> +    QemuMutex mutex;
> +    bool run;
> +} RdmaBackendThread;
> +
> +typedef struct RdmaBackendDev {
> +    struct ibv_device_attr dev_attr;
> +    RdmaBackendThread comp_thread;
> +    union ibv_gid gid;
> +    PCIDevice *dev;
> +    RdmaDeviceResources *rdma_dev_res;
> +    struct ibv_device *ib_dev;
> +    struct ibv_context *context;
> +    struct ibv_comp_channel *channel;
> +    uint8_t port_num;
> +    uint8_t backend_gid_idx;
> +} RdmaBackendDev;
> +
> +typedef struct RdmaBackendPD {
> +    struct ibv_pd *ibpd;
> +} RdmaBackendPD;
> +
> +typedef struct RdmaBackendMR {
> +    struct ibv_pd *ibpd;
> +    struct ibv_mr *ibmr;
> +} RdmaBackendMR;
> +
> +typedef struct RdmaBackendCQ {
> +    RdmaBackendDev *backend_dev;
> +    struct ibv_cq *ibcq;
> +} RdmaBackendCQ;
> +
> +typedef struct RdmaBackendQP {
> +    struct ibv_pd *ibpd;
> +    struct ibv_qp *ibqp;
> +} RdmaBackendQP;
> +
> +#endif
> diff --git a/hw/rdma/rdma_rm_defs.h b/hw/rdma/rdma_rm_defs.h
> new file mode 100644
> index 0000000000..6522dca68f
> --- /dev/null
> +++ b/hw/rdma/rdma_rm_defs.h
> @@ -0,0 +1,104 @@
> +/*
> + * RDMA device: Definitions of Resource Manager structures
> + *
> + * Copyright (C) 2018 Oracle
> + * Copyright (C) 2018 Red Hat Inc
> + *
> + * Authors:
> + *     Yuval Shaia <yuval.shaia@oracle.com>
> + *     Marcel Apfelbaum <marcel@redhat.com>
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2 or later.
> + * See the COPYING file in the top-level directory.
> + *
> + */
> +
> +#ifndef RDMA_RM_DEFS_H
> +#define RDMA_RM_DEFS_H
> +
> +#include "rdma_backend_defs.h"
> +
> +#define MAX_PORTS             1
> +#define MAX_PORT_GIDS         1
> +#define MAX_PORT_PKEYS        1
> +#define MAX_PKEYS             1
> +#define MAX_GIDS              2048
> +#define MAX_UCS               512
> +#define MAX_MR_SIZE           (1UL << 27)
> +#define MAX_QP                1024
> +#define MAX_SGE               4
> +#define MAX_CQ                2048
> +#define MAX_MR                1024
> +#define MAX_PD                1024
> +#define MAX_QP_RD_ATOM        16
> +#define MAX_QP_INIT_RD_ATOM   16
> +#define MAX_AH                64
> +
> +#define MAX_RMRESTBL_NAME_SZ 16
> +typedef struct RdmaRmResTbl {
> +    char name[MAX_RMRESTBL_NAME_SZ];
> +    QemuMutex lock;
> +    unsigned long *bitmap;
> +    size_t tbl_sz;
> +    size_t res_sz;
> +    void *tbl;
> +} RdmaRmResTbl;
> +
> +typedef struct RdmaRmPD {
> +    RdmaBackendPD backend_pd;
> +    uint32_t ctx_handle;
> +} RdmaRmPD;
> +
> +typedef struct RdmaRmCQ {
> +    RdmaBackendCQ backend_cq;
> +    void *opaque;
> +    bool notify;
> +} RdmaRmCQ;
> +
> +typedef struct RdmaRmUserMR {
> +    uint64_t host_virt;
> +    uint64_t guest_start;
> +    size_t length;
> +} RdmaRmUserMR;
> +
> +/* MR (DMA region) */
> +typedef struct RdmaRmMR {
> +    RdmaBackendMR backend_mr;
> +    RdmaRmUserMR user_mr;
> +    uint32_t pd_handle;
> +    uint32_t lkey;
> +    uint32_t rkey;
> +} RdmaRmMR;
> +
> +typedef struct RdmaRmUC {
> +    uint64_t uc_handle;
> +} RdmaRmUC;
> +
> +typedef struct RdmaRmQP {
> +    RdmaBackendQP backend_qp;
> +    void *opaque;
> +    uint32_t qp_type;
> +    uint32_t qpn;
> +    uint32_t send_cq_handle;
> +    uint32_t recv_cq_handle;
> +    enum ibv_qp_state qp_state;
> +} RdmaRmQP;
> +
> +typedef struct RdmaRmPort {
> +    union ibv_gid gid_tbl[MAX_PORT_GIDS];
> +    enum ibv_port_state state;
> +    int *pkey_tbl; /* TODO: Not yet supported */
> +} RdmaRmPort;
> +
> +typedef struct RdmaDeviceResources {
> +    RdmaRmPort ports[MAX_PORTS];
> +    RdmaRmResTbl pd_tbl;
> +    RdmaRmResTbl mr_tbl;
> +    RdmaRmResTbl uc_tbl;
> +    RdmaRmResTbl qp_tbl;
> +    RdmaRmResTbl cq_tbl;
> +    RdmaRmResTbl cqe_ctx_tbl;
> +    GHashTable *qp_hash; /* Keeps mapping between real and emulated */
> +} RdmaDeviceResources;
> +
> +#endif