* [PATCH V8 libibverbs 1/7] Infrastructure to support verbs extensions
[not found] ` <1374741488-30895-1-git-send-email-yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
@ 2013-07-25 8:38 ` Yishai Hadas
[not found] ` <1374741488-30895-2-git-send-email-yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2013-07-25 8:38 ` [PATCH V8 libibverbs 2/7] Introduce XRC domains Yishai Hadas
` (5 subsequent siblings)
6 siblings, 1 reply; 15+ messages in thread
From: Yishai Hadas @ 2013-07-25 8:38 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA, roland-BHEL68pLQRGGvPXPguhicg
Cc: ogerlitz-VPRAkNaXOzVWk0Htik3J/w, tzahio-VPRAkNaXOzVWk0Htik3J/w,
yishaih-VPRAkNaXOzVWk0Htik3J/w, Sean Hefty
Infrastructure to support extended verbs capabilities in a forward/backward
manner.
Support for extensions is determeind by the provider calling
verbs_register_driver in place of ibv_register_driver. When
extensions are enabled, ibverbs sets the current alloc_context /
free_context device operations to NULL. These are used to
indicate that the struct ibv_device may be cast to struct
verbs_device.
With extensions, ibverbs allocates the ibv_context structure
and calls into the provider to initialize it. The init call
is part of the verbs_device struct.
The abi_compat field of struct ibv_context is used to determine
support of verbs extensions. As a result, support for ABI version <
2 is removed (corresponds to kernel releases 2.6.11-2.6.14 no
longer being supported). The lowest ABI now supported is 3 (really
4 since 2.6.15 was ABI 4, I don't see that ABI 3 was in a release).
Signed-off-by: Yishai Hadas <yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
Signed-off-by: Tzahi Oved <tzahio-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Change from v7:
Added an helper macro for v.ext, named verbs_set_ctx_op, being used in
farther patches.
Fixed "container_of" to match c99 standard.
include/infiniband/driver.h | 3 +
include/infiniband/kern-abi.h | 43 +--------------
include/infiniband/verbs.h | 60 ++++++++++++++++++++
src/cmd.c | 125 +----------------------------------------
src/device.c | 53 +++++++++++++----
src/init.c | 41 +++++++++++--
src/kern_abi.h | 101 ---------------------------------
src/libibverbs.map | 1 +
8 files changed, 142 insertions(+), 285 deletions(-)
delete mode 100644 src/kern_abi.h
diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index 9a81416..f22f287 100644
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -55,8 +55,11 @@
typedef struct ibv_device *(*ibv_driver_init_func)(const char *uverbs_sys_path,
int abi_version);
+typedef struct verbs_device *(*verbs_driver_init_func)(const char *uverbs_sys_path,
+ int abi_version);
void ibv_register_driver(const char *name, ibv_driver_init_func init_func);
+void verbs_register_driver(const char *name, verbs_driver_init_func init_func);
int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd,
size_t cmd_size, struct ibv_get_context_resp *resp,
size_t resp_size);
diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index 619ea7e..e7f8981 100644
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -45,7 +45,7 @@
/*
* The minimum and maximum kernel ABI that we can handle.
*/
-#define IB_USER_VERBS_MIN_ABI_VERSION 1
+#define IB_USER_VERBS_MIN_ABI_VERSION 3
#define IB_USER_VERBS_MAX_ABI_VERSION 6
enum {
@@ -806,47 +806,6 @@ enum {
IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL_V2 = -1,
};
-struct ibv_destroy_cq_v1 {
- __u32 command;
- __u16 in_words;
- __u16 out_words;
- __u32 cq_handle;
-};
-
-struct ibv_destroy_qp_v1 {
- __u32 command;
- __u16 in_words;
- __u16 out_words;
- __u32 qp_handle;
-};
-
-struct ibv_destroy_srq_v1 {
- __u32 command;
- __u16 in_words;
- __u16 out_words;
- __u32 srq_handle;
-};
-
-struct ibv_get_context_v2 {
- __u32 command;
- __u16 in_words;
- __u16 out_words;
- __u64 response;
- __u64 cq_fd_tab;
- __u64 driver_data[0];
-};
-
-struct ibv_create_cq_v2 {
- __u32 command;
- __u16 in_words;
- __u16 out_words;
- __u64 response;
- __u64 user_handle;
- __u32 cqe;
- __u32 event_handler;
- __u64 driver_data[0];
-};
-
struct ibv_modify_srq_v3 {
__u32 command;
__u16 in_words;
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 4b1ab57..ca300af 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -38,6 +38,7 @@
#include <stdint.h>
#include <pthread.h>
+#include <stddef.h>
#ifdef __cplusplus
# define BEGIN_C_DECLS extern "C" {
@@ -63,6 +64,20 @@ union ibv_gid {
} global;
};
+#ifndef container_of
+/**
+ * container_of - cast a member of a structure out to the containing structure
+ * @ptr: the pointer to the member.
+ * @type: the type of the container struct this is embedded in.
+ * @member: the name of the member within the struct.
+ *
+ */
+#define container_of(ptr, type, member) \
+ ((type *) ((char *)(ptr) - offsetof(type, member)))
+#endif
+
+#define vext_field_avail(type, fld, sz) (offsetof(type, fld) < (sz))
+
enum ibv_node_type {
IBV_NODE_UNKNOWN = -1,
IBV_NODE_CA = 1,
@@ -656,6 +671,17 @@ struct ibv_device {
char ibdev_path[IBV_SYSFS_PATH_MAX];
};
+struct verbs_device {
+ struct ibv_device device; /* Must be first */
+ size_t sz;
+ size_t size_of_context;
+ int (*init_context)(struct verbs_device *device,
+ struct ibv_context *ctx, int cmd_fd);
+ void (*uninit_context)(struct verbs_device *device,
+ struct ibv_context *ctx);
+ /* future fields added here */
+};
+
struct ibv_context_ops {
int (*query_device)(struct ibv_context *context,
struct ibv_device_attr *device_attr);
@@ -724,6 +750,40 @@ struct ibv_context {
void *abi_compat;
};
+enum verbs_context_mask {
+ VERBS_CONTEXT_RESERVED = 1 << 0
+};
+
+struct verbs_context {
+ /* "grows up" - new fields go here */
+ uint64_t has_comp_mask;
+ size_t sz; /* Must be immediately before struct ibv_context */
+ struct ibv_context context;/* Must be last field in the struct */
+};
+
+static inline struct verbs_context *verbs_get_ctx(
+ const struct ibv_context *ctx)
+{
+ return (ctx->abi_compat != ((uint8_t *)NULL) - 1) ?
+ NULL : container_of(ctx, struct verbs_context, context);
+}
+
+#define verbs_get_ctx_op(ctx, op) ({ \
+ struct verbs_context *vctx = verbs_get_ctx(ctx); \
+ (!vctx || (vctx->sz < sizeof(*vctx) - offsetof(struct verbs_context, op)) || \
+ !vctx->op) ? NULL : vctx; })
+
+#define verbs_set_ctx_op(vctx, op, ptr) ({ \
+ if (vctx && (vctx->sz >= sizeof(*vctx) - offsetof(struct verbs_context, op))) \
+ vctx->op = ptr; })
+
+static inline struct verbs_device *verbs_get_device(
+ const struct ibv_device *dev)
+{
+ return (dev->ops.alloc_context) ?
+ NULL : container_of(dev, struct verbs_device, device);
+}
+
/**
* ibv_get_device_list - Get list of IB devices currently available
* @num_devices: optional. if non-NULL, set to the number of devices
diff --git a/src/cmd.c b/src/cmd.c
index 9789092..86350fd 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -45,52 +45,13 @@
#include "ibverbs.h"
-static int ibv_cmd_get_context_v2(struct ibv_context *context,
- struct ibv_get_context *new_cmd,
- size_t new_cmd_size,
- struct ibv_get_context_resp *resp,
- size_t resp_size)
-{
- struct ibv_abi_compat_v2 *t;
- struct ibv_get_context_v2 *cmd;
- size_t cmd_size;
- uint32_t cq_fd;
-
- t = malloc(sizeof *t);
- if (!t)
- return ENOMEM;
- pthread_mutex_init(&t->in_use, NULL);
-
- cmd_size = sizeof *cmd + new_cmd_size - sizeof *new_cmd;
- cmd = alloca(cmd_size);
- memcpy(cmd->driver_data, new_cmd->driver_data, new_cmd_size - sizeof *new_cmd);
-
- IBV_INIT_CMD_RESP(cmd, cmd_size, GET_CONTEXT, resp, resp_size);
- cmd->cq_fd_tab = (uintptr_t) &cq_fd;
-
- if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) {
- free(t);
- return errno;
- }
-
- (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
-
- context->async_fd = resp->async_fd;
- context->num_comp_vectors = 1;
- t->channel.context = context;
- t->channel.fd = cq_fd;
- t->channel.refcnt = 0;
- context->abi_compat = t;
-
- return 0;
-}
int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd,
size_t cmd_size, struct ibv_get_context_resp *resp,
size_t resp_size)
{
- if (abi_ver <= 2)
- return ibv_cmd_get_context_v2(context, cmd, cmd_size, resp, resp_size);
+ if (abi_ver < IB_USER_VERBS_MIN_ABI_VERSION)
+ return ENOSYS;
IBV_INIT_CMD_RESP(cmd, cmd_size, GET_CONTEXT, resp, resp_size);
@@ -274,45 +235,12 @@ int ibv_cmd_dereg_mr(struct ibv_mr *mr)
return 0;
}
-static int ibv_cmd_create_cq_v2(struct ibv_context *context, int cqe,
- struct ibv_cq *cq,
- struct ibv_create_cq *new_cmd, size_t new_cmd_size,
- struct ibv_create_cq_resp *resp, size_t resp_size)
-{
- struct ibv_create_cq_v2 *cmd;
- size_t cmd_size;
-
- cmd_size = sizeof *cmd + new_cmd_size - sizeof *new_cmd;
- cmd = alloca(cmd_size);
- memcpy(cmd->driver_data, new_cmd->driver_data, new_cmd_size - sizeof *new_cmd);
-
- IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_CQ, resp, resp_size);
- cmd->user_handle = (uintptr_t) cq;
- cmd->cqe = cqe;
- cmd->event_handler = 0;
-
- if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
- return errno;
-
- (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
-
- cq->handle = resp->cq_handle;
- cq->cqe = resp->cqe;
- cq->context = context;
-
- return 0;
-}
-
int ibv_cmd_create_cq(struct ibv_context *context, int cqe,
struct ibv_comp_channel *channel,
int comp_vector, struct ibv_cq *cq,
struct ibv_create_cq *cmd, size_t cmd_size,
struct ibv_create_cq_resp *resp, size_t resp_size)
{
- if (abi_ver <= 2)
- return ibv_cmd_create_cq_v2(context, cqe, cq,
- cmd, cmd_size, resp, resp_size);
-
IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_CQ, resp, resp_size);
cmd->user_handle = (uintptr_t) cq;
cmd->cqe = cqe;
@@ -397,7 +325,6 @@ int ibv_cmd_resize_cq(struct ibv_cq *cq, int cqe,
struct ibv_resize_cq *cmd, size_t cmd_size,
struct ibv_resize_cq_resp *resp, size_t resp_size)
{
-
IBV_INIT_CMD_RESP(cmd, cmd_size, RESIZE_CQ, resp, resp_size);
cmd->cq_handle = cq->handle;
cmd->cqe = cqe;
@@ -412,27 +339,11 @@ int ibv_cmd_resize_cq(struct ibv_cq *cq, int cqe,
return 0;
}
-static int ibv_cmd_destroy_cq_v1(struct ibv_cq *cq)
-{
- struct ibv_destroy_cq_v1 cmd;
-
- IBV_INIT_CMD(&cmd, sizeof cmd, DESTROY_CQ);
- cmd.cq_handle = cq->handle;
-
- if (write(cq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
- return errno;
-
- return 0;
-}
-
int ibv_cmd_destroy_cq(struct ibv_cq *cq)
{
struct ibv_destroy_cq cmd;
struct ibv_destroy_cq_resp resp;
- if (abi_ver == 1)
- return ibv_cmd_destroy_cq_v1(cq);
-
IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_CQ, &resp, sizeof resp);
cmd.cq_handle = cq->handle;
cmd.reserved = 0;
@@ -557,27 +468,11 @@ int ibv_cmd_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr,
return 0;
}
-static int ibv_cmd_destroy_srq_v1(struct ibv_srq *srq)
-{
- struct ibv_destroy_srq_v1 cmd;
-
- IBV_INIT_CMD(&cmd, sizeof cmd, DESTROY_SRQ);
- cmd.srq_handle = srq->handle;
-
- if (write(srq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
- return errno;
-
- return 0;
-}
-
int ibv_cmd_destroy_srq(struct ibv_srq *srq)
{
struct ibv_destroy_srq cmd;
struct ibv_destroy_srq_resp resp;
- if (abi_ver == 1)
- return ibv_cmd_destroy_srq_v1(srq);
-
IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_SRQ, &resp, sizeof resp);
cmd.srq_handle = srq->handle;
cmd.reserved = 0;
@@ -799,19 +694,6 @@ int ibv_cmd_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
return 0;
}
-static int ibv_cmd_destroy_qp_v1(struct ibv_qp *qp)
-{
- struct ibv_destroy_qp_v1 cmd;
-
- IBV_INIT_CMD(&cmd, sizeof cmd, DESTROY_QP);
- cmd.qp_handle = qp->handle;
-
- if (write(qp->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
- return errno;
-
- return 0;
-}
-
int ibv_cmd_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
struct ibv_send_wr **bad_wr)
{
@@ -1074,9 +956,6 @@ int ibv_cmd_destroy_qp(struct ibv_qp *qp)
struct ibv_destroy_qp cmd;
struct ibv_destroy_qp_resp resp;
- if (abi_ver == 1)
- return ibv_cmd_destroy_qp_v1(qp);
-
IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_QP, &resp, sizeof resp);
cmd.qp_handle = qp->handle;
cmd.reserved = 0;
diff --git a/src/device.c b/src/device.c
index 1923fa5..970375e 100644
--- a/src/device.c
+++ b/src/device.c
@@ -124,9 +124,11 @@ default_symver(__ibv_get_device_guid, ibv_get_device_guid);
struct ibv_context *__ibv_open_device(struct ibv_device *device)
{
+ struct verbs_device *verbs_device = verbs_get_device(device);
char *devpath;
- int cmd_fd;
+ int cmd_fd, ret;
struct ibv_context *context;
+ struct verbs_context *context_ex;
if (asprintf(&devpath, "/dev/infiniband/%s", device->dev_name) < 0)
return NULL;
@@ -141,9 +143,33 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device)
if (cmd_fd < 0)
return NULL;
- context = device->ops.alloc_context(device, cmd_fd);
- if (!context)
- goto err;
+ if (!verbs_device) {
+ context = device->ops.alloc_context(device, cmd_fd);
+ if (!context)
+ goto err;
+ } else {
+ /* Library now allocates the context */
+ context_ex = calloc(1, sizeof(*context_ex) +
+ verbs_device->size_of_context);
+ if (!context_ex) {
+ errno = ENOMEM;
+ goto err;
+ }
+
+ context_ex->context.abi_compat = ((uint8_t *)NULL) - 1;
+ context_ex->sz = sizeof(*context_ex);
+
+ context = &context_ex->context;
+ ret = verbs_device->init_context(verbs_device, context, cmd_fd);
+ if (ret)
+ goto verbs_err;
+
+ /* initialize *all* library ops to either lib calls or
+ * directly to provider calls.
+ * context_ex->lib_new_func1 = __verbs_new_func1;
+ * context_ex->lib_new_func2 = __verbs_new_func2;
+ */
+ }
context->device = device;
context->cmd_fd = cmd_fd;
@@ -151,9 +177,10 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device)
return context;
+verbs_err:
+ free(context_ex);
err:
close(cmd_fd);
-
return NULL;
}
default_symver(__ibv_open_device, ibv_open_device);
@@ -163,15 +190,17 @@ int __ibv_close_device(struct ibv_context *context)
int async_fd = context->async_fd;
int cmd_fd = context->cmd_fd;
int cq_fd = -1;
-
- if (abi_ver <= 2) {
- struct ibv_abi_compat_v2 *t = context->abi_compat;
- cq_fd = t->channel.fd;
- free(context->abi_compat);
+ struct verbs_context *context_ex;
+
+ context_ex = verbs_get_ctx(context);
+ if (context_ex) {
+ struct verbs_device *verbs_device = verbs_get_device(context->device);
+ verbs_device->uninit_context(verbs_device, context);
+ free(context_ex);
+ } else {
+ context->device->ops.free_context(context);
}
- context->device->ops.free_context(context);
-
close(async_fd);
close(cmd_fd);
if (abi_ver <= 2)
diff --git a/src/init.c b/src/init.c
index 8e93f3f..9fcb1ee 100644
--- a/src/init.c
+++ b/src/init.c
@@ -70,6 +70,7 @@ struct ibv_driver_name {
struct ibv_driver {
const char *name;
ibv_driver_init_func init_func;
+ verbs_driver_init_func verbs_init_func;
struct ibv_driver *next;
};
@@ -153,7 +154,8 @@ static int find_sysfs_devs(void)
return ret;
}
-void ibv_register_driver(const char *name, ibv_driver_init_func init_func)
+static void register_driver(const char *name, ibv_driver_init_func init_func,
+ verbs_driver_init_func verbs_init_func)
{
struct ibv_driver *driver;
@@ -163,9 +165,10 @@ void ibv_register_driver(const char *name, ibv_driver_init_func init_func)
return;
}
- driver->name = name;
- driver->init_func = init_func;
- driver->next = NULL;
+ driver->name = name;
+ driver->init_func = init_func;
+ driver->verbs_init_func = verbs_init_func;
+ driver->next = NULL;
if (tail_driver)
tail_driver->next = driver;
@@ -174,6 +177,19 @@ void ibv_register_driver(const char *name, ibv_driver_init_func init_func)
tail_driver = driver;
}
+void ibv_register_driver(const char *name, ibv_driver_init_func init_func)
+{
+ register_driver(name, init_func, NULL);
+}
+
+/* New registration symbol with same functionality - used by providers to
+ * validate that library supports verbs extension.
+ */
+void verbs_register_driver(const char *name, verbs_driver_init_func init_func)
+{
+ register_driver(name, NULL, init_func);
+}
+
static void load_driver(const char *name)
{
char *so_name;
@@ -333,12 +349,23 @@ out:
static struct ibv_device *try_driver(struct ibv_driver *driver,
struct ibv_sysfs_dev *sysfs_dev)
{
+ struct verbs_device *vdev;
struct ibv_device *dev;
char value[8];
- dev = driver->init_func(sysfs_dev->sysfs_path, sysfs_dev->abi_ver);
- if (!dev)
- return NULL;
+ if (driver->init_func) {
+ dev = driver->init_func(sysfs_dev->sysfs_path, sysfs_dev->abi_ver);
+ if (!dev)
+ return NULL;
+ } else {
+ vdev = driver->verbs_init_func(sysfs_dev->sysfs_path, sysfs_dev->abi_ver);
+ if (!vdev)
+ return NULL;
+
+ dev = &vdev->device;
+ dev->ops.alloc_context = NULL;
+ dev->ops.free_context = NULL;
+ }
if (ibv_read_sysfs_file(sysfs_dev->ibdev_path, "node_type", value, sizeof value) < 0) {
fprintf(stderr, PFX "Warning: no node_type attr under %s.\n",
diff --git a/src/kern_abi.h b/src/kern_abi.h
deleted file mode 100644
index e055e75..0000000
--- a/src/kern_abi.h
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Copyright (c) 2005 Topspin Communications. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef KERN_ABI_H
-#define KERN_ABI_H
-
-#include <linux/types.h>
-
-/*
- * Increment this value if any changes that break userspace ABI
- * compatibility are made.
- */
-#define IB_USER_VERBS_ABI_VERSION 1
-
-enum {
- IB_USER_VERBS_CMD_GET_CONTEXT,
- IB_USER_VERBS_CMD_GET_EVENT_FDS,
- IB_USER_VERBS_CMD_ALLOC_PD,
- IB_USER_VERBS_CMD_DEALLOC_PD,
- IB_USER_VERBS_CMD_REG_MR,
- IB_USER_VERBS_CMD_DEREG_MR
-};
-
-/*
- * Make sure that all structs defined in this file remain laid out so
- * that they pack the same way on 32-bit and 64-bit architectures (to
- * avoid incompatibility between 32-bit userspace and 64-bit kernels).
- * In particular do not use pointer types -- pass pointers in __u64
- * instead.
- */
-
-struct ibv_kern_async_event {
- __u32 event_type;
- __u32 element;
-};
-
-struct ibv_comp_event {
- __u32 cq_handle;
-};
-
-/*
- * All commands from userspace should start with a __u32 command field
- * followed by __u16 in_words and out_words fields (which give the
- * length of the command block and response buffer if any in 32-bit
- * words). The kernel driver will read these fields first and read
- * the rest of the command struct based on these value.
- */
-
-struct ibv_get_context {
- __u32 command;
- __u16 in_words;
- __u16 out_words;
- __u64 response;
-};
-
-struct ibv_get_context_resp {
- __u32 num_cq_events;
-};
-
-struct ibv_get_event_fds {
- __u32 command;
- __u16 in_words;
- __u16 out_words;
- __u64 response;
-};
-
-struct ibv_get_event_fds_resp {
- __u32 async_fd;
- __u32 cq_fd[1];
-};
-
-#endif /* KERN_ABI_H */
diff --git a/src/libibverbs.map b/src/libibverbs.map
index 7e722f4..6e35c37 100644
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -91,6 +91,7 @@ IBVERBS_1.1 {
ibv_dontfork_range;
ibv_dofork_range;
ibv_register_driver;
+ verbs_register_driver;
ibv_node_type_str;
ibv_port_state_str;
--
1.7.1
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH V8 libibverbs 2/7] Introduce XRC domains
[not found] ` <1374741488-30895-1-git-send-email-yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2013-07-25 8:38 ` [PATCH V8 libibverbs 1/7] Infrastructure to support verbs extensions Yishai Hadas
@ 2013-07-25 8:38 ` Yishai Hadas
2013-07-25 8:38 ` [PATCH V8 libibverbs 3/7] Add support for XRC SRQs Yishai Hadas
` (4 subsequent siblings)
6 siblings, 0 replies; 15+ messages in thread
From: Yishai Hadas @ 2013-07-25 8:38 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA, roland-BHEL68pLQRGGvPXPguhicg
Cc: ogerlitz-VPRAkNaXOzVWk0Htik3J/w, tzahio-VPRAkNaXOzVWk0Htik3J/w,
yishaih-VPRAkNaXOzVWk0Htik3J/w, Sean Hefty
From: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
XRC introduces several new concepts and structures, one of
which is the XRC domain.
XRC domains: xrcd's are a type of protection domain used to
associate shared receive queues with xrc queue pairs. Since
xrcd are meant to be shared among multiple processes, we
introduce new APIs to open/close xrcd's.
The user to kernel ABI is extended to account for opening/
closing the xrcd.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Signed-off-by: Yishai Hadas <yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
---
Changes from V7:
verbs_context - remove place holder for ABI compatibility with OFED.
include/infiniband/driver.h | 17 ++++++++++++++
include/infiniband/kern-abi.h | 27 +++++++++++++++++++++-
include/infiniband/verbs.h | 50 ++++++++++++++++++++++++++++++++++++++--
src/cmd.c | 45 ++++++++++++++++++++++++++++++++++++
src/libibverbs.map | 3 ++
5 files changed, 138 insertions(+), 4 deletions(-)
diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index f22f287..08d8a82 100644
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -53,6 +53,17 @@
*/
#define IBV_DEVICE_LIBRARY_EXTENSION rdmav2
+enum verbs_xrcd_mask {
+ VERBS_XRCD_HANDLE = 1 << 0,
+ VERBS_XRCD_RESERVED = 1 << 1
+};
+
+struct verbs_xrcd {
+ struct ibv_xrcd xrcd;
+ uint32_t comp_mask;
+ uint32_t handle;
+};
+
typedef struct ibv_device *(*ibv_driver_init_func)(const char *uverbs_sys_path,
int abi_version);
typedef struct verbs_device *(*verbs_driver_init_func)(const char *uverbs_sys_path,
@@ -78,6 +89,12 @@ int ibv_cmd_alloc_pd(struct ibv_context *context, struct ibv_pd *pd,
struct ibv_alloc_pd *cmd, size_t cmd_size,
struct ibv_alloc_pd_resp *resp, size_t resp_size);
int ibv_cmd_dealloc_pd(struct ibv_pd *pd);
+int ibv_cmd_open_xrcd(struct ibv_context *context, struct verbs_xrcd *xrcd,
+ int vxrcd_size,
+ struct ibv_xrcd_init_attr *attr,
+ struct ibv_open_xrcd *cmd, size_t cmd_size,
+ struct ibv_open_xrcd_resp *resp, size_t resp_size);
+int ibv_cmd_close_xrcd(struct verbs_xrcd *xrcd);
#define IBV_CMD_REG_MR_HAS_RESP_PARAMS
int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
uint64_t hca_va, int access,
diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index e7f8981..6f40ad2 100644
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -85,7 +85,9 @@ enum {
IB_USER_VERBS_CMD_MODIFY_SRQ,
IB_USER_VERBS_CMD_QUERY_SRQ,
IB_USER_VERBS_CMD_DESTROY_SRQ,
- IB_USER_VERBS_CMD_POST_SRQ_RECV
+ IB_USER_VERBS_CMD_POST_SRQ_RECV,
+ IB_USER_VERBS_CMD_OPEN_XRCD,
+ IB_USER_VERBS_CMD_CLOSE_XRCD,
};
/*
@@ -246,6 +248,27 @@ struct ibv_dealloc_pd {
__u32 pd_handle;
};
+struct ibv_open_xrcd {
+ __u32 command;
+ __u16 in_words;
+ __u16 out_words;
+ __u64 response;
+ __u32 fd;
+ __u32 oflags;
+ __u64 driver_data[0];
+};
+
+struct ibv_open_xrcd_resp {
+ __u32 xrcd_handle;
+};
+
+struct ibv_close_xrcd {
+ __u32 command;
+ __u16 in_words;
+ __u16 out_words;
+ __u32 xrcd_handle;
+};
+
struct ibv_reg_mr {
__u32 command;
__u16 in_words;
@@ -804,6 +827,8 @@ enum {
* trick opcodes in IBV_INIT_CMD() doesn't break.
*/
IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL_V2 = -1,
+ IB_USER_VERBS_CMD_OPEN_XRCD_V2 = -1,
+ IB_USER_VERBS_CMD_CLOSE_XRCD_V2 = -1,
};
struct ibv_modify_srq_v3 {
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index ca300af..7459119 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
- * Copyright (c) 2004 Intel Corporation. All rights reserved.
+ * Copyright (c) 2004, 2011-2012 Intel Corporation. All rights reserved.
* Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2005 PathScale, Inc. All rights reserved.
*
@@ -39,6 +39,7 @@
#include <stdint.h>
#include <pthread.h>
#include <stddef.h>
+#include <errno.h>
#ifdef __cplusplus
# define BEGIN_C_DECLS extern "C" {
@@ -313,6 +314,22 @@ struct ibv_pd {
uint32_t handle;
};
+enum ibv_xrcd_init_attr_mask {
+ IBV_XRCD_INIT_ATTR_FD = 1 << 0,
+ IBV_XRCD_INIT_ATTR_OFLAGS = 1 << 1,
+ IBV_XRCD_INIT_ATTR_RESERVED = 1 << 2
+};
+
+struct ibv_xrcd_init_attr {
+ uint32_t comp_mask;
+ int fd;
+ int oflags;
+};
+
+struct ibv_xrcd {
+ struct ibv_context *context;
+};
+
enum ibv_rereg_mr_flags {
IBV_REREG_MR_CHANGE_TRANSLATION = (1 << 0),
IBV_REREG_MR_CHANGE_PD = (1 << 1),
@@ -751,11 +768,15 @@ struct ibv_context {
};
enum verbs_context_mask {
- VERBS_CONTEXT_RESERVED = 1 << 0
+ VERBS_CONTEXT_XRCD = 1 << 0,
+ VERBS_CONTEXT_RESERVED = 1 << 1
};
struct verbs_context {
/* "grows up" - new fields go here */
+ struct ibv_xrcd * (*open_xrcd)(struct ibv_context *context,
+ struct ibv_xrcd_init_attr *xrcd_init_attr);
+ int (*close_xrcd)(struct ibv_xrcd *xrcd);
uint64_t has_comp_mask;
size_t sz; /* Must be immediately before struct ibv_context */
struct ibv_context context;/* Must be last field in the struct */
@@ -862,7 +883,7 @@ static inline int ___ibv_query_port(struct ibv_context *context,
uint8_t port_num,
struct ibv_port_attr *port_attr)
{
- /* For compatability when running with old libibverbs */
+ /* For compatibility when running with old libibverbs */
port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED;
port_attr->reserved = 0;
@@ -895,6 +916,29 @@ struct ibv_pd *ibv_alloc_pd(struct ibv_context *context);
int ibv_dealloc_pd(struct ibv_pd *pd);
/**
+ * ibv_open_xrcd - Open an extended connection domain
+ */
+static inline struct ibv_xrcd *
+ibv_open_xrcd(struct ibv_context *context, struct ibv_xrcd_init_attr *xrcd_init_attr)
+{
+ struct verbs_context *vctx = verbs_get_ctx_op(context, open_xrcd);
+ if (!vctx) {
+ errno = ENOSYS;
+ return NULL;
+ }
+ return vctx->open_xrcd(context, xrcd_init_attr);
+}
+
+/**
+ * ibv_close_xrcd - Close an extended connection domain
+ */
+static inline int ibv_close_xrcd(struct ibv_xrcd *xrcd)
+{
+ struct verbs_context *vctx = verbs_get_ctx(xrcd->context);
+ return vctx->close_xrcd(xrcd);
+}
+
+/**
* ibv_reg_mr - Register a memory region
*/
struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr,
diff --git a/src/cmd.c b/src/cmd.c
index 86350fd..f3d590a 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -194,6 +194,51 @@ int ibv_cmd_dealloc_pd(struct ibv_pd *pd)
return 0;
}
+int ibv_cmd_open_xrcd(struct ibv_context *context, struct verbs_xrcd *xrcd,
+ int vxrcd_size,
+ struct ibv_xrcd_init_attr *attr,
+ struct ibv_open_xrcd *cmd, size_t cmd_size,
+ struct ibv_open_xrcd_resp *resp, size_t resp_size)
+{
+ IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_XRCD, resp, resp_size);
+
+ if (attr->comp_mask >= IBV_XRCD_INIT_ATTR_RESERVED)
+ return ENOSYS;
+
+ if (!(attr->comp_mask & IBV_XRCD_INIT_ATTR_FD) ||
+ !(attr->comp_mask & IBV_XRCD_INIT_ATTR_OFLAGS))
+ return EINVAL;
+
+ cmd->fd = attr->fd;
+ cmd->oflags = attr->oflags;
+ if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
+ return errno;
+
+ VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
+
+ xrcd->xrcd.context = context;
+ xrcd->comp_mask = 0;
+ if (vext_field_avail(struct verbs_xrcd, handle, vxrcd_size)) {
+ xrcd->comp_mask = VERBS_XRCD_HANDLE;
+ xrcd->handle = resp->xrcd_handle;
+ }
+
+ return 0;
+}
+
+int ibv_cmd_close_xrcd(struct verbs_xrcd *xrcd)
+{
+ struct ibv_close_xrcd cmd;
+
+ IBV_INIT_CMD(&cmd, sizeof cmd, CLOSE_XRCD);
+ cmd.xrcd_handle = xrcd->handle;
+
+ if (write(xrcd->xrcd.context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
+ return errno;
+
+ return 0;
+}
+
int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
uint64_t hca_va, int access,
struct ibv_mr *mr, struct ibv_reg_mr *cmd,
diff --git a/src/libibverbs.map b/src/libibverbs.map
index 6e35c37..c190eb9 100644
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -100,4 +100,7 @@ IBVERBS_1.1 {
ibv_rate_to_mbps;
mbps_to_ibv_rate;
+
+ ibv_cmd_open_xrcd;
+ ibv_cmd_close_xrcd;
} IBVERBS_1.0;
--
1.7.1
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH V8 libibverbs 3/7] Add support for XRC SRQs
[not found] ` <1374741488-30895-1-git-send-email-yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
2013-07-25 8:38 ` [PATCH V8 libibverbs 1/7] Infrastructure to support verbs extensions Yishai Hadas
2013-07-25 8:38 ` [PATCH V8 libibverbs 2/7] Introduce XRC domains Yishai Hadas
@ 2013-07-25 8:38 ` Yishai Hadas
2013-07-25 8:38 ` [PATCH V8 libibverbs 4/7] Add support for XRC QPs Yishai Hadas
` (3 subsequent siblings)
6 siblings, 0 replies; 15+ messages in thread
From: Yishai Hadas @ 2013-07-25 8:38 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA, roland-BHEL68pLQRGGvPXPguhicg
Cc: ogerlitz-VPRAkNaXOzVWk0Htik3J/w, tzahio-VPRAkNaXOzVWk0Htik3J/w,
yishaih-VPRAkNaXOzVWk0Htik3J/w, Sean Hefty
From: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
XRC support requires the use of a new type of SRQ.
XRC shared receive queues: xrc srq's are similar to normal
srq's, except that they are bound to an xrcd, rather
than to a protection domain. Based on the current spec
and implementation, they are only usable with xrc qps. To
support xrc srq's, we define a new srq_init_attr structure
to include an srq type and other needed information.
The kernel ABI is also updated to allow creating extended
SRQs.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Signed-off-by: Yishai Hadas <yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
---
include/infiniband/driver.h | 32 +++++++++++++++++
include/infiniband/kern-abi.h | 21 +++++++++++-
include/infiniband/verbs.h | 62 +++++++++++++++++++++++++++++++++-
src/cmd.c | 75 +++++++++++++++++++++++++++++++++++++++++
src/libibverbs.map | 2 +
5 files changed, 190 insertions(+), 2 deletions(-)
diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index 08d8a82..11f52c2 100644
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -64,6 +64,23 @@ struct verbs_xrcd {
uint32_t handle;
};
+enum verbs_srq_mask {
+ VERBS_SRQ_TYPE = 1 << 0,
+ VERBS_SRQ_XRCD = 1 << 1,
+ VERBS_SRQ_CQ = 1 << 2,
+ VERBS_SRQ_NUM = 1 << 3,
+ VERBS_SRQ_RESERVED = 1 << 4
+};
+
+struct verbs_srq {
+ struct ibv_srq srq;
+ uint32_t comp_mask;
+ enum ibv_srq_type srq_type;
+ struct verbs_xrcd *xrcd;
+ struct ibv_cq *cq;
+ uint32_t srq_num;
+};
+
typedef struct ibv_device *(*ibv_driver_init_func)(const char *uverbs_sys_path,
int abi_version);
typedef struct verbs_device *(*verbs_driver_init_func)(const char *uverbs_sys_path,
@@ -119,6 +136,11 @@ int ibv_cmd_create_srq(struct ibv_pd *pd,
struct ibv_srq *srq, struct ibv_srq_init_attr *attr,
struct ibv_create_srq *cmd, size_t cmd_size,
struct ibv_create_srq_resp *resp, size_t resp_size);
+int ibv_cmd_create_srq_ex(struct ibv_context *context,
+ struct verbs_srq *srq, int vsrq_sz,
+ struct ibv_srq_init_attr_ex *attr_ex,
+ struct ibv_create_xsrq *cmd, size_t cmd_size,
+ struct ibv_create_srq_resp *resp, size_t resp_size);
int ibv_cmd_modify_srq(struct ibv_srq *srq,
struct ibv_srq_attr *srq_attr,
int srq_attr_mask,
@@ -163,4 +185,14 @@ const char *ibv_get_sysfs_path(void);
int ibv_read_sysfs_file(const char *dir, const char *file,
char *buf, size_t size);
+static inline int verbs_get_srq_num(struct ibv_srq *srq, uint32_t *srq_num)
+{
+ struct verbs_srq *vsrq = container_of(srq, struct verbs_srq, srq);
+ if (vsrq->comp_mask & VERBS_SRQ_NUM) {
+ *srq_num = vsrq->srq_num;
+ return 0;
+ }
+ return ENOSYS;
+}
+
#endif /* INFINIBAND_DRIVER_H */
diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index 6f40ad2..ef93cc3 100644
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -88,6 +88,7 @@ enum {
IB_USER_VERBS_CMD_POST_SRQ_RECV,
IB_USER_VERBS_CMD_OPEN_XRCD,
IB_USER_VERBS_CMD_CLOSE_XRCD,
+ IB_USER_VERBS_CMD_CREATE_XSRQ
};
/*
@@ -730,11 +731,28 @@ struct ibv_create_srq {
__u64 driver_data[0];
};
+struct ibv_create_xsrq {
+ __u32 command;
+ __u16 in_words;
+ __u16 out_words;
+ __u64 response;
+ __u64 user_handle;
+ __u32 srq_type;
+ __u32 pd_handle;
+ __u32 max_wr;
+ __u32 max_sge;
+ __u32 srq_limit;
+ __u32 reserved;
+ __u32 xrcd_handle;
+ __u32 cq_handle;
+ __u64 driver_data[0];
+};
+
struct ibv_create_srq_resp {
__u32 srq_handle;
__u32 max_wr;
__u32 max_sge;
- __u32 reserved;
+ __u32 srqn;
};
struct ibv_modify_srq {
@@ -829,6 +847,7 @@ enum {
IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL_V2 = -1,
IB_USER_VERBS_CMD_OPEN_XRCD_V2 = -1,
IB_USER_VERBS_CMD_CLOSE_XRCD_V2 = -1,
+ IB_USER_VERBS_CMD_CREATE_XSRQ_V2 = -1
};
struct ibv_modify_srq_v3 {
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 7459119..8074bfe 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -449,6 +449,30 @@ struct ibv_srq_init_attr {
struct ibv_srq_attr attr;
};
+enum ibv_srq_type {
+ IBV_SRQT_BASIC,
+ IBV_SRQT_XRC
+};
+
+enum ibv_srq_init_attr_mask {
+ IBV_SRQ_INIT_ATTR_TYPE = 1 << 0,
+ IBV_SRQ_INIT_ATTR_PD = 1 << 1,
+ IBV_SRQ_INIT_ATTR_XRCD = 1 << 2,
+ IBV_SRQ_INIT_ATTR_CQ = 1 << 3,
+ IBV_SRQ_INIT_ATTR_RESERVED = 1 << 4
+};
+
+struct ibv_srq_init_attr_ex {
+ void *srq_context;
+ struct ibv_srq_attr attr;
+
+ uint32_t comp_mask;
+ enum ibv_srq_type srq_type;
+ struct ibv_pd *pd;
+ struct ibv_xrcd *xrcd;
+ struct ibv_cq *cq;
+};
+
enum ibv_qp_type {
IBV_QPT_RC = 2,
IBV_QPT_UC,
@@ -769,11 +793,15 @@ struct ibv_context {
enum verbs_context_mask {
VERBS_CONTEXT_XRCD = 1 << 0,
- VERBS_CONTEXT_RESERVED = 1 << 1
+ VERBS_CONTEXT_SRQ = 1 << 1,
+ VERBS_CONTEXT_RESERVED = 1 << 2
};
struct verbs_context {
/* "grows up" - new fields go here */
+ int (*get_srq_num)(struct ibv_srq *srq, uint32_t *srq_num);
+ struct ibv_srq * (*create_srq_ex)(struct ibv_context *context,
+ struct ibv_srq_init_attr_ex *srq_init_attr_ex);
struct ibv_xrcd * (*open_xrcd)(struct ibv_context *context,
struct ibv_xrcd_init_attr *xrcd_init_attr);
int (*close_xrcd)(struct ibv_xrcd *xrcd);
@@ -1061,6 +1089,28 @@ static inline int ibv_req_notify_cq(struct ibv_cq *cq, int solicited_only)
struct ibv_srq *ibv_create_srq(struct ibv_pd *pd,
struct ibv_srq_init_attr *srq_init_attr);
+static inline struct ibv_srq *
+ibv_create_srq_ex(struct ibv_context *context,
+ struct ibv_srq_init_attr_ex *srq_init_attr_ex)
+{
+ struct verbs_context *vctx;
+ uint32_t mask = srq_init_attr_ex->comp_mask;
+
+ if (!(mask & ~(IBV_SRQ_INIT_ATTR_PD | IBV_SRQ_INIT_ATTR_TYPE)) &&
+ (mask & IBV_SRQ_INIT_ATTR_PD) &&
+ (!(mask & IBV_SRQ_INIT_ATTR_TYPE) ||
+ (srq_init_attr_ex->srq_type == IBV_SRQT_BASIC)))
+ return ibv_create_srq(srq_init_attr_ex->pd,
+ (struct ibv_srq_init_attr *)srq_init_attr_ex);
+
+ vctx = verbs_get_ctx_op(context, create_srq_ex);
+ if (!vctx) {
+ errno = ENOSYS;
+ return NULL;
+ }
+ return vctx->create_srq_ex(context, srq_init_attr_ex);
+}
+
/**
* ibv_modify_srq - Modifies the attributes for the specified SRQ.
* @srq: The SRQ to modify.
@@ -1085,6 +1135,16 @@ int ibv_modify_srq(struct ibv_srq *srq,
*/
int ibv_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr);
+static inline int ibv_get_srq_num(struct ibv_srq *srq, uint32_t *srq_num)
+{
+ struct verbs_context *vctx = verbs_get_ctx_op(srq->context, get_srq_num);
+
+ if (!vctx)
+ return ENOSYS;
+
+ return vctx->get_srq_num(srq, srq_num);
+}
+
/**
* ibv_destroy_srq - Destroys the specified SRQ.
* @srq: The SRQ to destroy.
diff --git a/src/cmd.c b/src/cmd.c
index f3d590a..fae2846 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -442,6 +442,81 @@ int ibv_cmd_create_srq(struct ibv_pd *pd,
return 0;
}
+int ibv_cmd_create_srq_ex(struct ibv_context *context,
+ struct verbs_srq *srq, int vsrq_sz,
+ struct ibv_srq_init_attr_ex *attr_ex,
+ struct ibv_create_xsrq *cmd, size_t cmd_size,
+ struct ibv_create_srq_resp *resp, size_t resp_size)
+{
+ struct verbs_xrcd *vxrcd = NULL;
+
+ IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_XSRQ, resp, resp_size);
+
+ if (attr_ex->comp_mask >= IBV_SRQ_INIT_ATTR_RESERVED)
+ return ENOSYS;
+
+ if (!(attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_PD))
+ return EINVAL;
+
+ cmd->user_handle = (uintptr_t) srq;
+ cmd->pd_handle = attr_ex->pd->handle;
+ cmd->max_wr = attr_ex->attr.max_wr;
+ cmd->max_sge = attr_ex->attr.max_sge;
+ cmd->srq_limit = attr_ex->attr.srq_limit;
+
+ cmd->srq_type = (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) ?
+ attr_ex->srq_type : IBV_SRQT_BASIC;
+ if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_XRCD) {
+ if (!(attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_CQ))
+ return EINVAL;
+
+ vxrcd = container_of(attr_ex->xrcd, struct verbs_xrcd, xrcd);
+ cmd->xrcd_handle = vxrcd->handle;
+ cmd->cq_handle = attr_ex->cq->handle;
+ }
+
+ if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
+ return errno;
+
+ VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
+
+ srq->srq.handle = resp->srq_handle;
+ srq->srq.context = context;
+ srq->srq.srq_context = attr_ex->srq_context;
+ srq->srq.pd = attr_ex->pd;
+ srq->srq.events_completed = 0;
+ pthread_mutex_init(&srq->srq.mutex, NULL);
+ pthread_cond_init(&srq->srq.cond, NULL);
+
+ /*
+ * check that the last field is available.
+ * If it is than all the others exist as well
+ */
+ if (vext_field_avail(struct verbs_srq, srq_num, vsrq_sz)) {
+ srq->comp_mask = IBV_SRQ_INIT_ATTR_TYPE;
+ srq->srq_type = (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) ?
+ attr_ex->srq_type : IBV_SRQT_BASIC;
+ if (srq->srq_type == IBV_SRQT_XRC) {
+ srq->comp_mask |= VERBS_SRQ_NUM;
+ srq->srq_num = resp->srqn;
+ }
+ if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_XRCD) {
+ srq->comp_mask |= VERBS_SRQ_XRCD;
+ srq->xrcd = vxrcd;
+ }
+ if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_CQ) {
+ srq->comp_mask |= VERBS_SRQ_CQ;
+ srq->cq = attr_ex->cq;
+ }
+ }
+
+ attr_ex->attr.max_wr = resp->max_wr;
+ attr_ex->attr.max_sge = resp->max_sge;
+
+ return 0;
+}
+
+
static int ibv_cmd_modify_srq_v3(struct ibv_srq *srq,
struct ibv_srq_attr *srq_attr,
int srq_attr_mask,
diff --git a/src/libibverbs.map b/src/libibverbs.map
index c190eb9..a4b089f 100644
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -103,4 +103,6 @@ IBVERBS_1.1 {
ibv_cmd_open_xrcd;
ibv_cmd_close_xrcd;
+ ibv_cmd_create_srq_ex;
+
} IBVERBS_1.0;
--
1.7.1
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH V8 libibverbs 4/7] Add support for XRC QPs
[not found] ` <1374741488-30895-1-git-send-email-yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
` (2 preceding siblings ...)
2013-07-25 8:38 ` [PATCH V8 libibverbs 3/7] Add support for XRC SRQs Yishai Hadas
@ 2013-07-25 8:38 ` Yishai Hadas
2013-07-25 8:38 ` [PATCH V8 libibverbs 5/7] Add ibv_open_qp Yishai Hadas
` (2 subsequent siblings)
6 siblings, 0 replies; 15+ messages in thread
From: Yishai Hadas @ 2013-07-25 8:38 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA, roland-BHEL68pLQRGGvPXPguhicg
Cc: ogerlitz-VPRAkNaXOzVWk0Htik3J/w, tzahio-VPRAkNaXOzVWk0Htik3J/w,
yishaih-VPRAkNaXOzVWk0Htik3J/w, Sean Hefty
From: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
XRC queue pairs: xrc defines two new types of QPs. The
initiator, or send-side, xrc qp behaves similar to a send-
only RC qp. xrc send qp's are managed through the existing
QP functions. The send_wr structure is extended in a back-
wards compatible way to support posting sends on a send xrc
qp, which require specifying the remote xrc srq.
The target, or receive-side, xrc qp behaves differently
than other implemented qp's. A recv xrc qp can be created,
modified, and destroyed like other qp's through the existing
calls. The qp_init_attr structure is extended for xrc qp's.
Because xrc recv qp's are bound to an xrcd, rather than a pd,
it is intended to be used among multiple processes. Any process
with access to an xrcd may allocate and connect an xrc recv qp.
The actual xrc recv qp is allocated and managed by the kernel.
If the owning process explicit destroys the xrc recv qp, it is
destroyed. However, if the xrc recv qp is left open when the
user process exits or closes its device, then the lifetime of
the xrc recv qp is bound with the lifetime of the xrcd.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Signed-off-by: Yishai Hadas <yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
---
include/infiniband/driver.h | 15 ++++++
include/infiniband/kern-abi.h | 5 ++
include/infiniband/verbs.h | 55 ++++++++++++++++++++++-
src/cmd.c | 96 ++++++++++++++++++++++++++++++++++++++++-
src/libibverbs.map | 1 +
5 files changed, 168 insertions(+), 4 deletions(-)
diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index 11f52c2..91b5b12 100644
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -81,6 +81,16 @@ struct verbs_srq {
uint32_t srq_num;
};
+enum verbs_qp_mask {
+ VERBS_QP_XRCD = 1 << 0,
+ VERBS_QP_RESERVED = 1 << 1
+};
+
+struct verbs_qp {
+ struct ibv_qp qp;
+ uint32_t comp_mask;
+ struct verbs_xrcd *xrcd;
+};
typedef struct ibv_device *(*ibv_driver_init_func)(const char *uverbs_sys_path,
int abi_version);
typedef struct verbs_device *(*verbs_driver_init_func)(const char *uverbs_sys_path,
@@ -154,6 +164,11 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
struct ibv_create_qp *cmd, size_t cmd_size,
struct ibv_create_qp_resp *resp, size_t resp_size);
+int ibv_cmd_create_qp_ex(struct ibv_context *context,
+ struct verbs_qp *qp, int vqp_sz,
+ struct ibv_qp_init_attr_ex *attr_ex,
+ struct ibv_create_qp *cmd, size_t cmd_size,
+ struct ibv_create_qp_resp *resp, size_t resp_size);
int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *qp_attr,
int attr_mask,
struct ibv_qp_init_attr *qp_init_attr,
diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index ef93cc3..65f3ffd 100644
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -618,6 +618,11 @@ struct ibv_kern_send_wr {
__u32 reserved;
} ud;
} wr;
+ union {
+ struct {
+ __u32 remote_srqn;
+ } xrc;
+ } qp_type;
};
struct ibv_post_send {
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 8074bfe..aba8dd6 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -108,7 +108,8 @@ enum ibv_device_cap_flags {
IBV_DEVICE_SYS_IMAGE_GUID = 1 << 11,
IBV_DEVICE_RC_RNR_NAK_GEN = 1 << 12,
IBV_DEVICE_SRQ_RESIZE = 1 << 13,
- IBV_DEVICE_N_NOTIFY_CQ = 1 << 14
+ IBV_DEVICE_N_NOTIFY_CQ = 1 << 14,
+ IBV_DEVICE_XRC = 1 << 20
};
enum ibv_atomic_cap {
@@ -477,7 +478,9 @@ enum ibv_qp_type {
IBV_QPT_RC = 2,
IBV_QPT_UC,
IBV_QPT_UD,
- IBV_QPT_RAW_PACKET = 8
+ IBV_QPT_RAW_PACKET = 8,
+ IBV_QPT_XRC_SEND = 9,
+ IBV_QPT_XRC_RECV
};
struct ibv_qp_cap {
@@ -498,6 +501,26 @@ struct ibv_qp_init_attr {
int sq_sig_all;
};
+enum ibv_qp_init_attr_mask {
+ IBV_QP_INIT_ATTR_PD = 1 << 0,
+ IBV_QP_INIT_ATTR_XRCD = 1 << 1,
+ IBV_QP_INIT_ATTR_RESERVED = 1 << 2
+};
+
+struct ibv_qp_init_attr_ex {
+ void *qp_context;
+ struct ibv_cq *send_cq;
+ struct ibv_cq *recv_cq;
+ struct ibv_srq *srq;
+ struct ibv_qp_cap cap;
+ enum ibv_qp_type qp_type;
+ int sq_sig_all;
+
+ uint32_t comp_mask;
+ struct ibv_pd *pd;
+ struct ibv_xrcd *xrcd;
+};
+
enum ibv_qp_attr_mask {
IBV_QP_STATE = 1 << 0,
IBV_QP_CUR_STATE = 1 << 1,
@@ -614,6 +637,11 @@ struct ibv_send_wr {
uint32_t remote_qkey;
} ud;
} wr;
+ union {
+ struct {
+ uint32_t remote_srqn;
+ } xrc;
+ } qp_type;
};
struct ibv_recv_wr {
@@ -794,11 +822,14 @@ struct ibv_context {
enum verbs_context_mask {
VERBS_CONTEXT_XRCD = 1 << 0,
VERBS_CONTEXT_SRQ = 1 << 1,
- VERBS_CONTEXT_RESERVED = 1 << 2
+ VERBS_CONTEXT_QP = 1 << 2,
+ VERBS_CONTEXT_RESERVED = 1 << 3
};
struct verbs_context {
/* "grows up" - new fields go here */
+ struct ibv_qp *(*create_qp_ex)(struct ibv_context *context,
+ struct ibv_qp_init_attr_ex *qp_init_attr_ex);
int (*get_srq_num)(struct ibv_srq *srq, uint32_t *srq_num);
struct ibv_srq * (*create_srq_ex)(struct ibv_context *context,
struct ibv_srq_init_attr_ex *srq_init_attr_ex);
@@ -1171,6 +1202,24 @@ static inline int ibv_post_srq_recv(struct ibv_srq *srq,
struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
struct ibv_qp_init_attr *qp_init_attr);
+static inline struct ibv_qp *
+ibv_create_qp_ex(struct ibv_context *context, struct ibv_qp_init_attr_ex *qp_init_attr_ex)
+{
+ struct verbs_context *vctx;
+ uint32_t mask = qp_init_attr_ex->comp_mask;
+
+ if (mask == IBV_QP_INIT_ATTR_PD)
+ return ibv_create_qp(qp_init_attr_ex->pd,
+ (struct ibv_qp_init_attr *)qp_init_attr_ex);
+
+ vctx = verbs_get_ctx_op(context, create_qp_ex);
+ if (!vctx) {
+ errno = ENOSYS;
+ return NULL;
+ }
+ return vctx->create_qp_ex(context, qp_init_attr_ex);
+}
+
/**
* ibv_modify_qp - Modify a queue pair.
*/
diff --git a/src/cmd.c b/src/cmd.c
index fae2846..463b95b 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -610,6 +610,100 @@ int ibv_cmd_destroy_srq(struct ibv_srq *srq)
return 0;
}
+int ibv_cmd_create_qp_ex(struct ibv_context *context,
+ struct verbs_qp *qp, int vqp_sz,
+ struct ibv_qp_init_attr_ex *attr_ex,
+ struct ibv_create_qp *cmd, size_t cmd_size,
+ struct ibv_create_qp_resp *resp, size_t resp_size)
+{
+ struct verbs_xrcd *vxrcd = NULL;
+
+ IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size);
+
+ if (attr_ex->comp_mask >= IBV_QP_INIT_ATTR_RESERVED)
+ return ENOSYS;
+
+ cmd->user_handle = (uintptr_t) qp;
+
+ if (attr_ex->comp_mask & IBV_QP_INIT_ATTR_XRCD) {
+ vxrcd = container_of(attr_ex->xrcd, struct verbs_xrcd, xrcd);
+ cmd->pd_handle = vxrcd->handle;
+ } else {
+ if (!(attr_ex->comp_mask & IBV_QP_INIT_ATTR_PD))
+ return EINVAL;
+
+ cmd->pd_handle = attr_ex->pd->handle;
+ cmd->send_cq_handle = attr_ex->send_cq->handle;
+
+ if (attr_ex->qp_type != IBV_QPT_XRC_SEND) {
+ cmd->recv_cq_handle = attr_ex->recv_cq->handle;
+ cmd->srq_handle = attr_ex->srq ? attr_ex->srq->handle : 0;
+ }
+ }
+
+ cmd->max_send_wr = attr_ex->cap.max_send_wr;
+ cmd->max_recv_wr = attr_ex->cap.max_recv_wr;
+ cmd->max_send_sge = attr_ex->cap.max_send_sge;
+ cmd->max_recv_sge = attr_ex->cap.max_recv_sge;
+ cmd->max_inline_data = attr_ex->cap.max_inline_data;
+ cmd->sq_sig_all = attr_ex->sq_sig_all;
+ cmd->qp_type = attr_ex->qp_type;
+ cmd->is_srq = !!attr_ex->srq;
+ cmd->reserved = 0;
+
+ if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
+ return errno;
+
+ (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
+
+ if (abi_ver > 3) {
+ attr_ex->cap.max_recv_sge = resp->max_recv_sge;
+ attr_ex->cap.max_send_sge = resp->max_send_sge;
+ attr_ex->cap.max_recv_wr = resp->max_recv_wr;
+ attr_ex->cap.max_send_wr = resp->max_send_wr;
+ attr_ex->cap.max_inline_data = resp->max_inline_data;
+ }
+
+ if (abi_ver == 4) {
+ struct ibv_create_qp_resp_v4 *resp_v4 =
+ (struct ibv_create_qp_resp_v4 *)resp;
+
+ memmove((void *)resp + sizeof *resp,
+ (void *)resp_v4 + sizeof *resp_v4,
+ resp_size - sizeof *resp);
+ } else if (abi_ver <= 3) {
+ struct ibv_create_qp_resp_v3 *resp_v3 =
+ (struct ibv_create_qp_resp_v3 *)resp;
+
+ memmove((void *)resp + sizeof *resp,
+ (void *)resp_v3 + sizeof *resp_v3,
+ resp_size - sizeof *resp);
+ }
+
+ qp->qp.handle = resp->qp_handle;
+ qp->qp.qp_num = resp->qpn;
+ qp->qp.context = context;
+ qp->qp.qp_context = attr_ex->qp_context;
+ qp->qp.pd = attr_ex->pd;
+ qp->qp.send_cq = attr_ex->send_cq;
+ qp->qp.recv_cq = attr_ex->recv_cq;
+ qp->qp.srq = attr_ex->srq;
+ qp->qp.qp_type = attr_ex->qp_type;
+ qp->qp.state = IBV_QPS_RESET;
+ qp->qp.events_completed = 0;
+ pthread_mutex_init(&qp->qp.mutex, NULL);
+ pthread_cond_init(&qp->qp.cond, NULL);
+
+ qp->comp_mask = 0;
+ if (vext_field_avail(struct verbs_qp, xrcd, vqp_sz) &&
+ (attr_ex->comp_mask & IBV_QP_INIT_ATTR_XRCD)) {
+ qp->comp_mask |= VERBS_QP_XRCD;
+ qp->xrcd = vxrcd;
+ }
+
+ return 0;
+}
+
int ibv_cmd_create_qp(struct ibv_pd *pd,
struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
struct ibv_create_qp *cmd, size_t cmd_size,
@@ -618,7 +712,7 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size);
cmd->user_handle = (uintptr_t) qp;
- cmd->pd_handle = pd->handle;
+ cmd->pd_handle = pd->handle;
cmd->send_cq_handle = attr->send_cq->handle;
cmd->recv_cq_handle = attr->recv_cq->handle;
cmd->srq_handle = attr->srq ? attr->srq->handle : 0;
diff --git a/src/libibverbs.map b/src/libibverbs.map
index a4b089f..6db5291 100644
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -104,5 +104,6 @@ IBVERBS_1.1 {
ibv_cmd_open_xrcd;
ibv_cmd_close_xrcd;
ibv_cmd_create_srq_ex;
+ ibv_cmd_create_qp_ex;
} IBVERBS_1.0;
--
1.7.1
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH V8 libibverbs 5/7] Add ibv_open_qp
[not found] ` <1374741488-30895-1-git-send-email-yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
` (3 preceding siblings ...)
2013-07-25 8:38 ` [PATCH V8 libibverbs 4/7] Add support for XRC QPs Yishai Hadas
@ 2013-07-25 8:38 ` Yishai Hadas
2013-07-25 8:38 ` [PATCH V8 libibverbs 6/7] XRC man pages Yishai Hadas
2013-07-25 8:38 ` [PATCH V8 libibverbs 7/7] Add XRC sample application Yishai Hadas
6 siblings, 0 replies; 15+ messages in thread
From: Yishai Hadas @ 2013-07-25 8:38 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA, roland-BHEL68pLQRGGvPXPguhicg
Cc: ogerlitz-VPRAkNaXOzVWk0Htik3J/w, tzahio-VPRAkNaXOzVWk0Htik3J/w,
yishaih-VPRAkNaXOzVWk0Htik3J/w, Sean Hefty
From: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
XRC receive QPs are shareable across multiple processes. Allow
any process with access to the xrc domain to open an existing
QP. After opening the QP, the process will receive events
related to the QP and be able to modify the QP.
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Signed-off-by: Yishai Hadas <yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
---
include/infiniband/driver.h | 5 ++++
include/infiniband/kern-abi.h | 20 ++++++++++++++-
include/infiniband/verbs.h | 35 +++++++++++++++++++++++++++-
src/cmd.c | 50 +++++++++++++++++++++++++++++++++++++++++
src/libibverbs.map | 1 +
5 files changed, 108 insertions(+), 3 deletions(-)
diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index 91b5b12..f69962e 100644
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -169,6 +169,11 @@ int ibv_cmd_create_qp_ex(struct ibv_context *context,
struct ibv_qp_init_attr_ex *attr_ex,
struct ibv_create_qp *cmd, size_t cmd_size,
struct ibv_create_qp_resp *resp, size_t resp_size);
+int ibv_cmd_open_qp(struct ibv_context *context,
+ struct verbs_qp *qp, int vqp_sz,
+ struct ibv_qp_open_attr *attr,
+ struct ibv_open_qp *cmd, size_t cmd_size,
+ struct ibv_create_qp_resp *resp, size_t resp_size);
int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *qp_attr,
int attr_mask,
struct ibv_qp_init_attr *qp_init_attr,
diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index 65f3ffd..064f1aa 100644
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -88,7 +88,8 @@ enum {
IB_USER_VERBS_CMD_POST_SRQ_RECV,
IB_USER_VERBS_CMD_OPEN_XRCD,
IB_USER_VERBS_CMD_CLOSE_XRCD,
- IB_USER_VERBS_CMD_CREATE_XSRQ
+ IB_USER_VERBS_CMD_CREATE_XSRQ,
+ IB_USER_VERBS_CMD_OPEN_QP
};
/*
@@ -476,6 +477,20 @@ struct ibv_create_qp {
__u64 driver_data[0];
};
+struct ibv_open_qp {
+ __u32 command;
+ __u16 in_words;
+ __u16 out_words;
+ __u64 response;
+ __u64 user_handle;
+ __u32 pd_handle;
+ __u32 qpn;
+ __u8 qp_type;
+ __u8 reserved[7];
+ __u64 driver_data[0];
+};
+
+/* also used for open response */
struct ibv_create_qp_resp {
__u32 qp_handle;
__u32 qpn;
@@ -852,7 +867,8 @@ enum {
IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL_V2 = -1,
IB_USER_VERBS_CMD_OPEN_XRCD_V2 = -1,
IB_USER_VERBS_CMD_CLOSE_XRCD_V2 = -1,
- IB_USER_VERBS_CMD_CREATE_XSRQ_V2 = -1
+ IB_USER_VERBS_CMD_CREATE_XSRQ_V2 = -1,
+ IB_USER_VERBS_CMD_OPEN_QP_V2 = -1
};
struct ibv_modify_srq_v3 {
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index aba8dd6..6437811 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -521,6 +521,22 @@ struct ibv_qp_init_attr_ex {
struct ibv_xrcd *xrcd;
};
+enum ibv_qp_open_attr_mask {
+ IBV_QP_OPEN_ATTR_NUM = 1 << 0,
+ IBV_QP_OPEN_ATTR_XRCD = 1 << 1,
+ IBV_QP_OPEN_ATTR_CONTEXT = 1 << 2,
+ IBV_QP_OPEN_ATTR_TYPE = 1 << 3,
+ IBV_QP_OPEN_ATTR_RESERVED = 1 << 4
+};
+
+struct ibv_qp_open_attr {
+ uint32_t comp_mask;
+ uint32_t qp_num;
+ struct ibv_xrcd *xrcd;
+ void *qp_context;
+ enum ibv_qp_type qp_type;
+};
+
enum ibv_qp_attr_mask {
IBV_QP_STATE = 1 << 0,
IBV_QP_CUR_STATE = 1 << 1,
@@ -552,7 +568,8 @@ enum ibv_qp_state {
IBV_QPS_RTS,
IBV_QPS_SQD,
IBV_QPS_SQE,
- IBV_QPS_ERR
+ IBV_QPS_ERR,
+ IBV_QPS_UNKNOWN
};
enum ibv_mig_state {
@@ -828,6 +845,8 @@ enum verbs_context_mask {
struct verbs_context {
/* "grows up" - new fields go here */
+ struct ibv_qp *(*open_qp)(struct ibv_context *context,
+ struct ibv_qp_open_attr *attr);
struct ibv_qp *(*create_qp_ex)(struct ibv_context *context,
struct ibv_qp_init_attr_ex *qp_init_attr_ex);
int (*get_srq_num)(struct ibv_srq *srq, uint32_t *srq_num);
@@ -1221,6 +1240,20 @@ ibv_create_qp_ex(struct ibv_context *context, struct ibv_qp_init_attr_ex *qp_ini
}
/**
+ * ibv_open_qp - Open a shareable queue pair.
+ */
+static inline struct ibv_qp *
+ibv_open_qp(struct ibv_context *context, struct ibv_qp_open_attr *qp_open_attr)
+{
+ struct verbs_context *vctx = verbs_get_ctx_op(context, open_qp);
+ if (!vctx) {
+ errno = ENOSYS;
+ return NULL;
+ }
+ return vctx->open_qp(context, qp_open_attr);
+}
+
+/**
* ibv_modify_qp - Modify a queue pair.
*/
int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
diff --git a/src/cmd.c b/src/cmd.c
index 463b95b..7c88666 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -762,6 +762,56 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
return 0;
}
+int ibv_cmd_open_qp(struct ibv_context *context, struct verbs_qp *qp,
+ int vqp_sz,
+ struct ibv_qp_open_attr *attr,
+ struct ibv_open_qp *cmd, size_t cmd_size,
+ struct ibv_create_qp_resp *resp, size_t resp_size)
+{
+ struct verbs_xrcd *xrcd;
+ IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_QP, resp, resp_size);
+
+ if (attr->comp_mask >= IBV_QP_OPEN_ATTR_RESERVED)
+ return ENOSYS;
+
+ if (!(attr->comp_mask & IBV_QP_OPEN_ATTR_XRCD) ||
+ !(attr->comp_mask & IBV_QP_OPEN_ATTR_NUM) ||
+ !(attr->comp_mask & IBV_QP_OPEN_ATTR_TYPE))
+ return EINVAL;
+
+ xrcd = container_of(attr->xrcd, struct verbs_xrcd, xrcd);
+ cmd->user_handle = (uintptr_t) qp;
+ cmd->pd_handle = xrcd->handle;
+ cmd->qpn = attr->qp_num;
+ cmd->qp_type = attr->qp_type;
+
+ if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
+ return errno;
+
+ VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
+
+ qp->qp.handle = resp->qp_handle;
+ qp->qp.context = context;
+ qp->qp.qp_context = attr->qp_context;
+ qp->qp.pd = NULL;
+ qp->qp.send_cq = NULL;
+ qp->qp.recv_cq = NULL;
+ qp->qp.srq = NULL;
+ qp->qp.qp_num = attr->qp_num;
+ qp->qp.qp_type = attr->qp_type;
+ qp->qp.state = IBV_QPS_UNKNOWN;
+ qp->qp.events_completed = 0;
+ pthread_mutex_init(&qp->qp.mutex, NULL);
+ pthread_cond_init(&qp->qp.cond, NULL);
+ qp->comp_mask = 0;
+ if (vext_field_avail(struct verbs_qp, xrcd, vqp_sz)) {
+ qp->comp_mask = VERBS_QP_XRCD;
+ qp->xrcd = xrcd;
+ }
+
+ return 0;
+}
+
int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
int attr_mask,
struct ibv_qp_init_attr *init_attr,
diff --git a/src/libibverbs.map b/src/libibverbs.map
index 6db5291..bbc6de1 100644
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -105,5 +105,6 @@ IBVERBS_1.1 {
ibv_cmd_close_xrcd;
ibv_cmd_create_srq_ex;
ibv_cmd_create_qp_ex;
+ ibv_cmd_open_qp;
} IBVERBS_1.0;
--
1.7.1
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH V8 libibverbs 6/7] XRC man pages
[not found] ` <1374741488-30895-1-git-send-email-yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
` (4 preceding siblings ...)
2013-07-25 8:38 ` [PATCH V8 libibverbs 5/7] Add ibv_open_qp Yishai Hadas
@ 2013-07-25 8:38 ` Yishai Hadas
2013-07-25 8:38 ` [PATCH V8 libibverbs 7/7] Add XRC sample application Yishai Hadas
6 siblings, 0 replies; 15+ messages in thread
From: Yishai Hadas @ 2013-07-25 8:38 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA, roland-BHEL68pLQRGGvPXPguhicg
Cc: ogerlitz-VPRAkNaXOzVWk0Htik3J/w, tzahio-VPRAkNaXOzVWk0Htik3J/w,
yishaih-VPRAkNaXOzVWk0Htik3J/w, Sean Hefty
From: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Signed-off-by: Yishai Hadas <yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
---
Makefile.am | 8 +++-
man/ibv_create_qp_ex.3 | 83 +++++++++++++++++++++++++++++++++++++++++++++++
man/ibv_create_srq_ex.3 | 71 ++++++++++++++++++++++++++++++++++++++++
man/ibv_get_srq_num.3 | 32 ++++++++++++++++++
man/ibv_open_qp.3 | 51 +++++++++++++++++++++++++++++
man/ibv_open_xrcd.3 | 76 +++++++++++++++++++++++++++++++++++++++++++
6 files changed, 319 insertions(+), 2 deletions(-)
create mode 100644 man/ibv_create_qp_ex.3
create mode 100644 man/ibv_create_srq_ex.3
create mode 100644 man/ibv_get_srq_num.3
create mode 100644 man/ibv_open_qp.3
create mode 100644 man/ibv_open_xrcd.3
diff --git a/Makefile.am b/Makefile.am
index 40e83be..aa6097a 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -54,7 +54,9 @@ man_MANS = man/ibv_asyncwatch.1 man/ibv_devices.1 man/ibv_devinfo.1 \
man/ibv_post_srq_recv.3 man/ibv_query_device.3 man/ibv_query_gid.3 \
man/ibv_query_pkey.3 man/ibv_query_port.3 man/ibv_query_qp.3 \
man/ibv_query_srq.3 man/ibv_rate_to_mult.3 man/ibv_reg_mr.3 \
- man/ibv_req_notify_cq.3 man/ibv_resize_cq.3 man/ibv_rate_to_mbps.3
+ man/ibv_req_notify_cq.3 man/ibv_resize_cq.3 man/ibv_rate_to_mbps.3 \
+ man/ibv_create_qp_ex.3 man/ibv_create_srq_ex.3 man/ibv_open_xrcd.3 \
+ man/ibv_get_srq_num.3 man/ibv_open_qp.3
DEBIAN = debian/changelog debian/compat debian/control debian/copyright \
debian/ibverbs-utils.install debian/libibverbs1.install \
@@ -89,6 +91,7 @@ install-data-hook:
$(RM) ibv_node_type_str.3 && \
$(RM) ibv_port_state_str.3 && \
$(RM) mbps_to_ibv_rate.3 && \
+ $(RM) ibv_close_xrcd.3 && \
$(LN_S) ibv_get_async_event.3 ibv_ack_async_event.3 && \
$(LN_S) ibv_get_cq_event.3 ibv_ack_cq_events.3 && \
$(LN_S) ibv_open_device.3 ibv_close_device.3 && \
@@ -105,4 +108,5 @@ install-data-hook:
$(LN_S) ibv_rate_to_mult.3 mult_to_ibv_rate.3 && \
$(LN_S) ibv_event_type_str.3 ibv_node_type_str.3 && \
$(LN_S) ibv_event_type_str.3 ibv_port_state_str.3 && \
- $(LN_S) ibv_rate_to_mbps.3 mbps_to_ibv_rate.3
+ $(LN_S) ibv_rate_to_mbps.3 mbps_to_ibv_rate.3 && \
+ $(LN_S) ibv_open_xrcd.3 ibv_close_xrcd.3
diff --git a/man/ibv_create_qp_ex.3 b/man/ibv_create_qp_ex.3
new file mode 100644
index 0000000..f772a57
--- /dev/null
+++ b/man/ibv_create_qp_ex.3
@@ -0,0 +1,83 @@
+.\" -*- nroff -*-
+.\"
+.TH IBV_CREATE_QP_EX 3 2013-06-26 libibverbs "Libibverbs Programmer's Manual"
+.SH "NAME"
+ibv_create_qp_ex, ibv_destroy_qp \- create or destroy a queue pair (QP)
+.SH "SYNOPSIS"
+.nf
+.B #include <infiniband/verbs.h>
+.sp
+.BI "struct ibv_qp *ibv_create_qp_ex(struct ibv_context " "*context" ,
+.BI " struct ibv_qp_init_attr_ex " "*qp_init_attr" );
+.sp
+.BI "int ibv_destroy_qp(struct ibv_qp " "*qp" );
+.fi
+.SH "DESCRIPTION"
+.B ibv_create_qp_ex()
+creates a queue pair (QP) associated with the protection domain
+.I pd\fR.
+The argument
+.I qp_init_attr_ex
+is an ibv_qp_init_attr_ex struct, as defined in <infiniband/verbs.h>.
+.PP
+.nf
+struct ibv_qp_init_attr_ex {
+.in +8
+void *qp_context; /* Associated context of the QP */
+struct ibv_cq *send_cq; /* CQ to be associated with the Send Queue (SQ) */
+struct ibv_cq *recv_cq; /* CQ to be associated with the Receive Queue (RQ) */
+struct ibv_srq *srq; /* SRQ handle if QP is to be associated with an SRQ, otherwise NULL */
+struct ibv_qp_cap cap; /* QP capabilities */
+enum ibv_qp_type qp_type; /* QP Transport Service Type: IBV_QPT_RC, IBV_QPT_UC, IBV_QPT_UD or IBV_QPT_RAW_PACKET */
+int sq_sig_all; /* If set, each Work Request (WR) submitted to the SQ generates a completion entry */
+uint32_t comp_mask; /* Identifies valid fields */
+struct ibv_pd *pd; /* PD to be associated with the QP */
+struct ibv_xrcd *xrcd; /* XRC domain to be associated with the target QP */
+enum ibv_qp_create_flags create_flags; /* Creation flags for this QP */
+.in -8
+};
+.sp
+.nf
+struct ibv_qp_cap {
+.in +8
+uint32_t max_send_wr; /* Requested max number of outstanding WRs in the SQ */
+uint32_t max_recv_wr; /* Requested max number of outstanding WRs in the RQ */
+uint32_t max_send_sge; /* Requested max number of scatter/gather (s/g) elements in a WR in the SQ */
+uint32_t max_recv_sge; /* Requested max number of s/g elements in a WR in the SQ */
+uint32_t max_inline_data;/* Requested max number of data (bytes) that can be posted inline to the SQ, otherwise 0 */
+.in -8
+};
+.fi
+.PP
+The function
+.B ibv_create_qp_ex()
+will update the
+.I qp_init_attr_ex\fB\fR->cap
+struct with the actual \s-1QP\s0 values of the QP that was created;
+the values will be greater than or equal to the values requested.
+.PP
+.B ibv_destroy_qp()
+destroys the QP
+.I qp\fR.
+.SH "RETURN VALUE"
+.B ibv_create_qp_ex()
+returns a pointer to the created QP, or NULL if the request fails.
+Check the QP number (\fBqp_num\fR) in the returned QP.
+.PP
+.B ibv_destroy_qp()
+returns 0 on success, or the value of errno on failure (which indicates the failure reason).
+.SH "NOTES"
+.PP
+The attributes max_recv_wr and max_recv_sge are ignored by
+.B ibv_create_qp_ex()
+if the QP is to be associated with an SRQ.
+.PP
+.B ibv_destroy_qp()
+fails if the QP is attached to a multicast group.
+.SH "SEE ALSO"
+.BR ibv_alloc_pd (3),
+.BR ibv_modify_qp (3),
+.BR ibv_query_qp (3)
+.SH "AUTHORS"
+.TP
+Yishai Hadas <yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
diff --git a/man/ibv_create_srq_ex.3 b/man/ibv_create_srq_ex.3
new file mode 100644
index 0000000..9151d6e
--- /dev/null
+++ b/man/ibv_create_srq_ex.3
@@ -0,0 +1,71 @@
+.\" -*- nroff -*-
+.\"
+.TH IBV_CREATE_SRQ_EX 3 2013-06-26 libibverbs "Libibverbs Programmer's Manual"
+.SH "NAME"
+ibv_create_srq_ex, ibv_destroy_srq \- create or destroy a shared receive queue (SRQ)
+.SH "SYNOPSIS"
+.nf
+.B #include <infiniband/verbs.h>
+.sp
+.BI "struct ibv_srq *ibv_create_srq_ex(struct ibv_context " "*context" ", struct "
+.BI " ibv_srq_init_attr_ex " "*srq_init_attr_ex" );
+.sp
+.BI "int ibv_destroy_srq(struct ibv_srq " "*srq" );
+.fi
+.SH "DESCRIPTION"
+.B ibv_create_srq_ex()
+creates a shared receive queue (SRQ) supporting both basic and xrc modes.
+The argument
+.I srq_init_attr_ex
+is an ibv_srq_init_attr_ex struct, as defined in <infiniband/verbs.h>.
+.PP
+.nf
+struct ibv_srq_init_attr_ex {
+.in +8
+void *srq_context; /* Associated context of the SRQ */
+struct ibv_srq_attr attr; /* SRQ attributes */
+uint32_t comp_mask; /* Identifies valid fields */
+enum ibv_srq_type srq_type; /* Basic / XRC */
+struct ibv_pd *pd; /* PD associated with the SRQ */
+struct ibv_xrcd *xrcd; /* XRC domain to associate with the SRQ */
+struct ibv_cq *cq; /* CQ to associate with the SRQ for XRC mode */
+.in -8
+};
+.sp
+.nf
+struct ibv_srq_attr {
+.in +8
+uint32_t max_wr; /* Requested max number of outstanding work requests (WRs) in the SRQ */
+uint32_t max_sge; /* Requested max number of scatter elements per WR */
+uint32_t srq_limit; /* The limit value of the SRQ */
+.in -8
+};
+.fi
+.PP
+The function
+.B ibv_create_srq_ex()
+will update the
+.I srq_init_attr_ex
+struct with the original values of the SRQ that was created; the
+values of max_wr and max_sge will be greater than or equal to the
+values requested.
+.PP
+.B ibv_destroy_srq()
+destroys the SRQ
+.I srq\fR.
+.SH "RETURN VALUE"
+.B ibv_create_srq_ex()
+returns a pointer to the created SRQ, or NULL if the request fails.
+.PP
+.B ibv_destroy_srq()
+returns 0 on success, or the value of errno on failure (which indicates the failure reason).
+.SH "NOTES"
+.B ibv_destroy_srq()
+fails if any queue pair is still associated with this SRQ.
+.SH "SEE ALSO"
+.BR ibv_alloc_pd (3),
+.BR ibv_modify_srq (3),
+.BR ibv_query_srq (3)
+.SH "AUTHORS"
+.TP
+Yishai Hadas <yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
diff --git a/man/ibv_get_srq_num.3 b/man/ibv_get_srq_num.3
new file mode 100644
index 0000000..00e4fb4
--- /dev/null
+++ b/man/ibv_get_srq_num.3
@@ -0,0 +1,32 @@
+.\" -*- nroff -*-
+.\"
+.TH IBV_GET_SRQ_NUM 3 2013-06-26 libibverbs "Libibverbs Programmer's Manual"
+.SH "NAME"
+ibv_get_srq_num \- return srq number associated with the given shared receive queue (SRQ)
+.SH "SYNOPSIS"
+.nf
+.B #include <infiniband/verbs.h>
+.sp
+.BI "int ibv_get_srq_num(struct ibv_srq " "*srq" ,
+.BI " uint32_t " "*srq_num" );
+.fi
+.SH "DESCRIPTION"
+.B ibv_get_srq_num()
+return srq number associated with the given shared receive queue
+The argument
+.I srq
+is an ibv_srq struct, as defined in <infiniband/verbs.h>.
+.I srq_num
+is an output parameter that holds the returned srq number.
+.PP
+.nf
+.SH "RETURN VALUE"
+.B ibv_get_srq_num()
+returns 0 on success, or the value of errno on failure (which indicates the failure reason).
+.SH "SEE ALSO"
+.BR ibv_alloc_pd (3),
+.BR ibv_modify_srq (3),
+.BR ibv_create_srq_ex (3)
+.SH "AUTHORS"
+.TP
+Yishai Hadas <yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
diff --git a/man/ibv_open_qp.3 b/man/ibv_open_qp.3
new file mode 100644
index 0000000..71b5d46
--- /dev/null
+++ b/man/ibv_open_qp.3
@@ -0,0 +1,51 @@
+.\" -*- nroff -*-
+.\"
+.TH IBV_OPEN_QP 3 2011-08-12 libibverbs "Libibverbs Programmer's Manual"
+.SH "NAME"
+ibv_open_qp \- open a shareable queue pair (QP)
+.SH "SYNOPSIS"
+.nf
+.B #include <infiniband/verbs.h>
+.sp
+.BI "struct ibv_qp *ibv_open_qp(struct ibv_context " "*context" ,
+.BI " struct ibv_qp_open_attr " "*qp_open_attr" );
+.fi
+.SH "DESCRIPTION"
+.B ibv_open_qp()
+opens an existing queue pair (QP) associated with the extended protection domain
+.I xrcd\fR.
+The argument
+.I qp_open_attr
+is an ibv_qp_open_attr struct, as defined in <infiniband/verbs.h>.
+.PP
+.nf
+struct ibv_qp_open_attr {
+.in +8
+uint32_t comp_mask; /* Identifies valid fields */
+uint32_t qp_num; /* QP number */
+struct *ibv_xrcd; /* XRC domain */
+void *qp_context; /* User defined opaque value */
+enum ibv_qp_type qp_type; /* QP transport service type */
+.fi
+.PP
+.B ibv_destroy_qp()
+closes the opened QP and destroys the underlying QP if it has no
+other references.
+.I qp\fR.
+.SH "RETURN VALUE"
+.B ibv_open_qp()
+returns a pointer to the opened QP, or NULL if the request fails.
+Check the QP number (\fBqp_num\fR) in the returned QP.
+.SH "NOTES"
+.B ibv_open_qp()
+will fail if a it is asked to open a QP that does not exist within
+the xrcd with the specified qp_num and qp_type.
+.SH "SEE ALSO"
+.BR ibv_alloc_pd (3),
+.BR ibv_create_qp (3),
+.BR ibv_create_qp_ex (3),
+.BR ibv_modify_qp (3),
+.BR ibv_query_qp (3)
+.SH "AUTHORS"
+.TP
+Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
diff --git a/man/ibv_open_xrcd.3 b/man/ibv_open_xrcd.3
new file mode 100644
index 0000000..b9724bf
--- /dev/null
+++ b/man/ibv_open_xrcd.3
@@ -0,0 +1,76 @@
+.\" -*- nroff -*-
+.\"
+.TH IBV_OPEN_XRCD 3 2011-06-17 libibverbs "Libibverbs Programmer's Manual"
+.SH "NAME"
+ibv_open_xrcd, ibv_close_xrcd \- open or close an XRC protection domain (XRCDs)
+.SH "SYNOPSIS"
+.nf
+.B #include <infiniband/verbs.h>
+.sp
+.BI "struct ibv_xrcd *ibv_open_xrcd(struct ibv_context " "*context" ","
+.BI " struct ibv_xrcd_init_attr " "*xrcd_init_attr" );
+.sp
+.BI "int ibv_close_xrcd(struct ibv_xrcd " "*xrcd" );
+.fi
+.SH "DESCRIPTION"
+.B ibv_open_xrcd()
+open an XRC domain for the RDMA device context
+.I context
+.I xrcd_init_attr
+is an ibv_xrcd_init_attr struct, as defined in <infiniband/verbs.h>.
+.PP
+.nf
+struct ibv_xrcd_init_attr {
+.in +8
+uint32_t comp_mask; /* Identifies valid fields */
+int fd;
+int oflag;
+.fi
+.PP
+.I fd
+is the file descriptor to associate with the XRCD.
+.I oflag
+describes the desired creation attributes. It is a bitwise OR of zero or more
+of the following flags:
+.PP
+.TP
+.B O_CREAT
+Indicates that an XRCD should be created and associated with the inode referenced
+by the given fd. If the XRCD exists, this flag has no effect except as noted under
+.BR O_EXCL
+below.\fR
+.TP
+.B O_EXCL
+If
+.BR O_EXCL
+and
+.BR O_CREAT
+are set, open will fail if an XRCD associated with the inode exists.
+.PP
+If
+.I fd
+equals -1, no inode is associated with the XRCD. To indicate that XRCD should be created, use
+.I oflag
+=
+.B O_CREAT\fR.
+.PP
+.B ibv_close_xrcd()
+closes the XRCD
+.I xrcd\fR.
+If this is the last reference, the XRCD will be destroyed.
+.SH "RETURN VALUE"
+.B ibv_open_xrcd()
+returns a pointer to the opened XRCD, or NULL if the request fails.
+.PP
+.B ibv_close_xrcd()
+returns 0 on success, or the value of errno on failure (which indicates the
+failure reason).
+.SH "NOTES"
+.B ibv_close_xrcd()
+may fail if any other resource is still associated with the XRCD being closed.
+.SH "SEE ALSO"
+.BR ibv_create_srq_ex (3),
+.BR ibv_create_qp_ex (3),
+.SH "AUTHORS"
+.TP
+Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
--
1.7.1
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH V8 libibverbs 7/7] Add XRC sample application
[not found] ` <1374741488-30895-1-git-send-email-yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
` (5 preceding siblings ...)
2013-07-25 8:38 ` [PATCH V8 libibverbs 6/7] XRC man pages Yishai Hadas
@ 2013-07-25 8:38 ` Yishai Hadas
6 siblings, 0 replies; 15+ messages in thread
From: Yishai Hadas @ 2013-07-25 8:38 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA, roland-BHEL68pLQRGGvPXPguhicg
Cc: ogerlitz-VPRAkNaXOzVWk0Htik3J/w, tzahio-VPRAkNaXOzVWk0Htik3J/w,
yishaih-VPRAkNaXOzVWk0Htik3J/w, Jay Sternberg, Sean Hefty
From: Jay Sternberg <jay.e.sternberg-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Signed-off-by: Jay Sternberg <jay.e.sternberg-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Signed-off-by: Yishai Hadas <yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
---
Makefile.am | 4 +-
examples/xsrq_pingpong.c | 890 ++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 893 insertions(+), 1 deletions(-)
create mode 100644 examples/xsrq_pingpong.c
diff --git a/Makefile.am b/Makefile.am
index aa6097a..a6767de 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -18,7 +18,7 @@ src_libibverbs_la_DEPENDENCIES = $(srcdir)/src/libibverbs.map
bin_PROGRAMS = examples/ibv_devices examples/ibv_devinfo \
examples/ibv_asyncwatch examples/ibv_rc_pingpong examples/ibv_uc_pingpong \
- examples/ibv_ud_pingpong examples/ibv_srq_pingpong
+ examples/ibv_ud_pingpong examples/ibv_srq_pingpong examples/ibv_xsrq_pingpong
examples_ibv_devices_SOURCES = examples/device_list.c
examples_ibv_devices_LDADD = $(top_builddir)/src/libibverbs.la
examples_ibv_devinfo_SOURCES = examples/devinfo.c
@@ -31,6 +31,8 @@ examples_ibv_ud_pingpong_SOURCES = examples/ud_pingpong.c examples/pingpong.c
examples_ibv_ud_pingpong_LDADD = $(top_builddir)/src/libibverbs.la
examples_ibv_srq_pingpong_SOURCES = examples/srq_pingpong.c examples/pingpong.c
examples_ibv_srq_pingpong_LDADD = $(top_builddir)/src/libibverbs.la
+examples_ibv_xsrq_pingpong_SOURCES = examples/xsrq_pingpong.c examples/pingpong.c
+examples_ibv_xsrq_pingpong_LDADD = $(top_builddir)/src/libibverbs.la
examples_ibv_asyncwatch_SOURCES = examples/asyncwatch.c
examples_ibv_asyncwatch_LDADD = $(top_builddir)/src/libibverbs.la
diff --git a/examples/xsrq_pingpong.c b/examples/xsrq_pingpong.c
new file mode 100644
index 0000000..b5fc1b6
--- /dev/null
+++ b/examples/xsrq_pingpong.c
@@ -0,0 +1,890 @@
+/*
+ * Copyright (c) 2005 Topspin Communications. All rights reserved.
+ * Copyright (c) 2011 Intel Corporation, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <stdio.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <netdb.h>
+#include <malloc.h>
+#include <getopt.h>
+#include <arpa/inet.h>
+#include <time.h>
+
+#include "pingpong.h"
+
+#define MSG_FORMAT "%04x:%06x:%06x:%06x:%04x"
+#define MSG_SIZE 31
+#define MSG_SSCAN "%x:%x:%x:%x:%x"
+#define ADDR_FORMAT \
+ "%8s: LID %04x, QPN RECV %06x SEND %06x, PSN %06x, SRQN %04x\n"
+static int page_size;
+
+struct pingpong_dest {
+ int lid;
+ int recv_qpn;
+ int send_qpn;
+ int recv_psn;
+ int send_psn;
+ int srqn;
+ int pp_cnt;
+};
+
+struct pingpong_context {
+ struct ibv_context *context;
+ struct ibv_comp_channel *channel;
+ struct ibv_pd *pd;
+ struct ibv_mr *mr;
+ struct ibv_cq *send_cq;
+ struct ibv_cq *recv_cq;
+ struct ibv_srq *srq;
+ struct ibv_xrcd *xrcd;
+ struct ibv_qp **recv_qp;
+ struct ibv_qp **send_qp;
+ struct pingpong_dest *rem_dest;
+ void *buf;
+ int lid;
+ int sl;
+ enum ibv_mtu mtu;
+ int ib_port;
+ int fd;
+ int size;
+ int num_clients;
+ int num_tests;
+ int use_event;
+};
+
+struct pingpong_context ctx;
+
+
+static int open_device(char *ib_devname)
+{
+ struct ibv_device **dev_list;
+ int i = 0;
+
+ dev_list = ibv_get_device_list(NULL);
+ if (!dev_list) {
+ fprintf(stderr, "Failed to get IB devices list");
+ return -1;
+ }
+
+ if (ib_devname) {
+ for (; dev_list[i]; ++i) {
+ if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))
+ break;
+ }
+ }
+ if (!dev_list[i]) {
+ fprintf(stderr, "IB device %s not found\n",
+ ib_devname ? ib_devname : "");
+ return -1;
+ }
+
+ ctx.context = ibv_open_device(dev_list[i]);
+ if (!ctx.context) {
+ fprintf(stderr, "Couldn't get context for %s\n",
+ ibv_get_device_name(dev_list[i]));
+ return -1;
+ }
+
+ ibv_free_device_list(dev_list);
+ return 0;
+}
+
+static int create_qps(void)
+{
+ struct ibv_qp_init_attr_ex init;
+ struct ibv_qp_attr mod;
+ int i;
+
+ for (i = 0; i < ctx.num_clients; ++i) {
+
+ memset(&init, 0, sizeof init);
+ init.qp_type = IBV_QPT_XRC_RECV;
+ init.comp_mask = IBV_QP_INIT_ATTR_XRCD;
+ init.xrcd = ctx.xrcd;
+
+ ctx.recv_qp[i] = ibv_create_qp_ex(ctx.context, &init);
+ if (!ctx.recv_qp[i]) {
+ fprintf(stderr, "Couldn't create recv QP[%d] errno %d\n",
+ i, errno);
+ return 1;
+ }
+
+ mod.qp_state = IBV_QPS_INIT;
+ mod.pkey_index = 0;
+ mod.port_num = ctx.ib_port;
+ mod.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ;
+
+ if (ibv_modify_qp(ctx.recv_qp[i], &mod,
+ IBV_QP_STATE | IBV_QP_PKEY_INDEX |
+ IBV_QP_PORT | IBV_QP_ACCESS_FLAGS)) {
+ fprintf(stderr, "Failed to modify recv QP[%d] to INIT\n", i);
+ return 1;
+ }
+
+ memset(&init, 0, sizeof init);
+ init.qp_type = IBV_QPT_XRC_SEND;
+ init.send_cq = ctx.send_cq;
+ init.cap.max_send_wr = ctx.num_clients * ctx.num_tests;
+ init.cap.max_send_sge = 1;
+ init.comp_mask = IBV_QP_INIT_ATTR_PD;
+ init.pd = ctx.pd;
+
+ ctx.send_qp[i] = ibv_create_qp_ex(ctx.context, &init);
+ if (!ctx.send_qp[i]) {
+ fprintf(stderr, "Couldn't create send QP[%d] errno %d\n",
+ i, errno);
+ return 1;
+ }
+
+ mod.qp_state = IBV_QPS_INIT;
+ mod.pkey_index = 0;
+ mod.port_num = ctx.ib_port;
+ mod.qp_access_flags = 0;
+
+ if (ibv_modify_qp(ctx.send_qp[i], &mod,
+ IBV_QP_STATE | IBV_QP_PKEY_INDEX |
+ IBV_QP_PORT | IBV_QP_ACCESS_FLAGS)) {
+ fprintf(stderr, "Failed to modify send QP[%d] to INIT\n", i);
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+static int pp_init_ctx(char *ib_devname)
+{
+ struct ibv_srq_init_attr_ex attr;
+ struct ibv_xrcd_init_attr xrcd_attr;
+
+ ctx.recv_qp = calloc(ctx.num_clients, sizeof *ctx.recv_qp);
+ ctx.send_qp = calloc(ctx.num_clients, sizeof *ctx.send_qp);
+ ctx.rem_dest = calloc(ctx.num_clients, sizeof *ctx.rem_dest);
+ if (!ctx.recv_qp || !ctx.send_qp || !ctx.rem_dest)
+ return 1;
+
+ if (open_device(ib_devname)) {
+ fprintf(stderr, "Failed to open device\n");
+ return 1;
+ }
+
+ ctx.lid = pp_get_local_lid(ctx.context, ctx.ib_port);
+ if (ctx.lid < 0) {
+ fprintf(stderr, "Failed to get SLID\n");
+ return 1;
+ }
+
+ ctx.buf = memalign(page_size, ctx.size);
+ if (!ctx.buf) {
+ fprintf(stderr, "Couldn't allocate work buf.\n");
+ return 1;
+ }
+
+ memset(ctx.buf, 0, ctx.size);
+
+ if (ctx.use_event) {
+ ctx.channel = ibv_create_comp_channel(ctx.context);
+ if (!ctx.channel) {
+ fprintf(stderr, "Couldn't create completion channel\n");
+ return 1;
+ }
+ }
+
+ ctx.pd = ibv_alloc_pd(ctx.context);
+ if (!ctx.pd) {
+ fprintf(stderr, "Couldn't allocate PD\n");
+ return 1;
+ }
+
+ ctx.mr = ibv_reg_mr(ctx.pd, ctx.buf, ctx.size, IBV_ACCESS_LOCAL_WRITE);
+ if (!ctx.mr) {
+ fprintf(stderr, "Couldn't register MR\n");
+ return 1;
+ }
+
+ ctx.fd = open("/tmp/xrc_domain", O_RDONLY | O_CREAT, S_IRUSR | S_IRGRP);
+ if (ctx.fd < 0) {
+ fprintf(stderr,
+ "Couldn't create the file for the XRC Domain "
+ "but not stopping %d\n", errno);
+ ctx.fd = -1;
+ }
+
+ memset(&xrcd_attr, 0, sizeof xrcd_attr);
+ xrcd_attr.comp_mask = IBV_XRCD_INIT_ATTR_FD | IBV_XRCD_INIT_ATTR_OFLAGS;
+ xrcd_attr.fd = ctx.fd;
+ xrcd_attr.oflags = O_CREAT;
+ ctx.xrcd = ibv_open_xrcd(ctx.context, &xrcd_attr);
+ if (!ctx.xrcd) {
+ fprintf(stderr, "Couldn't Open the XRC Domain %d\n", errno);
+ return 1;
+ }
+
+ ctx.recv_cq = ibv_create_cq(ctx.context, ctx.num_clients, &ctx.recv_cq,
+ ctx.channel, 0);
+ if (!ctx.recv_cq) {
+ fprintf(stderr, "Couldn't create recv CQ\n");
+ return 1;
+ }
+
+ if (ctx.use_event) {
+ if (ibv_req_notify_cq(ctx.recv_cq, 0)) {
+ fprintf(stderr, "Couldn't request CQ notification\n");
+ return 1;
+ }
+ }
+
+ ctx.send_cq = ibv_create_cq(ctx.context, ctx.num_clients, NULL, NULL, 0);
+ if (!ctx.send_cq) {
+ fprintf(stderr, "Couldn't create send CQ\n");
+ return 1;
+ }
+
+ memset(&attr, 0, sizeof attr);
+ attr.attr.max_wr = ctx.num_clients;
+ attr.attr.max_sge = 1;
+ attr.comp_mask = IBV_SRQ_INIT_ATTR_TYPE | IBV_SRQ_INIT_ATTR_XRCD |
+ IBV_SRQ_INIT_ATTR_CQ | IBV_SRQ_INIT_ATTR_PD;
+ attr.srq_type = IBV_SRQT_XRC;
+ attr.xrcd = ctx.xrcd;
+ attr.cq = ctx.recv_cq;
+ attr.pd = ctx.pd;
+
+ ctx.srq = ibv_create_srq_ex(ctx.context, &attr);
+ if (!ctx.srq) {
+ fprintf(stderr, "Couldn't create SRQ\n");
+ return 1;
+ }
+
+ if (create_qps())
+ return 1;
+
+ return 0;
+}
+
+static int send_local_dest(int sockfd, int index)
+{
+ char msg[MSG_SIZE];
+ uint32_t srq_num;
+
+ ctx.rem_dest[index].recv_psn = lrand48() & 0xffffff;
+ if (ibv_get_srq_num(ctx.srq, &srq_num)) {
+ fprintf(stderr, "Couldn't get SRQ num\n");
+ return -1;
+ }
+
+ printf(ADDR_FORMAT, "local", ctx.lid, ctx.recv_qp[index]->qp_num,
+ ctx.send_qp[index]->qp_num, ctx.rem_dest[index].recv_psn,
+ srq_num);
+
+ sprintf(msg, MSG_FORMAT, ctx.lid, ctx.recv_qp[index]->qp_num,
+ ctx.send_qp[index]->qp_num, ctx.rem_dest[index].recv_psn,
+ srq_num);
+
+ if (write(sockfd, msg, MSG_SIZE) != MSG_SIZE) {
+ fprintf(stderr, "Couldn't send local address\n");
+ return -1;
+ }
+
+ return 0;
+}
+
+static int recv_remote_dest(int sockfd, int index)
+{
+ struct pingpong_dest *rem_dest;
+ char msg[MSG_SIZE];
+ int n = 0, r;
+
+ while (n < MSG_SIZE) {
+ r = read(sockfd, msg + n, MSG_SIZE - n);
+ if (r < 0) {
+ perror("client read");
+ fprintf(stderr,
+ "%d/%d: Couldn't read remote address [%d]\n",
+ n, MSG_SIZE, index);
+ return -1;
+ }
+ n += r;
+ }
+
+ rem_dest = &ctx.rem_dest[index];
+ sscanf(msg, MSG_SSCAN, &rem_dest->lid, &rem_dest->recv_qpn,
+ &rem_dest->send_qpn, &rem_dest->send_psn, &rem_dest->srqn);
+
+ printf(ADDR_FORMAT, "remote", rem_dest->lid, rem_dest->recv_qpn,
+ rem_dest->send_qpn, rem_dest->send_psn, rem_dest->srqn);
+
+ return 0;
+}
+
+static int connect_qps(int index)
+{
+ struct ibv_qp_attr attr;
+
+ memset(&attr, 0, sizeof attr);
+ attr.qp_state = IBV_QPS_RTR;
+ attr.dest_qp_num = ctx.rem_dest[index].send_qpn;
+ attr.path_mtu = ctx.mtu;
+ attr.rq_psn = ctx.rem_dest[index].send_psn;
+ attr.min_rnr_timer = 12;
+ attr.ah_attr.dlid = ctx.rem_dest[index].lid;
+ attr.ah_attr.sl = ctx.sl;
+ attr.ah_attr.port_num = ctx.ib_port;
+
+ if (ibv_modify_qp(ctx.recv_qp[index], &attr,
+ IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU |
+ IBV_QP_DEST_QPN | IBV_QP_RQ_PSN |
+ IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER)) {
+ fprintf(stderr, "Failed to modify send QP[%d] to RTR\n", index);
+ return 1;
+ }
+
+ memset(&attr, 0, sizeof attr);
+ attr.qp_state = IBV_QPS_RTS;
+ attr.timeout = 14;
+ attr.sq_psn = ctx.rem_dest[index].recv_psn;
+
+ if (ibv_modify_qp(ctx.recv_qp[index], &attr,
+ IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_SQ_PSN)) {
+ fprintf(stderr, "Failed to modify recv QP[%d] to RTS\n", index);
+ return 1;
+ }
+
+ memset(&attr, 0, sizeof attr);
+ attr.qp_state = IBV_QPS_RTR;
+ attr.dest_qp_num = ctx.rem_dest[index].recv_qpn;
+ attr.path_mtu = ctx.mtu;
+ attr.rq_psn = ctx.rem_dest[index].send_psn;
+ attr.ah_attr.dlid = ctx.rem_dest[index].lid;
+ attr.ah_attr.sl = ctx.sl;
+ attr.ah_attr.port_num = ctx.ib_port;
+
+ if (ibv_modify_qp(ctx.send_qp[index], &attr,
+ IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU |
+ IBV_QP_DEST_QPN | IBV_QP_RQ_PSN)) {
+ fprintf(stderr, "Failed to modify send QP[%d] to RTR\n", index);
+ return 1;
+ }
+
+ memset(&attr, 0, sizeof attr);
+ attr.qp_state = IBV_QPS_RTS;
+ attr.timeout = 14;
+ attr.retry_cnt = 7;
+ attr.rnr_retry = 7;
+ attr.sq_psn = ctx.rem_dest[index].recv_psn;
+
+ if (ibv_modify_qp(ctx.send_qp[index], &attr,
+ IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_SQ_PSN |
+ IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_MAX_QP_RD_ATOMIC)) {
+ fprintf(stderr, "Failed to modify send QP[%d] to RTS\n", index);
+ return 1;
+ }
+
+ return 0;
+}
+
+static int pp_client_connect(const char *servername, int port)
+{
+ struct addrinfo *res, *t;
+ char *service;
+ int ret;
+ int sockfd = -1;
+ struct addrinfo hints = {
+ .ai_family = AF_UNSPEC,
+ .ai_socktype = SOCK_STREAM
+ };
+
+ if (asprintf(&service, "%d", port) < 0)
+ return 1;
+
+ ret = getaddrinfo(servername, service, &hints, &res);
+ if (ret < 0) {
+ fprintf(stderr, "%s for %s:%d\n", gai_strerror(ret), servername, port);
+ free(service);
+ return 1;
+ }
+
+ for (t = res; t; t = t->ai_next) {
+ sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
+ if (sockfd >= 0) {
+ if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
+ break;
+ close(sockfd);
+ sockfd = -1;
+ }
+ }
+
+ freeaddrinfo(res);
+ free(service);
+
+ if (sockfd < 0) {
+ fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);
+ return 1;
+ }
+
+ if (send_local_dest(sockfd, 0))
+ return 1;
+
+ if (recv_remote_dest(sockfd, 0))
+ return 1;
+
+ if (connect_qps(0))
+ return 1;
+
+ shutdown(sockfd, SHUT_RDWR);
+ close(sockfd);
+ return 0;
+}
+
+static int pp_server_connect(int port)
+{
+ struct addrinfo *res, *t;
+ char *service;
+ int ret, i, n;
+ int sockfd = -1, connfd;
+ struct addrinfo hints = {
+ .ai_flags = AI_PASSIVE,
+ .ai_family = AF_UNSPEC,
+ .ai_socktype = SOCK_STREAM
+ };
+
+ if (asprintf(&service, "%d", port) < 0)
+ return 1;
+
+ ret = getaddrinfo(NULL, service, &hints, &res);
+ if (ret < 0) {
+ fprintf(stderr, "%s for port %d\n", gai_strerror(ret), port);
+ free(service);
+ return 1;
+ }
+
+ for (t = res; t; t = t->ai_next) {
+ sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
+ if (sockfd >= 0) {
+ n = 1;
+ setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);
+ if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
+ break;
+ close(sockfd);
+ sockfd = -1;
+ }
+ }
+
+ freeaddrinfo(res);
+ free(service);
+
+ if (sockfd < 0) {
+ fprintf(stderr, "Couldn't listen to port %d\n", port);
+ return 1;
+ }
+
+ listen(sockfd, ctx.num_clients);
+
+ for (i = 0; i < ctx.num_clients; i++) {
+ connfd = accept(sockfd, NULL, 0);
+ if (connfd < 0) {
+ fprintf(stderr, "accept() failed for client %d\n", i);
+ return 1;
+ }
+
+ if (recv_remote_dest(connfd, i))
+ return 1;
+
+ if (send_local_dest(connfd, i))
+ return 1;
+
+ if (connect_qps(i))
+ return 1;
+
+ shutdown(connfd, SHUT_RDWR);
+ close(connfd);
+ }
+
+ close(sockfd);
+ return 0;
+}
+
+
+static int pp_close_ctx(void)
+{
+ int i;
+
+ for (i = 0; i < ctx.num_clients; ++i) {
+ /*
+ * Receive QPs may be used by other clients. Leave them, they
+ * will be cleaned up when the xrcd is destroyed.
+ */
+ if (ibv_destroy_qp(ctx.send_qp[i])) {
+ fprintf(stderr, "Couldn't destroy INI QP[%d]\n", i);
+ return 1;
+ }
+ }
+
+ if (ibv_destroy_srq(ctx.srq)) {
+ fprintf(stderr, "Couldn't destroy SRQ\n");
+ return 1;
+ }
+
+ if (!ctx.xrcd && ibv_close_xrcd(ctx.xrcd)) {
+ fprintf(stderr, "Couldn't close the XRC Domain\n");
+ return 1;
+ }
+ if (ctx.fd >= 0 && close(ctx.fd)) {
+ fprintf(stderr, "Couldn't close the file for the XRC Domain\n");
+ return 1;
+ }
+
+ if (ibv_destroy_cq(ctx.send_cq)) {
+ fprintf(stderr, "Couldn't destroy send CQ\n");
+ return 1;
+ }
+
+ if (ibv_destroy_cq(ctx.recv_cq)) {
+ fprintf(stderr, "Couldn't destroy recv CQ\n");
+ return 1;
+ }
+
+ if (ibv_dereg_mr(ctx.mr)) {
+ fprintf(stderr, "Couldn't deregister MR\n");
+ return 1;
+ }
+
+ if (ibv_dealloc_pd(ctx.pd)) {
+ fprintf(stderr, "Couldn't deallocate PD\n");
+ return 1;
+ }
+
+ if (ctx.channel) {
+ if (ibv_destroy_comp_channel(ctx.channel)) {
+ fprintf(stderr,
+ "Couldn't destroy completion channel\n");
+ return 1;
+ }
+ }
+
+ if (ibv_close_device(ctx.context)) {
+ fprintf(stderr, "Couldn't release context\n");
+ return 1;
+ }
+
+ free(ctx.buf);
+ free(ctx.rem_dest);
+ free(ctx.send_qp);
+ free(ctx.recv_qp);
+ return 0;
+}
+
+static int pp_post_recv(int cnt)
+{
+ struct ibv_sge sge;
+ struct ibv_recv_wr wr, *bad_wr;
+
+ sge.addr = (uintptr_t) ctx.buf;
+ sge.length = ctx.size;
+ sge.lkey = ctx.mr->lkey;
+
+ wr.next = NULL;
+ wr.wr_id = (uintptr_t) &ctx;
+ wr.sg_list = &sge;
+ wr.num_sge = 1;
+
+ while (cnt--) {
+ if (ibv_post_srq_recv(ctx.srq, &wr, &bad_wr)) {
+ fprintf(stderr, "Failed to post receive to SRQ\n");
+ return 1;
+ }
+ }
+ return 0;
+}
+
+/*
+ * Send to each client round robin on each set of xrc send/recv qp.
+ * Generate a completion on the last send.
+ */
+static int pp_post_send(int index)
+{
+ struct ibv_sge sge;
+ struct ibv_send_wr wr, *bad_wr;
+ int qpi;
+
+ sge.addr = (uintptr_t) ctx.buf;
+ sge.length = ctx.size;
+ sge.lkey = ctx.mr->lkey;
+
+ wr.wr_id = (uintptr_t) index;
+ wr.next = NULL;
+ wr.sg_list = &sge;
+ wr.num_sge = 1;
+ wr.opcode = IBV_WR_SEND;
+ wr.qp_type.xrc.remote_srqn = ctx.rem_dest[index].srqn;
+
+ qpi = (index + ctx.rem_dest[index].pp_cnt) % ctx.num_clients;
+ wr.send_flags = (++ctx.rem_dest[index].pp_cnt >= ctx.num_tests) ?
+ IBV_SEND_SIGNALED : 0;
+
+ return ibv_post_send(ctx.send_qp[qpi], &wr, &bad_wr);
+}
+
+static int find_qp(int qpn)
+{
+ int i;
+
+ if (ctx.num_clients == 1)
+ return 0;
+
+ for (i = 0; i < ctx.num_clients; ++i)
+ if (ctx.recv_qp[i]->qp_num == qpn)
+ return i;
+
+ fprintf(stderr, "Unable to find qp %x\n", qpn);
+ return 0;
+}
+
+static int get_cq_event(void)
+{
+ struct ibv_cq *ev_cq;
+ void *ev_ctx;
+
+ if (ibv_get_cq_event(ctx.channel, &ev_cq, &ev_ctx)) {
+ fprintf(stderr, "Failed to get cq_event\n");
+ return 1;
+ }
+
+ if (ev_cq != ctx.recv_cq) {
+ fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
+ return 1;
+ }
+
+ if (ibv_req_notify_cq(ctx.recv_cq, 0)) {
+ fprintf(stderr, "Couldn't request CQ notification\n");
+ return 1;
+ }
+
+ return 0;
+}
+
+static void init(void)
+{
+ srand48(getpid() * time(NULL));
+
+ ctx.size = 4096;
+ ctx.ib_port = 1;
+ ctx.num_clients = 1;
+ ctx.num_tests = 5;
+ ctx.mtu = IBV_MTU_2048;
+ ctx.sl = 0;
+}
+
+static void usage(const char *argv0)
+{
+ printf("Usage:\n");
+ printf(" %s start a server and wait for connection\n", argv0);
+ printf(" %s <host> connect to server at <host>\n", argv0);
+ printf("\n");
+ printf("Options:\n");
+ printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n");
+ printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n");
+ printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n");
+ printf(" -s, --size=<size> size of message to exchange (default 4096)\n");
+ printf(" -m, --mtu=<size> path MTU (default 2048)\n");
+ printf(" -c, --clients=<n> number of clients (on server only, default 1)\n");
+ printf(" -n, --num_tests=<n> number of tests per client (default 5)\n");
+ printf(" -l, --sl=<sl> service level value\n");
+ printf(" -e, --events sleep on CQ events (default poll)\n");
+}
+
+int main(int argc, char *argv[])
+{
+ char *ib_devname = NULL;
+ char *servername = NULL;
+ int port = 18515;
+ int i, total, cnt = 0;
+ int ne, qpi, num_cq_events = 0;
+ struct ibv_wc wc;
+
+ init();
+ while (1) {
+ int c;
+
+ static struct option long_options[] = {
+ { .name = "port", .has_arg = 1, .val = 'p' },
+ { .name = "ib-dev", .has_arg = 1, .val = 'd' },
+ { .name = "ib-port", .has_arg = 1, .val = 'i' },
+ { .name = "size", .has_arg = 1, .val = 's' },
+ { .name = "mtu", .has_arg = 1, .val = 'm' },
+ { .name = "clients", .has_arg = 1, .val = 'c' },
+ { .name = "num_tests", .has_arg = 1, .val = 'n' },
+ { .name = "sl", .has_arg = 1, .val = 'l' },
+ { .name = "events", .has_arg = 0, .val = 'e' },
+ { 0 }
+ };
+
+ c = getopt_long(argc, argv, "p:d:i:s:m:c:n:l:e", long_options,
+ NULL);
+ if (c == -1)
+ break;
+
+ switch (c) {
+ case 'p':
+ port = strtol(optarg, NULL, 0);
+ if (port < 0 || port > 65535) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
+ case 'd':
+ ib_devname = strdupa(optarg);
+ break;
+ case 'i':
+ ctx.ib_port = strtol(optarg, NULL, 0);
+ if (ctx.ib_port < 0) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
+ case 's':
+ ctx.size = strtol(optarg, NULL, 0);
+ break;
+ case 'm':
+ ctx.mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0));
+ if (ctx.mtu < 0) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
+ case 'c':
+ ctx.num_clients = strtol(optarg, NULL, 0);
+ break;
+ case 'n':
+ ctx.num_tests = strtol(optarg, NULL, 0);
+ break;
+ case 'l':
+ ctx.sl = strtol(optarg, NULL, 0);
+ break;
+ case 'e':
+ ctx.use_event = 1;
+ break;
+ default:
+ usage(argv[0]);
+ return 1;
+ }
+ }
+
+ if (optind == argc - 1) {
+ servername = strdupa(argv[optind]);
+ ctx.num_clients = 1;
+ } else if (optind < argc) {
+ usage(argv[0]);
+ return 1;
+ }
+
+ page_size = sysconf(_SC_PAGESIZE);
+
+ if (pp_init_ctx(ib_devname))
+ return 1;
+
+ if (pp_post_recv(ctx.num_clients)) {
+ fprintf(stderr, "Couldn't post receives\n");
+ return 1;
+ }
+
+ if (servername) {
+ if (pp_client_connect(servername, port))
+ return 1;
+ } else {
+ if (pp_server_connect(port))
+ return 1;
+
+ for (i = 0; i < ctx.num_clients; i++)
+ pp_post_send(i);
+ }
+
+ total = ctx.num_clients * ctx.num_tests;
+ while (cnt < total) {
+ if (ctx.use_event) {
+ if (get_cq_event())
+ return 1;
+
+ ++num_cq_events;
+ }
+
+ do {
+ ne = ibv_poll_cq(ctx.recv_cq, 1, &wc);
+ if (ne < 0) {
+ fprintf(stderr, "Error polling cq %d\n", ne);
+ return 1;
+ } else if (ne == 0) {
+ break;
+ }
+
+ if (wc.status) {
+ fprintf(stderr, "Work completion error %d\n", wc.status);
+ return 1;
+ }
+
+ pp_post_recv(ne);
+ qpi = find_qp(wc.qp_num);
+ if (ctx.rem_dest[qpi].pp_cnt < ctx.num_tests)
+ pp_post_send(qpi);
+ cnt += ne;
+ } while (ne > 0);
+ }
+
+ for (cnt = 0; cnt < ctx.num_clients; cnt += ne) {
+ ne = ibv_poll_cq(ctx.send_cq, 1, &wc);
+ if (ne < 0) {
+ fprintf(stderr, "Error polling cq %d\n", ne);
+ return 1;
+ }
+ }
+
+ if (ctx.use_event)
+ ibv_ack_cq_events(ctx.recv_cq, num_cq_events);
+
+ if (pp_close_ctx())
+ return 1;
+
+ printf("success\n");
+ return 0;
+}
--
1.7.1
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related [flat|nested] 15+ messages in thread