* [Bug 178581] New: v4.9-rc1: Deadlock while unloading IPoIB
From: bugzilla-daemon-590EEB7GvNiWaY/ihj7yzEB+6BGkLq7r @ 2016-10-19 20:31 UTC (permalink / raw)
To: linux-rdma-u79uwXL29TY76Z2rM5mHXA
https://bugzilla.kernel.org/show_bug.cgi?id=178581
Bug ID: 178581
Summary: v4.9-rc1: Deadlock while unloading IPoIB
Product: Drivers
Version: 2.5
Kernel Version: v4.9-rc1
Hardware: x86-64
OS: Linux
Tree: Mainline
Status: NEW
Severity: normal
Priority: P1
Component: Infiniband/RDMA
Assignee: drivers_infiniband-rdma-ztI5WcYan/vQLgFONoPN62D2FQJk+8+b@public.gmane.org
Reporter: bvanassche-HInyCGIudOg@public.gmane.org
Regression: No
Created attachment 241991
--> https://bugzilla.kernel.org/attachment.cgi?id=241991&action=edit
Lockdep output
The output produced by lockdep has been attached. Call trace from that
attachment:
Call Trace:
[<ffffffff81329695>] dump_stack+0x68/0x93
[<ffffffff810baafe>] print_circular_bug+0x1be/0x210
[<ffffffff810bd31c>] __lock_acquire+0x10fc/0x1270
[<ffffffff810bd8b9>] lock_acquire+0xe9/0x1d0
[<ffffffff8126275a>] __kernfs_remove+0x24a/0x310
[<ffffffff812632ae>] kernfs_remove+0x1e/0x30
[<ffffffff8126559c>] sysfs_remove_dir+0x4c/0x80
[<ffffffff8132bec8>] kobject_del+0x18/0x50
[<ffffffff81450fb0>] device_del+0x190/0x240
[<ffffffff81550a27>] netdev_unregister_kobject+0x57/0x60
[<ffffffff81527df0>] rollback_registered_many+0x220/0x370
[<ffffffff81527f6c>] rollback_registered+0x2c/0x40
[<ffffffff81528e54>] unregister_netdevice_queue+0x44/0x90
[<ffffffff81528eb7>] unregister_netdev+0x17/0x20
[<ffffffffa02bdc47>] ipoib_remove_one+0x97/0xe0 [ib_ipoib]
[<ffffffffa032c539>] ib_unregister_device+0xa9/0x150 [ib_core]
[<ffffffffa036d7e5>] mlx4_ib_remove+0x65/0x250 [mlx4_ib]
[<ffffffffa025131e>] mlx4_remove_device+0x6e/0x80 [mlx4_core]
[<ffffffffa0251738>] mlx4_unregister_device+0x48/0x90 [mlx4_core]
[<ffffffffa025364d>] mlx4_unload_one+0x7d/0x310 [mlx4_core]
[<ffffffffa02539cd>] mlx4_shutdown+0x5d/0x70 [mlx4_core]
[<ffffffff8136c768>] pci_device_shutdown+0x28/0x60
[<ffffffff81452f70>] device_shutdown+0xd0/0x1d0
[<ffffffff8108c7e1>] kernel_restart_prepare+0x31/0x40
[<ffffffff8108c88d>] kernel_restart+0xd/0x50
[<ffffffff8108cad9>] SYSC_reboot+0xf9/0x1e0
[<ffffffff8108cc09>] SyS_reboot+0x9/0x10
[<ffffffff8163ed2a>] entry_SYSCALL_64_fastpath+0x18/0xad
--
You are receiving this mail because:
You are watching the assignee of the bug.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* [PATCH] i40iw: Enable message packing
From: Henry Orosco @ 2016-10-19 20:32 UTC (permalink / raw)
To: dledford-H+wXaHxf7aLQT0dZR+AlfA
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA,
e1000-rdma-5NWGOfrQmneRv+LV9MX5uipxlwaOVQ5f, Henry Orosco
Remove the parameter to disable message packing and
always enable it.
Signed-off-by: Shiraz Saleem <shiraz.saleem-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Signed-off-by: Henry Orosco <henry.orosco-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
drivers/infiniband/hw/i40iw/i40iw_ctrl.c | 5 -----
drivers/infiniband/hw/i40iw/i40iw_main.c | 2 +-
drivers/infiniband/hw/i40iw/i40iw_type.h | 2 +-
3 files changed, 2 insertions(+), 7 deletions(-)
diff --git a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c b/drivers/infiniband/hw/i40iw/i40iw_ctrl.c
index 31c4a0c..6c6a1ef 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_ctrl.c
@@ -429,12 +429,10 @@ static enum i40iw_status_code i40iw_sc_cqp_init(struct i40iw_sc_cqp *cqp,
/**
* i40iw_sc_cqp_create - create cqp during bringup
* @cqp: struct for cqp hw
- * @disable_pfpdus: if pfpdu to be disabled
* @maj_err: If error, major err number
* @min_err: If error, minor err number
*/
static enum i40iw_status_code i40iw_sc_cqp_create(struct i40iw_sc_cqp *cqp,
- bool disable_pfpdus,
u16 *maj_err,
u16 *min_err)
{
@@ -453,9 +451,6 @@ static enum i40iw_status_code i40iw_sc_cqp_create(struct i40iw_sc_cqp *cqp,
temp = LS_64(cqp->hw_sq_size, I40IW_CQPHC_SQSIZE) |
LS_64(cqp->struct_ver, I40IW_CQPHC_SVER);
- if (disable_pfpdus)
- temp |= LS_64(1, I40IW_CQPHC_DISABLE_PFPDUS);
-
set_64bit_val(cqp->host_ctx, 0, temp);
set_64bit_val(cqp->host_ctx, 8, cqp->sq_pa);
temp = LS_64(cqp->enabled_vf_count, I40IW_CQPHC_ENABLED_VFS) |
diff --git a/drivers/infiniband/hw/i40iw/i40iw_main.c b/drivers/infiniband/hw/i40iw/i40iw_main.c
index e6dafc7..2a4ca8a 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_main.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_main.c
@@ -603,7 +603,7 @@ static enum i40iw_status_code i40iw_create_cqp(struct i40iw_device *iwdev)
i40iw_pr_err("cqp init status %d\n", status);
goto exit;
}
- status = dev->cqp_ops->cqp_create(dev->cqp, true, &maj_err, &min_err);
+ status = dev->cqp_ops->cqp_create(dev->cqp, &maj_err, &min_err);
if (status) {
i40iw_pr_err("cqp create status %d maj_err %d min_err %d\n",
status, maj_err, min_err);
diff --git a/drivers/infiniband/hw/i40iw/i40iw_type.h b/drivers/infiniband/hw/i40iw/i40iw_type.h
index b6f448a..d1847e6 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_type.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_type.h
@@ -988,7 +988,7 @@ struct i40iw_cqp_query_fpm_values {
struct i40iw_cqp_ops {
enum i40iw_status_code (*cqp_init)(struct i40iw_sc_cqp *,
struct i40iw_cqp_init_info *);
- enum i40iw_status_code (*cqp_create)(struct i40iw_sc_cqp *, bool, u16 *, u16 *);
+ enum i40iw_status_code (*cqp_create)(struct i40iw_sc_cqp *, u16 *, u16 *);
void (*cqp_post_sq)(struct i40iw_sc_cqp *);
u64 *(*cqp_get_next_send_wqe)(struct i40iw_sc_cqp *, u64 scratch);
enum i40iw_status_code (*cqp_destroy)(struct i40iw_sc_cqp *);
--
1.8.3.1
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH] i40iw: Remove workaround for pre-production errata
From: Henry Orosco @ 2016-10-19 20:33 UTC (permalink / raw)
To: dledford-H+wXaHxf7aLQT0dZR+AlfA
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA,
e1000-rdma-5NWGOfrQmneRv+LV9MX5uipxlwaOVQ5f, Henry Orosco
Pre-production silicon incorrectly truncates 4 bytes of the MPA
packet in UDP loopback case. Remove the workaround as it is no
longer necessary.
Signed-off-by: Shiraz Saleem <shiraz.saleem-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Signed-off-by: Henry Orosco <henry.orosco-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
drivers/infiniband/hw/i40iw/i40iw_cm.c | 26 +++-----------------------
drivers/infiniband/hw/i40iw/i40iw_cm.h | 2 --
drivers/infiniband/hw/i40iw/i40iw_utils.c | 2 +-
3 files changed, 4 insertions(+), 26 deletions(-)
diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.c b/drivers/infiniband/hw/i40iw/i40iw_cm.c
index 460a367..e202ff0 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_cm.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_cm.c
@@ -361,15 +361,6 @@ static void i40iw_cleanup_retrans_entry(struct i40iw_cm_node *cm_node)
spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
}
-static bool is_remote_ne020_or_chelsio(struct i40iw_cm_node *cm_node)
-{
- if ((cm_node->rem_mac[0] == 0x0) &&
- (((cm_node->rem_mac[1] == 0x12) && (cm_node->rem_mac[2] == 0x55)) ||
- ((cm_node->rem_mac[1] == 0x07 && (cm_node->rem_mac[2] == 0x43)))))
- return true;
- return false;
-}
-
/**
* i40iw_form_cm_frame - get a free packet and build frame
* @cm_node: connection's node ionfo to use in frame
@@ -410,11 +401,8 @@ static struct i40iw_puda_buf *i40iw_form_cm_frame(struct i40iw_cm_node *cm_node,
if (hdr)
hdr_len = hdr->size;
- if (pdata) {
+ if (pdata)
pd_len = pdata->size;
- if (!is_remote_ne020_or_chelsio(cm_node))
- pd_len += MPA_ZERO_PAD_LEN;
- }
if (cm_node->vlan_id < VLAN_TAG_PRESENT)
eth_hlen += 4;
@@ -3604,7 +3592,7 @@ int i40iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
iwqp->cm_node = (void *)cm_node;
cm_node->iwqp = iwqp;
- buf_len = conn_param->private_data_len + I40IW_MAX_IETF_SIZE + MPA_ZERO_PAD_LEN;
+ buf_len = conn_param->private_data_len + I40IW_MAX_IETF_SIZE;
status = i40iw_allocate_dma_mem(dev->hw, &iwqp->ietf_mem, buf_len, 1);
@@ -3638,18 +3626,10 @@ int i40iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
iwqp->lsmm_mr = ibmr;
if (iwqp->page)
iwqp->sc_qp.qp_uk.sq_base = kmap(iwqp->page);
- if (is_remote_ne020_or_chelsio(cm_node))
- dev->iw_priv_qp_ops->qp_send_lsmm(
- &iwqp->sc_qp,
+ dev->iw_priv_qp_ops->qp_send_lsmm(&iwqp->sc_qp,
iwqp->ietf_mem.va,
(accept.size + conn_param->private_data_len),
ibmr->lkey);
- else
- dev->iw_priv_qp_ops->qp_send_lsmm(
- &iwqp->sc_qp,
- iwqp->ietf_mem.va,
- (accept.size + conn_param->private_data_len + MPA_ZERO_PAD_LEN),
- ibmr->lkey);
} else {
if (iwqp->page)
diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.h b/drivers/infiniband/hw/i40iw/i40iw_cm.h
index 945ed26..24615c2 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_cm.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_cm.h
@@ -56,8 +56,6 @@
#define I40IW_MAX_IETF_SIZE 32
-#define MPA_ZERO_PAD_LEN 4
-
/* IETF RTR MSG Fields */
#define IETF_PEER_TO_PEER 0x8000
#define IETF_FLPDU_ZERO_LEN 0x4000
diff --git a/drivers/infiniband/hw/i40iw/i40iw_utils.c b/drivers/infiniband/hw/i40iw/i40iw_utils.c
index 218e9fd..805603b 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_utils.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_utils.c
@@ -1250,7 +1250,7 @@ enum i40iw_status_code i40iw_puda_get_tcpip_info(struct i40iw_puda_completion_in
buf->totallen = pkt_len + buf->maclen;
- if (info->payload_len < buf->totallen - 4) {
+ if (info->payload_len < buf->totallen) {
i40iw_pr_err("payload_len = 0x%x totallen expected0x%x\n",
info->payload_len, buf->totallen);
return I40IW_ERR_INVALID_SIZE;
--
1.8.3.1
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH] i40iw: Set MAX IRD, MAX ORD size to max supported value
From: Henry Orosco @ 2016-10-19 20:33 UTC (permalink / raw)
To: dledford-H+wXaHxf7aLQT0dZR+AlfA
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA,
e1000-rdma-5NWGOfrQmneRv+LV9MX5uipxlwaOVQ5f, Henry Orosco
Set the MAX_IRD and MAX_ORD size negotiated to the maximum
supported values.
Signed-off-by: Shiraz Saleem <shiraz.saleem-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Signed-off-by: Henry Orosco <henry.orosco-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
drivers/infiniband/hw/i40iw/i40iw_ctrl.c | 5 -----
drivers/infiniband/hw/i40iw/i40iw_user.h | 5 ++---
2 files changed, 2 insertions(+), 8 deletions(-)
diff --git a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c b/drivers/infiniband/hw/i40iw/i40iw_ctrl.c
index 31c4a0c..89eddbc 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_ctrl.c
@@ -2626,11 +2626,6 @@ static enum i40iw_status_code i40iw_sc_qp_setctx(
152,
LS_64(iw->last_byte_sent, I40IWQPC_LASTBYTESENT));
- /*
- * Hard-code IRD_SIZE to hw-limit, 128, in qpctx, i.e matching an
- *advertisable IRD of 64
- */
- iw->ird_size = I40IW_QPCTX_ENCD_MAXIRD;
set_64bit_val(qp_ctx,
160,
LS_64(iw->ord_size, I40IWQPC_ORDSIZE) |
diff --git a/drivers/infiniband/hw/i40iw/i40iw_user.h b/drivers/infiniband/hw/i40iw/i40iw_user.h
index 276bcef..e65c2baa 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_user.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_user.h
@@ -72,10 +72,9 @@ enum i40iw_device_capabilities_const {
I40IW_MAX_SQ_PAYLOAD_SIZE = 2145386496,
I40IW_MAX_INLINE_DATA_SIZE = 48,
I40IW_MAX_PUSHMODE_INLINE_DATA_SIZE = 48,
- I40IW_MAX_IRD_SIZE = 32,
- I40IW_QPCTX_ENCD_MAXIRD = 3,
+ I40IW_MAX_IRD_SIZE = 63,
+ I40IW_MAX_ORD_SIZE = 127,
I40IW_MAX_WQ_ENTRIES = 2048,
- I40IW_MAX_ORD_SIZE = 32,
I40IW_Q2_BUFFER_SIZE = (248 + 100),
I40IW_QP_CTX_SIZE = 248
};
--
1.8.3.1
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* Re: [PATCH v3 04/11] blk-mq: Introduce blk_mq_quiesce_queue()
From: Bart Van Assche @ 2016-10-19 21:04 UTC (permalink / raw)
To: Jens Axboe
Cc: Christoph Hellwig, James Bottomley, Martin K. Petersen,
Mike Snitzer, Doug Ledford, Keith Busch, Ming Lei,
Laurence Oberman, linux-block@vger.kernel.org,
linux-scsi@vger.kernel.org, linux-rdma@vger.kernel.org,
linux-nvme@lists.infradead.org
In-Reply-To: <e42a952c-f245-eb39-d0a1-0336035573f9@sandisk.com>
On 10/18/2016 02:50 PM, Bart Van Assche wrote:
> blk_mq_quiesce_queue() waits until ongoing .queue_rq() invocations
> have finished. This function does *not* wait until all outstanding
> requests have finished (this means invocation of request.end_io()).
(replying to my own e-mail)
The zero-day kernel test infrastructure reported to me that this patch
causes a build failure with CONFIG_SRCU=n. Should I add "select SRCU" to
block/Kconfig (excludes TINY_RCU) or should I rather modify this patch
such that a mutex or rwsem is used instead of SRCU?
Thanks,
Bart.
^ permalink raw reply
* RE: [PATCH rdma-core 2/4] glue/redhat: add udev/systemd/etc infrastructure bits
From: Weiny, Ira @ 2016-10-19 21:35 UTC (permalink / raw)
To: Jason Gunthorpe
Cc: Jarod Wilson, linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org,
Doug Ledford, Hefty, Sean
In-Reply-To: <20161019185920.GA20600-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org>
Udev below works great. Thanks!
But I have another cmake question.
I have specified the following:
install(FILES rdma-ndd.conf DESTINATION "${CMAKE_INSTALL_SYSCONFDIR}")
set(rdma_ndd_CONFIG_PATH "${CMAKE_INSTALL_SYSCONFDIR}")
configure_file ("${PROJECT_SOURCE_DIR}/rdma-ndd/rdma-config.h.in"
"${PROJECT_BINARY_DIR}/rdma-ndd/rdma-config.h")
include_directories("${PROJECT_BINARY_DIR}/rdma-ndd")
The idea here is to have an rdma-ndd.conf file which allows the user to configure their names. (previously this was the infiniband-diags.conf file). But I have ported over the code.
The value in rdma-config.h ends up being:
17:30:34 > cat build/rdma-ndd/rdma-config.h
/* Pick up configuration items from cmake */
#define RDMA_NDD_CONFIG_PATH "etc"
#define PACKAGE_VERSION 11
Which seems fine but after a "make install" I'm left with an incorrect path:
[root@phcppriv13 sbin]# pwd && ./rdma-ndd -f
/usr/local/sbin
rdma-ndd[91656]: Using config file (etc/rdma-ndd.conf)
...
So it is not reading the correct config file.
How do I link the install to this variable in the code?
Ira
> -----Original Message-----
> From: Jason Gunthorpe [mailto:jgunthorpe-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org]
> Sent: Wednesday, October 19, 2016 11:59 AM
> To: Weiny, Ira <ira.weiny-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> Cc: Jarod Wilson <jarod-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>; linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org; Doug
> Ledford <dledford-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>; Hefty, Sean <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> Subject: Re: [PATCH rdma-core 2/4] glue/redhat: add udev/systemd/etc
> infrastructure bits
>
> On Wed, Oct 19, 2016 at 06:38:52PM +0000, Weiny, Ira wrote:
>
> > I have the C code ported to rdma-core but how do I do this in cmake?
>
> Attached, you will also need
>
> target_link_libraries(rdma-ndd ${UDEV_LIBRARIES})
>
> > PKG_CHECK_EXISTS(libudev >= 218, [with_dev_logging=no],
> > [with_udev_logging=yes])
> > if test "$with_udev_logging" = "yes"; then
> > AC_DEFINE_UNQUOTED([HAVE_UDEV_LOGGING], 1,
> > [whether libudev logging can be used])
>
> Dump HAVE_UDEV_LOGGING from the code, upstream deleted support for it:
>
> https://www.redhat.com/archives/libvir-list/2014-December/msg00749.html
>
> src.c:3:2: warning: 'udev_set_log_fn' is deprecated (declared at
> /usr/include/libudev.h:41) [-Wdeprecated-declarations]
> int main(int argc,const char *argv[]) {udev_set_log_fn(NULL, NULL); return 0;}
> ^
>
> > I've found a "modules" file which looks like it has a compatible BSD
> > license and could be added but is there a better way?
>
> Basically the right idea, but I prefer to avoid pkgconfig whenever possible as it
> screws up cross compiling.
>
> > I also have to convert the man page from *.rst to man in some way...
> > Would it be ok if I put a dependency on rst2man in the repo?
>
> We have talked about it, maybe for now just include both the .rst and the
> rst2man output and we can revisit it..
>
> Jason
>
> From b0062acc238e06289aa946dacd5c534cf6c68d7d Mon Sep 17 00:00:00
> 2001
> From: Jason Gunthorpe <jgunthorpe-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org>
> Date: Wed, 19 Oct 2016 12:57:25 -0600
> Subject: [PATCH] Add a dependency on libudev
>
> incomplete, needs the RH stuff too.
>
> Signed-off-by: Jason Gunthorpe <jgunthorpe-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org>
> ---
> .travis.yml | 1 +
> CMakeLists.txt | 4 ++++
> README.md | 2 +-
> buildlib/FindUDev.cmake | 10 ++++++++++
> debian/control | 1 +
> 5 files changed, 17 insertions(+), 1 deletion(-) create mode 100644
> buildlib/FindUDev.cmake
>
> diff --git a/.travis.yml b/.travis.yml
> index d81b699294eb..d9c36cc9c649 100644
> --- a/.travis.yml
> +++ b/.travis.yml
> @@ -24,6 +24,7 @@ addons:
> - gcc-6
> - libnl-3-dev
> - libnl-route-3-dev
> + - libudev-dev
> - make
> - ninja-build
> - pkg-config
> diff --git a/CMakeLists.txt b/CMakeLists.txt index a23aa860e6d3..9402bacf70ce
> 100644
> --- a/CMakeLists.txt
> +++ b/CMakeLists.txt
> @@ -237,6 +237,10 @@ if (NOT NL_KIND EQUAL 0)
> set(CMAKE_REQUIRED_INCLUDES "${SAFE_CMAKE_REQUIRED_INCLUDES}")
> endif()
>
> +# udev
> +find_package(UDev REQUIRED)
> +include_directories(${UDEV_INCLUDE_DIRS})
> +
> # Statically determine sizeof(long), this is largely unnecessary, no new code #
> should rely on this.
> check_type_size("long" SIZEOF_LONG BUILTIN_TYPES_ONLY LANGUAGE C) diff
> --git a/README.md b/README.md index 66aee3f49f00..d24fd0bf2606 100644
> --- a/README.md
> +++ b/README.md
> @@ -48,7 +48,7 @@ only load from the system path.
> ### Debian Derived
>
> ```sh
> -$ apt-get install build-essential cmake gcc libnl-3-dev libnl-route-3-dev ninja-
> build pkg-config valgrind
> +$ apt-get install build-essential cmake gcc libudev-dev libnl-3-dev
> +libnl-route-3-dev ninja-build pkg-config valgrind
> ```
>
> ### Fedora
> diff --git a/buildlib/FindUDev.cmake b/buildlib/FindUDev.cmake new file mode
> 100644 index 000000000000..ce05ddf991a1
> --- /dev/null
> +++ b/buildlib/FindUDev.cmake
> @@ -0,0 +1,10 @@
> +# COPYRIGHT (c) 2016 Obsidian Research Corporation. See COPYING file
> +
> +find_library(LIBUDEV_LIBRARY NAMES udev libudev)
> +
> +set(UDEV_LIBRARIES ${LIBUDEV_LIBRARY})
> +
> +include(FindPackageHandleStandardArgs)
> +find_package_handle_standard_args(UDev REQUIRED_VARS
> LIBUDEV_LIBRARY)
> +
> +mark_as_advanced(LIBUDEV_LIBRARY)
> diff --git a/debian/control b/debian/control index
> 2335d1f4814d..ed9850a348be 100644
> --- a/debian/control
> +++ b/debian/control
> @@ -8,6 +8,7 @@ Build-Depends: build-essential,
> dh-systemd,
> dpkg-dev (>= 1.17),
> gcc,
> + libudev-dev,
> libnl-3-dev,
> libnl-route-3-dev,
> make,
> --
> 2.1.4
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: [PATCH v3 0/11] Fix race conditions related to stopping block layer queues
From: Keith Busch @ 2016-10-19 22:24 UTC (permalink / raw)
To: Bart Van Assche
Cc: Jens Axboe, Christoph Hellwig, James Bottomley,
Martin K. Petersen, Mike Snitzer, Doug Ledford, Ming Lin,
Laurence Oberman, linux-block@vger.kernel.org,
linux-scsi@vger.kernel.org, linux-rdma@vger.kernel.org,
linux-nvme@lists.infradead.org
In-Reply-To: <b39eb0e7-1007-eb63-8e7f-9a7f08508379@sandisk.com>
Hi Bart,
I'm running linux 4.9-rc1 + linux-block/for-linus, and alternating tests
with and without this series.
Without this, I'm not seeing any problems in a link-down test while
running fio after ~30 runs.
With this series, I only see the test pass infrequently. Most of the
time I observe one of several failures. In all cases, it looks like the
rq->queuelist is in an unexpected state.
I think I've almost got this tracked down, but I have to leave for the
day soon. Rather than having a more useful suggestion, I've put the two
failures below.
First failure:
[ 214.782075] ------------[ cut here ]------------
[ 214.782098] kernel BUG at block/blk-mq.c:498!
[ 214.782117] invalid opcode: 0000 [#1] SMP
[ 214.782133] Modules linked in: nvme nvme_core nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_raw ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security iptable_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security ebtable_filter ebtables ip6table_filter ip6_tables vfat fat
[ 214.782356] CPU: 6 PID: 160 Comm: kworker/u16:6 Not tainted 4.9.0-rc1+ #28
[ 214.782383] Hardware name: Gigabyte Technology Co., Ltd. Z97X-UD5H/Z97X-UD5H, BIOS F8 06/17/2014
[ 214.782419] Workqueue: nvme nvme_reset_work [nvme]
[ 214.782440] task: ffff8c0815403b00 task.stack: ffffb6ad01384000
[ 214.782463] RIP: 0010:[<ffffffff9f3b88a5>] [<ffffffff9f3b88a5>] blk_mq_requeue_request+0x35/0x40
[ 214.782502] RSP: 0018:ffffb6ad01387b88 EFLAGS: 00010287
[ 214.782524] RAX: ffff8c0814b98400 RBX: ffff8c0814b98200 RCX: 0000000000007530
[ 214.782551] RDX: 0000000000000007 RSI: 0000000000000001 RDI: ffff8c0814b98200
[ 214.782578] RBP: ffffb6ad01387b98 R08: 0000000000000000 R09: ffffffff9f408680
[ 214.783394] R10: 0000000000000394 R11: 0000000000000388 R12: 0000000000000001
[ 214.784212] R13: ffff8c081593a000 R14: 0000000000000001 R15: ffff8c080cdea740
[ 214.785033] FS: 0000000000000000(0000) GS:ffff8c081fb80000(0000) knlGS:0000000000000000
[ 214.785869] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 214.786710] CR2: 00007ffae4497f34 CR3: 00000001dfe06000 CR4: 00000000001406e0
[ 214.787559] Stack:
[ 214.788406] ffff8c0814b98200 0000000000000000 ffffb6ad01387ba8 ffffffffc03451b3
[ 214.789287] ffffb6ad01387bd0 ffffffffc0357a4a ffff8c0814b98200 ffffd6acffc81a00
[ 214.790174] 0000000000000006 ffffb6ad01387bf8 ffffffff9f3b8e22 ffff8c0814b98200
[ 214.791066] Call Trace:
[ 214.791935] [<ffffffffc03451b3>] nvme_requeue_req+0x13/0x20 [nvme_core]
[ 214.792810] [<ffffffffc0357a4a>] nvme_complete_rq+0x16a/0x1d0 [nvme]
[ 214.793680] [<ffffffff9f3b8e22>] __blk_mq_complete_request+0x72/0xe0
[ 214.794551] [<ffffffff9f3b8eac>] blk_mq_complete_request+0x1c/0x20
[ 214.795422] [<ffffffffc0345e70>] nvme_cancel_request+0x50/0x90 [nvme_core]
[ 214.796299] [<ffffffff9f3bc09e>] bt_tags_iter+0x2e/0x40
[ 214.797157] [<ffffffff9f3bc523>] blk_mq_tagset_busy_iter+0x173/0x1e0
[ 214.798005] [<ffffffffc0345e20>] ? nvme_shutdown_ctrl+0x100/0x100 [nvme_core]
[ 214.798852] [<ffffffffc0345e20>] ? nvme_shutdown_ctrl+0x100/0x100 [nvme_core]
[ 214.799682] [<ffffffffc035603d>] nvme_dev_disable+0x11d/0x380 [nvme]
[ 214.800511] [<ffffffff9f0479fa>] ? acpi_unregister_gsi_ioapic+0x3a/0x40
[ 214.801344] [<ffffffff9f52d33c>] ? dev_warn+0x6c/0x90
[ 214.802157] [<ffffffffc0356bc4>] nvme_reset_work+0xa4/0xdc0 [nvme]
[ 214.802961] [<ffffffff9f025736>] ? __switch_to+0x2b6/0x5f0
[ 214.803773] [<ffffffff9f0bb1bf>] process_one_work+0x15f/0x430
[ 214.804593] [<ffffffff9f0bb4de>] worker_thread+0x4e/0x490
[ 214.805419] [<ffffffff9f0bb490>] ? process_one_work+0x430/0x430
[ 214.806255] [<ffffffff9f0c0d09>] kthread+0xd9/0xf0
[ 214.807096] [<ffffffff9f0c0c30>] ? kthread_park+0x60/0x60
[ 214.807946] [<ffffffff9f81dc15>] ret_from_fork+0x25/0x30
[ 214.808801] Code: 54 53 48 89 fb 41 89 f4 e8 a9 fa ff ff 48 8b 03 48 39 c3 75 16 41 0f b6 d4 48 89 df be 01 00 00 00 e8 10 ff ff ff 5b 41 5c 5d c3 <0f> 0b 66 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 be 40 00 00
[ 214.810714] RIP [<ffffffff9f3b88a5>] blk_mq_requeue_request+0x35/0x40
[ 214.811628] RSP <ffffb6ad01387b88>
[ 214.812545] ---[ end trace 6ef3a3b6f8cea418 ]---
[ 214.813437] ------------[ cut here ]------------
Second failure, warning followed by NMI watchdog:
[ 410.736619] ------------[ cut here ]------------
[ 410.736624] WARNING: CPU: 2 PID: 577 at lib/list_debug.c:29 __list_add+0x62/0xb0
[ 410.736883] list_add corruption. next->prev should be prev (ffffacf481847d78), but was ffff931f8fb78000. (next=ffff931f8fb78000).
[ 410.736884] Modules linked in: nvme nvme_core nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_REJECT nf_reject_ipv6 ip6t_rpfilter xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_security ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_raw ip6table_mangle iptable_security iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_raw iptable_mangle ebtable_filter ebtables ip6table_filter ip6_tables vfat fat
[ 410.736902] CPU: 2 PID: 577 Comm: kworker/2:1H Not tainted 4.9.0-rc1+ #28
[ 410.736903] Hardware name: Gigabyte Technology Co., Ltd. Z97X-UD5H/Z97X-UD5H, BIOS F8 06/17/2014
[ 410.736906] Workqueue: kblockd blk_mq_run_work_fn
[ 410.736907] ffffacf481847c80 ffffffffae3dce7e ffffacf481847cd0 0000000000000000
[ 410.736909] ffffacf481847cc0 ffffffffae0a116b 0000001dae0b9cac ffff931f8fb78000
[ 410.736910] ffffacf481847d78 ffff931f8fb78000 ffff931f8fb78000 0000000000000000
[ 410.736912] Call Trace:
[ 410.736916] [<ffffffffae3dce7e>] dump_stack+0x63/0x85
[ 410.736918] [<ffffffffae0a116b>] __warn+0xcb/0xf0
[ 410.736920] [<ffffffffae0a11ef>] warn_slowpath_fmt+0x5f/0x80
[ 410.736921] [<ffffffffae3fc362>] __list_add+0x62/0xb0
[ 410.736923] [<ffffffffae3ba108>] blk_mq_process_rq_list+0x258/0x350
[ 410.736925] [<ffffffffae3ba289>] __blk_mq_run_hw_queue+0x89/0x90
[ 410.736926] [<ffffffffae3ba2d2>] blk_mq_run_work_fn+0x12/0x20
[ 410.736928] [<ffffffffae0bb1bf>] process_one_work+0x15f/0x430
[ 410.736929] [<ffffffffae0bb4de>] worker_thread+0x4e/0x490
[ 410.736931] [<ffffffffae0bb490>] ? process_one_work+0x430/0x430
[ 410.736932] [<ffffffffae0bb490>] ? process_one_work+0x430/0x430
[ 410.736934] [<ffffffffae003c27>] ? do_syscall_64+0x67/0x180
[ 410.736936] [<ffffffffae0c0d09>] kthread+0xd9/0xf0
[ 410.736937] [<ffffffffae0c0c30>] ? kthread_park+0x60/0x60
[ 410.736940] [<ffffffffae81dc15>] ret_from_fork+0x25/0x30
[ 410.736941] ---[ end trace 0d9c0b78654a9c5e ]---
[ 410.736942] ------------[ cut here ]-----------
[ 436.159108] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [kworker/2:1H:577]
[ 436.159126] Modules linked in: nvme nvme_core nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_REJECT nf_reject_ipv6 ip6t_rpfilter xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_security ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_raw ip6table_mangle iptable_security iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_raw iptable_mangle ebtable_filter ebtables ip6table_filter ip6_tables vfat fat
[ 436.159138] CPU: 2 PID: 577 Comm: kworker/2:1H Tainted: G W 4.9.0-rc1+ #28
[ 436.159138] Hardware name: Gigabyte Technology Co., Ltd. Z97X-UD5H/Z97X-UD5H, BIOS F8 06/17/2014
[ 436.159142] Workqueue: kblockd blk_mq_run_work_fn
[ 436.159143] task: ffff931f95411d80 task.stack: ffffacf481844000
[ 436.159143] RIP: 0010:[<ffffffffae3b7f11>] [<ffffffffae3b7f11>] __blk_mq_free_request+0x31/0x50
[ 436.159145] RSP: 0018:ffffacf481847d08 EFLAGS: 00000246
[ 436.159146] RAX: ffff931f8fb78000 RBX: ffff931f8f9f8000 RCX: 0000000000010000
[ 436.159146] RDX: 0000000000000040 RSI: ffffccf47fc81800 RDI: ffff931f8da45c00
[ 436.159147] RBP: ffffacf481847d10 R08: 0000000000000000 R09: ffff931f8fb78000
[ 436.159147] R10: 0000000000000000 R11: 0000000000000015 R12: 00000000fffffffb
[ 436.159147] R13: ffffacf481847d88 R14: ffff931f8fb78000 R15: 0000000000000000
[ 436.159148] FS: 0000000000000000(0000) GS:ffff931f9fa80000(0000) knlGS:0000000000000000
[ 436.159148] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 436.159149] CR2: 000055dab2dc8b70 CR3: 000000009de06000 CR4: 00000000001406e0
[ 436.159149] Stack:
[ 436.159150] ffff931f8fb78000 ffffacf481847d20 ffffffffae3b7f6d ffffacf481847d30
[ 436.159151] ffffffffae3b7fa2 ffffacf481847d50 ffffffffae3b8d93 ffff931f8da45c00
[ 436.159152] ffffacf481847d78 ffffacf481847de0 ffffffffae3ba1db ffff931f8f9f8000
[ 436.159153] Call Trace:
[ 436.159155] [<ffffffffae3b7f6d>] blk_mq_free_hctx_request+0x3d/0x40
[ 436.159156] [<ffffffffae3b7fa2>] blk_mq_free_request+0x32/0x40
[ 436.159157] [<ffffffffae3b8d93>] blk_mq_end_request+0x53/0x70
[ 436.159158] [<ffffffffae3ba1db>] blk_mq_process_rq_list+0x32b/0x350
[ 436.159159] [<ffffffffae3ba289>] __blk_mq_run_hw_queue+0x89/0x90
[ 436.159160] [<ffffffffae3ba2d2>] blk_mq_run_work_fn+0x12/0x20
[ 436.159162] [<ffffffffae0bb1bf>] process_one_work+0x15f/0x430
[ 436.159163] [<ffffffffae0bb4de>] worker_thread+0x4e/0x490
[ 436.159164] [<ffffffffae0bb490>] ? process_one_work+0x430/0x430
[ 436.159165] [<ffffffffae0bb490>] ? process_one_work+0x430/0x430
[ 436.159166] [<ffffffffae003c27>] ? do_syscall_64+0x67/0x180
[ 436.159168] [<ffffffffae0c0d09>] kthread+0xd9/0xf0
[ 436.159169] [<ffffffffae0c0c30>] ? kthread_park+0x60/0x60
[ 436.159171] [<ffffffffae81dc15>] ret_from_fork+0x25/0x30
[ 436.159172] Code: 89 d0 55 f6 40 4b 20 48 89 e5 53 8b 92 00 01 00 00 48 8b 58 30 74 07 f0 ff 8f e0 01 00 00 48 c7 40 48 00 00 00 00 f0 80 60 50 fd <e8> ba 47 00 00 48 89 df e8 d2 70 ff ff 5b 5d c3 66 66 66 66 66
^ permalink raw reply
* Re: [PATCH net-next 6/6] net: use core MTU range checking in misc drivers
From: Stefan Richter @ 2016-10-19 22:38 UTC (permalink / raw)
To: Sabrina Dubroca, Jarod Wilson
Cc: linux-kernel, netdev, Faisal Latif, linux-rdma, Cliff Whickman,
Robin Holt, Jes Sorensen, Marek Lindner, Simon Wunderlich,
Antonio Quartulli
In-Reply-To: <20161019160546.GC11224@bistromath.localdomain>
On Oct 19 Sabrina Dubroca wrote:
> 2016-10-18, 22:33:33 -0400, Jarod Wilson wrote:
> [...]
> > diff --git a/drivers/firewire/net.c b/drivers/firewire/net.c
> > index 309311b..b5f125c 100644
> > --- a/drivers/firewire/net.c
> > +++ b/drivers/firewire/net.c
> > @@ -1349,15 +1349,6 @@ static netdev_tx_t fwnet_tx(struct sk_buff *skb, struct net_device *net)
> > return NETDEV_TX_OK;
> > }
> >
> > -static int fwnet_change_mtu(struct net_device *net, int new_mtu)
> > -{
> > - if (new_mtu < 68)
> > - return -EINVAL;
> > -
> > - net->mtu = new_mtu;
> > - return 0;
> > -}
> > -
>
> This doesn't do any upper bound checking.
I need to check more closely, but I think the RFC 2734 encapsulation spec
and our implementation do not impose a particular upper limit. Though I
guess it's bad to let userland set arbitrarily large values here.
> > static const struct ethtool_ops fwnet_ethtool_ops = {
> > .get_link = ethtool_op_get_link,
> > };
> > @@ -1366,7 +1357,6 @@ static const struct net_device_ops fwnet_netdev_ops = {
> > .ndo_open = fwnet_open,
> > .ndo_stop = fwnet_stop,
> > .ndo_start_xmit = fwnet_tx,
> > - .ndo_change_mtu = fwnet_change_mtu,
> > };
> >
> > static void fwnet_init_dev(struct net_device *net)
> > @@ -1481,6 +1471,8 @@ static int fwnet_probe(struct fw_unit *unit,
> > max_mtu = (1 << (card->max_receive + 1))
> > - sizeof(struct rfc2734_header) - IEEE1394_GASP_HDR_SIZE;
> > net->mtu = min(1500U, max_mtu);
> > + net->min_mtu = ETH_MIN_MTU;
> > + net->max_mtu = net->mtu;
>
> But that will now prevent increasing the MTU above the initial value?
Indeed, therefore NAK.
PS:
If the IP packet plus encapsulation header fits into IEEE 1394 packet
payload, it is transported without link fragmentation. If it does not
fit, link fragmentation occurs (which reduces bandwidth a bit and
consumes additional buffering resources at the transmitter and the
receiver).
Broadcast and multicast packets are transmitted via IEEE 1394 asynchronous
stream packets at a low bus speed (because our code does not attempt to
find the maximum speed and size that is supported by all potential
listeners). This limits the payload to 512 bytes.
Unicast packets are transmitted via IEEE 1394 asynchronous write request
packets at optimum speed. In most cases, this means that 2048 bytes
payload is possible, in some cases 4096 bytes. Many CardBus FireWire
cards support only 1024 bytes payload of these packets though.
Furthermore, some low-speed long-haul cablings may cap the bus speed and
thereby the payload size to 1024 or 512 bytes, but this is uncommon in
practice.
--
Stefan Richter
-======----- =-=- =-=--
http://arcgraph.de/sr/
^ permalink raw reply
* Re: [PATCH 0/3] iopmem : A block device for PCIe memory
From: Stephen Bates @ 2016-10-19 22:54 UTC (permalink / raw)
To: Dan Williams
Cc: linux-kernel@vger.kernel.org, linux-nvdimm@lists.01.org,
linux-rdma, linux-block, Linux MM, Ross Zwisler, Matthew Wilcox,
Jason Gunthorpe, haggaie, Christoph Hellwig, Jens Axboe,
Jonathan Corbet, jim.macdonald, sbates, Logan Gunthorpe,
David Woodhouse, Raj, Ashok
In-Reply-To: <CAPcyv4ht=ZtQOyUp8khzzJtZhWcsaCgQi=feEuaj1AY3f9wd=g@mail.gmail.com>
> >>
> >> If you're only using the block-device as a entry-point to create
> >> dax-mappings then a device-dax (drivers/dax/) character-device might
> >> be a better fit.
> >>
> >
> > We chose a block device because we felt it was intuitive for users to
> > carve up a memory region but putting a DAX filesystem on it and creating
> > files on that DAX aware FS. It seemed like a convenient way to
> > partition up the region and to be easily able to get the DMA address
> > for the memory backing the device.
> >
> > That said I would be very keen to get other peoples thoughts on how
> > they would like to see this done. And I know some people have had some
> > reservations about using DAX mounted FS to do this in the past.
>
> I guess it depends on the expected size of these devices BARs, but I
> get the sense they may be smaller / more precious such that you
> wouldn't want to spend capacity on filesystem metadata? For the target
> use case is it assumed that these device BARs are always backed by
> non-volatile memory? Otherwise this is a mkfs each boot for a
> volatile device.
Dan
Fair point and this is a concern I share. We are not assuming that all
iopmem devices are backed by non-volatile memory so the mkfs
recreation comment is valid. All in all I think you are persuading us
to take a look at /dev/dax ;-). I will see if anyone else chips in
with their thoughts on this.
>
> >>
> >> > 2. Memory Segment Spacing. This patch has the same limitations that
> >> > ZONE_DEVICE does in that memory regions must be spaces at least
> >> > SECTION_SIZE bytes part. On x86 this is 128MB and there are cases where
> >> > BARs can be placed closer together than this. Thus ZONE_DEVICE would not
> >> > be usable on neighboring BARs. For our purposes, this is not an issue as
> >> > we'd only be looking at enabling a single BAR in a given PCIe device.
> >> > More exotic use cases may have problems with this.
> >>
> >> I'm working on patches for 4.10 to allow mixing multiple
> >> devm_memremap_pages() allocations within the same physical section.
> >> Hopefully this won't be a problem going forward.
> >>
> >
> > Thanks Dan. Your patches will help address the problem of how to
> > partition a /dev/dax device but they don't help the case then BARs
> > themselves are small, closely spaced and non-segment aligned. However
> > I think most people using iopmem will want to use reasonbly large
> > BARs so I am not sure item 2 is that big of an issue.
>
> I think you might have misunderstood what I'm proposing. The patches
> I'm working on are separate from a facility to carve up a /dev/dax
> device. The effort is to allow devm_memremap_pages() to maintain
> several allocations within the same 128MB section. I need this for
> persistent memory to handle platforms that mix pmem and system-ram in
> the same section. I want to be able to map ZONE_DEVICE pages for a
> portion of a section and be able to remove portions of section that
> may collide with allocations of a different lifetime.
Oh I did misunderstand. This is very cool and would be useful to us.
One more reason to consider moving to /dev/dax in the next spin of
this patchset ;-).
Thanks
Stephen
^ permalink raw reply
* RE: [PATCH rdma-core 2/4] glue/redhat: add udev/systemd/etc infrastructure bits
From: Weiny, Ira @ 2016-10-19 22:55 UTC (permalink / raw)
To: Weiny, Ira, Jason Gunthorpe
Cc: Jarod Wilson, linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org,
Doug Ledford, Hefty, Sean
In-Reply-To: <2807E5FD2F6FDA4886F6618EAC48510E24F0E288-8k97q/ur5Z2krb+BlOpmy7fspsVTdybXVpNB7YpNyf8@public.gmane.org>
Nevermind I think I have it...
Just need to test with an rpm build...
Ira
> -----Original Message-----
> From: linux-rdma-owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org [mailto:linux-rdma-
> owner-u79uwXL29TY76Z2rM5mHXA@public.gmane.org] On Behalf Of Weiny, Ira
> Sent: Wednesday, October 19, 2016 2:35 PM
> To: Jason Gunthorpe <jgunthorpe-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org>
> Cc: Jarod Wilson <jarod-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>; linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org; Doug
> Ledford <dledford-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>; Hefty, Sean <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> Subject: RE: [PATCH rdma-core 2/4] glue/redhat: add udev/systemd/etc
> infrastructure bits
>
> Udev below works great. Thanks!
>
> But I have another cmake question.
>
> I have specified the following:
>
> install(FILES rdma-ndd.conf DESTINATION "${CMAKE_INSTALL_SYSCONFDIR}")
> set(rdma_ndd_CONFIG_PATH "${CMAKE_INSTALL_SYSCONFDIR}")
> configure_file ("${PROJECT_SOURCE_DIR}/rdma-ndd/rdma-config.h.in"
> "${PROJECT_BINARY_DIR}/rdma-ndd/rdma-config.h")
> include_directories("${PROJECT_BINARY_DIR}/rdma-ndd")
>
> The idea here is to have an rdma-ndd.conf file which allows the user to
> configure their names. (previously this was the infiniband-diags.conf file). But I
> have ported over the code.
>
> The value in rdma-config.h ends up being:
>
> 17:30:34 > cat build/rdma-ndd/rdma-config.h
> /* Pick up configuration items from cmake */ #define
> RDMA_NDD_CONFIG_PATH "etc"
> #define PACKAGE_VERSION 11
>
> Which seems fine but after a "make install" I'm left with an incorrect path:
>
> [root@phcppriv13 sbin]# pwd && ./rdma-ndd -f /usr/local/sbin
> rdma-ndd[91656]: Using config file (etc/rdma-ndd.conf) ...
>
> So it is not reading the correct config file.
>
> How do I link the install to this variable in the code?
>
> Ira
>
>
> > -----Original Message-----
> > From: Jason Gunthorpe [mailto:jgunthorpe-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org]
> > Sent: Wednesday, October 19, 2016 11:59 AM
> > To: Weiny, Ira <ira.weiny-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> > Cc: Jarod Wilson <jarod-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>; linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org; Doug
> > Ledford <dledford-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>; Hefty, Sean <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
> > Subject: Re: [PATCH rdma-core 2/4] glue/redhat: add udev/systemd/etc
> > infrastructure bits
> >
> > On Wed, Oct 19, 2016 at 06:38:52PM +0000, Weiny, Ira wrote:
> >
> > > I have the C code ported to rdma-core but how do I do this in cmake?
> >
> > Attached, you will also need
> >
> > target_link_libraries(rdma-ndd ${UDEV_LIBRARIES})
> >
> > > PKG_CHECK_EXISTS(libudev >= 218, [with_dev_logging=no],
> > > [with_udev_logging=yes])
> > > if test "$with_udev_logging" = "yes"; then
> > > AC_DEFINE_UNQUOTED([HAVE_UDEV_LOGGING], 1,
> > > [whether libudev logging can be
> > > used])
> >
> > Dump HAVE_UDEV_LOGGING from the code, upstream deleted support for it:
> >
> > https://www.redhat.com/archives/libvir-list/2014-December/msg00749.htm
> > l
> >
> > src.c:3:2: warning: 'udev_set_log_fn' is deprecated (declared at
> > /usr/include/libudev.h:41) [-Wdeprecated-declarations]
> > int main(int argc,const char *argv[]) {udev_set_log_fn(NULL, NULL); return
> 0;}
> > ^
> >
> > > I've found a "modules" file which looks like it has a compatible BSD
> > > license and could be added but is there a better way?
> >
> > Basically the right idea, but I prefer to avoid pkgconfig whenever
> > possible as it screws up cross compiling.
> >
> > > I also have to convert the man page from *.rst to man in some way...
> > > Would it be ok if I put a dependency on rst2man in the repo?
> >
> > We have talked about it, maybe for now just include both the .rst and
> > the rst2man output and we can revisit it..
> >
> > Jason
> >
> > From b0062acc238e06289aa946dacd5c534cf6c68d7d Mon Sep 17 00:00:00
> > 2001
> > From: Jason Gunthorpe <jgunthorpe-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org>
> > Date: Wed, 19 Oct 2016 12:57:25 -0600
> > Subject: [PATCH] Add a dependency on libudev
> >
> > incomplete, needs the RH stuff too.
> >
> > Signed-off-by: Jason Gunthorpe <jgunthorpe-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org>
> > ---
> > .travis.yml | 1 +
> > CMakeLists.txt | 4 ++++
> > README.md | 2 +-
> > buildlib/FindUDev.cmake | 10 ++++++++++
> > debian/control | 1 +
> > 5 files changed, 17 insertions(+), 1 deletion(-) create mode 100644
> > buildlib/FindUDev.cmake
> >
> > diff --git a/.travis.yml b/.travis.yml index
> > d81b699294eb..d9c36cc9c649 100644
> > --- a/.travis.yml
> > +++ b/.travis.yml
> > @@ -24,6 +24,7 @@ addons:
> > - gcc-6
> > - libnl-3-dev
> > - libnl-route-3-dev
> > + - libudev-dev
> > - make
> > - ninja-build
> > - pkg-config
> > diff --git a/CMakeLists.txt b/CMakeLists.txt index
> > a23aa860e6d3..9402bacf70ce
> > 100644
> > --- a/CMakeLists.txt
> > +++ b/CMakeLists.txt
> > @@ -237,6 +237,10 @@ if (NOT NL_KIND EQUAL 0)
> > set(CMAKE_REQUIRED_INCLUDES
> "${SAFE_CMAKE_REQUIRED_INCLUDES}")
> > endif()
> >
> > +# udev
> > +find_package(UDev REQUIRED)
> > +include_directories(${UDEV_INCLUDE_DIRS})
> > +
> > # Statically determine sizeof(long), this is largely unnecessary, no
> > new code # should rely on this.
> > check_type_size("long" SIZEOF_LONG BUILTIN_TYPES_ONLY LANGUAGE C)
> > diff --git a/README.md b/README.md index 66aee3f49f00..d24fd0bf2606
> > 100644
> > --- a/README.md
> > +++ b/README.md
> > @@ -48,7 +48,7 @@ only load from the system path.
> > ### Debian Derived
> >
> > ```sh
> > -$ apt-get install build-essential cmake gcc libnl-3-dev
> > libnl-route-3-dev ninja- build pkg-config valgrind
> > +$ apt-get install build-essential cmake gcc libudev-dev libnl-3-dev
> > +libnl-route-3-dev ninja-build pkg-config valgrind
> > ```
> >
> > ### Fedora
> > diff --git a/buildlib/FindUDev.cmake b/buildlib/FindUDev.cmake new
> > file mode
> > 100644 index 000000000000..ce05ddf991a1
> > --- /dev/null
> > +++ b/buildlib/FindUDev.cmake
> > @@ -0,0 +1,10 @@
> > +# COPYRIGHT (c) 2016 Obsidian Research Corporation. See COPYING file
> > +
> > +find_library(LIBUDEV_LIBRARY NAMES udev libudev)
> > +
> > +set(UDEV_LIBRARIES ${LIBUDEV_LIBRARY})
> > +
> > +include(FindPackageHandleStandardArgs)
> > +find_package_handle_standard_args(UDev REQUIRED_VARS
> > LIBUDEV_LIBRARY)
> > +
> > +mark_as_advanced(LIBUDEV_LIBRARY)
> > diff --git a/debian/control b/debian/control index
> > 2335d1f4814d..ed9850a348be 100644
> > --- a/debian/control
> > +++ b/debian/control
> > @@ -8,6 +8,7 @@ Build-Depends: build-essential,
> > dh-systemd,
> > dpkg-dev (>= 1.17),
> > gcc,
> > + libudev-dev,
> > libnl-3-dev,
> > libnl-route-3-dev,
> > make,
> > --
> > 2.1.4
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body
> of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org More majordomo info at
> http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: [PATCH v3 04/11] blk-mq: Introduce blk_mq_quiesce_queue()
From: Ming Lei @ 2016-10-19 23:47 UTC (permalink / raw)
To: Bart Van Assche
Cc: Jens Axboe, Christoph Hellwig, James Bottomley,
Martin K. Petersen, Mike Snitzer, Doug Ledford, Keith Busch,
Laurence Oberman, linux-block@vger.kernel.org,
linux-scsi@vger.kernel.org, linux-rdma@vger.kernel.org,
linux-nvme@lists.infradead.org
In-Reply-To: <0718ec72-d525-2e75-a123-3bf9ea129319@sandisk.com>
On Thu, Oct 20, 2016 at 5:04 AM, Bart Van Assche
<bart.vanassche@sandisk.com> wrote:
> On 10/18/2016 02:50 PM, Bart Van Assche wrote:
>>
>> blk_mq_quiesce_queue() waits until ongoing .queue_rq() invocations
>> have finished. This function does *not* wait until all outstanding
>> requests have finished (this means invocation of request.end_io()).
>
>
> (replying to my own e-mail)
>
> The zero-day kernel test infrastructure reported to me that this patch
> causes a build failure with CONFIG_SRCU=n. Should I add "select SRCU" to
> block/Kconfig (excludes TINY_RCU) or should I rather modify this patch such
Select SRCU is fine, and you can see it is done in lots of
places(btrfs, net, quota, kvm, power,...)
> that a mutex or rwsem is used instead of SRCU?
Both should be much worse than SRCU, even not as good as atomic_t.
Thanks,
Ming
>
> Thanks,
>
> Bart.
>
--
Ming Lei
^ permalink raw reply
* Re: [PATCH v3 01/11] blk-mq: Do not invoke .queue_rq() for a stopped queue
From: Ming Lei @ 2016-10-19 23:48 UTC (permalink / raw)
To: Bart Van Assche
Cc: Jens Axboe, Christoph Hellwig, James Bottomley,
Martin K. Petersen, Mike Snitzer, Doug Ledford, Keith Busch,
Ming Lin, Laurence Oberman,
linux-block-u79uwXL29TY76Z2rM5mHXA@public.gmane.org,
linux-scsi-u79uwXL29TY76Z2rM5mHXA@public.gmane.org,
linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org,
linux-nvme-IAPFreCvJWM7uuMidbF8XUB+6BGkLq7r@public.gmane.org
In-Reply-To: <595d4b59-3892-ef24-ef91-b7cab6611af7-XdAiOPVOjttBDgjK7y7TUQ@public.gmane.org>
On Wed, Oct 19, 2016 at 5:48 AM, Bart Van Assche
<bart.vanassche-XdAiOPVOjttBDgjK7y7TUQ@public.gmane.org> wrote:
> The meaning of the BLK_MQ_S_STOPPED flag is "do not call
> .queue_rq()". Hence modify blk_mq_make_request() such that requests
> are queued instead of issued if a queue has been stopped.
>
> Signed-off-by: Bart Van Assche <bart.vanassche-XdAiOPVOjttBDgjK7y7TUQ@public.gmane.org>
> Cc: Christoph Hellwig <hch-jcswGhMUV9g@public.gmane.org>
> Cc: Hannes Reinecke <hare-IBi9RG/b67k@public.gmane.org>
> Cc: Sagi Grimberg <sagi-NQWnxTmZq1alnMjI0IkVqw@public.gmane.org>
> Cc: Johannes Thumshirn <jthumshirn-l3A5Bk7waGM@public.gmane.org>
> Cc: <stable-u79uwXL29TY76Z2rM5mHXA@public.gmane.org>
Reviewed-by: Ming Lei <tom.leiming-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org>
> ---
> block/blk-mq.c | 6 +++---
> 1 file changed, 3 insertions(+), 3 deletions(-)
>
> diff --git a/block/blk-mq.c b/block/blk-mq.c
> index ddc2eed..b5dcafb 100644
> --- a/block/blk-mq.c
> +++ b/block/blk-mq.c
> @@ -1332,9 +1332,9 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
> blk_mq_put_ctx(data.ctx);
> if (!old_rq)
> goto done;
> - if (!blk_mq_direct_issue_request(old_rq, &cookie))
> - goto done;
> - blk_mq_insert_request(old_rq, false, true, true);
> + if (test_bit(BLK_MQ_S_STOPPED, &data.hctx->state) ||
> + blk_mq_direct_issue_request(old_rq, &cookie) != 0)
> + blk_mq_insert_request(old_rq, false, true, true);
> goto done;
> }
>
> --
> 2.10.1
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-block" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
--
Ming Lei
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: [PATCH v3 0/11] Fix race conditions related to stopping block layer queues
From: Bart Van Assche @ 2016-10-19 23:51 UTC (permalink / raw)
To: Keith Busch
Cc: Jens Axboe, Christoph Hellwig, James Bottomley,
Martin K. Petersen, Mike Snitzer, Doug Ledford, Ming Lin,
Laurence Oberman, linux-block@vger.kernel.org,
linux-scsi@vger.kernel.org, linux-rdma@vger.kernel.org,
linux-nvme@lists.infradead.org
In-Reply-To: <20161019222454.GA1215@localhost.localdomain>
On 10/19/2016 03:14 PM, Keith Busch wrote:
> I'm running linux 4.9-rc1 + linux-block/for-linus, and alternating tests
> with and without this series.
>
> Without this, I'm not seeing any problems in a link-down test while
> running fio after ~30 runs.
>
> With this series, I only see the test pass infrequently. Most of the
> time I observe one of several failures. In all cases, it looks like the
> rq->queuelist is in an unexpected state.
>
> I think I've almost got this tracked down, but I have to leave for the
> day soon. Rather than having a more useful suggestion, I've put the two
> failures below.
>
> First failure:
>
> [ 214.782098] kernel BUG at block/blk-mq.c:498!
Hello Keith,
Thank you for having taken the time to test this patch series. Since I
think that the second and third failures are consequences of the first,
I will focus on the first failure triggered by your tests.
I assume that line 498 in blk-mq.c corresponds to
BUG_ON(blk_queued_rq(rq))? Anyway, it seems to me like this is a bug in
the NVMe code and also that this bug is completely unrelated to my patch
series. In nvme_complete_rq() I see that blk_mq_requeue_request() is
called. I don't think this is allowed from the context of
nvme_cancel_request() because blk_mq_requeue_request() assumes that a
request has already been removed from the request list. However, neither
blk_mq_tagset_busy_iter() nor nvme_cancel_request() remove a request
from the request list before nvme_complete_rq() is called. I think this
is what triggers the BUG_ON() statement in blk_mq_requeue_request().
Have you noticed that e.g. the scsi-mq code only calls
blk_mq_requeue_request() after __blk_mq_end_request() has finished? Have
you considered to follow the same approach in nvme_cancel_request()?
Thanks,
Bart.
^ permalink raw reply
* Re: [PATCH net-next 6/6] net: use core MTU range checking in misc drivers
From: Jarod Wilson @ 2016-10-20 3:16 UTC (permalink / raw)
To: Stefan Richter
Cc: Sabrina Dubroca, linux-kernel, netdev, Faisal Latif, linux-rdma,
Cliff Whickman, Robin Holt, Jes Sorensen, Marek Lindner,
Simon Wunderlich, Antonio Quartulli
In-Reply-To: <20161020003846.64f85f7e@kant>
On Thu, Oct 20, 2016 at 12:38:46AM +0200, Stefan Richter wrote:
> On Oct 19 Sabrina Dubroca wrote:
> > 2016-10-18, 22:33:33 -0400, Jarod Wilson wrote:
> > [...]
> > > diff --git a/drivers/firewire/net.c b/drivers/firewire/net.c
> > > index 309311b..b5f125c 100644
> > > --- a/drivers/firewire/net.c
> > > +++ b/drivers/firewire/net.c
> > > @@ -1349,15 +1349,6 @@ static netdev_tx_t fwnet_tx(struct sk_buff *skb, struct net_device *net)
> > > return NETDEV_TX_OK;
> > > }
> > >
> > > -static int fwnet_change_mtu(struct net_device *net, int new_mtu)
> > > -{
> > > - if (new_mtu < 68)
> > > - return -EINVAL;
> > > -
> > > - net->mtu = new_mtu;
> > > - return 0;
> > > -}
> > > -
> >
> > This doesn't do any upper bound checking.
>
> I need to check more closely, but I think the RFC 2734 encapsulation spec
> and our implementation do not impose a particular upper limit. Though I
> guess it's bad to let userland set arbitrarily large values here.
In which case, that would suggest using IP_MAX_MTU (65535) here.
> > > static const struct ethtool_ops fwnet_ethtool_ops = {
> > > .get_link = ethtool_op_get_link,
> > > };
> > > @@ -1366,7 +1357,6 @@ static const struct net_device_ops fwnet_netdev_ops = {
> > > .ndo_open = fwnet_open,
> > > .ndo_stop = fwnet_stop,
> > > .ndo_start_xmit = fwnet_tx,
> > > - .ndo_change_mtu = fwnet_change_mtu,
> > > };
> > >
> > > static void fwnet_init_dev(struct net_device *net)
> > > @@ -1481,6 +1471,8 @@ static int fwnet_probe(struct fw_unit *unit,
> > > max_mtu = (1 << (card->max_receive + 1))
> > > - sizeof(struct rfc2734_header) - IEEE1394_GASP_HDR_SIZE;
> > > net->mtu = min(1500U, max_mtu);
> > > + net->min_mtu = ETH_MIN_MTU;
> > > + net->max_mtu = net->mtu;
> >
> > But that will now prevent increasing the MTU above the initial value?
>
> Indeed, therefore NAK.
However, there's an explicit calculation for 'max_mtu' right there that I
glazed right over. It would seem perhaps *that* should be used for
net->max_mtu here, no?
> PS:
> If the IP packet plus encapsulation header fits into IEEE 1394 packet
> payload, it is transported without link fragmentation. If it does not
> fit, link fragmentation occurs (which reduces bandwidth a bit and
> consumes additional buffering resources at the transmitter and the
> receiver).
>
> Broadcast and multicast packets are transmitted via IEEE 1394 asynchronous
> stream packets at a low bus speed (because our code does not attempt to
> find the maximum speed and size that is supported by all potential
> listeners). This limits the payload to 512 bytes.
>
> Unicast packets are transmitted via IEEE 1394 asynchronous write request
> packets at optimum speed. In most cases, this means that 2048 bytes
> payload is possible, in some cases 4096 bytes. Many CardBus FireWire
> cards support only 1024 bytes payload of these packets though.
> Furthermore, some low-speed long-haul cablings may cap the bus speed and
> thereby the payload size to 1024 or 512 bytes, but this is uncommon in
> practice.
Thorough as always, Stefan! :)
--
Jarod Wilson
jarod@redhat.com
^ permalink raw reply
* Re: [PATCH rdma-core 2/4] glue/redhat: add udev/systemd/etc infrastructure bits
From: Jason Gunthorpe @ 2016-10-20 3:50 UTC (permalink / raw)
To: Weiny, Ira
Cc: Jarod Wilson, linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org,
Doug Ledford, Hefty, Sean
In-Reply-To: <2807E5FD2F6FDA4886F6618EAC48510E24F0E288-8k97q/ur5Z2krb+BlOpmy7fspsVTdybXVpNB7YpNyf8@public.gmane.org>
On Wed, Oct 19, 2016 at 09:35:15PM +0000, Weiny, Ira wrote:
> set(rdma_ndd_CONFIG_PATH "${CMAKE_INSTALL_SYSCONFDIR}")
> configure_file ("${PROJECT_SOURCE_DIR}/rdma-ndd/rdma-config.h.in"
> "${PROJECT_BINARY_DIR}/rdma-ndd/rdma-config.h")
> include_directories("${PROJECT_BINARY_DIR}/rdma-ndd")
Please just use the existing buildlib/config.h.in and put in:
#define RDMA_NDD_CONFIG_FILE "@CMAKE_INSTALL_FULL_SYSCONFDIR@/rdma-ndd.conf"
Nothing else is needed, please don't add more config.h's.
> #define RDMA_NDD_CONFIG_PATH "etc"
This is the difference between the 'FULL' and normal path versions in
the cmake scheme. The FULL version must be used everyplace needing an
absolute path, the other verions are used with install and have to do
with making DESTDIR work properly.
Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* RE: [PATCH rdma-core 2/4] glue/redhat: add udev/systemd/etc infrastructure bits
From: Weiny, Ira @ 2016-10-20 4:20 UTC (permalink / raw)
To: Jason Gunthorpe
Cc: Jarod Wilson, linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org,
Doug Ledford, Hefty, Sean
In-Reply-To: <20161020035054.GB28678-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org>
>
> On Wed, Oct 19, 2016 at 09:35:15PM +0000, Weiny, Ira wrote:
>
> > set(rdma_ndd_CONFIG_PATH "${CMAKE_INSTALL_SYSCONFDIR}")
> configure_file
> > ("${PROJECT_SOURCE_DIR}/rdma-ndd/rdma-config.h.in"
> > "${PROJECT_BINARY_DIR}/rdma-ndd/rdma-config.h")
> > include_directories("${PROJECT_BINARY_DIR}/rdma-ndd")
>
> Please just use the existing buildlib/config.h.in and put in:
>
> #define RDMA_NDD_CONFIG_FILE
> "@CMAKE_INSTALL_FULL_SYSCONFDIR@/rdma-ndd.conf"
>
> Nothing else is needed, please don't add more config.h's.
Fair enough, I missed that as it was buried in buildlib...
>
> > #define RDMA_NDD_CONFIG_PATH "etc"
>
> This is the difference between the 'FULL' and normal path versions in the cmake
> scheme. The FULL version must be used everyplace needing an absolute path,
> the other verions are used with install and have to do with making DESTDIR
> work properly.
Good to know...
I think the biggest issue I have with cmake is that I finally feel like I understand autotools and now I have to learn another system... :-P
Ira
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* systemd unit file location...
From: Weiny, Ira @ 2016-10-20 5:04 UTC (permalink / raw)
To: 'Jason Gunthorpe (jgunthorpe-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org)'
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
Why is this:
./build/CMakeCache.txt:CMAKE_INSTALL_SYSTEMD_SERVICEDIR:PATH=/usr/local/lib/systemd
Not defined as:
./build/CMakeCache.txt:CMAKE_INSTALL_SYSTEMD_SERVICEDIR:PATH=/usr/local/lib/systemd/system
??
And you add the "system" in the Debian rules?
./debian/iwpmd.install:lib/systemd/system/iwpmd.service
./debian/rules: -DCMAKE_INSTALL_SYSTEMD_SERVICEDIR:PATH=/lib/systemd/system \
I believe the same "system" sub dir is used on Fedora and RHEL, not sure about other distros?
Ira
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* RE: Introduction of libqedr to the Consolidated Userspace RDMA Library Repo
From: Amrani, Ram @ 2016-10-20 6:16 UTC (permalink / raw)
To: Jason Gunthorpe
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org, Elior, Ariel,
Kalderon, Michal, Borundia, Rajesh,
dledford-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org
In-Reply-To: <20161019172410.GB29879-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org>
> You also need to make sure it builds, Travis says the 32 bit builds are no good
> (look at the pull request and click on the red X)
That's true, I haven't checked those. I will.
> The github process from here is to make changes and then update your branch
> on your github, that will reflect in the pull request. Eg you can immediately fix
> the 32 bit issues and see that travis goes green.
OK
> I left some minor notes for you on github, the build system stuff looks fine to
> me, and I didn't notice anything too unusual in a casual browse. Didn't check if
> the code was any good..
OK
Thanks,
Ram
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* RE: [PATCH] qedr: return -EINVAL if pd is null and avoid null ptr dereference
From: Amrani, Ram @ 2016-10-20 7:12 UTC (permalink / raw)
To: Colin King, Doug Ledford, Sean Hefty, Hal Rosenstock,
Borundia, Rajesh,
linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
Cc: linux-kernel-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
In-Reply-To: <20161018183928.8539-1-colin.king-Z7WLFzj8eWMS+FvcfC7Uqw@public.gmane.org>
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset="utf-8", Size: 943 bytes --]
> drivers/infiniband/hw/qedr/verbs.c | 4 +++-
> 1 file changed, 3 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/infiniband/hw/qedr/verbs.c
> b/drivers/infiniband/hw/qedr/verbs.c
> index a615142..b2a0eb8 100644
> --- a/drivers/infiniband/hw/qedr/verbs.c
> +++ b/drivers/infiniband/hw/qedr/verbs.c
> @@ -511,8 +511,10 @@ int qedr_dealloc_pd(struct ib_pd *ibpd)
> struct qedr_dev *dev = get_qedr_dev(ibpd->device);
> struct qedr_pd *pd = get_qedr_pd(ibpd);
>
> - if (!pd)
> + if (!pd) {
> pr_err("Invalid PD received in dealloc_pd\n");
> + return -EINVAL;
> + }
>
> DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
> dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
> --
> 2.9.3
Thanks Colin.
Acked-by: Ram Amrani <ram.amrani@cavium.com>
N‹§²æìr¸›yúèšØb²X¬¶Ç§vØ^–)Þº{.nÇ+‰·¥Š{±ÙšŠ{ayº\x1dʇڙë,j\a¢f£¢·hš‹»öì\x17/oSc¾™Ú³9˜uÀ¦æå‰È&jw¨®\x03(階ŠÝ¢j"ú\x1a¶^[m§ÿïêäz¹Þ–Šàþf£¢·hšˆ§~ˆmš
^ permalink raw reply
* [Test fail]//Re: some test question//Re: [For help] configure crossbar build tool in CMakelist.txt
From: oulijun @ 2016-10-20 7:50 UTC (permalink / raw)
To: Jason Gunthorpe; +Cc: linux-rdma, Linuxarm
In-Reply-To: <20161019041102.GA28678-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org>
Hi, Jason
I am working for test the userspace code of hns in rdma-core.git with your guide.
I have met some quesiton for debugging in my environment.
after add some print lines in load_driver() in libibverbs/init.c, as follows:
printf("[%s, %d] %s, %d\n", __func__, __LINE__, VERBS_PROVIDER_DIR, sizeof(VERBS_PROVIDER_DIR));
/* If configured with a provider plugin path then try that next */
if (sizeof(VERBS_PROVIDER_DIR) >= 1) {
if (asprintf(&so_name, VERBS_PROVIDER_DIR "/lib%s" DLOPEN_TRAILER, name) <
0)
goto out_asprintf;
printf("[%s, %d] so_name: %s\n", __func__, __LINE__, so_name);// @1
printf("[%s, %d] so_name: %s\n", __func__, __LINE__, DLOPEN_TRAILER);//
dlhandle = dlopen(so_name, RTLD_NOW);
free(so_name);
if (dlhandle)
return;
}
/* Otherwise use the system libary search path. This is the historical
behavior of libibverbs */
if (asprintf(&so_name, "lib%s" DLOPEN_TRAILER, name) < 0)
goto out_asprintf;
printf("[%s, %d] so_name: %s\n", __func__, __LINE__, so_name);//@2
printf("[%s, %d] so_name: %s\n", __func__, __LINE__, DLOPEN_TRAILER);//
using the cmd:
CC=aarch64-linux-gnu-gcc cmake -GNinja -DENABLE_RESOLVE_NEIGH=0 -DVERBS_PROVIDER_DIR='' ..
ninja
after obtained the libhisi-rdmav2.so and hisi.driver, i put them into the /lib and /etc/libibverbs.d/
in my test environment.
Now, it is test fail. some question are not understanded.
1. when set the VERBS_PROVIDER_DIR to empty, the sizeof(VERBS_PROVIDER_DIR) should be 0 and the branch should not be run
2. the value of so_name should be /libhisi-rdmav2.so in @1 and libhisi-rdmav2.so in @2. in fact, the value of
so_name is /libhisi and libhisi
the test print log as follows:
-rdmav2.soer, 218] so_name: /libhisi
[load_driver, 219] so_name: -rdmav2.so
-rdmav2.soer, 229] so_name: libhisi
[load_driver, 230] so_name: -rdmav2.so
在 2016/10/19 12:11, Jason Gunthorpe 写道:
> On Wed, Oct 19, 2016 at 10:53:34AM +0800, oulijun wrote:
>
>> when use the cmd: CC=aarch64-linux-gnu-gcc cmake -GNinja -DENABLE_RESOLVE_NEIGH=0 ..
>>
>> the provider/hns/CMakelist.txt can be called correctly and generate
>> the library libhns-rdmav2.so(not be tested), but it will generate
>> warning with mlx5
>>
>> the print log as follows:
>>
>> Building C object providers/mlx5/CMakeFiles/mlx5-rdmav2.dir/cq.c.o
>> ../providers/mlx5/cq.c:414:13: warning: function declaration isn't a prototype [-Wstrict-prototypes]
>> static void mlx5_stall_poll_cq()
>
> Thanks, you may be the first person to compile on ARM64, so I'm glad
> to hear that is your only warning, I will get it fixed.
>
>> In summary, I have two questions:
>>
>> 1. if don't use libnl(use -DENABLE_RESOLVE_NEIGH=0), it will not
>> confluent the function of library?
>
> Correct, RoCEE will not work entirely as expected if you try and run
> the libibverbs from such a build.
>
>> 2. the rdma-core.git use the default build method by used build.sh,
>> and i need modify it by added the building item or modify the
>> README.md for hns, is the right approach?
>
> No.
>
> If you native compile on an actual ARM64 machine you will not have
> problems, simply follow the README.md directions to install the
> required packages on the actual machine.
>
> Your problem is cross compiling. Cross compiling is hard. You need to
> make libnl3 available in your cross compiler's 'sysroot'. I can't
> really give you generic good directions for that..
>
> One option is to cross compile libnl and 'cross-install' it into the
> compiler tree.
>
> Another option is to download the libnl3 ARM64 packages from Debian and
> then unpack and rename directories 'just so' to make it work.
>
> But if you don't intend to run the build binaries (eg when you go to
> test, you will build natively on ARM64), then don't worry about it.
>
> Jason
>
> .
>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: SQ overflow seen running isert traffic
From: Sagi Grimberg @ 2016-10-20 8:34 UTC (permalink / raw)
To: Jason Gunthorpe, Steve Wise
Cc: 'Potnuri Bharat Teja', target-devel, nab, linux-rdma
In-Reply-To: <20161018161313.GC24189@obsidianresearch.com>
Hey Jason,
>> 1) we believe the iSER + RW API correctly sizes the SQ, yet we're seeing SQ
>> overflows. So the SQ sizing needs more investigation.
>
> NFS had this sort of problem - in that case it was because the code
> was assuming that a RQ completion implied SQ space - that is not
> legal, only direct completions from SQ WCs can guide available space
> in the SQ..
Its not the same problem. iser-target does not tie SQ and RQ spaces.
The origin here is the difference between IB/RoCE and iWARP and the
chelsio HW that makes it hard to predict the SQ correct size.
iWARP needs extra registration for rdma reads and the chelsio device
seems to be limited in the number of pages per registration so this
configuration will need a larger send queue.
Another problem is that we don't have a correct retry flow.
Hopefully we can address that in the RW API which is designed to hide
these details from the ULP...
^ permalink raw reply
* RE: New providers in rdma-core
From: Amrani, Ram @ 2016-10-20 9:24 UTC (permalink / raw)
To: Jason Gunthorpe, Adit Ranadive
Cc: Christoph Hellwig, Leon Romanovsky, Lijun Ou, Knut Omang,
Doug Ledford, linux-rdma
In-Reply-To: <20161019200911.GA2666-ePGOBjL8dl3ta4EC/59zMFaTQe2KTcn/@public.gmane.org>
> Perhaps aided by a macro so we get high uniformity:
>
> #define DECLARE_DRIVER_UDATA(struct_driver, struct_common)
> struct user # struct_driver {
> struct struct_common ibv_resp;
> struct struct_driver udata;
> };
>
> Where mlx4_alloc_ucontext_resp_v3 comes from the uapi header, and is not
> inlined, so each driver gets an 'abi wrapper' header that just builds all these
> structs.
The idea looks good to me.
But still, I'm missing something - in order for libraries and the kernel to use the
same headers they should reside in the same repository.
Is that the long-term goal? And wouldn't that be mixing user/kernel spaces?
Thanks,
Ram
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* [PATCH rdma-core 0/6] libqedr: userspace library for qedr
From: Ram Amrani @ 2016-10-20 9:49 UTC (permalink / raw)
To: dledford-H+wXaHxf7aLQT0dZR+AlfA,
linux-rdma-u79uwXL29TY76Z2rM5mHXA
Cc: Ariel.Elior-YGCgFSpz5w/QT0dZR+AlfA,
Michal.Kalderon-YGCgFSpz5w/QT0dZR+AlfA, Ram Amrani
From: Ram Amrani <Ram.Amrani-74tsMCuadCbQT0dZR+AlfA@public.gmane.org>
This series introduces RoCE RDMA userspace library for the 579xx RDMA products
by Qlogic.
The libqedr library will support both RoCE and iWarp, although this series only
adds RoCE support.
This libqedr version matches the qedr patches submitted for upstream recently-
commit: 993d1b52615e1a549e55875c3b74308391672d9f.
Built on top of rdma-core 29e6d0ebc56cd089d9a200ba33a958899fc84bbe.
Ram Amrani (6):
libqedr (qelr) chains
libqedr (qelr) verbs
libqedr (qelr) HSI
libqedr (qelr) main
libqedr (qelr) abi
libqedr (qelr) addition to consolidated repo
CMakeLists.txt | 1 +
MAINTAINERS | 7 +
README.md | 1 +
providers/qedr/CMakeLists.txt | 5 +
providers/qedr/common_hsi.h | 1502 +++++++++++++++++++++++++++++++
providers/qedr/qelr.h | 320 +++++++
providers/qedr/qelr_abi.h | 120 +++
providers/qedr/qelr_chain.c | 107 +++
providers/qedr/qelr_chain.h | 163 ++++
providers/qedr/qelr_hsi.h | 67 ++
providers/qedr/qelr_hsi_rdma.h | 914 +++++++++++++++++++
providers/qedr/qelr_main.c | 286 ++++++
providers/qedr/qelr_main.h | 83 ++
providers/qedr/qelr_verbs.c | 1948 ++++++++++++++++++++++++++++++++++++++++
providers/qedr/qelr_verbs.h | 83 ++
providers/qedr/rdma_common.h | 74 ++
providers/qedr/roce_common.h | 50 ++
17 files changed, 5731 insertions(+)
create mode 100644 providers/qedr/CMakeLists.txt
create mode 100644 providers/qedr/common_hsi.h
create mode 100644 providers/qedr/qelr.h
create mode 100644 providers/qedr/qelr_abi.h
create mode 100644 providers/qedr/qelr_chain.c
create mode 100644 providers/qedr/qelr_chain.h
create mode 100644 providers/qedr/qelr_hsi.h
create mode 100644 providers/qedr/qelr_hsi_rdma.h
create mode 100644 providers/qedr/qelr_main.c
create mode 100644 providers/qedr/qelr_main.h
create mode 100644 providers/qedr/qelr_verbs.c
create mode 100644 providers/qedr/qelr_verbs.h
create mode 100644 providers/qedr/rdma_common.h
create mode 100644 providers/qedr/roce_common.h
--
2.7.4
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* [PATCH rdma-core 1/6] libqedr: chains
From: Ram Amrani @ 2016-10-20 9:49 UTC (permalink / raw)
To: dledford-H+wXaHxf7aLQT0dZR+AlfA,
linux-rdma-u79uwXL29TY76Z2rM5mHXA
Cc: Ariel.Elior-YGCgFSpz5w/QT0dZR+AlfA,
Michal.Kalderon-YGCgFSpz5w/QT0dZR+AlfA, Ram Amrani, Ram Amrani
In-Reply-To: <1476956952-17388-1-git-send-email-Ram.Amrani-YGCgFSpz5w/QT0dZR+AlfA@public.gmane.org>
From: Ram Amrani <Ram.Amrani-74tsMCuadCbQT0dZR+AlfA@public.gmane.org>
Introducing chains for managing user queues - SQ, RQ and CQ.
Signed-off-by: Ram Amrani <Ram.Amrani-YGCgFSpz5w/QT0dZR+AlfA@public.gmane.org>
---
providers/qedr/qelr_chain.c | 107 +++++++++++++++++++++++++++++
providers/qedr/qelr_chain.h | 163 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 270 insertions(+)
create mode 100644 providers/qedr/qelr_chain.c
create mode 100644 providers/qedr/qelr_chain.h
diff --git a/providers/qedr/qelr_chain.c b/providers/qedr/qelr_chain.c
new file mode 100644
index 0000000..6101f74
--- /dev/null
+++ b/providers/qedr/qelr_chain.c
@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2015-2016 QLogic Corporation
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and /or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <stdio.h>
+#include <string.h>
+#include <endian.h>
+#include <errno.h>
+
+#include "qelr.h"
+
+void *qelr_chain_get_last_elem(struct qelr_chain *p_chain)
+{
+ void *p_virt_addr = NULL;
+ uint32_t size;
+
+ if (!p_chain->addr)
+ goto out;
+
+ size = p_chain->elem_size * (p_chain->n_elems - 1);
+ p_virt_addr = ((uint8_t *)p_chain->addr + size);
+out:
+ return p_virt_addr;
+}
+
+void qelr_chain_reset(struct qelr_chain *p_chain)
+{
+ p_chain->prod_idx = 0;
+ p_chain->cons_idx = 0;
+
+ p_chain->p_cons_elem = p_chain->addr;
+ p_chain->p_prod_elem = p_chain->addr;
+}
+
+#define QELR_ANON_FD (-1) /* MAP_ANONYMOUS => file desc.= -1 */
+#define QELR_ANON_OFFSET (0) /* MAP_ANONYMOUS => offset = d/c */
+
+int qelr_chain_alloc(struct qelr_chain *chain, int chain_size, int page_size,
+ uint16_t elem_size)
+{
+ int ret, a_chain_size;
+ void *addr;
+
+ /* alloc aligned page aligned chain */
+ a_chain_size = (chain_size + page_size - 1) & ~(page_size - 1);
+ addr = mmap(NULL, a_chain_size, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, QELR_ANON_FD,
+ QELR_ANON_OFFSET);
+ if (chain->addr == MAP_FAILED)
+ return errno;
+
+ ret = ibv_dontfork_range(addr, a_chain_size);
+ if (ret) {
+ munmap(addr, a_chain_size);
+ return ret;
+ }
+
+ /* init chain */
+ memset(chain, 0, sizeof(*chain));
+ memset(chain->addr, 0, chain->size);
+ chain->addr = addr;
+ chain->size = a_chain_size;
+ chain->p_cons_elem = chain->addr;
+ chain->p_prod_elem = chain->addr;
+ chain->elem_size = elem_size;
+ chain->n_elems = chain->size / elem_size;
+
+ return 0;
+}
+
+void qelr_chain_free(struct qelr_chain *chain)
+{
+ if (chain->size) {
+ ibv_dofork_range(chain->addr, chain->size);
+ munmap(chain->addr, chain->size);
+ }
+}
diff --git a/providers/qedr/qelr_chain.h b/providers/qedr/qelr_chain.h
new file mode 100644
index 0000000..2ff5bf5
--- /dev/null
+++ b/providers/qedr/qelr_chain.h
@@ -0,0 +1,163 @@
+/*
+ * Copyright (c) 2015-2016 QLogic Corporation
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and /or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef __QELR_CHAIN_H__
+#define __QELR_CHAIN_H__
+
+struct qelr_chain {
+ /* Address of first page of the chain */
+ void *addr;
+
+ /* Point to next element to produce/consume */
+ void *p_prod_elem;
+ void *p_cons_elem;
+
+ uint32_t prod_idx;
+ uint32_t cons_idx;
+
+ uint32_t n_elems;
+ uint32_t size;
+ uint16_t elem_size;
+};
+
+/* fast path functions are inline */
+
+static inline uint32_t qelr_chain_get_cons_idx_u32(struct qelr_chain *p_chain)
+{
+ return p_chain->cons_idx;
+}
+
+static inline void *qelr_chain_produce(struct qelr_chain *p_chain)
+{
+ void *p_ret = NULL;
+
+ p_chain->prod_idx++;
+
+ p_ret = p_chain->p_prod_elem;
+
+ if (p_chain->prod_idx % p_chain->n_elems == 0)
+ p_chain->p_prod_elem = p_chain->addr;
+ else
+ p_chain->p_prod_elem = (void *)(((uint8_t *)p_chain->p_prod_elem) +
+ p_chain->elem_size);
+
+ return p_ret;
+}
+
+static inline void *qelr_chain_produce_n(struct qelr_chain *p_chain, int n)
+{
+ void *p_ret = NULL;
+ int n_wrap;
+
+ p_chain->prod_idx++;
+ p_ret = p_chain->p_prod_elem;
+
+ n_wrap = p_chain->prod_idx % p_chain->n_elems;
+ if (n_wrap < n)
+ p_chain->p_prod_elem = (void *)(((uint8_t *)p_chain->addr) +
+ (p_chain->elem_size * n_wrap));
+ else
+ p_chain->p_prod_elem = (void *)(((uint8_t *)p_chain->p_prod_elem) +
+ (p_chain->elem_size * n));
+
+ return p_ret;
+}
+
+static inline void *qelr_chain_consume(struct qelr_chain *p_chain)
+{
+ void *p_ret = NULL;
+
+ p_chain->cons_idx++;
+
+ p_ret = p_chain->p_cons_elem;
+
+ if (p_chain->cons_idx % p_chain->n_elems == 0)
+ p_chain->p_cons_elem = p_chain->addr;
+ else
+ p_chain->p_cons_elem = (void *)
+ (((uint8_t *)p_chain->p_cons_elem) +
+ p_chain->elem_size);
+
+ return p_ret;
+}
+
+static inline void *qelr_chain_consume_n(struct qelr_chain *p_chain, int n)
+{
+ void *p_ret = NULL;
+ int n_wrap;
+
+ p_chain->cons_idx += n;
+ p_ret = p_chain->p_cons_elem;
+
+ n_wrap = p_chain->cons_idx % p_chain->n_elems;
+ if (n_wrap < n)
+ p_chain->p_cons_elem = (void *)(((uint8_t *)p_chain->addr) +
+ (p_chain->elem_size * n_wrap));
+ else
+ p_chain->p_cons_elem = (void *)(((uint8_t *)p_chain->p_cons_elem) +
+ (p_chain->elem_size * n));
+
+ return p_ret;
+}
+
+static inline uint32_t qelr_chain_get_elem_left_u32(struct qelr_chain *p_chain)
+{
+ uint32_t used;
+
+ used = (uint32_t)(((uint64_t)((uint64_t) ~0U) + 1 +
+ (uint64_t)(p_chain->prod_idx)) -
+ (uint64_t)p_chain->cons_idx);
+
+ return p_chain->n_elems - used;
+}
+
+static inline uint8_t qelr_chain_is_full(struct qelr_chain *p_chain)
+{
+ return qelr_chain_get_elem_left_u32(p_chain) == p_chain->n_elems;
+}
+
+static inline void qelr_chain_set_prod(
+ struct qelr_chain *p_chain,
+ uint32_t prod_idx,
+ void *p_prod_elem)
+{
+ p_chain->prod_idx = prod_idx;
+ p_chain->p_prod_elem = p_prod_elem;
+}
+
+void *qelr_chain_get_last_elem(struct qelr_chain *p_chain);
+void qelr_chain_reset(struct qelr_chain *p_chain);
+int qelr_chain_alloc(struct qelr_chain *chain, int chain_size, int page_size,
+ uint16_t elem_size);
+void qelr_chain_free(struct qelr_chain *buf);
+
+#endif /* __QELR_CHAIN_H__ */
--
2.7.4
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH rdma-core 2/6] libqedr: verbs
From: Ram Amrani @ 2016-10-20 9:49 UTC (permalink / raw)
To: dledford-H+wXaHxf7aLQT0dZR+AlfA,
linux-rdma-u79uwXL29TY76Z2rM5mHXA
Cc: Ariel.Elior-YGCgFSpz5w/QT0dZR+AlfA,
Michal.Kalderon-YGCgFSpz5w/QT0dZR+AlfA, Ram Amrani, Ram Amrani
In-Reply-To: <1476956952-17388-1-git-send-email-Ram.Amrani-YGCgFSpz5w/QT0dZR+AlfA@public.gmane.org>
From: Ram Amrani <Ram.Amrani-74tsMCuadCbQT0dZR+AlfA@public.gmane.org>
Introducing verbs - create, modify, query and destroy for QPs CQs and etc.
Signed-off-by: Ram Amrani <Ram.Amrani-YGCgFSpz5w/QT0dZR+AlfA@public.gmane.org>
---
providers/qedr/qelr_verbs.c | 1948 +++++++++++++++++++++++++++++++++++++++++++
providers/qedr/qelr_verbs.h | 83 ++
2 files changed, 2031 insertions(+)
create mode 100644 providers/qedr/qelr_verbs.c
create mode 100644 providers/qedr/qelr_verbs.h
diff --git a/providers/qedr/qelr_verbs.c b/providers/qedr/qelr_verbs.c
new file mode 100644
index 0000000..496493a
--- /dev/null
+++ b/providers/qedr/qelr_verbs.c
@@ -0,0 +1,1948 @@
+/*
+ * Copyright (c) 2015-2016 QLogic Corporation
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and /or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <config.h>
+
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <signal.h>
+#include <errno.h>
+#include <pthread.h>
+#include <malloc.h>
+#include <sys/mman.h>
+#include <netinet/in.h>
+#include <unistd.h>
+
+#include "qelr.h"
+#include "qelr_abi.h"
+#include "qelr_chain.h"
+#include "qelr_verbs.h"
+
+#define PTR_LO(x) ((uint32_t)(((uint64_t)(x)) & 0xffffffff))
+#define PTR_HI(x) ((uint32_t)(((uint64_t)(x)) >> 32))
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <execinfo.h>
+
+/* Fast path debug prints */
+#define FP_DP_VERBOSE(...)
+/* #define FP_DP_VERBOSE(...) DP_VERBOSE(__VA_ARGS__) */
+
+#define QELR_SQE_ELEMENT_SIZE (sizeof(struct rdma_sq_sge))
+#define QELR_RQE_ELEMENT_SIZE (sizeof(struct rdma_rq_sge))
+#define QELR_CQE_SIZE (sizeof(union rdma_cqe))
+
+static void qelr_inc_sw_cons_u16(struct qelr_qp_hwq_info *info)
+{
+ info->cons = (info->cons + 1) % info->max_wr;
+ info->wqe_cons++;
+}
+
+static void qelr_inc_sw_prod_u16(struct qelr_qp_hwq_info *info)
+{
+ info->prod = (info->prod + 1) % info->max_wr;
+}
+
+int qelr_query_device(struct ibv_context *context,
+ struct ibv_device_attr *attr)
+{
+ struct ibv_query_device cmd;
+ uint64_t fw_ver;
+ unsigned int major, minor, revision, eng;
+ int status;
+
+ bzero(attr, sizeof(*attr));
+ status = ibv_cmd_query_device(context, attr, &fw_ver, &cmd,
+ sizeof(cmd));
+
+ major = (fw_ver >> 24) & 0xff;
+ minor = (fw_ver >> 16) & 0xff;
+ revision = (fw_ver >> 8) & 0xff;
+ eng = fw_ver & 0xff;
+
+ snprintf(attr->fw_ver, sizeof(attr->fw_ver),
+ "%d.%d.%d.%d", major, minor, revision, eng);
+
+ return status;
+}
+
+int qelr_query_port(struct ibv_context *context, uint8_t port,
+ struct ibv_port_attr *attr)
+{
+ struct ibv_query_port cmd;
+ int status;
+
+ status = ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd));
+ return status;
+}
+
+struct ibv_pd *qelr_alloc_pd(struct ibv_context *context)
+{
+ struct qelr_alloc_pd_req cmd;
+ struct qelr_alloc_pd_resp resp;
+ struct qelr_pd *pd;
+ struct qelr_devctx *cxt = get_qelr_ctx(context);
+
+ pd = malloc(sizeof(*pd));
+ if (!pd)
+ return NULL;
+
+ bzero(pd, sizeof(*pd));
+ memset(&cmd, 0, sizeof(cmd));
+
+ if (ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd.cmd, sizeof(cmd),
+ &resp.ibv_resp, sizeof(resp))) {
+ free(pd);
+ return NULL;
+ }
+
+ pd->pd_id = resp.pd_id;
+
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_INIT, "Allocated pd: %d\n", pd->pd_id);
+
+ return &pd->ibv_pd;
+}
+
+int qelr_dealloc_pd(struct ibv_pd *ibpd)
+{
+ int rc = 0;
+ struct qelr_pd *pd = get_qelr_pd(ibpd);
+ struct qelr_devctx *cxt = get_qelr_ctx(ibpd->context);
+
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_INIT, "Deallocated pd: %d\n",
+ pd->pd_id);
+
+ rc = ibv_cmd_dealloc_pd(ibpd);
+
+ if (rc)
+ return rc;
+
+ free(pd);
+
+ return rc;
+}
+
+struct ibv_mr *qelr_reg_mr(struct ibv_pd *ibpd, void *addr,
+ size_t len, int access)
+{
+ struct qelr_mr *mr;
+ struct ibv_reg_mr cmd;
+ struct qelr_reg_mr_resp resp;
+ struct qelr_pd *pd = get_qelr_pd(ibpd);
+ struct qelr_devctx *cxt = get_qelr_ctx(ibpd->context);
+
+ uint64_t hca_va = (uintptr_t) addr;
+
+ mr = malloc(sizeof(*mr));
+ if (!mr)
+ return NULL;
+
+ bzero(mr, sizeof(*mr));
+
+ if (ibv_cmd_reg_mr(ibpd, addr, len, hca_va,
+ access, &mr->ibv_mr, &cmd, sizeof(cmd),
+ &resp.ibv_resp, sizeof(resp))) {
+ free(mr);
+ return NULL;
+ }
+
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_MR,
+ "MR Register %p completed succesfully pd_id=%d addr=%p len=%zu access=%d lkey=%x rkey=%x\n",
+ mr, pd->pd_id, addr, len, access, mr->ibv_mr.lkey,
+ mr->ibv_mr.rkey);
+
+ return &mr->ibv_mr;
+}
+
+int qelr_dereg_mr(struct ibv_mr *mr)
+{
+ struct qelr_devctx *cxt = get_qelr_ctx(mr->context);
+ int rc;
+
+ rc = ibv_cmd_dereg_mr(mr);
+ if (rc)
+ return rc;
+
+ free(mr);
+
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_MR,
+ "MR DERegister %p completed succesfully\n", mr);
+ return 0;
+}
+
+static void consume_cqe(struct qelr_cq *cq)
+{
+ if (cq->latest_cqe == cq->toggle_cqe)
+ cq->chain_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
+
+ cq->latest_cqe = qelr_chain_consume(&cq->chain);
+}
+
+static inline int qelr_cq_entries(int entries)
+{
+ /* FW requires an extra entry */
+ return entries + 1;
+}
+
+struct ibv_cq *qelr_create_cq(struct ibv_context *context, int cqe,
+ struct ibv_comp_channel *channel,
+ int comp_vector)
+{
+ struct qelr_devctx *cxt = get_qelr_ctx(context);
+ struct qelr_create_cq_resp resp;
+ struct qelr_create_cq_req cmd;
+ struct qelr_cq *cq;
+ int chain_size;
+ int rc;
+
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_CQ,
+ "create cq: context=%p, cqe=%d, channel=%p, comp_vector=%d\n",
+ context, cqe, channel, comp_vector);
+
+ if (!cqe || cqe > cxt->max_cqes) {
+ DP_ERR(cxt->dbg_fp,
+ "create cq: failed. attempted to allocate %d cqes but valid range is 1...%d\n",
+ cqe, cqe > cxt->max_cqes);
+ return NULL;
+ }
+
+ /* allocate CQ structure */
+ cq = calloc(1, sizeof(*cq));
+ if (!cq)
+ return NULL;
+
+ /* allocate CQ buffer */
+ chain_size = qelr_cq_entries(cqe) * QELR_CQE_SIZE;
+ rc = qelr_chain_alloc(&cq->chain, chain_size, cxt->kernel_page_size,
+ QELR_CQE_SIZE);
+ if (rc)
+ goto err_0;
+
+ cmd.addr = (uintptr_t) cq->chain.addr;
+ cmd.len = cq->chain.size;
+ rc = ibv_cmd_create_cq(context, cqe, channel, comp_vector,
+ &cq->ibv_cq, &cmd.ibv_cmd, sizeof(cmd),
+ &resp.ibv_resp, sizeof(resp));
+ if (rc) {
+ DP_ERR(cxt->dbg_fp, "create cq: failed with rc = %d\n", rc);
+ goto err_1;
+ }
+
+ /* map the doorbell and prepare its data */
+ cq->db.data.icid = htole16(resp.icid);
+ cq->db.data.params = DB_AGG_CMD_SET <<
+ RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
+ cq->db_addr = cxt->db_addr + resp.db_offset;
+
+ /* point to the very last element, passing this we will toggle */
+ cq->toggle_cqe = qelr_chain_get_last_elem(&cq->chain);
+ cq->chain_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
+ cq->latest_cqe = NULL; /* must be different from chain_toggle */
+ consume_cqe(cq);
+
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_CQ,
+ "create cq: successfully created %p\n", cq);
+
+ return &cq->ibv_cq;
+
+err_1:
+ qelr_chain_free(&cq->chain);
+err_0:
+ free(cq);
+
+ return NULL;
+}
+
+int qelr_destroy_cq(struct ibv_cq *ibv_cq)
+{
+ struct qelr_devctx *cxt = get_qelr_ctx(ibv_cq->context);
+ struct qelr_cq *cq = get_qelr_cq(ibv_cq);
+ int rc;
+
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_CQ, "destroy cq: %p\n", cq);
+
+ rc = ibv_cmd_destroy_cq(ibv_cq);
+ if (rc) {
+ DP_ERR(cxt->dbg_fp,
+ "destroy cq: failed to destroy %p, got %d.\n", cq, rc);
+ return rc;
+ }
+
+ qelr_chain_free(&cq->chain);
+ free(cq);
+
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_CQ,
+ "destroy cq: successfully destroyed %p\n", cq);
+
+ return 0;
+}
+
+static void qelr_free_rq(struct qelr_qp *qp)
+{
+ free(qp->rqe_wr_id);
+}
+
+static void qelr_free_sq(struct qelr_qp *qp)
+{
+ free(qp->wqe_wr_id);
+}
+
+static void qelr_chain_free_sq(struct qelr_qp *qp)
+{
+ qelr_chain_free(&qp->sq.chain);
+}
+
+static void qelr_chain_free_rq(struct qelr_qp *qp)
+{
+ qelr_chain_free(&qp->rq.chain);
+}
+
+static inline int qelr_create_qp_buffers_sq(struct qelr_devctx *cxt,
+ struct qelr_qp *qp,
+ struct ibv_qp_init_attr *attrs)
+{
+ uint32_t max_send_wr, max_send_sges, max_send_buf;
+ int chain_size;
+ int rc;
+
+ /* SQ */
+ max_send_wr = attrs->cap.max_send_wr;
+ max_send_wr = max_t(uint32_t, max_send_wr, 1);
+ max_send_wr = min_t(uint32_t, max_send_wr, cxt->max_send_wr);
+ max_send_sges = max_send_wr * cxt->sges_per_send_wr;
+ max_send_buf = max_send_sges * QELR_SQE_ELEMENT_SIZE;
+
+ chain_size = max_send_buf;
+ rc = qelr_chain_alloc(&qp->sq.chain, chain_size, cxt->kernel_page_size,
+ QELR_SQE_ELEMENT_SIZE);
+ if (rc)
+ DP_ERR(cxt->dbg_fp, "create qp: failed to map SQ, got %d", rc);
+
+ qp->sq.max_wr = max_send_wr;
+ qp->sq.max_sges = cxt->sges_per_send_wr;
+
+ return rc;
+}
+
+static inline int qelr_create_qp_buffers_rq(struct qelr_devctx *cxt,
+ struct qelr_qp *qp,
+ struct ibv_qp_init_attr *attrs)
+{
+ uint32_t max_recv_wr, max_recv_sges, max_recv_buf;
+ int chain_size;
+ int rc;
+
+ /* RQ */
+ max_recv_wr = attrs->cap.max_recv_wr;
+ max_recv_wr = max_t(uint32_t, max_recv_wr, 1);
+ max_recv_wr = min_t(uint32_t, max_recv_wr, cxt->max_recv_wr);
+ max_recv_sges = max_recv_wr * cxt->sges_per_recv_wr;
+ max_recv_buf = max_recv_sges * QELR_RQE_ELEMENT_SIZE;
+ qp->rq.max_wr = max_recv_wr;
+ qp->rq.max_sges = RDMA_MAX_SGE_PER_RQ_WQE;
+
+ chain_size = max_recv_buf;
+ rc = qelr_chain_alloc(&qp->rq.chain, chain_size, cxt->kernel_page_size,
+ QELR_RQE_ELEMENT_SIZE);
+ if (rc)
+ DP_ERR(cxt->dbg_fp, "create qp: failed to map RQ, got %d", rc);
+
+ qp->rq.max_wr = max_recv_wr;
+ qp->rq.max_sges = cxt->sges_per_recv_wr;
+
+ return rc;
+}
+
+static inline int qelr_create_qp_buffers(struct qelr_devctx *cxt,
+ struct qelr_qp *qp,
+ struct ibv_qp_init_attr *attrs)
+{
+ int rc;
+
+ rc = qelr_create_qp_buffers_sq(cxt, qp, attrs);
+ if (rc)
+ return rc;
+
+ rc = qelr_create_qp_buffers_rq(cxt, qp, attrs);
+ if (rc) {
+ qelr_chain_free_sq(qp);
+ return rc;
+ }
+
+ return 0;
+}
+
+static inline int qelr_configure_qp_sq(struct qelr_devctx *cxt,
+ struct qelr_qp *qp,
+ struct ibv_qp_init_attr *attrs,
+ struct qelr_create_qp_resp *resp)
+{
+ qp->sq.icid = resp->sq_icid;
+ qp->sq.db_data.data.icid = htole16(resp->sq_icid);
+ qp->sq.prod = 0;
+ qp->sq.db = cxt->db_addr + resp->sq_db_offset;
+ qp->sq.edpm_db = cxt->db_addr;
+
+ /* shadow SQ */
+ qp->wqe_wr_id = calloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id));
+ if (!qp->wqe_wr_id) {
+ DP_ERR(cxt->dbg_fp,
+ "create qp: failed shdow SQ memory allocation\n");
+ return -ENOMEM;
+ }
+ return 0;
+}
+
+static inline int qelr_configure_qp_rq(struct qelr_devctx *cxt,
+ struct qelr_qp *qp,
+ struct ibv_qp_init_attr *attrs,
+ struct qelr_create_qp_resp *resp)
+{
+ /* RQ */
+ qp->rq.icid = resp->rq_icid;
+ qp->rq.db_data.data.icid = htole16(resp->rq_icid);
+ qp->rq.db = cxt->db_addr + resp->rq_db_offset;
+ qp->rq.prod = 0;
+
+ /* shadow RQ */
+ qp->rqe_wr_id = calloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id));
+ if (!qp->rqe_wr_id) {
+ DP_ERR(cxt->dbg_fp,
+ "create qp: failed shdow RQ memory allocation\n");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static inline int qelr_configure_qp(struct qelr_devctx *cxt, struct qelr_qp *qp,
+ struct ibv_qp_init_attr *attrs,
+ struct qelr_create_qp_resp *resp)
+{
+ int rc;
+
+ /* general */
+ pthread_spin_init(&qp->q_lock, PTHREAD_PROCESS_PRIVATE);
+ qp->qp_id = resp->qp_id;
+ qp->state = QELR_QPS_RST;
+ qp->sq_sig_all = attrs->sq_sig_all;
+ qp->atomic_supported = resp->atomic_supported;
+
+ rc = qelr_configure_qp_sq(cxt, qp, attrs, resp);
+ if (rc)
+ return rc;
+ rc = qelr_configure_qp_rq(cxt, qp, attrs, resp);
+ if (rc)
+ qelr_free_sq(qp);
+
+ return rc;
+}
+
+static inline void qelr_print_qp_init_attr(
+ struct qelr_devctx *cxt,
+ struct ibv_qp_init_attr *attr)
+{
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_QP,
+ "create qp: send_cq=%p, recv_cq=%p, srq=%p, max_inline_data=%d, max_recv_sge=%d, max_recv_wr=%d, max_send_sge=%d, max_send_wr=%d, qp_type=%d, sq_sig_all=%d\n",
+ attr->send_cq, attr->recv_cq, attr->srq,
+ attr->cap.max_inline_data, attr->cap.max_recv_sge,
+ attr->cap.max_recv_wr, attr->cap.max_send_sge,
+ attr->cap.max_send_wr, attr->qp_type, attr->sq_sig_all);
+}
+
+static inline void
+qelr_create_qp_configure_sq_req(struct qelr_qp *qp,
+ struct qelr_create_qp_req *req)
+{
+ req->sq_addr = (uintptr_t)qp->sq.chain.addr;
+ req->sq_len = qp->sq.chain.size;
+}
+
+static inline void
+qelr_create_qp_configure_rq_req(struct qelr_qp *qp,
+ struct qelr_create_qp_req *req)
+{
+ req->rq_addr = (uintptr_t)qp->rq.chain.addr;
+ req->rq_len = qp->rq.chain.size;
+}
+
+static inline void
+qelr_create_qp_configure_req(struct qelr_qp *qp,
+ struct qelr_create_qp_req *req)
+{
+ memset(req, 0, sizeof(*req));
+ req->qp_handle_hi = PTR_HI(qp);
+ req->qp_handle_lo = PTR_LO(qp);
+ qelr_create_qp_configure_sq_req(qp, req);
+ qelr_create_qp_configure_rq_req(qp, req);
+}
+
+struct ibv_qp *qelr_create_qp(struct ibv_pd *pd,
+ struct ibv_qp_init_attr *attrs)
+{
+ struct qelr_devctx *cxt = get_qelr_ctx(pd->context);
+ struct qelr_create_qp_resp resp;
+ struct qelr_create_qp_req req;
+ struct qelr_qp *qp;
+ int rc;
+
+ qelr_print_qp_init_attr(cxt, attrs);
+
+ qp = calloc(1, sizeof(*qp));
+ if (!qp)
+ return NULL;
+
+ rc = qelr_create_qp_buffers(cxt, qp, attrs);
+ if (rc)
+ goto err0;
+
+ qelr_create_qp_configure_req(qp, &req);
+
+ rc = ibv_cmd_create_qp(pd, &qp->ibv_qp, attrs, &req.ibv_qp, sizeof(req),
+ &resp.ibv_resp, sizeof(resp));
+ if (rc) {
+ DP_ERR(cxt->dbg_fp,
+ "create qp: failed on ibv_cmd_create_qp with %d\n", rc);
+ goto err1;
+ }
+
+ rc = qelr_configure_qp(cxt, qp, attrs, &resp);
+ if (rc)
+ goto err2;
+
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_QP,
+ "create qp: successfully created %p. handle_hi=%x handle_lo=%x\n",
+ qp, req.qp_handle_hi, req.qp_handle_lo);
+
+ return &qp->ibv_qp;
+
+err2:
+ rc = ibv_cmd_destroy_qp(&qp->ibv_qp);
+ if (rc)
+ DP_ERR(cxt->dbg_fp, "create qp: fatal fault. rc=%d\n", rc);
+err1:
+ qelr_chain_free_sq(qp);
+ qelr_chain_free_rq(qp);
+err0:
+ free(qp);
+
+ return NULL;
+}
+
+static void qelr_print_ah_attr(struct qelr_devctx *cxt, struct ibv_ah_attr *attr)
+{
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_QP,
+ "grh.dgid=[%lx:%lx], grh.flow_label=%d, grh.sgid_index=%d, grh.hop_limit=%d, grh.traffic_class=%d, dlid=%d, sl=%d, src_path_bits=%d, static_rate = %d, port_num=%d\n",
+ attr->grh.dgid.global.interface_id,
+ attr->grh.dgid.global.subnet_prefix,
+ attr->grh.flow_label, attr->grh.hop_limit,
+ attr->grh.sgid_index, attr->grh.traffic_class, attr->dlid,
+ attr->sl, attr->src_path_bits,
+ attr->static_rate, attr->port_num);
+}
+
+static void qelr_print_qp_attr(struct qelr_devctx *cxt, struct ibv_qp_attr *attr)
+{
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_QP,
+ "\tqp_state=%d\tcur_qp_state=%d\tpath_mtu=%d\tpath_mig_state=%d\tqkey=%d\trq_psn=%d\tsq_psn=%d\tdest_qp_num=%d\tqp_access_flags=%d\tmax_inline_data=%d\tmax_recv_sge=%d\tmax_recv_wr=%d\tmax_send_sge=%d\tmax_send_wr=%d\tpkey_index=%d\talt_pkey_index=%d\ten_sqd_async_notify=%d\tsq_draining=%d\tmax_rd_atomic=%d\tmax_dest_rd_atomic=%d\tmin_rnr_timer=%d\tport_num=%d\ttimeout=%d\tretry_cnt=%d\trnr_retry=%d\talt_port_num=%d\talt_timeout=%d\n",
+ attr->qp_state, attr->cur_qp_state, attr->path_mtu,
+ attr->path_mig_state, attr->qkey, attr->rq_psn, attr->sq_psn,
+ attr->dest_qp_num, attr->qp_access_flags,
+ attr->cap.max_inline_data, attr->cap.max_recv_sge,
+ attr->cap.max_recv_wr, attr->cap.max_send_sge,
+ attr->cap.max_send_wr, attr->pkey_index,
+ attr->alt_pkey_index, attr->en_sqd_async_notify,
+ attr->sq_draining, attr->max_rd_atomic,
+ attr->max_dest_rd_atomic, attr->min_rnr_timer,
+ attr->port_num, attr->timeout, attr->retry_cnt,
+ attr->rnr_retry, attr->alt_port_num, attr->alt_timeout);
+
+ qelr_print_ah_attr(cxt, &attr->ah_attr);
+ qelr_print_ah_attr(cxt, &attr->alt_ah_attr);
+}
+
+int qelr_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+ int attr_mask, struct ibv_qp_init_attr *init_attr)
+{
+ struct ibv_query_qp cmd;
+ struct qelr_devctx *cxt = get_qelr_ctx(qp->context);
+ int rc;
+
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_QP, "QP Query %p, attr_mask=0x%x\n",
+ get_qelr_qp(qp), attr_mask);
+
+ rc = ibv_cmd_query_qp(qp, attr, attr_mask,
+ init_attr, &cmd, sizeof(cmd));
+
+ qelr_print_qp_attr(cxt, attr);
+
+ return rc;
+}
+
+static enum qelr_qp_state get_qelr_qp_state(enum ibv_qp_state qps)
+{
+ switch (qps) {
+ case IBV_QPS_RESET:
+ return QELR_QPS_RST;
+ case IBV_QPS_INIT:
+ return QELR_QPS_INIT;
+ case IBV_QPS_RTR:
+ return QELR_QPS_RTR;
+ case IBV_QPS_RTS:
+ return QELR_QPS_RTS;
+ case IBV_QPS_SQD:
+ return QELR_QPS_SQD;
+ case IBV_QPS_SQE:
+ return QELR_QPS_SQE;
+ case IBV_QPS_ERR:
+ default:
+ return QELR_QPS_ERR;
+ };
+}
+
+static void qelr_reset_qp_hwq_info(struct qelr_qp_hwq_info *q)
+{
+ qelr_chain_reset(&q->chain);
+ q->prod = 0;
+ q->cons = 0;
+ q->wqe_cons = 0;
+ q->db_data.data.value = 0;
+}
+
+static int qelr_update_qp_state(struct qelr_qp *qp,
+ enum ibv_qp_state new_ib_state)
+{
+ int status = 0;
+ enum qelr_qp_state new_state;
+
+ new_state = get_qelr_qp_state(new_ib_state);
+
+ pthread_spin_lock(&qp->q_lock);
+
+ if (new_state == qp->state) {
+ pthread_spin_unlock(&qp->q_lock);
+ return 0;
+ }
+
+ switch (qp->state) {
+ case QELR_QPS_RST:
+ switch (new_state) {
+ case QELR_QPS_INIT:
+ qp->prev_wqe_size = 0;
+ qelr_reset_qp_hwq_info(&qp->sq);
+ qelr_reset_qp_hwq_info(&qp->rq);
+ break;
+ default:
+ status = -EINVAL;
+ break;
+ };
+ break;
+ case QELR_QPS_INIT:
+ /* INIT->XXX */
+ switch (new_state) {
+ case QELR_QPS_RTR:
+ /* Update doorbell (in case post_recv was done before
+ * move to RTR)
+ */
+ wmb();
+ writel(qp->rq.db_data.raw, qp->rq.db);
+ wc_wmb();
+ break;
+ case QELR_QPS_ERR:
+ break;
+ default:
+ /* invalid state change. */
+ status = -EINVAL;
+ break;
+ };
+ break;
+ case QELR_QPS_RTR:
+ /* RTR->XXX */
+ switch (new_state) {
+ case QELR_QPS_RTS:
+ break;
+ case QELR_QPS_ERR:
+ break;
+ default:
+ /* invalid state change. */
+ status = -EINVAL;
+ break;
+ };
+ break;
+ case QELR_QPS_RTS:
+ /* RTS->XXX */
+ switch (new_state) {
+ case QELR_QPS_SQD:
+ case QELR_QPS_SQE:
+ break;
+ case QELR_QPS_ERR:
+ break;
+ default:
+ /* invalid state change. */
+ status = -EINVAL;
+ break;
+ };
+ break;
+ case QELR_QPS_SQD:
+ /* SQD->XXX */
+ switch (new_state) {
+ case QELR_QPS_RTS:
+ case QELR_QPS_SQE:
+ case QELR_QPS_ERR:
+ break;
+ default:
+ /* invalid state change. */
+ status = -EINVAL;
+ break;
+ };
+ break;
+ case QELR_QPS_SQE:
+ switch (new_state) {
+ case QELR_QPS_RTS:
+ case QELR_QPS_ERR:
+ break;
+ default:
+ /* invalid state change. */
+ status = -EINVAL;
+ break;
+ };
+ break;
+ case QELR_QPS_ERR:
+ /* ERR->XXX */
+ switch (new_state) {
+ case QELR_QPS_RST:
+ break;
+ default:
+ status = -EINVAL;
+ break;
+ };
+ break;
+ default:
+ status = -EINVAL;
+ break;
+ };
+ if (!status)
+ qp->state = new_state;
+
+ pthread_spin_unlock(&qp->q_lock);
+
+ return status;
+}
+
+int qelr_modify_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr,
+ int attr_mask)
+{
+ struct ibv_modify_qp cmd;
+ struct qelr_qp *qp = get_qelr_qp(ibqp);
+ struct qelr_devctx *cxt = get_qelr_ctx(ibqp->context);
+ int rc;
+
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_QP, "QP Modify %p, attr_mask=0x%x\n",
+ qp, attr_mask);
+
+ qelr_print_qp_attr(cxt, attr);
+
+ rc = ibv_cmd_modify_qp(ibqp, attr, attr_mask, &cmd, sizeof(cmd));
+
+ if (!rc && (attr_mask & IBV_QP_STATE)) {
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_QP, "QP Modify state %d->%d\n",
+ qp->state, attr->qp_state);
+ qelr_update_qp_state(qp, attr->qp_state);
+ }
+
+ return rc;
+}
+
+int qelr_destroy_qp(struct ibv_qp *ibqp)
+{
+ struct qelr_devctx *cxt = get_qelr_ctx(ibqp->context);
+ struct qelr_qp *qp = get_qelr_qp(ibqp);
+ int rc = 0;
+
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_QP, "destroy qp: %p\n", qp);
+
+ rc = ibv_cmd_destroy_qp(ibqp);
+ if (rc) {
+ DP_ERR(cxt->dbg_fp,
+ "destroy qp: failed to destroy %p, got %d.\n", qp, rc);
+ return rc;
+ }
+
+ qelr_free_sq(qp);
+ qelr_free_rq(qp);
+ qelr_chain_free_sq(qp);
+ qelr_chain_free_rq(qp);
+ free(qp);
+
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_QP,
+ "destroy cq: succesfully destroyed %p\n", qp);
+
+ return 0;
+}
+
+static int sge_data_len(struct ibv_sge *sg_list, int num_sge)
+{
+ int i, len = 0;
+
+ for (i = 0; i < num_sge; i++)
+ len += sg_list[i].length;
+ return len;
+}
+
+static void swap_wqe_data64(uint64_t *p)
+{
+ int i;
+
+ for (i = 0; i < ROCE_WQE_ELEM_SIZE / sizeof(uint64_t); i++, p++)
+ *p = htobe64(htole64(*p));
+}
+
+static void qelr_init_edpm_info(struct qelr_qp *qp, struct qelr_devctx *cxt)
+{
+ memset(&qp->edpm, 0, sizeof(qp->edpm));
+
+ qp->edpm.rdma_ext = (struct qelr_rdma_ext *)&qp->edpm.dpm_payload;
+ if (qelr_chain_is_full(&qp->sq.chain))
+ qp->edpm.is_edpm = 1;
+}
+
+#define QELR_IB_OPCODE_SEND_ONLY 0x04
+#define QELR_IB_OPCODE_SEND_ONLY_WITH_IMMEDIATE 0x05
+#define QELR_IB_OPCODE_RDMA_WRITE_ONLY 0x0a
+#define QELR_IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE 0x0b
+#define QELR_IS_IMM(opcode) \
+ ((opcode == QELR_IB_OPCODE_SEND_ONLY_WITH_IMMEDIATE) || \
+ (opcode == QELR_IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE))
+
+static inline void qelr_edpm_set_msg_data(struct qelr_qp *qp,
+ uint8_t opcode,
+ uint16_t length,
+ uint8_t se,
+ uint8_t comp)
+{
+ uint32_t wqe_size = length +
+ (QELR_IS_IMM(opcode) ? sizeof(uint32_t) : 0);
+ uint32_t dpm_size = wqe_size + sizeof(struct db_roce_dpm_data);
+
+ if (!qp->edpm.is_edpm)
+ return;
+
+ SET_FIELD(qp->edpm.msg.data.params.params,
+ DB_ROCE_DPM_PARAMS_SIZE,
+ (dpm_size + sizeof(uint64_t) - 1) / sizeof(uint64_t));
+
+ SET_FIELD(qp->edpm.msg.data.params.params,
+ DB_ROCE_DPM_PARAMS_DPM_TYPE, DPM_ROCE);
+
+ SET_FIELD(qp->edpm.msg.data.params.params,
+ DB_ROCE_DPM_PARAMS_OPCODE,
+ opcode);
+
+ SET_FIELD(qp->edpm.msg.data.params.params,
+ DB_ROCE_DPM_PARAMS_WQE_SIZE,
+ wqe_size);
+
+ SET_FIELD(qp->edpm.msg.data.params.params,
+ DB_ROCE_DPM_PARAMS_COMPLETION_FLG, comp ? 1 : 0);
+
+ SET_FIELD(qp->edpm.msg.data.params.params,
+ DB_ROCE_DPM_PARAMS_S_FLG,
+ se ? 1 : 0);
+}
+
+static inline void qelr_edpm_set_inv_imm(struct qelr_qp *qp,
+ uint32_t inv_key_or_imm_data)
+{
+ if (!qp->edpm.is_edpm)
+ return;
+
+ memcpy(&qp->edpm.dpm_payload[qp->edpm.dpm_payload_offset],
+ &inv_key_or_imm_data, sizeof(inv_key_or_imm_data));
+
+ qp->edpm.dpm_payload_offset += sizeof(inv_key_or_imm_data);
+ qp->edpm.dpm_payload_size += sizeof(inv_key_or_imm_data);
+}
+
+static inline void qelr_edpm_set_rdma_ext(struct qelr_qp *qp,
+ uint64_t remote_addr,
+ uint32_t rkey)
+{
+ if (!qp->edpm.is_edpm)
+ return;
+
+ qp->edpm.rdma_ext->remote_va = htonll(remote_addr);
+ qp->edpm.rdma_ext->remote_key = htonl(rkey);
+ qp->edpm.dpm_payload_offset += sizeof(*qp->edpm.rdma_ext);
+ qp->edpm.dpm_payload_size += sizeof(*qp->edpm.rdma_ext);
+}
+
+static inline void qelr_edpm_set_payload(struct qelr_qp *qp, char *buf,
+ uint32_t length)
+{
+ if (!qp->edpm.is_edpm)
+ return;
+
+ memcpy(&qp->edpm.dpm_payload[qp->edpm.dpm_payload_offset],
+ buf,
+ length);
+
+ qp->edpm.dpm_payload_offset += length;
+}
+
+#define MIN(X, Y) (((X) < (Y)) ? (X) : (Y))
+
+static uint32_t qelr_prepare_sq_inline_data(struct qelr_qp *qp,
+ uint8_t *wqe_size,
+ struct ibv_send_wr *wr,
+ struct ibv_send_wr **bad_wr,
+ uint8_t *bits, uint8_t bit)
+{
+ int i, seg_siz;
+ char *seg_prt, *wqe;
+ uint32_t data_size = sge_data_len(wr->sg_list, wr->num_sge);
+
+ if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
+ DP_ERR(stderr, "Too much inline data in WR: %d\n", data_size);
+ *bad_wr = wr;
+ return 0;
+ }
+
+ if (!data_size)
+ return data_size;
+
+ /* set the bit */
+ *bits |= bit;
+
+ seg_prt = NULL;
+ wqe = NULL;
+ seg_siz = 0;
+
+ /* copy data inline */
+ for (i = 0; i < wr->num_sge; i++) {
+ uint32_t len = wr->sg_list[i].length;
+ void *src = (void *)wr->sg_list[i].addr;
+
+ qelr_edpm_set_payload(qp, src, wr->sg_list[i].length);
+
+ while (len > 0) {
+ uint32_t cur;
+
+ /* new segment required */
+ if (!seg_siz) {
+ wqe = (char *)qelr_chain_produce(&qp->sq.chain);
+ seg_prt = wqe;
+ seg_siz = sizeof(struct rdma_sq_common_wqe);
+ (*wqe_size)++;
+ }
+
+ /* calculate currently allowed length */
+ cur = MIN(len, seg_siz);
+
+ memcpy(seg_prt, src, cur);
+
+ /* update segment variables */
+ seg_prt += cur;
+ seg_siz -= cur;
+ /* update sge variables */
+ src += cur;
+ len -= cur;
+
+ /* swap fully-completed segments */
+ if (!seg_siz)
+ swap_wqe_data64((uint64_t *)wqe);
+ }
+ }
+
+ /* swap last not completed segment */
+ if (seg_siz)
+ swap_wqe_data64((uint64_t *)wqe);
+
+ if (qp->edpm.is_edpm) {
+ qp->edpm.dpm_payload_size += data_size;
+ qp->edpm.rdma_ext->dma_length = htonl(data_size);
+ }
+
+ return data_size;
+}
+
+static uint32_t qelr_prepare_sq_sges(struct qelr_qp *qp,
+ uint8_t *wqe_size,
+ struct ibv_send_wr *wr)
+{
+ uint32_t data_size = 0;
+ int i;
+
+ for (i = 0; i < wr->num_sge; i++) {
+ struct rdma_sq_sge *sge = qelr_chain_produce(&qp->sq.chain);
+
+ TYPEPTR_ADDR_SET(sge, addr, wr->sg_list[i].addr);
+ sge->l_key = htole32(wr->sg_list[i].lkey);
+ sge->length = htole32(wr->sg_list[i].length);
+ data_size += wr->sg_list[i].length;
+ }
+
+ if (wqe_size)
+ *wqe_size += wr->num_sge;
+
+ return data_size;
+}
+
+static uint32_t qelr_prepare_sq_rdma_data(struct qelr_qp *qp,
+ struct rdma_sq_rdma_wqe_1st *rwqe,
+ struct rdma_sq_rdma_wqe_2nd *rwqe2,
+ struct ibv_send_wr *wr,
+ struct ibv_send_wr **bad_wr)
+{
+ memset(rwqe2, 0, sizeof(*rwqe2));
+ rwqe2->r_key = htole32(wr->wr.rdma.rkey);
+ TYPEPTR_ADDR_SET(rwqe2, remote_va, wr->wr.rdma.remote_addr);
+
+ if (wr->send_flags & IBV_SEND_INLINE) {
+ uint8_t flags = 0;
+
+ SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
+ return qelr_prepare_sq_inline_data(qp, &rwqe->wqe_size, wr,
+ bad_wr, &rwqe->flags, flags);
+ }
+ /* else */
+ qp->edpm.is_edpm = 0;
+
+ return qelr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
+}
+
+static uint32_t qelr_prepare_sq_send_data(struct qelr_qp *qp,
+ struct rdma_sq_send_wqe_1st *swqe,
+ struct rdma_sq_send_wqe_2st *swqe2,
+ struct ibv_send_wr *wr,
+ struct ibv_send_wr **bad_wr)
+{
+ memset(swqe2, 0, sizeof(*swqe2));
+ if (wr->send_flags & IBV_SEND_INLINE) {
+ uint8_t flags = 0;
+
+ SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
+ return qelr_prepare_sq_inline_data(qp, &swqe->wqe_size, wr,
+ bad_wr, &swqe->flags, flags);
+ }
+
+ qp->edpm.is_edpm = 0;
+
+ /* else */
+
+ return qelr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
+}
+
+static enum ibv_wc_opcode qelr_ibv_to_wc_opcode(enum ibv_wr_opcode opcode)
+{
+ switch (opcode) {
+ case IBV_WR_RDMA_WRITE:
+ case IBV_WR_RDMA_WRITE_WITH_IMM:
+ return IBV_WC_RDMA_WRITE;
+ case IBV_WR_SEND_WITH_IMM:
+ case IBV_WR_SEND:
+ return IBV_WC_SEND;
+ case IBV_WR_RDMA_READ:
+ return IBV_WC_RDMA_READ;
+ case IBV_WR_ATOMIC_CMP_AND_SWP:
+ return IBV_WC_COMP_SWAP;
+ case IBV_WR_ATOMIC_FETCH_AND_ADD:
+ return IBV_WC_FETCH_ADD;
+ default:
+ return IBV_WC_SEND;
+ }
+}
+
+static void doorbell_edpm_qp(struct qelr_qp *qp)
+{
+ uint32_t offset = 0;
+ uint64_t data;
+ uint64_t *dpm_payload = (uint64_t *)qp->edpm.dpm_payload;
+ uint32_t num_dwords;
+ int bytes = 0;
+
+ if (!qp->edpm.is_edpm)
+ return;
+
+ wmb();
+
+ qp->edpm.msg.data.icid = qp->sq.db_data.data.icid;
+ qp->edpm.msg.data.prod_val = qp->sq.db_data.data.value;
+
+ writeq(qp->edpm.msg.raw, qp->sq.edpm_db);
+
+ bytes += sizeof(uint64_t);
+
+ num_dwords = (qp->edpm.dpm_payload_size + sizeof(uint64_t) - 1) /
+ sizeof(uint64_t);
+
+ while (offset < num_dwords) {
+ data = dpm_payload[offset];
+
+ writeq(data,
+ qp->sq.edpm_db + sizeof(qp->edpm.msg.data) + offset *
+ sizeof(uint64_t));
+
+ bytes += sizeof(uint64_t);
+ /* Need to place a barrier after every 64 bytes */
+ if (bytes == 64) {
+ wc_wmb();
+ bytes = 0;
+ }
+ offset++;
+ }
+
+ wc_wmb();
+}
+
+int qelr_post_send(struct ibv_qp *ib_qp, struct ibv_send_wr *wr,
+ struct ibv_send_wr **bad_wr)
+{
+ int status = 0;
+ struct qelr_qp *qp = get_qelr_qp(ib_qp);
+ struct qelr_devctx *cxt = get_qelr_ctx(ib_qp->context);
+ uint8_t se, comp, fence;
+ uint16_t db_val;
+ *bad_wr = NULL;
+
+ pthread_spin_lock(&qp->q_lock);
+
+ if (qp->state != QELR_QPS_RTS && qp->state != QELR_QPS_SQD) {
+ pthread_spin_unlock(&qp->q_lock);
+ *bad_wr = wr;
+ return -EINVAL;
+ }
+
+ while (wr) {
+ struct rdma_sq_common_wqe *wqe;
+ struct rdma_sq_send_wqe_1st *swqe;
+ struct rdma_sq_send_wqe_2st *swqe2;
+ struct rdma_sq_rdma_wqe_1st *rwqe;
+ struct rdma_sq_rdma_wqe_2nd *rwqe2;
+ struct rdma_sq_atomic_wqe_1st *awqe1;
+ struct rdma_sq_atomic_wqe_2nd *awqe2;
+ struct rdma_sq_atomic_wqe_3rd *awqe3;
+
+ if ((qelr_chain_get_elem_left_u32(&qp->sq.chain) <
+ QELR_MAX_SQ_WQE_SIZE) ||
+ (wr->num_sge > qp->sq.max_sges)) {
+ status = -ENOMEM;
+ *bad_wr = wr;
+ break;
+ }
+
+ qelr_init_edpm_info(qp, cxt);
+
+ wqe = qelr_chain_produce(&qp->sq.chain);
+
+ comp = (!!(wr->send_flags & IBV_SEND_SIGNALED)) ||
+ (!!qp->sq_sig_all);
+ qp->wqe_wr_id[qp->sq.prod].signaled = comp;
+
+ /* common fields */
+ wqe->flags = 0;
+ se = !!(wr->send_flags & IBV_SEND_SOLICITED);
+ fence = !!(wr->send_flags & IBV_SEND_FENCE);
+ SET_FIELD2(wqe->flags, RDMA_SQ_COMMON_WQE_SE_FLG, se);
+ SET_FIELD2(wqe->flags, RDMA_SQ_COMMON_WQE_COMP_FLG, comp);
+ SET_FIELD2(wqe->flags, RDMA_SQ_COMMON_WQE_RD_FENCE_FLG, fence);
+ wqe->prev_wqe_size = qp->prev_wqe_size;
+
+ qp->wqe_wr_id[qp->sq.prod].opcode =
+ qelr_ibv_to_wc_opcode(wr->opcode);
+
+ switch (wr->opcode) {
+ case IBV_WR_SEND_WITH_IMM:
+ wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
+ swqe = (struct rdma_sq_send_wqe_1st *)wqe;
+
+ swqe->wqe_size = 2;
+ swqe2 = (struct rdma_sq_send_wqe_2st *)
+ qelr_chain_produce(&qp->sq.chain);
+ swqe->inv_key_or_imm_data =
+ htonl(htole32(wr->imm_data));
+ qelr_edpm_set_inv_imm(qp, swqe->inv_key_or_imm_data);
+ swqe->length = htole32(
+ qelr_prepare_sq_send_data(qp, swqe,
+ swqe2, wr,
+ bad_wr));
+ qelr_edpm_set_msg_data(qp,
+ QELR_IB_OPCODE_SEND_ONLY_WITH_IMMEDIATE,
+ swqe->length,
+ se, comp);
+ qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
+ qp->prev_wqe_size = swqe->wqe_size;
+ qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
+ FP_DP_VERBOSE(cxt->dbg_fp, QELR_MSG_CQ,
+ "SEND w/ IMM length = %d imm data=%x\n",
+ swqe->length, wr->imm_data);
+ break;
+
+ case IBV_WR_SEND:
+ wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
+ swqe = (struct rdma_sq_send_wqe_1st *)wqe;
+
+ swqe->wqe_size = 2;
+ swqe2 = (struct rdma_sq_send_wqe_2st *)
+ qelr_chain_produce(&qp->sq.chain);
+ swqe->length = htole32(
+ qelr_prepare_sq_send_data(qp, swqe,
+ swqe2, wr,
+ bad_wr));
+ qelr_edpm_set_msg_data(qp, QELR_IB_OPCODE_SEND_ONLY,
+ swqe->length,
+ se, comp);
+ qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
+ qp->prev_wqe_size = swqe->wqe_size;
+ qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
+ FP_DP_VERBOSE(cxt->dbg_fp, QELR_MSG_CQ,
+ "SEND w/o IMM length = %d\n",
+ swqe->length);
+ break;
+
+ case IBV_WR_RDMA_WRITE_WITH_IMM:
+ wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
+ rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
+
+ rwqe->wqe_size = 2;
+ rwqe->imm_data = htonl(htole32(wr->imm_data));
+ qelr_edpm_set_rdma_ext(qp, wr->wr.rdma.remote_addr,
+ wr->wr.rdma.rkey);
+ qelr_edpm_set_inv_imm(qp, rwqe->imm_data);
+ rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
+ qelr_chain_produce(&qp->sq.chain);
+ rwqe->length = htole32(
+ qelr_prepare_sq_rdma_data(qp, rwqe,
+ rwqe2, wr,
+ bad_wr));
+ qelr_edpm_set_msg_data(qp,
+ QELR_IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE,
+ rwqe->length + sizeof(*qp->edpm.rdma_ext),
+ se, comp);
+ qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
+ qp->prev_wqe_size = rwqe->wqe_size;
+ qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
+ FP_DP_VERBOSE(cxt->dbg_fp, QELR_MSG_CQ,
+ "RDMA WRITE w/ IMM length = %d imm data=%x\n",
+ rwqe->length, rwqe->imm_data);
+ break;
+
+ case IBV_WR_RDMA_WRITE:
+ wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
+ rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
+
+ rwqe->wqe_size = 2;
+ qelr_edpm_set_rdma_ext(qp, wr->wr.rdma.remote_addr,
+ wr->wr.rdma.rkey);
+ rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
+ qelr_chain_produce(&qp->sq.chain);
+ rwqe->length = htole32(
+ qelr_prepare_sq_rdma_data(qp, rwqe, rwqe2, wr,
+ bad_wr));
+ qelr_edpm_set_msg_data(qp,
+ QELR_IB_OPCODE_RDMA_WRITE_ONLY,
+ rwqe->length + sizeof(*qp->edpm.rdma_ext),
+ se, comp);
+ qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
+ qp->prev_wqe_size = rwqe->wqe_size;
+ qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
+ FP_DP_VERBOSE(cxt->dbg_fp, QELR_MSG_CQ,
+ "RDMA WRITE w/o IMM length = %d\n",
+ rwqe->length);
+ break;
+
+ case IBV_WR_RDMA_READ:
+ wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
+ rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
+
+ rwqe->wqe_size = 2;
+ rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
+ qelr_chain_produce(&qp->sq.chain);
+ rwqe->length = htole32(
+ qelr_prepare_sq_rdma_data(qp, rwqe,
+ rwqe2, wr,
+ bad_wr));
+
+ qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
+ qp->prev_wqe_size = rwqe->wqe_size;
+ qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
+ FP_DP_VERBOSE(cxt->dbg_fp, QELR_MSG_CQ,
+ "RDMA READ length = %d\n", rwqe->length);
+ break;
+
+ case IBV_WR_ATOMIC_CMP_AND_SWP:
+ case IBV_WR_ATOMIC_FETCH_AND_ADD:
+ FP_DP_VERBOSE(cxt->dbg_fp, QELR_MSG_CQ, "ATOMIC\n");
+ if (!qp->atomic_supported) {
+ DP_ERR(cxt->dbg_fp,
+ "Atomic not supported on this machine\n");
+ status = -EINVAL;
+ *bad_wr = wr;
+ break;
+ }
+ awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
+ awqe1->wqe_size = 4;
+
+ awqe2 = (struct rdma_sq_atomic_wqe_2nd *)
+ qelr_chain_produce(&qp->sq.chain);
+ TYPEPTR_ADDR_SET(awqe2, remote_va,
+ wr->wr.atomic.remote_addr);
+ awqe2->r_key = htole32(wr->wr.atomic.rkey);
+
+ awqe3 = (struct rdma_sq_atomic_wqe_3rd *)
+ qelr_chain_produce(&qp->sq.chain);
+
+ if (wr->opcode == IBV_WR_ATOMIC_FETCH_AND_ADD) {
+ wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
+ TYPEPTR_ADDR_SET(awqe3, swap_data,
+ wr->wr.atomic.compare_add);
+ } else {
+ wqe->req_type =
+ RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
+ TYPEPTR_ADDR_SET(awqe3, swap_data,
+ wr->wr.atomic.swap);
+ TYPEPTR_ADDR_SET(awqe3, cmp_data,
+ wr->wr.atomic.compare_add);
+ }
+
+ qelr_prepare_sq_sges(qp, NULL, wr);
+
+ qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
+ qp->prev_wqe_size = awqe1->wqe_size;
+
+ break;
+
+ default:
+ *bad_wr = wr;
+ break;
+ }
+
+ if (*bad_wr) {
+ /* restore prod to its position before this WR was
+ * processed
+ */
+ qelr_chain_set_prod(&qp->sq.chain,
+ le16toh(qp->sq.db_data.data.value),
+ wqe);
+ /* restore prev_wqe_size */
+ qp->prev_wqe_size = wqe->prev_wqe_size;
+ status = -EINVAL;
+ DP_ERR(cxt->dbg_fp, "POST SEND FAILED\n");
+ break; /* out of the loop */
+ }
+
+ qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
+
+ qelr_inc_sw_prod_u16(&qp->sq);
+
+ db_val = le16toh(qp->sq.db_data.data.value) + 1;
+ qp->sq.db_data.data.value = htole16(db_val);
+
+ wr = wr->next;
+
+ /* Doorbell */
+ doorbell_edpm_qp(qp);
+ }
+
+ if (!qp->edpm.is_edpm) {
+ wmb();
+
+ writel(qp->sq.db_data.raw, qp->sq.db);
+
+ wc_wmb();
+ }
+
+ pthread_spin_unlock(&qp->q_lock);
+
+ return status;
+}
+
+int qelr_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
+ struct ibv_recv_wr **bad_wr)
+{
+ int status = 0;
+ struct qelr_qp *qp = get_qelr_qp(ibqp);
+ struct qelr_devctx *cxt = get_qelr_ctx(ibqp->context);
+ uint16_t db_val;
+
+ pthread_spin_lock(&qp->q_lock);
+
+ if (qp->state == QELR_QPS_RST || qp->state == QELR_QPS_ERR) {
+ pthread_spin_unlock(&qp->q_lock);
+ *bad_wr = wr;
+ return -EINVAL;
+ }
+
+ while (wr) {
+ int i;
+
+ if (qelr_chain_get_elem_left_u32(&qp->rq.chain) <
+ QELR_MAX_RQ_WQE_SIZE || wr->num_sge > qp->rq.max_sges) {
+ DP_ERR(cxt->dbg_fp,
+ "Can't post WR (%d < %d) || (%d > %d)\n",
+ qelr_chain_get_elem_left_u32(&qp->rq.chain),
+ QELR_MAX_RQ_WQE_SIZE, wr->num_sge,
+ qp->rq.max_sges);
+ status = -ENOMEM;
+ *bad_wr = wr;
+ break;
+ }
+ FP_DP_VERBOSE(cxt->dbg_fp, QELR_MSG_CQ,
+ "RQ WR: SGEs: %d with wr_id[%d] = %lx\n",
+ wr->num_sge, qp->rq.prod, wr->wr_id);
+ for (i = 0; i < wr->num_sge; i++) {
+ uint32_t flags = 0;
+ struct rdma_rq_sge *rqe;
+
+ /* first one must include the number of SGE in the
+ * list
+ */
+ if (!i)
+ SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
+ wr->num_sge);
+
+ SET_FIELD(flags, RDMA_RQ_SGE_L_KEY,
+ wr->sg_list[i].lkey);
+ rqe = qelr_chain_produce(&qp->rq.chain);
+ RQ_SGE_SET(rqe, wr->sg_list[i].addr,
+ wr->sg_list[i].length, flags);
+ FP_DP_VERBOSE(cxt->dbg_fp, QELR_MSG_CQ,
+ "[%d]: len %d key %x addr %x:%x\n", i,
+ rqe->length, rqe->flags, rqe->addr.hi,
+ rqe->addr.lo);
+ }
+ /* Special case of no sges. FW requires between 1-4 sges...
+ * in this case we need to post 1 sge with length zero. this is
+ * because rdma write with immediate consumes an RQ.
+ */
+ if (!wr->num_sge) {
+ uint32_t flags = 0;
+ struct rdma_rq_sge *rqe;
+
+ /* first one must include the number of SGE in the
+ * list
+ */
+ SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
+ SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
+
+ rqe = qelr_chain_produce(&qp->rq.chain);
+ RQ_SGE_SET(rqe, 0, 0, flags);
+ i = 1;
+ }
+
+ qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
+ qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
+
+ qelr_inc_sw_prod_u16(&qp->rq);
+
+ wmb();
+
+ db_val = le16toh(qp->rq.db_data.data.value) + 1;
+ qp->rq.db_data.data.value = htole16(db_val);
+
+ writel(qp->rq.db_data.raw, qp->rq.db);
+
+ wc_wmb();
+
+ wr = wr->next;
+ }
+
+ FP_DP_VERBOSE(cxt->dbg_fp, QELR_MSG_CQ, "POST: Elements in RespQ: %d\n",
+ qelr_chain_get_elem_left_u32(&qp->rq.chain));
+ pthread_spin_unlock(&qp->q_lock);
+
+ return status;
+}
+
+static int is_valid_cqe(struct qelr_cq *cq, union rdma_cqe *cqe)
+{
+ struct rdma_cqe_requester *resp_cqe = &cqe->req;
+
+ return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
+ cq->chain_toggle;
+}
+
+static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
+{
+ struct rdma_cqe_requester *resp_cqe = &cqe->req;
+
+ return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
+}
+
+static struct qelr_qp *cqe_get_qp(union rdma_cqe *cqe)
+{
+ struct rdma_cqe_requester *resp_cqe = &cqe->req;
+ struct qelr_qp *qp;
+
+ qp = (struct qelr_qp *)HILO_U64(resp_cqe->qp_handle.hi,
+ resp_cqe->qp_handle.lo);
+ return qp;
+}
+
+static int process_req(struct qelr_qp *qp, struct qelr_cq *cq, int num_entries,
+ struct ibv_wc *wc, uint16_t hw_cons,
+ enum ibv_wc_status status, int force)
+{
+ struct qelr_devctx *cxt = get_qelr_ctx(qp->ibv_qp.context);
+ uint16_t cnt = 0;
+
+ while (num_entries && qp->sq.wqe_cons != hw_cons) {
+ if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
+ /* skip WC */
+ goto next_cqe;
+ }
+
+ /* fill WC */
+ wc->status = status;
+ wc->wc_flags = 0;
+ wc->qp_num = qp->qp_id;
+
+ /* common section */
+ wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
+ wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
+
+ switch (wc->opcode) {
+ case IBV_WC_RDMA_WRITE:
+ wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_CQ,
+ "POLL REQ CQ: IBV_WC_RDMA_WRITE byte_len=%d\n",
+ qp->wqe_wr_id[qp->sq.cons].bytes_len);
+ break;
+ case IBV_WC_COMP_SWAP:
+ case IBV_WC_FETCH_ADD:
+ wc->byte_len = 8;
+ break;
+ case IBV_WC_RDMA_READ:
+ case IBV_WC_SEND:
+ case IBV_WC_BIND_MW:
+ DP_VERBOSE(cxt->dbg_fp, QELR_MSG_CQ,
+ "POLL REQ CQ: IBV_WC_RDMA_READ / IBV_WC_SEND\n");
+ break;
+ default:
+ break;
+ }
+
+ num_entries--;
+ wc++;
+ cnt++;
+next_cqe:
+ while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
+ qelr_chain_consume(&qp->sq.chain);
+ qelr_inc_sw_cons_u16(&qp->sq);
+ }
+
+ return cnt;
+}
+
+static int qelr_poll_cq_req(struct qelr_qp *qp, struct qelr_cq *cq,
+ int num_entries, struct ibv_wc *wc,
+ struct rdma_cqe_requester *req)
+{
+ struct qelr_devctx *cxt = get_qelr_ctx(qp->ibv_qp.context);
+ int cnt = 0;
+
+ switch (req->status) {
+ case RDMA_CQE_REQ_STS_OK:
+ cnt = process_req(qp, cq, num_entries, wc, req->sq_cons,
+ IBV_WC_SUCCESS, 0);
+ break;
+ case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
+ DP_ERR(cxt->dbg_fp,
+ "Error: POLL CQ with ROCE_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. QP icid=0x%x\n",
+ qp->sq.icid);
+ cnt = process_req(qp, cq, num_entries, wc, req->sq_cons,
+ IBV_WC_WR_FLUSH_ERR, 0);
+ break;
+ default: /* other errors case */
+ /* process all WQE before the consumer */
+ qp->state = QELR_QPS_ERR;
+ cnt = process_req(qp, cq, num_entries, wc, req->sq_cons - 1,
+ IBV_WC_SUCCESS, 0);
+ wc += cnt;
+ /* if we have extra WC fill it with actual error info */
+ if (cnt < num_entries) {
+ enum ibv_wc_status wc_status;
+
+ switch (req->status) {
+ case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
+ DP_ERR(cxt->dbg_fp,
+ "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. QP icid=0x%x\n",
+ qp->sq.icid);
+ wc_status = IBV_WC_BAD_RESP_ERR;
+ break;
+ case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
+ DP_ERR(cxt->dbg_fp,
+ "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. QP icid=0x%x\n",
+ qp->sq.icid);
+ wc_status = IBV_WC_LOC_LEN_ERR;
+ break;
+ case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
+ DP_ERR(cxt->dbg_fp,
+ "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. QP icid=0x%x\n",
+ qp->sq.icid);
+ wc_status = IBV_WC_LOC_QP_OP_ERR;
+ break;
+ case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
+ DP_ERR(cxt->dbg_fp,
+ "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. QP icid=0x%x\n",
+ qp->sq.icid);
+ wc_status = IBV_WC_LOC_PROT_ERR;
+ break;
+ case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
+ DP_ERR(cxt->dbg_fp,
+ "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. QP icid=0x%x\n",
+ qp->sq.icid);
+ wc_status = IBV_WC_MW_BIND_ERR;
+ break;
+ case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
+ DP_ERR(cxt->dbg_fp,
+ "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. QP icid=0x%x\n",
+ qp->sq.icid);
+ wc_status = IBV_WC_REM_INV_REQ_ERR;
+ break;
+ case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
+ DP_ERR(cxt->dbg_fp,
+ "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. QP icid=0x%x\n",
+ qp->sq.icid);
+ wc_status = IBV_WC_REM_ACCESS_ERR;
+ break;
+ case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
+ DP_ERR(cxt->dbg_fp,
+ "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. QP icid=0x%x\n",
+ qp->sq.icid);
+ wc_status = IBV_WC_REM_OP_ERR;
+ break;
+ case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
+ DP_ERR(cxt->dbg_fp,
+ "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. QP icid=0x%x\n",
+ qp->sq.icid);
+ wc_status = IBV_WC_RNR_RETRY_EXC_ERR;
+ break;
+ case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
+ DP_ERR(cxt->dbg_fp,
+ "RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. QP icid=0x%x\n",
+ qp->sq.icid);
+ wc_status = IBV_WC_RETRY_EXC_ERR;
+ break;
+ default:
+ DP_ERR(cxt->dbg_fp,
+ "IBV_WC_GENERAL_ERR. QP icid=0x%x\n",
+ qp->sq.icid);
+ wc_status = IBV_WC_GENERAL_ERR;
+ }
+
+ cnt += process_req(qp, cq, 1, wc, req->sq_cons,
+ wc_status, 1 /* force use of WC */);
+ }
+ }
+
+ return cnt;
+}
+
+static void __process_resp_one(struct qelr_qp *qp, struct qelr_cq *cq,
+ struct ibv_wc *wc,
+ struct rdma_cqe_responder *resp, uint64_t wr_id)
+{
+ struct qelr_devctx *cxt = get_qelr_ctx(qp->ibv_qp.context);
+ enum ibv_wc_status wc_status = IBV_WC_SUCCESS;
+ uint8_t flags;
+
+ wc->opcode = IBV_WC_RECV;
+
+ FP_DP_VERBOSE(cxt->dbg_fp, QELR_MSG_CQ, "\n");
+
+ switch (resp->status) {
+ case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
+ wc_status = IBV_WC_LOC_ACCESS_ERR;
+ break;
+ case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
+ wc_status = IBV_WC_LOC_LEN_ERR;
+ break;
+ case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
+ wc_status = IBV_WC_LOC_QP_OP_ERR;
+ break;
+ case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
+ wc_status = IBV_WC_LOC_PROT_ERR;
+ break;
+ case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
+ wc_status = IBV_WC_MW_BIND_ERR;
+ break;
+ case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
+ wc_status = IBV_WC_REM_INV_RD_REQ_ERR;
+ break;
+ case RDMA_CQE_RESP_STS_OK:
+ wc_status = IBV_WC_SUCCESS;
+ wc->byte_len = le32toh(resp->length);
+
+ flags = resp->flags & QELR_RESP_RDMA_IMM;
+
+ switch (flags) {
+ case QELR_RESP_RDMA_IMM:
+ /* update opcode */
+ wc->opcode = IBV_WC_RECV_RDMA_WITH_IMM;
+ /* fall to set imm data */
+ case QELR_RESP_IMM:
+ wc->imm_data =
+ ntohl(le32toh(resp->imm_data_or_inv_r_Key));
+ wc->wc_flags |= IBV_WC_WITH_IMM;
+ FP_DP_VERBOSE(cxt->dbg_fp, QELR_MSG_CQ,
+ "POLL CQ RQ2: RESP_RDMA_IMM imm_data = %x resp_len=%d\n",
+ wc->imm_data, wc->byte_len);
+ break;
+ case QELR_RESP_RDMA:
+ DP_ERR(cxt->dbg_fp, "Invalid flags detected\n");
+ break;
+ default:
+ /* valid configuration, but nothing to do here */
+ break;
+ }
+
+ wc->wr_id = wr_id;
+ break;
+ default:
+ wc->status = IBV_WC_GENERAL_ERR;
+ DP_ERR(cxt->dbg_fp, "Invalid CQE status detected\n");
+ }
+
+ /* fill WC */
+ wc->status = wc_status;
+ wc->qp_num = qp->qp_id;
+}
+
+static int process_resp_one(struct qelr_qp *qp, struct qelr_cq *cq,
+ struct ibv_wc *wc, struct rdma_cqe_responder *resp)
+{
+ uint64_t wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
+
+ __process_resp_one(qp, cq, wc, resp, wr_id);
+
+ while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
+ qelr_chain_consume(&qp->rq.chain);
+
+ qelr_inc_sw_cons_u16(&qp->rq);
+
+ return 1;
+}
+
+static int process_resp_flush(struct qelr_qp *qp, struct qelr_cq *cq,
+ int num_entries, struct ibv_wc *wc,
+ uint16_t hw_cons)
+{
+ uint16_t cnt = 0;
+
+ while (num_entries && qp->rq.wqe_cons != hw_cons) {
+ /* fill WC */
+ wc->status = IBV_WC_WR_FLUSH_ERR;
+ wc->qp_num = qp->qp_id;
+ wc->byte_len = 0;
+ wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
+ num_entries--;
+ wc++;
+ cnt++;
+ while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
+ qelr_chain_consume(&qp->rq.chain);
+ qelr_inc_sw_cons_u16(&qp->rq);
+ }
+
+ return cnt;
+}
+
+/* return latest CQE (needs processing) */
+static union rdma_cqe *get_cqe(struct qelr_cq *cq)
+{
+ return cq->latest_cqe;
+}
+
+static void try_consume_req_cqe(struct qelr_cq *cq, struct qelr_qp *qp,
+ struct rdma_cqe_requester *req, int *update)
+{
+ if (le16toh(req->sq_cons) == qp->sq.wqe_cons) {
+ consume_cqe(cq);
+ *update |= 1;
+ }
+}
+
+/* used with flush only, when resp->rq_cons is valid */
+static void try_consume_resp_cqe(struct qelr_cq *cq, struct qelr_qp *qp,
+ struct rdma_cqe_responder *resp, int *update)
+{
+ if (le16toh(resp->rq_cons) == qp->rq.wqe_cons) {
+ consume_cqe(cq);
+ *update |= 1;
+ }
+}
+
+static int qelr_poll_cq_resp(struct qelr_qp *qp, struct qelr_cq *cq,
+ int num_entries, struct ibv_wc *wc,
+ struct rdma_cqe_responder *resp, int *update)
+{
+ int cnt;
+
+ if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
+ cnt = process_resp_flush(qp, cq, num_entries, wc,
+ resp->rq_cons);
+ try_consume_resp_cqe(cq, qp, resp, update);
+ } else {
+ cnt = process_resp_one(qp, cq, wc, resp);
+ consume_cqe(cq);
+ *update |= 1;
+ }
+
+ return cnt;
+}
+
+static void doorbell_cq(struct qelr_cq *cq, uint32_t cons, uint8_t flags)
+{
+ wmb();
+ cq->db.data.agg_flags = flags;
+ cq->db.data.value = htole32(cons);
+
+ writeq(cq->db.raw, cq->db_addr);
+ wc_wmb();
+}
+
+int qelr_poll_cq(struct ibv_cq *ibcq, int num_entries, struct ibv_wc *wc)
+{
+ struct qelr_cq *cq = get_qelr_cq(ibcq);
+ int done = 0;
+ union rdma_cqe *cqe = get_cqe(cq);
+ int update = 0;
+ uint32_t db_cons;
+
+ while (num_entries && is_valid_cqe(cq, cqe)) {
+ int cnt = 0;
+ struct qelr_qp *qp;
+
+ /* prevent speculative reads of any field of CQE */
+ rmb();
+
+ qp = cqe_get_qp(cqe);
+ if (!qp) {
+ DP_ERR(stderr,
+ "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
+ break;
+ }
+
+ switch (cqe_get_type(cqe)) {
+ case RDMA_CQE_TYPE_REQUESTER:
+ cnt = qelr_poll_cq_req(qp, cq, num_entries, wc,
+ &cqe->req);
+ try_consume_req_cqe(cq, qp, &cqe->req, &update);
+ break;
+ case RDMA_CQE_TYPE_RESPONDER_RQ:
+ cnt = qelr_poll_cq_resp(qp, cq, num_entries, wc,
+ &cqe->resp, &update);
+ break;
+ case RDMA_CQE_TYPE_INVALID:
+ default:
+ printf("Error: invalid CQE type = %d\n",
+ cqe_get_type(cqe));
+ }
+ num_entries -= cnt;
+ wc += cnt;
+ done += cnt;
+
+ cqe = get_cqe(cq);
+ }
+
+ db_cons = qelr_chain_get_cons_idx_u32(&cq->chain) - 1;
+ if (update) {
+ /* doorbell notifies about latest VALID entry,
+ * but chain already point to the next INVALID one
+ */
+ doorbell_cq(cq, db_cons, cq->arm_flags);
+ FP_DP_VERBOSE(stderr, QELR_MSG_CQ, "doorbell_cq cons=%x\n",
+ db_cons);
+ }
+
+ return done;
+}
+
+void qelr_cq_event(struct ibv_cq *ibcq)
+{
+ /* Trigger received, can reset arm flags */
+ struct qelr_cq *cq = get_qelr_cq(ibcq);
+
+ cq->arm_flags = 0;
+}
+
+int qelr_arm_cq(struct ibv_cq *ibcq, int solicited)
+{
+ struct qelr_cq *cq = get_qelr_cq(ibcq);
+ uint32_t db_cons;
+
+ db_cons = qelr_chain_get_cons_idx_u32(&cq->chain) - 1;
+ FP_DP_VERBOSE(get_qelr_ctx(ibcq->context)->dbg_fp, QELR_MSG_CQ,
+ "Arm CQ cons=%x solicited=%d\n", db_cons, solicited);
+
+ cq->arm_flags = solicited ? DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD :
+ DQ_UCM_ROCE_CQ_ARM_CF_CMD;
+
+ doorbell_cq(cq, db_cons, cq->arm_flags);
+
+ return 0;
+}
+
+void qelr_async_event(struct ibv_async_event *event)
+{
+ struct qelr_cq *cq = NULL;
+ struct qelr_qp *qp = NULL;
+
+ switch (event->event_type) {
+ case IBV_EVENT_CQ_ERR:
+ cq = get_qelr_cq(event->element.cq);
+ break;
+ case IBV_EVENT_QP_FATAL:
+ case IBV_EVENT_QP_REQ_ERR:
+ case IBV_EVENT_QP_ACCESS_ERR:
+ case IBV_EVENT_PATH_MIG_ERR:{
+ qp = get_qelr_qp(event->element.qp);
+ break;
+ }
+ case IBV_EVENT_SQ_DRAINED:
+ case IBV_EVENT_PATH_MIG:
+ case IBV_EVENT_COMM_EST:
+ case IBV_EVENT_QP_LAST_WQE_REACHED:
+ break;
+ case IBV_EVENT_PORT_ACTIVE:
+ case IBV_EVENT_PORT_ERR:
+ break;
+ default:
+ break;
+ }
+
+ fprintf(stderr, "qelr_async_event not implemented yet cq=%p qp=%p\n",
+ cq, qp);
+}
diff --git a/providers/qedr/qelr_verbs.h b/providers/qedr/qelr_verbs.h
new file mode 100644
index 0000000..f10b76b
--- /dev/null
+++ b/providers/qedr/qelr_verbs.h
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2015-2016 QLogic Corporation
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and /or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef __QELR_VERBS_H__
+#define __QELR_VERBS_H__
+
+#include <inttypes.h>
+#include <stddef.h>
+#include <endian.h>
+
+#include <infiniband/driver.h>
+#include <infiniband/arch.h>
+
+struct ibv_device *qelr_driver_init(const char *, int);
+
+int qelr_query_device(struct ibv_context *, struct ibv_device_attr *);
+int qelr_query_port(struct ibv_context *, uint8_t, struct ibv_port_attr *);
+
+struct ibv_pd *qelr_alloc_pd(struct ibv_context *);
+int qelr_dealloc_pd(struct ibv_pd *);
+
+struct ibv_mr *qelr_reg_mr(struct ibv_pd *, void *, size_t,
+ int ibv_access_flags);
+int qelr_dereg_mr(struct ibv_mr *);
+
+struct ibv_cq *qelr_create_cq(struct ibv_context *, int,
+ struct ibv_comp_channel *, int);
+int qelr_destroy_cq(struct ibv_cq *);
+int qelr_poll_cq(struct ibv_cq *, int, struct ibv_wc *);
+void qelr_cq_event(struct ibv_cq *);
+int qelr_arm_cq(struct ibv_cq *, int);
+
+int qelr_query_srq(struct ibv_srq *ibv_srq, struct ibv_srq_attr *attr);
+int qelr_modify_srq(struct ibv_srq *ibv_srq, struct ibv_srq_attr *attr,
+ int attr_mask);
+struct ibv_srq *qelr_create_srq(struct ibv_pd *, struct ibv_srq_init_attr *);
+int qelr_destroy_srq(struct ibv_srq *ibv_srq);
+int qelr_post_srq_recv(struct ibv_srq *, struct ibv_recv_wr *,
+ struct ibv_recv_wr **bad_wr);
+
+struct ibv_qp *qelr_create_qp(struct ibv_pd *, struct ibv_qp_init_attr *);
+int qelr_modify_qp(struct ibv_qp *, struct ibv_qp_attr *,
+ int ibv_qp_attr_mask);
+int qelr_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask,
+ struct ibv_qp_init_attr *init_attr);
+int qelr_destroy_qp(struct ibv_qp *);
+
+int qelr_post_send(struct ibv_qp *, struct ibv_send_wr *,
+ struct ibv_send_wr **);
+int qelr_post_recv(struct ibv_qp *, struct ibv_recv_wr *,
+ struct ibv_recv_wr **);
+
+void qelr_async_event(struct ibv_async_event *event);
+#endif /* __QELR_VERBS_H__ */
--
2.7.4
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox