[PATCH 0/4] Last NFS/RDMA client patches for v4.15

[PATCH 0/4] Last NFS/RDMA client patches for v4.15

[Chuck Lever]

Hi Anna-

Here's my final set of NFS/RDMA client patches for v4.15.

Three of these are clean up patches. The first patch is an important
change, however.

In the earlier set of patches, I changed the Receive CQ to run in
workqueue context. Here, I change the Send CQ to also run in
workqueue context.

In addition, I move it to a different compvec than the Receive CQ.
My performance measurements show that the two completion handlers
interfere with each other if they run in the same workqueue thread.

Please consider this set for v4.15.

---

Chuck Lever (4):
      xprtrdma: Put Send CQ in IB_POLL_WORKQUEUE mode
      rpcrdma: Remove C structure definitions of XDR data items
      xprtrdma: Remove include for linux/prefetch.h
      xprtrdma: Update copyright notices


 include/linux/sunrpc/rpc_rdma.h |   60 +--------------------------------------
 net/sunrpc/xprtrdma/rpc_rdma.c  |    7 +++--
 net/sunrpc/xprtrdma/transport.c |    1 +
 net/sunrpc/xprtrdma/verbs.c     |    4 +--
 net/sunrpc/xprtrdma/xprt_rdma.h |    7 +----
 5 files changed, 9 insertions(+), 70 deletions(-)

--
Chuck Lever

[PATCH 1/4] xprtrdma: Put Send CQ in IB_POLL_WORKQUEUE mode

[Chuck Lever]

Lift the Send and LocalInv completion handlers out of soft IRQ mode
to make room for other work. Also, move the Send CQ to a different
CPU than the CPU where the Receive CQ is running, for improved
scalability.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
 net/sunrpc/xprtrdma/verbs.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index 22128a8..4cfa893 100644
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -558,7 +558,7 @@
 
 	sendcq = ib_alloc_cq(ia->ri_device, NULL,
 			     ep->rep_attr.cap.max_send_wr + 1,
-			     0, IB_POLL_SOFTIRQ);
+			     1, IB_POLL_WORKQUEUE);
 	if (IS_ERR(sendcq)) {
 		rc = PTR_ERR(sendcq);
 		dprintk("RPC:       %s: failed to create send CQ: %i\n",

[PATCH 2/4] rpcrdma: Remove C structure definitions of XDR data items

[Chuck Lever]

Clean up: XDR structures have been replaced in all client and server
transport code.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
 include/linux/sunrpc/rpc_rdma.h |   59 ---------------------------------------
 net/sunrpc/xprtrdma/rpc_rdma.c  |    6 ++--
 net/sunrpc/xprtrdma/xprt_rdma.h |    6 ----
 3 files changed, 3 insertions(+), 68 deletions(-)

diff --git a/include/linux/sunrpc/rpc_rdma.h b/include/linux/sunrpc/rpc_rdma.h
index b7e85b3..840afac 100644
--- a/include/linux/sunrpc/rpc_rdma.h
+++ b/include/linux/sunrpc/rpc_rdma.h
@@ -50,65 +50,6 @@ enum {
 	RPCRDMA_V1_DEF_INLINE_SIZE	= 1024,
 };
 
-struct rpcrdma_segment {
-	__be32 rs_handle;	/* Registered memory handle */
-	__be32 rs_length;	/* Length of the chunk in bytes */
-	__be64 rs_offset;	/* Chunk virtual address or offset */
-};
-
-/*
- * read chunk(s), encoded as a linked list.
- */
-struct rpcrdma_read_chunk {
-	__be32 rc_discrim;	/* 1 indicates presence */
-	__be32 rc_position;	/* Position in XDR stream */
-	struct rpcrdma_segment rc_target;
-};
-
-/*
- * write chunk, and reply chunk.
- */
-struct rpcrdma_write_chunk {
-	struct rpcrdma_segment wc_target;
-};
-
-/*
- * write chunk(s), encoded as a counted array.
- */
-struct rpcrdma_write_array {
-	__be32 wc_discrim;	/* 1 indicates presence */
-	__be32 wc_nchunks;	/* Array count */
-	struct rpcrdma_write_chunk wc_array[0];
-};
-
-struct rpcrdma_msg {
-	__be32 rm_xid;	/* Mirrors the RPC header xid */
-	__be32 rm_vers;	/* Version of this protocol */
-	__be32 rm_credit;	/* Buffers requested/granted */
-	__be32 rm_type;	/* Type of message (enum rpcrdma_proc) */
-	union {
-
-		struct {			/* no chunks */
-			__be32 rm_empty[3];	/* 3 empty chunk lists */
-		} rm_nochunks;
-
-		struct {			/* no chunks and padded */
-			__be32 rm_align;	/* Padding alignment */
-			__be32 rm_thresh;	/* Padding threshold */
-			__be32 rm_pempty[3];	/* 3 empty chunk lists */
-		} rm_padded;
-
-		struct {
-			__be32 rm_err;
-			__be32 rm_vers_low;
-			__be32 rm_vers_high;
-		} rm_error;
-
-		__be32 rm_chunks[0];	/* read, write and reply chunks */
-
-	} rm_body;
-};
-
 /*
  * XDR sizes, in quads
  */
diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c
index 4fdeaac..45cb549 100644
--- a/net/sunrpc/xprtrdma/rpc_rdma.c
+++ b/net/sunrpc/xprtrdma/rpc_rdma.c
@@ -75,11 +75,11 @@ static unsigned int rpcrdma_max_call_header_size(unsigned int maxsegs)
 
 	/* Maximum Read list size */
 	maxsegs += 2;	/* segment for head and tail buffers */
-	size = maxsegs * sizeof(struct rpcrdma_read_chunk);
+	size = maxsegs * rpcrdma_readchunk_maxsz * sizeof(__be32);
 
 	/* Minimal Read chunk size */
 	size += sizeof(__be32);	/* segment count */
-	size += sizeof(struct rpcrdma_segment);
+	size += rpcrdma_segment_maxsz * sizeof(__be32);
 	size += sizeof(__be32);	/* list discriminator */
 
 	dprintk("RPC:       %s: max call header size = %u\n",
@@ -102,7 +102,7 @@ static unsigned int rpcrdma_max_reply_header_size(unsigned int maxsegs)
 	/* Maximum Write list size */
 	maxsegs += 2;	/* segment for head and tail buffers */
 	size = sizeof(__be32);		/* segment count */
-	size += maxsegs * sizeof(struct rpcrdma_segment);
+	size += maxsegs * rpcrdma_segment_maxsz * sizeof(__be32);
 	size += sizeof(__be32);	/* list discriminator */
 
 	dprintk("RPC:       %s: max reply header size = %u\n",
diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
index 6e64c82..8b9954c 100644
--- a/net/sunrpc/xprtrdma/xprt_rdma.h
+++ b/net/sunrpc/xprtrdma/xprt_rdma.h
@@ -145,12 +145,6 @@ struct rpcrdma_regbuf {
 	return rb->rg_iov.lkey;
 }
 
-static inline struct rpcrdma_msg *
-rdmab_to_msg(struct rpcrdma_regbuf *rb)
-{
-	return (struct rpcrdma_msg *)rb->rg_base;
-}
-
 static inline struct ib_device *
 rdmab_device(struct rpcrdma_regbuf *rb)
 {

[PATCH 3/4] xprtrdma: Remove include for linux/prefetch.h

[Chuck Lever]

Clean up. This include should have been removed by
commit 23826c7aeac7 ("xprtrdma: Serialize credit accounting again").

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
 net/sunrpc/xprtrdma/verbs.c |    1 -
 1 file changed, 1 deletion(-)

diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index 4cfa893..be61c29 100644
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -49,7 +49,6 @@
 
 #include <linux/interrupt.h>
 #include <linux/slab.h>
-#include <linux/prefetch.h>
 #include <linux/sunrpc/addr.h>
 #include <linux/sunrpc/svc_rdma.h>
 

[PATCH 4/4] xprtrdma: Update copyright notices

[Chuck Lever]

Credit work contributed by Oracle engineers since 2014.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
 include/linux/sunrpc/rpc_rdma.h |    1 +
 net/sunrpc/xprtrdma/rpc_rdma.c  |    1 +
 net/sunrpc/xprtrdma/transport.c |    1 +
 net/sunrpc/xprtrdma/verbs.c     |    1 +
 net/sunrpc/xprtrdma/xprt_rdma.h |    1 +
 5 files changed, 5 insertions(+)

diff --git a/include/linux/sunrpc/rpc_rdma.h b/include/linux/sunrpc/rpc_rdma.h
index 840afac..8f144db 100644
--- a/include/linux/sunrpc/rpc_rdma.h
+++ b/include/linux/sunrpc/rpc_rdma.h
@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2015-2017 Oracle. All rights reserved.
  * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c
index 45cb549..ed34dc0 100644
--- a/net/sunrpc/xprtrdma/rpc_rdma.c
+++ b/net/sunrpc/xprtrdma/rpc_rdma.c
@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2014-2017 Oracle.  All rights reserved.
  * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c
index dcc2df0..04494bb 100644
--- a/net/sunrpc/xprtrdma/transport.c
+++ b/net/sunrpc/xprtrdma/transport.c
@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2014-2017 Oracle.  All rights reserved.
  * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index be61c29..710b3f7 100644
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2014-2017 Oracle.  All rights reserved.
  * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
index 8b9954c..51686d9 100644
--- a/net/sunrpc/xprtrdma/xprt_rdma.h
+++ b/net/sunrpc/xprtrdma/xprt_rdma.h
@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2014-2017 Oracle.  All rights reserved.
  * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two

Re: [PATCH 1/4] xprtrdma: Put Send CQ in IB_POLL_WORKQUEUE mode

[Devesh Sharma]

Looks good
Reviewed-By: Devesh Sharma <devesh.sharma@broadcom.com>

On Fri, Oct 27, 2017 at 10:13 PM, Chuck Lever <chuck.lever@oracle.com> wrote:
> Lift the Send and LocalInv completion handlers out of soft IRQ mode
> to make room for other work. Also, move the Send CQ to a different
> CPU than the CPU where the Receive CQ is running, for improved
> scalability.
>
> Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
> ---
>  net/sunrpc/xprtrdma/verbs.c |    2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
> index 22128a8..4cfa893 100644
> --- a/net/sunrpc/xprtrdma/verbs.c
> +++ b/net/sunrpc/xprtrdma/verbs.c
> @@ -558,7 +558,7 @@
>
>         sendcq = ib_alloc_cq(ia->ri_device, NULL,
>                              ep->rep_attr.cap.max_send_wr + 1,
> -                            0, IB_POLL_SOFTIRQ);
> +                            1, IB_POLL_WORKQUEUE);
>         if (IS_ERR(sendcq)) {
>                 rc = PTR_ERR(sendcq);
>                 dprintk("RPC:       %s: failed to create send CQ: %i\n",
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 3/4] xprtrdma: Remove include for linux/prefetch.h

[Devesh Sharma]

looks good,

Reveiwed-By: Devesh Sharma <devesh.sharma@broadcom.com>

On Fri, Oct 27, 2017 at 10:14 PM, Chuck Lever <chuck.lever@oracle.com> wrote:
> Clean up. This include should have been removed by
> commit 23826c7aeac7 ("xprtrdma: Serialize credit accounting again").
>
> Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
> ---
>  net/sunrpc/xprtrdma/verbs.c |    1 -
>  1 file changed, 1 deletion(-)
>
> diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
> index 4cfa893..be61c29 100644
> --- a/net/sunrpc/xprtrdma/verbs.c
> +++ b/net/sunrpc/xprtrdma/verbs.c
> @@ -49,7 +49,6 @@
>
>  #include <linux/interrupt.h>
>  #include <linux/slab.h>
> -#include <linux/prefetch.h>
>  #include <linux/sunrpc/addr.h>
>  #include <linux/sunrpc/svc_rdma.h>
>
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 2/4] rpcrdma: Remove C structure definitions of XDR data items

[Devesh Sharma]

Hi Chuck,

Could add a pointer to the patch which replaced things with the new
data-structure in the commit message

On Fri, Oct 27, 2017 at 10:14 PM, Chuck Lever <chuck.lever@oracle.com> wrote:
> Clean up: XDR structures have been replaced in all client and server
> transport code.
>
> Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
> ---
>  include/linux/sunrpc/rpc_rdma.h |   59 ---------------------------------------
>  net/sunrpc/xprtrdma/rpc_rdma.c  |    6 ++--
>  net/sunrpc/xprtrdma/xprt_rdma.h |    6 ----
>  3 files changed, 3 insertions(+), 68 deletions(-)
>
> diff --git a/include/linux/sunrpc/rpc_rdma.h b/include/linux/sunrpc/rpc_rdma.h
> index b7e85b3..840afac 100644
> --- a/include/linux/sunrpc/rpc_rdma.h
> +++ b/include/linux/sunrpc/rpc_rdma.h
> @@ -50,65 +50,6 @@ enum {
>         RPCRDMA_V1_DEF_INLINE_SIZE      = 1024,
>  };
>
> -struct rpcrdma_segment {
> -       __be32 rs_handle;       /* Registered memory handle */
> -       __be32 rs_length;       /* Length of the chunk in bytes */
> -       __be64 rs_offset;       /* Chunk virtual address or offset */
> -};
> -
> -/*
> - * read chunk(s), encoded as a linked list.
> - */
> -struct rpcrdma_read_chunk {
> -       __be32 rc_discrim;      /* 1 indicates presence */
> -       __be32 rc_position;     /* Position in XDR stream */
> -       struct rpcrdma_segment rc_target;
> -};
> -
> -/*
> - * write chunk, and reply chunk.
> - */
> -struct rpcrdma_write_chunk {
> -       struct rpcrdma_segment wc_target;
> -};
> -
> -/*
> - * write chunk(s), encoded as a counted array.
> - */
> -struct rpcrdma_write_array {
> -       __be32 wc_discrim;      /* 1 indicates presence */
> -       __be32 wc_nchunks;      /* Array count */
> -       struct rpcrdma_write_chunk wc_array[0];
> -};
> -
> -struct rpcrdma_msg {
> -       __be32 rm_xid;  /* Mirrors the RPC header xid */
> -       __be32 rm_vers; /* Version of this protocol */
> -       __be32 rm_credit;       /* Buffers requested/granted */
> -       __be32 rm_type; /* Type of message (enum rpcrdma_proc) */
> -       union {
> -
> -               struct {                        /* no chunks */
> -                       __be32 rm_empty[3];     /* 3 empty chunk lists */
> -               } rm_nochunks;
> -
> -               struct {                        /* no chunks and padded */
> -                       __be32 rm_align;        /* Padding alignment */
> -                       __be32 rm_thresh;       /* Padding threshold */
> -                       __be32 rm_pempty[3];    /* 3 empty chunk lists */
> -               } rm_padded;
> -
> -               struct {
> -                       __be32 rm_err;
> -                       __be32 rm_vers_low;
> -                       __be32 rm_vers_high;
> -               } rm_error;
> -
> -               __be32 rm_chunks[0];    /* read, write and reply chunks */
> -
> -       } rm_body;
> -};
> -
>  /*
>   * XDR sizes, in quads
>   */
> diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c
> index 4fdeaac..45cb549 100644
> --- a/net/sunrpc/xprtrdma/rpc_rdma.c
> +++ b/net/sunrpc/xprtrdma/rpc_rdma.c
> @@ -75,11 +75,11 @@ static unsigned int rpcrdma_max_call_header_size(unsigned int maxsegs)
>
>         /* Maximum Read list size */
>         maxsegs += 2;   /* segment for head and tail buffers */
> -       size = maxsegs * sizeof(struct rpcrdma_read_chunk);
> +       size = maxsegs * rpcrdma_readchunk_maxsz * sizeof(__be32);
>
>         /* Minimal Read chunk size */
>         size += sizeof(__be32); /* segment count */
> -       size += sizeof(struct rpcrdma_segment);
> +       size += rpcrdma_segment_maxsz * sizeof(__be32);
>         size += sizeof(__be32); /* list discriminator */
>
>         dprintk("RPC:       %s: max call header size = %u\n",
> @@ -102,7 +102,7 @@ static unsigned int rpcrdma_max_reply_header_size(unsigned int maxsegs)
>         /* Maximum Write list size */
>         maxsegs += 2;   /* segment for head and tail buffers */
>         size = sizeof(__be32);          /* segment count */
> -       size += maxsegs * sizeof(struct rpcrdma_segment);
> +       size += maxsegs * rpcrdma_segment_maxsz * sizeof(__be32);
>         size += sizeof(__be32); /* list discriminator */
>
>         dprintk("RPC:       %s: max reply header size = %u\n",
> diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
> index 6e64c82..8b9954c 100644
> --- a/net/sunrpc/xprtrdma/xprt_rdma.h
> +++ b/net/sunrpc/xprtrdma/xprt_rdma.h
> @@ -145,12 +145,6 @@ struct rpcrdma_regbuf {
>         return rb->rg_iov.lkey;
>  }
>
> -static inline struct rpcrdma_msg *
> -rdmab_to_msg(struct rpcrdma_regbuf *rb)
> -{
> -       return (struct rpcrdma_msg *)rb->rg_base;
> -}
> -
>  static inline struct ib_device *
>  rdmab_device(struct rpcrdma_regbuf *rb)
>  {
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 2/4] rpcrdma: Remove C structure definitions of XDR data items

[Chuck Lever]

> On Oct 28, 2017, at 3:38 AM, Devesh Sharma <devesh.sharma@broadcom.com> wrote:
> 
> Hi Chuck,

Hi Devesh, thanks for the review!


> Could add a pointer to the patch which replaced things with the new
> data-structure in the commit message

How about this:

"Clean up: C-structure style XDR encoding and decoding logic has
been replaced over the past several merge windows on both the
client and server. These data structures are no longer used."


> On Fri, Oct 27, 2017 at 10:14 PM, Chuck Lever <chuck.lever@oracle.com> wrote:
>> Clean up: XDR structures have been replaced in all client and server
>> transport code.
>> 
>> Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
>> ---
>> include/linux/sunrpc/rpc_rdma.h |   59 ---------------------------------------
>> net/sunrpc/xprtrdma/rpc_rdma.c  |    6 ++--
>> net/sunrpc/xprtrdma/xprt_rdma.h |    6 ----
>> 3 files changed, 3 insertions(+), 68 deletions(-)
>> 
>> diff --git a/include/linux/sunrpc/rpc_rdma.h b/include/linux/sunrpc/rpc_rdma.h
>> index b7e85b3..840afac 100644
>> --- a/include/linux/sunrpc/rpc_rdma.h
>> +++ b/include/linux/sunrpc/rpc_rdma.h
>> @@ -50,65 +50,6 @@ enum {
>>        RPCRDMA_V1_DEF_INLINE_SIZE      = 1024,
>> };
>> 
>> -struct rpcrdma_segment {
>> -       __be32 rs_handle;       /* Registered memory handle */
>> -       __be32 rs_length;       /* Length of the chunk in bytes */
>> -       __be64 rs_offset;       /* Chunk virtual address or offset */
>> -};
>> -
>> -/*
>> - * read chunk(s), encoded as a linked list.
>> - */
>> -struct rpcrdma_read_chunk {
>> -       __be32 rc_discrim;      /* 1 indicates presence */
>> -       __be32 rc_position;     /* Position in XDR stream */
>> -       struct rpcrdma_segment rc_target;
>> -};
>> -
>> -/*
>> - * write chunk, and reply chunk.
>> - */
>> -struct rpcrdma_write_chunk {
>> -       struct rpcrdma_segment wc_target;
>> -};
>> -
>> -/*
>> - * write chunk(s), encoded as a counted array.
>> - */
>> -struct rpcrdma_write_array {
>> -       __be32 wc_discrim;      /* 1 indicates presence */
>> -       __be32 wc_nchunks;      /* Array count */
>> -       struct rpcrdma_write_chunk wc_array[0];
>> -};
>> -
>> -struct rpcrdma_msg {
>> -       __be32 rm_xid;  /* Mirrors the RPC header xid */
>> -       __be32 rm_vers; /* Version of this protocol */
>> -       __be32 rm_credit;       /* Buffers requested/granted */
>> -       __be32 rm_type; /* Type of message (enum rpcrdma_proc) */
>> -       union {
>> -
>> -               struct {                        /* no chunks */
>> -                       __be32 rm_empty[3];     /* 3 empty chunk lists */
>> -               } rm_nochunks;
>> -
>> -               struct {                        /* no chunks and padded */
>> -                       __be32 rm_align;        /* Padding alignment */
>> -                       __be32 rm_thresh;       /* Padding threshold */
>> -                       __be32 rm_pempty[3];    /* 3 empty chunk lists */
>> -               } rm_padded;
>> -
>> -               struct {
>> -                       __be32 rm_err;
>> -                       __be32 rm_vers_low;
>> -                       __be32 rm_vers_high;
>> -               } rm_error;
>> -
>> -               __be32 rm_chunks[0];    /* read, write and reply chunks */
>> -
>> -       } rm_body;
>> -};
>> -
>> /*
>>  * XDR sizes, in quads
>>  */
>> diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c
>> index 4fdeaac..45cb549 100644
>> --- a/net/sunrpc/xprtrdma/rpc_rdma.c
>> +++ b/net/sunrpc/xprtrdma/rpc_rdma.c
>> @@ -75,11 +75,11 @@ static unsigned int rpcrdma_max_call_header_size(unsigned int maxsegs)
>> 
>>        /* Maximum Read list size */
>>        maxsegs += 2;   /* segment for head and tail buffers */
>> -       size = maxsegs * sizeof(struct rpcrdma_read_chunk);
>> +       size = maxsegs * rpcrdma_readchunk_maxsz * sizeof(__be32);
>> 
>>        /* Minimal Read chunk size */
>>        size += sizeof(__be32); /* segment count */
>> -       size += sizeof(struct rpcrdma_segment);
>> +       size += rpcrdma_segment_maxsz * sizeof(__be32);
>>        size += sizeof(__be32); /* list discriminator */
>> 
>>        dprintk("RPC:       %s: max call header size = %u\n",
>> @@ -102,7 +102,7 @@ static unsigned int rpcrdma_max_reply_header_size(unsigned int maxsegs)
>>        /* Maximum Write list size */
>>        maxsegs += 2;   /* segment for head and tail buffers */
>>        size = sizeof(__be32);          /* segment count */
>> -       size += maxsegs * sizeof(struct rpcrdma_segment);
>> +       size += maxsegs * rpcrdma_segment_maxsz * sizeof(__be32);
>>        size += sizeof(__be32); /* list discriminator */
>> 
>>        dprintk("RPC:       %s: max reply header size = %u\n",
>> diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
>> index 6e64c82..8b9954c 100644
>> --- a/net/sunrpc/xprtrdma/xprt_rdma.h
>> +++ b/net/sunrpc/xprtrdma/xprt_rdma.h
>> @@ -145,12 +145,6 @@ struct rpcrdma_regbuf {
>>        return rb->rg_iov.lkey;
>> }
>> 
>> -static inline struct rpcrdma_msg *
>> -rdmab_to_msg(struct rpcrdma_regbuf *rb)
>> -{
>> -       return (struct rpcrdma_msg *)rb->rg_base;
>> -}
>> -
>> static inline struct ib_device *
>> rdmab_device(struct rpcrdma_regbuf *rb)
>> {
>> 
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
>> the body of a message to majordomo@vger.kernel.org
>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

--
Chuck Lever

Re: [PATCH 2/4] rpcrdma: Remove C structure definitions of XDR data items

[Devesh Sharma]

Okay looks good to me.

-Regards
Devesh

On Sun, Oct 29, 2017 at 10:58 PM, Chuck Lever <chuck.lever@oracle.com> wrote:
>
>> On Oct 28, 2017, at 3:38 AM, Devesh Sharma <devesh.sharma@broadcom.com> wrote:
>>
>> Hi Chuck,
>
> Hi Devesh, thanks for the review!
>
>
>> Could add a pointer to the patch which replaced things with the new
>> data-structure in the commit message
>
> How about this:
>
> "Clean up: C-structure style XDR encoding and decoding logic has
> been replaced over the past several merge windows on both the
> client and server. These data structures are no longer used."
>
>
>> On Fri, Oct 27, 2017 at 10:14 PM, Chuck Lever <chuck.lever@oracle.com> wrote:
>>> Clean up: XDR structures have been replaced in all client and server
>>> transport code.
>>>
>>> Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
>>> ---
>>> include/linux/sunrpc/rpc_rdma.h |   59 ---------------------------------------
>>> net/sunrpc/xprtrdma/rpc_rdma.c  |    6 ++--
>>> net/sunrpc/xprtrdma/xprt_rdma.h |    6 ----
>>> 3 files changed, 3 insertions(+), 68 deletions(-)
>>>
>>> diff --git a/include/linux/sunrpc/rpc_rdma.h b/include/linux/sunrpc/rpc_rdma.h
>>> index b7e85b3..840afac 100644
>>> --- a/include/linux/sunrpc/rpc_rdma.h
>>> +++ b/include/linux/sunrpc/rpc_rdma.h
>>> @@ -50,65 +50,6 @@ enum {
>>>        RPCRDMA_V1_DEF_INLINE_SIZE      = 1024,
>>> };
>>>
>>> -struct rpcrdma_segment {
>>> -       __be32 rs_handle;       /* Registered memory handle */
>>> -       __be32 rs_length;       /* Length of the chunk in bytes */
>>> -       __be64 rs_offset;       /* Chunk virtual address or offset */
>>> -};
>>> -
>>> -/*
>>> - * read chunk(s), encoded as a linked list.
>>> - */
>>> -struct rpcrdma_read_chunk {
>>> -       __be32 rc_discrim;      /* 1 indicates presence */
>>> -       __be32 rc_position;     /* Position in XDR stream */
>>> -       struct rpcrdma_segment rc_target;
>>> -};
>>> -
>>> -/*
>>> - * write chunk, and reply chunk.
>>> - */
>>> -struct rpcrdma_write_chunk {
>>> -       struct rpcrdma_segment wc_target;
>>> -};
>>> -
>>> -/*
>>> - * write chunk(s), encoded as a counted array.
>>> - */
>>> -struct rpcrdma_write_array {
>>> -       __be32 wc_discrim;      /* 1 indicates presence */
>>> -       __be32 wc_nchunks;      /* Array count */
>>> -       struct rpcrdma_write_chunk wc_array[0];
>>> -};
>>> -
>>> -struct rpcrdma_msg {
>>> -       __be32 rm_xid;  /* Mirrors the RPC header xid */
>>> -       __be32 rm_vers; /* Version of this protocol */
>>> -       __be32 rm_credit;       /* Buffers requested/granted */
>>> -       __be32 rm_type; /* Type of message (enum rpcrdma_proc) */
>>> -       union {
>>> -
>>> -               struct {                        /* no chunks */
>>> -                       __be32 rm_empty[3];     /* 3 empty chunk lists */
>>> -               } rm_nochunks;
>>> -
>>> -               struct {                        /* no chunks and padded */
>>> -                       __be32 rm_align;        /* Padding alignment */
>>> -                       __be32 rm_thresh;       /* Padding threshold */
>>> -                       __be32 rm_pempty[3];    /* 3 empty chunk lists */
>>> -               } rm_padded;
>>> -
>>> -               struct {
>>> -                       __be32 rm_err;
>>> -                       __be32 rm_vers_low;
>>> -                       __be32 rm_vers_high;
>>> -               } rm_error;
>>> -
>>> -               __be32 rm_chunks[0];    /* read, write and reply chunks */
>>> -
>>> -       } rm_body;
>>> -};
>>> -
>>> /*
>>>  * XDR sizes, in quads
>>>  */
>>> diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c
>>> index 4fdeaac..45cb549 100644
>>> --- a/net/sunrpc/xprtrdma/rpc_rdma.c
>>> +++ b/net/sunrpc/xprtrdma/rpc_rdma.c
>>> @@ -75,11 +75,11 @@ static unsigned int rpcrdma_max_call_header_size(unsigned int maxsegs)
>>>
>>>        /* Maximum Read list size */
>>>        maxsegs += 2;   /* segment for head and tail buffers */
>>> -       size = maxsegs * sizeof(struct rpcrdma_read_chunk);
>>> +       size = maxsegs * rpcrdma_readchunk_maxsz * sizeof(__be32);
>>>
>>>        /* Minimal Read chunk size */
>>>        size += sizeof(__be32); /* segment count */
>>> -       size += sizeof(struct rpcrdma_segment);
>>> +       size += rpcrdma_segment_maxsz * sizeof(__be32);
>>>        size += sizeof(__be32); /* list discriminator */
>>>
>>>        dprintk("RPC:       %s: max call header size = %u\n",
>>> @@ -102,7 +102,7 @@ static unsigned int rpcrdma_max_reply_header_size(unsigned int maxsegs)
>>>        /* Maximum Write list size */
>>>        maxsegs += 2;   /* segment for head and tail buffers */
>>>        size = sizeof(__be32);          /* segment count */
>>> -       size += maxsegs * sizeof(struct rpcrdma_segment);
>>> +       size += maxsegs * rpcrdma_segment_maxsz * sizeof(__be32);
>>>        size += sizeof(__be32); /* list discriminator */
>>>
>>>        dprintk("RPC:       %s: max reply header size = %u\n",
>>> diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
>>> index 6e64c82..8b9954c 100644
>>> --- a/net/sunrpc/xprtrdma/xprt_rdma.h
>>> +++ b/net/sunrpc/xprtrdma/xprt_rdma.h
>>> @@ -145,12 +145,6 @@ struct rpcrdma_regbuf {
>>>        return rb->rg_iov.lkey;
>>> }
>>>
>>> -static inline struct rpcrdma_msg *
>>> -rdmab_to_msg(struct rpcrdma_regbuf *rb)
>>> -{
>>> -       return (struct rpcrdma_msg *)rb->rg_base;
>>> -}
>>> -
>>> static inline struct ib_device *
>>> rdmab_device(struct rpcrdma_regbuf *rb)
>>> {
>>>
>>> --
>>> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
>>> the body of a message to majordomo@vger.kernel.org
>>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
>> the body of a message to majordomo@vger.kernel.org
>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>
> --
> Chuck Lever
>
>
>