Re: [PATCH v2 03/10] sunrpc: Replace the rq_pages array with dynamically-allocated memory

[Jeff Layton <jlayton@kernel.org>]

On Sat, 2025-04-19 at 13:28 -0400, cel@kernel.org wrote:
> From: Chuck Lever <chuck.lever@oracle.com>
> 
> As a step towards making NFSD's maximum rsize and wsize variable at
> run-time, replace the fixed-size rq_vec[] array in struct svc_rqst
> with a chunk of dynamically-allocated memory.
> 
> On a system with 8-byte pointers and 4KB pages, pahole reports that
> the rq_pages[] array is 2080 bytes. Replacing it with a single
> pointer reduces the size of struct svc_rqst to just over 9500 bytes.
> 
> Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
> ---
>  include/linux/sunrpc/svc.h        |  3 ++-
>  net/sunrpc/svc.c                  | 34 ++++++++++++++++++-------------
>  net/sunrpc/svc_xprt.c             | 10 +--------
>  net/sunrpc/xprtrdma/svc_rdma_rw.c |  2 +-
>  4 files changed, 24 insertions(+), 25 deletions(-)
> 
> diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
> index 5b879c31d7b8..96ac12dbb04d 100644
> --- a/include/linux/sunrpc/svc.h
> +++ b/include/linux/sunrpc/svc.h
> @@ -200,7 +200,8 @@ struct svc_rqst {
>  	struct xdr_stream	rq_res_stream;
>  	struct page		*rq_scratch_page;
>  	struct xdr_buf		rq_res;
> -	struct page		*rq_pages[RPCSVC_MAXPAGES + 1];
> +	unsigned long		rq_maxpages;	/* num of entries in rq_pages */
> +	struct page *		*rq_pages;
>  	struct page *		*rq_respages;	/* points into rq_pages */
>  	struct page *		*rq_next_page; /* next reply page to use */
>  	struct page *		*rq_page_end;  /* one past the last page */
> diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
> index 8ce3e6b3df6a..682e11c9be36 100644
> --- a/net/sunrpc/svc.c
> +++ b/net/sunrpc/svc.c
> @@ -636,20 +636,25 @@ svc_destroy(struct svc_serv **servp)
>  EXPORT_SYMBOL_GPL(svc_destroy);
>  
>  static bool
> -svc_init_buffer(struct svc_rqst *rqstp, unsigned int size, int node)
> +svc_init_buffer(struct svc_rqst *rqstp, const struct svc_serv *serv, int node)
>  {
> -	unsigned long pages, ret;
> +	unsigned long ret;
>  
> -	pages = size / PAGE_SIZE + 1; /* extra page as we hold both request and reply.
> -				       * We assume one is at most one page
> -				       */
> -	WARN_ON_ONCE(pages > RPCSVC_MAXPAGES);
> -	if (pages > RPCSVC_MAXPAGES)
> -		pages = RPCSVC_MAXPAGES;
> +	/* Add an extra page, as rq_pages holds both request and reply.
> +	 * We assume one of those is at most one page.
> +	 */
> +	rqstp->rq_maxpages = svc_serv_maxpages(serv) + 1;
>  
> -	ret = alloc_pages_bulk_node(GFP_KERNEL, node, pages,
> +	/* rq_pages' last entry is NULL for historical reasons. */
> +	rqstp->rq_pages = kcalloc_node(rqstp->rq_maxpages + 1,
> +				       sizeof(struct page *),
> +				       GFP_KERNEL, node);
> +	if (!rqstp->rq_pages)
> +		return false;
> +
> +	ret = alloc_pages_bulk_node(GFP_KERNEL, node, rqstp->rq_maxpages,
>  				    rqstp->rq_pages);
> -	return ret == pages;
> +	return ret == rqstp->rq_maxpages;
>  }
>  
>  /*
> @@ -658,11 +663,12 @@ svc_init_buffer(struct svc_rqst *rqstp, unsigned int size, int node)
>  static void
>  svc_release_buffer(struct svc_rqst *rqstp)
>  {
> -	unsigned int i;
> +	unsigned long i;
>  
> -	for (i = 0; i < ARRAY_SIZE(rqstp->rq_pages); i++)
> +	for (i = 0; i < rqstp->rq_maxpages; i++)
>  		if (rqstp->rq_pages[i])
>  			put_page(rqstp->rq_pages[i]);
> +	kfree(rqstp->rq_pages);
>  }
>  
>  static void
> @@ -704,7 +710,7 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)
>  	if (!rqstp->rq_resp)
>  		goto out_enomem;
>  
> -	if (!svc_init_buffer(rqstp, serv->sv_max_mesg, node))
> +	if (!svc_init_buffer(rqstp, serv, node))
>  		goto out_enomem;
>  
>  	rqstp->rq_err = -EAGAIN; /* No error yet */
> @@ -896,7 +902,7 @@ EXPORT_SYMBOL_GPL(svc_set_num_threads);
>  bool svc_rqst_replace_page(struct svc_rqst *rqstp, struct page *page)
>  {
>  	struct page **begin = rqstp->rq_pages;
> -	struct page **end = &rqstp->rq_pages[RPCSVC_MAXPAGES];
> +	struct page **end = &rqstp->rq_pages[rqstp->rq_maxpages];
>  
>  	if (unlikely(rqstp->rq_next_page < begin || rqstp->rq_next_page > end)) {
>  		trace_svc_replace_page_err(rqstp);
> diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c
> index ae25405d8bd2..23547ed25269 100644
> --- a/net/sunrpc/svc_xprt.c
> +++ b/net/sunrpc/svc_xprt.c
> @@ -651,18 +651,10 @@ static void svc_check_conn_limits(struct svc_serv *serv)
>  
>  static bool svc_alloc_arg(struct svc_rqst *rqstp)
>  {
> -	struct svc_serv *serv = rqstp->rq_server;
>  	struct xdr_buf *arg = &rqstp->rq_arg;
>  	unsigned long pages, filled, ret;
>  
> -	pages = (serv->sv_max_mesg + 2 * PAGE_SIZE) >> PAGE_SHIFT;
> -	if (pages > RPCSVC_MAXPAGES) {
> -		pr_warn_once("svc: warning: pages=%lu > RPCSVC_MAXPAGES=%lu\n",
> -			     pages, RPCSVC_MAXPAGES);
> -		/* use as many pages as possible */
> -		pages = RPCSVC_MAXPAGES;
> -	}
> -
> +	pages = rqstp->rq_maxpages;
>  	for (filled = 0; filled < pages; filled = ret) {
>  		ret = alloc_pages_bulk(GFP_KERNEL, pages, rqstp->rq_pages);
>  		if (ret > filled)
> diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
> index 40797114d50a..661b3fe2779f 100644
> --- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
> +++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
> @@ -765,7 +765,7 @@ static int svc_rdma_build_read_segment(struct svc_rqst *rqstp,
>  		}
>  		len -= seg_len;
>  
> -		if (len && ((head->rc_curpage + 1) > ARRAY_SIZE(rqstp->rq_pages)))
> +		if (len && ((head->rc_curpage + 1) > rqstp->rq_maxpages))
>  			goto out_overrun;
>  	}
>  

nit: I'd probably squash 2 and 3 together, but they both look right:

Reviewed-by: Jeff Layton <jlayton@kernel.org>