Re: [PATCH v2 04/10] sunrpc: Replace the rq_vec array with dynamically-allocated memory

[Chuck Lever <cel@kernel.org>]

On 4/21/25 8:22 AM, Jeff Layton wrote:
> On Sat, 2025-04-19 at 13:28 -0400, cel@kernel.org wrote:
>> From: Chuck Lever <chuck.lever@oracle.com>
>>
>> As a step towards making NFSD's maximum rsize and wsize variable at
>> run-time, replace the fixed-size rq_vec[] array in struct svc_rqst
>> with a chunk of dynamically-allocated memory.
>>
>> The rq_vec array is sized assuming request processing will need at
>> most one kvec per page in a maximum-sized RPC message.
>>
>> On a system with 8-byte pointers and 4KB pages, pahole reports that
>> the rq_vec[] array is 4144 bytes. Replacing it with a single
>> pointer reduces the size of struct svc_rqst to about 5400 bytes.
> 
> nit: so I guess the current struct is ~9k or so?

11K plus, on my test system.


> If you're going to
> post numbers here, they should probably refer to the same thing.
> 
> I'm lazy -- don't make me do math.

Well it's a running total. Each patch in this part of the series reduces
the size of struct svc_rqst.

But I can update the descriptions. What math were you attempting to do?


>> Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
>> ---
>>  fs/nfsd/nfs4proc.c         | 1 -
>>  fs/nfsd/vfs.c              | 2 +-
>>  include/linux/sunrpc/svc.h | 2 +-
>>  net/sunrpc/svc.c           | 8 +++++++-
>>  4 files changed, 9 insertions(+), 4 deletions(-)
>>
>> diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
>> index b397246dae7b..d1be58b557d1 100644
>> --- a/fs/nfsd/nfs4proc.c
>> +++ b/fs/nfsd/nfs4proc.c
>> @@ -1228,7 +1228,6 @@ nfsd4_write(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
>>  	write->wr_how_written = write->wr_stable_how;
>>  
>>  	nvecs = svc_fill_write_vector(rqstp, &write->wr_payload);
>> -	WARN_ON_ONCE(nvecs > ARRAY_SIZE(rqstp->rq_vec));
>>  
>>  	status = nfsd_vfs_write(rqstp, &cstate->current_fh, nf,
>>  				write->wr_offset, rqstp->rq_vec, nvecs, &cnt,
>> diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c
>> index 9abdc4b75813..4eaac3aa7e15 100644
>> --- a/fs/nfsd/vfs.c
>> +++ b/fs/nfsd/vfs.c
>> @@ -1094,7 +1094,7 @@ __be32 nfsd_iter_read(struct svc_rqst *rqstp, struct svc_fh *fhp,
>>  		++v;
>>  		base = 0;
>>  	}
>> -	WARN_ON_ONCE(v > ARRAY_SIZE(rqstp->rq_vec));
>> +	WARN_ON_ONCE(v > rqstp->rq_maxpages);
>>  
>>  	trace_nfsd_read_vector(rqstp, fhp, offset, *count);
>>  	iov_iter_kvec(&iter, ITER_DEST, rqstp->rq_vec, v, *count);
>> diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
>> index 96ac12dbb04d..72d016772711 100644
>> --- a/include/linux/sunrpc/svc.h
>> +++ b/include/linux/sunrpc/svc.h
>> @@ -207,7 +207,7 @@ struct svc_rqst {
>>  	struct page *		*rq_page_end;  /* one past the last page */
>>  
>>  	struct folio_batch	rq_fbatch;
>> -	struct kvec		rq_vec[RPCSVC_MAXPAGES]; /* generally useful.. */
>> +	struct kvec		*rq_vec;
>>  	struct bio_vec		rq_bvec[RPCSVC_MAXPAGES];
>>  
>>  	__be32			rq_xid;		/* transmission id */
>> diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
>> index 682e11c9be36..5808d4b97547 100644
>> --- a/net/sunrpc/svc.c
>> +++ b/net/sunrpc/svc.c
>> @@ -675,6 +675,7 @@ static void
>>  svc_rqst_free(struct svc_rqst *rqstp)
>>  {
>>  	folio_batch_release(&rqstp->rq_fbatch);
>> +	kfree(rqstp->rq_vec);
>>  	svc_release_buffer(rqstp);
>>  	if (rqstp->rq_scratch_page)
>>  		put_page(rqstp->rq_scratch_page);
>> @@ -713,6 +714,11 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)
>>  	if (!svc_init_buffer(rqstp, serv, node))
>>  		goto out_enomem;
>>  
>> +	rqstp->rq_vec = kcalloc_node(rqstp->rq_maxpages, sizeof(struct kvec),
>> +				      GFP_KERNEL, node);
>> +	if (!rqstp->rq_vec)
>> +		goto out_enomem;
>> +
>>  	rqstp->rq_err = -EAGAIN; /* No error yet */
>>  
>>  	serv->sv_nrthreads += 1;
>> @@ -1750,7 +1756,7 @@ unsigned int svc_fill_write_vector(struct svc_rqst *rqstp,
>>  		++pages;
>>  	}
>>  
>> -	WARN_ON_ONCE(i > ARRAY_SIZE(rqstp->rq_vec));
>> +	WARN_ON_ONCE(i > rqstp->rq_maxpages);
>>  	return i;
>>  }
>>  EXPORT_SYMBOL_GPL(svc_fill_write_vector);
> 


-- 
Chuck Lever