From: Benjamin Coddington <ben.coddington@hammerspace.com>
To: Trond Myklebust <trondmy@kernel.org>, Anna Schumaker <anna@kernel.org>
Cc: linux-nfs@vger.kernel.org,
Jonathan Curley <jcurley@purestorage.com>,
Mike Snitzer <snitzer@kernel.org>,
Jeff Layton <jlayton@kernel.org>
Subject: [PATCH 21/21] NFSv4/flexfiles: Add a dataserver_nconnect cap
Date: Thu, 13 Aug 2026 16:43:17 -0400 [thread overview]
Message-ID: <1ee15f2bd1099ddace5e2baaa0d5bb31b8c4908c.1786653063.git.bcodding@hammerspace.com> (raw)
In-Reply-To: <cover.1786653063.git.bcodding@hammerspace.com>
Data-server clients inherit the MDS nconnect setting. A striping
mount multiplies that by every distinct data server: at ~1000 DSes
and nconnect=16 the client opens ~16k sockets plus their sunrpc slot
tables, and striping already spreads I/O across the data servers, so
a high per-DS transport count buys little for that workload.
Add a dataserver_nconnect module parameter to the flexfiles layout
driver alongside its existing dataserver_timeo and dataserver_retrans
knobs, and thread the value through nfs4_pnfs_ds_connect() to both
the v3 and v4 data-server client setup paths. The default of 0
preserves today's inherit-from-MDS behavior, so the cap is opt-in
and non-regressing. The files layout passes 0, unchanged.
Assisted-by: Claude:claude-fable-5
Signed-off-by: Benjamin Coddington <bcodding@hammerspace.com>
---
fs/nfs/filelayout/filelayoutdev.c | 2 +-
fs/nfs/flexfilelayout/flexfilelayoutdev.c | 7 +++++++
fs/nfs/internal.h | 3 ++-
fs/nfs/nfs3client.c | 9 +++++++--
fs/nfs/nfs4client.c | 5 ++++-
fs/nfs/pnfs.h | 3 ++-
fs/nfs/pnfs_nfs.c | 20 +++++++++++++-------
7 files changed, 36 insertions(+), 13 deletions(-)
diff --git a/fs/nfs/filelayout/filelayoutdev.c b/fs/nfs/filelayout/filelayoutdev.c
index 7226989ee4d5..35c7bcfaa773 100644
--- a/fs/nfs/filelayout/filelayoutdev.c
+++ b/fs/nfs/filelayout/filelayoutdev.c
@@ -279,7 +279,7 @@ nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx)
goto out_test_devid;
status = nfs4_pnfs_ds_connect(s, ds, devid, dataserver_timeo,
- dataserver_retrans, 4,
+ dataserver_retrans, 0, 4,
s->nfs_client->cl_minorversion);
if (status) {
nfs4_mark_deviceid_unavailable(devid);
diff --git a/fs/nfs/flexfilelayout/flexfilelayoutdev.c b/fs/nfs/flexfilelayout/flexfilelayoutdev.c
index 4ab55f0874d5..a870c9303bf8 100644
--- a/fs/nfs/flexfilelayout/flexfilelayoutdev.c
+++ b/fs/nfs/flexfilelayout/flexfilelayoutdev.c
@@ -20,6 +20,7 @@
static unsigned int dataserver_timeo = NFS_DEF_TCP_TIMEO;
static unsigned int dataserver_retrans;
+static unsigned int dataserver_nconnect;
static bool ff_layout_has_available_ds(struct pnfs_layout_segment *lseg);
@@ -439,6 +440,7 @@ nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg,
*/
status = nfs4_pnfs_ds_connect(s, ds, &mirror_ds->id_node,
dataserver_timeo, dataserver_retrans,
+ dataserver_nconnect,
mirror_ds->ds_versions[0].version,
mirror_ds->ds_versions[0].minor_version);
@@ -696,3 +698,8 @@ module_param(dataserver_timeo, uint, 0644);
MODULE_PARM_DESC(dataserver_timeo, "The time (in tenths of a second) the "
"NFSv4.1 client waits for a response from a "
" data server before it retries an NFS request.");
+module_param(dataserver_nconnect, uint, 0644);
+MODULE_PARM_DESC(dataserver_nconnect, "The maximum number of connections "
+ "the NFSv4.1 client opens to each data server, "
+ "capping the value inherited from the MDS nconnect "
+ "mount option. 0 (default) applies no cap.");
diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index e4533f583632..21007ac2eb0a 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -250,6 +250,7 @@ extern struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv,
int ds_addrlen, int ds_proto,
unsigned int ds_timeo,
unsigned int ds_retrans,
+ unsigned int ds_nconnect,
u32 minor_version);
extern struct rpc_clnt *nfs4_find_or_create_ds_client(struct nfs_client *,
struct inode *);
@@ -258,7 +259,7 @@ extern void nfs4_session_limit_xasize(struct nfs_server *server);
extern struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv,
const struct sockaddr_storage *ds_addr, int ds_addrlen,
int ds_proto, unsigned int ds_timeo,
- unsigned int ds_retrans);
+ unsigned int ds_retrans, unsigned int ds_nconnect);
#ifdef CONFIG_PROC_FS
extern int __init nfs_fs_proc_init(void);
extern void nfs_fs_proc_exit(void);
diff --git a/fs/nfs/nfs3client.c b/fs/nfs/nfs3client.c
index 5d97c1d38bb6..cf2f7be4b435 100644
--- a/fs/nfs/nfs3client.c
+++ b/fs/nfs/nfs3client.c
@@ -84,7 +84,8 @@ struct nfs_server *nfs3_clone_server(struct nfs_server *source,
*/
struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv,
const struct sockaddr_storage *ds_addr, int ds_addrlen,
- int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans)
+ int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans,
+ unsigned int ds_nconnect)
{
struct rpc_timeout ds_timeout;
unsigned long connect_timeout = ds_timeo * (ds_retrans + 1) * HZ / 10;
@@ -124,8 +125,12 @@ struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv,
fallthrough;
case XPRT_TRANSPORT_RDMA:
case XPRT_TRANSPORT_TCP:
- if (mds_clp->cl_nconnect > 1)
+ if (mds_clp->cl_nconnect > 1) {
cl_init.nconnect = mds_clp->cl_nconnect;
+ if (ds_nconnect)
+ cl_init.nconnect = min(cl_init.nconnect,
+ ds_nconnect);
+ }
}
if (mds_srv->flags & NFS_MOUNT_NORESVPORT)
diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c
index 6a2f7522179c..56a2a9c57cb2 100644
--- a/fs/nfs/nfs4client.c
+++ b/fs/nfs/nfs4client.c
@@ -793,7 +793,7 @@ static int nfs4_set_client(struct nfs_server *server,
struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv,
const struct sockaddr_storage *ds_addr, int ds_addrlen,
int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans,
- u32 minor_version)
+ unsigned int ds_nconnect, u32 minor_version)
{
struct rpc_timeout ds_timeout;
struct nfs_client *mds_clp = mds_srv->nfs_client;
@@ -831,6 +831,9 @@ struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv,
case XPRT_TRANSPORT_TCP:
if (mds_clp->cl_nconnect > 1) {
cl_init.nconnect = mds_clp->cl_nconnect;
+ if (ds_nconnect)
+ cl_init.nconnect = min(cl_init.nconnect,
+ ds_nconnect);
cl_init.max_connect = NFS_MAX_TRANSPORTS;
}
}
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index e0a7ae5fe8e9..0ae5329afdab 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -497,7 +497,8 @@ struct nfs4_pnfs_ds *nfs4_pnfs_ds_add(const struct net *net,
void nfs4_pnfs_v3_ds_connect_unload(void);
int nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds,
struct nfs4_deviceid_node *devid, unsigned int timeo,
- unsigned int retrans, u32 version, u32 minor_version);
+ unsigned int retrans, unsigned int nconnect,
+ u32 version, u32 minor_version);
struct nfs4_pnfs_ds_addr *nfs4_decode_mp_ds_addr(struct net *net,
struct xdr_stream *xdr,
gfp_t gfp_flags);
diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c
index a94f4a2933cd..4740d41ef1c5 100644
--- a/fs/nfs/pnfs_nfs.c
+++ b/fs/nfs/pnfs_nfs.c
@@ -842,7 +842,8 @@ static struct nfs_client *(*get_v3_ds_connect)(
int ds_addrlen,
int ds_proto,
unsigned int ds_timeo,
- unsigned int ds_retrans);
+ unsigned int ds_retrans,
+ unsigned int ds_nconnect);
static bool load_v3_ds_connect(void)
{
@@ -865,7 +866,8 @@ void nfs4_pnfs_v3_ds_connect_unload(void)
static int _nfs4_pnfs_v3_ds_connect(struct nfs_server *mds_srv,
struct nfs4_pnfs_ds *ds,
unsigned int timeo,
- unsigned int retrans)
+ unsigned int retrans,
+ unsigned int nconnect)
{
struct nfs_client *clp = ERR_PTR(-EIO);
struct nfs_client *mds_clp = mds_srv->nfs_client;
@@ -917,7 +919,7 @@ static int _nfs4_pnfs_v3_ds_connect(struct nfs_server *mds_srv,
ds_proto = XPRT_TRANSPORT_TCP_TLS;
clp = get_v3_ds_connect(mds_srv, &da->da_addr, da->da_addrlen,
- ds_proto, timeo, retrans);
+ ds_proto, timeo, retrans, nconnect);
if (IS_ERR(clp))
continue;
clp->cl_rpcclient->cl_softerr = 0;
@@ -940,6 +942,7 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv,
struct nfs4_pnfs_ds *ds,
unsigned int timeo,
unsigned int retrans,
+ unsigned int nconnect,
u32 minor_version)
{
struct nfs_client *clp = ERR_PTR(-EIO);
@@ -1030,7 +1033,8 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv,
clp = nfs4_set_ds_client(mds_srv, &da->da_addr,
da->da_addrlen, ds_proto,
- timeo, retrans, minor_version);
+ timeo, retrans, nconnect,
+ minor_version);
if (IS_ERR(clp))
continue;
@@ -1063,7 +1067,8 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv,
*/
int nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds,
struct nfs4_deviceid_node *devid, unsigned int timeo,
- unsigned int retrans, u32 version, u32 minor_version)
+ unsigned int retrans, unsigned int nconnect,
+ u32 version, u32 minor_version)
{
int err;
@@ -1082,11 +1087,12 @@ int nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds,
switch (version) {
case 3:
- err = _nfs4_pnfs_v3_ds_connect(mds_srv, ds, timeo, retrans);
+ err = _nfs4_pnfs_v3_ds_connect(mds_srv, ds, timeo, retrans,
+ nconnect);
break;
case 4:
err = _nfs4_pnfs_v4_ds_connect(mds_srv, ds, timeo, retrans,
- minor_version);
+ nconnect, minor_version);
break;
default:
dprintk("%s: unsupported DS version %d\n", __func__, version);
--
2.53.0
next prev parent reply other threads:[~2026-08-13 20:43 UTC|newest]
Thread overview: 23+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-08-13 20:42 [PATCH 00/21] NFS: flexfiles device notifications and caching for wide striped layouts Benjamin Coddington
2026-08-13 20:42 ` [PATCH 01/21] pNFS: Fix CB_NOTIFY_DEVICEID CHANGE to consume ndc_immediate Benjamin Coddington
2026-08-13 20:42 ` [PATCH 02/21] NFSv4/flexfiles: Use the full 64-bit offset for read DS selection Benjamin Coddington
2026-08-13 20:42 ` [PATCH 03/21] NFSv4/flexfiles: Bound page coalescing on the absolute stripe offset Benjamin Coddington
2026-08-13 20:43 ` [PATCH 04/21] NFSv4/filelayout: Anchor page coalescing on pattern_offset Benjamin Coddington
2026-08-13 20:43 ` [PATCH 05/21] NFSv4/flexfiles: Reference the device node across DS setup Benjamin Coddington
2026-08-13 20:43 ` [PATCH 06/21] NFSv4/flexfiles: Carry the device node reference across each I/O Benjamin Coddington
2026-08-13 20:43 ` [PATCH 07/21] NFSv4/flexfiles: Hold a device node reference for layoutstats encoding Benjamin Coddington
2026-08-13 20:43 ` [PATCH 08/21] NFSv4/flexfiles: Make the pinned device node pointer RCU-managed Benjamin Coddington
2026-08-13 20:43 ` [PATCH 09/21] pNFS: Add a reresolve_deviceid layout driver hook Benjamin Coddington
2026-08-13 20:43 ` [PATCH 10/21] NFSv4/flexfiles: Implement in-place device re-resolve on CHANGE Benjamin Coddington
2026-08-13 20:43 ` [PATCH 11/21] NFSv4: Dispatch CB_NOTIFY_DEVICEID CHANGE to an in-place refresh Benjamin Coddington
2026-08-13 20:43 ` [PATCH 12/21] NFSv4/flexfiles: Honor ndc_immediate on CB_NOTIFY_DEVICEID CHANGE Benjamin Coddington
2026-08-13 20:43 ` [PATCH 13/21] pNFS: Discard a GETDEVICEINFO reply that raced a CHANGE notification Benjamin Coddington
2026-08-13 20:43 ` [PATCH 14/21] pNFS: Add deviceid reference query and collection walkers Benjamin Coddington
2026-08-13 20:43 ` [PATCH 15/21] NFSv4/pnfs: Recover revoked layouts on a deleted deviceID Benjamin Coddington
2026-08-13 20:43 ` [PATCH 16/21] NFSv4/pnfs: Confirm a deviceID delete via GETDEVICEINFO Benjamin Coddington
2026-08-13 20:43 ` [PATCH 17/21] NFSv4/pnfs: Dispatch CB_NOTIFY_DEVICEID DELETE to race recovery Benjamin Coddington
2026-08-13 20:43 ` [PATCH 18/21] NFSv4/pnfs: Grow the deviceid cache hash table Benjamin Coddington
2026-08-13 20:43 ` [PATCH 19/21] NFSv4/pnfs: Re-home the data-server cache onto hash buckets Benjamin Coddington
2026-08-13 20:43 ` [PATCH 20/21] NFSv4/pnfs: Key the data-server cache by its address set Benjamin Coddington
2026-08-13 20:43 ` Benjamin Coddington [this message]
2026-08-14 14:35 ` [PATCH 00/21] NFS: flexfiles device notifications and caching for wide striped layouts Benjamin Coddington
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1ee15f2bd1099ddace5e2baaa0d5bb31b8c4908c.1786653063.git.bcodding@hammerspace.com \
--to=ben.coddington@hammerspace.com \
--cc=anna@kernel.org \
--cc=jcurley@purestorage.com \
--cc=jlayton@kernel.org \
--cc=linux-nfs@vger.kernel.org \
--cc=snitzer@kernel.org \
--cc=trondmy@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox