From: Edward Srouji <edwards@nvidia.com>
To: Jason Gunthorpe <jgg@ziepe.ca>, Leon Romanovsky <leon@kernel.org>,
"Saeed Mahameed" <saeedm@nvidia.com>,
Tariq Toukan <tariqt@nvidia.com>, Mark Bloch <mbloch@nvidia.com>,
Andrew Lunn <andrew+netdev@lunn.ch>,
"David S. Miller" <davem@davemloft.net>,
Eric Dumazet <edumazet@google.com>,
Jakub Kicinski <kuba@kernel.org>, Paolo Abeni <pabeni@redhat.com>
Cc: <linux-kernel@vger.kernel.org>, <linux-rdma@vger.kernel.org>,
<netdev@vger.kernel.org>,
Michael Guralnik <michaelgur@nvidia.com>,
"Edward Srouji" <edwards@nvidia.com>,
Patrisious Haddad <phaddad@nvidia.com>
Subject: [PATCH rdma-next v3 09/11] RDMA/core: Add netlink command to modify FRMR aging
Date: Mon, 2 Feb 2026 18:00:01 +0200 [thread overview]
Message-ID: <20260202-frmr_pools-v3-9-b8405ed9deba@nvidia.com> (raw)
In-Reply-To: <20260202-frmr_pools-v3-0-b8405ed9deba@nvidia.com>
From: Michael Guralnik <michaelgur@nvidia.com>
Allow users to set FRMR pools aging timer through netlink.
This functionality will allow user to control how long handles reside in
the kernel before being destroyed, thus being able to tune the tradeoff
between memory and HW object consumption and memory registration
optimization.
Since FRMR pools is highly beneficial for application restart scenarios,
this command allows users to modify the aging timer to their application
restart time, making sure the FRMR handles deregistered on application
teardown are kept for long enough in the pools for reuse in the
application startup.
Signed-off-by: Michael Guralnik <michaelgur@nvidia.com>
Reviewed-by: Patrisious Haddad <phaddad@nvidia.com>
Signed-off-by: Edward Srouji <edwards@nvidia.com>
---
drivers/infiniband/core/frmr_pools.c | 31 ++++++++++++++++++++++++++++--
drivers/infiniband/core/frmr_pools.h | 2 ++
drivers/infiniband/core/nldev.c | 37 ++++++++++++++++++++++++++++++++++++
include/uapi/rdma/rdma_netlink.h | 3 +++
4 files changed, 71 insertions(+), 2 deletions(-)
diff --git a/drivers/infiniband/core/frmr_pools.c b/drivers/infiniband/core/frmr_pools.c
index c5abfc59f9d6..1493bf2cb064 100644
--- a/drivers/infiniband/core/frmr_pools.c
+++ b/drivers/infiniband/core/frmr_pools.c
@@ -174,7 +174,7 @@ static void pool_aging_work(struct work_struct *work)
if (has_work)
queue_delayed_work(
pools->aging_wq, &pool->aging_work,
- secs_to_jiffies(FRMR_POOLS_DEFAULT_AGING_PERIOD_SECS));
+ secs_to_jiffies(READ_ONCE(pools->aging_period_sec)));
}
static void destroy_frmr_pool(struct ib_device *device,
@@ -214,6 +214,8 @@ int ib_frmr_pools_init(struct ib_device *device,
return -ENOMEM;
}
+ pools->aging_period_sec = FRMR_POOLS_DEFAULT_AGING_PERIOD_SECS;
+
device->frmr_pools = pools;
return 0;
}
@@ -249,6 +251,31 @@ void ib_frmr_pools_cleanup(struct ib_device *device)
}
EXPORT_SYMBOL(ib_frmr_pools_cleanup);
+int ib_frmr_pools_set_aging_period(struct ib_device *device, u32 period_sec)
+{
+ struct ib_frmr_pools *pools = device->frmr_pools;
+ struct ib_frmr_pool *pool;
+ struct rb_node *node;
+
+ if (!pools)
+ return -EINVAL;
+
+ if (period_sec == 0)
+ return -EINVAL;
+
+ WRITE_ONCE(pools->aging_period_sec, period_sec);
+
+ read_lock(&pools->rb_lock);
+ for (node = rb_first(&pools->rb_root); node; node = rb_next(node)) {
+ pool = rb_entry(node, struct ib_frmr_pool, node);
+ mod_delayed_work(pools->aging_wq, &pool->aging_work,
+ secs_to_jiffies(period_sec));
+ }
+ read_unlock(&pools->rb_lock);
+
+ return 0;
+}
+
static inline int compare_keys(struct ib_frmr_key *key1,
struct ib_frmr_key *key2)
{
@@ -517,7 +544,7 @@ int ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr)
if (ret == 0 && schedule_aging)
queue_delayed_work(pools->aging_wq, &pool->aging_work,
- secs_to_jiffies(FRMR_POOLS_DEFAULT_AGING_PERIOD_SECS));
+ secs_to_jiffies(READ_ONCE(pools->aging_period_sec)));
return ret;
}
diff --git a/drivers/infiniband/core/frmr_pools.h b/drivers/infiniband/core/frmr_pools.h
index b144273ee347..81149ff15e00 100644
--- a/drivers/infiniband/core/frmr_pools.h
+++ b/drivers/infiniband/core/frmr_pools.h
@@ -54,8 +54,10 @@ struct ib_frmr_pools {
const struct ib_frmr_pool_ops *pool_ops;
struct workqueue_struct *aging_wq;
+ u32 aging_period_sec;
};
int ib_frmr_pools_set_pinned(struct ib_device *device, struct ib_frmr_key *key,
u32 pinned_handles);
+int ib_frmr_pools_set_aging_period(struct ib_device *device, u32 period_sec);
#endif /* RDMA_CORE_FRMR_POOLS_H */
diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c
index 6637c76165be..8d004b7568b7 100644
--- a/drivers/infiniband/core/nldev.c
+++ b/drivers/infiniband/core/nldev.c
@@ -184,6 +184,7 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
[RDMA_NLDEV_ATTR_FRMR_POOL_QUEUE_HANDLES] = { .type = NLA_U32 },
[RDMA_NLDEV_ATTR_FRMR_POOL_MAX_IN_USE] = { .type = NLA_U64 },
[RDMA_NLDEV_ATTR_FRMR_POOL_IN_USE] = { .type = NLA_U64 },
+ [RDMA_NLDEV_ATTR_FRMR_POOLS_AGING_PERIOD] = { .type = NLA_U32 },
};
static int put_driver_name_print_type(struct sk_buff *msg, const char *name,
@@ -2799,6 +2800,38 @@ static int nldev_frmr_pools_get_dumpit(struct sk_buff *skb,
return ret;
}
+static int nldev_frmr_pools_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
+ struct netlink_ext_ack *extack)
+{
+ struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
+ struct ib_device *device;
+ u32 aging_period;
+ int err;
+
+ err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, nldev_policy,
+ extack);
+ if (err)
+ return err;
+
+ if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX])
+ return -EINVAL;
+
+ if (!tb[RDMA_NLDEV_ATTR_FRMR_POOLS_AGING_PERIOD])
+ return -EINVAL;
+
+ device = ib_device_get_by_index(
+ sock_net(skb->sk), nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]));
+ if (!device)
+ return -EINVAL;
+
+ aging_period = nla_get_u32(tb[RDMA_NLDEV_ATTR_FRMR_POOLS_AGING_PERIOD]);
+
+ err = ib_frmr_pools_set_aging_period(device, aging_period);
+
+ ib_device_put(device);
+ return err;
+}
+
static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
[RDMA_NLDEV_CMD_GET] = {
.doit = nldev_get_doit,
@@ -2908,6 +2941,10 @@ static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
[RDMA_NLDEV_CMD_FRMR_POOLS_GET] = {
.dump = nldev_frmr_pools_get_dumpit,
},
+ [RDMA_NLDEV_CMD_FRMR_POOLS_SET] = {
+ .doit = nldev_frmr_pools_set_doit,
+ .flags = RDMA_NL_ADMIN_PERM,
+ },
};
static int fill_mon_netdev_rename(struct sk_buff *msg,
diff --git a/include/uapi/rdma/rdma_netlink.h b/include/uapi/rdma/rdma_netlink.h
index 8f17ffe0190c..f9c295caf2b1 100644
--- a/include/uapi/rdma/rdma_netlink.h
+++ b/include/uapi/rdma/rdma_netlink.h
@@ -310,6 +310,8 @@ enum rdma_nldev_command {
RDMA_NLDEV_CMD_FRMR_POOLS_GET, /* can dump */
+ RDMA_NLDEV_CMD_FRMR_POOLS_SET,
+
RDMA_NLDEV_NUM_OPS
};
@@ -598,6 +600,7 @@ enum rdma_nldev_attr {
RDMA_NLDEV_ATTR_FRMR_POOL_QUEUE_HANDLES, /* u32 */
RDMA_NLDEV_ATTR_FRMR_POOL_MAX_IN_USE, /* u64 */
RDMA_NLDEV_ATTR_FRMR_POOL_IN_USE, /* u64 */
+ RDMA_NLDEV_ATTR_FRMR_POOLS_AGING_PERIOD, /* u32 */
/*
* Always the end
--
2.47.1
next prev parent reply other threads:[~2026-02-02 16:01 UTC|newest]
Thread overview: 14+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-02-02 15:59 [PATCH rdma-next v3 00/11] RDMA/core: Introduce FRMR pools infrastructure Edward Srouji
2026-02-02 15:59 ` [PATCH rdma-next v3 01/11] RDMA/mlx5: Move device async_ctx initialization Edward Srouji
2026-02-02 15:59 ` [PATCH rdma-next v3 02/11] IB/core: Introduce FRMR pools Edward Srouji
2026-02-02 15:59 ` [PATCH rdma-next v3 03/11] RDMA/core: Add aging to " Edward Srouji
2026-02-02 15:59 ` [PATCH rdma-next v3 04/11] RDMA/core: Add FRMR pools statistics Edward Srouji
2026-02-02 15:59 ` [PATCH rdma-next v3 05/11] RDMA/core: Add pinned handles to FRMR pools Edward Srouji
2026-02-02 15:59 ` [PATCH rdma-next v3 06/11] RDMA/mlx5: Switch from MR cache " Edward Srouji
2026-02-02 15:59 ` [PATCH rdma-next v3 07/11] net/mlx5: Drop MR cache related code Edward Srouji
2026-02-02 16:00 ` [PATCH rdma-next v3 08/11] RDMA/nldev: Add command to get FRMR pools Edward Srouji
2026-02-02 16:00 ` Edward Srouji [this message]
2026-02-02 16:00 ` [PATCH rdma-next v3 10/11] RDMA/nldev: Add command to set pinned FRMR handles Edward Srouji
2026-02-02 16:00 ` [PATCH rdma-next v3 11/11] RDMA/nldev: Expose kernel-internal FRMR pools in netlink Edward Srouji
2026-02-25 11:47 ` [PATCH rdma-next v3 00/11] RDMA/core: Introduce FRMR pools infrastructure Leon Romanovsky
2026-02-26 13:32 ` Edward Srouji
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260202-frmr_pools-v3-9-b8405ed9deba@nvidia.com \
--to=edwards@nvidia.com \
--cc=andrew+netdev@lunn.ch \
--cc=davem@davemloft.net \
--cc=edumazet@google.com \
--cc=jgg@ziepe.ca \
--cc=kuba@kernel.org \
--cc=leon@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-rdma@vger.kernel.org \
--cc=mbloch@nvidia.com \
--cc=michaelgur@nvidia.com \
--cc=netdev@vger.kernel.org \
--cc=pabeni@redhat.com \
--cc=phaddad@nvidia.com \
--cc=saeedm@nvidia.com \
--cc=tariqt@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox