From: Michael Guralnik <michaelgur@nvidia.com>
To: <jgg@nvidia.com>, <leonro@nvidia.com>
Cc: <maorg@nvidia.com>, <linux-rdma@vger.kernel.org>,
<saeedm@nvidia.com>, Aharon Landau <aharonl@nvidia.com>,
Michael Guralnik <michaelgur@nvidia.com>
Subject: [PATCH rdma-next 5/8] RDMA/mlx5: Introduce mlx5r_cache_rb_key
Date: Thu, 8 Sep 2022 23:54:18 +0300 [thread overview]
Message-ID: <20220908205421.210048-6-michaelgur@nvidia.com> (raw)
In-Reply-To: <20220908205421.210048-1-michaelgur@nvidia.com>
From: Aharon Landau <aharonl@nvidia.com>
In the next patch, I will change the cache structure to an RB tree.
Introducing the key of the tree.
The key is the mkey properties that UMR operations can't modify.
Using this key to define the cache entries and to search and create
cache mkeys.
Signed-off-by: Aharon Landau <aharonl@nvidia.com>
Reviewed-by: Michael Guralnik <michaelgur@nvidia.com>
---
drivers/infiniband/hw/mlx5/mlx5_ib.h | 14 +++++-
drivers/infiniband/hw/mlx5/mr.c | 73 +++++++++++++++++++---------
drivers/infiniband/hw/mlx5/odp.c | 8 ++-
3 files changed, 69 insertions(+), 26 deletions(-)
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index cb8006b854e0..5b57b2a24b47 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -613,6 +613,16 @@ enum mlx5_mkey_type {
MLX5_MKEY_INDIRECT_DEVX,
};
+struct mlx5r_cache_rb_key {
+ unsigned int access_mode;
+ unsigned int access_flags;
+ /*
+ * keep ndescs as the last member so entries with about the same ndescs
+ * will be close in the tree
+ */
+ unsigned int ndescs;
+};
+
struct mlx5_ib_mkey {
u32 key;
enum mlx5_mkey_type type;
@@ -731,10 +741,10 @@ struct mlx5_cache_ent {
unsigned long stored;
unsigned long reserved;
+ struct mlx5r_cache_rb_key rb_key;
+
char name[4];
u32 order;
- u32 access_mode;
- unsigned int ndescs;
u8 disabled:1;
u8 fill_to_high_water:1;
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index ea8634cafa9c..bb3d5a766cb8 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -287,18 +287,18 @@ static int get_mkc_octo_size(unsigned int access_mode, unsigned int ndescs)
return ret;
}
-static void set_cache_mkc(struct mlx5_ib_dev *dev, u8 access_mode,
- unsigned int access_flags, unsigned int ndescs,
- void *mkc)
+static void set_cache_mkc(struct mlx5_ib_dev *dev,
+ struct mlx5r_cache_rb_key rb_key, void *mkc)
{
- set_mkc_access_pd_addr_fields(mkc, access_flags, 0, dev->umrc.pd);
+ set_mkc_access_pd_addr_fields(mkc, rb_key.access_flags, 0,
+ dev->umrc.pd);
MLX5_SET(mkc, mkc, free, 1);
MLX5_SET(mkc, mkc, umr_en, 1);
- MLX5_SET(mkc, mkc, access_mode_1_0, access_mode & 0x3);
- MLX5_SET(mkc, mkc, access_mode_4_2, (access_mode >> 2) & 0x7);
+ MLX5_SET(mkc, mkc, access_mode_1_0, rb_key.access_mode & 0x3);
+ MLX5_SET(mkc, mkc, access_mode_4_2, (rb_key.access_mode >> 2) & 0x7);
MLX5_SET(mkc, mkc, translations_octword_size,
- get_mkc_octo_size(access_mode, ndescs));
+ get_mkc_octo_size(rb_key.access_mode, rb_key.ndescs));
MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
}
@@ -317,7 +317,7 @@ static int add_keys(struct mlx5_cache_ent *ent, unsigned int num)
return -ENOMEM;
mkc = MLX5_ADDR_OF(create_mkey_in, async_create->in,
memory_key_mkey_entry);
- set_cache_mkc(ent->dev, ent->access_mode, 0, ent->ndescs, mkc);
+ set_cache_mkc(ent->dev, ent->rb_key, mkc);
async_create->ent = ent;
err = push_mkey(ent, true, NULL);
@@ -343,8 +343,8 @@ static int add_keys(struct mlx5_cache_ent *ent, unsigned int num)
}
/* Synchronously create a cacheable mkey */
-static int create_cache_mkey(struct mlx5_ib_dev *dev, u8 access_mode,
- unsigned int access_flags, unsigned int ndescs,
+static int create_cache_mkey(struct mlx5_ib_dev *dev,
+ struct mlx5r_cache_rb_key rb_key,
struct mlx5_ib_mkey *mkey)
{
size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
@@ -356,7 +356,7 @@ static int create_cache_mkey(struct mlx5_ib_dev *dev, u8 access_mode,
if (!in)
return -ENOMEM;
mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
- set_cache_mkc(dev, access_mode, access_flags, ndescs, mkc);
+ set_cache_mkc(dev, rb_key, mkc);
err = mlx5_ib_create_mkey(dev, mkey, in, inlen);
kfree(in);
@@ -647,7 +647,7 @@ static bool mlx5_ent_get_mkey(struct mlx5_cache_ent *ent, struct mlx5_ib_mr *mr)
}
mr->mmkey.key = pop_stored_mkey(ent);
- mr->mmkey.ndescs = ent->ndescs;
+ mr->mmkey.ndescs = ent->rb_key.ndescs;
mr->mmkey.cache_ent = ent;
queue_adjust_cache_locked(ent);
ent->in_use++;
@@ -668,29 +668,57 @@ static struct mlx5_cache_ent *mkey_cache_ent_from_order(struct mlx5_ib_dev *dev,
return &cache->ent[order];
}
-static bool mlx5_cache_get_mkey(struct mlx5_ib_dev *dev, u8 access_mode,
- u8 access_flags, unsigned int ndescs,
+static bool mlx5_cache_get_mkey(struct mlx5_ib_dev *dev,
+ struct mlx5r_cache_rb_key rb_key,
struct mlx5_ib_mr *mr)
{
struct mlx5_cache_ent *ent;
- if (!mlx5r_umr_can_reconfig(dev, 0, access_flags))
+ if (!mlx5r_umr_can_reconfig(dev, 0, rb_key.access_flags))
return false;
- if (access_mode == MLX5_MKC_ACCESS_MODE_KSM)
+ if (rb_key.access_mode == MLX5_MKC_ACCESS_MODE_KSM)
ent = &dev->cache.ent[MLX5_IMR_KSM_CACHE_ENTRY];
- ent = mkey_cache_ent_from_order(dev, order_base_2(ndescs));
+ ent = mkey_cache_ent_from_order(dev, order_base_2(rb_key.ndescs));
if (!ent)
return false;
return mlx5_ent_get_mkey(ent, mr);
}
+static int get_uchangeable_access_flags(struct mlx5_ib_dev *dev,
+ int access_flags)
+{
+ int ret = 0;
+
+ if ((access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
+ MLX5_CAP_GEN(dev->mdev, atomic) &&
+ MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled))
+ ret |= IB_ACCESS_REMOTE_ATOMIC;
+
+ if ((access_flags & IB_ACCESS_RELAXED_ORDERING) &&
+ MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write) &&
+ !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr))
+ ret |= IB_ACCESS_RELAXED_ORDERING;
+
+ if ((access_flags & IB_ACCESS_RELAXED_ORDERING) &&
+ MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) &&
+ !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr))
+ ret |= IB_ACCESS_RELAXED_ORDERING;
+
+ return ret;
+}
+
struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, u8 access_mode,
unsigned int access_flags,
unsigned int ndescs)
{
+ struct mlx5r_cache_rb_key rb_key = {
+ .access_mode = access_mode,
+ .access_flags = get_uchangeable_access_flags(dev, access_flags),
+ .ndescs = ndescs
+ };
struct mlx5_ib_mr *mr;
int err;
@@ -698,13 +726,12 @@ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, u8 access_mode,
if (!mr)
return ERR_PTR(-ENOMEM);
- if (!mlx5_cache_get_mkey(dev, access_mode, access_flags, ndescs, mr)) {
+ if (!mlx5_cache_get_mkey(dev, rb_key, mr)) {
/*
* Didn't find an mkey in cache.
* Create an mkey with the exact needed size.
*/
- err = create_cache_mkey(dev, access_mode, access_flags, ndescs,
- &mr->mmkey);
+ err = create_cache_mkey(dev, rb_key, &mr->mmkey);
if (err) {
kfree(mr);
return ERR_PTR(err);
@@ -774,6 +801,8 @@ static void delay_time_func(struct timer_list *t)
int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev)
{
+ struct mlx5r_cache_rb_key rb_key = { .access_mode =
+ MLX5_MKC_ACCESS_MODE_MTT };
struct mlx5_mkey_cache *cache = &dev->cache;
struct mlx5_cache_ent *ent;
int i;
@@ -804,8 +833,8 @@ int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev)
if (ent->order > mkey_cache_max_order(dev))
continue;
- ent->ndescs = 1 << ent->order;
- ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
+ rb_key.ndescs = 1 << ent->order;
+ ent->rb_key = rb_key;
if ((dev->mdev->profile.mask & MLX5_PROF_MASK_MR_CACHE) &&
!dev->is_rep && mlx5_core_is_pf(dev->mdev) &&
mlx5r_umr_can_load_pas(dev, 0))
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index 137143da5959..90339edddfed 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -1589,10 +1589,14 @@ mlx5_ib_odp_destroy_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq)
void mlx5_odp_init_mkey_cache_entry(struct mlx5_cache_ent *ent)
{
+ struct mlx5r_cache_rb_key rb_key = {
+ .access_mode = MLX5_MKC_ACCESS_MODE_KSM,
+ .ndescs = mlx5_imr_ksm_entries
+ };
+
if (!(ent->dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT))
return;
- ent->ndescs = mlx5_imr_ksm_entries;
- ent->access_mode = MLX5_MKC_ACCESS_MODE_KSM;
+ ent->rb_key = rb_key;
}
static const struct ib_device_ops mlx5_ib_dev_odp_ops = {
--
2.17.2
next prev parent reply other threads:[~2022-09-08 20:55 UTC|newest]
Thread overview: 10+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-09-08 20:54 [PATCH rdma-next 0/8] RDMA/mlx5: Switch MR cache to use RB-tree Michael Guralnik
2022-09-08 20:54 ` [PATCH rdma-next 1/8] RDMA/mlx5: Don't keep umrable 'page_shift' in cache entries Michael Guralnik
2022-09-08 20:54 ` [PATCH rdma-next 2/8] RDMA/mlx5: Generalize mlx5_cache_cache_mr() to fit all cacheable mkeys Michael Guralnik
2022-09-09 14:47 ` Jason Gunthorpe
2022-09-08 20:54 ` [PATCH rdma-next 3/8] RDMA/mlx5: Remove explicit ODP cache entry Michael Guralnik
2022-09-08 20:54 ` [PATCH rdma-next 4/8] RDMA/mlx5: Allow rereg all the mkeys that can load pas with UMR Michael Guralnik
2022-09-08 20:54 ` Michael Guralnik [this message]
2022-09-08 20:54 ` [PATCH rdma-next 6/8] RDMA/mlx5: Change the cache structure to an RB-tree Michael Guralnik
2022-09-08 20:54 ` [PATCH rdma-next 7/8] RDMA/mlx5: Cache all user cacheable mkeys on dereg MR flow Michael Guralnik
2022-09-08 20:54 ` [PATCH rdma-next 8/8] RDMA/mlx5: Add work to remove temporary entries from the cache Michael Guralnik
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220908205421.210048-6-michaelgur@nvidia.com \
--to=michaelgur@nvidia.com \
--cc=aharonl@nvidia.com \
--cc=jgg@nvidia.com \
--cc=leonro@nvidia.com \
--cc=linux-rdma@vger.kernel.org \
--cc=maorg@nvidia.com \
--cc=saeedm@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox