From: Aurelien Aptel <aaptel@nvidia.com>
To: netdev@vger.kernel.org, davem@davemloft.net, kuba@kernel.org,
edumazet@google.com, pabeni@redhat.com, saeedm@nvidia.com,
tariqt@nvidia.com, linux-nvme@lists.infradead.org,
sagi@grimberg.me, hch@lst.de, kbusch@kernel.org, axboe@fb.com,
chaitanyak@nvidia.com
Cc: smalin@nvidia.com, aaptel@nvidia.com, ogerlitz@nvidia.com,
yorayz@nvidia.com, borisp@nvidia.com, aurelien.aptel@gmail.com,
malin1024@gmail.com
Subject: [PATCH v6 18/23] net/mlx5e: NVMEoTCP, use KLM UMRs for buffer registration
Date: Thu, 20 Oct 2022 13:18:33 +0300 [thread overview]
Message-ID: <20221020101838.2712846-19-aaptel@nvidia.com> (raw)
In-Reply-To: <20221020101838.2712846-1-aaptel@nvidia.com>
From: Ben Ben-Ishay <benishay@nvidia.com>
NVMEoTCP offload uses buffer registration for ddp operation.
Every request comprises from SG list that might consist from elements
with multiple combination sizes, thus the appropriate way to perform
buffer registration is with KLM UMRs.
UMR stands for user-mode memory registration, it is a mechanism to alter
address translation properties of MKEY by posting WorkQueueElement
aka WQE on send queue.
MKEY stands for memory key, MKEY are used to describe a region in memory
that can be later used by HW.
KLM stands for {Key, Length, MemVa}, KLM_MKEY is indirect MKEY that
enables to map multiple memory spaces with different sizes in unified MKEY.
KLM UMR is a UMR that use to update a KLM_MKEY.
Signed-off-by: Ben Ben-Ishay <benishay@nvidia.com>
Signed-off-by: Boris Pismenny <borisp@nvidia.com>
Signed-off-by: Or Gerlitz <ogerlitz@nvidia.com>
Signed-off-by: Yoray Zack <yorayz@nvidia.com>
Signed-off-by: Aurelien Aptel <aaptel@nvidia.com>
Reviewed-by: Tariq Toukan <tariqt@nvidia.com>
---
.../net/ethernet/mellanox/mlx5/core/en/txrx.h | 3 +
.../mellanox/mlx5/core/en_accel/nvmeotcp.c | 123 ++++++++++++++++++
.../mlx5/core/en_accel/nvmeotcp_utils.h | 25 ++++
.../net/ethernet/mellanox/mlx5/core/en_rx.c | 4 +
4 files changed, 155 insertions(+)
create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp_utils.h
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h
index 101b7630b046..8e437d98565a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h
@@ -29,6 +29,9 @@ enum mlx5e_icosq_wqe_type {
MLX5E_ICOSQ_WQE_SET_PSV_TLS,
MLX5E_ICOSQ_WQE_GET_PSV_TLS,
#endif
+#ifdef CONFIG_MLX5_EN_NVMEOTCP
+ MLX5E_ICOSQ_WQE_UMR_NVMEOTCP,
+#endif
};
/* General */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.c
index b00dc46c7c3c..30c0a50f5dac 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp.c
@@ -4,6 +4,7 @@
#include <linux/netdevice.h>
#include <linux/idr.h>
#include "en_accel/nvmeotcp.h"
+#include "en_accel/nvmeotcp_utils.h"
#include "en_accel/fs_tcp.h"
#include "en/txrx.h"
@@ -19,6 +20,120 @@ static const struct rhashtable_params rhash_queues = {
.max_size = MAX_NUM_NVMEOTCP_QUEUES,
};
+static void
+fill_nvmeotcp_klm_wqe(struct mlx5e_nvmeotcp_queue *queue, struct mlx5e_umr_wqe *wqe, u16 ccid,
+ u32 klm_entries, u16 klm_offset)
+{
+ struct scatterlist *sgl_mkey;
+ u32 lkey, i;
+
+ lkey = queue->priv->mdev->mlx5e_res.hw_objs.mkey;
+ for (i = 0; i < klm_entries; i++) {
+ sgl_mkey = &queue->ccid_table[ccid].sgl[i + klm_offset];
+ wqe->inline_klms[i].bcount = cpu_to_be32(sg_dma_len(sgl_mkey));
+ wqe->inline_klms[i].key = cpu_to_be32(lkey);
+ wqe->inline_klms[i].va = cpu_to_be64(sgl_mkey->dma_address);
+ }
+
+ for (; i < ALIGN(klm_entries, MLX5_UMR_KLM_ALIGNMENT); i++) {
+ wqe->inline_klms[i].bcount = 0;
+ wqe->inline_klms[i].key = 0;
+ wqe->inline_klms[i].va = 0;
+ }
+}
+
+static void
+build_nvmeotcp_klm_umr(struct mlx5e_nvmeotcp_queue *queue, struct mlx5e_umr_wqe *wqe,
+ u16 ccid, int klm_entries, u32 klm_offset, u32 len,
+ enum wqe_type klm_type)
+{
+ u32 id = (klm_type == KLM_UMR) ? queue->ccid_table[ccid].klm_mkey :
+ (mlx5e_tir_get_tirn(&queue->tir) << MLX5_WQE_CTRL_TIR_TIS_INDEX_SHIFT);
+ u8 opc_mod = (klm_type == KLM_UMR) ? MLX5_CTRL_SEGMENT_OPC_MOD_UMR_UMR :
+ MLX5_OPC_MOD_TRANSPORT_TIR_STATIC_PARAMS;
+ u32 ds_cnt = MLX5E_KLM_UMR_DS_CNT(ALIGN(klm_entries, MLX5_UMR_KLM_ALIGNMENT));
+ struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
+ struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
+ struct mlx5_mkey_seg *mkc = &wqe->mkc;
+ u32 sqn = queue->sq.sqn;
+ u16 pc = queue->sq.pc;
+
+ cseg->opmod_idx_opcode = cpu_to_be32((pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
+ MLX5_OPCODE_UMR | (opc_mod) << 24);
+ cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) | ds_cnt);
+ cseg->general_id = cpu_to_be32(id);
+
+ if (klm_type == KLM_UMR && !klm_offset) {
+ ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_XLT_OCT_SIZE |
+ MLX5_MKEY_MASK_LEN | MLX5_MKEY_MASK_FREE);
+ mkc->xlt_oct_size = cpu_to_be32(ALIGN(len, MLX5_UMR_KLM_ALIGNMENT));
+ mkc->len = cpu_to_be64(queue->ccid_table[ccid].size);
+ }
+
+ ucseg->flags = MLX5_UMR_INLINE | MLX5_UMR_TRANSLATION_OFFSET_EN;
+ ucseg->xlt_octowords = cpu_to_be16(ALIGN(klm_entries, MLX5_UMR_KLM_ALIGNMENT));
+ ucseg->xlt_offset = cpu_to_be16(klm_offset);
+ fill_nvmeotcp_klm_wqe(queue, wqe, ccid, klm_entries, klm_offset);
+}
+
+static void
+mlx5e_nvmeotcp_fill_wi(struct mlx5e_icosq *sq, u32 wqebbs, u16 pi)
+{
+ struct mlx5e_icosq_wqe_info *wi = &sq->db.wqe_info[pi];
+
+ memset(wi, 0, sizeof(*wi));
+
+ wi->num_wqebbs = wqebbs;
+ wi->wqe_type = MLX5E_ICOSQ_WQE_UMR_NVMEOTCP;
+}
+
+static u32
+post_klm_wqe(struct mlx5e_nvmeotcp_queue *queue,
+ enum wqe_type wqe_type,
+ u16 ccid,
+ u32 klm_length,
+ u32 klm_offset)
+{
+ struct mlx5e_icosq *sq = &queue->sq;
+ u32 wqebbs, cur_klm_entries;
+ struct mlx5e_umr_wqe *wqe;
+ u16 pi, wqe_sz;
+
+ cur_klm_entries = min_t(int, queue->max_klms_per_wqe, klm_length - klm_offset);
+ wqe_sz = MLX5E_KLM_UMR_WQE_SZ(ALIGN(cur_klm_entries, MLX5_UMR_KLM_ALIGNMENT));
+ wqebbs = DIV_ROUND_UP(wqe_sz, MLX5_SEND_WQE_BB);
+ pi = mlx5e_icosq_get_next_pi(sq, wqebbs);
+ wqe = MLX5E_NVMEOTCP_FETCH_KLM_WQE(sq, pi);
+ mlx5e_nvmeotcp_fill_wi(sq, wqebbs, pi);
+ build_nvmeotcp_klm_umr(queue, wqe, ccid, cur_klm_entries, klm_offset,
+ klm_length, wqe_type);
+ sq->pc += wqebbs;
+ sq->doorbell_cseg = &wqe->ctrl;
+ return cur_klm_entries;
+}
+
+static void
+mlx5e_nvmeotcp_post_klm_wqe(struct mlx5e_nvmeotcp_queue *queue, enum wqe_type wqe_type,
+ u16 ccid, u32 klm_length)
+{
+ struct mlx5e_icosq *sq = &queue->sq;
+ u32 klm_offset = 0, wqes, i;
+
+ wqes = DIV_ROUND_UP(klm_length, queue->max_klms_per_wqe);
+
+ spin_lock_bh(&queue->sq_lock);
+
+ for (i = 0; i < wqes; i++)
+ klm_offset += post_klm_wqe(queue, wqe_type, ccid, klm_length, klm_offset);
+
+ if (wqe_type == KLM_UMR) /* not asking for completion on ddp_setup UMRs */
+ __mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, sq->doorbell_cseg, 0);
+ else
+ mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, sq->doorbell_cseg);
+
+ spin_unlock_bh(&queue->sq_lock);
+}
+
static int
mlx5e_nvmeotcp_offload_limits(struct net_device *netdev,
struct ulp_ddp_limits *ulp_limits)
@@ -45,6 +160,14 @@ mlx5e_nvmeotcp_ddp_setup(struct net_device *netdev,
struct sock *sk,
struct ulp_ddp_io *ddp)
{
+ struct mlx5e_nvmeotcp_queue *queue;
+
+ queue = container_of(ulp_ddp_get_ctx(sk),
+ struct mlx5e_nvmeotcp_queue, ulp_ddp_ctx);
+
+ /* Placeholder - map_sg and initializing the count */
+
+ mlx5e_nvmeotcp_post_klm_wqe(queue, KLM_UMR, ddp->command_id, 0);
return 0;
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp_utils.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp_utils.h
new file mode 100644
index 000000000000..c7b176577167
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/nvmeotcp_utils.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. */
+#ifndef __MLX5E_NVMEOTCP_UTILS_H__
+#define __MLX5E_NVMEOTCP_UTILS_H__
+
+#include "en.h"
+
+#define MLX5E_NVMEOTCP_FETCH_KLM_WQE(sq, pi) \
+ ((struct mlx5e_umr_wqe *)\
+ mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_umr_wqe)))
+
+#define MLX5_CTRL_SEGMENT_OPC_MOD_UMR_NVMEOTCP_TIR_PROGRESS_PARAMS 0x4
+
+#define MLX5_CTRL_SEGMENT_OPC_MOD_UMR_TIR_PARAMS 0x2
+#define MLX5_CTRL_SEGMENT_OPC_MOD_UMR_UMR 0x0
+
+enum wqe_type {
+ KLM_UMR,
+ BSF_KLM_UMR,
+ SET_PSV_UMR,
+ BSF_UMR,
+ KLM_INV_UMR,
+};
+
+#endif /* __MLX5E_NVMEOTCP_UTILS_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 261802579791..039eeb3b3e45 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -921,6 +921,10 @@ int mlx5e_poll_ico_cq(struct mlx5e_cq *cq, int budget)
case MLX5E_ICOSQ_WQE_GET_PSV_TLS:
mlx5e_ktls_handle_get_psv_completion(wi, sq);
break;
+#endif
+#ifdef CONFIG_MLX5_EN_NVMEOTCP
+ case MLX5E_ICOSQ_WQE_UMR_NVMEOTCP:
+ break;
#endif
default:
netdev_WARN_ONCE(cq->netdev,
--
2.31.1
next prev parent reply other threads:[~2022-10-20 10:22 UTC|newest]
Thread overview: 28+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-10-20 10:18 [PATCH v6 00/23] nvme-tcp receive offloads Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 01/23] net: Introduce direct data placement tcp offload Aurelien Aptel
2022-10-21 5:05 ` Jakub Kicinski
2022-10-21 11:19 ` Aurelien Aptel
2022-10-24 11:22 ` Leon Romanovsky
2022-10-24 13:09 ` Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 02/23] iov_iter: DDP copy to iter/pages Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 03/23] net/tls: export get_netdev_for_sock Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 04/23] Revert "nvme-tcp: remove the unused queue_size member in nvme_tcp_queue" Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 05/23] nvme-tcp: Add DDP offload control path Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 06/23] nvme-tcp: Add DDP data-path Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 07/23] nvme-tcp: RX DDGST offload Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 08/23] nvme-tcp: Deal with netdevice DOWN events Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 09/23] nvme-tcp: Add ulp_offload modparam to control enablement of ULP offload Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 10/23] Documentation: add ULP DDP offload documentation Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 11/23] net/mlx5e: Rename from tls to transport static params Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 12/23] net/mlx5e: Refactor ico sq polling to get budget Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 13/23] net/mlx5e: Have mdev pointer directly on the icosq structure Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 14/23] net/mlx5e: Refactor doorbell function to allow avoiding a completion Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 15/23] net/mlx5: Add NVMEoTCP caps, HW bits, 128B CQE and enumerations Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 16/23] net/mlx5e: NVMEoTCP, offload initialization Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 17/23] net/mlx5e: TCP flow steering for nvme-tcp acceleration Aurelien Aptel
2022-10-20 10:18 ` Aurelien Aptel [this message]
2022-10-20 10:18 ` [PATCH v6 19/23] net/mlx5e: NVMEoTCP, queue init/teardown Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 20/23] net/mlx5e: NVMEoTCP, ddp setup and resync Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 21/23] net/mlx5e: NVMEoTCP, async ddp invalidation Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 22/23] net/mlx5e: NVMEoTCP, data-path for DDP+DDGST offload Aurelien Aptel
2022-10-20 10:18 ` [PATCH v6 23/23] net/mlx5e: NVMEoTCP, statistics Aurelien Aptel
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20221020101838.2712846-19-aaptel@nvidia.com \
--to=aaptel@nvidia.com \
--cc=aurelien.aptel@gmail.com \
--cc=axboe@fb.com \
--cc=borisp@nvidia.com \
--cc=chaitanyak@nvidia.com \
--cc=davem@davemloft.net \
--cc=edumazet@google.com \
--cc=hch@lst.de \
--cc=kbusch@kernel.org \
--cc=kuba@kernel.org \
--cc=linux-nvme@lists.infradead.org \
--cc=malin1024@gmail.com \
--cc=netdev@vger.kernel.org \
--cc=ogerlitz@nvidia.com \
--cc=pabeni@redhat.com \
--cc=saeedm@nvidia.com \
--cc=sagi@grimberg.me \
--cc=smalin@nvidia.com \
--cc=tariqt@nvidia.com \
--cc=yorayz@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).