From: Saeed Mahameed <saeedm@mellanox.com>
To: "David S. Miller" <davem@davemloft.net>,
Jakub Kicinski <kuba@kernel.org>
Cc: netdev@vger.kernel.org, Tariq Toukan <tariqt@mellanox.com>,
Maxim Mikityanskiy <maximmi@mellanox.com>,
Saeed Mahameed <saeedm@mellanox.com>
Subject: [net-next 12/15] net/mlx5e: XDP, Avoid indirect call in TX flow
Date: Thu, 16 Jul 2020 14:33:18 -0700 [thread overview]
Message-ID: <20200716213321.29468-13-saeedm@mellanox.com> (raw)
In-Reply-To: <20200716213321.29468-1-saeedm@mellanox.com>
From: Tariq Toukan <tariqt@mellanox.com>
Use INDIRECT_CALL_2() helper to avoid the cost of the indirect call
when/if CONFIG_RETPOLINE=y.
Signed-off-by: Tariq Toukan <tariqt@mellanox.com>
Reviewed-by: Maxim Mikityanskiy <maximmi@mellanox.com>
Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>
---
.../net/ethernet/mellanox/mlx5/core/en/xdp.c | 36 ++++++++++++-------
.../ethernet/mellanox/mlx5/core/en/xsk/tx.c | 22 ++++++++++--
2 files changed, 44 insertions(+), 14 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
index c9d308e919655..3514208111555 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
@@ -34,6 +34,7 @@
#include <net/xdp_sock_drv.h>
#include "en/xdp.h"
#include "en/params.h"
+#include <linux/indirect_call_wrapper.h>
int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk)
{
@@ -55,6 +56,15 @@ int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk)
return MLX5E_HW2SW_MTU(params, SKB_MAX_HEAD(hr));
}
+INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq,
+ struct mlx5e_xdp_xmit_data *xdptxd,
+ struct mlx5e_xdp_info *xdpi,
+ int check_result));
+INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq,
+ struct mlx5e_xdp_xmit_data *xdptxd,
+ struct mlx5e_xdp_info *xdpi,
+ int check_result));
+
static inline bool
mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq,
struct mlx5e_dma_info *di, struct xdp_buff *xdp)
@@ -114,7 +124,8 @@ mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq,
xdpi.page.di = *di;
}
- return sq->xmit_xdp_frame(sq, &xdptxd, &xdpi, 0);
+ return INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe,
+ mlx5e_xmit_xdp_frame, sq, &xdptxd, &xdpi, 0);
}
/* returns true if packet was consumed by xdp */
@@ -237,7 +248,7 @@ enum {
MLX5E_XDP_CHECK_START_MPWQE = 2,
};
-static int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq)
+INDIRECT_CALLABLE_SCOPE int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq)
{
if (unlikely(!sq->mpwqe.wqe)) {
const u16 stop_room = mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS);
@@ -256,10 +267,9 @@ static int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq)
return MLX5E_XDP_CHECK_OK;
}
-static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq,
- struct mlx5e_xdp_xmit_data *xdptxd,
- struct mlx5e_xdp_info *xdpi,
- int check_result)
+INDIRECT_CALLABLE_SCOPE bool
+mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_xmit_data *xdptxd,
+ struct mlx5e_xdp_info *xdpi, int check_result)
{
struct mlx5e_xdp_mpwqe *session = &sq->mpwqe;
struct mlx5e_xdpsq_stats *stats = sq->stats;
@@ -293,7 +303,7 @@ static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq,
return true;
}
-static int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq)
+INDIRECT_CALLABLE_SCOPE int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq)
{
if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, 1))) {
/* SQ is full, ring doorbell */
@@ -305,10 +315,9 @@ static int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq)
return MLX5E_XDP_CHECK_OK;
}
-static bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq,
- struct mlx5e_xdp_xmit_data *xdptxd,
- struct mlx5e_xdp_info *xdpi,
- int check_result)
+INDIRECT_CALLABLE_SCOPE bool
+mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_xmit_data *xdptxd,
+ struct mlx5e_xdp_info *xdpi, int check_result)
{
struct mlx5_wq_cyc *wq = &sq->wq;
u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
@@ -506,6 +515,7 @@ int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
struct xdp_frame *xdpf = frames[i];
struct mlx5e_xdp_xmit_data xdptxd;
struct mlx5e_xdp_info xdpi;
+ bool ret;
xdptxd.data = xdpf->data;
xdptxd.len = xdpf->len;
@@ -522,7 +532,9 @@ int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
xdpi.frame.xdpf = xdpf;
xdpi.frame.dma_addr = xdptxd.dma_addr;
- if (unlikely(!sq->xmit_xdp_frame(sq, &xdptxd, &xdpi, 0))) {
+ ret = INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe,
+ mlx5e_xmit_xdp_frame, sq, &xdptxd, &xdpi, 0);
+ if (unlikely(!ret)) {
dma_unmap_single(sq->pdev, xdptxd.dma_addr,
xdptxd.len, DMA_TO_DEVICE);
xdp_return_frame_rx_napi(xdpf);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
index e0b3c61af93ea..44f2d7bcf4bb2 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
@@ -6,6 +6,7 @@
#include "en/xdp.h"
#include "en/params.h"
#include <net/xdp_sock_drv.h>
+#include <linux/indirect_call_wrapper.h>
int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags)
{
@@ -64,6 +65,17 @@ static void mlx5e_xsk_tx_post_err(struct mlx5e_xdpsq *sq,
sq->doorbell_cseg = &nopwqe->ctrl;
}
+INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq,
+ struct mlx5e_xdp_xmit_data *xdptxd,
+ struct mlx5e_xdp_info *xdpi,
+ int check_result));
+INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq,
+ struct mlx5e_xdp_xmit_data *xdptxd,
+ struct mlx5e_xdp_info *xdpi,
+ int check_result));
+INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq));
+INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq));
+
bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget)
{
struct xdp_umem *umem = sq->umem;
@@ -75,8 +87,12 @@ bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget)
xdpi.mode = MLX5E_XDP_XMIT_MODE_XSK;
for (; budget; budget--) {
- int check_result = sq->xmit_xdp_frame_check(sq);
+ int check_result = INDIRECT_CALL_2(sq->xmit_xdp_frame_check,
+ mlx5e_xmit_xdp_frame_check_mpwqe,
+ mlx5e_xmit_xdp_frame_check,
+ sq);
struct xdp_desc desc;
+ bool ret;
if (unlikely(check_result < 0)) {
work_done = false;
@@ -98,7 +114,9 @@ bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget)
xsk_buff_raw_dma_sync_for_device(umem, xdptxd.dma_addr, xdptxd.len);
- if (unlikely(!sq->xmit_xdp_frame(sq, &xdptxd, &xdpi, check_result))) {
+ ret = INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe,
+ mlx5e_xmit_xdp_frame, sq, &xdptxd, &xdpi, check_result);
+ if (unlikely(!ret)) {
if (sq->mpwqe.wqe)
mlx5e_xdp_mpwqe_complete(sq);
--
2.26.2
next prev parent reply other threads:[~2020-07-16 21:34 UTC|newest]
Thread overview: 18+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-07-16 21:33 [pull request][net-next 00/15] mlx5 updates 2020-07-16 Saeed Mahameed
2020-07-16 21:33 ` [net-next 01/15] net/mlx5e: Fix missing switch_id for representors Saeed Mahameed
2020-07-16 21:33 ` [net-next 02/15] net/mlx5e: Fix build break when CONFIG_XPS is not set Saeed Mahameed
2020-07-16 21:33 ` [net-next 03/15] net/mlx5: Make MLX5_EN_TLS non-prompt Saeed Mahameed
2020-07-16 21:33 ` [net-next 04/15] net/mlx5: E-switch, Avoid function change handler for non ECPF Saeed Mahameed
2020-07-16 21:33 ` [net-next 05/15] net/mlx5: E-switch, Reduce dependency on num_vfs during mode set Saeed Mahameed
2020-07-16 21:33 ` [net-next 06/15] net/mlx5: Accel, Add core IPsec support for the Connect-X family Saeed Mahameed
2020-07-16 21:33 ` [net-next 07/15] net/mlx5: IPsec: Add HW crypto offload support Saeed Mahameed
2020-07-16 21:33 ` [net-next 08/15] net/mlx5: Add IPsec related Flow steering entry's fields Saeed Mahameed
2020-07-16 21:33 ` [net-next 09/15] net/mlx5e: IPsec: Add IPsec steering in local NIC RX Saeed Mahameed
2020-07-16 21:33 ` [net-next 10/15] net/mlx5e: IPsec: Add Connect-X IPsec Rx data path offload Saeed Mahameed
2020-07-16 21:33 ` [net-next 11/15] net/mlx5e: IPsec: Add Connect-X IPsec ESN update offload support Saeed Mahameed
2020-07-16 21:33 ` Saeed Mahameed [this message]
2020-07-16 22:26 ` [net-next 12/15] net/mlx5e: XDP, Avoid indirect call in TX flow Jakub Kicinski
2020-07-16 23:07 ` Saeed Mahameed
2020-07-16 21:33 ` [net-next 13/15] net/mlx5e: RX, Avoid indirect call in representor CQE handling Saeed Mahameed
2020-07-16 21:33 ` [net-next 14/15] net/mlx5e: Do not request completion on every single UMR WQE Saeed Mahameed
2020-07-16 21:33 ` [net-next 15/15] net/mlx5e: CT: Map 128 bits labels to 32 bit map ID Saeed Mahameed
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200716213321.29468-13-saeedm@mellanox.com \
--to=saeedm@mellanox.com \
--cc=davem@davemloft.net \
--cc=kuba@kernel.org \
--cc=maximmi@mellanox.com \
--cc=netdev@vger.kernel.org \
--cc=tariqt@mellanox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.