From: Alexander Lobakin <aleksander.lobakin@intel.com>
To: "David S. Miller" <davem@davemloft.net>,
Eric Dumazet <edumazet@google.com>,
Jakub Kicinski <kuba@kernel.org>, Paolo Abeni <pabeni@redhat.com>
Cc: Alexander Lobakin <aleksander.lobakin@intel.com>,
Maciej Fijalkowski <maciej.fijalkowski@intel.com>,
Michal Kubiak <michal.kubiak@intel.com>,
Larysa Zaremba <larysa.zaremba@intel.com>,
Alexei Starovoitov <ast@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
Willem de Bruijn <willemdebruijn.kernel@gmail.com>,
intel-wired-lan@lists.osuosl.org, netdev@vger.kernel.org,
linux-kernel@vger.kernel.org
Subject: [PATCH RFC net-next 24/34] idpf: add support for XDP on Rx
Date: Sat, 23 Dec 2023 03:55:44 +0100 [thread overview]
Message-ID: <20231223025554.2316836-25-aleksander.lobakin@intel.com> (raw)
In-Reply-To: <20231223025554.2316836-1-aleksander.lobakin@intel.com>
Use libie XDP infra to support running XDP program on Rx polling.
This includes all of the possible verdicts/actions.
XDP Tx queues are cleaned only in "lazy" mode when there are less than
1/4 free descriptors left on the ring. Some functions are oneliners
around libie's __always_inlines, so that the compiler could uninline
them when needed.
Co-developed-by: Michal Kubiak <michal.kubiak@intel.com>
Signed-off-by: Michal Kubiak <michal.kubiak@intel.com>
Signed-off-by: Alexander Lobakin <aleksander.lobakin@intel.com>
---
drivers/net/ethernet/intel/idpf/idpf_lib.c | 6 +
drivers/net/ethernet/intel/idpf/idpf_txrx.c | 10 +-
drivers/net/ethernet/intel/idpf/idpf_txrx.h | 55 ++++++++
drivers/net/ethernet/intel/idpf/idpf_xdp.c | 140 ++++++++++++++++++++
drivers/net/ethernet/intel/idpf/idpf_xdp.h | 20 ++-
5 files changed, 227 insertions(+), 4 deletions(-)
diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c
index 01130e7c4d2e..a19704c4c421 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_lib.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c
@@ -840,6 +840,12 @@ static int idpf_cfg_netdev(struct idpf_vport *vport)
netdev->features |= dflt_features;
netdev->hw_features |= dflt_features | offloads;
netdev->hw_enc_features |= dflt_features | offloads;
+
+ if (idpf_is_queue_model_split(vport->rxq_model))
+ xdp_set_features_flag(netdev, NETDEV_XDP_ACT_BASIC |
+ NETDEV_XDP_ACT_REDIRECT |
+ NETDEV_XDP_ACT_RX_SG);
+
idpf_set_ethtool_ops(netdev);
SET_NETDEV_DEV(netdev, &adapter->pdev->dev);
diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c
index cbbb6bf85b19..99c9b889507b 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c
@@ -1522,7 +1522,7 @@ int idpf_vport_queues_alloc(struct idpf_vport *vport)
* idpf_tx_handle_sw_marker - Handle queue marker packet
* @tx_q: tx queue to handle software marker
*/
-static void idpf_tx_handle_sw_marker(struct idpf_queue *tx_q)
+void idpf_tx_handle_sw_marker(struct idpf_queue *tx_q)
{
struct idpf_vport *vport = tx_q->vport;
int i;
@@ -3045,8 +3045,11 @@ static int idpf_rx_splitq_clean(struct idpf_queue *rxq, int budget)
int total_rx_bytes = 0, total_rx_pkts = 0;
struct idpf_queue *rx_bufq = NULL;
u16 ntc = rxq->next_to_clean;
+ struct libie_xdp_tx_bulk bq;
struct xdp_buff xdp;
+ libie_xdp_tx_init_bulk(&bq, rxq->xdp_prog, rxq->xdp_rxq.dev,
+ rxq->xdpqs, rxq->num_xdp_txq);
libie_xdp_init_buff(&xdp, &rxq->xdp, &rxq->xdp_rxq);
/* Process Rx packets bounded by budget */
@@ -3161,6 +3164,9 @@ static int idpf_rx_splitq_clean(struct idpf_queue *rxq, int budget)
total_rx_bytes += xdp_get_buff_len(&xdp);
total_rx_pkts++;
+ if (!idpf_xdp_run_prog(&xdp, &bq))
+ continue;
+
skb = xdp_build_skb_from_buff(&xdp);
if (unlikely(!skb)) {
xdp_return_buff(&xdp);
@@ -3182,7 +3188,9 @@ static int idpf_rx_splitq_clean(struct idpf_queue *rxq, int budget)
}
rxq->next_to_clean = ntc;
+
libie_xdp_save_buff(&rxq->xdp, &xdp);
+ idpf_xdp_finalize_rx(&bq);
u64_stats_update_begin(&rxq->stats_sync);
u64_stats_add(&rxq->q_stats.rx.packets, total_rx_pkts);
diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h
index 318241020347..20f484712ac2 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h
+++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h
@@ -135,6 +135,8 @@ do { \
((++(txq)->compl_tag_cur_gen) >= (txq)->compl_tag_gen_max ? \
0 : (txq)->compl_tag_cur_gen)
+#define IDPF_QUEUE_QUARTER(Q) ((Q)->desc_count >> 2)
+
#define IDPF_TXD_LAST_DESC_CMD (IDPF_TX_DESC_CMD_EOP | IDPF_TX_DESC_CMD_RS)
#define IDPF_TX_FLAGS_TSO BIT(0)
@@ -939,5 +941,58 @@ netdev_tx_t idpf_tx_singleq_start(struct sk_buff *skb,
bool idpf_rx_singleq_buf_hw_alloc_all(struct idpf_queue *rxq,
u16 cleaned_count);
int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off);
+void idpf_tx_handle_sw_marker(struct idpf_queue *tx_q);
+
+/**
+ * idpf_xdpq_update_tail - Updates the XDP Tx queue tail register
+ * @xdpq: XDP Tx queue
+ *
+ * This function updates the XDP Tx queue tail register.
+ */
+static inline void idpf_xdpq_update_tail(const struct idpf_queue *xdpq)
+{
+ /* Force memory writes to complete before letting h/w
+ * know there are new descriptors to fetch.
+ */
+ wmb();
+ writel_relaxed(xdpq->next_to_use, xdpq->tail);
+}
+
+/**
+ * idpf_set_rs_bit - set RS bit on last produced descriptor.
+ * @xdpq: XDP queue to produce the HW Tx descriptors on
+ *
+ * Returns the index of descriptor RS bit was set on (one behind current NTU).
+ */
+static inline void idpf_set_rs_bit(const struct idpf_queue *xdpq)
+{
+ int rs_idx = xdpq->next_to_use ? xdpq->next_to_use - 1 :
+ xdpq->desc_count - 1;
+ union idpf_tx_flex_desc *tx_desc;
+
+ tx_desc = &xdpq->flex_tx[rs_idx];
+ tx_desc->q.qw1.cmd_dtype |= le16_encode_bits(IDPF_TXD_LAST_DESC_CMD,
+ IDPF_FLEX_TXD_QW1_CMD_M);
+}
+
+/**
+ * idpf_xdp_tx_finalize - Bump XDP Tx tail and/or flush redirect map
+ * @xdpq: XDP Tx queue
+ *
+ * This function bumps XDP Tx tail and should be called when a batch of packets
+ * has been processed in the napi loop.
+ */
+static inline void idpf_xdp_tx_finalize(void *_xdpq, bool tail)
+{
+ struct idpf_queue *xdpq = _xdpq;
+
+ libie_xdp_sq_lock(&xdpq->xdp_lock);
+
+ idpf_set_rs_bit(xdpq);
+ if (tail)
+ idpf_xdpq_update_tail(xdpq);
+
+ libie_xdp_sq_unlock(&xdpq->xdp_lock);
+}
#endif /* !_IDPF_TXRX_H_ */
diff --git a/drivers/net/ethernet/intel/idpf/idpf_xdp.c b/drivers/net/ethernet/intel/idpf/idpf_xdp.c
index 87d147e80047..b9952ebda4fb 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_xdp.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_xdp.c
@@ -173,6 +173,146 @@ void idpf_vport_xdpq_put(const struct idpf_vport *vport)
cpus_read_unlock();
}
+/**
+ * idpf_clean_xdp_irq - Reclaim a batch of TX resources from completed XDP_TX
+ * @xdpq: XDP Tx queue
+ *
+ * Returns number of cleaned descriptors.
+ */
+static u32 idpf_clean_xdp_irq(struct idpf_queue *xdpq)
+{
+ struct idpf_queue *complq = xdpq->txq_grp->complq, *txq;
+ struct idpf_splitq_4b_tx_compl_desc *last_rs_desc;
+ struct libie_sq_onstack_stats ss = { };
+ int complq_budget = complq->desc_count;
+ u32 tx_ntc = xdpq->next_to_clean;
+ u32 ntc = complq->next_to_clean;
+ u32 cnt = xdpq->desc_count;
+ u32 done_frames = 0, i = 0;
+ struct xdp_frame_bulk bq;
+ int head = tx_ntc;
+ bool gen_flag;
+
+ last_rs_desc = &complq->comp_4b[ntc];
+ gen_flag = test_bit(__IDPF_Q_GEN_CHK, complq->flags);
+
+ do {
+ int ctype = idpf_parse_compl_desc(last_rs_desc, complq,
+ &txq, gen_flag);
+ if (likely(ctype == IDPF_TXD_COMPLT_RS)) {
+ head = le16_to_cpu(last_rs_desc->q_head_compl_tag.q_head);
+ goto fetch_next_desc;
+ }
+
+ switch (ctype) {
+ case IDPF_TXD_COMPLT_SW_MARKER:
+ idpf_tx_handle_sw_marker(xdpq);
+ break;
+ case -ENODATA:
+ goto exit_xdp_irq;
+ case -EINVAL:
+ break;
+ default:
+ dev_err(&xdpq->vport->adapter->pdev->dev,
+ "Unsupported completion type for XDP\n");
+ break;
+ }
+
+fetch_next_desc:
+ last_rs_desc++;
+ ntc++;
+ if (unlikely(ntc == complq->desc_count)) {
+ ntc = 0;
+ last_rs_desc = &complq->comp_4b[0];
+ gen_flag = !gen_flag;
+ change_bit(__IDPF_Q_GEN_CHK, complq->flags);
+ }
+ prefetch(last_rs_desc);
+ complq_budget--;
+ } while (likely(complq_budget));
+
+exit_xdp_irq:
+ complq->next_to_clean = ntc;
+ done_frames = head >= tx_ntc ? head - tx_ntc :
+ head + cnt - tx_ntc;
+
+ xdp_frame_bulk_init(&bq);
+
+ for (i = 0; i < done_frames; i++) {
+ libie_xdp_complete_tx_buf(&xdpq->tx_buf[tx_ntc], xdpq->dev,
+ true, &bq, &xdpq->xdp_tx_active,
+ &ss);
+
+ if (unlikely(++tx_ntc == cnt))
+ tx_ntc = 0;
+ }
+
+ xdpq->next_to_clean = tx_ntc;
+
+ xdp_flush_frame_bulk(&bq);
+ libie_sq_napi_stats_add((struct libie_sq_stats *)&xdpq->q_stats.tx,
+ &ss);
+
+ return i;
+}
+
+static u32 idpf_xdp_tx_prep(void *_xdpq, struct libie_xdp_tx_queue *sq)
+{
+ struct idpf_queue *xdpq = _xdpq;
+ u32 free;
+
+ libie_xdp_sq_lock(&xdpq->xdp_lock);
+
+ free = IDPF_DESC_UNUSED(xdpq);
+ if (unlikely(free < IDPF_QUEUE_QUARTER(xdpq)))
+ free += idpf_clean_xdp_irq(xdpq);
+
+ *sq = (struct libie_xdp_tx_queue){
+ .dev = xdpq->dev,
+ .tx_buf = xdpq->tx_buf,
+ .desc_ring = xdpq->desc_ring,
+ .xdp_lock = &xdpq->xdp_lock,
+ .next_to_use = &xdpq->next_to_use,
+ .desc_count = xdpq->desc_count,
+ .xdp_tx_active = &xdpq->xdp_tx_active,
+ };
+
+ return free;
+}
+
+static void idpf_xdp_tx_xmit(struct libie_xdp_tx_desc desc,
+ const struct libie_xdp_tx_queue *sq)
+{
+ union idpf_tx_flex_desc *tx_desc = sq->desc_ring;
+ struct idpf_tx_splitq_params tx_params = {
+ .dtype = IDPF_TX_DESC_DTYPE_FLEX_L2TAG1_L2TAG2,
+ .eop_cmd = IDPF_TX_DESC_CMD_EOP,
+ };
+
+ tx_desc = &tx_desc[*sq->next_to_use];
+ tx_desc->q.buf_addr = cpu_to_le64(desc.addr);
+
+ idpf_tx_splitq_build_desc(tx_desc, &tx_params,
+ tx_params.eop_cmd | tx_params.offload.td_cmd,
+ desc.len);
+}
+
+static bool idpf_xdp_tx_flush_bulk(struct libie_xdp_tx_bulk *bq)
+{
+ return libie_xdp_tx_flush_bulk(bq, idpf_xdp_tx_prep, idpf_xdp_tx_xmit);
+}
+
+void __idpf_xdp_finalize_rx(struct libie_xdp_tx_bulk *bq)
+{
+ libie_xdp_finalize_rx(bq, idpf_xdp_tx_flush_bulk,
+ idpf_xdp_tx_finalize);
+}
+
+bool __idpf_xdp_run_prog(struct xdp_buff *xdp, struct libie_xdp_tx_bulk *bq)
+{
+ return libie_xdp_run_prog(xdp, bq, idpf_xdp_tx_flush_bulk);
+}
+
/**
* idpf_xdp_reconfig_queues - reconfigure queues after the XDP setup
* @vport: vport to load or unload XDP for
diff --git a/drivers/net/ethernet/intel/idpf/idpf_xdp.h b/drivers/net/ethernet/intel/idpf/idpf_xdp.h
index 1d102b1fd2ac..1f299c268ca5 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_xdp.h
+++ b/drivers/net/ethernet/intel/idpf/idpf_xdp.h
@@ -4,10 +4,9 @@
#ifndef _IDPF_XDP_H_
#define _IDPF_XDP_H_
-struct bpf_prog;
+#include <linux/net/intel/libie/xdp.h>
+
struct idpf_vport;
-struct net_device;
-struct netdev_bpf;
int idpf_xdp_rxq_info_init_all(const struct idpf_vport *vport);
void idpf_xdp_rxq_info_deinit_all(const struct idpf_vport *vport);
@@ -17,6 +16,21 @@ void idpf_copy_xdp_prog_to_qs(const struct idpf_vport *vport,
void idpf_vport_xdpq_get(const struct idpf_vport *vport);
void idpf_vport_xdpq_put(const struct idpf_vport *vport);
+bool __idpf_xdp_run_prog(struct xdp_buff *xdp, struct libie_xdp_tx_bulk *bq);
+void __idpf_xdp_finalize_rx(struct libie_xdp_tx_bulk *bq);
+
+static inline bool idpf_xdp_run_prog(struct xdp_buff *xdp,
+ struct libie_xdp_tx_bulk *bq)
+{
+ return bq->prog ? __idpf_xdp_run_prog(xdp, bq) : true;
+}
+
+static inline void idpf_xdp_finalize_rx(struct libie_xdp_tx_bulk *bq)
+{
+ if (bq->act_mask >= LIBIE_XDP_TX)
+ __idpf_xdp_finalize_rx(bq);
+}
+
int idpf_xdp(struct net_device *netdev, struct netdev_bpf *xdp);
#endif /* _IDPF_XDP_H_ */
--
2.43.0
next prev parent reply other threads:[~2023-12-23 2:59 UTC|newest]
Thread overview: 45+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-12-23 2:55 [PATCH RFC net-next 00/34] Christmas 3-serie XDP for idpf (+generic stuff) Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 01/34] idpf: reuse libie's definitions of parsed ptype structures Alexander Lobakin
2023-12-27 15:43 ` Willem de Bruijn
2024-01-08 16:04 ` Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 02/34] idpf: pack &idpf_queue way more efficiently Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 03/34] idpf: remove legacy Page Pool Ethtool stats Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 04/34] libie: support different types of buffers for Rx Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 05/34] idpf: convert header split mode to libie + napi_build_skb() Alexander Lobakin
2023-12-27 15:30 ` Willem de Bruijn
2024-01-08 16:17 ` Alexander Lobakin
2024-01-09 13:59 ` Willem de Bruijn
2024-01-11 13:09 ` Alexander Lobakin
2024-01-09 14:43 ` Eric Dumazet
2023-12-23 2:55 ` [PATCH RFC net-next 06/34] idpf: use libie Rx buffer management for payload buffer Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 07/34] libie: add Tx buffer completion helpers Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 08/34] idpf: convert to libie Tx buffer completion Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 09/34] bpf, xdp: constify some bpf_prog * function arguments Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 10/34] xdp: constify read-only arguments of some static inline helpers Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 11/34] xdp: allow attaching already registered memory model to xdp_rxq_info Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 12/34] xdp: add generic xdp_buff_add_frag() Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 13/34] xdp: add generic xdp_build_skb_from_buff() Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 14/34] xdp: get rid of xdp_frame::mem.id Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 15/34] page_pool: add inline helper to sync VA for device (for XDP_TX) Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 16/34] jump_label: export static_key_slow_{inc,dec}_cpuslocked() Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 17/34] libie: support native XDP and register memory model Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 18/34] libie: add a couple of XDP helpers Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 19/34] idpf: stop using macros for accessing queue descriptors Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 20/34] idpf: make complq cleaning dependent on scheduling mode Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 21/34] idpf: prepare structures to support xdp Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 22/34] idpf: implement XDP_SETUP_PROG in ndo_bpf for splitq Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 23/34] idpf: use generic functions to build xdp_buff and skb Alexander Lobakin
2023-12-23 2:55 ` Alexander Lobakin [this message]
2023-12-23 2:55 ` [PATCH RFC net-next 25/34] idpf: add support for .ndo_xdp_xmit() Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 26/34] xdp: add generic XSk xdp_buff -> skb conversion Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 27/34] idpf: add support for sw interrupt Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 28/34] idpf: add relative queue id member to idpf_queue Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 29/34] idpf: add vc functions to manage selected queues Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 30/34] idpf: move search rx and tx queues to header Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 31/34] idpf: add XSk pool initialization Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 32/34] idpf: implement Tx path for AF_XDP Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 33/34] idpf: implement Rx " Alexander Lobakin
2023-12-23 2:55 ` [PATCH RFC net-next 34/34] idpf: enable XSk features and ndo_xsk_wakeup Alexander Lobakin
2023-12-26 20:23 ` [PATCH RFC net-next 00/34] Christmas 3-serie XDP for idpf (+generic stuff) Willem de Bruijn
2024-01-08 16:01 ` Alexander Lobakin
2024-01-08 16:09 ` Willem de Bruijn
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20231223025554.2316836-25-aleksander.lobakin@intel.com \
--to=aleksander.lobakin@intel.com \
--cc=ast@kernel.org \
--cc=daniel@iogearbox.net \
--cc=davem@davemloft.net \
--cc=edumazet@google.com \
--cc=intel-wired-lan@lists.osuosl.org \
--cc=kuba@kernel.org \
--cc=larysa.zaremba@intel.com \
--cc=linux-kernel@vger.kernel.org \
--cc=maciej.fijalkowski@intel.com \
--cc=michal.kubiak@intel.com \
--cc=netdev@vger.kernel.org \
--cc=pabeni@redhat.com \
--cc=willemdebruijn.kernel@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox