* [PATCH net-next 1/9] ionic: set adminq irq affinity
2024-01-30 1:30 [PATCH net-next 0/9] ionic: add XDP support Shannon Nelson
@ 2024-01-30 1:30 ` Shannon Nelson
2024-01-30 1:30 ` [PATCH net-next 2/9] ionic: add helpers for accessing buffer info Shannon Nelson
` (7 subsequent siblings)
8 siblings, 0 replies; 12+ messages in thread
From: Shannon Nelson @ 2024-01-30 1:30 UTC (permalink / raw)
To: netdev, davem, kuba, edumazet, pabeni
Cc: brett.creeley, drivers, Shannon Nelson
We claim to have the AdminQ on our irq0 and thus cpu id 0,
but we need to be sure we set the affinity hint to try to
keep it there.
Signed-off-by: Shannon Nelson <shannon.nelson@amd.com>
---
drivers/net/ethernet/pensando/ionic/ionic_lif.c | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
index cf2d5ad7b68c..d92f8734d153 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
@@ -3391,9 +3391,12 @@ static int ionic_lif_adminq_init(struct ionic_lif *lif)
napi_enable(&qcq->napi);
- if (qcq->flags & IONIC_QCQ_F_INTR)
+ if (qcq->flags & IONIC_QCQ_F_INTR) {
+ irq_set_affinity_hint(qcq->intr.vector,
+ &qcq->intr.affinity_mask);
ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
IONIC_INTR_MASK_CLEAR);
+ }
qcq->flags |= IONIC_QCQ_F_INITED;
--
2.17.1
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH net-next 2/9] ionic: add helpers for accessing buffer info
2024-01-30 1:30 [PATCH net-next 0/9] ionic: add XDP support Shannon Nelson
2024-01-30 1:30 ` [PATCH net-next 1/9] ionic: set adminq irq affinity Shannon Nelson
@ 2024-01-30 1:30 ` Shannon Nelson
2024-01-30 1:30 ` [PATCH net-next 3/9] ionic: use dma range APIs Shannon Nelson
` (6 subsequent siblings)
8 siblings, 0 replies; 12+ messages in thread
From: Shannon Nelson @ 2024-01-30 1:30 UTC (permalink / raw)
To: netdev, davem, kuba, edumazet, pabeni
Cc: brett.creeley, drivers, Shannon Nelson
These helpers clean up some of the code around DMA mapping
and other buffer references, and will be used in the next
few patches for the XDP support.
Signed-off-by: Shannon Nelson <shannon.nelson@amd.com>
---
.../net/ethernet/pensando/ionic/ionic_txrx.c | 37 ++++++++++++-------
1 file changed, 24 insertions(+), 13 deletions(-)
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
index 54cd96b035d6..19a7a8a8e1b3 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
@@ -88,6 +88,21 @@ static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q)
return netdev_get_tx_queue(q->lif->netdev, q->index);
}
+static inline void *ionic_rx_buf_va(struct ionic_buf_info *buf_info)
+{
+ return page_address(buf_info->page) + buf_info->page_offset;
+}
+
+static inline dma_addr_t ionic_rx_buf_pa(struct ionic_buf_info *buf_info)
+{
+ return buf_info->dma_addr + buf_info->page_offset;
+}
+
+static inline unsigned int ionic_rx_buf_size(struct ionic_buf_info *buf_info)
+{
+ return min_t(u32, IONIC_MAX_BUF_LEN, IONIC_PAGE_SIZE - buf_info->page_offset);
+}
+
static int ionic_rx_page_alloc(struct ionic_queue *q,
struct ionic_buf_info *buf_info)
{
@@ -207,12 +222,11 @@ static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,
return NULL;
}
- frag_len = min_t(u16, len, min_t(u32, IONIC_MAX_BUF_LEN,
- IONIC_PAGE_SIZE - buf_info->page_offset));
+ frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info));
len -= frag_len;
dma_sync_single_for_cpu(dev,
- buf_info->dma_addr + buf_info->page_offset,
+ ionic_rx_buf_pa(buf_info),
frag_len, DMA_FROM_DEVICE);
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
@@ -262,10 +276,10 @@ static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,
return NULL;
}
- dma_sync_single_for_cpu(dev, buf_info->dma_addr + buf_info->page_offset,
+ dma_sync_single_for_cpu(dev, ionic_rx_buf_pa(buf_info),
len, DMA_FROM_DEVICE);
- skb_copy_to_linear_data(skb, page_address(buf_info->page) + buf_info->page_offset, len);
- dma_sync_single_for_device(dev, buf_info->dma_addr + buf_info->page_offset,
+ skb_copy_to_linear_data(skb, ionic_rx_buf_va(buf_info), len);
+ dma_sync_single_for_device(dev, ionic_rx_buf_pa(buf_info),
len, DMA_FROM_DEVICE);
skb_put(skb, len);
@@ -452,9 +466,8 @@ void ionic_rx_fill(struct ionic_queue *q)
}
/* fill main descriptor - buf[0] */
- desc->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset);
- frag_len = min_t(u16, len, min_t(u32, IONIC_MAX_BUF_LEN,
- IONIC_PAGE_SIZE - buf_info->page_offset));
+ desc->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info));
+ frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info));
desc->len = cpu_to_le16(frag_len);
remain_len -= frag_len;
buf_info++;
@@ -472,10 +485,8 @@ void ionic_rx_fill(struct ionic_queue *q)
}
}
- sg_elem->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset);
- frag_len = min_t(u16, remain_len, min_t(u32, IONIC_MAX_BUF_LEN,
- IONIC_PAGE_SIZE -
- buf_info->page_offset));
+ sg_elem->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info));
+ frag_len = min_t(u16, remain_len, ionic_rx_buf_size(buf_info));
sg_elem->len = cpu_to_le16(frag_len);
remain_len -= frag_len;
buf_info++;
--
2.17.1
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH net-next 3/9] ionic: use dma range APIs
2024-01-30 1:30 [PATCH net-next 0/9] ionic: add XDP support Shannon Nelson
2024-01-30 1:30 ` [PATCH net-next 1/9] ionic: set adminq irq affinity Shannon Nelson
2024-01-30 1:30 ` [PATCH net-next 2/9] ionic: add helpers for accessing buffer info Shannon Nelson
@ 2024-01-30 1:30 ` Shannon Nelson
2024-01-30 1:30 ` [PATCH net-next 4/9] ionic: add initial framework for XDP support Shannon Nelson
` (5 subsequent siblings)
8 siblings, 0 replies; 12+ messages in thread
From: Shannon Nelson @ 2024-01-30 1:30 UTC (permalink / raw)
To: netdev, davem, kuba, edumazet, pabeni
Cc: brett.creeley, drivers, Shannon Nelson
Convert Rx datapath handling to use the DMA range APIs
in preparation for adding XDP handling.
Signed-off-by: Shannon Nelson <shannon.nelson@amd.com>
---
drivers/net/ethernet/pensando/ionic/ionic_txrx.c | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
index 19a7a8a8e1b3..aee38979a9d7 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
@@ -225,9 +225,8 @@ static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,
frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info));
len -= frag_len;
- dma_sync_single_for_cpu(dev,
- ionic_rx_buf_pa(buf_info),
- frag_len, DMA_FROM_DEVICE);
+ dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info),
+ 0, frag_len, DMA_FROM_DEVICE);
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
buf_info->page, buf_info->page_offset, frag_len,
@@ -276,11 +275,11 @@ static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,
return NULL;
}
- dma_sync_single_for_cpu(dev, ionic_rx_buf_pa(buf_info),
- len, DMA_FROM_DEVICE);
+ dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info),
+ 0, len, DMA_FROM_DEVICE);
skb_copy_to_linear_data(skb, ionic_rx_buf_va(buf_info), len);
- dma_sync_single_for_device(dev, ionic_rx_buf_pa(buf_info),
- len, DMA_FROM_DEVICE);
+ dma_sync_single_range_for_device(dev, ionic_rx_buf_pa(buf_info),
+ 0, len, DMA_FROM_DEVICE);
skb_put(skb, len);
skb->protocol = eth_type_trans(skb, q->lif->netdev);
--
2.17.1
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH net-next 4/9] ionic: add initial framework for XDP support
2024-01-30 1:30 [PATCH net-next 0/9] ionic: add XDP support Shannon Nelson
` (2 preceding siblings ...)
2024-01-30 1:30 ` [PATCH net-next 3/9] ionic: use dma range APIs Shannon Nelson
@ 2024-01-30 1:30 ` Shannon Nelson
2024-01-30 1:30 ` [PATCH net-next 5/9] ionic: Add XDP packet headroom Shannon Nelson
` (4 subsequent siblings)
8 siblings, 0 replies; 12+ messages in thread
From: Shannon Nelson @ 2024-01-30 1:30 UTC (permalink / raw)
To: netdev, davem, kuba, edumazet, pabeni
Cc: brett.creeley, drivers, Shannon Nelson
Set up the basics for running Rx packets through XDP programs.
Add new queue setup and teardown steps for adding/removing an
XDP program, and add the call to run the XDP on a packet.
The XDP frame size needs to be the MTU plus standard ethernet
header, plus head room for XDP scribblings and tail room for a
struct skb_shared_info. Also, at this point, we don't support
XDP frags, only a single contiguous Rx buffer. This means
that our page splitting is not very useful, so when XDP is in
use we need to use the full Rx buffer size and not do sharing.
Co-developed-by: Brett Creeley <brett.creeley@amd.com>
Signed-off-by: Brett Creeley <brett.creeley@amd.com>
Signed-off-by: Shannon Nelson <shannon.nelson@amd.com>
---
.../net/ethernet/pensando/ionic/ionic_dev.h | 7 +
.../ethernet/pensando/ionic/ionic_ethtool.c | 5 +
.../net/ethernet/pensando/ionic/ionic_lif.c | 172 +++++++++++++++++-
.../net/ethernet/pensando/ionic/ionic_lif.h | 7 +
.../net/ethernet/pensando/ionic/ionic_stats.c | 9 +
.../net/ethernet/pensando/ionic/ionic_txrx.c | 61 ++++++-
6 files changed, 255 insertions(+), 6 deletions(-)
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.h b/drivers/net/ethernet/pensando/ionic/ionic_dev.h
index 2667e1cde16b..70f5725fdfe8 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_dev.h
+++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.h
@@ -8,6 +8,7 @@
#include <linux/mutex.h>
#include <linux/workqueue.h>
#include <linux/skbuff.h>
+#include <linux/bpf_trace.h>
#include "ionic_if.h"
#include "ionic_regs.h"
@@ -195,6 +196,11 @@ typedef void (*ionic_desc_cb)(struct ionic_queue *q,
#define IONIC_PAGE_GFP_MASK (GFP_ATOMIC | __GFP_NOWARN |\
__GFP_COMP | __GFP_MEMALLOC)
+#define IONIC_XDP_MAX_LINEAR_MTU (IONIC_PAGE_SIZE - \
+ (VLAN_ETH_HLEN + \
+ XDP_PACKET_HEADROOM + \
+ SKB_DATA_ALIGN(sizeof(struct skb_shared_info))))
+
struct ionic_buf_info {
struct page *page;
dma_addr_t dma_addr;
@@ -256,6 +262,7 @@ struct ionic_queue {
struct ionic_txq_sg_desc *txq_sgl;
struct ionic_rxq_sg_desc *rxq_sgl;
};
+ struct xdp_rxq_info *xdp_rxq_info;
dma_addr_t base_pa;
dma_addr_t cmb_base_pa;
dma_addr_t sg_base_pa;
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c
index cd3c0b01402e..98df2ee11c51 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c
@@ -721,6 +721,11 @@ static int ionic_set_channels(struct net_device *netdev,
ionic_init_queue_params(lif, &qparam);
+ if ((ch->rx_count || ch->tx_count) && lif->xdp_prog) {
+ netdev_info(lif->netdev, "Split Tx/Rx interrupts not available when using XDP\n");
+ return -EOPNOTSUPP;
+ }
+
if (ch->rx_count != ch->tx_count) {
netdev_info(netdev, "The rx and tx count must be equal\n");
return -EINVAL;
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
index d92f8734d153..6e9065dd149e 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
@@ -46,6 +46,9 @@ static int ionic_start_queues(struct ionic_lif *lif);
static void ionic_stop_queues(struct ionic_lif *lif);
static void ionic_lif_queue_identify(struct ionic_lif *lif);
+static int ionic_xdp_queues_config(struct ionic_lif *lif);
+static void ionic_xdp_unregister_rxq_info(struct ionic_queue *q);
+
static void ionic_dim_work(struct work_struct *work)
{
struct dim *dim = container_of(work, struct dim, work);
@@ -422,6 +425,7 @@ static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
qcq->sg_base_pa = 0;
}
+ ionic_xdp_unregister_rxq_info(&qcq->q);
ionic_qcq_intr_free(lif, qcq);
vfree(qcq->cq.info);
@@ -862,8 +866,7 @@ static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
.type = q->type,
.ver = lif->qtype_info[q->type].version,
.index = cpu_to_le32(q->index),
- .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
- IONIC_QINIT_F_SG),
+ .flags = cpu_to_le16(IONIC_QINIT_F_IRQ),
.intr_index = cpu_to_le16(cq->bound_intr->index),
.pid = cpu_to_le16(q->pid),
.ring_size = ilog2(q->num_descs),
@@ -875,6 +878,9 @@ static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
};
int err;
+ if (!lif->xdp_prog)
+ ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_SG);
+
if (qcq->flags & IONIC_QCQ_F_CMB_RINGS) {
ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_CMB);
ctx.cmd.q_init.ring_base = cpu_to_le64(qcq->cmb_q_base_pa);
@@ -1640,6 +1646,8 @@ static int ionic_init_nic_features(struct ionic_lif *lif)
netdev->priv_flags |= IFF_UNICAST_FLT |
IFF_LIVE_ADDR_CHANGE;
+ netdev->xdp_features = NETDEV_XDP_ACT_BASIC;
+
return 0;
}
@@ -1777,6 +1785,18 @@ static int ionic_start_queues_reconfig(struct ionic_lif *lif)
return err;
}
+static bool ionic_xdp_is_valid_mtu(struct ionic_lif *lif, u32 mtu,
+ struct bpf_prog *xdp_prog)
+{
+ if (!xdp_prog)
+ return true;
+
+ if (mtu <= IONIC_XDP_MAX_LINEAR_MTU)
+ return true;
+
+ return false;
+}
+
static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
{
struct ionic_lif *lif = netdev_priv(netdev);
@@ -1789,8 +1809,13 @@ static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
.mtu = cpu_to_le32(new_mtu),
},
};
+ struct bpf_prog *xdp_prog;
int err;
+ xdp_prog = READ_ONCE(lif->xdp_prog);
+ if (!ionic_xdp_is_valid_mtu(lif, new_mtu, xdp_prog))
+ return -EINVAL;
+
err = ionic_adminq_post_wait(lif, &ctx);
if (err)
return err;
@@ -2166,6 +2191,10 @@ static int ionic_txrx_enable(struct ionic_lif *lif)
int derr = 0;
int i, err;
+ err = ionic_xdp_queues_config(lif);
+ if (err)
+ return err;
+
for (i = 0; i < lif->nxqs; i++) {
if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {
dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i);
@@ -2211,6 +2240,8 @@ static int ionic_txrx_enable(struct ionic_lif *lif)
derr = ionic_qcq_disable(lif, lif->rxqcqs[i], derr);
}
+ ionic_xdp_queues_config(lif);
+
return err;
}
@@ -2668,11 +2699,147 @@ static void ionic_vf_attr_replay(struct ionic_lif *lif)
ionic_vf_start(ionic);
}
+static void ionic_xdp_unregister_rxq_info(struct ionic_queue *q)
+{
+ struct xdp_rxq_info *xi;
+
+ if (!q->xdp_rxq_info)
+ return;
+
+ xi = q->xdp_rxq_info;
+ q->xdp_rxq_info = NULL;
+
+ xdp_rxq_info_unreg(xi);
+ kfree(xi);
+}
+
+static int ionic_xdp_register_rxq_info(struct ionic_queue *q, unsigned int napi_id)
+{
+ struct xdp_rxq_info *rxq_info;
+ int err;
+
+ rxq_info = kzalloc(sizeof(*rxq_info), GFP_KERNEL);
+ if (!rxq_info)
+ return -ENOMEM;
+
+ err = xdp_rxq_info_reg(rxq_info, q->lif->netdev, q->index, napi_id);
+ if (err) {
+ dev_err(q->dev, "Queue %d xdp_rxq_info_reg failed, err %d\n",
+ q->index, err);
+ kfree(rxq_info);
+ return err;
+ }
+
+ err = xdp_rxq_info_reg_mem_model(rxq_info, MEM_TYPE_PAGE_ORDER0, NULL);
+ if (err) {
+ dev_err(q->dev, "Queue %d xdp_rxq_info_reg_mem_model failed, err %d\n",
+ q->index, err);
+ xdp_rxq_info_unreg(rxq_info);
+ kfree(rxq_info);
+ return err;
+ }
+
+ q->xdp_rxq_info = rxq_info;
+
+ return 0;
+}
+
+static int ionic_xdp_queues_config(struct ionic_lif *lif)
+{
+ unsigned int i;
+ int err;
+
+ if (!lif->rxqcqs)
+ return 0;
+
+ /* There's no need to rework memory if not going to/from NULL program.
+ * If there is no lif->xdp_prog, there should also be no q.xdp_rxq_info
+ * This way we don't need to keep an *xdp_prog in every queue struct.
+ */
+ if (!lif->xdp_prog == !lif->rxqcqs[0]->q.xdp_rxq_info)
+ return 0;
+
+ for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
+ struct ionic_queue *q = &lif->rxqcqs[i]->q;
+
+ if (q->xdp_rxq_info) {
+ ionic_xdp_unregister_rxq_info(q);
+ } else {
+ err = ionic_xdp_register_rxq_info(q, lif->rxqcqs[i]->napi.napi_id);
+ if (err) {
+ dev_err(lif->ionic->dev, "failed to register RX queue %d info for XDP, err %d\n",
+ i, err);
+ goto err_out;
+ }
+ }
+ }
+
+ return 0;
+
+err_out:
+ for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++)
+ ionic_xdp_unregister_rxq_info(&lif->rxqcqs[i]->q);
+
+ return err;
+}
+
+static int ionic_xdp_config(struct net_device *netdev, struct netdev_bpf *bpf)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct bpf_prog *old_prog;
+ u32 maxfs;
+
+ if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state)) {
+#define XDP_ERR_SPLIT "XDP not available with split Tx/Rx interrupts"
+ NL_SET_ERR_MSG_MOD(bpf->extack, XDP_ERR_SPLIT);
+ netdev_info(lif->netdev, XDP_ERR_SPLIT);
+ return -EOPNOTSUPP;
+ }
+
+ if (!ionic_xdp_is_valid_mtu(lif, netdev->mtu, bpf->prog)) {
+#define XDP_ERR_MTU "MTU is too large for XDP without frags support"
+ NL_SET_ERR_MSG_MOD(bpf->extack, XDP_ERR_MTU);
+ netdev_info(lif->netdev, XDP_ERR_MTU);
+ return -EINVAL;
+ }
+
+ maxfs = __le32_to_cpu(lif->identity->eth.max_frame_size) - VLAN_ETH_HLEN;
+ if (bpf->prog)
+ maxfs = min_t(u32, maxfs, IONIC_XDP_MAX_LINEAR_MTU);
+ netdev->max_mtu = maxfs;
+
+ if (!netif_running(netdev)) {
+ old_prog = xchg(&lif->xdp_prog, bpf->prog);
+ } else {
+ mutex_lock(&lif->queue_lock);
+ ionic_stop_queues_reconfig(lif);
+ old_prog = xchg(&lif->xdp_prog, bpf->prog);
+ ionic_start_queues_reconfig(lif);
+ mutex_unlock(&lif->queue_lock);
+ }
+
+ if (old_prog)
+ bpf_prog_put(old_prog);
+
+ return 0;
+}
+
+static int ionic_xdp(struct net_device *netdev, struct netdev_bpf *bpf)
+{
+ switch (bpf->command) {
+ case XDP_SETUP_PROG:
+ return ionic_xdp_config(netdev, bpf);
+ default:
+ return -EINVAL;
+ }
+}
+
static const struct net_device_ops ionic_netdev_ops = {
.ndo_open = ionic_open,
.ndo_stop = ionic_stop,
.ndo_eth_ioctl = ionic_eth_ioctl,
.ndo_start_xmit = ionic_start_xmit,
+ .ndo_bpf = ionic_xdp,
.ndo_get_stats64 = ionic_get_stats64,
.ndo_set_rx_mode = ionic_ndo_set_rx_mode,
.ndo_set_features = ionic_set_features,
@@ -2755,6 +2922,7 @@ static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
swap(a->q.base, b->q.base);
swap(a->q.base_pa, b->q.base_pa);
swap(a->q.info, b->q.info);
+ swap(a->q.xdp_rxq_info, b->q.xdp_rxq_info);
swap(a->q_base, b->q_base);
swap(a->q_base_pa, b->q_base_pa);
swap(a->q_size, b->q_size);
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.h b/drivers/net/ethernet/pensando/ionic/ionic_lif.h
index 61548b3eea93..61fa4ea4f04c 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_lif.h
+++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.h
@@ -51,6 +51,9 @@ struct ionic_rx_stats {
u64 alloc_err;
u64 hwstamp_valid;
u64 hwstamp_invalid;
+ u64 xdp_drop;
+ u64 xdp_aborted;
+ u64 xdp_pass;
};
#define IONIC_QCQ_F_INITED BIT(0)
@@ -135,6 +138,9 @@ struct ionic_lif_sw_stats {
u64 hw_rx_over_errors;
u64 hw_rx_missed_errors;
u64 hw_tx_aborted_errors;
+ u64 xdp_drop;
+ u64 xdp_aborted;
+ u64 xdp_pass;
};
enum ionic_lif_state_flags {
@@ -230,6 +236,7 @@ struct ionic_lif {
struct ionic_phc *phc;
struct dentry *dentry;
+ struct bpf_prog *xdp_prog;
};
struct ionic_phc {
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_stats.c b/drivers/net/ethernet/pensando/ionic/ionic_stats.c
index 1f6022fb7679..2fb20173b2c6 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_stats.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_stats.c
@@ -27,6 +27,9 @@ static const struct ionic_stat_desc ionic_lif_stats_desc[] = {
IONIC_LIF_STAT_DESC(hw_rx_over_errors),
IONIC_LIF_STAT_DESC(hw_rx_missed_errors),
IONIC_LIF_STAT_DESC(hw_tx_aborted_errors),
+ IONIC_LIF_STAT_DESC(xdp_drop),
+ IONIC_LIF_STAT_DESC(xdp_aborted),
+ IONIC_LIF_STAT_DESC(xdp_pass),
};
static const struct ionic_stat_desc ionic_port_stats_desc[] = {
@@ -149,6 +152,9 @@ static const struct ionic_stat_desc ionic_rx_stats_desc[] = {
IONIC_RX_STAT_DESC(hwstamp_invalid),
IONIC_RX_STAT_DESC(dropped),
IONIC_RX_STAT_DESC(vlan_stripped),
+ IONIC_RX_STAT_DESC(xdp_drop),
+ IONIC_RX_STAT_DESC(xdp_aborted),
+ IONIC_RX_STAT_DESC(xdp_pass),
};
#define IONIC_NUM_LIF_STATS ARRAY_SIZE(ionic_lif_stats_desc)
@@ -185,6 +191,9 @@ static void ionic_add_lif_rxq_stats(struct ionic_lif *lif, int q_num,
stats->rx_csum_error += rxstats->csum_error;
stats->rx_hwstamp_valid += rxstats->hwstamp_valid;
stats->rx_hwstamp_invalid += rxstats->hwstamp_invalid;
+ stats->xdp_drop += rxstats->xdp_drop;
+ stats->xdp_aborted += rxstats->xdp_aborted;
+ stats->xdp_pass += rxstats->xdp_pass;
}
static void ionic_get_lif_stats(struct ionic_lif *lif,
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
index aee38979a9d7..07a17be94d4d 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
@@ -177,7 +177,7 @@ static bool ionic_rx_buf_recycle(struct ionic_queue *q,
if (page_to_nid(buf_info->page) != numa_mem_id())
return false;
- size = ALIGN(used, IONIC_PAGE_SPLIT_SZ);
+ size = ALIGN(used, q->xdp_rxq_info ? IONIC_PAGE_SIZE : IONIC_PAGE_SPLIT_SZ);
buf_info->page_offset += size;
if (buf_info->page_offset >= IONIC_PAGE_SIZE)
return false;
@@ -287,6 +287,54 @@ static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,
return skb;
}
+static bool ionic_run_xdp(struct ionic_rx_stats *stats,
+ struct net_device *netdev,
+ struct ionic_queue *rxq,
+ struct ionic_buf_info *buf_info,
+ int len)
+{
+ u32 xdp_action = XDP_ABORTED;
+ struct bpf_prog *xdp_prog;
+ struct xdp_buff xdp_buf;
+
+ xdp_prog = READ_ONCE(rxq->lif->xdp_prog);
+ if (!xdp_prog)
+ return false;
+
+ xdp_init_buff(&xdp_buf, IONIC_PAGE_SIZE, rxq->xdp_rxq_info);
+ xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info),
+ 0, len, false);
+
+ dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(buf_info),
+ 0, len,
+ DMA_FROM_DEVICE);
+
+ prefetchw(&xdp_buf.data_hard_start);
+
+ xdp_action = bpf_prog_run_xdp(xdp_prog, &xdp_buf);
+
+ switch (xdp_action) {
+ case XDP_PASS:
+ stats->xdp_pass++;
+ return false; /* false = we didn't consume the packet */
+
+ case XDP_DROP:
+ ionic_rx_page_free(rxq, buf_info);
+ stats->xdp_drop++;
+ break;
+
+ case XDP_REDIRECT:
+ case XDP_TX:
+ case XDP_ABORTED:
+ default:
+ trace_xdp_exception(netdev, xdp_prog, xdp_action);
+ ionic_rx_page_free(rxq, buf_info);
+ stats->xdp_aborted++;
+ }
+
+ return true;
+}
+
static void ionic_rx_clean(struct ionic_queue *q,
struct ionic_desc_info *desc_info,
struct ionic_cq_info *cq_info,
@@ -297,6 +345,7 @@ static void ionic_rx_clean(struct ionic_queue *q,
struct ionic_rx_stats *stats;
struct ionic_rxq_comp *comp;
struct sk_buff *skb;
+ u16 len;
comp = cq_info->cq_desc + qcq->cq.desc_size - sizeof(*comp);
@@ -307,10 +356,14 @@ static void ionic_rx_clean(struct ionic_queue *q,
return;
}
+ len = le16_to_cpu(comp->len);
stats->pkts++;
- stats->bytes += le16_to_cpu(comp->len);
+ stats->bytes += len;
+
+ if (ionic_run_xdp(stats, netdev, q, desc_info->bufs, len))
+ return;
- if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
+ if (len <= q->lif->rx_copybreak)
skb = ionic_rx_copybreak(q, desc_info, comp);
else
skb = ionic_rx_frags(q, desc_info, comp);
@@ -380,7 +433,7 @@ static void ionic_rx_clean(struct ionic_queue *q,
}
}
- if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
+ if (len <= q->lif->rx_copybreak)
napi_gro_receive(&qcq->napi, skb);
else
napi_gro_frags(&qcq->napi);
--
2.17.1
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH net-next 5/9] ionic: Add XDP packet headroom
2024-01-30 1:30 [PATCH net-next 0/9] ionic: add XDP support Shannon Nelson
` (3 preceding siblings ...)
2024-01-30 1:30 ` [PATCH net-next 4/9] ionic: add initial framework for XDP support Shannon Nelson
@ 2024-01-30 1:30 ` Shannon Nelson
2024-01-30 1:30 ` [PATCH net-next 6/9] ionic: Add XDP_TX support Shannon Nelson
` (3 subsequent siblings)
8 siblings, 0 replies; 12+ messages in thread
From: Shannon Nelson @ 2024-01-30 1:30 UTC (permalink / raw)
To: netdev, davem, kuba, edumazet, pabeni
Cc: brett.creeley, drivers, Shannon Nelson
If an xdp program is loaded, add headroom at the beginning
of the frame to allow for editing and insertions that an XDP
program might need room for, and tailroom used later for XDP
frame tracking. These are only needed in the first Rx buffer
in a packet, not for any trailing frags.
Co-developed-by: Brett Creeley <brett.creeley@amd.com>
Signed-off-by: Brett Creeley <brett.creeley@amd.com>
Signed-off-by: Shannon Nelson <shannon.nelson@amd.com>
---
.../net/ethernet/pensando/ionic/ionic_txrx.c | 67 ++++++++++++-------
1 file changed, 44 insertions(+), 23 deletions(-)
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
index 07a17be94d4d..072a9e376b39 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
@@ -189,7 +189,9 @@ static bool ionic_rx_buf_recycle(struct ionic_queue *q,
static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,
struct ionic_desc_info *desc_info,
- struct ionic_rxq_comp *comp)
+ unsigned int headroom,
+ unsigned int len,
+ unsigned int num_sg_elems)
{
struct net_device *netdev = q->lif->netdev;
struct ionic_buf_info *buf_info;
@@ -198,12 +200,10 @@ static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,
struct sk_buff *skb;
unsigned int i;
u16 frag_len;
- u16 len;
stats = q_to_rx_stats(q);
buf_info = &desc_info->bufs[0];
- len = le16_to_cpu(comp->len);
prefetchw(buf_info->page);
@@ -215,22 +215,25 @@ static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,
return NULL;
}
- i = comp->num_sg_elems + 1;
+ i = num_sg_elems + 1;
do {
if (unlikely(!buf_info->page)) {
dev_kfree_skb(skb);
return NULL;
}
- frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info));
+ if (headroom)
+ frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN);
+ else
+ frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info));
len -= frag_len;
dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info),
- 0, frag_len, DMA_FROM_DEVICE);
+ headroom, frag_len, DMA_FROM_DEVICE);
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
- buf_info->page, buf_info->page_offset, frag_len,
- IONIC_PAGE_SIZE);
+ buf_info->page, buf_info->page_offset + headroom,
+ frag_len, IONIC_PAGE_SIZE);
if (!ionic_rx_buf_recycle(q, buf_info, frag_len)) {
dma_unmap_page(dev, buf_info->dma_addr,
@@ -238,6 +241,10 @@ static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,
buf_info->page = NULL;
}
+ /* only needed on the first buffer */
+ if (headroom)
+ headroom = 0;
+
buf_info++;
i--;
@@ -248,19 +255,18 @@ static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,
static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,
struct ionic_desc_info *desc_info,
- struct ionic_rxq_comp *comp)
+ unsigned int headroom,
+ unsigned int len)
{
struct net_device *netdev = q->lif->netdev;
struct ionic_buf_info *buf_info;
struct ionic_rx_stats *stats;
struct device *dev = q->dev;
struct sk_buff *skb;
- u16 len;
stats = q_to_rx_stats(q);
buf_info = &desc_info->bufs[0];
- len = le16_to_cpu(comp->len);
skb = napi_alloc_skb(&q_to_qcq(q)->napi, len);
if (unlikely(!skb)) {
@@ -276,10 +282,10 @@ static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,
}
dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info),
- 0, len, DMA_FROM_DEVICE);
- skb_copy_to_linear_data(skb, ionic_rx_buf_va(buf_info), len);
+ headroom, len, DMA_FROM_DEVICE);
+ skb_copy_to_linear_data(skb, ionic_rx_buf_va(buf_info) + headroom, len);
dma_sync_single_range_for_device(dev, ionic_rx_buf_pa(buf_info),
- 0, len, DMA_FROM_DEVICE);
+ headroom, len, DMA_FROM_DEVICE);
skb_put(skb, len);
skb->protocol = eth_type_trans(skb, q->lif->netdev);
@@ -303,10 +309,10 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats,
xdp_init_buff(&xdp_buf, IONIC_PAGE_SIZE, rxq->xdp_rxq_info);
xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info),
- 0, len, false);
+ XDP_PACKET_HEADROOM, len, false);
dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(buf_info),
- 0, len,
+ XDP_PACKET_HEADROOM, len,
DMA_FROM_DEVICE);
prefetchw(&xdp_buf.data_hard_start);
@@ -344,6 +350,7 @@ static void ionic_rx_clean(struct ionic_queue *q,
struct ionic_qcq *qcq = q_to_qcq(q);
struct ionic_rx_stats *stats;
struct ionic_rxq_comp *comp;
+ unsigned int headroom;
struct sk_buff *skb;
u16 len;
@@ -363,10 +370,11 @@ static void ionic_rx_clean(struct ionic_queue *q,
if (ionic_run_xdp(stats, netdev, q, desc_info->bufs, len))
return;
+ headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0;
if (len <= q->lif->rx_copybreak)
- skb = ionic_rx_copybreak(q, desc_info, comp);
+ skb = ionic_rx_copybreak(q, desc_info, headroom, len);
else
- skb = ionic_rx_frags(q, desc_info, comp);
+ skb = ionic_rx_frags(q, desc_info, headroom, len, comp->num_sg_elems);
if (unlikely(!skb)) {
stats->dropped++;
@@ -490,8 +498,9 @@ void ionic_rx_fill(struct ionic_queue *q)
unsigned int frag_len;
unsigned int nfrags;
unsigned int n_fill;
- unsigned int i, j;
unsigned int len;
+ unsigned int i;
+ unsigned int j;
n_fill = ionic_q_space_avail(q);
@@ -500,9 +509,12 @@ void ionic_rx_fill(struct ionic_queue *q)
if (n_fill < fill_threshold)
return;
- len = netdev->mtu + ETH_HLEN + VLAN_HLEN;
+ len = netdev->mtu + VLAN_ETH_HLEN;
for (i = n_fill; i; i--) {
+ unsigned int headroom;
+ unsigned int buf_len;
+
nfrags = 0;
remain_len = len;
desc_info = &q->info[q->head_idx];
@@ -517,9 +529,18 @@ void ionic_rx_fill(struct ionic_queue *q)
}
}
- /* fill main descriptor - buf[0] */
- desc->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info));
- frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info));
+ /* fill main descriptor - buf[0]
+ * XDP uses space in the first buffer, so account for
+ * head room, tail room, and ip header in the first frag size.
+ */
+ headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0;
+ if (q->xdp_rxq_info)
+ buf_len = IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN;
+ else
+ buf_len = ionic_rx_buf_size(buf_info);
+ frag_len = min_t(u16, len, buf_len);
+
+ desc->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info) + headroom);
desc->len = cpu_to_le16(frag_len);
remain_len -= frag_len;
buf_info++;
--
2.17.1
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH net-next 6/9] ionic: Add XDP_TX support
2024-01-30 1:30 [PATCH net-next 0/9] ionic: add XDP support Shannon Nelson
` (4 preceding siblings ...)
2024-01-30 1:30 ` [PATCH net-next 5/9] ionic: Add XDP packet headroom Shannon Nelson
@ 2024-01-30 1:30 ` Shannon Nelson
2024-02-01 2:25 ` Jakub Kicinski
2024-01-30 1:30 ` [PATCH net-next 7/9] ionic: Add XDP_REDIRECT support Shannon Nelson
` (2 subsequent siblings)
8 siblings, 1 reply; 12+ messages in thread
From: Shannon Nelson @ 2024-01-30 1:30 UTC (permalink / raw)
To: netdev, davem, kuba, edumazet, pabeni
Cc: brett.creeley, drivers, Shannon Nelson
The XDP_TX packets get fed back into the Rx queue's partnered
Tx queue as an xdp_frame.
Co-developed-by: Brett Creeley <brett.creeley@amd.com>
Signed-off-by: Brett Creeley <brett.creeley@amd.com>
Signed-off-by: Shannon Nelson <shannon.nelson@amd.com>
---
.../net/ethernet/pensando/ionic/ionic_dev.h | 3 +
.../net/ethernet/pensando/ionic/ionic_lif.c | 4 +
.../net/ethernet/pensando/ionic/ionic_lif.h | 4 +
.../net/ethernet/pensando/ionic/ionic_stats.c | 6 +
.../net/ethernet/pensando/ionic/ionic_txrx.c | 137 +++++++++++++++++-
5 files changed, 151 insertions(+), 3 deletions(-)
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.h b/drivers/net/ethernet/pensando/ionic/ionic_dev.h
index 70f5725fdfe8..76425bb546ba 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_dev.h
+++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.h
@@ -228,6 +228,8 @@ struct ionic_desc_info {
struct ionic_buf_info bufs[MAX_SKB_FRAGS + 1];
ionic_desc_cb cb;
void *cb_arg;
+ struct xdp_frame *xdpf;
+ enum xdp_action act;
};
#define IONIC_QUEUE_NAME_MAX_SZ 16
@@ -263,6 +265,7 @@ struct ionic_queue {
struct ionic_rxq_sg_desc *rxq_sgl;
};
struct xdp_rxq_info *xdp_rxq_info;
+ struct ionic_queue *partner;
dma_addr_t base_pa;
dma_addr_t cmb_base_pa;
dma_addr_t sg_base_pa;
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
index 6e9065dd149e..997141321c40 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
@@ -878,6 +878,9 @@ static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
};
int err;
+ q->partner = &lif->txqcqs[q->index]->q;
+ q->partner->partner = q;
+
if (!lif->xdp_prog)
ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_SG);
@@ -2923,6 +2926,7 @@ static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
swap(a->q.base_pa, b->q.base_pa);
swap(a->q.info, b->q.info);
swap(a->q.xdp_rxq_info, b->q.xdp_rxq_info);
+ swap(a->q.partner, b->q.partner);
swap(a->q_base, b->q_base);
swap(a->q_base_pa, b->q_base_pa);
swap(a->q_size, b->q_size);
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.h b/drivers/net/ethernet/pensando/ionic/ionic_lif.h
index 61fa4ea4f04c..092ff08fc7e0 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_lif.h
+++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.h
@@ -37,6 +37,7 @@ struct ionic_tx_stats {
u64 dma_map_err;
u64 hwstamp_valid;
u64 hwstamp_invalid;
+ u64 xdp_frames;
};
struct ionic_rx_stats {
@@ -54,6 +55,7 @@ struct ionic_rx_stats {
u64 xdp_drop;
u64 xdp_aborted;
u64 xdp_pass;
+ u64 xdp_tx;
};
#define IONIC_QCQ_F_INITED BIT(0)
@@ -141,6 +143,8 @@ struct ionic_lif_sw_stats {
u64 xdp_drop;
u64 xdp_aborted;
u64 xdp_pass;
+ u64 xdp_tx;
+ u64 xdp_frames;
};
enum ionic_lif_state_flags {
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_stats.c b/drivers/net/ethernet/pensando/ionic/ionic_stats.c
index 2fb20173b2c6..5d48226e66cd 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_stats.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_stats.c
@@ -30,6 +30,8 @@ static const struct ionic_stat_desc ionic_lif_stats_desc[] = {
IONIC_LIF_STAT_DESC(xdp_drop),
IONIC_LIF_STAT_DESC(xdp_aborted),
IONIC_LIF_STAT_DESC(xdp_pass),
+ IONIC_LIF_STAT_DESC(xdp_tx),
+ IONIC_LIF_STAT_DESC(xdp_frames),
};
static const struct ionic_stat_desc ionic_port_stats_desc[] = {
@@ -138,6 +140,7 @@ static const struct ionic_stat_desc ionic_tx_stats_desc[] = {
IONIC_TX_STAT_DESC(csum_none),
IONIC_TX_STAT_DESC(csum),
IONIC_TX_STAT_DESC(vlan_inserted),
+ IONIC_TX_STAT_DESC(xdp_frames),
};
static const struct ionic_stat_desc ionic_rx_stats_desc[] = {
@@ -155,6 +158,7 @@ static const struct ionic_stat_desc ionic_rx_stats_desc[] = {
IONIC_RX_STAT_DESC(xdp_drop),
IONIC_RX_STAT_DESC(xdp_aborted),
IONIC_RX_STAT_DESC(xdp_pass),
+ IONIC_RX_STAT_DESC(xdp_tx),
};
#define IONIC_NUM_LIF_STATS ARRAY_SIZE(ionic_lif_stats_desc)
@@ -177,6 +181,7 @@ static void ionic_add_lif_txq_stats(struct ionic_lif *lif, int q_num,
stats->tx_csum += txstats->csum;
stats->tx_hwstamp_valid += txstats->hwstamp_valid;
stats->tx_hwstamp_invalid += txstats->hwstamp_invalid;
+ stats->xdp_frames += txstats->xdp_frames;
}
static void ionic_add_lif_rxq_stats(struct ionic_lif *lif, int q_num,
@@ -194,6 +199,7 @@ static void ionic_add_lif_rxq_stats(struct ionic_lif *lif, int q_num,
stats->xdp_drop += rxstats->xdp_drop;
stats->xdp_aborted += rxstats->xdp_aborted;
stats->xdp_pass += rxstats->xdp_pass;
+ stats->xdp_tx += rxstats->xdp_tx;
}
static void ionic_get_lif_stats(struct ionic_lif *lif,
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
index 072a9e376b39..a68f659e416d 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
@@ -10,6 +10,16 @@
#include "ionic_lif.h"
#include "ionic_txrx.h"
+static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs);
+
+static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
+ void *data, size_t len);
+
+static void ionic_tx_clean(struct ionic_queue *q,
+ struct ionic_desc_info *desc_info,
+ struct ionic_cq_info *cq_info,
+ void *cb_arg);
+
static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell,
ionic_desc_cb cb_func, void *cb_arg)
{
@@ -293,6 +303,75 @@ static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,
return skb;
}
+static void ionic_xdp_tx_desc_clean(struct ionic_queue *q,
+ struct ionic_desc_info *desc_info)
+{
+ unsigned int nbufs = desc_info->nbufs;
+ struct ionic_buf_info *buf_info;
+ struct device *dev = q->dev;
+
+ if (!nbufs)
+ return;
+
+ buf_info = desc_info->bufs;
+ dma_unmap_single(dev, buf_info->dma_addr,
+ buf_info->len, DMA_TO_DEVICE);
+ __free_pages(buf_info->page, 0);
+ buf_info->page = NULL;
+
+ desc_info->nbufs = 0;
+ desc_info->xdpf = NULL;
+ desc_info->act = 0;
+}
+
+static int ionic_xdp_post_frame(struct net_device *netdev,
+ struct ionic_queue *q, struct xdp_frame *frame,
+ enum xdp_action act, struct page *page, int off,
+ bool ring_doorbell)
+{
+ struct ionic_desc_info *desc_info;
+ struct ionic_buf_info *buf_info;
+ struct ionic_tx_stats *stats;
+ struct ionic_txq_desc *desc;
+ size_t len = frame->len;
+ dma_addr_t dma_addr;
+ u64 cmd;
+
+ desc_info = &q->info[q->head_idx];
+ desc = desc_info->txq_desc;
+ buf_info = desc_info->bufs;
+ stats = q_to_tx_stats(q);
+
+ dma_addr = ionic_tx_map_single(q, frame->data, len);
+ if (dma_mapping_error(q->dev, dma_addr)) {
+ stats->dma_map_err++;
+ return -EIO;
+ }
+ buf_info->dma_addr = dma_addr;
+ buf_info->len = len;
+ buf_info->page = page;
+ buf_info->page_offset = off;
+
+ desc_info->nbufs = 1;
+ desc_info->xdpf = frame;
+ desc_info->act = act;
+
+ cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,
+ 0, 0, buf_info->dma_addr);
+ desc->cmd = cpu_to_le64(cmd);
+ desc->len = cpu_to_le16(len);
+ desc->csum_start = 0;
+ desc->csum_offset = 0;
+
+ stats->xdp_frames++;
+ stats->pkts++;
+ stats->bytes += len;
+
+ ionic_txq_post(q, ring_doorbell, ionic_tx_clean, NULL);
+
+ return 0;
+}
+
static bool ionic_run_xdp(struct ionic_rx_stats *stats,
struct net_device *netdev,
struct ionic_queue *rxq,
@@ -302,6 +381,10 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats,
u32 xdp_action = XDP_ABORTED;
struct bpf_prog *xdp_prog;
struct xdp_buff xdp_buf;
+ struct ionic_queue *txq;
+ struct netdev_queue *nq;
+ struct xdp_frame *xdpf;
+ int err = 0;
xdp_prog = READ_ONCE(rxq->lif->xdp_prog);
if (!xdp_prog)
@@ -330,14 +413,52 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats,
break;
case XDP_REDIRECT:
+ goto out_xdp_abort;
+
case XDP_TX:
+ xdpf = xdp_convert_buff_to_frame(&xdp_buf);
+ if (!xdpf)
+ goto out_xdp_abort;
+
+ txq = rxq->partner;
+ nq = netdev_get_tx_queue(netdev, txq->index);
+ __netif_tx_lock(nq, smp_processor_id());
+
+ if (netif_tx_queue_stopped(nq) ||
+ unlikely(ionic_maybe_stop_tx(txq, 1))) {
+ __netif_tx_unlock(nq);
+ goto out_xdp_abort;
+ }
+
+ dma_unmap_page(rxq->dev, buf_info->dma_addr,
+ IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
+
+ err = ionic_xdp_post_frame(netdev, txq, xdpf, XDP_TX,
+ buf_info->page,
+ buf_info->page_offset,
+ true);
+ __netif_tx_unlock(nq);
+ if (err) {
+ netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err);
+ goto out_xdp_abort;
+ }
+ stats->xdp_tx++;
+
+ /* the Tx completion will free the buffers */
+ break;
+
case XDP_ABORTED:
default:
- trace_xdp_exception(netdev, xdp_prog, xdp_action);
- ionic_rx_page_free(rxq, buf_info);
- stats->xdp_aborted++;
+ goto out_xdp_abort;
}
+ return true;
+
+out_xdp_abort:
+ trace_xdp_exception(netdev, xdp_prog, xdp_action);
+ ionic_rx_page_free(rxq, buf_info);
+ stats->xdp_aborted++;
+
return true;
}
@@ -880,6 +1001,16 @@ static void ionic_tx_clean(struct ionic_queue *q,
struct sk_buff *skb = cb_arg;
u16 qi;
+ if (desc_info->xdpf) {
+ ionic_xdp_tx_desc_clean(q->partner, desc_info);
+ stats->clean++;
+
+ if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index)))
+ netif_wake_subqueue(q->lif->netdev, q->index);
+
+ return;
+ }
+
ionic_tx_desc_unmap_bufs(q, desc_info);
if (!skb)
--
2.17.1
^ permalink raw reply related [flat|nested] 12+ messages in thread* Re: [PATCH net-next 6/9] ionic: Add XDP_TX support
2024-01-30 1:30 ` [PATCH net-next 6/9] ionic: Add XDP_TX support Shannon Nelson
@ 2024-02-01 2:25 ` Jakub Kicinski
2024-02-01 17:41 ` Nelson, Shannon
0 siblings, 1 reply; 12+ messages in thread
From: Jakub Kicinski @ 2024-02-01 2:25 UTC (permalink / raw)
To: Shannon Nelson; +Cc: netdev, davem, edumazet, pabeni, brett.creeley, drivers
On Mon, 29 Jan 2024 17:30:39 -0800 Shannon Nelson wrote:
> case XDP_TX:
> + xdpf = xdp_convert_buff_to_frame(&xdp_buf);
> + if (!xdpf)
> + goto out_xdp_abort;
> +
> + txq = rxq->partner;
> + nq = netdev_get_tx_queue(netdev, txq->index);
> + __netif_tx_lock(nq, smp_processor_id());
> +
> + if (netif_tx_queue_stopped(nq) ||
> + unlikely(ionic_maybe_stop_tx(txq, 1))) {
> + __netif_tx_unlock(nq);
> + goto out_xdp_abort;
> + }
> +
> + dma_unmap_page(rxq->dev, buf_info->dma_addr,
> + IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
> +
> + err = ionic_xdp_post_frame(netdev, txq, xdpf, XDP_TX,
> + buf_info->page,
> + buf_info->page_offset,
> + true);
I think that you need txq_trans_cond_update() somewhere, otherwise
if XDP starves stack Tx the stack will think the queue is stalled.
> + __netif_tx_unlock(nq);
> + if (err) {
> + netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err);
> + goto out_xdp_abort;
> + }
> + stats->xdp_tx++;
> +
> + /* the Tx completion will free the buffers */
> + break;
> +
> case XDP_ABORTED:
> default:
> - trace_xdp_exception(netdev, xdp_prog, xdp_action);
> - ionic_rx_page_free(rxq, buf_info);
> - stats->xdp_aborted++;
> + goto out_xdp_abort;
> }
>
> + return true;
> +
> +out_xdp_abort:
> + trace_xdp_exception(netdev, xdp_prog, xdp_action);
> + ionic_rx_page_free(rxq, buf_info);
> + stats->xdp_aborted++;
> +
> return true;
> }
>
> @@ -880,6 +1001,16 @@ static void ionic_tx_clean(struct ionic_queue *q,
> struct sk_buff *skb = cb_arg;
> u16 qi;
>
> + if (desc_info->xdpf) {
> + ionic_xdp_tx_desc_clean(q->partner, desc_info);
> + stats->clean++;
> +
> + if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index)))
> + netif_wake_subqueue(q->lif->netdev, q->index);
> +
> + return;
> + }
You can't complete XDP if NAPI budget is 0, you may be in hard IRQ
context if its netpoll calling :(
^ permalink raw reply [flat|nested] 12+ messages in thread* Re: [PATCH net-next 6/9] ionic: Add XDP_TX support
2024-02-01 2:25 ` Jakub Kicinski
@ 2024-02-01 17:41 ` Nelson, Shannon
0 siblings, 0 replies; 12+ messages in thread
From: Nelson, Shannon @ 2024-02-01 17:41 UTC (permalink / raw)
To: Jakub Kicinski; +Cc: netdev, davem, edumazet, pabeni, brett.creeley, drivers
On 1/31/2024 6:25 PM, Jakub Kicinski wrote:
>
> On Mon, 29 Jan 2024 17:30:39 -0800 Shannon Nelson wrote:
>> case XDP_TX:
>> + xdpf = xdp_convert_buff_to_frame(&xdp_buf);
>> + if (!xdpf)
>> + goto out_xdp_abort;
>> +
>> + txq = rxq->partner;
>> + nq = netdev_get_tx_queue(netdev, txq->index);
>> + __netif_tx_lock(nq, smp_processor_id());
>> +
>> + if (netif_tx_queue_stopped(nq) ||
>> + unlikely(ionic_maybe_stop_tx(txq, 1))) {
>> + __netif_tx_unlock(nq);
>> + goto out_xdp_abort;
>> + }
>> +
>> + dma_unmap_page(rxq->dev, buf_info->dma_addr,
>> + IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
>> +
>> + err = ionic_xdp_post_frame(netdev, txq, xdpf, XDP_TX,
>> + buf_info->page,
>> + buf_info->page_offset,
>> + true);
>
> I think that you need txq_trans_cond_update() somewhere, otherwise
> if XDP starves stack Tx the stack will think the queue is stalled.
Thanks, I'll look into that.
>
>> + __netif_tx_unlock(nq);
>> + if (err) {
>> + netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err);
>> + goto out_xdp_abort;
>> + }
>> + stats->xdp_tx++;
>> +
>> + /* the Tx completion will free the buffers */
>> + break;
>> +
>> case XDP_ABORTED:
>> default:
>> - trace_xdp_exception(netdev, xdp_prog, xdp_action);
>> - ionic_rx_page_free(rxq, buf_info);
>> - stats->xdp_aborted++;
>> + goto out_xdp_abort;
>> }
>>
>> + return true;
>> +
>> +out_xdp_abort:
>> + trace_xdp_exception(netdev, xdp_prog, xdp_action);
>> + ionic_rx_page_free(rxq, buf_info);
>> + stats->xdp_aborted++;
>> +
>> return true;
>> }
>>
>> @@ -880,6 +1001,16 @@ static void ionic_tx_clean(struct ionic_queue *q,
>> struct sk_buff *skb = cb_arg;
>> u16 qi;
>>
>> + if (desc_info->xdpf) {
>> + ionic_xdp_tx_desc_clean(q->partner, desc_info);
>> + stats->clean++;
>> +
>> + if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index)))
>> + netif_wake_subqueue(q->lif->netdev, q->index);
>> +
>> + return;
>> + }
>
> You can't complete XDP if NAPI budget is 0, you may be in hard IRQ
> context if its netpoll calling :(
Right - I remember a mention of this a few weeks ago on some other
driver. I'll try to dig up the reference and adjust.
Thanks,
sln
>
^ permalink raw reply [flat|nested] 12+ messages in thread
* [PATCH net-next 7/9] ionic: Add XDP_REDIRECT support
2024-01-30 1:30 [PATCH net-next 0/9] ionic: add XDP support Shannon Nelson
` (5 preceding siblings ...)
2024-01-30 1:30 ` [PATCH net-next 6/9] ionic: Add XDP_TX support Shannon Nelson
@ 2024-01-30 1:30 ` Shannon Nelson
2024-01-30 1:30 ` [PATCH net-next 8/9] ionic: add ndo_xdp_xmit Shannon Nelson
2024-01-30 1:30 ` [PATCH net-next 9/9] ionic: implement xdp frags support Shannon Nelson
8 siblings, 0 replies; 12+ messages in thread
From: Shannon Nelson @ 2024-01-30 1:30 UTC (permalink / raw)
To: netdev, davem, kuba, edumazet, pabeni
Cc: brett.creeley, drivers, Shannon Nelson
The XDP_REDIRECT packets are given to the XDP stack and
we drop the use of the related page: it will get freed
by the driver that ends up doing the Tx. Because we have
some hardware configurations with limited queue resources,
we use the existing datapath Tx queues rather than creating
and managing a separate set of xdp_tx queues.
Co-developed-by: Brett Creeley <brett.creeley@amd.com>
Signed-off-by: Brett Creeley <brett.creeley@amd.com>
Signed-off-by: Shannon Nelson <shannon.nelson@amd.com>
---
.../net/ethernet/pensando/ionic/ionic_dev.h | 1 +
.../net/ethernet/pensando/ionic/ionic_lif.c | 3 ++-
.../net/ethernet/pensando/ionic/ionic_lif.h | 2 ++
.../net/ethernet/pensando/ionic/ionic_stats.c | 3 +++
.../net/ethernet/pensando/ionic/ionic_txrx.c | 24 ++++++++++++++++++-
5 files changed, 31 insertions(+), 2 deletions(-)
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.h b/drivers/net/ethernet/pensando/ionic/ionic_dev.h
index 76425bb546ba..bfcfc2d7bcbd 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_dev.h
+++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.h
@@ -266,6 +266,7 @@ struct ionic_queue {
};
struct xdp_rxq_info *xdp_rxq_info;
struct ionic_queue *partner;
+ bool xdp_flush;
dma_addr_t base_pa;
dma_addr_t cmb_base_pa;
dma_addr_t sg_base_pa;
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
index 997141321c40..bd65b4b2c7f8 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
@@ -1649,7 +1649,8 @@ static int ionic_init_nic_features(struct ionic_lif *lif)
netdev->priv_flags |= IFF_UNICAST_FLT |
IFF_LIVE_ADDR_CHANGE;
- netdev->xdp_features = NETDEV_XDP_ACT_BASIC;
+ netdev->xdp_features = NETDEV_XDP_ACT_BASIC |
+ NETDEV_XDP_ACT_REDIRECT;
return 0;
}
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.h b/drivers/net/ethernet/pensando/ionic/ionic_lif.h
index 092ff08fc7e0..42006de8069d 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_lif.h
+++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.h
@@ -56,6 +56,7 @@ struct ionic_rx_stats {
u64 xdp_aborted;
u64 xdp_pass;
u64 xdp_tx;
+ u64 xdp_redirect;
};
#define IONIC_QCQ_F_INITED BIT(0)
@@ -144,6 +145,7 @@ struct ionic_lif_sw_stats {
u64 xdp_aborted;
u64 xdp_pass;
u64 xdp_tx;
+ u64 xdp_redirect;
u64 xdp_frames;
};
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_stats.c b/drivers/net/ethernet/pensando/ionic/ionic_stats.c
index 5d48226e66cd..0107599a9dd4 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_stats.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_stats.c
@@ -31,6 +31,7 @@ static const struct ionic_stat_desc ionic_lif_stats_desc[] = {
IONIC_LIF_STAT_DESC(xdp_aborted),
IONIC_LIF_STAT_DESC(xdp_pass),
IONIC_LIF_STAT_DESC(xdp_tx),
+ IONIC_LIF_STAT_DESC(xdp_redirect),
IONIC_LIF_STAT_DESC(xdp_frames),
};
@@ -159,6 +160,7 @@ static const struct ionic_stat_desc ionic_rx_stats_desc[] = {
IONIC_RX_STAT_DESC(xdp_aborted),
IONIC_RX_STAT_DESC(xdp_pass),
IONIC_RX_STAT_DESC(xdp_tx),
+ IONIC_RX_STAT_DESC(xdp_redirect),
};
#define IONIC_NUM_LIF_STATS ARRAY_SIZE(ionic_lif_stats_desc)
@@ -200,6 +202,7 @@ static void ionic_add_lif_rxq_stats(struct ionic_lif *lif, int q_num,
stats->xdp_aborted += rxstats->xdp_aborted;
stats->xdp_pass += rxstats->xdp_pass;
stats->xdp_tx += rxstats->xdp_tx;
+ stats->xdp_redirect += rxstats->xdp_redirect;
}
static void ionic_get_lif_stats(struct ionic_lif *lif,
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
index a68f659e416d..c2be2215406a 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
@@ -413,7 +413,19 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats,
break;
case XDP_REDIRECT:
- goto out_xdp_abort;
+ /* unmap the pages before handing them to a different device */
+ dma_unmap_page(rxq->dev, buf_info->dma_addr,
+ IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
+
+ err = xdp_do_redirect(netdev, &xdp_buf, xdp_prog);
+ if (err) {
+ netdev_dbg(netdev, "xdp_do_redirect err %d\n", err);
+ goto out_xdp_abort;
+ }
+ buf_info->page = NULL;
+ rxq->xdp_flush = true;
+ stats->xdp_redirect++;
+ break;
case XDP_TX:
xdpf = xdp_convert_buff_to_frame(&xdp_buf);
@@ -803,6 +815,14 @@ int ionic_tx_napi(struct napi_struct *napi, int budget)
return work_done;
}
+static void ionic_xdp_do_flush(struct ionic_cq *cq)
+{
+ if (cq->bound_q->xdp_flush) {
+ xdp_do_flush();
+ cq->bound_q->xdp_flush = false;
+ }
+}
+
int ionic_rx_napi(struct napi_struct *napi, int budget)
{
struct ionic_qcq *qcq = napi_to_qcq(napi);
@@ -820,6 +840,7 @@ int ionic_rx_napi(struct napi_struct *napi, int budget)
ionic_rx_fill(cq->bound_q);
+ ionic_xdp_do_flush(cq);
if (work_done < budget && napi_complete_done(napi, work_done)) {
ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR);
flags |= IONIC_INTR_CRED_UNMASK;
@@ -866,6 +887,7 @@ int ionic_txrx_napi(struct napi_struct *napi, int budget)
ionic_rx_fill(rxcq->bound_q);
+ ionic_xdp_do_flush(rxcq);
if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) {
ionic_dim_update(rxqcq, 0);
flags |= IONIC_INTR_CRED_UNMASK;
--
2.17.1
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH net-next 8/9] ionic: add ndo_xdp_xmit
2024-01-30 1:30 [PATCH net-next 0/9] ionic: add XDP support Shannon Nelson
` (6 preceding siblings ...)
2024-01-30 1:30 ` [PATCH net-next 7/9] ionic: Add XDP_REDIRECT support Shannon Nelson
@ 2024-01-30 1:30 ` Shannon Nelson
2024-01-30 1:30 ` [PATCH net-next 9/9] ionic: implement xdp frags support Shannon Nelson
8 siblings, 0 replies; 12+ messages in thread
From: Shannon Nelson @ 2024-01-30 1:30 UTC (permalink / raw)
To: netdev, davem, kuba, edumazet, pabeni
Cc: brett.creeley, drivers, Shannon Nelson
When our ndo_xdp_xmit is called we mark the buffer with
XDP_REDIRECT so we know to return it to the XDP stack for
cleaning.
Co-developed-by: Brett Creeley <brett.creeley@amd.com>
Signed-off-by: Brett Creeley <brett.creeley@amd.com>
Signed-off-by: Shannon Nelson <shannon.nelson@amd.com>
---
.../net/ethernet/pensando/ionic/ionic_lif.c | 4 +-
.../net/ethernet/pensando/ionic/ionic_txrx.c | 62 ++++++++++++++++++-
.../net/ethernet/pensando/ionic/ionic_txrx.h | 1 +
3 files changed, 65 insertions(+), 2 deletions(-)
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
index bd65b4b2c7f8..ed5d792c4780 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
@@ -1650,7 +1650,8 @@ static int ionic_init_nic_features(struct ionic_lif *lif)
IFF_LIVE_ADDR_CHANGE;
netdev->xdp_features = NETDEV_XDP_ACT_BASIC |
- NETDEV_XDP_ACT_REDIRECT;
+ NETDEV_XDP_ACT_REDIRECT |
+ NETDEV_XDP_ACT_NDO_XMIT;
return 0;
}
@@ -2844,6 +2845,7 @@ static const struct net_device_ops ionic_netdev_ops = {
.ndo_eth_ioctl = ionic_eth_ioctl,
.ndo_start_xmit = ionic_start_xmit,
.ndo_bpf = ionic_xdp,
+ .ndo_xdp_xmit = ionic_xdp_xmit,
.ndo_get_stats64 = ionic_get_stats64,
.ndo_set_rx_mode = ionic_ndo_set_rx_mode,
.ndo_set_features = ionic_set_features,
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
index c2be2215406a..618d15fb8d95 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
@@ -316,9 +316,13 @@ static void ionic_xdp_tx_desc_clean(struct ionic_queue *q,
buf_info = desc_info->bufs;
dma_unmap_single(dev, buf_info->dma_addr,
buf_info->len, DMA_TO_DEVICE);
- __free_pages(buf_info->page, 0);
+ if (desc_info->act == XDP_TX)
+ __free_pages(buf_info->page, 0);
buf_info->page = NULL;
+ if (desc_info->act == XDP_REDIRECT)
+ xdp_return_frame(desc_info->xdpf);
+
desc_info->nbufs = 0;
desc_info->xdpf = NULL;
desc_info->act = 0;
@@ -372,6 +376,62 @@ static int ionic_xdp_post_frame(struct net_device *netdev,
return 0;
}
+int ionic_xdp_xmit(struct net_device *netdev, int n,
+ struct xdp_frame **xdp_frames, u32 flags)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic_queue *txq;
+ struct netdev_queue *nq;
+ int nxmit;
+ int space;
+ int cpu;
+ int qi;
+
+ if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state)))
+ return -ENETDOWN;
+
+ if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
+ return -EINVAL;
+
+ /* AdminQ is assumed on cpu 0, while we attempt to affinitize the
+ * TxRx queue pairs 0..n-1 on cpus 1..n. We try to keep with that
+ * affinitization here, but of course irqbalance and friends might
+ * have juggled things anyway, so we have to check for the 0 case.
+ */
+ cpu = smp_processor_id();
+ qi = cpu ? (cpu - 1) % lif->nxqs : cpu;
+
+ txq = &lif->txqcqs[qi]->q;
+ nq = netdev_get_tx_queue(netdev, txq->index);
+ __netif_tx_lock(nq, cpu);
+
+ if (netif_tx_queue_stopped(nq) ||
+ unlikely(ionic_maybe_stop_tx(txq, 1))) {
+ __netif_tx_unlock(nq);
+ return -EIO;
+ }
+
+ space = min_t(int, n, ionic_q_space_avail(txq));
+ for (nxmit = 0; nxmit < space ; nxmit++) {
+ if (ionic_xdp_post_frame(netdev, txq, xdp_frames[nxmit],
+ XDP_REDIRECT,
+ virt_to_page(xdp_frames[nxmit]->data),
+ 0, false)) {
+ nxmit--;
+ break;
+ }
+ }
+
+ if (flags & XDP_XMIT_FLUSH)
+ ionic_dbell_ring(lif->kern_dbpage, txq->hw_type,
+ txq->dbval | txq->head_idx);
+
+ ionic_maybe_stop_tx(txq, 4);
+ __netif_tx_unlock(nq);
+
+ return nxmit;
+}
+
static bool ionic_run_xdp(struct ionic_rx_stats *stats,
struct net_device *netdev,
struct ionic_queue *rxq,
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.h b/drivers/net/ethernet/pensando/ionic/ionic_txrx.h
index d7cbaad8a6fb..82fc38e0f573 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.h
+++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.h
@@ -17,4 +17,5 @@ netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev);
bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info);
bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info);
+int ionic_xdp_xmit(struct net_device *netdev, int n, struct xdp_frame **xdp, u32 flags);
#endif /* _IONIC_TXRX_H_ */
--
2.17.1
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH net-next 9/9] ionic: implement xdp frags support
2024-01-30 1:30 [PATCH net-next 0/9] ionic: add XDP support Shannon Nelson
` (7 preceding siblings ...)
2024-01-30 1:30 ` [PATCH net-next 8/9] ionic: add ndo_xdp_xmit Shannon Nelson
@ 2024-01-30 1:30 ` Shannon Nelson
8 siblings, 0 replies; 12+ messages in thread
From: Shannon Nelson @ 2024-01-30 1:30 UTC (permalink / raw)
To: netdev, davem, kuba, edumazet, pabeni
Cc: brett.creeley, drivers, Shannon Nelson
Add support for using scatter-gather / frags in XDP in both
Rx and Tx paths.
Co-developed-by: Brett Creeley <brett.creeley@amd.com>
Signed-off-by: Brett Creeley <brett.creeley@amd.com>
Signed-off-by: Shannon Nelson <shannon.nelson@amd.com>
---
.../net/ethernet/pensando/ionic/ionic_lif.c | 12 ++-
.../net/ethernet/pensando/ionic/ionic_txrx.c | 91 ++++++++++++++++++-
2 files changed, 98 insertions(+), 5 deletions(-)
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
index ed5d792c4780..11adf3f55fd0 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
@@ -881,7 +881,8 @@ static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
q->partner = &lif->txqcqs[q->index]->q;
q->partner->partner = q;
- if (!lif->xdp_prog)
+ if (!lif->xdp_prog ||
+ (lif->xdp_prog->aux && lif->xdp_prog->aux->xdp_has_frags))
ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_SG);
if (qcq->flags & IONIC_QCQ_F_CMB_RINGS) {
@@ -1651,7 +1652,9 @@ static int ionic_init_nic_features(struct ionic_lif *lif)
netdev->xdp_features = NETDEV_XDP_ACT_BASIC |
NETDEV_XDP_ACT_REDIRECT |
- NETDEV_XDP_ACT_NDO_XMIT;
+ NETDEV_XDP_ACT_RX_SG |
+ NETDEV_XDP_ACT_NDO_XMIT |
+ NETDEV_XDP_ACT_NDO_XMIT_SG;
return 0;
}
@@ -1799,6 +1802,9 @@ static bool ionic_xdp_is_valid_mtu(struct ionic_lif *lif, u32 mtu,
if (mtu <= IONIC_XDP_MAX_LINEAR_MTU)
return true;
+ if (xdp_prog->aux && xdp_prog->aux->xdp_has_frags)
+ return true;
+
return false;
}
@@ -2809,7 +2815,7 @@ static int ionic_xdp_config(struct net_device *netdev, struct netdev_bpf *bpf)
}
maxfs = __le32_to_cpu(lif->identity->eth.max_frame_size) - VLAN_ETH_HLEN;
- if (bpf->prog)
+ if (bpf->prog && !(bpf->prog->aux && bpf->prog->aux->xdp_has_frags))
maxfs = min_t(u32, maxfs, IONIC_XDP_MAX_LINEAR_MTU);
netdev->max_mtu = maxfs;
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
index 618d15fb8d95..f80ee7a870fb 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
@@ -15,6 +15,13 @@ static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs);
static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
void *data, size_t len);
+static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,
+ const skb_frag_t *frag,
+ size_t offset, size_t len);
+
+static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q,
+ struct ionic_desc_info *desc_info);
+
static void ionic_tx_clean(struct ionic_queue *q,
struct ionic_desc_info *desc_info,
struct ionic_cq_info *cq_info,
@@ -309,6 +316,7 @@ static void ionic_xdp_tx_desc_clean(struct ionic_queue *q,
unsigned int nbufs = desc_info->nbufs;
struct ionic_buf_info *buf_info;
struct device *dev = q->dev;
+ int i;
if (!nbufs)
return;
@@ -320,6 +328,15 @@ static void ionic_xdp_tx_desc_clean(struct ionic_queue *q,
__free_pages(buf_info->page, 0);
buf_info->page = NULL;
+ buf_info++;
+ for (i = 1; i < nbufs + 1 && buf_info->page; i++, buf_info++) {
+ dma_unmap_page(dev, buf_info->dma_addr,
+ buf_info->len, DMA_TO_DEVICE);
+ if (desc_info->act == XDP_TX)
+ __free_pages(buf_info->page, 0);
+ buf_info->page = NULL;
+ }
+
if (desc_info->act == XDP_REDIRECT)
xdp_return_frame(desc_info->xdpf);
@@ -360,8 +377,38 @@ static int ionic_xdp_post_frame(struct net_device *netdev,
desc_info->xdpf = frame;
desc_info->act = act;
+ if (xdp_frame_has_frags(frame)) {
+ struct ionic_txq_sg_elem *elem;
+ struct skb_shared_info *sinfo;
+ struct ionic_buf_info *bi;
+ skb_frag_t *frag;
+ int i;
+
+ bi = &buf_info[1];
+ sinfo = xdp_get_shared_info_from_frame(frame);
+ frag = sinfo->frags;
+ elem = desc_info->txq_sg_desc->elems;
+ for (i = 0; i < sinfo->nr_frags; i++, frag++, bi++) {
+ dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag));
+ if (dma_mapping_error(q->dev, dma_addr)) {
+ stats->dma_map_err++;
+ ionic_tx_desc_unmap_bufs(q, desc_info);
+ return -EIO;
+ }
+ bi->dma_addr = dma_addr;
+ bi->len = skb_frag_size(frag);
+ bi->page = skb_frag_page(frag);
+
+ elem->addr = cpu_to_le64(bi->dma_addr);
+ elem->len = cpu_to_le16(bi->len);
+ elem++;
+
+ desc_info->nbufs++;
+ }
+ }
+
cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,
- 0, 0, buf_info->dma_addr);
+ 0, (desc_info->nbufs - 1), buf_info->dma_addr);
desc->cmd = cpu_to_le64(cmd);
desc->len = cpu_to_le16(len);
desc->csum_start = 0;
@@ -444,6 +491,8 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats,
struct ionic_queue *txq;
struct netdev_queue *nq;
struct xdp_frame *xdpf;
+ int remain_len;
+ int frag_len;
int err = 0;
xdp_prog = READ_ONCE(rxq->lif->xdp_prog);
@@ -451,8 +500,9 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats,
return false;
xdp_init_buff(&xdp_buf, IONIC_PAGE_SIZE, rxq->xdp_rxq_info);
+ frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN);
xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info),
- XDP_PACKET_HEADROOM, len, false);
+ XDP_PACKET_HEADROOM, frag_len, false);
dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(buf_info),
XDP_PACKET_HEADROOM, len,
@@ -460,6 +510,43 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats,
prefetchw(&xdp_buf.data_hard_start);
+ /* We limit MTU size to one buffer if !xdp_has_frags, so
+ * if the recv len is bigger than one buffer
+ * then we know we have frag info to gather
+ */
+ remain_len = len - frag_len;
+ if (remain_len) {
+ struct skb_shared_info *sinfo;
+ struct ionic_buf_info *bi;
+ skb_frag_t *frag;
+
+ bi = buf_info;
+ sinfo = xdp_get_shared_info_from_buff(&xdp_buf);
+ sinfo->nr_frags = 0;
+ sinfo->xdp_frags_size = 0;
+ xdp_buff_set_frags_flag(&xdp_buf);
+
+ do {
+ if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) {
+ err = -ENOSPC;
+ goto out_xdp_abort;
+ }
+
+ frag = &sinfo->frags[sinfo->nr_frags];
+ sinfo->nr_frags++;
+ bi++;
+ frag_len = min_t(u16, remain_len, ionic_rx_buf_size(bi));
+ dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(bi),
+ 0, frag_len, DMA_FROM_DEVICE);
+ skb_frag_fill_page_desc(frag, bi->page, 0, frag_len);
+ sinfo->xdp_frags_size += frag_len;
+ remain_len -= frag_len;
+
+ if (page_is_pfmemalloc(bi->page))
+ xdp_buff_set_frag_pfmemalloc(&xdp_buf);
+ } while (remain_len > 0);
+ }
+
xdp_action = bpf_prog_run_xdp(xdp_prog, &xdp_buf);
switch (xdp_action) {
--
2.17.1
^ permalink raw reply related [flat|nested] 12+ messages in thread