* [net-next PATCH 1/6] octeontx2-pf: Add AF_XDP non-zero copy support
2025-01-07 10:46 [net-next PATCH 0/6] Add af_xdp support for cn10k Suman Ghosh
@ 2025-01-07 10:46 ` Suman Ghosh
2025-01-07 10:46 ` [net-next PATCH 2/6] octeontx2-pf: AF_XDP zero copy receive support Suman Ghosh
` (4 subsequent siblings)
5 siblings, 0 replies; 9+ messages in thread
From: Suman Ghosh @ 2025-01-07 10:46 UTC (permalink / raw)
To: sgoutham, gakula, sbhatta, hkelam, davem, edumazet, kuba, pabeni,
netdev, linux-kernel, lcherian, jerinj
Cc: Suman Ghosh
For XDP, page_pool APIs are getting used now. But the memory type was
not getting set due to which XDP_REDIRECT and hence AF_XDP was not
working. This patch ads the memory type MEM_TYPE_PAGE_POOL as the memory
model of the XDP program.
Signed-off-by: Suman Ghosh <sumang@marvell.com>
---
.../ethernet/marvell/octeontx2/nic/otx2_common.c | 8 +++++++-
.../ethernet/marvell/octeontx2/nic/otx2_common.h | 1 +
.../net/ethernet/marvell/octeontx2/nic/otx2_txrx.c | 13 ++++++++++---
3 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
index 2b49bfec7869..161cf33ef89e 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
@@ -1047,6 +1047,7 @@ static int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx)
int err, pool_id, non_xdp_queues;
struct nix_aq_enq_req *aq;
struct otx2_cq_queue *cq;
+ struct otx2_pool *pool;
cq = &qset->cq[qidx];
cq->cq_idx = qidx;
@@ -1055,8 +1056,13 @@ static int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx)
cq->cq_type = CQ_RX;
cq->cint_idx = qidx;
cq->cqe_cnt = qset->rqe_cnt;
- if (pfvf->xdp_prog)
+ if (pfvf->xdp_prog) {
+ pool = &qset->pool[qidx];
xdp_rxq_info_reg(&cq->xdp_rxq, pfvf->netdev, qidx, 0);
+ xdp_rxq_info_reg_mem_model(&cq->xdp_rxq,
+ MEM_TYPE_PAGE_POOL,
+ pool->page_pool);
+ }
} else if (qidx < non_xdp_queues) {
cq->cq_type = CQ_TX;
cq->cint_idx = qidx - pfvf->hw.rx_queues;
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
index 65814e3dc93f..3748941bbaf1 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
@@ -21,6 +21,7 @@
#include <linux/time64.h>
#include <linux/dim.h>
#include <uapi/linux/if_macsec.h>
+#include <net/page_pool/helpers.h>
#include <mbox.h>
#include <npc.h>
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
index 224cef938927..ed8b37eb2054 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
@@ -96,7 +96,7 @@ static unsigned int frag_num(unsigned int i)
static void otx2_xdp_snd_pkt_handler(struct otx2_nic *pfvf,
struct otx2_snd_queue *sq,
- struct nix_cqe_tx_s *cqe)
+ struct nix_cqe_tx_s *cqe)
{
struct nix_send_comp_s *snd_comp = &cqe->comp;
struct sg_list *sg;
@@ -109,6 +109,11 @@ static void otx2_xdp_snd_pkt_handler(struct otx2_nic *pfvf,
otx2_dma_unmap_page(pfvf, sg->dma_addr[0],
sg->size[0], DMA_TO_DEVICE);
page = virt_to_page(phys_to_virt(pa));
+ if (page->pp) {
+ page_pool_recycle_direct(page->pp, page);
+ return;
+ }
+
put_page(page);
}
@@ -1419,6 +1424,7 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
bool *need_xdp_flush)
{
unsigned char *hard_start;
+ struct otx2_pool *pool;
int qidx = cq->cq_idx;
struct xdp_buff xdp;
struct page *page;
@@ -1426,6 +1432,7 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
u32 act;
int err;
+ pool = &pfvf->qset.pool[qidx];
iova = cqe->sg.seg_addr - OTX2_HEAD_ROOM;
pa = otx2_iova_to_phys(pfvf->iommu_domain, iova);
page = virt_to_page(phys_to_virt(pa));
@@ -1456,7 +1463,7 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
*need_xdp_flush = true;
return true;
}
- put_page(page);
+ page_pool_recycle_direct(pool->page_pool, page);
break;
default:
bpf_warn_invalid_xdp_action(pfvf->netdev, prog, act);
@@ -1467,7 +1474,7 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
case XDP_DROP:
otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize,
DMA_FROM_DEVICE);
- put_page(page);
+ page_pool_recycle_direct(pool->page_pool, page);
cq->pool_ptrs++;
return true;
}
--
2.25.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [net-next PATCH 2/6] octeontx2-pf: AF_XDP zero copy receive support
2025-01-07 10:46 [net-next PATCH 0/6] Add af_xdp support for cn10k Suman Ghosh
2025-01-07 10:46 ` [net-next PATCH 1/6] octeontx2-pf: Add AF_XDP non-zero copy support Suman Ghosh
@ 2025-01-07 10:46 ` Suman Ghosh
2025-01-07 14:34 ` Simon Horman
2025-01-07 10:46 ` [net-next PATCH 3/6] octeontx2-pf: Reconfigure RSS table after enabling AF_XDP zerocopy on rx queue Suman Ghosh
` (3 subsequent siblings)
5 siblings, 1 reply; 9+ messages in thread
From: Suman Ghosh @ 2025-01-07 10:46 UTC (permalink / raw)
To: sgoutham, gakula, sbhatta, hkelam, davem, edumazet, kuba, pabeni,
netdev, linux-kernel, lcherian, jerinj
Cc: Suman Ghosh
This patch adds support to AF_XDP zero copy support for CN10K.
This patch specifically adds receive side support. In this approach once
a xdp program with zero copy support on a specific rx queue is enabled,
then that receive queue is disabled/detached from the existing kernel
queue and re-assigned to the umem memory.
Signed-off-by: Suman Ghosh <sumang@marvell.com>
---
.../ethernet/marvell/octeontx2/nic/Makefile | 2 +-
.../ethernet/marvell/octeontx2/nic/cn10k.c | 6 +-
.../marvell/octeontx2/nic/otx2_common.c | 120 +++++++++---
.../marvell/octeontx2/nic/otx2_common.h | 11 +-
.../ethernet/marvell/octeontx2/nic/otx2_pf.c | 17 +-
.../marvell/octeontx2/nic/otx2_txrx.c | 60 +++++-
.../marvell/octeontx2/nic/otx2_txrx.h | 6 +
.../ethernet/marvell/octeontx2/nic/otx2_vf.c | 4 +
.../ethernet/marvell/octeontx2/nic/otx2_xsk.c | 182 ++++++++++++++++++
.../ethernet/marvell/octeontx2/nic/otx2_xsk.h | 21 ++
.../ethernet/marvell/octeontx2/nic/qos_sq.c | 2 +-
11 files changed, 378 insertions(+), 53 deletions(-)
create mode 100644 drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c
create mode 100644 drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/Makefile b/drivers/net/ethernet/marvell/octeontx2/nic/Makefile
index cb6513ab35e7..69e0778f9ac1 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/Makefile
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/Makefile
@@ -9,7 +9,7 @@ obj-$(CONFIG_RVU_ESWITCH) += rvu_rep.o
rvu_nicpf-y := otx2_pf.o otx2_common.o otx2_txrx.o otx2_ethtool.o \
otx2_flows.o otx2_tc.o cn10k.o otx2_dmac_flt.o \
- otx2_devlink.o qos_sq.o qos.o
+ otx2_devlink.o qos_sq.o qos.o otx2_xsk.o
rvu_nicvf-y := otx2_vf.o
rvu_rep-y := rep.o
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c
index a15cc86635d6..9a2865c60850 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c
@@ -112,9 +112,12 @@ int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
struct otx2_nic *pfvf = dev;
int cnt = cq->pool_ptrs;
u64 ptrs[NPA_MAX_BURST];
+ struct otx2_pool *pool;
dma_addr_t bufptr;
int num_ptrs = 1;
+ pool = &pfvf->qset.pool[cq->cq_idx];
+
/* Refill pool with new buffers */
while (cq->pool_ptrs) {
if (otx2_alloc_buffer(pfvf, cq, &bufptr)) {
@@ -124,7 +127,8 @@ int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
break;
}
cq->pool_ptrs--;
- ptrs[num_ptrs] = (u64)bufptr + OTX2_HEAD_ROOM;
+ ptrs[num_ptrs] = pool->xsk_pool ? (u64)bufptr : (u64)bufptr + OTX2_HEAD_ROOM;
+
num_ptrs++;
if (num_ptrs == NPA_MAX_BURST || cq->pool_ptrs == 0) {
__cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs,
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
index 161cf33ef89e..d653e0136cf4 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
@@ -17,6 +17,7 @@
#include "otx2_common.h"
#include "otx2_struct.h"
#include "cn10k.h"
+#include "otx2_xsk.h"
static bool otx2_is_pfc_enabled(struct otx2_nic *pfvf)
{
@@ -549,10 +550,13 @@ static int otx2_alloc_pool_buf(struct otx2_nic *pfvf, struct otx2_pool *pool,
}
static int __otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool,
- dma_addr_t *dma)
+ dma_addr_t *dma, int qidx, int idx)
{
u8 *buf;
+ if (pool->xsk_pool)
+ return otx2_xsk_pool_alloc_buf(pfvf, pool, dma, idx);
+
if (pool->page_pool)
return otx2_alloc_pool_buf(pfvf, pool, dma);
@@ -571,12 +575,12 @@ static int __otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool,
}
int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool,
- dma_addr_t *dma)
+ dma_addr_t *dma, int qidx, int idx)
{
int ret;
local_bh_disable();
- ret = __otx2_alloc_rbuf(pfvf, pool, dma);
+ ret = __otx2_alloc_rbuf(pfvf, pool, dma, qidx, idx);
local_bh_enable();
return ret;
}
@@ -584,7 +588,8 @@ int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool,
int otx2_alloc_buffer(struct otx2_nic *pfvf, struct otx2_cq_queue *cq,
dma_addr_t *dma)
{
- if (unlikely(__otx2_alloc_rbuf(pfvf, cq->rbpool, dma)))
+ if (unlikely(__otx2_alloc_rbuf(pfvf, cq->rbpool, dma,
+ cq->cq_idx, cq->pool_ptrs - 1)))
return -ENOMEM;
return 0;
}
@@ -884,7 +889,7 @@ void otx2_sqb_flush(struct otx2_nic *pfvf)
#define RQ_PASS_LVL_AURA (255 - ((95 * 256) / 100)) /* RED when 95% is full */
#define RQ_DROP_LVL_AURA (255 - ((99 * 256) / 100)) /* Drop when 99% is full */
-static int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura)
+int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura)
{
struct otx2_qset *qset = &pfvf->qset;
struct nix_aq_enq_req *aq;
@@ -1041,7 +1046,7 @@ int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura)
}
-static int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx)
+int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx)
{
struct otx2_qset *qset = &pfvf->qset;
int err, pool_id, non_xdp_queues;
@@ -1057,11 +1062,18 @@ static int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx)
cq->cint_idx = qidx;
cq->cqe_cnt = qset->rqe_cnt;
if (pfvf->xdp_prog) {
- pool = &qset->pool[qidx];
xdp_rxq_info_reg(&cq->xdp_rxq, pfvf->netdev, qidx, 0);
- xdp_rxq_info_reg_mem_model(&cq->xdp_rxq,
- MEM_TYPE_PAGE_POOL,
- pool->page_pool);
+ pool = &qset->pool[qidx];
+ if (pool->xsk_pool) {
+ xdp_rxq_info_reg_mem_model(&cq->xdp_rxq,
+ MEM_TYPE_XSK_BUFF_POOL,
+ NULL);
+ xsk_pool_set_rxq_info(pool->xsk_pool, &cq->xdp_rxq);
+ } else if (pool->page_pool) {
+ xdp_rxq_info_reg_mem_model(&cq->xdp_rxq,
+ MEM_TYPE_PAGE_POOL,
+ pool->page_pool);
+ }
}
} else if (qidx < non_xdp_queues) {
cq->cq_type = CQ_TX;
@@ -1281,9 +1293,10 @@ void otx2_free_bufs(struct otx2_nic *pfvf, struct otx2_pool *pool,
pa = otx2_iova_to_phys(pfvf->iommu_domain, iova);
page = virt_to_head_page(phys_to_virt(pa));
-
if (pool->page_pool) {
page_pool_put_full_page(pool->page_pool, page, true);
+ } else if (pool->xsk_pool) {
+ /* Note: No way of identifying xdp_buff */
} else {
dma_unmap_page_attrs(pfvf->dev, iova, size,
DMA_FROM_DEVICE,
@@ -1298,6 +1311,7 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type)
int pool_id, pool_start = 0, pool_end = 0, size = 0;
struct otx2_pool *pool;
u64 iova;
+ int idx;
if (type == AURA_NIX_SQ) {
pool_start = otx2_get_pool_idx(pfvf, type, 0);
@@ -1312,8 +1326,8 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type)
/* Free SQB and RQB pointers from the aura pool */
for (pool_id = pool_start; pool_id < pool_end; pool_id++) {
- iova = otx2_aura_allocptr(pfvf, pool_id);
pool = &pfvf->qset.pool[pool_id];
+ iova = otx2_aura_allocptr(pfvf, pool_id);
while (iova) {
if (type == AURA_NIX_RQ)
iova -= OTX2_HEAD_ROOM;
@@ -1323,6 +1337,13 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type)
iova = otx2_aura_allocptr(pfvf, pool_id);
}
}
+
+ for (idx = 0 ; idx < pool->xdp_cnt; idx++) {
+ if (!pool->xdp[idx])
+ continue;
+
+ xsk_buff_free(pool->xdp[idx]);
+ }
}
void otx2_aura_pool_free(struct otx2_nic *pfvf)
@@ -1337,8 +1358,12 @@ void otx2_aura_pool_free(struct otx2_nic *pfvf)
pool = &pfvf->qset.pool[pool_id];
qmem_free(pfvf->dev, pool->stack);
qmem_free(pfvf->dev, pool->fc_addr);
- page_pool_destroy(pool->page_pool);
- pool->page_pool = NULL;
+ if (pool->page_pool) {
+ page_pool_destroy(pool->page_pool);
+ pool->page_pool = NULL;
+ }
+ devm_kfree(pfvf->dev, pool->xdp);
+ pool->xsk_pool = NULL;
}
devm_kfree(pfvf->dev, pfvf->qset.pool);
pfvf->qset.pool = NULL;
@@ -1425,9 +1450,10 @@ int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id,
int stack_pages, int numptrs, int buf_size, int type)
{
struct page_pool_params pp_params = { 0 };
+ struct xsk_buff_pool *xsk_pool;
struct npa_aq_enq_req *aq;
struct otx2_pool *pool;
- int err;
+ int err, sz;
pool = &pfvf->qset.pool[pool_id];
/* Alloc memory for stack which is used to store buffer pointers */
@@ -1468,21 +1494,36 @@ int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id,
aq->ctype = NPA_AQ_CTYPE_POOL;
aq->op = NPA_AQ_INSTOP_INIT;
- if (type != AURA_NIX_RQ) {
- pool->page_pool = NULL;
+ if (type != AURA_NIX_RQ)
+ return 0;
+
+ if (!test_bit(pool_id, pfvf->af_xdp_zc_qidx)) {
+ sz = ALIGN(ALIGN(SKB_DATA_ALIGN(buf_size), OTX2_ALIGN), PAGE_SIZE);
+ pp_params.order = (sz / PAGE_SIZE) - 1;
+ pp_params.flags = PP_FLAG_DMA_MAP;
+ pp_params.pool_size = min(OTX2_PAGE_POOL_SZ, numptrs);
+ pp_params.nid = NUMA_NO_NODE;
+ pp_params.dev = pfvf->dev;
+ pp_params.dma_dir = DMA_FROM_DEVICE;
+ pool->page_pool = page_pool_create(&pp_params);
+ if (IS_ERR(pool->page_pool)) {
+ netdev_err(pfvf->netdev, "Creation of page pool failed\n");
+ return PTR_ERR(pool->page_pool);
+ }
return 0;
}
- pp_params.order = get_order(buf_size);
- pp_params.flags = PP_FLAG_DMA_MAP;
- pp_params.pool_size = min(OTX2_PAGE_POOL_SZ, numptrs);
- pp_params.nid = NUMA_NO_NODE;
- pp_params.dev = pfvf->dev;
- pp_params.dma_dir = DMA_FROM_DEVICE;
- pool->page_pool = page_pool_create(&pp_params);
- if (IS_ERR(pool->page_pool)) {
- netdev_err(pfvf->netdev, "Creation of page pool failed\n");
- return PTR_ERR(pool->page_pool);
+ /* Set XSK pool to support AF_XDP zero-copy */
+ xsk_pool = xsk_get_pool_from_qid(pfvf->netdev, pool_id);
+ if (xsk_pool) {
+ pool->xsk_pool = xsk_pool;
+ pool->xdp_cnt = numptrs;
+ pool->xdp = devm_kcalloc(pfvf->dev,
+ numptrs, sizeof(struct xdp_buff *), GFP_KERNEL);
+ if (IS_ERR(pool->xdp)) {
+ netdev_err(pfvf->netdev, "Creation of xsk pool failed\n");
+ return PTR_ERR(pool->xdp);
+ }
}
return 0;
@@ -1543,9 +1584,18 @@ int otx2_sq_aura_pool_init(struct otx2_nic *pfvf)
}
for (ptr = 0; ptr < num_sqbs; ptr++) {
- err = otx2_alloc_rbuf(pfvf, pool, &bufptr);
- if (err)
+ err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, ptr);
+ if (err) {
+ if (pool->xsk_pool) {
+ ptr--;
+ while (ptr >= 0) {
+ xsk_buff_free(pool->xdp[ptr]);
+ ptr--;
+ }
+ }
goto err_mem;
+ }
+
pfvf->hw_ops->aura_freeptr(pfvf, pool_id, bufptr);
sq->sqb_ptrs[sq->sqb_count++] = (u64)bufptr;
}
@@ -1595,11 +1645,19 @@ int otx2_rq_aura_pool_init(struct otx2_nic *pfvf)
/* Allocate pointers and free them to aura/pool */
for (pool_id = 0; pool_id < hw->rqpool_cnt; pool_id++) {
pool = &pfvf->qset.pool[pool_id];
+
for (ptr = 0; ptr < num_ptrs; ptr++) {
- err = otx2_alloc_rbuf(pfvf, pool, &bufptr);
- if (err)
+ err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, ptr);
+ if (err) {
+ if (pool->xsk_pool) {
+ while (ptr)
+ xsk_buff_free(pool->xdp[--ptr]);
+ }
return -ENOMEM;
+ }
+
pfvf->hw_ops->aura_freeptr(pfvf, pool_id,
+ pool->xsk_pool ? bufptr :
bufptr + OTX2_HEAD_ROOM);
}
}
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
index 3748941bbaf1..40ee21ae6c72 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
@@ -532,6 +532,7 @@ struct otx2_nic {
/* Inline ipsec */
struct cn10k_ipsec ipsec;
+ unsigned long *af_xdp_zc_qidx;
};
static inline bool is_otx2_lbkvf(struct pci_dev *pdev)
@@ -1003,7 +1004,7 @@ void otx2_txschq_free_one(struct otx2_nic *pfvf, u16 lvl, u16 schq);
void otx2_free_pending_sqe(struct otx2_nic *pfvf);
void otx2_sqb_flush(struct otx2_nic *pfvf);
int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool,
- dma_addr_t *dma);
+ dma_addr_t *dma, int qidx, int idx);
int otx2_rxtx_enable(struct otx2_nic *pfvf, bool enable);
void otx2_ctx_disable(struct mbox *mbox, int type, bool npa);
int otx2_nix_config_bp(struct otx2_nic *pfvf, bool enable);
@@ -1033,6 +1034,14 @@ void otx2_pfaf_mbox_destroy(struct otx2_nic *pf);
void otx2_disable_mbox_intr(struct otx2_nic *pf);
void otx2_disable_napi(struct otx2_nic *pf);
irqreturn_t otx2_cq_intr_handler(int irq, void *cq_irq);
+int otx2_init_rsrc(struct pci_dev *pdev, struct otx2_nic *pf);
+int otx2_aura_aq_init(struct otx2_nic *pfvf, int aura_id,
+ int pool_id, int numptrs);
+int otx2_pool_aq_init(struct otx2_nic *pfvf, u16 pool_id,
+ int stack_pages, int numptrs, int buf_size, int type);
+int otx2_set_hw_capabilities(struct otx2_nic *pfvf);
+int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura);
+int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx);
/* RSS configuration APIs*/
int otx2_rss_init(struct otx2_nic *pfvf);
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
index e1dde93e8af8..e10c7841fd8b 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
@@ -27,6 +27,7 @@
#include "qos.h"
#include <rvu_trace.h>
#include "cn10k_ipsec.h"
+#include "otx2_xsk.h"
#define DRV_NAME "rvu_nicpf"
#define DRV_STRING "Marvell RVU NIC Physical Function Driver"
@@ -1662,9 +1663,7 @@ void otx2_free_hw_resources(struct otx2_nic *pf)
struct nix_lf_free_req *free_req;
struct mbox *mbox = &pf->mbox;
struct otx2_cq_queue *cq;
- struct otx2_pool *pool;
struct msg_req *req;
- int pool_id;
int qidx;
/* Ensure all SQE are processed */
@@ -1705,13 +1704,6 @@ void otx2_free_hw_resources(struct otx2_nic *pf)
/* Free RQ buffer pointers*/
otx2_free_aura_ptr(pf, AURA_NIX_RQ);
- for (qidx = 0; qidx < pf->hw.rx_queues; qidx++) {
- pool_id = otx2_get_pool_idx(pf, AURA_NIX_RQ, qidx);
- pool = &pf->qset.pool[pool_id];
- page_pool_destroy(pool->page_pool);
- pool->page_pool = NULL;
- }
-
otx2_free_cq_res(pf);
/* Free all ingress bandwidth profiles allocated */
@@ -2789,6 +2781,8 @@ static int otx2_xdp(struct net_device *netdev, struct netdev_bpf *xdp)
switch (xdp->command) {
case XDP_SETUP_PROG:
return otx2_xdp_setup(pf, xdp->prog);
+ case XDP_SETUP_XSK_POOL:
+ return otx2_xsk_pool_setup(pf, xdp->xsk.pool, xdp->xsk.queue_id);
default:
return -EINVAL;
}
@@ -2866,6 +2860,7 @@ static const struct net_device_ops otx2_netdev_ops = {
.ndo_set_vf_vlan = otx2_set_vf_vlan,
.ndo_get_vf_config = otx2_get_vf_config,
.ndo_bpf = otx2_xdp,
+ .ndo_xsk_wakeup = otx2_xsk_wakeup,
.ndo_xdp_xmit = otx2_xdp_xmit,
.ndo_setup_tc = otx2_setup_tc,
.ndo_set_vf_trust = otx2_ndo_set_vf_trust,
@@ -3204,6 +3199,10 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id)
/* Enable link notifications */
otx2_cgx_config_linkevents(pf, true);
+ pf->af_xdp_zc_qidx = bitmap_zalloc(qcount, GFP_KERNEL);
+ if (!pf->af_xdp_zc_qidx)
+ goto err_pf_sriov_init;
+
#ifdef CONFIG_DCB
err = otx2_dcbnl_set_ops(netdev);
if (err)
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
index ed8b37eb2054..741b4ce6d0ff 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
@@ -12,6 +12,7 @@
#include <linux/bpf_trace.h>
#include <net/ip6_checksum.h>
#include <net/xfrm.h>
+#include <net/xdp.h>
#include "otx2_reg.h"
#include "otx2_common.h"
@@ -535,6 +536,7 @@ int otx2_napi_handler(struct napi_struct *napi, int budget)
struct otx2_cq_poll *cq_poll;
int workdone = 0, cq_idx, i;
struct otx2_cq_queue *cq;
+ struct otx2_pool *pool;
struct otx2_qset *qset;
struct otx2_nic *pfvf;
int filled_cnt = -1;
@@ -559,6 +561,7 @@ int otx2_napi_handler(struct napi_struct *napi, int budget)
if (rx_cq && rx_cq->pool_ptrs)
filled_cnt = pfvf->hw_ops->refill_pool_ptrs(pfvf, rx_cq);
+
/* Clear the IRQ */
otx2_write64(pfvf, NIX_LF_CINTX_INT(cq_poll->cint_idx), BIT_ULL(0));
@@ -571,6 +574,7 @@ int otx2_napi_handler(struct napi_struct *napi, int budget)
if (pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED)
otx2_adjust_adaptive_coalese(pfvf, cq_poll);
+ pool = &pfvf->qset.pool[cq->cq_idx];
if (unlikely(!filled_cnt)) {
struct refill_work *work;
struct delayed_work *dwork;
@@ -584,7 +588,13 @@ int otx2_napi_handler(struct napi_struct *napi, int budget)
schedule_delayed_work(dwork,
msecs_to_jiffies(100));
}
+ /* Call for wake-up for not able to fill buffers */
+ if (pool->xsk_pool)
+ xsk_set_rx_need_wakeup(pool->xsk_pool);
} else {
+ /* Clear wake-up, since buffers are filled successfully */
+ if (pool->xsk_pool)
+ xsk_clear_rx_need_wakeup(pool->xsk_pool);
/* Re-enable interrupts */
otx2_write64(pfvf,
NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx),
@@ -1235,15 +1245,19 @@ void otx2_cleanup_rx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, int q
u16 pool_id;
u64 iova;
- if (pfvf->xdp_prog)
+ pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_RQ, qidx);
+ pool = &pfvf->qset.pool[pool_id];
+
+ if (pfvf->xdp_prog) {
+ if (pool->page_pool)
+ xdp_rxq_info_unreg_mem_model(&cq->xdp_rxq);
+
xdp_rxq_info_unreg(&cq->xdp_rxq);
+ }
if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
return;
- pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_RQ, qidx);
- pool = &pfvf->qset.pool[pool_id];
-
while (cq->pend_cqe) {
cqe = (struct nix_cqe_rx_s *)otx2_get_next_cqe(cq);
processed_cqe++;
@@ -1426,13 +1440,24 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
unsigned char *hard_start;
struct otx2_pool *pool;
int qidx = cq->cq_idx;
- struct xdp_buff xdp;
+ struct xdp_buff xdp, *xsk_buff = NULL;
struct page *page;
u64 iova, pa;
u32 act;
int err;
pool = &pfvf->qset.pool[qidx];
+
+ if (pool->xsk_pool) {
+ xsk_buff = pool->xdp[--cq->rbpool->xdp_top];
+ if (!xsk_buff)
+ return false;
+
+ xsk_buff->data_end = xsk_buff->data + cqe->sg.seg_size;
+ act = bpf_prog_run_xdp(prog, xsk_buff);
+ goto handle_xdp_verdict;
+ }
+
iova = cqe->sg.seg_addr - OTX2_HEAD_ROOM;
pa = otx2_iova_to_phys(pfvf->iommu_domain, iova);
page = virt_to_page(phys_to_virt(pa));
@@ -1445,6 +1470,7 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
act = bpf_prog_run_xdp(prog, &xdp);
+handle_xdp_verdict:
switch (act) {
case XDP_PASS:
break;
@@ -1455,8 +1481,16 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
cqe->sg.seg_size, qidx);
case XDP_REDIRECT:
cq->pool_ptrs++;
- err = xdp_do_redirect(pfvf->netdev, &xdp, prog);
+ if (xsk_buff) {
+ err = xdp_do_redirect(pfvf->netdev, xsk_buff, prog);
+ if (!err) {
+ *need_xdp_flush = true;
+ return true;
+ }
+ return false;
+ }
+ err = xdp_do_redirect(pfvf->netdev, &xdp, prog);
otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize,
DMA_FROM_DEVICE);
if (!err) {
@@ -1469,13 +1503,21 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
bpf_warn_invalid_xdp_action(pfvf->netdev, prog, act);
break;
case XDP_ABORTED:
+ if (xsk_buff)
+ xsk_buff_free(xsk_buff);
trace_xdp_exception(pfvf->netdev, prog, act);
break;
case XDP_DROP:
- otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize,
- DMA_FROM_DEVICE);
- page_pool_recycle_direct(pool->page_pool, page);
cq->pool_ptrs++;
+ if (xsk_buff) {
+ xsk_buff_free(xsk_buff);
+ } else if (page->pp) {
+ page_pool_recycle_direct(pool->page_pool, page);
+ } else {
+ otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize,
+ DMA_FROM_DEVICE);
+ put_page(page);
+ }
return true;
}
return false;
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
index d23810963fdb..401a4eb04441 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
@@ -12,6 +12,7 @@
#include <linux/iommu.h>
#include <linux/if_vlan.h>
#include <net/xdp.h>
+#include <net/xdp_sock_drv.h>
#define LBK_CHAN_BASE 0x000
#define SDP_CHAN_BASE 0x700
@@ -127,7 +128,11 @@ struct otx2_pool {
struct qmem *stack;
struct qmem *fc_addr;
struct page_pool *page_pool;
+ struct xsk_buff_pool *xsk_pool;
+ struct xdp_buff **xdp;
+ u16 xdp_cnt;
u16 rbsize;
+ u16 xdp_top;
};
struct otx2_cq_queue {
@@ -144,6 +149,7 @@ struct otx2_cq_queue {
void *cqe_base;
struct qmem *cqe;
struct otx2_pool *rbpool;
+ bool xsk_zc_en;
struct xdp_rxq_info xdp_rxq;
} ____cacheline_aligned_in_smp;
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
index e926c6ce96cf..9bb7e5c3e227 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
@@ -722,6 +722,10 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (err)
goto err_shutdown_tc;
+ vf->af_xdp_zc_qidx = bitmap_zalloc(qcount, GFP_KERNEL);
+ if (!vf->af_xdp_zc_qidx)
+ goto err_shutdown_tc;
+
#ifdef CONFIG_DCB
err = otx2_dcbnl_set_ops(netdev);
if (err)
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c
new file mode 100644
index 000000000000..69098c6a6fed
--- /dev/null
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c
@@ -0,0 +1,182 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Marvell RVU Ethernet driver
+ *
+ * Copyright (C) 2024 Marvell.
+ *
+ */
+
+#include <linux/bpf_trace.h>
+#include <linux/stringify.h>
+#include <net/xdp_sock_drv.h>
+#include <net/xdp.h>
+
+#include "otx2_common.h"
+#include "otx2_xsk.h"
+
+int otx2_xsk_pool_alloc_buf(struct otx2_nic *pfvf, struct otx2_pool *pool,
+ dma_addr_t *dma, int idx)
+{
+ struct xdp_buff *xdp;
+ int delta;
+
+ xdp = xsk_buff_alloc(pool->xsk_pool);
+ if (!xdp)
+ return -ENOMEM;
+
+ pool->xdp[pool->xdp_top++] = xdp;
+ *dma = OTX2_DATA_ALIGN(xsk_buff_xdp_get_dma(xdp));
+ /* Adjust xdp->data for unaligned addresses */
+ delta = *dma - xsk_buff_xdp_get_dma(xdp);
+ xdp->data += delta;
+
+ return 0;
+}
+
+static int otx2_xsk_ctx_disable(struct otx2_nic *pfvf, u16 qidx, int aura_id)
+{
+ struct nix_cn10k_aq_enq_req *cn10k_rq_aq;
+ struct npa_aq_enq_req *aura_aq;
+ struct npa_aq_enq_req *pool_aq;
+ struct nix_aq_enq_req *rq_aq;
+
+ if (test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) {
+ cn10k_rq_aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
+ if (!cn10k_rq_aq)
+ return -ENOMEM;
+ cn10k_rq_aq->qidx = qidx;
+ cn10k_rq_aq->rq.ena = 0;
+ cn10k_rq_aq->rq_mask.ena = 1;
+ cn10k_rq_aq->ctype = NIX_AQ_CTYPE_RQ;
+ cn10k_rq_aq->op = NIX_AQ_INSTOP_WRITE;
+ } else {
+ rq_aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox);
+ if (!rq_aq)
+ return -ENOMEM;
+ rq_aq->qidx = qidx;
+ rq_aq->sq.ena = 0;
+ rq_aq->sq_mask.ena = 1;
+ rq_aq->ctype = NIX_AQ_CTYPE_RQ;
+ rq_aq->op = NIX_AQ_INSTOP_WRITE;
+ }
+
+ aura_aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox);
+ if (!aura_aq) {
+ otx2_mbox_reset(&pfvf->mbox.mbox, 0);
+ return -ENOMEM;
+ }
+
+ aura_aq->aura_id = aura_id;
+ aura_aq->aura.ena = 0;
+ aura_aq->aura_mask.ena = 1;
+ aura_aq->ctype = NPA_AQ_CTYPE_AURA;
+ aura_aq->op = NPA_AQ_INSTOP_WRITE;
+
+ pool_aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox);
+ if (!pool_aq) {
+ otx2_mbox_reset(&pfvf->mbox.mbox, 0);
+ return -ENOMEM;
+ }
+
+ pool_aq->aura_id = aura_id;
+ pool_aq->pool.ena = 0;
+ pool_aq->pool_mask.ena = 1;
+
+ pool_aq->ctype = NPA_AQ_CTYPE_POOL;
+ pool_aq->op = NPA_AQ_INSTOP_WRITE;
+
+ return otx2_sync_mbox_msg(&pfvf->mbox);
+}
+
+static void otx2_clean_up_rq(struct otx2_nic *pfvf, int qidx)
+{
+ struct otx2_qset *qset = &pfvf->qset;
+ struct otx2_cq_queue *cq;
+ struct otx2_pool *pool;
+ u64 iova;
+
+ /* If the DOWN flag is set SQs are already freed */
+ if (pfvf->flags & OTX2_FLAG_INTF_DOWN)
+ return;
+
+ cq = &qset->cq[qidx];
+ if (cq)
+ otx2_cleanup_rx_cqes(pfvf, cq, qidx);
+
+ pool = &pfvf->qset.pool[qidx];
+ iova = otx2_aura_allocptr(pfvf, qidx);
+ while (iova) {
+ iova -= OTX2_HEAD_ROOM;
+ otx2_free_bufs(pfvf, pool, iova, pfvf->rbsize);
+ iova = otx2_aura_allocptr(pfvf, qidx);
+ }
+
+ mutex_lock(&pfvf->mbox.lock);
+ otx2_xsk_ctx_disable(pfvf, qidx, qidx);
+ mutex_unlock(&pfvf->mbox.lock);
+}
+
+int otx2_xsk_pool_enable(struct otx2_nic *pf, struct xsk_buff_pool *pool, u16 qidx)
+{
+ u16 rx_queues = pf->hw.rx_queues;
+ u16 tx_queues = pf->hw.tx_queues;
+ int err;
+
+ if (qidx >= rx_queues || qidx >= tx_queues)
+ return -EINVAL;
+
+ err = xsk_pool_dma_map(pool, pf->dev, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
+ if (err)
+ return err;
+
+ set_bit(qidx, pf->af_xdp_zc_qidx);
+ otx2_clean_up_rq(pf, qidx);
+ /* Kick start the NAPI context so that receiving will start */
+ return otx2_xsk_wakeup(pf->netdev, qidx, XDP_WAKEUP_RX);
+}
+
+int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qidx)
+{
+ struct net_device *netdev = pf->netdev;
+ struct xsk_buff_pool *pool;
+
+ pool = xsk_get_pool_from_qid(netdev, qidx);
+ if (!pool)
+ return -EINVAL;
+
+ otx2_clean_up_rq(pf, qidx);
+ clear_bit(qidx, pf->af_xdp_zc_qidx);
+ xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
+
+ return 0;
+}
+
+int otx2_xsk_pool_setup(struct otx2_nic *pf, struct xsk_buff_pool *pool, u16 qidx)
+{
+ if (pool)
+ return otx2_xsk_pool_enable(pf, pool, qidx);
+
+ return otx2_xsk_pool_disable(pf, qidx);
+}
+
+int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
+{
+ struct otx2_nic *pf = netdev_priv(dev);
+ struct otx2_cq_poll *cq_poll = NULL;
+ struct otx2_qset *qset = &pf->qset;
+
+ if (pf->flags & OTX2_FLAG_INTF_DOWN)
+ return -ENETDOWN;
+
+ if (queue_id >= pf->hw.rx_queues)
+ return -EINVAL;
+
+ cq_poll = &qset->napi[queue_id];
+ if (!cq_poll)
+ return -EINVAL;
+
+ /* Trigger interrupt */
+ if (!napi_if_scheduled_mark_missed(&cq_poll->napi))
+ otx2_write64(pf, NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx), BIT_ULL(0));
+
+ return 0;
+}
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h
new file mode 100644
index 000000000000..022b3433edbb
--- /dev/null
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Marvell RVU PF/VF Netdev Devlink
+ *
+ * Copyright (C) 2024 Marvell.
+ *
+ */
+
+#ifndef OTX2_XSK_H
+#define OTX2_XSK_H
+
+struct otx2_nic;
+struct xsk_buff_pool;
+
+int otx2_xsk_pool_setup(struct otx2_nic *pf, struct xsk_buff_pool *pool, u16 qid);
+int otx2_xsk_pool_enable(struct otx2_nic *pf, struct xsk_buff_pool *pool, u16 qid);
+int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qid);
+int otx2_xsk_pool_alloc_buf(struct otx2_nic *pfvf, struct otx2_pool *pool,
+ dma_addr_t *dma, int idx);
+int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags);
+
+#endif /* OTX2_XSK_H */
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/qos_sq.c b/drivers/net/ethernet/marvell/octeontx2/nic/qos_sq.c
index 9d887bfc3108..c5dbae0e513b 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/qos_sq.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/qos_sq.c
@@ -82,7 +82,7 @@ static int otx2_qos_sq_aura_pool_init(struct otx2_nic *pfvf, int qidx)
}
for (ptr = 0; ptr < num_sqbs; ptr++) {
- err = otx2_alloc_rbuf(pfvf, pool, &bufptr);
+ err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, ptr);
if (err)
goto sqb_free;
pfvf->hw_ops->aura_freeptr(pfvf, pool_id, bufptr);
--
2.25.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* Re: [net-next PATCH 2/6] octeontx2-pf: AF_XDP zero copy receive support
2025-01-07 10:46 ` [net-next PATCH 2/6] octeontx2-pf: AF_XDP zero copy receive support Suman Ghosh
@ 2025-01-07 14:34 ` Simon Horman
2025-01-08 14:02 ` [EXTERNAL] " Suman Ghosh
0 siblings, 1 reply; 9+ messages in thread
From: Simon Horman @ 2025-01-07 14:34 UTC (permalink / raw)
To: Suman Ghosh
Cc: sgoutham, gakula, sbhatta, hkelam, davem, edumazet, kuba, pabeni,
netdev, linux-kernel, lcherian, jerinj
On Tue, Jan 07, 2025 at 04:16:24PM +0530, Suman Ghosh wrote:
> This patch adds support to AF_XDP zero copy support for CN10K.
> This patch specifically adds receive side support. In this approach once
> a xdp program with zero copy support on a specific rx queue is enabled,
> then that receive queue is disabled/detached from the existing kernel
> queue and re-assigned to the umem memory.
>
> Signed-off-by: Suman Ghosh <sumang@marvell.com>
Hi Suman,
I'd like to bring to your attention a number of minor issues
flagged by Smatch.
...
> diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
...
> @@ -1298,6 +1311,7 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type)
> int pool_id, pool_start = 0, pool_end = 0, size = 0;
> struct otx2_pool *pool;
> u64 iova;
> + int idx;
>
> if (type == AURA_NIX_SQ) {
> pool_start = otx2_get_pool_idx(pfvf, type, 0);
> @@ -1312,8 +1326,8 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type)
>
> /* Free SQB and RQB pointers from the aura pool */
> for (pool_id = pool_start; pool_id < pool_end; pool_id++) {
> - iova = otx2_aura_allocptr(pfvf, pool_id);
> pool = &pfvf->qset.pool[pool_id];
> + iova = otx2_aura_allocptr(pfvf, pool_id);
> while (iova) {
> if (type == AURA_NIX_RQ)
> iova -= OTX2_HEAD_ROOM;
> @@ -1323,6 +1337,13 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type)
> iova = otx2_aura_allocptr(pfvf, pool_id);
> }
> }
> +
> + for (idx = 0 ; idx < pool->xdp_cnt; idx++) {
> + if (!pool->xdp[idx])
> + continue;
> +
> + xsk_buff_free(pool->xdp[idx]);
> + }
Looking over otx2_pool_init(), I am wondering if the loop above run should
over all (non AURA_NIX_RQ) pools, rather than the last pool covered by the
preceding loop.
This was flagged by Smatch, because it is concerned that pool may be used
unset above, presumably if the preceding loop iterates zero times (I'm
unsure if that can happen in practice).
...
> diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
...
> @@ -3204,6 +3199,10 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> /* Enable link notifications */
> otx2_cgx_config_linkevents(pf, true);
>
> + pf->af_xdp_zc_qidx = bitmap_zalloc(qcount, GFP_KERNEL);
> + if (!pf->af_xdp_zc_qidx)
> + goto err_pf_sriov_init;
> +
This goto will result in the function returning err.
However, here err is 0. Should it be set to a negative error value instead?
> #ifdef CONFIG_DCB
> err = otx2_dcbnl_set_ops(netdev);
> if (err)
> diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
...
> @@ -571,6 +574,7 @@ int otx2_napi_handler(struct napi_struct *napi, int budget)
> if (pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED)
> otx2_adjust_adaptive_coalese(pfvf, cq_poll);
>
> + pool = &pfvf->qset.pool[cq->cq_idx];
I am also unsure if this can happen in practice, but Smatch is concerned
that cq may be used uninitialised here. It does seem that could
theoretically be the case if, in the for loop towards the top of this
function, cq_poll->cq_ids[i] is always CINT_INVALID_CQ.
> if (unlikely(!filled_cnt)) {
> struct refill_work *work;
> struct delayed_work *dwork;
...
> @@ -1426,13 +1440,24 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
> unsigned char *hard_start;
> struct otx2_pool *pool;
> int qidx = cq->cq_idx;
> - struct xdp_buff xdp;
> + struct xdp_buff xdp, *xsk_buff = NULL;
> struct page *page;
> u64 iova, pa;
> u32 act;
> int err;
>
> pool = &pfvf->qset.pool[qidx];
> +
> + if (pool->xsk_pool) {
> + xsk_buff = pool->xdp[--cq->rbpool->xdp_top];
> + if (!xsk_buff)
> + return false;
> +
> + xsk_buff->data_end = xsk_buff->data + cqe->sg.seg_size;
> + act = bpf_prog_run_xdp(prog, xsk_buff);
> + goto handle_xdp_verdict;
iova is not initialised until a few lines further down,
which does not occur in the case of this condition.
> + }
> +
> iova = cqe->sg.seg_addr - OTX2_HEAD_ROOM;
> pa = otx2_iova_to_phys(pfvf->iommu_domain, iova);
> page = virt_to_page(phys_to_virt(pa));
> @@ -1445,6 +1470,7 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
>
> act = bpf_prog_run_xdp(prog, &xdp);
>
> +handle_xdp_verdict:
> switch (act) {
> case XDP_PASS:
> break;
The lines lines of this function, between the hunk above and the one below
looks like this:
case XDP_TX:
qidx += pfvf->hw.tx_queues;
cq->pool_ptrs++;
return otx2_xdp_sq_append_pkt(pfvf, iova,
The above uses iova, but in the case that we got here
via goto handle_xdp_verdict it is uninitialised.
...
> diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
> index e926c6ce96cf..9bb7e5c3e227 100644
> --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
> +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
> @@ -722,6 +722,10 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> if (err)
> goto err_shutdown_tc;
>
> + vf->af_xdp_zc_qidx = bitmap_zalloc(qcount, GFP_KERNEL);
> + if (!vf->af_xdp_zc_qidx)
> + goto err_shutdown_tc;
Along the same lines of my comment on otx2_probe():
should err be set to a negative error value here?
...
--
pw-bot: changes-requested
^ permalink raw reply [flat|nested] 9+ messages in thread* RE: [EXTERNAL] Re: [net-next PATCH 2/6] octeontx2-pf: AF_XDP zero copy receive support
2025-01-07 14:34 ` Simon Horman
@ 2025-01-08 14:02 ` Suman Ghosh
0 siblings, 0 replies; 9+ messages in thread
From: Suman Ghosh @ 2025-01-08 14:02 UTC (permalink / raw)
To: Simon Horman
Cc: Sunil Kovvuri Goutham, Geethasowjanya Akula,
Subbaraya Sundeep Bhatta, Hariprasad Kelam, davem@davemloft.net,
edumazet@google.com, kuba@kernel.org, pabeni@redhat.com,
netdev@vger.kernel.org, linux-kernel@vger.kernel.org,
Linu Cherian, Jerin Jacob
Hi Simon,
>> @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type)
>>
>> /* Free SQB and RQB pointers from the aura pool */
>> for (pool_id = pool_start; pool_id < pool_end; pool_id++) {
>> - iova = otx2_aura_allocptr(pfvf, pool_id);
>> pool = &pfvf->qset.pool[pool_id];
>> + iova = otx2_aura_allocptr(pfvf, pool_id);
>> while (iova) {
>> if (type == AURA_NIX_RQ)
>> iova -= OTX2_HEAD_ROOM;
>> @@ -1323,6 +1337,13 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf,
>int type)
>> iova = otx2_aura_allocptr(pfvf, pool_id);
>> }
>> }
>> +
>> + for (idx = 0 ; idx < pool->xdp_cnt; idx++) {
>> + if (!pool->xdp[idx])
>> + continue;
>> +
>> + xsk_buff_free(pool->xdp[idx]);
>> + }
>
>Looking over otx2_pool_init(), I am wondering if the loop above run
>should over all (non AURA_NIX_RQ) pools, rather than the last pool
>covered by the preceding loop.
[Suman] Yes, you are right. Thanks for catching this, will update in v2.
>
>This was flagged by Smatch, because it is concerned that pool may be
>used unset above, presumably if the preceding loop iterates zero times
>(I'm unsure if that can happen in practice).
[Suman] Will update the logic in v2.
>
>...
>
>> diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
>> b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
>
>...
>
>> @@ -3204,6 +3199,10 @@ static int otx2_probe(struct pci_dev *pdev,
>const struct pci_device_id *id)
>> /* Enable link notifications */
>> otx2_cgx_config_linkevents(pf, true);
>>
>> + pf->af_xdp_zc_qidx = bitmap_zalloc(qcount, GFP_KERNEL);
>> + if (!pf->af_xdp_zc_qidx)
>> + goto err_pf_sriov_init;
>> +
>
>This goto will result in the function returning err.
>However, here err is 0. Should it be set to a negative error value
>instead?
[Suman] Yes, will update in v2
>
>> #ifdef CONFIG_DCB
>> err = otx2_dcbnl_set_ops(netdev);
>> if (err)
>> diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
>> b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
>
>...
>
>> @@ -571,6 +574,7 @@ int otx2_napi_handler(struct napi_struct *napi,
>int budget)
>> if (pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED)
>> otx2_adjust_adaptive_coalese(pfvf, cq_poll);
>>
>> + pool = &pfvf->qset.pool[cq->cq_idx];
>
>I am also unsure if this can happen in practice, but Smatch is concerned
>that cq may be used uninitialised here. It does seem that could
>theoretically be the case if, in the for loop towards the top of this
>function, cq_poll->cq_ids[i] is always CINT_INVALID_CQ.
>
>> if (unlikely(!filled_cnt)) {
>> struct refill_work *work;
>> struct delayed_work *dwork;
>
>...
>
>> @@ -1426,13 +1440,24 @@ static bool otx2_xdp_rcv_pkt_handler(struct
>otx2_nic *pfvf,
>> unsigned char *hard_start;
>> struct otx2_pool *pool;
>> int qidx = cq->cq_idx;
>> - struct xdp_buff xdp;
>> + struct xdp_buff xdp, *xsk_buff = NULL;
>> struct page *page;
>> u64 iova, pa;
>> u32 act;
>> int err;
>>
>> pool = &pfvf->qset.pool[qidx];
>> +
>> + if (pool->xsk_pool) {
>> + xsk_buff = pool->xdp[--cq->rbpool->xdp_top];
>> + if (!xsk_buff)
>> + return false;
>> +
>> + xsk_buff->data_end = xsk_buff->data + cqe->sg.seg_size;
>> + act = bpf_prog_run_xdp(prog, xsk_buff);
>> + goto handle_xdp_verdict;
>
>iova is not initialised until a few lines further down, which does not
>occur in the case of this condition.
[Suman] I think the patch ordering is wrong.
"patch#4: octeontx2-pf: Don't unmap page pool buffer used by XDP" is fixing this. I will update the patch order.
>
>> + }
>> +
>> iova = cqe->sg.seg_addr - OTX2_HEAD_ROOM;
>> pa = otx2_iova_to_phys(pfvf->iommu_domain, iova);
>> page = virt_to_page(phys_to_virt(pa)); @@ -1445,6 +1470,7 @@ static
>> bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
>>
>> act = bpf_prog_run_xdp(prog, &xdp);
>>
>> +handle_xdp_verdict:
>> switch (act) {
>> case XDP_PASS:
>> break;
>
>The lines lines of this function, between the hunk above and the one
>below looks like this:
[Suman] replied above.
>
> case XDP_TX:
> qidx += pfvf->hw.tx_queues;
> cq->pool_ptrs++;
> return otx2_xdp_sq_append_pkt(pfvf, iova,
>
>The above uses iova, but in the case that we got here via goto
>handle_xdp_verdict it is uninitialised.
[Suman] same as above.
>
>...
>
>> diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
>> b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
>> index e926c6ce96cf..9bb7e5c3e227 100644
>> --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
>> +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
>> @@ -722,6 +722,10 @@ static int otx2vf_probe(struct pci_dev *pdev,
>const struct pci_device_id *id)
>> if (err)
>> goto err_shutdown_tc;
>>
>> + vf->af_xdp_zc_qidx = bitmap_zalloc(qcount, GFP_KERNEL);
>> + if (!vf->af_xdp_zc_qidx)
>> + goto err_shutdown_tc;
>
>Along the same lines of my comment on otx2_probe():
>should err be set to a negative error value here?
[Suman] yes, will update in v2
>
>...
>
>--
>pw-bot: changes-requested
^ permalink raw reply [flat|nested] 9+ messages in thread
* [net-next PATCH 3/6] octeontx2-pf: Reconfigure RSS table after enabling AF_XDP zerocopy on rx queue
2025-01-07 10:46 [net-next PATCH 0/6] Add af_xdp support for cn10k Suman Ghosh
2025-01-07 10:46 ` [net-next PATCH 1/6] octeontx2-pf: Add AF_XDP non-zero copy support Suman Ghosh
2025-01-07 10:46 ` [net-next PATCH 2/6] octeontx2-pf: AF_XDP zero copy receive support Suman Ghosh
@ 2025-01-07 10:46 ` Suman Ghosh
2025-01-07 10:46 ` [net-next PATCH 4/6] octeontx2-pf: Don't unmap page pool buffer used by XDP Suman Ghosh
` (2 subsequent siblings)
5 siblings, 0 replies; 9+ messages in thread
From: Suman Ghosh @ 2025-01-07 10:46 UTC (permalink / raw)
To: sgoutham, gakula, sbhatta, hkelam, davem, edumazet, kuba, pabeni,
netdev, linux-kernel, lcherian, jerinj
Cc: Suman Ghosh
RSS table needs to be reconfigured once a rx queue is enabled or
disabled for AF_XDP zerocopy support. After enabling UMEM on a rx queue,
that queue should not be part of RSS queue selection algorithm.
Similarly the queue should be considered again after UMEM is disabled.
Signed-off-by: Suman Ghosh <sumang@marvell.com>
---
drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c | 4 ++++
drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c | 6 +++++-
drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c | 4 ++++
3 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
index d653e0136cf4..9d7c0f0b0b56 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
@@ -331,6 +331,10 @@ int otx2_set_rss_table(struct otx2_nic *pfvf, int ctx_id)
rss_ctx = rss->rss_ctx[ctx_id];
/* Get memory to put this msg */
for (idx = 0; idx < rss->rss_size; idx++) {
+ /* Ignore the queue if AF_XDP zero copy is enabled */
+ if (test_bit(rss_ctx->ind_tbl[idx], pfvf->af_xdp_zc_qidx))
+ continue;
+
aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox);
if (!aq) {
/* The shared memory buffer can be full.
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c
index 2d53dc77ef1e..010385b29988 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c
@@ -910,8 +910,12 @@ static int otx2_get_rxfh(struct net_device *dev,
return -ENOENT;
if (indir) {
- for (idx = 0; idx < rss->rss_size; idx++)
+ for (idx = 0; idx < rss->rss_size; idx++) {
+ /* Ignore if the rx queue is AF_XDP zero copy enabled */
+ if (test_bit(rss_ctx->ind_tbl[idx], pfvf->af_xdp_zc_qidx))
+ continue;
indir[idx] = rss_ctx->ind_tbl[idx];
+ }
}
if (rxfh->key)
memcpy(rxfh->key, rss->key, sizeof(rss->key));
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c
index 69098c6a6fed..f6bbe18016ba 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c
@@ -130,6 +130,8 @@ int otx2_xsk_pool_enable(struct otx2_nic *pf, struct xsk_buff_pool *pool, u16 qi
set_bit(qidx, pf->af_xdp_zc_qidx);
otx2_clean_up_rq(pf, qidx);
+ /* Reconfigure RSS table as 'qidx' cannot be part of RSS now */
+ otx2_set_rss_table(pf, DEFAULT_RSS_CONTEXT_GROUP);
/* Kick start the NAPI context so that receiving will start */
return otx2_xsk_wakeup(pf->netdev, qidx, XDP_WAKEUP_RX);
}
@@ -146,6 +148,8 @@ int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qidx)
otx2_clean_up_rq(pf, qidx);
clear_bit(qidx, pf->af_xdp_zc_qidx);
xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
+ /* Reconfigure RSS table as 'qidx' now need to be part of RSS now */
+ otx2_set_rss_table(pf, DEFAULT_RSS_CONTEXT_GROUP);
return 0;
}
--
2.25.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [net-next PATCH 4/6] octeontx2-pf: Don't unmap page pool buffer used by XDP
2025-01-07 10:46 [net-next PATCH 0/6] Add af_xdp support for cn10k Suman Ghosh
` (2 preceding siblings ...)
2025-01-07 10:46 ` [net-next PATCH 3/6] octeontx2-pf: Reconfigure RSS table after enabling AF_XDP zerocopy on rx queue Suman Ghosh
@ 2025-01-07 10:46 ` Suman Ghosh
2025-01-07 10:46 ` [net-next PATCH 5/6] octeontx2-pf: Prepare for AF_XDP transmit Suman Ghosh
2025-01-07 10:46 ` [net-next PATCH 6/6] octeontx2-pf: AF_XDP zero copy transmit support Suman Ghosh
5 siblings, 0 replies; 9+ messages in thread
From: Suman Ghosh @ 2025-01-07 10:46 UTC (permalink / raw)
To: sgoutham, gakula, sbhatta, hkelam, davem, edumazet, kuba, pabeni,
netdev, linux-kernel, lcherian, jerinj
Cc: Suman Ghosh
From: Geetha sowjanya <gakula@marvell.com>
When xdp buffers are from page pool do not dma unmap
the buffers. DMA map/unmap are handled by the page_pool
APIs.
Signed-off-by: Geetha sowjanya <gakula@marvell.com>
Signed-off-by: Suman Ghosh <sumang@marvell.com>
---
.../marvell/octeontx2/nic/otx2_common.h | 3 +-
.../ethernet/marvell/octeontx2/nic/otx2_pf.c | 8 +++--
.../marvell/octeontx2/nic/otx2_txrx.c | 32 ++++++++++++-------
.../marvell/octeontx2/nic/otx2_txrx.h | 1 +
4 files changed, 30 insertions(+), 14 deletions(-)
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
index 40ee21ae6c72..dc68d2aa0a0e 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
@@ -1104,7 +1104,8 @@ int otx2_del_macfilter(struct net_device *netdev, const u8 *mac);
int otx2_add_macfilter(struct net_device *netdev, const u8 *mac);
int otx2_enable_rxvlan(struct otx2_nic *pf, bool enable);
int otx2_install_rxvlan_offload_flow(struct otx2_nic *pfvf);
-bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len, u16 qidx);
+bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len,
+ u16 qidx, u16 flags);
u16 otx2_get_max_mtu(struct otx2_nic *pfvf);
int otx2_handle_ntuple_tc_features(struct net_device *netdev,
netdev_features_t features);
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
index e10c7841fd8b..c6205cf2eb03 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
@@ -2693,11 +2693,15 @@ static int otx2_xdp_xmit_tx(struct otx2_nic *pf, struct xdp_frame *xdpf,
if (dma_mapping_error(pf->dev, dma_addr))
return -ENOMEM;
- err = otx2_xdp_sq_append_pkt(pf, dma_addr, xdpf->len, qidx);
+ err = otx2_xdp_sq_append_pkt(pf, dma_addr, xdpf->len,
+ qidx, XDP_REDIRECT);
if (!err) {
otx2_dma_unmap_page(pf, dma_addr, xdpf->len, DMA_TO_DEVICE);
page = virt_to_page(xdpf->data);
- put_page(page);
+ if (page->pp)
+ page_pool_recycle_direct(page->pp, page);
+ else
+ put_page(page);
return -ENOMEM;
}
return 0;
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
index 741b4ce6d0ff..c8b11d45debf 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
@@ -102,19 +102,20 @@ static void otx2_xdp_snd_pkt_handler(struct otx2_nic *pfvf,
struct nix_send_comp_s *snd_comp = &cqe->comp;
struct sg_list *sg;
struct page *page;
- u64 pa;
+ u64 pa, iova;
sg = &sq->sg[snd_comp->sqe_id];
- pa = otx2_iova_to_phys(pfvf->iommu_domain, sg->dma_addr[0]);
- otx2_dma_unmap_page(pfvf, sg->dma_addr[0],
- sg->size[0], DMA_TO_DEVICE);
+ iova = sg->dma_addr[0] - OTX2_HEAD_ROOM;
+ pa = otx2_iova_to_phys(pfvf->iommu_domain, iova);
page = virt_to_page(phys_to_virt(pa));
+ if (sg->flags & XDP_REDIRECT)
+ otx2_dma_unmap_page(pfvf, sg->dma_addr[0], sg->size[0], DMA_TO_DEVICE);
+
if (page->pp) {
page_pool_recycle_direct(page->pp, page);
return;
}
-
put_page(page);
}
@@ -1379,7 +1380,7 @@ void otx2_free_pending_sqe(struct otx2_nic *pfvf)
}
static void otx2_xdp_sqe_add_sg(struct otx2_snd_queue *sq, u64 dma_addr,
- int len, int *offset)
+ int len, int *offset, u16 flags)
{
struct nix_sqe_sg_s *sg = NULL;
u64 *iova = NULL;
@@ -1396,9 +1397,11 @@ static void otx2_xdp_sqe_add_sg(struct otx2_snd_queue *sq, u64 dma_addr,
sq->sg[sq->head].dma_addr[0] = dma_addr;
sq->sg[sq->head].size[0] = len;
sq->sg[sq->head].num_segs = 1;
+ sq->sg[sq->head].flags = flags;
}
-bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len, u16 qidx)
+bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len,
+ u16 qidx, u16 flags)
{
struct nix_sqe_hdr_s *sqe_hdr;
struct otx2_snd_queue *sq;
@@ -1424,7 +1427,7 @@ bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len, u16 qidx)
offset = sizeof(*sqe_hdr);
- otx2_xdp_sqe_add_sg(sq, iova, len, &offset);
+ otx2_xdp_sqe_add_sg(sq, iova, len, &offset, flags);
sqe_hdr->sizem1 = (offset / 16) - 1;
pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);
@@ -1477,8 +1480,8 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
case XDP_TX:
qidx += pfvf->hw.tx_queues;
cq->pool_ptrs++;
- return otx2_xdp_sq_append_pkt(pfvf, iova,
- cqe->sg.seg_size, qidx);
+ return otx2_xdp_sq_append_pkt(pfvf, cqe->sg.seg_addr,
+ cqe->sg.seg_size, qidx, XDP_TX);
case XDP_REDIRECT:
cq->pool_ptrs++;
if (xsk_buff) {
@@ -1497,7 +1500,14 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
*need_xdp_flush = true;
return true;
}
- page_pool_recycle_direct(pool->page_pool, page);
+ if (page->pp) {
+ page_pool_recycle_direct(pool->page_pool, page);
+ return false;
+ }
+
+ otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize,
+ DMA_FROM_DEVICE);
+ put_page(page);
break;
default:
bpf_warn_invalid_xdp_action(pfvf->netdev, prog, act);
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
index 401a4eb04441..8f346fbc8221 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
@@ -77,6 +77,7 @@ struct otx2_rcv_queue {
struct sg_list {
u16 num_segs;
+ u16 flags;
u64 skb;
u64 size[OTX2_MAX_FRAGS_IN_SQE];
u64 dma_addr[OTX2_MAX_FRAGS_IN_SQE];
--
2.25.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [net-next PATCH 5/6] octeontx2-pf: Prepare for AF_XDP transmit
2025-01-07 10:46 [net-next PATCH 0/6] Add af_xdp support for cn10k Suman Ghosh
` (3 preceding siblings ...)
2025-01-07 10:46 ` [net-next PATCH 4/6] octeontx2-pf: Don't unmap page pool buffer used by XDP Suman Ghosh
@ 2025-01-07 10:46 ` Suman Ghosh
2025-01-07 10:46 ` [net-next PATCH 6/6] octeontx2-pf: AF_XDP zero copy transmit support Suman Ghosh
5 siblings, 0 replies; 9+ messages in thread
From: Suman Ghosh @ 2025-01-07 10:46 UTC (permalink / raw)
To: sgoutham, gakula, sbhatta, hkelam, davem, edumazet, kuba, pabeni,
netdev, linux-kernel, lcherian, jerinj
Cc: Suman Ghosh
From: Hariprasad Kelam <hkelam@marvell.com>
Implement necessary APIs required for AF_XDP transmit.
Signed-off-by: Hariprasad Kelam <hkelam@marvell.com>
Signed-off-by: Suman Ghosh <sumang@marvell.com>
---
.../marvell/octeontx2/nic/otx2_common.h | 1 +
.../marvell/octeontx2/nic/otx2_txrx.c | 25 +++++++++++++++++--
2 files changed, 24 insertions(+), 2 deletions(-)
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
index dc68d2aa0a0e..24faf9dc41e6 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
@@ -1186,4 +1186,5 @@ static inline int mcam_entry_cmp(const void *a, const void *b)
dma_addr_t otx2_dma_map_skb_frag(struct otx2_nic *pfvf,
struct sk_buff *skb, int seg, int *len);
void otx2_dma_unmap_skb_frags(struct otx2_nic *pfvf, struct sg_list *sg);
+int otx2_read_free_sqe(struct otx2_nic *pfvf, u16 qidx);
#endif /* OTX2_COMMON_H */
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
index c8b11d45debf..80769c8ffb9a 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
@@ -22,6 +22,12 @@
#include "cn10k.h"
#define CQE_ADDR(CQ, idx) ((CQ)->cqe_base + ((CQ)->cqe_size * (idx)))
+#define READ_FREE_SQE(SQ, free_sqe) \
+ do { \
+ typeof(SQ) _SQ = (SQ); \
+ free_sqe = (((_SQ)->cons_head - (_SQ)->head - 1 + (_SQ)->sqe_cnt) \
+ & ((_SQ)->sqe_cnt - 1)); \
+ } while (0)
#define PTP_PORT 0x13F
/* PTPv2 header Original Timestamp starts at byte offset 34 and
* contains 6 byte seconds field and 4 byte nano seconds field.
@@ -1163,7 +1169,7 @@ bool otx2_sq_append_skb(void *dev, struct netdev_queue *txq,
/* Check if there is enough room between producer
* and consumer index.
*/
- free_desc = (sq->cons_head - sq->head - 1 + sq->sqe_cnt) & (sq->sqe_cnt - 1);
+ READ_FREE_SQE(sq, free_desc);
if (free_desc < sq->sqe_thresh)
return false;
@@ -1400,6 +1406,21 @@ static void otx2_xdp_sqe_add_sg(struct otx2_snd_queue *sq, u64 dma_addr,
sq->sg[sq->head].flags = flags;
}
+int otx2_read_free_sqe(struct otx2_nic *pfvf, u16 qidx)
+{
+ struct otx2_snd_queue *sq;
+ int free_sqe;
+
+ sq = &pfvf->qset.sq[qidx];
+ READ_FREE_SQE(sq, free_sqe);
+ if (free_sqe < sq->sqe_thresh) {
+ netdev_warn(pfvf->netdev, "No free sqe for Send queue%d\n", qidx);
+ return 0;
+ }
+
+ return free_sqe - sq->sqe_thresh;
+}
+
bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len,
u16 qidx, u16 flags)
{
@@ -1408,7 +1429,7 @@ bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len,
int offset, free_sqe;
sq = &pfvf->qset.sq[qidx];
- free_sqe = (sq->num_sqbs - *sq->aura_fc_addr) * sq->sqe_per_sqb;
+ READ_FREE_SQE(sq, free_sqe);
if (free_sqe < sq->sqe_thresh)
return false;
--
2.25.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [net-next PATCH 6/6] octeontx2-pf: AF_XDP zero copy transmit support
2025-01-07 10:46 [net-next PATCH 0/6] Add af_xdp support for cn10k Suman Ghosh
` (4 preceding siblings ...)
2025-01-07 10:46 ` [net-next PATCH 5/6] octeontx2-pf: Prepare for AF_XDP transmit Suman Ghosh
@ 2025-01-07 10:46 ` Suman Ghosh
5 siblings, 0 replies; 9+ messages in thread
From: Suman Ghosh @ 2025-01-07 10:46 UTC (permalink / raw)
To: sgoutham, gakula, sbhatta, hkelam, davem, edumazet, kuba, pabeni,
netdev, linux-kernel, lcherian, jerinj
Cc: Suman Ghosh
From: Hariprasad Kelam <hkelam@marvell.com>
This patch implements below changes,
1. To avoid concurrency with normal traffic uses
XDP queues.
2. Since there are chances that XDP and AF_XDP can
fall under same queue uses separate flags to handle
dma buffers.
Signed-off-by: Hariprasad Kelam <hkelam@marvell.com>
Signed-off-by: Suman Ghosh <sumang@marvell.com>
---
.../marvell/octeontx2/nic/otx2_common.c | 4 ++
.../marvell/octeontx2/nic/otx2_common.h | 6 +++
.../ethernet/marvell/octeontx2/nic/otx2_pf.c | 2 +-
.../marvell/octeontx2/nic/otx2_txrx.c | 45 +++++++++++++++----
.../marvell/octeontx2/nic/otx2_txrx.h | 1 +
.../ethernet/marvell/octeontx2/nic/otx2_xsk.c | 43 +++++++++++++++++-
.../ethernet/marvell/octeontx2/nic/otx2_xsk.h | 3 ++
7 files changed, 92 insertions(+), 12 deletions(-)
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
index 9d7c0f0b0b56..c1fdb8b943cd 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
@@ -1037,6 +1037,10 @@ int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura)
sq->stats.bytes = 0;
sq->stats.pkts = 0;
+ /* Attach XSK_BUFF_POOL to XDP queue */
+ if (qidx > pfvf->hw.xdp_queues)
+ otx2_attach_xsk_buff(pfvf, sq, (qidx - pfvf->hw.xdp_queues));
+
chan_offset = qidx % pfvf->hw.tx_chan_cnt;
err = pfvf->hw_ops->sq_aq_init(pfvf, qidx, chan_offset, sqb_aura);
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
index 24faf9dc41e6..69a8c6132cd3 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
@@ -129,6 +129,12 @@ enum otx2_errcodes_re {
ERRCODE_IL4_CSUM = 0x22,
};
+enum otx2_xdp_action {
+ OTX2_XDP_TX = BIT(0),
+ OTX2_XDP_REDIRECT = BIT(1),
+ OTX2_AF_XDP_FRAME = BIT(2),
+};
+
struct otx2_dev_stats {
u64 rx_bytes;
u64 rx_frames;
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
index c6205cf2eb03..15bbd7e39eb0 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
@@ -2694,7 +2694,7 @@ static int otx2_xdp_xmit_tx(struct otx2_nic *pf, struct xdp_frame *xdpf,
return -ENOMEM;
err = otx2_xdp_sq_append_pkt(pf, dma_addr, xdpf->len,
- qidx, XDP_REDIRECT);
+ qidx, OTX2_XDP_REDIRECT);
if (!err) {
otx2_dma_unmap_page(pf, dma_addr, xdpf->len, DMA_TO_DEVICE);
page = virt_to_page(xdpf->data);
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
index 80769c8ffb9a..03446e8055be 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
@@ -20,6 +20,7 @@
#include "otx2_txrx.h"
#include "otx2_ptp.h"
#include "cn10k.h"
+#include "otx2_xsk.h"
#define CQE_ADDR(CQ, idx) ((CQ)->cqe_base + ((CQ)->cqe_size * (idx)))
#define READ_FREE_SQE(SQ, free_sqe) \
@@ -103,7 +104,8 @@ static unsigned int frag_num(unsigned int i)
static void otx2_xdp_snd_pkt_handler(struct otx2_nic *pfvf,
struct otx2_snd_queue *sq,
- struct nix_cqe_tx_s *cqe)
+ struct nix_cqe_tx_s *cqe,
+ int *xsk_frames)
{
struct nix_send_comp_s *snd_comp = &cqe->comp;
struct sg_list *sg;
@@ -112,10 +114,15 @@ static void otx2_xdp_snd_pkt_handler(struct otx2_nic *pfvf,
sg = &sq->sg[snd_comp->sqe_id];
+ if (sg->flags & OTX2_AF_XDP_FRAME) {
+ (*xsk_frames)++;
+ return;
+ }
+
iova = sg->dma_addr[0] - OTX2_HEAD_ROOM;
pa = otx2_iova_to_phys(pfvf->iommu_domain, iova);
page = virt_to_page(phys_to_virt(pa));
- if (sg->flags & XDP_REDIRECT)
+ if (sg->flags & OTX2_XDP_REDIRECT)
otx2_dma_unmap_page(pfvf, sg->dma_addr[0], sg->size[0], DMA_TO_DEVICE);
if (page->pp) {
@@ -444,6 +451,18 @@ int otx2_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
return cnt - cq->pool_ptrs;
}
+static void otx2_zc_submit_pkts(struct otx2_nic *pfvf, struct xsk_buff_pool *xsk_pool,
+ int *xsk_frames, int qidx, int budget)
+{
+ if (*xsk_frames)
+ xsk_tx_completed(xsk_pool, *xsk_frames);
+
+ if (xsk_uses_need_wakeup(xsk_pool))
+ xsk_set_tx_need_wakeup(xsk_pool);
+
+ otx2_zc_napi_handler(pfvf, xsk_pool, qidx, budget);
+}
+
static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
struct otx2_cq_queue *cq, int budget)
{
@@ -452,16 +471,22 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
struct nix_cqe_tx_s *cqe;
struct net_device *ndev;
int processed_cqe = 0;
+ int xsk_frames = 0;
+
+ qidx = cq->cq_idx - pfvf->hw.rx_queues;
+ sq = &pfvf->qset.sq[qidx];
if (cq->pend_cqe >= budget)
goto process_cqe;
- if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
+ if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe) {
+ if (sq->xsk_pool)
+ otx2_zc_submit_pkts(pfvf, sq->xsk_pool, &xsk_frames,
+ qidx, budget);
return 0;
+ }
process_cqe:
- qidx = cq->cq_idx - pfvf->hw.rx_queues;
- sq = &pfvf->qset.sq[qidx];
while (likely(processed_cqe < budget) && cq->pend_cqe) {
cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq);
@@ -471,10 +496,8 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
break;
}
- qidx = cq->cq_idx - pfvf->hw.rx_queues;
-
if (cq->cq_type == CQ_XDP)
- otx2_xdp_snd_pkt_handler(pfvf, sq, cqe);
+ otx2_xdp_snd_pkt_handler(pfvf, sq, cqe, &xsk_frames);
else
otx2_snd_pkt_handler(pfvf, cq, &pfvf->qset.sq[qidx],
cqe, budget, &tx_pkts, &tx_bytes);
@@ -515,6 +538,10 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
netif_carrier_ok(ndev))
netif_tx_wake_queue(txq);
}
+
+ if (sq->xsk_pool)
+ otx2_zc_submit_pkts(pfvf, sq->xsk_pool, &xsk_frames, qidx, budget);
+
return 0;
}
@@ -1502,7 +1529,7 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
qidx += pfvf->hw.tx_queues;
cq->pool_ptrs++;
return otx2_xdp_sq_append_pkt(pfvf, cqe->sg.seg_addr,
- cqe->sg.seg_size, qidx, XDP_TX);
+ cqe->sg.seg_size, qidx, OTX2_XDP_TX);
case XDP_REDIRECT:
cq->pool_ptrs++;
if (xsk_buff) {
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
index 8f346fbc8221..2fd8b768e8c7 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
@@ -106,6 +106,7 @@ struct otx2_snd_queue {
/* SQE ring and CPT response queue for Inline IPSEC */
struct qmem *sqe_ring;
struct qmem *cpt_resp;
+ struct xsk_buff_pool *xsk_pool;
} ____cacheline_aligned_in_smp;
enum cq_type {
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c
index f6bbe18016ba..64035d0689de 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c
@@ -140,11 +140,14 @@ int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qidx)
{
struct net_device *netdev = pf->netdev;
struct xsk_buff_pool *pool;
+ struct otx2_snd_queue *sq;
pool = xsk_get_pool_from_qid(netdev, qidx);
if (!pool)
return -EINVAL;
+ sq = &pf->qset.sq[qidx + pf->hw.tx_queues];
+ sq->xsk_pool = NULL;
otx2_clean_up_rq(pf, qidx);
clear_bit(qidx, pf->af_xdp_zc_qidx);
xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
@@ -171,7 +174,7 @@ int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
if (pf->flags & OTX2_FLAG_INTF_DOWN)
return -ENETDOWN;
- if (queue_id >= pf->hw.rx_queues)
+ if (queue_id >= pf->hw.rx_queues || queue_id >= pf->hw.tx_queues)
return -EINVAL;
cq_poll = &qset->napi[queue_id];
@@ -179,8 +182,44 @@ int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
return -EINVAL;
/* Trigger interrupt */
- if (!napi_if_scheduled_mark_missed(&cq_poll->napi))
+ if (!napi_if_scheduled_mark_missed(&cq_poll->napi)) {
otx2_write64(pf, NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx), BIT_ULL(0));
+ otx2_write64(pf, NIX_LF_CINTX_INT_W1S(cq_poll->cint_idx), BIT_ULL(0));
+ }
return 0;
}
+
+void otx2_attach_xsk_buff(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, int qidx)
+{
+ if (test_bit(qidx, pfvf->af_xdp_zc_qidx))
+ sq->xsk_pool = xsk_get_pool_from_qid(pfvf->netdev, qidx);
+}
+
+void otx2_zc_napi_handler(struct otx2_nic *pfvf, struct xsk_buff_pool *pool,
+ int queue, int budget)
+{
+ struct xdp_desc *xdp_desc = pool->tx_descs;
+ int err, i, work_done = 0, batch;
+
+ budget = min(budget, otx2_read_free_sqe(pfvf, queue));
+ batch = xsk_tx_peek_release_desc_batch(pool, budget);
+ if (!batch)
+ return;
+
+ for (i = 0; i < batch; i++) {
+ dma_addr_t dma_addr;
+
+ dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc[i].addr);
+ err = otx2_xdp_sq_append_pkt(pfvf, dma_addr, xdp_desc[i].len,
+ queue, OTX2_AF_XDP_FRAME);
+ if (!err) {
+ netdev_err(pfvf->netdev, "AF_XDP: Unable to transfer packet err%d\n", err);
+ break;
+ }
+ work_done++;
+ }
+
+ if (work_done)
+ xsk_tx_release(pool);
+}
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h
index 022b3433edbb..8047fafee8fe 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h
@@ -17,5 +17,8 @@ int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qid);
int otx2_xsk_pool_alloc_buf(struct otx2_nic *pfvf, struct otx2_pool *pool,
dma_addr_t *dma, int idx);
int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags);
+void otx2_zc_napi_handler(struct otx2_nic *pfvf, struct xsk_buff_pool *pool,
+ int queue, int budget);
+void otx2_attach_xsk_buff(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, int qidx);
#endif /* OTX2_XSK_H */
--
2.25.1
^ permalink raw reply related [flat|nested] 9+ messages in thread