[net-next, PATCH 0/2, v2] net: socionext: add XDP support

[net-next, PATCH 0/2, v2] net: socionext: add XDP support

[Ilias Apalodimas]

This patch series adds AF_XDP support socionext netsec driver
In addition the new dma allocation scheme offers a 10% boost on Rx
pps rate using 64b packets

- patch [1/2]: Use a different allocation scheme for Rx DMA buffers to prepare
the driver for AF_XDP support
- patch [2/2]: Add XDP support without zero-copy

Changes since v1:
- patch [2/2]: 
	- Rephrased commit message
	- As pointed out by Toshiaki Makita: 
	 - Added XDP_PACKET_HEADROOM
	 - Fixed a bug on XDP_PASS case as pointed out by Toshiaki Makita
	 - Using extact for error messaging instead of netdev_warn, when 
	tryint to setup XDP as suggested by Toshiaki Makita

Ilias Apalodimas (2):
  net: socionext: different approach on DMA
  net: socionext: add XDP support

 drivers/net/ethernet/socionext/netsec.c | 449 ++++++++++++++++++++++++--------
 1 file changed, 334 insertions(+), 115 deletions(-)

-- 
2.7.4

[net-next, PATCH 1/2, v2] net: socionext: different approach on DMA

[Ilias Apalodimas]

Current driver dynamically allocates an skb and maps it as DMA rx buffer.
A following patch introduces AF_XDP functionality, so we need a
different allocation scheme. Buffers are allocated dynamically and
mapped into hardware. During the Rx operation the driver uses
build_skb() to produce the necessary buffers for the network stack

Signed-off-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>
---
 drivers/net/ethernet/socionext/netsec.c | 239 +++++++++++++++++---------------
 1 file changed, 130 insertions(+), 109 deletions(-)

diff --git a/drivers/net/ethernet/socionext/netsec.c b/drivers/net/ethernet/socionext/netsec.c
index 7aa5ebb..666fee2 100644
--- a/drivers/net/ethernet/socionext/netsec.c
+++ b/drivers/net/ethernet/socionext/netsec.c
@@ -296,6 +296,11 @@ struct netsec_rx_pkt_info {
 	bool err_flag;
 };
 
+static void netsec_rx_fill(struct netsec_priv *priv, u16 from, u16 num);
+
+static void *netsec_alloc_rx_data(struct netsec_priv *priv,
+				  dma_addr_t *dma_addr, u16 *len);
+
 static void netsec_write(struct netsec_priv *priv, u32 reg_addr, u32 val)
 {
 	writel(val, priv->ioaddr + reg_addr);
@@ -556,34 +561,10 @@ static const struct ethtool_ops netsec_ethtool_ops = {
 
 /************* NETDEV_OPS FOLLOW *************/
 
-static struct sk_buff *netsec_alloc_skb(struct netsec_priv *priv,
-					struct netsec_desc *desc)
-{
-	struct sk_buff *skb;
-
-	if (device_get_dma_attr(priv->dev) == DEV_DMA_COHERENT) {
-		skb = netdev_alloc_skb_ip_align(priv->ndev, desc->len);
-	} else {
-		desc->len = L1_CACHE_ALIGN(desc->len);
-		skb = netdev_alloc_skb(priv->ndev, desc->len);
-	}
-	if (!skb)
-		return NULL;
-
-	desc->addr = skb->data;
-	desc->dma_addr = dma_map_single(priv->dev, desc->addr, desc->len,
-					DMA_FROM_DEVICE);
-	if (dma_mapping_error(priv->dev, desc->dma_addr)) {
-		dev_kfree_skb_any(skb);
-		return NULL;
-	}
-	return skb;
-}
 
 static void netsec_set_rx_de(struct netsec_priv *priv,
 			     struct netsec_desc_ring *dring, u16 idx,
-			     const struct netsec_desc *desc,
-			     struct sk_buff *skb)
+			     const struct netsec_desc *desc)
 {
 	struct netsec_de *de = dring->vaddr + DESC_SZ * idx;
 	u32 attr = (1 << NETSEC_RX_PKT_OWN_FIELD) |
@@ -602,59 +583,6 @@ static void netsec_set_rx_de(struct netsec_priv *priv,
 	dring->desc[idx].dma_addr = desc->dma_addr;
 	dring->desc[idx].addr = desc->addr;
 	dring->desc[idx].len = desc->len;
-	dring->desc[idx].skb = skb;
-}
-
-static struct sk_buff *netsec_get_rx_de(struct netsec_priv *priv,
-					struct netsec_desc_ring *dring,
-					u16 idx,
-					struct netsec_rx_pkt_info *rxpi,
-					struct netsec_desc *desc, u16 *len)
-{
-	struct netsec_de de = {};
-
-	memcpy(&de, dring->vaddr + DESC_SZ * idx, DESC_SZ);
-
-	*len = de.buf_len_info >> 16;
-
-	rxpi->err_flag = (de.attr >> NETSEC_RX_PKT_ER_FIELD) & 1;
-	rxpi->rx_cksum_result = (de.attr >> NETSEC_RX_PKT_CO_FIELD) & 3;
-	rxpi->err_code = (de.attr >> NETSEC_RX_PKT_ERR_FIELD) &
-							NETSEC_RX_PKT_ERR_MASK;
-	*desc = dring->desc[idx];
-	return desc->skb;
-}
-
-static struct sk_buff *netsec_get_rx_pkt_data(struct netsec_priv *priv,
-					      struct netsec_rx_pkt_info *rxpi,
-					      struct netsec_desc *desc,
-					      u16 *len)
-{
-	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
-	struct sk_buff *tmp_skb, *skb = NULL;
-	struct netsec_desc td;
-	int tail;
-
-	*rxpi = (struct netsec_rx_pkt_info){};
-
-	td.len = priv->ndev->mtu + 22;
-
-	tmp_skb = netsec_alloc_skb(priv, &td);
-
-	tail = dring->tail;
-
-	if (!tmp_skb) {
-		netsec_set_rx_de(priv, dring, tail, &dring->desc[tail],
-				 dring->desc[tail].skb);
-	} else {
-		skb = netsec_get_rx_de(priv, dring, tail, rxpi, desc, len);
-		netsec_set_rx_de(priv, dring, tail, &td, tmp_skb);
-	}
-
-	/* move tail ahead */
-	dring->tail = (dring->tail + 1) % DESC_NUM;
-
-	return skb;
 }
 
 static int netsec_clean_tx_dring(struct netsec_priv *priv, int budget)
@@ -721,19 +649,29 @@ static int netsec_process_tx(struct netsec_priv *priv, int budget)
 	return done;
 }
 
+static void nsetsec_adv_desc(u16 *idx)
+{
+	*idx = *idx + 1;
+	if (unlikely(*idx >= DESC_NUM))
+		*idx = 0;
+}
+
 static int netsec_process_rx(struct netsec_priv *priv, int budget)
 {
 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
 	struct net_device *ndev = priv->ndev;
-	struct netsec_rx_pkt_info rx_info;
-	int done = 0;
-	struct netsec_desc desc;
 	struct sk_buff *skb;
-	u16 len;
+	int done = 0;
 
 	while (done < budget) {
 		u16 idx = dring->tail;
 		struct netsec_de *de = dring->vaddr + (DESC_SZ * idx);
+		struct netsec_desc *desc = &dring->desc[idx];
+		struct netsec_rx_pkt_info rpi;
+		dma_addr_t dma_handle;
+		void *buf_addr;
+		u16 pkt_len;
+		u16 desc_len;
 
 		if (de->attr & (1U << NETSEC_RX_PKT_OWN_FIELD))
 			break;
@@ -744,28 +682,62 @@ static int netsec_process_rx(struct netsec_priv *priv, int budget)
 		 */
 		dma_rmb();
 		done++;
-		skb = netsec_get_rx_pkt_data(priv, &rx_info, &desc, &len);
-		if (unlikely(!skb) || rx_info.err_flag) {
+
+		pkt_len = de->buf_len_info >> 16;
+		rpi.err_code = (de->attr >> NETSEC_RX_PKT_ERR_FIELD) &
+			NETSEC_RX_PKT_ERR_MASK;
+		rpi.err_flag = (de->attr >> NETSEC_RX_PKT_ER_FIELD) & 1;
+		if (rpi.err_flag) {
 			netif_err(priv, drv, priv->ndev,
-				  "%s: rx fail err(%d)\n",
-				  __func__, rx_info.err_code);
+				  "%s: rx fail err(%d)\n", __func__,
+				  rpi.err_code);
 			ndev->stats.rx_dropped++;
+			nsetsec_adv_desc(&dring->tail);
+			/* reuse buffer page frag */
+			netsec_rx_fill(priv, idx, 1);
 			continue;
 		}
+		rpi.rx_cksum_result = (de->attr >> NETSEC_RX_PKT_CO_FIELD) & 3;
 
-		dma_unmap_single(priv->dev, desc.dma_addr, desc.len,
-				 DMA_FROM_DEVICE);
-		skb_put(skb, len);
+		dma_sync_single_for_cpu(priv->dev, desc->dma_addr, pkt_len,
+					DMA_FROM_DEVICE);
+
+		prefetch(desc->addr);
+		buf_addr = netsec_alloc_rx_data(priv, &dma_handle, &desc_len);
+		if (unlikely(!buf_addr))
+			break;
+
+		skb = build_skb(desc->addr, desc->len);
+		if (unlikely(!skb)) {
+			dma_unmap_single(priv->dev, dma_handle, desc_len,
+					 DMA_TO_DEVICE);
+			skb_free_frag(buf_addr);
+			netif_err(priv, drv, priv->ndev,
+				  "rx failed to alloc skb\n");
+			break;
+		}
+		dma_unmap_single_attrs(priv->dev, desc->dma_addr, desc->len,
+				       DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
+
+		/* Update the descriptor with fresh buffers */
+		desc->len = desc_len;
+		desc->dma_addr = dma_handle;
+		desc->addr = buf_addr;
+
+		skb_put(skb, pkt_len);
 		skb->protocol = eth_type_trans(skb, priv->ndev);
 
 		if (priv->rx_cksum_offload_flag &&
-		    rx_info.rx_cksum_result == NETSEC_RX_CKSUM_OK)
+		    rpi.rx_cksum_result == NETSEC_RX_CKSUM_OK)
 			skb->ip_summed = CHECKSUM_UNNECESSARY;
 
 		if (napi_gro_receive(&priv->napi, skb) != GRO_DROP) {
 			ndev->stats.rx_packets++;
-			ndev->stats.rx_bytes += len;
+			ndev->stats.rx_bytes += pkt_len;
 		}
+
+		netsec_rx_fill(priv, idx, 1);
+		nsetsec_adv_desc(&dring->tail);
 	}
 
 	return done;
@@ -928,7 +900,10 @@ static void netsec_uninit_pkt_dring(struct netsec_priv *priv, int id)
 		dma_unmap_single(priv->dev, desc->dma_addr, desc->len,
 				 id == NETSEC_RING_RX ? DMA_FROM_DEVICE :
 							      DMA_TO_DEVICE);
-		dev_kfree_skb(desc->skb);
+		if (id == NETSEC_RING_RX)
+			skb_free_frag(desc->addr);
+		else if (id == NETSEC_RING_TX)
+			dev_kfree_skb(desc->skb);
 	}
 
 	memset(dring->desc, 0, sizeof(struct netsec_desc) * DESC_NUM);
@@ -953,50 +928,96 @@ static void netsec_free_dring(struct netsec_priv *priv, int id)
 	dring->desc = NULL;
 }
 
+static void *netsec_alloc_rx_data(struct netsec_priv *priv,
+				  dma_addr_t *dma_handle, u16 *desc_len)
+{
+	size_t len = priv->ndev->mtu + ETH_HLEN + VLAN_HLEN * 2 + NET_SKB_PAD +
+		NET_IP_ALIGN;
+	dma_addr_t mapping;
+	void *buf;
+
+	len = SKB_DATA_ALIGN(len);
+	len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+
+	buf = napi_alloc_frag(len);
+	if (!buf)
+		return NULL;
+
+	mapping = dma_map_single(priv->dev, buf, len, DMA_FROM_DEVICE);
+	if (unlikely(dma_mapping_error(priv->dev, mapping)))
+		goto err_out;
+
+	*dma_handle = mapping;
+	*desc_len = len;
+
+	return buf;
+
+err_out:
+	skb_free_frag(buf);
+	return NULL;
+}
+
+static void netsec_rx_fill(struct netsec_priv *priv, u16 from, u16 num)
+{
+	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
+	u16 idx = from;
+
+	while (num) {
+		netsec_set_rx_de(priv, dring, idx, &dring->desc[idx]);
+		idx++;
+		if (idx >= DESC_NUM)
+			idx = 0;
+		num--;
+	}
+}
+
 static int netsec_alloc_dring(struct netsec_priv *priv, enum ring_id id)
 {
 	struct netsec_desc_ring *dring = &priv->desc_ring[id];
-	int ret = 0;
 
 	dring->vaddr = dma_zalloc_coherent(priv->dev, DESC_SZ * DESC_NUM,
 					   &dring->desc_dma, GFP_KERNEL);
-	if (!dring->vaddr) {
-		ret = -ENOMEM;
+	if (!dring->vaddr)
 		goto err;
-	}
 
 	dring->desc = kcalloc(DESC_NUM, sizeof(*dring->desc), GFP_KERNEL);
-	if (!dring->desc) {
-		ret = -ENOMEM;
+	if (!dring->desc)
 		goto err;
-	}
 
 	return 0;
 err:
 	netsec_free_dring(priv, id);
 
-	return ret;
+	return -ENOMEM;
 }
 
 static int netsec_setup_rx_dring(struct netsec_priv *priv)
 {
 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
-	struct netsec_desc desc;
-	struct sk_buff *skb;
-	int n;
+	int i;
 
-	desc.len = priv->ndev->mtu + 22;
+	for (i = 0; i < DESC_NUM; i++) {
+		struct netsec_desc *desc = &dring->desc[i];
+		dma_addr_t dma_handle;
+		void *buf;
+		u16 len;
 
-	for (n = 0; n < DESC_NUM; n++) {
-		skb = netsec_alloc_skb(priv, &desc);
-		if (!skb) {
+		buf = netsec_alloc_rx_data(priv, &dma_handle, &len);
+		if (!buf) {
 			netsec_uninit_pkt_dring(priv, NETSEC_RING_RX);
-			return -ENOMEM;
+			goto err_out;
 		}
-		netsec_set_rx_de(priv, dring, n, &desc, skb);
+		desc->dma_addr = dma_handle;
+		desc->addr = buf;
+		desc->len = len;
 	}
 
+	netsec_rx_fill(priv, 0, DESC_NUM);
+
 	return 0;
+
+err_out:
+	return -ENOMEM;
 }
 
 static int netsec_netdev_load_ucode_region(struct netsec_priv *priv, u32 reg,
-- 
2.7.4

[net-next, PATCH 2/2, v2] net: socionext: add XDP support

[Ilias Apalodimas]

Add basic XDP support

Signed-off-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>
---
 drivers/net/ethernet/socionext/netsec.c | 234 +++++++++++++++++++++++++++++---
 1 file changed, 216 insertions(+), 18 deletions(-)

diff --git a/drivers/net/ethernet/socionext/netsec.c b/drivers/net/ethernet/socionext/netsec.c
index 666fee2..1f4594f 100644
--- a/drivers/net/ethernet/socionext/netsec.c
+++ b/drivers/net/ethernet/socionext/netsec.c
@@ -9,6 +9,9 @@
 #include <linux/etherdevice.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
+#include <linux/netlink.h>
+#include <linux/bpf.h>
+#include <linux/bpf_trace.h>
 
 #include <net/tcp.h>
 #include <net/ip6_checksum.h>
@@ -238,6 +241,11 @@
 
 #define NETSEC_F_NETSEC_VER_MAJOR_NUM(x)	((x) & 0xffff0000)
 
+#define NETSEC_XDP_PASS          0
+#define NETSEC_XDP_CONSUMED      BIT(0)
+#define NETSEC_XDP_TX            BIT(1)
+#define NETSEC_XDP_REDIR         BIT(2)
+
 enum ring_id {
 	NETSEC_RING_TX = 0,
 	NETSEC_RING_RX
@@ -256,11 +264,14 @@ struct netsec_desc_ring {
 	void *vaddr;
 	u16 pkt_cnt;
 	u16 head, tail;
+	bool is_xdp;
+	struct xdp_rxq_info xdp_rxq;
 };
 
 struct netsec_priv {
 	struct netsec_desc_ring desc_ring[NETSEC_RING_MAX];
 	struct ethtool_coalesce et_coalesce;
+	struct bpf_prog *xdp_prog;
 	spinlock_t reglock; /* protect reg access */
 	struct napi_struct napi;
 	phy_interface_t phy_interface;
@@ -297,6 +308,8 @@ struct netsec_rx_pkt_info {
 };
 
 static void netsec_rx_fill(struct netsec_priv *priv, u16 from, u16 num);
+static u32 netsec_run_xdp(struct netsec_desc *desc, struct netsec_priv *priv,
+			  struct bpf_prog *prog, struct xdp_buff *xdp);
 
 static void *netsec_alloc_rx_data(struct netsec_priv *priv,
 				  dma_addr_t *dma_addr, u16 *len);
@@ -613,13 +626,23 @@ static int netsec_clean_tx_dring(struct netsec_priv *priv, int budget)
 
 		eop = (entry->attr >> NETSEC_TX_LAST) & 1;
 
-		dma_unmap_single(priv->dev, desc->dma_addr, desc->len,
-				 DMA_TO_DEVICE);
-		if (eop) {
-			pkts++;
+		if (desc->skb)
+			dma_unmap_single(priv->dev,
+					 desc->dma_addr - XDP_PACKET_HEADROOM,
+					 desc->len, DMA_TO_DEVICE);
+
+		if (!eop) {
+			*desc = (struct netsec_desc){};
+			continue;
+		}
+
+		if (!desc->skb) {
+			skb_free_frag(desc->addr);
+		} else {
 			bytes += desc->skb->len;
 			dev_kfree_skb(desc->skb);
 		}
+		pkts++;
 		*desc = (struct netsec_desc){};
 	}
 	dring->pkt_cnt -= budget;
@@ -659,19 +682,22 @@ static void nsetsec_adv_desc(u16 *idx)
 static int netsec_process_rx(struct netsec_priv *priv, int budget)
 {
 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
+	struct bpf_prog *xdp_prog = READ_ONCE(priv->xdp_prog);
 	struct net_device *ndev = priv->ndev;
-	struct sk_buff *skb;
+	struct sk_buff *skb = NULL;
+	u32 xdp_flush = 0;
+	u32 xdp_result;
 	int done = 0;
 
 	while (done < budget) {
 		u16 idx = dring->tail;
 		struct netsec_de *de = dring->vaddr + (DESC_SZ * idx);
 		struct netsec_desc *desc = &dring->desc[idx];
+		dma_addr_t dma_handle, dma_unmap;
 		struct netsec_rx_pkt_info rpi;
-		dma_addr_t dma_handle;
+		u16 pkt_len, desc_len;
+		struct xdp_buff xdp;
 		void *buf_addr;
-		u16 pkt_len;
-		u16 desc_len;
 
 		if (de->attr & (1U << NETSEC_RX_PKT_OWN_FIELD))
 			break;
@@ -704,10 +730,40 @@ static int netsec_process_rx(struct netsec_priv *priv, int budget)
 
 		prefetch(desc->addr);
 		buf_addr = netsec_alloc_rx_data(priv, &dma_handle, &desc_len);
+
 		if (unlikely(!buf_addr))
 			break;
 
-		skb = build_skb(desc->addr, desc->len);
+		dma_unmap = dring->is_xdp ?
+			desc->dma_addr - XDP_PACKET_HEADROOM : desc->dma_addr;
+
+		xdp.data_hard_start = desc->addr;
+		xdp.data = desc->addr;
+		xdp_set_data_meta_invalid(&xdp);
+		xdp.data_end = xdp.data + pkt_len;
+		xdp.rxq = &dring->xdp_rxq;
+
+		if (xdp_prog) {
+			xdp.data = desc->addr + XDP_PACKET_HEADROOM;
+			xdp.data_end = xdp.data + pkt_len;
+			xdp_result = netsec_run_xdp(desc, priv, xdp_prog, &xdp);
+			if (xdp_result != NETSEC_XDP_PASS) {
+				xdp_flush |= xdp_result & NETSEC_XDP_REDIR;
+
+				dma_unmap_single_attrs(priv->dev, dma_unmap,
+						       desc->len, DMA_TO_DEVICE,
+						       DMA_ATTR_SKIP_CPU_SYNC);
+
+				desc->len = desc_len;
+				desc->dma_addr = dma_handle;
+				desc->addr = buf_addr;
+				netsec_rx_fill(priv, idx, 1);
+				nsetsec_adv_desc(&dring->tail);
+				continue;
+			}
+		}
+
+		skb = build_skb(xdp.data_hard_start, desc->len);
 		if (unlikely(!skb)) {
 			dma_unmap_single(priv->dev, dma_handle, desc_len,
 					 DMA_TO_DEVICE);
@@ -716,7 +772,7 @@ static int netsec_process_rx(struct netsec_priv *priv, int budget)
 				  "rx failed to alloc skb\n");
 			break;
 		}
-		dma_unmap_single_attrs(priv->dev, desc->dma_addr, desc->len,
+		dma_unmap_single_attrs(priv->dev, dma_unmap, desc->len,
 				       DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
 
 		/* Update the descriptor with fresh buffers */
@@ -724,7 +780,8 @@ static int netsec_process_rx(struct netsec_priv *priv, int budget)
 		desc->dma_addr = dma_handle;
 		desc->addr = buf_addr;
 
-		skb_put(skb, pkt_len);
+		skb_reserve(skb, xdp.data - xdp.data_hard_start);
+		skb_put(skb, xdp.data_end - xdp.data);
 		skb->protocol = eth_type_trans(skb, priv->ndev);
 
 		if (priv->rx_cksum_offload_flag &&
@@ -733,13 +790,16 @@ static int netsec_process_rx(struct netsec_priv *priv, int budget)
 
 		if (napi_gro_receive(&priv->napi, skb) != GRO_DROP) {
 			ndev->stats.rx_packets++;
-			ndev->stats.rx_bytes += pkt_len;
+			ndev->stats.rx_bytes += xdp.data_end - xdp.data;
 		}
 
 		netsec_rx_fill(priv, idx, 1);
 		nsetsec_adv_desc(&dring->tail);
 	}
 
+	if (xdp_flush & NETSEC_XDP_REDIR)
+		xdp_do_flush_map();
+
 	return done;
 }
 
@@ -892,6 +952,9 @@ static void netsec_uninit_pkt_dring(struct netsec_priv *priv, int id)
 	if (!dring->vaddr || !dring->desc)
 		return;
 
+	if (xdp_rxq_info_is_reg(&dring->xdp_rxq))
+		xdp_rxq_info_unreg(&dring->xdp_rxq);
+
 	for (idx = 0; idx < DESC_NUM; idx++) {
 		desc = &dring->desc[idx];
 		if (!desc->addr)
@@ -931,11 +994,14 @@ static void netsec_free_dring(struct netsec_priv *priv, int id)
 static void *netsec_alloc_rx_data(struct netsec_priv *priv,
 				  dma_addr_t *dma_handle, u16 *desc_len)
 {
+	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
 	size_t len = priv->ndev->mtu + ETH_HLEN + VLAN_HLEN * 2 + NET_SKB_PAD +
 		NET_IP_ALIGN;
 	dma_addr_t mapping;
 	void *buf;
 
+	if (dring->is_xdp)
+		len += XDP_PACKET_HEADROOM;
 	len = SKB_DATA_ALIGN(len);
 	len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 
@@ -943,11 +1009,12 @@ static void *netsec_alloc_rx_data(struct netsec_priv *priv,
 	if (!buf)
 		return NULL;
 
-	mapping = dma_map_single(priv->dev, buf, len, DMA_FROM_DEVICE);
+	mapping = dma_map_single(priv->dev, buf, len,
+				 DMA_FROM_DEVICE);
 	if (unlikely(dma_mapping_error(priv->dev, mapping)))
 		goto err_out;
 
-	*dma_handle = mapping;
+	*dma_handle = mapping + (dring->is_xdp ? XDP_PACKET_HEADROOM : 0);
 	*desc_len = len;
 
 	return buf;
@@ -994,7 +1061,13 @@ static int netsec_alloc_dring(struct netsec_priv *priv, enum ring_id id)
 static int netsec_setup_rx_dring(struct netsec_priv *priv)
 {
 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
-	int i;
+	struct bpf_prog *xdp_prog = READ_ONCE(priv->xdp_prog);
+	int i, err;
+
+	if (xdp_prog)
+		dring->is_xdp = true;
+	else
+		dring->is_xdp = false;
 
 	for (i = 0; i < DESC_NUM; i++) {
 		struct netsec_desc *desc = &dring->desc[i];
@@ -1003,20 +1076,29 @@ static int netsec_setup_rx_dring(struct netsec_priv *priv)
 		u16 len;
 
 		buf = netsec_alloc_rx_data(priv, &dma_handle, &len);
-		if (!buf) {
-			netsec_uninit_pkt_dring(priv, NETSEC_RING_RX);
+		if (!buf)
 			goto err_out;
-		}
 		desc->dma_addr = dma_handle;
 		desc->addr = buf;
 		desc->len = len;
 	}
 
 	netsec_rx_fill(priv, 0, DESC_NUM);
+	err = xdp_rxq_info_reg(&dring->xdp_rxq, priv->ndev, 0);
+	if (err)
+		goto err_out;
+
+	err = xdp_rxq_info_reg_mem_model(&dring->xdp_rxq, MEM_TYPE_PAGE_SHARED,
+					 NULL);
+	if (err) {
+		xdp_rxq_info_unreg(&dring->xdp_rxq);
+		goto err_out;
+	}
 
 	return 0;
 
 err_out:
+	netsec_uninit_pkt_dring(priv, NETSEC_RING_RX);
 	return -ENOMEM;
 }
 
@@ -1420,6 +1502,121 @@ static int netsec_netdev_ioctl(struct net_device *ndev, struct ifreq *ifr,
 	return phy_mii_ioctl(ndev->phydev, ifr, cmd);
 }
 
+static u32 netsec_xmit_xdp(struct netsec_priv *priv, struct xdp_buff *xdp,
+			   struct netsec_desc *rx_desc)
+{
+	struct netsec_desc_ring *tx_ring = &priv->desc_ring[NETSEC_RING_TX];
+	struct netsec_tx_pkt_ctrl tx_ctrl = {};
+	struct netsec_desc tx_desc;
+	int filled;
+	u32 len;
+
+	len = xdp->data_end - xdp->data;
+
+	if (tx_ring->head >= tx_ring->tail)
+		filled = tx_ring->head - tx_ring->tail;
+	else
+		filled = tx_ring->head + DESC_NUM - tx_ring->tail;
+
+	if (DESC_NUM - filled <= 1)
+		return NETSEC_XDP_CONSUMED;
+
+	dma_sync_single_for_device(priv->dev, rx_desc->dma_addr, len,
+				   DMA_TO_DEVICE);
+
+	tx_desc.dma_addr = rx_desc->dma_addr;
+	tx_desc.addr = xdp->data;
+	tx_desc.len = len;
+
+	netsec_set_tx_de(priv, tx_ring, &tx_ctrl, &tx_desc, NULL);
+	netsec_write(priv, NETSEC_REG_NRM_TX_PKTCNT, 1);
+
+	return NETSEC_XDP_TX;
+}
+
+static u32 netsec_run_xdp(struct netsec_desc *desc, struct netsec_priv *priv,
+			  struct bpf_prog *prog, struct xdp_buff *xdp)
+{
+	u32 ret = NETSEC_XDP_PASS;
+	int err;
+	u32 act;
+
+	rcu_read_lock();
+	act = bpf_prog_run_xdp(prog, xdp);
+
+	switch (act) {
+	case XDP_PASS:
+		ret = NETSEC_XDP_PASS;
+		break;
+	case XDP_TX:
+		ret = netsec_xmit_xdp(priv, xdp, desc);
+		break;
+	case XDP_REDIRECT:
+		err = xdp_do_redirect(priv->ndev, xdp, prog);
+		if (!err) {
+			ret = NETSEC_XDP_REDIR;
+		} else {
+			ret = NETSEC_XDP_CONSUMED;
+			xdp_return_buff(xdp);
+		}
+		break;
+	default:
+		bpf_warn_invalid_xdp_action(act);
+		/* fall through */
+	case XDP_ABORTED:
+		trace_xdp_exception(priv->ndev, prog, act);
+		/* fall through -- handle aborts by dropping packet */
+	case XDP_DROP:
+		ret = NETSEC_XDP_CONSUMED;
+		break;
+	}
+
+	rcu_read_unlock();
+
+	return ret;
+}
+
+static int netsec_xdp_setup(struct netsec_priv *priv, struct bpf_prog *prog,
+			    struct netlink_ext_ack *extack)
+{
+	struct net_device *dev = priv->ndev;
+	struct bpf_prog *old_prog;
+
+	/* For now just support only the usual MTU sized frames */
+	if (prog && dev->mtu > 1500) {
+		NL_SET_ERR_MSG_MOD(extack, "Jumbo frames not supported on XDP");
+		return -EOPNOTSUPP;
+	}
+
+	if (netif_running(dev))
+		netsec_netdev_stop(dev);
+
+	/* Detach old prog, if any */
+	old_prog = xchg(&priv->xdp_prog, prog);
+	if (old_prog)
+		bpf_prog_put(old_prog);
+
+	if (netif_running(dev))
+		netsec_netdev_open(dev);
+
+	return 0;
+}
+
+static int netsec_xdp(struct net_device *ndev, struct netdev_bpf *xdp)
+{
+	struct netsec_priv *priv = netdev_priv(ndev);
+
+	switch (xdp->command) {
+	case XDP_SETUP_PROG:
+		return netsec_xdp_setup(priv, xdp->prog, xdp->extack);
+	case XDP_QUERY_PROG:
+		xdp->prog_id = priv->xdp_prog ? priv->xdp_prog->aux->id : 0;
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
 static const struct net_device_ops netsec_netdev_ops = {
 	.ndo_init		= netsec_netdev_init,
 	.ndo_uninit		= netsec_netdev_uninit,
@@ -1430,6 +1627,7 @@ static const struct net_device_ops netsec_netdev_ops = {
 	.ndo_set_mac_address    = eth_mac_addr,
 	.ndo_validate_addr	= eth_validate_addr,
 	.ndo_do_ioctl		= netsec_netdev_ioctl,
+	.ndo_bpf		= netsec_xdp,
 };
 
 static int netsec_of_probe(struct platform_device *pdev,
-- 
2.7.4

Re: [net-next, PATCH 2/2, v2] net: socionext: add XDP support

[Jesper Dangaard Brouer]

On Wed, 12 Sep 2018 12:02:38 +0300
Ilias Apalodimas <ilias.apalodimas@linaro.org> wrote:

> @@ -1003,20 +1076,29 @@ static int netsec_setup_rx_dring(struct netsec_priv *priv)
>  		u16 len;
>  
>  		buf = netsec_alloc_rx_data(priv, &dma_handle, &len);
> -		if (!buf) {
> -			netsec_uninit_pkt_dring(priv, NETSEC_RING_RX);
> +		if (!buf)
>  			goto err_out;
> -		}
>  		desc->dma_addr = dma_handle;
>  		desc->addr = buf;
>  		desc->len = len;
>  	}
>  
>  	netsec_rx_fill(priv, 0, DESC_NUM);
> +	err = xdp_rxq_info_reg(&dring->xdp_rxq, priv->ndev, 0);

Do you only have 1 RX queue? (last arg to xdp_rxq_info_reg is 0),


> +	if (err)
> +		goto err_out;
> +
> +	err = xdp_rxq_info_reg_mem_model(&dring->xdp_rxq, MEM_TYPE_PAGE_SHARED,
> +					 NULL);
> +	if (err) {
> +		xdp_rxq_info_unreg(&dring->xdp_rxq);
> +		goto err_out;
> +	}
>  
>  	return 0;
>  


-- 
Best regards,
  Jesper Dangaard Brouer
  MSc.CS, Principal Kernel Engineer at Red Hat
  LinkedIn: http://www.linkedin.com/in/brouer

Re: [net-next, PATCH 2/2, v2] net: socionext: add XDP support

[Ilias Apalodimas]

On Wed, Sep 12, 2018 at 11:14:57AM +0200, Jesper Dangaard Brouer wrote:
> On Wed, 12 Sep 2018 12:02:38 +0300
> Ilias Apalodimas <ilias.apalodimas@linaro.org> wrote:
> 
> > @@ -1003,20 +1076,29 @@ static int netsec_setup_rx_dring(struct netsec_priv *priv)
> >  		u16 len;
> >  
> >  		buf = netsec_alloc_rx_data(priv, &dma_handle, &len);
> > -		if (!buf) {
> > -			netsec_uninit_pkt_dring(priv, NETSEC_RING_RX);
> > +		if (!buf)
> >  			goto err_out;
> > -		}
> >  		desc->dma_addr = dma_handle;
> >  		desc->addr = buf;
> >  		desc->len = len;
> >  	}
> >  
> >  	netsec_rx_fill(priv, 0, DESC_NUM);
> > +	err = xdp_rxq_info_reg(&dring->xdp_rxq, priv->ndev, 0);
> 
> Do you only have 1 RX queue? (last arg to xdp_rxq_info_reg is 0),
> 
> 
Yes the current driver is only supporting a single queue (same for Tx)
> > +	if (err)
> > +		goto err_out;
> > +
> > +	err = xdp_rxq_info_reg_mem_model(&dring->xdp_rxq, MEM_TYPE_PAGE_SHARED,
> > +					 NULL);
> > +	if (err) {
> > +		xdp_rxq_info_unreg(&dring->xdp_rxq);
> > +		goto err_out;
> > +	}
> >  
> >  	return 0;
> >  
> 
> 
> -- 
> Best regards,
>   Jesper Dangaard Brouer
>   MSc.CS, Principal Kernel Engineer at Red Hat
>   LinkedIn: http://www.linkedin.com/in/brouer


Thanks for looking at this

/Ilias

Re: [net-next, PATCH 2/2, v2] net: socionext: add XDP support

[Björn Töpel]

Den ons 12 sep. 2018 kl 11:21 skrev Ilias Apalodimas
<ilias.apalodimas@linaro.org>:
>
> On Wed, Sep 12, 2018 at 11:14:57AM +0200, Jesper Dangaard Brouer wrote:
> > On Wed, 12 Sep 2018 12:02:38 +0300
> > Ilias Apalodimas <ilias.apalodimas@linaro.org> wrote:
> >
> > > @@ -1003,20 +1076,29 @@ static int netsec_setup_rx_dring(struct netsec_priv *priv)
> > >             u16 len;
> > >
> > >             buf = netsec_alloc_rx_data(priv, &dma_handle, &len);
> > > -           if (!buf) {
> > > -                   netsec_uninit_pkt_dring(priv, NETSEC_RING_RX);
> > > +           if (!buf)
> > >                     goto err_out;
> > > -           }
> > >             desc->dma_addr = dma_handle;
> > >             desc->addr = buf;
> > >             desc->len = len;
> > >     }
> > >
> > >     netsec_rx_fill(priv, 0, DESC_NUM);
> > > +   err = xdp_rxq_info_reg(&dring->xdp_rxq, priv->ndev, 0);
> >
> > Do you only have 1 RX queue? (last arg to xdp_rxq_info_reg is 0),
> >
> >
> Yes the current driver is only supporting a single queue (same for Tx)

XDP and skbuff path sharing the same queue? You'll probably need some
means of synchronization between the .ndo_xmit_xdp and .ndo_start_xmit
implementations. And it looks like .ndo_xmit_xdp is missing!


Björn

> > > +   if (err)
> > > +           goto err_out;
> > > +
> > > +   err = xdp_rxq_info_reg_mem_model(&dring->xdp_rxq, MEM_TYPE_PAGE_SHARED,
> > > +                                    NULL);
> > > +   if (err) {
> > > +           xdp_rxq_info_unreg(&dring->xdp_rxq);
> > > +           goto err_out;
> > > +   }
> > >
> > >     return 0;
> > >
> >
> >
> > --
> > Best regards,
> >   Jesper Dangaard Brouer
> >   MSc.CS, Principal Kernel Engineer at Red Hat
> >   LinkedIn: http://www.linkedin.com/in/brouer
>
>
> Thanks for looking at this
>
> /Ilias

Re: [net-next, PATCH 2/2, v2] net: socionext: add XDP support

[Jesper Dangaard Brouer]

On Wed, 12 Sep 2018 12:02:38 +0300
Ilias Apalodimas <ilias.apalodimas@linaro.org> wrote:

>  static const struct net_device_ops netsec_netdev_ops = {
>  	.ndo_init		= netsec_netdev_init,
>  	.ndo_uninit		= netsec_netdev_uninit,
> @@ -1430,6 +1627,7 @@ static const struct net_device_ops netsec_netdev_ops = {
>  	.ndo_set_mac_address    = eth_mac_addr,
>  	.ndo_validate_addr	= eth_validate_addr,
>  	.ndo_do_ioctl		= netsec_netdev_ioctl,
> +	.ndo_bpf		= netsec_xdp,
>  };
>  

You have not implemented ndo_xdp_xmit.

Thus, you have "only" implemented the RX side of XDP_REDIRECT.  Which
allows you to do, cpumap and AF_XDP redirects, but not allowing other
drivers to XDP send out this device.

-- 
Best regards,
  Jesper Dangaard Brouer
  MSc.CS, Principal Kernel Engineer at Red Hat
  LinkedIn: http://www.linkedin.com/in/brouer

Re: [net-next, PATCH 2/2, v2] net: socionext: add XDP support

[Ilias Apalodimas]

On Wed, Sep 12, 2018 at 11:25:24AM +0200, Jesper Dangaard Brouer wrote:
> On Wed, 12 Sep 2018 12:02:38 +0300
> Ilias Apalodimas <ilias.apalodimas@linaro.org> wrote:
> 
> >  static const struct net_device_ops netsec_netdev_ops = {
> >  	.ndo_init		= netsec_netdev_init,
> >  	.ndo_uninit		= netsec_netdev_uninit,
> > @@ -1430,6 +1627,7 @@ static const struct net_device_ops netsec_netdev_ops = {
> >  	.ndo_set_mac_address    = eth_mac_addr,
> >  	.ndo_validate_addr	= eth_validate_addr,
> >  	.ndo_do_ioctl		= netsec_netdev_ioctl,
> > +	.ndo_bpf		= netsec_xdp,
> >  };
> >  
> 
> You have not implemented ndo_xdp_xmit.
> 
> Thus, you have "only" implemented the RX side of XDP_REDIRECT.  Which
> allows you to do, cpumap and AF_XDP redirects, but not allowing other
> drivers to XDP send out this device.
Correct, that was the planning, is ndo_xdp_xmit() needed for the patch or
is the patch message just misleading and i should change that ?

Thanks
/Ilias

Re: [net-next, PATCH 2/2, v2] net: socionext: add XDP support

[Jesper Dangaard Brouer]

On Wed, 12 Sep 2018 12:29:15 +0300
Ilias Apalodimas <ilias.apalodimas@linaro.org> wrote:

> On Wed, Sep 12, 2018 at 11:25:24AM +0200, Jesper Dangaard Brouer wrote:
> > On Wed, 12 Sep 2018 12:02:38 +0300
> > Ilias Apalodimas <ilias.apalodimas@linaro.org> wrote:
> >   
> > >  static const struct net_device_ops netsec_netdev_ops = {
> > >  	.ndo_init		= netsec_netdev_init,
> > >  	.ndo_uninit		= netsec_netdev_uninit,
> > > @@ -1430,6 +1627,7 @@ static const struct net_device_ops netsec_netdev_ops = {
> > >  	.ndo_set_mac_address    = eth_mac_addr,
> > >  	.ndo_validate_addr	= eth_validate_addr,
> > >  	.ndo_do_ioctl		= netsec_netdev_ioctl,
> > > +	.ndo_bpf		= netsec_xdp,
> > >  };
> > >    
> > 
> > You have not implemented ndo_xdp_xmit.
> > 
> > Thus, you have "only" implemented the RX side of XDP_REDIRECT.  Which
> > allows you to do, cpumap and AF_XDP redirects, but not allowing other
> > drivers to XDP send out this device.  
>
> Correct, that was the planning, is ndo_xdp_xmit() needed for the patch or
> is the patch message just misleading and i should change that ?

Yes, I think you should ALSO implement ndo_xdp_xmit, maybe as a separate
patch, but in the same series. (Our experience is that if we don't
require this, people forget to complete this part of the XDP support).

Also you XDP_TX is not optimal, as it (looks like) you flush TX on
every send.

BTW, do you have any performance numbers?

-- 
Best regards,
  Jesper Dangaard Brouer
  MSc.CS, Principal Kernel Engineer at Red Hat
  LinkedIn: http://www.linkedin.com/in/brouer

Re: [net-next, PATCH 2/2, v2] net: socionext: add XDP support

[Ilias Apalodimas]

On Thu, Sep 13, 2018 at 04:32:06PM +0200, Jesper Dangaard Brouer wrote:
> On Wed, 12 Sep 2018 12:29:15 +0300
> Ilias Apalodimas <ilias.apalodimas@linaro.org> wrote:
> 
> > On Wed, Sep 12, 2018 at 11:25:24AM +0200, Jesper Dangaard Brouer wrote:
> > > On Wed, 12 Sep 2018 12:02:38 +0300
> > > Ilias Apalodimas <ilias.apalodimas@linaro.org> wrote:
> > >   
> > > >  static const struct net_device_ops netsec_netdev_ops = {
> > > >  	.ndo_init		= netsec_netdev_init,
> > > >  	.ndo_uninit		= netsec_netdev_uninit,
> > > > @@ -1430,6 +1627,7 @@ static const struct net_device_ops netsec_netdev_ops = {
> > > >  	.ndo_set_mac_address    = eth_mac_addr,
> > > >  	.ndo_validate_addr	= eth_validate_addr,
> > > >  	.ndo_do_ioctl		= netsec_netdev_ioctl,
> > > > +	.ndo_bpf		= netsec_xdp,
> > > >  };
> > > >    
> > > 
> > > You have not implemented ndo_xdp_xmit.
> > > 
> > > Thus, you have "only" implemented the RX side of XDP_REDIRECT.  Which
> > > allows you to do, cpumap and AF_XDP redirects, but not allowing other
> > > drivers to XDP send out this device.  
> >
> > Correct, that was the planning, is ndo_xdp_xmit() needed for the patch or
> > is the patch message just misleading and i should change that ?
> 
> Yes, I think you should ALSO implement ndo_xdp_xmit, maybe as a separate
> patch, but in the same series. (Our experience is that if we don't
> require this, people forget to complete this part of the XDP support).
Ok makes sense. Already started on that i should have something soon
> 
> Also you XDP_TX is not optimal, as it (looks like) you flush TX on
> every send.
Yes i do, the driver is queueing packet by packet (in it's default skb 
implemetation) so i just did the same. Agree it's far from optimal though
i'll see if i can change than on the next version
> 
> BTW, do you have any performance numbers?
Yes XDP_TX is doing ~330kpps and XDP_REDIRECT ~340kpps(dropping packets)
using 64b packets.  I am not really sure if this is a hardware limitation 
due to only using a single queue. I used ./samples/bpf/xdpsock for AF_XDP
and ./samples/bpf/xdp2 for XDP_TX. I hope i am doing the right tests

The default Rx path is doing ~220kpps with the improved memory allocation
scheme so we do have some improvement although we are far away from line 
rate

The default Tx seems to hang after some point with a txq full message so 
i don't have any precice numbers for that

This change on the driver started as an investigation of using AF_XDP
for Time Sensitive networking setups. The offloading seems to work wonders
there since the latency is reduced *A LOT* (more than 10x in my case) in 
Rx path

Another thing i did consider is that Bjorn is right. Since i only have 1 
shared txq i need locking to avoid race conditions. 

Once again thanks for reviewing this

/Ilias