[PATCH net-next 0/2] gve: adopt page pool

[PATCH net-next 0/2] gve: adopt page pool

[Praveen Kaligineedi]

From: Harshitha Ramamurthy <hramamurthy@google.com>

This patchset implements page pool support for gve.
The first patch deals with movement of code to make
page pool adoption easier in the next patch. The
second patch adopts the page pool API.

Harshitha Ramamurthy (2):
  gve: move DQO rx buffer management related code to a new file
  gve: adopt page pool for DQ RDA mode

 drivers/net/ethernet/google/Kconfig           |   1 +
 drivers/net/ethernet/google/gve/Makefile      |   3 +-
 drivers/net/ethernet/google/gve/gve.h         |  27 ++
 .../ethernet/google/gve/gve_buffer_mgmt_dqo.c | 298 +++++++++++++++++
 drivers/net/ethernet/google/gve/gve_rx_dqo.c  | 313 +++---------------
 5 files changed, 374 insertions(+), 268 deletions(-)
 create mode 100644 drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c

-- 
2.46.0.598.g6f2099f65c-goog

[PATCH net-next 1/2] gve: move DQO rx buffer management related code to a new file

[Praveen Kaligineedi]

From: Harshitha Ramamurthy <hramamurthy@google.com>

In preparation for the upcoming page pool adoption for DQO
raw addressing mode, move RX buffer management code to a new
file. In the follow on patches, page pool code will be added
to this file.

No functional change, just movement of code.

Reviewed-by: Praveen Kaligineedi <pkaligineedi@google.com>
Reviewed-by: Shailend Chand <shailend@google.com>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
 drivers/net/ethernet/google/gve/Makefile      |   3 +-
 drivers/net/ethernet/google/gve/gve.h         |  14 ++
 .../ethernet/google/gve/gve_buffer_mgmt_dqo.c | 226 ++++++++++++++++++
 drivers/net/ethernet/google/gve/gve_rx_dqo.c  | 225 -----------------
 4 files changed, 242 insertions(+), 226 deletions(-)
 create mode 100644 drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c

diff --git a/drivers/net/ethernet/google/gve/Makefile b/drivers/net/ethernet/google/gve/Makefile
index 9ed07080b38a..4520f1c07a63 100644
--- a/drivers/net/ethernet/google/gve/Makefile
+++ b/drivers/net/ethernet/google/gve/Makefile
@@ -1,4 +1,5 @@
 # Makefile for the Google virtual Ethernet (gve) driver
 
 obj-$(CONFIG_GVE) += gve.o
-gve-objs := gve_main.o gve_tx.o gve_tx_dqo.o gve_rx.o gve_rx_dqo.o gve_ethtool.o gve_adminq.o gve_utils.o gve_flow_rule.o
+gve-objs := gve_main.o gve_tx.o gve_tx_dqo.o gve_rx.o gve_rx_dqo.o gve_ethtool.o gve_adminq.o gve_utils.o gve_flow_rule.o \
+	    gve_buffer_mgmt_dqo.o
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index 301fa1ea4f51..387fd26ebc43 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -1162,6 +1162,20 @@ void gve_rx_stop_ring_gqi(struct gve_priv *priv, int idx);
 u16 gve_get_pkt_buf_size(const struct gve_priv *priv, bool enable_hplit);
 bool gve_header_split_supported(const struct gve_priv *priv);
 int gve_set_hsplit_config(struct gve_priv *priv, u8 tcp_data_split);
+/* rx buffer handling */
+int gve_buf_ref_cnt(struct gve_rx_buf_state_dqo *bs);
+void gve_free_page_dqo(struct gve_priv *priv, struct gve_rx_buf_state_dqo *bs, bool free_page);
+struct gve_rx_buf_state_dqo *gve_alloc_buf_state(struct gve_rx_ring *rx);
+bool gve_buf_state_is_allocated(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state);
+void gve_free_buf_state(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state);
+struct gve_rx_buf_state_dqo *gve_dequeue_buf_state(struct gve_rx_ring *rx,
+						   struct gve_index_list *list);
+void gve_enqueue_buf_state(struct gve_rx_ring *rx, struct gve_index_list *list,
+			   struct gve_rx_buf_state_dqo *buf_state);
+struct gve_rx_buf_state_dqo *gve_get_recycled_buf_state(struct gve_rx_ring *rx);
+int gve_alloc_page_dqo(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state);
+void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx,
+			 struct gve_rx_buf_state_dqo *buf_state);
 /* Reset */
 void gve_schedule_reset(struct gve_priv *priv);
 int gve_reset(struct gve_priv *priv, bool attempt_teardown);
diff --git a/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c b/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c
new file mode 100644
index 000000000000..a8ea23b407ed
--- /dev/null
+++ b/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c
@@ -0,0 +1,226 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Google virtual Ethernet (gve) driver
+ *
+ * Copyright (C) 2015-2024 Google, Inc.
+ */
+
+#include "gve.h"
+#include "gve_utils.h"
+
+int gve_buf_ref_cnt(struct gve_rx_buf_state_dqo *bs)
+{
+	return page_count(bs->page_info.page) - bs->page_info.pagecnt_bias;
+}
+
+void gve_free_page_dqo(struct gve_priv *priv, struct gve_rx_buf_state_dqo *bs, bool free_page)
+{
+	page_ref_sub(bs->page_info.page, bs->page_info.pagecnt_bias - 1);
+	if (free_page)
+		gve_free_page(&priv->pdev->dev, bs->page_info.page, bs->addr,
+			      DMA_FROM_DEVICE);
+	bs->page_info.page = NULL;
+}
+
+struct gve_rx_buf_state_dqo *gve_alloc_buf_state(struct gve_rx_ring *rx)
+{
+	struct gve_rx_buf_state_dqo *buf_state;
+	s16 buffer_id;
+
+	buffer_id = rx->dqo.free_buf_states;
+	if (unlikely(buffer_id == -1))
+		return NULL;
+
+	buf_state = &rx->dqo.buf_states[buffer_id];
+
+	/* Remove buf_state from free list */
+	rx->dqo.free_buf_states = buf_state->next;
+
+	/* Point buf_state to itself to mark it as allocated */
+	buf_state->next = buffer_id;
+
+	return buf_state;
+}
+
+bool gve_buf_state_is_allocated(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state)
+{
+	s16 buffer_id = buf_state - rx->dqo.buf_states;
+
+	return buf_state->next == buffer_id;
+}
+
+void gve_free_buf_state(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state)
+{
+	s16 buffer_id = buf_state - rx->dqo.buf_states;
+
+	buf_state->next = rx->dqo.free_buf_states;
+	rx->dqo.free_buf_states = buffer_id;
+}
+
+struct gve_rx_buf_state_dqo *gve_dequeue_buf_state(struct gve_rx_ring *rx,
+						   struct gve_index_list *list)
+{
+	struct gve_rx_buf_state_dqo *buf_state;
+	s16 buffer_id;
+
+	buffer_id = list->head;
+	if (unlikely(buffer_id == -1))
+		return NULL;
+
+	buf_state = &rx->dqo.buf_states[buffer_id];
+
+	/* Remove buf_state from list */
+	list->head = buf_state->next;
+	if (buf_state->next == -1)
+		list->tail = -1;
+
+	/* Point buf_state to itself to mark it as allocated */
+	buf_state->next = buffer_id;
+
+	return buf_state;
+}
+
+void gve_enqueue_buf_state(struct gve_rx_ring *rx, struct gve_index_list *list,
+			   struct gve_rx_buf_state_dqo *buf_state)
+{
+	s16 buffer_id = buf_state - rx->dqo.buf_states;
+
+	buf_state->next = -1;
+
+	if (list->head == -1) {
+		list->head = buffer_id;
+		list->tail = buffer_id;
+	} else {
+		int tail = list->tail;
+
+		rx->dqo.buf_states[tail].next = buffer_id;
+		list->tail = buffer_id;
+	}
+}
+
+struct gve_rx_buf_state_dqo *gve_get_recycled_buf_state(struct gve_rx_ring *rx)
+{
+	struct gve_rx_buf_state_dqo *buf_state;
+	int i;
+
+	/* Recycled buf states are immediately usable. */
+	buf_state = gve_dequeue_buf_state(rx, &rx->dqo.recycled_buf_states);
+	if (likely(buf_state))
+		return buf_state;
+
+	if (unlikely(rx->dqo.used_buf_states.head == -1))
+		return NULL;
+
+	/* Used buf states are only usable when ref count reaches 0, which means
+	 * no SKBs refer to them.
+	 *
+	 * Search a limited number before giving up.
+	 */
+	for (i = 0; i < 5; i++) {
+		buf_state = gve_dequeue_buf_state(rx, &rx->dqo.used_buf_states);
+		if (gve_buf_ref_cnt(buf_state) == 0) {
+			rx->dqo.used_buf_states_cnt--;
+			return buf_state;
+		}
+
+		gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state);
+	}
+
+	/* For QPL, we cannot allocate any new buffers and must
+	 * wait for the existing ones to be available.
+	 */
+	if (rx->dqo.qpl)
+		return NULL;
+
+	/* If there are no free buf states discard an entry from
+	 * `used_buf_states` so it can be used.
+	 */
+	if (unlikely(rx->dqo.free_buf_states == -1)) {
+		buf_state = gve_dequeue_buf_state(rx, &rx->dqo.used_buf_states);
+		if (gve_buf_ref_cnt(buf_state) == 0)
+			return buf_state;
+
+		gve_free_page_dqo(rx->gve, buf_state, true);
+		gve_free_buf_state(rx, buf_state);
+	}
+
+	return NULL;
+}
+
+int gve_alloc_page_dqo(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state)
+{
+	struct gve_priv *priv = rx->gve;
+	u32 idx;
+
+	if (!rx->dqo.qpl) {
+		int err;
+
+		err = gve_alloc_page(priv, &priv->pdev->dev,
+				     &buf_state->page_info.page,
+				     &buf_state->addr,
+				     DMA_FROM_DEVICE, GFP_ATOMIC);
+		if (err)
+			return err;
+	} else {
+		idx = rx->dqo.next_qpl_page_idx;
+		if (idx >= gve_get_rx_pages_per_qpl_dqo(priv->rx_desc_cnt)) {
+			net_err_ratelimited("%s: Out of QPL pages\n",
+					    priv->dev->name);
+			return -ENOMEM;
+		}
+		buf_state->page_info.page = rx->dqo.qpl->pages[idx];
+		buf_state->addr = rx->dqo.qpl->page_buses[idx];
+		rx->dqo.next_qpl_page_idx++;
+	}
+	buf_state->page_info.page_offset = 0;
+	buf_state->page_info.page_address =
+		page_address(buf_state->page_info.page);
+	buf_state->last_single_ref_offset = 0;
+
+	/* The page already has 1 ref. */
+	page_ref_add(buf_state->page_info.page, INT_MAX - 1);
+	buf_state->page_info.pagecnt_bias = INT_MAX;
+
+	return 0;
+}
+
+void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx,
+			 struct gve_rx_buf_state_dqo *buf_state)
+{
+	const u16 data_buffer_size = priv->data_buffer_size_dqo;
+	int pagecount;
+
+	/* Can't reuse if we only fit one buffer per page */
+	if (data_buffer_size * 2 > PAGE_SIZE)
+		goto mark_used;
+
+	pagecount = gve_buf_ref_cnt(buf_state);
+
+	/* Record the offset when we have a single remaining reference.
+	 *
+	 * When this happens, we know all of the other offsets of the page are
+	 * usable.
+	 */
+	if (pagecount == 1) {
+		buf_state->last_single_ref_offset =
+			buf_state->page_info.page_offset;
+	}
+
+	/* Use the next buffer sized chunk in the page. */
+	buf_state->page_info.page_offset += data_buffer_size;
+	buf_state->page_info.page_offset &= (PAGE_SIZE - 1);
+
+	/* If we wrap around to the same offset without ever dropping to 1
+	 * reference, then we don't know if this offset was ever freed.
+	 */
+	if (buf_state->page_info.page_offset ==
+	    buf_state->last_single_ref_offset) {
+		goto mark_used;
+	}
+
+	gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, buf_state);
+	return;
+
+mark_used:
+	gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state);
+	rx->dqo.used_buf_states_cnt++;
+}
diff --git a/drivers/net/ethernet/google/gve/gve_rx_dqo.c b/drivers/net/ethernet/google/gve/gve_rx_dqo.c
index 1154c1d8f66f..b343be2fb118 100644
--- a/drivers/net/ethernet/google/gve/gve_rx_dqo.c
+++ b/drivers/net/ethernet/google/gve/gve_rx_dqo.c
@@ -16,189 +16,6 @@
 #include <net/ipv6.h>
 #include <net/tcp.h>
 
-static int gve_buf_ref_cnt(struct gve_rx_buf_state_dqo *bs)
-{
-	return page_count(bs->page_info.page) - bs->page_info.pagecnt_bias;
-}
-
-static void gve_free_page_dqo(struct gve_priv *priv,
-			      struct gve_rx_buf_state_dqo *bs,
-			      bool free_page)
-{
-	page_ref_sub(bs->page_info.page, bs->page_info.pagecnt_bias - 1);
-	if (free_page)
-		gve_free_page(&priv->pdev->dev, bs->page_info.page, bs->addr,
-			      DMA_FROM_DEVICE);
-	bs->page_info.page = NULL;
-}
-
-static struct gve_rx_buf_state_dqo *gve_alloc_buf_state(struct gve_rx_ring *rx)
-{
-	struct gve_rx_buf_state_dqo *buf_state;
-	s16 buffer_id;
-
-	buffer_id = rx->dqo.free_buf_states;
-	if (unlikely(buffer_id == -1))
-		return NULL;
-
-	buf_state = &rx->dqo.buf_states[buffer_id];
-
-	/* Remove buf_state from free list */
-	rx->dqo.free_buf_states = buf_state->next;
-
-	/* Point buf_state to itself to mark it as allocated */
-	buf_state->next = buffer_id;
-
-	return buf_state;
-}
-
-static bool gve_buf_state_is_allocated(struct gve_rx_ring *rx,
-				       struct gve_rx_buf_state_dqo *buf_state)
-{
-	s16 buffer_id = buf_state - rx->dqo.buf_states;
-
-	return buf_state->next == buffer_id;
-}
-
-static void gve_free_buf_state(struct gve_rx_ring *rx,
-			       struct gve_rx_buf_state_dqo *buf_state)
-{
-	s16 buffer_id = buf_state - rx->dqo.buf_states;
-
-	buf_state->next = rx->dqo.free_buf_states;
-	rx->dqo.free_buf_states = buffer_id;
-}
-
-static struct gve_rx_buf_state_dqo *
-gve_dequeue_buf_state(struct gve_rx_ring *rx, struct gve_index_list *list)
-{
-	struct gve_rx_buf_state_dqo *buf_state;
-	s16 buffer_id;
-
-	buffer_id = list->head;
-	if (unlikely(buffer_id == -1))
-		return NULL;
-
-	buf_state = &rx->dqo.buf_states[buffer_id];
-
-	/* Remove buf_state from list */
-	list->head = buf_state->next;
-	if (buf_state->next == -1)
-		list->tail = -1;
-
-	/* Point buf_state to itself to mark it as allocated */
-	buf_state->next = buffer_id;
-
-	return buf_state;
-}
-
-static void gve_enqueue_buf_state(struct gve_rx_ring *rx,
-				  struct gve_index_list *list,
-				  struct gve_rx_buf_state_dqo *buf_state)
-{
-	s16 buffer_id = buf_state - rx->dqo.buf_states;
-
-	buf_state->next = -1;
-
-	if (list->head == -1) {
-		list->head = buffer_id;
-		list->tail = buffer_id;
-	} else {
-		int tail = list->tail;
-
-		rx->dqo.buf_states[tail].next = buffer_id;
-		list->tail = buffer_id;
-	}
-}
-
-static struct gve_rx_buf_state_dqo *
-gve_get_recycled_buf_state(struct gve_rx_ring *rx)
-{
-	struct gve_rx_buf_state_dqo *buf_state;
-	int i;
-
-	/* Recycled buf states are immediately usable. */
-	buf_state = gve_dequeue_buf_state(rx, &rx->dqo.recycled_buf_states);
-	if (likely(buf_state))
-		return buf_state;
-
-	if (unlikely(rx->dqo.used_buf_states.head == -1))
-		return NULL;
-
-	/* Used buf states are only usable when ref count reaches 0, which means
-	 * no SKBs refer to them.
-	 *
-	 * Search a limited number before giving up.
-	 */
-	for (i = 0; i < 5; i++) {
-		buf_state = gve_dequeue_buf_state(rx, &rx->dqo.used_buf_states);
-		if (gve_buf_ref_cnt(buf_state) == 0) {
-			rx->dqo.used_buf_states_cnt--;
-			return buf_state;
-		}
-
-		gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state);
-	}
-
-	/* For QPL, we cannot allocate any new buffers and must
-	 * wait for the existing ones to be available.
-	 */
-	if (rx->dqo.qpl)
-		return NULL;
-
-	/* If there are no free buf states discard an entry from
-	 * `used_buf_states` so it can be used.
-	 */
-	if (unlikely(rx->dqo.free_buf_states == -1)) {
-		buf_state = gve_dequeue_buf_state(rx, &rx->dqo.used_buf_states);
-		if (gve_buf_ref_cnt(buf_state) == 0)
-			return buf_state;
-
-		gve_free_page_dqo(rx->gve, buf_state, true);
-		gve_free_buf_state(rx, buf_state);
-	}
-
-	return NULL;
-}
-
-static int gve_alloc_page_dqo(struct gve_rx_ring *rx,
-			      struct gve_rx_buf_state_dqo *buf_state)
-{
-	struct gve_priv *priv = rx->gve;
-	u32 idx;
-
-	if (!rx->dqo.qpl) {
-		int err;
-
-		err = gve_alloc_page(priv, &priv->pdev->dev,
-				     &buf_state->page_info.page,
-				     &buf_state->addr,
-				     DMA_FROM_DEVICE, GFP_ATOMIC);
-		if (err)
-			return err;
-	} else {
-		idx = rx->dqo.next_qpl_page_idx;
-		if (idx >= gve_get_rx_pages_per_qpl_dqo(priv->rx_desc_cnt)) {
-			net_err_ratelimited("%s: Out of QPL pages\n",
-					    priv->dev->name);
-			return -ENOMEM;
-		}
-		buf_state->page_info.page = rx->dqo.qpl->pages[idx];
-		buf_state->addr = rx->dqo.qpl->page_buses[idx];
-		rx->dqo.next_qpl_page_idx++;
-	}
-	buf_state->page_info.page_offset = 0;
-	buf_state->page_info.page_address =
-		page_address(buf_state->page_info.page);
-	buf_state->last_single_ref_offset = 0;
-
-	/* The page already has 1 ref. */
-	page_ref_add(buf_state->page_info.page, INT_MAX - 1);
-	buf_state->page_info.pagecnt_bias = INT_MAX;
-
-	return 0;
-}
-
 static void gve_rx_free_hdr_bufs(struct gve_priv *priv, struct gve_rx_ring *rx)
 {
 	struct device *hdev = &priv->pdev->dev;
@@ -557,48 +374,6 @@ void gve_rx_post_buffers_dqo(struct gve_rx_ring *rx)
 	rx->fill_cnt += num_posted;
 }
 
-static void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx,
-				struct gve_rx_buf_state_dqo *buf_state)
-{
-	const u16 data_buffer_size = priv->data_buffer_size_dqo;
-	int pagecount;
-
-	/* Can't reuse if we only fit one buffer per page */
-	if (data_buffer_size * 2 > PAGE_SIZE)
-		goto mark_used;
-
-	pagecount = gve_buf_ref_cnt(buf_state);
-
-	/* Record the offset when we have a single remaining reference.
-	 *
-	 * When this happens, we know all of the other offsets of the page are
-	 * usable.
-	 */
-	if (pagecount == 1) {
-		buf_state->last_single_ref_offset =
-			buf_state->page_info.page_offset;
-	}
-
-	/* Use the next buffer sized chunk in the page. */
-	buf_state->page_info.page_offset += data_buffer_size;
-	buf_state->page_info.page_offset &= (PAGE_SIZE - 1);
-
-	/* If we wrap around to the same offset without ever dropping to 1
-	 * reference, then we don't know if this offset was ever freed.
-	 */
-	if (buf_state->page_info.page_offset ==
-	    buf_state->last_single_ref_offset) {
-		goto mark_used;
-	}
-
-	gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, buf_state);
-	return;
-
-mark_used:
-	gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state);
-	rx->dqo.used_buf_states_cnt++;
-}
-
 static void gve_rx_skb_csum(struct sk_buff *skb,
 			    const struct gve_rx_compl_desc_dqo *desc,
 			    struct gve_ptype ptype)
-- 
2.46.0.598.g6f2099f65c-goog

[PATCH net-next 2/2] gve: adopt page pool for DQ RDA mode

[Praveen Kaligineedi]

From: Harshitha Ramamurthy <hramamurthy@google.com>

For DQ queue format in raw DMA addressing(RDA) mode,
implement page pool recycling of buffers by leveraging
a few helper functions.

DQ QPL mode will continue to use the exisiting recycling
logic. This is because in QPL mode, the pages come from a
constant set of pages that the driver pre-allocates and
registers with the device.

Reviewed-by: Praveen Kaligineedi <pkaligineedi@google.com>
Reviewed-by: Shailend Chand <shailend@google.com>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
 drivers/net/ethernet/google/Kconfig           |   1 +
 drivers/net/ethernet/google/gve/gve.h         |  15 +-
 .../ethernet/google/gve/gve_buffer_mgmt_dqo.c | 166 +++++++++++++-----
 drivers/net/ethernet/google/gve/gve_rx_dqo.c  |  88 +++++-----
 4 files changed, 180 insertions(+), 90 deletions(-)

diff --git a/drivers/net/ethernet/google/Kconfig b/drivers/net/ethernet/google/Kconfig
index 8641a00f8e63..564862a57124 100644
--- a/drivers/net/ethernet/google/Kconfig
+++ b/drivers/net/ethernet/google/Kconfig
@@ -18,6 +18,7 @@ if NET_VENDOR_GOOGLE
 config GVE
 	tristate "Google Virtual NIC (gVNIC) support"
 	depends on (PCI_MSI && (X86 || CPU_LITTLE_ENDIAN))
+	select PAGE_POOL
 	help
 	  This driver supports Google Virtual NIC (gVNIC)"
 
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index 387fd26ebc43..5012ed6fbdb4 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -13,6 +13,7 @@
 #include <linux/netdevice.h>
 #include <linux/pci.h>
 #include <linux/u64_stats_sync.h>
+#include <net/page_pool/helpers.h>
 #include <net/xdp.h>
 
 #include "gve_desc.h"
@@ -60,6 +61,8 @@
 
 #define GVE_DEFAULT_RX_BUFFER_OFFSET 2048
 
+#define GVE_PAGE_POOL_SIZE_MULTIPLIER 4
+
 #define GVE_FLOW_RULES_CACHE_SIZE \
 	(GVE_ADMINQ_BUFFER_SIZE / sizeof(struct gve_adminq_queried_flow_rule))
 #define GVE_FLOW_RULE_IDS_CACHE_SIZE \
@@ -102,6 +105,7 @@ struct gve_rx_slot_page_info {
 	struct page *page;
 	void *page_address;
 	u32 page_offset; /* offset to write to in page */
+	unsigned int buf_size;
 	int pagecnt_bias; /* expected pagecnt if only the driver has a ref */
 	u16 pad; /* adjustment for rx padding */
 	u8 can_flip; /* tracks if the networking stack is using the page */
@@ -273,6 +277,8 @@ struct gve_rx_ring {
 
 			/* Address info of the buffers for header-split */
 			struct gve_header_buf hdr_bufs;
+
+			struct page_pool *page_pool;
 		} dqo;
 	};
 
@@ -1173,9 +1179,16 @@ struct gve_rx_buf_state_dqo *gve_dequeue_buf_state(struct gve_rx_ring *rx,
 void gve_enqueue_buf_state(struct gve_rx_ring *rx, struct gve_index_list *list,
 			   struct gve_rx_buf_state_dqo *buf_state);
 struct gve_rx_buf_state_dqo *gve_get_recycled_buf_state(struct gve_rx_ring *rx);
-int gve_alloc_page_dqo(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state);
 void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx,
 			 struct gve_rx_buf_state_dqo *buf_state);
+void gve_free_to_page_pool(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state);
+int gve_alloc_qpl_page_dqo(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state);
+void gve_free_qpl_page_dqo(struct gve_rx_buf_state_dqo *buf_state);
+void gve_reuse_buffer(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state);
+void gve_free_buffer(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state);
+int gve_alloc_buffer(struct gve_rx_ring *rx, struct gve_rx_desc_dqo *desc);
+struct page_pool *gve_rx_create_page_pool(struct gve_priv *priv, struct gve_rx_ring *rx);
+
 /* Reset */
 void gve_schedule_reset(struct gve_priv *priv);
 int gve_reset(struct gve_priv *priv, bool attempt_teardown);
diff --git a/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c b/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c
index a8ea23b407ed..c26461cb7cf4 100644
--- a/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c
+++ b/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c
@@ -12,15 +12,6 @@ int gve_buf_ref_cnt(struct gve_rx_buf_state_dqo *bs)
 	return page_count(bs->page_info.page) - bs->page_info.pagecnt_bias;
 }
 
-void gve_free_page_dqo(struct gve_priv *priv, struct gve_rx_buf_state_dqo *bs, bool free_page)
-{
-	page_ref_sub(bs->page_info.page, bs->page_info.pagecnt_bias - 1);
-	if (free_page)
-		gve_free_page(&priv->pdev->dev, bs->page_info.page, bs->addr,
-			      DMA_FROM_DEVICE);
-	bs->page_info.page = NULL;
-}
-
 struct gve_rx_buf_state_dqo *gve_alloc_buf_state(struct gve_rx_ring *rx)
 {
 	struct gve_rx_buf_state_dqo *buf_state;
@@ -125,55 +116,27 @@ struct gve_rx_buf_state_dqo *gve_get_recycled_buf_state(struct gve_rx_ring *rx)
 		gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state);
 	}
 
-	/* For QPL, we cannot allocate any new buffers and must
-	 * wait for the existing ones to be available.
-	 */
-	if (rx->dqo.qpl)
-		return NULL;
-
-	/* If there are no free buf states discard an entry from
-	 * `used_buf_states` so it can be used.
-	 */
-	if (unlikely(rx->dqo.free_buf_states == -1)) {
-		buf_state = gve_dequeue_buf_state(rx, &rx->dqo.used_buf_states);
-		if (gve_buf_ref_cnt(buf_state) == 0)
-			return buf_state;
-
-		gve_free_page_dqo(rx->gve, buf_state, true);
-		gve_free_buf_state(rx, buf_state);
-	}
-
 	return NULL;
 }
 
-int gve_alloc_page_dqo(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state)
+int gve_alloc_qpl_page_dqo(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state)
 {
 	struct gve_priv *priv = rx->gve;
 	u32 idx;
 
-	if (!rx->dqo.qpl) {
-		int err;
-
-		err = gve_alloc_page(priv, &priv->pdev->dev,
-				     &buf_state->page_info.page,
-				     &buf_state->addr,
-				     DMA_FROM_DEVICE, GFP_ATOMIC);
-		if (err)
-			return err;
-	} else {
-		idx = rx->dqo.next_qpl_page_idx;
-		if (idx >= gve_get_rx_pages_per_qpl_dqo(priv->rx_desc_cnt)) {
-			net_err_ratelimited("%s: Out of QPL pages\n",
-					    priv->dev->name);
-			return -ENOMEM;
-		}
-		buf_state->page_info.page = rx->dqo.qpl->pages[idx];
-		buf_state->addr = rx->dqo.qpl->page_buses[idx];
-		rx->dqo.next_qpl_page_idx++;
+	idx = rx->dqo.next_qpl_page_idx;
+	if (idx >= gve_get_rx_pages_per_qpl_dqo(priv->rx_desc_cnt)) {
+		net_err_ratelimited("%s: Out of QPL pages\n",
+				    priv->dev->name);
+		return -ENOMEM;
 	}
+	buf_state->page_info.page = rx->dqo.qpl->pages[idx];
+	buf_state->addr = rx->dqo.qpl->page_buses[idx];
+	rx->dqo.next_qpl_page_idx++;
 	buf_state->page_info.page_offset = 0;
 	buf_state->page_info.page_address =
 		page_address(buf_state->page_info.page);
+	buf_state->page_info.buf_size = priv->data_buffer_size_dqo;
 	buf_state->last_single_ref_offset = 0;
 
 	/* The page already has 1 ref. */
@@ -183,6 +146,15 @@ int gve_alloc_page_dqo(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_
 	return 0;
 }
 
+void gve_free_qpl_page_dqo(struct gve_rx_buf_state_dqo *buf_state)
+{
+	if (!buf_state->page_info.page)
+		return;
+
+	page_ref_sub(buf_state->page_info.page, buf_state->page_info.pagecnt_bias - 1);
+	buf_state->page_info.page = NULL;
+}
+
 void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx,
 			 struct gve_rx_buf_state_dqo *buf_state)
 {
@@ -224,3 +196,103 @@ void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx,
 	gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state);
 	rx->dqo.used_buf_states_cnt++;
 }
+
+void gve_free_to_page_pool(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state)
+{
+	struct page *page = buf_state->page_info.page;
+
+	if (!page)
+		return;
+
+	page_pool_put_page(page->pp, page, buf_state->page_info.buf_size, true);
+	buf_state->page_info.page = NULL;
+}
+
+static int gve_alloc_from_page_pool(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state)
+{
+	struct gve_priv *priv = rx->gve;
+	struct page *page;
+
+	buf_state->page_info.buf_size = priv->data_buffer_size_dqo;
+	page = page_pool_alloc(rx->dqo.page_pool, &buf_state->page_info.page_offset,
+			       &buf_state->page_info.buf_size, GFP_ATOMIC);
+
+	if (!page) {
+		priv->page_alloc_fail++;
+		return -ENOMEM;
+	}
+
+	buf_state->page_info.page = page;
+	buf_state->page_info.page_address = page_address(page);
+	buf_state->addr = page_pool_get_dma_addr(page);
+
+	return 0;
+}
+
+struct page_pool *gve_rx_create_page_pool(struct gve_priv *priv, struct gve_rx_ring *rx)
+{
+	struct page_pool_params pp = {
+		.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
+		.order = 0,
+		.pool_size = GVE_PAGE_POOL_SIZE_MULTIPLIER * priv->rx_desc_cnt,
+		.dev = &priv->pdev->dev,
+		.netdev = priv->dev,
+		.max_len = PAGE_SIZE,
+		.dma_dir = DMA_FROM_DEVICE,
+	};
+
+	return page_pool_create(&pp);
+}
+
+void gve_free_buffer(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state)
+{
+	if (rx->dqo.page_pool) {
+		gve_free_to_page_pool(rx, buf_state);
+		gve_free_buf_state(rx, buf_state);
+	} else {
+		gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, buf_state);
+	}
+}
+
+void gve_reuse_buffer(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state)
+{
+	if (rx->dqo.page_pool) {
+		buf_state->page_info.page = NULL;
+		gve_free_buf_state(rx, buf_state);
+	} else {
+		gve_dec_pagecnt_bias(&buf_state->page_info);
+		gve_try_recycle_buf(rx->gve, rx, buf_state);
+	}
+}
+
+int gve_alloc_buffer(struct gve_rx_ring *rx, struct gve_rx_desc_dqo *desc)
+{
+	struct gve_rx_buf_state_dqo *buf_state;
+
+	if (rx->dqo.page_pool) {
+		buf_state = gve_alloc_buf_state(rx);
+		if (WARN_ON_ONCE(!buf_state))
+			return -ENOMEM;
+
+		if (gve_alloc_from_page_pool(rx, buf_state))
+			goto free_buf_state;
+	} else {
+		buf_state = gve_get_recycled_buf_state(rx);
+		if (unlikely(!buf_state)) {
+			buf_state = gve_alloc_buf_state(rx);
+			if (unlikely(!buf_state))
+				return -ENOMEM;
+
+			if (unlikely(gve_alloc_qpl_page_dqo(rx, buf_state)))
+				goto free_buf_state;
+		}
+	}
+	desc->buf_id = cpu_to_le16(buf_state - rx->dqo.buf_states);
+	desc->buf_addr = cpu_to_le64(buf_state->addr + buf_state->page_info.page_offset);
+
+	return 0;
+
+free_buf_state:
+	gve_free_buf_state(rx, buf_state);
+	return -ENOMEM;
+}
diff --git a/drivers/net/ethernet/google/gve/gve_rx_dqo.c b/drivers/net/ethernet/google/gve/gve_rx_dqo.c
index b343be2fb118..250c0302664c 100644
--- a/drivers/net/ethernet/google/gve/gve_rx_dqo.c
+++ b/drivers/net/ethernet/google/gve/gve_rx_dqo.c
@@ -95,8 +95,10 @@ static void gve_rx_reset_ring_dqo(struct gve_priv *priv, int idx)
 		for (i = 0; i < rx->dqo.num_buf_states; i++) {
 			struct gve_rx_buf_state_dqo *bs = &rx->dqo.buf_states[i];
 
-			if (bs->page_info.page)
-				gve_free_page_dqo(priv, bs, !rx->dqo.qpl);
+			if (rx->dqo.page_pool)
+				gve_free_to_page_pool(rx, bs);
+			else
+				gve_free_qpl_page_dqo(bs);
 		}
 	}
 
@@ -138,9 +140,11 @@ void gve_rx_free_ring_dqo(struct gve_priv *priv, struct gve_rx_ring *rx,
 
 	for (i = 0; i < rx->dqo.num_buf_states; i++) {
 		struct gve_rx_buf_state_dqo *bs = &rx->dqo.buf_states[i];
-		/* Only free page for RDA. QPL pages are freed in gve_main. */
-		if (bs->page_info.page)
-			gve_free_page_dqo(priv, bs, !rx->dqo.qpl);
+
+		if (rx->dqo.page_pool)
+			gve_free_to_page_pool(rx, bs);
+		else
+			gve_free_qpl_page_dqo(bs);
 	}
 
 	if (rx->dqo.qpl) {
@@ -167,6 +171,11 @@ void gve_rx_free_ring_dqo(struct gve_priv *priv, struct gve_rx_ring *rx,
 	kvfree(rx->dqo.buf_states);
 	rx->dqo.buf_states = NULL;
 
+	if (rx->dqo.page_pool) {
+		page_pool_destroy(rx->dqo.page_pool);
+		rx->dqo.page_pool = NULL;
+	}
+
 	gve_rx_free_hdr_bufs(priv, rx);
 
 	netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx);
@@ -199,6 +208,7 @@ int gve_rx_alloc_ring_dqo(struct gve_priv *priv,
 			  int idx)
 {
 	struct device *hdev = &priv->pdev->dev;
+	struct page_pool *pool;
 	int qpl_page_cnt;
 	size_t size;
 	u32 qpl_id;
@@ -212,8 +222,7 @@ int gve_rx_alloc_ring_dqo(struct gve_priv *priv,
 	rx->gve = priv;
 	rx->q_num = idx;
 
-	rx->dqo.num_buf_states = cfg->raw_addressing ?
-		min_t(s16, S16_MAX, buffer_queue_slots * 4) :
+	rx->dqo.num_buf_states = cfg->raw_addressing ? buffer_queue_slots :
 		gve_get_rx_pages_per_qpl_dqo(cfg->ring_size);
 	rx->dqo.buf_states = kvcalloc(rx->dqo.num_buf_states,
 				      sizeof(rx->dqo.buf_states[0]),
@@ -241,7 +250,13 @@ int gve_rx_alloc_ring_dqo(struct gve_priv *priv,
 	if (!rx->dqo.bufq.desc_ring)
 		goto err;
 
-	if (!cfg->raw_addressing) {
+	if (cfg->raw_addressing) {
+		pool = gve_rx_create_page_pool(priv, rx);
+		if (IS_ERR(pool))
+			goto err;
+
+		rx->dqo.page_pool = pool;
+	} else {
 		qpl_id = gve_get_rx_qpl_id(cfg->qcfg_tx, rx->q_num);
 		qpl_page_cnt = gve_get_rx_pages_per_qpl_dqo(cfg->ring_size);
 
@@ -338,26 +353,14 @@ void gve_rx_post_buffers_dqo(struct gve_rx_ring *rx)
 	num_avail_slots = min_t(u32, num_avail_slots, complq->num_free_slots);
 	while (num_posted < num_avail_slots) {
 		struct gve_rx_desc_dqo *desc = &bufq->desc_ring[bufq->tail];
-		struct gve_rx_buf_state_dqo *buf_state;
-
-		buf_state = gve_get_recycled_buf_state(rx);
-		if (unlikely(!buf_state)) {
-			buf_state = gve_alloc_buf_state(rx);
-			if (unlikely(!buf_state))
-				break;
-
-			if (unlikely(gve_alloc_page_dqo(rx, buf_state))) {
-				u64_stats_update_begin(&rx->statss);
-				rx->rx_buf_alloc_fail++;
-				u64_stats_update_end(&rx->statss);
-				gve_free_buf_state(rx, buf_state);
-				break;
-			}
+
+		if (unlikely(gve_alloc_buffer(rx, desc))) {
+			u64_stats_update_begin(&rx->statss);
+			rx->rx_buf_alloc_fail++;
+			u64_stats_update_end(&rx->statss);
+			break;
 		}
 
-		desc->buf_id = cpu_to_le16(buf_state - rx->dqo.buf_states);
-		desc->buf_addr = cpu_to_le64(buf_state->addr +
-					     buf_state->page_info.page_offset);
 		if (rx->dqo.hdr_bufs.data)
 			desc->header_buf_addr =
 				cpu_to_le64(rx->dqo.hdr_bufs.addr +
@@ -488,6 +491,9 @@ static int gve_rx_append_frags(struct napi_struct *napi,
 		if (!skb)
 			return -1;
 
+		if (rx->dqo.page_pool)
+			skb_mark_for_recycle(skb);
+
 		if (rx->ctx.skb_tail == rx->ctx.skb_head)
 			skb_shinfo(rx->ctx.skb_head)->frag_list = skb;
 		else
@@ -498,7 +504,7 @@ static int gve_rx_append_frags(struct napi_struct *napi,
 	if (rx->ctx.skb_tail != rx->ctx.skb_head) {
 		rx->ctx.skb_head->len += buf_len;
 		rx->ctx.skb_head->data_len += buf_len;
-		rx->ctx.skb_head->truesize += priv->data_buffer_size_dqo;
+		rx->ctx.skb_head->truesize += buf_state->page_info.buf_size;
 	}
 
 	/* Trigger ondemand page allocation if we are running low on buffers */
@@ -508,13 +514,9 @@ static int gve_rx_append_frags(struct napi_struct *napi,
 	skb_add_rx_frag(rx->ctx.skb_tail, num_frags,
 			buf_state->page_info.page,
 			buf_state->page_info.page_offset,
-			buf_len, priv->data_buffer_size_dqo);
-	gve_dec_pagecnt_bias(&buf_state->page_info);
+			buf_len, buf_state->page_info.buf_size);
 
-	/* Advances buffer page-offset if page is partially used.
-	 * Marks buffer as used if page is full.
-	 */
-	gve_try_recycle_buf(priv, rx, buf_state);
+	gve_reuse_buffer(rx, buf_state);
 	return 0;
 }
 
@@ -548,8 +550,7 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
 	}
 
 	if (unlikely(compl_desc->rx_error)) {
-		gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states,
-				      buf_state);
+		gve_free_buffer(rx, buf_state);
 		return -EINVAL;
 	}
 
@@ -573,6 +574,9 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
 			if (unlikely(!rx->ctx.skb_head))
 				goto error;
 			rx->ctx.skb_tail = rx->ctx.skb_head;
+
+			if (rx->dqo.page_pool)
+				skb_mark_for_recycle(rx->ctx.skb_head);
 		} else {
 			unsplit = 1;
 		}
@@ -609,8 +613,7 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
 		rx->rx_copybreak_pkt++;
 		u64_stats_update_end(&rx->statss);
 
-		gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states,
-				      buf_state);
+		gve_free_buffer(rx, buf_state);
 		return 0;
 	}
 
@@ -625,16 +628,17 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
 		return 0;
 	}
 
+	if (rx->dqo.page_pool)
+		skb_mark_for_recycle(rx->ctx.skb_head);
+
 	skb_add_rx_frag(rx->ctx.skb_head, 0, buf_state->page_info.page,
 			buf_state->page_info.page_offset, buf_len,
-			priv->data_buffer_size_dqo);
-	gve_dec_pagecnt_bias(&buf_state->page_info);
-
-	gve_try_recycle_buf(priv, rx, buf_state);
+			buf_state->page_info.buf_size);
+	gve_reuse_buffer(rx, buf_state);
 	return 0;
 
 error:
-	gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, buf_state);
+	gve_free_buffer(rx, buf_state);
 	return -ENOMEM;
 }
 
-- 
2.46.0.598.g6f2099f65c-goog

Re: [PATCH net-next 2/2] gve: adopt page pool for DQ RDA mode

[Nelson, Shannon]

On 9/10/2024 10:53 AM, Praveen Kaligineedi wrote:
> 
> From: Harshitha Ramamurthy <hramamurthy@google.com>

Hi Harshita!

> 
> For DQ queue format in raw DMA addressing(RDA) mode,
> implement page pool recycling of buffers by leveraging
> a few helper functions.
> 
> DQ QPL mode will continue to use the exisiting recycling

s/exisiting/existing/

> logic. This is because in QPL mode, the pages come from a
> constant set of pages that the driver pre-allocates and
> registers with the device.

I didn't look too deeply into this, but it looks like the call to 
xdp_rxq_info_reg_mem_model() didn't change from using 
MEM_TYPE_PAGE_SHARED to MEM_TYPE_PAGE_POOL.  Was this intentional?  Or 
did I miss it somewhere?  I suspect you'll want the machinery that goes 
with MEM_TYPE_PAGE_POOL.

Also, it seems this is always calling page_pool_put_page() with 
allow_direct set to true: this implies no locking is needed - e.g. 
you're in the napi softirq context.  Is this always true in the calling 
sequences?

Cheers,
sln

> 
> Reviewed-by: Praveen Kaligineedi <pkaligineedi@google.com>
> Reviewed-by: Shailend Chand <shailend@google.com>
> Reviewed-by: Willem de Bruijn <willemb@google.com>
> Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
> ---
>   drivers/net/ethernet/google/Kconfig           |   1 +
>   drivers/net/ethernet/google/gve/gve.h         |  15 +-
>   .../ethernet/google/gve/gve_buffer_mgmt_dqo.c | 166 +++++++++++++-----
>   drivers/net/ethernet/google/gve/gve_rx_dqo.c  |  88 +++++-----
>   4 files changed, 180 insertions(+), 90 deletions(-)
> 
> diff --git a/drivers/net/ethernet/google/Kconfig b/drivers/net/ethernet/google/Kconfig
> index 8641a00f8e63..564862a57124 100644
> --- a/drivers/net/ethernet/google/Kconfig
> +++ b/drivers/net/ethernet/google/Kconfig
> @@ -18,6 +18,7 @@ if NET_VENDOR_GOOGLE
>   config GVE
>          tristate "Google Virtual NIC (gVNIC) support"
>          depends on (PCI_MSI && (X86 || CPU_LITTLE_ENDIAN))
> +       select PAGE_POOL
>          help
>            This driver supports Google Virtual NIC (gVNIC)"
> 
> diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
> index 387fd26ebc43..5012ed6fbdb4 100644
> --- a/drivers/net/ethernet/google/gve/gve.h
> +++ b/drivers/net/ethernet/google/gve/gve.h
> @@ -13,6 +13,7 @@
>   #include <linux/netdevice.h>
>   #include <linux/pci.h>
>   #include <linux/u64_stats_sync.h>
> +#include <net/page_pool/helpers.h>
>   #include <net/xdp.h>
> 
>   #include "gve_desc.h"
> @@ -60,6 +61,8 @@
> 
>   #define GVE_DEFAULT_RX_BUFFER_OFFSET 2048
> 
> +#define GVE_PAGE_POOL_SIZE_MULTIPLIER 4
> +
>   #define GVE_FLOW_RULES_CACHE_SIZE \
>          (GVE_ADMINQ_BUFFER_SIZE / sizeof(struct gve_adminq_queried_flow_rule))
>   #define GVE_FLOW_RULE_IDS_CACHE_SIZE \
> @@ -102,6 +105,7 @@ struct gve_rx_slot_page_info {
>          struct page *page;
>          void *page_address;
>          u32 page_offset; /* offset to write to in page */
> +       unsigned int buf_size;
>          int pagecnt_bias; /* expected pagecnt if only the driver has a ref */
>          u16 pad; /* adjustment for rx padding */
>          u8 can_flip; /* tracks if the networking stack is using the page */
> @@ -273,6 +277,8 @@ struct gve_rx_ring {
> 
>                          /* Address info of the buffers for header-split */
>                          struct gve_header_buf hdr_bufs;
> +
> +                       struct page_pool *page_pool;
>                  } dqo;
>          };
> 
> @@ -1173,9 +1179,16 @@ struct gve_rx_buf_state_dqo *gve_dequeue_buf_state(struct gve_rx_ring *rx,
>   void gve_enqueue_buf_state(struct gve_rx_ring *rx, struct gve_index_list *list,
>                             struct gve_rx_buf_state_dqo *buf_state);
>   struct gve_rx_buf_state_dqo *gve_get_recycled_buf_state(struct gve_rx_ring *rx);
> -int gve_alloc_page_dqo(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state);
>   void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx,
>                           struct gve_rx_buf_state_dqo *buf_state);
> +void gve_free_to_page_pool(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state);
> +int gve_alloc_qpl_page_dqo(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state);
> +void gve_free_qpl_page_dqo(struct gve_rx_buf_state_dqo *buf_state);
> +void gve_reuse_buffer(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state);
> +void gve_free_buffer(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state);
> +int gve_alloc_buffer(struct gve_rx_ring *rx, struct gve_rx_desc_dqo *desc);
> +struct page_pool *gve_rx_create_page_pool(struct gve_priv *priv, struct gve_rx_ring *rx);
> +
>   /* Reset */
>   void gve_schedule_reset(struct gve_priv *priv);
>   int gve_reset(struct gve_priv *priv, bool attempt_teardown);
> diff --git a/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c b/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c
> index a8ea23b407ed..c26461cb7cf4 100644
> --- a/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c
> +++ b/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c
> @@ -12,15 +12,6 @@ int gve_buf_ref_cnt(struct gve_rx_buf_state_dqo *bs)
>          return page_count(bs->page_info.page) - bs->page_info.pagecnt_bias;
>   }
> 
> -void gve_free_page_dqo(struct gve_priv *priv, struct gve_rx_buf_state_dqo *bs, bool free_page)
> -{
> -       page_ref_sub(bs->page_info.page, bs->page_info.pagecnt_bias - 1);
> -       if (free_page)
> -               gve_free_page(&priv->pdev->dev, bs->page_info.page, bs->addr,
> -                             DMA_FROM_DEVICE);
> -       bs->page_info.page = NULL;
> -}
> -
>   struct gve_rx_buf_state_dqo *gve_alloc_buf_state(struct gve_rx_ring *rx)
>   {
>          struct gve_rx_buf_state_dqo *buf_state;
> @@ -125,55 +116,27 @@ struct gve_rx_buf_state_dqo *gve_get_recycled_buf_state(struct gve_rx_ring *rx)
>                  gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state);
>          }
> 
> -       /* For QPL, we cannot allocate any new buffers and must
> -        * wait for the existing ones to be available.
> -        */
> -       if (rx->dqo.qpl)
> -               return NULL;
> -
> -       /* If there are no free buf states discard an entry from
> -        * `used_buf_states` so it can be used.
> -        */
> -       if (unlikely(rx->dqo.free_buf_states == -1)) {
> -               buf_state = gve_dequeue_buf_state(rx, &rx->dqo.used_buf_states);
> -               if (gve_buf_ref_cnt(buf_state) == 0)
> -                       return buf_state;
> -
> -               gve_free_page_dqo(rx->gve, buf_state, true);
> -               gve_free_buf_state(rx, buf_state);
> -       }
> -
>          return NULL;
>   }
> 
> -int gve_alloc_page_dqo(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state)
> +int gve_alloc_qpl_page_dqo(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state)
>   {
>          struct gve_priv *priv = rx->gve;
>          u32 idx;
> 
> -       if (!rx->dqo.qpl) {
> -               int err;
> -
> -               err = gve_alloc_page(priv, &priv->pdev->dev,
> -                                    &buf_state->page_info.page,
> -                                    &buf_state->addr,
> -                                    DMA_FROM_DEVICE, GFP_ATOMIC);
> -               if (err)
> -                       return err;
> -       } else {
> -               idx = rx->dqo.next_qpl_page_idx;
> -               if (idx >= gve_get_rx_pages_per_qpl_dqo(priv->rx_desc_cnt)) {
> -                       net_err_ratelimited("%s: Out of QPL pages\n",
> -                                           priv->dev->name);
> -                       return -ENOMEM;
> -               }
> -               buf_state->page_info.page = rx->dqo.qpl->pages[idx];
> -               buf_state->addr = rx->dqo.qpl->page_buses[idx];
> -               rx->dqo.next_qpl_page_idx++;
> +       idx = rx->dqo.next_qpl_page_idx;
> +       if (idx >= gve_get_rx_pages_per_qpl_dqo(priv->rx_desc_cnt)) {
> +               net_err_ratelimited("%s: Out of QPL pages\n",
> +                                   priv->dev->name);
> +               return -ENOMEM;
>          }
> +       buf_state->page_info.page = rx->dqo.qpl->pages[idx];
> +       buf_state->addr = rx->dqo.qpl->page_buses[idx];
> +       rx->dqo.next_qpl_page_idx++;
>          buf_state->page_info.page_offset = 0;
>          buf_state->page_info.page_address =
>                  page_address(buf_state->page_info.page);
> +       buf_state->page_info.buf_size = priv->data_buffer_size_dqo;
>          buf_state->last_single_ref_offset = 0;
> 
>          /* The page already has 1 ref. */
> @@ -183,6 +146,15 @@ int gve_alloc_page_dqo(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_
>          return 0;
>   }
> 
> +void gve_free_qpl_page_dqo(struct gve_rx_buf_state_dqo *buf_state)
> +{
> +       if (!buf_state->page_info.page)
> +               return;
> +
> +       page_ref_sub(buf_state->page_info.page, buf_state->page_info.pagecnt_bias - 1);
> +       buf_state->page_info.page = NULL;
> +}
> +
>   void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx,
>                           struct gve_rx_buf_state_dqo *buf_state)
>   {
> @@ -224,3 +196,103 @@ void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx,
>          gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state);
>          rx->dqo.used_buf_states_cnt++;
>   }
> +
> +void gve_free_to_page_pool(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state)
> +{
> +       struct page *page = buf_state->page_info.page;
> +
> +       if (!page)
> +               return;
> +
> +       page_pool_put_page(page->pp, page, buf_state->page_info.buf_size, true);
> +       buf_state->page_info.page = NULL;
> +}
> +
> +static int gve_alloc_from_page_pool(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state)
> +{
> +       struct gve_priv *priv = rx->gve;
> +       struct page *page;
> +
> +       buf_state->page_info.buf_size = priv->data_buffer_size_dqo;
> +       page = page_pool_alloc(rx->dqo.page_pool, &buf_state->page_info.page_offset,
> +                              &buf_state->page_info.buf_size, GFP_ATOMIC);
> +
> +       if (!page) {
> +               priv->page_alloc_fail++;
> +               return -ENOMEM;
> +       }
> +
> +       buf_state->page_info.page = page;
> +       buf_state->page_info.page_address = page_address(page);
> +       buf_state->addr = page_pool_get_dma_addr(page);
> +
> +       return 0;
> +}
> +
> +struct page_pool *gve_rx_create_page_pool(struct gve_priv *priv, struct gve_rx_ring *rx)
> +{
> +       struct page_pool_params pp = {
> +               .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
> +               .order = 0,
> +               .pool_size = GVE_PAGE_POOL_SIZE_MULTIPLIER * priv->rx_desc_cnt,
> +               .dev = &priv->pdev->dev,
> +               .netdev = priv->dev,
> +               .max_len = PAGE_SIZE,
> +               .dma_dir = DMA_FROM_DEVICE,
> +       };
> +
> +       return page_pool_create(&pp);
> +}
> +
> +void gve_free_buffer(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state)
> +{
> +       if (rx->dqo.page_pool) {
> +               gve_free_to_page_pool(rx, buf_state);
> +               gve_free_buf_state(rx, buf_state);
> +       } else {
> +               gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, buf_state);
> +       }
> +}
> +
> +void gve_reuse_buffer(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state)
> +{
> +       if (rx->dqo.page_pool) {
> +               buf_state->page_info.page = NULL;
> +               gve_free_buf_state(rx, buf_state);
> +       } else {
> +               gve_dec_pagecnt_bias(&buf_state->page_info);
> +               gve_try_recycle_buf(rx->gve, rx, buf_state);
> +       }
> +}
> +
> +int gve_alloc_buffer(struct gve_rx_ring *rx, struct gve_rx_desc_dqo *desc)
> +{
> +       struct gve_rx_buf_state_dqo *buf_state;
> +
> +       if (rx->dqo.page_pool) {
> +               buf_state = gve_alloc_buf_state(rx);
> +               if (WARN_ON_ONCE(!buf_state))
> +                       return -ENOMEM;
> +
> +               if (gve_alloc_from_page_pool(rx, buf_state))
> +                       goto free_buf_state;
> +       } else {
> +               buf_state = gve_get_recycled_buf_state(rx);
> +               if (unlikely(!buf_state)) {
> +                       buf_state = gve_alloc_buf_state(rx);
> +                       if (unlikely(!buf_state))
> +                               return -ENOMEM;
> +
> +                       if (unlikely(gve_alloc_qpl_page_dqo(rx, buf_state)))
> +                               goto free_buf_state;
> +               }
> +       }
> +       desc->buf_id = cpu_to_le16(buf_state - rx->dqo.buf_states);
> +       desc->buf_addr = cpu_to_le64(buf_state->addr + buf_state->page_info.page_offset);
> +
> +       return 0;
> +
> +free_buf_state:
> +       gve_free_buf_state(rx, buf_state);
> +       return -ENOMEM;
> +}
> diff --git a/drivers/net/ethernet/google/gve/gve_rx_dqo.c b/drivers/net/ethernet/google/gve/gve_rx_dqo.c
> index b343be2fb118..250c0302664c 100644
> --- a/drivers/net/ethernet/google/gve/gve_rx_dqo.c
> +++ b/drivers/net/ethernet/google/gve/gve_rx_dqo.c
> @@ -95,8 +95,10 @@ static void gve_rx_reset_ring_dqo(struct gve_priv *priv, int idx)
>                  for (i = 0; i < rx->dqo.num_buf_states; i++) {
>                          struct gve_rx_buf_state_dqo *bs = &rx->dqo.buf_states[i];
> 
> -                       if (bs->page_info.page)
> -                               gve_free_page_dqo(priv, bs, !rx->dqo.qpl);
> +                       if (rx->dqo.page_pool)
> +                               gve_free_to_page_pool(rx, bs);
> +                       else
> +                               gve_free_qpl_page_dqo(bs);
>                  }
>          }
> 
> @@ -138,9 +140,11 @@ void gve_rx_free_ring_dqo(struct gve_priv *priv, struct gve_rx_ring *rx,
> 
>          for (i = 0; i < rx->dqo.num_buf_states; i++) {
>                  struct gve_rx_buf_state_dqo *bs = &rx->dqo.buf_states[i];
> -               /* Only free page for RDA. QPL pages are freed in gve_main. */
> -               if (bs->page_info.page)
> -                       gve_free_page_dqo(priv, bs, !rx->dqo.qpl);
> +
> +               if (rx->dqo.page_pool)
> +                       gve_free_to_page_pool(rx, bs);
> +               else
> +                       gve_free_qpl_page_dqo(bs);
>          }
> 
>          if (rx->dqo.qpl) {
> @@ -167,6 +171,11 @@ void gve_rx_free_ring_dqo(struct gve_priv *priv, struct gve_rx_ring *rx,
>          kvfree(rx->dqo.buf_states);
>          rx->dqo.buf_states = NULL;
> 
> +       if (rx->dqo.page_pool) {
> +               page_pool_destroy(rx->dqo.page_pool);
> +               rx->dqo.page_pool = NULL;
> +       }
> +
>          gve_rx_free_hdr_bufs(priv, rx);
> 
>          netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx);
> @@ -199,6 +208,7 @@ int gve_rx_alloc_ring_dqo(struct gve_priv *priv,
>                            int idx)
>   {
>          struct device *hdev = &priv->pdev->dev;
> +       struct page_pool *pool;
>          int qpl_page_cnt;
>          size_t size;
>          u32 qpl_id;
> @@ -212,8 +222,7 @@ int gve_rx_alloc_ring_dqo(struct gve_priv *priv,
>          rx->gve = priv;
>          rx->q_num = idx;
> 
> -       rx->dqo.num_buf_states = cfg->raw_addressing ?
> -               min_t(s16, S16_MAX, buffer_queue_slots * 4) :
> +       rx->dqo.num_buf_states = cfg->raw_addressing ? buffer_queue_slots :
>                  gve_get_rx_pages_per_qpl_dqo(cfg->ring_size);
>          rx->dqo.buf_states = kvcalloc(rx->dqo.num_buf_states,
>                                        sizeof(rx->dqo.buf_states[0]),
> @@ -241,7 +250,13 @@ int gve_rx_alloc_ring_dqo(struct gve_priv *priv,
>          if (!rx->dqo.bufq.desc_ring)
>                  goto err;
> 
> -       if (!cfg->raw_addressing) {
> +       if (cfg->raw_addressing) {
> +               pool = gve_rx_create_page_pool(priv, rx);
> +               if (IS_ERR(pool))
> +                       goto err;
> +
> +               rx->dqo.page_pool = pool;
> +       } else {
>                  qpl_id = gve_get_rx_qpl_id(cfg->qcfg_tx, rx->q_num);
>                  qpl_page_cnt = gve_get_rx_pages_per_qpl_dqo(cfg->ring_size);
> 
> @@ -338,26 +353,14 @@ void gve_rx_post_buffers_dqo(struct gve_rx_ring *rx)
>          num_avail_slots = min_t(u32, num_avail_slots, complq->num_free_slots);
>          while (num_posted < num_avail_slots) {
>                  struct gve_rx_desc_dqo *desc = &bufq->desc_ring[bufq->tail];
> -               struct gve_rx_buf_state_dqo *buf_state;
> -
> -               buf_state = gve_get_recycled_buf_state(rx);
> -               if (unlikely(!buf_state)) {
> -                       buf_state = gve_alloc_buf_state(rx);
> -                       if (unlikely(!buf_state))
> -                               break;
> -
> -                       if (unlikely(gve_alloc_page_dqo(rx, buf_state))) {
> -                               u64_stats_update_begin(&rx->statss);
> -                               rx->rx_buf_alloc_fail++;
> -                               u64_stats_update_end(&rx->statss);
> -                               gve_free_buf_state(rx, buf_state);
> -                               break;
> -                       }
> +
> +               if (unlikely(gve_alloc_buffer(rx, desc))) {
> +                       u64_stats_update_begin(&rx->statss);
> +                       rx->rx_buf_alloc_fail++;
> +                       u64_stats_update_end(&rx->statss);
> +                       break;
>                  }
> 
> -               desc->buf_id = cpu_to_le16(buf_state - rx->dqo.buf_states);
> -               desc->buf_addr = cpu_to_le64(buf_state->addr +
> -                                            buf_state->page_info.page_offset);
>                  if (rx->dqo.hdr_bufs.data)
>                          desc->header_buf_addr =
>                                  cpu_to_le64(rx->dqo.hdr_bufs.addr +
> @@ -488,6 +491,9 @@ static int gve_rx_append_frags(struct napi_struct *napi,
>                  if (!skb)
>                          return -1;
> 
> +               if (rx->dqo.page_pool)
> +                       skb_mark_for_recycle(skb);
> +
>                  if (rx->ctx.skb_tail == rx->ctx.skb_head)
>                          skb_shinfo(rx->ctx.skb_head)->frag_list = skb;
>                  else
> @@ -498,7 +504,7 @@ static int gve_rx_append_frags(struct napi_struct *napi,
>          if (rx->ctx.skb_tail != rx->ctx.skb_head) {
>                  rx->ctx.skb_head->len += buf_len;
>                  rx->ctx.skb_head->data_len += buf_len;
> -               rx->ctx.skb_head->truesize += priv->data_buffer_size_dqo;
> +               rx->ctx.skb_head->truesize += buf_state->page_info.buf_size;
>          }
> 
>          /* Trigger ondemand page allocation if we are running low on buffers */
> @@ -508,13 +514,9 @@ static int gve_rx_append_frags(struct napi_struct *napi,
>          skb_add_rx_frag(rx->ctx.skb_tail, num_frags,
>                          buf_state->page_info.page,
>                          buf_state->page_info.page_offset,
> -                       buf_len, priv->data_buffer_size_dqo);
> -       gve_dec_pagecnt_bias(&buf_state->page_info);
> +                       buf_len, buf_state->page_info.buf_size);
> 
> -       /* Advances buffer page-offset if page is partially used.
> -        * Marks buffer as used if page is full.
> -        */
> -       gve_try_recycle_buf(priv, rx, buf_state);
> +       gve_reuse_buffer(rx, buf_state);
>          return 0;
>   }
> 
> @@ -548,8 +550,7 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
>          }
> 
>          if (unlikely(compl_desc->rx_error)) {
> -               gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states,
> -                                     buf_state);
> +               gve_free_buffer(rx, buf_state);
>                  return -EINVAL;
>          }
> 
> @@ -573,6 +574,9 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
>                          if (unlikely(!rx->ctx.skb_head))
>                                  goto error;
>                          rx->ctx.skb_tail = rx->ctx.skb_head;
> +
> +                       if (rx->dqo.page_pool)
> +                               skb_mark_for_recycle(rx->ctx.skb_head);
>                  } else {
>                          unsplit = 1;
>                  }
> @@ -609,8 +613,7 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
>                  rx->rx_copybreak_pkt++;
>                  u64_stats_update_end(&rx->statss);
> 
> -               gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states,
> -                                     buf_state);
> +               gve_free_buffer(rx, buf_state);
>                  return 0;
>          }
> 
> @@ -625,16 +628,17 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
>                  return 0;
>          }
> 
> +       if (rx->dqo.page_pool)
> +               skb_mark_for_recycle(rx->ctx.skb_head);
> +
>          skb_add_rx_frag(rx->ctx.skb_head, 0, buf_state->page_info.page,
>                          buf_state->page_info.page_offset, buf_len,
> -                       priv->data_buffer_size_dqo);
> -       gve_dec_pagecnt_bias(&buf_state->page_info);
> -
> -       gve_try_recycle_buf(priv, rx, buf_state);
> +                       buf_state->page_info.buf_size);
> +       gve_reuse_buffer(rx, buf_state);
>          return 0;
> 
>   error:
> -       gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, buf_state);
> +       gve_free_buffer(rx, buf_state);
>          return -ENOMEM;
>   }
> 
> --
> 2.46.0.598.g6f2099f65c-goog
> 
> 

Re: [PATCH net-next 2/2] gve: adopt page pool for DQ RDA mode

[Jakub Kicinski]

On Tue, 10 Sep 2024 10:53:15 -0700 Praveen Kaligineedi wrote:
> +static int gve_alloc_from_page_pool(struct gve_rx_ring *rx, struct gve_rx_buf_state_dqo *buf_state)
> +{
> +	struct gve_priv *priv = rx->gve;
> +	struct page *page;
> +
> +	buf_state->page_info.buf_size = priv->data_buffer_size_dqo;
> +	page = page_pool_alloc(rx->dqo.page_pool, &buf_state->page_info.page_offset,
> +			       &buf_state->page_info.buf_size, GFP_ATOMIC);
> +
> +	if (!page) {
> +		priv->page_alloc_fail++;

Is this counter global to the device? No locking or atomicity needed?

> +struct page_pool *gve_rx_create_page_pool(struct gve_priv *priv, struct gve_rx_ring *rx)
> +{
> +	struct page_pool_params pp = {
> +		.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
> +		.order = 0,
> +		.pool_size = GVE_PAGE_POOL_SIZE_MULTIPLIER * priv->rx_desc_cnt,
> +		.dev = &priv->pdev->dev,
> +		.netdev = priv->dev,
> +		.max_len = PAGE_SIZE,
> +		.dma_dir = DMA_FROM_DEVICE,

Can the allocation run from process context in parallel with the NAPI
that uses the pool? It's uncommon for drivers to do that. If not you
can set the NAPI pointer here and get lock-free recycling.

> +	};
> +
> +	return page_pool_create(&pp);
> +}

Could you make sure to wrap the new code at 80 chars?

./scripts/checkpatch.pl --strict --max-line-length=80 
-- 
pw-bot: cr