* [PATCH net-next v2 01/15] gve: don't pass in unused parameter to gve_adminq_free
2026-06-02 23:58 [PATCH net-next v2 00/15] gve: AdminQ mode related refactors Harshitha Ramamurthy
@ 2026-06-02 23:58 ` Harshitha Ramamurthy
2026-06-02 23:58 ` [PATCH net-next v2 02/15] gve: refactor initialization with helper functions Harshitha Ramamurthy
` (13 subsequent siblings)
14 siblings, 0 replies; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-02 23:58 UTC (permalink / raw)
To: netdev
Cc: joshwash, hramamurthy, andrew+netdev, davem, edumazet, kuba,
pabeni, ast, daniel, hawk, john.fastabend, sdf, willemb,
jordanrhee, jfraker, nktgrg, bpf, linux-kernel
Clean up gve_adminq_free to not take in an unused parameter.
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Jordan Rhee <jordanrhee@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
drivers/net/ethernet/google/gve/gve_adminq.c | 2 +-
drivers/net/ethernet/google/gve/gve_adminq.h | 2 +-
drivers/net/ethernet/google/gve/gve_main.c | 4 ++--
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index 08587bf40ed4..70ffed8b52c3 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -385,7 +385,7 @@ void gve_adminq_release(struct gve_priv *priv)
gve_clear_admin_queue_ok(priv);
}
-void gve_adminq_free(struct device *dev, struct gve_priv *priv)
+void gve_adminq_free(struct gve_priv *priv)
{
if (!gve_get_admin_queue_ok(priv))
return;
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
index 22a74b6aa17e..8e80f36116ec 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.h
+++ b/drivers/net/ethernet/google/gve/gve_adminq.h
@@ -620,7 +620,7 @@ union gve_adminq_command {
static_assert(sizeof(union gve_adminq_command) == 64);
int gve_adminq_alloc(struct device *dev, struct gve_priv *priv);
-void gve_adminq_free(struct device *dev, struct gve_priv *priv);
+void gve_adminq_free(struct gve_priv *priv);
void gve_adminq_release(struct gve_priv *priv);
int gve_adminq_describe_device(struct gve_priv *priv);
int gve_adminq_configure_device_resources(struct gve_priv *priv,
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index e4d78ae52daf..30bf6df4ebc5 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -2506,14 +2506,14 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
bitmap_free(priv->xsk_pools);
priv->xsk_pools = NULL;
err:
- gve_adminq_free(&priv->pdev->dev, priv);
+ gve_adminq_free(priv);
return err;
}
static void gve_teardown_priv_resources(struct gve_priv *priv)
{
gve_teardown_device_resources(priv);
- gve_adminq_free(&priv->pdev->dev, priv);
+ gve_adminq_free(priv);
bitmap_free(priv->xsk_pools);
priv->xsk_pools = NULL;
}
--
2.54.0.1013.g208068f2d8-goog
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH net-next v2 02/15] gve: refactor initialization with helper functions
2026-06-02 23:58 [PATCH net-next v2 00/15] gve: AdminQ mode related refactors Harshitha Ramamurthy
2026-06-02 23:58 ` [PATCH net-next v2 01/15] gve: don't pass in unused parameter to gve_adminq_free Harshitha Ramamurthy
@ 2026-06-02 23:58 ` Harshitha Ramamurthy
2026-06-03 23:59 ` sashiko-bot
2026-06-05 3:41 ` Harshitha Ramamurthy
2026-06-02 23:58 ` [PATCH net-next v2 03/15] gve: introduce gve_adminq_get_device_properties() Harshitha Ramamurthy
` (12 subsequent siblings)
14 siblings, 2 replies; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-02 23:58 UTC (permalink / raw)
To: netdev
Cc: joshwash, hramamurthy, andrew+netdev, davem, edumazet, kuba,
pabeni, ast, daniel, hawk, john.fastabend, sdf, willemb,
jordanrhee, jfraker, nktgrg, bpf, linux-kernel
In the interest of commonizing code, refactor gve_probe()
and gve_init_priv() with a few helper functions that can
be expanded and utilized in upcoming patches that add the
mailbox ABI to the driver. The helper functions are:
- gve_set_num_ntfy_blks()
- gve_set_num_queues()
Reorder code to combine lines that accomplish a similar objective
like setting defaults. Move setting HW-GRO and UDP GSO support out
of an Adminq method into gve_init_priv().
These changes are just code movement, no functional change.
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Jordan Rhee <jordanrhee@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
drivers/net/ethernet/google/gve/gve_adminq.c | 8 --
drivers/net/ethernet/google/gve/gve_main.c | 108 +++++++++++--------
2 files changed, 65 insertions(+), 51 deletions(-)
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index 70ffed8b52c3..dfeabddfb396 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -1117,14 +1117,6 @@ int gve_adminq_describe_device(struct gve_priv *priv)
gve_set_default_rss_sizes(priv);
- /* DQO supports HW-GRO and UDP_GSO */
- if (gve_is_dqo(priv)) {
- u64 additional_features = NETIF_F_GRO_HW | NETIF_F_GSO_UDP_L4;
-
- priv->dev->hw_features |= additional_features;
- priv->dev->features |= additional_features;
- }
-
priv->max_registered_pages =
be64_to_cpu(descriptor->max_registered_pages);
mtu = be16_to_cpu(descriptor->mtu);
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 30bf6df4ebc5..b9542ef36b29 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -2398,65 +2398,32 @@ static const struct xdp_metadata_ops gve_xdp_metadata_ops = {
.xmo_rx_timestamp = gve_xdp_rx_timestamp,
};
-static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
+static int gve_set_num_ntfy_blks(struct gve_priv *priv)
{
int num_ntfy;
- int err;
-
- /* Set up the adminq */
- err = gve_adminq_alloc(&priv->pdev->dev, priv);
- if (err) {
- dev_err(&priv->pdev->dev,
- "Failed to alloc admin queue: err=%d\n", err);
- return err;
- }
-
- err = gve_verify_driver_compatibility(priv);
- if (err) {
- dev_err(&priv->pdev->dev,
- "Could not verify driver compatibility: err=%d\n", err);
- goto err;
- }
-
- priv->num_registered_pages = 0;
-
- if (skip_describe_device)
- goto setup_device;
- priv->queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED;
- /* Get the initial information we need from the device */
- err = gve_adminq_describe_device(priv);
- if (err) {
- dev_err(&priv->pdev->dev,
- "Could not get device information: err=%d\n", err);
- goto err;
- }
- priv->dev->mtu = priv->dev->max_mtu;
num_ntfy = pci_msix_vec_count(priv->pdev);
if (num_ntfy <= 0) {
dev_err(&priv->pdev->dev,
"could not count MSI-x vectors: err=%d\n", num_ntfy);
- err = num_ntfy;
- goto err;
+ return num_ntfy;
} else if (num_ntfy < GVE_MIN_MSIX) {
dev_err(&priv->pdev->dev, "gve needs at least %d MSI-x vectors, but only has %d\n",
GVE_MIN_MSIX, num_ntfy);
- err = -EINVAL;
- goto err;
+ return -EINVAL;
}
- /* Big TCP is only supported on DQO */
- if (!gve_is_gqi(priv))
- netif_set_tso_max_size(priv->dev, GVE_DQO_TX_MAX);
-
- priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK;
/* gvnic has one Notification Block per MSI-x vector, except for the
* management vector
*/
priv->num_ntfy_blks = (num_ntfy - 1) & ~0x1;
priv->mgmt_msix_idx = priv->num_ntfy_blks;
- priv->numa_node = dev_to_node(&priv->pdev->dev);
+ return 0;
+}
+
+static void gve_set_num_queues(struct gve_priv *priv)
+{
priv->tx_cfg.max_queues =
min_t(int, priv->tx_cfg.max_queues, priv->num_ntfy_blks / 2);
priv->rx_cfg.max_queues =
@@ -2470,18 +2437,73 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
priv->rx_cfg.num_queues = min_t(int, priv->default_num_queues,
priv->rx_cfg.num_queues);
}
- priv->tx_cfg.num_xdp_queues = 0;
dev_info(&priv->pdev->dev, "TX queues %d, RX queues %d\n",
priv->tx_cfg.num_queues, priv->rx_cfg.num_queues);
dev_info(&priv->pdev->dev, "Max TX queues %d, Max RX queues %d\n",
priv->tx_cfg.max_queues, priv->rx_cfg.max_queues);
+}
+
+static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
+{
+ int err;
+
+ /* Set up the adminq */
+ err = gve_adminq_alloc(&priv->pdev->dev, priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Failed to alloc admin queue: err=%d\n", err);
+ return err;
+ }
+
+ priv->num_registered_pages = 0;
+
+ if (skip_describe_device)
+ goto setup_device;
+
+ err = gve_verify_driver_compatibility(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Could not verify driver compatibility: err=%d\n", err);
+ goto err;
+ }
+
+ /* Get the initial information we need from the device */
+ priv->queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED;
+ err = gve_adminq_describe_device(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Could not get device information: err=%d\n", err);
+ goto err;
+ }
+
+ err = gve_set_num_ntfy_blks(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Could not setup notify blocks: err=%d\n", err);
+ goto err;
+ }
+
+ gve_set_num_queues(priv);
+
+ if (gve_is_dqo(priv)) {
+ /* DQO supports HW-GRO and UDP_GSO */
+ u64 additional_features = NETIF_F_GRO_HW | NETIF_F_GSO_UDP_L4;
+
+ priv->dev->hw_features |= additional_features;
+ priv->dev->features |= additional_features;
- if (!gve_is_gqi(priv)) {
priv->tx_coalesce_usecs = GVE_TX_IRQ_RATELIMIT_US_DQO;
priv->rx_coalesce_usecs = GVE_RX_IRQ_RATELIMIT_US_DQO;
+
+ /* Big TCP is only supported on DQO */
+ netif_set_tso_max_size(priv->dev, GVE_DQO_TX_MAX);
}
+ priv->dev->mtu = priv->dev->max_mtu;
+ priv->numa_node = dev_to_node(&priv->pdev->dev);
+ priv->tx_cfg.num_xdp_queues = 0;
+ priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK;
priv->ts_config.tx_type = HWTSTAMP_TX_OFF;
priv->ts_config.rx_filter = HWTSTAMP_FILTER_NONE;
--
2.54.0.1013.g208068f2d8-goog
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH net-next v2 02/15] gve: refactor initialization with helper functions
2026-06-02 23:58 ` [PATCH net-next v2 02/15] gve: refactor initialization with helper functions Harshitha Ramamurthy
@ 2026-06-03 23:59 ` sashiko-bot
2026-06-05 3:41 ` Harshitha Ramamurthy
1 sibling, 0 replies; 33+ messages in thread
From: sashiko-bot @ 2026-06-03 23:59 UTC (permalink / raw)
To: Harshitha Ramamurthy; +Cc: bpf
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [High] gve_verify_driver_compatibility() is incorrectly skipped during device reset recovery, causing the device to lose driver capability context.
--
commit 8cee6a7e999b446ef0c02854dce95bbc44f444e8
Author: Harshitha Ramamurthy <hramamurthy@google.com>
gve: refactor initialization with helper functions
This commit refactors gve_probe() and gve_init_priv() to commonize code
using helper functions gve_set_num_ntfy_blks() and gve_set_num_queues().
It also reorders code to combine lines that accomplish similar objectives
and moves setting HW-GRO and UDP GSO support into gve_init_priv().
> diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
> index 30bf6df4ebc58..b9542ef36b297 100644
> --- a/drivers/net/ethernet/google/gve/gve_main.c
> +++ b/drivers/net/ethernet/google/gve/gve_main.c
[ ... ]
> @@ -2470,18 +2437,73 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
[ ... ]
> +static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
> +{
> + int err;
> +
> + /* Set up the adminq */
> + err = gve_adminq_alloc(&priv->pdev->dev, priv);
> + if (err) {
> + dev_err(&priv->pdev->dev,
> + "Failed to alloc admin queue: err=%d\n", err);
> + return err;
> + }
> +
> + priv->num_registered_pages = 0;
> +
> + if (skip_describe_device)
> + goto setup_device;
> +
> + err = gve_verify_driver_compatibility(priv);
[Severity: High]
Does moving this driver compatibility check below the early-exit bypass the
critical capability handshake during reset recovery?
When a device reset occurs, the hardware clears its internal state. In the
baseline code, gve_verify_driver_compatibility() was called unconditionally
at the beginning of gve_init_priv(), ensuring the device was repopulated
with the driver's capabilities.
By placing it after the skip_describe_device check, the driver capability
handshake appears to be skipped during reset recovery. This leaves the
hardware unaware of the driver's features, which can cause it to reject
subsequent configuration commands or misinterpret descriptors, potentially
resulting in a denial of service until the module is reloaded.
Is it intentional to omit this handshake during recovery?
--
Sashiko AI review · https://sashiko.dev/#/patchset/20260602235900.3050782-1-hramamurthy@google.com?part=2
^ permalink raw reply [flat|nested] 33+ messages in thread* Re: [PATCH net-next v2 02/15] gve: refactor initialization with helper functions
2026-06-02 23:58 ` [PATCH net-next v2 02/15] gve: refactor initialization with helper functions Harshitha Ramamurthy
2026-06-03 23:59 ` sashiko-bot
@ 2026-06-05 3:41 ` Harshitha Ramamurthy
1 sibling, 0 replies; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-05 3:41 UTC (permalink / raw)
To: netdev
Cc: joshwash, andrew+netdev, davem, edumazet, kuba, pabeni, ast,
daniel, hawk, john.fastabend, sdf, willemb, jordanrhee, jfraker,
nktgrg, bpf, linux-kernel
On Tue, Jun 2, 2026 at 4:59 PM Harshitha Ramamurthy
<hramamurthy@google.com> wrote:
>
> In the interest of commonizing code, refactor gve_probe()
> and gve_init_priv() with a few helper functions that can
> be expanded and utilized in upcoming patches that add the
> mailbox ABI to the driver. The helper functions are:
>
> - gve_set_num_ntfy_blks()
> - gve_set_num_queues()
>
> Reorder code to combine lines that accomplish a similar objective
> like setting defaults. Move setting HW-GRO and UDP GSO support out
> of an Adminq method into gve_init_priv().
>
> These changes are just code movement, no functional change.
>
> Reviewed-by: Willem de Bruijn <willemb@google.com>
> Reviewed-by: Jordan Rhee <jordanrhee@google.com>
> Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
> ---
> drivers/net/ethernet/google/gve/gve_adminq.c | 8 --
> drivers/net/ethernet/google/gve/gve_main.c | 108 +++++++++++--------
> 2 files changed, 65 insertions(+), 51 deletions(-)
>
> diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
> index 70ffed8b52c3..dfeabddfb396 100644
> --- a/drivers/net/ethernet/google/gve/gve_adminq.c
> +++ b/drivers/net/ethernet/google/gve/gve_adminq.c
> @@ -1117,14 +1117,6 @@ int gve_adminq_describe_device(struct gve_priv *priv)
>
> gve_set_default_rss_sizes(priv);
>
> - /* DQO supports HW-GRO and UDP_GSO */
> - if (gve_is_dqo(priv)) {
> - u64 additional_features = NETIF_F_GRO_HW | NETIF_F_GSO_UDP_L4;
> -
> - priv->dev->hw_features |= additional_features;
> - priv->dev->features |= additional_features;
> - }
> -
> priv->max_registered_pages =
> be64_to_cpu(descriptor->max_registered_pages);
> mtu = be16_to_cpu(descriptor->mtu);
> diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
> index 30bf6df4ebc5..b9542ef36b29 100644
> --- a/drivers/net/ethernet/google/gve/gve_main.c
> +++ b/drivers/net/ethernet/google/gve/gve_main.c
> @@ -2398,65 +2398,32 @@ static const struct xdp_metadata_ops gve_xdp_metadata_ops = {
> .xmo_rx_timestamp = gve_xdp_rx_timestamp,
> };
>
> -static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
> +static int gve_set_num_ntfy_blks(struct gve_priv *priv)
> {
> int num_ntfy;
> - int err;
> -
> - /* Set up the adminq */
> - err = gve_adminq_alloc(&priv->pdev->dev, priv);
> - if (err) {
> - dev_err(&priv->pdev->dev,
> - "Failed to alloc admin queue: err=%d\n", err);
> - return err;
> - }
> -
> - err = gve_verify_driver_compatibility(priv);
> - if (err) {
> - dev_err(&priv->pdev->dev,
> - "Could not verify driver compatibility: err=%d\n", err);
> - goto err;
> - }
> -
> - priv->num_registered_pages = 0;
> -
> - if (skip_describe_device)
> - goto setup_device;
>
> - priv->queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED;
> - /* Get the initial information we need from the device */
> - err = gve_adminq_describe_device(priv);
> - if (err) {
> - dev_err(&priv->pdev->dev,
> - "Could not get device information: err=%d\n", err);
> - goto err;
> - }
> - priv->dev->mtu = priv->dev->max_mtu;
> num_ntfy = pci_msix_vec_count(priv->pdev);
> if (num_ntfy <= 0) {
> dev_err(&priv->pdev->dev,
> "could not count MSI-x vectors: err=%d\n", num_ntfy);
> - err = num_ntfy;
> - goto err;
> + return num_ntfy;
> } else if (num_ntfy < GVE_MIN_MSIX) {
> dev_err(&priv->pdev->dev, "gve needs at least %d MSI-x vectors, but only has %d\n",
> GVE_MIN_MSIX, num_ntfy);
> - err = -EINVAL;
> - goto err;
> + return -EINVAL;
> }
>
> - /* Big TCP is only supported on DQO */
> - if (!gve_is_gqi(priv))
> - netif_set_tso_max_size(priv->dev, GVE_DQO_TX_MAX);
> -
> - priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK;
> /* gvnic has one Notification Block per MSI-x vector, except for the
> * management vector
> */
> priv->num_ntfy_blks = (num_ntfy - 1) & ~0x1;
> priv->mgmt_msix_idx = priv->num_ntfy_blks;
> - priv->numa_node = dev_to_node(&priv->pdev->dev);
>
> + return 0;
> +}
> +
> +static void gve_set_num_queues(struct gve_priv *priv)
> +{
> priv->tx_cfg.max_queues =
> min_t(int, priv->tx_cfg.max_queues, priv->num_ntfy_blks / 2);
> priv->rx_cfg.max_queues =
> @@ -2470,18 +2437,73 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
> priv->rx_cfg.num_queues = min_t(int, priv->default_num_queues,
> priv->rx_cfg.num_queues);
> }
> - priv->tx_cfg.num_xdp_queues = 0;
>
> dev_info(&priv->pdev->dev, "TX queues %d, RX queues %d\n",
> priv->tx_cfg.num_queues, priv->rx_cfg.num_queues);
> dev_info(&priv->pdev->dev, "Max TX queues %d, Max RX queues %d\n",
> priv->tx_cfg.max_queues, priv->rx_cfg.max_queues);
> +}
> +
> +static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
> +{
> + int err;
> +
> + /* Set up the adminq */
> + err = gve_adminq_alloc(&priv->pdev->dev, priv);
> + if (err) {
> + dev_err(&priv->pdev->dev,
> + "Failed to alloc admin queue: err=%d\n", err);
> + return err;
> + }
> +
> + priv->num_registered_pages = 0;
> +
> + if (skip_describe_device)
> + goto setup_device;
> +
> + err = gve_verify_driver_compatibility(priv);
> + if (err) {
> + dev_err(&priv->pdev->dev,
> + "Could not verify driver compatibility: err=%d\n", err);
> + goto err;
> + }
Addressing Sashiko's report:
https://sashiko.dev/#/patchset/20260602235900.3050782-1-hramamurthy%40google.com
"Does moving this driver compatibility check below the early-exit bypass the
critical capability handshake during reset recovery?"
Yes, this needs to also be called in the reset path. Will fix in v3.
> +
> + /* Get the initial information we need from the device */
> + priv->queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED;
> + err = gve_adminq_describe_device(priv);
> + if (err) {
> + dev_err(&priv->pdev->dev,
> + "Could not get device information: err=%d\n", err);
> + goto err;
> + }
> +
> + err = gve_set_num_ntfy_blks(priv);
> + if (err) {
> + dev_err(&priv->pdev->dev,
> + "Could not setup notify blocks: err=%d\n", err);
> + goto err;
> + }
> +
> + gve_set_num_queues(priv);
> +
> + if (gve_is_dqo(priv)) {
> + /* DQO supports HW-GRO and UDP_GSO */
> + u64 additional_features = NETIF_F_GRO_HW | NETIF_F_GSO_UDP_L4;
> +
> + priv->dev->hw_features |= additional_features;
> + priv->dev->features |= additional_features;
>
> - if (!gve_is_gqi(priv)) {
> priv->tx_coalesce_usecs = GVE_TX_IRQ_RATELIMIT_US_DQO;
> priv->rx_coalesce_usecs = GVE_RX_IRQ_RATELIMIT_US_DQO;
> +
> + /* Big TCP is only supported on DQO */
> + netif_set_tso_max_size(priv->dev, GVE_DQO_TX_MAX);
> }
>
> + priv->dev->mtu = priv->dev->max_mtu;
> + priv->numa_node = dev_to_node(&priv->pdev->dev);
> + priv->tx_cfg.num_xdp_queues = 0;
> + priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK;
> priv->ts_config.tx_type = HWTSTAMP_TX_OFF;
> priv->ts_config.rx_filter = HWTSTAMP_FILTER_NONE;
>
> --
> 2.54.0.1013.g208068f2d8-goog
>
^ permalink raw reply [flat|nested] 33+ messages in thread
* [PATCH net-next v2 03/15] gve: introduce gve_adminq_get_device_properties()
2026-06-02 23:58 [PATCH net-next v2 00/15] gve: AdminQ mode related refactors Harshitha Ramamurthy
2026-06-02 23:58 ` [PATCH net-next v2 01/15] gve: don't pass in unused parameter to gve_adminq_free Harshitha Ramamurthy
2026-06-02 23:58 ` [PATCH net-next v2 02/15] gve: refactor initialization with helper functions Harshitha Ramamurthy
@ 2026-06-02 23:58 ` Harshitha Ramamurthy
2026-06-02 23:58 ` [PATCH net-next v2 04/15] gve: add a few helper functions to set device properties Harshitha Ramamurthy
` (11 subsequent siblings)
14 siblings, 0 replies; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-02 23:58 UTC (permalink / raw)
To: netdev
Cc: joshwash, hramamurthy, andrew+netdev, davem, edumazet, kuba,
pabeni, ast, daniel, hawk, john.fastabend, sdf, willemb,
jordanrhee, jfraker, nktgrg, bpf, linux-kernel
Introduce gve_adminq_get_device_properties() which executes the first
two Adminq commands: VERIFY_DRIVER_COMPATIBILITY and DESCRIBE_DEVICE
so that this can be called during initialization.
Move these to Adminq specific files. This is just code movement, no
functional change.
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Jordan Rhee <jordanrhee@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
drivers/net/ethernet/google/gve/gve.h | 2 +
drivers/net/ethernet/google/gve/gve_adminq.c | 66 ++++++++++++++++++--
drivers/net/ethernet/google/gve/gve_adminq.h | 5 +-
drivers/net/ethernet/google/gve/gve_main.c | 59 +----------------
4 files changed, 66 insertions(+), 66 deletions(-)
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index 1d66d3834f7e..645175a5dfae 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -15,6 +15,8 @@
#include <linux/pci.h>
#include <linux/ptp_clock_kernel.h>
#include <linux/u64_stats_sync.h>
+#include <linux/utsname.h>
+#include <linux/version.h>
#include <net/page_pool/helpers.h>
#include <net/xdp.h>
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index dfeabddfb396..860d43650c6a 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -1144,6 +1144,27 @@ int gve_adminq_describe_device(struct gve_priv *priv)
return err;
}
+int gve_adminq_get_device_properties(struct gve_priv *priv)
+{
+ int err;
+
+ err = gve_adminq_verify_driver_compatibility(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Could not verify driver compatibility: err=%d\n", err);
+ return err;
+ }
+
+ /* Get the initial information we need from the device */
+ err = gve_adminq_describe_device(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Could not get device information: err=%d\n", err);
+ return err;
+ }
+ return 0;
+}
+
int gve_adminq_register_page_list(struct gve_priv *priv,
struct gve_queue_page_list *qpl)
{
@@ -1206,20 +1227,53 @@ int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len,
return gve_adminq_execute_cmd(priv, &cmd);
}
-int gve_adminq_verify_driver_compatibility(struct gve_priv *priv,
- u64 driver_info_len,
- dma_addr_t driver_info_addr)
+int gve_adminq_verify_driver_compatibility(struct gve_priv *priv)
{
+ struct gve_driver_info *driver_info;
union gve_adminq_command cmd;
+ dma_addr_t driver_info_bus;
+ int err;
+
+ driver_info = dma_alloc_coherent(&priv->pdev->dev,
+ sizeof(struct gve_driver_info),
+ &driver_info_bus, GFP_KERNEL);
+ if (!driver_info)
+ return -ENOMEM;
+
+ *driver_info = (struct gve_driver_info) {
+ .os_type = 1, /* Linux */
+ .os_version_major = cpu_to_be32(LINUX_VERSION_MAJOR),
+ .os_version_minor = cpu_to_be32(LINUX_VERSION_SUBLEVEL),
+ .os_version_sub = cpu_to_be32(LINUX_VERSION_PATCHLEVEL),
+ .driver_capability_flags = {
+ cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS1),
+ cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS2),
+ cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS3),
+ cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS4),
+ },
+ };
+ strscpy(driver_info->os_version_str1, utsname()->release,
+ sizeof(driver_info->os_version_str1));
+ strscpy(driver_info->os_version_str2, utsname()->version,
+ sizeof(driver_info->os_version_str2));
memset(&cmd, 0, sizeof(cmd));
cmd.opcode = cpu_to_be32(GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY);
cmd.verify_driver_compatibility = (struct gve_adminq_verify_driver_compatibility) {
- .driver_info_len = cpu_to_be64(driver_info_len),
- .driver_info_addr = cpu_to_be64(driver_info_addr),
+ .driver_info_len = cpu_to_be64(sizeof(struct gve_driver_info)),
+ .driver_info_addr = cpu_to_be64(driver_info_bus),
};
- return gve_adminq_execute_cmd(priv, &cmd);
+ err = gve_adminq_execute_cmd(priv, &cmd);
+
+ /* It's ok if the device doesn't support this */
+ if (err == -EOPNOTSUPP)
+ err = 0;
+
+ dma_free_coherent(&priv->pdev->dev,
+ sizeof(struct gve_driver_info),
+ driver_info, driver_info_bus);
+ return err;
}
int gve_adminq_report_link_speed(struct gve_priv *priv)
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
index 8e80f36116ec..831759d30f44 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.h
+++ b/drivers/net/ethernet/google/gve/gve_adminq.h
@@ -640,9 +640,8 @@ int gve_adminq_register_page_list(struct gve_priv *priv,
int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id);
int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len,
dma_addr_t stats_report_addr, u64 interval);
-int gve_adminq_verify_driver_compatibility(struct gve_priv *priv,
- u64 driver_info_len,
- dma_addr_t driver_info_addr);
+int gve_adminq_verify_driver_compatibility(struct gve_priv *priv);
+int gve_adminq_get_device_properties(struct gve_priv *priv);
int gve_adminq_report_link_speed(struct gve_priv *priv);
int gve_adminq_add_flow_rule(struct gve_priv *priv, struct gve_adminq_flow_rule *rule, u32 loc);
int gve_adminq_del_flow_rule(struct gve_priv *priv, u32 loc);
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index b9542ef36b29..eb2d768e5881 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -17,7 +17,6 @@
#include <linux/sched.h>
#include <linux/timer.h>
#include <linux/workqueue.h>
-#include <linux/utsname.h>
#include <linux/version.h>
#include <net/netdev_queues.h>
#include <net/sch_generic.h>
@@ -41,49 +40,6 @@ char gve_driver_name[] = "gve";
const char gve_version_str[] = GVE_VERSION;
static const char gve_version_prefix[] = GVE_VERSION_PREFIX;
-static int gve_verify_driver_compatibility(struct gve_priv *priv)
-{
- int err;
- struct gve_driver_info *driver_info;
- dma_addr_t driver_info_bus;
-
- driver_info = dma_alloc_coherent(&priv->pdev->dev,
- sizeof(struct gve_driver_info),
- &driver_info_bus, GFP_KERNEL);
- if (!driver_info)
- return -ENOMEM;
-
- *driver_info = (struct gve_driver_info) {
- .os_type = 1, /* Linux */
- .os_version_major = cpu_to_be32(LINUX_VERSION_MAJOR),
- .os_version_minor = cpu_to_be32(LINUX_VERSION_SUBLEVEL),
- .os_version_sub = cpu_to_be32(LINUX_VERSION_PATCHLEVEL),
- .driver_capability_flags = {
- cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS1),
- cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS2),
- cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS3),
- cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS4),
- },
- };
- strscpy(driver_info->os_version_str1, utsname()->release,
- sizeof(driver_info->os_version_str1));
- strscpy(driver_info->os_version_str2, utsname()->version,
- sizeof(driver_info->os_version_str2));
-
- err = gve_adminq_verify_driver_compatibility(priv,
- sizeof(struct gve_driver_info),
- driver_info_bus);
-
- /* It's ok if the device doesn't support this */
- if (err == -EOPNOTSUPP)
- err = 0;
-
- dma_free_coherent(&priv->pdev->dev,
- sizeof(struct gve_driver_info),
- driver_info, driver_info_bus);
- return err;
-}
-
static netdev_features_t gve_features_check(struct sk_buff *skb,
struct net_device *dev,
netdev_features_t features)
@@ -2461,21 +2417,10 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
if (skip_describe_device)
goto setup_device;
- err = gve_verify_driver_compatibility(priv);
- if (err) {
- dev_err(&priv->pdev->dev,
- "Could not verify driver compatibility: err=%d\n", err);
- goto err;
- }
-
- /* Get the initial information we need from the device */
priv->queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED;
- err = gve_adminq_describe_device(priv);
- if (err) {
- dev_err(&priv->pdev->dev,
- "Could not get device information: err=%d\n", err);
+ err = gve_adminq_get_device_properties(priv);
+ if (err)
goto err;
- }
err = gve_set_num_ntfy_blks(priv);
if (err) {
--
2.54.0.1013.g208068f2d8-goog
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH net-next v2 04/15] gve: add a few helper functions to set device properties
2026-06-02 23:58 [PATCH net-next v2 00/15] gve: AdminQ mode related refactors Harshitha Ramamurthy
` (2 preceding siblings ...)
2026-06-02 23:58 ` [PATCH net-next v2 03/15] gve: introduce gve_adminq_get_device_properties() Harshitha Ramamurthy
@ 2026-06-02 23:58 ` Harshitha Ramamurthy
2026-06-02 23:58 ` [PATCH net-next v2 05/15] gve: add struct gve_device_info to hold " Harshitha Ramamurthy
` (10 subsequent siblings)
14 siblings, 0 replies; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-02 23:58 UTC (permalink / raw)
To: netdev
Cc: joshwash, hramamurthy, andrew+netdev, davem, edumazet, kuba,
pabeni, ast, daniel, hawk, john.fastabend, sdf, willemb,
jordanrhee, jfraker, nktgrg, bpf, linux-kernel
For the mailbox ABI, device properties will come from a different
source compared to the AdminQ mode. To accommodate the new source
when the mailbox ABI is added, add a few helper functions to set a
few device properties. Those functions are:
- gve_set_queue_properties() to set no. of pages for QPL mode and
number of queues in general
- gve_set_mtu()
- gve_set_mac()
This is just code movement, no functional change.
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Jordan Rhee <jordanrhee@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
drivers/net/ethernet/google/gve/gve_adminq.c | 38 +++-------------
drivers/net/ethernet/google/gve/gve_adminq.h | 6 +++
drivers/net/ethernet/google/gve/gve_main.c | 46 ++++++++++++++++++++
3 files changed, 59 insertions(+), 31 deletions(-)
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index 860d43650c6a..77203184efbd 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -920,19 +920,6 @@ int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 num_queues)
return err;
}
-static void gve_set_default_desc_cnt(struct gve_priv *priv,
- const struct gve_device_descriptor *descriptor)
-{
- priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
- priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
-
- /* set default ranges */
- priv->max_tx_desc_cnt = priv->tx_desc_cnt;
- priv->max_rx_desc_cnt = priv->rx_desc_cnt;
- priv->min_tx_desc_cnt = priv->tx_desc_cnt;
- priv->min_rx_desc_cnt = priv->rx_desc_cnt;
-}
-
static void gve_set_default_rss_sizes(struct gve_priv *priv)
{
if (!gve_is_gqi(priv)) {
@@ -1049,8 +1036,6 @@ int gve_adminq_describe_device(struct gve_priv *priv)
union gve_adminq_command cmd;
dma_addr_t descriptor_bus;
int err = 0;
- u8 *mac;
- u16 mtu;
memset(&cmd, 0, sizeof(cmd));
descriptor = dma_pool_alloc(priv->adminq_pool, GFP_KERNEL,
@@ -1112,26 +1097,17 @@ int gve_adminq_describe_device(struct gve_priv *priv)
"Driver is running with GQI QPL queue format.\n");
}
- /* set default descriptor counts */
- gve_set_default_desc_cnt(priv, descriptor);
-
gve_set_default_rss_sizes(priv);
- priv->max_registered_pages =
- be64_to_cpu(descriptor->max_registered_pages);
- mtu = be16_to_cpu(descriptor->mtu);
- if (mtu < ETH_MIN_MTU) {
- dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu);
- err = -EINVAL;
+ err = gve_set_mtu(priv, descriptor);
+ if (err)
goto free_device_descriptor;
- }
- priv->dev->max_mtu = mtu;
+
priv->num_event_counters = be16_to_cpu(descriptor->counters);
- eth_hw_addr_set(priv->dev, descriptor->mac);
- mac = descriptor->mac;
- dev_info(&priv->pdev->dev, "MAC addr: %pM\n", mac);
- priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl);
- priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues);
+
+ gve_set_mac(priv, descriptor);
+
+ gve_set_queue_properties(priv, descriptor);
gve_enable_supported_features(priv, supported_features_mask,
dev_op_jumbo_frames, dev_op_dqo_qpl,
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
index 831759d30f44..ed5cb052a52d 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.h
+++ b/drivers/net/ethernet/google/gve/gve_adminq.h
@@ -655,5 +655,11 @@ int gve_adminq_report_nic_ts(struct gve_priv *priv,
struct gve_ptype_lut;
int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
struct gve_ptype_lut *ptype_lut);
+void gve_set_queue_properties(struct gve_priv *priv,
+ struct gve_device_descriptor *descriptor);
+int gve_set_mtu(struct gve_priv *priv,
+ struct gve_device_descriptor *descriptor);
+void gve_set_mac(struct gve_priv *priv,
+ struct gve_device_descriptor *descriptor);
#endif /* _GVE_ADMINQ_H */
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index eb2d768e5881..338dc0b3249a 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -2400,6 +2400,52 @@ static void gve_set_num_queues(struct gve_priv *priv)
priv->tx_cfg.max_queues, priv->rx_cfg.max_queues);
}
+static void gve_set_default_desc_cnt(struct gve_priv *priv,
+ const struct gve_device_descriptor *descriptor)
+{
+ priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
+ priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
+
+ /* set default ranges */
+ priv->max_tx_desc_cnt = priv->tx_desc_cnt;
+ priv->max_rx_desc_cnt = priv->rx_desc_cnt;
+ priv->min_tx_desc_cnt = priv->tx_desc_cnt;
+ priv->min_rx_desc_cnt = priv->rx_desc_cnt;
+}
+
+void gve_set_queue_properties(struct gve_priv *priv,
+ struct gve_device_descriptor *descriptor)
+{
+ /* set default descriptor counts */
+ gve_set_default_desc_cnt(priv, descriptor);
+
+ priv->max_registered_pages = be64_to_cpu(descriptor->max_registered_pages);
+ priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl);
+ priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues);
+}
+
+int gve_set_mtu(struct gve_priv *priv,
+ struct gve_device_descriptor *descriptor)
+{
+ u16 mtu;
+
+ mtu = be16_to_cpu(descriptor->mtu);
+ if (mtu < ETH_MIN_MTU) {
+ dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu);
+ return -EINVAL;
+ }
+ priv->dev->max_mtu = mtu;
+
+ return 0;
+}
+
+void gve_set_mac(struct gve_priv *priv,
+ struct gve_device_descriptor *descriptor)
+{
+ eth_hw_addr_set(priv->dev, descriptor->mac);
+ dev_info(&priv->pdev->dev, "MAC addr: %pM\n", descriptor->mac);
+}
+
static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
{
int err;
--
2.54.0.1013.g208068f2d8-goog
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH net-next v2 05/15] gve: add struct gve_device_info to hold device properties
2026-06-02 23:58 [PATCH net-next v2 00/15] gve: AdminQ mode related refactors Harshitha Ramamurthy
` (3 preceding siblings ...)
2026-06-02 23:58 ` [PATCH net-next v2 04/15] gve: add a few helper functions to set device properties Harshitha Ramamurthy
@ 2026-06-02 23:58 ` Harshitha Ramamurthy
2026-06-03 23:59 ` sashiko-bot
2026-06-05 3:44 ` Harshitha Ramamurthy
2026-06-02 23:58 ` [PATCH net-next v2 06/15] gve: introduce control plane operations structure Harshitha Ramamurthy
` (9 subsequent siblings)
14 siblings, 2 replies; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-02 23:58 UTC (permalink / raw)
To: netdev
Cc: joshwash, hramamurthy, andrew+netdev, davem, edumazet, kuba,
pabeni, ast, daniel, hawk, john.fastabend, sdf, willemb,
jordanrhee, jfraker, nktgrg, bpf, linux-kernel
In the current AdminQ mode, device properties are written into
struct gve_device_descriptor that is allocated in shared memory
between the driver and device. In the upcoming MailboxQ mode,
these properties will be returned in the response of a mailbox
message. Hence, add struct gve_device_info as the structure that
holds all the properties that are negotiated with the device in
either mode.
Change the AdminQ mode method gve_adminq_describe_device()
and the functions it calls to fill up device information
into this newly introduced struct gve_device_info. Move a
few helper functions and code that set device properties
in the priv structure into gve_init_priv(). So now
gve_init_priv() calls/does the following:
- gve_set_mtu()
- gve_set_mac()
- gve_set_queue_properties()
- gve_set_buf_sizes()
- set flow steering and RSS properties
- set other priv properties
When MailboxQ support is added, device information will be filled
into the same structure and the same gve_init_priv() path would be
used to set device properties to ensure common code reusage.
These changes are refactors only, no functional change.
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Jordan Rhee <jordanrhee@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
drivers/net/ethernet/google/gve/gve.h | 29 +++++
drivers/net/ethernet/google/gve/gve_adminq.c | 113 +++++++++++--------
drivers/net/ethernet/google/gve/gve_adminq.h | 6 -
drivers/net/ethernet/google/gve/gve_main.c | 101 ++++++++++++-----
4 files changed, 169 insertions(+), 80 deletions(-)
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index 645175a5dfae..086a315f42c1 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -796,6 +796,34 @@ struct gve_ptp {
struct gve_priv *priv;
};
+struct gve_device_info {
+ enum gve_queue_format queue_format;
+ u16 default_tx_queues;
+ u16 default_rx_queues;
+ u16 max_tx_queues;
+ u16 max_rx_queues;
+ u16 default_tx_ring_size;
+ u16 default_rx_ring_size;
+ u16 max_tx_ring_size;
+ u16 max_rx_ring_size;
+ u16 min_tx_ring_size;
+ u16 min_rx_ring_size;
+ u16 max_mtu;
+ u8 mac[ETH_ALEN];
+ u16 max_rx_buffer_size;
+ u16 header_buf_size;
+ u32 max_flow_rules;
+ u16 rss_key_size;
+ u16 rss_lut_size;
+ u16 tx_pages_per_qpl;
+ u16 num_event_counters;
+ u64 max_registered_pages;
+ bool default_min_ring_size;
+ bool nic_timestamp_supported;
+ bool modify_ring_size_enabled;
+ bool cache_rss_config;
+};
+
struct gve_priv {
struct net_device *dev;
struct gve_tx_ring *tx; /* array of tx_cfg.num_queues */
@@ -928,6 +956,7 @@ struct gve_priv {
struct gve_nic_ts_report *nic_ts_report;
dma_addr_t nic_ts_report_bus;
u64 last_sync_nic_counter; /* Clock counter from last NIC TS report */
+ struct gve_device_info device_info;
};
enum gve_service_task_flags_bit {
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index 77203184efbd..06db2ccd0b89 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -70,7 +70,7 @@ void gve_parse_device_option(struct gve_priv *priv,
dev_info(&priv->pdev->dev,
"Gqi raw addressing device option enabled.\n");
- priv->queue_format = GVE_GQI_RDA_FORMAT;
+ priv->device_info.queue_format = GVE_GQI_RDA_FORMAT;
break;
case GVE_DEV_OPT_ID_GQI_RDA:
if (option_length < sizeof(**dev_op_gqi_rda) ||
@@ -190,7 +190,7 @@ void gve_parse_device_option(struct gve_priv *priv,
/* device has not provided min ring size */
if (option_length == GVE_DEVICE_OPTION_NO_MIN_RING_SIZE)
- priv->default_min_ring_size = true;
+ priv->device_info.default_min_ring_size = true;
break;
case GVE_DEV_OPT_ID_FLOW_STEERING:
if (option_length < sizeof(**dev_op_flow_steering) ||
@@ -922,10 +922,13 @@ int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 num_queues)
static void gve_set_default_rss_sizes(struct gve_priv *priv)
{
- if (!gve_is_gqi(priv)) {
- priv->rss_key_size = GVE_RSS_KEY_SIZE;
- priv->rss_lut_size = GVE_RSS_INDIR_SIZE;
- priv->cache_rss_config = true;
+ struct gve_device_info *device_info = &priv->device_info;
+
+ if (device_info->queue_format == GVE_DQO_RDA_FORMAT ||
+ device_info->queue_format == GVE_DQO_QPL_FORMAT) {
+ device_info->rss_key_size = GVE_RSS_KEY_SIZE;
+ device_info->rss_lut_size = GVE_RSS_INDIR_SIZE;
+ device_info->cache_rss_config = true;
}
}
@@ -946,77 +949,105 @@ static void gve_enable_supported_features(struct gve_priv *priv,
const struct gve_device_option_modify_ring
*dev_op_modify_ring)
{
+ struct gve_device_info *info = &priv->device_info;
+
/* Before control reaches this point, the page-size-capped max MTU from
* the gve_device_descriptor field has already been stored in
- * priv->dev->max_mtu. We overwrite it with the true max MTU below.
+ * device_info->max_mtu. We overwrite it with the true max MTU below.
*/
if (dev_op_jumbo_frames &&
(supported_features_mask & GVE_SUP_JUMBO_FRAMES_MASK)) {
dev_info(&priv->pdev->dev,
"JUMBO FRAMES device option enabled.\n");
- priv->dev->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu);
+ info->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu);
}
if (dev_op_buffer_sizes &&
(supported_features_mask & GVE_SUP_BUFFER_SIZES_MASK)) {
- priv->max_rx_buffer_size =
+ info->max_rx_buffer_size =
be16_to_cpu(dev_op_buffer_sizes->packet_buffer_size);
- priv->header_buf_size =
+ info->header_buf_size =
be16_to_cpu(dev_op_buffer_sizes->header_buffer_size);
dev_info(&priv->pdev->dev,
"BUFFER SIZES device option enabled with max_rx_buffer_size of %u, header_buf_size of %u.\n",
- priv->max_rx_buffer_size, priv->header_buf_size);
- if (gve_is_dqo(priv) &&
- priv->max_rx_buffer_size > GVE_DEFAULT_RX_BUFFER_SIZE)
- priv->rx_cfg.packet_buffer_size =
- priv->max_rx_buffer_size;
+ info->max_rx_buffer_size, info->header_buf_size);
}
/* Read and store ring size ranges given by device */
if (dev_op_modify_ring &&
(supported_features_mask & GVE_SUP_MODIFY_RING_MASK)) {
- priv->modify_ring_size_enabled = true;
- priv->max_rx_desc_cnt =
+ info->modify_ring_size_enabled = true;
+ info->max_rx_ring_size =
be16_to_cpu(dev_op_modify_ring->max_rx_ring_size);
- priv->max_tx_desc_cnt =
+ info->max_tx_ring_size =
be16_to_cpu(dev_op_modify_ring->max_tx_ring_size);
if (priv->default_min_ring_size) {
/* If device hasn't provided minimums, use default minimums */
- priv->min_tx_desc_cnt = GVE_DEFAULT_MIN_TX_RING_SIZE;
- priv->min_rx_desc_cnt = GVE_DEFAULT_MIN_RX_RING_SIZE;
+ info->min_tx_ring_size = GVE_DEFAULT_MIN_TX_RING_SIZE;
+ info->min_rx_ring_size = GVE_DEFAULT_MIN_RX_RING_SIZE;
} else {
- priv->min_rx_desc_cnt = be16_to_cpu(dev_op_modify_ring->min_rx_ring_size);
- priv->min_tx_desc_cnt = be16_to_cpu(dev_op_modify_ring->min_tx_ring_size);
+ info->min_rx_ring_size =
+ be16_to_cpu(dev_op_modify_ring->min_rx_ring_size);
+ info->min_tx_ring_size =
+ be16_to_cpu(dev_op_modify_ring->min_tx_ring_size);
}
}
if (dev_op_flow_steering &&
(supported_features_mask & GVE_SUP_FLOW_STEERING_MASK)) {
if (dev_op_flow_steering->max_flow_rules) {
- priv->max_flow_rules =
+ info->max_flow_rules =
be32_to_cpu(dev_op_flow_steering->max_flow_rules);
- priv->dev->hw_features |= NETIF_F_NTUPLE;
dev_info(&priv->pdev->dev,
"FLOW STEERING device option enabled with max rule limit of %u.\n",
- priv->max_flow_rules);
+ info->max_flow_rules);
}
}
if (dev_op_rss_config &&
(supported_features_mask & GVE_SUP_RSS_CONFIG_MASK)) {
- priv->rss_key_size =
+ info->rss_key_size =
be16_to_cpu(dev_op_rss_config->hash_key_size);
- priv->rss_lut_size =
+ info->rss_lut_size =
be16_to_cpu(dev_op_rss_config->hash_lut_size);
- priv->cache_rss_config = false;
+ info->cache_rss_config = false;
dev_dbg(&priv->pdev->dev,
"RSS device option enabled with key size of %u, lut size of %u.\n",
- priv->rss_key_size, priv->rss_lut_size);
+ info->rss_key_size, info->rss_lut_size);
}
if (dev_op_nic_timestamp &&
(supported_features_mask & GVE_SUP_NIC_TIMESTAMP_MASK))
- priv->nic_timestamp_supported = true;
+ info->nic_timestamp_supported = true;
+}
+
+static void gve_fill_device_info(struct gve_priv *priv,
+ struct gve_device_descriptor *descriptor)
+{
+ struct gve_device_info *device_info = &priv->device_info;
+ u16 default_num_queues;
+
+ device_info->tx_pages_per_qpl =
+ be16_to_cpu(descriptor->tx_pages_per_qpl);
+ device_info->max_registered_pages =
+ be64_to_cpu(descriptor->max_registered_pages);
+ device_info->num_event_counters = be16_to_cpu(descriptor->counters);
+ ether_addr_copy(device_info->mac, descriptor->mac);
+ device_info->max_mtu = be16_to_cpu(descriptor->mtu);
+
+ default_num_queues = be16_to_cpu(descriptor->default_num_queues);
+ device_info->default_tx_queues = default_num_queues;
+ device_info->default_rx_queues = default_num_queues;
+ device_info->default_tx_ring_size =
+ be16_to_cpu(descriptor->tx_queue_entries);
+ device_info->default_rx_ring_size =
+ be16_to_cpu(descriptor->rx_queue_entries);
+
+ /* set default ranges */
+ device_info->max_tx_ring_size = device_info->default_tx_ring_size;
+ device_info->max_rx_ring_size = device_info->default_rx_ring_size;
+ device_info->min_tx_ring_size = device_info->default_tx_ring_size;
+ device_info->min_rx_ring_size = device_info->default_rx_ring_size;
}
int gve_adminq_describe_device(struct gve_priv *priv)
@@ -1027,6 +1058,7 @@ int gve_adminq_describe_device(struct gve_priv *priv)
struct gve_device_option_jumbo_frames *dev_op_jumbo_frames = NULL;
struct gve_device_option_modify_ring *dev_op_modify_ring = NULL;
struct gve_device_option_rss_config *dev_op_rss_config = NULL;
+ struct gve_device_info *device_info = &priv->device_info;
struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL;
struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL;
struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL;
@@ -1070,26 +1102,26 @@ int gve_adminq_describe_device(struct gve_priv *priv)
* DqoRda, DqoQpl, GqiRda, GqiQpl. Use GqiQpl as default.
*/
if (dev_op_dqo_rda) {
- priv->queue_format = GVE_DQO_RDA_FORMAT;
+ device_info->queue_format = GVE_DQO_RDA_FORMAT;
dev_info(&priv->pdev->dev,
"Driver is running with DQO RDA queue format.\n");
supported_features_mask =
be32_to_cpu(dev_op_dqo_rda->supported_features_mask);
} else if (dev_op_dqo_qpl) {
- priv->queue_format = GVE_DQO_QPL_FORMAT;
+ device_info->queue_format = GVE_DQO_QPL_FORMAT;
supported_features_mask =
be32_to_cpu(dev_op_dqo_qpl->supported_features_mask);
} else if (dev_op_gqi_rda) {
- priv->queue_format = GVE_GQI_RDA_FORMAT;
+ device_info->queue_format = GVE_GQI_RDA_FORMAT;
dev_info(&priv->pdev->dev,
"Driver is running with GQI RDA queue format.\n");
supported_features_mask =
be32_to_cpu(dev_op_gqi_rda->supported_features_mask);
- } else if (priv->queue_format == GVE_GQI_RDA_FORMAT) {
+ } else if (device_info->queue_format == GVE_GQI_RDA_FORMAT) {
dev_info(&priv->pdev->dev,
"Driver is running with GQI RDA queue format.\n");
} else {
- priv->queue_format = GVE_GQI_QPL_FORMAT;
+ device_info->queue_format = GVE_GQI_QPL_FORMAT;
if (dev_op_gqi_qpl)
supported_features_mask =
be32_to_cpu(dev_op_gqi_qpl->supported_features_mask);
@@ -1097,18 +1129,9 @@ int gve_adminq_describe_device(struct gve_priv *priv)
"Driver is running with GQI QPL queue format.\n");
}
+ gve_fill_device_info(priv, descriptor);
gve_set_default_rss_sizes(priv);
- err = gve_set_mtu(priv, descriptor);
- if (err)
- goto free_device_descriptor;
-
- priv->num_event_counters = be16_to_cpu(descriptor->counters);
-
- gve_set_mac(priv, descriptor);
-
- gve_set_queue_properties(priv, descriptor);
-
gve_enable_supported_features(priv, supported_features_mask,
dev_op_jumbo_frames, dev_op_dqo_qpl,
dev_op_buffer_sizes, dev_op_flow_steering,
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
index ed5cb052a52d..831759d30f44 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.h
+++ b/drivers/net/ethernet/google/gve/gve_adminq.h
@@ -655,11 +655,5 @@ int gve_adminq_report_nic_ts(struct gve_priv *priv,
struct gve_ptype_lut;
int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
struct gve_ptype_lut *ptype_lut);
-void gve_set_queue_properties(struct gve_priv *priv,
- struct gve_device_descriptor *descriptor);
-int gve_set_mtu(struct gve_priv *priv,
- struct gve_device_descriptor *descriptor);
-void gve_set_mac(struct gve_priv *priv,
- struct gve_device_descriptor *descriptor);
#endif /* _GVE_ADMINQ_H */
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 338dc0b3249a..1aeee916471f 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -2380,6 +2380,8 @@ static int gve_set_num_ntfy_blks(struct gve_priv *priv)
static void gve_set_num_queues(struct gve_priv *priv)
{
+ struct gve_device_info *device_info = &priv->device_info;
+
priv->tx_cfg.max_queues =
min_t(int, priv->tx_cfg.max_queues, priv->num_ntfy_blks / 2);
priv->rx_cfg.max_queues =
@@ -2387,12 +2389,14 @@ static void gve_set_num_queues(struct gve_priv *priv)
priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
- if (priv->default_num_queues > 0) {
- priv->tx_cfg.num_queues = min_t(int, priv->default_num_queues,
+ if (device_info->default_tx_queues > 0)
+ priv->tx_cfg.num_queues = min_t(int,
+ device_info->default_tx_queues,
priv->tx_cfg.num_queues);
- priv->rx_cfg.num_queues = min_t(int, priv->default_num_queues,
+ if (device_info->default_rx_queues > 0)
+ priv->rx_cfg.num_queues = min_t(int,
+ device_info->default_rx_queues,
priv->rx_cfg.num_queues);
- }
dev_info(&priv->pdev->dev, "TX queues %d, RX queues %d\n",
priv->tx_cfg.num_queues, priv->rx_cfg.num_queues);
@@ -2400,54 +2404,69 @@ static void gve_set_num_queues(struct gve_priv *priv)
priv->tx_cfg.max_queues, priv->rx_cfg.max_queues);
}
-static void gve_set_default_desc_cnt(struct gve_priv *priv,
- const struct gve_device_descriptor *descriptor)
+static void gve_set_desc_cnt(struct gve_priv *priv)
{
- priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
- priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
+ struct gve_device_info *device_info = &priv->device_info;
- /* set default ranges */
- priv->max_tx_desc_cnt = priv->tx_desc_cnt;
- priv->max_rx_desc_cnt = priv->rx_desc_cnt;
- priv->min_tx_desc_cnt = priv->tx_desc_cnt;
- priv->min_rx_desc_cnt = priv->rx_desc_cnt;
+ priv->tx_desc_cnt = device_info->default_tx_ring_size;
+ priv->rx_desc_cnt = device_info->default_rx_ring_size;
+ priv->max_tx_desc_cnt = device_info->max_tx_ring_size;
+ priv->max_rx_desc_cnt = device_info->max_rx_ring_size;
+ priv->min_tx_desc_cnt = device_info->min_tx_ring_size;
+ priv->min_rx_desc_cnt = device_info->min_rx_ring_size;
}
-void gve_set_queue_properties(struct gve_priv *priv,
- struct gve_device_descriptor *descriptor)
+static void gve_set_queue_properties(struct gve_priv *priv)
{
- /* set default descriptor counts */
- gve_set_default_desc_cnt(priv, descriptor);
+ struct gve_device_info *device_info = &priv->device_info;
- priv->max_registered_pages = be64_to_cpu(descriptor->max_registered_pages);
- priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl);
- priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues);
+ gve_set_desc_cnt(priv);
+ priv->max_registered_pages = device_info->max_registered_pages;
+ priv->tx_pages_per_qpl = device_info->tx_pages_per_qpl;
}
-int gve_set_mtu(struct gve_priv *priv,
- struct gve_device_descriptor *descriptor)
+static int gve_set_mtu(struct gve_priv *priv)
{
+ struct gve_device_info *device_info = &priv->device_info;
u16 mtu;
- mtu = be16_to_cpu(descriptor->mtu);
+ mtu = device_info->max_mtu;
if (mtu < ETH_MIN_MTU) {
dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu);
return -EINVAL;
}
priv->dev->max_mtu = mtu;
+ priv->dev->mtu = priv->dev->max_mtu;
return 0;
}
-void gve_set_mac(struct gve_priv *priv,
- struct gve_device_descriptor *descriptor)
+static void gve_set_mac(struct gve_priv *priv)
{
- eth_hw_addr_set(priv->dev, descriptor->mac);
- dev_info(&priv->pdev->dev, "MAC addr: %pM\n", descriptor->mac);
+ struct gve_device_info *device_info = &priv->device_info;
+
+ eth_hw_addr_set(priv->dev, device_info->mac);
+ dev_info(&priv->pdev->dev, "MAC addr: %pM\n", device_info->mac);
+}
+
+static void gve_set_buf_sizes(struct gve_priv *priv)
+{
+ struct gve_device_info *device_info = &priv->device_info;
+
+ if (device_info->max_rx_buffer_size > priv->max_rx_buffer_size)
+ priv->max_rx_buffer_size = device_info->max_rx_buffer_size;
+
+ if (gve_is_dqo(priv) &&
+ priv->max_rx_buffer_size > GVE_DEFAULT_RX_BUFFER_SIZE)
+ priv->rx_cfg.packet_buffer_size = priv->max_rx_buffer_size;
+
+ if (device_info->header_buf_size)
+ priv->header_buf_size = device_info->header_buf_size;
}
static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
{
+ struct gve_device_info *device_info = &priv->device_info;
int err;
/* Set up the adminq */
@@ -2463,11 +2482,13 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
if (skip_describe_device)
goto setup_device;
- priv->queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED;
+ device_info->queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED;
err = gve_adminq_get_device_properties(priv);
if (err)
goto err;
+ priv->queue_format = priv->device_info.queue_format;
+
err = gve_set_num_ntfy_blks(priv);
if (err) {
dev_err(&priv->pdev->dev,
@@ -2491,12 +2512,34 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
netif_set_tso_max_size(priv->dev, GVE_DQO_TX_MAX);
}
- priv->dev->mtu = priv->dev->max_mtu;
+ if (gve_set_mtu(priv)) {
+ err = -EINVAL;
+ goto err;
+ }
+
+ priv->num_event_counters = device_info->num_event_counters;
+
+ gve_set_mac(priv);
+
+ gve_set_queue_properties(priv);
+ priv->modify_ring_size_enabled = device_info->modify_ring_size_enabled;
+
+ gve_set_buf_sizes(priv);
+
+ priv->max_flow_rules = device_info->max_flow_rules;
+ if (priv->max_flow_rules)
+ priv->dev->hw_features |= NETIF_F_NTUPLE;
+
+ priv->rss_key_size = device_info->rss_key_size;
+ priv->rss_lut_size = device_info->rss_lut_size;
+ priv->cache_rss_config = device_info->cache_rss_config;
+
priv->numa_node = dev_to_node(&priv->pdev->dev);
priv->tx_cfg.num_xdp_queues = 0;
priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK;
priv->ts_config.tx_type = HWTSTAMP_TX_OFF;
priv->ts_config.rx_filter = HWTSTAMP_FILTER_NONE;
+ priv->nic_timestamp_supported = device_info->nic_timestamp_supported;
setup_device:
priv->xsk_pools = bitmap_zalloc(priv->rx_cfg.max_queues, GFP_KERNEL);
--
2.54.0.1013.g208068f2d8-goog
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH net-next v2 05/15] gve: add struct gve_device_info to hold device properties
2026-06-02 23:58 ` [PATCH net-next v2 05/15] gve: add struct gve_device_info to hold " Harshitha Ramamurthy
@ 2026-06-03 23:59 ` sashiko-bot
2026-06-05 3:44 ` Harshitha Ramamurthy
1 sibling, 0 replies; 33+ messages in thread
From: sashiko-bot @ 2026-06-03 23:59 UTC (permalink / raw)
To: Harshitha Ramamurthy; +Cc: bpf
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [High] Uninitialized boolean check leads to out-of-bounds read of device options for minimum ring sizes.
--
commit 81dd3f8c245cbe096c0cfd21c5c45aa1925ab1d3
Author: Harshitha Ramamurthy <hramamurthy@google.com>
gve: add struct gve_device_info to hold device properties
This commit refactors the driver to store device properties in a new
gve_device_info structure instead of gve_device_descriptor, paving the way
for the upcoming MailboxQ mode. Device option parsing and feature enabling
are updated to use this new structure.
> diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
> index 77203184efbdc..06db2ccd0b898 100644
> --- a/drivers/net/ethernet/google/gve/gve_adminq.c
> +++ b/drivers/net/ethernet/google/gve/gve_adminq.c
> @@ -190,7 +190,7 @@ void gve_parse_device_option(struct gve_priv *priv,
>
> /* device has not provided min ring size */
> if (option_length == GVE_DEVICE_OPTION_NO_MIN_RING_SIZE)
> - priv->default_min_ring_size = true;
> + priv->device_info.default_min_ring_size = true;
> break;
> case GVE_DEV_OPT_ID_FLOW_STEERING:
[ ... ]
> @@ -946,77 +949,105 @@ static void gve_enable_supported_features(struct gve_priv *priv,
> const struct gve_device_option_modify_ring
> *dev_op_modify_ring)
> {
> + struct gve_device_info *info = &priv->device_info;
[ ... ]
> /* Read and store ring size ranges given by device */
> if (dev_op_modify_ring &&
> (supported_features_mask & GVE_SUP_MODIFY_RING_MASK)) {
> - priv->modify_ring_size_enabled = true;
> - priv->max_rx_desc_cnt =
> + info->modify_ring_size_enabled = true;
> + info->max_rx_ring_size =
> be16_to_cpu(dev_op_modify_ring->max_rx_ring_size);
> - priv->max_tx_desc_cnt =
> + info->max_tx_ring_size =
> be16_to_cpu(dev_op_modify_ring->max_tx_ring_size);
> if (priv->default_min_ring_size) {
[Severity: High]
Does this code still read the correct boolean?
Since gve_parse_device_option() was updated to set
priv->device_info.default_min_ring_size, but this check still looks
at the un-refactored priv->default_min_ring_size (which remains false),
won't we unconditionally enter the else block?
If the device sends the short 8-byte MODIFY_RING option, could this
lead to an out-of-bounds read into the header of the next packed
device option, misinterpreting its option_id as the minimum ring sizes?
> /* If device hasn't provided minimums, use default minimums */
> - priv->min_tx_desc_cnt = GVE_DEFAULT_MIN_TX_RING_SIZE;
> - priv->min_rx_desc_cnt = GVE_DEFAULT_MIN_RX_RING_SIZE;
> + info->min_tx_ring_size = GVE_DEFAULT_MIN_TX_RING_SIZE;
> + info->min_rx_ring_size = GVE_DEFAULT_MIN_RX_RING_SIZE;
> } else {
> - priv->min_rx_desc_cnt = be16_to_cpu(dev_op_modify_ring->min_rx_ring_size);
> - priv->min_tx_desc_cnt = be16_to_cpu(dev_op_modify_ring->min_tx_ring_size);
> + info->min_rx_ring_size =
> + be16_to_cpu(dev_op_modify_ring->min_rx_ring_size);
> + info->min_tx_ring_size =
> + be16_to_cpu(dev_op_modify_ring->min_tx_ring_size);
> }
> }
--
Sashiko AI review · https://sashiko.dev/#/patchset/20260602235900.3050782-1-hramamurthy@google.com?part=5
^ permalink raw reply [flat|nested] 33+ messages in thread* Re: [PATCH net-next v2 05/15] gve: add struct gve_device_info to hold device properties
2026-06-02 23:58 ` [PATCH net-next v2 05/15] gve: add struct gve_device_info to hold " Harshitha Ramamurthy
2026-06-03 23:59 ` sashiko-bot
@ 2026-06-05 3:44 ` Harshitha Ramamurthy
1 sibling, 0 replies; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-05 3:44 UTC (permalink / raw)
To: netdev
Cc: joshwash, andrew+netdev, davem, edumazet, kuba, pabeni, ast,
daniel, hawk, john.fastabend, sdf, willemb, jordanrhee, jfraker,
nktgrg, bpf, linux-kernel
On Tue, Jun 2, 2026 at 4:59 PM Harshitha Ramamurthy
<hramamurthy@google.com> wrote:
>
> In the current AdminQ mode, device properties are written into
> struct gve_device_descriptor that is allocated in shared memory
> between the driver and device. In the upcoming MailboxQ mode,
> these properties will be returned in the response of a mailbox
> message. Hence, add struct gve_device_info as the structure that
> holds all the properties that are negotiated with the device in
> either mode.
>
> Change the AdminQ mode method gve_adminq_describe_device()
> and the functions it calls to fill up device information
> into this newly introduced struct gve_device_info. Move a
> few helper functions and code that set device properties
> in the priv structure into gve_init_priv(). So now
> gve_init_priv() calls/does the following:
>
> - gve_set_mtu()
> - gve_set_mac()
> - gve_set_queue_properties()
> - gve_set_buf_sizes()
> - set flow steering and RSS properties
> - set other priv properties
>
> When MailboxQ support is added, device information will be filled
> into the same structure and the same gve_init_priv() path would be
> used to set device properties to ensure common code reusage.
>
> These changes are refactors only, no functional change.
>
> Reviewed-by: Willem de Bruijn <willemb@google.com>
> Reviewed-by: Jordan Rhee <jordanrhee@google.com>
> Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
> ---
> drivers/net/ethernet/google/gve/gve.h | 29 +++++
> drivers/net/ethernet/google/gve/gve_adminq.c | 113 +++++++++++--------
> drivers/net/ethernet/google/gve/gve_adminq.h | 6 -
> drivers/net/ethernet/google/gve/gve_main.c | 101 ++++++++++++-----
> 4 files changed, 169 insertions(+), 80 deletions(-)
>
> diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
> index 645175a5dfae..086a315f42c1 100644
> --- a/drivers/net/ethernet/google/gve/gve.h
> +++ b/drivers/net/ethernet/google/gve/gve.h
> @@ -796,6 +796,34 @@ struct gve_ptp {
> struct gve_priv *priv;
> };
>
> +struct gve_device_info {
> + enum gve_queue_format queue_format;
> + u16 default_tx_queues;
> + u16 default_rx_queues;
> + u16 max_tx_queues;
> + u16 max_rx_queues;
> + u16 default_tx_ring_size;
> + u16 default_rx_ring_size;
> + u16 max_tx_ring_size;
> + u16 max_rx_ring_size;
> + u16 min_tx_ring_size;
> + u16 min_rx_ring_size;
> + u16 max_mtu;
> + u8 mac[ETH_ALEN];
> + u16 max_rx_buffer_size;
> + u16 header_buf_size;
> + u32 max_flow_rules;
> + u16 rss_key_size;
> + u16 rss_lut_size;
> + u16 tx_pages_per_qpl;
> + u16 num_event_counters;
> + u64 max_registered_pages;
> + bool default_min_ring_size;
> + bool nic_timestamp_supported;
> + bool modify_ring_size_enabled;
> + bool cache_rss_config;
> +};
> +
> struct gve_priv {
> struct net_device *dev;
> struct gve_tx_ring *tx; /* array of tx_cfg.num_queues */
> @@ -928,6 +956,7 @@ struct gve_priv {
> struct gve_nic_ts_report *nic_ts_report;
> dma_addr_t nic_ts_report_bus;
> u64 last_sync_nic_counter; /* Clock counter from last NIC TS report */
> + struct gve_device_info device_info;
> };
>
> enum gve_service_task_flags_bit {
> diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
> index 77203184efbd..06db2ccd0b89 100644
> --- a/drivers/net/ethernet/google/gve/gve_adminq.c
> +++ b/drivers/net/ethernet/google/gve/gve_adminq.c
> @@ -70,7 +70,7 @@ void gve_parse_device_option(struct gve_priv *priv,
>
> dev_info(&priv->pdev->dev,
> "Gqi raw addressing device option enabled.\n");
> - priv->queue_format = GVE_GQI_RDA_FORMAT;
> + priv->device_info.queue_format = GVE_GQI_RDA_FORMAT;
> break;
> case GVE_DEV_OPT_ID_GQI_RDA:
> if (option_length < sizeof(**dev_op_gqi_rda) ||
> @@ -190,7 +190,7 @@ void gve_parse_device_option(struct gve_priv *priv,
>
> /* device has not provided min ring size */
> if (option_length == GVE_DEVICE_OPTION_NO_MIN_RING_SIZE)
> - priv->default_min_ring_size = true;
> + priv->device_info.default_min_ring_size = true;
> break;
> case GVE_DEV_OPT_ID_FLOW_STEERING:
> if (option_length < sizeof(**dev_op_flow_steering) ||
> @@ -922,10 +922,13 @@ int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 num_queues)
>
> static void gve_set_default_rss_sizes(struct gve_priv *priv)
> {
> - if (!gve_is_gqi(priv)) {
> - priv->rss_key_size = GVE_RSS_KEY_SIZE;
> - priv->rss_lut_size = GVE_RSS_INDIR_SIZE;
> - priv->cache_rss_config = true;
> + struct gve_device_info *device_info = &priv->device_info;
> +
> + if (device_info->queue_format == GVE_DQO_RDA_FORMAT ||
> + device_info->queue_format == GVE_DQO_QPL_FORMAT) {
> + device_info->rss_key_size = GVE_RSS_KEY_SIZE;
> + device_info->rss_lut_size = GVE_RSS_INDIR_SIZE;
> + device_info->cache_rss_config = true;
> }
> }
>
> @@ -946,77 +949,105 @@ static void gve_enable_supported_features(struct gve_priv *priv,
> const struct gve_device_option_modify_ring
> *dev_op_modify_ring)
> {
> + struct gve_device_info *info = &priv->device_info;
> +
> /* Before control reaches this point, the page-size-capped max MTU from
> * the gve_device_descriptor field has already been stored in
> - * priv->dev->max_mtu. We overwrite it with the true max MTU below.
> + * device_info->max_mtu. We overwrite it with the true max MTU below.
> */
> if (dev_op_jumbo_frames &&
> (supported_features_mask & GVE_SUP_JUMBO_FRAMES_MASK)) {
> dev_info(&priv->pdev->dev,
> "JUMBO FRAMES device option enabled.\n");
> - priv->dev->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu);
> + info->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu);
> }
>
> if (dev_op_buffer_sizes &&
> (supported_features_mask & GVE_SUP_BUFFER_SIZES_MASK)) {
> - priv->max_rx_buffer_size =
> + info->max_rx_buffer_size =
> be16_to_cpu(dev_op_buffer_sizes->packet_buffer_size);
> - priv->header_buf_size =
> + info->header_buf_size =
> be16_to_cpu(dev_op_buffer_sizes->header_buffer_size);
> dev_info(&priv->pdev->dev,
> "BUFFER SIZES device option enabled with max_rx_buffer_size of %u, header_buf_size of %u.\n",
> - priv->max_rx_buffer_size, priv->header_buf_size);
> - if (gve_is_dqo(priv) &&
> - priv->max_rx_buffer_size > GVE_DEFAULT_RX_BUFFER_SIZE)
> - priv->rx_cfg.packet_buffer_size =
> - priv->max_rx_buffer_size;
> + info->max_rx_buffer_size, info->header_buf_size);
> }
>
> /* Read and store ring size ranges given by device */
> if (dev_op_modify_ring &&
> (supported_features_mask & GVE_SUP_MODIFY_RING_MASK)) {
> - priv->modify_ring_size_enabled = true;
> - priv->max_rx_desc_cnt =
> + info->modify_ring_size_enabled = true;
> + info->max_rx_ring_size =
> be16_to_cpu(dev_op_modify_ring->max_rx_ring_size);
> - priv->max_tx_desc_cnt =
> + info->max_tx_ring_size =
> be16_to_cpu(dev_op_modify_ring->max_tx_ring_size);
> if (priv->default_min_ring_size) {
Sashiko's report:
"Does this code still read the correct boolean?
Since gve_parse_device_option() was updated to set
priv->device_info.default_min_ring_size, but this check still looks
at the un-refactored priv->default_min_ring_size (which remains false),
won't we unconditionally enter the else block?"
Correct, this needs to now check for info->default_min_ring_size. Will
fix in v3.
> /* If device hasn't provided minimums, use default minimums */
> - priv->min_tx_desc_cnt = GVE_DEFAULT_MIN_TX_RING_SIZE;
> - priv->min_rx_desc_cnt = GVE_DEFAULT_MIN_RX_RING_SIZE;
> + info->min_tx_ring_size = GVE_DEFAULT_MIN_TX_RING_SIZE;
> + info->min_rx_ring_size = GVE_DEFAULT_MIN_RX_RING_SIZE;
> } else {
> - priv->min_rx_desc_cnt = be16_to_cpu(dev_op_modify_ring->min_rx_ring_size);
> - priv->min_tx_desc_cnt = be16_to_cpu(dev_op_modify_ring->min_tx_ring_size);
> + info->min_rx_ring_size =
> + be16_to_cpu(dev_op_modify_ring->min_rx_ring_size);
> + info->min_tx_ring_size =
> + be16_to_cpu(dev_op_modify_ring->min_tx_ring_size);
> }
> }
>
> if (dev_op_flow_steering &&
> (supported_features_mask & GVE_SUP_FLOW_STEERING_MASK)) {
> if (dev_op_flow_steering->max_flow_rules) {
> - priv->max_flow_rules =
> + info->max_flow_rules =
> be32_to_cpu(dev_op_flow_steering->max_flow_rules);
> - priv->dev->hw_features |= NETIF_F_NTUPLE;
> dev_info(&priv->pdev->dev,
> "FLOW STEERING device option enabled with max rule limit of %u.\n",
> - priv->max_flow_rules);
> + info->max_flow_rules);
> }
> }
>
> if (dev_op_rss_config &&
> (supported_features_mask & GVE_SUP_RSS_CONFIG_MASK)) {
> - priv->rss_key_size =
> + info->rss_key_size =
> be16_to_cpu(dev_op_rss_config->hash_key_size);
> - priv->rss_lut_size =
> + info->rss_lut_size =
> be16_to_cpu(dev_op_rss_config->hash_lut_size);
> - priv->cache_rss_config = false;
> + info->cache_rss_config = false;
> dev_dbg(&priv->pdev->dev,
> "RSS device option enabled with key size of %u, lut size of %u.\n",
> - priv->rss_key_size, priv->rss_lut_size);
> + info->rss_key_size, info->rss_lut_size);
> }
>
> if (dev_op_nic_timestamp &&
> (supported_features_mask & GVE_SUP_NIC_TIMESTAMP_MASK))
> - priv->nic_timestamp_supported = true;
> + info->nic_timestamp_supported = true;
> +}
> +
> +static void gve_fill_device_info(struct gve_priv *priv,
> + struct gve_device_descriptor *descriptor)
> +{
> + struct gve_device_info *device_info = &priv->device_info;
> + u16 default_num_queues;
> +
> + device_info->tx_pages_per_qpl =
> + be16_to_cpu(descriptor->tx_pages_per_qpl);
> + device_info->max_registered_pages =
> + be64_to_cpu(descriptor->max_registered_pages);
> + device_info->num_event_counters = be16_to_cpu(descriptor->counters);
> + ether_addr_copy(device_info->mac, descriptor->mac);
> + device_info->max_mtu = be16_to_cpu(descriptor->mtu);
> +
> + default_num_queues = be16_to_cpu(descriptor->default_num_queues);
> + device_info->default_tx_queues = default_num_queues;
> + device_info->default_rx_queues = default_num_queues;
> + device_info->default_tx_ring_size =
> + be16_to_cpu(descriptor->tx_queue_entries);
> + device_info->default_rx_ring_size =
> + be16_to_cpu(descriptor->rx_queue_entries);
> +
> + /* set default ranges */
> + device_info->max_tx_ring_size = device_info->default_tx_ring_size;
> + device_info->max_rx_ring_size = device_info->default_rx_ring_size;
> + device_info->min_tx_ring_size = device_info->default_tx_ring_size;
> + device_info->min_rx_ring_size = device_info->default_rx_ring_size;
> }
>
> int gve_adminq_describe_device(struct gve_priv *priv)
> @@ -1027,6 +1058,7 @@ int gve_adminq_describe_device(struct gve_priv *priv)
> struct gve_device_option_jumbo_frames *dev_op_jumbo_frames = NULL;
> struct gve_device_option_modify_ring *dev_op_modify_ring = NULL;
> struct gve_device_option_rss_config *dev_op_rss_config = NULL;
> + struct gve_device_info *device_info = &priv->device_info;
> struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL;
> struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL;
> struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL;
> @@ -1070,26 +1102,26 @@ int gve_adminq_describe_device(struct gve_priv *priv)
> * DqoRda, DqoQpl, GqiRda, GqiQpl. Use GqiQpl as default.
> */
> if (dev_op_dqo_rda) {
> - priv->queue_format = GVE_DQO_RDA_FORMAT;
> + device_info->queue_format = GVE_DQO_RDA_FORMAT;
> dev_info(&priv->pdev->dev,
> "Driver is running with DQO RDA queue format.\n");
> supported_features_mask =
> be32_to_cpu(dev_op_dqo_rda->supported_features_mask);
> } else if (dev_op_dqo_qpl) {
> - priv->queue_format = GVE_DQO_QPL_FORMAT;
> + device_info->queue_format = GVE_DQO_QPL_FORMAT;
> supported_features_mask =
> be32_to_cpu(dev_op_dqo_qpl->supported_features_mask);
> } else if (dev_op_gqi_rda) {
> - priv->queue_format = GVE_GQI_RDA_FORMAT;
> + device_info->queue_format = GVE_GQI_RDA_FORMAT;
> dev_info(&priv->pdev->dev,
> "Driver is running with GQI RDA queue format.\n");
> supported_features_mask =
> be32_to_cpu(dev_op_gqi_rda->supported_features_mask);
> - } else if (priv->queue_format == GVE_GQI_RDA_FORMAT) {
> + } else if (device_info->queue_format == GVE_GQI_RDA_FORMAT) {
> dev_info(&priv->pdev->dev,
> "Driver is running with GQI RDA queue format.\n");
> } else {
> - priv->queue_format = GVE_GQI_QPL_FORMAT;
> + device_info->queue_format = GVE_GQI_QPL_FORMAT;
> if (dev_op_gqi_qpl)
> supported_features_mask =
> be32_to_cpu(dev_op_gqi_qpl->supported_features_mask);
> @@ -1097,18 +1129,9 @@ int gve_adminq_describe_device(struct gve_priv *priv)
> "Driver is running with GQI QPL queue format.\n");
> }
>
> + gve_fill_device_info(priv, descriptor);
> gve_set_default_rss_sizes(priv);
>
> - err = gve_set_mtu(priv, descriptor);
> - if (err)
> - goto free_device_descriptor;
> -
> - priv->num_event_counters = be16_to_cpu(descriptor->counters);
> -
> - gve_set_mac(priv, descriptor);
> -
> - gve_set_queue_properties(priv, descriptor);
> -
> gve_enable_supported_features(priv, supported_features_mask,
> dev_op_jumbo_frames, dev_op_dqo_qpl,
> dev_op_buffer_sizes, dev_op_flow_steering,
> diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
> index ed5cb052a52d..831759d30f44 100644
> --- a/drivers/net/ethernet/google/gve/gve_adminq.h
> +++ b/drivers/net/ethernet/google/gve/gve_adminq.h
> @@ -655,11 +655,5 @@ int gve_adminq_report_nic_ts(struct gve_priv *priv,
> struct gve_ptype_lut;
> int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
> struct gve_ptype_lut *ptype_lut);
> -void gve_set_queue_properties(struct gve_priv *priv,
> - struct gve_device_descriptor *descriptor);
> -int gve_set_mtu(struct gve_priv *priv,
> - struct gve_device_descriptor *descriptor);
> -void gve_set_mac(struct gve_priv *priv,
> - struct gve_device_descriptor *descriptor);
>
> #endif /* _GVE_ADMINQ_H */
> diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
> index 338dc0b3249a..1aeee916471f 100644
> --- a/drivers/net/ethernet/google/gve/gve_main.c
> +++ b/drivers/net/ethernet/google/gve/gve_main.c
> @@ -2380,6 +2380,8 @@ static int gve_set_num_ntfy_blks(struct gve_priv *priv)
>
> static void gve_set_num_queues(struct gve_priv *priv)
> {
> + struct gve_device_info *device_info = &priv->device_info;
> +
> priv->tx_cfg.max_queues =
> min_t(int, priv->tx_cfg.max_queues, priv->num_ntfy_blks / 2);
> priv->rx_cfg.max_queues =
> @@ -2387,12 +2389,14 @@ static void gve_set_num_queues(struct gve_priv *priv)
>
> priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
> priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
> - if (priv->default_num_queues > 0) {
> - priv->tx_cfg.num_queues = min_t(int, priv->default_num_queues,
> + if (device_info->default_tx_queues > 0)
> + priv->tx_cfg.num_queues = min_t(int,
> + device_info->default_tx_queues,
> priv->tx_cfg.num_queues);
> - priv->rx_cfg.num_queues = min_t(int, priv->default_num_queues,
> + if (device_info->default_rx_queues > 0)
> + priv->rx_cfg.num_queues = min_t(int,
> + device_info->default_rx_queues,
> priv->rx_cfg.num_queues);
> - }
>
> dev_info(&priv->pdev->dev, "TX queues %d, RX queues %d\n",
> priv->tx_cfg.num_queues, priv->rx_cfg.num_queues);
> @@ -2400,54 +2404,69 @@ static void gve_set_num_queues(struct gve_priv *priv)
> priv->tx_cfg.max_queues, priv->rx_cfg.max_queues);
> }
>
> -static void gve_set_default_desc_cnt(struct gve_priv *priv,
> - const struct gve_device_descriptor *descriptor)
> +static void gve_set_desc_cnt(struct gve_priv *priv)
> {
> - priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
> - priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
> + struct gve_device_info *device_info = &priv->device_info;
>
> - /* set default ranges */
> - priv->max_tx_desc_cnt = priv->tx_desc_cnt;
> - priv->max_rx_desc_cnt = priv->rx_desc_cnt;
> - priv->min_tx_desc_cnt = priv->tx_desc_cnt;
> - priv->min_rx_desc_cnt = priv->rx_desc_cnt;
> + priv->tx_desc_cnt = device_info->default_tx_ring_size;
> + priv->rx_desc_cnt = device_info->default_rx_ring_size;
> + priv->max_tx_desc_cnt = device_info->max_tx_ring_size;
> + priv->max_rx_desc_cnt = device_info->max_rx_ring_size;
> + priv->min_tx_desc_cnt = device_info->min_tx_ring_size;
> + priv->min_rx_desc_cnt = device_info->min_rx_ring_size;
> }
>
> -void gve_set_queue_properties(struct gve_priv *priv,
> - struct gve_device_descriptor *descriptor)
> +static void gve_set_queue_properties(struct gve_priv *priv)
> {
> - /* set default descriptor counts */
> - gve_set_default_desc_cnt(priv, descriptor);
> + struct gve_device_info *device_info = &priv->device_info;
>
> - priv->max_registered_pages = be64_to_cpu(descriptor->max_registered_pages);
> - priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl);
> - priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues);
> + gve_set_desc_cnt(priv);
> + priv->max_registered_pages = device_info->max_registered_pages;
> + priv->tx_pages_per_qpl = device_info->tx_pages_per_qpl;
> }
>
> -int gve_set_mtu(struct gve_priv *priv,
> - struct gve_device_descriptor *descriptor)
> +static int gve_set_mtu(struct gve_priv *priv)
> {
> + struct gve_device_info *device_info = &priv->device_info;
> u16 mtu;
>
> - mtu = be16_to_cpu(descriptor->mtu);
> + mtu = device_info->max_mtu;
> if (mtu < ETH_MIN_MTU) {
> dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu);
> return -EINVAL;
> }
> priv->dev->max_mtu = mtu;
> + priv->dev->mtu = priv->dev->max_mtu;
>
> return 0;
> }
>
> -void gve_set_mac(struct gve_priv *priv,
> - struct gve_device_descriptor *descriptor)
> +static void gve_set_mac(struct gve_priv *priv)
> {
> - eth_hw_addr_set(priv->dev, descriptor->mac);
> - dev_info(&priv->pdev->dev, "MAC addr: %pM\n", descriptor->mac);
> + struct gve_device_info *device_info = &priv->device_info;
> +
> + eth_hw_addr_set(priv->dev, device_info->mac);
> + dev_info(&priv->pdev->dev, "MAC addr: %pM\n", device_info->mac);
> +}
> +
> +static void gve_set_buf_sizes(struct gve_priv *priv)
> +{
> + struct gve_device_info *device_info = &priv->device_info;
> +
> + if (device_info->max_rx_buffer_size > priv->max_rx_buffer_size)
> + priv->max_rx_buffer_size = device_info->max_rx_buffer_size;
> +
> + if (gve_is_dqo(priv) &&
> + priv->max_rx_buffer_size > GVE_DEFAULT_RX_BUFFER_SIZE)
> + priv->rx_cfg.packet_buffer_size = priv->max_rx_buffer_size;
> +
> + if (device_info->header_buf_size)
> + priv->header_buf_size = device_info->header_buf_size;
> }
>
> static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
> {
> + struct gve_device_info *device_info = &priv->device_info;
> int err;
>
> /* Set up the adminq */
> @@ -2463,11 +2482,13 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
> if (skip_describe_device)
> goto setup_device;
>
> - priv->queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED;
> + device_info->queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED;
> err = gve_adminq_get_device_properties(priv);
> if (err)
> goto err;
>
> + priv->queue_format = priv->device_info.queue_format;
> +
> err = gve_set_num_ntfy_blks(priv);
> if (err) {
> dev_err(&priv->pdev->dev,
> @@ -2491,12 +2512,34 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
> netif_set_tso_max_size(priv->dev, GVE_DQO_TX_MAX);
> }
>
> - priv->dev->mtu = priv->dev->max_mtu;
> + if (gve_set_mtu(priv)) {
> + err = -EINVAL;
> + goto err;
> + }
> +
> + priv->num_event_counters = device_info->num_event_counters;
> +
> + gve_set_mac(priv);
> +
> + gve_set_queue_properties(priv);
> + priv->modify_ring_size_enabled = device_info->modify_ring_size_enabled;
> +
> + gve_set_buf_sizes(priv);
> +
> + priv->max_flow_rules = device_info->max_flow_rules;
> + if (priv->max_flow_rules)
> + priv->dev->hw_features |= NETIF_F_NTUPLE;
> +
> + priv->rss_key_size = device_info->rss_key_size;
> + priv->rss_lut_size = device_info->rss_lut_size;
> + priv->cache_rss_config = device_info->cache_rss_config;
> +
> priv->numa_node = dev_to_node(&priv->pdev->dev);
> priv->tx_cfg.num_xdp_queues = 0;
> priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK;
> priv->ts_config.tx_type = HWTSTAMP_TX_OFF;
> priv->ts_config.rx_filter = HWTSTAMP_FILTER_NONE;
> + priv->nic_timestamp_supported = device_info->nic_timestamp_supported;
>
> setup_device:
> priv->xsk_pools = bitmap_zalloc(priv->rx_cfg.max_queues, GFP_KERNEL);
> --
> 2.54.0.1013.g208068f2d8-goog
>
^ permalink raw reply [flat|nested] 33+ messages in thread
* [PATCH net-next v2 06/15] gve: introduce control plane operations structure
2026-06-02 23:58 [PATCH net-next v2 00/15] gve: AdminQ mode related refactors Harshitha Ramamurthy
` (4 preceding siblings ...)
2026-06-02 23:58 ` [PATCH net-next v2 05/15] gve: add struct gve_device_info to hold " Harshitha Ramamurthy
@ 2026-06-02 23:58 ` Harshitha Ramamurthy
2026-06-02 23:58 ` [PATCH net-next v2 07/15] gve: introduce ctrl ops to set vectors and Qs Harshitha Ramamurthy
` (8 subsequent siblings)
14 siblings, 0 replies; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-02 23:58 UTC (permalink / raw)
To: netdev
Cc: joshwash, hramamurthy, andrew+netdev, davem, edumazet, kuba,
pabeni, ast, daniel, hawk, john.fastabend, sdf, willemb,
jordanrhee, jfraker, nktgrg, bpf, linux-kernel
To abstract out the difference in implementation of control
plane operations between the existing Adminq ABI and the upcoming
Mailbox ABI, introduce a new gve_ctrl_ops structure which will
contain the basic operations. At probe, these ops will be set based
on the ABI and the corresponding ops will be called in relevant
places.
As of this patch, only Adminq ops are set. In future patches,
corresponding ops will be set for the new mailbox mode.
Implement a ctrl op to map/unmap the doorbell bar. Since this
functionality has moved to a control op, call this op after control
ops are set for AdminQ mode.
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Jordan Rhee <jordanrhee@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
drivers/net/ethernet/google/gve/gve.h | 11 ++++++
drivers/net/ethernet/google/gve/gve_adminq.c | 21 ++++++++++++
drivers/net/ethernet/google/gve/gve_adminq.h | 2 ++
drivers/net/ethernet/google/gve/gve_main.c | 36 +++++++++++---------
4 files changed, 54 insertions(+), 16 deletions(-)
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index 086a315f42c1..c1e118241d2b 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -824,6 +824,16 @@ struct gve_device_info {
bool cache_rss_config;
};
+/**
+ * struct gve_ctrl_ops - Control plane operations structure
+ * @map_db_bar: Maps the doorbell BAR for the device and store in @priv.
+ * @unmap_db_bar: Unmaps the doorbell BAR previously mapped by @map_db_bar.
+ */
+struct gve_ctrl_ops {
+ int (*map_db_bar)(struct gve_priv *priv);
+ void (*unmap_db_bar)(struct gve_priv *priv);
+};
+
struct gve_priv {
struct net_device *dev;
struct gve_tx_ring *tx; /* array of tx_cfg.num_queues */
@@ -957,6 +967,7 @@ struct gve_priv {
dma_addr_t nic_ts_report_bus;
u64 last_sync_nic_counter; /* Clock counter from last NIC TS report */
struct gve_device_info device_info;
+ const struct gve_ctrl_ops *ctrl_ops;
};
enum gve_service_task_flags_bit {
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index 06db2ccd0b89..4617489d8714 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -1645,3 +1645,24 @@ int gve_adminq_query_rss_config(struct gve_priv *priv, struct ethtool_rxfh_param
dma_pool_free(priv->adminq_pool, descriptor, descriptor_bus);
return err;
}
+
+int gve_adminq_map_db_bar(struct gve_priv *priv)
+{
+ struct pci_dev *pdev = priv->pdev;
+ void __iomem *db_bar;
+
+ db_bar = pci_iomap(pdev, GVE_DOORBELL_BAR, 0);
+ if (!db_bar) {
+ dev_err(&pdev->dev, "Failed to map doorbell bar!\n");
+ return -ENOMEM;
+ }
+ priv->db_bar2 = db_bar;
+ return 0;
+}
+
+void gve_adminq_unmap_db_bar(struct gve_priv *priv)
+{
+ struct pci_dev *pdev = priv->pdev;
+
+ pci_iounmap(pdev, priv->db_bar2);
+}
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
index 831759d30f44..bc76976eb00f 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.h
+++ b/drivers/net/ethernet/google/gve/gve_adminq.h
@@ -655,5 +655,7 @@ int gve_adminq_report_nic_ts(struct gve_priv *priv,
struct gve_ptype_lut;
int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
struct gve_ptype_lut *ptype_lut);
+int gve_adminq_map_db_bar(struct gve_priv *priv);
+void gve_adminq_unmap_db_bar(struct gve_priv *priv);
#endif /* _GVE_ADMINQ_H */
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 1aeee916471f..51dd4f9b3022 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -2464,6 +2464,11 @@ static void gve_set_buf_sizes(struct gve_priv *priv)
priv->header_buf_size = device_info->header_buf_size;
}
+static const struct gve_ctrl_ops gve_adminq_ops = {
+ .map_db_bar = gve_adminq_map_db_bar,
+ .unmap_db_bar = gve_adminq_unmap_db_bar,
+};
+
static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
{
struct gve_device_info *device_info = &priv->device_info;
@@ -2850,7 +2855,6 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
{
int max_tx_queues, max_rx_queues;
struct net_device *dev;
- __be32 __iomem *db_bar;
struct gve_registers __iomem *reg_bar;
struct gve_priv *priv;
int err;
@@ -2878,13 +2882,6 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
goto abort_with_pci_region;
}
- db_bar = pci_iomap(pdev, GVE_DOORBELL_BAR, 0);
- if (!db_bar) {
- dev_err(&pdev->dev, "Failed to map doorbell bar!\n");
- err = -ENOMEM;
- goto abort_with_reg_bar;
- }
-
gve_write_version(®_bar->driver_version);
/* Get max queues to alloc etherdev */
max_tx_queues = ioread32be(®_bar->max_tx_queues);
@@ -2894,7 +2891,7 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
if (!dev) {
dev_err(&pdev->dev, "could not allocate netdev\n");
err = -ENOMEM;
- goto abort_with_db_bar;
+ goto abort_with_reg_bar;
}
SET_NETDEV_DEV(dev, &pdev->dev);
pci_set_drvdata(pdev, dev);
@@ -2926,19 +2923,27 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
priv->pdev = pdev;
priv->msg_enable = DEFAULT_MSG_LEVEL;
priv->reg_bar0 = reg_bar;
- priv->db_bar2 = db_bar;
priv->service_task_flags = 0x0;
priv->state_flags = 0x0;
priv->ethtool_flags = 0x0;
priv->rx_cfg.packet_buffer_size = GVE_DEFAULT_RX_BUFFER_SIZE;
priv->max_rx_buffer_size = GVE_DEFAULT_RX_BUFFER_SIZE;
+ /* Set adminq ctrl ops */
+ priv->ctrl_ops = &gve_adminq_ops;
+
+ err = priv->ctrl_ops->map_db_bar(priv);
+ if (err) {
+ err = -ENOMEM;
+ goto abort_with_netdev;
+ }
+
gve_set_probe_in_progress(priv);
priv->gve_wq = alloc_ordered_workqueue("gve", 0);
if (!priv->gve_wq) {
dev_err(&pdev->dev, "Could not allocate workqueue");
err = -ENOMEM;
- goto abort_with_netdev;
+ goto abort_with_unmap_db_bar;
}
INIT_WORK(&priv->service_task, gve_service_task);
INIT_WORK(&priv->stats_report_task, gve_stats_report_task);
@@ -2968,12 +2973,12 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
abort_with_wq:
destroy_workqueue(priv->gve_wq);
+abort_with_unmap_db_bar:
+ priv->ctrl_ops->unmap_db_bar(priv);
+
abort_with_netdev:
free_netdev(dev);
-abort_with_db_bar:
- pci_iounmap(pdev, db_bar);
-
abort_with_reg_bar:
pci_iounmap(pdev, reg_bar);
@@ -2989,14 +2994,13 @@ static void gve_remove(struct pci_dev *pdev)
{
struct net_device *netdev = pci_get_drvdata(pdev);
struct gve_priv *priv = netdev_priv(netdev);
- __be32 __iomem *db_bar = priv->db_bar2;
void __iomem *reg_bar = priv->reg_bar0;
unregister_netdev(netdev);
gve_teardown_priv_resources(priv);
destroy_workqueue(priv->gve_wq);
+ priv->ctrl_ops->unmap_db_bar(priv);
free_netdev(netdev);
- pci_iounmap(pdev, db_bar);
pci_iounmap(pdev, reg_bar);
pci_release_regions(pdev);
pci_disable_device(pdev);
--
2.54.0.1013.g208068f2d8-goog
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH net-next v2 07/15] gve: introduce ctrl ops to set vectors and Qs
2026-06-02 23:58 [PATCH net-next v2 00/15] gve: AdminQ mode related refactors Harshitha Ramamurthy
` (5 preceding siblings ...)
2026-06-02 23:58 ` [PATCH net-next v2 06/15] gve: introduce control plane operations structure Harshitha Ramamurthy
@ 2026-06-02 23:58 ` Harshitha Ramamurthy
2026-06-02 23:58 ` [PATCH net-next v2 08/15] gve: refactor gve_init_priv for reset path Harshitha Ramamurthy
` (7 subsequent siblings)
14 siblings, 0 replies; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-02 23:58 UTC (permalink / raw)
To: netdev
Cc: joshwash, hramamurthy, andrew+netdev, davem, edumazet, kuba,
pabeni, ast, daniel, hawk, john.fastabend, sdf, willemb,
jordanrhee, jfraker, nktgrg, bpf, linux-kernel
Introduce following ctrl ops for Adminq mode. This change does
not introduce any functional change, just movement of code into
ops.
- set_num_queues to set maximum and default TX/RX queues in the
alloc structures used in the initialization flow.
- set_num_ntfy_blks op to set number of vectors(ntfy_blks) in the
initialization flow.
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Jordan Rhee <jordanrhee@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
drivers/net/ethernet/google/gve/gve.h | 6 ++
drivers/net/ethernet/google/gve/gve_adminq.c | 45 ++++++++++++++
drivers/net/ethernet/google/gve/gve_adminq.h | 2 +
drivers/net/ethernet/google/gve/gve_main.c | 64 +++-----------------
4 files changed, 63 insertions(+), 54 deletions(-)
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index c1e118241d2b..bd48c3d7a2b2 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -828,10 +828,16 @@ struct gve_device_info {
* struct gve_ctrl_ops - Control plane operations structure
* @map_db_bar: Maps the doorbell BAR for the device and store in @priv.
* @unmap_db_bar: Unmaps the doorbell BAR previously mapped by @map_db_bar.
+ * @set_num_queues: Sets default and max TX/RX queues into allocation
+ * structures stored in @priv to be used during initialization.
+ * @set_num_ntfy_blks: Sets no. of vectors into @priv to be used during
+ * initialization.
*/
struct gve_ctrl_ops {
int (*map_db_bar)(struct gve_priv *priv);
void (*unmap_db_bar)(struct gve_priv *priv);
+ void (*set_num_queues)(struct gve_priv *priv);
+ int (*set_num_ntfy_blks)(struct gve_priv *priv);
};
struct gve_priv {
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index 4617489d8714..923a5c737258 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -1666,3 +1666,48 @@ void gve_adminq_unmap_db_bar(struct gve_priv *priv)
pci_iounmap(pdev, priv->db_bar2);
}
+
+int gve_adminq_set_num_ntfy_blks(struct gve_priv *priv)
+{
+ int num_ntfy;
+
+ num_ntfy = pci_msix_vec_count(priv->pdev);
+ if (num_ntfy <= 0) {
+ dev_err(&priv->pdev->dev,
+ "could not count MSI-x vectors: err=%d\n", num_ntfy);
+ return num_ntfy;
+ } else if (num_ntfy < GVE_MIN_MSIX) {
+ dev_err(&priv->pdev->dev, "gve needs at least %d MSI-x vectors, but only has %d\n",
+ GVE_MIN_MSIX, num_ntfy);
+ return -EINVAL;
+ }
+
+ /* gvnic has one Notification Block per MSI-x vector, except for the
+ * management vector
+ */
+ priv->num_ntfy_blks = (num_ntfy - 1) & ~0x1;
+ priv->mgmt_msix_idx = priv->num_ntfy_blks;
+
+ return 0;
+}
+
+void gve_adminq_set_num_queues(struct gve_priv *priv)
+{
+ struct gve_device_info *device_info = &priv->device_info;
+
+ priv->tx_cfg.max_queues =
+ min_t(int, priv->tx_cfg.max_queues, priv->num_ntfy_blks / 2);
+ priv->rx_cfg.max_queues =
+ min_t(int, priv->rx_cfg.max_queues, priv->num_ntfy_blks / 2);
+
+ priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
+ priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
+ if (device_info->default_tx_queues > 0)
+ priv->tx_cfg.num_queues = min_t(int,
+ device_info->default_tx_queues,
+ priv->tx_cfg.num_queues);
+ if (device_info->default_rx_queues > 0)
+ priv->rx_cfg.num_queues = min_t(int,
+ device_info->default_rx_queues,
+ priv->rx_cfg.num_queues);
+}
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
index bc76976eb00f..ac3ef9fd8d24 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.h
+++ b/drivers/net/ethernet/google/gve/gve_adminq.h
@@ -657,5 +657,7 @@ int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
struct gve_ptype_lut *ptype_lut);
int gve_adminq_map_db_bar(struct gve_priv *priv);
void gve_adminq_unmap_db_bar(struct gve_priv *priv);
+int gve_adminq_set_num_ntfy_blks(struct gve_priv *priv);
+void gve_adminq_set_num_queues(struct gve_priv *priv);
#endif /* _GVE_ADMINQ_H */
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 51dd4f9b3022..d693caed7e3d 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -2354,56 +2354,6 @@ static const struct xdp_metadata_ops gve_xdp_metadata_ops = {
.xmo_rx_timestamp = gve_xdp_rx_timestamp,
};
-static int gve_set_num_ntfy_blks(struct gve_priv *priv)
-{
- int num_ntfy;
-
- num_ntfy = pci_msix_vec_count(priv->pdev);
- if (num_ntfy <= 0) {
- dev_err(&priv->pdev->dev,
- "could not count MSI-x vectors: err=%d\n", num_ntfy);
- return num_ntfy;
- } else if (num_ntfy < GVE_MIN_MSIX) {
- dev_err(&priv->pdev->dev, "gve needs at least %d MSI-x vectors, but only has %d\n",
- GVE_MIN_MSIX, num_ntfy);
- return -EINVAL;
- }
-
- /* gvnic has one Notification Block per MSI-x vector, except for the
- * management vector
- */
- priv->num_ntfy_blks = (num_ntfy - 1) & ~0x1;
- priv->mgmt_msix_idx = priv->num_ntfy_blks;
-
- return 0;
-}
-
-static void gve_set_num_queues(struct gve_priv *priv)
-{
- struct gve_device_info *device_info = &priv->device_info;
-
- priv->tx_cfg.max_queues =
- min_t(int, priv->tx_cfg.max_queues, priv->num_ntfy_blks / 2);
- priv->rx_cfg.max_queues =
- min_t(int, priv->rx_cfg.max_queues, priv->num_ntfy_blks / 2);
-
- priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
- priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
- if (device_info->default_tx_queues > 0)
- priv->tx_cfg.num_queues = min_t(int,
- device_info->default_tx_queues,
- priv->tx_cfg.num_queues);
- if (device_info->default_rx_queues > 0)
- priv->rx_cfg.num_queues = min_t(int,
- device_info->default_rx_queues,
- priv->rx_cfg.num_queues);
-
- dev_info(&priv->pdev->dev, "TX queues %d, RX queues %d\n",
- priv->tx_cfg.num_queues, priv->rx_cfg.num_queues);
- dev_info(&priv->pdev->dev, "Max TX queues %d, Max RX queues %d\n",
- priv->tx_cfg.max_queues, priv->rx_cfg.max_queues);
-}
-
static void gve_set_desc_cnt(struct gve_priv *priv)
{
struct gve_device_info *device_info = &priv->device_info;
@@ -2465,8 +2415,10 @@ static void gve_set_buf_sizes(struct gve_priv *priv)
}
static const struct gve_ctrl_ops gve_adminq_ops = {
- .map_db_bar = gve_adminq_map_db_bar,
- .unmap_db_bar = gve_adminq_unmap_db_bar,
+ .map_db_bar = gve_adminq_map_db_bar,
+ .unmap_db_bar = gve_adminq_unmap_db_bar,
+ .set_num_queues = gve_adminq_set_num_queues,
+ .set_num_ntfy_blks = gve_adminq_set_num_ntfy_blks,
};
static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
@@ -2494,14 +2446,18 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
priv->queue_format = priv->device_info.queue_format;
- err = gve_set_num_ntfy_blks(priv);
+ err = priv->ctrl_ops->set_num_ntfy_blks(priv);
if (err) {
dev_err(&priv->pdev->dev,
"Could not setup notify blocks: err=%d\n", err);
goto err;
}
- gve_set_num_queues(priv);
+ priv->ctrl_ops->set_num_queues(priv);
+ dev_info(&priv->pdev->dev, "TX queues %d, RX queues %d\n",
+ priv->tx_cfg.num_queues, priv->rx_cfg.num_queues);
+ dev_info(&priv->pdev->dev, "Max TX queues %d, Max RX queues %d\n",
+ priv->tx_cfg.max_queues, priv->rx_cfg.max_queues);
if (gve_is_dqo(priv)) {
/* DQO supports HW-GRO and UDP_GSO */
--
2.54.0.1013.g208068f2d8-goog
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH net-next v2 08/15] gve: refactor gve_init_priv for reset path
2026-06-02 23:58 [PATCH net-next v2 00/15] gve: AdminQ mode related refactors Harshitha Ramamurthy
` (6 preceding siblings ...)
2026-06-02 23:58 ` [PATCH net-next v2 07/15] gve: introduce ctrl ops to set vectors and Qs Harshitha Ramamurthy
@ 2026-06-02 23:58 ` Harshitha Ramamurthy
2026-06-03 23:59 ` sashiko-bot
2026-06-05 3:52 ` Harshitha Ramamurthy
2026-06-02 23:58 ` [PATCH net-next v2 09/15] gve: simplify reset logic Harshitha Ramamurthy
` (6 subsequent siblings)
14 siblings, 2 replies; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-02 23:58 UTC (permalink / raw)
To: netdev
Cc: joshwash, hramamurthy, andrew+netdev, davem, edumazet, kuba,
pabeni, ast, daniel, hawk, john.fastabend, sdf, willemb,
jordanrhee, jfraker, nktgrg, bpf, linux-kernel
The driver does not need to renegotiate all properties with
the device on a reset since those should stay constant through
a reset. Hence change gve_init_priv() into a method that only
sets these properties into the priv structure and hence needs
to be only called once during gve_probe().
To achieve this end state of gve_init_priv(), do the following:
- introduce gve_adminq_init() which writes the driver version register
and allocates the AdminQ and call it in gve_probe()
- move gve_adminq_get_device_properties() into gve_probe()
- introduce gve_setup_device() which deals with device setup logic and
call it in gve_probe()
- resetting no. of registered pages is moved into gve_register_qpls()
since that is a QPL specific property.
With these changes, gve_adminq_get_device_properties() and
gve_init_priv() are only called once during gve_probe.
gve_reset_recovery() can now bypass full initialization and call these
targeted setup functions directly.
This prepares the driver to add mailbox mode's control plane
initialization and device properties negotiation in the same place
as is done in AdminQ mode in the upcoming patches when adding the
mailbox ABI.
These changes are only code movement, no functional change.
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Jordan Rhee <jordanrhee@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
drivers/net/ethernet/google/gve/gve.h | 2 +
drivers/net/ethernet/google/gve/gve_adminq.c | 12 +-
drivers/net/ethernet/google/gve/gve_adminq.h | 2 +-
drivers/net/ethernet/google/gve/gve_main.c | 132 +++++++++++--------
4 files changed, 92 insertions(+), 56 deletions(-)
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index bd48c3d7a2b2..72588d201e5d 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -1252,6 +1252,8 @@ static inline bool gve_is_clock_enabled(struct gve_priv *priv)
return priv->nic_ts_report;
}
+void gve_adminq_write_version(u8 __iomem *driver_version_register);
+
/* gqi napi handler defined in gve_main.c */
int gve_napi_poll(struct napi_struct *napi, int budget);
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index 923a5c737258..e7956d2768b6 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -296,8 +296,10 @@ gve_process_device_options(struct gve_priv *priv,
return 0;
}
-int gve_adminq_alloc(struct device *dev, struct gve_priv *priv)
+static int gve_adminq_alloc(struct gve_priv *priv)
{
+ struct device *dev = &priv->pdev->dev;
+
priv->adminq_pool = dma_pool_create("adminq_pool", dev,
GVE_ADMINQ_BUFFER_SIZE, 0, 0);
if (unlikely(!priv->adminq_pool))
@@ -353,6 +355,14 @@ int gve_adminq_alloc(struct device *dev, struct gve_priv *priv)
return 0;
}
+int gve_adminq_init(struct gve_priv *priv)
+{
+ struct gve_registers __iomem *reg_bar = priv->reg_bar0;
+
+ gve_adminq_write_version(®_bar->driver_version);
+ return gve_adminq_alloc(priv);
+}
+
void gve_adminq_release(struct gve_priv *priv)
{
int i = 0;
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
index ac3ef9fd8d24..d07e9c6f279d 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.h
+++ b/drivers/net/ethernet/google/gve/gve_adminq.h
@@ -619,7 +619,7 @@ union gve_adminq_command {
static_assert(sizeof(union gve_adminq_command) == 64);
-int gve_adminq_alloc(struct device *dev, struct gve_priv *priv);
+int gve_adminq_init(struct gve_priv *priv);
void gve_adminq_free(struct gve_priv *priv);
void gve_adminq_release(struct gve_priv *priv);
int gve_adminq_describe_device(struct gve_priv *priv);
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index d693caed7e3d..746ff69a28dd 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -797,6 +797,8 @@ static int gve_register_qpls(struct gve_priv *priv)
int err;
int i;
+ priv->num_registered_pages = 0;
+
num_tx_qpls = gve_num_tx_qpls(&priv->tx_cfg, gve_is_qpl(priv));
num_rx_qpls = gve_num_rx_qpls(&priv->rx_cfg, gve_is_qpl(priv));
@@ -2414,6 +2416,33 @@ static void gve_set_buf_sizes(struct gve_priv *priv)
priv->header_buf_size = device_info->header_buf_size;
}
+static int gve_setup_device(struct gve_priv *priv)
+{
+ int err;
+
+ priv->xsk_pools = bitmap_zalloc(priv->rx_cfg.max_queues, GFP_KERNEL);
+ if (!priv->xsk_pools) {
+ err = -ENOMEM;
+ goto err;
+ }
+
+ gve_set_netdev_xdp_features(priv);
+ if (!gve_is_gqi(priv))
+ priv->dev->xdp_metadata_ops = &gve_xdp_metadata_ops;
+
+ err = gve_setup_device_resources(priv);
+ if (err)
+ goto err_free_xsk_bitmap;
+
+ return 0;
+
+err_free_xsk_bitmap:
+ bitmap_free(priv->xsk_pools);
+ priv->xsk_pools = NULL;
+err:
+ return err;
+}
+
static const struct gve_ctrl_ops gve_adminq_ops = {
.map_db_bar = gve_adminq_map_db_bar,
.unmap_db_bar = gve_adminq_unmap_db_bar,
@@ -2421,36 +2450,18 @@ static const struct gve_ctrl_ops gve_adminq_ops = {
.set_num_ntfy_blks = gve_adminq_set_num_ntfy_blks,
};
-static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
+static int gve_init_priv(struct gve_priv *priv)
{
struct gve_device_info *device_info = &priv->device_info;
int err;
- /* Set up the adminq */
- err = gve_adminq_alloc(&priv->pdev->dev, priv);
- if (err) {
- dev_err(&priv->pdev->dev,
- "Failed to alloc admin queue: err=%d\n", err);
- return err;
- }
-
- priv->num_registered_pages = 0;
-
- if (skip_describe_device)
- goto setup_device;
-
- device_info->queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED;
- err = gve_adminq_get_device_properties(priv);
- if (err)
- goto err;
-
priv->queue_format = priv->device_info.queue_format;
err = priv->ctrl_ops->set_num_ntfy_blks(priv);
if (err) {
dev_err(&priv->pdev->dev,
"Could not setup notify blocks: err=%d\n", err);
- goto err;
+ return err;
}
priv->ctrl_ops->set_num_queues(priv);
@@ -2473,10 +2484,8 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
netif_set_tso_max_size(priv->dev, GVE_DQO_TX_MAX);
}
- if (gve_set_mtu(priv)) {
- err = -EINVAL;
- goto err;
- }
+ if (gve_set_mtu(priv))
+ return -EINVAL;
priv->num_event_counters = device_info->num_event_counters;
@@ -2501,30 +2510,7 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
priv->ts_config.tx_type = HWTSTAMP_TX_OFF;
priv->ts_config.rx_filter = HWTSTAMP_FILTER_NONE;
priv->nic_timestamp_supported = device_info->nic_timestamp_supported;
-
-setup_device:
- priv->xsk_pools = bitmap_zalloc(priv->rx_cfg.max_queues, GFP_KERNEL);
- if (!priv->xsk_pools) {
- err = -ENOMEM;
- goto err;
- }
-
- gve_set_netdev_xdp_features(priv);
- if (!gve_is_gqi(priv))
- priv->dev->xdp_metadata_ops = &gve_xdp_metadata_ops;
-
- err = gve_setup_device_resources(priv);
- if (err)
- goto err_free_xsk_bitmap;
-
return 0;
-
-err_free_xsk_bitmap:
- bitmap_free(priv->xsk_pools);
- priv->xsk_pools = NULL;
-err:
- gve_adminq_free(priv);
- return err;
}
static void gve_teardown_priv_resources(struct gve_priv *priv)
@@ -2554,15 +2540,29 @@ static int gve_reset_recovery(struct gve_priv *priv, bool was_up)
{
int err;
- err = gve_init_priv(priv, true);
- if (err)
+ err = gve_adminq_init(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Failed to alloc admin queue: err=%d\n", err);
goto err;
+ }
+
+ err = gve_setup_device(priv);
+ if (err)
+ goto err_free_adminq;
if (was_up) {
err = gve_open(priv->dev);
if (err)
- goto err;
+ goto err_free_device;
}
return 0;
+
+err_free_device:
+ gve_teardown_device_resources(priv);
+ bitmap_free(priv->xsk_pools);
+ priv->xsk_pools = NULL;
+err_free_adminq:
+ gve_adminq_free(priv);
err:
dev_err(&priv->pdev->dev, "Reset failed! !!! DISABLING ALL QUEUES !!!\n");
gve_turndown(priv);
@@ -2605,7 +2605,7 @@ int gve_reset(struct gve_priv *priv, bool attempt_teardown)
return err;
}
-static void gve_write_version(u8 __iomem *driver_version_register)
+void gve_adminq_write_version(u8 __iomem *driver_version_register)
{
const char *c = gve_version_prefix;
@@ -2838,7 +2838,6 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
goto abort_with_pci_region;
}
- gve_write_version(®_bar->driver_version);
/* Get max queues to alloc etherdev */
max_tx_queues = ioread32be(®_bar->max_tx_queues);
max_rx_queues = ioread32be(®_bar->max_rx_queues);
@@ -2885,13 +2884,28 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
priv->rx_cfg.packet_buffer_size = GVE_DEFAULT_RX_BUFFER_SIZE;
priv->max_rx_buffer_size = GVE_DEFAULT_RX_BUFFER_SIZE;
+ err = gve_adminq_init(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Failed to alloc admin queue: err=%d\n", err);
+ goto abort_with_netdev;
+ }
+
+ priv->device_info.queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED;
+ err = gve_adminq_get_device_properties(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Could not get device information: err=%d\n", err);
+ goto abort_with_adminq;
+ }
+
/* Set adminq ctrl ops */
priv->ctrl_ops = &gve_adminq_ops;
err = priv->ctrl_ops->map_db_bar(priv);
if (err) {
err = -ENOMEM;
- goto abort_with_netdev;
+ goto abort_with_adminq;
}
gve_set_probe_in_progress(priv);
@@ -2906,10 +2920,17 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
priv->tx_cfg.max_queues = max_tx_queues;
priv->rx_cfg.max_queues = max_rx_queues;
- err = gve_init_priv(priv, false);
+ err = gve_init_priv(priv);
if (err)
goto abort_with_wq;
+ err = gve_setup_device(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Could not setup device: err=%d\n", err);
+ goto abort_with_wq;
+ }
+
if (!gve_is_gqi(priv) && !gve_is_qpl(priv))
dev->netmem_tx = NETMEM_TX_DMA;
@@ -2932,6 +2953,9 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
abort_with_unmap_db_bar:
priv->ctrl_ops->unmap_db_bar(priv);
+abort_with_adminq:
+ gve_adminq_free(priv);
+
abort_with_netdev:
free_netdev(dev);
--
2.54.0.1013.g208068f2d8-goog
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH net-next v2 08/15] gve: refactor gve_init_priv for reset path
2026-06-02 23:58 ` [PATCH net-next v2 08/15] gve: refactor gve_init_priv for reset path Harshitha Ramamurthy
@ 2026-06-03 23:59 ` sashiko-bot
2026-06-05 3:52 ` Harshitha Ramamurthy
1 sibling, 0 replies; 33+ messages in thread
From: sashiko-bot @ 2026-06-03 23:59 UTC (permalink / raw)
To: Harshitha Ramamurthy; +Cc: bpf
Thank you for your contribution! Sashiko AI review found 2 potential issue(s) to consider:
New issues:
- [High] gve_reset_recovery error path leaves device in a state where subsequent ndo_open causes Use-After-Free and NULL pointer dereferences.
Pre-existing issues:
- [High] Device reset leaks XSK pool DMA mappings and silently breaks XDP sockets.
--
commit b96244d58e5385fb7dc71f1bd8136017d84f17d7
Author: Harshitha Ramamurthy <hramamurthy@google.com>
gve: refactor gve_init_priv for reset path
This commit refactors gve_init_priv() into targeted setup functions to bypass
full initialization during the reset path. It moves admin queue allocation and
device properties negotiation into gve_probe(). This prepares the driver for
adding mailbox mode control plane initialization in upcoming patches.
> diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
> index d693caed7e3d1..746ff69a28ddc 100644
> --- a/drivers/net/ethernet/google/gve/gve_main.c
> +++ b/drivers/net/ethernet/google/gve/gve_main.c
> @@ -2414,6 +2416,33 @@ static void gve_set_buf_sizes(struct gve_priv *priv)
> priv->header_buf_size = device_info->header_buf_size;
> }
>
> +static int gve_setup_device(struct gve_priv *priv)
> +{
> + int err;
> +
> + priv->xsk_pools = bitmap_zalloc(priv->rx_cfg.max_queues, GFP_KERNEL);
[Severity: High]
This is a pre-existing issue, but does this allocation cause an XDP socket
DMA mapping leak and silently break XDP sockets during a device reset?
When a device reset occurs, such as from a transmit timeout, the teardown
process frees the priv->xsk_pools bitmap without unmapping the active XDP
socket DMA mappings.
Then gve_reset_recovery() calls gve_setup_device() here, which allocates a
fresh, zeroed xsk_pools bitmap. When the interface is brought back up, the
driver reads the zeroed bitmap, forgetting which queues had active XSK pools.
This causes XDP sockets to silently stop receiving traffic and permanently
leaks the DMA mappings established by gve_xsk_pool_enable().
> + if (!priv->xsk_pools) {
> + err = -ENOMEM;
> + goto err;
> + }
[ ... ]
> @@ -2554,15 +2540,29 @@ static int gve_reset_recovery(struct gve_priv *priv, bool was_up)
> {
> int err;
>
> - err = gve_init_priv(priv, true);
> - if (err)
> + err = gve_adminq_init(priv);
> + if (err) {
> + dev_err(&priv->pdev->dev,
> + "Failed to alloc admin queue: err=%d\n", err);
> goto err;
> + }
> +
> + err = gve_setup_device(priv);
> + if (err)
> + goto err_free_adminq;
> if (was_up) {
> err = gve_open(priv->dev);
> if (err)
> - goto err;
> + goto err_free_device;
> }
> return 0;
> +
> +err_free_device:
> + gve_teardown_device_resources(priv);
> + bitmap_free(priv->xsk_pools);
> + priv->xsk_pools = NULL;
> +err_free_adminq:
> + gve_adminq_free(priv);
[Severity: High]
If gve_open() fails here, the error path tears down device resources (freeing
priv->ntfy_blocks) and the admin queue DMA pool via gve_adminq_free().
However, the net_device remains registered. If a user subsequently attempts to
bring the interface up, gve_open() is invoked again. Because gve_open()
currently does not check if resources are initialized, it proceeds to call
gve_queues_start().
Could this trigger a use-after-free in gve_adminq_issue_cmd() (dereferencing
priv->adminq from the destroyed pool) and a null pointer dereference in
gve_turnup() or gve_reg_xdp_info() (dereferencing priv->ntfy_blocks)?
I notice this issue is fixed later in the series by commit d54cafae24e9
("gve: simplify reset logic") which adds a gve_get_device_resources_ok()
check at the start of gve_open().
> err:
> dev_err(&priv->pdev->dev, "Reset failed! !!! DISABLING ALL QUEUES !!!\n");
--
Sashiko AI review · https://sashiko.dev/#/patchset/20260602235900.3050782-1-hramamurthy@google.com?part=8
^ permalink raw reply [flat|nested] 33+ messages in thread* Re: [PATCH net-next v2 08/15] gve: refactor gve_init_priv for reset path
2026-06-02 23:58 ` [PATCH net-next v2 08/15] gve: refactor gve_init_priv for reset path Harshitha Ramamurthy
2026-06-03 23:59 ` sashiko-bot
@ 2026-06-05 3:52 ` Harshitha Ramamurthy
1 sibling, 0 replies; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-05 3:52 UTC (permalink / raw)
To: netdev
Cc: joshwash, andrew+netdev, davem, edumazet, kuba, pabeni, ast,
daniel, hawk, john.fastabend, sdf, willemb, jordanrhee, jfraker,
nktgrg, bpf, linux-kernel
On Tue, Jun 2, 2026 at 4:59 PM Harshitha Ramamurthy
<hramamurthy@google.com> wrote:
>
> The driver does not need to renegotiate all properties with
> the device on a reset since those should stay constant through
> a reset. Hence change gve_init_priv() into a method that only
> sets these properties into the priv structure and hence needs
> to be only called once during gve_probe().
>
> To achieve this end state of gve_init_priv(), do the following:
> - introduce gve_adminq_init() which writes the driver version register
> and allocates the AdminQ and call it in gve_probe()
> - move gve_adminq_get_device_properties() into gve_probe()
> - introduce gve_setup_device() which deals with device setup logic and
> call it in gve_probe()
> - resetting no. of registered pages is moved into gve_register_qpls()
> since that is a QPL specific property.
>
> With these changes, gve_adminq_get_device_properties() and
> gve_init_priv() are only called once during gve_probe.
> gve_reset_recovery() can now bypass full initialization and call these
> targeted setup functions directly.
>
> This prepares the driver to add mailbox mode's control plane
> initialization and device properties negotiation in the same place
> as is done in AdminQ mode in the upcoming patches when adding the
> mailbox ABI.
>
> These changes are only code movement, no functional change.
>
> Reviewed-by: Willem de Bruijn <willemb@google.com>
> Reviewed-by: Jordan Rhee <jordanrhee@google.com>
> Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
> ---
> drivers/net/ethernet/google/gve/gve.h | 2 +
> drivers/net/ethernet/google/gve/gve_adminq.c | 12 +-
> drivers/net/ethernet/google/gve/gve_adminq.h | 2 +-
> drivers/net/ethernet/google/gve/gve_main.c | 132 +++++++++++--------
> 4 files changed, 92 insertions(+), 56 deletions(-)
>
> diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
> index bd48c3d7a2b2..72588d201e5d 100644
> --- a/drivers/net/ethernet/google/gve/gve.h
> +++ b/drivers/net/ethernet/google/gve/gve.h
> @@ -1252,6 +1252,8 @@ static inline bool gve_is_clock_enabled(struct gve_priv *priv)
> return priv->nic_ts_report;
> }
>
> +void gve_adminq_write_version(u8 __iomem *driver_version_register);
> +
> /* gqi napi handler defined in gve_main.c */
> int gve_napi_poll(struct napi_struct *napi, int budget);
>
> diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
> index 923a5c737258..e7956d2768b6 100644
> --- a/drivers/net/ethernet/google/gve/gve_adminq.c
> +++ b/drivers/net/ethernet/google/gve/gve_adminq.c
> @@ -296,8 +296,10 @@ gve_process_device_options(struct gve_priv *priv,
> return 0;
> }
>
> -int gve_adminq_alloc(struct device *dev, struct gve_priv *priv)
> +static int gve_adminq_alloc(struct gve_priv *priv)
> {
> + struct device *dev = &priv->pdev->dev;
> +
> priv->adminq_pool = dma_pool_create("adminq_pool", dev,
> GVE_ADMINQ_BUFFER_SIZE, 0, 0);
> if (unlikely(!priv->adminq_pool))
> @@ -353,6 +355,14 @@ int gve_adminq_alloc(struct device *dev, struct gve_priv *priv)
> return 0;
> }
>
> +int gve_adminq_init(struct gve_priv *priv)
> +{
> + struct gve_registers __iomem *reg_bar = priv->reg_bar0;
> +
> + gve_adminq_write_version(®_bar->driver_version);
> + return gve_adminq_alloc(priv);
> +}
> +
> void gve_adminq_release(struct gve_priv *priv)
> {
> int i = 0;
> diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
> index ac3ef9fd8d24..d07e9c6f279d 100644
> --- a/drivers/net/ethernet/google/gve/gve_adminq.h
> +++ b/drivers/net/ethernet/google/gve/gve_adminq.h
> @@ -619,7 +619,7 @@ union gve_adminq_command {
>
> static_assert(sizeof(union gve_adminq_command) == 64);
>
> -int gve_adminq_alloc(struct device *dev, struct gve_priv *priv);
> +int gve_adminq_init(struct gve_priv *priv);
> void gve_adminq_free(struct gve_priv *priv);
> void gve_adminq_release(struct gve_priv *priv);
> int gve_adminq_describe_device(struct gve_priv *priv);
> diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
> index d693caed7e3d..746ff69a28dd 100644
> --- a/drivers/net/ethernet/google/gve/gve_main.c
> +++ b/drivers/net/ethernet/google/gve/gve_main.c
> @@ -797,6 +797,8 @@ static int gve_register_qpls(struct gve_priv *priv)
> int err;
> int i;
>
> + priv->num_registered_pages = 0;
> +
Commenting on Sashiko's report:
"When gve_reset_recovery() runs with was_up=false, gve_open() is not
called and gve_register_qpls() is not reached, so num_registered_pages
is no longer zeroed on that path. Is that intentional, given the
symmetry with gve_unregister_qpl() decrements?
Although this number will get reset every time queues are created, to
be more symmetric, resetting this in gve_recover() which calls
gve_setup_device or in gve_setup_device is better. Will fix in v3.
> num_tx_qpls = gve_num_tx_qpls(&priv->tx_cfg, gve_is_qpl(priv));
> num_rx_qpls = gve_num_rx_qpls(&priv->rx_cfg, gve_is_qpl(priv));
>
> @@ -2414,6 +2416,33 @@ static void gve_set_buf_sizes(struct gve_priv *priv)
> priv->header_buf_size = device_info->header_buf_size;
> }
>
> +static int gve_setup_device(struct gve_priv *priv)
> +{
> + int err;
> +
> + priv->xsk_pools = bitmap_zalloc(priv->rx_cfg.max_queues, GFP_KERNEL);
Commenting on Sashiko's report:
"This is a pre-existing issue, but does this allocation cause an XDP socket
DMA mapping leak and silently break XDP sockets during a device reset?
When a device reset occurs, such as from a transmit timeout, the teardown
process frees the priv->xsk_pools bitmap without unmapping the active XDP
socket DMA mappings."
Will send a separate net fix for this pre-existing issue.
> + if (!priv->xsk_pools) {
> + err = -ENOMEM;
> + goto err;
> + }
> +
> + gve_set_netdev_xdp_features(priv);
> + if (!gve_is_gqi(priv))
> + priv->dev->xdp_metadata_ops = &gve_xdp_metadata_ops;
> +
> + err = gve_setup_device_resources(priv);
> + if (err)
> + goto err_free_xsk_bitmap;
> +
> + return 0;
> +
> +err_free_xsk_bitmap:
> + bitmap_free(priv->xsk_pools);
> + priv->xsk_pools = NULL;
> +err:
> + return err;
> +}
> +
> static const struct gve_ctrl_ops gve_adminq_ops = {
> .map_db_bar = gve_adminq_map_db_bar,
> .unmap_db_bar = gve_adminq_unmap_db_bar,
> @@ -2421,36 +2450,18 @@ static const struct gve_ctrl_ops gve_adminq_ops = {
> .set_num_ntfy_blks = gve_adminq_set_num_ntfy_blks,
> };
>
> -static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
> +static int gve_init_priv(struct gve_priv *priv)
> {
> struct gve_device_info *device_info = &priv->device_info;
> int err;
>
> - /* Set up the adminq */
> - err = gve_adminq_alloc(&priv->pdev->dev, priv);
> - if (err) {
> - dev_err(&priv->pdev->dev,
> - "Failed to alloc admin queue: err=%d\n", err);
> - return err;
> - }
> -
> - priv->num_registered_pages = 0;
> -
> - if (skip_describe_device)
> - goto setup_device;
> -
> - device_info->queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED;
> - err = gve_adminq_get_device_properties(priv);
> - if (err)
> - goto err;
> -
> priv->queue_format = priv->device_info.queue_format;
>
> err = priv->ctrl_ops->set_num_ntfy_blks(priv);
> if (err) {
> dev_err(&priv->pdev->dev,
> "Could not setup notify blocks: err=%d\n", err);
> - goto err;
> + return err;
> }
>
> priv->ctrl_ops->set_num_queues(priv);
> @@ -2473,10 +2484,8 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
> netif_set_tso_max_size(priv->dev, GVE_DQO_TX_MAX);
> }
>
> - if (gve_set_mtu(priv)) {
> - err = -EINVAL;
> - goto err;
> - }
> + if (gve_set_mtu(priv))
> + return -EINVAL;
>
> priv->num_event_counters = device_info->num_event_counters;
>
> @@ -2501,30 +2510,7 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
> priv->ts_config.tx_type = HWTSTAMP_TX_OFF;
> priv->ts_config.rx_filter = HWTSTAMP_FILTER_NONE;
> priv->nic_timestamp_supported = device_info->nic_timestamp_supported;
> -
> -setup_device:
> - priv->xsk_pools = bitmap_zalloc(priv->rx_cfg.max_queues, GFP_KERNEL);
> - if (!priv->xsk_pools) {
> - err = -ENOMEM;
> - goto err;
> - }
> -
> - gve_set_netdev_xdp_features(priv);
> - if (!gve_is_gqi(priv))
> - priv->dev->xdp_metadata_ops = &gve_xdp_metadata_ops;
> -
> - err = gve_setup_device_resources(priv);
> - if (err)
> - goto err_free_xsk_bitmap;
> -
> return 0;
> -
> -err_free_xsk_bitmap:
> - bitmap_free(priv->xsk_pools);
> - priv->xsk_pools = NULL;
> -err:
> - gve_adminq_free(priv);
> - return err;
> }
>
> static void gve_teardown_priv_resources(struct gve_priv *priv)
> @@ -2554,15 +2540,29 @@ static int gve_reset_recovery(struct gve_priv *priv, bool was_up)
> {
> int err;
>
> - err = gve_init_priv(priv, true);
> - if (err)
> + err = gve_adminq_init(priv);
> + if (err) {
> + dev_err(&priv->pdev->dev,
> + "Failed to alloc admin queue: err=%d\n", err);
> goto err;
> + }
> +
> + err = gve_setup_device(priv);
> + if (err)
> + goto err_free_adminq;
> if (was_up) {
> err = gve_open(priv->dev);
> if (err)
> - goto err;
> + goto err_free_device;
> }
> return 0;
> +
> +err_free_device:
> + gve_teardown_device_resources(priv);
> + bitmap_free(priv->xsk_pools);
> + priv->xsk_pools = NULL;
> +err_free_adminq:
> + gve_adminq_free(priv);
"If gve_open() fails here, the error path tears down device resources (freeing
priv->ntfy_blocks) and the admin queue DMA pool via gve_adminq_free().
However, the net_device remains registered. If a user subsequently attempts to
bring the interface up, gve_open() is invoked again. Because gve_open()
currently does not check if resources are initialized, it proceeds to call
gve_queues_start()."
Correct, if gve_open() fails, will just return error instead. Will fix in v3.
> err:
> dev_err(&priv->pdev->dev, "Reset failed! !!! DISABLING ALL QUEUES !!!\n");
> gve_turndown(priv);
> @@ -2605,7 +2605,7 @@ int gve_reset(struct gve_priv *priv, bool attempt_teardown)
> return err;
> }
>
> -static void gve_write_version(u8 __iomem *driver_version_register)
> +void gve_adminq_write_version(u8 __iomem *driver_version_register)
> {
> const char *c = gve_version_prefix;
>
> @@ -2838,7 +2838,6 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
> goto abort_with_pci_region;
> }
>
> - gve_write_version(®_bar->driver_version);
> /* Get max queues to alloc etherdev */
> max_tx_queues = ioread32be(®_bar->max_tx_queues);
> max_rx_queues = ioread32be(®_bar->max_rx_queues);
> @@ -2885,13 +2884,28 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
> priv->rx_cfg.packet_buffer_size = GVE_DEFAULT_RX_BUFFER_SIZE;
> priv->max_rx_buffer_size = GVE_DEFAULT_RX_BUFFER_SIZE;
>
> + err = gve_adminq_init(priv);
> + if (err) {
> + dev_err(&priv->pdev->dev,
> + "Failed to alloc admin queue: err=%d\n", err);
> + goto abort_with_netdev;
> + }
> +
> + priv->device_info.queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED;
> + err = gve_adminq_get_device_properties(priv);
> + if (err) {
> + dev_err(&priv->pdev->dev,
> + "Could not get device information: err=%d\n", err);
> + goto abort_with_adminq;
> + }
> +
> /* Set adminq ctrl ops */
> priv->ctrl_ops = &gve_adminq_ops;
>
> err = priv->ctrl_ops->map_db_bar(priv);
> if (err) {
> err = -ENOMEM;
> - goto abort_with_netdev;
> + goto abort_with_adminq;
> }
>
> gve_set_probe_in_progress(priv);
> @@ -2906,10 +2920,17 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
> priv->tx_cfg.max_queues = max_tx_queues;
> priv->rx_cfg.max_queues = max_rx_queues;
>
> - err = gve_init_priv(priv, false);
> + err = gve_init_priv(priv);
> if (err)
> goto abort_with_wq;
>
> + err = gve_setup_device(priv);
> + if (err) {
> + dev_err(&priv->pdev->dev,
> + "Could not setup device: err=%d\n", err);
> + goto abort_with_wq;
> + }
> +
> if (!gve_is_gqi(priv) && !gve_is_qpl(priv))
> dev->netmem_tx = NETMEM_TX_DMA;
>
> @@ -2932,6 +2953,9 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
> abort_with_unmap_db_bar:
> priv->ctrl_ops->unmap_db_bar(priv);
>
> +abort_with_adminq:
> + gve_adminq_free(priv);
> +
> abort_with_netdev:
> free_netdev(dev);
>
> --
> 2.54.0.1013.g208068f2d8-goog
>
^ permalink raw reply [flat|nested] 33+ messages in thread
* [PATCH net-next v2 09/15] gve: simplify reset logic
2026-06-02 23:58 [PATCH net-next v2 00/15] gve: AdminQ mode related refactors Harshitha Ramamurthy
` (7 preceding siblings ...)
2026-06-02 23:58 ` [PATCH net-next v2 08/15] gve: refactor gve_init_priv for reset path Harshitha Ramamurthy
@ 2026-06-02 23:58 ` Harshitha Ramamurthy
2026-06-03 23:59 ` sashiko-bot
2026-06-07 22:20 ` Joshua Washington
2026-06-02 23:58 ` [PATCH net-next v2 10/15] gve: add gve_ctrl_ops for gve initialization/teardown sequences Harshitha Ramamurthy
` (5 subsequent siblings)
14 siblings, 2 replies; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-02 23:58 UTC (permalink / raw)
To: netdev
Cc: joshwash, hramamurthy, andrew+netdev, davem, edumazet, kuba,
pabeni, ast, daniel, hawk, john.fastabend, sdf, willemb,
jordanrhee, jfraker, nktgrg, bpf, linux-kernel
From: Joshua Washington <joshwash@google.com>
Current GVE reset logic is quite complex, with a number of methods with
similar names and functionalities. This complexity has allowed a number
of bugs to enter the reset/recovery path, including the potential for
reset loops if an operation fails during teardown.
Simplify the reset path by doing the following:
1) Removing recursive resets. Recursive resets have two major issues.
First, there is the potential for stack overflows if resets are
invoked too many times in a row. Second, long recursive calls mean
that GVE never gives up the RTNL lock, or at the very least holds it
for too long. If a reset must occur anywhere during the
reset/recovery path, it should be scheduled as a separate task.
2) Removing resets during teardown. This is partly covered by removing
recursive resets, but the primary goal in this case is to ensure that
the driver is capable of actually executing a hardware reset if
something goes wrong with a control plane operation. As it stands, if
`deconfigure_device_resources` fails, for example, GVE will preempt
its reset with another reset without actually invoking a hardware
reset, which could actually help with recovery.
3) Decompose allocation/de-allocation and setup/teardown. Performing
allocation and setup for each control plane system (RSS, ptype map,
etc) leaves many more error conditions to handle, causing teardown in
the case of failures to be much more complex than they need to be.
This will also be useful to better align a major behavioral change
in mailbox mode, which will use separate response buffers instead to
get data from the device instead of a pre-allocated shared memory
region.
With the new reset functionality, shared resources between the device
and driver are not freed until after the hardware reset has completed
in the event that `deconfigure_device_resources` fails, meaning that the
device could potentially still be holding on to shared memory.
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Jordan Rhee <jordanrhee@google.com>
Signed-off-by: Joshua Washington <joshwash@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
Changes in v2:
- Fixed typos in commit message (recursive, preempt)
- Fixed a kdoc warning
drivers/net/ethernet/google/gve/gve.h | 2 +-
drivers/net/ethernet/google/gve/gve_adminq.c | 5 +-
drivers/net/ethernet/google/gve/gve_adminq.h | 1 -
drivers/net/ethernet/google/gve/gve_ethtool.c | 2 +-
drivers/net/ethernet/google/gve/gve_main.c | 338 +++++++++---------
5 files changed, 167 insertions(+), 181 deletions(-)
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index 72588d201e5d..c2e68fd97de5 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -1347,7 +1347,7 @@ struct page_pool *gve_rx_create_page_pool(struct gve_priv *priv,
/* Reset */
void gve_schedule_reset(struct gve_priv *priv);
-int gve_reset(struct gve_priv *priv, bool attempt_teardown);
+int gve_reset(struct gve_priv *priv, bool skip_queue_setup);
void gve_get_curr_alloc_cfgs(struct gve_priv *priv,
struct gve_tx_alloc_rings_cfg *tx_alloc_cfg,
struct gve_rx_alloc_rings_cfg *rx_alloc_cfg);
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index e7956d2768b6..84b548467a0c 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -363,7 +363,7 @@ int gve_adminq_init(struct gve_priv *priv)
return gve_adminq_alloc(priv);
}
-void gve_adminq_release(struct gve_priv *priv)
+static void gve_adminq_release(struct gve_priv *priv)
{
int i = 0;
@@ -392,7 +392,6 @@ void gve_adminq_release(struct gve_priv *priv)
}
gve_clear_device_rings_ok(priv);
gve_clear_device_resources_ok(priv);
- gve_clear_admin_queue_ok(priv);
}
void gve_adminq_free(struct gve_priv *priv)
@@ -1377,7 +1376,7 @@ gve_adminq_configure_flow_rule(struct gve_priv *priv,
if (err == -ETIME) {
dev_err(&priv->pdev->dev, "Timeout to configure the flow rule, trigger reset");
- gve_reset(priv, true);
+ gve_reset(priv, false);
} else if (!err) {
priv->flow_rules_cache.rules_cache_synced = false;
}
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
index d07e9c6f279d..12d2d1347e21 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.h
+++ b/drivers/net/ethernet/google/gve/gve_adminq.h
@@ -621,7 +621,6 @@ static_assert(sizeof(union gve_adminq_command) == 64);
int gve_adminq_init(struct gve_priv *priv);
void gve_adminq_free(struct gve_priv *priv);
-void gve_adminq_release(struct gve_priv *priv);
int gve_adminq_describe_device(struct gve_priv *priv);
int gve_adminq_configure_device_resources(struct gve_priv *priv,
dma_addr_t counter_array_bus_addr,
diff --git a/drivers/net/ethernet/google/gve/gve_ethtool.c b/drivers/net/ethernet/google/gve/gve_ethtool.c
index dc2213b5ce24..54b623b678ce 100644
--- a/drivers/net/ethernet/google/gve/gve_ethtool.c
+++ b/drivers/net/ethernet/google/gve/gve_ethtool.c
@@ -651,7 +651,7 @@ static int gve_user_reset(struct net_device *netdev, u32 *flags)
if (*flags == ETH_RESET_ALL) {
*flags = 0;
- return gve_reset(priv, true);
+ return gve_reset(priv, false);
}
return -EOPNOTSUPP;
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 746ff69a28dd..f8289f478e5b 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -591,7 +591,22 @@ static void gve_free_notify_blocks(struct gve_priv *priv)
priv->msix_vectors = NULL;
}
-static int gve_setup_device_resources(struct gve_priv *priv)
+static void gve_free_control_plane_resources(struct gve_priv *priv)
+{
+ bitmap_free(priv->xsk_pools);
+ priv->xsk_pools = NULL;
+
+ kvfree(priv->ptype_lut_dqo);
+ priv->ptype_lut_dqo = NULL;
+
+ gve_free_stats_report(priv);
+ gve_free_notify_blocks(priv);
+ gve_free_counter_array(priv);
+ gve_free_rss_config_cache(priv);
+ gve_free_flow_rule_caches(priv);
+}
+
+static int gve_alloc_control_plane_resources(struct gve_priv *priv)
{
int err;
@@ -600,16 +615,42 @@ static int gve_setup_device_resources(struct gve_priv *priv)
return err;
err = gve_alloc_rss_config_cache(priv);
if (err)
- goto abort_with_flow_rule_caches;
+ goto abort;
err = gve_alloc_counter_array(priv);
if (err)
- goto abort_with_rss_config_cache;
+ goto abort;
err = gve_alloc_notify_blocks(priv);
if (err)
- goto abort_with_counter;
+ goto abort;
err = gve_alloc_stats_report(priv);
if (err)
- goto abort_with_ntfy_blocks;
+ goto abort;
+
+ if (!gve_is_gqi(priv)) {
+ priv->ptype_lut_dqo = kvzalloc_obj(*priv->ptype_lut_dqo,
+ GFP_KERNEL);
+ if (!priv->ptype_lut_dqo) {
+ err = -ENOMEM;
+ goto abort;
+ }
+ }
+
+ priv->xsk_pools = bitmap_zalloc(priv->rx_cfg.max_queues, GFP_KERNEL);
+ if (!priv->xsk_pools) {
+ err = -ENOMEM;
+ goto abort;
+ }
+
+ return 0;
+abort:
+ gve_free_control_plane_resources(priv);
+ return err;
+}
+
+static int gve_setup_control_plane_resources(struct gve_priv *priv)
+{
+ int err = 0;
+
err = gve_adminq_configure_device_resources(priv,
priv->counter_array_bus,
priv->num_event_counters,
@@ -619,20 +660,15 @@ static int gve_setup_device_resources(struct gve_priv *priv)
dev_err(&priv->pdev->dev,
"could not setup device_resources: err=%d\n", err);
err = -ENXIO;
- goto abort_with_stats_report;
+ return err;
}
if (!gve_is_gqi(priv)) {
- priv->ptype_lut_dqo = kvzalloc_obj(*priv->ptype_lut_dqo);
- if (!priv->ptype_lut_dqo) {
- err = -ENOMEM;
- goto abort_with_stats_report;
- }
err = gve_adminq_get_ptype_map_dqo(priv, priv->ptype_lut_dqo);
if (err) {
dev_err(&priv->pdev->dev,
"Failed to get ptype map: err=%d\n", err);
- goto abort_with_ptype_lut;
+ goto deconfigure_device;
}
}
@@ -647,7 +683,7 @@ static int gve_setup_device_resources(struct gve_priv *priv)
err = gve_init_rss_config(priv, priv->rx_cfg.num_queues);
if (err) {
dev_err(&priv->pdev->dev, "Failed to init RSS config");
- goto abort_with_clock;
+ goto teardown_clock;
}
err = gve_adminq_report_stats(priv, priv->stats_report_len,
@@ -659,67 +695,61 @@ static int gve_setup_device_resources(struct gve_priv *priv)
gve_set_device_resources_ok(priv);
return 0;
-abort_with_clock:
+teardown_clock:
gve_teardown_clock(priv);
-abort_with_ptype_lut:
- kvfree(priv->ptype_lut_dqo);
- priv->ptype_lut_dqo = NULL;
-abort_with_stats_report:
- gve_free_stats_report(priv);
-abort_with_ntfy_blocks:
- gve_free_notify_blocks(priv);
-abort_with_counter:
- gve_free_counter_array(priv);
-abort_with_rss_config_cache:
- gve_free_rss_config_cache(priv);
-abort_with_flow_rule_caches:
- gve_free_flow_rule_caches(priv);
-
+deconfigure_device:
+ gve_adminq_deconfigure_device_resources(priv);
return err;
}
-static void gve_trigger_reset(struct gve_priv *priv);
-
-static void gve_teardown_device_resources(struct gve_priv *priv)
+/*
+ * Request the device to release any allocated shared resources.
+ *
+ * If any part of the teardown step fails, the failure is documented, but is
+ * otherwise ignored. It is expected that a device reset is triggered
+ * immediately after tearing down device resources, which would clear any
+ * lingering state on the device.
+ */
+static void gve_teardown_control_plane_resources(struct gve_priv *priv)
{
int err;
/* Tell device its resources are being freed */
if (gve_get_device_resources_ok(priv)) {
err = gve_flow_rules_reset(priv);
- if (err) {
+ if (err)
dev_err(&priv->pdev->dev,
"Failed to reset flow rules: err=%d\n", err);
- gve_trigger_reset(priv);
- }
/* detach the stats report */
err = gve_adminq_report_stats(priv, 0, 0x0, GVE_STATS_REPORT_TIMER_PERIOD);
- if (err) {
+ if (err)
dev_err(&priv->pdev->dev,
"Failed to detach stats report: err=%d\n", err);
- gve_trigger_reset(priv);
- }
+ gve_teardown_clock(priv);
err = gve_adminq_deconfigure_device_resources(priv);
- if (err) {
+ if (err)
dev_err(&priv->pdev->dev,
"Could not deconfigure device resources: err=%d\n",
err);
- gve_trigger_reset(priv);
- }
}
- kvfree(priv->ptype_lut_dqo);
- priv->ptype_lut_dqo = NULL;
-
- gve_free_flow_rule_caches(priv);
- gve_free_rss_config_cache(priv);
- gve_free_counter_array(priv);
- gve_free_notify_blocks(priv);
- gve_free_stats_report(priv);
- gve_teardown_clock(priv);
gve_clear_device_resources_ok(priv);
}
+static void gve_teardown_device(struct gve_priv *priv)
+{
+ gve_teardown_control_plane_resources(priv);
+ gve_adminq_free(priv);
+ /*
+ * Free any resources shared with the device only after we have a
+ * guarantee that the device will not try to access such resources.
+ * Device commands in gve_teardown_control_plane_resources can fail, in
+ * which case, device resources won't be relinquished until
+ * gve_adminq_free is called to trigger a device reset.
+ */
+ gve_free_control_plane_resources(priv);
+}
+
static int gve_unregister_qpl(struct gve_priv *priv,
struct gve_queue_page_list *qpl)
{
@@ -1160,8 +1190,6 @@ void gve_schedule_reset(struct gve_priv *priv)
queue_work(priv->gve_wq, &priv->service_task);
}
-static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up);
-static int gve_reset_recovery(struct gve_priv *priv, bool was_up);
static void gve_turndown(struct gve_priv *priv);
static void gve_turnup(struct gve_priv *priv);
@@ -1272,11 +1300,12 @@ static int gve_reg_xdp_info(struct gve_priv *priv, struct net_device *dev)
return err;
}
-
static void gve_drain_page_cache(struct gve_priv *priv)
{
int i;
+ if (!priv->rx)
+ return;
for (i = 0; i < priv->rx_cfg.num_queues; i++)
page_frag_cache_drain(&priv->rx[i].page_cache);
}
@@ -1419,10 +1448,11 @@ static int gve_queues_start(struct gve_priv *priv,
reset:
if (gve_get_reset_in_progress(priv))
goto stop_and_free_rings;
- gve_reset_and_teardown(priv, true);
- /* if this fails there is nothing we can do so just ignore the return */
- gve_reset_recovery(priv, false);
- /* return the original error */
+
+ /* Attempt to reset. If reset is successful, gve_queues_start was
+ * successful.
+ */
+ err = gve_reset(priv, false);
return err;
stop_and_free_rings:
gve_tx_stop_rings(priv, gve_num_tx_queues(priv));
@@ -1438,6 +1468,12 @@ static int gve_open(struct net_device *dev)
struct gve_priv *priv = netdev_priv(dev);
int err;
+ if (!gve_get_device_resources_ok(priv)) {
+ dev_err(&priv->pdev->dev,
+ "Attempting to open netdev without resources. Device must be reset.");
+ return -ENODEV;
+ }
+
gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg);
err = gve_queues_mem_alloc(priv, &tx_alloc_cfg, &rx_alloc_cfg);
@@ -1454,41 +1490,18 @@ static int gve_open(struct net_device *dev)
return 0;
}
-static int gve_queues_stop(struct gve_priv *priv)
+static void gve_queues_stop(struct gve_priv *priv)
{
- int err;
-
- netif_carrier_off(priv->dev);
- if (gve_get_device_rings_ok(priv)) {
- gve_turndown(priv);
- gve_drain_page_cache(priv);
- err = gve_destroy_rings(priv);
- if (err)
- goto err;
- err = gve_unregister_qpls(priv);
- if (err)
- goto err;
- gve_clear_device_rings_ok(priv);
- }
- timer_delete_sync(&priv->stats_report_timer);
+ gve_turndown(priv);
gve_unreg_xdp_info(priv);
+ gve_drain_page_cache(priv);
+
+ timer_delete_sync(&priv->stats_report_timer);
+ cancel_work_sync(&priv->stats_report_task);
gve_tx_stop_rings(priv, gve_num_tx_queues(priv));
gve_rx_stop_rings(priv, priv->rx_cfg.num_queues);
-
- priv->interface_down_cnt++;
- return 0;
-
-err:
- /* This must have been called from a reset due to the rtnl lock
- * so just return at this point.
- */
- if (gve_get_reset_in_progress(priv))
- return err;
- /* Otherwise reset before returning */
- gve_reset_and_teardown(priv, true);
- return gve_reset_recovery(priv, false);
}
static int gve_close(struct net_device *dev)
@@ -1496,12 +1509,28 @@ static int gve_close(struct net_device *dev)
struct gve_priv *priv = netdev_priv(dev);
int err;
- err = gve_queues_stop(priv);
- if (err)
- return err;
+ gve_queues_stop(priv);
+
+ /* Surrender to reset if the queue destroying adminq cmds fail. Reset
+ * will not re-enable the interface.
+ */
+ if (gve_get_device_rings_ok(priv)) {
+ gve_clear_device_rings_ok(priv);
+ err = gve_destroy_rings(priv);
+ if (err)
+ goto reset;
+ err = gve_unregister_qpls(priv);
+ if (err)
+ goto reset;
+ }
gve_queues_mem_remove(priv);
+ priv->interface_down_cnt++;
return 0;
+
+reset:
+ err = gve_reset(priv, true);
+ return err;
}
static void gve_handle_link_status(struct gve_priv *priv, bool link_status)
@@ -2420,25 +2449,17 @@ static int gve_setup_device(struct gve_priv *priv)
{
int err;
- priv->xsk_pools = bitmap_zalloc(priv->rx_cfg.max_queues, GFP_KERNEL);
- if (!priv->xsk_pools) {
- err = -ENOMEM;
- goto err;
- }
-
gve_set_netdev_xdp_features(priv);
if (!gve_is_gqi(priv))
priv->dev->xdp_metadata_ops = &gve_xdp_metadata_ops;
- err = gve_setup_device_resources(priv);
+ err = gve_alloc_control_plane_resources(priv);
if (err)
- goto err_free_xsk_bitmap;
-
+ goto err;
+ err = gve_setup_control_plane_resources(priv);
+ if (err)
+ goto err;
return 0;
-
-err_free_xsk_bitmap:
- bitmap_free(priv->xsk_pools);
- priv->xsk_pools = NULL;
err:
return err;
}
@@ -2513,30 +2534,7 @@ static int gve_init_priv(struct gve_priv *priv)
return 0;
}
-static void gve_teardown_priv_resources(struct gve_priv *priv)
-{
- gve_teardown_device_resources(priv);
- gve_adminq_free(priv);
- bitmap_free(priv->xsk_pools);
- priv->xsk_pools = NULL;
-}
-
-static void gve_trigger_reset(struct gve_priv *priv)
-{
- /* Reset the device by releasing the AQ */
- gve_adminq_release(priv);
-}
-
-static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up)
-{
- gve_trigger_reset(priv);
- /* With the reset having already happened, close cannot fail */
- if (was_up)
- gve_close(priv->dev);
- gve_teardown_priv_resources(priv);
-}
-
-static int gve_reset_recovery(struct gve_priv *priv, bool was_up)
+static int gve_recover(struct gve_priv *priv, bool setup_queues)
{
int err;
@@ -2549,54 +2547,52 @@ static int gve_reset_recovery(struct gve_priv *priv, bool was_up)
err = gve_setup_device(priv);
if (err)
- goto err_free_adminq;
- if (was_up) {
+ goto err;
+ if (setup_queues) {
err = gve_open(priv->dev);
if (err)
- goto err_free_device;
+ goto err;
}
return 0;
-err_free_device:
- gve_teardown_device_resources(priv);
- bitmap_free(priv->xsk_pools);
- priv->xsk_pools = NULL;
-err_free_adminq:
- gve_adminq_free(priv);
err:
- dev_err(&priv->pdev->dev, "Reset failed! !!! DISABLING ALL QUEUES !!!\n");
- gve_turndown(priv);
+ dev_err(&priv->pdev->dev, "Recover failed! !!! DISABLING ALL QUEUES !!!\n");
+ gve_teardown_device(priv);
return err;
}
-int gve_reset(struct gve_priv *priv, bool attempt_teardown)
+int gve_reset(struct gve_priv *priv, bool skip_queue_setup)
{
bool was_up = netif_running(priv->dev);
int err;
+ if (gve_get_reset_in_progress(priv))
+ return 0;
+
dev_info(&priv->pdev->dev, "Performing reset\n");
gve_clear_do_reset(priv);
gve_set_reset_in_progress(priv);
- /* If we aren't attempting to teardown normally, just go turndown and
- * reset right away.
- */
- if (!attempt_teardown) {
- gve_turndown(priv);
- gve_reset_and_teardown(priv, was_up);
- } else {
- /* Otherwise attempt to close normally */
- if (was_up) {
- err = gve_close(priv->dev);
- /* If that fails reset as we did above */
- if (err)
- gve_reset_and_teardown(priv, was_up);
- }
- /* Clean up any remaining resources */
- gve_teardown_priv_resources(priv);
+
+ if (was_up) {
+ gve_queues_stop(priv);
+ gve_destroy_rings(priv);
+ gve_unregister_qpls(priv);
}
- /* Set it all back up */
- err = gve_reset_recovery(priv, was_up);
+ /* Reset the device by releasing the AQ. Rings and other resources
+ * within the NIC are implicitly destroyed if commands fail.
+ */
+ gve_adminq_free(priv);
+
+ gve_queues_mem_remove(priv);
+ gve_teardown_clock(priv);
+ gve_free_control_plane_resources(priv);
+
+ err = gve_recover(priv, was_up && !skip_queue_setup);
+ if (err)
+ dev_info(&priv->pdev->dev,
+ "Failed to recover in reset: %d\n", err);
+
gve_clear_reset_in_progress(priv);
priv->reset_cnt++;
priv->interface_up_cnt = 0;
@@ -2928,7 +2924,7 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
if (err) {
dev_err(&priv->pdev->dev,
"Could not setup device: err=%d\n", err);
- goto abort_with_wq;
+ goto abort_teardown_device;
}
if (!gve_is_gqi(priv) && !gve_is_qpl(priv))
@@ -2936,7 +2932,7 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
err = register_netdev(dev);
if (err)
- goto abort_with_gve_init;
+ goto abort_teardown_device;
dev_info(&pdev->dev, "GVE version %s\n", gve_version_str);
dev_info(&pdev->dev, "GVE queue format %d\n", (int)priv->queue_format);
@@ -2944,8 +2940,8 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
queue_work(priv->gve_wq, &priv->service_task);
return 0;
-abort_with_gve_init:
- gve_teardown_priv_resources(priv);
+abort_teardown_device:
+ gve_teardown_device(priv);
abort_with_wq:
destroy_workqueue(priv->gve_wq);
@@ -2977,8 +2973,8 @@ static void gve_remove(struct pci_dev *pdev)
void __iomem *reg_bar = priv->reg_bar0;
unregister_netdev(netdev);
- gve_teardown_priv_resources(priv);
destroy_workqueue(priv->gve_wq);
+ gve_teardown_device(priv);
priv->ctrl_ops->unmap_db_bar(priv);
free_netdev(netdev);
pci_iounmap(pdev, reg_bar);
@@ -2996,13 +2992,9 @@ static void gve_shutdown(struct pci_dev *pdev)
rtnl_lock();
netdev_lock(netdev);
- if (was_up && gve_close(priv->dev)) {
- /* If the dev was up, attempt to close, if close fails, reset */
- gve_reset_and_teardown(priv, was_up);
- } else {
- /* If the dev wasn't up or close worked, finish tearing down */
- gve_teardown_priv_resources(priv);
- }
+ if (was_up)
+ gve_close(priv->dev);
+ gve_teardown_device(priv);
netdev_unlock(netdev);
rtnl_unlock();
}
@@ -3017,13 +3009,9 @@ static int gve_suspend(struct device *dev)
priv->suspend_cnt++;
rtnl_lock();
netdev_lock(netdev);
- if (was_up && gve_close(priv->dev)) {
- /* If the dev was up, attempt to close, if close fails, reset */
- gve_reset_and_teardown(priv, was_up);
- } else {
- /* If the dev wasn't up or close worked, finish tearing down */
- gve_teardown_priv_resources(priv);
- }
+ if (was_up)
+ gve_close(priv->dev);
+ gve_teardown_device(priv);
priv->up_before_suspend = was_up;
netdev_unlock(netdev);
rtnl_unlock();
@@ -3040,7 +3028,7 @@ static int gve_resume(struct device *dev)
priv->resume_cnt++;
rtnl_lock();
netdev_lock(netdev);
- err = gve_reset_recovery(priv, priv->up_before_suspend);
+ err = gve_recover(priv, priv->up_before_suspend);
netdev_unlock(netdev);
rtnl_unlock();
return err;
--
2.54.0.1013.g208068f2d8-goog
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH net-next v2 09/15] gve: simplify reset logic
2026-06-02 23:58 ` [PATCH net-next v2 09/15] gve: simplify reset logic Harshitha Ramamurthy
@ 2026-06-03 23:59 ` sashiko-bot
2026-06-07 22:20 ` Joshua Washington
1 sibling, 0 replies; 33+ messages in thread
From: sashiko-bot @ 2026-06-03 23:59 UTC (permalink / raw)
To: Harshitha Ramamurthy; +Cc: bpf
Thank you for your contribution! Sashiko AI review found 3 potential issue(s) to consider:
New issues:
- [Critical] In `gve_remove()`, the workqueue is destroyed before interrupts are unregistered, leading to a Use-After-Free if an interrupt fires and queues work.
- [High] Missing state check in `gve_reset()` leads to Use-After-Free of `priv->adminq` when called concurrently or after system suspend/shutdown.
Pre-existing issues:
- [High] XDP rxq info is leaked when gve_queues_start fails during an in-progress reset, leaving dangling pointers in the network stack.
--
commit d54cafae24e9e84884315921d064dc0a6c4c8cc6
Author: Joshua Washington <joshwash@google.com>
gve: simplify reset logic
This commit simplifies the reset logic in the GVE driver by removing
recursive resets, avoiding resets during teardown, and decomposing allocation
and setup steps.
> diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
> index 746ff69a28ddc..f94f3536be4c6 100644
> --- a/drivers/net/ethernet/google/gve/gve_main.c
> +++ b/drivers/net/ethernet/google/gve/gve_main.c
> @@ -1419,10 +1448,11 @@ static int gve_queues_start(struct gve_priv *priv,
> reset:
> if (gve_get_reset_in_progress(priv))
> goto stop_and_free_rings;
> - gve_reset_and_teardown(priv, true);
> - /* if this fails there is nothing we can do so just ignore the return */
> - gve_reset_recovery(priv, false);
> - /* return the original error */
> +
> + /* Attempt to reset. If reset is successful, gve_queues_start was
> + * successful.
> + */
> + err = gve_reset(priv, false);
> return err;
> stop_and_free_rings:
> gve_tx_stop_rings(priv, gve_num_tx_queues(priv));
> gve_rx_stop_rings(priv, priv->rx_cfg.num_queues);
[Severity: High]
This is a pre-existing issue, but in the stop_and_free_rings error path
inside gve_queues_start(), the rx arrays are freed via
gve_queues_mem_remove(priv) without unregistering the XDP rxq structures.
Could this leak xdp_rxq info and leave dangling pointers in the network
stack?
[ ... ]
> @@ -2549,54 +2547,52 @@ static int gve_reset_recovery(struct gve_priv *priv, bool was_up)
[ ... ]
> -int gve_reset(struct gve_priv *priv, bool attempt_teardown)
> +int gve_reset(struct gve_priv *priv, bool skip_queue_setup)
> {
> bool was_up = netif_running(priv->dev);
> int err;
>
> + if (gve_get_reset_in_progress(priv))
> + return 0;
> +
> dev_info(&priv->pdev->dev, "Performing reset\n");
> gve_clear_do_reset(priv);
> gve_set_reset_in_progress(priv);
> - /* If we aren't attempting to teardown normally, just go turndown and
> - * reset right away.
> - */
> - if (!attempt_teardown) {
> - gve_turndown(priv);
> - gve_reset_and_teardown(priv, was_up);
> - } else {
> - /* Otherwise attempt to close normally */
> - if (was_up) {
> - err = gve_close(priv->dev);
> - /* If that fails reset as we did above */
> - if (err)
> - gve_reset_and_teardown(priv, was_up);
> - }
> - /* Clean up any remaining resources */
> - gve_teardown_priv_resources(priv);
> +
> + if (was_up) {
> + gve_queues_stop(priv);
> + gve_destroy_rings(priv);
> + gve_unregister_qpls(priv);
> }
[Severity: High]
If a background task executes gve_reset() after gve_suspend() or
gve_shutdown() has already freed priv->adminq, wouldn't this unconditional
call to gve_destroy_rings(priv) write to the freed admin queue ring buffer?
The previous code appeared to verify that device rings were still okay
before attempting to tear them down.
[ ... ]
> @@ -2977,8 +2973,8 @@ static void gve_remove(struct pci_dev *pdev)
> void __iomem *reg_bar = priv->reg_bar0;
>
> unregister_netdev(netdev);
> - gve_teardown_priv_resources(priv);
> destroy_workqueue(priv->gve_wq);
> + gve_teardown_device(priv);
> priv->ctrl_ops->unmap_db_bar(priv);
> free_netdev(netdev);
> pci_iounmap(pdev, reg_bar);
[Severity: Critical]
Since gve_teardown_device() is responsible for freeing the notify blocks and
interrupts, does tearing down the workqueue first expose us to a
use-after-free if gve_mgmnt_intr() fires and calls
queue_work(priv->gve_wq, &priv->service_task)?
--
Sashiko AI review · https://sashiko.dev/#/patchset/20260602235900.3050782-1-hramamurthy@google.com?part=9
^ permalink raw reply [flat|nested] 33+ messages in thread* Re: [PATCH net-next v2 09/15] gve: simplify reset logic
2026-06-02 23:58 ` [PATCH net-next v2 09/15] gve: simplify reset logic Harshitha Ramamurthy
2026-06-03 23:59 ` sashiko-bot
@ 2026-06-07 22:20 ` Joshua Washington
1 sibling, 0 replies; 33+ messages in thread
From: Joshua Washington @ 2026-06-07 22:20 UTC (permalink / raw)
To: Harshitha Ramamurthy
Cc: netdev, andrew+netdev, davem, edumazet, kuba, pabeni, ast, daniel,
hawk, john.fastabend, sdf, willemb, jordanrhee, jfraker, nktgrg,
bpf, linux-kernel
On Tue, Jun 2, 2026 at 4:59 PM Harshitha Ramamurthy
<hramamurthy@google.com> wrote:
>
> From: Joshua Washington <joshwash@google.com>
>
> Current GVE reset logic is quite complex, with a number of methods with
> similar names and functionalities. This complexity has allowed a number
> of bugs to enter the reset/recovery path, including the potential for
> reset loops if an operation fails during teardown.
>
> Simplify the reset path by doing the following:
> 1) Removing recursive resets. Recursive resets have two major issues.
> First, there is the potential for stack overflows if resets are
> invoked too many times in a row. Second, long recursive calls mean
> that GVE never gives up the RTNL lock, or at the very least holds it
> for too long. If a reset must occur anywhere during the
> reset/recovery path, it should be scheduled as a separate task.
> 2) Removing resets during teardown. This is partly covered by removing
> recursive resets, but the primary goal in this case is to ensure that
> the driver is capable of actually executing a hardware reset if
> something goes wrong with a control plane operation. As it stands, if
> `deconfigure_device_resources` fails, for example, GVE will preempt
> its reset with another reset without actually invoking a hardware
> reset, which could actually help with recovery.
> 3) Decompose allocation/de-allocation and setup/teardown. Performing
> allocation and setup for each control plane system (RSS, ptype map,
> etc) leaves many more error conditions to handle, causing teardown in
> the case of failures to be much more complex than they need to be.
> This will also be useful to better align a major behavioral change
> in mailbox mode, which will use separate response buffers instead to
> get data from the device instead of a pre-allocated shared memory
> region.
>
> With the new reset functionality, shared resources between the device
> and driver are not freed until after the hardware reset has completed
> in the event that `deconfigure_device_resources` fails, meaning that the
> device could potentially still be holding on to shared memory.
>
> Reviewed-by: Willem de Bruijn <willemb@google.com>
> Reviewed-by: Jordan Rhee <jordanrhee@google.com>
> Signed-off-by: Joshua Washington <joshwash@google.com>
> Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
> ---
> Changes in v2:
> - Fixed typos in commit message (recursive, preempt)
> - Fixed a kdoc warning
>
> drivers/net/ethernet/google/gve/gve.h | 2 +-
> drivers/net/ethernet/google/gve/gve_adminq.c | 5 +-
> drivers/net/ethernet/google/gve/gve_adminq.h | 1 -
> drivers/net/ethernet/google/gve/gve_ethtool.c | 2 +-
> drivers/net/ethernet/google/gve/gve_main.c | 338 +++++++++---------
> 5 files changed, 167 insertions(+), 181 deletions(-)
>
> diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
> index 72588d201e5d..c2e68fd97de5 100644
> --- a/drivers/net/ethernet/google/gve/gve.h
> +++ b/drivers/net/ethernet/google/gve/gve.h
> @@ -1347,7 +1347,7 @@ struct page_pool *gve_rx_create_page_pool(struct gve_priv *priv,
>
> /* Reset */
> void gve_schedule_reset(struct gve_priv *priv);
> -int gve_reset(struct gve_priv *priv, bool attempt_teardown);
> +int gve_reset(struct gve_priv *priv, bool skip_queue_setup);
> void gve_get_curr_alloc_cfgs(struct gve_priv *priv,
> struct gve_tx_alloc_rings_cfg *tx_alloc_cfg,
> struct gve_rx_alloc_rings_cfg *rx_alloc_cfg);
> diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
> index e7956d2768b6..84b548467a0c 100644
> --- a/drivers/net/ethernet/google/gve/gve_adminq.c
> +++ b/drivers/net/ethernet/google/gve/gve_adminq.c
> @@ -363,7 +363,7 @@ int gve_adminq_init(struct gve_priv *priv)
> return gve_adminq_alloc(priv);
> }
>
> -void gve_adminq_release(struct gve_priv *priv)
> +static void gve_adminq_release(struct gve_priv *priv)
> {
> int i = 0;
>
> @@ -392,7 +392,6 @@ void gve_adminq_release(struct gve_priv *priv)
> }
> gve_clear_device_rings_ok(priv);
> gve_clear_device_resources_ok(priv);
> - gve_clear_admin_queue_ok(priv);
> }
>
> void gve_adminq_free(struct gve_priv *priv)
> @@ -1377,7 +1376,7 @@ gve_adminq_configure_flow_rule(struct gve_priv *priv,
>
> if (err == -ETIME) {
> dev_err(&priv->pdev->dev, "Timeout to configure the flow rule, trigger reset");
> - gve_reset(priv, true);
> + gve_reset(priv, false);
Sashiko says:
The commit message states recursive resets and resets during teardown
are removed, but this site still calls gve_reset() from inside an adminq
command issued during teardown.
The reachable chain is:
gve_remove() / gve_shutdown() / gve_suspend()
-> gve_teardown_device()
-> gve_teardown_control_plane_resources()
-> gve_flow_rules_reset()
-> gve_adminq_reset_flow_rules()
-> gve_adminq_configure_flow_rule()
-> gve_reset(priv, false) /* on -ETIME */
The spirit of this patch was to disallow resets during teardown _in
the reset path_, but the commit message was verbose enough as it was.
Resets due to errors as part of graceful teardown are not inherently
problematic, as the point is to teardown the device regardless. In a
case like this, only a single reset would be triggered.
However, Claude does allude to a good point in that the scope of
-ETIME failures to destroy flow rules could instead be limited to
ethtool operations. I will make this change in v3.
> } else if (!err) {
> priv->flow_rules_cache.rules_cache_synced = false;
> }
> diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
> index d07e9c6f279d..12d2d1347e21 100644
> --- a/drivers/net/ethernet/google/gve/gve_adminq.h
> +++ b/drivers/net/ethernet/google/gve/gve_adminq.h
> @@ -621,7 +621,6 @@ static_assert(sizeof(union gve_adminq_command) == 64);
>
> int gve_adminq_init(struct gve_priv *priv);
> void gve_adminq_free(struct gve_priv *priv);
> -void gve_adminq_release(struct gve_priv *priv);
> int gve_adminq_describe_device(struct gve_priv *priv);
> int gve_adminq_configure_device_resources(struct gve_priv *priv,
> dma_addr_t counter_array_bus_addr,
> diff --git a/drivers/net/ethernet/google/gve/gve_ethtool.c b/drivers/net/ethernet/google/gve/gve_ethtool.c
> index dc2213b5ce24..54b623b678ce 100644
> --- a/drivers/net/ethernet/google/gve/gve_ethtool.c
> +++ b/drivers/net/ethernet/google/gve/gve_ethtool.c
> @@ -651,7 +651,7 @@ static int gve_user_reset(struct net_device *netdev, u32 *flags)
>
> if (*flags == ETH_RESET_ALL) {
> *flags = 0;
> - return gve_reset(priv, true);
> + return gve_reset(priv, false);
> }
>
> return -EOPNOTSUPP;
> diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
> index 746ff69a28dd..f8289f478e5b 100644
> --- a/drivers/net/ethernet/google/gve/gve_main.c
> +++ b/drivers/net/ethernet/google/gve/gve_main.c
> @@ -591,7 +591,22 @@ static void gve_free_notify_blocks(struct gve_priv *priv)
> priv->msix_vectors = NULL;
> }
>
> -static int gve_setup_device_resources(struct gve_priv *priv)
> +static void gve_free_control_plane_resources(struct gve_priv *priv)
> +{
> + bitmap_free(priv->xsk_pools);
> + priv->xsk_pools = NULL;
> +
> + kvfree(priv->ptype_lut_dqo);
> + priv->ptype_lut_dqo = NULL;
> +
> + gve_free_stats_report(priv);
> + gve_free_notify_blocks(priv);
> + gve_free_counter_array(priv);
> + gve_free_rss_config_cache(priv);
> + gve_free_flow_rule_caches(priv);
> +}
> +
> +static int gve_alloc_control_plane_resources(struct gve_priv *priv)
> {
> int err;
>
> @@ -600,16 +615,42 @@ static int gve_setup_device_resources(struct gve_priv *priv)
> return err;
> err = gve_alloc_rss_config_cache(priv);
> if (err)
> - goto abort_with_flow_rule_caches;
> + goto abort;
> err = gve_alloc_counter_array(priv);
> if (err)
> - goto abort_with_rss_config_cache;
> + goto abort;
> err = gve_alloc_notify_blocks(priv);
> if (err)
> - goto abort_with_counter;
> + goto abort;
> err = gve_alloc_stats_report(priv);
> if (err)
> - goto abort_with_ntfy_blocks;
> + goto abort;
> +
> + if (!gve_is_gqi(priv)) {
> + priv->ptype_lut_dqo = kvzalloc_obj(*priv->ptype_lut_dqo,
> + GFP_KERNEL);
> + if (!priv->ptype_lut_dqo) {
> + err = -ENOMEM;
> + goto abort;
> + }
> + }
> +
> + priv->xsk_pools = bitmap_zalloc(priv->rx_cfg.max_queues, GFP_KERNEL);
> + if (!priv->xsk_pools) {
> + err = -ENOMEM;
> + goto abort;
> + }
> +
> + return 0;
> +abort:
> + gve_free_control_plane_resources(priv);
> + return err;
> +}
> +
> +static int gve_setup_control_plane_resources(struct gve_priv *priv)
> +{
> + int err = 0;
> +
> err = gve_adminq_configure_device_resources(priv,
> priv->counter_array_bus,
> priv->num_event_counters,
> @@ -619,20 +660,15 @@ static int gve_setup_device_resources(struct gve_priv *priv)
> dev_err(&priv->pdev->dev,
> "could not setup device_resources: err=%d\n", err);
> err = -ENXIO;
> - goto abort_with_stats_report;
> + return err;
> }
>
> if (!gve_is_gqi(priv)) {
> - priv->ptype_lut_dqo = kvzalloc_obj(*priv->ptype_lut_dqo);
> - if (!priv->ptype_lut_dqo) {
> - err = -ENOMEM;
> - goto abort_with_stats_report;
> - }
> err = gve_adminq_get_ptype_map_dqo(priv, priv->ptype_lut_dqo);
> if (err) {
> dev_err(&priv->pdev->dev,
> "Failed to get ptype map: err=%d\n", err);
> - goto abort_with_ptype_lut;
> + goto deconfigure_device;
> }
> }
>
> @@ -647,7 +683,7 @@ static int gve_setup_device_resources(struct gve_priv *priv)
> err = gve_init_rss_config(priv, priv->rx_cfg.num_queues);
> if (err) {
> dev_err(&priv->pdev->dev, "Failed to init RSS config");
> - goto abort_with_clock;
> + goto teardown_clock;
> }
>
> err = gve_adminq_report_stats(priv, priv->stats_report_len,
> @@ -659,67 +695,61 @@ static int gve_setup_device_resources(struct gve_priv *priv)
> gve_set_device_resources_ok(priv);
> return 0;
>
> -abort_with_clock:
> +teardown_clock:
> gve_teardown_clock(priv);
> -abort_with_ptype_lut:
> - kvfree(priv->ptype_lut_dqo);
> - priv->ptype_lut_dqo = NULL;
> -abort_with_stats_report:
> - gve_free_stats_report(priv);
> -abort_with_ntfy_blocks:
> - gve_free_notify_blocks(priv);
> -abort_with_counter:
> - gve_free_counter_array(priv);
> -abort_with_rss_config_cache:
> - gve_free_rss_config_cache(priv);
> -abort_with_flow_rule_caches:
> - gve_free_flow_rule_caches(priv);
> -
> +deconfigure_device:
> + gve_adminq_deconfigure_device_resources(priv);
> return err;
> }
>
> -static void gve_trigger_reset(struct gve_priv *priv);
> -
> -static void gve_teardown_device_resources(struct gve_priv *priv)
> +/*
> + * Request the device to release any allocated shared resources.
> + *
> + * If any part of the teardown step fails, the failure is documented, but is
> + * otherwise ignored. It is expected that a device reset is triggered
> + * immediately after tearing down device resources, which would clear any
> + * lingering state on the device.
> + */
> +static void gve_teardown_control_plane_resources(struct gve_priv *priv)
> {
> int err;
>
> /* Tell device its resources are being freed */
> if (gve_get_device_resources_ok(priv)) {
> err = gve_flow_rules_reset(priv);
> - if (err) {
> + if (err)
> dev_err(&priv->pdev->dev,
> "Failed to reset flow rules: err=%d\n", err);
> - gve_trigger_reset(priv);
> - }
> /* detach the stats report */
> err = gve_adminq_report_stats(priv, 0, 0x0, GVE_STATS_REPORT_TIMER_PERIOD);
> - if (err) {
> + if (err)
> dev_err(&priv->pdev->dev,
> "Failed to detach stats report: err=%d\n", err);
> - gve_trigger_reset(priv);
> - }
> + gve_teardown_clock(priv);
> err = gve_adminq_deconfigure_device_resources(priv);
> - if (err) {
> + if (err)
> dev_err(&priv->pdev->dev,
> "Could not deconfigure device resources: err=%d\n",
> err);
> - gve_trigger_reset(priv);
> - }
> }
>
> - kvfree(priv->ptype_lut_dqo);
> - priv->ptype_lut_dqo = NULL;
> -
> - gve_free_flow_rule_caches(priv);
> - gve_free_rss_config_cache(priv);
> - gve_free_counter_array(priv);
> - gve_free_notify_blocks(priv);
> - gve_free_stats_report(priv);
> - gve_teardown_clock(priv);
> gve_clear_device_resources_ok(priv);
> }
>
> +static void gve_teardown_device(struct gve_priv *priv)
> +{
> + gve_teardown_control_plane_resources(priv);
> + gve_adminq_free(priv);
> + /*
> + * Free any resources shared with the device only after we have a
> + * guarantee that the device will not try to access such resources.
> + * Device commands in gve_teardown_control_plane_resources can fail, in
> + * which case, device resources won't be relinquished until
> + * gve_adminq_free is called to trigger a device reset.
> + */
> + gve_free_control_plane_resources(priv);
> +}
> +
> static int gve_unregister_qpl(struct gve_priv *priv,
> struct gve_queue_page_list *qpl)
> {
> @@ -1160,8 +1190,6 @@ void gve_schedule_reset(struct gve_priv *priv)
> queue_work(priv->gve_wq, &priv->service_task);
> }
>
> -static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up);
> -static int gve_reset_recovery(struct gve_priv *priv, bool was_up);
> static void gve_turndown(struct gve_priv *priv);
> static void gve_turnup(struct gve_priv *priv);
>
> @@ -1272,11 +1300,12 @@ static int gve_reg_xdp_info(struct gve_priv *priv, struct net_device *dev)
> return err;
> }
>
> -
> static void gve_drain_page_cache(struct gve_priv *priv)
> {
> int i;
>
> + if (!priv->rx)
> + return;
> for (i = 0; i < priv->rx_cfg.num_queues; i++)
> page_frag_cache_drain(&priv->rx[i].page_cache);
> }
> @@ -1419,10 +1448,11 @@ static int gve_queues_start(struct gve_priv *priv,
> reset:
> if (gve_get_reset_in_progress(priv))
> goto stop_and_free_rings;
> - gve_reset_and_teardown(priv, true);
> - /* if this fails there is nothing we can do so just ignore the return */
> - gve_reset_recovery(priv, false);
> - /* return the original error */
> +
> + /* Attempt to reset. If reset is successful, gve_queues_start was
> + * successful.
> + */
> + err = gve_reset(priv, false);
> return err;
> stop_and_free_rings:
> gve_tx_stop_rings(priv, gve_num_tx_queues(priv));
Sashiko says:
This is a pre-existing issue, but in the stop_and_free_rings error path
inside gve_queues_start(), the rx arrays are freed via
gve_queues_mem_remove(priv) without unregistering the XDP rxq structures.
Could this leak xdp_rxq info and leave dangling pointers in the network
stack?
I will submit a fix separately for this behavior, since it is a
pre-existing issue.
> @@ -1438,6 +1468,12 @@ static int gve_open(struct net_device *dev)
> struct gve_priv *priv = netdev_priv(dev);
> int err;
>
> + if (!gve_get_device_resources_ok(priv)) {
> + dev_err(&priv->pdev->dev,
> + "Attempting to open netdev without resources. Device must be reset.");
> + return -ENODEV;
> + }
> +
> gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg);
>
> err = gve_queues_mem_alloc(priv, &tx_alloc_cfg, &rx_alloc_cfg);
> @@ -1454,41 +1490,18 @@ static int gve_open(struct net_device *dev)
> return 0;
> }
>
> -static int gve_queues_stop(struct gve_priv *priv)
> +static void gve_queues_stop(struct gve_priv *priv)
> {
> - int err;
> -
> - netif_carrier_off(priv->dev);
> - if (gve_get_device_rings_ok(priv)) {
> - gve_turndown(priv);
> - gve_drain_page_cache(priv);
> - err = gve_destroy_rings(priv);
> - if (err)
> - goto err;
> - err = gve_unregister_qpls(priv);
> - if (err)
> - goto err;
> - gve_clear_device_rings_ok(priv);
> - }
> - timer_delete_sync(&priv->stats_report_timer);
> + gve_turndown(priv);
>
> gve_unreg_xdp_info(priv);
> + gve_drain_page_cache(priv);
> +
> + timer_delete_sync(&priv->stats_report_timer);
> + cancel_work_sync(&priv->stats_report_task);
>
> gve_tx_stop_rings(priv, gve_num_tx_queues(priv));
> gve_rx_stop_rings(priv, priv->rx_cfg.num_queues);
> -
> - priv->interface_down_cnt++;
> - return 0;
> -
> -err:
> - /* This must have been called from a reset due to the rtnl lock
> - * so just return at this point.
> - */
> - if (gve_get_reset_in_progress(priv))
> - return err;
> - /* Otherwise reset before returning */
> - gve_reset_and_teardown(priv, true);
> - return gve_reset_recovery(priv, false);
> }
>
> static int gve_close(struct net_device *dev)
> @@ -1496,12 +1509,28 @@ static int gve_close(struct net_device *dev)
> struct gve_priv *priv = netdev_priv(dev);
> int err;
>
> - err = gve_queues_stop(priv);
> - if (err)
> - return err;
> + gve_queues_stop(priv);
> +
> + /* Surrender to reset if the queue destroying adminq cmds fail. Reset
> + * will not re-enable the interface.
> + */
> + if (gve_get_device_rings_ok(priv)) {
> + gve_clear_device_rings_ok(priv);
> + err = gve_destroy_rings(priv);
> + if (err)
> + goto reset;
> + err = gve_unregister_qpls(priv);
> + if (err)
> + goto reset;
> + }
>
> gve_queues_mem_remove(priv);
> + priv->interface_down_cnt++;
> return 0;
> +
> +reset:
> + err = gve_reset(priv, true);
> + return err;
> }
>
> static void gve_handle_link_status(struct gve_priv *priv, bool link_status)
> @@ -2420,25 +2449,17 @@ static int gve_setup_device(struct gve_priv *priv)
> {
> int err;
>
> - priv->xsk_pools = bitmap_zalloc(priv->rx_cfg.max_queues, GFP_KERNEL);
> - if (!priv->xsk_pools) {
> - err = -ENOMEM;
> - goto err;
> - }
> -
> gve_set_netdev_xdp_features(priv);
> if (!gve_is_gqi(priv))
> priv->dev->xdp_metadata_ops = &gve_xdp_metadata_ops;
>
> - err = gve_setup_device_resources(priv);
> + err = gve_alloc_control_plane_resources(priv);
> if (err)
> - goto err_free_xsk_bitmap;
> -
> + goto err;
> + err = gve_setup_control_plane_resources(priv);
> + if (err)
> + goto err;
> return 0;
> -
> -err_free_xsk_bitmap:
> - bitmap_free(priv->xsk_pools);
> - priv->xsk_pools = NULL;
> err:
> return err;
> }
> @@ -2513,30 +2534,7 @@ static int gve_init_priv(struct gve_priv *priv)
> return 0;
> }
>
> -static void gve_teardown_priv_resources(struct gve_priv *priv)
> -{
> - gve_teardown_device_resources(priv);
> - gve_adminq_free(priv);
> - bitmap_free(priv->xsk_pools);
> - priv->xsk_pools = NULL;
> -}
> -
> -static void gve_trigger_reset(struct gve_priv *priv)
> -{
> - /* Reset the device by releasing the AQ */
> - gve_adminq_release(priv);
> -}
> -
> -static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up)
> -{
> - gve_trigger_reset(priv);
> - /* With the reset having already happened, close cannot fail */
> - if (was_up)
> - gve_close(priv->dev);
> - gve_teardown_priv_resources(priv);
> -}
> -
> -static int gve_reset_recovery(struct gve_priv *priv, bool was_up)
> +static int gve_recover(struct gve_priv *priv, bool setup_queues)
> {
> int err;
>
> @@ -2549,54 +2547,52 @@ static int gve_reset_recovery(struct gve_priv *priv, bool was_up)
>
> err = gve_setup_device(priv);
> if (err)
> - goto err_free_adminq;
> - if (was_up) {
> + goto err;
> + if (setup_queues) {
> err = gve_open(priv->dev);
> if (err)
> - goto err_free_device;
> + goto err;
> }
> return 0;
>
> -err_free_device:
> - gve_teardown_device_resources(priv);
> - bitmap_free(priv->xsk_pools);
> - priv->xsk_pools = NULL;
> -err_free_adminq:
> - gve_adminq_free(priv);
> err:
> - dev_err(&priv->pdev->dev, "Reset failed! !!! DISABLING ALL QUEUES !!!\n");
> - gve_turndown(priv);
> + dev_err(&priv->pdev->dev, "Recover failed! !!! DISABLING ALL QUEUES !!!\n");
> + gve_teardown_device(priv);
> return err;
> }
>
> -int gve_reset(struct gve_priv *priv, bool attempt_teardown)
> +int gve_reset(struct gve_priv *priv, bool skip_queue_setup)
> {
> bool was_up = netif_running(priv->dev);
> int err;
>
> + if (gve_get_reset_in_progress(priv))
> + return 0;
> +
> dev_info(&priv->pdev->dev, "Performing reset\n");
> gve_clear_do_reset(priv);
> gve_set_reset_in_progress(priv);
> - /* If we aren't attempting to teardown normally, just go turndown and
> - * reset right away.
> - */
> - if (!attempt_teardown) {
> - gve_turndown(priv);
> - gve_reset_and_teardown(priv, was_up);
> - } else {
> - /* Otherwise attempt to close normally */
> - if (was_up) {
> - err = gve_close(priv->dev);
> - /* If that fails reset as we did above */
> - if (err)
> - gve_reset_and_teardown(priv, was_up);
> - }
> - /* Clean up any remaining resources */
> - gve_teardown_priv_resources(priv);
> +
> + if (was_up) {
> + gve_queues_stop(priv);
> + gve_destroy_rings(priv);
> + gve_unregister_qpls(priv);
> }
Sashiko says:
If a background task executes gve_reset() after gve_suspend() or
gve_shutdown() has already freed priv->adminq, wouldn't this unconditional
call to gve_destroy_rings(priv) write to the freed admin queue ring buffer?
The previous code appeared to verify that device rings were still okay
before attempting to tear them down.
This is a legitimate concern. Will add a check on device_rings_ok() in v3.
>
> - /* Set it all back up */
> - err = gve_reset_recovery(priv, was_up);
> + /* Reset the device by releasing the AQ. Rings and other resources
> + * within the NIC are implicitly destroyed if commands fail.
> + */
> + gve_adminq_free(priv);
> +
> + gve_queues_mem_remove(priv);
> + gve_teardown_clock(priv);
> + gve_free_control_plane_resources(priv);
> +
> + err = gve_recover(priv, was_up && !skip_queue_setup);
> + if (err)
> + dev_info(&priv->pdev->dev,
> + "Failed to recover in reset: %d\n", err);
> +
> gve_clear_reset_in_progress(priv);
> priv->reset_cnt++;
> priv->interface_up_cnt = 0;
> @@ -2928,7 +2924,7 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
> if (err) {
> dev_err(&priv->pdev->dev,
> "Could not setup device: err=%d\n", err);
> - goto abort_with_wq;
> + goto abort_teardown_device;
> }
>
> if (!gve_is_gqi(priv) && !gve_is_qpl(priv))
> @@ -2936,7 +2932,7 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
>
> err = register_netdev(dev);
> if (err)
> - goto abort_with_gve_init;
> + goto abort_teardown_device;
>
> dev_info(&pdev->dev, "GVE version %s\n", gve_version_str);
> dev_info(&pdev->dev, "GVE queue format %d\n", (int)priv->queue_format);
> @@ -2944,8 +2940,8 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
> queue_work(priv->gve_wq, &priv->service_task);
> return 0;
>
> -abort_with_gve_init:
> - gve_teardown_priv_resources(priv);
> +abort_teardown_device:
> + gve_teardown_device(priv);
>
> abort_with_wq:
> destroy_workqueue(priv->gve_wq);
> @@ -2977,8 +2973,8 @@ static void gve_remove(struct pci_dev *pdev)
> void __iomem *reg_bar = priv->reg_bar0;
>
> unregister_netdev(netdev);
> - gve_teardown_priv_resources(priv);
> destroy_workqueue(priv->gve_wq);
> + gve_teardown_device(priv);
Sashiko says:
Since gve_teardown_device() is responsible for freeing the notify blocks and
interrupts, does tearing down the workqueue first expose us to a
use-after-free if gve_mgmnt_intr() fires and calls
queue_work(priv->gve_wq, &priv->service_task)?
If I understand correctly, instead of destroying the work queue first,
the following order is better:
1) All work items on the queue should be disabled. disable_work()
ensures that no subsequent work items using a given work struct can be
scheduled.
2) gve_teardown_device(), disabling anything that would attempt to schedule work
3) destroy_workqueue()
If there are no objections to this, I will implement in v3.
> priv->ctrl_ops->unmap_db_bar(priv);
> free_netdev(netdev);
> pci_iounmap(pdev, reg_bar);
> @@ -2996,13 +2992,9 @@ static void gve_shutdown(struct pci_dev *pdev)
>
> rtnl_lock();
> netdev_lock(netdev);
> - if (was_up && gve_close(priv->dev)) {
> - /* If the dev was up, attempt to close, if close fails, reset */
> - gve_reset_and_teardown(priv, was_up);
> - } else {
> - /* If the dev wasn't up or close worked, finish tearing down */
> - gve_teardown_priv_resources(priv);
> - }
> + if (was_up)
> + gve_close(priv->dev);
> + gve_teardown_device(priv);
> netdev_unlock(netdev);
> rtnl_unlock();
> }
> @@ -3017,13 +3009,9 @@ static int gve_suspend(struct device *dev)
> priv->suspend_cnt++;
> rtnl_lock();
> netdev_lock(netdev);
> - if (was_up && gve_close(priv->dev)) {
> - /* If the dev was up, attempt to close, if close fails, reset */
> - gve_reset_and_teardown(priv, was_up);
> - } else {
> - /* If the dev wasn't up or close worked, finish tearing down */
> - gve_teardown_priv_resources(priv);
> - }
> + if (was_up)
> + gve_close(priv->dev);
> + gve_teardown_device(priv);
Sashiko says:
With the new structure, when gve_close() hits its reset: label during
shutdown or suspend it calls gve_reset(priv, true), which runs
gve_recover() and re-initializes adminq plus the full control plane via
gve_setup_device(). gve_shutdown()/gve_suspend() then immediately call
gve_teardown_device(), which performs a second adminq_free and a second
hardware reset.
The previous gve_reset_and_teardown() path on this same condition did
not invoke gve_reset_recovery(). On a wedged device during PCI shutdown
or suspend, every adminq command can take its full timeout, and the new
flow runs through these commands twice.
Is this re-setup-then-teardown sequence on the shutdown/suspend close
failure path intended? Would it be preferable to skip recovery when the
caller is about to immediately tear the device down?
While skipping recovery would be nice, I would prefer to do it in a
way that does not greatly increase the complexity of the reset
functionality.
I am considering following up on this work, adding
netif_device_attach/detach semantics, which the reset patch can
leverage to understand whether it should attempt to recreate queues as
part of recovery from a reset without adding unnecessary driver state
or branching logic.
> priv->up_before_suspend = was_up;
> netdev_unlock(netdev);
> rtnl_unlock();
> @@ -3040,7 +3028,7 @@ static int gve_resume(struct device *dev)
> priv->resume_cnt++;
> rtnl_lock();
> netdev_lock(netdev);
> - err = gve_reset_recovery(priv, priv->up_before_suspend);
> + err = gve_recover(priv, priv->up_before_suspend);
> netdev_unlock(netdev);
> rtnl_unlock();
> return err;
> --
> 2.54.0.1013.g208068f2d8-goog
>
--
Joshua Washington | Software Engineer | joshwash@google.com | (414) 366-4423
^ permalink raw reply [flat|nested] 33+ messages in thread
* [PATCH net-next v2 10/15] gve: add gve_ctrl_ops for gve initialization/teardown sequences
2026-06-02 23:58 [PATCH net-next v2 00/15] gve: AdminQ mode related refactors Harshitha Ramamurthy
` (8 preceding siblings ...)
2026-06-02 23:58 ` [PATCH net-next v2 09/15] gve: simplify reset logic Harshitha Ramamurthy
@ 2026-06-02 23:58 ` Harshitha Ramamurthy
2026-06-02 23:58 ` [PATCH net-next v2 11/15] gve: split up notify block allocation and setup paths Harshitha Ramamurthy
` (4 subsequent siblings)
14 siblings, 0 replies; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-02 23:58 UTC (permalink / raw)
To: netdev
Cc: joshwash, hramamurthy, andrew+netdev, davem, edumazet, kuba,
pabeni, ast, daniel, hawk, john.fastabend, sdf, willemb,
jordanrhee, jfraker, nktgrg, bpf, linux-kernel
From: Joshua Washington <joshwash@google.com>
Driver initialization and teardown involve a number of control plane
operations that need to be defined for gve_probe to operate in both
mailbox and adminq modes. This list includes:
- get_ptype_map: a mapping of packet types (L3+L4) held in RX completion
descriptors
- configure_rss: set up default RSS configuration if the device is not
queryable
- setup_stats_report: set up DMA region for stats report (AQ-only)
- reset_flow_rules: needed in teardown; flushes all flow rules from
device
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Jordan Rhee <jordanrhee@google.com>
Signed-off-by: Joshua Washington <joshwash@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
drivers/net/ethernet/google/gve/gve.h | 12 ++++++++++
drivers/net/ethernet/google/gve/gve_adminq.c | 7 +++---
drivers/net/ethernet/google/gve/gve_adminq.h | 3 +--
drivers/net/ethernet/google/gve/gve_main.c | 24 +++++++++++++++-----
4 files changed, 34 insertions(+), 12 deletions(-)
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index c2e68fd97de5..939b245ed562 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -832,12 +832,24 @@ struct gve_device_info {
* structures stored in @priv to be used during initialization.
* @set_num_ntfy_blks: Sets no. of vectors into @priv to be used during
* initialization.
+ * @get_ptype_map: Learn packet type map from device and store it in @priv
+ * @configure_rss: Set up default RSS configuration
+ * @setup_stats_report: Set up DMA region for stats report (AdminQ only)
+ * @reset_flow_rules: Flush all flow rules from device
*/
struct gve_ctrl_ops {
int (*map_db_bar)(struct gve_priv *priv);
void (*unmap_db_bar)(struct gve_priv *priv);
void (*set_num_queues)(struct gve_priv *priv);
int (*set_num_ntfy_blks)(struct gve_priv *priv);
+ int (*get_ptype_map)(struct gve_priv *priv);
+ int (*configure_rss)(struct gve_priv *priv,
+ struct ethtool_rxfh_param *param);
+ int (*setup_stats_report)(struct gve_priv *priv,
+ u64 stats_report_len,
+ dma_addr_t stats_report_addr,
+ u64 interval_ms); /* AQ-specific */
+ int (*reset_flow_rules)(struct gve_priv *priv);
};
struct gve_priv {
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index 84b548467a0c..d814108deeef 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -1327,8 +1327,7 @@ int gve_adminq_report_nic_ts(struct gve_priv *priv,
return gve_adminq_execute_cmd(priv, &cmd);
}
-int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
- struct gve_ptype_lut *ptype_lut)
+int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv)
{
struct gve_ptype_map *ptype_map;
union gve_adminq_command cmd;
@@ -1354,9 +1353,9 @@ int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
/* Populate ptype_lut. */
for (i = 0; i < GVE_NUM_PTYPES; i++) {
- ptype_lut->ptypes[i].l3_type =
+ priv->ptype_lut_dqo->ptypes[i].l3_type =
ptype_map->ptypes[i].l3_type;
- ptype_lut->ptypes[i].l4_type =
+ priv->ptype_lut_dqo->ptypes[i].l4_type =
ptype_map->ptypes[i].l4_type;
}
err:
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
index 12d2d1347e21..d5f98a80be46 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.h
+++ b/drivers/net/ethernet/google/gve/gve_adminq.h
@@ -652,8 +652,7 @@ int gve_adminq_report_nic_ts(struct gve_priv *priv,
dma_addr_t nic_ts_report_addr);
struct gve_ptype_lut;
-int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
- struct gve_ptype_lut *ptype_lut);
+int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv);
int gve_adminq_map_db_bar(struct gve_priv *priv);
void gve_adminq_unmap_db_bar(struct gve_priv *priv);
int gve_adminq_set_num_ntfy_blks(struct gve_priv *priv);
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index f8289f478e5b..06df6d8ad429 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -649,7 +649,8 @@ static int gve_alloc_control_plane_resources(struct gve_priv *priv)
static int gve_setup_control_plane_resources(struct gve_priv *priv)
{
- int err = 0;
+ const struct gve_ctrl_ops *ops = priv->ctrl_ops;
+ int err;
err = gve_adminq_configure_device_resources(priv,
priv->counter_array_bus,
@@ -664,7 +665,7 @@ static int gve_setup_control_plane_resources(struct gve_priv *priv)
}
if (!gve_is_gqi(priv)) {
- err = gve_adminq_get_ptype_map_dqo(priv, priv->ptype_lut_dqo);
+ err = ops->get_ptype_map(priv);
if (err) {
dev_err(&priv->pdev->dev,
"Failed to get ptype map: err=%d\n", err);
@@ -686,12 +687,13 @@ static int gve_setup_control_plane_resources(struct gve_priv *priv)
goto teardown_clock;
}
- err = gve_adminq_report_stats(priv, priv->stats_report_len,
+ err = ops->setup_stats_report(priv, priv->stats_report_len,
priv->stats_report_bus,
GVE_STATS_REPORT_TIMER_PERIOD);
if (err)
dev_err(&priv->pdev->dev,
"Failed to report stats: err=%d\n", err);
+
gve_set_device_resources_ok(priv);
return 0;
@@ -712,6 +714,7 @@ static int gve_setup_control_plane_resources(struct gve_priv *priv)
*/
static void gve_teardown_control_plane_resources(struct gve_priv *priv)
{
+ const struct gve_ctrl_ops *ops = priv->ctrl_ops;
int err;
/* Tell device its resources are being freed */
@@ -721,11 +724,13 @@ static void gve_teardown_control_plane_resources(struct gve_priv *priv)
dev_err(&priv->pdev->dev,
"Failed to reset flow rules: err=%d\n", err);
/* detach the stats report */
- err = gve_adminq_report_stats(priv, 0, 0x0, GVE_STATS_REPORT_TIMER_PERIOD);
+ err = ops->setup_stats_report(priv, 0, 0x0,
+ GVE_STATS_REPORT_TIMER_PERIOD);
if (err)
dev_err(&priv->pdev->dev,
"Failed to detach stats report: err=%d\n", err);
gve_teardown_clock(priv);
+
err = gve_adminq_deconfigure_device_resources(priv);
if (err)
dev_err(&priv->pdev->dev,
@@ -1811,6 +1816,7 @@ static int gve_xdp(struct net_device *dev, struct netdev_bpf *xdp)
int gve_init_rss_config(struct gve_priv *priv, u16 num_queues)
{
+ const struct gve_ctrl_ops *ops = priv->ctrl_ops;
struct gve_rss_config *rss_config = &priv->rss_config;
struct ethtool_rxfh_param rxfh = {0};
u16 i;
@@ -1826,15 +1832,17 @@ int gve_init_rss_config(struct gve_priv *priv, u16 num_queues)
rxfh.hfunc = ETH_RSS_HASH_TOP;
- return gve_adminq_configure_rss(priv, &rxfh);
+ return ops->configure_rss(priv, &rxfh);
}
int gve_flow_rules_reset(struct gve_priv *priv)
{
+ const struct gve_ctrl_ops *ops = priv->ctrl_ops;
+
if (!priv->max_flow_rules)
return 0;
- return gve_adminq_reset_flow_rules(priv);
+ return ops->reset_flow_rules(priv);
}
int gve_adjust_config(struct gve_priv *priv,
@@ -2469,6 +2477,10 @@ static const struct gve_ctrl_ops gve_adminq_ops = {
.unmap_db_bar = gve_adminq_unmap_db_bar,
.set_num_queues = gve_adminq_set_num_queues,
.set_num_ntfy_blks = gve_adminq_set_num_ntfy_blks,
+ .get_ptype_map = gve_adminq_get_ptype_map_dqo,
+ .reset_flow_rules = gve_adminq_reset_flow_rules,
+ .setup_stats_report = gve_adminq_report_stats,
+ .configure_rss = gve_adminq_configure_rss,
};
static int gve_init_priv(struct gve_priv *priv)
--
2.54.0.1013.g208068f2d8-goog
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH net-next v2 11/15] gve: split up notify block allocation and setup paths
2026-06-02 23:58 [PATCH net-next v2 00/15] gve: AdminQ mode related refactors Harshitha Ramamurthy
` (9 preceding siblings ...)
2026-06-02 23:58 ` [PATCH net-next v2 10/15] gve: add gve_ctrl_ops for gve initialization/teardown sequences Harshitha Ramamurthy
@ 2026-06-02 23:58 ` Harshitha Ramamurthy
2026-06-03 23:59 ` sashiko-bot
2026-06-07 22:29 ` Joshua Washington
2026-06-02 23:58 ` [PATCH net-next v2 12/15] gve: introduce new methods to handle IRQ doorbells Harshitha Ramamurthy
` (3 subsequent siblings)
14 siblings, 2 replies; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-02 23:58 UTC (permalink / raw)
To: netdev
Cc: joshwash, hramamurthy, andrew+netdev, davem, edumazet, kuba,
pabeni, ast, daniel, hawk, john.fastabend, sdf, willemb,
jordanrhee, jfraker, nktgrg, bpf, linux-kernel
From: Joshua Washington <joshwash@google.com>
Before this patch, notify block allocation and setup occurred in the same
method. This all occurred before gve_adminq_configure_device_resources,
which populates the irq_db_indicies array, a DMA region with BAR offsets
for MSI-X vectors.
The coming mailbox mode will require notify blocks to be set up only
after receiving the IRQ doorbell offsets, as the request does not work
with a supplied DMA buffer in the way that admin queue mode does. The
intended flow in that case would be:
1) allocate notify blocks
2) request doorbell information
3) set up MSI-X vectors based on doorbell info
This ordering also works for admin queue mode, so it will be updated to
match.
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Jordan Rhee <jordanrhee@google.com>
Signed-off-by: Joshua Washington <joshwash@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
drivers/net/ethernet/google/gve/gve.h | 2 +
drivers/net/ethernet/google/gve/gve_main.c | 158 ++++++++++++---------
2 files changed, 89 insertions(+), 71 deletions(-)
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index 939b245ed562..e173c14bdc08 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -673,6 +673,7 @@ struct gve_notify_block {
struct gve_tx_ring *tx; /* tx rings on this block */
struct gve_rx_ring *rx; /* rx rings on this block */
u32 irq;
+ bool irq_requested;
};
/* Tracks allowed and current rx queue settings */
@@ -953,6 +954,7 @@ struct gve_priv {
u64 link_speed;
bool up_before_suspend; /* True if dev was up before suspend */
+ bool mgmt_irq_requested;
struct gve_ptype_lut *ptype_lut_dqo;
/* Must be a power of two. */
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 06df6d8ad429..98970508ae54 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -425,6 +425,24 @@ int gve_napi_poll_dqo(struct napi_struct *napi, int budget)
return work_done;
}
+static void gve_free_notify_blocks(struct gve_priv *priv)
+{
+ pci_disable_msix(priv->pdev);
+ if (priv->irq_db_indices) {
+ dma_free_coherent(&priv->pdev->dev,
+ priv->num_ntfy_blks *
+ sizeof(*priv->irq_db_indices),
+ priv->irq_db_indices,
+ priv->irq_db_indices_bus);
+ priv->irq_db_indices = NULL;
+ }
+
+ kvfree(priv->ntfy_blocks);
+ priv->ntfy_blocks = NULL;
+ kvfree(priv->msix_vectors);
+ priv->msix_vectors = NULL;
+}
+
static const struct cpumask *gve_get_node_mask(struct gve_priv *priv)
{
if (priv->numa_node == NUMA_NO_NODE)
@@ -436,11 +454,9 @@ static const struct cpumask *gve_get_node_mask(struct gve_priv *priv)
static int gve_alloc_notify_blocks(struct gve_priv *priv)
{
int num_vecs_requested = priv->num_ntfy_blks + 1;
- const struct cpumask *node_mask;
- unsigned int cur_cpu;
int vecs_enabled;
- int i, j;
int err;
+ int i;
priv->msix_vectors = kvzalloc_objs(*priv->msix_vectors,
num_vecs_requested);
@@ -454,7 +470,7 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
dev_err(&priv->pdev->dev, "Could not enable min msix %d/%d\n",
GVE_MIN_MSIX, vecs_enabled);
err = vecs_enabled;
- goto abort_with_msix_vectors;
+ goto abort;
}
if (vecs_enabled != num_vecs_requested) {
int new_num_ntfy_blks = (vecs_enabled - 1) & ~0x1;
@@ -477,15 +493,6 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
}
- /* Setup Management Vector - the last vector */
- snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name), "gve-mgmnt@pci:%s",
- pci_name(priv->pdev));
- err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector,
- gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv);
- if (err) {
- dev_err(&priv->pdev->dev, "Did not receive management vector.\n");
- goto abort_with_msix_enabled;
- }
priv->irq_db_indices =
dma_alloc_coherent(&priv->pdev->dev,
priv->num_ntfy_blks *
@@ -493,15 +500,65 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
&priv->irq_db_indices_bus, GFP_KERNEL);
if (!priv->irq_db_indices) {
err = -ENOMEM;
- goto abort_with_mgmt_vector;
+ goto abort;
}
priv->ntfy_blocks = kvzalloc(priv->num_ntfy_blks *
sizeof(*priv->ntfy_blocks), GFP_KERNEL);
if (!priv->ntfy_blocks) {
err = -ENOMEM;
- goto abort_with_irq_db_indices;
+ goto abort;
+ }
+ return 0;
+
+abort:
+ gve_free_notify_blocks(priv);
+ return err;
+}
+
+static void gve_teardown_notify_blocks(struct gve_priv *priv)
+{
+ int i;
+
+ if (!priv->ntfy_blocks)
+ return;
+
+ for (i = 0; i < priv->num_ntfy_blks; i++) {
+ struct gve_notify_block *block = &priv->ntfy_blocks[i];
+
+ if (!block->irq_requested)
+ continue;
+
+ irq_set_affinity_hint(priv->msix_vectors[i].vector,
+ NULL);
+ free_irq(priv->msix_vectors[i].vector, block);
+ block->irq = 0;
+ block->irq_requested = false;
+ }
+
+ if (priv->mgmt_irq_requested) {
+ free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
+ priv->mgmt_irq_requested = false;
+ }
+}
+
+static int gve_setup_notify_blocks(struct gve_priv *priv)
+{
+ const struct cpumask *node_mask;
+ unsigned int cur_cpu;
+ int i;
+ int err;
+
+ /* Setup Management Vector - the last vector */
+ snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name),
+ "gve-mgmnt@pci:%s", pci_name(priv->pdev));
+ err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector,
+ gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv);
+ if (err) {
+ dev_err(&priv->pdev->dev, "Did not receive management vector.\n");
+ return err;
}
+ priv->mgmt_irq_requested = true;
/* Setup the other blocks - the first n-1 vectors */
node_mask = gve_get_node_mask(priv);
@@ -519,9 +576,10 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
if (err) {
dev_err(&priv->pdev->dev,
"Failed to receive msix vector %d\n", i);
- goto abort_with_some_ntfy_blocks;
+ goto abort;
}
block->irq = priv->msix_vectors[msix_idx].vector;
+ block->irq_requested = true;
irq_set_affinity_and_hint(block->irq,
cpumask_of(cur_cpu));
block->irq_db_index = &priv->irq_db_indices[i].index;
@@ -535,61 +593,12 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
cur_cpu = cpumask_first(node_mask);
}
return 0;
-abort_with_some_ntfy_blocks:
- for (j = 0; j < i; j++) {
- struct gve_notify_block *block = &priv->ntfy_blocks[j];
- int msix_idx = j;
- irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
- NULL);
- free_irq(priv->msix_vectors[msix_idx].vector, block);
- block->irq = 0;
- }
- kvfree(priv->ntfy_blocks);
- priv->ntfy_blocks = NULL;
-abort_with_irq_db_indices:
- dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
- sizeof(*priv->irq_db_indices),
- priv->irq_db_indices, priv->irq_db_indices_bus);
- priv->irq_db_indices = NULL;
-abort_with_mgmt_vector:
- free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
-abort_with_msix_enabled:
- pci_disable_msix(priv->pdev);
-abort_with_msix_vectors:
- kvfree(priv->msix_vectors);
- priv->msix_vectors = NULL;
+abort:
+ gve_teardown_notify_blocks(priv);
return err;
}
-static void gve_free_notify_blocks(struct gve_priv *priv)
-{
- int i;
-
- if (!priv->msix_vectors)
- return;
-
- /* Free the irqs */
- for (i = 0; i < priv->num_ntfy_blks; i++) {
- struct gve_notify_block *block = &priv->ntfy_blocks[i];
- int msix_idx = i;
-
- irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
- NULL);
- free_irq(priv->msix_vectors[msix_idx].vector, block);
- block->irq = 0;
- }
- free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
- kvfree(priv->ntfy_blocks);
- priv->ntfy_blocks = NULL;
- dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
- sizeof(*priv->irq_db_indices),
- priv->irq_db_indices, priv->irq_db_indices_bus);
- priv->irq_db_indices = NULL;
- pci_disable_msix(priv->pdev);
- kvfree(priv->msix_vectors);
- priv->msix_vectors = NULL;
-}
static void gve_free_control_plane_resources(struct gve_priv *priv)
{
@@ -743,6 +752,7 @@ static void gve_teardown_control_plane_resources(struct gve_priv *priv)
static void gve_teardown_device(struct gve_priv *priv)
{
+ gve_teardown_notify_blocks(priv);
gve_teardown_control_plane_resources(priv);
gve_adminq_free(priv);
/*
@@ -2463,13 +2473,16 @@ static int gve_setup_device(struct gve_priv *priv)
err = gve_alloc_control_plane_resources(priv);
if (err)
- goto err;
+ return err;
+
err = gve_setup_control_plane_resources(priv);
if (err)
- goto err;
+ return err;
+
+ err = gve_setup_notify_blocks(priv);
+ if (err)
+ return err;
return 0;
-err:
- return err;
}
static const struct gve_ctrl_ops gve_adminq_ops = {
@@ -2591,6 +2604,9 @@ int gve_reset(struct gve_priv *priv, bool skip_queue_setup)
gve_unregister_qpls(priv);
}
+ gve_teardown_notify_blocks(priv);
+ gve_teardown_control_plane_resources(priv);
+
/* Reset the device by releasing the AQ. Rings and other resources
* within the NIC are implicitly destroyed if commands fail.
*/
--
2.54.0.1013.g208068f2d8-goog
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH net-next v2 11/15] gve: split up notify block allocation and setup paths
2026-06-02 23:58 ` [PATCH net-next v2 11/15] gve: split up notify block allocation and setup paths Harshitha Ramamurthy
@ 2026-06-03 23:59 ` sashiko-bot
2026-06-07 22:29 ` Joshua Washington
1 sibling, 0 replies; 33+ messages in thread
From: sashiko-bot @ 2026-06-03 23:59 UTC (permalink / raw)
To: Harshitha Ramamurthy; +Cc: bpf
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [High] The management interrupt can trigger a Use-After-Free by queueing work on a destroyed workqueue during device removal.
--
commit 3245301c6f7eba646dce95ef2e71441d61e4f037
Author: Joshua Washington <joshwash@google.com>
gve: split up notify block allocation and setup paths
Before this patch, notify block allocation and setup occurred in the same
method. This all occurred before gve_adminq_configure_device_resources,
which populates the irq_db_indicies array, a DMA region with BAR offsets
for MSI-X vectors. The coming mailbox mode will require notify blocks to be
set up only after receiving the IRQ doorbell offsets. This ordering also
works for admin queue mode, so it is updated to match.
> diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
> index 0e88a04348490..2f92dc8176305 100644
> --- a/drivers/net/ethernet/google/gve/gve_main.c
> +++ b/drivers/net/ethernet/google/gve/gve_main.c
[ ... ]
> @@ -743,6 +752,7 @@ static void gve_teardown_control_plane_resources(struct gve_priv *priv)
>
> static void gve_teardown_device(struct gve_priv *priv)
> {
> + gve_teardown_notify_blocks(priv);
[Severity: High]
In gve_remove(), the patch series reordered the teardown sequence to call
destroy_workqueue() before gve_teardown_device():
drivers/net/ethernet/google/gve/gve_main.c:gve_remove() {
...
unregister_netdev(netdev);
destroy_workqueue(priv->gve_wq);
gve_teardown_device(priv);
...
}
Because gve_teardown_device() (via gve_teardown_notify_blocks()) is what
calls free_irq() to release the management interrupt, the interrupt remains
active while and after the workqueue is destroyed.
If a management interrupt fires during this window, gve_mgmnt_intr() will
execute:
drivers/net/ethernet/google/gve/gve_main.c:gve_mgmnt_intr() {
...
queue_work(priv->gve_wq, &priv->service_task);
return IRQ_HANDLED;
}
Can this sequence trigger a use-after-free by enqueuing work on the destroyed
workqueue?
> gve_teardown_control_plane_resources(priv);
> gve_adminq_free(priv);
> /*
--
Sashiko AI review · https://sashiko.dev/#/patchset/20260602235900.3050782-1-hramamurthy@google.com?part=11
^ permalink raw reply [flat|nested] 33+ messages in thread* Re: [PATCH net-next v2 11/15] gve: split up notify block allocation and setup paths
2026-06-02 23:58 ` [PATCH net-next v2 11/15] gve: split up notify block allocation and setup paths Harshitha Ramamurthy
2026-06-03 23:59 ` sashiko-bot
@ 2026-06-07 22:29 ` Joshua Washington
1 sibling, 0 replies; 33+ messages in thread
From: Joshua Washington @ 2026-06-07 22:29 UTC (permalink / raw)
To: Harshitha Ramamurthy
Cc: netdev, andrew+netdev, davem, edumazet, kuba, pabeni, ast, daniel,
hawk, john.fastabend, sdf, willemb, jordanrhee, jfraker, nktgrg,
bpf, linux-kernel
On Tue, Jun 2, 2026 at 4:59 PM Harshitha Ramamurthy
<hramamurthy@google.com> wrote:
>
> From: Joshua Washington <joshwash@google.com>
>
> Before this patch, notify block allocation and setup occurred in the same
> method. This all occurred before gve_adminq_configure_device_resources,
> which populates the irq_db_indicies array, a DMA region with BAR offsets
> for MSI-X vectors.
>
> The coming mailbox mode will require notify blocks to be set up only
> after receiving the IRQ doorbell offsets, as the request does not work
> with a supplied DMA buffer in the way that admin queue mode does. The
> intended flow in that case would be:
>
> 1) allocate notify blocks
> 2) request doorbell information
> 3) set up MSI-X vectors based on doorbell info
>
> This ordering also works for admin queue mode, so it will be updated to
> match.
>
> Reviewed-by: Willem de Bruijn <willemb@google.com>
> Reviewed-by: Jordan Rhee <jordanrhee@google.com>
> Signed-off-by: Joshua Washington <joshwash@google.com>
> Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
> ---
> drivers/net/ethernet/google/gve/gve.h | 2 +
> drivers/net/ethernet/google/gve/gve_main.c | 158 ++++++++++++---------
> 2 files changed, 89 insertions(+), 71 deletions(-)
>
> diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
> index 939b245ed562..e173c14bdc08 100644
> --- a/drivers/net/ethernet/google/gve/gve.h
> +++ b/drivers/net/ethernet/google/gve/gve.h
> @@ -673,6 +673,7 @@ struct gve_notify_block {
> struct gve_tx_ring *tx; /* tx rings on this block */
> struct gve_rx_ring *rx; /* rx rings on this block */
> u32 irq;
> + bool irq_requested;
> };
>
> /* Tracks allowed and current rx queue settings */
> @@ -953,6 +954,7 @@ struct gve_priv {
> u64 link_speed;
> bool up_before_suspend; /* True if dev was up before suspend */
>
> + bool mgmt_irq_requested;
> struct gve_ptype_lut *ptype_lut_dqo;
>
> /* Must be a power of two. */
> diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
> index 06df6d8ad429..98970508ae54 100644
> --- a/drivers/net/ethernet/google/gve/gve_main.c
> +++ b/drivers/net/ethernet/google/gve/gve_main.c
> @@ -425,6 +425,24 @@ int gve_napi_poll_dqo(struct napi_struct *napi, int budget)
> return work_done;
> }
>
> +static void gve_free_notify_blocks(struct gve_priv *priv)
> +{
> + pci_disable_msix(priv->pdev);
> + if (priv->irq_db_indices) {
> + dma_free_coherent(&priv->pdev->dev,
> + priv->num_ntfy_blks *
> + sizeof(*priv->irq_db_indices),
> + priv->irq_db_indices,
> + priv->irq_db_indices_bus);
> + priv->irq_db_indices = NULL;
> + }
> +
> + kvfree(priv->ntfy_blocks);
> + priv->ntfy_blocks = NULL;
> + kvfree(priv->msix_vectors);
> + priv->msix_vectors = NULL;
> +}
> +
> static const struct cpumask *gve_get_node_mask(struct gve_priv *priv)
> {
> if (priv->numa_node == NUMA_NO_NODE)
> @@ -436,11 +454,9 @@ static const struct cpumask *gve_get_node_mask(struct gve_priv *priv)
> static int gve_alloc_notify_blocks(struct gve_priv *priv)
> {
> int num_vecs_requested = priv->num_ntfy_blks + 1;
> - const struct cpumask *node_mask;
> - unsigned int cur_cpu;
> int vecs_enabled;
> - int i, j;
> int err;
> + int i;
>
> priv->msix_vectors = kvzalloc_objs(*priv->msix_vectors,
> num_vecs_requested);
> @@ -454,7 +470,7 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
> dev_err(&priv->pdev->dev, "Could not enable min msix %d/%d\n",
> GVE_MIN_MSIX, vecs_enabled);
> err = vecs_enabled;
> - goto abort_with_msix_vectors;
> + goto abort;
> }
> if (vecs_enabled != num_vecs_requested) {
> int new_num_ntfy_blks = (vecs_enabled - 1) & ~0x1;
> @@ -477,15 +493,6 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
> priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
> }
>
> - /* Setup Management Vector - the last vector */
> - snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name), "gve-mgmnt@pci:%s",
> - pci_name(priv->pdev));
> - err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector,
> - gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv);
> - if (err) {
> - dev_err(&priv->pdev->dev, "Did not receive management vector.\n");
> - goto abort_with_msix_enabled;
> - }
> priv->irq_db_indices =
> dma_alloc_coherent(&priv->pdev->dev,
> priv->num_ntfy_blks *
> @@ -493,15 +500,65 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
> &priv->irq_db_indices_bus, GFP_KERNEL);
> if (!priv->irq_db_indices) {
> err = -ENOMEM;
> - goto abort_with_mgmt_vector;
> + goto abort;
> }
>
> priv->ntfy_blocks = kvzalloc(priv->num_ntfy_blks *
> sizeof(*priv->ntfy_blocks), GFP_KERNEL);
> if (!priv->ntfy_blocks) {
> err = -ENOMEM;
> - goto abort_with_irq_db_indices;
> + goto abort;
> + }
> + return 0;
> +
> +abort:
> + gve_free_notify_blocks(priv);
> + return err;
> +}
> +
> +static void gve_teardown_notify_blocks(struct gve_priv *priv)
> +{
> + int i;
> +
> + if (!priv->ntfy_blocks)
> + return;
> +
> + for (i = 0; i < priv->num_ntfy_blks; i++) {
> + struct gve_notify_block *block = &priv->ntfy_blocks[i];
> +
> + if (!block->irq_requested)
> + continue;
> +
> + irq_set_affinity_hint(priv->msix_vectors[i].vector,
> + NULL);
> + free_irq(priv->msix_vectors[i].vector, block);
> + block->irq = 0;
> + block->irq_requested = false;
> + }
> +
> + if (priv->mgmt_irq_requested) {
> + free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
> + priv->mgmt_irq_requested = false;
> + }
> +}
> +
> +static int gve_setup_notify_blocks(struct gve_priv *priv)
> +{
> + const struct cpumask *node_mask;
> + unsigned int cur_cpu;
> + int i;
> + int err;
> +
> + /* Setup Management Vector - the last vector */
> + snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name),
> + "gve-mgmnt@pci:%s", pci_name(priv->pdev));
> + err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector,
> + gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv);
> + if (err) {
> + dev_err(&priv->pdev->dev, "Did not receive management vector.\n");
> + return err;
> }
> + priv->mgmt_irq_requested = true;
>
> /* Setup the other blocks - the first n-1 vectors */
> node_mask = gve_get_node_mask(priv);
> @@ -519,9 +576,10 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
> if (err) {
> dev_err(&priv->pdev->dev,
> "Failed to receive msix vector %d\n", i);
> - goto abort_with_some_ntfy_blocks;
> + goto abort;
> }
> block->irq = priv->msix_vectors[msix_idx].vector;
> + block->irq_requested = true;
> irq_set_affinity_and_hint(block->irq,
> cpumask_of(cur_cpu));
> block->irq_db_index = &priv->irq_db_indices[i].index;
> @@ -535,61 +593,12 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
> cur_cpu = cpumask_first(node_mask);
> }
> return 0;
> -abort_with_some_ntfy_blocks:
> - for (j = 0; j < i; j++) {
> - struct gve_notify_block *block = &priv->ntfy_blocks[j];
> - int msix_idx = j;
>
> - irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
> - NULL);
> - free_irq(priv->msix_vectors[msix_idx].vector, block);
> - block->irq = 0;
> - }
> - kvfree(priv->ntfy_blocks);
> - priv->ntfy_blocks = NULL;
> -abort_with_irq_db_indices:
> - dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
> - sizeof(*priv->irq_db_indices),
> - priv->irq_db_indices, priv->irq_db_indices_bus);
> - priv->irq_db_indices = NULL;
> -abort_with_mgmt_vector:
> - free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
> -abort_with_msix_enabled:
> - pci_disable_msix(priv->pdev);
> -abort_with_msix_vectors:
> - kvfree(priv->msix_vectors);
> - priv->msix_vectors = NULL;
> +abort:
> + gve_teardown_notify_blocks(priv);
> return err;
> }
>
> -static void gve_free_notify_blocks(struct gve_priv *priv)
> -{
> - int i;
> -
> - if (!priv->msix_vectors)
> - return;
> -
> - /* Free the irqs */
> - for (i = 0; i < priv->num_ntfy_blks; i++) {
> - struct gve_notify_block *block = &priv->ntfy_blocks[i];
> - int msix_idx = i;
> -
> - irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
> - NULL);
> - free_irq(priv->msix_vectors[msix_idx].vector, block);
> - block->irq = 0;
> - }
> - free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
> - kvfree(priv->ntfy_blocks);
> - priv->ntfy_blocks = NULL;
> - dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
> - sizeof(*priv->irq_db_indices),
> - priv->irq_db_indices, priv->irq_db_indices_bus);
> - priv->irq_db_indices = NULL;
> - pci_disable_msix(priv->pdev);
> - kvfree(priv->msix_vectors);
> - priv->msix_vectors = NULL;
> -}
>
> static void gve_free_control_plane_resources(struct gve_priv *priv)
> {
> @@ -743,6 +752,7 @@ static void gve_teardown_control_plane_resources(struct gve_priv *priv)
>
> static void gve_teardown_device(struct gve_priv *priv)
> {
> + gve_teardown_notify_blocks(priv);
Sashiko says:
In gve_remove(), the patch series reordered the teardown sequence to call
destroy_workqueue() before gve_teardown_device().
Because gve_teardown_device() (via gve_teardown_notify_blocks()) is what
calls free_irq() to release the management interrupt, the interrupt remains
active while and after the workqueue is destroyed.
If a management interrupt fires during this window, gve_mgmnt_intr() will
[queue the service task on the work queue].
Can this sequence trigger a use-after-free by enqueuing work on the destroyed
workqueue?
This will be fixed by the disable_work() change in patch 9.
> gve_teardown_control_plane_resources(priv);
> gve_adminq_free(priv);
> /*
> @@ -2463,13 +2473,16 @@ static int gve_setup_device(struct gve_priv *priv)
>
> err = gve_alloc_control_plane_resources(priv);
> if (err)
> - goto err;
> + return err;
> +
> err = gve_setup_control_plane_resources(priv);
> if (err)
> - goto err;
> + return err;
> +
> + err = gve_setup_notify_blocks(priv);
> + if (err)
> + return err;
> return 0;
> -err:
> - return err;
> }
>
> static const struct gve_ctrl_ops gve_adminq_ops = {
> @@ -2591,6 +2604,9 @@ int gve_reset(struct gve_priv *priv, bool skip_queue_setup)
> gve_unregister_qpls(priv);
> }
>
> + gve_teardown_notify_blocks(priv);
> + gve_teardown_control_plane_resources(priv);
> +
Sashiko says:
...
The newly added gve_teardown_control_plane_resources(priv) call already
invokes gve_teardown_clock() internally when device_resources_ok is
set, so is the standalone gve_teardown_clock(priv) call further down
now redundant?
Will remove the redundant call.
> /* Reset the device by releasing the AQ. Rings and other resources
> * within the NIC are implicitly destroyed if commands fail.
> */
> --
> 2.54.0.1013.g208068f2d8-goog
>
--
Joshua Washington | Software Engineer | joshwash@google.com | (414) 366-4423
^ permalink raw reply [flat|nested] 33+ messages in thread
* [PATCH net-next v2 12/15] gve: introduce new methods to handle IRQ doorbells
2026-06-02 23:58 [PATCH net-next v2 00/15] gve: AdminQ mode related refactors Harshitha Ramamurthy
` (10 preceding siblings ...)
2026-06-02 23:58 ` [PATCH net-next v2 11/15] gve: split up notify block allocation and setup paths Harshitha Ramamurthy
@ 2026-06-02 23:58 ` Harshitha Ramamurthy
2026-06-03 23:59 ` sashiko-bot
2026-06-07 22:35 ` Joshua Washington
2026-06-02 23:58 ` [PATCH net-next v2 13/15] gve: setup and teardown management interrupts Harshitha Ramamurthy
` (2 subsequent siblings)
14 siblings, 2 replies; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-02 23:58 UTC (permalink / raw)
To: netdev
Cc: joshwash, hramamurthy, andrew+netdev, davem, edumazet, kuba,
pabeni, ast, daniel, hawk, john.fastabend, sdf, willemb,
jordanrhee, jfraker, nktgrg, bpf, linux-kernel
From: Joshua Washington <joshwash@google.com>
Introduce `request_db_info` and `free_db_resources` to
`struct gve_ctrl_ops`. These encapsulate the configuration of device
resources (counter arrays and IRQ doorbell indices) which vary between
Admin Queue and Mailbox modes. All behaviors related to the IRQ doorbell
indices will be managed by these new methods instead of occurring
directly in notify_block setup/teardown methods. Similarly, GQ ring
counters will be managed in `request_db_info`.
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Jordan Rhee <jordanrhee@google.com>
Signed-off-by: Joshua Washington <joshwash@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
drivers/net/ethernet/google/gve/gve.h | 12 ++++
drivers/net/ethernet/google/gve/gve_adminq.c | 71 ++++++++++++++++++++
drivers/net/ethernet/google/gve/gve_adminq.h | 2 +
drivers/net/ethernet/google/gve/gve_main.c | 70 +++++--------------
4 files changed, 103 insertions(+), 52 deletions(-)
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index e173c14bdc08..6db5fbc0b321 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -833,6 +833,8 @@ struct gve_device_info {
* structures stored in @priv to be used during initialization.
* @set_num_ntfy_blks: Sets no. of vectors into @priv to be used during
* initialization.
+ * @request_db_info: Request and store doorbell information into @priv
+ * @free_db_resources: Free DMA memory holding doorbell info (AdminQ only)
* @get_ptype_map: Learn packet type map from device and store it in @priv
* @configure_rss: Set up default RSS configuration
* @setup_stats_report: Set up DMA region for stats report (AdminQ only)
@@ -843,6 +845,8 @@ struct gve_ctrl_ops {
void (*unmap_db_bar)(struct gve_priv *priv);
void (*set_num_queues)(struct gve_priv *priv);
int (*set_num_ntfy_blks)(struct gve_priv *priv);
+ int (*request_db_info)(struct gve_priv *priv);
+ void (*free_db_resources)(struct gve_priv *priv);
int (*get_ptype_map)(struct gve_priv *priv);
int (*configure_rss)(struct gve_priv *priv,
struct ethtool_rxfh_param *param);
@@ -1163,6 +1167,11 @@ static inline u32 gve_rx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx)
return (priv->num_ntfy_blks / 2) + queue_idx;
}
+static inline u32 gve_ntfy_to_msix_idx(struct gve_priv *priv, u32 ntfy_blk_idx)
+{
+ return ntfy_blk_idx;
+}
+
static inline bool gve_is_qpl(struct gve_priv *priv)
{
return priv->queue_format == GVE_GQI_QPL_FORMAT ||
@@ -1375,6 +1384,9 @@ int gve_adjust_queues(struct gve_priv *priv,
struct gve_rx_queue_config new_rx_config,
struct gve_tx_queue_config new_tx_config,
bool reset_rss);
+/* Initialization sequence */
+int gve_alloc_counter_array(struct gve_priv *priv);
+void gve_free_counter_array(struct gve_priv *priv);
/* flow steering rule */
int gve_get_flow_rule_entry(struct gve_priv *priv, struct ethtool_rxnfc *cmd);
int gve_get_flow_rule_ids(struct gve_priv *priv, struct ethtool_rxnfc *cmd, u32 *rule_locs);
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index d814108deeef..259dcd617216 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -1719,3 +1719,74 @@ void gve_adminq_set_num_queues(struct gve_priv *priv)
device_info->default_rx_queues,
priv->rx_cfg.num_queues);
}
+
+int gve_adminq_request_db_info(struct gve_priv *priv)
+{
+ int err;
+ int i;
+
+ /* Alloc dma addrs needed for shm regions */
+ err = gve_alloc_counter_array(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Failed to alloc db counter array.");
+ return err;
+ }
+
+ priv->irq_db_indices =
+ dma_alloc_coherent(&priv->pdev->dev,
+ priv->num_ntfy_blks *
+ sizeof(*priv->irq_db_indices),
+ &priv->irq_db_indices_bus, GFP_KERNEL);
+ if (!priv->irq_db_indices) {
+ err = -ENOMEM;
+ goto abort_with_counter_array;
+ }
+
+ err = gve_adminq_configure_device_resources(priv,
+ priv->counter_array_bus,
+ priv->num_event_counters,
+ priv->irq_db_indices_bus,
+ priv->num_ntfy_blks);
+ if (unlikely(err)) {
+ dev_err(&priv->pdev->dev,
+ "could not setup device_resources: err=%d\n", err);
+ err = -ENXIO;
+ goto abort_with_irq_db_indices;
+ }
+
+ for (i = 0; i < priv->num_ntfy_blks; i++)
+ priv->ntfy_blocks[i].irq_db_index =
+ &priv->irq_db_indices[i].index;
+ return 0;
+
+abort_with_irq_db_indices:
+ dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
+ sizeof(*priv->irq_db_indices),
+ priv->irq_db_indices, priv->irq_db_indices_bus);
+ priv->irq_db_indices = NULL;
+abort_with_counter_array:
+ gve_free_counter_array(priv);
+ return err;
+}
+
+void gve_adminq_free_db_resources(struct gve_priv *priv)
+{
+ int err;
+
+ /* Log error in deconfigure device, but don't fail. This is only ever
+ * called as a reset is about to be triggered, so it would be redundant
+ * to trigger a reset.
+ */
+ err = gve_adminq_deconfigure_device_resources(priv);
+ if (err)
+ dev_err(&priv->pdev->dev,
+ "Could not deconfigure device resources: err=%d\n",
+ err);
+
+ dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
+ sizeof(*priv->irq_db_indices),
+ priv->irq_db_indices, priv->irq_db_indices_bus);
+ priv->irq_db_indices = NULL;
+ gve_free_counter_array(priv);
+}
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
index d5f98a80be46..985b1aefea6e 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.h
+++ b/drivers/net/ethernet/google/gve/gve_adminq.h
@@ -657,5 +657,7 @@ int gve_adminq_map_db_bar(struct gve_priv *priv);
void gve_adminq_unmap_db_bar(struct gve_priv *priv);
int gve_adminq_set_num_ntfy_blks(struct gve_priv *priv);
void gve_adminq_set_num_queues(struct gve_priv *priv);
+int gve_adminq_request_db_info(struct gve_priv *priv);
+void gve_adminq_free_db_resources(struct gve_priv *priv);
#endif /* _GVE_ADMINQ_H */
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 98970508ae54..55f48aee125e 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -179,7 +179,7 @@ static void gve_free_rss_config_cache(struct gve_priv *priv)
memset(rss_config, 0, sizeof(*rss_config));
}
-static int gve_alloc_counter_array(struct gve_priv *priv)
+int gve_alloc_counter_array(struct gve_priv *priv)
{
priv->counter_array =
dma_alloc_coherent(&priv->pdev->dev,
@@ -192,7 +192,7 @@ static int gve_alloc_counter_array(struct gve_priv *priv)
return 0;
}
-static void gve_free_counter_array(struct gve_priv *priv)
+void gve_free_counter_array(struct gve_priv *priv)
{
if (!priv->counter_array)
return;
@@ -428,15 +428,6 @@ int gve_napi_poll_dqo(struct napi_struct *napi, int budget)
static void gve_free_notify_blocks(struct gve_priv *priv)
{
pci_disable_msix(priv->pdev);
- if (priv->irq_db_indices) {
- dma_free_coherent(&priv->pdev->dev,
- priv->num_ntfy_blks *
- sizeof(*priv->irq_db_indices),
- priv->irq_db_indices,
- priv->irq_db_indices_bus);
- priv->irq_db_indices = NULL;
- }
-
kvfree(priv->ntfy_blocks);
priv->ntfy_blocks = NULL;
kvfree(priv->msix_vectors);
@@ -493,24 +484,14 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
}
- priv->irq_db_indices =
- dma_alloc_coherent(&priv->pdev->dev,
- priv->num_ntfy_blks *
- sizeof(*priv->irq_db_indices),
- &priv->irq_db_indices_bus, GFP_KERNEL);
- if (!priv->irq_db_indices) {
- err = -ENOMEM;
- goto abort;
- }
-
priv->ntfy_blocks = kvzalloc(priv->num_ntfy_blks *
sizeof(*priv->ntfy_blocks), GFP_KERNEL);
if (!priv->ntfy_blocks) {
err = -ENOMEM;
goto abort;
}
- return 0;
+ return 0;
abort:
gve_free_notify_blocks(priv);
return err;
@@ -525,13 +506,14 @@ static void gve_teardown_notify_blocks(struct gve_priv *priv)
for (i = 0; i < priv->num_ntfy_blks; i++) {
struct gve_notify_block *block = &priv->ntfy_blocks[i];
+ int msix_idx = gve_ntfy_to_msix_idx(priv, i);
if (!block->irq_requested)
continue;
- irq_set_affinity_hint(priv->msix_vectors[i].vector,
+ irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
NULL);
- free_irq(priv->msix_vectors[i].vector, block);
+ free_irq(priv->msix_vectors[msix_idx].vector, block);
block->irq = 0;
block->irq_requested = false;
}
@@ -560,12 +542,11 @@ static int gve_setup_notify_blocks(struct gve_priv *priv)
}
priv->mgmt_irq_requested = true;
- /* Setup the other blocks - the first n-1 vectors */
node_mask = gve_get_node_mask(priv);
cur_cpu = cpumask_first(node_mask);
for (i = 0; i < priv->num_ntfy_blks; i++) {
struct gve_notify_block *block = &priv->ntfy_blocks[i];
- int msix_idx = i;
+ int msix_idx = gve_ntfy_to_msix_idx(priv, i);
snprintf(block->name, sizeof(block->name), "gve-ntfy-blk%d@pci:%s",
i, pci_name(priv->pdev));
@@ -575,14 +556,13 @@ static int gve_setup_notify_blocks(struct gve_priv *priv)
IRQF_NO_AUTOEN, block->name, block);
if (err) {
dev_err(&priv->pdev->dev,
- "Failed to receive msix vector %d\n", i);
+ "Failed to receive msix vector %d\n", msix_idx);
goto abort;
}
block->irq = priv->msix_vectors[msix_idx].vector;
block->irq_requested = true;
irq_set_affinity_and_hint(block->irq,
cpumask_of(cur_cpu));
- block->irq_db_index = &priv->irq_db_indices[i].index;
cur_cpu = cpumask_next(cur_cpu, node_mask);
/* Wrap once CPUs in the node have been exhausted, or when
@@ -599,7 +579,6 @@ static int gve_setup_notify_blocks(struct gve_priv *priv)
return err;
}
-
static void gve_free_control_plane_resources(struct gve_priv *priv)
{
bitmap_free(priv->xsk_pools);
@@ -608,9 +587,8 @@ static void gve_free_control_plane_resources(struct gve_priv *priv)
kvfree(priv->ptype_lut_dqo);
priv->ptype_lut_dqo = NULL;
- gve_free_stats_report(priv);
gve_free_notify_blocks(priv);
- gve_free_counter_array(priv);
+ gve_free_stats_report(priv);
gve_free_rss_config_cache(priv);
gve_free_flow_rule_caches(priv);
}
@@ -623,9 +601,6 @@ static int gve_alloc_control_plane_resources(struct gve_priv *priv)
if (err)
return err;
err = gve_alloc_rss_config_cache(priv);
- if (err)
- goto abort;
- err = gve_alloc_counter_array(priv);
if (err)
goto abort;
err = gve_alloc_notify_blocks(priv);
@@ -661,15 +636,9 @@ static int gve_setup_control_plane_resources(struct gve_priv *priv)
const struct gve_ctrl_ops *ops = priv->ctrl_ops;
int err;
- err = gve_adminq_configure_device_resources(priv,
- priv->counter_array_bus,
- priv->num_event_counters,
- priv->irq_db_indices_bus,
- priv->num_ntfy_blks);
- if (unlikely(err)) {
- dev_err(&priv->pdev->dev,
- "could not setup device_resources: err=%d\n", err);
- err = -ENXIO;
+ err = ops->request_db_info(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev, "Failed to get db info");
return err;
}
@@ -678,7 +647,7 @@ static int gve_setup_control_plane_resources(struct gve_priv *priv)
if (err) {
dev_err(&priv->pdev->dev,
"Failed to get ptype map: err=%d\n", err);
- goto deconfigure_device;
+ goto free_db_resources;
}
}
@@ -708,8 +677,8 @@ static int gve_setup_control_plane_resources(struct gve_priv *priv)
teardown_clock:
gve_teardown_clock(priv);
-deconfigure_device:
- gve_adminq_deconfigure_device_resources(priv);
+free_db_resources:
+ ops->free_db_resources(priv);
return err;
}
@@ -739,12 +708,7 @@ static void gve_teardown_control_plane_resources(struct gve_priv *priv)
dev_err(&priv->pdev->dev,
"Failed to detach stats report: err=%d\n", err);
gve_teardown_clock(priv);
-
- err = gve_adminq_deconfigure_device_resources(priv);
- if (err)
- dev_err(&priv->pdev->dev,
- "Could not deconfigure device resources: err=%d\n",
- err);
+ ops->free_db_resources(priv);
}
gve_clear_device_resources_ok(priv);
@@ -2494,6 +2458,8 @@ static const struct gve_ctrl_ops gve_adminq_ops = {
.reset_flow_rules = gve_adminq_reset_flow_rules,
.setup_stats_report = gve_adminq_report_stats,
.configure_rss = gve_adminq_configure_rss,
+ .request_db_info = gve_adminq_request_db_info,
+ .free_db_resources = gve_adminq_free_db_resources,
};
static int gve_init_priv(struct gve_priv *priv)
--
2.54.0.1013.g208068f2d8-goog
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH net-next v2 12/15] gve: introduce new methods to handle IRQ doorbells
2026-06-02 23:58 ` [PATCH net-next v2 12/15] gve: introduce new methods to handle IRQ doorbells Harshitha Ramamurthy
@ 2026-06-03 23:59 ` sashiko-bot
2026-06-07 22:35 ` Joshua Washington
1 sibling, 0 replies; 33+ messages in thread
From: sashiko-bot @ 2026-06-03 23:59 UTC (permalink / raw)
To: Harshitha Ramamurthy; +Cc: bpf
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [High] DMA memory for irq_db_indices and counter_array is freed before the hardware is guaranteed to be stopped, allowing the hardware to corrupt kernel memory if admin queue commands time out or fail.
--
commit e39d080d71706978b42ac097e883fbae5157b9ec
Author: Joshua Washington <joshwash@google.com>
gve: introduce new methods to handle IRQ doorbells
This commit introduces request_db_info and free_db_resources to
struct gve_ctrl_ops to encapsulate the configuration of device resources
(counter arrays and IRQ doorbell indices). It also manages GQ ring counters
in request_db_info.
> diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
> index d814108deeefe..259dcd617216c 100644
> --- a/drivers/net/ethernet/google/gve/gve_adminq.c
> +++ b/drivers/net/ethernet/google/gve/gve_adminq.c
[ ... ]
> @@ -1719,3 +1719,74 @@ void gve_adminq_set_num_queues(struct gve_priv *priv)
[ ... ]
> +int gve_adminq_request_db_info(struct gve_priv *priv)
> +{
[ ... ]
> + err = gve_adminq_configure_device_resources(priv,
> + priv->counter_array_bus,
> + priv->num_event_counters,
> + priv->irq_db_indices_bus,
> + priv->num_ntfy_blks);
> + if (unlikely(err)) {
> + dev_err(&priv->pdev->dev,
> + "could not setup device_resources: err=%d\n", err);
> + err = -ENXIO;
> + goto abort_with_irq_db_indices;
> + }
> +
> + for (i = 0; i < priv->num_ntfy_blks; i++)
> + priv->ntfy_blocks[i].irq_db_index =
> + &priv->irq_db_indices[i].index;
> + return 0;
> +
> +abort_with_irq_db_indices:
> + dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
> + sizeof(*priv->irq_db_indices),
> + priv->irq_db_indices, priv->irq_db_indices_bus);
> + priv->irq_db_indices = NULL;
> +abort_with_counter_array:
> + gve_free_counter_array(priv);
> + return err;
> +}
[Severity: High]
Does this error path introduce a use-after-free window?
If the configure command in gve_adminq_configure_device_resources() times
out (e.g. returns -ETIME), the hardware might still be running and attempting
to access the memory.
Since we immediately free the DMA memory via dma_free_coherent() and
gve_free_counter_array() before the hardware is guaranteed to be stopped, can
the hardware overwrite reallocated kernel memory?
> +
> +void gve_adminq_free_db_resources(struct gve_priv *priv)
> +{
> + int err;
> +
> + /* Log error in deconfigure device, but don't fail. This is only ever
> + * called as a reset is about to be triggered, so it would be redundant
> + * to trigger a reset.
> + */
> + err = gve_adminq_deconfigure_device_resources(priv);
> + if (err)
> + dev_err(&priv->pdev->dev,
> + "Could not deconfigure device resources: err=%d\n",
> + err);
> +
> + dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
> + sizeof(*priv->irq_db_indices),
> + priv->irq_db_indices, priv->irq_db_indices_bus);
> + priv->irq_db_indices = NULL;
> + gve_free_counter_array(priv);
> +}
[Severity: High]
Could freeing the DMA memory here cause memory corruption?
In an earlier patch in this series ('gve: simplify reset logic'), the teardown
logic was explicitly fixed to delay freeing shared resources until after
gve_adminq_free() triggers a device reset.
However, gve_teardown_device() executes gve_teardown_control_plane_resources()
(which now calls ops->free_db_resources()) before gve_adminq_free():
drivers/net/ethernet/google/gve/gve_main.c:gve_teardown_device() {
...
gve_teardown_notify_blocks(priv);
gve_teardown_control_plane_resources(priv);
gve_adminq_free(priv);
...
}
If the deconfigure command in gve_adminq_deconfigure_device_resources() times
out or fails, the active hardware might continue writing to the freed DMA
memory since the device hasn't been reset yet.
--
Sashiko AI review · https://sashiko.dev/#/patchset/20260602235900.3050782-1-hramamurthy@google.com?part=12
^ permalink raw reply [flat|nested] 33+ messages in thread* Re: [PATCH net-next v2 12/15] gve: introduce new methods to handle IRQ doorbells
2026-06-02 23:58 ` [PATCH net-next v2 12/15] gve: introduce new methods to handle IRQ doorbells Harshitha Ramamurthy
2026-06-03 23:59 ` sashiko-bot
@ 2026-06-07 22:35 ` Joshua Washington
1 sibling, 0 replies; 33+ messages in thread
From: Joshua Washington @ 2026-06-07 22:35 UTC (permalink / raw)
To: Harshitha Ramamurthy
Cc: netdev, andrew+netdev, davem, edumazet, kuba, pabeni, ast, daniel,
hawk, john.fastabend, sdf, willemb, jordanrhee, jfraker, nktgrg,
bpf, linux-kernel
On Tue, Jun 2, 2026 at 4:59 PM Harshitha Ramamurthy
<hramamurthy@google.com> wrote:
>
> From: Joshua Washington <joshwash@google.com>
>
> Introduce `request_db_info` and `free_db_resources` to
> `struct gve_ctrl_ops`. These encapsulate the configuration of device
> resources (counter arrays and IRQ doorbell indices) which vary between
> Admin Queue and Mailbox modes. All behaviors related to the IRQ doorbell
> indices will be managed by these new methods instead of occurring
> directly in notify_block setup/teardown methods. Similarly, GQ ring
> counters will be managed in `request_db_info`.
>
> Reviewed-by: Willem de Bruijn <willemb@google.com>
> Reviewed-by: Jordan Rhee <jordanrhee@google.com>
> Signed-off-by: Joshua Washington <joshwash@google.com>
> Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
> ---
> drivers/net/ethernet/google/gve/gve.h | 12 ++++
> drivers/net/ethernet/google/gve/gve_adminq.c | 71 ++++++++++++++++++++
> drivers/net/ethernet/google/gve/gve_adminq.h | 2 +
> drivers/net/ethernet/google/gve/gve_main.c | 70 +++++--------------
> 4 files changed, 103 insertions(+), 52 deletions(-)
>
> diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
> index e173c14bdc08..6db5fbc0b321 100644
> --- a/drivers/net/ethernet/google/gve/gve.h
> +++ b/drivers/net/ethernet/google/gve/gve.h
> @@ -833,6 +833,8 @@ struct gve_device_info {
> * structures stored in @priv to be used during initialization.
> * @set_num_ntfy_blks: Sets no. of vectors into @priv to be used during
> * initialization.
> + * @request_db_info: Request and store doorbell information into @priv
> + * @free_db_resources: Free DMA memory holding doorbell info (AdminQ only)
> * @get_ptype_map: Learn packet type map from device and store it in @priv
> * @configure_rss: Set up default RSS configuration
> * @setup_stats_report: Set up DMA region for stats report (AdminQ only)
> @@ -843,6 +845,8 @@ struct gve_ctrl_ops {
> void (*unmap_db_bar)(struct gve_priv *priv);
> void (*set_num_queues)(struct gve_priv *priv);
> int (*set_num_ntfy_blks)(struct gve_priv *priv);
> + int (*request_db_info)(struct gve_priv *priv);
> + void (*free_db_resources)(struct gve_priv *priv);
> int (*get_ptype_map)(struct gve_priv *priv);
> int (*configure_rss)(struct gve_priv *priv,
> struct ethtool_rxfh_param *param);
> @@ -1163,6 +1167,11 @@ static inline u32 gve_rx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx)
> return (priv->num_ntfy_blks / 2) + queue_idx;
> }
>
> +static inline u32 gve_ntfy_to_msix_idx(struct gve_priv *priv, u32 ntfy_blk_idx)
> +{
> + return ntfy_blk_idx;
> +}
> +
> static inline bool gve_is_qpl(struct gve_priv *priv)
> {
> return priv->queue_format == GVE_GQI_QPL_FORMAT ||
> @@ -1375,6 +1384,9 @@ int gve_adjust_queues(struct gve_priv *priv,
> struct gve_rx_queue_config new_rx_config,
> struct gve_tx_queue_config new_tx_config,
> bool reset_rss);
> +/* Initialization sequence */
> +int gve_alloc_counter_array(struct gve_priv *priv);
> +void gve_free_counter_array(struct gve_priv *priv);
> /* flow steering rule */
> int gve_get_flow_rule_entry(struct gve_priv *priv, struct ethtool_rxnfc *cmd);
> int gve_get_flow_rule_ids(struct gve_priv *priv, struct ethtool_rxnfc *cmd, u32 *rule_locs);
> diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
> index d814108deeef..259dcd617216 100644
> --- a/drivers/net/ethernet/google/gve/gve_adminq.c
> +++ b/drivers/net/ethernet/google/gve/gve_adminq.c
> @@ -1719,3 +1719,74 @@ void gve_adminq_set_num_queues(struct gve_priv *priv)
> device_info->default_rx_queues,
> priv->rx_cfg.num_queues);
> }
> +
> +int gve_adminq_request_db_info(struct gve_priv *priv)
> +{
> + int err;
> + int i;
> +
> + /* Alloc dma addrs needed for shm regions */
> + err = gve_alloc_counter_array(priv);
> + if (err) {
> + dev_err(&priv->pdev->dev,
> + "Failed to alloc db counter array.");
> + return err;
> + }
> +
> + priv->irq_db_indices =
> + dma_alloc_coherent(&priv->pdev->dev,
> + priv->num_ntfy_blks *
> + sizeof(*priv->irq_db_indices),
> + &priv->irq_db_indices_bus, GFP_KERNEL);
> + if (!priv->irq_db_indices) {
> + err = -ENOMEM;
> + goto abort_with_counter_array;
> + }
> +
> + err = gve_adminq_configure_device_resources(priv,
> + priv->counter_array_bus,
> + priv->num_event_counters,
> + priv->irq_db_indices_bus,
> + priv->num_ntfy_blks);
> + if (unlikely(err)) {
> + dev_err(&priv->pdev->dev,
> + "could not setup device_resources: err=%d\n", err);
> + err = -ENXIO;
> + goto abort_with_irq_db_indices;
> + }
> +
> + for (i = 0; i < priv->num_ntfy_blks; i++)
> + priv->ntfy_blocks[i].irq_db_index =
> + &priv->irq_db_indices[i].index;
> + return 0;
> +
> +abort_with_irq_db_indices:
> + dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
> + sizeof(*priv->irq_db_indices),
> + priv->irq_db_indices, priv->irq_db_indices_bus);
> + priv->irq_db_indices = NULL;
> +abort_with_counter_array:
> + gve_free_counter_array(priv);
> + return err;
> +}
Sashiko says:
Does this error path introduce a use-after-free window?
If the configure command in gve_adminq_configure_device_resources() times
out (e.g. returns -ETIME), the hardware might still be running and attempting
to access the memory.
Since we immediately free the DMA memory via dma_free_coherent() and
gve_free_counter_array() before the hardware is guaranteed to be stopped, can
the hardware overwrite reallocated kernel memory?
Will move the allocation/free to alloc/free_control_plane_resources.
See the separate concern regarding performing the freeing in
`free_db_resources`.
> +
> +void gve_adminq_free_db_resources(struct gve_priv *priv)
> +{
> + int err;
> +
> + /* Log error in deconfigure device, but don't fail. This is only ever
> + * called as a reset is about to be triggered, so it would be redundant
> + * to trigger a reset.
> + */
> + err = gve_adminq_deconfigure_device_resources(priv);
> + if (err)
> + dev_err(&priv->pdev->dev,
> + "Could not deconfigure device resources: err=%d\n",
> + err);
> +
> + dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
> + sizeof(*priv->irq_db_indices),
> + priv->irq_db_indices, priv->irq_db_indices_bus);
> + priv->irq_db_indices = NULL;
> + gve_free_counter_array(priv);
> +}
Sashiko says:
Could freeing the DMA memory here cause memory corruption?
In an earlier patch in this series ('gve: simplify reset logic'), the teardown
logic was explicitly fixed to delay freeing shared resources until after
gve_adminq_free() triggers a device reset.
However, gve_teardown_device() executes gve_teardown_control_plane_resources()
(which now calls ops->free_db_resources()) before gve_adminq_free().
If the deconfigure command in gve_adminq_deconfigure_device_resources() times
out or fails, the active hardware might continue writing to the freed DMA
memory since the device hasn't been reset yet.
This is a tricky problem. It looks like there is no real choice but to
re-introduce admin queue-specific allocation paths back into
`alloc_control_plane_resources` :/
> diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
> index d5f98a80be46..985b1aefea6e 100644
> --- a/drivers/net/ethernet/google/gve/gve_adminq.h
> +++ b/drivers/net/ethernet/google/gve/gve_adminq.h
> @@ -657,5 +657,7 @@ int gve_adminq_map_db_bar(struct gve_priv *priv);
> void gve_adminq_unmap_db_bar(struct gve_priv *priv);
> int gve_adminq_set_num_ntfy_blks(struct gve_priv *priv);
> void gve_adminq_set_num_queues(struct gve_priv *priv);
> +int gve_adminq_request_db_info(struct gve_priv *priv);
> +void gve_adminq_free_db_resources(struct gve_priv *priv);
>
> #endif /* _GVE_ADMINQ_H */
> diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
> index 98970508ae54..55f48aee125e 100644
> --- a/drivers/net/ethernet/google/gve/gve_main.c
> +++ b/drivers/net/ethernet/google/gve/gve_main.c
> @@ -179,7 +179,7 @@ static void gve_free_rss_config_cache(struct gve_priv *priv)
> memset(rss_config, 0, sizeof(*rss_config));
> }
>
> -static int gve_alloc_counter_array(struct gve_priv *priv)
> +int gve_alloc_counter_array(struct gve_priv *priv)
> {
> priv->counter_array =
> dma_alloc_coherent(&priv->pdev->dev,
> @@ -192,7 +192,7 @@ static int gve_alloc_counter_array(struct gve_priv *priv)
> return 0;
> }
>
> -static void gve_free_counter_array(struct gve_priv *priv)
> +void gve_free_counter_array(struct gve_priv *priv)
> {
> if (!priv->counter_array)
> return;
> @@ -428,15 +428,6 @@ int gve_napi_poll_dqo(struct napi_struct *napi, int budget)
> static void gve_free_notify_blocks(struct gve_priv *priv)
> {
> pci_disable_msix(priv->pdev);
> - if (priv->irq_db_indices) {
> - dma_free_coherent(&priv->pdev->dev,
> - priv->num_ntfy_blks *
> - sizeof(*priv->irq_db_indices),
> - priv->irq_db_indices,
> - priv->irq_db_indices_bus);
> - priv->irq_db_indices = NULL;
> - }
> -
> kvfree(priv->ntfy_blocks);
> priv->ntfy_blocks = NULL;
> kvfree(priv->msix_vectors);
> @@ -493,24 +484,14 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
> priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
> }
>
> - priv->irq_db_indices =
> - dma_alloc_coherent(&priv->pdev->dev,
> - priv->num_ntfy_blks *
> - sizeof(*priv->irq_db_indices),
> - &priv->irq_db_indices_bus, GFP_KERNEL);
> - if (!priv->irq_db_indices) {
> - err = -ENOMEM;
> - goto abort;
> - }
> -
> priv->ntfy_blocks = kvzalloc(priv->num_ntfy_blks *
> sizeof(*priv->ntfy_blocks), GFP_KERNEL);
> if (!priv->ntfy_blocks) {
> err = -ENOMEM;
> goto abort;
> }
> - return 0;
>
> + return 0;
> abort:
> gve_free_notify_blocks(priv);
> return err;
> @@ -525,13 +506,14 @@ static void gve_teardown_notify_blocks(struct gve_priv *priv)
>
> for (i = 0; i < priv->num_ntfy_blks; i++) {
> struct gve_notify_block *block = &priv->ntfy_blocks[i];
> + int msix_idx = gve_ntfy_to_msix_idx(priv, i);
>
> if (!block->irq_requested)
> continue;
>
> - irq_set_affinity_hint(priv->msix_vectors[i].vector,
> + irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
> NULL);
> - free_irq(priv->msix_vectors[i].vector, block);
> + free_irq(priv->msix_vectors[msix_idx].vector, block);
> block->irq = 0;
> block->irq_requested = false;
> }
> @@ -560,12 +542,11 @@ static int gve_setup_notify_blocks(struct gve_priv *priv)
> }
> priv->mgmt_irq_requested = true;
>
> - /* Setup the other blocks - the first n-1 vectors */
> node_mask = gve_get_node_mask(priv);
> cur_cpu = cpumask_first(node_mask);
> for (i = 0; i < priv->num_ntfy_blks; i++) {
> struct gve_notify_block *block = &priv->ntfy_blocks[i];
> - int msix_idx = i;
> + int msix_idx = gve_ntfy_to_msix_idx(priv, i);
>
> snprintf(block->name, sizeof(block->name), "gve-ntfy-blk%d@pci:%s",
> i, pci_name(priv->pdev));
> @@ -575,14 +556,13 @@ static int gve_setup_notify_blocks(struct gve_priv *priv)
> IRQF_NO_AUTOEN, block->name, block);
> if (err) {
> dev_err(&priv->pdev->dev,
> - "Failed to receive msix vector %d\n", i);
> + "Failed to receive msix vector %d\n", msix_idx);
> goto abort;
> }
> block->irq = priv->msix_vectors[msix_idx].vector;
> block->irq_requested = true;
> irq_set_affinity_and_hint(block->irq,
> cpumask_of(cur_cpu));
> - block->irq_db_index = &priv->irq_db_indices[i].index;
>
> cur_cpu = cpumask_next(cur_cpu, node_mask);
> /* Wrap once CPUs in the node have been exhausted, or when
> @@ -599,7 +579,6 @@ static int gve_setup_notify_blocks(struct gve_priv *priv)
> return err;
> }
>
> -
> static void gve_free_control_plane_resources(struct gve_priv *priv)
> {
> bitmap_free(priv->xsk_pools);
> @@ -608,9 +587,8 @@ static void gve_free_control_plane_resources(struct gve_priv *priv)
> kvfree(priv->ptype_lut_dqo);
> priv->ptype_lut_dqo = NULL;
>
> - gve_free_stats_report(priv);
> gve_free_notify_blocks(priv);
> - gve_free_counter_array(priv);
> + gve_free_stats_report(priv);
> gve_free_rss_config_cache(priv);
> gve_free_flow_rule_caches(priv);
> }
> @@ -623,9 +601,6 @@ static int gve_alloc_control_plane_resources(struct gve_priv *priv)
> if (err)
> return err;
> err = gve_alloc_rss_config_cache(priv);
> - if (err)
> - goto abort;
> - err = gve_alloc_counter_array(priv);
> if (err)
> goto abort;
> err = gve_alloc_notify_blocks(priv);
> @@ -661,15 +636,9 @@ static int gve_setup_control_plane_resources(struct gve_priv *priv)
> const struct gve_ctrl_ops *ops = priv->ctrl_ops;
> int err;
>
> - err = gve_adminq_configure_device_resources(priv,
> - priv->counter_array_bus,
> - priv->num_event_counters,
> - priv->irq_db_indices_bus,
> - priv->num_ntfy_blks);
> - if (unlikely(err)) {
> - dev_err(&priv->pdev->dev,
> - "could not setup device_resources: err=%d\n", err);
> - err = -ENXIO;
> + err = ops->request_db_info(priv);
> + if (err) {
> + dev_err(&priv->pdev->dev, "Failed to get db info");
> return err;
> }
>
> @@ -678,7 +647,7 @@ static int gve_setup_control_plane_resources(struct gve_priv *priv)
> if (err) {
> dev_err(&priv->pdev->dev,
> "Failed to get ptype map: err=%d\n", err);
> - goto deconfigure_device;
> + goto free_db_resources;
> }
> }
>
> @@ -708,8 +677,8 @@ static int gve_setup_control_plane_resources(struct gve_priv *priv)
>
> teardown_clock:
> gve_teardown_clock(priv);
> -deconfigure_device:
> - gve_adminq_deconfigure_device_resources(priv);
> +free_db_resources:
> + ops->free_db_resources(priv);
> return err;
> }
>
> @@ -739,12 +708,7 @@ static void gve_teardown_control_plane_resources(struct gve_priv *priv)
> dev_err(&priv->pdev->dev,
> "Failed to detach stats report: err=%d\n", err);
> gve_teardown_clock(priv);
> -
> - err = gve_adminq_deconfigure_device_resources(priv);
> - if (err)
> - dev_err(&priv->pdev->dev,
> - "Could not deconfigure device resources: err=%d\n",
> - err);
> + ops->free_db_resources(priv);
> }
>
> gve_clear_device_resources_ok(priv);
> @@ -2494,6 +2458,8 @@ static const struct gve_ctrl_ops gve_adminq_ops = {
> .reset_flow_rules = gve_adminq_reset_flow_rules,
> .setup_stats_report = gve_adminq_report_stats,
> .configure_rss = gve_adminq_configure_rss,
> + .request_db_info = gve_adminq_request_db_info,
> + .free_db_resources = gve_adminq_free_db_resources,
> };
>
> static int gve_init_priv(struct gve_priv *priv)
> --
> 2.54.0.1013.g208068f2d8-goog
>
--
Joshua Washington | Software Engineer | joshwash@google.com | (414) 366-4423
^ permalink raw reply [flat|nested] 33+ messages in thread
* [PATCH net-next v2 13/15] gve: setup and teardown management interrupts
2026-06-02 23:58 [PATCH net-next v2 00/15] gve: AdminQ mode related refactors Harshitha Ramamurthy
` (11 preceding siblings ...)
2026-06-02 23:58 ` [PATCH net-next v2 12/15] gve: introduce new methods to handle IRQ doorbells Harshitha Ramamurthy
@ 2026-06-02 23:58 ` Harshitha Ramamurthy
2026-06-03 23:59 ` sashiko-bot
2026-06-07 22:39 ` Joshua Washington
2026-06-02 23:58 ` [PATCH net-next v2 14/15] gve: add ctrl ops to for queue operations Harshitha Ramamurthy
2026-06-02 23:58 ` [PATCH net-next v2 15/15] gve: add link status/speed ctrl ops Harshitha Ramamurthy
14 siblings, 2 replies; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-02 23:58 UTC (permalink / raw)
To: netdev
Cc: joshwash, hramamurthy, andrew+netdev, davem, edumazet, kuba,
pabeni, ast, daniel, hawk, john.fastabend, sdf, willemb,
jordanrhee, jfraker, nktgrg, bpf, linux-kernel
From: Joshua Washington <joshwash@google.com>
Introduce control ops to setup/teardown control plane IRQs. Admin
queue-specific functionality is moved to admin queue files.
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Jordan Rhee <jordanrhee@google.com>
Signed-off-by: Joshua Washington <joshwash@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
drivers/net/ethernet/google/gve/gve.h | 4 +++
drivers/net/ethernet/google/gve/gve_adminq.c | 32 ++++++++++++++++++++
drivers/net/ethernet/google/gve/gve_adminq.h | 2 ++
drivers/net/ethernet/google/gve/gve_main.c | 28 ++++++-----------
4 files changed, 47 insertions(+), 19 deletions(-)
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index 6db5fbc0b321..b2e578b3d450 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -835,6 +835,8 @@ struct gve_device_info {
* initialization.
* @request_db_info: Request and store doorbell information into @priv
* @free_db_resources: Free DMA memory holding doorbell info (AdminQ only)
+ * @setup_mgmt_irq: Setup control plane IRQ
+ * @teardown_mgmt_irq: Teardown control plane IRQ
* @get_ptype_map: Learn packet type map from device and store it in @priv
* @configure_rss: Set up default RSS configuration
* @setup_stats_report: Set up DMA region for stats report (AdminQ only)
@@ -847,6 +849,8 @@ struct gve_ctrl_ops {
int (*set_num_ntfy_blks)(struct gve_priv *priv);
int (*request_db_info)(struct gve_priv *priv);
void (*free_db_resources)(struct gve_priv *priv);
+ int (*setup_mgmt_irq)(struct gve_priv *priv);
+ void (*teardown_mgmt_irq)(struct gve_priv *priv);
int (*get_ptype_map)(struct gve_priv *priv);
int (*configure_rss)(struct gve_priv *priv,
struct ethtool_rxfh_param *param);
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index 259dcd617216..5dc5c63cee32 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -1790,3 +1790,35 @@ void gve_adminq_free_db_resources(struct gve_priv *priv)
priv->irq_db_indices = NULL;
gve_free_counter_array(priv);
}
+
+static irqreturn_t gve_mgmnt_intr(int irq, void *arg)
+{
+ struct gve_priv *priv = arg;
+
+ queue_work(priv->gve_wq, &priv->service_task);
+ return IRQ_HANDLED;
+}
+
+int gve_adminq_setup_mgmt_irq(struct gve_priv *priv)
+{
+ int err;
+
+ snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name),
+ "gve-mgmnt@pci:%s", pci_name(priv->pdev));
+ err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector,
+ gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv);
+ if (err)
+ return err;
+
+ priv->mgmt_irq_requested = true;
+
+ return 0;
+}
+
+void gve_adminq_teardown_mgmt_irq(struct gve_priv *priv)
+{
+ if (priv->mgmt_irq_requested) {
+ free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
+ priv->mgmt_irq_requested = false;
+ }
+}
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
index 985b1aefea6e..ff656fb4f113 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.h
+++ b/drivers/net/ethernet/google/gve/gve_adminq.h
@@ -659,5 +659,7 @@ int gve_adminq_set_num_ntfy_blks(struct gve_priv *priv);
void gve_adminq_set_num_queues(struct gve_priv *priv);
int gve_adminq_request_db_info(struct gve_priv *priv);
void gve_adminq_free_db_resources(struct gve_priv *priv);
+int gve_adminq_setup_mgmt_irq(struct gve_priv *priv);
+void gve_adminq_teardown_mgmt_irq(struct gve_priv *priv);
#endif /* _GVE_ADMINQ_H */
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 55f48aee125e..9d00093826e3 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -267,14 +267,6 @@ static void gve_free_stats_report(struct gve_priv *priv)
priv->stats_report = NULL;
}
-static irqreturn_t gve_mgmnt_intr(int irq, void *arg)
-{
- struct gve_priv *priv = arg;
-
- queue_work(priv->gve_wq, &priv->service_task);
- return IRQ_HANDLED;
-}
-
static irqreturn_t gve_intr(int irq, void *arg)
{
struct gve_notify_block *block = arg;
@@ -499,6 +491,7 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
static void gve_teardown_notify_blocks(struct gve_priv *priv)
{
+ const struct gve_ctrl_ops *ops = priv->ctrl_ops;
int i;
if (!priv->ntfy_blocks)
@@ -518,29 +511,24 @@ static void gve_teardown_notify_blocks(struct gve_priv *priv)
block->irq_requested = false;
}
- if (priv->mgmt_irq_requested) {
- free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
- priv->mgmt_irq_requested = false;
- }
+ ops->teardown_mgmt_irq(priv);
}
static int gve_setup_notify_blocks(struct gve_priv *priv)
{
+ const struct gve_ctrl_ops *ops = priv->ctrl_ops;
const struct cpumask *node_mask;
unsigned int cur_cpu;
int i;
int err;
- /* Setup Management Vector - the last vector */
- snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name),
- "gve-mgmnt@pci:%s", pci_name(priv->pdev));
- err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector,
- gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv);
+ /* Setup Management Vector */
+ err = ops->setup_mgmt_irq(priv);
if (err) {
- dev_err(&priv->pdev->dev, "Did not receive management vector.\n");
+ dev_err(&priv->pdev->dev,
+ "Did not receive management vector.\n");
return err;
}
- priv->mgmt_irq_requested = true;
node_mask = gve_get_node_mask(priv);
cur_cpu = cpumask_first(node_mask);
@@ -2460,6 +2448,8 @@ static const struct gve_ctrl_ops gve_adminq_ops = {
.configure_rss = gve_adminq_configure_rss,
.request_db_info = gve_adminq_request_db_info,
.free_db_resources = gve_adminq_free_db_resources,
+ .setup_mgmt_irq = gve_adminq_setup_mgmt_irq,
+ .teardown_mgmt_irq = gve_adminq_teardown_mgmt_irq,
};
static int gve_init_priv(struct gve_priv *priv)
--
2.54.0.1013.g208068f2d8-goog
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH net-next v2 13/15] gve: setup and teardown management interrupts
2026-06-02 23:58 ` [PATCH net-next v2 13/15] gve: setup and teardown management interrupts Harshitha Ramamurthy
@ 2026-06-03 23:59 ` sashiko-bot
2026-06-07 22:39 ` Joshua Washington
1 sibling, 0 replies; 33+ messages in thread
From: sashiko-bot @ 2026-06-03 23:59 UTC (permalink / raw)
To: Harshitha Ramamurthy; +Cc: bpf
Thank you for your contribution! Sashiko AI review found 2 potential issue(s) to consider:
Pre-existing issues:
- [High] The workqueue `priv->gve_wq` is destroyed before the management IRQ is freed in `gve_remove()`, leading to a use-after-free if the interrupt fires during device teardown.
- [High] `stats_report_task` is not canceled when freeing the stats report, leading to a NULL pointer dereference if the interface is reset while down.
--
commit bb3bcdd68440531b8ea4f0e46ef000aabc8d097d
Author: Joshua Washington <joshwash@google.com>
gve: setup and teardown management interrupts
Introduce control ops to setup/teardown control plane IRQs. Admin
queue-specific functionality is moved to admin queue files.
> diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
> index 7b034b439cebb..8005373e91bde 100644
> --- a/drivers/net/ethernet/google/gve/gve_main.c
> +++ b/drivers/net/ethernet/google/gve/gve_main.c
> @@ -267,14 +267,6 @@ static void gve_free_stats_report(struct gve_priv *priv)
> priv->stats_report = NULL;
> }
[Severity: High]
This is a pre-existing issue, but does this function fail to cancel
the stats_report_task?
gve_free_stats_report() calls timer_delete_sync(&priv->stats_report_timer),
but it does not call cancel_work_sync(&priv->stats_report_task).
If the user enables stats reporting while the interface is down, the timer
starts. If a reset is triggered while the interface is down, gve_reset() skips
gve_queues_stop() (where the task is normally canceled) and proceeds to
teardown.
gve_free_control_plane_resources() then calls gve_free_stats_report(), which
frees priv->stats_report and sets it to NULL.
If the stats_report_task was queued just before the timer was deleted, it would
execute asynchronously and call gve_handle_report_stats(), which unconditionally
dereferences priv->stats_report->stats.
Could this lead to a NULL pointer dereference?
[ ... ]
> @@ -499,6 +491,7 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
>
> static void gve_teardown_notify_blocks(struct gve_priv *priv)
> {
> + const struct gve_ctrl_ops *ops = priv->ctrl_ops;
> int i;
>
> if (!priv->ntfy_blocks)
> @@ -518,29 +511,24 @@ static void gve_teardown_notify_blocks(struct gve_priv *priv)
> block->irq_requested = false;
> }
>
> - if (priv->mgmt_irq_requested) {
> - free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
> - priv->mgmt_irq_requested = false;
> - }
> + ops->teardown_mgmt_irq(priv);
[Severity: High]
This isn't a bug introduced by this patch, but is it possible for the
management IRQ to fire after the workqueue is destroyed?
In gve_remove() in drivers/net/ethernet/google/gve/gve_main.c,
destroy_workqueue(priv->gve_wq) is called before gve_teardown_device(priv).
gve_teardown_device() eventually reaches this code to tear down the management
IRQ. Because the IRQ is still active and unmasked after destroy_workqueue(),
an interrupt could fire during this window.
If that happens, gve_mgmnt_intr() calls:
queue_work(priv->gve_wq, &priv->service_task);
Could this result in a use-after-free regression on the destroyed workqueue
pointer?
--
Sashiko AI review · https://sashiko.dev/#/patchset/20260602235900.3050782-1-hramamurthy@google.com?part=13
^ permalink raw reply [flat|nested] 33+ messages in thread* Re: [PATCH net-next v2 13/15] gve: setup and teardown management interrupts
2026-06-02 23:58 ` [PATCH net-next v2 13/15] gve: setup and teardown management interrupts Harshitha Ramamurthy
2026-06-03 23:59 ` sashiko-bot
@ 2026-06-07 22:39 ` Joshua Washington
1 sibling, 0 replies; 33+ messages in thread
From: Joshua Washington @ 2026-06-07 22:39 UTC (permalink / raw)
To: Harshitha Ramamurthy
Cc: netdev, andrew+netdev, davem, edumazet, kuba, pabeni, ast, daniel,
hawk, john.fastabend, sdf, willemb, jordanrhee, jfraker, nktgrg,
bpf, linux-kernel
On Tue, Jun 2, 2026 at 4:59 PM Harshitha Ramamurthy
<hramamurthy@google.com> wrote:
>
> From: Joshua Washington <joshwash@google.com>
>
> Introduce control ops to setup/teardown control plane IRQs. Admin
> queue-specific functionality is moved to admin queue files.
>
> Reviewed-by: Willem de Bruijn <willemb@google.com>
> Reviewed-by: Jordan Rhee <jordanrhee@google.com>
> Signed-off-by: Joshua Washington <joshwash@google.com>
> Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
> ---
> drivers/net/ethernet/google/gve/gve.h | 4 +++
> drivers/net/ethernet/google/gve/gve_adminq.c | 32 ++++++++++++++++++++
> drivers/net/ethernet/google/gve/gve_adminq.h | 2 ++
> drivers/net/ethernet/google/gve/gve_main.c | 28 ++++++-----------
> 4 files changed, 47 insertions(+), 19 deletions(-)
>
> diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
> index 6db5fbc0b321..b2e578b3d450 100644
> --- a/drivers/net/ethernet/google/gve/gve.h
> +++ b/drivers/net/ethernet/google/gve/gve.h
> @@ -835,6 +835,8 @@ struct gve_device_info {
> * initialization.
> * @request_db_info: Request and store doorbell information into @priv
> * @free_db_resources: Free DMA memory holding doorbell info (AdminQ only)
> + * @setup_mgmt_irq: Setup control plane IRQ
> + * @teardown_mgmt_irq: Teardown control plane IRQ
> * @get_ptype_map: Learn packet type map from device and store it in @priv
> * @configure_rss: Set up default RSS configuration
> * @setup_stats_report: Set up DMA region for stats report (AdminQ only)
> @@ -847,6 +849,8 @@ struct gve_ctrl_ops {
> int (*set_num_ntfy_blks)(struct gve_priv *priv);
> int (*request_db_info)(struct gve_priv *priv);
> void (*free_db_resources)(struct gve_priv *priv);
> + int (*setup_mgmt_irq)(struct gve_priv *priv);
> + void (*teardown_mgmt_irq)(struct gve_priv *priv);
> int (*get_ptype_map)(struct gve_priv *priv);
> int (*configure_rss)(struct gve_priv *priv,
> struct ethtool_rxfh_param *param);
> diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
> index 259dcd617216..5dc5c63cee32 100644
> --- a/drivers/net/ethernet/google/gve/gve_adminq.c
> +++ b/drivers/net/ethernet/google/gve/gve_adminq.c
> @@ -1790,3 +1790,35 @@ void gve_adminq_free_db_resources(struct gve_priv *priv)
> priv->irq_db_indices = NULL;
> gve_free_counter_array(priv);
> }
> +
> +static irqreturn_t gve_mgmnt_intr(int irq, void *arg)
> +{
> + struct gve_priv *priv = arg;
> +
> + queue_work(priv->gve_wq, &priv->service_task);
> + return IRQ_HANDLED;
> +}
> +
> +int gve_adminq_setup_mgmt_irq(struct gve_priv *priv)
> +{
> + int err;
> +
> + snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name),
> + "gve-mgmnt@pci:%s", pci_name(priv->pdev));
> + err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector,
> + gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv);
> + if (err)
> + return err;
> +
> + priv->mgmt_irq_requested = true;
> +
> + return 0;
> +}
> +
> +void gve_adminq_teardown_mgmt_irq(struct gve_priv *priv)
> +{
> + if (priv->mgmt_irq_requested) {
> + free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
> + priv->mgmt_irq_requested = false;
> + }
> +}
> diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
> index 985b1aefea6e..ff656fb4f113 100644
> --- a/drivers/net/ethernet/google/gve/gve_adminq.h
> +++ b/drivers/net/ethernet/google/gve/gve_adminq.h
> @@ -659,5 +659,7 @@ int gve_adminq_set_num_ntfy_blks(struct gve_priv *priv);
> void gve_adminq_set_num_queues(struct gve_priv *priv);
> int gve_adminq_request_db_info(struct gve_priv *priv);
> void gve_adminq_free_db_resources(struct gve_priv *priv);
> +int gve_adminq_setup_mgmt_irq(struct gve_priv *priv);
> +void gve_adminq_teardown_mgmt_irq(struct gve_priv *priv);
>
> #endif /* _GVE_ADMINQ_H */
> diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
> index 55f48aee125e..9d00093826e3 100644
> --- a/drivers/net/ethernet/google/gve/gve_main.c
> +++ b/drivers/net/ethernet/google/gve/gve_main.c
> @@ -267,14 +267,6 @@ static void gve_free_stats_report(struct gve_priv *priv)
> priv->stats_report = NULL;
> }
Sashiko says:
This is a pre-existing issue, but does this function fail to cancel
the stats_report_task?
This issue predates this patch series; will have to introduce a net
patch to fix.
>
> -static irqreturn_t gve_mgmnt_intr(int irq, void *arg)
> -{
> - struct gve_priv *priv = arg;
> -
> - queue_work(priv->gve_wq, &priv->service_task);
> - return IRQ_HANDLED;
> -}
> -
> static irqreturn_t gve_intr(int irq, void *arg)
> {
> struct gve_notify_block *block = arg;
> @@ -499,6 +491,7 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
>
> static void gve_teardown_notify_blocks(struct gve_priv *priv)
> {
> + const struct gve_ctrl_ops *ops = priv->ctrl_ops;
> int i;
>
> if (!priv->ntfy_blocks)
> @@ -518,29 +511,24 @@ static void gve_teardown_notify_blocks(struct gve_priv *priv)
> block->irq_requested = false;
> }
>
> - if (priv->mgmt_irq_requested) {
> - free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
> - priv->mgmt_irq_requested = false;
> - }
> + ops->teardown_mgmt_irq(priv);
> }
Sashiko says:
This isn't a bug introduced by this patch, but is it possible for the
management IRQ to fire after the workqueue is destroyed?
Again, this will be fixed with changes in gve: Simplify Reset Logic.
>
> static int gve_setup_notify_blocks(struct gve_priv *priv)
> {
> + const struct gve_ctrl_ops *ops = priv->ctrl_ops;
> const struct cpumask *node_mask;
> unsigned int cur_cpu;
> int i;
> int err;
>
> - /* Setup Management Vector - the last vector */
> - snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name),
> - "gve-mgmnt@pci:%s", pci_name(priv->pdev));
> - err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector,
> - gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv);
> + /* Setup Management Vector */
> + err = ops->setup_mgmt_irq(priv);
> if (err) {
> - dev_err(&priv->pdev->dev, "Did not receive management vector.\n");
> + dev_err(&priv->pdev->dev,
> + "Did not receive management vector.\n");
> return err;
> }
> - priv->mgmt_irq_requested = true;
>
> node_mask = gve_get_node_mask(priv);
> cur_cpu = cpumask_first(node_mask);
> @@ -2460,6 +2448,8 @@ static const struct gve_ctrl_ops gve_adminq_ops = {
> .configure_rss = gve_adminq_configure_rss,
> .request_db_info = gve_adminq_request_db_info,
> .free_db_resources = gve_adminq_free_db_resources,
> + .setup_mgmt_irq = gve_adminq_setup_mgmt_irq,
> + .teardown_mgmt_irq = gve_adminq_teardown_mgmt_irq,
> };
>
> static int gve_init_priv(struct gve_priv *priv)
> --
> 2.54.0.1013.g208068f2d8-goog
>
--
Joshua Washington | Software Engineer | joshwash@google.com | (414) 366-4423
^ permalink raw reply [flat|nested] 33+ messages in thread
* [PATCH net-next v2 14/15] gve: add ctrl ops to for queue operations
2026-06-02 23:58 [PATCH net-next v2 00/15] gve: AdminQ mode related refactors Harshitha Ramamurthy
` (12 preceding siblings ...)
2026-06-02 23:58 ` [PATCH net-next v2 13/15] gve: setup and teardown management interrupts Harshitha Ramamurthy
@ 2026-06-02 23:58 ` Harshitha Ramamurthy
2026-06-05 3:56 ` Harshitha Ramamurthy
2026-06-02 23:58 ` [PATCH net-next v2 15/15] gve: add link status/speed ctrl ops Harshitha Ramamurthy
14 siblings, 1 reply; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-02 23:58 UTC (permalink / raw)
To: netdev
Cc: joshwash, hramamurthy, andrew+netdev, davem, edumazet, kuba,
pabeni, ast, daniel, hawk, john.fastabend, sdf, willemb,
jordanrhee, jfraker, nktgrg, bpf, linux-kernel
Add the next few control ops to
- create queues
- destroy queues
These changes are just moving existing code into ops, no functional
change.
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Jordan Rhee <jordanrhee@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
Changes in v2:
- Fixed typo in commit message (exisiting -> existing)
drivers/net/ethernet/google/gve/gve.h | 4 ++
drivers/net/ethernet/google/gve/gve_adminq.c | 50 ++++++++++++++++++++
drivers/net/ethernet/google/gve/gve_adminq.h | 3 +-
drivers/net/ethernet/google/gve/gve_main.c | 41 ++++------------
4 files changed, 65 insertions(+), 33 deletions(-)
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index b2e578b3d450..4283cb9b49de 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -841,6 +841,8 @@ struct gve_device_info {
* @configure_rss: Set up default RSS configuration
* @setup_stats_report: Set up DMA region for stats report (AdminQ only)
* @reset_flow_rules: Flush all flow rules from device
+ * @create_queues: Sends commands to the device to create TX/RX queues.
+ * @destroy_queues: Sends commands to the device to destroy TX/RX queues.
*/
struct gve_ctrl_ops {
int (*map_db_bar)(struct gve_priv *priv);
@@ -859,6 +861,8 @@ struct gve_ctrl_ops {
dma_addr_t stats_report_addr,
u64 interval_ms); /* AQ-specific */
int (*reset_flow_rules)(struct gve_priv *priv);
+ int (*create_queues)(struct gve_priv *priv);
+ int (*destroy_queues)(struct gve_priv *priv);
};
struct gve_priv {
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index 5dc5c63cee32..799381d24c71 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -941,6 +941,30 @@ static void gve_set_default_rss_sizes(struct gve_priv *priv)
}
}
+int gve_adminq_destroy_queues(struct gve_priv *priv)
+{
+ int num_tx_queues = gve_num_tx_queues(priv);
+ int err;
+
+ err = gve_adminq_destroy_tx_queues(priv, 0, num_tx_queues);
+ if (err) {
+ netif_err(priv, drv, priv->dev,
+ "failed to destroy tx queues\n");
+ /* This failure will trigger a reset - no need to clean up */
+ return err;
+ }
+ netif_dbg(priv, drv, priv->dev, "destroyed tx queues\n");
+ err = gve_adminq_destroy_rx_queues(priv, priv->rx_cfg.num_queues);
+ if (err) {
+ netif_err(priv, drv, priv->dev,
+ "failed to destroy rx queues\n");
+ /* This failure will trigger a reset - no need to clean up */
+ return err;
+ }
+ netif_dbg(priv, drv, priv->dev, "destroyed rx queues\n");
+ return err;
+}
+
static void gve_enable_supported_features(struct gve_priv *priv,
u32 supported_features_mask,
const struct gve_device_option_jumbo_frames
@@ -1822,3 +1846,29 @@ void gve_adminq_teardown_mgmt_irq(struct gve_priv *priv)
priv->mgmt_irq_requested = false;
}
}
+
+int gve_adminq_create_queues(struct gve_priv *priv)
+{
+ int num_tx_queues = gve_num_tx_queues(priv);
+ int err;
+
+ err = gve_adminq_create_tx_queues(priv, 0, num_tx_queues);
+ if (err) {
+ netif_err(priv, drv, priv->dev, "failed to create %d tx queues\n",
+ num_tx_queues);
+ return err;
+ }
+ netif_dbg(priv, drv, priv->dev, "created %d tx queues\n",
+ num_tx_queues);
+
+ err = gve_adminq_create_rx_queues(priv, priv->rx_cfg.num_queues);
+ if (err) {
+ netif_err(priv, drv, priv->dev, "failed to create %d rx queues\n",
+ priv->rx_cfg.num_queues);
+ return err;
+ }
+ netif_dbg(priv, drv, priv->dev, "created %d rx queues\n",
+ priv->rx_cfg.num_queues);
+
+ return err;
+}
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
index ff656fb4f113..03e8318f7327 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.h
+++ b/drivers/net/ethernet/google/gve/gve_adminq.h
@@ -628,6 +628,8 @@ int gve_adminq_configure_device_resources(struct gve_priv *priv,
dma_addr_t db_array_bus_addr,
u32 num_ntfy_blks);
int gve_adminq_deconfigure_device_resources(struct gve_priv *priv);
+int gve_adminq_create_queues(struct gve_priv *priv);
+int gve_adminq_destroy_queues(struct gve_priv *priv);
int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues);
int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues);
int gve_adminq_create_single_rx_queue(struct gve_priv *priv, u32 queue_index);
@@ -661,5 +663,4 @@ int gve_adminq_request_db_info(struct gve_priv *priv);
void gve_adminq_free_db_resources(struct gve_priv *priv);
int gve_adminq_setup_mgmt_irq(struct gve_priv *priv);
void gve_adminq_teardown_mgmt_irq(struct gve_priv *priv);
-
#endif /* _GVE_ADMINQ_H */
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 9d00093826e3..dfc734d8bec9 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -841,33 +841,18 @@ static int gve_unregister_qpls(struct gve_priv *priv)
static int gve_create_rings(struct gve_priv *priv)
{
- int num_tx_queues = gve_num_tx_queues(priv);
int err;
int i;
- err = gve_adminq_create_tx_queues(priv, 0, num_tx_queues);
+ err = priv->ctrl_ops->create_queues(priv);
if (err) {
- netif_err(priv, drv, priv->dev, "failed to create %d tx queues\n",
- num_tx_queues);
- /* This failure will trigger a reset - no need to clean
- * up
- */
- return err;
- }
- netif_dbg(priv, drv, priv->dev, "created %d tx queues\n",
- num_tx_queues);
-
- err = gve_adminq_create_rx_queues(priv, priv->rx_cfg.num_queues);
- if (err) {
- netif_err(priv, drv, priv->dev, "failed to create %d rx queues\n",
- priv->rx_cfg.num_queues);
+ netif_err(priv, drv, priv->dev,
+ "failed to create queues\n");
/* This failure will trigger a reset - no need to clean
* up
*/
return err;
}
- netif_dbg(priv, drv, priv->dev, "created %d rx queues\n",
- priv->rx_cfg.num_queues);
if (gve_is_gqi(priv)) {
/* Rx data ring has been prefilled with packet buffers at queue
@@ -1022,26 +1007,16 @@ static int gve_queues_mem_alloc(struct gve_priv *priv,
static int gve_destroy_rings(struct gve_priv *priv)
{
- int num_tx_queues = gve_num_tx_queues(priv);
int err;
- err = gve_adminq_destroy_tx_queues(priv, 0, num_tx_queues);
+ err = priv->ctrl_ops->destroy_queues(priv);
+
if (err) {
- netif_err(priv, drv, priv->dev,
- "failed to destroy tx queues\n");
/* This failure will trigger a reset - no need to clean up */
- return err;
- }
- netif_dbg(priv, drv, priv->dev, "destroyed tx queues\n");
- err = gve_adminq_destroy_rx_queues(priv, priv->rx_cfg.num_queues);
- if (err) {
netif_err(priv, drv, priv->dev,
- "failed to destroy rx queues\n");
- /* This failure will trigger a reset - no need to clean up */
- return err;
+ "failed to destroy queues\n");
}
- netif_dbg(priv, drv, priv->dev, "destroyed rx queues\n");
- return 0;
+ return err;
}
static void gve_queues_mem_free(struct gve_priv *priv,
@@ -2450,6 +2425,8 @@ static const struct gve_ctrl_ops gve_adminq_ops = {
.free_db_resources = gve_adminq_free_db_resources,
.setup_mgmt_irq = gve_adminq_setup_mgmt_irq,
.teardown_mgmt_irq = gve_adminq_teardown_mgmt_irq,
+ .create_queues = gve_adminq_create_queues,
+ .destroy_queues = gve_adminq_destroy_queues,
};
static int gve_init_priv(struct gve_priv *priv)
--
2.54.0.1013.g208068f2d8-goog
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH net-next v2 14/15] gve: add ctrl ops to for queue operations
2026-06-02 23:58 ` [PATCH net-next v2 14/15] gve: add ctrl ops to for queue operations Harshitha Ramamurthy
@ 2026-06-05 3:56 ` Harshitha Ramamurthy
0 siblings, 0 replies; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-05 3:56 UTC (permalink / raw)
To: netdev
Cc: joshwash, andrew+netdev, davem, edumazet, kuba, pabeni, ast,
daniel, hawk, john.fastabend, sdf, willemb, jordanrhee, jfraker,
nktgrg, bpf, linux-kernel
On Tue, Jun 2, 2026 at 4:59 PM Harshitha Ramamurthy
<hramamurthy@google.com> wrote:
>
> Add the next few control ops to
>
> - create queues
> - destroy queues
>
> These changes are just moving existing code into ops, no functional
> change.
>
> Reviewed-by: Willem de Bruijn <willemb@google.com>
> Reviewed-by: Jordan Rhee <jordanrhee@google.com>
> Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
> ---
> Changes in v2:
> - Fixed typo in commit message (exisiting -> existing)
>
> drivers/net/ethernet/google/gve/gve.h | 4 ++
> drivers/net/ethernet/google/gve/gve_adminq.c | 50 ++++++++++++++++++++
> drivers/net/ethernet/google/gve/gve_adminq.h | 3 +-
> drivers/net/ethernet/google/gve/gve_main.c | 41 ++++------------
> 4 files changed, 65 insertions(+), 33 deletions(-)
>
> diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
> index b2e578b3d450..4283cb9b49de 100644
> --- a/drivers/net/ethernet/google/gve/gve.h
> +++ b/drivers/net/ethernet/google/gve/gve.h
> @@ -841,6 +841,8 @@ struct gve_device_info {
> * @configure_rss: Set up default RSS configuration
> * @setup_stats_report: Set up DMA region for stats report (AdminQ only)
> * @reset_flow_rules: Flush all flow rules from device
> + * @create_queues: Sends commands to the device to create TX/RX queues.
> + * @destroy_queues: Sends commands to the device to destroy TX/RX queues.
> */
> struct gve_ctrl_ops {
> int (*map_db_bar)(struct gve_priv *priv);
> @@ -859,6 +861,8 @@ struct gve_ctrl_ops {
> dma_addr_t stats_report_addr,
> u64 interval_ms); /* AQ-specific */
> int (*reset_flow_rules)(struct gve_priv *priv);
> + int (*create_queues)(struct gve_priv *priv);
> + int (*destroy_queues)(struct gve_priv *priv);
> };
>
> struct gve_priv {
> diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
> index 5dc5c63cee32..799381d24c71 100644
> --- a/drivers/net/ethernet/google/gve/gve_adminq.c
> +++ b/drivers/net/ethernet/google/gve/gve_adminq.c
> @@ -941,6 +941,30 @@ static void gve_set_default_rss_sizes(struct gve_priv *priv)
> }
> }
>
> +int gve_adminq_destroy_queues(struct gve_priv *priv)
> +{
> + int num_tx_queues = gve_num_tx_queues(priv);
> + int err;
> +
> + err = gve_adminq_destroy_tx_queues(priv, 0, num_tx_queues);
> + if (err) {
> + netif_err(priv, drv, priv->dev,
> + "failed to destroy tx queues\n");
> + /* This failure will trigger a reset - no need to clean up */
> + return err;
> + }
> + netif_dbg(priv, drv, priv->dev, "destroyed tx queues\n");
> + err = gve_adminq_destroy_rx_queues(priv, priv->rx_cfg.num_queues);
> + if (err) {
> + netif_err(priv, drv, priv->dev,
> + "failed to destroy rx queues\n");
> + /* This failure will trigger a reset - no need to clean up */
> + return err;
> + }
> + netif_dbg(priv, drv, priv->dev, "destroyed rx queues\n");
> + return err;
> +}
> +
> static void gve_enable_supported_features(struct gve_priv *priv,
> u32 supported_features_mask,
> const struct gve_device_option_jumbo_frames
> @@ -1822,3 +1846,29 @@ void gve_adminq_teardown_mgmt_irq(struct gve_priv *priv)
> priv->mgmt_irq_requested = false;
> }
> }
> +
> +int gve_adminq_create_queues(struct gve_priv *priv)
> +{
> + int num_tx_queues = gve_num_tx_queues(priv);
> + int err;
> +
> + err = gve_adminq_create_tx_queues(priv, 0, num_tx_queues);
> + if (err) {
> + netif_err(priv, drv, priv->dev, "failed to create %d tx queues\n",
> + num_tx_queues);
> + return err;
> + }
> + netif_dbg(priv, drv, priv->dev, "created %d tx queues\n",
> + num_tx_queues);
> +
> + err = gve_adminq_create_rx_queues(priv, priv->rx_cfg.num_queues);
> + if (err) {
> + netif_err(priv, drv, priv->dev, "failed to create %d rx queues\n",
> + priv->rx_cfg.num_queues);
> + return err;
> + }
> + netif_dbg(priv, drv, priv->dev, "created %d rx queues\n",
> + priv->rx_cfg.num_queues);
> +
> + return err;
> +}
> diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
> index ff656fb4f113..03e8318f7327 100644
> --- a/drivers/net/ethernet/google/gve/gve_adminq.h
> +++ b/drivers/net/ethernet/google/gve/gve_adminq.h
> @@ -628,6 +628,8 @@ int gve_adminq_configure_device_resources(struct gve_priv *priv,
> dma_addr_t db_array_bus_addr,
> u32 num_ntfy_blks);
> int gve_adminq_deconfigure_device_resources(struct gve_priv *priv);
> +int gve_adminq_create_queues(struct gve_priv *priv);
> +int gve_adminq_destroy_queues(struct gve_priv *priv);
> int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues);
> int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues);
> int gve_adminq_create_single_rx_queue(struct gve_priv *priv, u32 queue_index);
> @@ -661,5 +663,4 @@ int gve_adminq_request_db_info(struct gve_priv *priv);
> void gve_adminq_free_db_resources(struct gve_priv *priv);
> int gve_adminq_setup_mgmt_irq(struct gve_priv *priv);
> void gve_adminq_teardown_mgmt_irq(struct gve_priv *priv);
> -
> #endif /* _GVE_ADMINQ_H */
> diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
> index 9d00093826e3..dfc734d8bec9 100644
> --- a/drivers/net/ethernet/google/gve/gve_main.c
> +++ b/drivers/net/ethernet/google/gve/gve_main.c
> @@ -841,33 +841,18 @@ static int gve_unregister_qpls(struct gve_priv *priv)
>
> static int gve_create_rings(struct gve_priv *priv)
> {
> - int num_tx_queues = gve_num_tx_queues(priv);
> int err;
> int i;
>
> - err = gve_adminq_create_tx_queues(priv, 0, num_tx_queues);
> + err = priv->ctrl_ops->create_queues(priv);
> if (err) {
> - netif_err(priv, drv, priv->dev, "failed to create %d tx queues\n",
> - num_tx_queues);
> - /* This failure will trigger a reset - no need to clean
> - * up
> - */
> - return err;
> - }
> - netif_dbg(priv, drv, priv->dev, "created %d tx queues\n",
> - num_tx_queues);
> -
> - err = gve_adminq_create_rx_queues(priv, priv->rx_cfg.num_queues);
> - if (err) {
> - netif_err(priv, drv, priv->dev, "failed to create %d rx queues\n",
> - priv->rx_cfg.num_queues);
> + netif_err(priv, drv, priv->dev,
> + "failed to create queues\n");
Commenting on Sashiko's report:
https://netdev-ai.bots.linux.dev/sashiko/#/patchset/20260602235900.3050782-1-hramamurthy%40google.com
"The commit message states "These changes are just moving existing code
into ops, no functional change." but on the failure path the driver now
emits two error messages instead of one. Is this intended?"
Will remove double logging in v3.
> /* This failure will trigger a reset - no need to clean
> * up
> */
> return err;
> }
> - netif_dbg(priv, drv, priv->dev, "created %d rx queues\n",
> - priv->rx_cfg.num_queues);
>
> if (gve_is_gqi(priv)) {
> /* Rx data ring has been prefilled with packet buffers at queue
> @@ -1022,26 +1007,16 @@ static int gve_queues_mem_alloc(struct gve_priv *priv,
>
> static int gve_destroy_rings(struct gve_priv *priv)
> {
> - int num_tx_queues = gve_num_tx_queues(priv);
> int err;
>
> - err = gve_adminq_destroy_tx_queues(priv, 0, num_tx_queues);
> + err = priv->ctrl_ops->destroy_queues(priv);
> +
> if (err) {
> - netif_err(priv, drv, priv->dev,
> - "failed to destroy tx queues\n");
> /* This failure will trigger a reset - no need to clean up */
> - return err;
> - }
> - netif_dbg(priv, drv, priv->dev, "destroyed tx queues\n");
> - err = gve_adminq_destroy_rx_queues(priv, priv->rx_cfg.num_queues);
> - if (err) {
> netif_err(priv, drv, priv->dev,
> - "failed to destroy rx queues\n");
> - /* This failure will trigger a reset - no need to clean up */
> - return err;
> + "failed to destroy queues\n");
> }
> - netif_dbg(priv, drv, priv->dev, "destroyed rx queues\n");
> - return 0;
> + return err;
> }
>
> static void gve_queues_mem_free(struct gve_priv *priv,
> @@ -2450,6 +2425,8 @@ static const struct gve_ctrl_ops gve_adminq_ops = {
> .free_db_resources = gve_adminq_free_db_resources,
> .setup_mgmt_irq = gve_adminq_setup_mgmt_irq,
> .teardown_mgmt_irq = gve_adminq_teardown_mgmt_irq,
> + .create_queues = gve_adminq_create_queues,
> + .destroy_queues = gve_adminq_destroy_queues,
> };
>
> static int gve_init_priv(struct gve_priv *priv)
> --
> 2.54.0.1013.g208068f2d8-goog
>
^ permalink raw reply [flat|nested] 33+ messages in thread
* [PATCH net-next v2 15/15] gve: add link status/speed ctrl ops
2026-06-02 23:58 [PATCH net-next v2 00/15] gve: AdminQ mode related refactors Harshitha Ramamurthy
` (13 preceding siblings ...)
2026-06-02 23:58 ` [PATCH net-next v2 14/15] gve: add ctrl ops to for queue operations Harshitha Ramamurthy
@ 2026-06-02 23:58 ` Harshitha Ramamurthy
2026-06-03 23:59 ` sashiko-bot
2026-06-07 22:45 ` Joshua Washington
14 siblings, 2 replies; 33+ messages in thread
From: Harshitha Ramamurthy @ 2026-06-02 23:58 UTC (permalink / raw)
To: netdev
Cc: joshwash, hramamurthy, andrew+netdev, davem, edumazet, kuba,
pabeni, ast, daniel, hawk, john.fastabend, sdf, willemb,
jordanrhee, jfraker, nktgrg, bpf, linux-kernel
From: Joshua Washington <joshwash@google.com>
Refactor link status check to use a control plane op. Introduce
new op for retrieving the link status in AQ mode. This op reads
the link status from the device status register and stores the
value in priv.
Also add an op for retrieving link speed in AQ mode which calls into
gve_adminq_report_link_speed.
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Jordan Rhee <jordanrhee@google.com>
Signed-off-by: Joshua Washington <joshwash@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
drivers/net/ethernet/google/gve/gve.h | 5 ++++
drivers/net/ethernet/google/gve/gve_adminq.c | 9 ++++++++
drivers/net/ethernet/google/gve/gve_adminq.h | 1 +
drivers/net/ethernet/google/gve/gve_ethtool.c | 3 +--
drivers/net/ethernet/google/gve/gve_main.c | 23 +++++++++++--------
5 files changed, 29 insertions(+), 12 deletions(-)
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index 4283cb9b49de..c8ebbbb93942 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -843,6 +843,8 @@ struct gve_device_info {
* @reset_flow_rules: Flush all flow rules from device
* @create_queues: Sends commands to the device to create TX/RX queues.
* @destroy_queues: Sends commands to the device to destroy TX/RX queues.
+ * @report_link_status: Set link status into @priv->link_up
+ * @report_link_speed: Set link status into @priv->link_speed
*/
struct gve_ctrl_ops {
int (*map_db_bar)(struct gve_priv *priv);
@@ -863,6 +865,8 @@ struct gve_ctrl_ops {
int (*reset_flow_rules)(struct gve_priv *priv);
int (*create_queues)(struct gve_priv *priv);
int (*destroy_queues)(struct gve_priv *priv);
+ int (*report_link_status)(struct gve_priv *priv);
+ int (*report_link_speed)(struct gve_priv *priv);
};
struct gve_priv {
@@ -964,6 +968,7 @@ struct gve_priv {
/* Gvnic device link speed from hypervisor. */
u64 link_speed;
+ bool link_up;
bool up_before_suspend; /* True if dev was up before suspend */
bool mgmt_irq_requested;
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index 799381d24c71..b33633555b71 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -1872,3 +1872,12 @@ int gve_adminq_create_queues(struct gve_priv *priv)
return err;
}
+
+int gve_adminq_report_link_status(struct gve_priv *priv)
+{
+ u32 status;
+
+ status = ioread32be(&priv->reg_bar0->device_status);
+ priv->link_up = !!(GVE_DEVICE_STATUS_LINK_STATUS_MASK & status);
+ return 0;
+}
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
index 03e8318f7327..82413369dec9 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.h
+++ b/drivers/net/ethernet/google/gve/gve_adminq.h
@@ -643,6 +643,7 @@ int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len,
dma_addr_t stats_report_addr, u64 interval);
int gve_adminq_verify_driver_compatibility(struct gve_priv *priv);
int gve_adminq_get_device_properties(struct gve_priv *priv);
+int gve_adminq_report_link_status(struct gve_priv *priv);
int gve_adminq_report_link_speed(struct gve_priv *priv);
int gve_adminq_add_flow_rule(struct gve_priv *priv, struct gve_adminq_flow_rule *rule, u32 loc);
int gve_adminq_del_flow_rule(struct gve_priv *priv, u32 loc);
diff --git a/drivers/net/ethernet/google/gve/gve_ethtool.c b/drivers/net/ethernet/google/gve/gve_ethtool.c
index 54b623b678ce..aaf5b8b1671b 100644
--- a/drivers/net/ethernet/google/gve/gve_ethtool.c
+++ b/drivers/net/ethernet/google/gve/gve_ethtool.c
@@ -749,12 +749,11 @@ static int gve_get_link_ksettings(struct net_device *netdev,
int err = 0;
if (priv->link_speed == 0)
- err = gve_adminq_report_link_speed(priv);
+ err = priv->ctrl_ops->report_link_speed(priv);
cmd->base.speed = priv->link_speed;
cmd->base.duplex = DUPLEX_FULL;
-
return err;
}
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index dfc734d8bec9..181ec6919051 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -1475,15 +1475,15 @@ static int gve_close(struct net_device *dev)
return err;
}
-static void gve_handle_link_status(struct gve_priv *priv, bool link_status)
+static void gve_handle_link_status(struct gve_priv *priv)
{
if (!gve_get_napi_enabled(priv))
return;
- if (link_status == netif_carrier_ok(priv->dev))
+ if (priv->link_up == netif_carrier_ok(priv->dev))
return;
- if (link_status) {
+ if (priv->link_up) {
netdev_info(priv->dev, "Device link is up.\n");
netif_carrier_on(priv->dev);
} else {
@@ -1510,7 +1510,6 @@ static int gve_set_xdp(struct gve_priv *priv, struct bpf_prog *prog,
{
struct bpf_prog *old_prog;
int err = 0;
- u32 status;
old_prog = READ_ONCE(priv->xdp_prog);
if (!netif_running(priv->dev)) {
@@ -1537,8 +1536,8 @@ static int gve_set_xdp(struct gve_priv *priv, struct bpf_prog *prog,
bpf_prog_put(old_prog);
out:
- status = ioread32be(&priv->reg_bar0->device_status);
- gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status);
+ priv->ctrl_ops->report_link_status(priv);
+ gve_handle_link_status(priv);
return err;
}
@@ -1971,11 +1970,11 @@ static void gve_turnup(struct gve_priv *priv)
static void gve_turnup_and_check_status(struct gve_priv *priv)
{
- u32 status;
+ const struct gve_ctrl_ops *ops = priv->ctrl_ops;
gve_turnup(priv);
- status = ioread32be(&priv->reg_bar0->device_status);
- gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status);
+ ops->report_link_status(priv);
+ gve_handle_link_status(priv);
}
static struct gve_notify_block *gve_get_tx_notify_block(struct gve_priv *priv,
@@ -2299,12 +2298,14 @@ static void gve_service_task(struct work_struct *work)
{
struct gve_priv *priv = container_of(work, struct gve_priv,
service_task);
+ const struct gve_ctrl_ops *ops = priv->ctrl_ops;
u32 status = ioread32be(&priv->reg_bar0->device_status);
gve_handle_status(priv, status);
gve_handle_reset(priv);
- gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status);
+ ops->report_link_status(priv);
+ gve_handle_link_status(priv);
}
static void gve_set_netdev_xdp_features(struct gve_priv *priv)
@@ -2427,6 +2428,8 @@ static const struct gve_ctrl_ops gve_adminq_ops = {
.teardown_mgmt_irq = gve_adminq_teardown_mgmt_irq,
.create_queues = gve_adminq_create_queues,
.destroy_queues = gve_adminq_destroy_queues,
+ .report_link_status = gve_adminq_report_link_status,
+ .report_link_speed = gve_adminq_report_link_speed,
};
static int gve_init_priv(struct gve_priv *priv)
--
2.54.0.1013.g208068f2d8-goog
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH net-next v2 15/15] gve: add link status/speed ctrl ops
2026-06-02 23:58 ` [PATCH net-next v2 15/15] gve: add link status/speed ctrl ops Harshitha Ramamurthy
@ 2026-06-03 23:59 ` sashiko-bot
2026-06-07 22:45 ` Joshua Washington
1 sibling, 0 replies; 33+ messages in thread
From: sashiko-bot @ 2026-06-03 23:59 UTC (permalink / raw)
To: Harshitha Ramamurthy; +Cc: bpf
Thank you for your contribution! Sashiko AI review found 4 potential issue(s) to consider:
New issues:
- [Medium] Data race and double-read TOCTOU on unprotected `priv->link_up` boolean field.
Pre-existing issues:
- [Critical] DMA memory corruption risk in AdminQ command timeout handling.
- [Critical] Use-After-Free of `priv->adminq` when an ethtool command is issued after a failed device reset.
- [High] The driver fails to cancel the `service_task` workqueue during device suspend, allowing it to run while the device is in D3hot, triggering AER crashes.
--
commit ef66cc6af8d7947d84b4c6d611c745d03fb318bf
Author: Joshua Washington <joshwash@google.com>
gve: add link status/speed ctrl ops
Refactor link status check to use a control plane op. Introduce
new op for retrieving the link status in AQ mode. This op reads
the link status from the device status register and stores the
value in priv.
Also add an op for retrieving link speed in AQ mode which calls into
gve_adminq_report_link_speed.
> diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
> index 799381d24c71f..b33633555b710 100644
> --- a/drivers/net/ethernet/google/gve/gve_adminq.c
> +++ b/drivers/net/ethernet/google/gve/gve_adminq.c
> @@ -1872,3 +1872,12 @@ int gve_adminq_create_queues(struct gve_priv *priv)
>
> return err;
> }
> +
> +int gve_adminq_report_link_status(struct gve_priv *priv)
> +{
> + u32 status;
> +
> + status = ioread32be(&priv->reg_bar0->device_status);
> + priv->link_up = !!(GVE_DEVICE_STATUS_LINK_STATUS_MASK & status);
[Severity: Medium]
Can this cause a data race? This function directly writes to priv->link_up
without locks, while it is read concurrently in other contexts.
> + return 0;
> +}
[ ... ]
> diff --git a/drivers/net/ethernet/google/gve/gve_ethtool.c b/drivers/net/ethernet/google/gve/gve_ethtool.c
> index 54b623b678ce3..aaf5b8b1671b5 100644
> --- a/drivers/net/ethernet/google/gve/gve_ethtool.c
> +++ b/drivers/net/ethernet/google/gve/gve_ethtool.c
> @@ -749,12 +749,11 @@ static int gve_get_link_ksettings(struct net_device *netdev,
> int err = 0;
>
> if (priv->link_speed == 0)
> - err = gve_adminq_report_link_speed(priv);
> + err = priv->ctrl_ops->report_link_speed(priv);
[Severity: Critical]
This is a pre-existing issue, but I have a question about the timeout handling
inside gve_adminq_report_link_speed().
If gve_adminq_execute_cmd() returns a timeout error, the function drops the
error and immediately frees the DMA memory via dma_free_coherent(). However,
since a host CPU timeout does not guarantee the NIC has aborted the DMA
operation, could the hardware eventually execute a DMA write to the provided
bus address after the memory has been reallocated, causing memory corruption?
[Severity: Critical]
This is another pre-existing issue, but could this ethtool call trigger a
use-after-free if a previous device reset failed?
If a device reset fails, gve_adminq_free() destroys the adminq_pool and frees
the DMA memory backing priv->adminq. If user-space subsequently issues an
ethtool command like gve_get_link_ksettings, the driver invokes
report_link_speed() and eventually gve_adminq_issue_cmd().
If the device's tail counter was reset to 0, and the host's head counter is
still 0, could gve_adminq_issue_cmd() write the command payload into the
now-dangling priv->adminq buffer?
>
> cmd->base.speed = priv->link_speed;
>
> cmd->base.duplex = DUPLEX_FULL;
[ ... ]
> diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
> index a103a6304b495..74778ee4d4d0e 100644
> --- a/drivers/net/ethernet/google/gve/gve_main.c
> +++ b/drivers/net/ethernet/google/gve/gve_main.c
> @@ -1475,15 +1475,15 @@ static int gve_close(struct net_device *dev)
> return err;
> }
>
> -static void gve_handle_link_status(struct gve_priv *priv, bool link_status)
> +static void gve_handle_link_status(struct gve_priv *priv)
> {
> if (!gve_get_napi_enabled(priv))
> return;
>
> - if (link_status == netif_carrier_ok(priv->dev))
> + if (priv->link_up == netif_carrier_ok(priv->dev))
> return;
>
> - if (link_status) {
> + if (priv->link_up) {
[Severity: Medium]
Is there a time-of-check to time-of-use race here?
Because priv->link_up is read twice in quick succession without protection,
and could be modified concurrently by gve_adminq_report_link_status() running
in the lockless service_task workqueue, can the value change between the two
checks?
Would using READ_ONCE() and saving the result to a local variable prevent
unintended compiler optimizations and state changes?
> netdev_info(priv->dev, "Device link is up.\n");
> netif_carrier_on(priv->dev);
> } else {
[ ... ]
> @@ -2299,12 +2298,14 @@ static void gve_service_task(struct work_struct *work)
> {
> struct gve_priv *priv = container_of(work, struct gve_priv,
> service_task);
> + const struct gve_ctrl_ops *ops = priv->ctrl_ops;
> u32 status = ioread32be(&priv->reg_bar0->device_status);
>
> gve_handle_status(priv, status);
>
> gve_handle_reset(priv);
> - gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status);
> + ops->report_link_status(priv);
> + gve_handle_link_status(priv);
[Severity: High]
This is a pre-existing issue, but is the service_task safely stopped during
system suspend?
Looking at gve_suspend(), the function does not call cancel_work_sync() for
the service_task. Because the priv->gve_wq workqueue is allocated without the
WQ_FREEZABLE flag, it will not be frozen by the PM core.
If this service_task runs while the device is in D3hot, could ioread32be()
return all Fs (0xFFFFFFFF), causing the GVE_DEVICE_STATUS_RESET_MASK check to
pass? If so, would this trigger gve_reset() to write to MMIO registers of a
suspended PCIe device, resulting in a PCIe Master Abort?
> }
>
> static void gve_set_netdev_xdp_features(struct gve_priv *priv)
--
Sashiko AI review · https://sashiko.dev/#/patchset/20260602235900.3050782-1-hramamurthy@google.com?part=15
^ permalink raw reply [flat|nested] 33+ messages in thread* Re: [PATCH net-next v2 15/15] gve: add link status/speed ctrl ops
2026-06-02 23:58 ` [PATCH net-next v2 15/15] gve: add link status/speed ctrl ops Harshitha Ramamurthy
2026-06-03 23:59 ` sashiko-bot
@ 2026-06-07 22:45 ` Joshua Washington
1 sibling, 0 replies; 33+ messages in thread
From: Joshua Washington @ 2026-06-07 22:45 UTC (permalink / raw)
To: Harshitha Ramamurthy
Cc: netdev, andrew+netdev, davem, edumazet, kuba, pabeni, ast, daniel,
hawk, john.fastabend, sdf, willemb, jordanrhee, jfraker, nktgrg,
bpf, linux-kernel
On Tue, Jun 2, 2026 at 4:59 PM Harshitha Ramamurthy
<hramamurthy@google.com> wrote:
>
> From: Joshua Washington <joshwash@google.com>
>
> Refactor link status check to use a control plane op. Introduce
> new op for retrieving the link status in AQ mode. This op reads
> the link status from the device status register and stores the
> value in priv.
>
> Also add an op for retrieving link speed in AQ mode which calls into
> gve_adminq_report_link_speed.
>
> Reviewed-by: Willem de Bruijn <willemb@google.com>
> Reviewed-by: Jordan Rhee <jordanrhee@google.com>
> Signed-off-by: Joshua Washington <joshwash@google.com>
> Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
> ---
> drivers/net/ethernet/google/gve/gve.h | 5 ++++
> drivers/net/ethernet/google/gve/gve_adminq.c | 9 ++++++++
> drivers/net/ethernet/google/gve/gve_adminq.h | 1 +
> drivers/net/ethernet/google/gve/gve_ethtool.c | 3 +--
> drivers/net/ethernet/google/gve/gve_main.c | 23 +++++++++++--------
> 5 files changed, 29 insertions(+), 12 deletions(-)
>
> diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
> index 4283cb9b49de..c8ebbbb93942 100644
> --- a/drivers/net/ethernet/google/gve/gve.h
> +++ b/drivers/net/ethernet/google/gve/gve.h
> @@ -843,6 +843,8 @@ struct gve_device_info {
> * @reset_flow_rules: Flush all flow rules from device
> * @create_queues: Sends commands to the device to create TX/RX queues.
> * @destroy_queues: Sends commands to the device to destroy TX/RX queues.
> + * @report_link_status: Set link status into @priv->link_up
> + * @report_link_speed: Set link status into @priv->link_speed
Sashiko says:
Should this read "Set link speed into @priv->link_speed" instead? ...
Will fix in v3.
> */
> struct gve_ctrl_ops {
> int (*map_db_bar)(struct gve_priv *priv);
> @@ -863,6 +865,8 @@ struct gve_ctrl_ops {
> int (*reset_flow_rules)(struct gve_priv *priv);
> int (*create_queues)(struct gve_priv *priv);
> int (*destroy_queues)(struct gve_priv *priv);
> + int (*report_link_status)(struct gve_priv *priv);
> + int (*report_link_speed)(struct gve_priv *priv);
> };
>
> struct gve_priv {
> @@ -964,6 +968,7 @@ struct gve_priv {
>
> /* Gvnic device link speed from hypervisor. */
> u64 link_speed;
> + bool link_up;
> bool up_before_suspend; /* True if dev was up before suspend */
>
> bool mgmt_irq_requested;
> diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
> index 799381d24c71..b33633555b71 100644
> --- a/drivers/net/ethernet/google/gve/gve_adminq.c
> +++ b/drivers/net/ethernet/google/gve/gve_adminq.c
> @@ -1872,3 +1872,12 @@ int gve_adminq_create_queues(struct gve_priv *priv)
>
> return err;
> }
> +
> +int gve_adminq_report_link_status(struct gve_priv *priv)
> +{
> + u32 status;
> +
> + status = ioread32be(&priv->reg_bar0->device_status);
> + priv->link_up = !!(GVE_DEVICE_STATUS_LINK_STATUS_MASK & status);
> + return 0;
> +}
> diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
> index 03e8318f7327..82413369dec9 100644
> --- a/drivers/net/ethernet/google/gve/gve_adminq.h
> +++ b/drivers/net/ethernet/google/gve/gve_adminq.h
> @@ -643,6 +643,7 @@ int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len,
> dma_addr_t stats_report_addr, u64 interval);
> int gve_adminq_verify_driver_compatibility(struct gve_priv *priv);
> int gve_adminq_get_device_properties(struct gve_priv *priv);
> +int gve_adminq_report_link_status(struct gve_priv *priv);
> int gve_adminq_report_link_speed(struct gve_priv *priv);
> int gve_adminq_add_flow_rule(struct gve_priv *priv, struct gve_adminq_flow_rule *rule, u32 loc);
> int gve_adminq_del_flow_rule(struct gve_priv *priv, u32 loc);
> diff --git a/drivers/net/ethernet/google/gve/gve_ethtool.c b/drivers/net/ethernet/google/gve/gve_ethtool.c
> index 54b623b678ce..aaf5b8b1671b 100644
> --- a/drivers/net/ethernet/google/gve/gve_ethtool.c
> +++ b/drivers/net/ethernet/google/gve/gve_ethtool.c
> @@ -749,12 +749,11 @@ static int gve_get_link_ksettings(struct net_device *netdev,
> int err = 0;
>
> if (priv->link_speed == 0)
> - err = gve_adminq_report_link_speed(priv);
> + err = priv->ctrl_ops->report_link_speed(priv);
Sashiko says:
---
This is a pre-existing issue, but I have a question about the timeout handling
inside gve_adminq_report_link_speed().
If gve_adminq_execute_cmd() returns a timeout error, the function drops the
error and immediately frees the DMA memory via dma_free_coherent(). However,
since a host CPU timeout does not guarantee the NIC has aborted the DMA
operation, could the hardware eventually execute a DMA write to the provided
bus address after the memory has been reallocated, causing memory corruption?
This is another pre-existing issue, but could this ethtool call trigger a
use-after-free if a previous device reset failed?
If a device reset fails, gve_adminq_free() destroys the adminq_pool and frees
the DMA memory backing priv->adminq. If user-space subsequently issues an
ethtool command like gve_get_link_ksettings, the driver invokes
report_link_speed() and eventually gve_adminq_issue_cmd().
If the device's tail counter was reset to 0, and the host's head counter is
still 0, could gve_adminq_issue_cmd() write the command payload into the
now-dangling priv->adminq buffer?
---
I think that this is a larger scale issue that affects all admin queue
commands that could time out. I think a solution here would be:
1) submit a net fix to allocate from dma pool in gve_amdinq_report_link_speed
2) follow up with a patch that invokes a reset on a timed out AQ command.
As far as any cross-reset failures go, that would require a much
larger behavioral change in the driver, as the driver assumes that if
the actual hardware reset fails, it is an “unrecoverable platform
error” and makes no promises about future driver behavior.
>
> cmd->base.speed = priv->link_speed;
>
> cmd->base.duplex = DUPLEX_FULL;
> -
> return err;
> }
>
> diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
> index dfc734d8bec9..181ec6919051 100644
> --- a/drivers/net/ethernet/google/gve/gve_main.c
> +++ b/drivers/net/ethernet/google/gve/gve_main.c
> @@ -1475,15 +1475,15 @@ static int gve_close(struct net_device *dev)
> return err;
> }
>
> -static void gve_handle_link_status(struct gve_priv *priv, bool link_status)
> +static void gve_handle_link_status(struct gve_priv *priv)
> {
> if (!gve_get_napi_enabled(priv))
> return;
>
> - if (link_status == netif_carrier_ok(priv->dev))
> + if (priv->link_up == netif_carrier_ok(priv->dev))
> return;
>
> - if (link_status) {
> + if (priv->link_up) {
Sashiko says:
Is there a time-of-check to time-of-use race here?
Because priv->link_up is read twice in quick succession without protection,
and could be modified concurrently by gve_adminq_report_link_status() running
in the lockless service_task workqueue, can the value change between the two
checks?
Would using READ_ONCE() and saving the result to a local variable prevent
unintended compiler optimizations and state changes?
Will fix in v3.
> netdev_info(priv->dev, "Device link is up.\n");
> netif_carrier_on(priv->dev);
> } else {
> @@ -1510,7 +1510,6 @@ static int gve_set_xdp(struct gve_priv *priv, struct bpf_prog *prog,
> {
> struct bpf_prog *old_prog;
> int err = 0;
> - u32 status;
>
> old_prog = READ_ONCE(priv->xdp_prog);
> if (!netif_running(priv->dev)) {
> @@ -1537,8 +1536,8 @@ static int gve_set_xdp(struct gve_priv *priv, struct bpf_prog *prog,
> bpf_prog_put(old_prog);
>
> out:
> - status = ioread32be(&priv->reg_bar0->device_status);
> - gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status);
> + priv->ctrl_ops->report_link_status(priv);
> + gve_handle_link_status(priv);
> return err;
> }
>
> @@ -1971,11 +1970,11 @@ static void gve_turnup(struct gve_priv *priv)
>
> static void gve_turnup_and_check_status(struct gve_priv *priv)
> {
> - u32 status;
> + const struct gve_ctrl_ops *ops = priv->ctrl_ops;
>
> gve_turnup(priv);
> - status = ioread32be(&priv->reg_bar0->device_status);
> - gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status);
> + ops->report_link_status(priv);
> + gve_handle_link_status(priv);
> }
>
> static struct gve_notify_block *gve_get_tx_notify_block(struct gve_priv *priv,
> @@ -2299,12 +2298,14 @@ static void gve_service_task(struct work_struct *work)
> {
> struct gve_priv *priv = container_of(work, struct gve_priv,
> service_task);
> + const struct gve_ctrl_ops *ops = priv->ctrl_ops;
> u32 status = ioread32be(&priv->reg_bar0->device_status);
>
> gve_handle_status(priv, status);
>
> gve_handle_reset(priv);
> - gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status);
> + ops->report_link_status(priv);
> + gve_handle_link_status(priv);
> }
>
> static void gve_set_netdev_xdp_features(struct gve_priv *priv)
> @@ -2427,6 +2428,8 @@ static const struct gve_ctrl_ops gve_adminq_ops = {
> .teardown_mgmt_irq = gve_adminq_teardown_mgmt_irq,
> .create_queues = gve_adminq_create_queues,
> .destroy_queues = gve_adminq_destroy_queues,
> + .report_link_status = gve_adminq_report_link_status,
> + .report_link_speed = gve_adminq_report_link_speed,
> };
>
> static int gve_init_priv(struct gve_priv *priv)
> --
> 2.54.0.1013.g208068f2d8-goog
>
--
Joshua Washington | Software Engineer | joshwash@google.com | (414) 366-4423
^ permalink raw reply [flat|nested] 33+ messages in thread