From: Harshitha Ramamurthy <hramamurthy@google.com>
To: netdev@vger.kernel.org
Cc: joshwash@google.com, hramamurthy@google.com,
andrew+netdev@lunn.ch, davem@davemloft.net, edumazet@google.com,
kuba@kernel.org, pabeni@redhat.com, ast@kernel.org,
daniel@iogearbox.net, hawk@kernel.org, john.fastabend@gmail.com,
sdf@fomichev.me, jordanrhee@google.com, willemb@google.com,
nktgrg@google.com, maolson@google.com, thostet@google.com,
jacob.e.keller@intel.com, debarghyak@google.com,
kees@kernel.org, bpf@vger.kernel.org,
linux-kernel@vger.kernel.org, przemyslaw.kitszel@intel.com
Subject: [PATCH net-next v9 08/12] gve: split up notify block allocation and setup paths
Date: Wed, 30 Sep 2026 19:04:24 +0000 [thread overview]
Message-ID: <20260930190428.1130007-9-hramamurthy@google.com> (raw)
In-Reply-To: <20260930190428.1130007-1-hramamurthy@google.com>
From: Joshua Washington <joshwash@google.com>
Before this patch, notify block allocation and setup occurred in the same
method. This all occurred before gve_adminq_configure_device_resources,
which populates the irq_db_indicies array, a DMA region with BAR offsets
for MSI-X vectors.
The coming mailbox mode will require notify blocks to be set up only
after receiving the IRQ doorbell offsets, as the request does not work
with a supplied DMA buffer in the way that admin queue mode does. The
intended flow in that case would be:
1) allocate notify blocks
2) request doorbell information
3) set up MSI-X vectors based on doorbell info
This ordering also works for admin queue mode, so it will be updated to
match.
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Jordan Rhee <jordanrhee@google.com>
Reviewed-by: Przemek Kitszel <przemyslaw.kitszel@intel.com>
Signed-off-by: Joshua Washington <joshwash@google.com>
Signed-off-by: Harshitha Ramamurthy <hramamurthy@google.com>
---
v8:
- gate mgmt irq vs ntfy block teardown on different conditions
- move gve_teardown_notify_blocks() to gve_teardown_device() from
gve_reset_device()
- don't set block->irq when tearing down notify blocks
- guard disable_irq() call in gve_remove_napi() on whether the irq was
requested
- teardown notify blocks as part of device reset
v3:
- remove redundant call to gve_teardown_clock()
drivers/net/ethernet/google/gve/gve.h | 2 +
drivers/net/ethernet/google/gve/gve_main.c | 155 +++++++++++---------
drivers/net/ethernet/google/gve/gve_utils.c | 4 +-
3 files changed, 89 insertions(+), 72 deletions(-)
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index e0583e8cd2cd..f624a3e385e4 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -674,6 +674,7 @@ struct gve_notify_block {
struct gve_tx_ring *tx; /* tx rings on this block */
struct gve_rx_ring *rx; /* rx rings on this block */
u32 irq;
+ bool irq_requested;
};
/* Tracks allowed and current rx queue settings */
@@ -954,6 +955,7 @@ struct gve_priv {
u64 link_speed;
bool up_before_suspend; /* True if dev was up before suspend */
+ bool mgmt_irq_requested;
struct gve_ptype_lut *ptype_lut_dqo;
/* Must be a power of two. */
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 156bee612ba6..01a271680d78 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -425,6 +425,24 @@ int gve_napi_poll_dqo(struct napi_struct *napi, int budget)
return work_done;
}
+static void gve_free_notify_blocks(struct gve_priv *priv)
+{
+ pci_disable_msix(priv->pdev);
+ if (priv->irq_db_indices) {
+ dma_free_coherent(&priv->pdev->dev,
+ priv->num_ntfy_blks *
+ sizeof(*priv->irq_db_indices),
+ priv->irq_db_indices,
+ priv->irq_db_indices_bus);
+ priv->irq_db_indices = NULL;
+ }
+
+ kvfree(priv->ntfy_blocks);
+ priv->ntfy_blocks = NULL;
+ kvfree(priv->msix_vectors);
+ priv->msix_vectors = NULL;
+}
+
static const struct cpumask *gve_get_node_mask(struct gve_priv *priv)
{
if (priv->numa_node == NUMA_NO_NODE)
@@ -436,11 +454,9 @@ static const struct cpumask *gve_get_node_mask(struct gve_priv *priv)
static int gve_alloc_notify_blocks(struct gve_priv *priv)
{
int num_vecs_requested = priv->num_ntfy_blks + 1;
- const struct cpumask *node_mask;
- unsigned int cur_cpu;
int vecs_enabled;
- int i, j;
int err;
+ int i;
priv->msix_vectors = kvzalloc_objs(*priv->msix_vectors,
num_vecs_requested);
@@ -454,7 +470,7 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
dev_err(&priv->pdev->dev, "Could not enable min msix %d/%d\n",
GVE_MIN_MSIX, vecs_enabled);
err = vecs_enabled;
- goto abort_with_msix_vectors;
+ goto abort;
}
if (vecs_enabled != num_vecs_requested) {
int new_num_ntfy_blks = (vecs_enabled - 1) & ~0x1;
@@ -477,15 +493,6 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
}
- /* Setup Management Vector - the last vector */
- snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name), "gve-mgmnt@pci:%s",
- pci_name(priv->pdev));
- err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector,
- gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv);
- if (err) {
- dev_err(&priv->pdev->dev, "Did not receive management vector.\n");
- goto abort_with_msix_enabled;
- }
priv->irq_db_indices =
dma_alloc_coherent(&priv->pdev->dev,
priv->num_ntfy_blks *
@@ -493,15 +500,64 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
&priv->irq_db_indices_bus, GFP_KERNEL);
if (!priv->irq_db_indices) {
err = -ENOMEM;
- goto abort_with_mgmt_vector;
+ goto abort;
}
priv->ntfy_blocks = kvzalloc(priv->num_ntfy_blks *
sizeof(*priv->ntfy_blocks), GFP_KERNEL);
if (!priv->ntfy_blocks) {
err = -ENOMEM;
- goto abort_with_irq_db_indices;
+ goto abort;
+ }
+ return 0;
+
+abort:
+ gve_free_notify_blocks(priv);
+ return err;
+}
+
+static void gve_teardown_notify_blocks(struct gve_priv *priv)
+{
+ int i;
+
+ if (priv->mgmt_irq_requested) {
+ free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
+ priv->mgmt_irq_requested = false;
+ }
+
+ if (!priv->ntfy_blocks)
+ return;
+
+ for (i = 0; i < priv->num_ntfy_blks; i++) {
+ struct gve_notify_block *block = &priv->ntfy_blocks[i];
+
+ if (!block->irq_requested)
+ continue;
+
+ irq_set_affinity_hint(priv->msix_vectors[i].vector,
+ NULL);
+ free_irq(priv->msix_vectors[i].vector, block);
+ block->irq_requested = false;
}
+}
+
+static int gve_setup_notify_blocks(struct gve_priv *priv)
+{
+ const struct cpumask *node_mask;
+ unsigned int cur_cpu;
+ int i;
+ int err;
+
+ /* Setup Management Vector - the last vector */
+ snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name),
+ "gve-mgmnt@pci:%s", pci_name(priv->pdev));
+ err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector,
+ gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv);
+ if (err) {
+ dev_err(&priv->pdev->dev, "Did not receive management vector.\n");
+ return err;
+ }
+ priv->mgmt_irq_requested = true;
/* Setup the other blocks - the first n-1 vectors */
node_mask = gve_get_node_mask(priv);
@@ -519,9 +575,10 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
if (err) {
dev_err(&priv->pdev->dev,
"Failed to receive msix vector %d\n", i);
- goto abort_with_some_ntfy_blocks;
+ goto abort;
}
block->irq = priv->msix_vectors[msix_idx].vector;
+ block->irq_requested = true;
irq_set_affinity_and_hint(block->irq,
cpumask_of(cur_cpu));
block->irq_db_index = &priv->irq_db_indices[i].index;
@@ -535,61 +592,12 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
cur_cpu = cpumask_first(node_mask);
}
return 0;
-abort_with_some_ntfy_blocks:
- for (j = 0; j < i; j++) {
- struct gve_notify_block *block = &priv->ntfy_blocks[j];
- int msix_idx = j;
- irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
- NULL);
- free_irq(priv->msix_vectors[msix_idx].vector, block);
- block->irq = 0;
- }
- kvfree(priv->ntfy_blocks);
- priv->ntfy_blocks = NULL;
-abort_with_irq_db_indices:
- dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
- sizeof(*priv->irq_db_indices),
- priv->irq_db_indices, priv->irq_db_indices_bus);
- priv->irq_db_indices = NULL;
-abort_with_mgmt_vector:
- free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
-abort_with_msix_enabled:
- pci_disable_msix(priv->pdev);
-abort_with_msix_vectors:
- kvfree(priv->msix_vectors);
- priv->msix_vectors = NULL;
+abort:
+ gve_teardown_notify_blocks(priv);
return err;
}
-static void gve_free_notify_blocks(struct gve_priv *priv)
-{
- int i;
-
- if (!priv->msix_vectors)
- return;
-
- /* Free the irqs */
- for (i = 0; i < priv->num_ntfy_blks; i++) {
- struct gve_notify_block *block = &priv->ntfy_blocks[i];
- int msix_idx = i;
-
- irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
- NULL);
- free_irq(priv->msix_vectors[msix_idx].vector, block);
- block->irq = 0;
- }
- free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
- kvfree(priv->ntfy_blocks);
- priv->ntfy_blocks = NULL;
- dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
- sizeof(*priv->irq_db_indices),
- priv->irq_db_indices, priv->irq_db_indices_bus);
- priv->irq_db_indices = NULL;
- pci_disable_msix(priv->pdev);
- kvfree(priv->msix_vectors);
- priv->msix_vectors = NULL;
-}
static void gve_tx_get_curr_alloc_cfg(struct gve_priv *priv,
struct gve_tx_alloc_rings_cfg *cfg)
@@ -824,6 +832,7 @@ static void gve_teardown_device(struct gve_priv *priv)
/* Free any resources shared with the device only after we have a
* guarantee that the device will not try to access such resources.
*/
+ gve_teardown_notify_blocks(priv);
gve_free_control_plane_resources(priv);
gve_queues_mem_remove(priv);
}
@@ -2477,13 +2486,16 @@ static int gve_setup_device(struct gve_priv *priv)
err = gve_alloc_control_plane_resources(priv);
if (err)
- goto err;
+ return err;
+
err = gve_setup_control_plane_resources(priv);
if (err)
- goto err;
+ return err;
+
+ err = gve_setup_notify_blocks(priv);
+ if (err)
+ return err;
return 0;
-err:
- return err;
}
static const struct gve_ctrl_ops gve_adminq_ops = {
@@ -2633,6 +2645,7 @@ int gve_reset(struct gve_priv *priv, bool skip_queue_setup)
gve_reset_device(priv);
gve_queues_stop(priv);
gve_queues_mem_remove(priv);
+ gve_teardown_notify_blocks(priv);
gve_free_control_plane_resources(priv);
enable_work(&priv->service_task);
diff --git a/drivers/net/ethernet/google/gve/gve_utils.c b/drivers/net/ethernet/google/gve/gve_utils.c
index b53b7fcdcdaf..caed38cd5f2b 100644
--- a/drivers/net/ethernet/google/gve/gve_utils.c
+++ b/drivers/net/ethernet/google/gve/gve_utils.c
@@ -119,6 +119,8 @@ void gve_remove_napi(struct gve_priv *priv, int ntfy_idx)
{
struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
- disable_irq(block->irq);
+ if (block->irq_requested)
+ disable_irq(block->irq);
+
netif_napi_del_locked(&block->napi);
}
--
2.56.0.rc1.315.gc6ed9934b7-goog
next prev parent reply other threads:[~2026-09-30 19:04 UTC|newest]
Thread overview: 21+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-09-30 19:04 [PATCH net-next v9 00/12] gve: AdminQ mode related refactors Harshitha Ramamurthy
2026-09-30 19:04 ` [PATCH net-next v9 01/12] gve: add struct gve_device_info to hold device properties Harshitha Ramamurthy
2026-10-02 10:06 ` netdev-bot+sashiko
2026-09-30 19:04 ` [PATCH net-next v9 02/12] gve: introduce control plane operations structure Harshitha Ramamurthy
2026-09-30 19:04 ` [PATCH net-next v9 03/12] gve: introduce ctrl ops to set vectors and Qs Harshitha Ramamurthy
2026-09-30 19:04 ` [PATCH net-next v9 04/12] gve: introduce gve_adminq_get_device_properties() Harshitha Ramamurthy
2026-09-30 19:04 ` [PATCH net-next v9 05/12] gve: refactor gve_init_priv for reset path Harshitha Ramamurthy
2026-10-02 10:06 ` netdev-bot+sashiko
2026-10-02 23:23 ` Harshitha Ramamurthy
2026-09-30 19:04 ` [PATCH net-next v9 06/12] gve: simplify reset logic Harshitha Ramamurthy
2026-10-02 10:06 ` netdev-bot+sashiko
2026-10-03 1:33 ` Joshua Washington
2026-09-30 19:04 ` [PATCH net-next v9 07/12] gve: add gve_ctrl_ops for gve initialization/teardown sequences Harshitha Ramamurthy
2026-10-02 10:06 ` netdev-bot+sashiko
2026-09-30 19:04 ` Harshitha Ramamurthy [this message]
2026-09-30 19:04 ` [PATCH net-next v9 09/12] gve: introduce new methods to handle IRQ doorbells Harshitha Ramamurthy
2026-10-02 10:06 ` netdev-bot+sashiko
2026-09-30 19:04 ` [PATCH net-next v9 10/12] gve: setup and teardown management interrupts Harshitha Ramamurthy
2026-09-30 19:04 ` [PATCH net-next v9 11/12] gve: add ctrl ops for queue operations Harshitha Ramamurthy
2026-09-30 19:04 ` [PATCH net-next v9 12/12] gve: add link status/speed ctrl ops Harshitha Ramamurthy
2026-10-05 23:40 ` [PATCH net-next v9 00/12] gve: AdminQ mode related refactors patchwork-bot+netdevbpf
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260930190428.1130007-9-hramamurthy@google.com \
--to=hramamurthy@google.com \
--cc=andrew+netdev@lunn.ch \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=davem@davemloft.net \
--cc=debarghyak@google.com \
--cc=edumazet@google.com \
--cc=hawk@kernel.org \
--cc=jacob.e.keller@intel.com \
--cc=john.fastabend@gmail.com \
--cc=jordanrhee@google.com \
--cc=joshwash@google.com \
--cc=kees@kernel.org \
--cc=kuba@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=maolson@google.com \
--cc=netdev@vger.kernel.org \
--cc=nktgrg@google.com \
--cc=pabeni@redhat.com \
--cc=przemyslaw.kitszel@intel.com \
--cc=sdf@fomichev.me \
--cc=thostet@google.com \
--cc=willemb@google.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.