[net-next, v3] net: mana: Support HW link state events

[net-next, v3] net: mana: Support HW link state events

[Haiyang Zhang]

From: Haiyang Zhang <haiyangz@microsoft.com>

Handle the HW link state events received from HW channel, and
set the proper link state accordingly.

Signed-off-by: Haiyang Zhang <haiyangz@microsoft.com>

---
v3:
  Don't stop / start the queues, and use disable_work_sync() as
  suggested by Jakub Kicinski.

v2:
  Updated link up/down to be symmetric, and other minor changes based
  on comments from Andrew Lunn.

---
 .../net/ethernet/microsoft/mana/gdma_main.c   |  1 +
 .../net/ethernet/microsoft/mana/hw_channel.c  | 12 +++++
 drivers/net/ethernet/microsoft/mana/mana_en.c | 53 +++++++++++++++++--
 include/net/mana/gdma.h                       |  4 +-
 include/net/mana/hw_channel.h                 |  2 +
 include/net/mana/mana.h                       |  4 ++
 6 files changed, 70 insertions(+), 6 deletions(-)

diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c
index 43f034e180c4..effe0a2f207a 100644
--- a/drivers/net/ethernet/microsoft/mana/gdma_main.c
+++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c
@@ -528,6 +528,7 @@ static void mana_gd_process_eqe(struct gdma_queue *eq)
 	case GDMA_EQE_HWC_INIT_DONE:
 	case GDMA_EQE_HWC_SOC_SERVICE:
 	case GDMA_EQE_RNIC_QP_FATAL:
+	case GDMA_EQE_HWC_SOC_RECONFIG_DATA:
 		if (!eq->eq.callback)
 			break;
 
diff --git a/drivers/net/ethernet/microsoft/mana/hw_channel.c b/drivers/net/ethernet/microsoft/mana/hw_channel.c
index ada6c78a2bef..367e18d71413 100644
--- a/drivers/net/ethernet/microsoft/mana/hw_channel.c
+++ b/drivers/net/ethernet/microsoft/mana/hw_channel.c
@@ -118,6 +118,7 @@ static void mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self,
 	struct gdma_dev *gd = hwc->gdma_dev;
 	union hwc_init_type_data type_data;
 	union hwc_init_eq_id_db eq_db;
+	struct mana_context *ac;
 	u32 type, val;
 	int ret;
 
@@ -196,6 +197,17 @@ static void mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self,
 			hwc->hwc_timeout = val;
 			break;
 
+		case HWC_DATA_HW_LINK_CONNECT:
+		case HWC_DATA_HW_LINK_DISCONNECT:
+			ac = gd->gdma_context->mana.driver_data;
+			if (!ac)
+				break;
+
+			ac->link_event = type;
+			schedule_work(&ac->link_change_work);
+
+			break;
+
 		default:
 			dev_warn(hwc->dev, "Received unknown reconfig type %u\n", type);
 			break;
diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c
index 0142fd98392c..949aedebc8c3 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_en.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_en.c
@@ -20,6 +20,7 @@
 
 #include <net/mana/mana.h>
 #include <net/mana/mana_auxiliary.h>
+#include <net/mana/hw_channel.h>
 
 static DEFINE_IDA(mana_adev_ida);
 
@@ -84,7 +85,6 @@ static int mana_open(struct net_device *ndev)
 	/* Ensure port state updated before txq state */
 	smp_wmb();
 
-	netif_carrier_on(ndev);
 	netif_tx_wake_all_queues(ndev);
 	netdev_dbg(ndev, "%s successful\n", __func__);
 	return 0;
@@ -100,6 +100,45 @@ static int mana_close(struct net_device *ndev)
 	return mana_detach(ndev, true);
 }
 
+static void mana_link_state_handle(struct work_struct *w)
+{
+	struct mana_context *ac;
+	struct net_device *ndev;
+	bool link_up;
+	int i;
+
+	ac = container_of(w, struct mana_context, link_change_work);
+
+	rtnl_lock();
+
+	if (ac->link_event == HWC_DATA_HW_LINK_CONNECT)
+		link_up = true;
+	else if (ac->link_event == HWC_DATA_HW_LINK_DISCONNECT)
+		link_up = false;
+	else
+		goto out;
+
+	/* Process all ports */
+	for (i = 0; i < ac->num_ports; i++) {
+		ndev = ac->ports[i];
+		if (!ndev)
+			continue;
+
+		if (link_up) {
+			if (!netif_carrier_ok(ndev))
+				netif_carrier_on(ndev);
+
+			__netdev_notify_peers(ndev);
+		} else {
+			if (netif_carrier_ok(ndev))
+				netif_carrier_off(ndev);
+		}
+	}
+
+out:
+	rtnl_unlock();
+}
+
 static bool mana_can_tx(struct gdma_queue *wq)
 {
 	return mana_gd_wq_avail_space(wq) >= MAX_TX_WQE_SIZE;
@@ -3059,9 +3098,6 @@ int mana_attach(struct net_device *ndev)
 	/* Ensure port state updated before txq state */
 	smp_wmb();
 
-	if (apc->port_is_up)
-		netif_carrier_on(ndev);
-
 	netif_device_attach(ndev);
 
 	return 0;
@@ -3154,7 +3190,6 @@ int mana_detach(struct net_device *ndev, bool from_close)
 	smp_wmb();
 
 	netif_tx_disable(ndev);
-	netif_carrier_off(ndev);
 
 	if (apc->port_st_save) {
 		err = mana_dealloc_queues(ndev);
@@ -3243,6 +3278,8 @@ static int mana_probe_port(struct mana_context *ac, int port_idx,
 		goto free_indir;
 	}
 
+	netif_carrier_on(ndev);
+
 	debugfs_create_u32("current_speed", 0400, apc->mana_port_debugfs, &apc->speed);
 
 	return 0;
@@ -3431,6 +3468,8 @@ int mana_probe(struct gdma_dev *gd, bool resuming)
 
 	if (!resuming) {
 		ac->num_ports = num_ports;
+
+		INIT_WORK(&ac->link_change_work, mana_link_state_handle);
 	} else {
 		if (ac->num_ports != num_ports) {
 			dev_err(dev, "The number of vPorts changed: %d->%d\n",
@@ -3438,6 +3477,8 @@ int mana_probe(struct gdma_dev *gd, bool resuming)
 			err = -EPROTO;
 			goto out;
 		}
+
+		enable_work(&ac->link_change_work);
 	}
 
 	if (ac->num_ports == 0)
@@ -3500,6 +3541,8 @@ void mana_remove(struct gdma_dev *gd, bool suspending)
 	int err;
 	int i;
 
+	disable_work_sync(&ac->link_change_work);
+
 	/* adev currently doesn't support suspending, always remove it */
 	if (gd->adev)
 		remove_adev(gd);
diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h
index 57df78cfbf82..637f42485dba 100644
--- a/include/net/mana/gdma.h
+++ b/include/net/mana/gdma.h
@@ -590,6 +590,7 @@ enum {
 
 /* Driver can self reset on FPGA Reconfig EQE notification */
 #define GDMA_DRV_CAP_FLAG_1_HANDLE_RECONFIG_EQE BIT(17)
+#define GDMA_DRV_CAP_FLAG_1_HW_VPORT_LINK_AWARE BIT(6)
 
 #define GDMA_DRV_CAP_FLAGS1 \
 	(GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \
@@ -599,7 +600,8 @@ enum {
 	 GDMA_DRV_CAP_FLAG_1_DEV_LIST_HOLES_SUP | \
 	 GDMA_DRV_CAP_FLAG_1_DYNAMIC_IRQ_ALLOC_SUPPORT | \
 	 GDMA_DRV_CAP_FLAG_1_SELF_RESET_ON_EQE | \
-	 GDMA_DRV_CAP_FLAG_1_HANDLE_RECONFIG_EQE)
+	 GDMA_DRV_CAP_FLAG_1_HANDLE_RECONFIG_EQE | \
+	 GDMA_DRV_CAP_FLAG_1_HW_VPORT_LINK_AWARE)
 
 #define GDMA_DRV_CAP_FLAGS2 0
 
diff --git a/include/net/mana/hw_channel.h b/include/net/mana/hw_channel.h
index 83cf93338eb3..16feb39616c1 100644
--- a/include/net/mana/hw_channel.h
+++ b/include/net/mana/hw_channel.h
@@ -24,6 +24,8 @@
 #define HWC_INIT_DATA_PF_DEST_CQ_ID	11
 
 #define HWC_DATA_CFG_HWC_TIMEOUT 1
+#define HWC_DATA_HW_LINK_CONNECT 2
+#define HWC_DATA_HW_LINK_DISCONNECT 3
 
 #define HW_CHANNEL_WAIT_RESOURCE_TIMEOUT_MS 30000
 
diff --git a/include/net/mana/mana.h b/include/net/mana/mana.h
index 0921485565c0..8906901535f5 100644
--- a/include/net/mana/mana.h
+++ b/include/net/mana/mana.h
@@ -477,6 +477,10 @@ struct mana_context {
 	struct dentry *mana_eqs_debugfs;
 
 	struct net_device *ports[MAX_PORTS_IN_MANA_DEV];
+
+	/* Link state change work */
+	struct work_struct link_change_work;
+	u32 link_event;
 };
 
 struct mana_port_context {
-- 
2.34.1

Re: [net-next, v3] net: mana: Support HW link state events

[Paolo Abeni]

On 10/24/25 3:41 AM, Haiyang Zhang wrote:
> @@ -3243,6 +3278,8 @@ static int mana_probe_port(struct mana_context *ac, int port_idx,
>  		goto free_indir;
>  	}
>  
> +	netif_carrier_on(ndev);

Why is  the above needed? I thought mana_link_state_handle() should kick
and set the carrier on as needed???

/P

RE: [EXTERNAL] Re: [net-next, v3] net: mana: Support HW link state events

[Haiyang Zhang]

> -----Original Message-----
> From: Paolo Abeni <pabeni@redhat.com>
> Sent: Tuesday, October 28, 2025 10:31 AM
> To: Haiyang Zhang <haiyangz@linux.microsoft.com>; linux-
> hyperv@vger.kernel.org; netdev@vger.kernel.org
> Cc: Haiyang Zhang <haiyangz@microsoft.com>; Paul Rosswurm
> <paulros@microsoft.com>; Dexuan Cui <DECUI@microsoft.com>; KY Srinivasan
> <kys@microsoft.com>; wei.liu@kernel.org; edumazet@google.com;
> davem@davemloft.net; kuba@kernel.org; Long Li <longli@microsoft.com>;
> ssengar@linux.microsoft.com; ernis@linux.microsoft.com;
> dipayanroy@linux.microsoft.com; Konstantin Taranov
> <kotaranov@microsoft.com>; horms@kernel.org;
> shradhagupta@linux.microsoft.com; leon@kernel.org; mlevitsk@redhat.com;
> yury.norov@gmail.com; Shiraz Saleem <shirazsaleem@microsoft.com>;
> andrew+netdev@lunn.ch; linux-rdma@vger.kernel.org; linux-
> kernel@vger.kernel.org
> Subject: [EXTERNAL] Re: [net-next, v3] net: mana: Support HW link state
> events
> 
> On 10/24/25 3:41 AM, Haiyang Zhang wrote:
> > @@ -3243,6 +3278,8 @@ static int mana_probe_port(struct mana_context
> *ac, int port_idx,
> >  		goto free_indir;
> >  	}
> >
> > +	netif_carrier_on(ndev);
> 
> Why is  the above needed? I thought mana_link_state_handle() should kick
> and set the carrier on as needed???

Thanks for the question -- our MANA NIC only sends out the link state down/up 
messages when need to let the VM rerun DHCP client and change IP address...

So, I need to add netif_carrier_on(ndev) in the probe(), otherwise the 
/sys/class/net/ethX/operstate will remain "unknown" until it receives the 
Link down/up messages which do NOT always happen.

Cc: @Paul Rosswurm

- Haiyang

Re: [EXTERNAL] Re: [net-next, v3] net: mana: Support HW link state events

[Jakub Kicinski]

On Tue, 28 Oct 2025 19:36:02 +0000 Haiyang Zhang wrote:
> > Why is  the above needed? I thought mana_link_state_handle() should kick
> > and set the carrier on as needed???  
> 
> Thanks for the question -- our MANA NIC only sends out the link state down/up 
> messages when need to let the VM rerun DHCP client and change IP address...
> 
> So, I need to add netif_carrier_on(ndev) in the probe(), otherwise the 
> /sys/class/net/ethX/operstate will remain "unknown" until it receives the 
> Link down/up messages which do NOT always happen.

Oh that makes the code make much more sense.
Please add this and more detail into the commit message.

> +			if (!netif_carrier_ok(ndev))
> +				netif_carrier_on(ndev);

Testing carrier_ok() before calling carrier_on/off is entirely
pointless, please see the relevant implementations.

BTW I think the ac->link_event accesses are technically racy,
wrap them in READ_ONCE() / WRITE_ONCE() while you respin.
(Unless mana_hwc_init_event_handler() is somehow under rtnl_lock)
-- 
pw-bot: cr

RE: [EXTERNAL] Re: [net-next, v3] net: mana: Support HW link state events

[Haiyang Zhang]

> -----Original Message-----
> From: Jakub Kicinski <kuba@kernel.org>
> Sent: Tuesday, October 28, 2025 6:02 PM
> To: Haiyang Zhang <haiyangz@microsoft.com>
> Cc: Paolo Abeni <pabeni@redhat.com>; Haiyang Zhang
> <haiyangz@linux.microsoft.com>; linux-hyperv@vger.kernel.org;
> netdev@vger.kernel.org; Paul Rosswurm <paulros@microsoft.com>; Dexuan Cui
> <DECUI@microsoft.com>; KY Srinivasan <kys@microsoft.com>;
> wei.liu@kernel.org; edumazet@google.com; davem@davemloft.net; Long Li
> <longli@microsoft.com>; ssengar@linux.microsoft.com;
> ernis@linux.microsoft.com; dipayanroy@linux.microsoft.com; Konstantin
> Taranov <kotaranov@microsoft.com>; horms@kernel.org;
> shradhagupta@linux.microsoft.com; leon@kernel.org; mlevitsk@redhat.com;
> yury.norov@gmail.com; Shiraz Saleem <shirazsaleem@microsoft.com>;
> andrew+netdev@lunn.ch; linux-rdma@vger.kernel.org; linux-
> kernel@vger.kernel.org
> Subject: Re: [EXTERNAL] Re: [net-next, v3] net: mana: Support HW link
> state events
> 
> On Tue, 28 Oct 2025 19:36:02 +0000 Haiyang Zhang wrote:
> > > Why is  the above needed? I thought mana_link_state_handle() should
> kick
> > > and set the carrier on as needed???
> >
> > Thanks for the question -- our MANA NIC only sends out the link state
> down/up
> > messages when need to let the VM rerun DHCP client and change IP
> address...
> >
> > So, I need to add netif_carrier_on(ndev) in the probe(), otherwise the
> > /sys/class/net/ethX/operstate will remain "unknown" until it receives
> the
> > Link down/up messages which do NOT always happen.
> 
> Oh that makes the code make much more sense.
> Please add this and more detail into the commit message.

Will do.

> 
> > +			if (!netif_carrier_ok(ndev))
> > +				netif_carrier_on(ndev);
> 
> Testing carrier_ok() before calling carrier_on/off is entirely
> pointless, please see the relevant implementations.
> 
> BTW I think the ac->link_event accesses are technically racy,
> wrap them in READ_ONCE() / WRITE_ONCE() while you respin.
> (Unless mana_hwc_init_event_handler() is somehow under rtnl_lock)

I will remove the netif_carrier_ok(), and add READ/WRITE_ONCE.

Thanks,
- Haiyang