[PATCH v1 10/32] net/ntnic: add link handling API

[Serhii Iliushyk <sil-plv@napatech.com>]

From: Danylo Vodopianov <dvo-plv@napatech.com>

Add link state management and NIM reset functionality

- Implement phy_get_link_state to retrieve PHY link state.
- Implement adjust_maturing_delay to adjust link maturing delay.
- Implement get/set to operate current link state.

Signed-off-by: Danylo Vodopianov <dvo-plv@napatech.com>
---
 .../link_agx_100g/nt4ga_agx_link_100g.c       | 171 ++++++++++++++++++
 drivers/net/ntnic/meson.build                 |   1 +
 .../ntnic/nthw/core/include/nthw_pca9532.h    |  22 +++
 .../ntnic/nthw/core/include/nthw_phy_tile.h   |   9 +
 drivers/net/ntnic/nthw/core/nthw_pca9532.c    |  40 ++++
 drivers/net/ntnic/nthw/core/nthw_phy_tile.c   |  51 +++++-
 drivers/net/ntnic/nthw/nthw_drv.h             |   2 +
 7 files changed, 295 insertions(+), 1 deletion(-)
 create mode 100644 drivers/net/ntnic/nthw/core/include/nthw_pca9532.h
 create mode 100644 drivers/net/ntnic/nthw/core/nthw_pca9532.c

diff --git a/drivers/net/ntnic/link_mgmt/link_agx_100g/nt4ga_agx_link_100g.c b/drivers/net/ntnic/link_mgmt/link_agx_100g/nt4ga_agx_link_100g.c
index 9193f21f6f..00a30f24a5 100644
--- a/drivers/net/ntnic/link_mgmt/link_agx_100g/nt4ga_agx_link_100g.c
+++ b/drivers/net/ntnic/link_mgmt/link_agx_100g/nt4ga_agx_link_100g.c
@@ -43,6 +43,38 @@ void link_agx_100g_init(void)
  * Phy handling
  */
 
+static void phy_get_link_state(adapter_info_t *drv,
+	int port,
+	nt_link_state_p p_curr_link_state,
+	nt_link_state_p p_latched_links_state,
+	uint32_t *p_local_fault,
+	uint32_t *p_remote_fault)
+{
+	uint32_t status = 0;
+	uint32_t status_latch = 0;
+
+	nthw_phy_tile_t *p = drv->fpga_info.mp_nthw_agx.p_phy_tile;
+
+	nthw_phy_tile_get_link_summary(p,
+		&status,
+		&status_latch,
+		p_local_fault,
+		p_remote_fault,
+		port);
+
+	if (status == 0x0)
+		*p_curr_link_state = NT_LINK_STATE_DOWN;
+
+	else
+		*p_curr_link_state = NT_LINK_STATE_UP;
+
+	if (status_latch == 0x0)
+		*p_latched_links_state = NT_LINK_STATE_DOWN;
+
+	else
+		*p_latched_links_state = NT_LINK_STATE_UP;
+}
+
 static void phy_rx_path_rst(adapter_info_t *drv, int port, bool reset)
 {
 	nthw_phy_tile_t *p = drv->fpga_info.mp_nthw_agx.p_phy_tile;
@@ -242,6 +274,135 @@ static void set_nim_low_power(nim_i2c_ctx_p ctx, uint8_t nim_idx, bool low_power
 	}
 }
 
+/*
+ * Link handling
+ */
+
+static void adjust_maturing_delay(adapter_info_t *drv, int port)
+{
+	nthw_phy_tile_t *p = drv->fpga_info.mp_nthw_agx.p_phy_tile;
+	nthw_rpf_t *p_rpf = drv->fpga_info.mp_nthw_agx.p_rpf;
+	/*
+	 * Find the maximum of the absolute values of the RX componensation for all
+	 * ports (the RX componensation may not be set for some of the other ports).
+	 */
+	const int16_t unset_rx_compensation = -1;	/* 0xffff */
+	int16_t max_comp = 0;
+
+	for (int i = 0; i < drv->fpga_info.n_phy_ports; i++) {
+		int16_t comp = (int16_t)nthw_phy_tile_get_timestamp_comp_rx(p, i);
+
+		if (comp != unset_rx_compensation && abs(comp) > abs(max_comp))
+			max_comp = comp;
+	}
+
+	int delay = nthw_rpf_get_maturing_delay(p_rpf) - max_comp;
+
+	/*
+	 * For SOF time-stamping account for jumbo frame.
+	 * Frame size = 80000 b. divided by Gb link speed = processing time in ns.
+	 */
+	if (!nthw_rpf_get_ts_at_eof(p_rpf)) {
+		uint32_t jumbo_frame_processing_time_ns = 80000U / 100U;
+		delay -= (int)jumbo_frame_processing_time_ns;
+	}
+
+	const unsigned int delay_bit_width = 19;/* 19 bits maturing delay */
+	const int min_delay = -(1 << (delay_bit_width - 1));
+	const int max_delay = 0;
+
+	if (delay >= min_delay && delay <= max_delay) {
+		nthw_rpf_set_maturing_delay(p_rpf, delay);
+
+	} else {
+		NT_LOG(WRN, NTNIC,
+			"Port %u: Cannot set the RPF adjusted maturing delay to %i because "
+			"that value is outside the legal range [%i:%i]",
+			port, delay, min_delay, max_delay);
+	}
+}
+
+static void set_link_state(adapter_info_t *drv, nim_i2c_ctx_p ctx, link_state_t *state, int port)
+{
+	nthw_phy_tile_t *p = drv->fpga_info.mp_nthw_agx.p_phy_tile;
+	/*
+	 * 100G: 4 LEDs per port
+	 */
+
+	bool led_on = state->nim_present && state->link_state == NT_LINK_STATE_UP;
+	uint8_t led_pos = (uint8_t)(port * ctx->lane_count);
+
+	for (uint8_t i = 0; i < ctx->lane_count; i++) {
+		nthw_pca9532_set_led_on(drv->fpga_info.mp_nthw_agx.p_pca9532_led, led_pos + 1,
+			led_on);
+	}
+
+	nthw_phy_tile_set_timestamp_comp_rx(p, port, 0);
+
+	if (ctx->specific_u.qsfp.specific_u.qsfp28.media_side_fec_ena) {
+		/* Updated after mail from JKH 2023-02-07 */
+		nthw_phy_tile_set_timestamp_comp_rx(p, port, 0x9E);
+		/* TODO Hermosa Awaiting comp values to use */
+
+	} else {
+		/* TODO Hermosa Awaiting comp values to use */
+		nthw_phy_tile_set_timestamp_comp_rx(p, port, 0);
+	}
+
+	adjust_maturing_delay(drv, port);	/* MUST be called after timestampCompRx */
+}
+
+static void get_link_state(adapter_info_t *drv, nim_i2c_ctx_p ctx, link_state_t *state, int port)
+{
+	uint32_t local_fault;
+	uint32_t remote_fault;
+	nt_link_state_t curr_link_state;
+
+	nthw_phy_tile_t *p = drv->fpga_info.mp_nthw_agx.p_phy_tile;
+
+	/* Save the current state before reading the new */
+	curr_link_state = state->link_state;
+
+	phy_get_link_state(drv, port, &state->link_state, &state->link_state_latched, &local_fault,
+		&remote_fault);
+
+	if (curr_link_state != state->link_state)
+		NT_LOG(DBG, NTNIC, "Port %d: Faults(Local = %d, Remote = %d)", port, local_fault,
+			remote_fault);
+
+	state->nim_present = nim_is_present(ctx, port);
+
+	if (!state->nim_present)
+		return;	/* No nim so no need to do anything */
+
+	state->link_up = state->link_state == NT_LINK_STATE_UP ? true : false;
+
+	if (state->link_state == NT_LINK_STATE_UP)
+		return;	/* The link is up so no need to do anything else */
+
+	if (remote_fault == 0) {
+		phy_reset_rx(drv, port);
+		NT_LOG(DBG, NTNIC, "Port %u: resetRx due to local fault.", port);
+		return;
+	}
+
+	/* In case of too many errors perform a reset */
+	if (nthw_phy_tile_get_rx_hi_ber(p, port)) {
+		NT_LOG(INF, NTNIC, "Port %u: HiBer", port);
+		phy_reset_rx(drv, port);
+		return;
+	}
+
+	/* If FEC is not enabled then no reason to look at FEC state */
+	if (!ctx->specific_u.qsfp.specific_u.qsfp28.media_side_fec_ena ||
+		nthw_phy_tile_read_fec_enabled_by_scratch(p, port)) {
+		return;
+	}
+
+	if (!nthw_phy_tile_get_rx_am_lock(p, port))
+		phy_reset_rx(drv, port);
+}
+
 /*
  * Utility functions
  */
@@ -645,6 +806,7 @@ static void *_common_ptp_nim_state_machine(void *data)
 
 	while (monitor_task_is_running[adapter_no]) {
 		int i;
+		static bool reported_link[NUM_ADAPTER_PORTS_MAX] = { false };
 
 		for (i = 0; i < nb_ports; i++) {
 			const bool is_port_disabled = link_info->port_action[i].port_disable;
@@ -691,12 +853,14 @@ static void *_common_ptp_nim_state_machine(void *data)
 				continue;
 			}
 
+			get_link_state(drv, nim_ctx, &link_state[i], i);
 			link_state[i].link_disabled = is_port_disabled;
 
 			if (!link_state[i].nim_present) {
 				if (!link_state[i].lh_nim_absent) {
 					NT_LOG(INF, NTNIC, "%s: NIM module removed",
 						drv->mp_port_id_str[i]);
+					reported_link[i] = false;
 					link_state[i].link_up = false;
 					link_state[i].lh_nim_absent = true;
 
@@ -707,6 +871,13 @@ static void *_common_ptp_nim_state_machine(void *data)
 
 				continue;
 			}
+
+			if (reported_link[i] != link_state[i].link_up) {
+				NT_LOG(INF, NTNIC, "%s: link is %s", drv->mp_port_id_str[i],
+					(link_state[i].link_up ? "up" : "down"));
+				reported_link[i] = link_state[i].link_up;
+				set_link_state(drv, nim_ctx, &link_state[i], i);
+			}
 		}
 
 		if (monitor_task_is_running[adapter_no])
diff --git a/drivers/net/ntnic/meson.build b/drivers/net/ntnic/meson.build
index b1505c5549..92aad6f94d 100644
--- a/drivers/net/ntnic/meson.build
+++ b/drivers/net/ntnic/meson.build
@@ -55,6 +55,7 @@ sources = files(
         'nthw/core/nthw_iic.c',
         'nthw/core/nthw_mac_pcs.c',
         'nthw/core/nthw_pcie3.c',
+        'nthw/core/nthw_pca9532.c',
         'nthw/core/nthw_pcal6416a.c',
         'nthw/core/nthw_phy_tile.c',
         'nthw/core/nthw_si5332_si5156.c',
diff --git a/drivers/net/ntnic/nthw/core/include/nthw_pca9532.h b/drivers/net/ntnic/nthw/core/include/nthw_pca9532.h
new file mode 100644
index 0000000000..24da937a18
--- /dev/null
+++ b/drivers/net/ntnic/nthw/core/include/nthw_pca9532.h
@@ -0,0 +1,22 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Napatech A/S
+ */
+
+#ifndef __NTHW_PCA9532_H__
+#define __NTHW_PCA9532_H__
+
+#include "nthw_si5332_si5156.h"
+
+struct nthw_pca9532 {
+	nthw_i2cm_t *mp_nt_i2cm;
+	uint8_t m_dev_address;
+	nthw_pca9849_t *mp_ca9849;
+	uint8_t m_mux_channel;
+};
+
+typedef struct nthw_pca9532 nthw_pca9532_t;
+
+void nthw_pca9532_set_led_on(nthw_pca9532_t *p, uint8_t led_pos, bool state_on);
+
+#endif	/* __NTHW_PCA9532_H__ */
diff --git a/drivers/net/ntnic/nthw/core/include/nthw_phy_tile.h b/drivers/net/ntnic/nthw/core/include/nthw_phy_tile.h
index c810f55c0d..f84aef44a9 100644
--- a/drivers/net/ntnic/nthw/core/include/nthw_phy_tile.h
+++ b/drivers/net/ntnic/nthw/core/include/nthw_phy_tile.h
@@ -74,8 +74,17 @@ void nthw_phy_tile_set_host_loopback(nthw_phy_tile_t *p, uint8_t intf_no, uint8_
 void nthw_phy_tile_set_tx_equalization(nthw_phy_tile_t *p, uint8_t intf_no, uint8_t lane,
 	uint32_t pre_tap2, uint32_t main_tap, uint32_t pre_tap1,
 	uint32_t post_tap1);
+void nthw_phy_tile_get_link_summary(nthw_phy_tile_t *p, uint32_t *p_nt_phy_link_state,
+	uint32_t *p_ll_nt_phy_link_state, uint32_t *p_lh_local_fault,
+	uint32_t *p_lh_remote_fault, uint8_t index);
 void nthw_phy_tile_set_tx_reset(nthw_phy_tile_t *p, uint8_t intf_no, bool reset);
 void nthw_phy_tile_set_rx_reset(nthw_phy_tile_t *p, uint8_t intf_no, bool reset);
+bool nthw_phy_tile_read_fec_enabled_by_scratch(nthw_phy_tile_t *p, uint8_t intf_no);
+
+bool nthw_phy_tile_get_rx_hi_ber(nthw_phy_tile_t *p, uint8_t intf_no);
+bool nthw_phy_tile_get_rx_am_lock(nthw_phy_tile_t *p, uint8_t intf_no);
+void nthw_phy_tile_set_timestamp_comp_rx(nthw_phy_tile_t *p, uint8_t intf_no, uint32_t value);
+uint32_t nthw_phy_tile_get_timestamp_comp_rx(nthw_phy_tile_t *p, uint8_t intf_no);
 
 uint32_t nthw_phy_tile_read_eth(nthw_phy_tile_t *p, uint8_t intf_no, uint32_t address);
 void nthw_phy_tile_write_eth(nthw_phy_tile_t *p, uint8_t intf_no, uint32_t address, uint32_t data);
diff --git a/drivers/net/ntnic/nthw/core/nthw_pca9532.c b/drivers/net/ntnic/nthw/core/nthw_pca9532.c
new file mode 100644
index 0000000000..fcf5463fcb
--- /dev/null
+++ b/drivers/net/ntnic/nthw/core/nthw_pca9532.c
@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Napatech A/S
+ */
+
+#include <pthread.h>
+
+#include "nt_util.h"
+#include "ntlog.h"
+
+#include "nthw_drv.h"
+#include "nthw_register.h"
+
+#include "nthw_pca9532.h"
+
+static const uint8_t led_sel_reg[4] = { 6, 7, 8, 9 };
+
+void nthw_pca9532_set_led_on(nthw_pca9532_t *p, uint8_t led_pos, bool state_on)
+{
+	if (led_pos >= 16) {
+		NT_LOG(ERR, NTHW, "Led pos (%u) out of range", led_pos);
+		return;
+	}
+
+	rte_spinlock_lock(&p->mp_nt_i2cm->i2cmmutex);
+	nthw_pca9849_set_channel(p->mp_ca9849, p->m_mux_channel);
+
+	uint8_t reg_addr = led_sel_reg[led_pos / 4];
+	uint8_t bit_pos = (uint8_t)((led_pos % 4) * 2);
+	uint8_t data = 0U;
+
+	nthw_i2cm_read(p->mp_nt_i2cm, p->m_dev_address, reg_addr, &data);
+	data = data & (uint8_t)(~(0b11 << bit_pos));	/* Set bits to "00" aka off */
+
+	if (state_on)
+		data = (uint8_t)(data | (0b01 << bit_pos));
+
+	nthw_i2cm_write(p->mp_nt_i2cm, p->m_dev_address, reg_addr, data);
+	rte_spinlock_unlock(&p->mp_nt_i2cm->i2cmmutex);
+}
diff --git a/drivers/net/ntnic/nthw/core/nthw_phy_tile.c b/drivers/net/ntnic/nthw/core/nthw_phy_tile.c
index 42e2782f99..b057e9e88c 100644
--- a/drivers/net/ntnic/nthw/core/nthw_phy_tile.c
+++ b/drivers/net/ntnic/nthw/core/nthw_phy_tile.c
@@ -263,7 +263,34 @@ void nthw_phy_tile_set_tx_equalization(nthw_phy_tile_t *p, uint8_t intf_no, uint
 	nthw_phy_tile_write_xcvr(p, intf_no, lane, tx_eq_addr + 0x8000U * lane, data);
 }
 
-static bool nthw_phy_tile_read_fec_enabled_by_scratch(nthw_phy_tile_t *p, uint8_t intf_no)
+void nthw_phy_tile_get_link_summary(nthw_phy_tile_t *p, uint32_t *p_nt_phy_link_state,
+	uint32_t *p_ll_nt_phy_link_state, uint32_t *p_lh_local_fault,
+	uint32_t *p_lh_remote_fault, uint8_t index)
+{
+	nthw_register_update(p->mp_reg_link_summary[index]);
+
+	if (p_nt_phy_link_state) {
+		*p_nt_phy_link_state =
+			nthw_field_get_val32(p->mp_fld_link_summary_nt_phy_link_state[index]);
+	}
+
+	if (p_ll_nt_phy_link_state) {
+		*p_ll_nt_phy_link_state =
+			nthw_field_get_val32(p->mp_fld_link_summary_ll_nt_phy_link_state[index]);
+	}
+
+	if (p_lh_local_fault) {
+		*p_lh_local_fault =
+			nthw_field_get_val32(p->mp_fld_link_summary_lh_received_local_fault[index]);
+	}
+
+	if (p_lh_remote_fault) {
+		*p_lh_remote_fault =
+			nthw_field_get_val32(p->mp_fld_link_summary_lh_remote_fault[index]);
+	}
+}
+
+bool nthw_phy_tile_read_fec_enabled_by_scratch(nthw_phy_tile_t *p, uint8_t intf_no)
 {
 	bool fec_enabled = false;
 
@@ -292,6 +319,28 @@ static void nthw_phy_tile_write_fec_enabled_by_scratch(nthw_phy_tile_t *p, uint8
 	nthw_field_set_val_flush32(p->mp_fld_scratch_data, val);
 }
 
+bool nthw_phy_tile_get_rx_hi_ber(nthw_phy_tile_t *p, uint8_t intf_no)
+{
+	return nthw_field_get_updated(p->mp_fld_port_status_rx_hi_ber[intf_no]);
+}
+
+bool nthw_phy_tile_get_rx_am_lock(nthw_phy_tile_t *p, uint8_t intf_no)
+{
+	return nthw_field_get_updated(p->mp_fld_port_status_rx_am_lock[intf_no]);
+}
+
+void nthw_phy_tile_set_timestamp_comp_rx(nthw_phy_tile_t *p, uint8_t intf_no, uint32_t value)
+{
+	nthw_field_get_updated(p->mp_fld_port_comp_rx_compensation[intf_no]);
+	nthw_field_set_val_flush32(p->mp_fld_port_comp_rx_compensation[intf_no], value);
+}
+
+uint32_t nthw_phy_tile_get_timestamp_comp_rx(nthw_phy_tile_t *p, uint8_t intf_no)
+{
+	nthw_field_get_updated(p->mp_fld_port_comp_rx_compensation[intf_no]);
+	return nthw_field_get_val32(p->mp_fld_port_comp_rx_compensation[intf_no]);
+}
+
 uint32_t nthw_phy_tile_read_eth(nthw_phy_tile_t *p, uint8_t intf_no, uint32_t address)
 {
 	nthw_register_update(p->mp_reg_port_eth_base[intf_no]);
diff --git a/drivers/net/ntnic/nthw/nthw_drv.h b/drivers/net/ntnic/nthw/nthw_drv.h
index 1fc8df52ce..a3c54846f5 100644
--- a/drivers/net/ntnic/nthw/nthw_drv.h
+++ b/drivers/net/ntnic/nthw/nthw_drv.h
@@ -11,6 +11,7 @@
 
 #include "nthw_si5332_si5156.h"
 #include "nthw_pcal6416a.h"
+#include "nthw_pca9532.h"
 #include "nthw_phy_tile.h"
 #include "nthw_rpf.h"
 #include "nthw_phy_tile.h"
@@ -22,6 +23,7 @@ typedef struct nthw_agx_s {
 	nthw_i2cm_t *p_i2cm;
 	nthw_pca9849_t *p_pca9849;
 	nthw_pcal6416a_t *p_io_nim;	/* PCAL6416A I/O expander for controlling TS */
+	nthw_pca9532_t *p_pca9532_led;
 	nthw_phy_tile_t *p_phy_tile;
 	nthw_rpf_t *p_rpf;
 } nthw_agx_t;
-- 
2.45.0