[PATCH net-next 1/2] mlx4: update mlx4_clock_read() to provide pre/post tstamps

[PATCH net-next 1/2] mlx4: update mlx4_clock_read() to provide pre/post tstamps

[Mahesh Bandewar]

The mlx4_clock_read() function, when called by cycle_counter->read(),
previously only returned the raw cycle count. However, for PTP helpers
like gettimex64(), which require pre- and post-timestamps, simply
returning raw cycles is insufficient. It also needs to provide the
necessary timestamps.

This update modifies mlx4_clock_read() to return both the cycles and
the required timestamps. Additionally, mlx4_en_read_clock() is now
responsible for reading and updating the clock_cache. This allows
another function, mlx4_en_read_clock_cache(), to act as the cycle
reader for cycle_counter->read(), preserving the same interface.

Signed-off-by: Mahesh Bandewar <maheshb@google.com>
---
 drivers/net/ethernet/mellanox/mlx4/en_clock.c | 29 +++++++++++++++----
 drivers/net/ethernet/mellanox/mlx4/main.c     | 12 ++++++--
 drivers/net/ethernet/mellanox/mlx4/mlx4_en.h  |  1 +
 include/linux/mlx4/device.h                   |  3 +-
 4 files changed, 36 insertions(+), 9 deletions(-)

diff --git a/drivers/net/ethernet/mellanox/mlx4/en_clock.c b/drivers/net/ethernet/mellanox/mlx4/en_clock.c
index cd754cd76bde..69c5e4c5e036 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_clock.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_clock.c
@@ -36,15 +36,23 @@
 
 #include "mlx4_en.h"
 
-/* mlx4_en_read_clock - read raw cycle counter (to be used by time counter)
+/* mlx4_en_read_clock_cache - read cached raw cycle counter (to be
+ * used by time counter)
  */
-static u64 mlx4_en_read_clock(const struct cyclecounter *tc)
+static u64 mlx4_en_read_clock_cache(const struct cyclecounter *tc)
 {
 	struct mlx4_en_dev *mdev =
 		container_of(tc, struct mlx4_en_dev, cycles);
-	struct mlx4_dev *dev = mdev->dev;
 
-	return mlx4_read_clock(dev) & tc->mask;
+	return READ_ONCE(mdev->clock_cache) & tc->mask;
+}
+
+static void mlx4_en_read_clock(struct mlx4_en_dev *mdev,
+			       struct ptp_system_timestamp *sts)
+{
+	u64 cycles = mlx4_read_clock(mdev->dev, sts);
+
+	WRITE_ONCE(mdev->clock_cache, cycles);
 }
 
 u64 mlx4_en_get_cqe_ts(struct mlx4_cqe *cqe)
@@ -109,6 +117,9 @@ void mlx4_en_ptp_overflow_check(struct mlx4_en_dev *mdev)
 
 	if (timeout) {
 		write_seqlock_irqsave(&mdev->clock_lock, flags);
+		/* refresh the clock_cache */
+		mlx4_en_read_clock(mdev, NULL);
+
 		timecounter_read(&mdev->clock);
 		write_sequnlock_irqrestore(&mdev->clock_lock, flags);
 		mdev->last_overflow_check = jiffies;
@@ -135,6 +146,8 @@ static int mlx4_en_phc_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)
 	mult = (u32)adjust_by_scaled_ppm(mdev->nominal_c_mult, scaled_ppm);
 
 	write_seqlock_irqsave(&mdev->clock_lock, flags);
+	/* refresh the clock_cache */
+	mlx4_en_read_clock(mdev, NULL);
 	timecounter_read(&mdev->clock);
 	mdev->cycles.mult = mult;
 	write_sequnlock_irqrestore(&mdev->clock_lock, flags);
@@ -179,6 +192,8 @@ static int mlx4_en_phc_gettime(struct ptp_clock_info *ptp,
 	u64 ns;
 
 	write_seqlock_irqsave(&mdev->clock_lock, flags);
+	/* refresh the clock_cache */
+	mlx4_en_read_clock(mdev, NULL);
 	ns = timecounter_read(&mdev->clock);
 	write_sequnlock_irqrestore(&mdev->clock_lock, flags);
 
@@ -205,6 +220,8 @@ static int mlx4_en_phc_settime(struct ptp_clock_info *ptp,
 
 	/* reset the timecounter */
 	write_seqlock_irqsave(&mdev->clock_lock, flags);
+	/* refresh the clock_cache */
+	mlx4_en_read_clock(mdev, NULL);
 	timecounter_init(&mdev->clock, &mdev->cycles, ns);
 	write_sequnlock_irqrestore(&mdev->clock_lock, flags);
 
@@ -273,7 +290,7 @@ void mlx4_en_init_timestamp(struct mlx4_en_dev *mdev)
 	seqlock_init(&mdev->clock_lock);
 
 	memset(&mdev->cycles, 0, sizeof(mdev->cycles));
-	mdev->cycles.read = mlx4_en_read_clock;
+	mdev->cycles.read = mlx4_en_read_clock_cache;
 	mdev->cycles.mask = CLOCKSOURCE_MASK(48);
 	mdev->cycles.shift = freq_to_shift(dev->caps.hca_core_clock);
 	mdev->cycles.mult =
@@ -281,6 +298,8 @@ void mlx4_en_init_timestamp(struct mlx4_en_dev *mdev)
 	mdev->nominal_c_mult = mdev->cycles.mult;
 
 	write_seqlock_irqsave(&mdev->clock_lock, flags);
+	/* initialize the clock_cache */
+	mlx4_en_read_clock(mdev, NULL);
 	timecounter_init(&mdev->clock, &mdev->cycles,
 			 ktime_to_ns(ktime_get_real()));
 	write_sequnlock_irqrestore(&mdev->clock_lock, flags);
diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c
index febeadfdd5a5..d9ef6006ada3 100644
--- a/drivers/net/ethernet/mellanox/mlx4/main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/main.c
@@ -43,6 +43,7 @@
 #include <linux/io-mapping.h>
 #include <linux/delay.h>
 #include <linux/etherdevice.h>
+#include <linux/ptp_clock_kernel.h>
 #include <net/devlink.h>
 
 #include <uapi/rdma/mlx4-abi.h>
@@ -1925,7 +1926,7 @@ static void unmap_bf_area(struct mlx4_dev *dev)
 		io_mapping_free(mlx4_priv(dev)->bf_mapping);
 }
 
-u64 mlx4_read_clock(struct mlx4_dev *dev)
+u64 mlx4_read_clock(struct mlx4_dev *dev, struct ptp_system_timestamp *sts)
 {
 	u32 clockhi, clocklo, clockhi1;
 	u64 cycles;
@@ -1933,7 +1934,13 @@ u64 mlx4_read_clock(struct mlx4_dev *dev)
 	struct mlx4_priv *priv = mlx4_priv(dev);
 
 	for (i = 0; i < 10; i++) {
-		clockhi = swab32(readl(priv->clock_mapping));
+		if (sts) {
+			ptp_read_system_prets(sts);
+			clockhi = swab32(readl(priv->clock_mapping));
+			ptp_read_system_postts(sts);
+		} else {
+			clockhi = swab32(readl(priv->clock_mapping));
+		}
 		clocklo = swab32(readl(priv->clock_mapping + 4));
 		clockhi1 = swab32(readl(priv->clock_mapping));
 		if (clockhi == clockhi1)
@@ -1946,7 +1953,6 @@ u64 mlx4_read_clock(struct mlx4_dev *dev)
 }
 EXPORT_SYMBOL_GPL(mlx4_read_clock);
 
-
 static int map_internal_clock(struct mlx4_dev *dev)
 {
 	struct mlx4_priv *priv = mlx4_priv(dev);
diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
index 28b70dcc652e..077b529eb01a 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
@@ -435,6 +435,7 @@ struct mlx4_en_dev {
 	unsigned long		last_overflow_check;
 	struct ptp_clock	*ptp_clock;
 	struct ptp_clock_info	ptp_clock_info;
+	u64			clock_cache;
 	struct notifier_block	netdev_nb;
 	struct notifier_block	mlx_nb;
 };
diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h
index 27f42f713c89..265accc4e606 100644
--- a/include/linux/mlx4/device.h
+++ b/include/linux/mlx4/device.h
@@ -44,6 +44,7 @@
 #include <linux/refcount.h>
 
 #include <linux/timecounter.h>
+#include <linux/ptp_clock_kernel.h>
 
 #define DEFAULT_UAR_PAGE_SHIFT  12
 
@@ -1483,7 +1484,7 @@ int mlx4_get_roce_gid_from_slave(struct mlx4_dev *dev, int port, int slave_id,
 int mlx4_FLOW_STEERING_IB_UC_QP_RANGE(struct mlx4_dev *dev, u32 min_range_qpn,
 				      u32 max_range_qpn);
 
-u64 mlx4_read_clock(struct mlx4_dev *dev);
+u64 mlx4_read_clock(struct mlx4_dev *dev, struct ptp_system_timestamp *sts);
 
 struct mlx4_active_ports {
 	DECLARE_BITMAP(ports, MLX4_MAX_PORTS);
-- 
2.47.0.rc0.187.ge670bccf7e-goog

Re: [PATCH net-next 1/2] mlx4: update mlx4_clock_read() to provide pre/post tstamps

[Paolo Abeni]

On 10/8/24 12:46, Mahesh Bandewar wrote:
> The mlx4_clock_read() function, when called by cycle_counter->read(),
> previously only returned the raw cycle count. However, for PTP helpers
> like gettimex64(), which require pre- and post-timestamps, simply
> returning raw cycles is insufficient. It also needs to provide the
> necessary timestamps.
> 
> This update modifies mlx4_clock_read() to return both the cycles and
> the required timestamps. Additionally, mlx4_en_read_clock() is now
> responsible for reading and updating the clock_cache. This allows
> another function, mlx4_en_read_clock_cache(), to act as the cycle
> reader for cycle_counter->read(), preserving the same interface.

It looks like this patch should be split in two, the first one could be 
possibly 'net' material and just fix gettimex64()/mlx4_read_clock() and 
the other one introduces the cache.

Thanks,

Paolo

Re: [PATCH net-next 1/2] mlx4: update mlx4_clock_read() to provide pre/post tstamps

[Paolo Abeni]

On 10/10/24 10:36, Paolo Abeni wrote:
> On 10/8/24 12:46, Mahesh Bandewar wrote:
>> The mlx4_clock_read() function, when called by cycle_counter->read(),
>> previously only returned the raw cycle count. However, for PTP helpers
>> like gettimex64(), which require pre- and post-timestamps, simply
>> returning raw cycles is insufficient. It also needs to provide the
>> necessary timestamps.
>>
>> This update modifies mlx4_clock_read() to return both the cycles and
>> the required timestamps. Additionally, mlx4_en_read_clock() is now
>> responsible for reading and updating the clock_cache. This allows
>> another function, mlx4_en_read_clock_cache(), to act as the cycle
>> reader for cycle_counter->read(), preserving the same interface.
> 
> It looks like this patch should be split in two, the first one could be
> possibly 'net' material and just fix gettimex64()/mlx4_read_clock() and
> the other one introduces the cache.

My bad, I was too hasty and actually missed that the gettimex64() 
callback is implemented in the next patch.

The main point still remains: the cache infra should be in a separate 
patch: it can introduce side effects and we want to be able to bisect.

Thanks,

Paolo

Re: [PATCH net-next 1/2] mlx4: update mlx4_clock_read() to provide pre/post tstamps

[Mahesh Bandewar (महेश बंडेवार)]

On Thu, Oct 10, 2024 at 1:44 AM Paolo Abeni <pabeni@redhat.com> wrote:
>
>
>
> On 10/10/24 10:36, Paolo Abeni wrote:
> > On 10/8/24 12:46, Mahesh Bandewar wrote:
> >> The mlx4_clock_read() function, when called by cycle_counter->read(),
> >> previously only returned the raw cycle count. However, for PTP helpers
> >> like gettimex64(), which require pre- and post-timestamps, simply
> >> returning raw cycles is insufficient. It also needs to provide the
> >> necessary timestamps.
> >>
> >> This update modifies mlx4_clock_read() to return both the cycles and
> >> the required timestamps. Additionally, mlx4_en_read_clock() is now
> >> responsible for reading and updating the clock_cache. This allows
> >> another function, mlx4_en_read_clock_cache(), to act as the cycle
> >> reader for cycle_counter->read(), preserving the same interface.
> >
> > It looks like this patch should be split in two, the first one could be
> > possibly 'net' material and just fix gettimex64()/mlx4_read_clock() and
> > the other one introduces the cache.
>
> My bad, I was too hasty and actually missed that the gettimex64()
> callback is implemented in the next patch.
>
> The main point still remains: the cache infra should be in a separate
> patch: it can introduce side effects and we want to be able to bisect.
>
Hi Paolo,

Thanks for the comment. I'll send the v2 with the separation where one
patch introduces just the time-cache
and the other piece adds the pre-/post-timestamps.

thanks,
> Thanks,
>
> Paolo
>