[PATCH net-next v9 2/7] net_sched: sch_fq: check device pacing offload

[Willem de Bruijn <willemdebruijn.kernel@gmail.com>]

From: Willem de Bruijn <willemb@google.com>

Network devices can modify pacing offload administratively or device
renegotiation.

In FQ, only offload if the feature is enabled and within the supported
device horizon.

Signed-off-by: Willem de Bruijn <willemb@google.com>
---
 net/sched/sch_fq.c | 53 +++++++++++++++++++++++++++++++++++-----------
 1 file changed, 41 insertions(+), 12 deletions(-)

diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c
index 6144b5686f13..fdd9380e445c 100644
--- a/net/sched/sch_fq.c
+++ b/net/sched/sch_fq.c
@@ -301,6 +301,26 @@ static void fq_gc(struct fq_sched_data *q,
 	q->stat_gc_flows += fcnt;
 }
 
+static u64 fq_offload_horizon(const struct Qdisc *sch,
+			      const struct fq_sched_data *q)
+{
+	const struct net_device *dev;
+	u64 offload_horizon;
+
+	offload_horizon = READ_ONCE(q->offload_horizon);
+	if (!offload_horizon)
+		return 0;
+
+	dev = qdisc_dev(sch);
+	if (!dev->pacing_offload)
+		return 0;
+
+	if (offload_horizon > READ_ONCE(dev->max_pacing_offload_horizon))
+		return 0;
+
+	return offload_horizon;
+}
+
 /* Fast path can be used if :
  * 1) Packet tstamp is in the past, or within the pacing offload horizon.
  * 2) FQ qlen == 0   OR
@@ -312,12 +332,12 @@ static void fq_gc(struct fq_sched_data *q,
  * FQ can not use generic TCQ_F_CAN_BYPASS infrastructure.
  */
 static bool fq_fastpath_check(const struct Qdisc *sch, struct sk_buff *skb,
-			      u64 now)
+			      u64 now, u64 offload_horizon)
 {
 	const struct fq_sched_data *q = qdisc_priv(sch);
 	const struct sock *sk;
 
-	if (fq_skb_cb(skb)->time_to_send > now + q->offload_horizon)
+	if (fq_skb_cb(skb)->time_to_send > now + offload_horizon)
 		return false;
 
 	if (sch->q.qlen != 0) {
@@ -338,7 +358,7 @@ static bool fq_fastpath_check(const struct Qdisc *sch, struct sk_buff *skb,
 		/* Ordering invariants fall apart if some delayed flows
 		 * are ready but we haven't serviced them, yet.
 		 */
-		if (q->time_next_delayed_flow <= now + q->offload_horizon)
+		if (q->time_next_delayed_flow <= now + offload_horizon)
 			return false;
 	}
 
@@ -357,6 +377,7 @@ static struct fq_flow *fq_classify(struct Qdisc *sch, struct sk_buff *skb,
 				   u64 now)
 {
 	struct fq_sched_data *q = qdisc_priv(sch);
+	u64 offload_horizon = fq_offload_horizon(sch, q);
 	struct rb_node **p, *parent;
 	struct sock *sk = skb->sk;
 	struct rb_root *root;
@@ -393,12 +414,13 @@ static struct fq_flow *fq_classify(struct Qdisc *sch, struct sk_buff *skb,
 		sk = (struct sock *)((hash << 1) | 1UL);
 	}
 
-	if (fq_fastpath_check(sch, skb, now)) {
+	if (fq_fastpath_check(sch, skb, now, offload_horizon)) {
 		q->internal.stat_fastpath_packets++;
 		if (skb->sk == sk && q->rate_enable &&
 		    READ_ONCE(sk->sk_pacing_status) != SK_PACING_FQ)
 			smp_store_release(&sk->sk_pacing_status,
 					  SK_PACING_FQ);
+
 		return &q->internal;
 	}
 
@@ -661,12 +683,13 @@ static int fq_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 	return NET_XMIT_SUCCESS;
 }
 
-static void fq_check_throttled(struct fq_sched_data *q, u64 now)
+static void fq_check_throttled(struct fq_sched_data *q, u64 now,
+			       u64 offload_horizon)
 {
 	unsigned long sample;
 	struct rb_node *p;
 
-	if (q->time_next_delayed_flow > now + q->offload_horizon)
+	if (q->time_next_delayed_flow > now + offload_horizon)
 		return;
 
 	/* Update unthrottle latency EWMA.
@@ -677,7 +700,7 @@ static void fq_check_throttled(struct fq_sched_data *q, u64 now)
 		q->unthrottle_latency_ns -= q->unthrottle_latency_ns >> 3;
 		q->unthrottle_latency_ns += sample >> 3;
 	}
-	now += q->offload_horizon;
+	now += offload_horizon;
 
 	q->time_next_delayed_flow = ~0ULL;
 	while ((p = rb_first(&q->delayed)) != NULL) {
@@ -705,6 +728,7 @@ static struct fq_flow_head *fq_pband_head_select(struct fq_perband_flows *pband)
 static struct sk_buff *fq_dequeue(struct Qdisc *sch)
 {
 	struct fq_sched_data *q = qdisc_priv(sch);
+	u64 offload_horizon = fq_offload_horizon(sch, q);
 	struct fq_perband_flows *pband;
 	struct fq_flow_head *head;
 	struct sk_buff *skb;
@@ -725,7 +749,7 @@ static struct sk_buff *fq_dequeue(struct Qdisc *sch)
 	}
 
 	now = ktime_get_ns();
-	fq_check_throttled(q, now);
+	fq_check_throttled(q, now, offload_horizon);
 	retry = 0;
 	pband = &q->band_flows[q->band_nr];
 begin:
@@ -761,7 +785,7 @@ static struct sk_buff *fq_dequeue(struct Qdisc *sch)
 		u64 time_next_packet = max_t(u64, fq_skb_cb(skb)->time_to_send,
 					     f->time_next_packet);
 
-		if (now + q->offload_horizon < time_next_packet) {
+		if (now + offload_horizon < time_next_packet) {
 			head->first = f->next;
 			f->time_next_packet = time_next_packet;
 			fq_flow_set_throttled(q, f);
@@ -836,6 +860,7 @@ static struct sk_buff *fq_dequeue(struct Qdisc *sch)
 		}
 		f->time_next_packet = now + len;
 	}
+
 out:
 	return skb;
 }
@@ -1183,11 +1208,15 @@ static int fq_change(struct Qdisc *sch, struct nlattr *opt,
 		u64 offload_horizon = (u64)NSEC_PER_USEC *
 				      nla_get_u32(tb[TCA_FQ_OFFLOAD_HORIZON]);
 
-		if (offload_horizon <= qdisc_dev(sch)->max_pacing_offload_horizon) {
-			WRITE_ONCE(q->offload_horizon, offload_horizon);
-		} else {
+		if (offload_horizon && !qdisc_dev(sch)->pacing_offload) {
+			NL_SET_ERR_MSG_MOD(extack, "device pacing offload is disabled");
+			err = -EINVAL;
+		} else if (offload_horizon >
+			   qdisc_dev(sch)->max_pacing_offload_horizon) {
 			NL_SET_ERR_MSG_MOD(extack, "invalid offload_horizon");
 			err = -EINVAL;
+		} else {
+			WRITE_ONCE(q->offload_horizon, offload_horizon);
 		}
 	}
 	if (!err) {
-- 
2.55.0.1007.g17ff1f9808-goog