[RFC PATCH 10/11] ptq: Hook up receive side of Per Queue Threads

[Tom Herbert <tom@herbertland.com>]

Add code to set the queue in an rflow as opposed to just setting the
CPU in an rps_dev_flow entry. set_rps_qid is the analogue for
set_rps_cpu but for setting queues. In get_rps_cpu, a check is
performed that identifier in the sock_flow_table refers to a queue;
when it does call set_rps_qid after converting the global qid in the
sock_flow_table to a device qid.

In rps_record_sock_flow check is there is a per task receive queue
for current (i.e. current->ptq_queues.rxq_id != NO_QUEUE). If there
is a queue then set in sock_flow_table instead of setting the running
CPU. Subsequently, the receive queue for the flow can be programmed
by aRFS logic (ndo_rx_flow_steer).
---
 include/linux/netdevice.h | 28 ++++++++++++++++++++++++----
 net/core/dev.c            | 36 ++++++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+), 4 deletions(-)

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index ca163925211a..3b39be470720 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -731,12 +731,25 @@ static inline void rps_dev_flow_set_cpu(struct rps_dev_flow *dev_flow, u16 cpu)
 	if (WARN_ON(cpu > RPS_MAX_CPU))
 		return;
 
-	/* Set the rflow target to the CPU atomically */
+	/* Set the device flow target to the CPU atomically */
 	cpu_qid.use_qid = 0;
 	cpu_qid.cpu = cpu;
 	dev_flow->cpu_qid = cpu_qid;
 }
 
+static inline void rps_dev_flow_set_qid(struct rps_dev_flow *dev_flow, u16 qid)
+{
+	struct rps_cpu_qid cpu_qid;
+
+	if (WARN_ON(qid > RPS_MAX_QID))
+		return;
+
+	/* Set the device flow target to the CPU atomically */
+	cpu_qid.use_qid = 1;
+	cpu_qid.qid = qid;
+	dev_flow->cpu_qid = cpu_qid;
+}
+
 /*
  * The rps_dev_flow_table structure contains a table of flow mappings.
  */
@@ -797,11 +810,18 @@ static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
 					u32 hash)
 {
 	if (table && hash) {
-		u32 val = hash & table->cpu_masks.hash_mask;
 		unsigned int index = hash & table->mask;
+		u32 val;
 
-		/* We only give a hint, preemption can change CPU under us */
-		val |= raw_smp_processor_id();
+#ifdef CONFIG_PER_THREAD_QUEUES
+		if (current->ptq_queues.rxq_id != NO_QUEUE)
+			val = RPS_SOCK_FLOW_USE_QID |
+			      (hash & table->queue_masks.hash_mask) |
+			      current->ptq_queues.rxq_id;
+		else
+#endif
+			val = (hash & table->cpu_masks.hash_mask) |
+			      raw_smp_processor_id();
 
 		if (table->ents[index] != val)
 			table->ents[index] = val;
diff --git a/net/core/dev.c b/net/core/dev.c
index f4478c9b1c9c..1cad776e8847 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4308,6 +4308,25 @@ set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
 	return rflow;
 }
 
+static struct rps_dev_flow *
+set_rps_qid(struct net_device *dev, struct sk_buff *skb,
+	    struct rps_dev_flow *rflow, u16 qid)
+{
+	if (qid > RPS_MAX_QID) {
+		rps_dev_flow_clear(rflow);
+		return rflow;
+	}
+
+#ifdef CONFIG_RFS_ACCEL
+	/* Should we steer this flow to a different hardware queue? */
+	if (skb_rx_queue_recorded(skb) && (dev->features & NETIF_F_NTUPLE) &&
+	    qid != skb_get_rx_queue(skb) && qid < dev->real_num_rx_queues)
+		set_arfs_queue(dev, skb, rflow, qid);
+#endif
+	rps_dev_flow_set_qid(rflow, qid);
+	return rflow;
+}
+
 /*
  * get_rps_cpu is called from netif_receive_skb and returns the target
  * CPU from the RPS map of the receiving queue for a given skb.
@@ -4356,6 +4375,10 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
 
 		/* First check into global flow table if there is a match */
 		ident = sock_flow_table->ents[hash & sock_flow_table->mask];
+
+		if (ident == RPS_SOCK_FLOW_NO_IDENT)
+			goto try_rps;
+
 		comparator = ((ident & RPS_SOCK_FLOW_USE_QID) ?
 				sock_flow_table->queue_masks.hash_mask :
 				sock_flow_table->cpu_masks.hash_mask);
@@ -4372,8 +4395,21 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
 		 * CPU. Proceed accordingly.
 		 */
 		if (ident & RPS_SOCK_FLOW_USE_QID) {
+			u16 dqid, gqid;
+
 			/* A queue identifier is in the sock_flow_table entry */
 
+			gqid = ident & sock_flow_table->queue_masks.mask;
+			dqid = netdev_rx_gqid_to_dqid(dev, gqid);
+
+			/* rflow has desired receive qid. Just set the qid in
+			 * HW and return to use current CPU. Note that we
+			 * don't consider OOO in this case.
+			 */
+			rflow = set_rps_qid(dev, skb, rflow, dqid);
+
+			*rflowp = rflow;
+
 			/* Don't use aRFS to set CPU in this case, skip to
 			 * trying RPS
 			 */
-- 
2.25.1