[PATCH bpf-next] bpf: add sock_ops callbacks for data send/recv/acked events

[Philo Lu <lulie@linux.alibaba.com>]

Add 3 sock_ops operators, namely BPF_SOCK_OPS_DATA_SEND_CB,
BPF_SOCK_OPS_DATA_RECV_CB, and BPF_SOCK_OPS_DATA_ACKED_CB. A flag
BPF_SOCK_OPS_DATA_EVENT_CB_FLAG is provided to minimize the performance
impact. The flag must be explicitly set to enable these callbacks.

If the flag is enabled, bpf sock_ops program will be called every time a
tcp data packet is sent, received, and acked.
BPF_SOCK_OPS_DATA_SEND_CB: call bpf after a data packet is sent.
BPF_SOCK_OPS_DATA_RECV_CB: call bpf after a data packet is receviced.
BPF_SOCK_OPS_DATA_ACKED_CB: call bpf after a valid ack packet is
processed (some sent data are ackknowledged).

We use these callbacks for fine-grained tcp monitoring, which collects
and analyses every tcp request/response event information. The whole
system has been described in SIGMOD'18 (see
https://dl.acm.org/doi/pdf/10.1145/3183713.3190659 for details). To
achieve this with bpf, we require hooks for data events that call
sock_ops bpf (1) when any data packet is sent/received/acked, and (2)
after critical tcp state variables have been updated (e.g., snd_una,
snd_nxt, rcv_nxt). However, existing sock_ops operators cannot meet our
requirements.

Besides, these hooks also help to debug tcp when data send/recv/acked.

Signed-off-by: Philo Lu <lulie@linux.alibaba.com>
---
 include/net/tcp.h        |  9 +++++++++
 include/uapi/linux/bpf.h | 14 +++++++++++++-
 net/ipv4/tcp_input.c     |  4 ++++
 net/ipv4/tcp_output.c    |  2 ++
 4 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/include/net/tcp.h b/include/net/tcp.h
index d2f0736b76b8..73eda03fdda5 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -2660,6 +2660,15 @@ static inline void tcp_bpf_rtt(struct sock *sk)
 		tcp_call_bpf(sk, BPF_SOCK_OPS_RTT_CB, 0, NULL);
 }
 
+/* op must be one of BPF_SOCK_OPS_DATA_SEND_CB, BPF_SOCK_OPS_DATA_RECV_CB,
+ * or BPF_SOCK_OPS_DATA_ACKED_CB.
+ */
+static inline void tcp_bpf_data_event(struct sock *sk, int op)
+{
+	if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_DATA_EVENT_CB_FLAG))
+		tcp_call_bpf(sk, op, 0, NULL);
+}
+
 #if IS_ENABLED(CONFIG_SMC)
 extern struct static_key_false tcp_have_smc;
 #endif
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 7cf8bcf9f6a2..2154a6235901 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -3016,6 +3016,7 @@ union bpf_attr {
  * 		* **BPF_SOCK_OPS_RETRANS_CB_FLAG** (retransmission)
  * 		* **BPF_SOCK_OPS_STATE_CB_FLAG** (TCP state change)
  * 		* **BPF_SOCK_OPS_RTT_CB_FLAG** (every RTT)
+ * 		* **BPF_SOCK_OPS_DATA_EVENT_CB_FLAG** (data packet send/recv/acked)
  *
  * 		Therefore, this function can be used to clear a callback flag by
  * 		setting the appropriate bit to zero. e.g. to disable the RTO
@@ -6755,8 +6756,10 @@ enum {
 	 * options first before the BPF program does.
 	 */
 	BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG = (1<<6),
+	/* Call bpf when data send/recv/acked. */
+	BPF_SOCK_OPS_DATA_EVENT_CB_FLAG = (1<<7),
 /* Mask of all currently supported cb flags */
-	BPF_SOCK_OPS_ALL_CB_FLAGS       = 0x7F,
+	BPF_SOCK_OPS_ALL_CB_FLAGS       = 0xFF,
 };
 
 /* List of known BPF sock_ops operators.
@@ -6869,6 +6872,15 @@ enum {
 					 * by the kernel or the
 					 * earlier bpf-progs.
 					 */
+	BPF_SOCK_OPS_DATA_SEND_CB,		/* Calls BPF program when a
+					 * data packet is sent. Pure ack is ignored.
+					 */
+	BPF_SOCK_OPS_DATA_RECV_CB,		/* Calls BPF program when a
+					 * data packet is received. Pure ack is ignored.
+					 */
+	BPF_SOCK_OPS_DATA_ACKED_CB,		/* Calls BPF program when sent
+					 * data are acknowledged.
+					 */
 };
 
 /* List of TCP states. There is a build check in net/ipv4/tcp.c to detect
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index bcb55d98004c..72c6192e7cd0 100644
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -824,6 +824,8 @@ static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb)
 
 	now = tcp_jiffies32;
 
+	tcp_bpf_data_event(sk, BPF_SOCK_OPS_DATA_RECV_CB);
+
 	if (!icsk->icsk_ack.ato) {
 		/* The _first_ data packet received, initialize
 		 * delayed ACK engine.
@@ -3454,6 +3456,8 @@ static int tcp_clean_rtx_queue(struct sock *sk, const struct sk_buff *ack_skb,
 		flag |= FLAG_SET_XMIT_TIMER;  /* set TLP or RTO timer */
 	}
 
+	tcp_bpf_data_event(sk, BPF_SOCK_OPS_DATA_ACKED_CB);
+
 	if (icsk->icsk_ca_ops->pkts_acked) {
 		struct ack_sample sample = { .pkts_acked = pkts_acked,
 					     .rtt_us = sack->rate->rtt_us };
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index eb13a55d660c..ddd6a9c2150f 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -2821,6 +2821,8 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
 		/* Send one loss probe per tail loss episode. */
 		if (push_one != 2)
 			tcp_schedule_loss_probe(sk, false);
+
+		tcp_bpf_data_event(sk, BPF_SOCK_OPS_DATA_SEND_CB);
 		return false;
 	}
 	return !tp->packets_out && !tcp_write_queue_empty(sk);
-- 
2.32.0.3.g01195cf9f