All of lore.kernel.org
 help / color / mirror / Atom feed
From: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
To: linux-kernel@vger.kernel.org
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
	stable@vger.kernel.org, Eric Dumazet <edumazet@googl.com>,
	liujian <liujian56@huawei.com>,
	Neal Cardwell <ncardwell@google.com>,
	Yuchung Cheng <ycheng@google.com>,
	Soheil Hassas Yeganeh <soheil@google.com>,
	"David S. Miller" <davem@davemloft.net>
Subject: [PATCH 4.9 049/105] tcp: update skb->skb_mstamp more carefully
Date: Tue, 10 Oct 2017 21:50:20 +0200	[thread overview]
Message-ID: <20171010192541.463529188@linuxfoundation.org> (raw)
In-Reply-To: <20171010192533.717049376@linuxfoundation.org>

4.9-stable review patch.  If anyone has any objections, please let me know.

------------------

From: Eric Dumazet <edumazet@googl.com>


[ Upstream commit 8c72c65b426b47b3c166a8fef0d8927fe5e8a28d ]

liujian reported a problem in TCP_USER_TIMEOUT processing with a patch
in tcp_probe_timer() :
      https://www.spinics.net/lists/netdev/msg454496.html

After investigations, the root cause of the problem is that we update
skb->skb_mstamp of skbs in write queue, even if the attempt to send a
clone or copy of it failed. One reason being a routing problem.

This patch prevents this, solving liujian issue.

It also removes a potential RTT miscalculation, since
__tcp_retransmit_skb() is not OR-ing TCP_SKB_CB(skb)->sacked with
TCPCB_EVER_RETRANS if a failure happens, but skb->skb_mstamp has
been changed.

A future ACK would then lead to a very small RTT sample and min_rtt
would then be lowered to this too small value.

Tested:

# cat user_timeout.pkt
--local_ip=192.168.102.64

    0 socket(..., SOCK_STREAM, IPPROTO_TCP) = 3
   +0 setsockopt(3, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
   +0 bind(3, ..., ...) = 0
   +0 listen(3, 1) = 0

   +0 `ifconfig tun0 192.168.102.64/16; ip ro add 192.0.2.1 dev tun0`

   +0 < S 0:0(0) win 0 <mss 1460>
   +0 > S. 0:0(0) ack 1 <mss 1460>

  +.1 < . 1:1(0) ack 1 win 65530
   +0 accept(3, ..., ...) = 4

   +0 setsockopt(4, SOL_TCP, TCP_USER_TIMEOUT, [3000], 4) = 0
   +0 write(4, ..., 24) = 24
   +0 > P. 1:25(24) ack 1 win 29200
   +.1 < . 1:1(0) ack 25 win 65530

//change the ipaddress
   +1 `ifconfig tun0 192.168.0.10/16`

   +1 write(4, ..., 24) = 24
   +1 write(4, ..., 24) = 24
   +1 write(4, ..., 24) = 24
   +1 write(4, ..., 24) = 24

   +0 `ifconfig tun0 192.168.102.64/16`
   +0 < . 1:2(1) ack 25 win 65530
   +0 `ifconfig tun0 192.168.0.10/16`

   +3 write(4, ..., 24) = -1

# ./packetdrill user_timeout.pkt

Signed-off-by: Eric Dumazet <edumazet@googl.com>
Reported-by: liujian <liujian56@huawei.com>
Acked-by: Neal Cardwell <ncardwell@google.com>
Acked-by: Yuchung Cheng <ycheng@google.com>
Acked-by: Soheil Hassas Yeganeh <soheil@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
---
 net/ipv4/tcp_output.c |   19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -914,6 +914,7 @@ static int tcp_transmit_skb(struct sock
 	struct tcp_skb_cb *tcb;
 	struct tcp_out_options opts;
 	unsigned int tcp_options_size, tcp_header_size;
+	struct sk_buff *oskb = NULL;
 	struct tcp_md5sig_key *md5;
 	struct tcphdr *th;
 	int err;
@@ -922,11 +923,11 @@ static int tcp_transmit_skb(struct sock
 	tp = tcp_sk(sk);
 
 	if (clone_it) {
-		skb_mstamp_get(&skb->skb_mstamp);
 		TCP_SKB_CB(skb)->tx.in_flight = TCP_SKB_CB(skb)->end_seq
 			- tp->snd_una;
 		tcp_rate_skb_sent(sk, skb);
 
+		oskb = skb;
 		if (unlikely(skb_cloned(skb)))
 			skb = pskb_copy(skb, gfp_mask);
 		else
@@ -934,6 +935,7 @@ static int tcp_transmit_skb(struct sock
 		if (unlikely(!skb))
 			return -ENOBUFS;
 	}
+	skb_mstamp_get(&skb->skb_mstamp);
 
 	inet = inet_sk(sk);
 	tcb = TCP_SKB_CB(skb);
@@ -1035,12 +1037,14 @@ static int tcp_transmit_skb(struct sock
 
 	err = icsk->icsk_af_ops->queue_xmit(sk, skb, &inet->cork.fl);
 
-	if (likely(err <= 0))
-		return err;
-
-	tcp_enter_cwr(sk);
+	if (unlikely(err > 0)) {
+		tcp_enter_cwr(sk);
+		err = net_xmit_eval(err);
+	}
+	if (!err && oskb)
+		skb_mstamp_get(&oskb->skb_mstamp);
 
-	return net_xmit_eval(err);
+	return err;
 }
 
 /* This routine just queues the buffer for sending.
@@ -2709,10 +2713,11 @@ int __tcp_retransmit_skb(struct sock *sk
 		     skb_headroom(skb) >= 0xFFFF)) {
 		struct sk_buff *nskb;
 
-		skb_mstamp_get(&skb->skb_mstamp);
 		nskb = __pskb_copy(skb, MAX_TCP_HEADER, GFP_ATOMIC);
 		err = nskb ? tcp_transmit_skb(sk, nskb, 0, GFP_ATOMIC) :
 			     -ENOBUFS;
+		if (!err)
+			skb_mstamp_get(&skb->skb_mstamp);
 	} else {
 		err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
 	}

  parent reply	other threads:[~2017-10-10 20:55 UTC|newest]

Thread overview: 121+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2017-10-10 19:49 [PATCH 4.9 000/105] 4.9.55-stable review Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 001/105] USB: gadgetfs: Fix crash caused by inadequate synchronization Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 002/105] USB: gadgetfs: fix copy_to_user while holding spinlock Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 003/105] usb: gadget: udc: atmel: set vbus irqflags explicitly Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 004/105] usb: gadget: udc: renesas_usb3: fix for no-data control transfer Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 005/105] usb: gadget: udc: renesas_usb3: fix Pn_RAMMAP.Pn_MPKT value Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 006/105] usb: gadget: udc: renesas_usb3: Fix return value of usb3_write_pipe() Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 007/105] usb-storage: unusual_devs entry to fix write-access regression for Seagate external drives Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 008/105] usb-storage: fix bogus hardware error messages for ATA pass-thru devices Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 009/105] usb: renesas_usbhs: fix the BCLR setting condition for non-DCP pipe Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 010/105] usb: renesas_usbhs: fix usbhsf_fifo_clear() for RX direction Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 011/105] ALSA: usb-audio: Check out-of-bounds access by corrupted buffer descriptor Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 012/105] usb: pci-quirks.c: Corrected timeout values used in handshake Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 014/105] USB: dummy-hcd: fix connection failures (wrong speed) Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 015/105] USB: dummy-hcd: fix infinite-loop resubmission bug Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 016/105] USB: dummy-hcd: Fix erroneous synchronization change Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 017/105] USB: devio: Dont corrupt user memory Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 018/105] usb: gadget: mass_storage: set msg_registered after msg registered Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 019/105] USB: g_mass_storage: Fix deadlock when driver is unbound Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 020/105] USB: uas: fix bug in handling of alternate settings Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 021/105] USB: core: harden cdc_parse_cdc_header Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 022/105] usb: Increase quirk delay for USB devices Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 023/105] USB: fix out-of-bounds in usb_set_configuration Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 024/105] xhci: fix finding correct bus_state structure for USB 3.1 hosts Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 025/105] xhci: Fix sleeping with spin_lock_irq() held in ASmedia 1042A workaround Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 026/105] xhci: set missing SuperSpeedPlus Link Protocol bit in roothub descriptor Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 027/105] Revert "xhci: Limit USB2 port wake support for AMD Promontory hosts" Greg Kroah-Hartman
2017-10-10 19:49 ` [PATCH 4.9 028/105] iio: adc: twl4030: Fix an error handling path in twl4030_madc_probe() Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 029/105] iio: adc: twl4030: Disable the vusb3v1 rugulator in the error handling path of twl4030_madc_probe() Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 030/105] iio: ad_sigma_delta: Implement a dedicated reset function Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 031/105] staging: iio: ad7192: Fix - use the dedicated reset function avoiding dma from stack Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 032/105] iio: core: Return error for failed read_reg Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 033/105] IIO: BME280: Updates to Humidity readings need ctrl_reg write! Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 034/105] iio: ad7793: Fix the serial interface reset Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 035/105] iio: adc: mcp320x: Fix readout of negative voltages Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 036/105] iio: adc: mcp320x: Fix oops on module unload Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 037/105] uwb: properly check kthread_run return value Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 038/105] uwb: ensure that endpoint is interrupt Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 039/105] staging: vchiq_2835_arm: Fix NULL ptr dereference in free_pagelist Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 040/105] mm, oom_reaper: skip mm structs with mmu notifiers Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 041/105] lib/ratelimit.c: use deferred printk() version Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 042/105] lsm: fix smack_inode_removexattr and xattr_getsecurity memleak Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 043/105] ALSA: compress: Remove unused variable Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 044/105] Revert "ALSA: echoaudio: purge contradictions between dimension matrix members and total number of members" Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 045/105] ALSA: usx2y: Suppress kernel warning at page allocation failures Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 046/105] mlxsw: spectrum: Prevent mirred-related crash on removal Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 047/105] net: sched: fix use-after-free in tcf_action_destroy and tcf_del_walker Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 048/105] sctp: potential read out of bounds in sctp_ulpevent_type_enabled() Greg Kroah-Hartman
2017-10-10 19:50 ` Greg Kroah-Hartman [this message]
2017-10-10 19:50 ` [PATCH 4.9 050/105] bpf/verifier: reject BPF_ALU64|BPF_END Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 051/105] tcp: fix data delivery rate Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 052/105] udpv6: Fix the checksum computation when HW checksum does not apply Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 053/105] ip6_gre: skb_push ipv6hdr before packing the header in ip6gre_header Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 054/105] net: phy: Fix mask value write on gmii2rgmii converter speed register Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 055/105] ip6_tunnel: do not allow loading ip6_tunnel if ipv6 is disabled in cmdline Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 056/105] net/sched: cls_matchall: fix crash when used with classful qdisc Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 057/105] tcp: fastopen: fix on syn-data transmit failure Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 058/105] net: emac: Fix napi poll list corruption Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 059/105] packet: hold bind lock when rebinding to fanout hook Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 060/105] bpf: one perf event close wont free bpf program attached by another perf event Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 061/105] isdn/i4l: fetch the ppp_write buffer in one shot Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 062/105] net_sched: always reset qdisc backlog in qdisc_reset() Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 063/105] net: qcom/emac: specify the correct size when mapping a DMA buffer Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 064/105] vti: fix use after free in vti_tunnel_xmit/vti6_tnl_xmit Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 065/105] l2tp: Avoid schedule while atomic in exit_net Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 066/105] l2tp: fix race condition in l2tp_tunnel_delete Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 067/105] tun: bail out from tun_get_user() if the skb is empty Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 068/105] net: dsa: Fix network device registration order Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 069/105] packet: in packet_do_bind, test fanout with bind_lock held Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 070/105] packet: only test po->has_vnet_hdr once in packet_snd Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 071/105] net: Set sk_prot_creator when cloning sockets to the right proto Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 072/105] netlink: do not proceed if dumps start() errs Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 073/105] ip6_gre: ip6gre_tap device should keep dst Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 074/105] ip6_tunnel: update mtu properly for ARPHRD_ETHER tunnel device in tx path Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 075/105] tipc: use only positive error codes in messages Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 076/105] net: rtnetlink: fix info leak in RTM_GETSTATS call Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 077/105] socket, bpf: fix possible use after free Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 078/105] powerpc/64s: Use emergency stack for kernel TM Bad Thing program checks Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 079/105] powerpc/tm: Fix illegal TM state in signal handler Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 080/105] percpu: make this_cpu_generic_read() atomic w.r.t. interrupts Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 081/105] driver core: platform: Dont read past the end of "driver_override" buffer Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 082/105] Drivers: hv: fcopy: restore correct transfer length Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 083/105] stm class: Fix a use-after-free Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 084/105] ftrace: Fix kmemleak in unregister_ftrace_graph Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 085/105] HID: i2c-hid: allocate hid buffers for real worst case Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 086/105] HID: wacom: leds: Dont try to control the EKRs read-only LEDs Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 087/105] HID: wacom: Always increment hdev refcount within wacom_get_hdev_data Greg Kroah-Hartman
2017-10-10 19:50 ` [PATCH 4.9 088/105] HID: wacom: bits shifted too much for 9th and 10th buttons Greg Kroah-Hartman
2017-10-10 19:51 ` [PATCH 4.9 089/105] rocker: fix rocker_tlv_put_* functions for KASAN Greg Kroah-Hartman
2017-10-10 19:51 ` [PATCH 4.9 090/105] netlink: fix nla_put_{u8,u16,u32} " Greg Kroah-Hartman
2017-10-11  9:54   ` Arnd Bergmann
2017-10-10 19:51 ` [PATCH 4.9 091/105] iwlwifi: mvm: use IWL_HCMD_NOCOPY for MCAST_FILTER_CMD Greg Kroah-Hartman
2017-10-10 19:51 ` [PATCH 4.9 092/105] iwlwifi: add workaround to disable wide channels in 5GHz Greg Kroah-Hartman
2017-10-10 19:51 ` [PATCH 4.9 093/105] scsi: sd: Do not override max_sectors_kb sysfs setting Greg Kroah-Hartman
2017-10-10 19:51 ` [PATCH 4.9 094/105] brcmfmac: add length check in brcmf_cfg80211_escan_handler() Greg Kroah-Hartman
2017-10-10 19:51 ` [PATCH 4.9 095/105] brcmfmac: setup passive scan if requested by user-space Greg Kroah-Hartman
2017-10-10 19:51 ` [PATCH 4.9 097/105] nvme-pci: Use PCI bus address for data/queues in CMB Greg Kroah-Hartman
2017-10-10 19:51 ` [PATCH 4.9 098/105] mmc: core: add driver strength selection when selecting hs400es Greg Kroah-Hartman
2017-10-10 19:51 ` [PATCH 4.9 099/105] sched/cpuset/pm: Fix cpuset vs. suspend-resume bugs Greg Kroah-Hartman
2017-10-10 19:51 ` [PATCH 4.9 100/105] vfs: deny copy_file_range() for non regular files Greg Kroah-Hartman
2017-10-10 19:51 ` [PATCH 4.9 101/105] ext4: fix data corruption for mmap writes Greg Kroah-Hartman
2017-10-10 19:51 ` [PATCH 4.9 102/105] ext4: Dont clear SGID when inheriting ACLs Greg Kroah-Hartman
2017-10-10 19:51 ` [PATCH 4.9 103/105] ext4: dont allow encrypted operations without keys Greg Kroah-Hartman
2017-10-10 19:51 ` [PATCH 4.9 104/105] f2fs: " Greg Kroah-Hartman
2017-10-11  0:58 ` [PATCH 4.9 000/105] 4.9.55-stable review Shuah Khan
2017-10-11 12:56   ` Greg Kroah-Hartman
2017-10-11 22:54     ` Shuah Khan
2017-10-12 10:39       ` Greg Kroah-Hartman
2017-10-12 11:54         ` Greg Kroah-Hartman
2017-10-12 12:13           ` Greg Kroah-Hartman
2017-10-12 13:45             ` Shuah Khan
2017-10-12 14:10               ` Xin Long
2017-10-12 15:18                 ` Andreas Radke
2017-10-12 15:25                   ` Xin Long
2017-10-12 15:35                     ` Shuah Khan
2017-10-12 16:54                       ` Shuah Khan
2017-10-12 17:08                         ` Ben Hutchings
2017-10-12 17:16                     ` David Miller
2017-10-12 19:18                       ` Greg KH
2017-10-11 13:19 ` Guenter Roeck
2017-10-11 20:25 ` Tom Gall

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20171010192541.463529188@linuxfoundation.org \
    --to=gregkh@linuxfoundation.org \
    --cc=davem@davemloft.net \
    --cc=edumazet@googl.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=liujian56@huawei.com \
    --cc=ncardwell@google.com \
    --cc=soheil@google.com \
    --cc=stable@vger.kernel.org \
    --cc=ycheng@google.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.