[PATCH v3 08/12] svcrdma: Fire reply-acknowledged callback on Send completion

[Chuck Lever <cel@kernel.org>]

svcrdma posts RDMA Write work requests ahead of the Send, so a
successful Send completion confirms that the transport header and the
reply payload have both reached the client.

Store rq_ack_cookie in the send context before posting and report the
reply as delivered from the Send completion handler, so the DRC can
evict the entry without waiting for RC_EXPIRE. Set XPT_REPLY_ACK on
RDMA transports so the upper layer expects these reports. A flushed
Send reports the reply as undelivered: the connection is closing, and
the entry stays cached for a retransmit on the next connection.

Signed-off-by: Chuck Lever <cel@kernel.org>
---
 include/linux/sunrpc/svc_rdma.h          |  1 +
 net/sunrpc/xprtrdma/svc_rdma_sendto.c    | 14 ++++++++++++++
 net/sunrpc/xprtrdma/svc_rdma_transport.c |  1 +
 3 files changed, 16 insertions(+)

diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
index 76aa5ec4ab40..7261dd3a892e 100644
--- a/include/linux/sunrpc/svc_rdma.h
+++ b/include/linux/sunrpc/svc_rdma.h
@@ -248,6 +248,7 @@ struct svc_rdma_send_ctxt {
 	struct list_head	sc_write_info_list;
 	struct svc_rdma_write_info sc_reply_info;
 
+	svc_ack_cookie_t	sc_ack_cookie;
 	void			*sc_xprt_buf;
 	int			sc_page_count;
 	int			sc_cur_sge_no;
diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c
index c09659b17351..884e2d76cb6f 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c
@@ -220,6 +220,7 @@ struct svc_rdma_send_ctxt *svc_rdma_send_ctxt_get(struct svcxprt_rdma *rdma)
 	ctxt->sc_send_wr.num_sge = 0;
 	ctxt->sc_cur_sge_no = 0;
 	ctxt->sc_page_count = 0;
+	ctxt->sc_ack_cookie = (svc_ack_cookie_t){};
 	ctxt->sc_wr_chain = &ctxt->sc_send_wr;
 	ctxt->sc_sqecount = 1;
 
@@ -470,11 +471,17 @@ static void svc_rdma_wc_send(struct ib_cq *cq, struct ib_wc *wc)
 	if (unlikely(wc->status != IB_WC_SUCCESS))
 		goto flushed;
 
+	if (svc_ack_cookie_present(&ctxt->sc_ack_cookie))
+		svc_reply_acked(rdma->sc_xprt.xpt_server,
+				&ctxt->sc_ack_cookie, true);
 	trace_svcrdma_wc_send(&ctxt->sc_cid);
 	svc_rdma_send_ctxt_put(rdma, ctxt);
 	return;
 
 flushed:
+	if (svc_ack_cookie_present(&ctxt->sc_ack_cookie))
+		svc_reply_acked(rdma->sc_xprt.xpt_server,
+				&ctxt->sc_ack_cookie, false);
 	if (wc->status != IB_WC_WR_FLUSH_ERR)
 		trace_svcrdma_wc_send_err(wc, &ctxt->sc_cid);
 	else
@@ -1198,6 +1205,7 @@ int svc_rdma_sendto(struct svc_rqst *rqstp)
 	if (ret < 0)
 		goto put_ctxt;
 
+	sctxt->sc_ack_cookie = rqstp->rq_ack_cookie;
 	ret = svc_rdma_send_reply_msg(rdma, sctxt, rctxt, rqstp);
 	if (ret < 0)
 		goto send_err;
@@ -1207,6 +1215,12 @@ int svc_rdma_sendto(struct svc_rqst *rqstp)
 	if (ret != -E2BIG && ret != -EINVAL)
 		goto put_ctxt;
 
+	/* The sctxt is reused for the RDMA_ERROR message. Clear the
+	 * ack cookie so that message's Send completion does not
+	 * report the unsent reply as delivered.
+	 */
+	sctxt->sc_ack_cookie = (svc_ack_cookie_t){};
+
 	/* Send completion releases payload pages that were part
 	 * of previously posted RDMA Writes.
 	 */
diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c
index 610df78f9176..64e964ac941c 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_transport.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c
@@ -218,6 +218,7 @@ static struct svcxprt_rdma *svc_rdma_create_xprt(struct svc_serv *serv,
 	 * transports are suitable here.
 	 */
 	set_bit(XPT_CONG_CTRL, &cma_xprt->sc_xprt.xpt_flags);
+	set_bit(XPT_REPLY_ACK, &cma_xprt->sc_xprt.xpt_flags);
 
 	return cma_xprt;
 }

-- 
2.55.0