Re: [PATCH v5 5/6] nvme-tcp: Support KeyUpdate

Sagi Grimberg
2025-11-30T22:31:47Z

On 12/11/2025 6:27, alistair23@gmail.com wrote:
> From: Alistair Francis <alistair.francis@wdc.com>
>
> If the nvme_tcp_try_send() or nvme_tcp_try_recv() functions return
> EKEYEXPIRED then the underlying TLS keys need to be updated. This occurs
> on an KeyUpdate event as described in RFC8446
> https://datatracker.ietf.org/doc/html/rfc8446#section-4.6.3.
>
> If the NVMe Target (TLS server) initiates a KeyUpdate this patch will
> allow the NVMe layer to process the KeyUpdate request and forward the
> request to userspace. Userspace must then update the key to keep the
> connection alive.
>
> This patch allows us to handle the NVMe target sending a KeyUpdate
> request without aborting the connection. At this time we don't support
> initiating a KeyUpdate.
>
> Signed-off-by: Alistair Francis <alistair.francis@wdc.com>
> ---

I see this is on top of Hannes recvmsg patch. Worth noting in the patch
here at least.

> v5:
> - Cleanup code flow
> - Check for MSG_CTRUNC in the msg_flags return from recvmsg
> and use that to determine if it's a control message
> v4:
> - Remove all support for initiating KeyUpdate
> - Don't call cancel_work() when updating keys
> v3:
> - Don't cancel existing handshake requests
> v2:
> - Don't change the state
> - Use a helper function for KeyUpdates
> - Continue sending in nvme_tcp_send_all() after a KeyUpdate
> - Remove command message using recvmsg
>
> drivers/nvme/host/tcp.c | 85 +++++++++++++++++++++++++++++++++--------
> 1 file changed, 70 insertions(+), 15 deletions(-)
>
> diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
> index 4797a4532b0d..5cec5a974bbf 100644
> --- a/drivers/nvme/host/tcp.c
> +++ b/drivers/nvme/host/tcp.c
> @@ -172,6 +172,7 @@ struct nvme_tcp_queue {
> bool tls_enabled;
> u32 rcv_crc;
> u32 snd_crc;
> + key_serial_t handshake_session_id;
> __le32 exp_ddgst;
> __le32 recv_ddgst;
> struct completion tls_complete;
> @@ -858,7 +859,10 @@ static void nvme_tcp_handle_c2h_term(struct nvme_tcp_queue *queue,
> static int nvme_tcp_recvmsg_pdu(struct nvme_tcp_queue *queue)
> {
> char *pdu = queue->pdu;
> + char cbuf[CMSG_LEN(sizeof(char))] = {};
> struct msghdr msg = {
> + .msg_control = cbuf,
> + .msg_controllen = sizeof(cbuf),
> .msg_flags = MSG_DONTWAIT,
> };
> struct kvec iov = {
> @@ -873,12 +877,17 @@ static int nvme_tcp_recvmsg_pdu(struct nvme_tcp_queue *queue)
> if (ret <= 0)
> return ret;
>
> + hdr = queue->pdu;
> + if (hdr->type == TLS_HANDSHAKE_KEYUPDATE) {
> + dev_err(queue->ctrl->ctrl.device, "KeyUpdate message\n");
> + return 1;
> + }
> +
> queue->pdu_remaining -= ret;
> queue->pdu_offset += ret;
> if (queue->pdu_remaining)
> return 0;
>
> - hdr = queue->pdu;
> if (unlikely(hdr->hlen != sizeof(struct nvme_tcp_rsp_pdu))) {
> if (!nvme_tcp_recv_pdu_supported(hdr->type))
> goto unsupported_pdu;
> @@ -944,6 +953,7 @@ static int nvme_tcp_recvmsg_data(struct nvme_tcp_queue *queue)
> struct request *rq =
> nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
> struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
> + char cbuf[CMSG_LEN(sizeof(char))] = {};
>
> if (nvme_tcp_recv_state(queue) != NVME_TCP_RECV_DATA)
> return 0;
> @@ -976,10 +986,26 @@ static int nvme_tcp_recvmsg_data(struct nvme_tcp_queue *queue)
>
> ret = sock_recvmsg(queue->sock, &msg, msg.msg_flags);
> if (ret < 0) {
> - dev_err(queue->ctrl->ctrl.device,
> - "queue %d failed to receive request %#x data",
> - nvme_tcp_queue_id(queue), rq->tag);
> - return ret;
> + /* If MSG_CTRUNC is set, it's a control message,
> + * so let's read the control message.
> + */
> + if (msg.msg_flags & MSG_CTRUNC) {
> + memset(&msg, 0, sizeof(msg));
> + msg.msg_flags = MSG_DONTWAIT;
> + msg.msg_control = cbuf;
> + msg.msg_controllen = sizeof(cbuf);
> +
> + ret = sock_recvmsg(queue->sock, &msg, msg.msg_flags);
> + }
> +
> + if (ret < 0) {
> + dev_dbg(queue->ctrl->ctrl.device,
> + "queue %d failed to receive request %#x data, %d",
> + nvme_tcp_queue_id(queue), rq->tag, ret);
> + return ret;
> + }
> +
> + return 0;
> }
> if (queue->data_digest)
> nvme_tcp_ddgst_calc(req, &queue->rcv_crc, ret);
> @@ -1384,15 +1410,39 @@ static int nvme_tcp_try_recvmsg(struct nvme_tcp_queue *queue)
> }
> } while (result >= 0);
>
> - if (result < 0 && result != -EAGAIN) {
> - dev_err(queue->ctrl->ctrl.device,
> - "receive failed: %d\n", result);
> - queue->rd_enabled = false;
> - nvme_tcp_error_recovery(&queue->ctrl->ctrl);
> - } else if (result == -EAGAIN)
> - result = 0;
> + if (result < 0) {
> + if (result != -EKEYEXPIRED && result != -EAGAIN) {
> + dev_err(queue->ctrl->ctrl.device,
> + "receive failed: %d\n", result);
> + queue->rd_enabled = false;
> + nvme_tcp_error_recovery(&queue->ctrl->ctrl);
> + }
> + return result;
> + }
> +
> + queue->nr_cqe = nr_cqe;
> + return nr_cqe;
> +}
> +
> +static void update_tls_keys(struct nvme_tcp_queue *queue)
> +{
> + int qid = nvme_tcp_queue_id(queue);
> + int ret;
> +
> + dev_dbg(queue->ctrl->ctrl.device,
> + "updating key for queue %d\n", qid);
>
> - return result < 0 ? result : (queue->nr_cqe = nr_cqe);
> + flush_work(&(queue->ctrl->ctrl).async_event_work);
> +
> + ret = nvme_tcp_start_tls(&(queue->ctrl->ctrl),
> + queue, queue->ctrl->ctrl.tls_pskid,
> + HANDSHAKE_KEY_UPDATE_TYPE_RECEIVED);

No need to quiesce the queue or anything like that? everything continues
as usual?
Why are you flushing async_event_work?