[PATCH] libceph: reset OSD session when keepalive2 acks stop arriving

[PATCH] libceph: reset OSD session when keepalive2 acks stop arriving

[Chris J Arges]

handle_timeout() in osd_client.c sends CEPH_MSGR2_TAG_KEEPALIVE2 frames
to OSDs with stalled requests, but libceph never verifies if the ACKs
actually return.

Consequently, if an OSD messenger queue wedges while the underlying
TCP socket remains ESTABLISHED, the client will block indefinitely in
ceph_osdc_wait_request(), causing tasks to hang in D state.

Fix this by introducing a watchdog in handle_timeout() that checks
ceph_con_keepalive_expired() against a new CEPH_OSD_PING_TIMEOUT (60s).
On expiry, reset the sparse-read state, call reopen_osd(), and kick
outstanding requests when the session is reopened.

Because OSD keepalives are only sent to OSDs with stalled requests,
last_keepalive_ack can be stale on an otherwise healthy connection that
has simply been idle. Track the start of each slow/probing episode per
OSD and require the episode to last CEPH_OSD_PING_TIMEOUT before checking
for an expired keepalive ack, so the watchdog only fires after we have
been actively pinging.

Additionally, seed last_keepalive_ack to the current time in
ceph_con_open() to prevent the watchdog from firing spuriously on fresh
connections for both OSD and monitor clients.

Fixes: 8b9558aab853 ("libceph: use keepalive2 to verify the mon session is alive")
Link: https://tracker.ceph.com/issues/76202
Co-developed-by: Andrew DeMaria <ademaria@cloudflare.com>
Signed-off-by: Andrew DeMaria <ademaria@cloudflare.com>
Signed-off-by: Chris J Arges <carges@cloudflare.com>
---
This patch fixes an issue where the kernel rbd client and an OSD have an
ESTABLISHED TCP connection, but keepalive2 ACKs stop returning from the
OSD. When that happens, outstanding OSD requests can remain held by the
client and callers can hang in D state. We were able to mitgiate this
issue by using ss -K to kill the affected OSD TCP connection which
reopened the OSD session.

We were able to reproduce this issue in production a few times, and
synthetically by dropping OSD to rbd application frames while allowing
TCP ACKs through.

https://tracker.ceph.com/issues/76202 describes the same situation.

The following patch addresses this by creating a watchdog that resets the
OSD session if this situation is detected.
---
 include/linux/ceph/libceph.h    |  1 +
 include/linux/ceph/osd_client.h |  1 +
 net/ceph/messenger.c            |  3 +++
 net/ceph/osd_client.c           | 23 ++++++++++++++++++++++-
 4 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h
index 63e0e2aa1ce9..1a117c5f1964 100644
--- a/include/linux/ceph/libceph.h
+++ b/include/linux/ceph/libceph.h
@@ -74,6 +74,7 @@ struct ceph_options {
  */
 #define CEPH_MOUNT_TIMEOUT_DEFAULT	msecs_to_jiffies(60 * 1000)
 #define CEPH_OSD_KEEPALIVE_DEFAULT	msecs_to_jiffies(5 * 1000)
+#define CEPH_OSD_PING_TIMEOUT		msecs_to_jiffies(60 * 1000)
 #define CEPH_OSD_IDLE_TTL_DEFAULT	msecs_to_jiffies(60 * 1000)
 #define CEPH_OSD_REQUEST_TIMEOUT_DEFAULT 0  /* no timeout */
 #define CEPH_READ_FROM_REPLICA_DEFAULT	0  /* read from primary */
diff --git a/include/linux/ceph/osd_client.h b/include/linux/ceph/osd_client.h
index 50b14a5661c7..52eb76e9d62a 100644
--- a/include/linux/ceph/osd_client.h
+++ b/include/linux/ceph/osd_client.h
@@ -94,6 +94,7 @@ struct ceph_osd {
 	struct ceph_auth_handshake o_auth;
 	unsigned long lru_ttl;
 	struct list_head o_keepalive_item;
+	unsigned long o_keepalive_stamp;
 	struct mutex lock;
 	struct ceph_sparse_read	o_sparse_read;
 };
diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c
index 34b3097b4c7b..f7776d83d506 100644
--- a/net/ceph/messenger.c
+++ b/net/ceph/messenger.c
@@ -610,6 +610,9 @@ void ceph_con_open(struct ceph_connection *con,
 
 	memcpy(&con->peer_addr, addr, sizeof(*addr));
 	con->delay = 0;      /* reset backoff memory */
+
+	ktime_get_real_ts64(&con->last_keepalive_ack);
+
 	mutex_unlock(&con->mutex);
 	queue_con(con);
 }
diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
index 2ff00070c181..6d9530beb2e3 100644
--- a/net/ceph/osd_client.c
+++ b/net/ceph/osd_client.c
@@ -53,6 +53,7 @@ static void link_linger(struct ceph_osd *osd,
 static void unlink_linger(struct ceph_osd *osd,
 			  struct ceph_osd_linger_request *lreq);
 static void clear_backoffs(struct ceph_osd *osd);
+static void kick_osd_requests(struct ceph_osd *osd);
 
 #if 1
 static inline bool rwsem_is_wrlocked(struct rw_semaphore *sem)
@@ -3480,8 +3481,11 @@ static void handle_timeout(struct work_struct *work)
 			mutex_unlock(&lreq->lock);
 		}
 
-		if (found)
+		if (found) {
 			list_move_tail(&osd->o_keepalive_item, &slow_osds);
+		} else {
+			osd->o_keepalive_stamp = 0;
+		}
 	}
 
 	if (opts->osd_request_timeout) {
@@ -3507,6 +3511,23 @@ static void handle_timeout(struct work_struct *work)
 							struct ceph_osd,
 							o_keepalive_item);
 		list_del_init(&osd->o_keepalive_item);
+
+		/* Record start of ping timeout from the first slow tick. */
+		if (!osd->o_keepalive_stamp) {
+			osd->o_keepalive_stamp = jiffies;
+		} else if (time_after_eq(jiffies,
+				  osd->o_keepalive_stamp + CEPH_OSD_PING_TIMEOUT) &&
+		    ceph_con_keepalive_expired(&osd->o_con,
+					       CEPH_OSD_PING_TIMEOUT)) {
+			pr_warn_ratelimited("osd%d not responding to keepalives, resetting session\n",
+					    osd->o_osd);
+			osd->o_sparse_op_idx = -1;
+			ceph_init_sparse_read(&osd->o_sparse_read);
+			if (!reopen_osd(osd))
+				kick_osd_requests(osd);
+			continue;
+		}
+
 		ceph_con_keepalive(&osd->o_con);
 	}
 

---
base-commit: 0e35b9b6ec0ffcc5e23cbdec09f5c622ad532b53
change-id: 20260707-fix-rbd-keepalives-664fe9266c35

Best regards,
--  
Chris J Arges <carges@cloudflare.com>

Re: [PATCH] libceph: reset OSD session when keepalive2 acks stop arriving

[Viacheslav Dubeyko]

On Tue, 2026-07-07 at 19:46 -0500, Chris J Arges wrote:
> handle_timeout() in osd_client.c sends CEPH_MSGR2_TAG_KEEPALIVE2 frames
> to OSDs with stalled requests, but libceph never verifies if the ACKs
> actually return.
> 
> Consequently, if an OSD messenger queue wedges while the underlying
> TCP socket remains ESTABLISHED, the client will block indefinitely in
> ceph_osdc_wait_request(), causing tasks to hang in D state.
> 
> Fix this by introducing a watchdog in handle_timeout() that checks
> ceph_con_keepalive_expired() against a new CEPH_OSD_PING_TIMEOUT (60s).
> On expiry, reset the sparse-read state, call reopen_osd(), and kick
> outstanding requests when the session is reopened.
> 
> Because OSD keepalives are only sent to OSDs with stalled requests,
> last_keepalive_ack can be stale on an otherwise healthy connection that
> has simply been idle. Track the start of each slow/probing episode per
> OSD and require the episode to last CEPH_OSD_PING_TIMEOUT before checking
> for an expired keepalive ack, so the watchdog only fires after we have
> been actively pinging.
> 
> Additionally, seed last_keepalive_ack to the current time in
> ceph_con_open() to prevent the watchdog from firing spuriously on fresh
> connections for both OSD and monitor clients.
> 
> Fixes: 8b9558aab853 ("libceph: use keepalive2 to verify the mon session is alive")
> Link: https://urldefense.proofpoint.com/v2/url?u=https-3A__tracker.ceph.com_issues_76202&d=DwICaQ&c=BSDicqBQBDjDI9RkVyTcHQ&r=q5bIm4AXMzc8NJu1_RGmnQ2fMWKq4Y4RAkElvUgSs00&m=8udRqH37aQXbOFlaT2eCFf9V5N2umZEpRHyjRWqPY9nOaaxnBb5-_kuP0k6YrT4O&s=4IAHOQNcSzsOsmpIm3nIsgqXGHCx-bxmgd1yDJ3m-VU&e= 
> Co-developed-by: Andrew DeMaria <ademaria@cloudflare.com>
> Signed-off-by: Andrew DeMaria <ademaria@cloudflare.com>
> Signed-off-by: Chris J Arges <carges@cloudflare.com>
> ---
> This patch fixes an issue where the kernel rbd client and an OSD have an
> ESTABLISHED TCP connection, but keepalive2 ACKs stop returning from the
> OSD. When that happens, outstanding OSD requests can remain held by the
> client and callers can hang in D state. We were able to mitgiate this
> issue by using ss -K to kill the affected OSD TCP connection which
> reopened the OSD session.
> 
> We were able to reproduce this issue in production a few times, and
> synthetically by dropping OSD to rbd application frames while allowing
> TCP ACKs through.
> 
> https://urldefense.proofpoint.com/v2/url?u=https-3A__tracker.ceph.com_issues_76202&d=DwICaQ&c=BSDicqBQBDjDI9RkVyTcHQ&r=q5bIm4AXMzc8NJu1_RGmnQ2fMWKq4Y4RAkElvUgSs00&m=8udRqH37aQXbOFlaT2eCFf9V5N2umZEpRHyjRWqPY9nOaaxnBb5-_kuP0k6YrT4O&s=4IAHOQNcSzsOsmpIm3nIsgqXGHCx-bxmgd1yDJ3m-VU&e=  describes the same situation.
> 
> The following patch addresses this by creating a watchdog that resets the
> OSD session if this situation is detected.
> ---
>  include/linux/ceph/libceph.h    |  1 +
>  include/linux/ceph/osd_client.h |  1 +
>  net/ceph/messenger.c            |  3 +++
>  net/ceph/osd_client.c           | 23 ++++++++++++++++++++++-
>  4 files changed, 27 insertions(+), 1 deletion(-)
> 
> diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h
> index 63e0e2aa1ce9..1a117c5f1964 100644
> --- a/include/linux/ceph/libceph.h
> +++ b/include/linux/ceph/libceph.h
> @@ -74,6 +74,7 @@ struct ceph_options {
>   */
>  #define CEPH_MOUNT_TIMEOUT_DEFAULT	msecs_to_jiffies(60 * 1000)
>  #define CEPH_OSD_KEEPALIVE_DEFAULT	msecs_to_jiffies(5 * 1000)
> +#define CEPH_OSD_PING_TIMEOUT		msecs_to_jiffies(60 * 1000)

Could we reuse the user-configurable osd_keepalive_timeout (default 5s)? Why
exactly 60s has been selected?

Thanks,
Slava.

>  #define CEPH_OSD_IDLE_TTL_DEFAULT	msecs_to_jiffies(60 * 1000)
>  #define CEPH_OSD_REQUEST_TIMEOUT_DEFAULT 0  /* no timeout */
>  #define CEPH_READ_FROM_REPLICA_DEFAULT	0  /* read from primary */
> diff --git a/include/linux/ceph/osd_client.h b/include/linux/ceph/osd_client.h
> index 50b14a5661c7..52eb76e9d62a 100644
> --- a/include/linux/ceph/osd_client.h
> +++ b/include/linux/ceph/osd_client.h
> @@ -94,6 +94,7 @@ struct ceph_osd {
>  	struct ceph_auth_handshake o_auth;
>  	unsigned long lru_ttl;
>  	struct list_head o_keepalive_item;
> +	unsigned long o_keepalive_stamp;
>  	struct mutex lock;
>  	struct ceph_sparse_read	o_sparse_read;
>  };
> diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c
> index 34b3097b4c7b..f7776d83d506 100644
> --- a/net/ceph/messenger.c
> +++ b/net/ceph/messenger.c
> @@ -610,6 +610,9 @@ void ceph_con_open(struct ceph_connection *con,
>  
>  	memcpy(&con->peer_addr, addr, sizeof(*addr));
>  	con->delay = 0;      /* reset backoff memory */
> +
> +	ktime_get_real_ts64(&con->last_keepalive_ack);
> +
>  	mutex_unlock(&con->mutex);
>  	queue_con(con);
>  }
> diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
> index 2ff00070c181..6d9530beb2e3 100644
> --- a/net/ceph/osd_client.c
> +++ b/net/ceph/osd_client.c
> @@ -53,6 +53,7 @@ static void link_linger(struct ceph_osd *osd,
>  static void unlink_linger(struct ceph_osd *osd,
>  			  struct ceph_osd_linger_request *lreq);
>  static void clear_backoffs(struct ceph_osd *osd);
> +static void kick_osd_requests(struct ceph_osd *osd);
>  
>  #if 1
>  static inline bool rwsem_is_wrlocked(struct rw_semaphore *sem)
> @@ -3480,8 +3481,11 @@ static void handle_timeout(struct work_struct *work)
>  			mutex_unlock(&lreq->lock);
>  		}
>  
> -		if (found)
> +		if (found) {
>  			list_move_tail(&osd->o_keepalive_item, &slow_osds);
> +		} else {
> +			osd->o_keepalive_stamp = 0;
> +		}
>  	}
>  
>  	if (opts->osd_request_timeout) {
> @@ -3507,6 +3511,23 @@ static void handle_timeout(struct work_struct *work)
>  							struct ceph_osd,
>  							o_keepalive_item);
>  		list_del_init(&osd->o_keepalive_item);
> +
> +		/* Record start of ping timeout from the first slow tick. */
> +		if (!osd->o_keepalive_stamp) {
> +			osd->o_keepalive_stamp = jiffies;
> +		} else if (time_after_eq(jiffies,
> +				  osd->o_keepalive_stamp + CEPH_OSD_PING_TIMEOUT) &&
> +		    ceph_con_keepalive_expired(&osd->o_con,
> +					       CEPH_OSD_PING_TIMEOUT)) {
> +			pr_warn_ratelimited("osd%d not responding to keepalives, resetting session\n",
> +					    osd->o_osd);
> +			osd->o_sparse_op_idx = -1;
> +			ceph_init_sparse_read(&osd->o_sparse_read);
> +			if (!reopen_osd(osd))
> +				kick_osd_requests(osd);
> +			continue;
> +		}
> +
>  		ceph_con_keepalive(&osd->o_con);
>  	}
>  
> 
> ---
> base-commit: 0e35b9b6ec0ffcc5e23cbdec09f5c622ad532b53
> change-id: 20260707-fix-rbd-keepalives-664fe9266c35
> 
> Best regards,
> --  
> Chris J Arges <carges@cloudflare.com>
> 

Re: [PATCH] libceph: reset OSD session when keepalive2 acks stop arriving

[Chris Arges]

On 2026-07-08 21:07:53, Viacheslav Dubeyko wrote:
> On Tue, 2026-07-07 at 19:46 -0500, Chris J Arges wrote:
> > handle_timeout() in osd_client.c sends CEPH_MSGR2_TAG_KEEPALIVE2 frames
> > to OSDs with stalled requests, but libceph never verifies if the ACKs
> > actually return.
> > 
> > Consequently, if an OSD messenger queue wedges while the underlying
> > TCP socket remains ESTABLISHED, the client will block indefinitely in
> > ceph_osdc_wait_request(), causing tasks to hang in D state.
> > 
> > Fix this by introducing a watchdog in handle_timeout() that checks
> > ceph_con_keepalive_expired() against a new CEPH_OSD_PING_TIMEOUT (60s).
> > On expiry, reset the sparse-read state, call reopen_osd(), and kick
> > outstanding requests when the session is reopened.
> > 
> > Because OSD keepalives are only sent to OSDs with stalled requests,
> > last_keepalive_ack can be stale on an otherwise healthy connection that
> > has simply been idle. Track the start of each slow/probing episode per
> > OSD and require the episode to last CEPH_OSD_PING_TIMEOUT before checking
> > for an expired keepalive ack, so the watchdog only fires after we have
> > been actively pinging.
> > 
> > Additionally, seed last_keepalive_ack to the current time in
> > ceph_con_open() to prevent the watchdog from firing spuriously on fresh
> > connections for both OSD and monitor clients.
> > 
> > Fixes: 8b9558aab853 ("libceph: use keepalive2 to verify the mon session is alive")
> > Link: https://urldefense.proofpoint.com/v2/url?u=https-3A__tracker.ceph.com_issues_76202&d=DwICaQ&c=BSDicqBQBDjDI9RkVyTcHQ&r=q5bIm4AXMzc8NJu1_RGmnQ2fMWKq4Y4RAkElvUgSs00&m=8udRqH37aQXbOFlaT2eCFf9V5N2umZEpRHyjRWqPY9nOaaxnBb5-_kuP0k6YrT4O&s=4IAHOQNcSzsOsmpIm3nIsgqXGHCx-bxmgd1yDJ3m-VU&e= 
> > Co-developed-by: Andrew DeMaria <ademaria@cloudflare.com>
> > Signed-off-by: Andrew DeMaria <ademaria@cloudflare.com>
> > Signed-off-by: Chris J Arges <carges@cloudflare.com>
> > ---
> > This patch fixes an issue where the kernel rbd client and an OSD have an
> > ESTABLISHED TCP connection, but keepalive2 ACKs stop returning from the
> > OSD. When that happens, outstanding OSD requests can remain held by the
> > client and callers can hang in D state. We were able to mitgiate this
> > issue by using ss -K to kill the affected OSD TCP connection which
> > reopened the OSD session.
> > 
> > We were able to reproduce this issue in production a few times, and
> > synthetically by dropping OSD to rbd application frames while allowing
> > TCP ACKs through.
> > 
> > https://urldefense.proofpoint.com/v2/url?u=https-3A__tracker.ceph.com_issues_76202&d=DwICaQ&c=BSDicqBQBDjDI9RkVyTcHQ&r=q5bIm4AXMzc8NJu1_RGmnQ2fMWKq4Y4RAkElvUgSs00&m=8udRqH37aQXbOFlaT2eCFf9V5N2umZEpRHyjRWqPY9nOaaxnBb5-_kuP0k6YrT4O&s=4IAHOQNcSzsOsmpIm3nIsgqXGHCx-bxmgd1yDJ3m-VU&e=  describes the same situation.
> > 
> > The following patch addresses this by creating a watchdog that resets the
> > OSD session if this situation is detected.
> > ---
> >  include/linux/ceph/libceph.h    |  1 +
> >  include/linux/ceph/osd_client.h |  1 +
> >  net/ceph/messenger.c            |  3 +++
> >  net/ceph/osd_client.c           | 23 ++++++++++++++++++++++-
> >  4 files changed, 27 insertions(+), 1 deletion(-)
> > 
> > diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h
> > index 63e0e2aa1ce9..1a117c5f1964 100644
> > --- a/include/linux/ceph/libceph.h
> > +++ b/include/linux/ceph/libceph.h
> > @@ -74,6 +74,7 @@ struct ceph_options {
> >   */
> >  #define CEPH_MOUNT_TIMEOUT_DEFAULT	msecs_to_jiffies(60 * 1000)
> >  #define CEPH_OSD_KEEPALIVE_DEFAULT	msecs_to_jiffies(5 * 1000)
> > +#define CEPH_OSD_PING_TIMEOUT		msecs_to_jiffies(60 * 1000)
> 
> Could we reuse the user-configurable osd_keepalive_timeout (default 5s)? Why
> exactly 60s has been selected?
> 
> Thanks,
> Slava.
> 

I think CEPH_OSD_KEEPALIVE_TIMEOUT should be distinct from
CEPH_OSD_PING_TIMEOUT. This way keepalive_timeout fires more frequently, and
can still catch a TCP session going bad. The ping_timeout should be longer so
we can try a few application-layer ACKs and if the TCP session is still good
then it shows a session reset may be warranted.

We chose 60s specifically because other timeouts had that value and it worked
in test.

--chris

> >  #define CEPH_OSD_IDLE_TTL_DEFAULT	msecs_to_jiffies(60 * 1000)
> >  #define CEPH_OSD_REQUEST_TIMEOUT_DEFAULT 0  /* no timeout */
> >  #define CEPH_READ_FROM_REPLICA_DEFAULT	0  /* read from primary */
> > diff --git a/include/linux/ceph/osd_client.h b/include/linux/ceph/osd_client.h
> > index 50b14a5661c7..52eb76e9d62a 100644
> > --- a/include/linux/ceph/osd_client.h
> > +++ b/include/linux/ceph/osd_client.h
> > @@ -94,6 +94,7 @@ struct ceph_osd {
> >  	struct ceph_auth_handshake o_auth;
> >  	unsigned long lru_ttl;
> >  	struct list_head o_keepalive_item;
> > +	unsigned long o_keepalive_stamp;
> >  	struct mutex lock;
> >  	struct ceph_sparse_read	o_sparse_read;
> >  };
> > diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c
> > index 34b3097b4c7b..f7776d83d506 100644
> > --- a/net/ceph/messenger.c
> > +++ b/net/ceph/messenger.c
> > @@ -610,6 +610,9 @@ void ceph_con_open(struct ceph_connection *con,
> >  
> >  	memcpy(&con->peer_addr, addr, sizeof(*addr));
> >  	con->delay = 0;      /* reset backoff memory */
> > +
> > +	ktime_get_real_ts64(&con->last_keepalive_ack);
> > +
> >  	mutex_unlock(&con->mutex);
> >  	queue_con(con);
> >  }
> > diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
> > index 2ff00070c181..6d9530beb2e3 100644
> > --- a/net/ceph/osd_client.c
> > +++ b/net/ceph/osd_client.c
> > @@ -53,6 +53,7 @@ static void link_linger(struct ceph_osd *osd,
> >  static void unlink_linger(struct ceph_osd *osd,
> >  			  struct ceph_osd_linger_request *lreq);
> >  static void clear_backoffs(struct ceph_osd *osd);
> > +static void kick_osd_requests(struct ceph_osd *osd);
> >  
> >  #if 1
> >  static inline bool rwsem_is_wrlocked(struct rw_semaphore *sem)
> > @@ -3480,8 +3481,11 @@ static void handle_timeout(struct work_struct *work)
> >  			mutex_unlock(&lreq->lock);
> >  		}
> >  
> > -		if (found)
> > +		if (found) {
> >  			list_move_tail(&osd->o_keepalive_item, &slow_osds);
> > +		} else {
> > +			osd->o_keepalive_stamp = 0;
> > +		}
> >  	}
> >  
> >  	if (opts->osd_request_timeout) {
> > @@ -3507,6 +3511,23 @@ static void handle_timeout(struct work_struct *work)
> >  							struct ceph_osd,
> >  							o_keepalive_item);
> >  		list_del_init(&osd->o_keepalive_item);
> > +
> > +		/* Record start of ping timeout from the first slow tick. */
> > +		if (!osd->o_keepalive_stamp) {
> > +			osd->o_keepalive_stamp = jiffies;
> > +		} else if (time_after_eq(jiffies,
> > +				  osd->o_keepalive_stamp + CEPH_OSD_PING_TIMEOUT) &&
> > +		    ceph_con_keepalive_expired(&osd->o_con,
> > +					       CEPH_OSD_PING_TIMEOUT)) {
> > +			pr_warn_ratelimited("osd%d not responding to keepalives, resetting session\n",
> > +					    osd->o_osd);
> > +			osd->o_sparse_op_idx = -1;
> > +			ceph_init_sparse_read(&osd->o_sparse_read);
> > +			if (!reopen_osd(osd))
> > +				kick_osd_requests(osd);
> > +			continue;
> > +		}
> > +
> >  		ceph_con_keepalive(&osd->o_con);
> >  	}
> >  
> > 
> > ---
> > base-commit: 0e35b9b6ec0ffcc5e23cbdec09f5c622ad532b53
> > change-id: 20260707-fix-rbd-keepalives-664fe9266c35
> > 
> > Best regards,
> > --  
> > Chris J Arges <carges@cloudflare.com>
> > 

RE: [PATCH] libceph: reset OSD session when keepalive2 acks stop arriving

[Viacheslav Dubeyko]

On Wed, 2026-07-08 at 17:17 -0500, Chris Arges wrote:
> On 2026-07-08 21:07:53, Viacheslav Dubeyko wrote:
> > On Tue, 2026-07-07 at 19:46 -0500, Chris J Arges wrote:
> > > handle_timeout() in osd_client.c sends CEPH_MSGR2_TAG_KEEPALIVE2 frames
> > > to OSDs with stalled requests, but libceph never verifies if the ACKs
> > > actually return.
> > > 
> > > Consequently, if an OSD messenger queue wedges while the underlying
> > > TCP socket remains ESTABLISHED, the client will block indefinitely in
> > > ceph_osdc_wait_request(), causing tasks to hang in D state.
> > > 
> > > Fix this by introducing a watchdog in handle_timeout() that checks
> > > ceph_con_keepalive_expired() against a new CEPH_OSD_PING_TIMEOUT (60s).
> > > On expiry, reset the sparse-read state, call reopen_osd(), and kick
> > > outstanding requests when the session is reopened.
> > > 
> > > Because OSD keepalives are only sent to OSDs with stalled requests,
> > > last_keepalive_ack can be stale on an otherwise healthy connection that
> > > has simply been idle. Track the start of each slow/probing episode per
> > > OSD and require the episode to last CEPH_OSD_PING_TIMEOUT before checking
> > > for an expired keepalive ack, so the watchdog only fires after we have
> > > been actively pinging.
> > > 
> > > Additionally, seed last_keepalive_ack to the current time in
> > > ceph_con_open() to prevent the watchdog from firing spuriously on fresh
> > > connections for both OSD and monitor clients.
> > > 
> > > Fixes: 8b9558aab853 ("libceph: use keepalive2 to verify the mon session is alive")
> > > Link: https://urldefense.proofpoint.com/v2/url?u=https-3A__tracker.ceph.com_issues_76202&d=DwICaQ&c=BSDicqBQBDjDI9RkVyTcHQ&r=q5bIm4AXMzc8NJu1_RGmnQ2fMWKq4Y4RAkElvUgSs00&m=8udRqH37aQXbOFlaT2eCFf9V5N2umZEpRHyjRWqPY9nOaaxnBb5-_kuP0k6YrT4O&s=4IAHOQNcSzsOsmpIm3nIsgqXGHCx-bxmgd1yDJ3m-VU&e= 
> > > Co-developed-by: Andrew DeMaria <ademaria@cloudflare.com>
> > > Signed-off-by: Andrew DeMaria <ademaria@cloudflare.com>
> > > Signed-off-by: Chris J Arges <carges@cloudflare.com>
> > > ---
> > > This patch fixes an issue where the kernel rbd client and an OSD have an
> > > ESTABLISHED TCP connection, but keepalive2 ACKs stop returning from the
> > > OSD. When that happens, outstanding OSD requests can remain held by the
> > > client and callers can hang in D state. We were able to mitgiate this
> > > issue by using ss -K to kill the affected OSD TCP connection which
> > > reopened the OSD session.
> > > 
> > > We were able to reproduce this issue in production a few times, and
> > > synthetically by dropping OSD to rbd application frames while allowing
> > > TCP ACKs through.
> > > 
> > > https://urldefense.proofpoint.com/v2/url?u=https-3A__tracker.ceph.com_issues_76202&d=DwICaQ&c=BSDicqBQBDjDI9RkVyTcHQ&r=q5bIm4AXMzc8NJu1_RGmnQ2fMWKq4Y4RAkElvUgSs00&m=8udRqH37aQXbOFlaT2eCFf9V5N2umZEpRHyjRWqPY9nOaaxnBb5-_kuP0k6YrT4O&s=4IAHOQNcSzsOsmpIm3nIsgqXGHCx-bxmgd1yDJ3m-VU&e=  describes the same situation.
> > > 
> > > The following patch addresses this by creating a watchdog that resets the
> > > OSD session if this situation is detected.
> > > ---
> > >  include/linux/ceph/libceph.h    |  1 +
> > >  include/linux/ceph/osd_client.h |  1 +
> > >  net/ceph/messenger.c            |  3 +++
> > >  net/ceph/osd_client.c           | 23 ++++++++++++++++++++++-
> > >  4 files changed, 27 insertions(+), 1 deletion(-)
> > > 
> > > diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h
> > > index 63e0e2aa1ce9..1a117c5f1964 100644
> > > --- a/include/linux/ceph/libceph.h
> > > +++ b/include/linux/ceph/libceph.h
> > > @@ -74,6 +74,7 @@ struct ceph_options {
> > >   */
> > >  #define CEPH_MOUNT_TIMEOUT_DEFAULT	msecs_to_jiffies(60 * 1000)
> > >  #define CEPH_OSD_KEEPALIVE_DEFAULT	msecs_to_jiffies(5 * 1000)
> > > +#define CEPH_OSD_PING_TIMEOUT		msecs_to_jiffies(60 * 1000)
> > 
> > Could we reuse the user-configurable osd_keepalive_timeout (default 5s)? Why
> > exactly 60s has been selected?
> > 
> > Thanks,
> > Slava.
> > 
> 
> I think CEPH_OSD_KEEPALIVE_TIMEOUT should be distinct from
> CEPH_OSD_PING_TIMEOUT. This way keepalive_timeout fires more frequently, and
> can still catch a TCP session going bad. The ping_timeout should be longer so
> we can try a few application-layer ACKs and if the TCP session is still good
> then it shows a session reset may be warranted.
> 
> We chose 60s specifically because other timeouts had that value and it worked
> in test.
> 
> --chris
> 
> > >  #define CEPH_OSD_IDLE_TTL_DEFAULT	msecs_to_jiffies(60 * 1000)
> > >  #define CEPH_OSD_REQUEST_TIMEOUT_DEFAULT 0  /* no timeout */
> > >  #define CEPH_READ_FROM_REPLICA_DEFAULT	0  /* read from primary */
> > > diff --git a/include/linux/ceph/osd_client.h b/include/linux/ceph/osd_client.h
> > > index 50b14a5661c7..52eb76e9d62a 100644
> > > --- a/include/linux/ceph/osd_client.h
> > > +++ b/include/linux/ceph/osd_client.h
> > > @@ -94,6 +94,7 @@ struct ceph_osd {
> > >  	struct ceph_auth_handshake o_auth;
> > >  	unsigned long lru_ttl;
> > >  	struct list_head o_keepalive_item;
> > > +	unsigned long o_keepalive_stamp;
> > >  	struct mutex lock;
> > >  	struct ceph_sparse_read	o_sparse_read;
> > >  };
> > > diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c
> > > index 34b3097b4c7b..f7776d83d506 100644
> > > --- a/net/ceph/messenger.c
> > > +++ b/net/ceph/messenger.c
> > > @@ -610,6 +610,9 @@ void ceph_con_open(struct ceph_connection *con,
> > >  
> > >  	memcpy(&con->peer_addr, addr, sizeof(*addr));
> > >  	con->delay = 0;      /* reset backoff memory */
> > > +
> > > +	ktime_get_real_ts64(&con->last_keepalive_ack);
> > > +
> > >  	mutex_unlock(&con->mutex);
> > >  	queue_con(con);
> > >  }
> > > diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
> > > index 2ff00070c181..6d9530beb2e3 100644
> > > --- a/net/ceph/osd_client.c
> > > +++ b/net/ceph/osd_client.c
> > > @@ -53,6 +53,7 @@ static void link_linger(struct ceph_osd *osd,
> > >  static void unlink_linger(struct ceph_osd *osd,
> > >  			  struct ceph_osd_linger_request *lreq);
> > >  static void clear_backoffs(struct ceph_osd *osd);
> > > +static void kick_osd_requests(struct ceph_osd *osd);
> > >  
> > >  #if 1
> > >  static inline bool rwsem_is_wrlocked(struct rw_semaphore *sem)
> > > @@ -3480,8 +3481,11 @@ static void handle_timeout(struct work_struct *work)
> > >  			mutex_unlock(&lreq->lock);
> > >  		}
> > >  
> > > -		if (found)
> > > +		if (found) {
> > >  			list_move_tail(&osd->o_keepalive_item, &slow_osds);
> > > +		} else {
> > > +			osd->o_keepalive_stamp = 0;
> > > +		}
> > >  	}
> > >  
> > >  	if (opts->osd_request_timeout) {
> > > @@ -3507,6 +3511,23 @@ static void handle_timeout(struct work_struct *work)
> > >  							struct ceph_osd,
> > >  							o_keepalive_item);
> > >  		list_del_init(&osd->o_keepalive_item);
> > > +
> > > +		/* Record start of ping timeout from the first slow tick. */
> > > +		if (!osd->o_keepalive_stamp) {
> > > +			osd->o_keepalive_stamp = jiffies;
> > > +		} else if (time_after_eq(jiffies,
> > > +				  osd->o_keepalive_stamp + CEPH_OSD_PING_TIMEOUT) &&
> > > +		    ceph_con_keepalive_expired(&osd->o_con,
> > > +					       CEPH_OSD_PING_TIMEOUT)) {

I don't very like this multi-line check. :) Could we rework it somehow? What
about static inline method?

Thanks,
Slava. 

> > > +			pr_warn_ratelimited("osd%d not responding to keepalives, resetting session\n",
> > > +					    osd->o_osd);
> > > +			osd->o_sparse_op_idx = -1;
> > > +			ceph_init_sparse_read(&osd->o_sparse_read);
> > > +			if (!reopen_osd(osd))
> > > +				kick_osd_requests(osd);
> > > +			continue;
> > > +		}
> > > +
> > >  		ceph_con_keepalive(&osd->o_con);
> > >  	}
> > >  
> > > 
> > > ---
> > > base-commit: 0e35b9b6ec0ffcc5e23cbdec09f5c622ad532b53
> > > change-id: 20260707-fix-rbd-keepalives-664fe9266c35
> > > 
> > > Best regards,
> > > --  
> > > Chris J Arges <carges@cloudflare.com>


> >