[PATCH v2 3/4] fs/ceph/caps: mark cap remove with RB_CLEAR_NODE() instead of ci=NULL

[Max Kellermann <max.kellermann@ionos.com>]

__ceph_remove_cap() erases the ceph_cap object from the RB tree, thus
it seems natural to use RB_CLEAR_NODE() / RB_EMPTY_NODE() for the
removal check.

Signed-off-by: Max Kellermann <max.kellermann@ionos.com>
---
v1->v2:
- move code to new patch "add helper function ceph_cap_is_removed()"
- add locking documentation to ceph_cap.ci_node
- fix data race by calling RB_CLEAR_NODE() under s_cap_lock

Signed-off-by: Max Kellermann <max.kellermann@ionos.com>
---
 fs/ceph/caps.c  |  7 +++++--
 fs/ceph/super.h | 16 ++++++++++++++--
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c
index f6f3e817b712..a6a59bc91b79 100644
--- a/fs/ceph/caps.c
+++ b/fs/ceph/caps.c
@@ -1157,8 +1157,11 @@ static void __ceph_remove_cap(struct ceph_cap *cap, bool queue_release)
 		cap->session = NULL;
 		removed = 1;
 	}
-	/* protect backpointer with s_cap_lock: see iterate_session_caps */
-	cap->ci = NULL;
+
+	/* protect removal marker with both i_ceph_lock and
+	   s_cap_lock, so either one can be used to check for
+	   removal */
+	RB_CLEAR_NODE(&cap->ci_node);
 
 	/*
 	 * s_cap_reconnect is protected by s_cap_lock. no one changes
diff --git a/fs/ceph/super.h b/fs/ceph/super.h
index 628d153601a5..509839c1db0b 100644
--- a/fs/ceph/super.h
+++ b/fs/ceph/super.h
@@ -203,7 +203,19 @@ struct ceph_fs_client {
  */
 struct ceph_cap {
 	struct ceph_inode_info *ci;
-	struct rb_node ci_node;          /* per-ci cap tree */
+
+	/**
+	 * Per-ci cap tree.  Protected with
+	 * `ceph_inode_info.i_ceph_lock`.
+	 *
+	 * Clearing this field with RB_CLEAR_NODE() requires holding
+	 * both `ceph_inode_info.i_ceph_lock` and
+	 * `ceph_mds_session->s_cap_lock`.  Calling RB_EMPTY_NODE()
+	 * (via ceph_cap_is_removed()) requires holding at least one
+	 * of these.
+	 */
+	struct rb_node ci_node;
+
 	struct ceph_mds_session *session;
 	struct list_head session_caps;   /* per-session caplist */
 	u64 cap_id;       /* unique cap id (mds provided) */
@@ -1280,7 +1292,7 @@ extern void ceph_add_cap(struct inode *inode,
  */
 static inline bool ceph_cap_is_removed(const struct ceph_cap *cap)
 {
-	return !cap->ci;
+	return RB_EMPTY_NODE(&cap->ci_node);
 }
 
 extern void ceph_remove_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap,
-- 
2.47.3