[Cluster-devel] [GFS2 PATCH 1/2] GFS2: Introduce EXSH (exclusively shared on one node)

[Bob Peterson <rpeterso@redhat.com>]

This patch is a first step in rgrp sharing. It allows for a new
type of glock mode called EXSH, which stands for a lock that is
Exclusive to one node, but shared amongst processes on that node.
Like a Shared glock, multiple processes may acquire the lock in
EXSH mode at the same time, provided they're all on the same
node. All other nodes will see this as an EX lock. In other words,
to the DLM, the lock is granted to the node in EX, but at the
glock layer, they may be shared.

For now, there are no users of the new EXSH glock mode.
Future patches will use it to improve performance with rgrp sharing.

Signed-off-by: Bob Peterson <rpeterso@redhat.com>
---
 fs/gfs2/glock.c      | 12 +++++++++++-
 fs/gfs2/glock.h      | 16 ++++++++++++----
 fs/gfs2/incore.h     |  8 ++++----
 fs/gfs2/lock_dlm.c   |  5 ++++-
 fs/gfs2/trace_gfs2.h |  2 ++
 5 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c
index 097bd3c0f270..3e3c3e7fada8 100644
--- a/fs/gfs2/glock.c
+++ b/fs/gfs2/glock.c
@@ -287,7 +287,7 @@ static inline int may_grant(const struct gfs2_glock *gl, const struct gfs2_holde
 		return 1;
 	if (gh->gh_flags & GL_EXACT)
 		return 0;
-	if (gl->gl_state == LM_ST_EXCLUSIVE) {
+	if (gl->gl_state == LM_ST_EXCLUSIVE || gl->gl_state == LM_ST_EXSH) {
 		if (gh->gh_state == LM_ST_SHARED && gh_head->gh_state == LM_ST_SHARED)
 			return 1;
 		if (gh->gh_state == LM_ST_DEFERRED && gh_head->gh_state == LM_ST_DEFERRED)
@@ -453,6 +453,13 @@ static void finish_xmote(struct gfs2_glock *gl, unsigned int ret)
 	unsigned state = ret & LM_OUT_ST_MASK;
 	int rv;
 
+	/**
+	 * If we asked DLM to grant EX, it might have been EX or EXSH.
+	 * Here we adjust to compensate for the difference between them.
+	 */
+	if (state == LM_ST_EXCLUSIVE && gl->gl_target == LM_ST_EXSH)
+		state = LM_ST_EXSH;
+
 	spin_lock(&gl->gl_lockref.lock);
 	trace_gfs2_glock_state_change(gl, state);
 	state_change(gl, state);
@@ -557,6 +564,7 @@ __acquires(&gl->gl_lockref.lock)
 	set_bit(GLF_BLOCKING, &gl->gl_flags);
 	if ((gl->gl_req == LM_ST_UNLOCKED) ||
 	    (gl->gl_state == LM_ST_EXCLUSIVE) ||
+	    (gl->gl_state == LM_ST_EXSH) ||
 	    (lck_flags & (LM_FLAG_TRY|LM_FLAG_TRY_1CB)))
 		clear_bit(GLF_BLOCKING, &gl->gl_flags);
 	spin_unlock(&gl->gl_lockref.lock);
@@ -1663,6 +1671,8 @@ static const char *state2str(unsigned state)
 		return "DF";
 	case LM_ST_EXCLUSIVE:
 		return "EX";
+	case LM_ST_EXSH:
+		return "ES";
 	}
 	return "??";
 }
diff --git a/fs/gfs2/glock.h b/fs/gfs2/glock.h
index 5e12220cc0c2..351df3905e70 100644
--- a/fs/gfs2/glock.h
+++ b/fs/gfs2/glock.h
@@ -45,12 +45,15 @@ enum {
  *
  * SHARED is compatible with SHARED, not with DEFERRED or EX.
  * DEFERRED is compatible with DEFERRED, not with SHARED or EX.
+ * EXSH looks like an EX lock to DLM and thus, all other nodes, but
+ *      to the node to which it's granted, it acts like an SHARED lock.
  */
 
 #define LM_ST_UNLOCKED		0
 #define LM_ST_EXCLUSIVE		1
 #define LM_ST_DEFERRED		2
 #define LM_ST_SHARED		3
+#define LM_ST_EXSH		4 /* Exclusively shared on one node) */
 
 /*
  * lm_lock() flags
@@ -94,16 +97,16 @@ enum {
  * lm_async_cb return flags
  *
  * LM_OUT_ST_MASK
- * Masks the lower two bits of lock state in the returned value.
+ * Masks the lower three bits of lock state in the returned value.
  *
  * LM_OUT_CANCELED
  * The lock request was canceled.
  *
  */
 
-#define LM_OUT_ST_MASK		0x00000003
-#define LM_OUT_CANCELED		0x00000008
-#define LM_OUT_ERROR		0x00000004
+#define LM_OUT_ST_MASK		0x00000007
+#define LM_OUT_CANCELED		0x00000010
+#define LM_OUT_ERROR		0x00000008
 
 /*
  * lm_recovery_done() messages
@@ -162,6 +165,11 @@ static inline int gfs2_glock_is_held_excl(struct gfs2_glock *gl)
 	return gl->gl_state == LM_ST_EXCLUSIVE;
 }
 
+static inline int gfs2_glock_is_held_exsh(struct gfs2_glock *gl)
+{
+	return gl->gl_state == LM_ST_EXSH;
+}
+
 static inline int gfs2_glock_is_held_dfrd(struct gfs2_glock *gl)
 {
 	return gl->gl_state == LM_ST_DEFERRED;
diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h
index 0bbbaa9b05cb..58b3ed8531ee 100644
--- a/fs/gfs2/incore.h
+++ b/fs/gfs2/incore.h
@@ -338,10 +338,10 @@ struct gfs2_glock {
 	struct lockref gl_lockref;
 
 	/* State fields protected by gl_lockref.lock */
-	unsigned int gl_state:2,	/* Current state */
-		     gl_target:2,	/* Target state */
-		     gl_demote_state:2,	/* State requested by remote node */
-		     gl_req:2,		/* State in last dlm request */
+	unsigned int gl_state:4,	/* Current state */
+		     gl_target:4,	/* Target state */
+		     gl_demote_state:4,	/* State requested by remote node */
+		     gl_req:4,		/* State in last dlm request */
 		     gl_reply:8;	/* Last reply from the dlm */
 
 	unsigned long gl_demote_time; /* time of first demote request */
diff --git a/fs/gfs2/lock_dlm.c b/fs/gfs2/lock_dlm.c
index 006c6164f759..e2f1968435a2 100644
--- a/fs/gfs2/lock_dlm.c
+++ b/fs/gfs2/lock_dlm.c
@@ -191,6 +191,8 @@ static int make_mode(const unsigned int lmstate)
 		return DLM_LOCK_NL;
 	case LM_ST_EXCLUSIVE:
 		return DLM_LOCK_EX;
+	case LM_ST_EXSH:
+		return DLM_LOCK_EX;
 	case LM_ST_DEFERRED:
 		return DLM_LOCK_CW;
 	case LM_ST_SHARED:
@@ -297,7 +299,8 @@ static void gdlm_put_lock(struct gfs2_glock *gl)
 
 	/* don't want to skip dlm_unlock writing the lvb when lock is ex */
 
-	if (gl->gl_lksb.sb_lvbptr && (gl->gl_state == LM_ST_EXCLUSIVE))
+	if (gl->gl_lksb.sb_lvbptr && (gl->gl_state == LM_ST_EXCLUSIVE ||
+				      gl->gl_state == LM_ST_EXSH))
 		lvb_needs_unlock = 1;
 
 	if (test_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags) &&
diff --git a/fs/gfs2/trace_gfs2.h b/fs/gfs2/trace_gfs2.h
index cb10b95efe0f..6a46cee64508 100644
--- a/fs/gfs2/trace_gfs2.h
+++ b/fs/gfs2/trace_gfs2.h
@@ -72,6 +72,8 @@ static inline u8 glock_trace_state(unsigned int state)
 		return DLM_LOCK_CW;
 	case LM_ST_EXCLUSIVE:
 		return DLM_LOCK_EX;
+	case LM_ST_EXSH:
+		return DLM_LOCK_EX;
 	}
 	return DLM_LOCK_NL;
 }
-- 
2.14.3