[Cluster-devel] GFS2: First bash at lockdep for glocks

[Steven Whitehouse <swhiteho@redhat.com>]

Hi,

So knowing that lockdep doesn't quite match up with GFS2's expectations
wrt glock semantics, I had a go anyway. Its a bit of a hack, but it
compiles ok.... I'm just about to start doing a little bit of testing as
I hope to use it in order to debug another problem that we've been
looking into recently. There is "just enough" here to do that I think,
but we might well want to hold off before pushing this anywhere else I
think.

Caveats:

 o We ignore all LM_ST_DEFERRED locks. Obviously that mean we won't
detect some conditions, but this lock mode is used only for direct I/O
so we still cover most possible things.

 o We only deal with the "local" end of glocks. It would be good to have
similar support in the dlm for its locks. The main thing there is that
lockdep doesn't support the VAX lock modes. So again, a similar thing to
the above.

 o This is a first draft, and I may well have missed something out.
Please let me know if anything is obviously broken.

Steve.


diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c
index 4cbb695..fcb290d 100644
--- a/fs/gfs2/glock.c
+++ b/fs/gfs2/glock.c
@@ -29,6 +29,7 @@
 #include <linux/freezer.h>
 #include <linux/workqueue.h>
 #include <linux/jiffies.h>
+#include <linux/lockdep.h>
 
 #include "gfs2.h"
 #include "incore.h"
@@ -41,6 +42,32 @@
 #include "super.h"
 #include "util.h"
 
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+#define __gfs2_lock_acquire(gl, gh, p)	lock_acquire(&(gl)->gl_depmap, \
+		0,                                                     \
+		(gh)->gh_flags & LM_FLAG_TRY ? 1 : 0,                  \
+		(gh)->gh_state == LM_ST_SHARED,                        \
+		(p), NULL,                                             \
+		(unsigned long)__builtin_return_address(0));
+#define __gfs2_may_acquire(gl, gh, p)	do {			\
+		if ((gh)->gh_state != LM_ST_DEFERRED)		\
+			__gfs2_lock_acquire((gl), (gh), (p));	\
+		} while (0)
+# ifdef CONFIG_PROVE_LOCKING
+#  define gfs2_lock_acquire(gl, gh)	__gfs2_may_acquire(gl, gh, 2);
+# else
+#  define gfs2_lock_acquire(gl, gh)	__gfs2_may_acquire(gl, gh, 1);
+# endif
+# define gfs2_lock_release(gl, gh)		do {\
+		if ((gh)->gh_state != LM_ST_DEFERRED)                        \
+			lock_release(&(gl)->gl_depmap, 0,                    \
+				(unsigned long)__builtin_return_address(0)); \
+		} while(0);
+#else
+# define gfs2_lock_acquire(gl, gh)	do { } while (0)
+# define gfs2_lock_release(gl, gh)	do { } while (0)
+#endif
+
 struct gfs2_gl_hash_bucket {
         struct hlist_head hb_list;
 };
@@ -671,6 +698,7 @@ int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number,
 	struct gfs2_glock *gl, *tmp;
 	unsigned int hash = gl_hash(sdp, &name);
 	int error;
+	static struct lock_class_key glock_keys[10];
 
 	read_lock(gl_lock_addr(hash));
 	gl = search_bucket(hash, sdp, &name);
@@ -714,6 +742,13 @@ int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number,
 	if (error)
 		goto fail_aspace;
 
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+	sprintf(gl->gl_lockname, "glock %x/%llx", glops->go_type,
+		(unsigned long long)number);
+	lockdep_init_map(&gl->gl_depmap, gl->gl_lockname,
+			 &glock_keys[glops->go_type], 0);
+#endif
+
 	write_lock(gl_lock_addr(hash));
 	tmp = search_bucket(hash, sdp, &name);
 	if (tmp) {
@@ -847,6 +882,8 @@ static void handle_callback(struct gfs2_glock *gl, unsigned int state,
 int gfs2_glock_wait(struct gfs2_holder *gh)
 {
 	wait_on_holder(gh);
+	if (gh->gh_error)
+		gfs2_lock_release(gh->gh_gl, gh);
 	return gh->gh_error;
 }
 
@@ -961,6 +998,7 @@ int gfs2_glock_nq(struct gfs2_holder *gh)
 		return -EIO;
 
 	spin_lock(&gl->gl_spin);
+	gfs2_lock_acquire(gl, gh);
 	add_to_queue(gh);
 	run_queue(gl, 1);
 	spin_unlock(&gl->gl_spin);
@@ -1016,6 +1054,7 @@ void gfs2_glock_dq(struct gfs2_holder *gh)
 			fast_path = 1;
 	}
 	spin_unlock(&gl->gl_spin);
+	gfs2_lock_release(gl, gh);
 	if (likely(fast_path))
 		return;
 
@@ -1673,7 +1712,7 @@ static int __dump_glock(struct seq_file *seq, const struct gfs2_glock *gl)
 	dtime *= 1000000/HZ; /* demote time in uSec */
 	if (!test_bit(GLF_DEMOTE, &gl->gl_flags))
 		dtime = 0;
-	gfs2_print_dbg(seq, "G:  s:%s n:%u/%llu f:%s t:%s d:%s/%llu l:%d a:%d r:%d\n",
+	gfs2_print_dbg(seq, "G:  s:%s n:%x/%llx f:%s t:%s d:%s/%llu l:%d a:%d r:%d\n",
 		  state2str(gl->gl_state),
 		  gl->gl_name.ln_type,
 		  (unsigned long long)gl->gl_name.ln_number,
diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h
index a1777a1..de36523 100644
--- a/fs/gfs2/incore.h
+++ b/fs/gfs2/incore.h
@@ -12,6 +12,7 @@
 
 #include <linux/fs.h>
 #include <linux/workqueue.h>
+#include <linux/lockdep.h>
 
 #define DIO_WAIT	0x00000010
 #define DIO_METADATA	0x00000020
@@ -202,6 +203,10 @@ struct gfs2_glock {
 	struct list_head gl_ail_list;
 	atomic_t gl_ail_count;
 	struct delayed_work gl_work;
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+	struct lockdep_map gl_depmap;
+	char gl_lockname[32];
+#endif
 };
 
 #define GFS2_MIN_LVB_SIZE 32	/* Min size of LVB that gfs2 supports */