[Cluster-devel] cluster/cmirror-kernel/src dm-clog.c

[Cluster-devel] cluster/cmirror-kernel/src dm-clog.c

[jbrassow]

CVSROOT:	/cvs/cluster
Module name:	cluster
Branch: 	RHEL5
Changes by:	jbrassow at sourceware.org	2008-02-13 15:06:23

Modified files:
	cmirror-kernel/src: dm-clog.c 

Log message:
	- change the way 'is_remote_recovering' works to improve overall
	performance.
	
	Before a mirror issues a write, it must call 'is_remote_recovering'
	to ensure that another machine will not be recovering the region during
	the write.  This function can dramatically slow things down.  One way to
	increase performance is to note when the mirror is in-sync - then
	is_remote_recovering can return 0 without having to send the request
	around the cluster.  (This has already been done.)  This greatly speeds up
	I/O during nominal mirror operation.  However, I/O during mirror resyncing
	is still greatly reduced.  The problem is that the cluster network is
	consumed with handling 'is_remote_recovering' calls that it becomes hard
	to actually do the recovery.  The fix is to only allow one
	is_remote_recovering call to go to the cluster every 1/4 sec.  When the
	call goes up to userspace, it also retrieves info about how far along
	the resync is.  If a request is determined to already be in sync by that
	info, then the region is not recovering and can safely be answered without
	having to send the request on to the cluster.  This approach has greatly
	improved both the recovery and nominal throughput.

Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror-kernel/src/dm-clog.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.2.2.9&r2=1.2.2.10

--- cluster/cmirror-kernel/src/dm-clog.c	2008/02/08 14:21:04	1.2.2.9
+++ cluster/cmirror-kernel/src/dm-clog.c	2008/02/13 15:06:23	1.2.2.10
@@ -23,7 +23,7 @@
 	char *ctr_str; /* Gives ability to restart if userspace dies */
 	uint32_t ctr_size;
 
-	uint32_t in_sync_hint;
+	uint64_t in_sync_hint;
 
 	spinlock_t flush_lock;
 	struct list_head flush_list;  /* only for clear and mark requests */
@@ -588,8 +588,8 @@
 	if (r)
 		return 0;
 
-	if (sync_count == lc->region_count)
-		lc->in_sync_hint = 1;
+	if (sync_count >= lc->region_count)
+		lc->in_sync_hint = lc->region_count;
 	/*
 	 * get_sync_count is never called after the
 	 * initial sync=1
@@ -644,9 +644,10 @@
 static int cluster_is_remote_recovering(struct dirty_log *log, region_t region)
 {
 	int r;
-	int is_recovering;
-	int rdata_size;
 	struct log_c *lc = (struct log_c *)log->context;
+	static unsigned long long limit = 0;
+	struct { int is_recovering; uint64_t sync_search; } pkg;
+	int rdata_size = sizeof(pkg);
 
 	/*
 	 * Once the mirror has been reported to be in-sync,
@@ -655,14 +656,21 @@
 	 * recovering if the device is in-sync.  (in_sync_hint
 	 * must be reset at resume time.)
 	 */
-	if (lc->in_sync_hint)
+	if (region < lc->in_sync_hint)
 		return 0;
+	else if (jiffies < limit)
+		return 1;
 
-	rdata_size = sizeof(is_recovering);
+	limit = jiffies + (HZ / 4);
 	r = cluster_do_request(lc, lc->uuid, DM_CLOG_IS_REMOTE_RECOVERING,
 			       (char *)&region, sizeof(region),
-			       (char *)&is_recovering, &rdata_size);
-	return (r) ? 1 : is_recovering;
+			       (char *)&pkg, &rdata_size);
+	if (r)
+		return 1;
+
+	lc->in_sync_hint = pkg.sync_search;
+
+	return pkg.is_recovering;
 }
 
 static struct dirty_log_type _clustered_core_type = {

[Cluster-devel] cluster/cmirror-kernel/src dm-clog.c

[jbrassow]

CVSROOT:	/cvs/cluster
Module name:	cluster
Branch: 	RHEL5
Changes by:	jbrassow at sourceware.org	2008-01-25 16:23:25

Modified files:
	cmirror-kernel/src: dm-clog.c 

Log message:
	- calling dm_get_device fixes rename bug 205641
	
	- caching extra state in the kernel helps reduce cluster traffic 90%,
	this improves performance

Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror-kernel/src/dm-clog.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.2.2.7&r2=1.2.2.8

--- cluster/cmirror-kernel/src/dm-clog.c	2008/01/23 21:22:28	1.2.2.7
+++ cluster/cmirror-kernel/src/dm-clog.c	2008/01/25 16:23:24	1.2.2.8
@@ -23,8 +23,12 @@
 	char *ctr_str; /* Gives ability to restart if userspace dies */
 	uint32_t ctr_size;
 
+	uint32_t in_sync_hint;
+
 	spinlock_t flush_lock;
 	struct list_head flush_list;  /* only for clear and mark requests */
+
+	struct dm_dev *disk_log;
 };
 
 static mempool_t *flush_entry_pool = NULL;
@@ -78,21 +82,21 @@
 }
 
 static int cluster_ctr(struct dirty_log *log, struct dm_target *ti,
-		       unsigned int argc, char **argv, int disk_log)
+		       unsigned int argc, char **argv,
+		       struct dm_dev *disk_log)
 {
 	int i;
 	int r = 0;
 	int str_size;
+	int offset = (disk_log) ? 1 : 0;
 	char *ctr_str = NULL;
 	struct log_c *lc = NULL;
 	uint32_t region_size;
 	region_t region_count;
 
 	/* Already checked argument count */
-	if (disk_log != 0 && disk_log != 1)
-		return -EINVAL;
 
-	if (sscanf(argv[disk_log], "%u", &region_size) != 1) {
+	if (sscanf(argv[offset], "%u", &region_size) != 1) {
 		DMWARN("Invalid region size string");
 		return -EINVAL;
 	}
@@ -108,9 +112,10 @@
 	lc->ti = ti;
 	lc->region_size = region_size;
 	lc->region_count = region_count;
+	lc->disk_log = disk_log;
 
 	/* FIXME: Need to check size of uuid arg */
-	memcpy(lc->uuid, argv[1 + disk_log], DM_UUID_LEN);
+	memcpy(lc->uuid, argv[1 + offset], DM_UUID_LEN);
 	spin_lock_init(&lc->flush_lock);
 	INIT_LIST_HEAD(&lc->flush_list);
 
@@ -174,7 +179,7 @@
 		return -EINVAL;
 	}
 
-	r = cluster_ctr(log, ti, argc, argv, 0);
+	r = cluster_ctr(log, ti, argc, argv, NULL);
 
 	return r;
 }
@@ -195,7 +200,9 @@
 static int cluster_disk_ctr(struct dirty_log *log, struct dm_target *ti,
 			    unsigned int argc, char **argv)
 {
-	int i;
+	int r, i;
+	struct dm_dev *dev;
+
 	if ((argc < 4) || (argc > 5)) {
 		DMERR("Too %s arguments to clustered-disk mirror log type.",
 		      (argc < 3) ? "few" : "many");
@@ -205,7 +212,15 @@
 		return -EINVAL;
 	}
 
-	return cluster_ctr(log, ti, argc, argv, 1);
+	r = dm_get_device(ti, argv[0], 0, 0, FMODE_READ | FMODE_WRITE, &dev);
+	if (r)
+		return r;
+
+	r = cluster_ctr(log, ti, argc, argv, dev);
+	if (r)
+		dm_put_device(ti, dev);
+
+	return r;
 }
 
 /*
@@ -222,6 +237,8 @@
 				   NULL, NULL);
 
 	/* FIXME: What do we do on failure? */
+	if (lc->disk_log)
+		dm_put_device(lc->ti, lc->disk_log);
 	kfree(lc->ctr_str);
 	kfree(lc);
 
@@ -269,6 +286,7 @@
 	int r;
 	struct log_c *lc = (struct log_c *)log->context;
 
+	lc->in_sync_hint = 0;
 	r = dm_clog_consult_server(lc->uuid, DM_CLOG_RESUME,
 				   NULL, 0,
 				   NULL, NULL);
@@ -335,6 +353,19 @@
 	int rdata_size;
 	struct log_c *lc = (struct log_c *)log->context;
 
+	/*
+	 * We can never respond directly - even if in_sync_hint is
+	 * set.  This is because another machine could see a device
+	 * failure and mark the region out-of-sync.  If we don't go
+	 * to userspace to ask, we might think the region is in-sync
+	 * and allow a read to pick up data that is stale.  (This is
+	 * very unlikely if a device actually fails; but it is very
+	 * likely if a connection to one device from one machine fails.)
+	 *
+	 * There still might be a problem if the mirror caches the region
+	 * state as in-sync... but then this call would not be made.  So,
+	 * that is a mirror problem.
+	 */
 	if (!can_block)
 		return -EWOULDBLOCK;
 
@@ -559,7 +590,19 @@
 			       NULL, 0,
 			       (char *)&sync_count, &rdata_size);
 
-	return (r) ? 0 : sync_count;
+	if (r)
+		return 0;
+
+	if (sync_count == lc->region_count)
+		lc->in_sync_hint = 1;
+	/*
+	 * get_sync_count is never called after the
+	 * initial sync=1
+	else
+		lc->in_sync_hint = 0;
+	*/
+
+	return sync_count;
 }
 
 /*
@@ -610,6 +653,16 @@
 	int rdata_size;
 	struct log_c *lc = (struct log_c *)log->context;
 
+	/*
+	 * Once the mirror has been reported to be in-sync,
+	 * it will never again ask for recovery work.  So,
+	 * we can safely say there is not a remote machine
+	 * recovering if the device is in-sync.  (in_sync_hint
+	 * must be reset at resume time.)
+	 */
+	if (lc->in_sync_hint)
+		return 0;
+
 	rdata_size = sizeof(is_recovering);
 	r = cluster_do_request(lc, lc->uuid, DM_CLOG_IS_REMOTE_RECOVERING,
 			       (char *)&region, sizeof(region),

[Cluster-devel] cluster/cmirror-kernel/src dm-clog.c

[jbrassow]

CVSROOT:	/cvs/cluster
Module name:	cluster
Branch: 	RHEL5
Changes by:	jbrassow at sourceware.org	2008-01-23 21:22:28

Modified files:
	cmirror-kernel/src: dm-clog.c 

Log message:
	- remember CTR string so if userspace server dies and restarts, we
	can pick up where we left off.

Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror-kernel/src/dm-clog.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.2.2.6&r2=1.2.2.7

--- cluster/cmirror-kernel/src/dm-clog.c	2008/01/21 20:37:03	1.2.2.6
+++ cluster/cmirror-kernel/src/dm-clog.c	2008/01/23 21:22:28	1.2.2.7
@@ -20,6 +20,9 @@
 	region_t region_count;
 	char uuid[DM_UUID_LEN];
 
+	char *ctr_str; /* Gives ability to restart if userspace dies */
+	uint32_t ctr_size;
+
 	spinlock_t flush_lock;
 	struct list_head flush_list;  /* only for clear and mark requests */
 };
@@ -36,6 +39,44 @@
 	kfree(element);
 }
 
+int cluster_do_request(struct log_c *lc, const char *uuid, int request_type,
+		       char *data, int data_size, char *rdata, int *rdata_size)
+{
+	int r;
+
+	/*
+	 * If the server isn't there, -ESRCH is returned,
+	 * and we must keep trying until the server is
+	 * restored.
+	 */
+retry:
+	r = dm_clog_consult_server(uuid, request_type, data,
+				   data_size, rdata, rdata_size);
+
+	if (r != -ESRCH)
+		return r;
+
+	DMERR(" Userspace cluster log server not found.");
+	while (1) {
+		set_current_state(TASK_INTERRUPTIBLE);
+		schedule_timeout(2*HZ);
+		DMWARN("Attempting to contact cluster log server...");
+		r = dm_clog_consult_server(uuid, DM_CLOG_CTR, lc->ctr_str,
+					   lc->ctr_size, NULL, NULL);
+		if (!r)
+			break;
+	}
+	DMINFO("Reconnected to cluster log server... CTR complete");
+	r = dm_clog_consult_server(uuid, DM_CLOG_RESUME, NULL,
+				   0, NULL, NULL);
+	if (!r)
+		goto retry;
+
+	DMERR("Error trying to resume cluster log: %d", r);
+
+	return -ESRCH;
+}
+
 static int cluster_ctr(struct dirty_log *log, struct dm_target *ti,
 		       unsigned int argc, char **argv, int disk_log)
 {
@@ -96,12 +137,14 @@
 	if (r == -ESRCH)
 		DMERR(" Userspace cluster log server not found");
 
-	log->context = lc;
-
-	if (r && lc)
+	if (r) {
 		kfree(lc);
-	if (ctr_str)
 		kfree(ctr_str);
+	} else {
+		lc->ctr_str = ctr_str;
+		lc->ctr_size = str_size;
+		log->context = lc;
+	}
 
 	return r;
 }
@@ -179,7 +222,7 @@
 				   NULL, NULL);
 
 	/* FIXME: What do we do on failure? */
-
+	kfree(lc->ctr_str);
 	kfree(lc);
 
 	return;
@@ -266,9 +309,9 @@
 	struct log_c *lc = (struct log_c *)log->context;
 
 	rdata_size = sizeof(is_clean);
-	r = dm_clog_consult_server(lc->uuid, DM_CLOG_IS_CLEAN,
-				   (char *)&region, sizeof(region),
-				   (char *)&is_clean, &rdata_size);
+	r = cluster_do_request(lc, lc->uuid, DM_CLOG_IS_CLEAN,
+			       (char *)&region, sizeof(region),
+			       (char *)&is_clean, &rdata_size);
 
 	return (r) ? 0 : is_clean;
 }
@@ -296,9 +339,9 @@
 		return -EWOULDBLOCK;
 
 	rdata_size = sizeof(in_sync);
-	r = dm_clog_consult_server(lc->uuid, DM_CLOG_IN_SYNC,
-				   (char *)&region, sizeof(region),
-				   (char *)&in_sync, &rdata_size);
+	r = cluster_do_request(lc, lc->uuid, DM_CLOG_IN_SYNC,
+			       (char *)&region, sizeof(region),
+			       (char *)&in_sync, &rdata_size);
 	return (r) ? 0 : in_sync;
 }
 
@@ -343,17 +386,17 @@
 	 */
 
 	list_for_each_entry(fe, &flush_list, list) {
-		r = dm_clog_consult_server(lc->uuid, fe->type,
-					   (char *)&fe->region,
-					   sizeof(fe->region),
-					   NULL, NULL);
+		r = cluster_do_request(lc, lc->uuid, fe->type,
+				       (char *)&fe->region,
+				       sizeof(fe->region),
+				       NULL, NULL);
 		if (r)
 			goto fail;
 	}
 
 	do {
-		r = dm_clog_consult_server(lc->uuid, DM_CLOG_FLUSH,
-					   NULL, 0, NULL, NULL);
+		r = cluster_do_request(lc, lc->uuid, DM_CLOG_FLUSH,
+				       NULL, 0, NULL, NULL);
 		if (r != -EAGAIN)
 			break;
 
@@ -460,9 +503,9 @@
 	struct { int i; region_t r; } pkg;
 
 	rdata_size = sizeof(pkg);
-	r = dm_clog_consult_server(lc->uuid, DM_CLOG_GET_RESYNC_WORK,
-				   NULL, 0,
-				   (char *)&pkg, &rdata_size);
+	r = cluster_do_request(lc, lc->uuid, DM_CLOG_GET_RESYNC_WORK,
+			       NULL, 0,
+			       (char *)&pkg, &rdata_size);
 
 	*region = pkg.r;
 	return (r) ? r : pkg.i;
@@ -487,9 +530,9 @@
 	pkg.r = region;
 	pkg.i = in_sync;
 
-	r = dm_clog_consult_server(lc->uuid, DM_CLOG_SET_REGION_SYNC,
-				   (char *)&pkg, sizeof(pkg),
-				   NULL, NULL);
+	r = cluster_do_request(lc, lc->uuid, DM_CLOG_SET_REGION_SYNC,
+			       (char *)&pkg, sizeof(pkg),
+			       NULL, NULL);
 
 	/* FIXME: It would be nice to be able to report failures */
 	return;
@@ -512,9 +555,9 @@
 	struct log_c *lc = (struct log_c *)log->context;
 
 	rdata_size = sizeof(sync_count);
-	r = dm_clog_consult_server(lc->uuid, DM_CLOG_GET_SYNC_COUNT,
-				   NULL, 0,
-				   (char *)&sync_count, &rdata_size);
+	r = cluster_do_request(lc, lc->uuid, DM_CLOG_GET_SYNC_COUNT,
+			       NULL, 0,
+			       (char *)&sync_count, &rdata_size);
 
 	return (r) ? 0 : sync_count;
 }
@@ -537,18 +580,18 @@
 
 	switch(status_type) {
 	case STATUSTYPE_INFO:
-		r = dm_clog_consult_server(lc->uuid, DM_CLOG_STATUS_INFO,
-					   NULL, 0,
-					   result, &sz);
+		r = cluster_do_request(lc, lc->uuid, DM_CLOG_STATUS_INFO,
+				       NULL, 0,
+				       result, &sz);
 		/*
 		 * FIXME: If we fail to contact server, we should still
 		 * populate this with parsible results
 		 */
 		break;
 	case STATUSTYPE_TABLE:
-		r = dm_clog_consult_server(lc->uuid, DM_CLOG_STATUS_TABLE,
-					   NULL, 0,
-					   result, &sz);
+		r = cluster_do_request(lc, lc->uuid, DM_CLOG_STATUS_TABLE,
+				       NULL, 0,
+				       result, &sz);
 		break;
 	}
 	return (r) ? 0: sz;
@@ -568,9 +611,9 @@
 	struct log_c *lc = (struct log_c *)log->context;
 
 	rdata_size = sizeof(is_recovering);
-	r = dm_clog_consult_server(lc->uuid, DM_CLOG_IS_REMOTE_RECOVERING,
-				   (char *)&region, sizeof(region),
-				   (char *)&is_recovering, &rdata_size);
+	r = cluster_do_request(lc, lc->uuid, DM_CLOG_IS_REMOTE_RECOVERING,
+			       (char *)&region, sizeof(region),
+			       (char *)&is_recovering, &rdata_size);
 	return (r) ? 1 : is_recovering;
 }
 

[Cluster-devel] cluster/cmirror-kernel/src dm-clog.c

[jbrassow]

CVSROOT:	/cvs/cluster
Module name:	cluster
Branch: 	RHEL5
Changes by:	jbrassow at sourceware.org	2008-01-21 20:37:03

Modified files:
	cmirror-kernel/src: dm-clog.c 

Log message:
	- name change s/clustered_/clustered-/

Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror-kernel/src/dm-clog.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.2.2.5&r2=1.2.2.6

--- cluster/cmirror-kernel/src/dm-clog.c	2008/01/14 22:46:58	1.2.2.5
+++ cluster/cmirror-kernel/src/dm-clog.c	2008/01/21 20:37:03	1.2.2.6
@@ -123,7 +123,7 @@
 {
 	int i, r;
 	if ((argc < 3) || (argc > 4)) {
-		DMERR("Too %s arguments to clustered_core mirror log type.",
+		DMERR("Too %s arguments to clustered-core mirror log type.",
 		      (argc < 2) ? "few" : "many");
 		DMERR("  %d arguments supplied:", argc);
 		for (i = 0; i < argc; i++)
@@ -154,7 +154,7 @@
 {
 	int i;
 	if ((argc < 4) || (argc > 5)) {
-		DMERR("Too %s arguments to clustered_disk mirror log type.",
+		DMERR("Too %s arguments to clustered-disk mirror log type.",
 		      (argc < 3) ? "few" : "many");
 		DMERR("  %d arguments supplied:", argc);
 		for (i = 0; i < argc; i++)
@@ -575,7 +575,7 @@
 }
 
 static struct dirty_log_type _clustered_core_type = {
-	.name = "clustered_core",
+	.name = "clustered-core",
 	.module = THIS_MODULE,
 	.ctr = cluster_core_ctr,
 	.dtr = cluster_dtr,
@@ -597,7 +597,7 @@
 };
 
 static struct dirty_log_type _clustered_disk_type = {
-	.name = "clustered_disk",
+	.name = "clustered-disk",
 	.module = THIS_MODULE,
 	.ctr = cluster_disk_ctr,
 	.dtr = cluster_dtr,
@@ -639,7 +639,7 @@
 
 	r = dm_register_dirty_log_type(&_clustered_core_type);
 	if (r) {
-		DMWARN("Couldn't register clustered_core dirty log type");
+		DMWARN("Couldn't register clustered-core dirty log type");
 		dm_clog_tfr_exit();
 		mempool_destroy(flush_entry_pool);
 		return r;
@@ -647,7 +647,7 @@
 
 	r = dm_register_dirty_log_type(&_clustered_disk_type);
 	if (r) {
-		DMWARN("Couldn't register clustered_disk dirty log type");
+		DMWARN("Couldn't register clustered-disk dirty log type");
 		dm_unregister_dirty_log_type(&_clustered_core_type);
 		dm_clog_tfr_exit();
 		mempool_destroy(flush_entry_pool);

[Cluster-devel] cluster/cmirror-kernel/src dm-clog.c

[jbrassow]

CVSROOT:	/cvs/cluster
Module name:	cluster
Branch: 	RHEL5
Changes by:	jbrassow at sourceware.org	2007-08-30 18:26:22

Modified files:
	cmirror-kernel/src: dm-clog.c 

Log message:
	- kmalloc -> kzalloc

Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror-kernel/src/dm-clog.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.2.2.2&r2=1.2.2.3

--- cluster/cmirror-kernel/src/dm-clog.c	2007/08/30 15:49:32	1.2.2.2
+++ cluster/cmirror-kernel/src/dm-clog.c	2007/08/30 18:26:22	1.2.2.3
@@ -78,7 +78,7 @@
 
 	str_size += 20; /* Max number of chars in a printed u64 number */
 
-	ctr_str = kmalloc(str_size, GFP_KERNEL);
+	ctr_str = kzalloc(str_size, GFP_KERNEL);
 	if (!ctr_str) {
 		DMWARN("Unable to allocate memory for constructor string");
 		kfree(lc);

[Cluster-devel] cluster/cmirror-kernel/src dm-clog.c

[jbrassow]

CVSROOT:	/cvs/cluster
Module name:	cluster
Changes by:	jbrassow at sourceware.org	2006-06-26 20:10:53

Modified files:
	cmirror-kernel/src: dm-clog.c 

Log message:
	- filling out client side logging implementation (patches sent previously)
	
	Work remaining:
	1) client (kernel) side netlink implementation
	2) server implementation

Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror-kernel/src/dm-clog.c.diff?cvsroot=cluster&r1=1.1&r2=1.2

--- cluster/cmirror-kernel/src/dm-clog.c	2006/06/21 01:41:43	1.1
+++ cluster/cmirror-kernel/src/dm-clog.c	2006/06/26 20:10:51	1.2
@@ -120,7 +120,9 @@
  *
  * argv contains:
  *   <disk> <region_size> <uuid> [[no]sync] "block_on_error"
- *--------------------------------------------------------------*/
+ *
+ * Returns: 0 on success, -XXX on failure
+ */
 static int cluster_disk_ctr(struct dirty_log *log, struct dm_target *ti,
 			    unsigned int argc, char **argv)
 {
@@ -137,29 +139,71 @@
 	return cluster_ctr(log, ti, argc, argv, 1);
 }
 
+/*
+ * cluster_dtr
+ * @log
+ */
 static void cluster_dtr(struct dirty_log *log)
 {
+	int r;
 	struct log_c *lc = (struct log_c *)log->context;
 
-	/* FIXME: Send shutdown to server */
+	r = dm_clog_consult_server(lc->uuid, DM_CLOG_DTR,
+				   NULL, 0,
+				   NULL, NULL);
+
+	/* FIXME: What do we do on failure? */
 	kfree(lc);
 
 	return;
 }
 
+/*
+ * cluster_presuspend
+ * @log
+ */
 static int cluster_presuspend(struct dirty_log *log)
 {
-	return -ENOSYS;
+	int r;
+	struct log_c *lc = (struct log_c *)log->context;
+
+	r = dm_clog_consult_server(lc->uuid, DM_CLOG_PRESUSPEND,
+				   NULL, 0,
+				   NULL, NULL);
+
+	return (r > 0) ? -r : r;
 }
 
+/*
+ * cluster_postsuspend
+ * @log
+ */
 static int cluster_postsuspend(struct dirty_log *log)
 {
-	return -ENOSYS;
+	int r;
+	struct log_c *lc = (struct log_c *)log->context;
+
+	r = dm_clog_consult_server(lc->uuid, DM_CLOG_POSTSUSPEND,
+				   NULL, 0,
+				   NULL, NULL);
+
+	return (r > 0) ? -r : r;
 }
 
+/*
+ * cluster_resume
+ * @log
+ */
 static int cluster_resume(struct dirty_log *log)
 {
-	return -ENOSYS;
+	int r;
+	struct log_c *lc = (struct log_c *)log->context;
+
+	r = dm_clog_consult_server(lc->uuid, DM_CLOG_RESUME,
+				   NULL, 0,
+				   NULL, NULL);
+
+	return (r > 0) ? -r : r;
 }
 
 /*
@@ -177,14 +221,55 @@
 	return lc->region_size;
 }
 
+/*
+ * cluster_is_clean
+ * @log
+ * @region
+ *
+ * Check whether a region is clean.  If there is any sort of
+ * failure when consulting the server, we return not clean.
+ *
+ * Returns: 1 if clean, 0 otherwise
+ */
 static int cluster_is_clean(struct dirty_log *log, region_t region)
 {
-	return 0; /* not clean for now */
+	int r;
+	int is_clean;
+	int rdata_size;
+	struct log_c *lc = (struct log_c *)log->context;
+
+	rdata_size = sizeof(is_clean);
+	r = dm_clog_consult_server(lc->uuid, DM_CLOG_IS_CLEAN,
+				   (char *)&region, sizeof(region),
+				   (char *)&is_clean, &rdata_size);
+
+	return (r) ? 0 : is_clean;
 }
 
+/*
+ * cluster_is_remote_recovering
+ * @log
+ * @region
+ *
+ * Check whether a region is being resync'ed on a remote node.
+ * If there is any sort of failure when consulting the server,
+ * we assume that the region is being remotely recovered.
+ *
+ * Returns: 1 if remote recovering, 0 otherwise
+ */
 static int cluster_is_remote_recovering(struct dirty_log *log, region_t region)
 {
-	return 1; /* yes for now */
+	int r;
+	int is_recovering;
+	int rdata_size;
+	struct log_c *lc = (struct log_c *)log->context;
+
+	rdata_size = sizeof(is_recovering);
+	r = dm_clog_consult_server(lc->uuid, DM_CLOG_IS_REMOTE_RECOVERING,
+				   (char *)&region, sizeof(region),
+				   (char *)&is_recovering, &rdata_size);
+
+	return (r) ? 1 : is_recovering;
 }
 
 /*
@@ -193,14 +278,28 @@
  * @region
  * @can_block: if set, return immediately
  *
- * Returns: 1 if in-sync, 0 if not-in-sync, < 0 on error
+ * Check if the region is in-sync.  If there is any sort
+ * of failure when consulting the server, we assume that
+ * the region is not in sync.
+ *
+ * Returns: 1 if in-sync, 0 if not-in-sync, -EWOULDBLOCK
  */
 static int cluster_in_sync(struct dirty_log *log, region_t region, int can_block)
 {
+	int r;
+	int in_sync;
+	int rdata_size;
+	struct log_c *lc = (struct log_c *)log->context;
+
 	if (!can_block)
 		return -EWOULDBLOCK;
 
-	return 0; /* not in sync for now */
+	rdata_size = sizeof(in_sync);
+	r = dm_clog_consult_server(lc->uuid, DM_CLOG_IN_SYNC,
+				   (char *)&region, sizeof(region),
+				   (char *)&in_sync, &rdata_size);
+
+	return (r) ? 0 : in_sync;
 }
 
 /*
@@ -245,7 +344,7 @@
 		r = dm_clog_consult_server(lc->uuid, fe->type,
 					   (char *)&fe->region,
 					   sizeof(fe->region),
-					   NULL, 0);
+					   NULL, NULL);
 		if (r) {
 			r = (r > 0) ? -r : r;
 			goto fail;
@@ -253,7 +352,7 @@
 	}
 
 	r = dm_clog_consult_server(lc->uuid, DM_CLOG_FLUSH,
-				   NULL, 0, NULL, 0);
+				   NULL, 0, NULL, NULL);
 	if (r)
 		r = (r > 0) ? -r : r;
 
@@ -327,26 +426,115 @@
 	return;
 }
 
+/*
+ * cluster_get_resync_work
+ * @log
+ * @region
+ *
+ * Get a region that needs recovery.  It is valid to return
+ * an error for this function.
+ *
+ * Returns: 1 if region filled, 0 if no work, <0 on error
+ */
 static int cluster_get_resync_work(struct dirty_log *log, region_t *region)
 {
-	return -ENOSYS;
+	int r;
+	int rdata_size;
+	struct log_c *lc = (struct log_c *)log->context;
+	struct { int i; region_t r; } pkg;
+
+	rdata_size = sizeof(pkg);
+	r = dm_clog_consult_server(lc->uuid, DM_CLOG_GET_RESYNC_WORK,
+				   NULL, 0,
+				   &pkg, &rdata_size);
+
+	r = (r > 0) ? -r : r;
+
+	*region = pkg.r;
+
+	return (r) ? r : pkg.i;
 }
 
+/*
+ * cluster_set_region_sync
+ * @log
+ * @region
+ * @in_sync
+ *
+ * Set the sync status of a given region.  This function
+ * must not fail.
+ */
 static void cluster_set_region_sync(struct dirty_log *log,
 				    region_t region, int in_sync)
 {
+	int r;
+	struct log_c *lc = (struct log_c *)log->context;
+	struct { region_t r; int i; } pkg;
+
+	pkg.r = region;
+	pkg.i = in_sync;
+
+	r = dm_clog_consult_server(lc->uuid, DM_CLOG_SET_REGION_SYNC,
+				   &pkg, sizeof(pkg),
+				   NULL, NULL);
+
+	/* FIXME: It would be nice to be able to report failures */
 	return;
 }
 
+/*
+ * cluster_get_sync_count
+ * @log
+ *
+ * If there is any sort of failure when consulting the server,
+ * we assume that the sync count is zero.
+ *
+ * Returns: sync count on success, 0 on failure
+ */
 static region_t cluster_get_sync_count(struct dirty_log *log)
 {
-	return 0;
+	int r;
+	int rdata_size;
+	region_t sync_count;
+	struct log_c *lc = (struct log_c *)log->context;
+
+	rdata_size = sizeof(sync_count);
+	r = dm_clog_consult_server(lc->uuid, DM_CLOG_GET_SYNC_COUNT,
+				   NULL, 0,
+				   (char *)&sync_count, &rdata_size);
+
+	return (r) ? 0 : sync_count;
 }
 
+/*
+ * cluster_status
+ * @log
+ * @status_type
+ * @result
+ * @maxlen
+ *
+ * Returns: amount of space consumed
+ */
 static int cluster_status(struct dirty_log *log, status_type_t status_type,
 			  char *result, unsigned int maxlen)
 {
-	return -ENOSYS;
+	int r;
+	unsigned int sz = maxlen;
+	struct log_c *lc = (struct log_c *)log->context;
+
+	switch(status) {
+	case STATUSTYPE_INFO:
+		r = dm_clog_consult_server(lc->uuid, DM_CLOG_STATUS_INFO,
+					   NULL, 0,
+					   result, &sz);
+		break;
+	case STATUSTYPE_TABLE:
+		r = dm_clog_consult_server(lc->uuid, DM_CLOG_STATUS_INFO,
+					   NULL, 0,
+					   result, &sz);
+		break;
+	}
+	return (r) ? 0: sz;
 }
 
 status int cluster_get_failure_response(struct dirty_log *log)