[Cluster-devel] cluster/cmirror/src clogd.c cluster.c cluster. ...

[jbrassow@sourceware.org <jbrassow@sourceware.org>]

CVSROOT:	/cvs/cluster
Module name:	cluster
Branch: 	RHEL5
Changes by:	jbrassow at sourceware.org	2008-01-23 21:21:06

Modified files:
	cmirror/src    : clogd.c cluster.c cluster.h functions.c local.c 
	                 local.h queues.c queues.h 

Log message:
	- better cleanup on exit
	- pull rbtrees in favor of simple list until later

Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror/src/clogd.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.2.4&r2=1.1.2.5
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror/src/cluster.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.2.10&r2=1.1.2.11
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror/src/cluster.h.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.2.1&r2=1.1.2.2
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror/src/functions.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.2.9&r2=1.1.2.10
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror/src/local.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.2.8&r2=1.1.2.9
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror/src/local.h.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.2.1&r2=1.1.2.2
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror/src/queues.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.2.3&r2=1.1.2.4
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror/src/queues.h.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.2.2&r2=1.1.2.3

--- cluster/cmirror/src/Attic/clogd.c	2008/01/21 20:18:44	1.1.2.4
+++ cluster/cmirror/src/Attic/clogd.c	2008/01/23 21:21:06	1.1.2.5
@@ -30,6 +30,7 @@
 static void process_signals(void);
 static void daemonize(void);
 static void init_all(void);
+static void cleanup_all(void);
 
 int main(int argc, char *argv[])
 {
@@ -129,6 +130,7 @@
 
 	if (!r) {
 		LOG_DBG("No current cluster logs... safe to exit.");
+		cleanup_all();
 		exit(EXIT_SUCCESS);
 	}
 
@@ -247,3 +249,14 @@
 	}
 }
 
+/*
+ * cleanup_all
+ *
+ * Clean up before exiting
+ */
+static void cleanup_all(void)
+{
+	cleanup_local();
+	cleanup_cluster();
+	cleanup_queues();
+}
--- cluster/cmirror/src/Attic/cluster.c	2008/01/18 21:00:15	1.1.2.10
+++ cluster/cmirror/src/Attic/cluster.c	2008/01/23 21:21:06	1.1.2.11
@@ -68,6 +68,9 @@
 	cpg_flow_control_state_t flow_control_state;
 	cpg_error_t error;
 	
+	/* FIXME: no flow control for now (cmirror should self regulate) */
+	return 0;
+
 	error = cpg_flow_control_state_get(handle, &flow_control_state);
 	if (error != CPG_OK) {
 		LOG_ERROR("Failed to get flow control state.  Reason: %d", error);
@@ -981,3 +984,12 @@
 	EXIT();
 	return 0;
 }
+
+void cleanup_cluster(void)
+{
+	SaAisErrorT err;
+
+	err = saCkptFinalize(ckpt_handle);
+	if (err != SA_AIS_OK)
+		LOG_ERROR("Failed to finalize checkpoint handle");
+}
--- cluster/cmirror/src/Attic/cluster.h	2007/08/23 19:57:31	1.1.2.1
+++ cluster/cmirror/src/Attic/cluster.h	2008/01/23 21:21:06	1.1.2.2
@@ -2,6 +2,7 @@
 #define __CLUSTER_LOG_CLUSTER_DOT_H__
 
 int init_cluster(void);
+void cleanup_cluster(void);
 
 int create_cluster_cpg(char *str);
 int destroy_cluster_cpg(char *str);
--- cluster/cmirror/src/Attic/functions.c	2008/01/18 17:11:07	1.1.2.9
+++ cluster/cmirror/src/Attic/functions.c	2008/01/23 21:21:06	1.1.2.10
@@ -15,7 +15,6 @@
 #include "common.h"
 #include "cluster.h"
 #include "logging.h"
-#include "rbtree.h"
 
 #define BYTE_SHIFT 3
 
@@ -67,7 +66,7 @@
 
 	uint32_t state;         /* current operational state of the log */
 
-	struct rb_tree mark_tree; /* Tree that tracks all mark requests */
+	struct list_head mark_list;
 
 	uint32_t recovery_halted;
 	struct recovery_request *recovery_request_list;
@@ -80,6 +79,7 @@
 };
 
 struct mark_entry {
+	struct list_head list;
 	uint32_t nodeid;
 	uint64_t region;
 };
@@ -288,47 +288,6 @@
 	return 0;
 }
 
-static void *get_mark_entry_region(void *data)
-{
-	struct mark_entry *m = data;
-
-	return (void *)&m->region;
-}
-
-static int cmp_mark_entry_regions(void *a, void *b)
-{
-	uint64_t _a = *((uint64_t *)a);
-	uint64_t _b = *((uint64_t *)b);
-
-	return (_a == _b) ? 0 : (_a < _b) ? -1 : 1;
-}
-
-/*
- * srsm_count - Same Region, Same Machine count
- * @data - data held in the tree node (a mark_entry ptr)
- * @adata - additional data passed in (nodeid)
- *
- * This function always returns 1 - allowing the RBT search
- * to continuing finding additional matches.  It's useful
- * feature is that it counts all the tree nodes that match
- * the given machine (not just the region).  Results of the
- * count is placed in 'srsm_count_var'.
- *
- * Returns: 1
- */
-static int srsm_count_var = 0;
-static int srsm_count(void *data, void *adata)
-{
-	uint32_t nodeid = *((uint32_t *)adata);
-	struct mark_entry *m = data;
-
-	if (nodeid == m->nodeid)
-		srsm_count_var++;
-
-	return 1;
-}
-
-
 static int find_disk_path(char *major_minor_str, char *path_rtn, int *unlink_path)
 {
 	int r;
@@ -467,8 +426,7 @@
 	lc->disk_fd = -1;
 	lc->log_dev_failed = 0;
 
-	rbt_init(&lc->mark_tree, sizeof(struct mark_entry),
-		 get_mark_entry_region, cmp_mark_entry_regions);
+	INIT_LIST_HEAD(&lc->mark_list);
 
 	lc->bitset_uint32_count = region_count / 
 		(sizeof(*lc->clean_bits) << BYTE_SHIFT);
@@ -523,8 +481,6 @@
 
 	list_add(&lc->list, &log_pending_list);
 
-	LOG_DBG("Log UUID = %s, mark_tree = 0x%p",
-		SHORT_UUID(lc->uuid), &lc->mark_tree);
 	EXIT();
 	return 0;
 fail:
@@ -965,39 +921,37 @@
  */
 static int mark_region(struct log_c *lc, uint64_t region, uint32_t who)
 {
-	struct rb_node *new;
+	int found = 0;
 	struct mark_entry *m;
+	struct list_head *p, *n;
 
-	/*
-	 * The search will find every node in the tree that has
-	 * the same region marked.  The additional function
-	 * passed to 'rbt_search_plus' detects if the matching
-	 * nodes also are from the machine who is performing this
-	 * request.
-	 */
-	srsm_count_var = 0;
-	if (!rbt_search_plus(&lc->mark_tree, &region, srsm_count, &who))
-		log_clear_bit(lc, lc->clean_bits, region);
+	list_for_each_safe(p, n, &lc->mark_list) {
+		/* FIXME: Use proper macros */
+		m = (struct mark_entry *)p;
+		if (m->region == region) {
+			found = 1;
+			if (m->nodeid == who)
+				return 0;
+		}
+	}
 
-	/* Requesting region/nodeid is already in the tree */
-	if (srsm_count_var)
-		return 0;
+	if (!found)
+		log_clear_bit(lc, lc->clean_bits, region);
 
 	/*
 	 * Save allocation until here - if there is a failure,
 	 * at least we have cleared the bit.
 	 */
-	new = rbt_alloc_node(&lc->mark_tree);
-	if (!new) {
+	m = malloc(sizeof(*m));
+	if (!m) {
 		LOG_ERROR("Unable to allocate space for mark_entry: %llu/%u",
 			  (unsigned long long)region, who);
 		return -ENOMEM;
 	}
 
-	m = new->rb_data;
 	m->nodeid = who;
 	m->region = region;
-	rbt_insert(&lc->mark_tree, new);
+	list_add_tail(&m->list, &lc->mark_list);
 
 	return 0;
 }
@@ -1042,40 +996,28 @@
 
 static int clear_region(struct log_c *lc, uint64_t region, uint32_t who)
 {
-	int set_bit = 1;
-	struct rb_node *mark_list;
+	int other_matches = 0;
 	struct mark_entry *m;
+	struct list_head *p, *n;
 
-	srsm_count_var = 0;
-	mark_list = rbt_search_plus(&lc->mark_tree, &region, srsm_count, &who);
-	if (!mark_list || !srsm_count_var) {
-		LOG_DBG("[%s] Clear issued by %u on region not marked: %llu",
-			SHORT_UUID(lc->uuid), who, (unsigned long long)region);
-		goto set_bit;
-	}
-
-	/* If rb_next is set, it means more than one node has this marked */
-	if (mark_list->rb_next)
-		set_bit = 0;
-
-	/* Must find this machine's entry to remove it */
-	for (; mark_list; mark_list = mark_list->rb_next) {
-		m = mark_list->rb_data;
-		if (m->nodeid == who)
-			break;
-	}
-
-	if (!mark_list) {
-		LOG_ERROR("Bad programming: searches disagree on results");
-		goto set_bit;
-	}
-
-	rbt_remove(&lc->mark_tree, mark_list);
-	rbt_free_node(&lc->mark_tree, mark_list);
+	list_for_each_safe(p, n, &lc->mark_list) {
+		/* FIXME: Use proper macros */
+		m = (struct mark_entry *)p;
+		if (m->region == region) {
+			if (m->nodeid == who) {
+				list_del_init(&m->list);
+				free(m);
+			} else
+				other_matches = 1;
+		}
+	}			
 
-set_bit:
-	/* Only clear the region if it is also in-sync */
-	if (set_bit && log_test_bit(lc->sync_bits, region))
+	/*
+	 * Clear region if:
+	 *  1) It is in-sync
+	 *  2) There are no other machines that have it marked
+	 */
+	if (!other_matches && log_test_bit(lc->sync_bits, region))
 		log_set_bit(lc, lc->clean_bits, region);
 
 	return 0;
--- cluster/cmirror/src/Attic/local.c	2008/01/18 17:11:07	1.1.2.8
+++ cluster/cmirror/src/Attic/local.c	2008/01/23 21:21:06	1.1.2.9
@@ -16,6 +16,7 @@
 #include "local.h"
 
 static int cn_fd;  /* Connector (netlink) socket fd */
+static int request_array[20];  /* for request counting (debugging) */
 
 static int kernel_recv_helper(void *data, int in_size)
 {
@@ -143,7 +144,8 @@
  */
 static int do_local_work(void *data)
 {
-	int r;
+	static int request_count = 0;
+	int r, i;
 	struct clog_tfr *tfr = NULL;
 
 	ENTER();
@@ -151,6 +153,12 @@
 	if (r)
 		return r;
 
+	request_array[tfr->request_type]++;
+	request_count++;
+	if (!(request_count % 10000))
+		for (i = 0; i < 20; i++)
+			LOG_PRINT("  %s: %d", RQ_TYPE(i), request_array[i]);
+
 	LOG_DBG("Request from kernel recieved [%s/%s/%llu]",
 		RQ_TYPE(tfr->request_type), SHORT_UUID(tfr->uuid),
 		(unsigned long long)tfr->seq);
@@ -287,6 +295,8 @@
 
 	ENTER();
 
+	memset(request_array, 0, sizeof(int)*20);
+
 	cn_fd = socket(PF_NETLINK, SOCK_DGRAM, NETLINK_CONNECTOR);
 	if (cn_fd < 0) {
 		EXIT();
@@ -321,3 +331,14 @@
 	EXIT();
 	return 0;
 }
+
+/*
+ * cleanup_local
+ *
+ * Clean up before exiting
+ */
+void cleanup_local(void)
+{
+	links_unregister(cn_fd);
+	close(cn_fd);
+}
--- cluster/cmirror/src/Attic/local.h	2007/08/23 19:57:31	1.1.2.1
+++ cluster/cmirror/src/Attic/local.h	2008/01/23 21:21:06	1.1.2.2
@@ -2,6 +2,7 @@
 #define __CLUSTER_LOG_LOCAL_DOT_H__
 
 int init_local(void);
+void cleanup_local(void);
 
 int kernel_send(struct clog_tfr *tfr);
 
--- cluster/cmirror/src/Attic/queues.c	2008/01/21 20:18:44	1.1.2.3
+++ cluster/cmirror/src/Attic/queues.c	2008/01/23 21:21:06	1.1.2.4
@@ -123,6 +123,17 @@
 }
 
 /*
+ * cleanup_queues
+ *
+ * Clean up before exiting
+ */
+void cleanup_queues(void)
+{
+	queue_dtr(cluster_queue);
+	queue_dtr(free_queue);
+}
+
+/*
  * queue_add_tail
  * @tfr: item to add
  * @q: receiving queue
--- cluster/cmirror/src/Attic/queues.h	2008/01/14 22:52:17	1.1.2.2
+++ cluster/cmirror/src/Attic/queues.h	2008/01/23 21:21:06	1.1.2.3
@@ -16,6 +16,7 @@
 extern struct queue *free_queue;
 
 int init_queues(void);
+void cleanup_queues(void);
 int queue_status(void);
 void queue_add_tail(struct clog_tfr *tfr, struct queue *q);
 void queue_add(struct clog_tfr *tfr, struct queue *q);