[Cluster-devel] cluster/fence/fenced fd.h group.c member_cman. ...

[Cluster-devel] cluster/fence/fenced fd.h group.c member_cman. ...

[teigland]

CVSROOT:	/cvs/cluster
Module name:	cluster
Changes by:	teigland at sourceware.org	2006-06-20 18:11:59

Modified files:
	fence/fenced   : fd.h group.c member_cman.c recover.c 

Log message:
	- keep cman member list updated by using cman callbacks instead
	of polling cman for the latest list every time we're interested
	- only bypass fencing of a node if it's both a cman member and
	in the groupd cpg (has started groupd past the point of checking
	for residual gfs/dlm state)

Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/fence/fenced/fd.h.diff?cvsroot=cluster&r1=1.20&r2=1.21
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/fence/fenced/group.c.diff?cvsroot=cluster&r1=1.8&r2=1.9
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/fence/fenced/member_cman.c.diff?cvsroot=cluster&r1=1.10&r2=1.11
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/fence/fenced/recover.c.diff?cvsroot=cluster&r1=1.22&r2=1.23

--- cluster/fence/fenced/fd.h	2006/03/17 19:19:56	1.20
+++ cluster/fence/fenced/fd.h	2006/06/20 18:11:58	1.21
@@ -179,6 +179,7 @@
 int setup_groupd(void);
 void exit_groupd(void);
 int process_groupd(void);
+int in_groupd_cpg(int nodeid);
 
 /* member_xxx.c */
 int setup_member(void);
--- cluster/fence/fenced/group.c	2006/03/02 20:00:09	1.8
+++ cluster/fence/fenced/group.c	2006/06/20 18:11:58	1.9
@@ -176,3 +176,22 @@
 	group_exit(gh);
 }
 
+int in_groupd_cpg(int nodeid)
+{
+	group_data_t data;
+	int i, rv;
+
+	memset(&data, 0, sizeof(data));
+
+	rv = group_get_group(-1, "groupd", &data);
+	if (rv)
+		return 0;
+
+	for (i = 0; i < data.member_count; i++) {
+		if (data.members[i] == nodeid)
+			return 1;
+	}
+
+	return 0;
+}
+
--- cluster/fence/fenced/member_cman.c	2006/03/20 19:21:26	1.10
+++ cluster/fence/fenced/member_cman.c	2006/06/20 18:11:58	1.11
@@ -16,12 +16,13 @@
 #define BUFLEN		128
 
 static cman_handle_t	ch;
-static cman_node_t	cluster_nodes[MAX_NODES];
-static cman_node_t	new_nodes[MAX_NODES];
-static int		cluster_count;
 static int		cman_cb;
 static int		cman_reason;
+static int		cman_quorate;
+static cman_node_t	cman_nodes[MAX_NODES];
+static int		cman_node_count;
 static char		name_buf[CMAN_MAX_NODENAME_LEN+1];
+
 extern struct list_head domains;
 
 char			*our_name;
@@ -69,14 +70,38 @@
 {
 	int i;
 
-	for (i = 0; i < cluster_count; i++) {
-		if (name_equal(cluster_nodes[i].cn_name, name))
-			return &cluster_nodes[i];
+	for (i = 0; i < cman_node_count; i++) {
+		if (name_equal(cman_nodes[i].cn_name, name))
+			return &cman_nodes[i];
 	}
 	return NULL;
 }
 
-static void member_callback(cman_handle_t h, void *private, int reason, int arg)
+static void statechange(void)
+{
+	int rv;
+
+	cman_quorate = cman_is_quorate(ch);
+	cman_node_count = 0;
+	memset(&cman_nodes, 0, sizeof(cman_nodes));
+
+	rv = cman_get_nodes(ch, MAX_NODES, &cman_node_count, cman_nodes);
+	if (rv < 0)
+		log_print("cman_get_nodes error %d %d", rv, errno);
+}
+
+static void process_cman_callback(void)
+{
+	switch (cman_reason) {
+	case CMAN_REASON_STATECHANGE:
+		statechange();
+		break;
+	default:
+		break;
+	}
+}
+
+static void cman_callback(cman_handle_t h, void *private, int reason, int arg)
 {
 	cman_cb = 1;
 	cman_reason = reason;
@@ -100,9 +125,10 @@
 		if (rv < 0)
 			break;
 
-		if (cman_cb)
+		if (cman_cb) {
 			cman_cb = 0;
-		else
+			process_cman_callback();
+		} else
 			break;
 	}
 
@@ -125,13 +151,15 @@
 		return -ENOTCONN;
 	}
 
-	rv = cman_start_notification(ch, member_callback);
+	rv = cman_start_notification(ch, cman_callback);
 	if (rv < 0) {
 		log_error("cman_start_notification error %d %d", rv, errno);
 		cman_finish(ch);
 		return rv;
 	}
 
+	statechange();
+
 	fd = cman_get_fd(ch);
 
 	/* FIXME: wait here for us to be a member of the cluster */
@@ -151,6 +179,7 @@
 
 	log_debug("our_nodeid %d our_name %s", our_nodeid, our_name);
 	rv = 0;
+
  out:
 	return fd;
 }
@@ -160,44 +189,20 @@
 	cman_finish(ch);
 }
 
-/* FIXME: just use cman callbacks to keep the cman membership list up to
-   date and don't bother with this function */
-
-int update_cluster_members(void)
-{
-	int rv, count;
-
-	count = 0;
-	memset(new_nodes, 0, sizeof(new_nodes));
-
-	rv = cman_get_nodes(ch, MAX_NODES, &count, new_nodes);
-	if (rv < 0) {
-		log_error("cman_get_nodes error %d %d", rv, errno);
-		return rv;
-	}
-
-	if (count < cluster_count)
-		log_error("decrease in cluster nodes %d %d",
-			  count, cluster_count);
-
-	cluster_count = count;
-	memcpy(cluster_nodes, new_nodes, sizeof(cluster_nodes));
-
-	log_debug("node count %d", count);
-	return 0;
-}
-
-/* update_cluster_members() is usually called prior to calling this */
-
 int is_member(char *name)
 {
 	cman_node_t *cn;
 
 	cn = find_cluster_node_name(name);
-	if (cn && cn->cn_member)
-		return TRUE;
-	log_debug("node \"%s\" not a member, cn %d", name, cn ? 1 : 0);
-	return FALSE;
+	if (cn && cn->cn_member) {
+		if (in_groupd_cpg(cn->cn_nodeid))
+			return 1;
+		log_debug("node \"%s\" not in groupd cpg", name);
+		return 0;
+	}
+
+	log_debug("node \"%s\" not a cman member, cn %d", name, cn ? 1 : 0);
+	return 0;
 }
 
 fd_node_t *get_new_node(fd_t *fd, int nodeid, char *in_name)
--- cluster/fence/fenced/recover.c	2005/06/01 10:16:08	1.22
+++ cluster/fence/fenced/recover.c	2006/06/20 18:11:58	1.23
@@ -155,8 +155,6 @@
 {
 	fd_node_t *prev_node, *safe;
 
-	update_cluster_members();
-
 	/* complete list initialised in init_nodes() to all nodes from ccs */
 	if (list_empty(&fd->complete))
 		log_debug("first complete list empty warning");
@@ -202,8 +200,6 @@
 
 	num_victims = list_count(&fd->victims);
 
-	update_cluster_members();
-
 	list_for_each_entry_safe(node, safe, &fd->victims, list) {
 		if (is_member(node->name)) {
 			list_del(&node->list);
@@ -303,7 +299,6 @@
 	while (!list_empty(&fd->victims)) {
 		node = list_entry(fd->victims.next, fd_node_t, list);
 
-		update_cluster_members();
 		if (is_member(node->name)) {
 			log_debug("averting fence of node %s", node->name);
 			list_del(&node->list);