* [Cluster-devel] cluster/group/daemon app.c cpg.c gd_internal.h
@ 2007-01-05 18:49 teigland
0 siblings, 0 replies; 5+ messages in thread
From: teigland @ 2007-01-05 18:49 UTC (permalink / raw)
To: cluster-devel.redhat.com
CVSROOT: /cvs/cluster
Module name: cluster
Changes by: teigland at sourceware.org 2007-01-05 18:49:00
Modified files:
group/daemon : app.c cpg.c gd_internal.h
Log message:
groupd creates uint32 global id's for each group. It doesn't
use them itself, but provides them to each registered app to use
if it wants. (The dlm and gfs each use the global id in messages
to distinguish between different lockspaces/fs's.) groupd's
method of creating these gid's (local counter | local nodeid)
can result in duplicate gid's in the cluster given a somewhat
uncommon sequence of events.
bz 221629
Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/app.c.diff?cvsroot=cluster&r1=1.53&r2=1.54
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/cpg.c.diff?cvsroot=cluster&r1=1.37&r2=1.38
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/gd_internal.h.diff?cvsroot=cluster&r1=1.45&r2=1.46
--- cluster/group/daemon/app.c 2006/12/01 20:26:01 1.53
+++ cluster/group/daemon/app.c 2007/01/05 18:49:00 1.54
@@ -21,6 +21,8 @@
return "recover";
case MSG_APP_INTERNAL:
return "internal";
+ case MSG_GLOBAL_ID:
+ return "global_id";
}
return "unknown";
}
--- cluster/group/daemon/cpg.c 2006/12/01 20:26:01 1.37
+++ cluster/group/daemon/cpg.c 2007/01/05 18:49:00 1.38
@@ -149,10 +149,47 @@
queue_app_leave(g, nodeid);
}
+static uint32_t max_global_id(uint32_t add_nodeid)
+{
+ group_t *g;
+ uint32_t nodeid, counter, max_counter = 0, max_gid = 0;
+
+ list_for_each_entry(g, &gd_groups, list) {
+ nodeid = g->global_id & 0x0000FFFF;
+ counter = (g->global_id >> 16) & 0x0000FFFF;
+ if (nodeid != add_nodeid)
+ continue;
+ if (!max_counter || counter > max_counter) {
+ max_counter = counter;
+ max_gid = g->global_id;
+ }
+ }
+ return max_gid;
+}
+
+static int send_gid(uint32_t gid)
+{
+ group_t g;
+ msg_t msg;
+
+ /* just so log_group will work */
+ memset(&g, 0, sizeof(group_t));
+ strcpy(g.name, "groupd");
+
+ memset(&msg, 0, sizeof(msg));
+ msg.ms_type = MSG_GLOBAL_ID;
+ msg.ms_global_id = gid;
+
+ msg_bswap_out(&msg);
+
+ return send_message_groupd(&g, &msg, sizeof(msg), MSG_GLOBAL_ID);
+}
+
void process_groupd_confchg(void)
{
struct recovery_set *rs;
int i, found = 0;
+ uint32_t gid;
log_debug("groupd confchg total %d left %d joined %d",
saved_member_count, saved_left_count, saved_joined_count);
@@ -167,6 +204,23 @@
}
}
+ if (!groupd_joined)
+ goto next;
+
+ /* find any groups that were created in the past by a new node
+ and send it the id it used so it can initialize global_id_counter
+ to avoid creating a new group with a duplicate id */
+
+ for (i = 0; i < saved_joined_count; i++) {
+ gid = max_global_id(saved_joined[i].nodeid);
+ if (!gid)
+ continue;
+ log_debug("joined node %d had old max gid %x",
+ saved_joined[i].nodeid, gid);
+ send_gid(gid);
+ }
+
+ next:
if (found)
groupd_joined = 1;
else
@@ -235,12 +289,26 @@
msg_t *msg = (msg_t *) data;
char *buf;
char name[MAX_NAMELEN+1];
+ uint32_t to_nodeid, counter;
int len;
memset(&name, 0, sizeof(name));
msg_bswap_in(msg);
+ if (msg->ms_type == MSG_GLOBAL_ID) {
+ to_nodeid = msg->ms_global_id & 0x0000FFFF;
+ counter = (msg->ms_global_id >> 16) & 0x0000FFFF;
+
+ if (to_nodeid == our_nodeid) {
+ log_debug("recv global_id %x from %u cur counter %u",
+ msg->ms_global_id, nodeid, global_id_counter);
+ if (counter > global_id_counter)
+ global_id_counter = counter;
+ }
+ return;
+ }
+
if (handle == groupd_handle) {
memcpy(&name, &msg->ms_name, MAX_NAMELEN);
--- cluster/group/daemon/gd_internal.h 2006/12/01 20:26:01 1.45
+++ cluster/group/daemon/gd_internal.h 2007/01/05 18:49:00 1.46
@@ -189,6 +189,7 @@
#define MSG_APP_STARTED 2
#define MSG_APP_RECOVER 3
#define MSG_APP_INTERNAL 4
+#define MSG_GLOBAL_ID 5
#define MSG_VER_MAJOR 1
#define MSG_VER_MINOR 0
^ permalink raw reply [flat|nested] 5+ messages in thread* [Cluster-devel] cluster/group/daemon app.c cpg.c gd_internal.h
@ 2007-01-05 19:56 teigland
0 siblings, 0 replies; 5+ messages in thread
From: teigland @ 2007-01-05 19:56 UTC (permalink / raw)
To: cluster-devel.redhat.com
CVSROOT: /cvs/cluster
Module name: cluster
Branch: RHEL50
Changes by: teigland at sourceware.org 2007-01-05 19:56:09
Modified files:
group/daemon : app.c cpg.c gd_internal.h
Log message:
groupd creates uint32 global id's for each group. It doesn't
use them itself, but provides them to each registered app to use
if it wants. (The dlm and gfs each use the global id in messages
to distinguish between different lockspaces/fs's.) groupd's
method of creating these gid's (local counter | local nodeid)
can result in duplicate gid's in the cluster given a somewhat
uncommon sequence of events.
bz 221629
Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/app.c.diff?cvsroot=cluster&only_with_tag=RHEL50&r1=1.52.4.1&r2=1.52.4.2
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/cpg.c.diff?cvsroot=cluster&only_with_tag=RHEL50&r1=1.36.4.1&r2=1.36.4.2
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/gd_internal.h.diff?cvsroot=cluster&only_with_tag=RHEL50&r1=1.44.4.1&r2=1.44.4.2
--- cluster/group/daemon/app.c 2006/12/01 20:27:36 1.52.4.1
+++ cluster/group/daemon/app.c 2007/01/05 19:56:09 1.52.4.2
@@ -21,6 +21,8 @@
return "recover";
case MSG_APP_INTERNAL:
return "internal";
+ case MSG_GLOBAL_ID:
+ return "global_id";
}
return "unknown";
}
--- cluster/group/daemon/cpg.c 2006/12/01 20:27:36 1.36.4.1
+++ cluster/group/daemon/cpg.c 2007/01/05 19:56:09 1.36.4.2
@@ -149,10 +149,47 @@
queue_app_leave(g, nodeid);
}
+static uint32_t max_global_id(uint32_t add_nodeid)
+{
+ group_t *g;
+ uint32_t nodeid, counter, max_counter = 0, max_gid = 0;
+
+ list_for_each_entry(g, &gd_groups, list) {
+ nodeid = g->global_id & 0x0000FFFF;
+ counter = (g->global_id >> 16) & 0x0000FFFF;
+ if (nodeid != add_nodeid)
+ continue;
+ if (!max_counter || counter > max_counter) {
+ max_counter = counter;
+ max_gid = g->global_id;
+ }
+ }
+ return max_gid;
+}
+
+static int send_gid(uint32_t gid)
+{
+ group_t g;
+ msg_t msg;
+
+ /* just so log_group will work */
+ memset(&g, 0, sizeof(group_t));
+ strcpy(g.name, "groupd");
+
+ memset(&msg, 0, sizeof(msg));
+ msg.ms_type = MSG_GLOBAL_ID;
+ msg.ms_global_id = gid;
+
+ msg_bswap_out(&msg);
+
+ return send_message_groupd(&g, &msg, sizeof(msg), MSG_GLOBAL_ID);
+}
+
void process_groupd_confchg(void)
{
struct recovery_set *rs;
int i, found = 0;
+ uint32_t gid;
log_debug("groupd confchg total %d left %d joined %d",
saved_member_count, saved_left_count, saved_joined_count);
@@ -167,6 +204,23 @@
}
}
+ if (!groupd_joined)
+ goto next;
+
+ /* find any groups that were created in the past by a new node
+ and send it the id it used so it can initialize global_id_counter
+ to avoid creating a new group with a duplicate id */
+
+ for (i = 0; i < saved_joined_count; i++) {
+ gid = max_global_id(saved_joined[i].nodeid);
+ if (!gid)
+ continue;
+ log_debug("joined node %d had old max gid %x",
+ saved_joined[i].nodeid, gid);
+ send_gid(gid);
+ }
+
+ next:
if (found)
groupd_joined = 1;
else
@@ -235,12 +289,26 @@
msg_t *msg = (msg_t *) data;
char *buf;
char name[MAX_NAMELEN+1];
+ uint32_t to_nodeid, counter;
int len;
memset(&name, 0, sizeof(name));
msg_bswap_in(msg);
+ if (msg->ms_type == MSG_GLOBAL_ID) {
+ to_nodeid = msg->ms_global_id & 0x0000FFFF;
+ counter = (msg->ms_global_id >> 16) & 0x0000FFFF;
+
+ if (to_nodeid == our_nodeid) {
+ log_debug("recv global_id %x from %u cur counter %u",
+ msg->ms_global_id, nodeid, global_id_counter);
+ if (counter > global_id_counter)
+ global_id_counter = counter;
+ }
+ return;
+ }
+
if (handle == groupd_handle) {
memcpy(&name, &msg->ms_name, MAX_NAMELEN);
--- cluster/group/daemon/gd_internal.h 2006/12/01 20:27:36 1.44.4.1
+++ cluster/group/daemon/gd_internal.h 2007/01/05 19:56:09 1.44.4.2
@@ -189,6 +189,7 @@
#define MSG_APP_STARTED 2
#define MSG_APP_RECOVER 3
#define MSG_APP_INTERNAL 4
+#define MSG_GLOBAL_ID 5
#define MSG_VER_MAJOR 1
#define MSG_VER_MINOR 0
^ permalink raw reply [flat|nested] 5+ messages in thread* [Cluster-devel] cluster/group/daemon app.c cpg.c gd_internal.h
@ 2007-01-05 18:50 teigland
0 siblings, 0 replies; 5+ messages in thread
From: teigland @ 2007-01-05 18:50 UTC (permalink / raw)
To: cluster-devel.redhat.com
CVSROOT: /cvs/cluster
Module name: cluster
Branch: RHEL5
Changes by: teigland at sourceware.org 2007-01-05 18:50:01
Modified files:
group/daemon : app.c cpg.c gd_internal.h
Log message:
groupd creates uint32 global id's for each group. It doesn't
use them itself, but provides them to each registered app to use
if it wants. (The dlm and gfs each use the global id in messages
to distinguish between different lockspaces/fs's.) groupd's
method of creating these gid's (local counter | local nodeid)
can result in duplicate gid's in the cluster given a somewhat
uncommon sequence of events.
bz 221629
Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/app.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.52.2.1&r2=1.52.2.2
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/cpg.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.36.2.1&r2=1.36.2.2
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/gd_internal.h.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.44.2.1&r2=1.44.2.2
--- cluster/group/daemon/app.c 2006/12/01 20:27:28 1.52.2.1
+++ cluster/group/daemon/app.c 2007/01/05 18:50:01 1.52.2.2
@@ -21,6 +21,8 @@
return "recover";
case MSG_APP_INTERNAL:
return "internal";
+ case MSG_GLOBAL_ID:
+ return "global_id";
}
return "unknown";
}
--- cluster/group/daemon/cpg.c 2006/12/01 20:27:28 1.36.2.1
+++ cluster/group/daemon/cpg.c 2007/01/05 18:50:01 1.36.2.2
@@ -149,10 +149,47 @@
queue_app_leave(g, nodeid);
}
+static uint32_t max_global_id(uint32_t add_nodeid)
+{
+ group_t *g;
+ uint32_t nodeid, counter, max_counter = 0, max_gid = 0;
+
+ list_for_each_entry(g, &gd_groups, list) {
+ nodeid = g->global_id & 0x0000FFFF;
+ counter = (g->global_id >> 16) & 0x0000FFFF;
+ if (nodeid != add_nodeid)
+ continue;
+ if (!max_counter || counter > max_counter) {
+ max_counter = counter;
+ max_gid = g->global_id;
+ }
+ }
+ return max_gid;
+}
+
+static int send_gid(uint32_t gid)
+{
+ group_t g;
+ msg_t msg;
+
+ /* just so log_group will work */
+ memset(&g, 0, sizeof(group_t));
+ strcpy(g.name, "groupd");
+
+ memset(&msg, 0, sizeof(msg));
+ msg.ms_type = MSG_GLOBAL_ID;
+ msg.ms_global_id = gid;
+
+ msg_bswap_out(&msg);
+
+ return send_message_groupd(&g, &msg, sizeof(msg), MSG_GLOBAL_ID);
+}
+
void process_groupd_confchg(void)
{
struct recovery_set *rs;
int i, found = 0;
+ uint32_t gid;
log_debug("groupd confchg total %d left %d joined %d",
saved_member_count, saved_left_count, saved_joined_count);
@@ -167,6 +204,23 @@
}
}
+ if (!groupd_joined)
+ goto next;
+
+ /* find any groups that were created in the past by a new node
+ and send it the id it used so it can initialize global_id_counter
+ to avoid creating a new group with a duplicate id */
+
+ for (i = 0; i < saved_joined_count; i++) {
+ gid = max_global_id(saved_joined[i].nodeid);
+ if (!gid)
+ continue;
+ log_debug("joined node %d had old max gid %x",
+ saved_joined[i].nodeid, gid);
+ send_gid(gid);
+ }
+
+ next:
if (found)
groupd_joined = 1;
else
@@ -235,12 +289,26 @@
msg_t *msg = (msg_t *) data;
char *buf;
char name[MAX_NAMELEN+1];
+ uint32_t to_nodeid, counter;
int len;
memset(&name, 0, sizeof(name));
msg_bswap_in(msg);
+ if (msg->ms_type == MSG_GLOBAL_ID) {
+ to_nodeid = msg->ms_global_id & 0x0000FFFF;
+ counter = (msg->ms_global_id >> 16) & 0x0000FFFF;
+
+ if (to_nodeid == our_nodeid) {
+ log_debug("recv global_id %x from %u cur counter %u",
+ msg->ms_global_id, nodeid, global_id_counter);
+ if (counter > global_id_counter)
+ global_id_counter = counter;
+ }
+ return;
+ }
+
if (handle == groupd_handle) {
memcpy(&name, &msg->ms_name, MAX_NAMELEN);
--- cluster/group/daemon/gd_internal.h 2006/12/01 20:27:28 1.44.2.1
+++ cluster/group/daemon/gd_internal.h 2007/01/05 18:50:01 1.44.2.2
@@ -189,6 +189,7 @@
#define MSG_APP_STARTED 2
#define MSG_APP_RECOVER 3
#define MSG_APP_INTERNAL 4
+#define MSG_GLOBAL_ID 5
#define MSG_VER_MAJOR 1
#define MSG_VER_MINOR 0
^ permalink raw reply [flat|nested] 5+ messages in thread* [Cluster-devel] cluster/group/daemon app.c cpg.c gd_internal.h
@ 2006-06-22 18:39 teigland
0 siblings, 0 replies; 5+ messages in thread
From: teigland @ 2006-06-22 18:39 UTC (permalink / raw)
To: cluster-devel.redhat.com
CVSROOT: /cvs/cluster
Module name: cluster
Changes by: teigland at sourceware.org 2006-06-22 18:39:27
Modified files:
group/daemon : app.c cpg.c gd_internal.h
Log message:
improvements to debug messages
Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/app.c.diff?cvsroot=cluster&r1=1.31&r2=1.32
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/cpg.c.diff?cvsroot=cluster&r1=1.26&r2=1.27
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/gd_internal.h.diff?cvsroot=cluster&r1=1.34&r2=1.35
--- cluster/group/daemon/app.c 2006/06/21 20:43:54 1.31
+++ cluster/group/daemon/app.c 2006/06/22 18:39:26 1.32
@@ -10,6 +10,19 @@
int nodeid;
};
+char *msg_type(int type)
+{
+ switch (type) {
+ case MSG_APP_STOPPED:
+ return "stopped";
+ case MSG_APP_STARTED:
+ return "started";
+ case MSG_APP_INTERNAL:
+ return "internal";
+ }
+ return "unknown";
+}
+
void msg_bswap_out(msg_t *msg)
{
msg->ms_version[0] = cpu_to_le32(MSG_VER_MAJOR);
@@ -547,23 +560,8 @@
int queue_app_message(group_t *g, struct save_msg *save)
{
- /*
- char *m = "unknown";
-
- switch (save->msg.ms_type) {
- case MSG_APP_STOPPED:
- m = "stopped";
- break;
- case MSG_APP_STARTED:
- m = "started";
- break;
- case MSG_APP_INTERNAL:
- m = "internal";
- break;
- }
- log_group(g, "queue message %s from %d", m, save->nodeid);
- */
-
+ /* log_group(g, "queue message %s from %d",
+ msg_type(save->msg.ms_type), save->nodeid); */
list_add_tail(&save->list, &g->messages);
return 0;
}
@@ -1092,9 +1090,12 @@
ev = a->current_event;
if (!ev || ev->id != save->msg.ms_event_id) {
- log_group(g, "ignore msg from %d id %llx type %d",
- save->nodeid, save->msg.ms_event_id,
- save->msg.ms_type);
+ if (!save->print_ignore) {
+ log_group(g, "ignore msg from %d id %llx %s",
+ save->nodeid, save->msg.ms_event_id,
+ msg_type(save->msg.ms_type));
+ save->print_ignore = 1;
+ }
continue;
}
--- cluster/group/daemon/cpg.c 2006/06/21 20:43:54 1.26
+++ cluster/group/daemon/cpg.c 2006/06/22 18:39:26 1.27
@@ -51,7 +51,7 @@
g->memb_count--;
free(node);
- log_group(g, "group del node %d total %d - down",
+ log_group(g, "cpg del node %d total %d - down",
nodeid, g->memb_count);
/* purge any queued join/leave events from the dead node */
@@ -97,7 +97,7 @@
node = new_node(saved_member[i].nodeId);
list_add_tail(&node->list, &g->memb);
g->memb_count++;
- log_group(g, "group add node %d total %d - init",
+ log_group(g, "cpg add node %d total %d",
node->nodeid, g->memb_count);
}
@@ -114,7 +114,7 @@
node = new_node(nodeid);
list_add_tail(&node->list, &g->memb);
g->memb_count++;
- log_group(g, "group add node %d total %d",
+ log_group(g, "cpg add node %d total %d",
node->nodeid, g->memb_count);
}
@@ -137,7 +137,7 @@
g->memb_count--;
free(node);
- log_group(g, "group del node %d total %d", nodeid, g->memb_count);
+ log_group(g, "cpg del node %d total %d", nodeid, g->memb_count);
queue_app_leave(g, nodeid);
}
@@ -146,7 +146,7 @@
{
int i, found = 0;
- log_debug("process_groupd_confchg members %d -%d +%d",
+ log_debug("groupd confchg total %d left %d joined %d",
saved_member_count, saved_left_count, saved_joined_count);
memcpy(&groupd_cpg_member, &saved_member, sizeof(saved_member));
@@ -234,7 +234,8 @@
}
/*
- log_group(g, "deliver from %d len %d", nodeid, data_len);
+ log_group(g, "deliver_cb from %d len %d type %s", nodeid, data_len,
+ msg_type(msg->ms_type));
*/
save = malloc(sizeof(struct save_msg));
@@ -265,19 +266,19 @@
g = find_group_by_handle(saved_handle);
if (!g) {
- log_debug("process_confchg: no group for handle %llx name %s",
+ log_debug("confchg: no group for handle %llx name %s",
saved_handle, saved_name.value);
return;
}
- log_group(g, "process_confchg members %d -%d +%d",
- saved_member_count, saved_left_count, saved_joined_count);
+ log_group(g, "confchg left %d joined %d total %d",
+ saved_left_count, saved_joined_count, saved_member_count);
for (i = 0; i < saved_joined_count; i++)
process_node_join(g, saved_joined[i].nodeId);
for (i = 0; i < saved_left_count; i++) {
- log_group(g, "node %d removed reason %d",
+ log_group(g, "confchg removed node %d reason %d",
saved_left[i].nodeId, saved_left[i].reason);
if (saved_left[i].reason == CPG_REASON_LEAVE)
@@ -306,8 +307,10 @@
}
}
- log_debug("%d:%s confchg members %d -%d +%d", level, name,
+ /*
+ log_debug("%d:%s confchg_cb total %d left %d joined %d", level, name,
member_list_entries, left_list_entries, joined_list_entries);
+ */
saved_handle = handle;
--- cluster/group/daemon/gd_internal.h 2006/06/21 20:43:54 1.34
+++ cluster/group/daemon/gd_internal.h 2006/06/22 18:39:26 1.35
@@ -201,6 +201,7 @@
struct save_msg {
struct list_head list;
int nodeid;
+ int print_ignore;
int msg_len;
msg_t msg;
char *msg_long;
@@ -250,6 +251,7 @@
void msg_bswap_in(msg_t *msg);
struct recovery_set *get_recovery_set(int nodeid);
void groupd_down(int nodeid);
+char *msg_type(int type);
/* main.c */
void app_stop(app_t *a);
^ permalink raw reply [flat|nested] 5+ messages in thread* [Cluster-devel] cluster/group/daemon app.c cpg.c gd_internal.h
@ 2006-06-21 18:10 teigland
0 siblings, 0 replies; 5+ messages in thread
From: teigland @ 2006-06-21 18:10 UTC (permalink / raw)
To: cluster-devel.redhat.com
CVSROOT: /cvs/cluster
Module name: cluster
Changes by: teigland at sourceware.org 2006-06-21 18:10:23
Modified files:
group/daemon : app.c cpg.c gd_internal.h
Log message:
Don't finalize/terminate a local group leave until we see that all
remaining group members have stopped.
Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/app.c.diff?cvsroot=cluster&r1=1.29&r2=1.30
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/cpg.c.diff?cvsroot=cluster&r1=1.24&r2=1.25
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/gd_internal.h.diff?cvsroot=cluster&r1=1.32&r2=1.33
--- cluster/group/daemon/app.c 2006/06/20 20:26:08 1.29
+++ cluster/group/daemon/app.c 2006/06/21 18:10:23 1.30
@@ -592,9 +592,11 @@
return (ev->nodeid == our_nodeid);
}
-/* called after the local app has acked that it is stopped as part
- of our own leave. We've gotten the final confchg for our leave
- so we can't send anything out to the group at this point. */
+/* Called after all nodes have acked that they're stopped for our
+ leave. We get their stopped messages even though we've left the
+ cpg because the messages are sent through the groupd cpg.
+ groupd_down() will fill in stops for us for nodes that fail before
+ sending stopped for our leave. */
void finalize_our_leave(group_t *g)
{
@@ -620,15 +622,6 @@
msg_t msg;
event_t *ev = g->app->current_event;
- /* FIXME: see other fixme that mentions that leaving nodes
- should also send a stopped message to be counted by the
- remaining nodes before they move on to restarted */
-
- if (ev && ev->state == EST_LEAVE_STOP_WAIT && is_our_leave(ev)) {
- finalize_our_leave(g);
- return 0;
- }
-
memset(&msg, 0, sizeof(msg));
msg.ms_type = MSG_APP_STOPPED;
msg.ms_global_id = g->global_id;
@@ -855,18 +848,6 @@
case EST_LEAVE_BEGIN:
ev->state = EST_LEAVE_STOP_WAIT;
app_stop(a);
-
- /* FIXME: have leaving node send a stopped message after
- the app acks that it's stopped, and then make the
- other nodes wait for this stopped message instead of
- just setting the leaving node as stopped here */
-
- if (!is_our_leave(ev)) {
- node = find_app_node(a, ev->nodeid);
- ASSERT(node);
- node->stopped = 1;
- }
-
break;
case EST_LEAVE_STOP_WAIT:
@@ -877,6 +858,12 @@
break;
case EST_LEAVE_ALL_STOPPED:
+ if (is_our_leave(ev)) {
+ /* frees group structure */
+ finalize_our_leave(g);
+ rv = -1;
+ break;
+ }
ev->state = EST_LEAVE_START_WAIT;
node = find_app_node(a, ev->nodeid);
@@ -1358,14 +1345,16 @@
{
app_t *a = g->app;
event_t *ev = NULL;
- int rv = 0;
+ int rv = 0, ret;
if (a->current_event) {
- /* this assumes that we never remove/free the group in
- process_current_event */
-
rv += process_app_messages(g);
- rv += process_current_event(g);
+
+ ret = process_current_event(g);
+ if (ret < 0)
+ goto out;
+ rv += ret;
+
rv += recover_current_event(g);
} else {
/* We only take on a new non-recovery event if there are
@@ -1407,3 +1396,30 @@
return rv;
}
+/* This is a bit of a hack that may not be entirely necessary. The problem
+ we're solving with this function is when a node leaves a group and is
+ collecting all the "stopped" messages from the remaining members, some
+ of those members may fail, so we wouldn't get a stopped message from
+ them and never finalize_our_leave (terminate the group). I'm not entirely
+ sure that we _need_ to wait for stopped messages from remaining members
+ before we do the finalize_our_leave/terminate... The reasoning@this
+ point is that when gfs is withdrawing, we want to be sure gfs is
+ suspended everywhere before we leave the lockspace (which happens at
+ terminate for the withdraw/leave) */
+
+void groupd_down(int nodeid)
+{
+ group_t *g;
+
+ list_for_each_entry(g, &gd_groups, list) {
+ if (g->app &&
+ g->app->current_event &&
+ g->app->current_event->state == EST_LEAVE_STOP_WAIT &&
+ is_our_leave(g->app->current_event)) {
+ log_group(g, "groupd down on %d, push our leave",
+ nodeid);
+ mark_node_stopped(g->app, nodeid);
+ }
+ }
+}
+
--- cluster/group/daemon/cpg.c 2006/06/20 20:26:08 1.24
+++ cluster/group/daemon/cpg.c 2006/06/21 18:10:23 1.25
@@ -171,8 +171,10 @@
where groupd exits but cman is still running. */
for (i = 0; i < saved_left_count; i++) {
- if (saved_left[i].reason != CPG_REASON_LEAVE)
+ if (saved_left[i].reason != CPG_REASON_LEAVE) {
add_recovery_set(saved_left[i].nodeId);
+ groupd_down(saved_left[i].nodeId);
+ }
}
}
--- cluster/group/daemon/gd_internal.h 2006/06/20 20:26:08 1.32
+++ cluster/group/daemon/gd_internal.h 2006/06/21 18:10:23 1.33
@@ -248,6 +248,7 @@
void msg_bswap_out(msg_t *msg);
void msg_bswap_in(msg_t *msg);
struct recovery_set *get_recovery_set(int nodeid);
+void groupd_down(int nodeid);
/* main.c */
void app_stop(app_t *a);
^ permalink raw reply [flat|nested] 5+ messages in thread
end of thread, other threads:[~2007-01-05 19:56 UTC | newest]
Thread overview: 5+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2007-01-05 18:49 [Cluster-devel] cluster/group/daemon app.c cpg.c gd_internal.h teigland
-- strict thread matches above, loose matches on Subject: below --
2007-01-05 19:56 teigland
2007-01-05 18:50 teigland
2006-06-22 18:39 teigland
2006-06-21 18:10 teigland
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).