[Cluster-devel] Re: [RESEND][PATCH] dlm: enhancing dlm_controld (pcmk) to be able to handle redundant rings

[David Teigland <teigland@redhat.com>]

On Wed, Oct 28, 2009 at 01:08:43PM +0800, Jiaju Zhang wrote:
> Hello David,
> 
> Per the discussion we had before, could we agree on the implementation
> like below? Or what other aspect I need to improve?
> 
> BTW, if the auto-detect bits can't be merged into upstream, can the
> command-line option bits be merged? Thanks a lot :-)

Hi, I've made some changes to the patch, and only tested that rrp_mode
none still results in tcp.  This patch is also for cluster.git STABLE3
branch, but I'll also push it to dlm.git if this works for you.
Dave


From e8c2ab811f02e891d6bc374b0d9aa43408d90456 Mon Sep 17 00:00:00 2001
From: David Teigland <teigland@redhat.com>
Date: Wed, 4 Nov 2009 15:20:47 -0600
Subject: [PATCH] dlm_controld: detect lowcomms protocol

based on value of totem/rrp_mode in confdb.
Also allow protocol to be set on command line.

Based on initial patch from Jiaju Zhang <jjzhang.linux@gmail.com>

Signed-off-by: Jiaju Zhang <jjzhang.linux@gmail.com>
Signed-off-by: David Teigland <teigland@redhat.com>
---
 group/dlm_controld/action.c     |   64 ++++++++++++++++++++++++++++++++++++++-
 group/dlm_controld/config.c     |    7 ++--
 group/dlm_controld/dlm_daemon.h |    6 ++++
 group/dlm_controld/main.c       |   16 ++++++----
 4 files changed, 82 insertions(+), 11 deletions(-)

diff --git a/group/dlm_controld/action.c b/group/dlm_controld/action.c
index 229823b..1069418 100644
--- a/group/dlm_controld/action.c
+++ b/group/dlm_controld/action.c
@@ -1,6 +1,9 @@
 #include "dlm_daemon.h"
 #include "config.h"
 
+#include <corosync/corotypes.h>
+#include <corosync/confdb.h>
+
 static int dir_members[MAX_NODES];
 static int dir_members_count;
 static int comms_nodes[MAX_NODES];
@@ -12,6 +15,58 @@ static char mg_name[DLM_LOCKSPACE_LEN+1];
 #define SPACES_DIR    "/sys/kernel/config/dlm/cluster/spaces"
 #define COMMS_DIR     "/sys/kernel/config/dlm/cluster/comms"
 
+static int detect_protocol(void)
+{
+	confdb_handle_t handle;
+	hdb_handle_t totem_handle;
+	char key_value[256];
+	size_t value_len;
+	int rv, proto = -1;
+	confdb_callbacks_t callbacks = {
+		.confdb_key_change_notify_fn = NULL,
+		.confdb_object_create_change_notify_fn = NULL,
+		.confdb_object_delete_change_notify_fn = NULL
+	};
+
+	rv = confdb_initialize(&handle, &callbacks);
+	if (rv != CS_OK) {
+		log_error("confdb_initialize error %d", rv);
+		return -1; 
+	}
+
+	rv = confdb_object_find_start(handle, OBJECT_PARENT_HANDLE);
+	if (rv != CS_OK) {
+		log_error("confdb_object_find_start error %d", rv);
+		goto out;
+	}
+
+	rv = confdb_object_find(handle, OBJECT_PARENT_HANDLE,
+				"totem", strlen("totem"), &totem_handle);
+	if (rv != CS_OK) {
+		log_error("confdb_object_find error %d", rv);
+		goto out;
+	}
+
+	rv = confdb_key_get(handle, totem_handle,
+			    "rrp_mode", strlen("rrp_mode"),
+			    key_value, &value_len);
+	if (rv != CS_OK) {
+		log_error("confdb_key_get error %d", rv);
+		goto out;
+	}
+
+	key_value[value_len] = '\0';
+	log_debug("totem/rrp_mode = '%s'", key_value);
+
+	if (!strcmp(key_value, "none"))
+		proto = PROTO_TCP;
+	else
+		proto = PROTO_SCTP;
+ out:
+	confdb_finalize(handle);
+	return proto;
+}
+
 /* look for an id that matches in e.g. /sys/fs/gfs/bull\:x/lock_module/id
    and then extract the "x" as the name */
 
@@ -824,7 +879,14 @@ int setup_configfs(void)
 		set_configfs_debug(cfgk_debug);
 	if (cfgk_timewarn != -1)
 		set_configfs_timewarn(cfgk_timewarn);
-	if (cfgk_protocol != -1)
+
+	if (cfgk_protocol == PROTO_DETECT) {
+		rv = detect_protocol();
+		if (rv == PROTO_TCP || rv == PROTO_SCTP)
+			cfgk_protocol = rv;
+	}
+
+	if (cfgk_protocol == PROTO_TCP || cfgk_protocol == PROTO_SCTP)
 		set_configfs_protocol(cfgk_protocol);
 
 	return 0;
diff --git a/group/dlm_controld/config.c b/group/dlm_controld/config.c
index 16c4efb..1720e7a 100644
--- a/group/dlm_controld/config.c
+++ b/group/dlm_controld/config.c
@@ -25,9 +25,6 @@
 #include "config.h"
 #include "ccs.h"
 
-#define PROTO_TCP  0
-#define PROTO_SCTP 1
-
 int ccs_handle;
 
 /* when not set in cluster.conf, a node's default weight is 1 */
@@ -199,6 +196,8 @@ static void read_ccs_protocol(const char *path, int *config_val)
 		val = PROTO_TCP;
 	else if (!strncasecmp(str, "sctp", 4))
 		val = PROTO_SCTP;
+	else if (!strncasecmp(str, "detect", 6))
+		val = PROTO_DETECT;
 	else {
 		log_error("ignore invalid value %s for %s", str, path);
 		return;
@@ -235,6 +234,7 @@ int setup_ccs(void)
 {
 	int cd, rv;
 
+	/* skip things that cannot be changed while running */
 	if (ccs_handle)
 		goto update;
 
@@ -270,7 +270,6 @@ int setup_ccs(void)
 			read_ccs_int(GFS_PLOCK_OWNERSHIP_PATH, &cfgd_plock_ownership);
 	}
 
-
 	/* The following can be changed while running */
  update:
 	if (!optd_plock_debug) {
diff --git a/group/dlm_controld/dlm_daemon.h b/group/dlm_controld/dlm_daemon.h
index 0ca895a..dd6c7cc 100644
--- a/group/dlm_controld/dlm_daemon.h
+++ b/group/dlm_controld/dlm_daemon.h
@@ -66,6 +66,12 @@
 #define GROUP_LIBGROUP	2
 #define GROUP_LIBCPG	3
 
+/* cfgk_protocol */
+
+#define PROTO_TCP  0
+#define PROTO_SCTP 1
+#define PROTO_DETECT 2
+
 extern int daemon_debug_opt;
 extern int daemon_quit;
 extern int cluster_down;
diff --git a/group/dlm_controld/main.c b/group/dlm_controld/main.c
index 712bed6..af96527 100644
--- a/group/dlm_controld/main.c
+++ b/group/dlm_controld/main.c
@@ -1089,6 +1089,9 @@ static void print_usage(void)
 	printf("  -D		Enable debugging to stderr and don't fork\n");
 	printf("  -L		Enable debugging to log file\n");
 	printf("  -K		Enable kernel dlm debugging messages\n");
+	printf("  -r <num>	dlm kernel lowcomms protocol, 0 tcp, 1 sctp, 2 detect\n");
+	printf("                2 selects tcp if corosync rrp_mode is \"none\", otherwise sctp\n");
+	printf("                Default is 2\n");
 	printf("  -g <num>	groupd compatibility mode, 0 off, 1 on, 2 detect\n");
 	printf("		0: use libcpg, no backward compat, best performance\n");
 	printf("		1: use libgroup for compat with cluster2/rhel5\n");
@@ -1118,17 +1121,13 @@ static void print_usage(void)
 	printf("  -V		Print program version information, then exit\n");
 }
 
-#define OPTION_STRING "LDKg:f:q:d:p:Pl:o:t:c:a:hV"
+#define OPTION_STRING "LDKg:f:q:d:p:Pl:o:t:c:a:hVr:"
 
 static void read_arguments(int argc, char **argv)
 {
 	int cont = 1;
 	int optchar;
 
-	/* we don't allow these to be set on command line, should we? */
-	optk_timewarn = 0;
-	optk_timewarn = 0;
-
 	while (cont) {
 		optchar = getopt(argc, argv, OPTION_STRING);
 
@@ -1153,6 +1152,11 @@ static void read_arguments(int argc, char **argv)
 			cfgk_debug = 1;
 			break;
 
+		case 'r':
+			optk_protocol = 1;
+			cfgk_protocol = atoi(optarg);
+			break;
+
 		case 'f':
 			optd_enable_fencing = 1;
 			cfgd_enable_fencing = atoi(optarg);
@@ -1359,7 +1363,7 @@ int optd_drop_resources_age;
 
 int cfgk_debug                  = -1;
 int cfgk_timewarn               = -1;
-int cfgk_protocol               = -1;
+int cfgk_protocol               = PROTO_DETECT;
 int cfgd_groupd_compat          = DEFAULT_GROUPD_COMPAT;
 int cfgd_debug_logfile		= DEFAULT_DEBUG_LOGFILE;
 int cfgd_enable_fencing         = DEFAULT_ENABLE_FENCING;
-- 
1.5.5.6