[Cluster-devel] [PATCH 1/5] rgmanager: DBus notifications for service state changes

[Cluster-devel] [PATCH 1/5] rgmanager: DBus notifications for service state changes

[Lon Hohberger]

Clean cherry-pick in to RHEL6 branch

Resolves: rhbz#657756

Signed-off-by: Lon Hohberger <lhh@redhat.com>
---
 rgmanager/include/rg_dbus.h         |   14 ++++
 rgmanager/man/rgmanager.8           |    2 +
 rgmanager/src/daemons/Makefile      |    7 +-
 rgmanager/src/daemons/main.c        |   88 ++++++++++++++++++++-
 rgmanager/src/daemons/update-dbus.c |  144 +++++++++++++++++++++++++++++++++++
 5 files changed, 250 insertions(+), 5 deletions(-)
 create mode 100644 rgmanager/include/rg_dbus.h
 create mode 100644 rgmanager/src/daemons/update-dbus.c

diff --git a/rgmanager/include/rg_dbus.h b/rgmanager/include/rg_dbus.h
new file mode 100644
index 0000000..6454f29
--- /dev/null
+++ b/rgmanager/include/rg_dbus.h
@@ -0,0 +1,14 @@
+#ifdef DBUS
+#ifndef _RGM_DBUS_H
+#define _RGM_DBUS_H
+
+int rgm_dbus_init(void);
+int rgm_dbus_release(void);
+int rgm_dbus_notify(const char *svcname,
+    		    const char *svcstatus,
+    		    const char *svcflags,
+    		    const char *svcowner,
+    		    const char *svclast);
+
+#endif
+#endif
diff --git a/rgmanager/man/rgmanager.8 b/rgmanager/man/rgmanager.8
index 39b2195..a4e2c73 100644
--- a/rgmanager/man/rgmanager.8
+++ b/rgmanager/man/rgmanager.8
@@ -371,6 +371,8 @@ will leave the virtual machine running.
 Run in the foreground (do not fork).
 .IP \-d
 Enable debug-level logging.
+.IP \-D
+Disable DBus signals which are normally sent when services change state.
 .IP \-w
 Disable internal process monitoring (for debugging).
 .IP \-N
diff --git a/rgmanager/src/daemons/Makefile b/rgmanager/src/daemons/Makefile
index 94e0dbb..4dbbe5a 100644
--- a/rgmanager/src/daemons/Makefile
+++ b/rgmanager/src/daemons/Makefile
@@ -29,6 +29,7 @@ OBJS1=	fo_domain.o \
 	service_op.o \
 	slang_event.o \
 	event_config.o \
+	update-dbus.o \
 	watchdog.o
 
 OBJS2=	test-noccs.o \
@@ -40,12 +41,13 @@ OBJS2=	test-noccs.o \
 	rg_locks-noccs.o \
 	event_config-noccs.o
 
-CFLAGS += -DSHAREDIR=\"${sharedir}\" -D_GNU_SOURCE
+CFLAGS += -DDBUS -DSHAREDIR=\"${sharedir}\" -D_GNU_SOURCE
 CFLAGS += -fPIC
 CFLAGS += -I${ccsincdir} -I${cmanincdir} -I${dlmincdir} -I${logtincdir}
 CFLAGS += `xml2-config --cflags` -I${slangincdir}
 CFLAGS += -I$(S)/../../include
 CFLAGS += -I${incdir}
+CFLAGS += `pkg-config --cflags dbus-1`
 
 NOCCS_CFLAGS += -DNO_CCS
 
@@ -59,6 +61,7 @@ DLM_LDFLAGS += -L${dlmlibdir} -ldlm
 XML2_LDFLAGS += `xml2-config --libs`
 SLANG_LDFLAGS += -L${slanglibdir} -lslang
 EXTRA_LDFLAGS += -lpthread
+DBUS_LDFLAGS += `pkg-config --libs dbus-1`
 
 LDDEPS += ../clulib/libclulib.a
 
@@ -66,7 +69,7 @@ ${TARGET1}: ${OBJS1} ${LDDEPS}
 	$(CC) -o $@ $^ $(CCS_LDFLAGS) $(CMAN_LDFLAGS) \
 			$(DLM_LDFLAGS) $(XML2_LDFLAGS) \
 			$(SLANG_LDFLAGS) $(EXTRA_LDFLAGS) \
-			$(LOGSYS_LDFLAGS) $(LD_FLAGS)
+			$(LOGSYS_LDFLAGS) $(LD_FLAGS) $(DBUS_LDFLAGS)
 
 #
 # Our test program links against the local allocator so that
diff --git a/rgmanager/src/daemons/main.c b/rgmanager/src/daemons/main.c
index 4e34246..303e0c0 100644
--- a/rgmanager/src/daemons/main.c
+++ b/rgmanager/src/daemons/main.c
@@ -23,6 +23,7 @@
 #include <members.h>
 #include <daemon_init.h>
 #include <groups.h>
+#include <rg_dbus.h>
 
 #ifdef WRAP_THREADS
 void dump_thread_states(FILE *);
@@ -42,7 +43,9 @@ static int signalled = 0;
 static uint8_t ALIGNED port = RG_PORT;
 static char *rgmanager_lsname = (char *)"rgmanager"; /* XXX default */
 static int status_poll_interval = DEFAULT_CHECK_INTERVAL;
-
+#ifdef DBUS
+static int state_notify = 1;
+#endif
 
 static void
 segfault(int __attribute__ ((unused)) sig)
@@ -935,6 +938,68 @@ shutdown_thread(void __attribute__ ((unused)) *arg)
 }
 
 
+#ifdef DBUS
+static int32_t
+svc_update(char *key, uint64_t view, void *data, uint32_t size)
+{
+	char flags[64];
+	rg_state_t *st;
+	cluster_member_list_t *m;
+	const char *owner;
+	const char *last;
+	int ret = 0;
+
+	if (!state_notify)
+		goto out_free;
+	if (view == 1)
+		goto out_free;
+	if (size != (sizeof(*st)))
+		goto out_free;
+
+	st = (rg_state_t *)data;
+	swab_rg_state_t(st);
+
+	/* Don't send transitional states */
+	if (st->rs_state == RG_STATE_STARTING ||
+	    st->rs_state == RG_STATE_STOPPING)
+		goto out_free;
+
+	m = member_list();
+	if (!m)
+		goto out_free;
+
+	owner = memb_id_to_name(m, st->rs_owner);
+	last = memb_id_to_name(m, st->rs_last_owner);
+
+	if (!owner)
+		owner = "(none)";
+	if (!last)
+		last = "(none)";
+
+	rg_flags_str(flags, sizeof(flags), st->rs_flags, (char *)" ");
+	if (flags[0] == 0)
+		snprintf(flags, sizeof(flags), "(none)");
+
+	ret = rgm_dbus_notify(st->rs_name,
+			      rg_state_str(st->rs_state),
+			      (char *)flags, owner, last);
+
+	if (ret < 0) {
+		logt_print(LOG_ERR, "Error sending update for %s; "
+			   "notifications disabled\n", key);
+		rgm_dbus_release();
+		state_notify = 0;
+	}
+
+out_free:
+	if (m)
+		free_member_list(m);
+	free(data);
+	return 0;
+}
+#endif
+
+
 #ifdef WRAP_THREADS
 void dump_thread_states(FILE *);
 #endif
@@ -949,7 +1014,7 @@ main(int argc, char **argv)
 	pthread_t th;
 	cman_handle_t clu = NULL;
 
-	while ((rv = getopt(argc, argv, "wfdN")) != EOF) {
+	while ((rv = getopt(argc, argv, "wfdND")) != EOF) {
 		switch (rv) {
 		case 'w':
 			wd = 0;
@@ -963,6 +1028,11 @@ main(int argc, char **argv)
 		case 'f':
 			foreground = 1;
 			break;
+		case 'D':
+#ifdef DBUS
+			state_notify = 0;
+#endif
+			break;
 		default:
 			return 1;
 			break;
@@ -1032,6 +1102,11 @@ main(int argc, char **argv)
 	configure_rgmanager(-1, debug, &cluster_timeout);
 	logt_print(LOG_NOTICE, "Resource Group Manager Starting\n");
 
+#ifdef DBUS
+	if (state_notify && rgm_dbus_init() != 0) 
+		logt_print(LOG_NOTICE, "Failed to initialize DBus\n");
+#endif
+
 	if (init_resource_groups(0, do_init) != 0) {
 		logt_print(LOG_CRIT, "#8: Couldn't initialize services\n");
 		goto out_ls;
@@ -1074,7 +1149,11 @@ main(int argc, char **argv)
 
 	ds_key_init("rg_lockdown", 32, 10);
 #else
-	if (vf_init(me.cn_nodeid, port, NULL, NULL, cluster_timeout) != 0) {
+	if (vf_init(me.cn_nodeid, port, NULL, 
+#ifdef DBUS /* Ugly */
+		    state_notify ? svc_update :
+#endif
+		    NULL, cluster_timeout) != 0) {
 		logt_print(LOG_CRIT, "#11: Couldn't set up VF listen socket\n");
 		goto out_ls;
 	}
@@ -1106,6 +1185,9 @@ out_ls:
 	clu_lock_finished(rgmanager_lsname);
 
 out:
+#ifdef DBUS
+	rgm_dbus_release();
+#endif
 	logt_print(LOG_NOTICE, "Shutdown complete, exiting\n");
 	cman_finish(clu);
 	
diff --git a/rgmanager/src/daemons/update-dbus.c b/rgmanager/src/daemons/update-dbus.c
new file mode 100644
index 0000000..9e3078d
--- /dev/null
+++ b/rgmanager/src/daemons/update-dbus.c
@@ -0,0 +1,144 @@
+/* DBus notifications */
+#include <stdio.h>
+#include <stdint.h>
+#include <resgroup.h>
+#include <poll.h>
+#include <dbus/dbus.h>
+#include <rg_dbus.h>
+#ifdef DBUS
+
+#define DBUS_RGM_NAME	"com.redhat.cluster.rgmanager"
+#define DBUS_RGM_IFACE	"com.redhat.cluster.rgmanager"
+#define DBUS_RGM_PATH	"/com/redhat/cluster/rgmanager"
+
+static DBusConnection *db = NULL;
+static pthread_mutex_t mu = PTHREAD_MUTEX_INITIALIZER;
+static pthread_t th = 0;
+static char _err[512];
+static int err_set = 0;
+
+int 
+rgm_dbus_init(void)
+{
+	DBusConnection *dbc = NULL;
+	DBusError err;
+
+	dbus_error_init(&err);
+
+	dbc = dbus_bus_get(DBUS_BUS_SYSTEM, &err);
+	if (!dbc) {
+		snprintf(_err, sizeof(_err),
+			 "dbus_bus_get: %s", err.message);
+		err_set = 1;
+		dbus_error_free(&err);
+		return -1;
+	}
+
+	dbus_connection_set_exit_on_disconnect(dbc, FALSE);
+
+	db = dbc;
+	return 0;
+}
+
+
+int
+rgm_dbus_release(void)
+{
+	pthread_t t;
+
+	if (!db)
+		return 0;
+
+	/* tell thread to exit - not sure how to tell dbus
+	 * to wake up, so just have it poll XXX */
+	if (th) {
+		t = th;
+		th = 0;
+		pthread_join(t, NULL);
+	}
+
+	dbus_connection_unref(db);
+	db = NULL;
+
+	return 0;
+}
+
+
+static void *
+_dbus_auto_flush(void *arg)
+{
+	/* DBus connection functions are thread safe */
+	dbus_connection_ref(db);
+
+	while (dbus_connection_read_write(db, 500)) {
+		if (!th)
+			break;	
+	}
+
+	dbus_connection_unref(db);
+
+	th = 0;
+	return NULL;
+}
+
+
+int
+rgm_dbus_notify(const char *svcname,
+		const char *svcstatus,
+		const char *svcflags,
+		const char *svcowner,
+		const char *svclast)
+{
+	DBusMessage *msg = NULL;
+	int ret = -1;
+
+	if (err_set) {
+		fprintf(stderr, "%s\n", _err);
+		err_set = 0;
+	}
+
+	if (!db) {
+		goto out_free;
+	}
+
+	pthread_mutex_lock(&mu);
+
+	if (dbus_connection_get_is_connected(db) != TRUE) {
+		err_set = 1;
+		snprintf(_err, sizeof(_err), "DBus connection lost");
+		rgm_dbus_release();
+		goto out_unlock;
+	}
+
+	if (!th) {
+		/* start auto-flush thread if needed */
+		pthread_create(&th, NULL, _dbus_auto_flush, NULL);
+	}
+
+	if (!(msg = dbus_message_new_signal(DBUS_RGM_PATH,
+	      				    DBUS_RGM_IFACE,
+	      				    "ServiceStateChange"))) {
+		goto out_unlock;
+	}
+
+	if (!dbus_message_append_args(msg,
+	 			      DBUS_TYPE_STRING, &svcname,
+	 			      DBUS_TYPE_STRING, &svcstatus,
+	 			      DBUS_TYPE_STRING, &svcflags,
+ 				      DBUS_TYPE_STRING, &svcowner,
+ 				      DBUS_TYPE_STRING, &svclast,
+	    			      DBUS_TYPE_INVALID)) {
+		goto out_unlock;
+	}
+
+	dbus_connection_send(db, msg, NULL);
+	ret = 0;
+
+out_unlock:
+	pthread_mutex_unlock(&mu);
+	if (msg)
+		dbus_message_unref(msg);
+out_free:
+	return ret;
+}
+#endif
-- 
1.7.2.3

[Cluster-devel] [PATCH 2/5] rgmanager: Clean up dbus notifications

[Lon Hohberger]

Clean cherry-pick in to RHEL6 branch

Resolves: rhbz#657756

Signed-off-by: Lon Hohberger <lhh@redhat.com>
---
 rgmanager/include/rg_dbus.h         |   20 ++++--
 rgmanager/src/daemons/Makefile      |    2 +-
 rgmanager/src/daemons/main.c        |   82 +--------------------------
 rgmanager/src/daemons/update-dbus.c |  105 ++++++++++++++++++++++++++++++++---
 4 files changed, 115 insertions(+), 94 deletions(-)

diff --git a/rgmanager/include/rg_dbus.h b/rgmanager/include/rg_dbus.h
index 6454f29..3ef5ae1 100644
--- a/rgmanager/include/rg_dbus.h
+++ b/rgmanager/include/rg_dbus.h
@@ -1,14 +1,20 @@
-#ifdef DBUS
 #ifndef _RGM_DBUS_H
 #define _RGM_DBUS_H
 
 int rgm_dbus_init(void);
 int rgm_dbus_release(void);
-int rgm_dbus_notify(const char *svcname,
-    		    const char *svcstatus,
-    		    const char *svcflags,
-    		    const char *svcowner,
-    		    const char *svclast);
+extern int rgm_dbus_notify;
 
-#endif
+#ifdef DBUS
+
+#define RGM_DBUS_DEFAULT 1
+#define RGM_DBUS_UPDATE (rgm_dbus_notify?rgm_dbus_update:0)
+int32_t rgm_dbus_update(char *key, uint64_t view, void *data, uint32_t size);
+
+#else
+
+#define RGM_DBUS_DEFAULT 0
+#define RGM_DBUS_UPDATE NULL
+
+#endif /* DBUS */
 #endif
diff --git a/rgmanager/src/daemons/Makefile b/rgmanager/src/daemons/Makefile
index 4dbbe5a..9619990 100644
--- a/rgmanager/src/daemons/Makefile
+++ b/rgmanager/src/daemons/Makefile
@@ -41,7 +41,7 @@ OBJS2=	test-noccs.o \
 	rg_locks-noccs.o \
 	event_config-noccs.o
 
-CFLAGS += -DDBUS -DSHAREDIR=\"${sharedir}\" -D_GNU_SOURCE
+CFLAGS += -DSHAREDIR=\"${sharedir}\" -D_GNU_SOURCE
 CFLAGS += -fPIC
 CFLAGS += -I${ccsincdir} -I${cmanincdir} -I${dlmincdir} -I${logtincdir}
 CFLAGS += `xml2-config --cflags` -I${slangincdir}
diff --git a/rgmanager/src/daemons/main.c b/rgmanager/src/daemons/main.c
index 303e0c0..1929072 100644
--- a/rgmanager/src/daemons/main.c
+++ b/rgmanager/src/daemons/main.c
@@ -43,9 +43,6 @@ static int signalled = 0;
 static uint8_t ALIGNED port = RG_PORT;
 static char *rgmanager_lsname = (char *)"rgmanager"; /* XXX default */
 static int status_poll_interval = DEFAULT_CHECK_INTERVAL;
-#ifdef DBUS
-static int state_notify = 1;
-#endif
 
 static void
 segfault(int __attribute__ ((unused)) sig)
@@ -938,68 +935,6 @@ shutdown_thread(void __attribute__ ((unused)) *arg)
 }
 
 
-#ifdef DBUS
-static int32_t
-svc_update(char *key, uint64_t view, void *data, uint32_t size)
-{
-	char flags[64];
-	rg_state_t *st;
-	cluster_member_list_t *m;
-	const char *owner;
-	const char *last;
-	int ret = 0;
-
-	if (!state_notify)
-		goto out_free;
-	if (view == 1)
-		goto out_free;
-	if (size != (sizeof(*st)))
-		goto out_free;
-
-	st = (rg_state_t *)data;
-	swab_rg_state_t(st);
-
-	/* Don't send transitional states */
-	if (st->rs_state == RG_STATE_STARTING ||
-	    st->rs_state == RG_STATE_STOPPING)
-		goto out_free;
-
-	m = member_list();
-	if (!m)
-		goto out_free;
-
-	owner = memb_id_to_name(m, st->rs_owner);
-	last = memb_id_to_name(m, st->rs_last_owner);
-
-	if (!owner)
-		owner = "(none)";
-	if (!last)
-		last = "(none)";
-
-	rg_flags_str(flags, sizeof(flags), st->rs_flags, (char *)" ");
-	if (flags[0] == 0)
-		snprintf(flags, sizeof(flags), "(none)");
-
-	ret = rgm_dbus_notify(st->rs_name,
-			      rg_state_str(st->rs_state),
-			      (char *)flags, owner, last);
-
-	if (ret < 0) {
-		logt_print(LOG_ERR, "Error sending update for %s; "
-			   "notifications disabled\n", key);
-		rgm_dbus_release();
-		state_notify = 0;
-	}
-
-out_free:
-	if (m)
-		free_member_list(m);
-	free(data);
-	return 0;
-}
-#endif
-
-
 #ifdef WRAP_THREADS
 void dump_thread_states(FILE *);
 #endif
@@ -1029,9 +964,7 @@ main(int argc, char **argv)
 			foreground = 1;
 			break;
 		case 'D':
-#ifdef DBUS
-			state_notify = 0;
-#endif
+			rgm_dbus_notify = 0;
 			break;
 		default:
 			return 1;
@@ -1102,10 +1035,8 @@ main(int argc, char **argv)
 	configure_rgmanager(-1, debug, &cluster_timeout);
 	logt_print(LOG_NOTICE, "Resource Group Manager Starting\n");
 
-#ifdef DBUS
-	if (state_notify && rgm_dbus_init() != 0) 
+	if (rgm_dbus_notify && rgm_dbus_init() != 0) 
 		logt_print(LOG_NOTICE, "Failed to initialize DBus\n");
-#endif
 
 	if (init_resource_groups(0, do_init) != 0) {
 		logt_print(LOG_CRIT, "#8: Couldn't initialize services\n");
@@ -1149,11 +1080,8 @@ main(int argc, char **argv)
 
 	ds_key_init("rg_lockdown", 32, 10);
 #else
-	if (vf_init(me.cn_nodeid, port, NULL, 
-#ifdef DBUS /* Ugly */
-		    state_notify ? svc_update :
-#endif
-		    NULL, cluster_timeout) != 0) {
+	if (vf_init(me.cn_nodeid, port, NULL, RGM_DBUS_UPDATE,
+		    cluster_timeout) != 0) {
 		logt_print(LOG_CRIT, "#11: Couldn't set up VF listen socket\n");
 		goto out_ls;
 	}
@@ -1185,9 +1113,7 @@ out_ls:
 	clu_lock_finished(rgmanager_lsname);
 
 out:
-#ifdef DBUS
 	rgm_dbus_release();
-#endif
 	logt_print(LOG_NOTICE, "Shutdown complete, exiting\n");
 	cman_finish(clu);
 	
diff --git a/rgmanager/src/daemons/update-dbus.c b/rgmanager/src/daemons/update-dbus.c
index 9e3078d..a3fb5e7 100644
--- a/rgmanager/src/daemons/update-dbus.c
+++ b/rgmanager/src/daemons/update-dbus.c
@@ -1,11 +1,18 @@
 /* DBus notifications */
+#include <stdint.h>
+#include <rg_dbus.h>
+#include <errno.h>
+
+#ifdef DBUS
+
 #include <stdio.h>
 #include <stdint.h>
 #include <resgroup.h>
 #include <poll.h>
 #include <dbus/dbus.h>
-#include <rg_dbus.h>
-#ifdef DBUS
+#include <liblogthread.h>
+#include <members.h>
+
 
 #define DBUS_RGM_NAME	"com.redhat.cluster.rgmanager"
 #define DBUS_RGM_IFACE	"com.redhat.cluster.rgmanager"
@@ -16,13 +23,22 @@ static pthread_mutex_t mu = PTHREAD_MUTEX_INITIALIZER;
 static pthread_t th = 0;
 static char _err[512];
 static int err_set = 0;
+#endif
+
+/* Set this to the desired value prior to calling rgm_dbus_init() */
+int rgm_dbus_notify = RGM_DBUS_DEFAULT;
+
 
 int 
 rgm_dbus_init(void)
+#ifdef DBUS
 {
 	DBusConnection *dbc = NULL;
 	DBusError err;
 
+	if (!rgm_dbus_notify)
+		return 0;
+
 	dbus_error_init(&err);
 
 	dbc = dbus_bus_get(DBUS_BUS_SYSTEM, &err);
@@ -39,10 +55,17 @@ rgm_dbus_init(void)
 	db = dbc;
 	return 0;
 }
+#else
+{
+	errno = ENOSYS;
+	return -1;
+}
+#endif
 
 
 int
 rgm_dbus_release(void)
+#ifdef DBUS
 {
 	pthread_t t;
 
@@ -62,8 +85,14 @@ rgm_dbus_release(void)
 
 	return 0;
 }
+#else
+{
+	return 0;
+}
+#endif
 
 
+#ifdef DBUS
 static void *
 _dbus_auto_flush(void *arg)
 {
@@ -82,12 +111,12 @@ _dbus_auto_flush(void *arg)
 }
 
 
-int
-rgm_dbus_notify(const char *svcname,
-		const char *svcstatus,
-		const char *svcflags,
-		const char *svcowner,
-		const char *svclast)
+static int
+_rgm_dbus_notify(const char *svcname,
+		 const char *svcstatus,
+		 const char *svcflags,
+		 const char *svcowner,
+		 const char *svclast)
 {
 	DBusMessage *msg = NULL;
 	int ret = -1;
@@ -141,4 +170,64 @@ out_unlock:
 out_free:
 	return ret;
 }
+
+
+int32_t
+rgm_dbus_update(char *key, uint64_t view, void *data, uint32_t size)
+{
+	char flags[64];
+	rg_state_t *st;
+	cluster_member_list_t *m = NULL;
+	const char *owner;
+	const char *last;
+	int ret = 0;
+
+	if (!rgm_dbus_notify)
+		goto out_free;
+	if (view == 1)
+		goto out_free;
+	if (size != (sizeof(*st)))
+		goto out_free;
+
+	st = (rg_state_t *)data;
+	swab_rg_state_t(st);
+
+	/* Don't send transitional states */
+	if (st->rs_state == RG_STATE_STARTING ||
+	    st->rs_state == RG_STATE_STOPPING)
+		goto out_free;
+
+	m = member_list();
+	if (!m)
+		goto out_free;
+
+	owner = memb_id_to_name(m, st->rs_owner);
+	last = memb_id_to_name(m, st->rs_last_owner);
+
+	if (!owner)
+		owner = "(none)";
+	if (!last)
+		last = "(none)";
+
+	rg_flags_str(flags, sizeof(flags), st->rs_flags, (char *)" ");
+	if (flags[0] == 0)
+		snprintf(flags, sizeof(flags), "(none)");
+
+	ret = _rgm_dbus_notify(st->rs_name,
+			       rg_state_str(st->rs_state),
+			       (char *)flags, owner, last);
+
+	if (ret < 0) {
+		logt_print(LOG_ERR, "Error sending update for %s; "
+			   "notifications disabled\n", key);
+		rgm_dbus_release();
+		rgm_dbus_notify = 0;
+	}
+
+out_free:
+	if (m)
+		free_member_list(m);
+	free(data);
+	return 0;
+}
 #endif
-- 
1.7.2.3

[Cluster-devel] [PATCH 3/5] rgmanager: Retry dbus if we get disconnected

[Lon Hohberger]

Clean cherry-pick in to RHEL6 branch

Resolves: rhbz#657756

Signed-off-by: Lon Hohberger <lhh@redhat.com>
---
 rgmanager/src/daemons/main.c        |    9 ++++-
 rgmanager/src/daemons/update-dbus.c |   60 ++++++++++++++++++++++++-----------
 2 files changed, 48 insertions(+), 21 deletions(-)

diff --git a/rgmanager/src/daemons/main.c b/rgmanager/src/daemons/main.c
index 1929072..dad379a 100644
--- a/rgmanager/src/daemons/main.c
+++ b/rgmanager/src/daemons/main.c
@@ -684,6 +684,8 @@ event_loop(msgctx_t *localctx, msgctx_t *clusterctx)
 		dump_internal_state("/var/lib/cluster/rgmanager-dump");
 	}
 
+	rgm_dbus_init();
+
 	while (running && (tv.tv_sec || tv.tv_usec)) {
 		FD_ZERO(&rfds);
 		max = -1;
@@ -1035,8 +1037,11 @@ main(int argc, char **argv)
 	configure_rgmanager(-1, debug, &cluster_timeout);
 	logt_print(LOG_NOTICE, "Resource Group Manager Starting\n");
 
-	if (rgm_dbus_notify && rgm_dbus_init() != 0) 
-		logt_print(LOG_NOTICE, "Failed to initialize DBus\n");
+	if (rgm_dbus_notify && rgm_dbus_init() != 0) {
+		rgm_dbus_notify = 0;
+		logt_print(LOG_NOTICE, "Failed to initialize DBus; "
+			   "notifications disabled\n");
+	}
 
 	if (init_resource_groups(0, do_init) != 0) {
 		logt_print(LOG_CRIT, "#8: Couldn't initialize services\n");
diff --git a/rgmanager/src/daemons/update-dbus.c b/rgmanager/src/daemons/update-dbus.c
index a3fb5e7..c9bbf40 100644
--- a/rgmanager/src/daemons/update-dbus.c
+++ b/rgmanager/src/daemons/update-dbus.c
@@ -21,8 +21,6 @@
 static DBusConnection *db = NULL;
 static pthread_mutex_t mu = PTHREAD_MUTEX_INITIALIZER;
 static pthread_t th = 0;
-static char _err[512];
-static int err_set = 0;
 #endif
 
 /* Set this to the desired value prior to calling rgm_dbus_init() */
@@ -39,20 +37,29 @@ rgm_dbus_init(void)
 	if (!rgm_dbus_notify)
 		return 0;
 
+	pthread_mutex_lock(&mu);
+	if (db) {
+		pthread_mutex_unlock(&mu);
+		return 0;
+	}
+
 	dbus_error_init(&err);
 
-	dbc = dbus_bus_get(DBUS_BUS_SYSTEM, &err);
+	dbc = dbus_bus_get_private(DBUS_BUS_SYSTEM, &err);
 	if (!dbc) {
-		snprintf(_err, sizeof(_err),
-			 "dbus_bus_get: %s", err.message);
-		err_set = 1;
+		logt_print(LOG_ERR,
+			   "DBus Failed to initialize: dbus_bus_get: %s\n",
+			   err.message);
 		dbus_error_free(&err);
+		pthread_mutex_unlock(&mu);
 		return -1;
 	}
 
 	dbus_connection_set_exit_on_disconnect(dbc, FALSE);
 
 	db = dbc;
+	pthread_mutex_unlock(&mu);
+	logt_print(LOG_DEBUG, "DBus Notifications Initialized\n");
 	return 0;
 }
 #else
@@ -63,9 +70,9 @@ rgm_dbus_init(void)
 #endif
 
 
-int
-rgm_dbus_release(void)
 #ifdef DBUS
+static int
+_rgm_dbus_release(int err)
 {
 	pthread_t t;
 
@@ -74,17 +81,39 @@ rgm_dbus_release(void)
 
 	/* tell thread to exit - not sure how to tell dbus
 	 * to wake up, so just have it poll XXX */
+
+	/* if the thread left because the dbus connection died,
+	   this block is avoided since the thread exits */
 	if (th) {
 		t = th;
 		th = 0;
 		pthread_join(t, NULL);
 	}
 
+	dbus_connection_close(db);
 	dbus_connection_unref(db);
 	db = NULL;
 
+	if (err)
+		logt_print(LOG_ERR, "DBus Connection Lost\n");
+	else
+		logt_print(LOG_DEBUG, "DBus Released\n");
 	return 0;
 }
+#endif
+
+
+int
+rgm_dbus_release(void)
+#ifdef DBUS
+{
+	int ret;
+
+	pthread_mutex_lock(&mu);
+	ret = _rgm_dbus_release(0);
+	pthread_mutex_unlock(&mu);
+	return ret;
+}
 #else
 {
 	return 0;
@@ -98,14 +127,12 @@ _dbus_auto_flush(void *arg)
 {
 	/* DBus connection functions are thread safe */
 	dbus_connection_ref(db);
-
 	while (dbus_connection_read_write(db, 500)) {
 		if (!th)
 			break;	
 	}
 
 	dbus_connection_unref(db);
-
 	th = 0;
 	return NULL;
 }
@@ -121,11 +148,6 @@ _rgm_dbus_notify(const char *svcname,
 	DBusMessage *msg = NULL;
 	int ret = -1;
 
-	if (err_set) {
-		fprintf(stderr, "%s\n", _err);
-		err_set = 0;
-	}
-
 	if (!db) {
 		goto out_free;
 	}
@@ -133,9 +155,7 @@ _rgm_dbus_notify(const char *svcname,
 	pthread_mutex_lock(&mu);
 
 	if (dbus_connection_get_is_connected(db) != TRUE) {
-		err_set = 1;
-		snprintf(_err, sizeof(_err), "DBus connection lost");
-		rgm_dbus_release();
+		_rgm_dbus_release(1);
 		goto out_unlock;
 	}
 
@@ -184,6 +204,8 @@ rgm_dbus_update(char *key, uint64_t view, void *data, uint32_t size)
 
 	if (!rgm_dbus_notify)
 		goto out_free;
+	if (!db)
+		goto out_free;
 	if (view == 1)
 		goto out_free;
 	if (size != (sizeof(*st)))
@@ -209,6 +231,7 @@ rgm_dbus_update(char *key, uint64_t view, void *data, uint32_t size)
 	if (!last)
 		last = "(none)";
 
+	flags[0] = 0;
 	rg_flags_str(flags, sizeof(flags), st->rs_flags, (char *)" ");
 	if (flags[0] == 0)
 		snprintf(flags, sizeof(flags), "(none)");
@@ -221,7 +244,6 @@ rgm_dbus_update(char *key, uint64_t view, void *data, uint32_t size)
 		logt_print(LOG_ERR, "Error sending update for %s; "
 			   "notifications disabled\n", key);
 		rgm_dbus_release();
-		rgm_dbus_notify = 0;
 	}
 
 out_free:
-- 
1.7.2.3

[Cluster-devel] [PATCH 4/5] rgmanager: minor dbus cleanups

[Lon Hohberger]

Clean cherry-pick in to RHEL6 branch

Resolves: rhbz#657756

Signed-off-by: Lon Hohberger <lhh@redhat.com>
---
 rgmanager/src/daemons/update-dbus.c |   31 +++++++++++++++++++++----------
 1 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/rgmanager/src/daemons/update-dbus.c b/rgmanager/src/daemons/update-dbus.c
index c9bbf40..f465d1d 100644
--- a/rgmanager/src/daemons/update-dbus.c
+++ b/rgmanager/src/daemons/update-dbus.c
@@ -47,7 +47,7 @@ rgm_dbus_init(void)
 
 	dbc = dbus_bus_get_private(DBUS_BUS_SYSTEM, &err);
 	if (!dbc) {
-		logt_print(LOG_ERR,
+		logt_print(LOG_DEBUG,
 			   "DBus Failed to initialize: dbus_bus_get: %s\n",
 			   err.message);
 		dbus_error_free(&err);
@@ -72,7 +72,7 @@ rgm_dbus_init(void)
 
 #ifdef DBUS
 static int
-_rgm_dbus_release(int err)
+_rgm_dbus_release(void)
 {
 	pthread_t t;
 
@@ -83,7 +83,7 @@ _rgm_dbus_release(int err)
 	 * to wake up, so just have it poll XXX */
 
 	/* if the thread left because the dbus connection died,
-	   this block is avoided since the thread exits */
+	   this block is avoided */
 	if (th) {
 		t = th;
 		th = 0;
@@ -94,15 +94,13 @@ _rgm_dbus_release(int err)
 	dbus_connection_unref(db);
 	db = NULL;
 
-	if (err)
-		logt_print(LOG_ERR, "DBus Connection Lost\n");
-	else
-		logt_print(LOG_DEBUG, "DBus Released\n");
+	logt_print(LOG_DEBUG, "DBus Released\n");
 	return 0;
 }
 #endif
 
 
+/* Clean shutdown (e.g. when exiting */
 int
 rgm_dbus_release(void)
 #ifdef DBUS
@@ -110,7 +108,7 @@ rgm_dbus_release(void)
 	int ret;
 
 	pthread_mutex_lock(&mu);
-	ret = _rgm_dbus_release(0);
+	ret = _rgm_dbus_release();
 	pthread_mutex_unlock(&mu);
 	return ret;
 }
@@ -122,6 +120,9 @@ rgm_dbus_release(void)
 
 
 #ifdef DBUS
+/* Auto-flush thread.  Since sending only guarantees queueing,
+ * we need this thread to push things out over dbus in the
+ * background */
 static void *
 _dbus_auto_flush(void *arg)
 {
@@ -154,8 +155,15 @@ _rgm_dbus_notify(const char *svcname,
 
 	pthread_mutex_lock(&mu);
 
+	/* Check to ensure the connection is still valid. If it
+	 * isn't, clean up and shut down the dbus connection.
+	 *
+	 * The main rgmanager thread will periodically try to
+	 * reinitialize the dbus notification subsystem unless
+	 * the administrator ran rgmanager with the -D command
+	 * line option.
+	 */
 	if (dbus_connection_get_is_connected(db) != TRUE) {
-		_rgm_dbus_release(1);
 		goto out_unlock;
 	}
 
@@ -192,6 +200,9 @@ out_free:
 }
 
 
+/*
+ * view-formation callback function
+ */
 int32_t
 rgm_dbus_update(char *key, uint64_t view, void *data, uint32_t size)
 {
@@ -242,7 +253,7 @@ rgm_dbus_update(char *key, uint64_t view, void *data, uint32_t size)
 
 	if (ret < 0) {
 		logt_print(LOG_ERR, "Error sending update for %s; "
-			   "notifications disabled\n", key);
+			   "DBus notifications disabled\n", key);
 		rgm_dbus_release();
 	}
 
-- 
1.7.2.3

[Cluster-devel] [PATCH 5/5] rgmanager: Match fenced's option to disable DBus

[Lon Hohberger]

Clean cherry-pick in to RHEL6 branch

Resolves: rhbz#657756

Signed-off-by: Lon Hohberger <lhh@redhat.com>
---
 rgmanager/man/rgmanager.8    |    2 +-
 rgmanager/src/daemons/main.c |    4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/rgmanager/man/rgmanager.8 b/rgmanager/man/rgmanager.8
index a4e2c73..2b018d2 100644
--- a/rgmanager/man/rgmanager.8
+++ b/rgmanager/man/rgmanager.8
@@ -371,7 +371,7 @@ will leave the virtual machine running.
 Run in the foreground (do not fork).
 .IP \-d
 Enable debug-level logging.
-.IP \-D
+.IP \-q
 Disable DBus signals which are normally sent when services change state.
 .IP \-w
 Disable internal process monitoring (for debugging).
diff --git a/rgmanager/src/daemons/main.c b/rgmanager/src/daemons/main.c
index dad379a..02cd2dc 100644
--- a/rgmanager/src/daemons/main.c
+++ b/rgmanager/src/daemons/main.c
@@ -951,7 +951,7 @@ main(int argc, char **argv)
 	pthread_t th;
 	cman_handle_t clu = NULL;
 
-	while ((rv = getopt(argc, argv, "wfdND")) != EOF) {
+	while ((rv = getopt(argc, argv, "wfdNq")) != EOF) {
 		switch (rv) {
 		case 'w':
 			wd = 0;
@@ -965,7 +965,7 @@ main(int argc, char **argv)
 		case 'f':
 			foreground = 1;
 			break;
-		case 'D':
+		case 'q':
 			rgm_dbus_notify = 0;
 			break;
 		default:
-- 
1.7.2.3

[Cluster-devel] [PATCH 2/5] rgmanager: Clean up dbus notifications

[Lon Hohberger]

On Wed, 2011-02-02 at 19:18 -0500, Lon Hohberger wrote:
> diff --git a/rgmanager/src/daemons/Makefile b/rgmanager/src/daemons/Makefile
> index 4dbbe5a..9619990 100644
> --- a/rgmanager/src/daemons/Makefile
> +++ b/rgmanager/src/daemons/Makefile
> @@ -41,7 +41,7 @@ OBJS2=	test-noccs.o \
>  	rg_locks-noccs.o \
>  	event_config-noccs.o
>  
> -CFLAGS += -DDBUS -DSHAREDIR=\"${sharedir}\" -D_GNU_SOURCE
> +CFLAGS += -DSHAREDIR=\"${sharedir}\" -D_GNU_SOURCE
>  CFLAGS += -fPIC
>  CFLAGS += -I${ccsincdir} -I${cmanincdir} -I${dlmincdir} -I${logtincdir}
>  CFLAGS += `xml2-config --cflags` -I${slangincdir}

Oops -- for RHEL6 branch, this part needs to be removed; we need the
-DDBUS.

-- Lon