Re: [PATCH v2] Samba RADOS service registration

[David Disseldorp <ddiss@suse.de>]

[-- Attachment #1: Type: text/plain, Size: 340 bytes --]

Please find a new version of the patchset attached, with the following
changes:
- use rados instace guid instead of hostname for instance string
- drop per-share metadata TODO
- avoid multiple rados_shutdown() calls from child processes
- add documentation for new ceph:service_user_id parameter

Feedback / push appreciated.

Cheers, David

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: add_rados_service_integration_v2.patchset --]
[-- Type: text/x-patch, Size: 11802 bytes --]

From 7b5a4217f40e00cf43bb681c8e673ae4451a2d0d Mon Sep 17 00:00:00 2001
From: David Disseldorp <ddiss@samba.org>
Date: Wed, 7 Feb 2018 14:58:48 +0100
Subject: [PATCH 1/2] ceph: add rados service integration

On startup, Samba advertises its presence to the Ceph Manager Service
via rados_service_register(). librados spawns a number of threads to
maintain the cluster connection and maintain periodic service
heartbeats.

Signed-off-by: David Disseldorp <ddiss@samba.org>
---
 source3/smbd/rados_service.c | 146 +++++++++++++++++++++++++++++++++++++++++++
 source3/smbd/rados_service.h |  30 +++++++++
 source3/smbd/server.c        |   7 +++
 source3/wscript              |  10 ++-
 source3/wscript_build        |  12 +++-
 5 files changed, 203 insertions(+), 2 deletions(-)
 create mode 100644 source3/smbd/rados_service.c
 create mode 100644 source3/smbd/rados_service.h

diff --git a/source3/smbd/rados_service.c b/source3/smbd/rados_service.c
new file mode 100644
index 00000000000..0e47e759ea7
--- /dev/null
+++ b/source3/smbd/rados_service.c
@@ -0,0 +1,146 @@
+/*
+   Copyright (C) David Disseldorp 2018
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "source3/include/includes.h"
+#include <rados/librados.h>
+#include "rados_service.h"
+
+struct ceph_service_state {
+	pid_t svc_reg_pid;
+	rados_t cluster;
+};
+
+static int ceph_service_destroy(struct ceph_service_state *state)
+{
+	/*
+	 * ensure that processes forked after spawning the service registration
+	 * connection don't additionally attempt cleanup.
+	 */
+	if (state->svc_reg_pid != getpid()) {
+		DBG_DEBUG("ignoring non-spawner destructor\n");
+		return 0;
+	}
+
+	DBG_DEBUG("ending Ceph cluster service registration\n");
+	rados_shutdown(state->cluster);
+	DBG_DEBUG("rados service connection shutdown\n");
+	return 0;
+}
+
+static int ceph_service_instance_gen(char *buf, size_t buflen,
+				     const char *daemon,
+				     uint64_t instance_guid)
+{
+	int ret;
+
+	ret = snprintf(buf, buflen, "%s:0x%016llx", daemon,
+			(unsigned long long)instance_guid);
+	if (ret >= buflen) {
+		DBG_ERR("Ceph instance name too long, skipping service "
+			"registration\n");
+		return -ENAMETOOLONG;
+	}
+
+	return 0;
+}
+
+int ceph_service_register(struct tevent_context *ev_ctx, const char *daemon)
+{
+	int ret;
+	const char *user_id = NULL;
+	const char *conf_file = NULL;
+	const char *cluster_name = NULL;
+	uint64_t instance_guid;
+	char instance_buf[128];
+	struct ceph_service_state *state = talloc(ev_ctx,
+						struct ceph_service_state);
+
+	if (state == NULL) {
+		return -ENOMEM;
+	}
+
+	state->svc_reg_pid = getpid();
+
+	/*
+	 * XXX This does not reuse the existing (share based) "ceph:user_id"
+	 * parameter, to allow for:
+	 * - running Samba with Ceph support, but without service registration.
+	 * - future mapping between Samba and Ceph users.
+	 */
+	user_id = lp_parm_const_string(GLOBAL_SECTION_SNUM, "ceph",
+				       "service_user_id", NULL);
+	if (user_id == NULL) {
+		DBG_DEBUG("built with Ceph support, but missing ceph:"
+			  "service_user_id configuration - skipping service "
+			  "registration\n");
+		ret = 0;
+		goto out_mem_free;
+	}
+
+	/* a NULL cluster_name or conf_file sees librados use the defaults */
+	cluster_name = lp_parm_const_string(GLOBAL_SECTION_SNUM, "ceph",
+					    "cluster_name", NULL);
+	conf_file = lp_parm_const_string(GLOBAL_SECTION_SNUM, "ceph",
+					 "config_file", NULL);
+
+	ret = rados_create2(&state->cluster, cluster_name, user_id, 0);
+	if (ret < 0) {
+		DBG_ERR("failed to create ceph cluster context\n");
+		goto out_mem_free;
+	}
+
+	ret = rados_conf_read_file(state->cluster, conf_file);
+	if (ret < 0) {
+		DBG_ERR("failed to parse Ceph cluster config: %s\n",
+			strerror(-ret));
+		goto err_cluster_free;
+	}
+
+	ret = rados_connect(state->cluster);
+	if (ret < 0) {
+		DBG_ERR("failed to connect to ceph cluster\n");
+		goto err_cluster_free;
+	}
+
+	instance_guid = rados_get_instance_id(state->cluster);
+
+	ret = ceph_service_instance_gen(instance_buf, sizeof(instance_buf),
+					daemon, instance_guid);
+	if (ret < 0) {
+		goto out_mem_free;
+	}
+
+	DBG_DEBUG("registering as %s with %s Ceph cluster\n", instance_buf,
+		  (cluster_name != NULL ? cluster_name : "default"));
+
+	ret = rados_service_register(state->cluster, "samba", instance_buf, "");
+	if (ret < 0) {
+		DBG_ERR("failed to register service with ceph cluster\n");
+		goto err_cluster_free;
+	}
+
+	/* close cluster conn and drop service listing on server exit */
+	talloc_set_destructor(state, ceph_service_destroy);
+
+	return 0;
+
+err_cluster_free:
+	rados_shutdown(state->cluster);
+out_mem_free:
+	talloc_free(state);
+	return ret;
+}
diff --git a/source3/smbd/rados_service.h b/source3/smbd/rados_service.h
new file mode 100644
index 00000000000..ef544b79798
--- /dev/null
+++ b/source3/smbd/rados_service.h
@@ -0,0 +1,30 @@
+/*
+   Copyright (C) David Disseldorp 2018
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _RADOS_SERVICE_H
+#define _RADOS_SERVICE_H
+
+int ceph_service_register(struct tevent_context *ev_ctx, const char *daemon);
+#if !defined(HAVE_LIBRADOS_SERVICES)
+/* built without librados service support, do nothing */
+int ceph_service_register(struct tevent_context *ev_ctx, const char *daemon)
+{
+	return 0;
+}
+#endif
+
+#endif /* _RADOS_SERVICE_H */
diff --git a/source3/smbd/server.c b/source3/smbd/server.c
index 99baf9d519d..d6629480eac 100644
--- a/source3/smbd/server.c
+++ b/source3/smbd/server.c
@@ -54,6 +54,7 @@
 #include "lib/util/sys_rw.h"
 #include "cleanupdb.h"
 #include "g_lock.h"
+#include "rados_service.h"
 
 #ifdef CLUSTER_SUPPORT
 #include "ctdb_protocol.h"
@@ -1593,6 +1594,7 @@ extern void build_options(bool screen);
 	struct smbd_parent_context *parent = NULL;
 	TALLOC_CTX *frame;
 	NTSTATUS status;
+	int ret;
 	struct tevent_context *ev_ctx;
 	struct messaging_context *msg_ctx;
 	struct server_id server_id;
@@ -2053,6 +2055,11 @@ extern void build_options(bool screen);
 	 * -- bad things will happen if smbd is launched via inetd
 	 *  and we fork a copy of ourselves here */
 	if (is_daemon && !interactive) {
+		ret = ceph_service_register(ev_ctx, "smbd");
+		if (ret < 0) {
+			exit_daemon("Samba failed to register with Ceph "
+				    "cluster", -ret);
+		}
 
 		if (rpc_lsasd_daemon() == RPC_DAEMON_FORK) {
 			start_lsasd(ev_ctx, msg_ctx);
diff --git a/source3/wscript b/source3/wscript
index 6d2d94bae87..605bdca6e01 100644
--- a/source3/wscript
+++ b/source3/wscript
@@ -1544,14 +1544,18 @@ main() {
 
     conf.env['CCFLAGS_CEPHFS'] = "-D_FILE_OFFSET_BITS=64"
     if Options.options.libcephfs_dir:
+        conf.env['CPPPATH_RADOS'] = Options.options.libcephfs_dir + '/include'
         conf.env['CPPPATH_CEPHFS'] = Options.options.libcephfs_dir + '/include'
         conf.env['LIBPATH_CEPHFS'] = Options.options.libcephfs_dir + '/lib'
         conf.env['LIBPATH_CEPH-COMMON'] = Options.options.libcephfs_dir + '/lib/ceph'
+        conf.env['LIBPATH_RADOS'] = Options.options.libcephfs_dir + '/lib'
 
     if (Options.options.with_cephfs and
         conf.CHECK_HEADERS('cephfs/libcephfs.h', False, False, 'cephfs') and
+        conf.CHECK_HEADERS('rados/librados.h', False, False, 'rados') and
         conf.CHECK_LIB('cephfs', shlib=True) and
-        conf.CHECK_LIB('ceph-common', shlib=True)):
+        conf.CHECK_LIB('ceph-common', shlib=True) and
+        conf.CHECK_LIB('rados', shlib=True)):
         if Options.options.with_acl_support:
             conf.DEFINE('HAVE_CEPH', '1')
             if conf.CHECK_FUNCS_IN('ceph_statx', 'cephfs ceph-common',
@@ -1561,6 +1565,10 @@ main() {
             Logs.warn("ceph support disabled due to --without-acl-support")
             conf.undefine('HAVE_CEPH')
 
+        if conf.CHECK_FUNCS_IN('rados_service_register', 'rados ceph-common',
+                               headers='rados/librados.h'):
+            conf.DEFINE('HAVE_LIBRADOS_SERVICES', '1')
+
     if Options.options.with_glusterfs:
         conf.CHECK_CFG(package='glusterfs-api', args='"glusterfs-api >= 4" --cflags --libs',
                        msg='Checking for glusterfs-api >= 4', uselib_store="GFAPI")
diff --git a/source3/wscript_build b/source3/wscript_build
index 76c5d6e203b..64f4b7ecebd 100644
--- a/source3/wscript_build
+++ b/source3/wscript_build
@@ -1052,6 +1052,16 @@ bld.SAMBA3_SUBSYSTEM('SPOOLSSD',
                          RPC_SOCK_HELPER
                          ''')
 
+smbd_rados_deps = ''
+if bld.CONFIG_SET('HAVE_LIBRADOS_SERVICES'):
+    bld.SAMBA_LIBRARY('rados_service',
+                      source='smbd/rados_service.c',
+                      deps='SMBCONF_PARAM samba-debug',
+                      local_include=False,
+                      public_deps='rados ceph-common',
+                      private_library=True)
+    smbd_rados_deps += ' rados_service'
+
 ########################## BINARIES #################################
 
 bld.SAMBA3_BINARY('smbd/smbd',
@@ -1064,7 +1074,7 @@ bld.SAMBA3_BINARY('smbd/smbd',
                       FSSD
                       MDSSD
                       SPOOLSSD
-                      ''',
+                      ''' + smbd_rados_deps,
                  install_path='${SBINDIR}')
 
 
-- 
2.13.6


From 463681ff53b5b0f12c5b6b3be5db4d27aceda881 Mon Sep 17 00:00:00 2001
From: David Disseldorp <ddiss@samba.org>
Date: Tue, 13 Feb 2018 20:16:48 +0100
Subject: [PATCH 2/2] docs/vfs_ceph: describe ceph:service_user_id parameter

Signed-off-by: David Disseldorp <ddiss@samba.org>
---
 docs-xml/manpages/vfs_ceph.8.xml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/docs-xml/manpages/vfs_ceph.8.xml b/docs-xml/manpages/vfs_ceph.8.xml
index 453030e50de..e84d75b0963 100644
--- a/docs-xml/manpages/vfs_ceph.8.xml
+++ b/docs-xml/manpages/vfs_ceph.8.xml
@@ -97,6 +97,20 @@
 		</listitem>
 		</varlistentry>
 
+		<varlistentry>
+		<term>ceph:service_user_id = id</term>
+		<listitem>
+		<para>
+			Allows one to explicitly set the client ID used for
+			registration of the Samba smbd daemon with the Ceph
+			Manager service.
+		</para>
+		<para>
+			Example: ceph:service_user_id = client.service_reg
+		</para>
+		</listitem>
+		</varlistentry>
+
 	</variablelist>
 
 </refsect1>
-- 
2.13.6