[Cluster-devel] [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Cluster-devel] [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Wendy Cheng]

This kernel patch (based on 2.6.21-rc4) should be paired with nfs-utils 
user mode changes (patch 4-4, based on nfs-utils-1.1.0-rc1) that is 
optional. If changes made in patch 4-4 is not presented in nfs-utils, 
the rpc.statd will ignore whatever this kernel patch does.

The changes record the ip interface that accepts the lock requests and 
passes the correct "my_name" (in standard IPV4 dot notation) to user 
mode statd (instead of system_utsname.nodename). This enables rpc.statd 
to add the correct taken-over IPv4 address into the 3rd parameter of 
ha_callout program. Current nfs-utils always resets "my_name" into 
loopback address (127.0.0.1), regardless the statement made in rpc.statd 
man page. Check out "man rpc.statd" and "man sm-notify" for details.

-- Wendy

-------------- next part --------------
A non-text attachment was scrubbed...
Name: 003_nlm_statd.patch
Type: text/x-patch
Size: 14486 bytes
Desc: not available
URL: <http://listman.redhat.com/archives/cluster-devel/attachments/20070405/7f776025/attachment.bin>

[PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Wendy Cheng]

[-- Attachment #1: Type: text/plain, Size: 780 bytes --]

This kernel patch (based on 2.6.21-rc4) should be paired with nfs-utils 
user mode changes (patch 4-4, based on nfs-utils-1.1.0-rc1) that is 
optional. If changes made in patch 4-4 is not presented in nfs-utils, 
the rpc.statd will ignore whatever this kernel patch does.

The changes record the ip interface that accepts the lock requests and 
passes the correct "my_name" (in standard IPV4 dot notation) to user 
mode statd (instead of system_utsname.nodename). This enables rpc.statd 
to add the correct taken-over IPv4 address into the 3rd parameter of 
ha_callout program. Current nfs-utils always resets "my_name" into 
loopback address (127.0.0.1), regardless the statement made in rpc.statd 
man page. Check out "man rpc.statd" and "man sm-notify" for details.

-- Wendy


[-- Attachment #2: 003_nlm_statd.patch --]
[-- Type: text/x-patch, Size: 14486 bytes --]

Signed-off-by: S. Wendy Cheng <wcheng@redhat.com>
Signed-off-by: Lon Hohberger  <lhh@redhat.com>

 fs/lockd/clntproc.c            |    2 
 fs/lockd/host.c                |   61 +++++++++++++++++++-----
 fs/lockd/mon.c                 |  104 +++++++++++++++++++++++++++++++++++------
 include/linux/lockd/lockd.h    |   11 +++-
 include/linux/lockd/sm_inter.h |    3 -
 net/sunrpc/svcsock.c           |   40 +++++++++++++++
 6 files changed, 191 insertions(+), 30 deletions(-)

--- linux-nlm-2/include/linux/lockd/sm_inter.h	2007-03-26 18:25:38.000000000 -0400
+++ linux/include/linux/lockd/sm_inter.h	2007-04-03 21:55:42.000000000 -0400
@@ -25,6 +25,7 @@
  */
 struct nsm_args {
 	__be32		addr;		/* remote address */
+	__be32          serv;		/* server ip address */
 	u32		prog;		/* RPC callback info */
 	u32		vers;
 	u32		proc;
@@ -40,7 +41,7 @@ struct nsm_res {
 	u32		state;
 };
 
-int		nsm_monitor(struct nlm_host *);
+int		nsm_monitor(struct nlm_host *, __be32 ip);
 int		nsm_unmonitor(struct nlm_host *);
 extern int	nsm_local_state;
 
--- linux-nlm-2/include/linux/lockd/lockd.h	2007-03-26 18:25:38.000000000 -0400
+++ linux/include/linux/lockd/lockd.h	2007-04-04 10:45:14.000000000 -0400
@@ -39,12 +39,12 @@
 struct nlm_host {
 	struct hlist_node	h_hash;		/* doubly linked list */
 	struct sockaddr_in	h_addr;		/* peer address */
+	__be32			h_server;	/* server ip for NLM failover */
 	struct rpc_clnt	*	h_rpcclnt;	/* RPC client to talk to peer */
 	char *			h_name;		/* remote hostname */
 	u32			h_version;	/* interface version */
 	unsigned short		h_proto;	/* transport proto */
 	unsigned short		h_reclaiming : 1,
-				h_server     : 1, /* server side, not client side */
 				h_inuse      : 1;
 	wait_queue_head_t	h_gracewait;	/* wait while reclaiming */
 	struct rw_semaphore	h_rwsem;	/* Reboot recovery lock */
@@ -62,11 +62,18 @@ struct nlm_host {
 	struct nsm_handle *	h_nsmhandle;	/* NSM status handle */
 };
 
+struct nsm_fo_monitored {
+	struct list_head	list;
+	__be32			addr;
+};
+
 struct nsm_handle {
 	struct list_head	sm_link;
 	atomic_t		sm_count;
 	char *			sm_name;
 	struct sockaddr_in	sm_addr;
+	struct mutex		sm_mutex;
+	struct nsm_fo_monitored sm_serverip;
 	unsigned int		sm_monitored : 1,
 				sm_sticky : 1;	/* don't unmonitor */
 };
@@ -254,7 +261,7 @@ static inline int
 nlmsvc_check_grace_period(struct nlm_args *argp)
 {
 	/* check for system wide grace period */
-	if (nlmsvc_grace_period)
+	if (nlmsvc_grace_period) 
 		return 1;
 
 	/* check for per exported fsid grace period */
--- linux-nlm-2/net/sunrpc/svcsock.c	2007-03-26 18:26:06.000000000 -0400
+++ linux/net/sunrpc/svcsock.c	2007-04-04 17:09:15.000000000 -0400
@@ -1111,6 +1111,44 @@ failed:
 	return;
 }
 
+/* Added for NLM-cluster failover implementation */ 
+static inline void svc_tcp_get_server_address(struct svc_rqst *rqstp)
+{
+	struct socket *sock = rqstp->rq_sock->sk_sock;
+	struct sockaddr_in6 sin6;
+	struct sockaddr_in *sin = (struct sockaddr_in *) &sin6;
+	int len, err;
+
+	/* ref: inet_getname, inet6_getname, and sys_getsockname */
+	err = sock->ops->getname(sock, (struct sockaddr *) sin, &len, 0);
+	if (err) {
+		dprintk("svc_tcp_get_server_address: getname err=%d\n", err);
+		return;
+	}
+		
+        switch (rqstp->rq_sock->sk_sk->sk_family) {
+        case AF_INET: 
+		/* sanity check */
+		if (sin->sin_family != AF_INET)
+			printk("sunrpc: inet address family mismatch %d\n",
+				(int) sin->sin_family); 
+                rqstp->rq_daddr.addr = sin->sin_addr;
+                break;
+        case AF_INET6: 
+		/* sanity check */
+		if (sin6.sin6_family != AF_INET6)
+			printk("sunrpc: inet6 address family mismatch %d\n",
+				(int) sin6.sin6_family); 
+                ipv6_addr_copy(&rqstp->rq_daddr.addr6, &sin6.sin6_addr); 
+                break;
+	default:
+		break;
+	}
+
+	/* no error return */
+	return;
+}
+
 /*
  * Receive data from a TCP socket.
  */
@@ -1260,6 +1298,8 @@ svc_tcp_recvfrom(struct svc_rqst *rqstp)
 	if (serv->sv_stats)
 		serv->sv_stats->nettcpcnt++;
 
+	svc_tcp_get_server_address(rqstp);
+
 	return len;
 
  err_delete:
--- linux-nlm-2/fs/lockd/host.c	2007-03-26 18:19:11.000000000 -0400
+++ linux/fs/lockd/host.c	2007-04-04 12:02:50.000000000 -0400
@@ -34,16 +34,16 @@ static DEFINE_MUTEX(nlm_host_mutex);
 
 static void			nlm_gc_hosts(void);
 static struct nsm_handle *	__nsm_find(const struct sockaddr_in *,
-					const char *, int, int);
+					const char *, int, int, __be32);
 static struct nsm_handle *	nsm_find(const struct sockaddr_in *sin,
 					 const char *hostname,
-					 int hostname_len);
+					 int hostname_len, __be32 ip);
 
 /*
  * Common host lookup routine for server & client
  */
 static struct nlm_host *
-nlm_lookup_host(int server, const struct sockaddr_in *sin,
+nlm_lookup_host(union svc_addr_u *server, const struct sockaddr_in *sin,
 					int proto, int version,
 					const char *hostname,
 					int hostname_len)
@@ -53,6 +53,7 @@ nlm_lookup_host(int server, const struct
 	struct nlm_host	*host;
 	struct nsm_handle *nsm = NULL;
 	int		hash;
+	__be32		server_ip;
 
 	dprintk("lockd: nlm_lookup_host(%u.%u.%u.%u, p=%d, v=%d, my role=%s, name=%.*s)\n",
 			NIPQUAD(sin->sin_addr.s_addr), proto, version,
@@ -60,6 +61,13 @@ nlm_lookup_host(int server, const struct
 			hostname_len,
 			hostname? hostname : "<none>");
 
+	/* NLM failover: ipv4 for now */
+	if (server)
+		server_ip = server->addr.s_addr;
+	else
+		server_ip = 0;
+
+	dprintk("lockd: server_ip = %u.%u.%u.%u\n", NIPQUAD(server_ip));
 
 	hash = NLM_ADDRHASH(sin->sin_addr.s_addr);
 
@@ -89,7 +97,7 @@ nlm_lookup_host(int server, const struct
 			continue;
 		if (host->h_version != version)
 			continue;
-		if (host->h_server != server)
+		if (host->h_server != server_ip)
 			continue;
 
 		/* Move to head of hash chain. */
@@ -107,7 +115,7 @@ nlm_lookup_host(int server, const struct
 	/* Sadly, the host isn't in our hash table yet. See if
 	 * we have an NSM handle for it. If not, create one.
 	 */
-	if (!nsm && !(nsm = nsm_find(sin, hostname, hostname_len)))
+	if (!nsm && !(nsm = nsm_find(sin, hostname, hostname_len, server_ip)))
 		goto out;
 
 	host = kzalloc(sizeof(*host), GFP_KERNEL);
@@ -130,7 +138,10 @@ nlm_lookup_host(int server, const struct
 	host->h_state      = 0;			/* pseudo NSM state */
 	host->h_nsmstate   = 0;			/* real NSM state */
 	host->h_nsmhandle  = nsm;
-	host->h_server	   = server;
+
+	/* NLM failover: only ipv4 for now */
+	host->h_server	   = server_ip;
+
 	hlist_add_head(&host->h_hash, chain);
 	INIT_LIST_HEAD(&host->h_lockowners);
 	spin_lock_init(&host->h_lock);
@@ -180,7 +191,7 @@ struct nlm_host *
 nlmclnt_lookup_host(const struct sockaddr_in *sin, int proto, int version,
 			const char *hostname, int hostname_len)
 {
-	return nlm_lookup_host(0, sin, proto, version,
+	return nlm_lookup_host(NULL, sin, proto, version,
 			       hostname, hostname_len);
 }
 
@@ -191,7 +202,7 @@ struct nlm_host *
 nlmsvc_lookup_host(struct svc_rqst *rqstp,
 			const char *hostname, int hostname_len)
 {
-	return nlm_lookup_host(1, svc_addr_in(rqstp),
+	return nlm_lookup_host(&rqstp->rq_daddr, svc_addr_in(rqstp),
 			       rqstp->rq_prot, rqstp->rq_vers,
 			       hostname, hostname_len);
 }
@@ -314,7 +325,7 @@ void nlm_host_rebooted(const struct sock
 			hostname, NIPQUAD(sin->sin_addr));
 
 	/* Find the NSM handle for this peer */
-	if (!(nsm = __nsm_find(sin, hostname, hostname_len, 0)))
+	if (!(nsm = __nsm_find(sin, hostname, hostname_len, 0, 0)))
 		return;
 
 	/* When reclaiming locks on this peer, make sure that
@@ -445,7 +456,7 @@ static DEFINE_MUTEX(nsm_mutex);
 static struct nsm_handle *
 __nsm_find(const struct sockaddr_in *sin,
 		const char *hostname, int hostname_len,
-		int create)
+		int create, __be32 server_ip)
 {
 	struct nsm_handle *nsm = NULL;
 	struct list_head *pos;
@@ -490,6 +501,11 @@ __nsm_find(const struct sockaddr_in *sin
 		atomic_set(&nsm->sm_count, 1);
 
 		list_add(&nsm->sm_link, &nsm_handles);
+
+		/* NLM failover */
+		mutex_init(&nsm->sm_mutex);
+		INIT_LIST_HEAD(&nsm->sm_serverip.list);
+		nsm->sm_serverip.addr = server_ip;
 	}
 
 out:
@@ -498,9 +514,28 @@ out:
 }
 
 static struct nsm_handle *
-nsm_find(const struct sockaddr_in *sin, const char *hostname, int hostname_len)
+nsm_find(const struct sockaddr_in *sin, const char *hostname, int hostname_len,
+		__be32 server_ip)
+{
+	return __nsm_find(sin, hostname, hostname_len, 1, server_ip);
+}
+
+/*
+ * NLM failover:
+ *	nsm_mutex should be obtained before entry
+ *	fo_ip not NULL
+ */
+void
+nsm_release_fo_ip(struct nsm_fo_monitored *fo_ip)
 {
-	return __nsm_find(sin, hostname, hostname_len, 1);
+	struct list_head *pos, *n, *head=&fo_ip->list;
+	struct nsm_fo_monitored *server_ip;
+
+	list_for_each_safe(pos, n, head) {
+		server_ip = list_entry(pos, struct nsm_fo_monitored, list);
+		kfree(server_ip);
+	}
+	return;
 }
 
 /*
@@ -515,6 +550,8 @@ nsm_release(struct nsm_handle *nsm)
 		mutex_lock(&nsm_mutex);
 		if (atomic_read(&nsm->sm_count) == 0) {
 			list_del(&nsm->sm_link);
+			if (!list_empty(&nsm->sm_serverip.list))
+				nsm_release_fo_ip(&nsm->sm_serverip);
 			kfree(nsm);
 		}
 		mutex_unlock(&nsm_mutex);
--- linux-nlm-2/fs/lockd/mon.c	2007-03-26 18:19:10.000000000 -0400
+++ linux/fs/lockd/mon.c	2007-04-04 16:11:05.000000000 -0400
@@ -30,7 +30,7 @@ int				nsm_local_state;
  * Common procedure for SM_MON/SM_UNMON calls
  */
 static int
-nsm_mon_unmon(struct nsm_handle *nsm, u32 proc, struct nsm_res *res)
+nsm_mon_unmon(struct nsm_handle *nsm, u32 proc, struct nsm_res *res, __be32 server_ip)
 {
 	struct rpc_clnt	*clnt;
 	int		status;
@@ -48,6 +48,12 @@ nsm_mon_unmon(struct nsm_handle *nsm, u3
 
 	memset(&args, 0, sizeof(args));
 	args.mon_name = nsm->sm_name;
+
+	/* NLM failover:
+	 * only IPV4 is supported at this moment 
+	 */
+	args.serv = server_ip;
+
 	args.addr = nsm->sm_addr.sin_addr.s_addr;
 	args.prog = NLM_PROGRAM;
 	args.vers = 3;
@@ -65,28 +71,71 @@ nsm_mon_unmon(struct nsm_handle *nsm, u3
 	return status;
 }
 
+static inline
+int nsm_is_monitored(struct nlm_host *host, __be32 server)
+{
+	struct nsm_handle *nsm = host->h_nsmhandle;
+	struct list_head *pos, *head;
+	struct nsm_fo_monitored *fo_entry;
+
+	/* client */
+	if (!server)
+		return nsm->sm_monitored;
+
+	/* server */
+	if (!nsm->sm_monitored)
+		return 0;
+
+	/* search for monitored list */
+	mutex_lock(&nsm->sm_mutex);
+	head = &nsm->sm_serverip.list;
+	list_for_each(pos, head) {
+		fo_entry = list_entry(pos, struct nsm_fo_monitored, list);
+		if (fo_entry->addr == server) {
+			mutex_unlock(&nsm->sm_mutex);
+			return 1;
+		}
+	}
+	mutex_unlock(&nsm->sm_mutex);
+
+	return 0;
+}
+
 /*
  * Set up monitoring of a remote host
  */
 int
-nsm_monitor(struct nlm_host *host)
+nsm_monitor(struct nlm_host *host, __be32 server)
 {
 	struct nsm_handle *nsm = host->h_nsmhandle;
 	struct nsm_res	res;
 	int		status;
+	struct nsm_fo_monitored *fo_entry;
 
 	dprintk("lockd: nsm_monitor(%s)\n", host->h_name);
 	BUG_ON(nsm == NULL);
 
-	if (nsm->sm_monitored)
+	if (nsm_is_monitored(host, server)) {
+		dprintk("nsm_monitor: sm_monitored is true - returning 0\n");
 		return 0;
+	}
 
-	status = nsm_mon_unmon(nsm, SM_MON, &res);
+	status = nsm_mon_unmon(nsm, SM_MON, &res, server);
 
 	if (status < 0 || res.status != 0)
 		printk(KERN_NOTICE "lockd: cannot monitor %s\n", host->h_name);
-	else
+	else if (nsm->sm_monitored) {
+		fo_entry = kzalloc(sizeof(struct nsm_fo_monitored), GFP_KERNEL);
+		if (!fo_entry) {
+			printk("lockd: out of memory, can't add fo_entry\n");
+			return -ENOMEM;
+		}
+		fo_entry->addr = server;
+		INIT_LIST_HEAD(&fo_entry->list);
+		list_add(&fo_entry->list, &nsm->sm_serverip.list);
+	} else
 		nsm->sm_monitored = 1;
+
 	return status;
 }
 
@@ -98,7 +147,9 @@ nsm_unmonitor(struct nlm_host *host)
 {
 	struct nsm_handle *nsm = host->h_nsmhandle;
 	struct nsm_res	res;
-	int		status = 0;
+	int		status = 0, error=0;
+	struct list_head *pos, *head=&nsm->sm_serverip.list;
+	struct nsm_fo_monitored *fo_entry;
 
 	if (nsm == NULL)
 		return 0;
@@ -108,11 +159,21 @@ nsm_unmonitor(struct nlm_host *host)
 	 && nsm->sm_monitored && !nsm->sm_sticky) {
 		dprintk("lockd: nsm_unmonitor(%s)\n", host->h_name);
 
-		status = nsm_mon_unmon(nsm, SM_UNMON, &res);
-		if (status < 0)
-			printk(KERN_NOTICE "lockd: cannot unmonitor %s\n",
+		/* Unmonitor each server IP 
+		 * 	todo: need to re-think error handling
+		 */
+		mutex_lock(&nsm->sm_mutex);
+		list_for_each(pos, head) {
+			fo_entry = list_entry(pos, struct nsm_fo_monitored, list);
+			status = nsm_mon_unmon(nsm, SM_UNMON, &res, fo_entry->addr);
+			if (status < 0) {
+				error++;
+				printk(KERN_NOTICE "lockd: cannot unmonitor %s\n",
 					host->h_name);
-		else
+			} 
+		}
+		mutex_unlock(&nsm->sm_mutex);
+		if (!error)
 			nsm->sm_monitored = 0;
 	}
 	nsm_release(nsm);
@@ -144,6 +205,13 @@ nsm_create(void)
 	return rpc_create(&args);
 }
 
+/* We want "buffer" in xdr_encode_common() to hold
+ * either the system_utsname.nodename string (__NEW_UTS_LEN+1)
+ * or IPv4 dot notation (16 bytes+1) for now.
+ */
+
+#define        XDR_ENCODE_BUF_LEN __NEW_UTS_LEN+1
+
 /*
  * XDR functions for NSM.
  */
@@ -151,7 +219,8 @@ nsm_create(void)
 static __be32 *
 xdr_encode_common(struct rpc_rqst *rqstp, __be32 *p, struct nsm_args *argp)
 {
-	char	buffer[20], *name;
+	char	*name;
+	char    buffer[XDR_ENCODE_BUF_LEN];
 
 	/*
 	 * Use the dotted-quad IP address of the remote host as
@@ -161,13 +230,20 @@ xdr_encode_common(struct rpc_rqst *rqstp
 	 */
 	if (nsm_use_hostnames) {
 		name = argp->mon_name;
-	} else {
+	} else { 
 		sprintf(buffer, "%u.%u.%u.%u", NIPQUAD(argp->addr));
 		name = buffer;
 	}
-	if (!(p = xdr_encode_string(p, name))
-	 || !(p = xdr_encode_string(p, utsname()->nodename)))
+	if (!(p = xdr_encode_string(p, name)))
+		return ERR_PTR(-EIO);
+
+	if (argp->serv)
+		sprintf(buffer, "%u.%u.%u.%u", NIPQUAD(argp->serv));
+	else
+		sprintf(buffer, "%s", utsname()->nodename);
+	if (!(p = xdr_encode_string(p, buffer)))
 		return ERR_PTR(-EIO);
+
 	*p++ = htonl(argp->prog);
 	*p++ = htonl(argp->vers);
 	*p++ = htonl(argp->proc);
--- linux-nlm-2/fs/lockd/clntproc.c	2007-03-26 18:19:10.000000000 -0400
+++ linux/fs/lockd/clntproc.c	2007-04-03 21:49:04.000000000 -0400
@@ -500,7 +500,7 @@ nlmclnt_lock(struct nlm_rqst *req, struc
 	unsigned char fl_flags = fl->fl_flags;
 	int status = -ENOLCK;
 
-	if (nsm_monitor(host) < 0) {
+	if (nsm_monitor(host, 0) < 0) {
 		printk(KERN_NOTICE "lockd: failed to monitor %s\n",
 					host->h_name);
 		goto out;



[-- Attachment #3: Type: text/plain, Size: 345 bytes --]

-------------------------------------------------------------------------
Take Surveys. Earn Cash. Influence the Future of IT
Join SourceForge.net's Techsay panel and you'll get the chance to share your
opinions on IT & business topics through brief surveys-and earn cash
http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV

[-- Attachment #4: Type: text/plain, Size: 140 bytes --]

_______________________________________________
NFS maillist  -  NFS@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/nfs

Re: [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Olaf Kirch]

[-- Attachment #1.1: Type: text/plain, Size: 1574 bytes --]

On Thursday 05 April 2007 23:52, Wendy Cheng wrote:
> The changes record the ip interface that accepts the lock requests and 
> passes the correct "my_name" (in standard IPV4 dot notation) to user 
> mode statd (instead of system_utsname.nodename). This enables rpc.statd 
> to add the correct taken-over IPv4 address into the 3rd parameter of 
> ha_callout program. Current nfs-utils always resets "my_name" into 
> loopback address (127.0.0.1), regardless the statement made in rpc.statd 
> man page. Check out "man rpc.statd" and "man sm-notify" for details.

I don't think this is the right approach. For one, there's not enough
room in the SM_MON request to accomodate an additional IPv6
address, so you would have to come up with something entirely
different for IPv6 anyway. But more importantly, I think we should
move away from associating all sorts of network level addresses
with lockd state - addresses are just smoke and mirrors. Despite
all of NLM/NSMs shortcomings, there's a vehicle to convey identity,
and that's mon_name. IMHO the focus should be on making it work
properly if it doesn't do what you do.

But - why do you need to record the address on which the request was
received. at all? Don't you know beforehand on which IP addresses you
will be servicing NFS requests, and which will need to be migrated?

Side note: should we think about replacing SM_MON with some new
design altogether (think netlink)?

Olaf
-- 
Olaf Kirch  |  --- o --- Nous sommes du soleil we love when we play
okir@lst.de |    / | \   sol.dhoop.naytheet.ah kin.ir.samse.qurax

[-- Attachment #2: Type: text/plain, Size: 345 bytes --]

-------------------------------------------------------------------------
Take Surveys. Earn Cash. Influence the Future of IT
Join SourceForge.net's Techsay panel and you'll get the chance to share your
opinions on IT & business topics through brief surveys-and earn cash
http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV

[-- Attachment #3: Type: text/plain, Size: 140 bytes --]

_______________________________________________
NFS maillist  -  NFS@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/nfs

[Cluster-devel] Re: [NFS] [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Olaf Kirch]

On Thursday 05 April 2007 23:52, Wendy Cheng wrote:
> The changes record the ip interface that accepts the lock requests and 
> passes the correct "my_name" (in standard IPV4 dot notation) to user 
> mode statd (instead of system_utsname.nodename). This enables rpc.statd 
> to add the correct taken-over IPv4 address into the 3rd parameter of 
> ha_callout program. Current nfs-utils always resets "my_name" into 
> loopback address (127.0.0.1), regardless the statement made in rpc.statd 
> man page. Check out "man rpc.statd" and "man sm-notify" for details.

I don't think this is the right approach. For one, there's not enough
room in the SM_MON request to accomodate an additional IPv6
address, so you would have to come up with something entirely
different for IPv6 anyway. But more importantly, I think we should
move away from associating all sorts of network level addresses
with lockd state - addresses are just smoke and mirrors. Despite
all of NLM/NSMs shortcomings, there's a vehicle to convey identity,
and that's mon_name. IMHO the focus should be on making it work
properly if it doesn't do what you do.

But - why do you need to record the address on which the request was
received. at all? Don't you know beforehand on which IP addresses you
will be servicing NFS requests, and which will need to be migrated?

Side note: should we think about replacing SM_MON with some new
design altogether (think netlink)?

Olaf
-- 
Olaf Kirch  |  --- o --- Nous sommes du soleil we love when we play
okir at lst.de |    / | \   sol.dhoop.naytheet.ah kin.ir.samse.qurax

[Cluster-devel] Re: [NFS] [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Lon Hohberger]

On Tue, Apr 10, 2007 at 11:09:43AM +0200, Olaf Kirch wrote:
> On Thursday 05 April 2007 23:52, Wendy Cheng wrote:
> > The changes record the ip interface that accepts the lock requests and 
> > passes the correct "my_name" (in standard IPV4 dot notation) to user 
> > mode statd (instead of system_utsname.nodename). This enables rpc.statd 
> > to add the correct taken-over IPv4 address into the 3rd parameter of 
> > ha_callout program. Current nfs-utils always resets "my_name" into 
> > loopback address (127.0.0.1), regardless the statement made in rpc.statd 
> > man page. Check out "man rpc.statd" and "man sm-notify" for details.
> 
> I don't think this is the right approach. For one, there's not enough
> room in the SM_MON request to accomodate an additional IPv6
> address, so you would have to come up with something entirely
> different for IPv6 anyway.

This is true.

> But more importantly, I think we should
> move away from associating all sorts of network level addresses
> with lockd state - addresses are just smoke and mirrors.

> Despite
> all of NLM/NSMs shortcomings, there's a vehicle to convey identity,
> and that's mon_name. IMHO the focus should be on making it work
> properly if it doesn't do what you do.

We'd have to give it an arbitrary name, completely disassociated from
all network addresses/hostnames/etc.  That could work.  The problems
don't go away, though - they just become different:

* multiple mon_names must be able to exist per-server, since
services in a cluster are not always advertised on the same node (and,
of course, multiple NFS services may exist and *MUST* operate
independently without affecting one-another)

* we have to tell clients our mon_name somehow; since it's not
associated with a server or an IP address

I guess the question is: How is mon_name determined currently by the
clients?


> But - why do you need to record the address on which the request was
> received. at all? Don't you know beforehand on which IP addresses you
> will be servicing NFS requests, and which will need to be migrated?

Here's an answer to the 'why'.  [Clearly, this is an IPv4-centric
example, but it's been implemented this way in the past, so we'll use
it.]

It matters if you have multiple virtual IPs servicing multiple file
systems.  Here's an overly complicated example, which indicates a 'why'
we might need per-address monitoring:

ip address 1
ip address 2
ip address 3
export 1 (file system 1)
export 2 (file system 1)
export 3 (file system 2)
export 4 (file system 2)
export 5 (file system 3)

client A mounts export 1 and 3 via IP address 1
client A mounts export 5 via IP address 2
client B mounts export 2 and 4 via IP address 2
client B mounts export 5 via IP address 1
client C mounts export 5 via IP address 3

Assume locks are taken in all cases.

The mapping we need to know becomes:

file system 1:
   client A via IP 1
   client B via IP 2
file system 2:
   client A via IP 1
   client B via IP 2
file system 3:
   client A via IP 2
   client B via IP 1
   client C via IP 3

For *correct* reclaim (no extraneous SM_NOTIFY requests to the wrong
clients, SM_NOTIFY correctly sent to each client), we must send the
following using the NSM design:

   SM_NOTIFY to client A via IP 1
   SM_NOTIFY to client A via IP 2
   SM_NOTIFY to client B via IP 1
   SM_NOTIFY to client B via IP 2
   SM_NOTIFY to client C via IP 3

Currently, if we do it by file system, fsid, etc - there is no
indication of the path SM_NOTIFY messages need to take.  I.e. If we
send to all clients via any old IP address for a specific file system,
rpc.statd on the remote host will drop the request, and locks won't get
reclaimed.  

One beautiful thing about the above (perhaps otherwise ugly) approach
(storing the entire fs/client/server-ip mapping) is that we maintain
compatibility with other NFS implementations.  We don't break anything
from the client's perspective; it works like it always did.

If we were able to use mon_name for the above set, it would be a single
message per client.  On the client side, there would need to be a
mapping of the mon_name to each network-level address.  When we send
SM_NOTIFY to each client, it can then reclaim the locks for all
associated server addresses (which may be different network protocols).

> Side note: should we think about replacing SM_MON with some new
> design altogether

Possibly.

> (think netlink)?

Redesign of the SM_MON messaging doesn't necessarily require rewriting
the underlying lockd->statd communication path.  That said, I have no
opinion about the merits (or not) of using netlink over the current
implementation.

We could also make a file system...

-- Lon

-- 
Lon Hohberger - Software Engineer - Red Hat, Inc.

Re: [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Lon Hohberger]

On Tue, Apr 10, 2007 at 11:09:43AM +0200, Olaf Kirch wrote:
> On Thursday 05 April 2007 23:52, Wendy Cheng wrote:
> > The changes record the ip interface that accepts the lock requests and 
> > passes the correct "my_name" (in standard IPV4 dot notation) to user 
> > mode statd (instead of system_utsname.nodename). This enables rpc.statd 
> > to add the correct taken-over IPv4 address into the 3rd parameter of 
> > ha_callout program. Current nfs-utils always resets "my_name" into 
> > loopback address (127.0.0.1), regardless the statement made in rpc.statd 
> > man page. Check out "man rpc.statd" and "man sm-notify" for details.
> 
> I don't think this is the right approach. For one, there's not enough
> room in the SM_MON request to accomodate an additional IPv6
> address, so you would have to come up with something entirely
> different for IPv6 anyway.

This is true.

> But more importantly, I think we should
> move away from associating all sorts of network level addresses
> with lockd state - addresses are just smoke and mirrors.

> Despite
> all of NLM/NSMs shortcomings, there's a vehicle to convey identity,
> and that's mon_name. IMHO the focus should be on making it work
> properly if it doesn't do what you do.

We'd have to give it an arbitrary name, completely disassociated from
all network addresses/hostnames/etc.  That could work.  The problems
don't go away, though - they just become different:

* multiple mon_names must be able to exist per-server, since
services in a cluster are not always advertised on the same node (and,
of course, multiple NFS services may exist and *MUST* operate
independently without affecting one-another)

* we have to tell clients our mon_name somehow; since it's not
associated with a server or an IP address

I guess the question is: How is mon_name determined currently by the
clients?


> But - why do you need to record the address on which the request was
> received. at all? Don't you know beforehand on which IP addresses you
> will be servicing NFS requests, and which will need to be migrated?

Here's an answer to the 'why'.  [Clearly, this is an IPv4-centric
example, but it's been implemented this way in the past, so we'll use
it.]

It matters if you have multiple virtual IPs servicing multiple file
systems.  Here's an overly complicated example, which indicates a 'why'
we might need per-address monitoring:

ip address 1
ip address 2
ip address 3
export 1 (file system 1)
export 2 (file system 1)
export 3 (file system 2)
export 4 (file system 2)
export 5 (file system 3)

client A mounts export 1 and 3 via IP address 1
client A mounts export 5 via IP address 2
client B mounts export 2 and 4 via IP address 2
client B mounts export 5 via IP address 1
client C mounts export 5 via IP address 3

Assume locks are taken in all cases.

The mapping we need to know becomes:

file system 1:
   client A via IP 1
   client B via IP 2
file system 2:
   client A via IP 1
   client B via IP 2
file system 3:
   client A via IP 2
   client B via IP 1
   client C via IP 3

For *correct* reclaim (no extraneous SM_NOTIFY requests to the wrong
clients, SM_NOTIFY correctly sent to each client), we must send the
following using the NSM design:

   SM_NOTIFY to client A via IP 1
   SM_NOTIFY to client A via IP 2
   SM_NOTIFY to client B via IP 1
   SM_NOTIFY to client B via IP 2
   SM_NOTIFY to client C via IP 3

Currently, if we do it by file system, fsid, etc - there is no
indication of the path SM_NOTIFY messages need to take.  I.e. If we
send to all clients via any old IP address for a specific file system,
rpc.statd on the remote host will drop the request, and locks won't get
reclaimed.  

One beautiful thing about the above (perhaps otherwise ugly) approach
(storing the entire fs/client/server-ip mapping) is that we maintain
compatibility with other NFS implementations.  We don't break anything
from the client's perspective; it works like it always did.

If we were able to use mon_name for the above set, it would be a single
message per client.  On the client side, there would need to be a
mapping of the mon_name to each network-level address.  When we send
SM_NOTIFY to each client, it can then reclaim the locks for all
associated server addresses (which may be different network protocols).

> Side note: should we think about replacing SM_MON with some new
> design altogether

Possibly.

> (think netlink)?

Redesign of the SM_MON messaging doesn't necessarily require rewriting
the underlying lockd->statd communication path.  That said, I have no
opinion about the merits (or not) of using netlink over the current
implementation.

We could also make a file system...

-- Lon

-- 
Lon Hohberger - Software Engineer - Red Hat, Inc.

-------------------------------------------------------------------------
Take Surveys. Earn Cash. Influence the Future of IT
Join SourceForge.net's Techsay panel and you'll get the chance to share your
opinions on IT & business topics through brief surveys-and earn cash
http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV
_______________________________________________
NFS maillist  -  NFS@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/nfs

[Cluster-devel] Re: [NFS] [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Wendy Cheng]

Olaf Kirch wrote:
> On Thursday 05 April 2007 23:52, Wendy Cheng wrote:
>   
>> The changes record the ip interface that accepts the lock requests and 
>> passes the correct "my_name" (in standard IPV4 dot notation) to user 
>> mode statd (instead of system_utsname.nodename). This enables rpc.statd 
>> to add the correct taken-over IPv4 address into the 3rd parameter of 
>> ha_callout program. Current nfs-utils always resets "my_name" into 
>> loopback address (127.0.0.1), regardless the statement made in rpc.statd 
>> man page. Check out "man rpc.statd" and "man sm-notify" for details.
>>     
>
> I don't think this is the right approach. For one, there's not enough
> room in the SM_MON request to accomodate an additional IPv6
> address, so you would have to come up with something entirely
> different for IPv6 anyway. 

The original plan was to pass fsid instead of floating ip but it 
required some major restructures on host lookup and file lookup (in 
nlmsvc_retrieve_args). I have been hoping by the time IPV6 is really 
required, NFS V4 would be mature enough to get deployed (so this would 
be a non-issue anyway).

If people doesn't mind to restructure the sequence of host and file 
lookup, passing fsid can be one of the strong candidates to get this right.

> But more importantly, I think we should
> move away from associating all sorts of network level addresses
> with lockd state - addresses are just smoke and mirrors. Despite
> all of NLM/NSMs shortcomings, there's a vehicle to convey identity,
> and that's mon_name. IMHO the focus should be on making it work
> properly if it doesn't do what you do.
>
> But - why do you need to record the address on which the request was
> received. at all? Don't you know beforehand on which IP addresses you
> will be servicing NFS requests, and which will need to be migrated?
>
> Side note: should we think about replacing SM_MON with some new
> design altogether (think netlink)?
>
>   
Totally agree ! More on this later when I'm back to office.

-- Wendy

Re: [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Wendy Cheng]

Olaf Kirch wrote:
> On Thursday 05 April 2007 23:52, Wendy Cheng wrote:
>   
>> The changes record the ip interface that accepts the lock requests and 
>> passes the correct "my_name" (in standard IPV4 dot notation) to user 
>> mode statd (instead of system_utsname.nodename). This enables rpc.statd 
>> to add the correct taken-over IPv4 address into the 3rd parameter of 
>> ha_callout program. Current nfs-utils always resets "my_name" into 
>> loopback address (127.0.0.1), regardless the statement made in rpc.statd 
>> man page. Check out "man rpc.statd" and "man sm-notify" for details.
>>     
>
> I don't think this is the right approach. For one, there's not enough
> room in the SM_MON request to accomodate an additional IPv6
> address, so you would have to come up with something entirely
> different for IPv6 anyway. 

The original plan was to pass fsid instead of floating ip but it 
required some major restructures on host lookup and file lookup (in 
nlmsvc_retrieve_args). I have been hoping by the time IPV6 is really 
required, NFS V4 would be mature enough to get deployed (so this would 
be a non-issue anyway).

If people doesn't mind to restructure the sequence of host and file 
lookup, passing fsid can be one of the strong candidates to get this right.

> But more importantly, I think we should
> move away from associating all sorts of network level addresses
> with lockd state - addresses are just smoke and mirrors. Despite
> all of NLM/NSMs shortcomings, there's a vehicle to convey identity,
> and that's mon_name. IMHO the focus should be on making it work
> properly if it doesn't do what you do.
>
> But - why do you need to record the address on which the request was
> received. at all? Don't you know beforehand on which IP addresses you
> will be servicing NFS requests, and which will need to be migrated?
>
> Side note: should we think about replacing SM_MON with some new
> design altogether (think netlink)?
>
>   
Totally agree ! More on this later when I'm back to office.

-- Wendy


-------------------------------------------------------------------------
Take Surveys. Earn Cash. Influence the Future of IT
Join SourceForge.net's Techsay panel and you'll get the chance to share your
opinions on IT & business topics through brief surveys-and earn cash
http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV
_______________________________________________
NFS maillist  -  NFS@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/nfs

[Cluster-devel] Re: [NFS] [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Wendy Cheng]

Olaf Kirch wrote:
> But - why do you need to record the address on which the request was
> received. at all? Don't you know beforehand on which IP addresses you
> will be servicing NFS requests, and which will need to be migrated?
>
>   

Sorry, somehow I missed this question ...

The intent is to avoid sending notifications to *all* clients.  The 
cluster user mode scripts will have to associate the floating IP(s) with 
fsid(s) and relies heavily on the information passed by rpc.statd's 
ha-callout program. My test ha-callout user mode program shows the 
following:

/mnt/lockd/ha-pg/hatest is invoked with 3 arguments.
    add-client
    192.168.24.146
    192.168.24.149
/mnt/lockd/ha-pg/hatest is invoked with 3 arguments.
    del-client
    192.168.24.145
    192.168.24.149

These info(s) will be used to construct the notification list when 
notification actually take places.

-- Wendy

Re: [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Wendy Cheng]

Olaf Kirch wrote:
> But - why do you need to record the address on which the request was
> received. at all? Don't you know beforehand on which IP addresses you
> will be servicing NFS requests, and which will need to be migrated?
>
>   

Sorry, somehow I missed this question ...

The intent is to avoid sending notifications to *all* clients.  The 
cluster user mode scripts will have to associate the floating IP(s) with 
fsid(s) and relies heavily on the information passed by rpc.statd's 
ha-callout program. My test ha-callout user mode program shows the 
following:

/mnt/lockd/ha-pg/hatest is invoked with 3 arguments.
    add-client
    192.168.24.146
    192.168.24.149
/mnt/lockd/ha-pg/hatest is invoked with 3 arguments.
    del-client
    192.168.24.145
    192.168.24.149

These info(s) will be used to construct the notification list when 
notification actually take places.

-- Wendy


-------------------------------------------------------------------------
Take Surveys. Earn Cash. Influence the Future of IT
Join SourceForge.net's Techsay panel and you'll get the chance to share your
opinions on IT & business topics through brief surveys-and earn cash
http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV
_______________________________________________
NFS maillist  -  NFS@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/nfs

[Cluster-devel] Re: [NFS] [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Neil Brown]

On Tuesday April 10, okir at lst.de wrote:
> On Thursday 05 April 2007 23:52, Wendy Cheng wrote:
> > The changes record the ip interface that accepts the lock requests and 
> > passes the correct "my_name" (in standard IPV4 dot notation) to user 
> > mode statd (instead of system_utsname.nodename). This enables rpc.statd 
> > to add the correct taken-over IPv4 address into the 3rd parameter of 
> > ha_callout program. Current nfs-utils always resets "my_name" into 
> > loopback address (127.0.0.1), regardless the statement made in rpc.statd 
> > man page. Check out "man rpc.statd" and "man sm-notify" for details.
> 
> I don't think this is the right approach. For one, there's not enough
> room in the SM_MON request to accomodate an additional IPv6
> address, so you would have to come up with something entirely
> different for IPv6 anyway. But more importantly, I think we should
> move away from associating all sorts of network level addresses
> with lockd state - addresses are just smoke and mirrors. Despite
> all of NLM/NSMs shortcomings, there's a vehicle to convey identity,
> and that's mon_name. IMHO the focus should be on making it work
> properly if it doesn't do what you do.

I don't understand your complaint.
You say there's "not enough room", but the extra information is being
placed in the 'my_name' string which is up to 1024 bytes long, which
I think is long enough.

You say that "mon_name" is the vehicle to convey identity and while
that is true, I don't think it is relevant as it is the identity of
the *server* that is being conveyed, rather than the identity of the
client (this on an NFS server).

Think of it like this.  The goal is (appears to be) to make it
possible to implement multiple independent NFS servers on the one
Linux host.
As a simplification, each server serves precisely one filesystem which
no other server serves, and each server has precisely one network
address which no other server shares.

So the 'servers' can be identified either by the filesystem (or FSID)
or by the network address.

Most NFS operations are file-local or at most filesystem-local and so
they don't need to care that there are multiple servers.  But locking
and peer-restart in particular is not.  It is server-local.  So for
the peer monitoring/notification operations, we need to enhance to
model to make the server name explicit rather than implicit ('this
host').

To allow a 'server' to migrate from one host to another (active-active
failover) we need to synthesise a reboot which means knowing which
clients are using which server.

lockd knows which is which either based on the destination network
address of the lock request, or the filesystem on which the lock was
taken.   Somehow this information needs to get communicated to statd
so that different 'sm' directories can be used.  my_name seems a
sensible place to put the information.


However:  now that I think I actually understand what is happening, I
wonder if FSID and IPaddress are really the right handles to use.  It
would seem to make more sense to use the filesystem name (i.e. a
path).

So I'm suggesting writing a directory name to
    /proc/fs/nfsd/nlm_unlock
and maybe also to
    /proc/fs/nlm_restart_grace_for_fs

and have 'my_name' in the SM_MON request be the path name of the
export point rather the network address.

Thinking more,  lockd would need to know whether each filesystem is an
independent server so that it knows if independent nsm objects are
needed.

So maybe we want an export flag "active_failover".
If this is set then the filesystem has an independent grace period
that starts on first export (except that 'first export' isn't really a
well defined concept) and lockd treats clients using that filesystem
as different from the same client using any other filesystem.

I'm not sure we really need the 'nlm_unlock' interface either.  We can
just synthesis incoming SM_NOTIFYs from all clients of that filesystem
and the locks will go away.

Not that I'm saying we have to use this approach rather than the
current one.  I'm just exploring the issue and making sure that I
understand it.


> 
> But - why do you need to record the address on which the request was
> received. at all? Don't you know beforehand on which IP addresses you
> will be servicing NFS requests, and which will need to be migrated?
> 
> Side note: should we think about replacing SM_MON with some new
> design altogether (think netlink)?

Well, I want something new to support the various state that needs to
be recorded by NFSv4, and what ever gets created could probably be
used for lockd/statd too.
But given that we have SM_MON implemented, what is so broken that it
needs to be replaced?

NeilBrown

Re: [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Neil Brown]

On Tuesday April 10, okir@lst.de wrote:
> On Thursday 05 April 2007 23:52, Wendy Cheng wrote:
> > The changes record the ip interface that accepts the lock requests and 
> > passes the correct "my_name" (in standard IPV4 dot notation) to user 
> > mode statd (instead of system_utsname.nodename). This enables rpc.statd 
> > to add the correct taken-over IPv4 address into the 3rd parameter of 
> > ha_callout program. Current nfs-utils always resets "my_name" into 
> > loopback address (127.0.0.1), regardless the statement made in rpc.statd 
> > man page. Check out "man rpc.statd" and "man sm-notify" for details.
> 
> I don't think this is the right approach. For one, there's not enough
> room in the SM_MON request to accomodate an additional IPv6
> address, so you would have to come up with something entirely
> different for IPv6 anyway. But more importantly, I think we should
> move away from associating all sorts of network level addresses
> with lockd state - addresses are just smoke and mirrors. Despite
> all of NLM/NSMs shortcomings, there's a vehicle to convey identity,
> and that's mon_name. IMHO the focus should be on making it work
> properly if it doesn't do what you do.

I don't understand your complaint.
You say there's "not enough room", but the extra information is being
placed in the 'my_name' string which is up to 1024 bytes long, which
I think is long enough.

You say that "mon_name" is the vehicle to convey identity and while
that is true, I don't think it is relevant as it is the identity of
the *server* that is being conveyed, rather than the identity of the
client (this on an NFS server).

Think of it like this.  The goal is (appears to be) to make it
possible to implement multiple independent NFS servers on the one
Linux host.
As a simplification, each server serves precisely one filesystem which
no other server serves, and each server has precisely one network
address which no other server shares.

So the 'servers' can be identified either by the filesystem (or FSID)
or by the network address.

Most NFS operations are file-local or at most filesystem-local and so
they don't need to care that there are multiple servers.  But locking
and peer-restart in particular is not.  It is server-local.  So for
the peer monitoring/notification operations, we need to enhance to
model to make the server name explicit rather than implicit ('this
host').

To allow a 'server' to migrate from one host to another (active-active
failover) we need to synthesise a reboot which means knowing which
clients are using which server.

lockd knows which is which either based on the destination network
address of the lock request, or the filesystem on which the lock was
taken.   Somehow this information needs to get communicated to statd
so that different 'sm' directories can be used.  my_name seems a
sensible place to put the information.


However:  now that I think I actually understand what is happening, I
wonder if FSID and IPaddress are really the right handles to use.  It
would seem to make more sense to use the filesystem name (i.e. a
path).

So I'm suggesting writing a directory name to
    /proc/fs/nfsd/nlm_unlock
and maybe also to
    /proc/fs/nlm_restart_grace_for_fs

and have 'my_name' in the SM_MON request be the path name of the
export point rather the network address.

Thinking more,  lockd would need to know whether each filesystem is an
independent server so that it knows if independent nsm objects are
needed.

So maybe we want an export flag "active_failover".
If this is set then the filesystem has an independent grace period
that starts on first export (except that 'first export' isn't really a
well defined concept) and lockd treats clients using that filesystem
as different from the same client using any other filesystem.

I'm not sure we really need the 'nlm_unlock' interface either.  We can
just synthesis incoming SM_NOTIFYs from all clients of that filesystem
and the locks will go away.

Not that I'm saying we have to use this approach rather than the
current one.  I'm just exploring the issue and making sure that I
understand it.


> 
> But - why do you need to record the address on which the request was
> received. at all? Don't you know beforehand on which IP addresses you
> will be servicing NFS requests, and which will need to be migrated?
> 
> Side note: should we think about replacing SM_MON with some new
> design altogether (think netlink)?

Well, I want something new to support the various state that needs to
be recorded by NFSv4, and what ever gets created could probably be
used for lockd/statd too.
But given that we have SM_MON implemented, what is so broken that it
needs to be replaced?

NeilBrown

-------------------------------------------------------------------------
Take Surveys. Earn Cash. Influence the Future of IT
Join SourceForge.net's Techsay panel and you'll get the chance to share your
opinions on IT & business topics through brief surveys-and earn cash
http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV
_______________________________________________
NFS maillist  -  NFS@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/nfs

[Cluster-devel] Re: [NFS] [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Lon Hohberger]

On Wed, Apr 11, 2007 at 02:50:04PM +1000, Neil Brown wrote:
> Think of it like this.  The goal is (appears to be) to make it
> possible to implement multiple independent NFS servers on the one
> Linux host.
> As a simplification, each server serves precisely one filesystem which
> no other server serves, and each server has precisely one network
> address which no other server shares.

The simplification here is a bit incorrect; we need to be able to handle
a given *set* of size >= 1 IPs which is paired with a *set* of size >= 1
of fsids.  The sets are not broken up (that is, we won't be moving 
file systems between IP addresses or vice-vers), but the number of
IPs:filesystems need not be 1:1.

   * 1 file system, 1 ip address
   * 2 fs, 1 ip
   * 1 ip, 2 fs
   * 18 ip, 42 fs.

... should all work without extraneous stuff being done.


> lockd knows which is which either based on the destination network
> address of the lock request, or the filesystem on which the lock was
> taken.   Somehow this information needs to get communicated to statd
> so that different 'sm' directories can be used.  my_name seems a
> sensible place to put the information.

Right.  Using the inbound-destination address instead of fsid or file
system will ensure the right address is used for sending the SM_NOTIFY
requests.  However, fsid, or export path - on a service with more than
one network address will not, I think...

Obviously, I agree with Olaf in that using all sorts of random network
protocol addresses is a hair ugly - but I think it's better than having
to send out multiple SM_NOTIFY messages on multiple net addrs just to
ensure that a particular client gets SM_NOTIFY from the *right* net
addr that it had in its monitor list...

If we could have a single 'name' per service that can get associated
with multiple network addresses, I'm all for it.

Just my opinion, of course.

> However:  now that I think I actually understand what is happening, I
> wonder if FSID and IPaddress are really the right handles to use.  It
> would seem to make more sense to use the filesystem name (i.e. a
> path).

This doesn't solve the 2 ip : 1 fs case very cleanly.  We need to send
SM_NOTIFY out on multiple IPs to ensure that each node gets it from the
right server address (that the client has on its monitor list)...?

Really, the goal is to have:

  * clients receive SM_NOTIFY from the ("virtual") host(s) they are
    monitoring when we start the service
  * No extraneous SM_NOTIFY requests are sent by server.
  * No extraneous SM_NOTIFY requests are received by clients.

-- 
Lon Hohberger - Software Engineer - Red Hat, Inc.

Re: [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Lon Hohberger]

On Wed, Apr 11, 2007 at 02:50:04PM +1000, Neil Brown wrote:
> Think of it like this.  The goal is (appears to be) to make it
> possible to implement multiple independent NFS servers on the one
> Linux host.
> As a simplification, each server serves precisely one filesystem which
> no other server serves, and each server has precisely one network
> address which no other server shares.

The simplification here is a bit incorrect; we need to be able to handle
a given *set* of size >= 1 IPs which is paired with a *set* of size >= 1
of fsids.  The sets are not broken up (that is, we won't be moving 
file systems between IP addresses or vice-vers), but the number of
IPs:filesystems need not be 1:1.

   * 1 file system, 1 ip address
   * 2 fs, 1 ip
   * 1 ip, 2 fs
   * 18 ip, 42 fs.

... should all work without extraneous stuff being done.


> lockd knows which is which either based on the destination network
> address of the lock request, or the filesystem on which the lock was
> taken.   Somehow this information needs to get communicated to statd
> so that different 'sm' directories can be used.  my_name seems a
> sensible place to put the information.

Right.  Using the inbound-destination address instead of fsid or file
system will ensure the right address is used for sending the SM_NOTIFY
requests.  However, fsid, or export path - on a service with more than
one network address will not, I think...

Obviously, I agree with Olaf in that using all sorts of random network
protocol addresses is a hair ugly - but I think it's better than having
to send out multiple SM_NOTIFY messages on multiple net addrs just to
ensure that a particular client gets SM_NOTIFY from the *right* net
addr that it had in its monitor list...

If we could have a single 'name' per service that can get associated
with multiple network addresses, I'm all for it.

Just my opinion, of course.

> However:  now that I think I actually understand what is happening, I
> wonder if FSID and IPaddress are really the right handles to use.  It
> would seem to make more sense to use the filesystem name (i.e. a
> path).

This doesn't solve the 2 ip : 1 fs case very cleanly.  We need to send
SM_NOTIFY out on multiple IPs to ensure that each node gets it from the
right server address (that the client has on its monitor list)...?

Really, the goal is to have:

  * clients receive SM_NOTIFY from the ("virtual") host(s) they are
    monitoring when we start the service
  * No extraneous SM_NOTIFY requests are sent by server.
  * No extraneous SM_NOTIFY requests are received by clients.

-- 
Lon Hohberger - Software Engineer - Red Hat, Inc.

-------------------------------------------------------------------------
Take Surveys. Earn Cash. Influence the Future of IT
Join SourceForge.net's Techsay panel and you'll get the chance to share your
opinions on IT & business topics through brief surveys-and earn cash
http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV
_______________________________________________
NFS maillist  -  NFS@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/nfs

[Cluster-devel] Re: [NFS] [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Wendy Cheng]

Lon Hohberger wrote:
> On Wed, Apr 11, 2007 at 02:50:04PM +1000, Neil Brown wrote:
>   
>> lockd knows which is which either based on the destination network
>> address of the lock request, or the filesystem on which the lock was
>> taken.   Somehow this information needs to get communicated to statd
>> so that different 'sm' directories can be used.  my_name seems a
>> sensible place to put the information.
>>     
>
> Right.  Using the inbound-destination address instead of fsid or file
> system will ensure the right address is used for sending the SM_NOTIFY
> requests.  However, fsid, or export path - on a service with more than
> one network address will not, I think...
>   

So I think we conclude here that inbound-destination address is required 
for SM_NOTIFY purpose.

Thanks for all the comments. Will summarize the issues on Monday and see 
what we should go from there.

-- Wendy

Re: [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Wendy Cheng]

Lon Hohberger wrote:
> On Wed, Apr 11, 2007 at 02:50:04PM +1000, Neil Brown wrote:
>   
>> lockd knows which is which either based on the destination network
>> address of the lock request, or the filesystem on which the lock was
>> taken.   Somehow this information needs to get communicated to statd
>> so that different 'sm' directories can be used.  my_name seems a
>> sensible place to put the information.
>>     
>
> Right.  Using the inbound-destination address instead of fsid or file
> system will ensure the right address is used for sending the SM_NOTIFY
> requests.  However, fsid, or export path - on a service with more than
> one network address will not, I think...
>   

So I think we conclude here that inbound-destination address is required 
for SM_NOTIFY purpose.

Thanks for all the comments. Will summarize the issues on Monday and see 
what we should go from there.

-- Wendy

-------------------------------------------------------------------------
Take Surveys. Earn Cash. Influence the Future of IT
Join SourceForge.net's Techsay panel and you'll get the chance to share your
opinions on IT & business topics through brief surveys-and earn cash
http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV
_______________________________________________
NFS maillist  -  NFS@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/nfs

[Cluster-devel] Re: [NFS] [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Olaf Kirch]

On Friday 13 April 2007 21:16, Lon Hohberger wrote:
> The simplification here is a bit incorrect; we need to be able to handle
> a given *set* of size >= 1 IPs which is paired with a *set* of size >= 1
> of fsids.  The sets are not broken up (that is, we won't be moving 
> file systems between IP addresses or vice-vers), but the number of
> IPs:filesystems need not be 1:1.
> 
>    * 1 file system, 1 ip address
>    * 2 fs, 1 ip
>    * 1 ip, 2 fs
>    * 18 ip, 42 fs.

So you move a bunch of file systems with a bunch of IPs, and that
information is recorded somewhere. So if you know you're moving
set X, which goes with IPs 1.2.3.4 and 1.2.3.5, you might as well
send notifications from both these IPs to all clients having established
locks on a file system from that set.

I think in term of correctness, it's better to send an SM_NOTIFY
for each IP associated with such a set, anyway.

To me, it seems the crucial point of information here when monitoring
a client, is which file system (or set of file systems) the client has
accessed.

Olaf
-- 
Olaf Kirch  |  --- o --- Nous sommes du soleil we love when we play
okir at lst.de |    / | \   sol.dhoop.naytheet.ah kin.ir.samse.qurax

Re: [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Olaf Kirch]

On Friday 13 April 2007 21:16, Lon Hohberger wrote:
> The simplification here is a bit incorrect; we need to be able to handle
> a given *set* of size >= 1 IPs which is paired with a *set* of size >= 1
> of fsids.  The sets are not broken up (that is, we won't be moving 
> file systems between IP addresses or vice-vers), but the number of
> IPs:filesystems need not be 1:1.
> 
>    * 1 file system, 1 ip address
>    * 2 fs, 1 ip
>    * 1 ip, 2 fs
>    * 18 ip, 42 fs.

So you move a bunch of file systems with a bunch of IPs, and that
information is recorded somewhere. So if you know you're moving
set X, which goes with IPs 1.2.3.4 and 1.2.3.5, you might as well
send notifications from both these IPs to all clients having established
locks on a file system from that set.

I think in term of correctness, it's better to send an SM_NOTIFY
for each IP associated with such a set, anyway.

To me, it seems the crucial point of information here when monitoring
a client, is which file system (or set of file systems) the client has
accessed.

Olaf
-- 
Olaf Kirch  |  --- o --- Nous sommes du soleil we love when we play
okir@lst.de |    / | \   sol.dhoop.naytheet.ah kin.ir.samse.qurax

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/
_______________________________________________
NFS maillist  -  NFS@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/nfs

[Cluster-devel] Re: [NFS] [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Wendy Cheng]

Olaf Kirch wrote:
> On Friday 13 April 2007 21:16, Lon Hohberger wrote:
>   
>> The simplification here is a bit incorrect; we need to be able to handle
>> a given *set* of size >= 1 IPs which is paired with a *set* of size >= 1
>> of fsids.  The sets are not broken up (that is, we won't be moving 
>> file systems between IP addresses or vice-vers), but the number of
>> IPs:filesystems need not be 1:1.
>>
>>    * 1 file system, 1 ip address
>>    * 2 fs, 1 ip
>>    * 1 ip, 2 fs
>>    * 18 ip, 42 fs.
>>     
>
> So you move a bunch of file systems with a bunch of IPs, and that
> information is recorded somewhere. So if you know you're moving
> set X, which goes with IPs 1.2.3.4 and 1.2.3.5, you might as well
> send notifications from both these IPs to all clients having established
> locks on a file system from that set.
>
> I think in term of correctness, it's better to send an SM_NOTIFY
> for each IP associated with such a set, anyway.
>   

That's exactly what we have been proposing... :) .. We'll rely heavily 
on HA callout program to tell us which client uses which (server) 
floating IP. 
> To me, it seems the crucial point of information here when monitoring
> a client, is which file system (or set of file systems) the client has
> accessed.
>
>   
The summary will be sent out shortly ... sorry - it is a little bit 
behind schedules due to other issues.

-- Wendy

Re: [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Wendy Cheng]

Olaf Kirch wrote:
> On Friday 13 April 2007 21:16, Lon Hohberger wrote:
>   
>> The simplification here is a bit incorrect; we need to be able to handle
>> a given *set* of size >= 1 IPs which is paired with a *set* of size >= 1
>> of fsids.  The sets are not broken up (that is, we won't be moving 
>> file systems between IP addresses or vice-vers), but the number of
>> IPs:filesystems need not be 1:1.
>>
>>    * 1 file system, 1 ip address
>>    * 2 fs, 1 ip
>>    * 1 ip, 2 fs
>>    * 18 ip, 42 fs.
>>     
>
> So you move a bunch of file systems with a bunch of IPs, and that
> information is recorded somewhere. So if you know you're moving
> set X, which goes with IPs 1.2.3.4 and 1.2.3.5, you might as well
> send notifications from both these IPs to all clients having established
> locks on a file system from that set.
>
> I think in term of correctness, it's better to send an SM_NOTIFY
> for each IP associated with such a set, anyway.
>   

That's exactly what we have been proposing... :) .. We'll rely heavily 
on HA callout program to tell us which client uses which (server) 
floating IP. 
> To me, it seems the crucial point of information here when monitoring
> a client, is which file system (or set of file systems) the client has
> accessed.
>
>   
The summary will be sent out shortly ... sorry - it is a little bit 
behind schedules due to other issues.

-- Wendy


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/
_______________________________________________
NFS maillist  -  NFS@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/nfs

[Cluster-devel] Re: [NFS] [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Olaf Kirch]

On Tuesday 17 April 2007 15:24, Wendy Cheng wrote:
> > I think in term of correctness, it's better to send an SM_NOTIFY
> > for each IP associated with such a set, anyway.
> >   
> 
> That's exactly what we have been proposing... :) .. We'll rely heavily 
> on HA callout program to tell us which client uses which (server) 
> floating IP. 

Correct me if I'm wrong, but it seems your patch records every IP
used by the client, rather than the *all* the IPs related to the set of
file systems being moved. So if there are several IPs for this set, you
will end up sending notifications only from those the client happened
to have talked to.

My point was, you move a set of file systems A, B and C, with
IPs X, Y, Z. You know what the addresses are, so from a
robustness point of view your best bet is to send SM_NOTIFY
messages from IPs X, Y, Z, regardless of whether the client has
been talking to all of them, or just one.

Olaf
-- 
Olaf Kirch  |  --- o --- Nous sommes du soleil we love when we play
okir at lst.de |    / | \   sol.dhoop.naytheet.ah kin.ir.samse.qurax

Re: [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Olaf Kirch]

On Tuesday 17 April 2007 15:24, Wendy Cheng wrote:
> > I think in term of correctness, it's better to send an SM_NOTIFY
> > for each IP associated with such a set, anyway.
> >   
> 
> That's exactly what we have been proposing... :) .. We'll rely heavily 
> on HA callout program to tell us which client uses which (server) 
> floating IP. 

Correct me if I'm wrong, but it seems your patch records every IP
used by the client, rather than the *all* the IPs related to the set of
file systems being moved. So if there are several IPs for this set, you
will end up sending notifications only from those the client happened
to have talked to.

My point was, you move a set of file systems A, B and C, with
IPs X, Y, Z. You know what the addresses are, so from a
robustness point of view your best bet is to send SM_NOTIFY
messages from IPs X, Y, Z, regardless of whether the client has
been talking to all of them, or just one.

Olaf
-- 
Olaf Kirch  |  --- o --- Nous sommes du soleil we love when we play
okir@lst.de |    / | \   sol.dhoop.naytheet.ah kin.ir.samse.qurax

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/
_______________________________________________
NFS maillist  -  NFS@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/nfs

[Cluster-devel] Re: [NFS] [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Wendy Cheng]

Olaf Kirch wrote:
> On Tuesday 17 April 2007 15:24, Wendy Cheng wrote:
>   
>>> I think in term of correctness, it's better to send an SM_NOTIFY
>>> for each IP associated with such a set, anyway.
>>>   
>>>       
>> That's exactly what we have been proposing... :) .. We'll rely heavily 
>> on HA callout program to tell us which client uses which (server) 
>> floating IP. 
>>     
>
> Correct me if I'm wrong, but it seems your patch records every IP
> used by the client, rather than the *all* the IPs related to the set of
> file systems being moved. So if there are several IPs for this set, you
> will end up sending notifications only from those the client happened
> to have talked to.
>   

Yes. Then why should we SM_NOTIFY the clients that do not have the 
associated locks (and introducing more possible reclaiming delay) ? Be 
aware that failover normally has timing constraints - it needs to get 
finished within a sensible time interval.

-- Wendy
> My point was, you move a set of file systems A, B and C, with
> IPs X, Y, Z. You know what the addresses are, so from a
> robustness point of view your best bet is to send SM_NOTIFY
> messages from IPs X, Y, Z, regardless of whether the client has
> been talking to all of them, or just one.
>
> Olaf
>   

Re: [PATCH 3/4 Revised] NLM - kernel lockd-statd changes

[Wendy Cheng]

Olaf Kirch wrote:
> On Tuesday 17 April 2007 15:24, Wendy Cheng wrote:
>   
>>> I think in term of correctness, it's better to send an SM_NOTIFY
>>> for each IP associated with such a set, anyway.
>>>   
>>>       
>> That's exactly what we have been proposing... :) .. We'll rely heavily 
>> on HA callout program to tell us which client uses which (server) 
>> floating IP. 
>>     
>
> Correct me if I'm wrong, but it seems your patch records every IP
> used by the client, rather than the *all* the IPs related to the set of
> file systems being moved. So if there are several IPs for this set, you
> will end up sending notifications only from those the client happened
> to have talked to.
>   

Yes. Then why should we SM_NOTIFY the clients that do not have the 
associated locks (and introducing more possible reclaiming delay) ? Be 
aware that failover normally has timing constraints - it needs to get 
finished within a sensible time interval.

-- Wendy
> My point was, you move a set of file systems A, B and C, with
> IPs X, Y, Z. You know what the addresses are, so from a
> robustness point of view your best bet is to send SM_NOTIFY
> messages from IPs X, Y, Z, regardless of whether the client has
> been talking to all of them, or just one.
>
> Olaf
>   


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/
_______________________________________________
NFS maillist  -  NFS@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/nfs