* [PATCH 1/8] NFS: Add a lame client-side function to display file handles
2011-03-10 17:29 [PATCH 0/8] Snapshot of client-side NFSv4 migration support Chuck Lever
@ 2011-03-10 17:29 ` Chuck Lever
2011-03-10 17:29 ` [PATCH 2/8] NFS: Save root file handle in nfs_server Chuck Lever
` (6 subsequent siblings)
7 siblings, 0 replies; 15+ messages in thread
From: Chuck Lever @ 2011-03-10 17:29 UTC (permalink / raw)
To: linux-nfs
nfs_display_fhandle() is a lame client-side function to display
file handles on the console. It is for debugging only.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
fs/nfs/inode.c | 44 ++++++++++++++++++++++++++++++++++++++++++++
include/linux/nfs_fs.h | 13 +++++++++++++
2 files changed, 57 insertions(+), 0 deletions(-)
diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
index 1cc600e..1325160 100644
--- a/fs/nfs/inode.c
+++ b/fs/nfs/inode.c
@@ -1033,6 +1033,50 @@ struct nfs_fh *nfs_alloc_fhandle(void)
}
/**
+ * _nfs_display_fhandle - display an NFS file handle on the console
+ *
+ * @fh: file handle to display
+ *
+ * For debugging only.
+ */
+#ifdef RPC_DEBUG
+void _nfs_display_fhandle(const struct nfs_fh *fh)
+{
+ unsigned short i;
+
+ if (fh->size == 0 || fh == NULL) {
+ printk(KERN_NOTICE "Empty NFS file handle at %p\n", fh);
+ return;
+ }
+
+ printk(KERN_NOTICE "NFS file handle at %p (size %u):\n", fh, fh->size);
+ for (i = 0; i < fh->size; i += 16) {
+ __be32 *pos = (__be32 *)&fh->data[i];
+
+ switch ((fh->size - i - 1) >> 2) {
+ case 0:
+ printk(KERN_NOTICE " %08x",
+ be32_to_cpup(pos));
+ break;
+ case 1:
+ printk(KERN_NOTICE " %08x %08x\n",
+ be32_to_cpup(pos), be32_to_cpup(pos + 1));
+ break;
+ case 2:
+ printk(KERN_NOTICE " %08x %08x %08x\n",
+ be32_to_cpup(pos), be32_to_cpup(pos + 1),
+ be32_to_cpup(pos + 2));
+ break;
+ default:
+ printk(KERN_NOTICE " %08x %08x %08x %08x\n",
+ be32_to_cpup(pos), be32_to_cpup(pos + 1),
+ be32_to_cpup(pos + 2), be32_to_cpup(pos + 3));
+ }
+ }
+}
+#endif
+
+/**
* nfs_inode_attrs_need_update - check if the inode attributes need updating
* @inode - pointer to inode
* @fattr - attributes
diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h
index 6023efa..511e22a 100644
--- a/include/linux/nfs_fs.h
+++ b/include/linux/nfs_fs.h
@@ -384,6 +384,19 @@ static inline void nfs_free_fhandle(const struct nfs_fh *fh)
kfree(fh);
}
+#ifdef RPC_DEBUG
+extern void _nfs_display_fhandle(const struct nfs_fh *fh);
+#define nfs_display_fhandle(fh) \
+ do { \
+ if (unlikely(nfs_debug & NFSDBG_FACILITY)) \
+ _nfs_display_fhandle(fh); \
+ } while(0)
+#else
+static inline void nfs_display_fhandle(const struct nfs_fh *fh)
+{
+}
+#endif
+
/*
* linux/fs/nfs/nfsroot.c
*/
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH 2/8] NFS: Save root file handle in nfs_server
2011-03-10 17:29 [PATCH 0/8] Snapshot of client-side NFSv4 migration support Chuck Lever
2011-03-10 17:29 ` [PATCH 1/8] NFS: Add a lame client-side function to display file handles Chuck Lever
@ 2011-03-10 17:29 ` Chuck Lever
2011-03-10 17:29 ` [PATCH 3/8] NFS: Introduce nfs4_proc_get_mig_status() Chuck Lever
` (5 subsequent siblings)
7 siblings, 0 replies; 15+ messages in thread
From: Chuck Lever @ 2011-03-10 17:29 UTC (permalink / raw)
To: linux-nfs
Save each FSID's root directory file handle in the export's local
nfs_server structure on the client. This FH can later be used by the
migration recovery logic.
NB: Saving the root FH is done only for NFSv4 mounts.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
fs/nfs/client.c | 1 +
fs/nfs/getroot.c | 6 ++++++
include/linux/nfs_fs_sb.h | 1 +
3 files changed, 8 insertions(+), 0 deletions(-)
diff --git a/fs/nfs/client.c b/fs/nfs/client.c
index f8c95e4..a2f016a 100644
--- a/fs/nfs/client.c
+++ b/fs/nfs/client.c
@@ -1081,6 +1081,7 @@ void nfs_free_server(struct nfs_server *server)
nfs_put_client(server->nfs_client);
nfs_free_iostats(server->io_stats);
+ nfs_free_fhandle(server->rootfh);
bdi_destroy(&server->backing_dev_info);
kfree(server);
nfs_release_automount_timer();
diff --git a/fs/nfs/getroot.c b/fs/nfs/getroot.c
index b5ffe8f..4b9fc6e 100644
--- a/fs/nfs/getroot.c
+++ b/fs/nfs/getroot.c
@@ -205,6 +205,12 @@ struct dentry *nfs4_get_root(struct super_block *sb, struct nfs_fh *mntfh)
ret = ERR_CAST(inode);
goto out;
}
+ server->rootfh = nfs_alloc_fhandle();
+ if (server->rootfh != NULL) {
+ dprintk("nfs_get_root: saving root FH\n");
+ nfs_display_fhandle(mntfh);
+ nfs_copy_fh(server->rootfh, mntfh);
+ }
error = nfs_superblock_set_dummy_root(sb, inode);
if (error != 0) {
diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
index b197563..961ee85 100644
--- a/include/linux/nfs_fs_sb.h
+++ b/include/linux/nfs_fs_sb.h
@@ -153,6 +153,7 @@ struct nfs_server {
#endif
struct list_head delegations;
void (*destroy)(struct nfs_server *);
+ struct nfs_fh *rootfh;
atomic_t active; /* Keep trace of any activity to this server */
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH 3/8] NFS: Introduce nfs4_proc_get_mig_status()
2011-03-10 17:29 [PATCH 0/8] Snapshot of client-side NFSv4 migration support Chuck Lever
2011-03-10 17:29 ` [PATCH 1/8] NFS: Add a lame client-side function to display file handles Chuck Lever
2011-03-10 17:29 ` [PATCH 2/8] NFS: Save root file handle in nfs_server Chuck Lever
@ 2011-03-10 17:29 ` Chuck Lever
2011-03-10 19:58 ` Trond Myklebust
2011-03-10 17:30 ` [PATCH 4/8] NFS: Add functions to swap transports during migration recovery Chuck Lever
` (4 subsequent siblings)
7 siblings, 1 reply; 15+ messages in thread
From: Chuck Lever @ 2011-03-10 17:29 UTC (permalink / raw)
To: linux-nfs
The nfs4_proc_fs_locations() function is invoked during referral
processing to perform a GETATTR(fs_locations) on an object's parent
directory in order to discover the target of the referral. It
performs a LOOKUP, so the client needs to know the parent's file
handle a priori.
During a migration event, an NFS4ERR_MOVED can be returned to some
client contexts that don't have the parent file handle readily
available. For example, suppose the proc function handling the
reply doesn't have a pointer to the object's dentry. In such cases,
all we have is the file handle of the object that got the error.
Or, we may need to probe fs_locations information on a root directory.
In that case, the "parent" of the root may not make sense.
Finally, a minor version zero server needs to know what client ID is
requesting fs_locations information, so it can clear the flag that
forces it to continue returning NFS4ERR_LEASE_MOVED. This flag is set
per client ID and per FSID. The client ID is not an argument of the
PUTFH or GETATTR operations.
Later minor versions have client ID information embedded in the
session ID. By convention, minor version zero clients send a RENEW
operation in the same compound as the GETATTR(fs_locations). (We may
need to visit this for referrals too, in case the LOOKUP operation is
actually seeing a migration).
Because of all of this, we need a new variant of
nfs4_proc_fs_locations() that can operate directly on a target file
handle, rather than taking a name and doing a LOOKUP as part of
retrieving fs_locations from the server. It also must properly append
a RENEW operation as needed. Introduce nfs4_proc_get_mig_status() to
fill this role, and add the requisite XDR encoding and decoding
paraphenalia.
Our client now has two ways to retrieve fs_locations data. This means
that the name of the ATTR flag that indicates the presence of
fs_locations data, NFS_ATTR_FATTR_V4_REFERRAL, no longer aligns with
the flag's usage. Introduce a new flag for indicating only that
fs_locations data is present. Keep the NFS_ATTR_FATTR_V4_REFERRAL
flag for indicating additionally that, in fact, a referral has
occurred.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
fs/nfs/nfs4_fs.h | 2 +
fs/nfs/nfs4proc.c | 54 ++++++++++++++++++++++++++++++-
fs/nfs/nfs4xdr.c | 81 ++++++++++++++++++++++++++++++++++++++++++++++-
include/linux/nfs4.h | 1 +
include/linux/nfs_xdr.h | 7 +++-
5 files changed, 141 insertions(+), 4 deletions(-)
diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h
index 7a74740..253156e 100644
--- a/fs/nfs/nfs4_fs.h
+++ b/fs/nfs/nfs4_fs.h
@@ -240,6 +240,8 @@ extern int nfs4_do_close(struct path *path, struct nfs4_state *state, gfp_t gfp_
extern int nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *fhandle);
extern int nfs4_proc_fs_locations(struct inode *dir, const struct qstr *name,
struct nfs4_fs_locations *fs_locations, struct page *page);
+extern int nfs4_proc_get_mig_status(struct nfs_server *server,
+ struct nfs4_fs_locations *fs_locations, struct page *page);
extern void nfs4_release_lockowner(const struct nfs4_lock_state *);
extern const struct xattr_handler *nfs4_xattr_handlers[];
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 491b068..403d257 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -2202,7 +2202,8 @@ static int nfs4_get_referral(struct inode *dir, const struct qstr *name, struct
}
memcpy(fattr, &locations->fattr, sizeof(struct nfs_fattr));
- fattr->valid |= NFS_ATTR_FATTR_V4_REFERRAL;
+ if (fattr->valid & NFS_ATTR_FATTR_V4_LOCATIONS)
+ fattr->valid |= NFS_ATTR_FATTR_V4_REFERRAL;
if (!fattr->mode)
fattr->mode = S_IFDIR;
memset(fhandle, 0, sizeof(struct nfs_fh));
@@ -4517,7 +4518,7 @@ static void nfs_fixup_referral_attributes(struct nfs_fattr *fattr)
{
if (!((fattr->valid & NFS_ATTR_FATTR_FILEID) &&
(fattr->valid & NFS_ATTR_FATTR_FSID) &&
- (fattr->valid & NFS_ATTR_FATTR_V4_REFERRAL)))
+ (fattr->valid & NFS_ATTR_FATTR_V4_LOCATIONS)))
return;
fattr->valid |= NFS_ATTR_FATTR_TYPE | NFS_ATTR_FATTR_MODE |
@@ -4560,6 +4561,55 @@ int nfs4_proc_fs_locations(struct inode *dir, const struct qstr *name,
return status;
}
+/**
+ * nfs4_proc_get_mig_status - probe migration status of an export
+ *
+ * @server: local state for server
+ * @locations: result of query
+ * @page: buffer
+ *
+ * Returns zero on success, or a negative errno code
+ */
+int nfs4_proc_get_mig_status(struct nfs_server *server,
+ struct nfs4_fs_locations *locations,
+ struct page *page)
+{
+ u32 bitmask[2] = {
+ [0] = FATTR4_WORD0_FSID |
+ FATTR4_WORD0_FS_LOCATIONS,
+ };
+ struct nfs4_fs_locations_arg args = {
+ .client = server->nfs_client,
+ .fh = server->rootfh,
+ .page = page,
+ .bitmask = bitmask,
+ };
+ struct nfs4_fs_locations_res res = {
+ .fs_locations = locations,
+ };
+ struct rpc_message msg = {
+ .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_GET_MIG_STATUS],
+ .rpc_argp = &args,
+ .rpc_resp = &res,
+ };
+ int status;
+
+ dprintk("--> %s: FSID %llx:%llx on \"%s\"\n", __func__,
+ (unsigned long long)server->fsid.major,
+ (unsigned long long)server->fsid.minor,
+ args.client->cl_hostname);
+ nfs_display_fhandle(args.fh);
+
+ if (args.client->cl_mvops->minor_version == 0)
+ args.renew = res.renew = 1;
+ nfs_fattr_init(&locations->fattr);
+ locations->server = server;
+ locations->nlocations = 0;
+ status = nfs4_call_sync(server, &msg, &args, &res, 0);
+ dprintk("<-- %s status=%d\n", __func__, status);
+ return status;
+}
+
#ifdef CONFIG_NFS_V4_1
/*
* Check the exchange flags returned by the server for invalid flags, having
diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c
index 4e2c168..a39e17f 100644
--- a/fs/nfs/nfs4xdr.c
+++ b/fs/nfs/nfs4xdr.c
@@ -676,6 +676,18 @@ static int nfs4_stat_to_errno(int);
decode_putfh_maxsz + \
decode_lookup_maxsz + \
decode_fs_locations_maxsz)
+#define NFS4_enc_get_mig_status_sz \
+ (compound_encode_hdr_maxsz + \
+ encode_sequence_maxsz + \
+ encode_putfh_maxsz + \
+ encode_fs_locations_maxsz + \
+ encode_renew_maxsz)
+#define NFS4_dec_get_mig_status_sz \
+ (compound_decode_hdr_maxsz + \
+ decode_sequence_maxsz + \
+ decode_putfh_maxsz + \
+ decode_fs_locations_maxsz + \
+ encode_renew_maxsz)
#if defined(CONFIG_NFS_V4_1)
#define NFS4_enc_exchange_id_sz \
(compound_encode_hdr_maxsz + \
@@ -2460,6 +2472,41 @@ static void nfs4_xdr_enc_fs_locations(struct rpc_rqst *req,
encode_nops(&hdr);
}
+/*
+ * Encode migration status probe
+ */
+static int nfs4_xdr_enc_get_mig_status(struct rpc_rqst *req,
+ struct xdr_stream *xdr,
+ struct nfs4_fs_locations_arg *args)
+{
+ struct compound_hdr hdr = {
+ .minorversion = nfs4_xdr_minorversion(&args->seq_args),
+ };
+ uint32_t replen;
+
+ encode_compound_hdr(xdr, req, &hdr);
+ encode_sequence(xdr, &args->seq_args, &hdr);
+ encode_putfh(xdr, args->fh, &hdr);
+ replen = hdr.replen; /* get the attribute into args->page */
+ encode_fs_locations(xdr, args->bitmask, &hdr);
+
+ /* Minor version zero servers need to have a client ID
+ * associated with this GETATTR(fs_locations) operation,
+ * to clear the "keep returning LEASE_MOVED" flag for
+ * this client ID. By convention, a RENEW operation is
+ * appended to the GETATTR(fs_locations) compound, as
+ * RENEW takes a client ID argument. Later minor
+ * versions get the client ID from the session. */
+ if (args->renew)
+ encode_renew(xdr, args->client, &hdr);
+
+ /* Set up reply kvec to capture returned fs_locations array. */
+ xdr_inline_pages(&req->rq_rcv_buf, replen << 2, &args->page,
+ 0, PAGE_SIZE);
+ encode_nops(&hdr);
+ return 0;
+}
+
#if defined(CONFIG_NFS_V4_1)
/*
* EXCHANGE_ID request
@@ -3204,7 +3251,7 @@ static int decode_attr_fs_locations(struct xdr_stream *xdr, uint32_t *bitmap, st
res->nlocations++;
}
if (res->nlocations != 0)
- status = NFS_ATTR_FATTR_V4_REFERRAL;
+ status = NFS_ATTR_FATTR_V4_LOCATIONS;
out:
dprintk("%s: fs_locations done, error = %d\n", __func__, status);
return status;
@@ -5915,6 +5962,37 @@ out:
return status;
}
+/*
+ * Decode migration status probe
+ */
+static int nfs4_xdr_dec_get_mig_status(struct rpc_rqst *req,
+ struct xdr_stream *xdr,
+ struct nfs4_fs_locations_res *res)
+{
+ struct compound_hdr hdr;
+ int status;
+
+ status = decode_compound_hdr(xdr, &hdr);
+ if (status)
+ goto out;
+ status = decode_sequence(xdr, &res->seq_res, req);
+ if (status)
+ goto out;
+ status = decode_putfh(xdr);
+ if (status)
+ goto out;
+ xdr_enter_page(xdr, PAGE_SIZE);
+ status = decode_getfattr(xdr, &res->fs_locations->fattr,
+ res->fs_locations->server,
+ !RPC_IS_ASYNC(req->rq_task));
+ if (status)
+ goto out;
+ if (res->renew)
+ status = decode_renew(xdr);
+out:
+ return status;
+}
+
#if defined(CONFIG_NFS_V4_1)
/*
* Decode EXCHANGE_ID response
@@ -6255,6 +6333,7 @@ struct rpc_procinfo nfs4_procedures[] = {
PROC(GETACL, enc_getacl, dec_getacl),
PROC(SETACL, enc_setacl, dec_setacl),
PROC(FS_LOCATIONS, enc_fs_locations, dec_fs_locations),
+ PROC(GET_MIG_STATUS, enc_get_mig_status, dec_get_mig_status),
PROC(RELEASE_LOCKOWNER, enc_release_lockowner, dec_release_lockowner),
#if defined(CONFIG_NFS_V4_1)
PROC(EXCHANGE_ID, enc_exchange_id, dec_exchange_id),
diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h
index 134716e..02a10df 100644
--- a/include/linux/nfs4.h
+++ b/include/linux/nfs4.h
@@ -549,6 +549,7 @@ enum {
NFSPROC4_CLNT_GETACL,
NFSPROC4_CLNT_SETACL,
NFSPROC4_CLNT_FS_LOCATIONS,
+ NFSPROC4_CLNT_GET_MIG_STATUS,
NFSPROC4_CLNT_RELEASE_LOCKOWNER,
/* nfs41 */
diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h
index b006857..632e869 100644
--- a/include/linux/nfs_xdr.h
+++ b/include/linux/nfs_xdr.h
@@ -77,7 +77,8 @@ struct nfs_fattr {
#define NFS_ATTR_FATTR_PRECTIME (1U << 16)
#define NFS_ATTR_FATTR_CHANGE (1U << 17)
#define NFS_ATTR_FATTR_PRECHANGE (1U << 18)
-#define NFS_ATTR_FATTR_V4_REFERRAL (1U << 19) /* NFSv4 referral */
+#define NFS_ATTR_FATTR_V4_REFERRAL (1U << 19)
+#define NFS_ATTR_FATTR_V4_LOCATIONS (1U << 20)
#define NFS_ATTR_FATTR (NFS_ATTR_FATTR_TYPE \
| NFS_ATTR_FATTR_MODE \
@@ -924,14 +925,18 @@ struct nfs4_fs_locations {
};
struct nfs4_fs_locations_arg {
+ const struct nfs_client *client;
const struct nfs_fh *dir_fh;
+ const struct nfs_fh *fh;
const struct qstr *name;
struct page *page;
const u32 *bitmask;
+ int renew;
struct nfs4_sequence_args seq_args;
};
struct nfs4_fs_locations_res {
+ int renew;
struct nfs4_fs_locations *fs_locations;
struct nfs4_sequence_res seq_res;
};
^ permalink raw reply related [flat|nested] 15+ messages in thread* Re: [PATCH 3/8] NFS: Introduce nfs4_proc_get_mig_status()
2011-03-10 17:29 ` [PATCH 3/8] NFS: Introduce nfs4_proc_get_mig_status() Chuck Lever
@ 2011-03-10 19:58 ` Trond Myklebust
2011-03-10 20:14 ` Chuck Lever
0 siblings, 1 reply; 15+ messages in thread
From: Trond Myklebust @ 2011-03-10 19:58 UTC (permalink / raw)
To: Chuck Lever; +Cc: linux-nfs
On Thu, 2011-03-10 at 12:29 -0500, Chuck Lever wrote:
> @@ -5915,6 +5962,37 @@ out:
> return status;
> }
>
> +/*
> + * Decode migration status probe
> + */
> +static int nfs4_xdr_dec_get_mig_status(struct rpc_rqst *req,
> + struct xdr_stream *xdr,
> + struct nfs4_fs_locations_res *res)
> +{
> + struct compound_hdr hdr;
> + int status;
> +
> + status = decode_compound_hdr(xdr, &hdr);
> + if (status)
> + goto out;
> + status = decode_sequence(xdr, &res->seq_res, req);
> + if (status)
> + goto out;
> + status = decode_putfh(xdr);
> + if (status)
> + goto out;
> + xdr_enter_page(xdr, PAGE_SIZE);
> + status = decode_getfattr(xdr, &res->fs_locations->fattr,
> + res->fs_locations->server,
> + !RPC_IS_ASYNC(req->rq_task));
> + if (status)
> + goto out;
> + if (res->renew)
> + status = decode_renew(xdr);
> +out:
> + return status;
> +}
> +
> #if defined(CONFIG_NFS_V4_1)
> /*
> * Decode EXCHANGE_ID response
> @@ -6255,6 +6333,7 @@ struct rpc_procinfo nfs4_procedures[] = {
> PROC(GETACL, enc_getacl, dec_getacl),
> PROC(SETACL, enc_setacl, dec_setacl),
> PROC(FS_LOCATIONS, enc_fs_locations, dec_fs_locations),
> + PROC(GET_MIG_STATUS, enc_get_mig_status, dec_get_mig_status)
Yet more trouble for nfsstat...
> ,
> PROC(RELEASE_LOCKOWNER, enc_release_lockowner, dec_release_lockowner),
> #if defined(CONFIG_NFS_V4_1)
> PROC(EXCHANGE_ID, enc_exchange_id, dec_exchange_id),
So looking at what you are doing in GET_MIG_STATUS, it appears to me
that
1) We don't need it for NFSv4.1: the existing fs_locations request just
works for that.
2) The only new thing in NFSv4.0 is the renew operation...
So how about if we just modify the existing fs_locations call to append
a RENEW request if/when we supply a clientid?
Cheers
Trond
--
Trond Myklebust
Linux NFS client maintainer
NetApp
Trond.Myklebust@netapp.com
www.netapp.com
^ permalink raw reply [flat|nested] 15+ messages in thread* Re: [PATCH 3/8] NFS: Introduce nfs4_proc_get_mig_status()
2011-03-10 19:58 ` Trond Myklebust
@ 2011-03-10 20:14 ` Chuck Lever
2011-03-10 20:21 ` Chuck Lever
0 siblings, 1 reply; 15+ messages in thread
From: Chuck Lever @ 2011-03-10 20:14 UTC (permalink / raw)
To: Trond Myklebust; +Cc: linux-nfs
On Mar 10, 2011, at 2:58 PM, Trond Myklebust wrote:
> On Thu, 2011-03-10 at 12:29 -0500, Chuck Lever wrote:
>> @@ -5915,6 +5962,37 @@ out:
>> return status;
>> }
>>
>> +/*
>> + * Decode migration status probe
>> + */
>> +static int nfs4_xdr_dec_get_mig_status(struct rpc_rqst *req,
>> + struct xdr_stream *xdr,
>> + struct nfs4_fs_locations_res *res)
>> +{
>> + struct compound_hdr hdr;
>> + int status;
>> +
>> + status = decode_compound_hdr(xdr, &hdr);
>> + if (status)
>> + goto out;
>> + status = decode_sequence(xdr, &res->seq_res, req);
>> + if (status)
>> + goto out;
>> + status = decode_putfh(xdr);
>> + if (status)
>> + goto out;
>> + xdr_enter_page(xdr, PAGE_SIZE);
>> + status = decode_getfattr(xdr, &res->fs_locations->fattr,
>> + res->fs_locations->server,
>> + !RPC_IS_ASYNC(req->rq_task));
>> + if (status)
>> + goto out;
>> + if (res->renew)
>> + status = decode_renew(xdr);
>> +out:
>> + return status;
>> +}
>> +
>> #if defined(CONFIG_NFS_V4_1)
>> /*
>> * Decode EXCHANGE_ID response
>> @@ -6255,6 +6333,7 @@ struct rpc_procinfo nfs4_procedures[] = {
>> PROC(GETACL, enc_getacl, dec_getacl),
>> PROC(SETACL, enc_setacl, dec_setacl),
>> PROC(FS_LOCATIONS, enc_fs_locations, dec_fs_locations),
>> + PROC(GET_MIG_STATUS, enc_get_mig_status, dec_get_mig_status)
>
> Yet more trouble for nfsstat...
Indeed.
>> ,
>> PROC(RELEASE_LOCKOWNER, enc_release_lockowner, dec_release_lockowner),
>> #if defined(CONFIG_NFS_V4_1)
>> PROC(EXCHANGE_ID, enc_exchange_id, dec_exchange_id),
>
> So looking at what you are doing in GET_MIG_STATUS, it appears to me
> that
>
> 1) We don't need it for NFSv4.1: the existing fs_locations request just
> works for that.
> 2) The only new thing in NFSv4.0 is the renew operation...
It's true that NFSv4.1 doesn't require a RENEW, but we do still need a new op for migration support. The original op uses PUTFH;LOOKUP;GETATTR(fs_locations). We can't use a LOOKUP request to retrieve the fs_locations array for the root directory of an FSID. So, the new op is substantively different than the original in more ways than the addition of the RENEW.
> So how about if we just modify the existing fs_locations call to append
> a RENEW request if/when we supply a clientid?
I'm willing to think more about this, but I don't see an easy way to reuse the existing fs_locations proc.
--
Chuck Lever
chuck[dot]lever[at]oracle[dot]com
^ permalink raw reply [flat|nested] 15+ messages in thread* Re: [PATCH 3/8] NFS: Introduce nfs4_proc_get_mig_status()
2011-03-10 20:14 ` Chuck Lever
@ 2011-03-10 20:21 ` Chuck Lever
0 siblings, 0 replies; 15+ messages in thread
From: Chuck Lever @ 2011-03-10 20:21 UTC (permalink / raw)
To: Trond Myklebust; +Cc: linux-nfs
On Mar 10, 2011, at 3:14 PM, Chuck Lever wrote:
>
> On Mar 10, 2011, at 2:58 PM, Trond Myklebust wrote:
>
>> On Thu, 2011-03-10 at 12:29 -0500, Chuck Lever wrote:
>>> @@ -5915,6 +5962,37 @@ out:
>>> return status;
>>> }
>>>
>>> +/*
>>> + * Decode migration status probe
>>> + */
>>> +static int nfs4_xdr_dec_get_mig_status(struct rpc_rqst *req,
>>> + struct xdr_stream *xdr,
>>> + struct nfs4_fs_locations_res *res)
>>> +{
>>> + struct compound_hdr hdr;
>>> + int status;
>>> +
>>> + status = decode_compound_hdr(xdr, &hdr);
>>> + if (status)
>>> + goto out;
>>> + status = decode_sequence(xdr, &res->seq_res, req);
>>> + if (status)
>>> + goto out;
>>> + status = decode_putfh(xdr);
>>> + if (status)
>>> + goto out;
>>> + xdr_enter_page(xdr, PAGE_SIZE);
>>> + status = decode_getfattr(xdr, &res->fs_locations->fattr,
>>> + res->fs_locations->server,
>>> + !RPC_IS_ASYNC(req->rq_task));
>>> + if (status)
>>> + goto out;
>>> + if (res->renew)
>>> + status = decode_renew(xdr);
>>> +out:
>>> + return status;
>>> +}
>>> +
>>> #if defined(CONFIG_NFS_V4_1)
>>> /*
>>> * Decode EXCHANGE_ID response
>>> @@ -6255,6 +6333,7 @@ struct rpc_procinfo nfs4_procedures[] = {
>>> PROC(GETACL, enc_getacl, dec_getacl),
>>> PROC(SETACL, enc_setacl, dec_setacl),
>>> PROC(FS_LOCATIONS, enc_fs_locations, dec_fs_locations),
>>> + PROC(GET_MIG_STATUS, enc_get_mig_status, dec_get_mig_status)
>>
>> Yet more trouble for nfsstat...
>
> Indeed.
>
>>> ,
>>> PROC(RELEASE_LOCKOWNER, enc_release_lockowner, dec_release_lockowner),
>>> #if defined(CONFIG_NFS_V4_1)
>>> PROC(EXCHANGE_ID, enc_exchange_id, dec_exchange_id),
>>
>> So looking at what you are doing in GET_MIG_STATUS, it appears to me
>> that
>>
>> 1) We don't need it for NFSv4.1: the existing fs_locations request just
>> works for that.
>
>> 2) The only new thing in NFSv4.0 is the renew operation...
>
> It's true that NFSv4.1 doesn't require a RENEW, but we do still need a new op for migration support. The original op uses PUTFH;LOOKUP;GETATTR(fs_locations). We can't use a LOOKUP request to retrieve the fs_locations array for the root directory of an FSID. So, the new op is substantively different than the original in more ways than the addition of the RENEW.
>
>> So how about if we just modify the existing fs_locations call to append
>> a RENEW request if/when we supply a clientid?
>
> I'm willing to think more about this, but I don't see an easy way to reuse the existing fs_locations proc.
On second thought... I can probably switch the behavior of the XDR codecs based on the presence of a dir_fh. Maybe we can use a single proc here.
--
Chuck Lever
chuck[dot]lever[at]oracle[dot]com
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH 4/8] NFS: Add functions to swap transports during migration recovery
2011-03-10 17:29 [PATCH 0/8] Snapshot of client-side NFSv4 migration support Chuck Lever
` (2 preceding siblings ...)
2011-03-10 17:29 ` [PATCH 3/8] NFS: Introduce nfs4_proc_get_mig_status() Chuck Lever
@ 2011-03-10 17:30 ` Chuck Lever
2011-03-10 17:30 ` [PATCH 5/8] NFS: Add basic migration support to state manager thread Chuck Lever
` (3 subsequent siblings)
7 siblings, 0 replies; 15+ messages in thread
From: Chuck Lever @ 2011-03-10 17:30 UTC (permalink / raw)
To: linux-nfs
Introduce functions that can walk through an array of returned
fs_locations information and connect a transport to one of the
destination servers listed therein.
Note that minor version 1 introduces "fs_locations_info" which
extends the sorting information available to clients. This is
not supported yet.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
fs/nfs/internal.h | 2 +
fs/nfs/nfs4namespace.c | 162 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 164 insertions(+), 0 deletions(-)
diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index 58a9ad7..bf2a4e7 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -164,6 +164,8 @@ static inline void nfs_fs_proc_exit(void)
/* nfs4namespace.c */
#ifdef CONFIG_NFS_V4
extern struct vfsmount *nfs_do_refmount(const struct vfsmount *mnt_parent, struct dentry *dentry);
+extern int nfs4_replace_transport(struct nfs_server *server,
+ const struct nfs4_fs_locations *locations);
#else
static inline
struct vfsmount *nfs_do_refmount(const struct vfsmount *mnt_parent, struct dentry *dentry)
diff --git a/fs/nfs/nfs4namespace.c b/fs/nfs/nfs4namespace.c
index 3c2a172..ea591ff 100644
--- a/fs/nfs/nfs4namespace.c
+++ b/fs/nfs/nfs4namespace.c
@@ -270,3 +270,165 @@ out:
dprintk("%s: done\n", __func__);
return mnt;
}
+
+#undef NFSDBG_FACILITY
+#define NFSDBG_FACILITY NFSDBG_CLIENT
+
+/*
+ * Returns zero on success, or a negative errno value.
+ */
+static int nfs4_update_server(struct nfs_server *server, const char *hostname,
+ struct sockaddr *sap, size_t salen)
+{
+ struct nfs_client *clp = server->nfs_client;
+ struct rpc_clnt *clnt = server->client;
+ struct xprt_create xargs = {
+ .ident = clp->cl_proto,
+ .net = &init_net,
+ .dstaddr = sap,
+ .addrlen = salen,
+ .servername = hostname,
+ };
+ int error;
+
+ dprintk("--> %s: move FSID %llx:%llx to \"%s\")\n", __func__,
+ (unsigned long long)server->fsid.major,
+ (unsigned long long)server->fsid.minor,
+ hostname);
+
+ /*
+ * rpc_lock_client() deadlocks here. This is because the tasks
+ * that received NFS4ERR_MOVED are waiting for us to wake them
+ * when we are done recovering. But they have bumped active_tasks,
+ * so rpc_lock_client() can't make any progress.
+ */
+#if 0
+ error = rpc_lock_client(clnt, clnt->cl_timeout->to_maxval);
+ if (error != 0)
+ goto out;
+#endif
+
+ error = rpc_switch_client_transport(clnt, &xargs, clnt->cl_timeout);
+
+out:
+#if 0
+ rpc_unlock_client(clnt);
+#endif
+ dprintk("<-- %s() = %d\n", __func__, error);
+ return error;
+}
+
+/*
+ * Try one location from the fs_locations array.
+ *
+ * Returns zero on success, or a negative errno value.
+ */
+static int nfs4_try_replacing_one_location(struct nfs_server *server,
+ char *page, char *page2,
+ const struct nfs4_fs_location *location)
+{
+ const size_t addr_bufsize = sizeof(struct sockaddr_storage);
+ struct sockaddr *sap;
+ unsigned int s;
+ size_t salen;
+ int error;
+
+ dprintk("--> %s(%llx:%llx)\n", __func__,
+ (unsigned long long)server->fsid.major,
+ (unsigned long long)server->fsid.minor);
+
+ error = -ENOMEM;
+ sap = kmalloc(addr_bufsize, GFP_KERNEL);
+ if (sap == NULL)
+ goto out;
+
+ error = -ENOENT;
+ for (s = 0; s < location->nservers; s++) {
+ const struct nfs4_string *buf = &location->servers[s];
+ char *hostname;
+
+ if (buf->len <= 0 || buf->len > PAGE_SIZE)
+ continue;
+
+ /* XXX: IPv6 not supported? */
+ if (memchr(buf->data, IPV6_SCOPE_DELIMITER, buf->len) != NULL)
+ continue;
+
+ salen = nfs_parse_server_name(buf->data, buf->len,
+ sap, addr_bufsize);
+ if (salen == 0)
+ continue;
+ rpc_set_port(sap, NFS_PORT);
+
+ error = -ENOMEM;
+ hostname = kstrndup(buf->data, buf->len, GFP_KERNEL);
+ if (hostname == NULL)
+ break;
+
+ error = nfs4_update_server(server, hostname, sap, salen);
+ kfree(hostname);
+ if (error == 0)
+ break;
+ }
+
+ kfree(sap);
+out:
+ dprintk("<-- %s() = %d\n", __func__, error);
+ return error;
+}
+
+/**
+ * nfs4_replace_transport - set up transport to destination server
+ *
+ * @server: export being migrated
+ * @locations: fs_locations array
+ *
+ * Returns zero on success, or a negative errno value.
+ *
+ * The client tries all the entries in the "locations" array, in the
+ * order returned by the server, until one works or the end of the
+ * array is reached.
+ */
+int nfs4_replace_transport(struct nfs_server *server,
+ const struct nfs4_fs_locations *locations)
+{
+ char *page = NULL, *page2 = NULL;
+ int loc, error;
+
+ dprintk("--> %s(%llx:%llx)\n", __func__,
+ (unsigned long long)server->fsid.major,
+ (unsigned long long)server->fsid.minor);
+
+ error = -ENOENT;
+ if (locations == NULL || locations->nlocations <= 0)
+ goto out;
+
+ error = -ENOMEM;
+ page = (char *) __get_free_page(GFP_USER);
+ if (!page)
+ goto out;
+ page2 = (char *) __get_free_page(GFP_USER);
+ if (!page2)
+ goto out;
+
+ for (loc = 0; loc < locations->nlocations; loc++) {
+ const struct nfs4_fs_location *location =
+ &locations->locations[loc];
+
+ if (location == NULL || location->nservers <= 0 ||
+ location->rootpath.ncomponents == 0)
+ continue;
+
+ error = nfs4_try_replacing_one_location(server, page,
+ page2, location);
+ if (error == 0)
+ break;
+ }
+
+out:
+ free_page((unsigned long)page);
+ free_page((unsigned long)page2);
+
+ dprintk("<-- %s() = %d\n", __func__, error);
+ return error;
+}
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH 5/8] NFS: Add basic migration support to state manager thread
2011-03-10 17:29 [PATCH 0/8] Snapshot of client-side NFSv4 migration support Chuck Lever
` (3 preceding siblings ...)
2011-03-10 17:30 ` [PATCH 4/8] NFS: Add functions to swap transports during migration recovery Chuck Lever
@ 2011-03-10 17:30 ` Chuck Lever
2011-03-10 20:35 ` Trond Myklebust
2011-03-10 17:30 ` [PATCH 6/8] NFS: Remove "const" from "struct nfs_server *" fields Chuck Lever
` (2 subsequent siblings)
7 siblings, 1 reply; 15+ messages in thread
From: Chuck Lever @ 2011-03-10 17:30 UTC (permalink / raw)
To: linux-nfs
Migration recovery will be handled separately from the normal
synchronous and asynchronous NFS processes, much like the existing
state manager thread. In fact state and migration recovery will
have to be serialized.
The best approach, then, is to add migration recovery support to
the existing state manager infrastructure, reusing its rendevous
mechanism and finite state machine.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
fs/nfs/nfs4_fs.h | 2 +
fs/nfs/nfs4state.c | 94 ++++++++++++++++++++++++++++++++++++++++++---
include/linux/nfs_fs_sb.h | 3 +
3 files changed, 93 insertions(+), 6 deletions(-)
diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h
index 253156e..d8724bf 100644
--- a/fs/nfs/nfs4_fs.h
+++ b/fs/nfs/nfs4_fs.h
@@ -47,6 +47,7 @@ enum nfs4_client_state {
NFS4CLNT_LAYOUTRECALL,
NFS4CLNT_SESSION_RESET,
NFS4CLNT_RECALL_SLOT,
+ NFS4CLNT_MOVED,
};
enum nfs4_session_state {
@@ -310,6 +311,7 @@ extern void nfs4_close_state(struct path *, struct nfs4_state *, fmode_t);
extern void nfs4_close_sync(struct path *, struct nfs4_state *, fmode_t);
extern void nfs4_state_set_mode_locked(struct nfs4_state *, fmode_t);
extern void nfs4_schedule_state_recovery(struct nfs_client *);
+extern void nfs4_schedule_migration_recovery(struct nfs_server *);
extern void nfs4_schedule_state_manager(struct nfs_client *);
extern int nfs4_state_mark_reclaim_nograce(struct nfs_client *clp, struct nfs4_state *state);
extern int nfs4_state_mark_reclaim_reboot(struct nfs_client *clp, struct nfs4_state *state);
diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
index e6742b5..a788ec9 100644
--- a/fs/nfs/nfs4state.c
+++ b/fs/nfs/nfs4state.c
@@ -56,6 +56,8 @@
#include "internal.h"
#include "pnfs.h"
+#define NFSDBG_FACILITY NFSDBG_CLIENT
+
#define OPENOWNER_POOL_SIZE 8
const nfs4_stateid zero_stateid;
@@ -1013,9 +1015,33 @@ void nfs4_schedule_state_recovery(struct nfs_client *clp)
{
if (!clp)
return;
+ dprintk("--> %s: \"%s\" (client ID %llx)\n",
+ __func__, clp->cl_hostname, clp->cl_clientid);
if (!test_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state))
set_bit(NFS4CLNT_CHECK_LEASE, &clp->cl_state);
nfs4_schedule_state_manager(clp);
+ dprintk("<-- %s\n", __func__);
+}
+
+/**
+ * nfs4_schedule_migration_recovery - start background migration recovery
+ *
+ * @server: nfs_server representing remote file system that is migrating
+ *
+ */
+void nfs4_schedule_migration_recovery(struct nfs_server *server)
+{
+ struct nfs_client *clp = server->nfs_client;
+
+ dprintk("--> %s(%llx:%llx)\n", __func__,
+ (unsigned long long)server->fsid.major,
+ (unsigned long long)server->fsid.minor);
+ if (test_and_set_bit(NFS4CLNT_MOVED, &clp->cl_state) == 0) {
+ set_bit(NFS4CLNT_SESSION_RESET, &clp->cl_state);
+ clp->cl_moved_server = server;
+ nfs4_schedule_state_manager(clp);
+ }
+ dprintk("<-- %s\n", __func__);
}
int nfs4_state_mark_reclaim_reboot(struct nfs_client *clp, struct nfs4_state *state)
@@ -1435,6 +1461,64 @@ static int nfs4_reclaim_lease(struct nfs_client *clp)
return status;
}
+/*
+ * Try remote migration of one FSID from a source server to a
+ * destination server. The source server provides a list of
+ * potential destinations.
+ */
+static void nfs4_try_migration(struct nfs_server *server)
+{
+ struct nfs_client *clp = server->nfs_client;
+ struct nfs4_fs_locations *locations = NULL;
+ struct page *page;
+ int status;
+
+ dprintk("--> %s: FSID %llx:%llx on \"%s\"\n", __func__,
+ (unsigned long long)server->fsid.major,
+ (unsigned long long)server->fsid.minor,
+ clp->cl_hostname);
+
+ status = -ENOMEM;
+ page = alloc_page(GFP_KERNEL);
+ locations = kmalloc(sizeof(struct nfs4_fs_locations), GFP_KERNEL);
+ if (page == NULL || locations == NULL) {
+ dprintk("<-- %s: no memory\n", __func__);
+ goto out;
+ }
+
+ status = nfs4_proc_get_mig_status(server, locations, page);
+ if (status != 0) {
+ dprintk("<-- %s: get migration status: %d\n",
+ __func__, status);
+ goto out;
+ }
+ if (!(locations->fattr.valid & NFS_ATTR_FATTR_V4_LOCATIONS)) {
+ dprintk("<-- %s: No fs_locations data available, "
+ "migration skipped\n", __func__);
+ goto out;
+ }
+
+ status = nfs4_replace_transport(server, locations);
+ if (status != 0) {
+ dprintk("<-- %s: failed to replace transport: %d\n",
+ __func__, status);
+ goto out;
+ }
+
+ /* Force an update of our callback info on the destination server */
+ set_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state);
+
+ /* XXX: would like to update /proc/mounts, but that's entirely
+ * optional. */
+
+ dprintk("<-- %s: migration succeeded\n", __func__);
+
+out:
+ if (page != NULL)
+ __free_page(page);
+ kfree(locations);
+}
+
#ifdef CONFIG_NFS_V4_1
void nfs41_handle_recall_slot(struct nfs_client *clp)
{
@@ -1664,6 +1748,10 @@ static void nfs4_state_manager(struct nfs_client *clp)
continue;
}
+ if (test_and_clear_bit(NFS4CLNT_MOVED, &clp->cl_state)) {
+ nfs4_try_migration(clp->cl_moved_server);
+ continue;
+ }
nfs4_clear_state_manager_bit(clp);
/* Did we race with an attempt to give us more work? */
@@ -1690,9 +1778,3 @@ static int nfs4_run_state_manager(void *ptr)
module_put_and_exit(0);
return 0;
}
-
-/*
- * Local variables:
- * c-basic-offset: 8
- * End:
- */
diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
index 961ee85..3741928 100644
--- a/include/linux/nfs_fs_sb.h
+++ b/include/linux/nfs_fs_sb.h
@@ -55,6 +55,9 @@ struct nfs_client {
struct rpc_wait_queue cl_rpcwaitq;
+ /* accessed only when NFS4CLNT_MOVED bit is set */
+ struct nfs_server * cl_moved_server;
+
/* used for the setclientid verifier */
struct timespec cl_boot_time;
^ permalink raw reply related [flat|nested] 15+ messages in thread* Re: [PATCH 5/8] NFS: Add basic migration support to state manager thread
2011-03-10 17:30 ` [PATCH 5/8] NFS: Add basic migration support to state manager thread Chuck Lever
@ 2011-03-10 20:35 ` Trond Myklebust
2011-03-10 20:40 ` Chuck Lever
2011-03-11 20:53 ` Chuck Lever
0 siblings, 2 replies; 15+ messages in thread
From: Trond Myklebust @ 2011-03-10 20:35 UTC (permalink / raw)
To: Chuck Lever; +Cc: linux-nfs
On Thu, 2011-03-10 at 12:30 -0500, Chuck Lever wrote:
> diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
> index e6742b5..a788ec9 100644
> --- a/fs/nfs/nfs4state.c
> +++ b/fs/nfs/nfs4state.c
> @@ -56,6 +56,8 @@
> #include "internal.h"
> #include "pnfs.h"
>
> +#define NFSDBG_FACILITY NFSDBG_CLIENT
> +
> #define OPENOWNER_POOL_SIZE 8
>
> const nfs4_stateid zero_stateid;
> @@ -1013,9 +1015,33 @@ void nfs4_schedule_state_recovery(struct nfs_client *clp)
> {
> if (!clp)
> return;
> + dprintk("--> %s: \"%s\" (client ID %llx)\n",
> + __func__, clp->cl_hostname, clp->cl_clientid);
> if (!test_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state))
> set_bit(NFS4CLNT_CHECK_LEASE, &clp->cl_state);
> nfs4_schedule_state_manager(clp);
> + dprintk("<-- %s\n", __func__);
> +}
> +
> +/**
> + * nfs4_schedule_migration_recovery - start background migration recovery
> + *
> + * @server: nfs_server representing remote file system that is migrating
> + *
> + */
> +void nfs4_schedule_migration_recovery(struct nfs_server *server)
> +{
> + struct nfs_client *clp = server->nfs_client;
> +
> + dprintk("--> %s(%llx:%llx)\n", __func__,
> + (unsigned long long)server->fsid.major,
> + (unsigned long long)server->fsid.minor);
> + if (test_and_set_bit(NFS4CLNT_MOVED, &clp->cl_state) == 0) {
> + set_bit(NFS4CLNT_SESSION_RESET, &clp->cl_state);
Hmm... Why do we need to reset the NFSv4.1 session here?
> + clp->cl_moved_server = server;
> + nfs4_schedule_state_manager(clp);
> + }
> + dprintk("<-- %s\n", __func__);
> }
>
> int nfs4_state_mark_reclaim_reboot(struct nfs_client *clp, struct nfs4_state *state)
> @@ -1435,6 +1461,64 @@ static int nfs4_reclaim_lease(struct nfs_client *clp)
> return status;
> }
>
> +/*
> + * Try remote migration of one FSID from a source server to a
> + * destination server. The source server provides a list of
> + * potential destinations.
> + */
> +static void nfs4_try_migration(struct nfs_server *server)
> +{
> + struct nfs_client *clp = server->nfs_client;
> + struct nfs4_fs_locations *locations = NULL;
> + struct page *page;
> + int status;
> +
> + dprintk("--> %s: FSID %llx:%llx on \"%s\"\n", __func__,
> + (unsigned long long)server->fsid.major,
> + (unsigned long long)server->fsid.minor,
> + clp->cl_hostname);
> +
> + status = -ENOMEM;
> + page = alloc_page(GFP_KERNEL);
> + locations = kmalloc(sizeof(struct nfs4_fs_locations), GFP_KERNEL);
> + if (page == NULL || locations == NULL) {
> + dprintk("<-- %s: no memory\n", __func__);
> + goto out;
> + }
> +
> + status = nfs4_proc_get_mig_status(server, locations, page);
> + if (status != 0) {
> + dprintk("<-- %s: get migration status: %d\n",
> + __func__, status);
> + goto out;
> + }
> + if (!(locations->fattr.valid & NFS_ATTR_FATTR_V4_LOCATIONS)) {
> + dprintk("<-- %s: No fs_locations data available, "
> + "migration skipped\n", __func__);
> + goto out;
> + }
> +
> + status = nfs4_replace_transport(server, locations);
> + if (status != 0) {
> + dprintk("<-- %s: failed to replace transport: %d\n",
> + __func__, status);
> + goto out;
> + }
> +
> + /* Force an update of our callback info on the destination server */
> + set_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state);
I wonder if this should be made into a separate flag. That would enable
us to reuse it for the NFS4ERR_CB_PATH_DOWN handling...
> +
> + /* XXX: would like to update /proc/mounts, but that's entirely
> + * optional. */
> +
> + dprintk("<-- %s: migration succeeded\n", __func__);
> +
> +out:
> + if (page != NULL)
> + __free_page(page);
> + kfree(locations);
> +}
> +
> #ifdef CONFIG_NFS_V4_1
> void nfs41_handle_recall_slot(struct nfs_client *clp)
> {
> @@ -1664,6 +1748,10 @@ static void nfs4_state_manager(struct nfs_client *clp)
--
Trond Myklebust
Linux NFS client maintainer
NetApp
Trond.Myklebust@netapp.com
www.netapp.com
^ permalink raw reply [flat|nested] 15+ messages in thread* Re: [PATCH 5/8] NFS: Add basic migration support to state manager thread
2011-03-10 20:35 ` Trond Myklebust
@ 2011-03-10 20:40 ` Chuck Lever
2011-03-11 20:53 ` Chuck Lever
1 sibling, 0 replies; 15+ messages in thread
From: Chuck Lever @ 2011-03-10 20:40 UTC (permalink / raw)
To: Trond Myklebust; +Cc: linux-nfs
On Mar 10, 2011, at 3:35 PM, Trond Myklebust wrote:
> On Thu, 2011-03-10 at 12:30 -0500, Chuck Lever wrote:
>
>> diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
>> index e6742b5..a788ec9 100644
>> --- a/fs/nfs/nfs4state.c
>> +++ b/fs/nfs/nfs4state.c
>> @@ -56,6 +56,8 @@
>> #include "internal.h"
>> #include "pnfs.h"
>>
>> +#define NFSDBG_FACILITY NFSDBG_CLIENT
>> +
>> #define OPENOWNER_POOL_SIZE 8
>>
>> const nfs4_stateid zero_stateid;
>> @@ -1013,9 +1015,33 @@ void nfs4_schedule_state_recovery(struct nfs_client *clp)
>> {
>> if (!clp)
>> return;
>> + dprintk("--> %s: \"%s\" (client ID %llx)\n",
>> + __func__, clp->cl_hostname, clp->cl_clientid);
>> if (!test_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state))
>> set_bit(NFS4CLNT_CHECK_LEASE, &clp->cl_state);
>> nfs4_schedule_state_manager(clp);
>> + dprintk("<-- %s\n", __func__);
>> +}
>> +
>> +/**
>> + * nfs4_schedule_migration_recovery - start background migration recovery
>> + *
>> + * @server: nfs_server representing remote file system that is migrating
>> + *
>> + */
>> +void nfs4_schedule_migration_recovery(struct nfs_server *server)
>> +{
>> + struct nfs_client *clp = server->nfs_client;
>> +
>> + dprintk("--> %s(%llx:%llx)\n", __func__,
>> + (unsigned long long)server->fsid.major,
>> + (unsigned long long)server->fsid.minor);
>> + if (test_and_set_bit(NFS4CLNT_MOVED, &clp->cl_state) == 0) {
>> + set_bit(NFS4CLNT_SESSION_RESET, &clp->cl_state);
>
> Hmm... Why do we need to reset the NFSv4.1 session here?
That's an assumption on my part. It's not really clear yet what kind of changes will be needed for minor version 1. I can pull this out for now.
>> + clp->cl_moved_server = server;
>> + nfs4_schedule_state_manager(clp);
>> + }
>> + dprintk("<-- %s\n", __func__);
>> }
>>
>> int nfs4_state_mark_reclaim_reboot(struct nfs_client *clp, struct nfs4_state *state)
>> @@ -1435,6 +1461,64 @@ static int nfs4_reclaim_lease(struct nfs_client *clp)
>> return status;
>> }
>>
>> +/*
>> + * Try remote migration of one FSID from a source server to a
>> + * destination server. The source server provides a list of
>> + * potential destinations.
>> + */
>> +static void nfs4_try_migration(struct nfs_server *server)
>> +{
>> + struct nfs_client *clp = server->nfs_client;
>> + struct nfs4_fs_locations *locations = NULL;
>> + struct page *page;
>> + int status;
>> +
>> + dprintk("--> %s: FSID %llx:%llx on \"%s\"\n", __func__,
>> + (unsigned long long)server->fsid.major,
>> + (unsigned long long)server->fsid.minor,
>> + clp->cl_hostname);
>> +
>> + status = -ENOMEM;
>> + page = alloc_page(GFP_KERNEL);
>> + locations = kmalloc(sizeof(struct nfs4_fs_locations), GFP_KERNEL);
>> + if (page == NULL || locations == NULL) {
>> + dprintk("<-- %s: no memory\n", __func__);
>> + goto out;
>> + }
>> +
>> + status = nfs4_proc_get_mig_status(server, locations, page);
>> + if (status != 0) {
>> + dprintk("<-- %s: get migration status: %d\n",
>> + __func__, status);
>> + goto out;
>> + }
>> + if (!(locations->fattr.valid & NFS_ATTR_FATTR_V4_LOCATIONS)) {
>> + dprintk("<-- %s: No fs_locations data available, "
>> + "migration skipped\n", __func__);
>> + goto out;
>> + }
>> +
>> + status = nfs4_replace_transport(server, locations);
>> + if (status != 0) {
>> + dprintk("<-- %s: failed to replace transport: %d\n",
>> + __func__, status);
>> + goto out;
>> + }
>> +
>> + /* Force an update of our callback info on the destination server */
>> + set_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state);
>
> I wonder if this should be made into a separate flag. That would enable
> us to reuse it for the NFS4ERR_CB_PATH_DOWN handling...
Instead, I think we will need a function call here to perform a fresh SETCLIENTID, rather than looping through the state manager's FSM again. For one thing, we need to construct a new long-form client ID for this nfs_server: the callback endpoint information will have changed.
I just wonder, though, if that is enough. Where will nfs4_init_clientid() send the SETCLIENTID request? To the source server or the destination server?
</speculation>
>
>> +
>> + /* XXX: would like to update /proc/mounts, but that's entirely
>> + * optional. */
>> +
>> + dprintk("<-- %s: migration succeeded\n", __func__);
>> +
>> +out:
>> + if (page != NULL)
>> + __free_page(page);
>> + kfree(locations);
>> +}
>> +
>> #ifdef CONFIG_NFS_V4_1
>> void nfs41_handle_recall_slot(struct nfs_client *clp)
>> {
>> @@ -1664,6 +1748,10 @@ static void nfs4_state_manager(struct nfs_client *clp)
>
> --
> Trond Myklebust
> Linux NFS client maintainer
>
> NetApp
> Trond.Myklebust@netapp.com
> www.netapp.com
>
--
Chuck Lever
chuck[dot]lever[at]oracle[dot]com
^ permalink raw reply [flat|nested] 15+ messages in thread* Re: [PATCH 5/8] NFS: Add basic migration support to state manager thread
2011-03-10 20:35 ` Trond Myklebust
2011-03-10 20:40 ` Chuck Lever
@ 2011-03-11 20:53 ` Chuck Lever
1 sibling, 0 replies; 15+ messages in thread
From: Chuck Lever @ 2011-03-11 20:53 UTC (permalink / raw)
To: Trond Myklebust; +Cc: linux-nfs
On Mar 10, 2011, at 3:35 PM, Trond Myklebust wrote:
> On Thu, 2011-03-10 at 12:30 -0500, Chuck Lever wrote:
>
>> diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
>> index e6742b5..a788ec9 100644
>> --- a/fs/nfs/nfs4state.c
>> +++ b/fs/nfs/nfs4state.c
>> @@ -56,6 +56,8 @@
>> #include "internal.h"
>> #include "pnfs.h"
>>
>> +#define NFSDBG_FACILITY NFSDBG_CLIENT
>> +
>> #define OPENOWNER_POOL_SIZE 8
>>
>> const nfs4_stateid zero_stateid;
>> @@ -1013,9 +1015,33 @@ void nfs4_schedule_state_recovery(struct nfs_client *clp)
>> {
>> if (!clp)
>> return;
>> + dprintk("--> %s: \"%s\" (client ID %llx)\n",
>> + __func__, clp->cl_hostname, clp->cl_clientid);
>> if (!test_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state))
>> set_bit(NFS4CLNT_CHECK_LEASE, &clp->cl_state);
>> nfs4_schedule_state_manager(clp);
>> + dprintk("<-- %s\n", __func__);
>> +}
>> +
>> +/**
>> + * nfs4_schedule_migration_recovery - start background migration recovery
>> + *
>> + * @server: nfs_server representing remote file system that is migrating
>> + *
>> + */
>> +void nfs4_schedule_migration_recovery(struct nfs_server *server)
>> +{
>> + struct nfs_client *clp = server->nfs_client;
>> +
>> + dprintk("--> %s(%llx:%llx)\n", __func__,
>> + (unsigned long long)server->fsid.major,
>> + (unsigned long long)server->fsid.minor);
>> + if (test_and_set_bit(NFS4CLNT_MOVED, &clp->cl_state) == 0) {
>> + set_bit(NFS4CLNT_SESSION_RESET, &clp->cl_state);
>
> Hmm... Why do we need to reset the NFSv4.1 session here?
>
>> + clp->cl_moved_server = server;
>> + nfs4_schedule_state_manager(clp);
>> + }
>> + dprintk("<-- %s\n", __func__);
>> }
>>
>> int nfs4_state_mark_reclaim_reboot(struct nfs_client *clp, struct nfs4_state *state)
>> @@ -1435,6 +1461,64 @@ static int nfs4_reclaim_lease(struct nfs_client *clp)
>> return status;
>> }
>>
>> +/*
>> + * Try remote migration of one FSID from a source server to a
>> + * destination server. The source server provides a list of
>> + * potential destinations.
>> + */
>> +static void nfs4_try_migration(struct nfs_server *server)
>> +{
>> + struct nfs_client *clp = server->nfs_client;
>> + struct nfs4_fs_locations *locations = NULL;
>> + struct page *page;
>> + int status;
>> +
>> + dprintk("--> %s: FSID %llx:%llx on \"%s\"\n", __func__,
>> + (unsigned long long)server->fsid.major,
>> + (unsigned long long)server->fsid.minor,
>> + clp->cl_hostname);
>> +
>> + status = -ENOMEM;
>> + page = alloc_page(GFP_KERNEL);
>> + locations = kmalloc(sizeof(struct nfs4_fs_locations), GFP_KERNEL);
>> + if (page == NULL || locations == NULL) {
>> + dprintk("<-- %s: no memory\n", __func__);
>> + goto out;
>> + }
>> +
>> + status = nfs4_proc_get_mig_status(server, locations, page);
>> + if (status != 0) {
>> + dprintk("<-- %s: get migration status: %d\n",
>> + __func__, status);
>> + goto out;
>> + }
>> + if (!(locations->fattr.valid & NFS_ATTR_FATTR_V4_LOCATIONS)) {
>> + dprintk("<-- %s: No fs_locations data available, "
>> + "migration skipped\n", __func__);
>> + goto out;
>> + }
>> +
>> + status = nfs4_replace_transport(server, locations);
>> + if (status != 0) {
>> + dprintk("<-- %s: failed to replace transport: %d\n",
>> + __func__, status);
>> + goto out;
>> + }
>> +
>> + /* Force an update of our callback info on the destination server */
>> + set_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state);
>
> I wonder if this should be made into a separate flag. That would enable
> us to reuse it for the NFS4ERR_CB_PATH_DOWN handling...
So... how exactly would this mechanism be different than nfs4[1]_init_clientid() ? I'm trying to get a handle on what this new flag will need to do.
>
>> +
>> + /* XXX: would like to update /proc/mounts, but that's entirely
>> + * optional. */
>> +
>> + dprintk("<-- %s: migration succeeded\n", __func__);
>> +
>> +out:
>> + if (page != NULL)
>> + __free_page(page);
>> + kfree(locations);
>> +}
>> +
>> #ifdef CONFIG_NFS_V4_1
>> void nfs41_handle_recall_slot(struct nfs_client *clp)
>> {
>> @@ -1664,6 +1748,10 @@ static void nfs4_state_manager(struct nfs_client *clp)
>
> --
> Trond Myklebust
> Linux NFS client maintainer
>
> NetApp
> Trond.Myklebust@netapp.com
> www.netapp.com
>
--
Chuck Lever
chuck[dot]lever[at]oracle[dot]com
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH 6/8] NFS: Remove "const" from "struct nfs_server *" fields
2011-03-10 17:29 [PATCH 0/8] Snapshot of client-side NFSv4 migration support Chuck Lever
` (4 preceding siblings ...)
2011-03-10 17:30 ` [PATCH 5/8] NFS: Add basic migration support to state manager thread Chuck Lever
@ 2011-03-10 17:30 ` Chuck Lever
2011-03-10 17:30 ` [PATCH 7/8] NFS: Add migration recovery callouts in nfs4proc.c Chuck Lever
2011-03-10 17:30 ` [PATCH 8/8] NFS: Implement support for NFS4ERR_LEASE_MOVED Chuck Lever
7 siblings, 0 replies; 15+ messages in thread
From: Chuck Lever @ 2011-03-10 17:30 UTC (permalink / raw)
To: linux-nfs
We're about to pass the nfs_server pointer in some NFSv4 argument and
result structures to functions that may change it, so it's no longer
"const".
The preference here is to maintain existing whitespace style rather
than answer all the nits called out by checkpatch.pl.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
include/linux/nfs_xdr.h | 28 ++++++++++++++--------------
1 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h
index 632e869..04b9ef7 100644
--- a/include/linux/nfs_xdr.h
+++ b/include/linux/nfs_xdr.h
@@ -256,7 +256,7 @@ struct nfs_openargs {
fmode_t delegation_type; /* CLAIM_PREVIOUS */
} u;
const struct qstr * name;
- const struct nfs_server *server; /* Needed for ID mapping */
+ struct nfs_server * server; /* Needed for ID mapping */
const u32 * bitmask;
__u32 claim;
struct nfs4_sequence_args seq_args;
@@ -270,7 +270,7 @@ struct nfs_openres {
struct nfs_fattr * f_attr;
struct nfs_fattr * dir_attr;
struct nfs_seqid * seqid;
- const struct nfs_server *server;
+ struct nfs_server * server;
fmode_t delegation_type;
nfs4_stateid delegation;
__u32 do_recall;
@@ -309,7 +309,7 @@ struct nfs_closeres {
nfs4_stateid stateid;
struct nfs_fattr * fattr;
struct nfs_seqid * seqid;
- const struct nfs_server *server;
+ struct nfs_server * server;
struct nfs4_sequence_res seq_res;
};
/*
@@ -381,7 +381,7 @@ struct nfs4_delegreturnargs {
struct nfs4_delegreturnres {
struct nfs_fattr * fattr;
- const struct nfs_server *server;
+ struct nfs_server * server;
struct nfs4_sequence_res seq_res;
};
@@ -431,7 +431,7 @@ struct nfs_writeres {
struct nfs_fattr * fattr;
struct nfs_writeverf * verf;
__u32 count;
- const struct nfs_server *server;
+ struct nfs_server * server;
struct nfs4_sequence_res seq_res;
};
@@ -446,7 +446,7 @@ struct nfs_removeargs {
};
struct nfs_removeres {
- const struct nfs_server *server;
+ struct nfs_server *server;
struct nfs_fattr *dir_attr;
struct nfs4_change_info cinfo;
struct nfs4_sequence_res seq_res;
@@ -465,7 +465,7 @@ struct nfs_renameargs {
};
struct nfs_renameres {
- const struct nfs_server *server;
+ struct nfs_server *server;
struct nfs4_change_info old_cinfo;
struct nfs_fattr *old_fattr;
struct nfs4_change_info new_cinfo;
@@ -546,7 +546,7 @@ struct nfs_getaclres {
struct nfs_setattrres {
struct nfs_fattr * fattr;
- const struct nfs_server * server;
+ struct nfs_server * server;
struct nfs4_sequence_res seq_res;
};
@@ -719,7 +719,7 @@ struct nfs4_accessargs {
};
struct nfs4_accessres {
- const struct nfs_server * server;
+ struct nfs_server * server;
struct nfs_fattr * fattr;
u32 supported;
u32 access;
@@ -747,7 +747,7 @@ struct nfs4_create_arg {
};
struct nfs4_create_res {
- const struct nfs_server * server;
+ struct nfs_server * server;
struct nfs_fh * fh;
struct nfs_fattr * fattr;
struct nfs4_change_info dir_cinfo;
@@ -773,7 +773,7 @@ struct nfs4_getattr_arg {
};
struct nfs4_getattr_res {
- const struct nfs_server * server;
+ struct nfs_server * server;
struct nfs_fattr * fattr;
struct nfs4_sequence_res seq_res;
};
@@ -787,7 +787,7 @@ struct nfs4_link_arg {
};
struct nfs4_link_res {
- const struct nfs_server * server;
+ struct nfs_server * server;
struct nfs_fattr * fattr;
struct nfs4_change_info cinfo;
struct nfs_fattr * dir_attr;
@@ -803,7 +803,7 @@ struct nfs4_lookup_arg {
};
struct nfs4_lookup_res {
- const struct nfs_server * server;
+ struct nfs_server * server;
struct nfs_fattr * fattr;
struct nfs_fh * fh;
struct nfs4_sequence_res seq_res;
@@ -918,7 +918,7 @@ struct nfs4_fs_location {
#define NFS4_FS_LOCATIONS_MAXENTRIES 10
struct nfs4_fs_locations {
struct nfs_fattr fattr;
- const struct nfs_server *server;
+ struct nfs_server *server;
struct nfs4_pathname fs_path;
int nlocations;
struct nfs4_fs_location locations[NFS4_FS_LOCATIONS_MAXENTRIES];
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH 7/8] NFS: Add migration recovery callouts in nfs4proc.c
2011-03-10 17:29 [PATCH 0/8] Snapshot of client-side NFSv4 migration support Chuck Lever
` (5 preceding siblings ...)
2011-03-10 17:30 ` [PATCH 6/8] NFS: Remove "const" from "struct nfs_server *" fields Chuck Lever
@ 2011-03-10 17:30 ` Chuck Lever
2011-03-10 17:30 ` [PATCH 8/8] NFS: Implement support for NFS4ERR_LEASE_MOVED Chuck Lever
7 siblings, 0 replies; 15+ messages in thread
From: Chuck Lever @ 2011-03-10 17:30 UTC (permalink / raw)
To: linux-nfs
Finally, to enable support for migration, insert migration recovery
callouts in the synchronous and asynchronous error handling paths for
NFSv4 procedures.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
fs/nfs/nfs4proc.c | 61 ++++++++++++++++++++++++++++++++++++++++-------------
1 files changed, 46 insertions(+), 15 deletions(-)
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 403d257..711b8ce 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -71,7 +71,8 @@ struct nfs4_opendata;
static int _nfs4_proc_open(struct nfs4_opendata *data);
static int _nfs4_recover_proc_open(struct nfs4_opendata *data);
static int nfs4_do_fsinfo(struct nfs_server *, struct nfs_fh *, struct nfs_fsinfo *);
-static int nfs4_async_handle_error(struct rpc_task *, const struct nfs_server *, struct nfs4_state *);
+static int nfs4_async_handle_error(struct rpc_task *, struct nfs_server *,
+ struct nfs4_state *);
static int _nfs4_proc_lookup(struct inode *dir, const struct qstr *name, struct nfs_fh *fhandle, struct nfs_fattr *fattr);
static int _nfs4_proc_getattr(struct nfs_server *server, struct nfs_fh *fhandle, struct nfs_fattr *fattr);
static int nfs4_do_setattr(struct inode *inode, struct rpc_cred *cred,
@@ -239,10 +240,17 @@ static int nfs4_delay(struct rpc_clnt *clnt, long *timeout)
return res;
}
-/* This is the error handling routine for processes that are allowed
- * to sleep.
+/**
+ * nfs4_handle_exception - Common error handling for callers allowed to sleep
+ *
+ * @server: local state context for the server
+ * @errorcode: NFS4ERR value returned from the server
+ * @exception: exception handling state
+ *
+ * Returns zero on success, or a negative errno value.
*/
-static int nfs4_handle_exception(const struct nfs_server *server, int errorcode, struct nfs4_exception *exception)
+static int nfs4_handle_exception(struct nfs_server *server, int errorcode,
+ struct nfs4_exception *exception)
{
struct nfs_client *clp = server->nfs_client;
struct nfs4_state *state = exception->state;
@@ -277,6 +285,9 @@ static int nfs4_handle_exception(const struct nfs_server *server, int errorcode,
exception->retry = 1;
break;
#endif /* defined(CONFIG_NFS_V4_1) */
+ case -NFS4ERR_MOVED:
+ nfs4_schedule_migration_recovery(server);
+ goto recovery_wait;
case -NFS4ERR_FILE_OPEN:
if (exception->timeout > HZ) {
/* We have retried a decent amount, time to
@@ -298,6 +309,7 @@ static int nfs4_handle_exception(const struct nfs_server *server, int errorcode,
return nfs4_map_errors(ret);
do_state_recovery:
nfs4_schedule_state_recovery(clp);
+recovery_wait:
ret = nfs4_wait_clnt_recover(clp);
if (ret == 0)
exception->retry = 1;
@@ -2194,10 +2206,14 @@ static int nfs4_get_referral(struct inode *dir, const struct qstr *name, struct
status = nfs4_proc_fs_locations(dir, name, locations, page);
if (status != 0)
goto out;
- /* Make sure server returned a different fsid for the referral */
+
+ /*
+ * If the fsid didn't change, this is a migration event, not a
+ * referral. Cause us to drop into the exception handler, which
+ * will kick off migration recovery.
+ */
if (nfs_fsid_equal(&NFS_SERVER(dir)->fsid, &locations->fattr.fsid)) {
- dprintk("%s: server did not return a different fsid for a referral at %s\n", __func__, name->name);
- status = -EIO;
+ status = -NFS4ERR_MOVED;
goto out;
}
@@ -3492,8 +3508,18 @@ static int nfs4_proc_set_acl(struct inode *inode, const void *buf, size_t buflen
return err;
}
-static int
-nfs4_async_handle_error(struct rpc_task *task, const struct nfs_server *server, struct nfs4_state *state)
+/**
+ * nfs4_async_handle_error - Common error handling for callers who cannot sleep
+ *
+ * @task: active RPC task
+ * @server: local state for the server
+ * @state: NFSv4 state for active operation
+ *
+ * Returns zero on success, or a negative errno value.
+ */
+static int nfs4_async_handle_error(struct rpc_task *task,
+ struct nfs_server *server,
+ struct nfs4_state *state)
{
struct nfs_client *clp = server->nfs_client;
@@ -3522,19 +3548,23 @@ nfs4_async_handle_error(struct rpc_task *task, const struct nfs_server *server,
dprintk("%s ERROR %d, Reset session\n", __func__,
task->tk_status);
nfs4_schedule_state_recovery(clp);
- task->tk_status = 0;
- return -EAGAIN;
+ goto restart_call;
#endif /* CONFIG_NFS_V4_1 */
+ case -NFS4ERR_MOVED:
+ rpc_sleep_on(&clp->cl_rpcwaitq, task, NULL);
+ nfs4_schedule_migration_recovery(server);
+ if (test_bit(NFS4CLNT_MANAGER_RUNNING,
+ &clp->cl_state) == 0)
+ rpc_wake_up_queued_task(&clp->cl_rpcwaitq, task);
+ goto restart_call;
case -NFS4ERR_DELAY:
nfs_inc_server_stats(server, NFSIOS_DELAY);
case -NFS4ERR_GRACE:
case -EKEYEXPIRED:
rpc_delay(task, NFS4_POLL_RETRY_MAX);
- task->tk_status = 0;
- return -EAGAIN;
+ goto restart_call;
case -NFS4ERR_OLD_STATEID:
- task->tk_status = 0;
- return -EAGAIN;
+ goto restart_call;
}
task->tk_status = nfs4_map_errors(task->tk_status);
return 0;
@@ -3543,6 +3573,7 @@ do_state_recovery:
nfs4_schedule_state_recovery(clp);
if (test_bit(NFS4CLNT_MANAGER_RUNNING, &clp->cl_state) == 0)
rpc_wake_up_queued_task(&clp->cl_rpcwaitq, task);
+restart_call:
task->tk_status = 0;
return -EAGAIN;
}
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH 8/8] NFS: Implement support for NFS4ERR_LEASE_MOVED
2011-03-10 17:29 [PATCH 0/8] Snapshot of client-side NFSv4 migration support Chuck Lever
` (6 preceding siblings ...)
2011-03-10 17:30 ` [PATCH 7/8] NFS: Add migration recovery callouts in nfs4proc.c Chuck Lever
@ 2011-03-10 17:30 ` Chuck Lever
7 siblings, 0 replies; 15+ messages in thread
From: Chuck Lever @ 2011-03-10 17:30 UTC (permalink / raw)
To: linux-nfs
To recover from NFS4ERR_LEASE_MOVED, walk the cl_superblocks list and
invoke nfs4_handle_migration() on each server's root file handle.
nfs4_handle_migration() should automatically determine whether that
file system has migrated, and then perform recovery for it.
The per-filesystem migration probe also informs minor version zero
servers that this client should no longer receive NFS4ERR_LEASE_MOVED.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
fs/nfs/client.c | 1 +
fs/nfs/nfs4_fs.h | 2 ++
fs/nfs/nfs4proc.c | 10 ++++++++
fs/nfs/nfs4state.c | 56 ++++++++++++++++++++++++++++++++++++++++++++-
include/linux/nfs_fs_sb.h | 2 ++
5 files changed, 70 insertions(+), 1 deletions(-)
diff --git a/fs/nfs/client.c b/fs/nfs/client.c
index a2f016a..f628526 100644
--- a/fs/nfs/client.c
+++ b/fs/nfs/client.c
@@ -179,6 +179,7 @@ static struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_
clp->cl_state = 1 << NFS4CLNT_LEASE_EXPIRED;
clp->cl_minorversion = cl_init->minorversion;
clp->cl_mvops = nfs_v4_minor_ops[cl_init->minorversion];
+ clp->cl_mig_counter = 1;
#endif
cred = rpc_lookup_machine_cred();
if (!IS_ERR(cred))
diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h
index d8724bf..dc976bd 100644
--- a/fs/nfs/nfs4_fs.h
+++ b/fs/nfs/nfs4_fs.h
@@ -48,6 +48,7 @@ enum nfs4_client_state {
NFS4CLNT_SESSION_RESET,
NFS4CLNT_RECALL_SLOT,
NFS4CLNT_MOVED,
+ NFS4CLNT_LEASE_MOVED,
};
enum nfs4_session_state {
@@ -312,6 +313,7 @@ extern void nfs4_close_sync(struct path *, struct nfs4_state *, fmode_t);
extern void nfs4_state_set_mode_locked(struct nfs4_state *, fmode_t);
extern void nfs4_schedule_state_recovery(struct nfs_client *);
extern void nfs4_schedule_migration_recovery(struct nfs_server *);
+extern void nfs4_schedule_lease_moved_recovery(struct nfs_client *);
extern void nfs4_schedule_state_manager(struct nfs_client *);
extern int nfs4_state_mark_reclaim_nograce(struct nfs_client *clp, struct nfs4_state *state);
extern int nfs4_state_mark_reclaim_reboot(struct nfs_client *clp, struct nfs4_state *state);
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 711b8ce..e139778 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -288,6 +288,9 @@ static int nfs4_handle_exception(struct nfs_server *server, int errorcode,
case -NFS4ERR_MOVED:
nfs4_schedule_migration_recovery(server);
goto recovery_wait;
+ case -NFS4ERR_LEASE_MOVED:
+ nfs4_schedule_lease_moved_recovery(clp);
+ goto recovery_wait;
case -NFS4ERR_FILE_OPEN:
if (exception->timeout > HZ) {
/* We have retried a decent amount, time to
@@ -3557,6 +3560,13 @@ static int nfs4_async_handle_error(struct rpc_task *task,
&clp->cl_state) == 0)
rpc_wake_up_queued_task(&clp->cl_rpcwaitq, task);
goto restart_call;
+ case -NFS4ERR_LEASE_MOVED:
+ rpc_sleep_on(&clp->cl_rpcwaitq, task, NULL);
+ nfs4_schedule_lease_moved_recovery(clp);
+ if (test_bit(NFS4CLNT_MANAGER_RUNNING,
+ &clp->cl_state) == 0)
+ rpc_wake_up_queued_task(&clp->cl_rpcwaitq, task);
+ goto restart_call;
case -NFS4ERR_DELAY:
nfs_inc_server_stats(server, NFSIOS_DELAY);
case -NFS4ERR_GRACE:
diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
index a788ec9..e2a83e8 100644
--- a/fs/nfs/nfs4state.c
+++ b/fs/nfs/nfs4state.c
@@ -60,6 +60,8 @@
#define OPENOWNER_POOL_SIZE 8
+static void nfs4_handle_lease_moved(struct nfs_client *clp);
+
const nfs4_stateid zero_stateid;
static LIST_HEAD(nfs4_clientid_list);
@@ -1044,6 +1046,23 @@ void nfs4_schedule_migration_recovery(struct nfs_server *server)
dprintk("<-- %s\n", __func__);
}
+/**
+ * nfs4_schedule_lease_moved_recovery - start lease moved recovery
+ *
+ * @clp: nfs_client of server that may have migrated file systems
+ *
+ */
+void nfs4_schedule_lease_moved_recovery(struct nfs_client *clp)
+{
+ dprintk("--> %s: \"%s\" (client ID %llx)\n",
+ __func__, clp->cl_hostname, clp->cl_clientid);
+
+ if (test_and_set_bit(NFS4CLNT_LEASE_MOVED, &clp->cl_state) == 0)
+ nfs4_schedule_state_manager(clp);
+
+ dprintk("<-- %s\n", __func__);
+}
+
int nfs4_state_mark_reclaim_reboot(struct nfs_client *clp, struct nfs4_state *state)
{
@@ -1349,11 +1368,13 @@ static int nfs4_recovery_handle_error(struct nfs_client *clp, int error)
nfs4_state_end_reclaim_reboot(clp);
return 0;
case -NFS4ERR_STALE_CLIENTID:
- case -NFS4ERR_LEASE_MOVED:
set_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state);
nfs4_state_clear_reclaim_reboot(clp);
nfs4_state_start_reclaim_reboot(clp);
break;
+ case -NFS4ERR_LEASE_MOVED:
+ nfs4_handle_lease_moved(clp);
+ return 0;
case -NFS4ERR_EXPIRED:
set_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state);
nfs4_state_start_reclaim_nograce(clp);
@@ -1519,6 +1540,34 @@ out:
kfree(locations);
}
+static void nfs4_handle_lease_moved(struct nfs_client *clp)
+{
+ struct nfs_server *server;
+
+ dprintk("--> %s: \"%s\" (client ID %llx)\n",
+ __func__, clp->cl_hostname, clp->cl_clientid);
+
+ /*
+ * rcu_read_lock() must be dropped before trying each individual
+ * migration. cl_mig_counter is used to skip servers that have
+ * already been visited for this lease_moved event when the list
+ * walk is restarted.
+ */
+ clp->cl_mig_counter++;
+restart:
+ rcu_read_lock();
+ list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link)
+ if (server->mig_counter != clp->cl_mig_counter) {
+ server->mig_counter = clp->cl_mig_counter;
+ rcu_read_unlock();
+ nfs4_try_migration(server);
+ goto restart;
+ }
+ rcu_read_unlock();
+
+ dprintk("<-- %s\n", __func__);
+}
+
#ifdef CONFIG_NFS_V4_1
void nfs41_handle_recall_slot(struct nfs_client *clp)
{
@@ -1748,6 +1797,11 @@ static void nfs4_state_manager(struct nfs_client *clp)
continue;
}
+ if (test_and_clear_bit(NFS4CLNT_LEASE_MOVED, &clp->cl_state)) {
+ nfs4_handle_lease_moved(clp);
+ continue;
+ }
+
if (test_and_clear_bit(NFS4CLNT_MOVED, &clp->cl_state)) {
nfs4_try_migration(clp->cl_moved_server);
continue;
diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
index 3741928..0ce4aad 100644
--- a/include/linux/nfs_fs_sb.h
+++ b/include/linux/nfs_fs_sb.h
@@ -57,6 +57,7 @@ struct nfs_client {
/* accessed only when NFS4CLNT_MOVED bit is set */
struct nfs_server * cl_moved_server;
+ unsigned long cl_mig_counter;
/* used for the setclientid verifier */
struct timespec cl_boot_time;
@@ -157,6 +158,7 @@ struct nfs_server {
struct list_head delegations;
void (*destroy)(struct nfs_server *);
struct nfs_fh *rootfh;
+ unsigned long mig_counter;
atomic_t active; /* Keep trace of any activity to this server */
^ permalink raw reply related [flat|nested] 15+ messages in thread