Re: [PATCH v2 11/35] pnfsd: get device list/info

[Benny Halevy <bhalevy@panasas.com>]

On Dec. 10, 2009, 19:30 +0200, " J. Bruce Fields" <bfields@citi.umich.edu> wrote:
> On Mon, Dec 07, 2009 at 11:32:10AM +0200, Benny Halevy wrote:
>> +static __be32
>> +nfsd4_getdevinfo(struct svc_rqst *rqstp,
>> +		struct nfsd4_compound_state *cstate,
>> +		struct nfsd4_pnfs_getdevinfo *gdp)
>> +{
>> +	struct super_block *sb;
>> +	struct svc_export *exp = NULL;
>> +	u32 fsidv = gdp->gd_devid.fsid;
>> +	int status;
>> +
>> +	dprintk("%s: layout_type %u dev_id %llx:%llx maxcnt %u\n",
>> +	       __func__, gdp->gd_layout_type, gdp->gd_devid.fsid,
>> +	       gdp->gd_devid.devid, gdp->gd_maxcount);
>> +
>> +	status = nfserr_inval;
>> +	exp = rqst_exp_find(rqstp, FSID_NUM, &fsidv);
> 
> As I said before, this seems to require an fsid= option on every pnfs
> export.  We shouldn't need that.

If there a better way to get to the sb given the fsid?
Since we don't have a current_fh for getdeviceinfo the alternative
might be to map the deviceids to sb's in the generic layer.

Benny

> 
> --b.
> 
> 
>> +	dprintk("%s: exp %p\n", __func__, exp);
>> +	if (IS_ERR(exp)) {
>> +		status = nfserrno(PTR_ERR(exp));
>> +		exp = NULL;
>> +		goto out;
>> +	}
>> +	sb = exp->ex_path.dentry->d_inode->i_sb;
>> +	dprintk("%s: sb %p\n", __func__, sb);
>> +	if (!sb)
>> +		goto out;
>> +
>> +	/* Ensure underlying file system supports pNFS and,
>> +	 * if so, the requested layout type
>> +	 */
>> +	status = nfsd4_layout_verify(sb, exp, gdp->gd_layout_type);
>> +	if (status)
>> +		goto out;
>> +
>> +	/* Set up arguments so device can be retrieved at encode time */
>> +	gdp->gd_sb = sb;
>> +out:
>> +	if (exp)
>> +		exp_put(exp);
>> +	return status;
>> +}
>>  #endif /* CONFIG_PNFSD */
>>  
>>  /*
>> @@ -1330,6 +1420,17 @@ static struct nfsd4_operation nfsd4_ops[] = {
>>  		.op_flags = ALLOWED_WITHOUT_FH | ALLOWED_AS_FIRST_OP,
>>  		.op_name = "OP_SEQUENCE",
>>  	},
>> +#if defined(CONFIG_PNFSD)
>> +	[OP_GETDEVICELIST] = {
>> +		.op_func = (nfsd4op_func)nfsd4_getdevlist,
>> +		.op_name = "OP_GETDEVICELIST",
>> +	},
>> +	[OP_GETDEVICEINFO] = {
>> +		.op_func = (nfsd4op_func)nfsd4_getdevinfo,
>> +		.op_flags = ALLOWED_WITHOUT_FH,
>> +		.op_name = "OP_GETDEVICEINFO",
>> +	},
>> +#endif /* CONFIG_PNFSD */
>>  };
>>  
>>  static const char *nfsd4_op_name(unsigned opnum)
>> diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
>> index a8587e9..955f583 100644
>> --- a/fs/nfsd/nfs4xdr.c
>> +++ b/fs/nfsd/nfs4xdr.c
>> @@ -46,6 +46,7 @@
>>  #include <linux/nfsd_idmap.h>
>>  #include <linux/nfs4_acl.h>
>>  #include <linux/sunrpc/svcauth_gss.h>
>> +#include <linux/exportfs.h>
>>  
>>  #include "xdr4.h"
>>  #include "vfs.h"
>> @@ -1233,6 +1234,42 @@ nfsd4_decode_sequence(struct nfsd4_compoundargs *argp,
>>  	DECODE_TAIL;
>>  }
>>  
>> +#if defined(CONFIG_PNFSD)
>> +static __be32
>> +nfsd4_decode_getdevlist(struct nfsd4_compoundargs *argp,
>> +			struct nfsd4_pnfs_getdevlist *gdevl)
>> +{
>> +	DECODE_HEAD;
>> +
>> +	READ_BUF(16 + sizeof(nfs4_verifier));
>> +	READ32(gdevl->gd_layout_type);
>> +	READ32(gdevl->gd_maxdevices);
>> +	READ64(gdevl->gd_cookie);
>> +	COPYMEM(&gdevl->gd_verf, sizeof(nfs4_verifier));
>> +
>> +	DECODE_TAIL;
>> +}
>> +
>> +static __be32
>> +nfsd4_decode_getdevinfo(struct nfsd4_compoundargs *argp,
>> +			struct nfsd4_pnfs_getdevinfo *gdev)
>> +{
>> +	u32 num;
>> +	DECODE_HEAD;
>> +
>> +	READ_BUF(12 + sizeof(struct nfsd4_pnfs_deviceid));
>> +	READ64(gdev->gd_devid.fsid);
>> +	READ64(gdev->gd_devid.devid);
>> +	READ32(gdev->gd_layout_type);
>> +	READ32(gdev->gd_maxcount);
>> +	READ32(num);
>> +	if (num)
>> +		READ_BUF(4); /* TODO: for now, just skip notify_types */
>> +
>> +	DECODE_TAIL;
>> +}
>> +#endif /* CONFIG_PNFSD */
>> +
>>  static __be32
>>  nfsd4_decode_noop(struct nfsd4_compoundargs *argp, void *p)
>>  {
>> @@ -1334,11 +1371,19 @@ static nfsd4_dec nfsd41_dec_ops[] = {
>>  	[OP_DESTROY_SESSION]	= (nfsd4_dec)nfsd4_decode_destroy_session,
>>  	[OP_FREE_STATEID]	= (nfsd4_dec)nfsd4_decode_notsupp,
>>  	[OP_GET_DIR_DELEGATION]	= (nfsd4_dec)nfsd4_decode_notsupp,
>> +#if defined(CONFIG_PNFSD)
>> +	[OP_GETDEVICEINFO]	= (nfsd4_dec)nfsd4_decode_getdevinfo,
>> +	[OP_GETDEVICELIST]	= (nfsd4_dec)nfsd4_decode_getdevlist,
>> +	[OP_LAYOUTCOMMIT]	= (nfsd4_dec)nfsd4_decode_notsupp,
>> +	[OP_LAYOUTGET]		= (nfsd4_dec)nfsd4_decode_notsupp,
>> +	[OP_LAYOUTRETURN]	= (nfsd4_dec)nfsd4_decode_notsupp,
>> +#else  /* CONFIG_PNFSD */
>>  	[OP_GETDEVICEINFO]	= (nfsd4_dec)nfsd4_decode_notsupp,
>>  	[OP_GETDEVICELIST]	= (nfsd4_dec)nfsd4_decode_notsupp,
>>  	[OP_LAYOUTCOMMIT]	= (nfsd4_dec)nfsd4_decode_notsupp,
>>  	[OP_LAYOUTGET]		= (nfsd4_dec)nfsd4_decode_notsupp,
>>  	[OP_LAYOUTRETURN]	= (nfsd4_dec)nfsd4_decode_notsupp,
>> +#endif /* CONFIG_PNFSD */
>>  	[OP_SECINFO_NO_NAME]	= (nfsd4_dec)nfsd4_decode_notsupp,
>>  	[OP_SEQUENCE]		= (nfsd4_dec)nfsd4_decode_sequence,
>>  	[OP_SET_SSV]		= (nfsd4_dec)nfsd4_decode_notsupp,
>> @@ -3062,6 +3107,207 @@ nfsd4_encode_sequence(struct nfsd4_compoundres *resp, int nfserr,
>>  	return 0;
>>  }
>>  
>> +#if defined(CONFIG_PNFSD)
>> +
>> +/* Uses the export interface to iterate through the available devices
>> + * and encodes them on the response stream.
>> + */
>> +static  __be32
>> +nfsd4_encode_devlist_iterator(struct nfsd4_compoundres *resp,
>> +			      struct nfsd4_pnfs_getdevlist *gdevl,
>> +			      unsigned int *dev_count)
>> +{
>> +	struct super_block *sb = gdevl->gd_fhp->fh_dentry->d_inode->i_sb;
>> +	__be32 nfserr;
>> +	int status;
>> +	__be32 *p;
>> +	struct nfsd4_pnfs_dev_iter_res res = {
>> +		.gd_cookie = gdevl->gd_cookie,
>> +		.gd_verf = gdevl->gd_verf,
>> +		.gd_eof = 0
>> +	};
>> +
>> +	dprintk("%s: Begin\n", __func__);
>> +
>> +	*dev_count = 0;
>> +	do {
>> +		status = sb->s_pnfs_op->get_device_iter(sb,
>> +							gdevl->gd_layout_type,
>> +							&res);
>> +		if (status) {
>> +			if (status == -ENOENT) {
>> +				res.gd_eof = 1;
>> +				/* return success */
>> +				break;
>> +			}
>> +			nfserr = nfserrno(status);
>> +			goto out_err;
>> +		}
>> +
>> +		/* Encode device id and layout type */
>> +		RESERVE_SPACE(sizeof(struct nfsd4_pnfs_deviceid));
>> +		WRITE64((__be64)gdevl->gd_fhp->fh_export->ex_fsid);
>> +		WRITE64(res.gd_devid);	/* devid minor */
>> +		ADJUST_ARGS();
>> +		(*dev_count)++;
>> +	} while (*dev_count < gdevl->gd_maxdevices && !res.gd_eof);
>> +	gdevl->gd_cookie = res.gd_cookie;
>> +	gdevl->gd_verf = res.gd_verf;
>> +	gdevl->gd_eof = res.gd_eof;
>> +	nfserr = nfs_ok;
>> +out_err:
>> +	dprintk("%s: Encoded %u devices\n", __func__, *dev_count);
>> +	return nfserr;
>> +}
>> +
>> +/* Encodes the response of get device list.
>> +*/
>> +static __be32
>> +nfsd4_encode_getdevlist(struct nfsd4_compoundres *resp, int nfserr,
>> +			struct nfsd4_pnfs_getdevlist *gdevl)
>> +{
>> +	unsigned int dev_count = 0, lead_count;
>> +	u32 *p_in = resp->p;
>> +	__be32 *p;
>> +
>> +	dprintk("%s: err %d\n", __func__, nfserr);
>> +	if (nfserr)
>> +		return nfserr;
>> +
>> +	/* Ensure we have room for cookie, verifier, and devlist len,
>> +	 * which we will backfill in after we encode as many devices as possible
>> +	 */
>> +	lead_count = 8 + sizeof(nfs4_verifier) + 4;
>> +	RESERVE_SPACE(lead_count);
>> +	/* skip past these values */
>> +	p += XDR_QUADLEN(lead_count);
>> +	ADJUST_ARGS();
>> +
>> +	/* Iterate over as many device ids as possible on the xdr stream */
>> +	nfserr = nfsd4_encode_devlist_iterator(resp, gdevl, &dev_count);
>> +	if (nfserr)
>> +		goto out_err;
>> +
>> +	/* Backfill in cookie, verf and number of devices encoded */
>> +	p = p_in;
>> +	WRITE64(gdevl->gd_cookie);
>> +	WRITEMEM(&gdevl->gd_verf, sizeof(nfs4_verifier));
>> +	WRITE32(dev_count);
>> +
>> +	/* Skip over devices */
>> +	p += XDR_QUADLEN(dev_count * sizeof(struct nfsd4_pnfs_deviceid));
>> +	ADJUST_ARGS();
>> +
>> +	/* are we at the end of devices? */
>> +	RESERVE_SPACE(4);
>> +	WRITE32(gdevl->gd_eof);
>> +	ADJUST_ARGS();
>> +
>> +	dprintk("%s: done.\n", __func__);
>> +
>> +	nfserr = nfs_ok;
>> +out:
>> +	return nfserr;
>> +out_err:
>> +	p = p_in;
>> +	ADJUST_ARGS();
>> +	goto out;
>> +}
>> +
>> +/* For a given device id, have the file system retrieve and encode the
>> + * associated device.  For file layout, the encoding function is
>> + * passed down to the file system.  The file system then has the option
>> + * of using this encoding function or one of its own.
>> + *
>> + * Note: the file system must return the XDR size of struct device_addr4
>> + * da_addr_body in pnfs_xdr_info.bytes_written on NFS4ERR_TOOSMALL for the
>> + * gdir_mincount calculation.
>> + */
>> +static __be32
>> +nfsd4_encode_getdevinfo(struct nfsd4_compoundres *resp, int nfserr,
>> +			struct nfsd4_pnfs_getdevinfo *gdev)
>> +{
>> +	struct super_block *sb;
>> +	int maxcount = 0, type_notify_len = 12;
>> +	__be32 *p, *p_save = NULL, *p_in = resp->p;
>> +	struct exp_xdr_stream xdr;
>> +
>> +	dprintk("%s: err %d\n", __func__, nfserr);
>> +	if (nfserr)
>> +		return nfserr;
>> +
>> +	sb = gdev->gd_sb;
>> +
>> +	if (gdev->gd_maxcount != 0) {
>> +		/* FIXME: this will be bound by the session max response */
>> +		maxcount = svc_max_payload(resp->rqstp);
>> +		if (maxcount > gdev->gd_maxcount)
>> +			maxcount = gdev->gd_maxcount;
>> +
>> +		/* Ensure have room for type and notify field */
>> +		maxcount -= type_notify_len;
>> +		if (maxcount < 0) {
>> +			nfserr = -ETOOSMALL;
>> +			goto toosmall;
>> +		}
>> +	}
>> +
>> +	RESERVE_SPACE(4);
>> +	WRITE32(gdev->gd_layout_type);
>> +	ADJUST_ARGS();
>> +
>> +	/* If maxcount is 0 then just update notifications */
>> +	if (gdev->gd_maxcount == 0)
>> +		goto handle_notifications;
>> +
>> +	xdr.p = p_save = resp->p;
>> +	xdr.end = resp->end;
>> +	if (xdr.end - xdr.p > exp_xdr_qwords(maxcount & ~3))
>> +		xdr.end = xdr.p + exp_xdr_qwords(maxcount & ~3);
>> +
>> +	nfserr = sb->s_pnfs_op->get_device_info(sb, &xdr, gdev->gd_layout_type,
>> +						&gdev->gd_devid);
>> +	if (nfserr) {
>> +		/* Rewind to the beginning */
>> +		p = p_in;
>> +		ADJUST_ARGS();
>> +		if (nfserr == -ETOOSMALL)
>> +			goto toosmall;
>> +		printk(KERN_ERR "%s: export ERROR %d\n", __func__, nfserr);
>> +		goto out;
>> +	}
>> +
>> +	/* The file system should never write 0 bytes without
>> +	 * returning an error
>> +	 */
>> +	BUG_ON(xdr.p == p_save);
>> +	BUG_ON(xdr.p > xdr.end);
>> +
>> +	/* Update the xdr stream with the number of bytes encoded
>> +	 * by the file system.
>> +	 */
>> +	p = xdr.p;
>> +	ADJUST_ARGS();
>> +
>> +handle_notifications:
>> +	/* Encode supported device notifications.
>> +	 * Note: Currently none are supported.
>> +	 */
>> +	RESERVE_SPACE(4);
>> +	WRITE32(0);
>> +	ADJUST_ARGS();
>> +
>> +out:
>> +	return nfserrno(nfserr);
>> +toosmall:
>> +	dprintk("%s: maxcount too small\n", __func__);
>> +	RESERVE_SPACE(4);
>> +	WRITE32((p_save ? (xdr.p - p_save) * 4 : 0) + type_notify_len);
>> +	ADJUST_ARGS();
>> +	goto out;
>> +}
>> +#endif /* CONFIG_PNFSD */
>> +
>>  static __be32
>>  nfsd4_encode_noop(struct nfsd4_compoundres *resp, __be32 nfserr, void *p)
>>  {
>> @@ -3122,11 +3368,19 @@ static nfsd4_enc nfsd4_enc_ops[] = {
>>  	[OP_DESTROY_SESSION]	= (nfsd4_enc)nfsd4_encode_destroy_session,
>>  	[OP_FREE_STATEID]	= (nfsd4_enc)nfsd4_encode_noop,
>>  	[OP_GET_DIR_DELEGATION]	= (nfsd4_enc)nfsd4_encode_noop,
>> +#if defined(CONFIG_PNFSD)
>> +	[OP_GETDEVICEINFO]	= (nfsd4_enc)nfsd4_encode_getdevinfo,
>> +	[OP_GETDEVICELIST]	= (nfsd4_enc)nfsd4_encode_getdevlist,
>> +	[OP_LAYOUTCOMMIT]	= (nfsd4_enc)nfsd4_encode_noop,
>> +	[OP_LAYOUTGET]		= (nfsd4_enc)nfsd4_encode_noop,
>> +	[OP_LAYOUTRETURN]	= (nfsd4_enc)nfsd4_encode_noop,
>> +#else  /* CONFIG_PNFSD */
>>  	[OP_GETDEVICEINFO]	= (nfsd4_enc)nfsd4_encode_noop,
>>  	[OP_GETDEVICELIST]	= (nfsd4_enc)nfsd4_encode_noop,
>>  	[OP_LAYOUTCOMMIT]	= (nfsd4_enc)nfsd4_encode_noop,
>>  	[OP_LAYOUTGET]		= (nfsd4_enc)nfsd4_encode_noop,
>>  	[OP_LAYOUTRETURN]	= (nfsd4_enc)nfsd4_encode_noop,
>> +#endif /* CONFIG_PNFSD */
>>  	[OP_SECINFO_NO_NAME]	= (nfsd4_enc)nfsd4_encode_noop,
>>  	[OP_SEQUENCE]		= (nfsd4_enc)nfsd4_encode_sequence,
>>  	[OP_SET_SSV]		= (nfsd4_enc)nfsd4_encode_noop,
>> diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
>> index 83202a1..acb215a 100644
>> --- a/fs/nfsd/xdr4.h
>> +++ b/fs/nfsd/xdr4.h
>> @@ -39,6 +39,8 @@
>>  #ifndef _LINUX_NFSD_XDR4_H
>>  #define _LINUX_NFSD_XDR4_H
>>  
>> +#include <linux/nfsd/nfsd4_pnfs.h>
>> +
>>  #include "state.h"
>>  #include "nfsd.h"
>>  
>> @@ -383,6 +385,22 @@ struct nfsd4_destroy_session {
>>  	struct nfs4_sessionid	sessionid;
>>  };
>>  
>> +struct nfsd4_pnfs_getdevinfo {
>> +	struct nfsd4_pnfs_deviceid gd_devid;	/* request */
>> +	u32			gd_layout_type;	/* request */
>> +	u32			gd_maxcount;	/* request */
>> +	struct super_block	*gd_sb;
>> +};
>> +
>> +struct nfsd4_pnfs_getdevlist {
>> +	u32             gd_layout_type;	/* request */
>> +	u32		gd_maxdevices;	/* request */
>> +	u64		gd_cookie;	/* request - response */
>> +	u64		gd_verf;	/* request - response */
>> +	struct svc_fh 	*gd_fhp;	/* response */
>> +	u32		gd_eof;		/* response */
>> +};
>> +
>>  struct nfsd4_op {
>>  	int					opnum;
>>  	__be32					status;
>> @@ -423,6 +441,10 @@ struct nfsd4_op {
>>  		struct nfsd4_create_session	create_session;
>>  		struct nfsd4_destroy_session	destroy_session;
>>  		struct nfsd4_sequence		sequence;
>> +#if defined(CONFIG_PNFSD)
>> +		struct nfsd4_pnfs_getdevlist	pnfs_getdevlist;
>> +		struct nfsd4_pnfs_getdevinfo	pnfs_getdevinfo;
>> +#endif /* CONFIG_PNFSD */
>>  	} u;
>>  	struct nfs4_replay *			replay;
>>  };
>> diff --git a/include/linux/nfsd/nfsd4_pnfs.h b/include/linux/nfsd/nfsd4_pnfs.h
>> index c44e13d..d68fd14 100644
>> --- a/include/linux/nfsd/nfsd4_pnfs.h
>> +++ b/include/linux/nfsd/nfsd4_pnfs.h
>> @@ -34,6 +34,21 @@
>>  #ifndef _LINUX_NFSD_NFSD4_PNFS_H
>>  #define _LINUX_NFSD_NFSD4_PNFS_H
>>  
>> +#include <linux/exportfs.h>
>> +#include <linux/exp_xdr.h>
>> +
>> +struct nfsd4_pnfs_deviceid {
>> +	u64	fsid;			/* filesystem ID */
>> +	u64	devid;			/* filesystem-wide unique device ID */
>> +};
>> +
>> +struct nfsd4_pnfs_dev_iter_res {
>> +	u64		gd_cookie;	/* request/repsonse */
>> +	u64		gd_verf;	/* request/repsonse */
>> +	u64		gd_devid;	/* response */
>> +	u32		gd_eof;		/* response */
>> +};
>> +
>>  /*
>>   * pNFS export operations vector.
>>   *
>> @@ -47,6 +62,25 @@
>>  struct pnfs_export_operations {
>>  	/* Returns the supported pnfs_layouttype4. */
>>  	int (*layout_type) (struct super_block *);
>> +
>> +	/* Encode device info onto the xdr stream. */
>> +	int (*get_device_info) (struct super_block *,
>> +				struct exp_xdr_stream *,
>> +				u32 layout_type,
>> +				const struct nfsd4_pnfs_deviceid *);
>> +
>> +	/* Retrieve all available devices via an iterator.
>> +	 * arg->cookie == 0 indicates the beginning of the list,
>> +	 * otherwise arg->verf is used to verify that the list hasn't changed
>> +	 * while retrieved.
>> +	 *
>> +	 * On output, the filesystem sets the devid based on the current cookie
>> +	 * and sets res->cookie and res->verf corresponding to the next entry.
>> +	 * When the last entry in the list is retrieved, res->eof is set to 1.
>> +	 */
>> +	int (*get_device_iter) (struct super_block *,
>> +				u32 layout_type,
>> +				struct nfsd4_pnfs_dev_iter_res *);
>>  };
>>  
>>  #endif /* _LINUX_NFSD_NFSD4_PNFS_H */
>> -- 
>> 1.6.5.1
>>