From: andros@netapp.com
To: bhalevy@panasas.com
Cc: linux-nfs@vger.kernel.org, Andy Adamson <andros@netapp.com>
Subject: [PATCH 24/50] pnfs_submit: filelayout layout segment alloc and free
Date: Fri, 13 Aug 2010 17:31:36 -0400 [thread overview]
Message-ID: <1281735122-1496-25-git-send-email-andros@netapp.com> (raw)
In-Reply-To: <1281735122-1496-24-git-send-email-andros@netapp.com>
From: The pNFS Team <linux-nfs@vger.kernel.org>
Signed-off-by: Andy Adamson <andros@netapp.com>
---
fs/nfs/nfs4filelayout.c | 201 +++++++++++++++++++++++++++++++++++++++++++++
fs/nfs/nfs4filelayout.h | 16 ++++
fs/nfs/pnfs.c | 22 +++++
include/linux/nfs4.h | 5 +
include/linux/nfs4_pnfs.h | 12 +++
5 files changed, 256 insertions(+), 0 deletions(-)
diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
index e1a09a8..50620f4 100644
--- a/fs/nfs/nfs4filelayout.c
+++ b/fs/nfs/nfs4filelayout.c
@@ -46,6 +46,7 @@
#include "nfs4filelayout.h"
#include "nfs4_fs.h"
#include "internal.h"
+#include "pnfs.h"
#define NFSDBG_FACILITY NFSDBG_PNFS_LD
@@ -104,9 +105,209 @@ filelayout_free_layout(struct pnfs_layout_type *lo)
dprintk("NFS_FILELAYOUT: freeing layout\n");
kfree(FILE_LO(lo));
}
+
+/*
+ * filelayout_check_layout()
+ *
+ * Make sure layout segment parameters are sane WRT the device.
+ *
+ * Notes:
+ * 1) current code insists that # stripe index = # data servers in ds_list
+ * which is wrong.
+ * 2) pattern_offset is ignored and must == 0 which is wrong;
+ * 3) the pattern_offset needs to be a mutliple of the stripe unit.
+ * 4) stripe unit is multiple of page size
+ */
+
+static int
+filelayout_check_layout(struct pnfs_layout_type *lo,
+ struct pnfs_layout_segment *lseg)
+{
+ struct nfs4_filelayout_segment *fl = LSEG_LD_DATA(lseg);
+ struct nfs4_file_layout_dsaddr *dsaddr;
+ int status = -EINVAL;
+ struct nfs_server *nfss = NFS_SERVER(PNFS_INODE(lo));
+
+ dprintk("--> %s\n", __func__);
+ dsaddr = nfs4_pnfs_device_item_find(nfss->nfs_client, &fl->dev_id);
+ if (dsaddr == NULL) {
+ dsaddr = get_device_info(PNFS_INODE(lo), &fl->dev_id);
+ if (dsaddr == NULL) {
+ dprintk("%s NO device for dev_id %s\n",
+ __func__, deviceid_fmt(&fl->dev_id));
+ goto out;
+ }
+ }
+ if (fl->first_stripe_index < 0 ||
+ fl->first_stripe_index > dsaddr->stripe_count) {
+ dprintk("%s Bad first_stripe_index %d\n",
+ __func__, fl->first_stripe_index);
+ goto out;
+ }
+
+ if (fl->pattern_offset != 0) {
+ dprintk("%s Unsupported no-zero pattern_offset %Ld\n",
+ __func__, fl->pattern_offset);
+ goto out;
+ }
+
+ if (fl->stripe_unit % PAGE_SIZE) {
+ dprintk("%s Stripe unit (%u) not page aligned\n",
+ __func__, fl->stripe_unit);
+ goto out;
+ }
+
+ /* XXX only support SPARSE packing. Don't support use MDS open fh */
+ if (!(fl->num_fh == 1 || fl->num_fh == dsaddr->ds_num)) {
+ dprintk("%s num_fh %u not equal to 1 or ds_num %u\n",
+ __func__, fl->num_fh, dsaddr->ds_num);
+ goto out;
+ }
+
+ if (fl->stripe_unit % nfss->rsize || fl->stripe_unit % nfss->wsize) {
+ dprintk("%s Stripe unit (%u) not aligned with rsize %u "
+ "wsize %u\n", __func__, fl->stripe_unit, nfss->rsize,
+ nfss->wsize);
+ }
+
+ /* reference the device */
+ nfs4_set_layout_deviceid(lseg, &dsaddr->deviceid);
+
+ status = 0;
+out:
+ dprintk("--> %s returns %d\n", __func__, status);
+ return status;
+}
+
+static void _filelayout_free_lseg(struct pnfs_layout_segment *lseg);
+static void filelayout_free_fh_array(struct nfs4_filelayout_segment *fl);
+
+/* Decode layout and store in layoutid. Overwrite any existing layout
+ * information for this file.
+ */
+static int
+filelayout_set_layout(struct nfs4_filelayout *flo,
+ struct nfs4_filelayout_segment *fl,
+ struct nfs4_pnfs_layoutget_res *lgr)
+{
+ uint32_t *p = (uint32_t *)lgr->layout.buf;
+ uint32_t nfl_util;
+ int i;
+
+ dprintk("%s: set_layout_map Begin\n", __func__);
+
+ memcpy(&fl->dev_id, p, NFS4_PNFS_DEVICEID4_SIZE);
+ p += XDR_QUADLEN(NFS4_PNFS_DEVICEID4_SIZE);
+ nfl_util = be32_to_cpup(p++);
+ if (nfl_util & NFL4_UFLG_COMMIT_THRU_MDS)
+ fl->commit_through_mds = 1;
+ if (nfl_util & NFL4_UFLG_DENSE)
+ fl->stripe_type = STRIPE_DENSE;
+ else
+ fl->stripe_type = STRIPE_SPARSE;
+ fl->stripe_unit = nfl_util & ~NFL4_UFLG_MASK;
+
+ if (!flo->stripe_unit)
+ flo->stripe_unit = fl->stripe_unit;
+ else if (flo->stripe_unit != fl->stripe_unit) {
+ printk(KERN_NOTICE "%s: updating strip_unit from %u to %u\n",
+ __func__, flo->stripe_unit, fl->stripe_unit);
+ flo->stripe_unit = fl->stripe_unit;
+ }
+
+ fl->first_stripe_index = be32_to_cpup(p++);
+ p = xdr_decode_hyper(p, &fl->pattern_offset);
+ fl->num_fh = be32_to_cpup(p++);
+
+ dprintk("%s: nfl_util 0x%X num_fh %u fsi %u po %llu dev_id %s\n",
+ __func__, nfl_util, fl->num_fh, fl->first_stripe_index,
+ fl->pattern_offset, deviceid_fmt(&fl->dev_id));
+
+ if (fl->num_fh * sizeof(struct nfs_fh) > 2*PAGE_SIZE) {
+ fl->fh_array = vmalloc(fl->num_fh * sizeof(struct nfs_fh));
+ if (fl->fh_array)
+ memset(fl->fh_array, 0,
+ fl->num_fh * sizeof(struct nfs_fh));
+ } else {
+ fl->fh_array = kzalloc(fl->num_fh * sizeof(struct nfs_fh),
+ GFP_KERNEL);
+ }
+ if (!fl->fh_array)
+ return -ENOMEM;
+
+ for (i = 0; i < fl->num_fh; i++) {
+ /* fh */
+ fl->fh_array[i].size = be32_to_cpup(p++);
+ if (sizeof(struct nfs_fh) < fl->fh_array[i].size) {
+ printk(KERN_ERR "Too big fh %d received %d\n",
+ i, fl->fh_array[i].size);
+ /* Layout is now invalid, pretend it doesn't exist */
+ filelayout_free_fh_array(fl);
+ fl->num_fh = 0;
+ break;
+ }
+ memcpy(fl->fh_array[i].data, p, fl->fh_array[i].size);
+ p += XDR_QUADLEN(fl->fh_array[i].size);
+ dprintk("DEBUG: %s: fh len %d\n", __func__,
+ fl->fh_array[i].size);
+ }
+
+ return 0;
+}
+
+static struct pnfs_layout_segment *
+filelayout_alloc_lseg(struct pnfs_layout_type *layoutid,
+ struct nfs4_pnfs_layoutget_res *lgr)
+{
+ struct nfs4_filelayout *flo = FILE_LO(layoutid);
+ struct pnfs_layout_segment *lseg;
+ int rc;
+
+ dprintk("--> %s\n", __func__);
+ lseg = kzalloc(sizeof(struct pnfs_layout_segment) +
+ sizeof(struct nfs4_filelayout_segment), GFP_KERNEL);
+ if (!lseg)
+ return NULL;
+
+ rc = filelayout_set_layout(flo, LSEG_LD_DATA(lseg), lgr);
+
+ if (rc != 0 || filelayout_check_layout(layoutid, lseg)) {
+ _filelayout_free_lseg(lseg);
+ lseg = NULL;
+ }
+ return lseg;
+}
+
+static void filelayout_free_fh_array(struct nfs4_filelayout_segment *fl)
+{
+ if (fl->num_fh * sizeof(struct nfs_fh) > 2*PAGE_SIZE)
+ vfree(fl->fh_array);
+ else
+ kfree(fl->fh_array);
+
+ fl->fh_array = NULL;
+}
+
+static void
+_filelayout_free_lseg(struct pnfs_layout_segment *lseg)
+{
+ filelayout_free_fh_array(LSEG_LD_DATA(lseg));
+ kfree(lseg);
+}
+
+static void
+filelayout_free_lseg(struct pnfs_layout_segment *lseg)
+{
+ dprintk("--> %s\n", __func__);
+ nfs4_unset_layout_deviceid(lseg, lseg->deviceid,
+ nfs4_fl_free_deviceid_callback);
+ _filelayout_free_lseg(lseg);
+}
struct layoutdriver_io_operations filelayout_io_operations = {
.alloc_layout = filelayout_alloc_layout,
.free_layout = filelayout_free_layout,
+ .alloc_lseg = filelayout_alloc_lseg,
+ .free_lseg = filelayout_free_lseg,
.initialize_mountpoint = filelayout_initialize_mountpoint,
.uninitialize_mountpoint = filelayout_uninitialize_mountpoint,
};
diff --git a/fs/nfs/nfs4filelayout.h b/fs/nfs/nfs4filelayout.h
index ad975fd..aeb2147 100644
--- a/fs/nfs/nfs4filelayout.h
+++ b/fs/nfs/nfs4filelayout.h
@@ -23,6 +23,11 @@
#define NFS4_PNFS_MAX_STRIPE_CNT 4096
#define NFS4_PNFS_MAX_MULTI_CNT 64 /* 256 fit into a u8 stripe_index */
+enum stripetype4 {
+ STRIPE_SPARSE = 1,
+ STRIPE_DENSE = 2
+};
+
/* Individual ip address */
struct nfs4_pnfs_ds {
struct list_head ds_node; /* nfs4_pnfs_dev_hlist dev_dslist */
@@ -41,6 +46,17 @@ struct nfs4_file_layout_dsaddr {
struct nfs4_pnfs_ds *ds_list[1];
};
+struct nfs4_filelayout_segment {
+ u32 stripe_type;
+ u32 commit_through_mds;
+ u32 stripe_unit;
+ u32 first_stripe_index;
+ u64 pattern_offset;
+ struct pnfs_deviceid dev_id;
+ unsigned int num_fh;
+ struct nfs_fh *fh_array;
+};
+
struct nfs4_filelayout {
struct pnfs_layout_type fl_layout;
u32 stripe_unit;
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 0f98261..33be484 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -972,6 +972,28 @@ nfs4_init_deviceid_node(struct nfs4_deviceid *d)
}
EXPORT_SYMBOL(nfs4_init_deviceid_node);
+/* Called from layoutdriver_io_operations->alloc_lseg */
+void
+nfs4_set_layout_deviceid(struct pnfs_layout_segment *l, struct nfs4_deviceid *d)
+{
+ dprintk("%s [%d]\n", __func__, atomic_read(&d->de_kref.refcount));
+ l->deviceid = d;
+ kref_get(&d->de_kref);
+}
+EXPORT_SYMBOL(nfs4_set_layout_deviceid);
+
+/* Called from layoutdriver_io_operations->free_lseg */
+void
+nfs4_unset_layout_deviceid(struct pnfs_layout_segment *l,
+ struct nfs4_deviceid *d,
+ void (*free_callback)(struct kref *))
+{
+ dprintk("%s [%d]\n", __func__, atomic_read(&d->de_kref.refcount));
+ l->deviceid = NULL;
+ kref_put(&d->de_kref, free_callback);
+}
+EXPORT_SYMBOL(nfs4_unset_layout_deviceid);
+
struct nfs4_deviceid *
nfs4_find_deviceid(struct nfs4_deviceid_cache *c, struct pnfs_deviceid *id)
{
diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h
index a5f5c94..2e11a3d 100644
--- a/include/linux/nfs4.h
+++ b/include/linux/nfs4.h
@@ -576,6 +576,11 @@ enum pnfs_iomode {
IOMODE_ANY = 3,
};
+#define NFL4_UFLG_MASK 0x0000003F
+#define NFL4_UFLG_DENSE 0x00000001
+#define NFL4_UFLG_COMMIT_THRU_MDS 0x00000002
+#define NFL4_UFLG_STRIPE_UNIT_SIZE_MASK 0xFFFFFFC0
+
#endif
#endif
diff --git a/include/linux/nfs4_pnfs.h b/include/linux/nfs4_pnfs.h
index 287a7dc..1ed509c 100644
--- a/include/linux/nfs4_pnfs.h
+++ b/include/linux/nfs4_pnfs.h
@@ -67,6 +67,12 @@ struct pnfs_layout_segment {
u8 ld_data[]; /* layout driver private data */
};
+static inline void *
+LSEG_LD_DATA(struct pnfs_layout_segment *lseg)
+{
+ return lseg->ld_data;
+}
+
/* Layout driver I/O operations.
* Either the pagecache or non-pagecache read/write operations must be implemented
*/
@@ -142,6 +148,12 @@ extern struct nfs4_deviceid *nfs4_find_deviceid(struct nfs4_deviceid_cache *,
struct pnfs_deviceid *);
extern struct nfs4_deviceid *nfs4_add_deviceid(struct nfs4_deviceid_cache *,
struct nfs4_deviceid *);
+extern void nfs4_set_layout_deviceid(struct pnfs_layout_segment *,
+ struct nfs4_deviceid *);
+extern void nfs4_unset_layout_deviceid(struct pnfs_layout_segment *,
+ struct nfs4_deviceid *,
+ void (*free_callback)(struct kref *));
+
/* pNFS client callback functions.
* These operations allow the layout driver to access pNFS client
* specific information or call pNFS client->server operations.
--
1.6.2.5
next prev parent reply other threads:[~2010-08-13 21:32 UTC|newest]
Thread overview: 69+ messages / expand[flat|nested] mbox.gz Atom feed top
2010-08-13 21:31 [PATCH 0/50] Squashed and re-organized pnfs-submit tree andros
2010-08-13 21:31 ` [PATCH 01/50] nfs41: prevent exchange_id from sending server-only flag andros
2010-08-13 21:31 ` [PATCH 02/50] sunrpc: define xdr_decode_opaque_fixed andros
2010-08-13 21:31 ` [PATCH 03/50] sunrpc: don't reset buflen twice in xdr_shrink_pagelen andros
2010-08-13 21:31 ` [PATCH 04/50] nfsd: remove duplicate NFS4_STATEID_SIZE declaration andros
2010-08-13 21:31 ` [PATCH 05/50] pnfs_submit: pnfs and nfslayoutdriver kconfig andros
2010-08-13 21:31 ` [PATCH 06/50] pnfs_submit: introduce include/linux/nfs4_pnfs.h andros
2010-08-13 21:31 ` [PATCH 07/50] pnfs_submit: introduce include/linux/pnfs_xdr.h andros
2010-08-13 21:31 ` [PATCH 08/50] pnfs_submit: introduce fs/nfs/pnfs.h andros
2010-08-13 21:31 ` [PATCH 09/50] pnfs_submit: introduce fs/nfs/pnfs.c andros
2010-08-13 21:31 ` [PATCH 10/50] pnfs_submit: register unregister pnfs module andros
2010-08-13 21:31 ` [PATCH 11/50] pnfs_submit: set and unset pnfs layoutdriver modules andros
2010-08-13 21:31 ` [PATCH 12/50] pnfs_submit: generic pnfs deviceid cache andros
2010-08-13 21:31 ` [PATCH 13/50] pnfs_submit: introduce nfs4layoutdriver module andros
2010-08-13 21:31 ` [PATCH 14/50] pnfs_submit: filelayout data server cache andros
2010-08-13 21:31 ` [PATCH 15/50] pnfs_submit: filelayout deviceid cache andros
2010-08-13 21:31 ` [PATCH 16/50] pnfs_submit: generic getdeviceinfo andros
2010-08-13 21:31 ` [PATCH 17/50] pnfs_submit: filelayout getdeviceinfo andros
2010-08-13 21:31 ` [PATCH 18/50] pnfs-submit: change stateid to be a union andros
2010-08-13 21:31 ` [PATCH 19/50] pnfs_submit: layout header alloc,reference, and destroy andros
2010-08-13 21:31 ` [PATCH 20/50] pnfs_submit: filelayout alloc_layout and free_layout andros
2010-08-13 21:31 ` [PATCH 21/50] pnfs_submit: layout segment alloc, reference, destroy andros
2010-08-13 21:31 ` [PATCH 22/50] pnfs_submit: layoutget andros
2010-08-13 21:31 ` [PATCH 23/50] pnfs_submit: layout helper functions andros
2010-08-13 21:31 ` andros [this message]
2010-08-13 21:31 ` [PATCH 25/50] pnfs_submit: layoutcommit " andros
2010-08-13 21:31 ` [PATCH 26/50] pnfs_submit: layoutcommit andros
2010-08-13 21:31 ` [PATCH 27/50] pnfs_submit: layoutreturn helper functions andros
2010-08-13 21:31 ` [PATCH 28/50] pnfs_submit: layoutreturn andros
2010-08-13 21:31 ` [PATCH 29/50] pnfs_submit: add data server session to nfs4_setup_sequence andros
2010-08-13 21:31 ` [PATCH 30/50] pnfs_submit: update nfs4_async_handle_error for data server andros
2010-08-13 21:31 ` [PATCH 31/50] pnfs_submit: update state renewal for data servers andros
2010-08-13 21:31 ` [PATCH 32/50] pnfs_submit-pageio-helpers.patch andros
2010-08-13 21:31 ` [PATCH 33/50] pnfs_submit: associate layout segmennt with nfs_page andros
2010-08-13 21:31 ` [PATCH 34/50] pnfs_submit: filelayout policy operations andros
2010-08-13 21:31 ` [PATCH 35/50] pnfs_submit: filelayout i/o helpers andros
2010-08-13 21:31 ` [PATCH 36/50] pnfs_submit: generic read andros
2010-08-13 21:31 ` [PATCH 37/50] pnfs_submit: filelayout read andros
2010-08-13 21:31 ` [PATCH 38/50] pnfs_submit: generic write andros
2010-08-13 21:31 ` [PATCH 39/50] pnfs_submit: data server write with no getattr andros
2010-08-13 21:31 ` [PATCH 40/50] pnfs_submit: filelayout write andros
2010-08-13 21:31 ` [PATCH 41/50] pnfs_submit: signal layoutdriver commit andros
2010-08-13 21:31 ` [PATCH 42/50] pnfs_submit: generic commit andros
2010-08-13 21:31 ` [PATCH 43/50] pnfs_submit: data server commit with no getattr andros
2010-08-13 21:31 ` [PATCH 44/50] pnfs_submit: filelayout commit andros
2010-08-13 21:31 ` [PATCH 45/50] pnfs_submit: cb_layoutrecall andros
2010-08-13 21:31 ` [PATCH 46/50] pnfs_submit: increase NFS_MAX_FILE_IO_SIZE andros
2010-08-13 21:31 ` [PATCH 47/50] SQUASHME pnfs_post_submit: direct i/o andros
2010-08-13 21:32 ` [PATCH 48/50] SQUASHME pnfs_post_submit: layout type enum andros
2010-08-13 21:32 ` [PATCH 49/50] SQUASHME pnfs_post_submit: cb notify deviceid declarations andros
2010-08-13 21:32 ` [PATCH 50/50] SQUASHME pnfs_submit: remove this unused code andros
2010-08-19 20:25 ` Benny Halevy
2010-08-31 16:32 ` Boaz Harrosh
2010-08-31 15:52 ` [PATCH 48/50] SQUASHME pnfs_post_submit: layout type enum Boaz Harrosh
2010-08-18 20:31 ` [PATCH 11/50] pnfs_submit: set and unset pnfs layoutdriver modules Christoph Hellwig
2010-08-18 20:46 ` Benny Halevy
2010-08-19 9:43 ` Christoph Hellwig
2010-08-18 20:29 ` [PATCH 10/50] pnfs_submit: register unregister pnfs module Christoph Hellwig
2010-08-18 20:49 ` Benny Halevy
2010-08-18 20:28 ` [PATCH 09/50] pnfs_submit: introduce fs/nfs/pnfs.c Christoph Hellwig
2010-08-19 17:21 ` J. Bruce Fields
2010-08-18 20:27 ` [PATCH 06/50] pnfs_submit: introduce include/linux/nfs4_pnfs.h Christoph Hellwig
2010-08-18 20:48 ` William A. (Andy) Adamson
2010-08-18 20:50 ` Benny Halevy
2010-08-18 20:25 ` [PATCH 05/50] pnfs_submit: pnfs and nfslayoutdriver kconfig Christoph Hellwig
2010-08-18 21:09 ` Benny Halevy
2010-08-19 9:45 ` Christoph Hellwig
2010-08-20 22:13 ` [PATCH 04/50] nfsd: remove duplicate NFS4_STATEID_SIZE declaration J. Bruce Fields
2010-08-19 20:50 ` [PATCH 0/50] Squashed and re-organized pnfs-submit tree Benny Halevy
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1281735122-1496-25-git-send-email-andros@netapp.com \
--to=andros@netapp.com \
--cc=bhalevy@panasas.com \
--cc=linux-nfs@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).