[PATCH] xfsprogs/io: add getdents command

[PATCH] xfsprogs/io: add getdents command

[Brian Foster]

getdents reads the directory entries from an open directory from
the provided offset (or 0 if not specified). On completion,
getdents prints summary information regarding the number of
operations and bytes transferred. Options are available to specify
the starting offset, buffer size and verbose mode to dump directory
entry information from the buffer.

Signed-off-by: Brian Foster <bfoster@redhat.com>
---

Hi all,

This is something I hacked together for testing something for gluster. Thoughts
appreciated!

Brian

 io/Makefile   |   4 ++
 io/getdents.c | 175 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 io/init.c     |   1 +
 io/io.h       |   1 +
 4 files changed, 181 insertions(+)
 create mode 100644 io/getdents.c

diff --git a/io/Makefile b/io/Makefile
index 50edf91..cfd1b11 100644
--- a/io/Makefile
+++ b/io/Makefile
@@ -80,6 +80,10 @@ ifeq ($(HAVE_PREADV),yes)
 LCFLAGS += -DHAVE_PREADV -DHAVE_PWRITEV
 endif
 
+ifeq ($(PKG_PLATFORM),linux)
+CFILES += getdents.c
+endif
+
 default: depend $(LTCOMMAND)
 
 include $(BUILDRULES)
diff --git a/io/getdents.c b/io/getdents.c
new file mode 100644
index 0000000..2b167f2
--- /dev/null
+++ b/io/getdents.c
@@ -0,0 +1,175 @@
+/*
+ * Copyright (c) 2013 Red Hat, Inc.
+ * All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it would be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write the Free Software Foundation,
+ * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+#include <xfs/xfs.h>
+#include <xfs/command.h>
+#include <xfs/input.h>
+#include "init.h"
+#include "io.h"
+
+#include <sys/syscall.h>
+
+static struct cmdinfo getdents_cmd;
+
+static char *gd_buf;
+static size_t gd_bufsz;
+
+struct linux_dirent {
+	unsigned long	d_ino;     /* Inode number */
+	unsigned long	d_off;     /* Offset to next linux_dirent */
+	unsigned short	d_reclen;  /* Length of this linux_dirent */
+	char		d_name[];  /* Filename (null-terminated) */
+};
+
+static void
+dump_dir_buffer(
+	struct linux_dirent *dirp,
+	long long offset,
+	long long length)
+{
+	while (length > 0) {
+		printf("%08llx: 0x%lx (%s)\n",
+			offset, dirp->d_ino, dirp->d_name);
+
+		offset = dirp->d_off;
+		length -= dirp->d_reclen;
+
+		dirp = (struct linux_dirent *) ((char *) dirp + dirp->d_reclen);
+	}
+}
+
+static int
+do_getdents(char *buf, unsigned int len)
+{
+	return syscall(SYS_getdents, file->fd, buf, len);
+}
+
+static int
+read_directory(
+	long long offset,
+	long long length,
+	int dump,
+	long long *total)
+{
+	int bytes;
+	int count = 0;
+
+	offset = lseek(file->fd, offset, SEEK_SET);
+	if (offset < 0)
+		perror("lseek");
+
+	*total = 0;
+	while (1) {
+		bytes = do_getdents(gd_buf, gd_bufsz);
+		if (!bytes)
+			break;
+		if (bytes < 0)
+			perror("getdents");
+
+		if (dump)
+			dump_dir_buffer((struct linux_dirent *) gd_buf, offset, bytes);
+
+		*total += bytes;
+		count++;
+
+		/*
+		 * Keep filling the buffer until we've read at least as much
+		 * data as requested.
+		 */
+		if (length > 0 && *total >= length)
+			break;
+	}
+
+	return count;
+}
+
+static int
+getdents_f(
+	int argc,
+	char **argv)
+{
+	int cnt;
+	long long total;
+	int c;
+	size_t fsblocksize, fssectsize;
+	struct timeval t1, t2;
+	char s1[64], s2[64], ts[64];
+	long long offset = -1;
+	long long length = -1;
+	int verbose = 0;
+
+	init_cvtnum(&fsblocksize, &fssectsize);
+	gd_bufsz = fsblocksize;
+
+	while ((c = getopt(argc, argv, "b:l:o:v")) != EOF) {
+		switch (c) {
+		case 'b':
+			gd_bufsz = cvtnum(fsblocksize, fssectsize, optarg);
+			break;
+		case 'l':
+			length = cvtnum(fsblocksize, fssectsize, optarg);
+			break;
+		case 'o':
+			offset = cvtnum(fsblocksize, fssectsize, optarg);
+			break;
+		case 'v':
+			verbose = 1;
+			break;
+		default:
+			return command_usage(&getdents_cmd);
+		}
+	}
+
+	gd_buf = malloc(gd_bufsz);
+	if (!gd_buf)
+		return -1;
+
+	if (offset == -1)
+		offset = lseek(file->fd, 0, SEEK_CUR);
+
+	gettimeofday(&t1, NULL);
+	cnt = read_directory(offset, length, verbose, &total);
+	gettimeofday(&t2, NULL);
+
+	t2 = tsub(t2, t1);
+	timestr(&t2, ts, sizeof(ts), 0);
+
+	cvtstr(total, s1, sizeof(s1));
+	cvtstr(tdiv(total, t2), s2, sizeof(s2));
+
+	printf(_("read %lld bytes from offset %lld\n"), total, offset);
+	printf(_("%s, %d ops, %s (%s/sec and %.4f ops/sec)\n"),
+		s1, cnt, ts, s2, tdiv(cnt, t2));
+
+	free(gd_buf);
+
+	return 0;
+}
+
+void
+getdents_init(void)
+{
+	getdents_cmd.name = "getdents";
+	getdents_cmd.cfunc = getdents_f;
+	getdents_cmd.argmax = 6;
+	getdents_cmd.flags = CMD_NOMAP_OK|CMD_FOREIGN_OK;
+	getdents_cmd.args = _("[-v][-b buffer size][-o offset][-l length]");
+	getdents_cmd.oneline = _("read directory entries");
+
+	add_command(&getdents_cmd);
+}
diff --git a/io/init.c b/io/init.c
index ca3055a..44cce44 100644
--- a/io/init.c
+++ b/io/init.c
@@ -60,6 +60,7 @@ init_commands(void)
 	file_init();
 	freeze_init();
 	fsync_init();
+	getdents_init();
 	getrusage_init();
 	help_init();
 	imap_init();
diff --git a/io/io.h b/io/io.h
index 91f0e3e..5658b13 100644
--- a/io/io.h
+++ b/io/io.h
@@ -94,6 +94,7 @@ extern void		bmap_init(void);
 extern void		file_init(void);
 extern void		freeze_init(void);
 extern void		fsync_init(void);
+extern void		getdents_init(void);
 extern void		getrusage_init(void);
 extern void		help_init(void);
 extern void		imap_init(void);
-- 
1.8.1.4

_______________________________________________
xfs mailing list
xfs@oss.sgi.com
http://oss.sgi.com/mailman/listinfo/xfs

Re: [PATCH] xfsprogs/io: add getdents command

[Zach Brown]

> +ifeq ($(PKG_PLATFORM),linux)
> +CFILES += getdents.c
> +endif

I'd make a real test for getdents() rather than tying it to Linux.  Just
copy what's done for sync_file_range :). 

> +struct linux_dirent {
> +	unsigned long	d_ino;     /* Inode number */
> +	unsigned long	d_off;     /* Offset to next linux_dirent */
> +	unsigned short	d_reclen;  /* Length of this linux_dirent */
> +	char		d_name[];  /* Filename (null-terminated) */
> +};

If we're only going to use one syscall interface, I'd use getdents64().

struct linux_dirent64 {
        u64             d_ino;
        s64             d_off;
        unsigned short  d_reclen;
        unsigned char   d_type;
        char            d_name[0];
};      

But it could also be useful to see the native 'long' interface for 32bit
apps, though, say if you're trying to debug ext4 htree d_off values
which are magically truncated for 32bit compat tasks.

> +static void
> +dump_dir_buffer(
> +	struct linux_dirent *dirp,
> +	long long offset,
> +	long long length)
> +{
> +	while (length > 0) {
> +		printf("%08llx: 0x%lx (%s)\n",
> +			offset, dirp->d_ino, dirp->d_name);
> +
> +		offset = dirp->d_off;
> +		length -= dirp->d_reclen;
> +
> +		dirp = (struct linux_dirent *) ((char *) dirp + dirp->d_reclen);
> +	}
> +}

You could also print out d_type.  In getdents64() it's a proper struct
member, in the bad old 'long' interface it's hidden in padding after the
null in d_name.

It'd be nice to see d_off for the last entry.  btrfs, for example, sets
it to CRAZY and that can be good to know.  In the past it has caused
32bit glibc to return -EOVERFLOW when reading entries from getdents64()
whose d_off overflowed 32bit off_t.

Interestingly, notice that with getdents() you don't know what the
offset of the first entry is.  You just know the offset that you started
reading from and the offset of the next entry.

- z
(*throws another bag of nickels in the readdir-design-disaster jar*)

_______________________________________________
xfs mailing list
xfs@oss.sgi.com
http://oss.sgi.com/mailman/listinfo/xfs

Re: [PATCH] xfsprogs/io: add getdents command

[Brian Foster]

On 07/15/2013 02:38 PM, Zach Brown wrote:
>> +ifeq ($(PKG_PLATFORM),linux)
>> +CFILES += getdents.c
>> +endif
> 
> I'd make a real test for getdents() rather than tying it to Linux.  Just
> copy what's done for sync_file_range :). 
> 

Having a look back, I think a reason why I avoided this was I didn't
have a libc interface for getdents. Perhaps there's still a way to test
that, but I'm also wondering if this would be more broadly useful if I
just converted it over to generic readdir(). Thoughts?

Brian

>> +struct linux_dirent {
>> +	unsigned long	d_ino;     /* Inode number */
>> +	unsigned long	d_off;     /* Offset to next linux_dirent */
>> +	unsigned short	d_reclen;  /* Length of this linux_dirent */
>> +	char		d_name[];  /* Filename (null-terminated) */
>> +};
> 
> If we're only going to use one syscall interface, I'd use getdents64().
> 
> struct linux_dirent64 {
>         u64             d_ino;
>         s64             d_off;
>         unsigned short  d_reclen;
>         unsigned char   d_type;
>         char            d_name[0];
> };      
> 
> But it could also be useful to see the native 'long' interface for 32bit
> apps, though, say if you're trying to debug ext4 htree d_off values
> which are magically truncated for 32bit compat tasks.
> 
>> +static void
>> +dump_dir_buffer(
>> +	struct linux_dirent *dirp,
>> +	long long offset,
>> +	long long length)
>> +{
>> +	while (length > 0) {
>> +		printf("%08llx: 0x%lx (%s)\n",
>> +			offset, dirp->d_ino, dirp->d_name);
>> +
>> +		offset = dirp->d_off;
>> +		length -= dirp->d_reclen;
>> +
>> +		dirp = (struct linux_dirent *) ((char *) dirp + dirp->d_reclen);
>> +	}
>> +}
> 
> You could also print out d_type.  In getdents64() it's a proper struct
> member, in the bad old 'long' interface it's hidden in padding after the
> null in d_name.
> 
> It'd be nice to see d_off for the last entry.  btrfs, for example, sets
> it to CRAZY and that can be good to know.  In the past it has caused
> 32bit glibc to return -EOVERFLOW when reading entries from getdents64()
> whose d_off overflowed 32bit off_t.
> 
> Interestingly, notice that with getdents() you don't know what the
> offset of the first entry is.  You just know the offset that you started
> reading from and the offset of the next entry.
> 
> - z
> (*throws another bag of nickels in the readdir-design-disaster jar*)
> 

_______________________________________________
xfs mailing list
xfs@oss.sgi.com
http://oss.sgi.com/mailman/listinfo/xfs

Re: [PATCH] xfsprogs/io: add getdents command

[Zach Brown]

On Tue, Jul 23, 2013 at 11:01:29AM -0400, Brian Foster wrote:
> On 07/15/2013 02:38 PM, Zach Brown wrote:
> >> +ifeq ($(PKG_PLATFORM),linux)
> >> +CFILES += getdents.c
> >> +endif
> > 
> > I'd make a real test for getdents() rather than tying it to Linux.  Just
> > copy what's done for sync_file_range :). 
> > 
> 
> Having a look back, I think a reason why I avoided this was I didn't
> have a libc interface for getdents.

Bleh, yeah.  I guess you could test for the syscall(__NR,) signature.

> Perhaps there's still a way to test that, but I'm also wondering if
> this would be more broadly useful if I just converted it over to
> generic readdir(). Thoughts?

That seems reasonable, given that's what apps actually use.  If tests
really wanted to inspect getdents specifically they can worry about it.

- z

_______________________________________________
xfs mailing list
xfs@oss.sgi.com
http://oss.sgi.com/mailman/listinfo/xfs