support for vfat long filenames

support for vfat long filenames

[Ryan Lortie]

[-- Attachment #1: Type: text/plain, Size: 176 bytes --]

hello.

here's a first go at a patch to implement long filename support for vfat
in libvolume_id.

i'm not on the list, so please keep me cc:'d in on any discussion...

cheers

[-- Attachment #2: vfat-lfn.patch --]
[-- Type: text/x-patch, Size: 5442 bytes --]

--- a/extras/volume_id/lib/fat.c	2007-03-28 14:47:26.000000000 -0400
+++ b/extras/volume_id/lib/fat.c	2007-05-23 02:12:42.000000000 -0400
@@ -16,6 +16,7 @@
 #  include <config.h>
 #endif
 
+#include <assert.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
@@ -34,6 +35,16 @@
 #define FAT_ATTR_MASK			0x3f
 #define FAT_ENTRY_FREE			0xe5
 
+#define VFAT_LFN_SEQ_MASK		0x3f
+#define VFAT_LFN_SEQ_LAST		0x40
+#define VFAT_LFN_SEQ_MAX		20
+#define VFAT_LFN_CHARS_PER_ENTRY	(5 + 6 + 2)
+#define VFAT_LOWERCASE_NAME		0x10
+#define VFAT_LOWERCASE_EXT		0x08
+
+#define TRUE				1
+#define FALSE				0
+
 struct vfat_super_block {
 	uint8_t		boot_jump[3];
 	uint8_t		sysid[8];
@@ -88,9 +99,10 @@
 struct vfat_dir_entry {
 	uint8_t		name[11];
 	uint8_t		attr;
+	uint8_t		lowercase;
+	uint8_t		fine_time_creat;
 	uint16_t	time_creat;
 	uint16_t	date_creat;
-	uint16_t	time_acc;
 	uint16_t	date_acc;
 	uint16_t	cluster_high;
 	uint16_t	time_write;
@@ -99,6 +111,139 @@
 	uint32_t	size;
 } PACKED;
 
+
+struct vfat_lfn_entry {
+	uint8_t		seq;
+	uint16_t	name0[5];
+	uint8_t		attr;
+	uint8_t		reserved;
+	uint8_t		cksum;
+	uint16_t	name1[6];
+	uint16_t	cluster;
+	uint16_t	name2[2];
+} PACKED;
+
+/* output must be at least 3 times longer than input */
+static void
+utf8_from_ucs16 (uint8_t *utf8, uint16_t *ucs16)
+{
+	while (*ucs16) {
+		if (*ucs16 & 0xf800) {
+			/* three-byte encoding */
+			*utf8++ = 0xe0 | ((*ucs16 >> 12) & 0x0f);
+			*utf8++ = 0x80 | ((*ucs16 >>  6) & 0x3f);
+			*utf8++ = 0x80 | ((*ucs16 >>  0) & 0x3f);
+		} else if (*ucs16 & 0xff80) {
+			/* two-byte encoding */
+			*utf8++ = 0xc0 | ((*ucs16 >> 6) & 0x1f);
+			*utf8++ = 0x80 | ((*ucs16 >> 0) & 0x3f);
+		} else {
+			/* ascii */
+			*utf8++ = *ucs16;
+		}
+
+		ucs16++;
+	}
+}
+
+static uint8_t
+fat_lfn_checksum (const uint8_t name[11])
+{
+  uint8_t cksum = 0;
+  int i;
+
+  /* http://en.wikipedia.org/wiki/File_Allocation_Table */
+  for (i = 0; i < 11; i++)
+    cksum = ((cksum & 1) ? 0x80 : 0) + (cksum >> 1) + name[i];
+
+  return cksum;
+}
+
+static int
+fat_read_lfn (struct vfat_dir_entry *dir,
+	      struct vfat_dir_entry *entry,
+	      uint8_t *filename)
+{
+	uint16_t buffer[VFAT_LFN_SEQ_MAX*VFAT_LFN_CHARS_PER_ENTRY];
+	uint8_t expected_seq = 1;
+	uint8_t cksum;
+	int len = 0;
+
+	cksum = fat_lfn_checksum (entry->name);
+
+	while (--entry >= dir)
+	{
+		struct vfat_lfn_entry *lfn = (struct vfat_lfn_entry *) entry;
+		unsigned int i;
+
+		if (expected_seq > VFAT_LFN_SEQ_MAX)
+			break;
+
+		if ((lfn->attr & FAT_ATTR_MASK) != FAT_ATTR_LONG_NAME)
+			break;
+
+		if (lfn->cksum != cksum)
+			break;
+
+		if ((lfn->seq & VFAT_LFN_SEQ_MASK) != expected_seq++)
+			break;
+
+		if (lfn->cluster != 0)
+			break;
+
+		/* extra paranoia */
+		assert (((char *) &buffer[len]) - ((char *) &buffer[0]) +
+			sizeof lfn->name0 +
+			sizeof lfn->name1 +
+			sizeof lfn->name2 <= sizeof buffer);
+
+		for (i = 0; i < sizeof lfn->name0 / sizeof (uint16_t); i++)
+			buffer[len++] = le16_to_cpu (lfn->name0[i]);
+		for (i = 0; i < sizeof lfn->name1 / sizeof (uint16_t); i++)
+			buffer[len++] = le16_to_cpu (lfn->name1[i]);
+		for (i = 0; i < sizeof lfn->name2 / sizeof (uint16_t); i++)
+			buffer[len++] = le16_to_cpu (lfn->name2[i]);
+
+		if (lfn->seq & VFAT_LFN_SEQ_LAST)
+		{
+			/* length is 255 including null terminator */
+			buffer[255] = '\0';
+
+			if (expected_seq <= VFAT_LFN_SEQ_MAX)
+				buffer[len] = '\0';
+
+			utf8_from_ucs16 (filename, buffer);
+
+			return TRUE;
+		}
+	}
+
+	return FALSE;
+}
+
+static uint8_t *
+fat_read_filename (struct vfat_dir_entry *dir, struct vfat_dir_entry *entry)
+{
+	static uint8_t filename[3*255];
+        int i;
+
+	/* check if maybe we have LFN entries */
+	if (fat_read_lfn (dir, entry, filename))
+		return filename;
+
+	/* else, read the normal 8.3 name */
+	for (i = 0; i < 11; i++) {
+		if (entry->lowercase & ((i < 8) ? VFAT_LOWERCASE_NAME :
+		    				  VFAT_LOWERCASE_EXT))
+			filename[i] = tolower (entry->name[i]);
+		else
+			filename[i] = entry->name[i];
+	}
+
+	return filename;
+}
+
+
 static uint8_t *get_attr_volume_id(struct vfat_dir_entry *dir, unsigned int count)
 {
 	unsigned int i;
@@ -124,7 +269,7 @@
 				continue;
 
 			dbg("found ATTR_VOLUME_ID id in root dir");
-			return dir[i].name;
+			return fat_read_filename (dir, &dir[i]);
 		}
 
 		dbg("skip dir entry");
@@ -282,8 +427,13 @@
 		return -1;
 
 	if (label != NULL && memcmp(label, "NO NAME    ", 11) != 0) {
-		volume_id_set_label_raw(id, label, 11);
-		volume_id_set_label_string(id, label, 11);
+		int length = strlen ((char *) label);
+
+		if (length > 64)
+			length = 64;
+
+		volume_id_set_label_raw(id, label, length);
+		volume_id_set_label_string(id, label, length);
 	} else if (memcmp(vs->type.fat.label, "NO NAME    ", 11) != 0) {
 		volume_id_set_label_raw(id, vs->type.fat.label, 11);
 		volume_id_set_label_string(id, vs->type.fat.label, 11);
@@ -360,8 +510,13 @@
 		return -1;
 
 	if (label != NULL && memcmp(label, "NO NAME    ", 11) != 0) {
-		volume_id_set_label_raw(id, label, 11);
-		volume_id_set_label_string(id, label, 11);
+		int length = strlen ((char *) label);
+
+		if (length > 64)
+			length = 64;
+
+		volume_id_set_label_raw(id, label, length);
+		volume_id_set_label_string(id, label, length);
 	} else if (memcmp(vs->type.fat32.label, "NO NAME    ", 11) != 0) {
 		volume_id_set_label_raw(id, vs->type.fat32.label, 11);
 		volume_id_set_label_string(id, vs->type.fat32.label, 11);

[-- Attachment #3: Type: text/plain, Size: 286 bytes --]

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

[-- Attachment #4: Type: text/plain, Size: 226 bytes --]

_______________________________________________
Linux-hotplug-devel mailing list  http://linux-hotplug.sourceforge.net
Linux-hotplug-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-hotplug-devel

Re: support for vfat long filenames

[Kay Sievers]

On 5/23/07, Ryan Lortie <desrt@desrt.ca> wrote:
> here's a first go at a patch to implement long filename support for vfat
> in libvolume_id.
>
> i'm not on the list, so please keep me cc:'d in on any discussion...

Oh, does that mean that fat can have labels with long filenames? What
tool can create these long labels, can you set such labels with
Windows now?

Can you use volume_id_set_unicode16() in utils.c, or is that different?

And please don't use assert(), just return no label if something goes wrong.

Thanks,
Kay

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/
_______________________________________________
Linux-hotplug-devel mailing list  http://linux-hotplug.sourceforge.net
Linux-hotplug-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-hotplug-devel

Re: support for vfat long filenames

[Ryan Lortie]

[-- Attachment #1: Type: text/plain, Size: 1154 bytes --]

On Wed, 2007-23-05 at 08:42 +0200, Kay Sievers wrote:
> On 5/23/07, Ryan Lortie <desrt@desrt.ca> wrote:
> > here's a first go at a patch to implement long filename support for vfat
> > in libvolume_id.
> >
> > i'm not on the list, so please keep me cc:'d in on any discussion...
> 
> Oh, does that mean that fat can have labels with long filenames? What
> tool can create these long labels, can you set such labels with
> Windows now?
i've been using the mtools package (specifically, "mlabel") to test my
work.  it supports long labels in exactly the way you would expect.

desrt@moonpix:~$ sudo mdir -i /dev/sdc2 ::
 Volume in drive : is My iPod has a very long name! (abbr=MY IPOD HAS)

i don't know about windows since i don't have access to a windows box,
but i assume it's true there too....?

> Can you use volume_id_set_unicode16() in utils.c, or is that different?
ah.  i didn't know about this.  done :)


> And please don't use assert(), just return no label if something goes wrong.
done -- this part is reworked a little to use memcpy anyway.

also fixed some small indentation and { placement issues.

thanks for the swift review.

cheers


[-- Attachment #2: vfat-lfn.patch --]
[-- Type: text/x-patch, Size: 4554 bytes --]

--- a/extras/volume_id/lib/fat.c	2007-03-28 14:47:26.000000000 -0400
+++ b/extras/volume_id/lib/fat.c	2007-05-23 10:16:50.000000000 -0400
@@ -34,6 +34,16 @@
 #define FAT_ATTR_MASK			0x3f
 #define FAT_ENTRY_FREE			0xe5
 
+#define VFAT_LFN_SEQ_MASK		0x3f
+#define VFAT_LFN_SEQ_LAST		0x40
+#define VFAT_LFN_SEQ_MAX		20
+#define VFAT_LFN_CHARS_PER_ENTRY	(5 + 6 + 2)
+#define VFAT_LOWERCASE_NAME		0x10
+#define VFAT_LOWERCASE_EXT		0x08
+
+#define TRUE				1
+#define FALSE				0
+
 struct vfat_super_block {
 	uint8_t		boot_jump[3];
 	uint8_t		sysid[8];
@@ -88,9 +98,10 @@
 struct vfat_dir_entry {
 	uint8_t		name[11];
 	uint8_t		attr;
+	uint8_t		lowercase;
+	uint8_t		fine_time_creat;
 	uint16_t	time_creat;
 	uint16_t	date_creat;
-	uint16_t	time_acc;
 	uint16_t	date_acc;
 	uint16_t	cluster_high;
 	uint16_t	time_write;
@@ -99,6 +110,110 @@
 	uint32_t	size;
 } PACKED;
 
+
+struct vfat_lfn_entry {
+	uint8_t		seq;
+	uint16_t	name0[5];
+	uint8_t		attr;
+	uint8_t		reserved;
+	uint8_t		cksum;
+	uint16_t	name1[6];
+	uint16_t	cluster;
+	uint16_t	name2[2];
+} PACKED;
+
+static uint8_t
+fat_lfn_checksum (const uint8_t name[11])
+{
+  uint8_t cksum = 0;
+  int i;
+
+  /* http://en.wikipedia.org/wiki/File_Allocation_Table */
+  for (i = 0; i < 11; i++)
+    cksum = ((cksum & 1) ? 0x80 : 0) + (cksum >> 1) + name[i];
+
+  return cksum;
+}
+
+static int
+fat_read_lfn (struct vfat_dir_entry *dir,
+	      struct vfat_dir_entry *entry,
+	      uint8_t *filename)
+{
+	uint8_t buffer[VFAT_LFN_SEQ_MAX*VFAT_LFN_CHARS_PER_ENTRY*2];
+	uint8_t expected_seq = 1;
+	uint8_t cksum;
+	int len = 0;
+
+	cksum = fat_lfn_checksum (entry->name);
+
+	while (--entry >= dir) {
+		struct vfat_lfn_entry *lfn = (struct vfat_lfn_entry *) entry;
+
+		if (expected_seq > VFAT_LFN_SEQ_MAX)
+			break;
+
+		if ((lfn->attr & FAT_ATTR_MASK) != FAT_ATTR_LONG_NAME)
+			break;
+
+		if (lfn->cksum != cksum)
+			break;
+
+		if ((lfn->seq & VFAT_LFN_SEQ_MASK) != expected_seq++)
+			break;
+
+		if (lfn->cluster != 0)
+			break;
+
+		/* extra paranoia -- should never happen */
+		if (len + sizeof lfn->name0 + sizeof sizeof lfn->name1 +
+		    sizeof lfn->name2 > sizeof buffer)
+			break;
+
+		memcpy (&buffer[len], lfn->name0, sizeof lfn->name0);
+		len += sizeof lfn->name0;
+		memcpy (&buffer[len], lfn->name1, sizeof lfn->name1);
+		len += sizeof lfn->name1;
+		memcpy (&buffer[len], lfn->name2, sizeof lfn->name2);
+		len += sizeof lfn->name2;
+
+		if (lfn->seq & VFAT_LFN_SEQ_LAST) {
+			if (len > 510)
+				len = 510;
+
+			volume_id_set_unicode16 ((char *) filename, 3*255,
+						 buffer, LE, len);
+
+			return TRUE;
+		}
+	}
+
+	return FALSE;
+}
+
+static uint8_t *
+fat_read_filename (struct vfat_dir_entry *dir, struct vfat_dir_entry *entry)
+{
+	static uint8_t filename[3*255];
+	int i;
+
+	/* check if maybe we have LFN entries */
+	if (fat_read_lfn (dir, entry, filename))
+		return filename;
+
+	/* else, read the normal 8.3 name */
+	for (i = 0; i < 11; i++) {
+		if (entry->lowercase & ((i < 8) ? VFAT_LOWERCASE_NAME :
+		    				  VFAT_LOWERCASE_EXT))
+			filename[i] = tolower (entry->name[i]);
+		else
+			filename[i] = entry->name[i];
+	}
+
+	return filename;
+}
+
+
 static uint8_t *get_attr_volume_id(struct vfat_dir_entry *dir, unsigned int count)
 {
 	unsigned int i;
@@ -124,7 +239,7 @@
 				continue;
 
 			dbg("found ATTR_VOLUME_ID id in root dir");
-			return dir[i].name;
+			return fat_read_filename (dir, &dir[i]);
 		}
 
 		dbg("skip dir entry");
@@ -282,8 +397,13 @@
 		return -1;
 
 	if (label != NULL && memcmp(label, "NO NAME    ", 11) != 0) {
-		volume_id_set_label_raw(id, label, 11);
-		volume_id_set_label_string(id, label, 11);
+		int length = strlen ((char *) label);
+
+		if (length > 64)
+			length = 64;
+
+		volume_id_set_label_raw(id, label, length);
+		volume_id_set_label_string(id, label, length);
 	} else if (memcmp(vs->type.fat.label, "NO NAME    ", 11) != 0) {
 		volume_id_set_label_raw(id, vs->type.fat.label, 11);
 		volume_id_set_label_string(id, vs->type.fat.label, 11);
@@ -360,8 +480,13 @@
 		return -1;
 
 	if (label != NULL && memcmp(label, "NO NAME    ", 11) != 0) {
-		volume_id_set_label_raw(id, label, 11);
-		volume_id_set_label_string(id, label, 11);
+		int length = strlen ((char *) label);
+
+		if (length > 64)
+			length = 64;
+
+		volume_id_set_label_raw(id, label, length);
+		volume_id_set_label_string(id, label, length);
 	} else if (memcmp(vs->type.fat32.label, "NO NAME    ", 11) != 0) {
 		volume_id_set_label_raw(id, vs->type.fat32.label, 11);
 		volume_id_set_label_string(id, vs->type.fat32.label, 11);

[-- Attachment #3: Type: text/plain, Size: 286 bytes --]

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

[-- Attachment #4: Type: text/plain, Size: 226 bytes --]

_______________________________________________
Linux-hotplug-devel mailing list  http://linux-hotplug.sourceforge.net
Linux-hotplug-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-hotplug-devel

Re: support for vfat long filenames

[Kay Sievers]

On Wed, 2007-05-23 at 10:18 -0400, Ryan Lortie wrote: 
> On Wed, 2007-23-05 at 08:42 +0200, Kay Sievers wrote:
> > On 5/23/07, Ryan Lortie <desrt@desrt.ca> wrote:
> > > here's a first go at a patch to implement long filename support for vfat
> > > in libvolume_id.
> > >
> > > i'm not on the list, so please keep me cc:'d in on any discussion...
> > 
> > Oh, does that mean that fat can have labels with long filenames? What
> > tool can create these long labels, can you set such labels with
> > Windows now?
> i've been using the mtools package (specifically, "mlabel") to test my
> work.  it supports long labels in exactly the way you would expect.

Ah nice. Unfortunately, seems it doesn't handle UTF8 correctly. It
encodes all multibyte chars to individual 16bit values and writes these
to the disk.

> desrt@moonpix:~$ sudo mdir -i /dev/sdc2 ::
>  Volume in drive : is My iPod has a very long name! (abbr=MY IPOD HAS)
> 
> i don't know about windows since i don't have access to a windows box,
> but i assume it's true there too....?

I tried it, both Windows XP and MacOS can't read or set the long
labels. :)

> > Can you use volume_id_set_unicode16() in utils.c, or is that different?
> ah.  i didn't know about this.  done :)
> 
> 
> > And please don't use assert(), just return no label if something goes wrong.
> done -- this part is reworked a little to use memcpy anyway.
> 
> also fixed some small indentation and { placement issues.

I've committed it with a few small changes. We better don't use
statically allocated strings, it may cause problems in threaded apps.
Please check, if it still works for you. :)

Thanks,
Kay


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/
_______________________________________________
Linux-hotplug-devel mailing list  http://linux-hotplug.sourceforge.net
Linux-hotplug-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-hotplug-devel

Re: support for vfat long filenames

[Ryan Lortie]

On Fri, 2007-25-05 at 04:04 +0200, Kay Sievers wrote:
> Ah nice. Unfortunately, seems it doesn't handle UTF8 correctly. It
> encodes all multibyte chars to individual 16bit values and writes these
> to the disk.
One day, this bug will be closed......

http://bugs.freedesktop.org/show_bug.cgi?id\x10363


> I tried it, both Windows XP and MacOS can't read or set the long
> labels. :)
Ah well.  No reason for us to be limited. :)

> I've committed it with a few small changes. We better don't use
> statically allocated strings, it may cause problems in threaded apps.
> Please check, if it still works for you. :)
Eep.  Sorry about this.  It didn't occur to me that applications outside
of vol_id might use this library (which is a silly assumption, of
course).

Can you please send me the patch as you committed it so I can test it
out?

Thanks for the (very) quick action on this patch.

Cheers



-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/
_______________________________________________
Linux-hotplug-devel mailing list  http://linux-hotplug.sourceforge.net
Linux-hotplug-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-hotplug-devel

Re: support for vfat long filenames

[Kay Sievers]

On Thu, 2007-05-24 at 22:15 -0400, Ryan Lortie wrote:
> On Fri, 2007-25-05 at 04:04 +0200, Kay Sievers wrote:
> > Ah nice. Unfortunately, seems it doesn't handle UTF8 correctly. It
> > encodes all multibyte chars to individual 16bit values and writes these
> > to the disk.
> One day, this bug will be closed......
> 
> http://bugs.freedesktop.org/show_bug.cgi?id\x10363

Yes, that will be nice.

> > I tried it, both Windows XP and MacOS can't read or set the long
> > labels. :)
> Ah well.  No reason for us to be limited. :)

Yeah, that's what I thought too. :)

> > I've committed it with a few small changes. We better don't use
> > statically allocated strings, it may cause problems in threaded apps.
> > Please check, if it still works for you. :)
> Eep.  Sorry about this.  It didn't occur to me that applications outside
> of vol_id might use this library (which is a silly assumption, of
> course).
> 
> Can you please send me the patch as you committed it so I can test it
> out?

It's in the udev git tree:
  http://git.kernel.org/?p=linux/hotplug/udev.git;a=commitdiff;h35b8aace33a2018c947381d32c33bcebe6cbe37

You can clone the whole tree, or just download the
patch (and the commit before your patch).

Thanks,
Kay


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/
_______________________________________________
Linux-hotplug-devel mailing list  http://linux-hotplug.sourceforge.net
Linux-hotplug-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-hotplug-devel