[PATCH] Breaking ext2 file size limit of 2TB

[PATCH] Breaking ext2 file size limit of 2TB

[Goldwyn Rodrigues]

Hi,

I have made a patch to enable file creation greater than 2TB. I tested it using sparse files and it works good.

Working:
The file size limit of the ext3 filesystem is limited to 2TB because of i_blocks, a variable which stores the number of 512 blocks in the inode. i_blocks is a 32 which limits the number it can hold. The patch makes use of l_i_reserved1 field to keep the higher order bits of i_blocks.

This has been developed and tested on kernel version 2.6.5 using sparse files.

Advantages: 
1. Patch is compatible with the existing filesystem and does not need re-formatting of the device.

Disadvantages: 

1. The patch uses l_i_reserved1 field to keep higher order 32-bits of i_blocks. This means the patch cannot be used with HURD filesystems, because it is occupied with a translator field.
2. Changes in fs.h, which is not really required.

I have also developed a patch to take the limits till 8TB, but there are lot of changes in that, including the memory copy of the inode structure. Moreover, it breaks with usual filesystems utils like fsck.

Feedback welcome.

Thanks,

-- 
Goldwyn :o)


diff -Nru linux-2.6.5-orig/fs/ext3/inode.c linux-2.6.5-4TB/fs/ext3/inode.c
--- linux-2.6.5-orig/fs/ext3/inode.c	2004-04-04 09:07:36.000000000 +0530
+++ linux-2.6.5-4TB/fs/ext3/inode.c	2004-06-23 12:30:41.000000000 +0530
@@ -355,7 +355,7 @@
  */
 
 static int ext3_block_to_path(struct inode *inode,
-			long i_block, int offsets[4], int *boundary)
+			sector_t i_block, int offsets[4], int *boundary)
 {
 	int ptrs = EXT3_ADDR_PER_BLOCK(inode->i_sb);
 	int ptrs_bits = EXT3_ADDR_PER_BLOCK_BITS(inode->i_sb);
@@ -906,7 +906,7 @@
  * `handle' can be NULL if create is zero
  */
 struct buffer_head *ext3_getblk(handle_t *handle, struct inode * inode,
-				long block, int create, int * errp)
+				sector_t block, int create, int * errp)
 {
 	struct buffer_head dummy;
 	int fatal = 0, err;
@@ -956,10 +956,10 @@
 }
 
 struct buffer_head *ext3_bread(handle_t *handle, struct inode * inode,
-			       int block, int create, int *err)
+			       sector_t block, int create, int *err)
 {
 	struct buffer_head * bh;
-	int prev_blocks;
+	sector_t prev_blocks;
 
 	prev_blocks = inode->i_blocks;
 
@@ -2126,7 +2126,7 @@
 	Indirect *partial;
 	int nr = 0;
 	int n;
-	long last_block;
+	sector_t last_block;
 	unsigned blocksize = inode->i_sb->s_blocksize;
 	struct page *page;
 
@@ -2516,6 +2516,8 @@
 					 * (for stat), not the fs block
 					 * size */  
 	inode->i_blocks = le32_to_cpu(raw_inode->i_blocks);
+	inode->i_blocks |= ((__u64)le32_to_cpu(raw_inode->i_blocks_high)) << 32;
+
 	ei->i_flags = le32_to_cpu(raw_inode->i_flags);
 #ifdef EXT3_FRAGMENTS
 	ei->i_faddr = le32_to_cpu(raw_inode->i_faddr);
@@ -2542,6 +2544,7 @@
 	 */
 	for (block = 0; block < EXT3_N_BLOCKS; block++)
 		ei->i_data[block] = raw_inode->i_block[block];
+	
 	INIT_LIST_HEAD(&ei->i_orphan);
 
 	if (S_ISREG(inode->i_mode)) {
@@ -2628,6 +2631,9 @@
 	raw_inode->i_ctime = cpu_to_le32(inode->i_ctime.tv_sec);
 	raw_inode->i_mtime = cpu_to_le32(inode->i_mtime.tv_sec);
 	raw_inode->i_blocks = cpu_to_le32(inode->i_blocks);
+	raw_inode->i_blocks_high = cpu_to_le32(inode->i_blocks >> 32);
+
+
 	raw_inode->i_dtime = cpu_to_le32(ei->i_dtime);
 	raw_inode->i_flags = cpu_to_le32(ei->i_flags);
 #ifdef EXT3_FRAGMENTS
diff -Nru linux-2.6.5-orig/fs/ext3/super.c linux-2.6.5-4TB/fs/ext3/super.c
--- linux-2.6.5-orig/fs/ext3/super.c	2004-04-04 09:08:14.000000000 +0530
+++ linux-2.6.5-4TB/fs/ext3/super.c	2004-06-23 12:32:01.000000000 +0530
@@ -1003,9 +1003,9 @@
 	res += 1LL << (bits-2);
 	res += 1LL << (2*(bits-2));
 	res += 1LL << (3*(bits-2));
-	res <<= bits;
-	if (res > (512LL << 32) - (1 << bits))
-		res = (512LL << 32) - (1 << bits);
+	/* Since another block is added, we add the same number again */
+	res += 1LL << (3*(bits-2));
+	res <<=bits;
 	return res;
 }
 
diff -Nru linux-2.6.5-orig/fs/Kconfig linux-2.6.5-4TB/fs/Kconfig
--- linux-2.6.5-orig/fs/Kconfig	2004-04-04 09:07:23.000000000 +0530
+++ linux-2.6.5-4TB/fs/Kconfig	2004-06-23 12:19:00.000000000 +0530
@@ -114,6 +114,21 @@
 	  of your root partition (the one containing the directory /) cannot
 	  be compiled as a module, and so this may be dangerous.
 
+config EXT3_LARGE_FILE_SUPPORT
+	bool "Large File (>2TB) Support (EXPERIMENTAL)"
+	depends on EXT3_FS
+	depends on LBD
+	default n
+	help
+	  Ext3 filesystem currently has a limit of 2TB. This experimental
+	  release extends this limit to 8TB by using the reserved fields
+	  in the inode. Thus, this feature can be used under Linux only.
+	  This feature is compatible with existing EXT3 filesystem.
+
+	  If unsure say N.
+		
+
+
 config EXT3_FS_XATTR
 	bool "Ext3 extended attributes"
 	depends on EXT3_FS
diff -Nru linux-2.6.5-orig/include/linux/ext3_fs.h linux-2.6.5-4TB/include/linux/ext3_fs.h
--- linux-2.6.5-orig/include/linux/ext3_fs.h	2004-04-04 09:07:23.000000000 +0530
+++ linux-2.6.5-4TB/include/linux/ext3_fs.h	2004-06-23 12:35:26.000000000 +0530
@@ -278,6 +278,10 @@
 #define i_gid_high	osd2.linux2.l_i_gid_high
 #define i_reserved2	osd2.linux2.l_i_reserved2
 
+#ifdef CONFIG_EXT3_LARGE_FILE_SUPPORT
+#define i_blocks_high		osd1.linux1.l_i_reserved1
+#endif
+
 #elif defined(__GNU__)
 
 #define i_translator	osd1.hurd1.h_i_translator
@@ -718,8 +722,8 @@
 
 /* inode.c */
 extern int ext3_forget(handle_t *, int, struct inode *, struct buffer_head *, int);
-extern struct buffer_head * ext3_getblk (handle_t *, struct inode *, long, int, int *);
-extern struct buffer_head * ext3_bread (handle_t *, struct inode *, int, int, int *);
+extern struct buffer_head * ext3_getblk (handle_t *, struct inode *, sector_t, int, int *);
+extern struct buffer_head * ext3_bread (handle_t *, struct inode *, sector_t, int, int *);
 
 extern void ext3_read_inode (struct inode *);
 extern void ext3_write_inode (struct inode *, int);
diff -Nru linux-2.6.5-orig/include/linux/fs.h linux-2.6.5-4TB/include/linux/fs.h
--- linux-2.6.5-orig/include/linux/fs.h	2004-04-04 09:06:52.000000000 +0530
+++ linux-2.6.5-4TB/include/linux/fs.h	2004-06-23 12:18:43.000000000 +0530
@@ -393,7 +393,11 @@
 	unsigned int		i_blkbits;
 	unsigned long		i_blksize;
 	unsigned long		i_version;
+#if !defined(CONFIG_EXT3_LARGE_FILE_SUPPORT) || defined(CONFIG_64BIT)
 	unsigned long		i_blocks;
+#else
+	unsigned long long	i_blocks;
+#endif /* CONFIG_EXT3_LARGE_FILE_SUPPORT */
 	unsigned short          i_bytes;
 	spinlock_t		i_lock;	/* i_blocks, i_bytes, maybe i_size */
 	struct semaphore	i_sem;

Re: [PATCH] Breaking ext2 file size limit of 2TB

[Jan-Benedict Glaw]

[-- Attachment #1: Type: text/plain, Size: 1570 bytes --]

On Fri, 2004-06-25 18:34:06 +0530, Goldwyn Rodrigues <goldwyn_r@myrealbox.com>
wrote in message <1088168646.d642871cgoldwyn_r@myrealbox.com>:
> Hi,
> 
> I have made a patch to enable file creation greater than 2TB. I
> tested it using sparse files and it works good.

Generally, a good idea, but...

> Advantages: 
> 1. Patch is compatible with the existing filesystem and does not
> need re-formatting of the device.

You're using a reserved field; how do you mean "compatible" in this
situation? Think of a filesystem with real files > 2TB. How will an
unpatched ext3fs driver handle those files? You'll only see the <2TB
content, right?

May an unpatched version under any circumstances clear the high-order
bits of the newly introduced 64bit integer, just because it doesn't know
to preserve this reserved field's value?

> Disadvantages: 
> 
> 1. The patch uses l_i_reserved1 field to keep higher order 32-bits of
> i_blocks. This means the patch cannot be used with HURD filesystems,
> because it is occupied with a translator field.

Being unfamiliar eith ext3's internals, are there other
reserved/free-for-future-use fields that don't clash with the HURD?

Are you proposing a patch like this for ext2, too?

MfG, JBG

-- 
   Jan-Benedict Glaw       jbglaw@lug-owl.de    . +49-172-7608481
   "Eine Freie Meinung in  einem Freien Kopf    | Gegen Zensur | Gegen Krieg
    fuer einen Freien Staat voll Freier Bürger" | im Internet! |   im Irak!
   ret = do_actions((curr | FREE_SPEECH) & ~(NEW_COPYRIGHT_LAW | DRM | TCPA));

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH] Breaking ext2 file size limit of 2TB

[Andreas Dilger]

[-- Attachment #1: Type: text/plain, Size: 3885 bytes --]

On Jun 25, 2004  18:34 +0530, Goldwyn Rodrigues wrote:
> I have made a patch to enable file creation greater than 2TB. I tested it
> using sparse files and it works good.
> 
> Working:
> The file size limit of the ext3 filesystem is limited to 2TB because of
> i_blocks, a variable which stores the number of 512 blocks in the inode.
> i_blocks is a 32 which limits the number it can hold. The patch makes
> use of l_i_reserved1 field to keep the higher order bits of i_blocks.

Do you have a real demand for doing this?  Given that block devices are
limited to 16TB on 32-bit architectures (page size * long), and ext3
files themselves are limited to 4TB+ (i386 page size again) because of
the triple-indirect block limit this isn't much of a win until we go to
something like extents.  Are you using a non-i386 architecture?

If we started using larger blocksizes for systems that have larger than
4kB pages (i.e. not i386) this would become an issue.  At some point
having giant files w/o extents is pointless (performance is too bad),
so we could also put the high blocks count in as part of the extent data
(e.g. i_blocks[14]) since the format would be gratuitously incompatible
anyways.

> @@ -1003,9 +1003,9 @@
>  	res += 1LL << (bits-2);
>  	res += 1LL << (2*(bits-2));
>  	res += 1LL << (3*(bits-2));
> -	res <<= bits;
> -	if (res > (512LL << 32) - (1 << bits))
> -		res = (512LL << 32) - (1 << bits);
> +	/* Since another block is added, we add the same number again */
> +	res += 1LL << (3*(bits-2));
> +	res <<=bits;

This is incorrect.  All that this change does is remove the extra
"res > (512LL << 32) - (1 << bits)" limit.  Even that could be removed
for sparse files without any of these changes if we wanted to check
at block allocation time whether we would overflow the i_blocks limit.

>  struct buffer_head *ext3_bread(handle_t *handle, struct inode * inode,
> -                              int block, int create, int *err)
> +                              sector_t block, int create, int *err)
> {
>         struct buffer_head * bh;
> -       int prev_blocks;
> +       sector_t prev_blocks;

This is a good fix regardless (at least change it to long from int).

> This has been developed and tested on kernel version 2.6.5 using sparse files.

That isn't really a test of anything, since a sparse file will not use 
more than 2^32 blocks.

> #define i_blocks_high		osd1.linux1.l_i_reserved1

If we really wanted to avoid being incompatible with Hurd (I personally
don't care about that, but someone who knows more should comment on how
badly this will screw things for it) we could use one of the other fields in
the inode like m_i_frag + m_i_fsize, or i_faddr as none of them is actually
used.  We also only really need 24 bits of this word before we hit the
64-bit byte i_size limit so we may as well be prudent and mask off the high
byte for later use.

Does anyone know if Hurd actually use both i_translator (i_reserved1) and
i_mode_high field (i_pad1)?

In any case, we need to wrap this with some sort of COMPAT flag in the
superblock, and probably a per-inode flag as well, so we know to trust
this value.

> --- linux-2.6.5-orig/include/linux/fs.h	2004-04-04 09:06:52.000000000 +0530
> +++ linux-2.6.5-4TB/include/linux/fs.h	2004-06-23 12:18:43.000000000 +0530
> @@ -393,7 +393,11 @@
>  	unsigned int		i_blkbits;
>  	unsigned long		i_blksize;
>  	unsigned long		i_version;
> +#if !defined(CONFIG_EXT3_LARGE_FILE_SUPPORT) || defined(CONFIG_64BIT)
>  	unsigned long		i_blocks;
> +#else
> +	unsigned long long	i_blocks;
> +#endif /* CONFIG_EXT3_LARGE_FILE_SUPPORT */

Why not just declare this as sector_t?

Cheers, Andreas
--
Andreas Dilger
http://sourceforge.net/projects/ext2resize/
http://members.shaw.ca/adilger/             http://members.shaw.ca/golinux/


[-- Attachment #2: Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH] Breaking ext2 file size limit of 2TB

[Goldwyn Rodrigues]

> Do you have a real demand for doing this?  Given that block devices are
> limited to 16TB on 32-bit architectures (page size * long), and ext3
> files themselves are limited to 4TB+ (i386 page size again) because of
> the triple-indirect block limit this isn't much of a win until we go to
> something like extents.  Are you using a non-i386 architecture?


As for the demad, I am not sure. But I have heard of enterprise level guys crying for bigger files, and thus trying to move their entire enterprise linux solution to different filesystems such as GFS. I don't know the architecture they are using.

I am using i386 architecture. As mentioned in my previous mail, I went ahead and made a few more changes to add an extra triple indirect field by replacing l_i_reserved2 to push the limits to 8TB but filesystem utilities like fsck din't like the change and changed the file size of the sparse file back to 4TB. If we really want to push it to extents it would require a radical change, which I am not sure if the community would accept. However, I am ready with it and could post the same.


> If we started using larger blocksizes for systems that have larger than
> 4kB pages (i.e. not i386) this would become an issue.  At some point
> having giant files w/o extents is pointless (performance is too bad),
> so we could also put the high blocks count in as part of the extent data
> (e.g. i_blocks[14]) since the format would be gratuitously incompatible
> anyways.

I din't quite understand this point. Do you mean to say that we keep such data elsewhere if required, and then read such data only for large systems. As in, do another block read?


> 
>>@@ -1003,9 +1003,9 @@
>> 	res += 1LL << (bits-2);
>> 	res += 1LL << (2*(bits-2));
>> 	res += 1LL << (3*(bits-2));
>>-	res <<= bits;
>>-	if (res > (512LL << 32) - (1 << bits))
>>-		res = (512LL << 32) - (1 << bits);
>>+	/* Since another block is added, we add the same number again */
>>+	res += 1LL << (3*(bits-2));
>>+	res <<=bits;
> 
> 
> This is incorrect.  All that this change does is remove the extra
> "res > (512LL << 32) - (1 << bits)" limit.  Even that could be removed
> for sparse files without any of these changes if we wanted to check
> at block allocation time whether we would overflow the i_blocks limit.
> 


Oops. Sorry, I mixed it with my 8TB code. The actual patch should look like:

diff -Nru /usr/src/linux-2.6.5-orig/fs/ext3/super.c /usr/src/linux-2.6.5-4TB/fs/ext3/super.c
--- /usr/src/linux-2.6.5-orig/fs/ext3/super.c   2004-04-04 09:08:14.000000000 +0530
+++ /usr/src/linux-2.6.5-4TB/fs/ext3/super.c    2004-06-26 10:39:05.000000000 +0530
@@ -1003,9 +1003,7 @@
        res += 1LL << (bits-2);
        res += 1LL << (2*(bits-2));
        res += 1LL << (3*(bits-2));
-       res <<= bits;
-       if (res > (512LL << 32) - (1 << bits))
-               res = (512LL << 32) - (1 << bits);
        return res;
 }




>>{
>>        struct buffer_head * bh;
>>-       int prev_blocks;
>>+       sector_t prev_blocks;
> 
> 
> This is a good fix regardless (at least change it to long from int).

long carries only 32 bits on i386 arch so I had to change it sector_t.

> 
> That isn't really a test of anything, since a sparse file will not use 
> more than 2^32 blocks.

I agree with you, but i386 is all I have for now. :)


>>#define i_blocks_high		osd1.linux1.l_i_reserved1
> 
> 
> If we really wanted to avoid being incompatible with Hurd (I personally
> don't care about that, but someone who knows more should comment on how
> badly this will screw things for it) we could use one of the other
> fields in
> the inode like m_i_frag + m_i_fsize, or i_faddr as none of them is
> actually
> used.  We also only really need 24 bits of this word before we hit the
> 64-bit byte i_size limit so we may as well be prudent and mask off the
> high
> byte for later use.

Its a good idea, if we can pick one 16bit field and another 8bit field. If fragments are not really used we could use l_i_frag and i_pad1 from the union. Please don't use variables wrt to masix in this context, it confused me.

But I need to know for sure that these variables are not used at all.


> In any case, we need to wrap this with some sort of COMPAT flag in the
> superblock, and probably a per-inode flag as well, so we know to trust
> this value.

Thanks, will work on it.

> 
>>@@ -393,7 +393,11 @@
>> 	unsigned int		i_blkbits;
>> 	unsigned long		i_blksize;
>> 	unsigned long		i_version;
>>+#if !defined(CONFIG_EXT3_LARGE_FILE_SUPPORT) || defined(CONFIG_64BIT)
>> 	unsigned long		i_blocks;
>>+#else
>>+	unsigned long long	i_blocks;
>>+#endif /* CONFIG_EXT3_LARGE_FILE_SUPPORT */
> 
> 
> Why not just declare this as sector_t?
> 

Okay. Will do that. Thanks again.
I will work on the suggested changes till then.

-- 
Goldwyn :o)

Re: [PATCH] Breaking ext2 file size limit of 2TB

[Goldwyn Rodrigues]

> You're using a reserved field; how do you mean "compatible" in this
> situation? Think of a filesystem with real files > 2TB. How will an
> unpatched ext3fs driver handle those files? You'll only see the <2TB
> content, right?


When I say compatible, I mean the the filesystem need not be formatted. All you need to do patch the kernel, and re-coompile the module/kernel, and remove the old module and insert the new one (if you are using ext3 as a module). 

Currently there is a function in ext3_max_size() which limits the file size to 2TB because of the number of blocks, namely i_blocks. 


> May an unpatched version under any circumstances clear the high-order
> bits of the newly introduced 64bit integer, just because it doesn't know
> to preserve this reserved field's value?

With an unpatched version you would not be able to create a file greater than 2TB at all and considers that all files are below 2TB.


> Being unfamiliar eith ext3's internals, are there other
> reserved/free-for-future-use fields that don't clash with the HURD?

Andreas has proposed a few fields but I want to make sure that those fields as well are not used.


> Are you proposing a patch like this for ext2, too?
> 

Once approved, it can be used for ext2 as well. Converting the current patch for ext2 would be childs play. ext2 and ext3 use almost the same structures (actually from my point of view, exactly the same but am afraid to use the word "exactly").

-- 
Goldwyn :o)

Re: [PATCH] Breaking ext2 file size limit of 2TB

[Jan-Benedict Glaw]

[-- Attachment #1: Type: text/plain, Size: 3151 bytes --]

On Sat, 2004-06-26 11:41:58 +0530, Goldwyn Rodrigues <goldwyn_r@myrealbox.com>
wrote in message <1088230318.9825981cgoldwyn_r@myrealbox.com>:
> > You're using a reserved field; how do you mean "compatible" in this
> > situation? Think of a filesystem with real files > 2TB. How will an
> > unpatched ext3fs driver handle those files? You'll only see the <2TB
> > content, right?
> 
> When I say compatible, I mean the the filesystem need not be formatted.
> All you need to do patch the kernel, and re-coompile the module/kernel,
> and remove the old module and insert the new one (if you are using ext3
> as a module).
> 
> Currently there is a function in ext3_max_size() which limits the file
> size to 2TB because of the number of blocks, namely i_blocks. 

So it's an incompatible extension, which tells me that you'll need to
set another incompatibility flag in case there's any >2TB file (like
ext3 does that when the filesystem's journal contains data).

> > May an unpatched version under any circumstances clear the high-order
> > bits of the newly introduced 64bit integer, just because it doesn't know
> > to preserve this reserved field's value?
> 
> With an unpatched version you would not be able to create a file
> greater than 2TB at all and considers that all files are below 2TB.

...but consider what's happening in the case you're first running a
patched kernel, create some >2TB files, then start an unpatched kernel
and work with those (formerly >2TB) files.

First of all, the files will look like being %= 2TB (albeit that, I
think with any >2TB file, you'd need to set an incompatibility flag so
that an unpatched ext3fs actually *refuses* to mount any FS containing a
file >2TB). What happens if you rename() such a file? What happens if
you use a pre-2TB ext2fs with it?

Sounds like an incompatible extension, so you need to mark it as one:)

> > Being unfamiliar eith ext3's internals, are there other
> > reserved/free-for-future-use fields that don't clash with the HURD?
> 
> Andreas has proposed a few fields but I want to make sure that those
> fields as well are not used.

If you were "offered" several chooses, why did you take this one, which
is used by the HURD? Were other possibilities already used for other
purposes?

> > Are you proposing a patch like this for ext2, too?
> 
> Once approved, it can be used for ext2 as well. Converting the
> current patch for ext2 would be childs play. ext2 and ext3 use
> almost the same structures (actually from my point of view, exactly
> the same but am afraid to use the word "exactly").

They're "compatible" as long as there's no data in the journal; with
data in the journal, ext3 marks itself as being incompatible ext2. This
is what >2TB filesize support probably needs to do, too.

MfG, JBG

-- 
   Jan-Benedict Glaw       jbglaw@lug-owl.de    . +49-172-7608481
   "Eine Freie Meinung in  einem Freien Kopf    | Gegen Zensur | Gegen Krieg
    fuer einen Freien Staat voll Freier Bürger" | im Internet! |   im Irak!
   ret = do_actions((curr | FREE_SPEECH) & ~(NEW_COPYRIGHT_LAW | DRM | TCPA));

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH] Breaking ext2 file size limit of 2TB

[Goldwyn Rodrigues]

> With an unpatched version you would not be able to create a file
>> greater than 2TB at all and considers that all files are below 2TB.
>
>
> ...but consider what's happening in the case you're first running a
> patched kernel, create some >2TB files, then start an unpatched kernel
> and work with those (formerly >2TB) files.
>
> First of all, the files will look like being %= 2TB (albeit that, I
> think with any >2TB file, you'd need to set an incompatibility flag so
> that an unpatched ext3fs actually *refuses* to mount any FS containing a
> file >2TB). What happens if you rename() such a file? What happens if
> you use a pre-2TB ext2fs with it?
>
> Sounds like an incompatible extension, so you need to mark it as one:)


I am working on the COMPAT fields now. Most probably, I will be able to give Read only access. I have to study more on this.

>> Andreas has proposed a few fields but I want to make sure that those
>> fields as well are not used.
>
>
>
> If you were "offered" several chooses, why did you take this one, which
> is used by the HURD? Were other possibilities already used for other
> purposes?


I got the mail just sometime back. I am working on it. The fields are
the ones for fragmented address. I do not know why they are placed there
and am trying to find if they are used in any other code. However, they
would still overlap with other OS. The fields proposed are:

__u32    i_faddr; used to keep fragmented addresses. I did not see any
reference to this data field in the linux code.
__u8 l_i_frag: fragment number.. conflicts with similar field in HURD
__u8 l_i_fsize: Fragment size.. conflicts with similar field in HURD
__u16 ipad1: unused. conflicts with i_mode_high in HURD

If anyone knows if these fields are used, in the Linux kernel source, or
any other operating system, let me know.


> They're "compatible" as long as there's no data in the journal; with
> data in the journal, ext3 marks itself as being incompatible ext2. This
> is what >2TB filesize support probably needs to do, too.
>

By "converting", I meant re-coding the patch for ext2. I consider the
ext2 and ext3 filesystems different, as they were meant to be. As far as
data structure which I am concerned with, for breaking the limits, ext2
and ext3 use the same inode data structures.

Thanks,

-- 
Goldwyn :o)

Re: [PATCH] Breaking ext2 file size limit of 2TB

[Jan-Benedict Glaw]

[-- Attachment #1: Type: text/plain, Size: 911 bytes --]

On Mon, 2004-06-28 12:42:01 +0530, Goldwyn Rodrigues <goldwyn_r@myrealbox.com>
wrote in message <1088406721.9804291cgoldwyn_r@myrealbox.com>:
> > With an unpatched version you would not be able to create a file
> >> greater than 2TB at all and considers that all files are below 2TB.
> > Sounds like an incompatible extension, so you need to mark it as one:)
> 
> I am working on the COMPAT fields now. Most probably, I will be able to give Read only access. I have to study more on this.

Just check if there's any code path that could clear your high-bits of
the block counter.

MfG, JBG

-- 
   Jan-Benedict Glaw       jbglaw@lug-owl.de    . +49-172-7608481
   "Eine Freie Meinung in  einem Freien Kopf    | Gegen Zensur | Gegen Krieg
    fuer einen Freien Staat voll Freier Bürger" | im Internet! |   im Irak!
   ret = do_actions((curr | FREE_SPEECH) & ~(NEW_COPYRIGHT_LAW | DRM | TCPA));

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH] Breaking ext2 file size limit of 2TB

[Andreas Dilger]

[-- Attachment #1: Type: text/plain, Size: 2319 bytes --]

On Jun 26, 2004  11:39 +0530, Goldwyn Rodrigues wrote:
> I am using i386 architecture.
> 
> > If we started using larger blocksizes for systems that have larger than
> > 4kB pages (i.e. not i386) this would become an issue.  At some point
> > having giant files w/o extents is pointless (performance is too bad),
> > so we could also put the high blocks count in as part of the extent data
> > (e.g. i_blocks[14]) since the format would be gratuitously incompatible
> > anyways.
> 
> I din't quite understand this point. Do you mean to say that we keep such
> data elsewhere if required, and then read such data only for large systems.
> As in, do another block read?

No, my point was that on i386 systems there is also a limit at 4TB so
your patch will only serve to double the maximum file size on such
systems.  On non-i386 systems this patch makes somewhat more sense,
but even so it is very inefficient to have enormous files allocating
a billion blocks or even more at larger block sizes.

On i386 we are limited to 4kB pages and this means after we fill the
triple indirect block we can't grow any larger (4096 / 4) ^ 3 * 4096 =
4TB.  However, on ia64 and other 64-bit platforms that have larger page
sizes we can use larger blocks for the filesystem (up to 64kB) and this
increases the triple indirect limit a lot (65536 / 4) ^ 3 * 65536 = 256PB
= 58 bits, at the sake of no longer being mountable on i386 machines.

I think that to have "useful" growth in file sizes we need to take a
hard look at the extents code already written by Alex.  Even with that
we need to be able to store a blocks count > 2^32 so this work isn't
incompatible with that, but if it takes 30 minutes to unlink a giant
file with a billion blocks I don't think just removing the i_blocks
limit will have helped us very much by itself.

Yes, the extent code is incompatible with the current layout (as is this
change), but if we add at least read support (write support separately
selectable if there is some objection to the incompatible change) for
extents to 2.6 then it can become a full-fledged member of ext3 in the
2.8 kernel.

Cheers, Andreas
--
Andreas Dilger
http://sourceforge.net/projects/ext2resize/
http://members.shaw.ca/adilger/             http://members.shaw.ca/golinux/


[-- Attachment #2: Type: application/pgp-signature, Size: 189 bytes --]