[RFC PATCH] erofs-utils: fsck: support fragments

[RFC PATCH] erofs-utils: fsck: support fragments

[Yue Hu]

From: Yue Hu <huyue2@coolpad.com>

Add compressed fragments support for fsck.erofs.

Signed-off-by: Yue Hu <huyue2@coolpad.com>
---
 fsck/main.c | 41 +++++++++++++++++++++++++++++++++++++++--
 lib/zmap.c  |  1 +
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/fsck/main.c b/fsck/main.c
index 2a9c501..9babc61 100644
--- a/fsck/main.c
+++ b/fsck/main.c
@@ -421,6 +421,31 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
 		if (!(map.m_flags & EROFS_MAP_MAPPED) || !fsckcfg.check_decomp)
 			continue;
 
+		if (map.m_flags & EROFS_MAP_FRAGMENT) {
+			struct erofs_inode packed_inode = {
+				.nid = sbi.packed_nid,
+			};
+
+			ret = erofs_read_inode_from_disk(&packed_inode);
+			if (ret) {
+				erofs_err("failed to read packed inode from disk");
+				goto out;
+			}
+
+			if (!buffer || map.m_llen > buffer_size) {
+				buffer_size = map.m_llen;
+				buffer = realloc(buffer, map.m_llen);
+				BUG_ON(!buffer);
+			}
+			ret = erofs_pread(&packed_inode, buffer, map.m_llen,
+					  inode->fragmentoff);
+			if (ret)
+				goto out;
+
+			compressed = true;	/* force using buffer */
+			goto write_outfd;
+		}
+
 		if (map.m_plen > raw_size) {
 			raw_size = map.m_plen;
 			raw = realloc(raw, raw_size);
@@ -476,6 +501,7 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
 			}
 		}
 
+write_outfd:
 		if (outfd >= 0 && write(outfd, compressed ? buffer : raw,
 					map.m_llen) < 0) {
 			erofs_err("I/O error occurred when verifying data chunk @ nid %llu",
@@ -486,8 +512,9 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
 	}
 
 	if (fsckcfg.print_comp_ratio) {
-		fsckcfg.logical_blocks += BLK_ROUND_UP(inode->i_size);
 		fsckcfg.physical_blocks += BLK_ROUND_UP(pchunk_len);
+		if (!erofs_is_packed_inode(inode))
+			fsckcfg.logical_blocks += BLK_ROUND_UP(inode->i_size);
 	}
 out:
 	if (raw)
@@ -732,6 +759,8 @@ static int erofsfsck_check_inode(erofs_nid_t pnid, erofs_nid_t nid)
 			ret = erofs_extract_dir(&inode);
 			break;
 		case S_IFREG:
+			if (erofs_is_packed_inode(&inode))
+				goto verify;
 			ret = erofs_extract_file(&inode);
 			break;
 		case S_IFLNK:
@@ -767,7 +796,7 @@ verify:
 		ret = erofs_iterate_dir(&ctx, true);
 	}
 
-	if (!ret)
+	if (!ret && !erofs_is_packed_inode(&inode))
 		erofsfsck_set_attributes(&inode, fsckcfg.extract_path);
 
 	if (ret == -ECANCELED)
@@ -822,6 +851,14 @@ int main(int argc, char **argv)
 		goto exit_put_super;
 	}
 
+	if (erofs_sb_has_fragments()) {
+		err = erofsfsck_check_inode(sbi.packed_nid, sbi.packed_nid);
+		if (err) {
+			erofs_err("failed to verify packed file");
+			goto exit_put_super;
+		}
+	}
+
 	err = erofsfsck_check_inode(sbi.root_nid, sbi.root_nid);
 	if (fsckcfg.corrupted) {
 		if (!fsckcfg.extract_path)
diff --git a/lib/zmap.c b/lib/zmap.c
index 41e0713..ca65038 100644
--- a/lib/zmap.c
+++ b/lib/zmap.c
@@ -639,6 +639,7 @@ static int z_erofs_do_map_blocks(struct erofs_inode *vi,
 		map->m_plen = vi->z_idata_size;
 	} else if (fragment && m.lcn == vi->z_tailextent_headlcn) {
 		map->m_flags |= EROFS_MAP_FRAGMENT;
+		map->m_plen = 0;
 	} else {
 		map->m_pa = blknr_to_addr(m.pblk);
 		err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
-- 
2.17.1

Re: [RFC PATCH] erofs-utils: fsck: support fragments

[Xiang Gao]

Hi Yue,

在 2022/12/24 17:43, Yue Hu 写道:
> From: Yue Hu <huyue2@coolpad.com>
> 
> Add compressed fragments support for fsck.erofs.
> 
> Signed-off-by: Yue Hu <huyue2@coolpad.com>
> ---
>   fsck/main.c | 41 +++++++++++++++++++++++++++++++++++++++--
>   lib/zmap.c  |  1 +
>   2 files changed, 40 insertions(+), 2 deletions(-)
> 
> diff --git a/fsck/main.c b/fsck/main.c
> index 2a9c501..9babc61 100644
> --- a/fsck/main.c
> +++ b/fsck/main.c
> @@ -421,6 +421,31 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
>   		if (!(map.m_flags & EROFS_MAP_MAPPED) || !fsckcfg.check_decomp)
>   			continue;
>   
> +		if (map.m_flags & EROFS_MAP_FRAGMENT) {
> +			struct erofs_inode packed_inode = {
> +				.nid = sbi.packed_nid,
> +			};

Sorry for late response.

a question... why don't we just rely on z_erofs_read_data()?


Thanks,
Gao Xiang

> +
> +			ret = erofs_read_inode_from_disk(&packed_inode);
> +			if (ret) {
> +				erofs_err("failed to read packed inode from disk");
> +				goto out;
> +			}
> +
> +			if (!buffer || map.m_llen > buffer_size) {
> +				buffer_size = map.m_llen;
> +				buffer = realloc(buffer, map.m_llen);
> +				BUG_ON(!buffer);
> +			}
> +			ret = erofs_pread(&packed_inode, buffer, map.m_llen,
> +					  inode->fragmentoff);
> +			if (ret)
> +				goto out;
> +
> +			compressed = true;	/* force using buffer */
> +			goto write_outfd;
> +		}
> +
>   		if (map.m_plen > raw_size) {
>   			raw_size = map.m_plen;
>   			raw = realloc(raw, raw_size);
> @@ -476,6 +501,7 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
>   			}
>   		}
>   
> +write_outfd:
>   		if (outfd >= 0 && write(outfd, compressed ? buffer : raw,
>   					map.m_llen) < 0) {
>   			erofs_err("I/O error occurred when verifying data chunk @ nid %llu",
> @@ -486,8 +512,9 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
>   	}
>   
>   	if (fsckcfg.print_comp_ratio) {
> -		fsckcfg.logical_blocks += BLK_ROUND_UP(inode->i_size);
>   		fsckcfg.physical_blocks += BLK_ROUND_UP(pchunk_len);
> +		if (!erofs_is_packed_inode(inode))
> +			fsckcfg.logical_blocks += BLK_ROUND_UP(inode->i_size);
>   	}
>   out:
>   	if (raw)
> @@ -732,6 +759,8 @@ static int erofsfsck_check_inode(erofs_nid_t pnid, erofs_nid_t nid)
>   			ret = erofs_extract_dir(&inode);
>   			break;
>   		case S_IFREG:
> +			if (erofs_is_packed_inode(&inode))
> +				goto verify;
>   			ret = erofs_extract_file(&inode);
>   			break;
>   		case S_IFLNK:
> @@ -767,7 +796,7 @@ verify:
>   		ret = erofs_iterate_dir(&ctx, true);
>   	}
>   
> -	if (!ret)
> +	if (!ret && !erofs_is_packed_inode(&inode))
>   		erofsfsck_set_attributes(&inode, fsckcfg.extract_path);
>   
>   	if (ret == -ECANCELED)
> @@ -822,6 +851,14 @@ int main(int argc, char **argv)
>   		goto exit_put_super;
>   	}
>   
> +	if (erofs_sb_has_fragments()) {
> +		err = erofsfsck_check_inode(sbi.packed_nid, sbi.packed_nid);
> +		if (err) {
> +			erofs_err("failed to verify packed file");
> +			goto exit_put_super;
> +		}
> +	}
> +
>   	err = erofsfsck_check_inode(sbi.root_nid, sbi.root_nid);
>   	if (fsckcfg.corrupted) {
>   		if (!fsckcfg.extract_path)
> diff --git a/lib/zmap.c b/lib/zmap.c
> index 41e0713..ca65038 100644
> --- a/lib/zmap.c
> +++ b/lib/zmap.c
> @@ -639,6 +639,7 @@ static int z_erofs_do_map_blocks(struct erofs_inode *vi,
>   		map->m_plen = vi->z_idata_size;
>   	} else if (fragment && m.lcn == vi->z_tailextent_headlcn) {
>   		map->m_flags |= EROFS_MAP_FRAGMENT;
> +		map->m_plen = 0;
>   	} else {
>   		map->m_pa = blknr_to_addr(m.pblk);
>   		err = z_erofs_get_extent_compressedlen(&m, initial_lcn);

Re: [RFC PATCH] erofs-utils: fsck: support fragments

[Yue Hu]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=GB18030, Size: 4058 bytes --]

Hi Xiang,

On Wed, 4 Jan 2023 10:48:40 +0800
Xiang Gao <hsiangkao@linux.alibaba.com> wrote:

> Hi Yue,
> 
> ÔÚ 2022/12/24 17:43, Yue Hu дµÀ:
> > From: Yue Hu <huyue2@coolpad.com>
> > 
> > Add compressed fragments support for fsck.erofs.
> > 
> > Signed-off-by: Yue Hu <huyue2@coolpad.com>
> > ---
> >   fsck/main.c | 41 +++++++++++++++++++++++++++++++++++++++--
> >   lib/zmap.c  |  1 +
> >   2 files changed, 40 insertions(+), 2 deletions(-)
> > 
> > diff --git a/fsck/main.c b/fsck/main.c
> > index 2a9c501..9babc61 100644
> > --- a/fsck/main.c
> > +++ b/fsck/main.c
> > @@ -421,6 +421,31 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
> >   		if (!(map.m_flags & EROFS_MAP_MAPPED) || !fsckcfg.check_decomp)
> >   			continue;
> >   
> > +		if (map.m_flags & EROFS_MAP_FRAGMENT) {
> > +			struct erofs_inode packed_inode = {
> > +				.nid = sbi.packed_nid,
> > +			};  
> 
> Sorry for late response.
> 
> a question... why don't we just rely on z_erofs_read_data()?

We may fall back to uncompressed mode for packed inode due to 'ENOSPC'.

> 
> 
> Thanks,
> Gao Xiang
> 
> > +
> > +			ret = erofs_read_inode_from_disk(&packed_inode);
> > +			if (ret) {
> > +				erofs_err("failed to read packed inode from disk");
> > +				goto out;
> > +			}
> > +
> > +			if (!buffer || map.m_llen > buffer_size) {
> > +				buffer_size = map.m_llen;
> > +				buffer = realloc(buffer, map.m_llen);
> > +				BUG_ON(!buffer);
> > +			}
> > +			ret = erofs_pread(&packed_inode, buffer, map.m_llen,
> > +					  inode->fragmentoff);
> > +			if (ret)
> > +				goto out;
> > +
> > +			compressed = true;	/* force using buffer */
> > +			goto write_outfd;
> > +		}
> > +
> >   		if (map.m_plen > raw_size) {
> >   			raw_size = map.m_plen;
> >   			raw = realloc(raw, raw_size);
> > @@ -476,6 +501,7 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
> >   			}
> >   		}
> >   
> > +write_outfd:
> >   		if (outfd >= 0 && write(outfd, compressed ? buffer : raw,
> >   					map.m_llen) < 0) {
> >   			erofs_err("I/O error occurred when verifying data chunk @ nid %llu",
> > @@ -486,8 +512,9 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
> >   	}
> >   
> >   	if (fsckcfg.print_comp_ratio) {
> > -		fsckcfg.logical_blocks += BLK_ROUND_UP(inode->i_size);
> >   		fsckcfg.physical_blocks += BLK_ROUND_UP(pchunk_len);
> > +		if (!erofs_is_packed_inode(inode))
> > +			fsckcfg.logical_blocks += BLK_ROUND_UP(inode->i_size);
> >   	}
> >   out:
> >   	if (raw)
> > @@ -732,6 +759,8 @@ static int erofsfsck_check_inode(erofs_nid_t pnid, erofs_nid_t nid)
> >   			ret = erofs_extract_dir(&inode);
> >   			break;
> >   		case S_IFREG:
> > +			if (erofs_is_packed_inode(&inode))
> > +				goto verify;
> >   			ret = erofs_extract_file(&inode);
> >   			break;
> >   		case S_IFLNK:
> > @@ -767,7 +796,7 @@ verify:
> >   		ret = erofs_iterate_dir(&ctx, true);
> >   	}
> >   
> > -	if (!ret)
> > +	if (!ret && !erofs_is_packed_inode(&inode))
> >   		erofsfsck_set_attributes(&inode, fsckcfg.extract_path);
> >   
> >   	if (ret == -ECANCELED)
> > @@ -822,6 +851,14 @@ int main(int argc, char **argv)
> >   		goto exit_put_super;
> >   	}
> >   
> > +	if (erofs_sb_has_fragments()) {
> > +		err = erofsfsck_check_inode(sbi.packed_nid, sbi.packed_nid);
> > +		if (err) {
> > +			erofs_err("failed to verify packed file");
> > +			goto exit_put_super;
> > +		}
> > +	}
> > +
> >   	err = erofsfsck_check_inode(sbi.root_nid, sbi.root_nid);
> >   	if (fsckcfg.corrupted) {
> >   		if (!fsckcfg.extract_path)
> > diff --git a/lib/zmap.c b/lib/zmap.c
> > index 41e0713..ca65038 100644
> > --- a/lib/zmap.c
> > +++ b/lib/zmap.c
> > @@ -639,6 +639,7 @@ static int z_erofs_do_map_blocks(struct erofs_inode *vi,
> >   		map->m_plen = vi->z_idata_size;
> >   	} else if (fragment && m.lcn == vi->z_tailextent_headlcn) {
> >   		map->m_flags |= EROFS_MAP_FRAGMENT;
> > +		map->m_plen = 0;
> >   	} else {
> >   		map->m_pa = blknr_to_addr(m.pblk);
> >   		err = z_erofs_get_extent_compressedlen(&m, initial_lcn);  

Re: [RFC PATCH] erofs-utils: fsck: support fragments

[Xiang Gao]

On 2023/1/4 11:24, Yue Hu wrote:
> Hi Xiang,
> 
> On Wed, 4 Jan 2023 10:48:40 +0800
> Xiang Gao <hsiangkao@linux.alibaba.com> wrote:
> 
>> Hi Yue,
>>
>> 在 2022/12/24 17:43, Yue Hu 写道:
>>> From: Yue Hu <huyue2@coolpad.com>
>>>
>>> Add compressed fragments support for fsck.erofs.
>>>
>>> Signed-off-by: Yue Hu <huyue2@coolpad.com>
>>> ---
>>>    fsck/main.c | 41 +++++++++++++++++++++++++++++++++++++++--
>>>    lib/zmap.c  |  1 +
>>>    2 files changed, 40 insertions(+), 2 deletions(-)
>>>
>>> diff --git a/fsck/main.c b/fsck/main.c
>>> index 2a9c501..9babc61 100644
>>> --- a/fsck/main.c
>>> +++ b/fsck/main.c
>>> @@ -421,6 +421,31 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
>>>    		if (!(map.m_flags & EROFS_MAP_MAPPED) || !fsckcfg.check_decomp)
>>>    			continue;
>>>    
>>> +		if (map.m_flags & EROFS_MAP_FRAGMENT) {
>>> +			struct erofs_inode packed_inode = {
>>> +				.nid = sbi.packed_nid,
>>> +			};
>>
>> Sorry for late response.
>>
>> a question... why don't we just rely on z_erofs_read_data()?
> 
> We may fall back to uncompressed mode for packed inode due to 'ENOSPC'.

I think we could just export parts of erofs_pread() to clean up the 
whole erofs_verify_inode_data()...

Thanks,
Gao Xiang

Re: [RFC PATCH] erofs-utils: fsck: support fragments

[Yue Hu]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=GB18030, Size: 1590 bytes --]

On Wed, 4 Jan 2023 11:32:36 +0800
Xiang Gao <hsiangkao@linux.alibaba.com> wrote:

> On 2023/1/4 11:24, Yue Hu wrote:
> > Hi Xiang,
> > 
> > On Wed, 4 Jan 2023 10:48:40 +0800
> > Xiang Gao <hsiangkao@linux.alibaba.com> wrote:
> >   
> >> Hi Yue,
> >>
> >> ÔÚ 2022/12/24 17:43, Yue Hu дµÀ:  
> >>> From: Yue Hu <huyue2@coolpad.com>
> >>>
> >>> Add compressed fragments support for fsck.erofs.
> >>>
> >>> Signed-off-by: Yue Hu <huyue2@coolpad.com>
> >>> ---
> >>>    fsck/main.c | 41 +++++++++++++++++++++++++++++++++++++++--
> >>>    lib/zmap.c  |  1 +
> >>>    2 files changed, 40 insertions(+), 2 deletions(-)
> >>>
> >>> diff --git a/fsck/main.c b/fsck/main.c
> >>> index 2a9c501..9babc61 100644
> >>> --- a/fsck/main.c
> >>> +++ b/fsck/main.c
> >>> @@ -421,6 +421,31 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
> >>>    		if (!(map.m_flags & EROFS_MAP_MAPPED) || !fsckcfg.check_decomp)
> >>>    			continue;
> >>>    
> >>> +		if (map.m_flags & EROFS_MAP_FRAGMENT) {
> >>> +			struct erofs_inode packed_inode = {
> >>> +				.nid = sbi.packed_nid,
> >>> +			};  
> >>
> >> Sorry for late response.
> >>
> >> a question... why don't we just rely on z_erofs_read_data()?  
> > 
> > We may fall back to uncompressed mode for packed inode due to 'ENOSPC'.  
> 
> I think we could just export parts of erofs_pread() to clean up the 
> whole erofs_verify_inode_data()...

What's the clean up referring to?

However, i think erofs_verify_inode_data() looks a little duplicate compared to erofs_read_raw_data/z_erofs_read_data().

> 
> Thanks,
> Gao Xiang
> 
> 

Re: [RFC PATCH] erofs-utils: fsck: support fragments

[Xiang Gao]

On 2023/1/4 15:26, Yue Hu wrote:

...

>>
>> I think we could just export parts of erofs_pread() to clean up the
>> whole erofs_verify_inode_data()...
> 
> What's the clean up referring to?
> 
> However, i think erofs_verify_inode_data() looks a little duplicate compared to erofs_read_raw_data/z_erofs_read_data().

We should reuse the main part (for example, by introducing an interface 
to accept mapping) of
erofs_read_raw_data() and z_erofs_read_data() to avoid duplicated code 
in erofs_verify_inode_data().

Thanks,
Gao Xiang

> 
>>
>> Thanks,
>> Gao Xiang
>>
>>

Re: [RFC PATCH] erofs-utils: fsck: support fragments

[Yue Hu]

On Wed, 4 Jan 2023 15:28:42 +0800
Xiang Gao <hsiangkao@linux.alibaba.com> wrote:

> On 2023/1/4 15:26, Yue Hu wrote:
> 
> ...
> 
> >>
> >> I think we could just export parts of erofs_pread() to clean up the
> >> whole erofs_verify_inode_data()...  
> > 
> > What's the clean up referring to?
> > 
> > However, i think erofs_verify_inode_data() looks a little duplicate compared to erofs_read_raw_data/z_erofs_read_data().  
> 
> We should reuse the main part (for example, by introducing an interface 
> to accept mapping) of
> erofs_read_raw_data() and z_erofs_read_data() to avoid duplicated code 
> in erofs_verify_inode_data().

Yes, it's. Let's do it.

> 
> Thanks,
> Gao Xiang
> 
> >   
> >>
> >> Thanks,
> >> Gao Xiang
> >>
> >>