* [RFC PATCH] erofs-utils: fsck: support fragments
@ 2022-12-24 9:43 Yue Hu
2023-01-04 2:48 ` Xiang Gao
0 siblings, 1 reply; 7+ messages in thread
From: Yue Hu @ 2022-12-24 9:43 UTC (permalink / raw)
To: linux-erofs; +Cc: Yue Hu, zhangwen
From: Yue Hu <huyue2@coolpad.com>
Add compressed fragments support for fsck.erofs.
Signed-off-by: Yue Hu <huyue2@coolpad.com>
---
fsck/main.c | 41 +++++++++++++++++++++++++++++++++++++++--
lib/zmap.c | 1 +
2 files changed, 40 insertions(+), 2 deletions(-)
diff --git a/fsck/main.c b/fsck/main.c
index 2a9c501..9babc61 100644
--- a/fsck/main.c
+++ b/fsck/main.c
@@ -421,6 +421,31 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
if (!(map.m_flags & EROFS_MAP_MAPPED) || !fsckcfg.check_decomp)
continue;
+ if (map.m_flags & EROFS_MAP_FRAGMENT) {
+ struct erofs_inode packed_inode = {
+ .nid = sbi.packed_nid,
+ };
+
+ ret = erofs_read_inode_from_disk(&packed_inode);
+ if (ret) {
+ erofs_err("failed to read packed inode from disk");
+ goto out;
+ }
+
+ if (!buffer || map.m_llen > buffer_size) {
+ buffer_size = map.m_llen;
+ buffer = realloc(buffer, map.m_llen);
+ BUG_ON(!buffer);
+ }
+ ret = erofs_pread(&packed_inode, buffer, map.m_llen,
+ inode->fragmentoff);
+ if (ret)
+ goto out;
+
+ compressed = true; /* force using buffer */
+ goto write_outfd;
+ }
+
if (map.m_plen > raw_size) {
raw_size = map.m_plen;
raw = realloc(raw, raw_size);
@@ -476,6 +501,7 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
}
}
+write_outfd:
if (outfd >= 0 && write(outfd, compressed ? buffer : raw,
map.m_llen) < 0) {
erofs_err("I/O error occurred when verifying data chunk @ nid %llu",
@@ -486,8 +512,9 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
}
if (fsckcfg.print_comp_ratio) {
- fsckcfg.logical_blocks += BLK_ROUND_UP(inode->i_size);
fsckcfg.physical_blocks += BLK_ROUND_UP(pchunk_len);
+ if (!erofs_is_packed_inode(inode))
+ fsckcfg.logical_blocks += BLK_ROUND_UP(inode->i_size);
}
out:
if (raw)
@@ -732,6 +759,8 @@ static int erofsfsck_check_inode(erofs_nid_t pnid, erofs_nid_t nid)
ret = erofs_extract_dir(&inode);
break;
case S_IFREG:
+ if (erofs_is_packed_inode(&inode))
+ goto verify;
ret = erofs_extract_file(&inode);
break;
case S_IFLNK:
@@ -767,7 +796,7 @@ verify:
ret = erofs_iterate_dir(&ctx, true);
}
- if (!ret)
+ if (!ret && !erofs_is_packed_inode(&inode))
erofsfsck_set_attributes(&inode, fsckcfg.extract_path);
if (ret == -ECANCELED)
@@ -822,6 +851,14 @@ int main(int argc, char **argv)
goto exit_put_super;
}
+ if (erofs_sb_has_fragments()) {
+ err = erofsfsck_check_inode(sbi.packed_nid, sbi.packed_nid);
+ if (err) {
+ erofs_err("failed to verify packed file");
+ goto exit_put_super;
+ }
+ }
+
err = erofsfsck_check_inode(sbi.root_nid, sbi.root_nid);
if (fsckcfg.corrupted) {
if (!fsckcfg.extract_path)
diff --git a/lib/zmap.c b/lib/zmap.c
index 41e0713..ca65038 100644
--- a/lib/zmap.c
+++ b/lib/zmap.c
@@ -639,6 +639,7 @@ static int z_erofs_do_map_blocks(struct erofs_inode *vi,
map->m_plen = vi->z_idata_size;
} else if (fragment && m.lcn == vi->z_tailextent_headlcn) {
map->m_flags |= EROFS_MAP_FRAGMENT;
+ map->m_plen = 0;
} else {
map->m_pa = blknr_to_addr(m.pblk);
err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
--
2.17.1
^ permalink raw reply related [flat|nested] 7+ messages in thread* Re: [RFC PATCH] erofs-utils: fsck: support fragments
2022-12-24 9:43 [RFC PATCH] erofs-utils: fsck: support fragments Yue Hu
@ 2023-01-04 2:48 ` Xiang Gao
2023-01-04 3:24 ` Yue Hu
0 siblings, 1 reply; 7+ messages in thread
From: Xiang Gao @ 2023-01-04 2:48 UTC (permalink / raw)
To: Yue Hu, linux-erofs; +Cc: Yue Hu, zhangwen
Hi Yue,
在 2022/12/24 17:43, Yue Hu 写道:
> From: Yue Hu <huyue2@coolpad.com>
>
> Add compressed fragments support for fsck.erofs.
>
> Signed-off-by: Yue Hu <huyue2@coolpad.com>
> ---
> fsck/main.c | 41 +++++++++++++++++++++++++++++++++++++++--
> lib/zmap.c | 1 +
> 2 files changed, 40 insertions(+), 2 deletions(-)
>
> diff --git a/fsck/main.c b/fsck/main.c
> index 2a9c501..9babc61 100644
> --- a/fsck/main.c
> +++ b/fsck/main.c
> @@ -421,6 +421,31 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
> if (!(map.m_flags & EROFS_MAP_MAPPED) || !fsckcfg.check_decomp)
> continue;
>
> + if (map.m_flags & EROFS_MAP_FRAGMENT) {
> + struct erofs_inode packed_inode = {
> + .nid = sbi.packed_nid,
> + };
Sorry for late response.
a question... why don't we just rely on z_erofs_read_data()?
Thanks,
Gao Xiang
> +
> + ret = erofs_read_inode_from_disk(&packed_inode);
> + if (ret) {
> + erofs_err("failed to read packed inode from disk");
> + goto out;
> + }
> +
> + if (!buffer || map.m_llen > buffer_size) {
> + buffer_size = map.m_llen;
> + buffer = realloc(buffer, map.m_llen);
> + BUG_ON(!buffer);
> + }
> + ret = erofs_pread(&packed_inode, buffer, map.m_llen,
> + inode->fragmentoff);
> + if (ret)
> + goto out;
> +
> + compressed = true; /* force using buffer */
> + goto write_outfd;
> + }
> +
> if (map.m_plen > raw_size) {
> raw_size = map.m_plen;
> raw = realloc(raw, raw_size);
> @@ -476,6 +501,7 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
> }
> }
>
> +write_outfd:
> if (outfd >= 0 && write(outfd, compressed ? buffer : raw,
> map.m_llen) < 0) {
> erofs_err("I/O error occurred when verifying data chunk @ nid %llu",
> @@ -486,8 +512,9 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
> }
>
> if (fsckcfg.print_comp_ratio) {
> - fsckcfg.logical_blocks += BLK_ROUND_UP(inode->i_size);
> fsckcfg.physical_blocks += BLK_ROUND_UP(pchunk_len);
> + if (!erofs_is_packed_inode(inode))
> + fsckcfg.logical_blocks += BLK_ROUND_UP(inode->i_size);
> }
> out:
> if (raw)
> @@ -732,6 +759,8 @@ static int erofsfsck_check_inode(erofs_nid_t pnid, erofs_nid_t nid)
> ret = erofs_extract_dir(&inode);
> break;
> case S_IFREG:
> + if (erofs_is_packed_inode(&inode))
> + goto verify;
> ret = erofs_extract_file(&inode);
> break;
> case S_IFLNK:
> @@ -767,7 +796,7 @@ verify:
> ret = erofs_iterate_dir(&ctx, true);
> }
>
> - if (!ret)
> + if (!ret && !erofs_is_packed_inode(&inode))
> erofsfsck_set_attributes(&inode, fsckcfg.extract_path);
>
> if (ret == -ECANCELED)
> @@ -822,6 +851,14 @@ int main(int argc, char **argv)
> goto exit_put_super;
> }
>
> + if (erofs_sb_has_fragments()) {
> + err = erofsfsck_check_inode(sbi.packed_nid, sbi.packed_nid);
> + if (err) {
> + erofs_err("failed to verify packed file");
> + goto exit_put_super;
> + }
> + }
> +
> err = erofsfsck_check_inode(sbi.root_nid, sbi.root_nid);
> if (fsckcfg.corrupted) {
> if (!fsckcfg.extract_path)
> diff --git a/lib/zmap.c b/lib/zmap.c
> index 41e0713..ca65038 100644
> --- a/lib/zmap.c
> +++ b/lib/zmap.c
> @@ -639,6 +639,7 @@ static int z_erofs_do_map_blocks(struct erofs_inode *vi,
> map->m_plen = vi->z_idata_size;
> } else if (fragment && m.lcn == vi->z_tailextent_headlcn) {
> map->m_flags |= EROFS_MAP_FRAGMENT;
> + map->m_plen = 0;
> } else {
> map->m_pa = blknr_to_addr(m.pblk);
> err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
^ permalink raw reply [flat|nested] 7+ messages in thread* Re: [RFC PATCH] erofs-utils: fsck: support fragments
2023-01-04 2:48 ` Xiang Gao
@ 2023-01-04 3:24 ` Yue Hu
2023-01-04 3:32 ` Xiang Gao
0 siblings, 1 reply; 7+ messages in thread
From: Yue Hu @ 2023-01-04 3:24 UTC (permalink / raw)
To: Xiang Gao; +Cc: Yue Hu, linux-erofs, zhangwen
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=GB18030, Size: 4058 bytes --]
Hi Xiang,
On Wed, 4 Jan 2023 10:48:40 +0800
Xiang Gao <hsiangkao@linux.alibaba.com> wrote:
> Hi Yue,
>
> ÔÚ 2022/12/24 17:43, Yue Hu дµÀ:
> > From: Yue Hu <huyue2@coolpad.com>
> >
> > Add compressed fragments support for fsck.erofs.
> >
> > Signed-off-by: Yue Hu <huyue2@coolpad.com>
> > ---
> > fsck/main.c | 41 +++++++++++++++++++++++++++++++++++++++--
> > lib/zmap.c | 1 +
> > 2 files changed, 40 insertions(+), 2 deletions(-)
> >
> > diff --git a/fsck/main.c b/fsck/main.c
> > index 2a9c501..9babc61 100644
> > --- a/fsck/main.c
> > +++ b/fsck/main.c
> > @@ -421,6 +421,31 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
> > if (!(map.m_flags & EROFS_MAP_MAPPED) || !fsckcfg.check_decomp)
> > continue;
> >
> > + if (map.m_flags & EROFS_MAP_FRAGMENT) {
> > + struct erofs_inode packed_inode = {
> > + .nid = sbi.packed_nid,
> > + };
>
> Sorry for late response.
>
> a question... why don't we just rely on z_erofs_read_data()?
We may fall back to uncompressed mode for packed inode due to 'ENOSPC'.
>
>
> Thanks,
> Gao Xiang
>
> > +
> > + ret = erofs_read_inode_from_disk(&packed_inode);
> > + if (ret) {
> > + erofs_err("failed to read packed inode from disk");
> > + goto out;
> > + }
> > +
> > + if (!buffer || map.m_llen > buffer_size) {
> > + buffer_size = map.m_llen;
> > + buffer = realloc(buffer, map.m_llen);
> > + BUG_ON(!buffer);
> > + }
> > + ret = erofs_pread(&packed_inode, buffer, map.m_llen,
> > + inode->fragmentoff);
> > + if (ret)
> > + goto out;
> > +
> > + compressed = true; /* force using buffer */
> > + goto write_outfd;
> > + }
> > +
> > if (map.m_plen > raw_size) {
> > raw_size = map.m_plen;
> > raw = realloc(raw, raw_size);
> > @@ -476,6 +501,7 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
> > }
> > }
> >
> > +write_outfd:
> > if (outfd >= 0 && write(outfd, compressed ? buffer : raw,
> > map.m_llen) < 0) {
> > erofs_err("I/O error occurred when verifying data chunk @ nid %llu",
> > @@ -486,8 +512,9 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
> > }
> >
> > if (fsckcfg.print_comp_ratio) {
> > - fsckcfg.logical_blocks += BLK_ROUND_UP(inode->i_size);
> > fsckcfg.physical_blocks += BLK_ROUND_UP(pchunk_len);
> > + if (!erofs_is_packed_inode(inode))
> > + fsckcfg.logical_blocks += BLK_ROUND_UP(inode->i_size);
> > }
> > out:
> > if (raw)
> > @@ -732,6 +759,8 @@ static int erofsfsck_check_inode(erofs_nid_t pnid, erofs_nid_t nid)
> > ret = erofs_extract_dir(&inode);
> > break;
> > case S_IFREG:
> > + if (erofs_is_packed_inode(&inode))
> > + goto verify;
> > ret = erofs_extract_file(&inode);
> > break;
> > case S_IFLNK:
> > @@ -767,7 +796,7 @@ verify:
> > ret = erofs_iterate_dir(&ctx, true);
> > }
> >
> > - if (!ret)
> > + if (!ret && !erofs_is_packed_inode(&inode))
> > erofsfsck_set_attributes(&inode, fsckcfg.extract_path);
> >
> > if (ret == -ECANCELED)
> > @@ -822,6 +851,14 @@ int main(int argc, char **argv)
> > goto exit_put_super;
> > }
> >
> > + if (erofs_sb_has_fragments()) {
> > + err = erofsfsck_check_inode(sbi.packed_nid, sbi.packed_nid);
> > + if (err) {
> > + erofs_err("failed to verify packed file");
> > + goto exit_put_super;
> > + }
> > + }
> > +
> > err = erofsfsck_check_inode(sbi.root_nid, sbi.root_nid);
> > if (fsckcfg.corrupted) {
> > if (!fsckcfg.extract_path)
> > diff --git a/lib/zmap.c b/lib/zmap.c
> > index 41e0713..ca65038 100644
> > --- a/lib/zmap.c
> > +++ b/lib/zmap.c
> > @@ -639,6 +639,7 @@ static int z_erofs_do_map_blocks(struct erofs_inode *vi,
> > map->m_plen = vi->z_idata_size;
> > } else if (fragment && m.lcn == vi->z_tailextent_headlcn) {
> > map->m_flags |= EROFS_MAP_FRAGMENT;
> > + map->m_plen = 0;
> > } else {
> > map->m_pa = blknr_to_addr(m.pblk);
> > err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
^ permalink raw reply [flat|nested] 7+ messages in thread* Re: [RFC PATCH] erofs-utils: fsck: support fragments
2023-01-04 3:24 ` Yue Hu
@ 2023-01-04 3:32 ` Xiang Gao
2023-01-04 7:26 ` Yue Hu
0 siblings, 1 reply; 7+ messages in thread
From: Xiang Gao @ 2023-01-04 3:32 UTC (permalink / raw)
To: Yue Hu; +Cc: Yue Hu, linux-erofs, zhangwen
On 2023/1/4 11:24, Yue Hu wrote:
> Hi Xiang,
>
> On Wed, 4 Jan 2023 10:48:40 +0800
> Xiang Gao <hsiangkao@linux.alibaba.com> wrote:
>
>> Hi Yue,
>>
>> 在 2022/12/24 17:43, Yue Hu 写道:
>>> From: Yue Hu <huyue2@coolpad.com>
>>>
>>> Add compressed fragments support for fsck.erofs.
>>>
>>> Signed-off-by: Yue Hu <huyue2@coolpad.com>
>>> ---
>>> fsck/main.c | 41 +++++++++++++++++++++++++++++++++++++++--
>>> lib/zmap.c | 1 +
>>> 2 files changed, 40 insertions(+), 2 deletions(-)
>>>
>>> diff --git a/fsck/main.c b/fsck/main.c
>>> index 2a9c501..9babc61 100644
>>> --- a/fsck/main.c
>>> +++ b/fsck/main.c
>>> @@ -421,6 +421,31 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
>>> if (!(map.m_flags & EROFS_MAP_MAPPED) || !fsckcfg.check_decomp)
>>> continue;
>>>
>>> + if (map.m_flags & EROFS_MAP_FRAGMENT) {
>>> + struct erofs_inode packed_inode = {
>>> + .nid = sbi.packed_nid,
>>> + };
>>
>> Sorry for late response.
>>
>> a question... why don't we just rely on z_erofs_read_data()?
>
> We may fall back to uncompressed mode for packed inode due to 'ENOSPC'.
I think we could just export parts of erofs_pread() to clean up the
whole erofs_verify_inode_data()...
Thanks,
Gao Xiang
^ permalink raw reply [flat|nested] 7+ messages in thread* Re: [RFC PATCH] erofs-utils: fsck: support fragments
2023-01-04 3:32 ` Xiang Gao
@ 2023-01-04 7:26 ` Yue Hu
2023-01-04 7:28 ` Xiang Gao
0 siblings, 1 reply; 7+ messages in thread
From: Yue Hu @ 2023-01-04 7:26 UTC (permalink / raw)
To: Xiang Gao; +Cc: Yue Hu, linux-erofs, zhangwen
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=GB18030, Size: 1590 bytes --]
On Wed, 4 Jan 2023 11:32:36 +0800
Xiang Gao <hsiangkao@linux.alibaba.com> wrote:
> On 2023/1/4 11:24, Yue Hu wrote:
> > Hi Xiang,
> >
> > On Wed, 4 Jan 2023 10:48:40 +0800
> > Xiang Gao <hsiangkao@linux.alibaba.com> wrote:
> >
> >> Hi Yue,
> >>
> >> ÔÚ 2022/12/24 17:43, Yue Hu дµÀ:
> >>> From: Yue Hu <huyue2@coolpad.com>
> >>>
> >>> Add compressed fragments support for fsck.erofs.
> >>>
> >>> Signed-off-by: Yue Hu <huyue2@coolpad.com>
> >>> ---
> >>> fsck/main.c | 41 +++++++++++++++++++++++++++++++++++++++--
> >>> lib/zmap.c | 1 +
> >>> 2 files changed, 40 insertions(+), 2 deletions(-)
> >>>
> >>> diff --git a/fsck/main.c b/fsck/main.c
> >>> index 2a9c501..9babc61 100644
> >>> --- a/fsck/main.c
> >>> +++ b/fsck/main.c
> >>> @@ -421,6 +421,31 @@ static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd)
> >>> if (!(map.m_flags & EROFS_MAP_MAPPED) || !fsckcfg.check_decomp)
> >>> continue;
> >>>
> >>> + if (map.m_flags & EROFS_MAP_FRAGMENT) {
> >>> + struct erofs_inode packed_inode = {
> >>> + .nid = sbi.packed_nid,
> >>> + };
> >>
> >> Sorry for late response.
> >>
> >> a question... why don't we just rely on z_erofs_read_data()?
> >
> > We may fall back to uncompressed mode for packed inode due to 'ENOSPC'.
>
> I think we could just export parts of erofs_pread() to clean up the
> whole erofs_verify_inode_data()...
What's the clean up referring to?
However, i think erofs_verify_inode_data() looks a little duplicate compared to erofs_read_raw_data/z_erofs_read_data().
>
> Thanks,
> Gao Xiang
>
>
^ permalink raw reply [flat|nested] 7+ messages in thread* Re: [RFC PATCH] erofs-utils: fsck: support fragments
2023-01-04 7:26 ` Yue Hu
@ 2023-01-04 7:28 ` Xiang Gao
2023-01-04 7:42 ` Yue Hu
0 siblings, 1 reply; 7+ messages in thread
From: Xiang Gao @ 2023-01-04 7:28 UTC (permalink / raw)
To: Yue Hu; +Cc: Yue Hu, linux-erofs, zhangwen
On 2023/1/4 15:26, Yue Hu wrote:
...
>>
>> I think we could just export parts of erofs_pread() to clean up the
>> whole erofs_verify_inode_data()...
>
> What's the clean up referring to?
>
> However, i think erofs_verify_inode_data() looks a little duplicate compared to erofs_read_raw_data/z_erofs_read_data().
We should reuse the main part (for example, by introducing an interface
to accept mapping) of
erofs_read_raw_data() and z_erofs_read_data() to avoid duplicated code
in erofs_verify_inode_data().
Thanks,
Gao Xiang
>
>>
>> Thanks,
>> Gao Xiang
>>
>>
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [RFC PATCH] erofs-utils: fsck: support fragments
2023-01-04 7:28 ` Xiang Gao
@ 2023-01-04 7:42 ` Yue Hu
0 siblings, 0 replies; 7+ messages in thread
From: Yue Hu @ 2023-01-04 7:42 UTC (permalink / raw)
To: Xiang Gao; +Cc: Yue Hu, linux-erofs, zhangwen
On Wed, 4 Jan 2023 15:28:42 +0800
Xiang Gao <hsiangkao@linux.alibaba.com> wrote:
> On 2023/1/4 15:26, Yue Hu wrote:
>
> ...
>
> >>
> >> I think we could just export parts of erofs_pread() to clean up the
> >> whole erofs_verify_inode_data()...
> >
> > What's the clean up referring to?
> >
> > However, i think erofs_verify_inode_data() looks a little duplicate compared to erofs_read_raw_data/z_erofs_read_data().
>
> We should reuse the main part (for example, by introducing an interface
> to accept mapping) of
> erofs_read_raw_data() and z_erofs_read_data() to avoid duplicated code
> in erofs_verify_inode_data().
Yes, it's. Let's do it.
>
> Thanks,
> Gao Xiang
>
> >
> >>
> >> Thanks,
> >> Gao Xiang
> >>
> >>
^ permalink raw reply [flat|nested] 7+ messages in thread
end of thread, other threads:[~2023-01-04 7:37 UTC | newest]
Thread overview: 7+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2022-12-24 9:43 [RFC PATCH] erofs-utils: fsck: support fragments Yue Hu
2023-01-04 2:48 ` Xiang Gao
2023-01-04 3:24 ` Yue Hu
2023-01-04 3:32 ` Xiang Gao
2023-01-04 7:26 ` Yue Hu
2023-01-04 7:28 ` Xiang Gao
2023-01-04 7:42 ` Yue Hu
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.