* [PATCH 01/13] erofs-utils: fix overriding of i_rdev for special device
2023-08-14 3:42 [PATCH 00/13] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
@ 2023-08-14 3:42 ` Jingbo Xu
2023-08-14 5:42 ` Gao Xiang
2023-08-14 3:42 ` [PATCH 02/13] erofs-utils: lib: add list_splice_tail() helper Jingbo Xu
` (11 subsequent siblings)
12 siblings, 1 reply; 23+ messages in thread
From: Jingbo Xu @ 2023-08-14 3:42 UTC (permalink / raw)
To: xiang, linux-erofs
In index mode for tarerofs, tarerofs_write_chunk_indexes() is still called
even for files with 0 i_size, and inode->u.chunkformat is initialized
accordingly. This will make the previously set inode->u.i_rdev be
overridden as u.chunkformat and u.i_rdev are reused in one union.
To tidy up the code of writing file, extract two helpers writing file in
index and non-index mode. Also fix the missing erofs_iput() when
tarerofs_write_chunkes() fails.
Fixes: 95d315fd7958 ("erofs-utils: introduce tarerofs")
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
lib/tar.c | 77 ++++++++++++++++++++++++++++++++-----------------------
1 file changed, 45 insertions(+), 32 deletions(-)
diff --git a/lib/tar.c b/lib/tar.c
index 54ee33f..42590d2 100644
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -359,6 +359,44 @@ void tarerofs_remove_inode(struct erofs_inode *inode)
--inode->i_parent->i_nlink;
}
+static int tarerofs_write_file_data(struct erofs_inode *inode,
+ struct erofs_tarfile *tar)
+{
+ unsigned int j, rem;
+ char buf[65536];
+
+ if (!inode->i_tmpfile) {
+ inode->i_tmpfile = tmpfile();
+ if (!inode->i_tmpfile)
+ return -ENOSPC;
+ }
+
+ for (j = inode->i_size; j; ) {
+ rem = min_t(unsigned int, sizeof(buf), j);
+
+ if (erofs_read_from_fd(tar->fd, buf, rem) != rem ||
+ fwrite(buf, rem, 1, inode->i_tmpfile) != 1)
+ return -EIO;
+ j -= rem;
+ }
+ fseek(inode->i_tmpfile, 0, SEEK_SET);
+ inode->with_tmpfile = true;
+ return 0;
+}
+
+static int tarerofs_write_file_index(struct erofs_inode *inode,
+ struct erofs_tarfile *tar, erofs_off_t data_offset)
+{
+ int ret;
+
+ ret = tarerofs_write_chunkes(inode, data_offset);
+ if (ret)
+ return ret;
+ if (erofs_lskip(tar->fd, inode->i_size))
+ return -EIO;
+ return 0;
+}
+
int tarerofs_parse_tar(struct erofs_inode *root, struct erofs_tarfile *tar)
{
char path[PATH_MAX];
@@ -672,41 +710,16 @@ new_inode:
inode->i_size = strlen(eh.link);
inode->i_link = malloc(inode->i_size + 1);
memcpy(inode->i_link, eh.link, inode->i_size + 1);
- } else if (tar->index_mode) {
- ret = tarerofs_write_chunkes(inode, data_offset);
- if (ret)
- goto out;
- if (erofs_lskip(tar->fd, inode->i_size)) {
+ } else if (inode->i_size) {
+ if (tar->index_mode)
+ ret = tarerofs_write_file_index(inode, tar,
+ data_offset);
+ else
+ ret = tarerofs_write_file_data(inode, tar);
+ if (ret) {
erofs_iput(inode);
- ret = -EIO;
goto out;
}
- } else {
- char buf[65536];
-
- if (!inode->i_tmpfile) {
- inode->i_tmpfile = tmpfile();
-
- if (!inode->i_tmpfile) {
- erofs_iput(inode);
- ret = -ENOSPC;
- goto out;
- }
- }
-
- for (j = inode->i_size; j; ) {
- rem = min_t(int, sizeof(buf), j);
-
- if (erofs_read_from_fd(tar->fd, buf, rem) != rem ||
- fwrite(buf, rem, 1, inode->i_tmpfile) != 1) {
- erofs_iput(inode);
- ret = -EIO;
- goto out;
- }
- j -= rem;
- }
- fseek(inode->i_tmpfile, 0, SEEK_SET);
- inode->with_tmpfile = true;
}
inode->i_nlink++;
ret = 0;
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 23+ messages in thread* Re: [PATCH 01/13] erofs-utils: fix overriding of i_rdev for special device
2023-08-14 3:42 ` [PATCH 01/13] erofs-utils: fix overriding of i_rdev for special device Jingbo Xu
@ 2023-08-14 5:42 ` Gao Xiang
0 siblings, 0 replies; 23+ messages in thread
From: Gao Xiang @ 2023-08-14 5:42 UTC (permalink / raw)
To: Jingbo Xu, xiang, linux-erofs
On 2023/8/14 11:42, Jingbo Xu wrote:
> In index mode for tarerofs, tarerofs_write_chunk_indexes() is still called
> even for files with 0 i_size, and inode->u.chunkformat is initialized
> accordingly. This will make the previously set inode->u.i_rdev be
> overridden as u.chunkformat and u.i_rdev are reused in one union.
>
> To tidy up the code of writing file, extract two helpers writing file in
> index and non-index mode. Also fix the missing erofs_iput() when
> tarerofs_write_chunkes() fails.
>
> Fixes: 95d315fd7958 ("erofs-utils: introduce tarerofs")
> Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
Reviewed-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Thanks,
Gao Xiang
^ permalink raw reply [flat|nested] 23+ messages in thread
* [PATCH 02/13] erofs-utils: lib: add list_splice_tail() helper
2023-08-14 3:42 [PATCH 00/13] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
2023-08-14 3:42 ` [PATCH 01/13] erofs-utils: fix overriding of i_rdev for special device Jingbo Xu
@ 2023-08-14 3:42 ` Jingbo Xu
2023-08-14 5:50 ` Gao Xiang
2023-08-14 3:42 ` [PATCH 03/13] erofs-utils: lib: read i_ino in erofs_read_inode_from_disk() Jingbo Xu
` (10 subsequent siblings)
12 siblings, 1 reply; 23+ messages in thread
From: Jingbo Xu @ 2023-08-14 3:42 UTC (permalink / raw)
To: xiang, linux-erofs
Add list_splice_tail() helper.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
include/erofs/list.h | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/include/erofs/list.h b/include/erofs/list.h
index 3f5da1a..d7a9fee 100644
--- a/include/erofs/list.h
+++ b/include/erofs/list.h
@@ -70,6 +70,26 @@ static inline int list_empty(struct list_head *head)
return head->next == head;
}
+static inline void __list_splice(struct list_head *list,
+ struct list_head *prev, struct list_head *next)
+{
+ struct list_head *first = list->next;
+ struct list_head *last = list->prev;
+
+ first->prev = prev;
+ prev->next = first;
+
+ last->next = next;
+ next->prev = last;
+}
+
+static inline void list_splice_tail(struct list_head *list,
+ struct list_head *head)
+{
+ if (!list_empty(list))
+ __list_splice(list, head->prev, head);
+}
+
#define list_entry(ptr, type, member) container_of(ptr, type, member)
#define list_first_entry(ptr, type, member) \
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 23+ messages in thread* [PATCH 03/13] erofs-utils: lib: read i_ino in erofs_read_inode_from_disk()
2023-08-14 3:42 [PATCH 00/13] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
2023-08-14 3:42 ` [PATCH 01/13] erofs-utils: fix overriding of i_rdev for special device Jingbo Xu
2023-08-14 3:42 ` [PATCH 02/13] erofs-utils: lib: add list_splice_tail() helper Jingbo Xu
@ 2023-08-14 3:42 ` Jingbo Xu
2023-08-14 6:03 ` Gao Xiang
2023-08-14 3:42 ` [PATCH 04/13] erofs-utils: lib: fix erofs_init_devices() in multidev mode Jingbo Xu
` (9 subsequent siblings)
12 siblings, 1 reply; 23+ messages in thread
From: Jingbo Xu @ 2023-08-14 3:42 UTC (permalink / raw)
To: xiang, linux-erofs
i_ino[0] is a unique inode serial number in the erofs filesystem where
the inode resides, and also serves as the on-disk inode number, while
i_ino[1] is a unique number identifying the source inode in the source
directory, which is usually derived from st->st_ino.
Read on-disk ino and store it in i_ino[0] given the above background.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
lib/namei.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lib/namei.c b/lib/namei.c
index 1023a9a..2bb1d4c 100644
--- a/lib/namei.c
+++ b/lib/namei.c
@@ -60,6 +60,7 @@ int erofs_read_inode_from_disk(struct erofs_inode *vi)
die = (struct erofs_inode_extended *)buf;
vi->xattr_isize = erofs_xattr_ibody_size(die->i_xattr_icount);
vi->i_mode = le16_to_cpu(die->i_mode);
+ vi->i_ino[0] = le32_to_cpu(die->i_ino);
switch (vi->i_mode & S_IFMT) {
case S_IFREG:
@@ -95,6 +96,7 @@ int erofs_read_inode_from_disk(struct erofs_inode *vi)
vi->inode_isize = sizeof(struct erofs_inode_compact);
vi->xattr_isize = erofs_xattr_ibody_size(dic->i_xattr_icount);
vi->i_mode = le16_to_cpu(dic->i_mode);
+ vi->i_ino[0] = le32_to_cpu(dic->i_ino);
switch (vi->i_mode & S_IFMT) {
case S_IFREG:
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 23+ messages in thread* Re: [PATCH 03/13] erofs-utils: lib: read i_ino in erofs_read_inode_from_disk()
2023-08-14 3:42 ` [PATCH 03/13] erofs-utils: lib: read i_ino in erofs_read_inode_from_disk() Jingbo Xu
@ 2023-08-14 6:03 ` Gao Xiang
0 siblings, 0 replies; 23+ messages in thread
From: Gao Xiang @ 2023-08-14 6:03 UTC (permalink / raw)
To: Jingbo Xu, xiang, linux-erofs
On 2023/8/14 11:42, Jingbo Xu wrote:
> i_ino[0] is a unique inode serial number in the erofs filesystem where
> the inode resides, and also serves as the on-disk inode number, while
> i_ino[1] is a unique number identifying the source inode in the source
> directory, which is usually derived from st->st_ino.
>
> Read on-disk ino and store it in i_ino[0] given the above background.
>
> Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
Fixes: 5e35b75ad499 ("erofs-utils: introduce fuse implementation")
Reviewed-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Thanks,
Gao Xiang
^ permalink raw reply [flat|nested] 23+ messages in thread
* [PATCH 04/13] erofs-utils: lib: fix erofs_init_devices() in multidev mode
2023-08-14 3:42 [PATCH 00/13] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (2 preceding siblings ...)
2023-08-14 3:42 ` [PATCH 03/13] erofs-utils: lib: read i_ino in erofs_read_inode_from_disk() Jingbo Xu
@ 2023-08-14 3:42 ` Jingbo Xu
2023-08-14 6:39 ` Gao Xiang
2023-08-14 3:42 ` [PATCH 05/13] erofs-utils: lib: keep self maintained devname Jingbo Xu
` (8 subsequent siblings)
12 siblings, 1 reply; 23+ messages in thread
From: Jingbo Xu @ 2023-08-14 3:42 UTC (permalink / raw)
To: xiang, linux-erofs
Fix the missing le32_to_cpu(). Read and cache blocks of each device for
later use.
Initialize sbi->extra_devices from on-disk extra_devices.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
lib/super.c | 11 ++++-------
1 file changed, 4 insertions(+), 7 deletions(-)
diff --git a/lib/super.c b/lib/super.c
index e8e84aa..54ab29b 100644
--- a/lib/super.c
+++ b/lib/super.c
@@ -36,11 +36,7 @@ static int erofs_init_devices(struct erofs_sb_info *sbi,
else
ondisk_extradevs = le16_to_cpu(dsb->extra_devices);
- if (ondisk_extradevs != sbi->extra_devices) {
- erofs_err("extra devices don't match (ondisk %u, given %u)",
- ondisk_extradevs, sbi->extra_devices);
- return -EINVAL;
- }
+ sbi->extra_devices = ondisk_extradevs;
if (!ondisk_extradevs)
return 0;
@@ -59,8 +55,9 @@ static int erofs_init_devices(struct erofs_sb_info *sbi,
return ret;
}
- sbi->devs[i].mapped_blkaddr = dis.mapped_blkaddr;
- sbi->total_blocks += dis.blocks;
+ sbi->devs[i].mapped_blkaddr = le32_to_cpu(dis.mapped_blkaddr);
+ sbi->devs[i].blocks = le32_to_cpu(dis.blocks);
+ sbi->total_blocks += sbi->devs[i].blocks;
pos += EROFS_DEVT_SLOT_SIZE;
}
return 0;
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 23+ messages in thread* [PATCH 05/13] erofs-utils: lib: keep self maintained devname
2023-08-14 3:42 [PATCH 00/13] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (3 preceding siblings ...)
2023-08-14 3:42 ` [PATCH 04/13] erofs-utils: lib: fix erofs_init_devices() in multidev mode Jingbo Xu
@ 2023-08-14 3:42 ` Jingbo Xu
2023-08-14 6:43 ` Gao Xiang
2023-08-14 3:42 ` [PATCH 06/13] erofs-utils: lib: make erofs_get_unhashed_chunk() global Jingbo Xu
` (7 subsequent siblings)
12 siblings, 1 reply; 23+ messages in thread
From: Jingbo Xu @ 2023-08-14 3:42 UTC (permalink / raw)
To: xiang, linux-erofs
Keep self allocated and maintained devname in erofs_sb_info.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
include/erofs/internal.h | 2 +-
lib/io.c | 14 ++++++++++++--
2 files changed, 13 insertions(+), 3 deletions(-)
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index a04e6a6..892dc96 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -61,7 +61,7 @@ struct erofs_device_info {
struct erofs_sb_info {
struct erofs_device_info *devs;
- const char *devname;
+ char *devname;
u64 total_blocks;
u64 primarydevice_blocks;
diff --git a/lib/io.c b/lib/io.c
index 8d84de2..1545436 100644
--- a/lib/io.c
+++ b/lib/io.c
@@ -10,6 +10,7 @@
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
+#include <stdlib.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include "erofs/io.h"
@@ -46,6 +47,7 @@ static int dev_get_blkdev_size(int fd, u64 *bytes)
void dev_close(struct erofs_sb_info *sbi)
{
close(sbi->devfd);
+ free(sbi->devname);
sbi->devname = NULL;
sbi->devfd = -1;
sbi->devsz = 0;
@@ -95,7 +97,11 @@ int dev_open(struct erofs_sb_info *sbi, const char *dev)
return -EINVAL;
}
- sbi->devname = dev;
+ sbi->devname = strdup(dev);
+ if (!sbi->devname) {
+ close(fd);
+ return -ENOMEM;
+ }
sbi->devfd = fd;
erofs_info("successfully to open %s", dev);
@@ -136,8 +142,12 @@ int dev_open_ro(struct erofs_sb_info *sbi, const char *dev)
return -errno;
}
+ sbi->devname = strdup(dev);
+ if (!sbi->devname) {
+ close(fd);
+ return -ENOMEM;
+ }
sbi->devfd = fd;
- sbi->devname = dev;
sbi->devsz = INT64_MAX;
return 0;
}
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 23+ messages in thread* Re: [PATCH 05/13] erofs-utils: lib: keep self maintained devname
2023-08-14 3:42 ` [PATCH 05/13] erofs-utils: lib: keep self maintained devname Jingbo Xu
@ 2023-08-14 6:43 ` Gao Xiang
2023-08-14 6:55 ` Gao Xiang
0 siblings, 1 reply; 23+ messages in thread
From: Gao Xiang @ 2023-08-14 6:43 UTC (permalink / raw)
To: Jingbo Xu, xiang, linux-erofs
On 2023/8/14 11:42, Jingbo Xu wrote:
> Keep self allocated and maintained devname in erofs_sb_info.
>
> Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
> ---
> include/erofs/internal.h | 2 +-
> lib/io.c | 14 ++++++++++++--
> 2 files changed, 13 insertions(+), 3 deletions(-)
>
> diff --git a/include/erofs/internal.h b/include/erofs/internal.h
> index a04e6a6..892dc96 100644
> --- a/include/erofs/internal.h
> +++ b/include/erofs/internal.h
> @@ -61,7 +61,7 @@ struct erofs_device_info {
>
> struct erofs_sb_info {
> struct erofs_device_info *devs;
> - const char *devname;
> + char *devname;
>
> u64 total_blocks;
> u64 primarydevice_blocks;
> diff --git a/lib/io.c b/lib/io.c
> index 8d84de2..1545436 100644
> --- a/lib/io.c
> +++ b/lib/io.c
> @@ -10,6 +10,7 @@
> #ifndef _GNU_SOURCE
> #define _GNU_SOURCE
> #endif
> +#include <stdlib.h>
> #include <sys/stat.h>
> #include <sys/ioctl.h>
> #include "erofs/io.h"
> @@ -46,6 +47,7 @@ static int dev_get_blkdev_size(int fd, u64 *bytes)
> void dev_close(struct erofs_sb_info *sbi)
> {
> close(sbi->devfd);
> + free(sbi->devname);
> sbi->devname = NULL;
> sbi->devfd = -1;
> sbi->devsz = 0;
> @@ -95,7 +97,11 @@ int dev_open(struct erofs_sb_info *sbi, const char *dev)
> return -EINVAL;
> }
>
> - sbi->devname = dev;
> + sbi->devname = strdup(dev);
Could we move sbi->devname assignment to the beginning of the function?
e.g.
..
sbi->devname = strdup(dev);
if (!sbi->devname)
return -ENOMEM;
fd = open(dev, O_RDWR | O_CREAT | O_BINARY, 0644);
...
> + if (!sbi->devname) {
> + close(fd);
> + return -ENOMEM;
> + }
> sbi->devfd = fd;
>
> erofs_info("successfully to open %s", dev);
> @@ -136,8 +142,12 @@ int dev_open_ro(struct erofs_sb_info *sbi, const char *dev)
> return -errno;
> }
>
> + sbi->devname = strdup(dev);
> + if (!sbi->devname) {
> + close(fd);
> + return -ENOMEM;
> + }
Same here.
Thanks,
Gao Xiang
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [PATCH 05/13] erofs-utils: lib: keep self maintained devname
2023-08-14 6:43 ` Gao Xiang
@ 2023-08-14 6:55 ` Gao Xiang
0 siblings, 0 replies; 23+ messages in thread
From: Gao Xiang @ 2023-08-14 6:55 UTC (permalink / raw)
To: Jingbo Xu, xiang, linux-erofs
On 2023/8/14 14:43, Gao Xiang wrote:
>
>
> On 2023/8/14 11:42, Jingbo Xu wrote:
>> Keep self allocated and maintained devname in erofs_sb_info.
>>
>> Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
>> ---
...
>> @@ -95,7 +97,11 @@ int dev_open(struct erofs_sb_info *sbi, const char *dev)
>> return -EINVAL;
>> }
>> - sbi->devname = dev;
>> + sbi->devname = strdup(dev);
>
> Could we move sbi->devname assignment to the beginning of the function?
> e.g.
>
> ..
> sbi->devname = strdup(dev);
> if (!sbi->devname)
> return -ENOMEM;
>
> fd = open(dev, O_RDWR | O_CREAT | O_BINARY, 0644);
> ...
>
>
>> + if (!sbi->devname) {
>> + close(fd);
>> + return -ENOMEM;
>> + }
After a second thought, since there are already several `close(fd);` in the
error paths and they are unavoidable. So it looks good to me now:
Reviewed-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Thanks,
Gao Xiang
^ permalink raw reply [flat|nested] 23+ messages in thread
* [PATCH 06/13] erofs-utils: lib: make erofs_get_unhashed_chunk() global
2023-08-14 3:42 [PATCH 00/13] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (4 preceding siblings ...)
2023-08-14 3:42 ` [PATCH 05/13] erofs-utils: lib: keep self maintained devname Jingbo Xu
@ 2023-08-14 3:42 ` Jingbo Xu
2023-08-14 3:42 ` [PATCH 07/13] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper Jingbo Xu
` (6 subsequent siblings)
12 siblings, 0 replies; 23+ messages in thread
From: Jingbo Xu @ 2023-08-14 3:42 UTC (permalink / raw)
To: xiang, linux-erofs
... so that it could be called from outside blobchunk.c later.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
include/erofs/blobchunk.h | 2 ++
lib/blobchunk.c | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/include/erofs/blobchunk.h b/include/erofs/blobchunk.h
index 010aee1..fb85d8e 100644
--- a/include/erofs/blobchunk.h
+++ b/include/erofs/blobchunk.h
@@ -14,6 +14,8 @@ extern "C"
#include "erofs/internal.h"
+struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
+ erofs_blk_t blkaddr, erofs_off_t sourceoffset);
int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, erofs_off_t off);
int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd);
int tarerofs_write_chunkes(struct erofs_inode *inode, erofs_off_t data_offset);
diff --git a/lib/blobchunk.c b/lib/blobchunk.c
index cada5bb..07f18bd 100644
--- a/lib/blobchunk.c
+++ b/lib/blobchunk.c
@@ -38,7 +38,7 @@ struct erofs_blobchunk erofs_holechunk = {
};
static LIST_HEAD(unhashed_blobchunks);
-static struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
+struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
erofs_blk_t blkaddr, erofs_off_t sourceoffset)
{
struct erofs_blobchunk *chunk;
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 23+ messages in thread* [PATCH 07/13] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
2023-08-14 3:42 [PATCH 00/13] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (5 preceding siblings ...)
2023-08-14 3:42 ` [PATCH 06/13] erofs-utils: lib: make erofs_get_unhashed_chunk() global Jingbo Xu
@ 2023-08-14 3:42 ` Jingbo Xu
2023-08-14 6:46 ` Gao Xiang
2023-08-14 3:42 ` [PATCH 08/13] erofs-utils: lib: add erofs_inode_tag_opaque() helper Jingbo Xu
` (5 subsequent siblings)
12 siblings, 1 reply; 23+ messages in thread
From: Jingbo Xu @ 2023-08-14 3:42 UTC (permalink / raw)
To: xiang, linux-erofs
Add erofs_read_xattrs_from_disk() helper to read extended attributes
from disk.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
include/erofs/xattr.h | 1 +
lib/xattr.c | 76 +++++++++++++++++++++++++++++++++++++++++++
2 files changed, 77 insertions(+)
diff --git a/include/erofs/xattr.h b/include/erofs/xattr.h
index dc27cf6..634daf9 100644
--- a/include/erofs/xattr.h
+++ b/include/erofs/xattr.h
@@ -85,6 +85,7 @@ int erofs_xattr_write_name_prefixes(struct erofs_sb_info *sbi, FILE *f);
int erofs_setxattr(struct erofs_inode *inode, char *key,
const void *value, size_t size);
+int erofs_read_xattrs_from_disk(struct erofs_inode *inode);
#ifdef __cplusplus
}
diff --git a/lib/xattr.c b/lib/xattr.c
index 12f580e..8d8f9f0 100644
--- a/lib/xattr.c
+++ b/lib/xattr.c
@@ -493,6 +493,82 @@ int erofs_scan_file_xattrs(struct erofs_inode *inode)
return erofs_droid_xattr_set_caps(inode);
}
+static struct xattr_item *erofs_read_xattr_from_disk(struct erofs_inode *inode,
+ char *key)
+{
+ ssize_t ret;
+ u8 prefix;
+ u16 prefixlen;
+ unsigned int len[2];
+ char *kvbuf;
+
+ if (!match_prefix(key, &prefix, &prefixlen))
+ return ERR_PTR(-ENODATA);
+
+ ret = erofs_getxattr(inode, key, NULL, 0);
+ if (ret < 0)
+ return ERR_PTR(-errno);
+
+ /* allocate key-value buffer */
+ len[0] = strlen(key) - prefixlen;
+ len[1] = ret;
+ kvbuf = malloc(len[0] + len[1]);
+ if (!kvbuf)
+ return ERR_PTR(-ENOMEM);
+ memcpy(kvbuf, key + prefixlen, len[0]);
+ if (len[1]) {
+ ret = erofs_getxattr(inode, key, kvbuf + len[0], len[1]);
+ if (ret < 0) {
+ free(kvbuf);
+ return ERR_PTR(-errno);
+ }
+ if (ret != len[1]) {
+ erofs_err("size of xattr value got changed just now (%u-> %ld)",
+ len[1], (long)ret);
+ len[1] = ret;
+ }
+ }
+ return get_xattritem(prefix, kvbuf, len);
+}
+
+int erofs_read_xattrs_from_disk(struct erofs_inode *inode)
+{
+ ssize_t kllen;
+ char *keylst, *key;
+ struct xattr_item *item;
+ int ret;
+
+ init_list_head(&inode->i_xattrs);
+ kllen = erofs_listxattr(inode, NULL, 0);
+ if (kllen < 0)
+ return kllen;
+ if (kllen <= 1)
+ return 0;
+
+ keylst = malloc(kllen);
+ if (!keylst)
+ return -ENOMEM;
+
+ ret = erofs_listxattr(inode, keylst, kllen);
+ if (ret < 0)
+ goto err;
+
+ for (key = keylst; key < keylst + kllen; key += strlen(key) + 1) {
+ item = erofs_read_xattr_from_disk(inode, key);
+ if (IS_ERR(item)) {
+ ret = PTR_ERR(item);
+ goto err;
+ }
+
+ ret = erofs_xattr_add(&inode->i_xattrs, item);
+ if (ret < 0)
+ goto err;
+ }
+err:
+ free(keylst);
+ return ret;
+}
+
int erofs_prepare_xattr_ibody(struct erofs_inode *inode)
{
int ret;
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 23+ messages in thread* Re: [PATCH 07/13] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
2023-08-14 3:42 ` [PATCH 07/13] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper Jingbo Xu
@ 2023-08-14 6:46 ` Gao Xiang
2023-08-14 6:59 ` Jingbo Xu
0 siblings, 1 reply; 23+ messages in thread
From: Gao Xiang @ 2023-08-14 6:46 UTC (permalink / raw)
To: Jingbo Xu, xiang, linux-erofs
On 2023/8/14 11:42, Jingbo Xu wrote:
> Add erofs_read_xattrs_from_disk() helper to read extended attributes
> from disk.
>
> Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
> ---
> include/erofs/xattr.h | 1 +
> lib/xattr.c | 76 +++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 77 insertions(+)
>
> diff --git a/include/erofs/xattr.h b/include/erofs/xattr.h
> index dc27cf6..634daf9 100644
> --- a/include/erofs/xattr.h
> +++ b/include/erofs/xattr.h
> @@ -85,6 +85,7 @@ int erofs_xattr_write_name_prefixes(struct erofs_sb_info *sbi, FILE *f);
>
> int erofs_setxattr(struct erofs_inode *inode, char *key,
> const void *value, size_t size);
> +int erofs_read_xattrs_from_disk(struct erofs_inode *inode);
>
> #ifdef __cplusplus
> }
> diff --git a/lib/xattr.c b/lib/xattr.c
> index 12f580e..8d8f9f0 100644
> --- a/lib/xattr.c
> +++ b/lib/xattr.c
> @@ -493,6 +493,82 @@ int erofs_scan_file_xattrs(struct erofs_inode *inode)
> return erofs_droid_xattr_set_caps(inode);
> }
>
> +static struct xattr_item *erofs_read_xattr_from_disk(struct erofs_inode *inode,
> + char *key)
> +{
> + ssize_t ret;
> + u8 prefix;
> + u16 prefixlen;
> + unsigned int len[2];
> + char *kvbuf;
> +
> + if (!match_prefix(key, &prefix, &prefixlen))
> + return ERR_PTR(-ENODATA);
> +
> + ret = erofs_getxattr(inode, key, NULL, 0);
> + if (ret < 0)
> + return ERR_PTR(-errno);
> +
> + /* allocate key-value buffer */
> + len[0] = strlen(key) - prefixlen;
> + len[1] = ret;
> + kvbuf = malloc(len[0] + len[1]);
> + if (!kvbuf)
> + return ERR_PTR(-ENOMEM);
> + memcpy(kvbuf, key + prefixlen, len[0]);
> + if (len[1]) {
> + ret = erofs_getxattr(inode, key, kvbuf + len[0], len[1]);
> + if (ret < 0) {
> + free(kvbuf);
> + return ERR_PTR(-errno);
> + }
> + if (ret != len[1]) {
> + erofs_err("size of xattr value got changed just now (%u-> %ld)",
> + len[1], (long)ret);
> + len[1] = ret;
> + }
Can this actually happen? Maybe just:
DBG_BUGON(ret != len[1]);
Thanks,
Gao Xiang
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [PATCH 07/13] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
2023-08-14 6:46 ` Gao Xiang
@ 2023-08-14 6:59 ` Jingbo Xu
0 siblings, 0 replies; 23+ messages in thread
From: Jingbo Xu @ 2023-08-14 6:59 UTC (permalink / raw)
To: Gao Xiang, xiang, linux-erofs
On 8/14/23 2:46 PM, Gao Xiang wrote:
>
>
> On 2023/8/14 11:42, Jingbo Xu wrote:
>> Add erofs_read_xattrs_from_disk() helper to read extended attributes
>> from disk.
>>
>> Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
>> ---
>> include/erofs/xattr.h | 1 +
>> lib/xattr.c | 76 +++++++++++++++++++++++++++++++++++++++++++
>> 2 files changed, 77 insertions(+)
>>
>> diff --git a/include/erofs/xattr.h b/include/erofs/xattr.h
>> index dc27cf6..634daf9 100644
>> --- a/include/erofs/xattr.h
>> +++ b/include/erofs/xattr.h
>> @@ -85,6 +85,7 @@ int erofs_xattr_write_name_prefixes(struct
>> erofs_sb_info *sbi, FILE *f);
>> int erofs_setxattr(struct erofs_inode *inode, char *key,
>> const void *value, size_t size);
>> +int erofs_read_xattrs_from_disk(struct erofs_inode *inode);
>> #ifdef __cplusplus
>> }
>> diff --git a/lib/xattr.c b/lib/xattr.c
>> index 12f580e..8d8f9f0 100644
>> --- a/lib/xattr.c
>> +++ b/lib/xattr.c
>> @@ -493,6 +493,82 @@ int erofs_scan_file_xattrs(struct erofs_inode
>> *inode)
>> return erofs_droid_xattr_set_caps(inode);
>> }
>> +static struct xattr_item *erofs_read_xattr_from_disk(struct
>> erofs_inode *inode,
>> + char *key)
>> +{
>> + ssize_t ret;
>> + u8 prefix;
>> + u16 prefixlen;
>> + unsigned int len[2];
>> + char *kvbuf;
>> +
>> + if (!match_prefix(key, &prefix, &prefixlen))
>> + return ERR_PTR(-ENODATA);
>> +
>> + ret = erofs_getxattr(inode, key, NULL, 0);
>> + if (ret < 0)
>> + return ERR_PTR(-errno);
>> +
>> + /* allocate key-value buffer */
>> + len[0] = strlen(key) - prefixlen;
>> + len[1] = ret;
>> + kvbuf = malloc(len[0] + len[1]);
>> + if (!kvbuf)
>> + return ERR_PTR(-ENOMEM);
>> + memcpy(kvbuf, key + prefixlen, len[0]);
>> + if (len[1]) {
>> + ret = erofs_getxattr(inode, key, kvbuf + len[0], len[1]);
>> + if (ret < 0) {
>> + free(kvbuf);
>> + return ERR_PTR(-errno);
>> + }
>> + if (ret != len[1]) {
>> + erofs_err("size of xattr value got changed just now (%u->
>> %ld)",
>> + len[1], (long)ret);
>> + len[1] = ret;
>> + }
>
> Can this actually happen? Maybe just:
> DBG_BUGON(ret != len[1]);
Make sense.
--
Thanks,
Jingbo
^ permalink raw reply [flat|nested] 23+ messages in thread
* [PATCH 08/13] erofs-utils: lib: add erofs_inode_tag_opaque() helper
2023-08-14 3:42 [PATCH 00/13] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (6 preceding siblings ...)
2023-08-14 3:42 ` [PATCH 07/13] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper Jingbo Xu
@ 2023-08-14 3:42 ` Jingbo Xu
2023-08-14 7:05 ` Gao Xiang
2023-08-14 3:42 ` [PATCH 09/13] erofs-utils: lib: add erofs_insert_ihash() helper Jingbo Xu
` (4 subsequent siblings)
12 siblings, 1 reply; 23+ messages in thread
From: Jingbo Xu @ 2023-08-14 3:42 UTC (permalink / raw)
To: xiang, linux-erofs
Add erofs_inode_tag_opaque() helper checking if it's an opaque directory.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
include/erofs/internal.h | 2 ++
include/erofs/xattr.h | 22 ++++++++++++++++++++
lib/tar.c | 5 -----
lib/xattr.c | 44 ++++++++++++++++++++++++++++++++++++++++
4 files changed, 68 insertions(+), 5 deletions(-)
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 892dc96..3631cc1 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -219,6 +219,8 @@ struct erofs_inode {
#endif
erofs_off_t fragmentoff;
unsigned int fragment_size;
+
+ bool opaque;
};
static inline erofs_off_t erofs_iloc(struct erofs_inode *inode)
diff --git a/include/erofs/xattr.h b/include/erofs/xattr.h
index 634daf9..21d669b 100644
--- a/include/erofs/xattr.h
+++ b/include/erofs/xattr.h
@@ -73,6 +73,27 @@ static inline unsigned int xattrblock_offset(struct erofs_inode *vi,
#ifndef XATTR_NAME_POSIX_ACL_DEFAULT
#define XATTR_NAME_POSIX_ACL_DEFAULT "system.posix_acl_default"
#endif
+#ifndef OVL_XATTR_NAMESPACE
+#define OVL_XATTR_NAMESPACE "overlay."
+#endif
+#ifndef OVL_XATTR_OPAQUE_POSTFIX
+#define OVL_XATTR_OPAQUE_POSTFIX "opaque"
+#endif
+#ifndef OVL_XATTR_TRUSTED_PREFIX
+#define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE
+#endif
+#ifndef OVL_XATTR_OPAQUE_SUFFIX
+#define OVL_XATTR_OPAQUE_SUFFIX OVL_XATTR_NAMESPACE OVL_XATTR_OPAQUE_POSTFIX
+#endif
+#ifndef OVL_XATTR_OPAQUE_SUFFIX_LEN
+#define OVL_XATTR_OPAQUE_SUFFIX_LEN (sizeof(OVL_XATTR_OPAQUE_SUFFIX) - 1)
+#endif
+#ifndef OVL_XATTR_OPAQUE
+#define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX
+#endif
+#ifndef OVL_XATTR_OPAQUE_LEN
+#define OVL_XATTR_OPAQUE_LEN (sizeof(OVL_XATTR_OPAQUE) - 1)
+#endif
int erofs_scan_file_xattrs(struct erofs_inode *inode);
int erofs_prepare_xattr_ibody(struct erofs_inode *inode);
@@ -86,6 +107,7 @@ int erofs_xattr_write_name_prefixes(struct erofs_sb_info *sbi, FILE *f);
int erofs_setxattr(struct erofs_inode *inode, char *key,
const void *value, size_t size);
int erofs_read_xattrs_from_disk(struct erofs_inode *inode);
+void erofs_inode_tag_opaque(struct erofs_inode *inode);
#ifdef __cplusplus
}
diff --git a/lib/tar.c b/lib/tar.c
index 42590d2..d7a58d2 100644
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -19,11 +19,6 @@
#include "erofs/xattr.h"
#include "erofs/blobchunk.h"
-#define OVL_XATTR_NAMESPACE "overlay."
-#define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE
-#define OVL_XATTR_OPAQUE_POSTFIX "opaque"
-#define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX
-
#define EROFS_WHITEOUT_DEV 0
static char erofs_libbuf[16384];
diff --git a/lib/xattr.c b/lib/xattr.c
index 8d8f9f0..e9aff53 100644
--- a/lib/xattr.c
+++ b/lib/xattr.c
@@ -1421,6 +1421,50 @@ int erofs_listxattr(struct erofs_inode *vi, char *buffer, size_t buffer_size)
return shared_listxattr(vi, &it);
}
+static bool erofs_xattr_is_opaque(struct xattr_item *item)
+{
+ struct ea_type_node *tnode;
+ unsigned int plen;
+ const char *prefix;
+
+ if (item->prefix == EROFS_XATTR_INDEX_TRUSTED &&
+ !strncmp(item->kvbuf, OVL_XATTR_OPAQUE_SUFFIX,
+ OVL_XATTR_OPAQUE_SUFFIX_LEN))
+ return true;
+
+ if (item->prefix & EROFS_XATTR_LONG_PREFIX) {
+ list_for_each_entry(tnode, &ea_name_prefixes, list) {
+ prefix = tnode->type.prefix;
+ plen = tnode->type.prefix_len;
+ if (tnode->index == item->prefix &&
+ plen + item->len[0] == OVL_XATTR_OPAQUE_LEN &&
+ !strncmp(prefix, OVL_XATTR_OPAQUE, plen) &&
+ !strncmp(item->kvbuf, OVL_XATTR_OPAQUE + plen,
+ item->len[0]))
+ return true;
+ }
+ }
+ return false;
+}
+
+void erofs_inode_tag_opaque(struct erofs_inode *inode)
+{
+ struct inode_xattr_node *node, *n;
+
+ list_for_each_entry_safe(node, n, &inode->i_xattrs, list) {
+ if (erofs_xattr_is_opaque(node->item)) {
+ if (S_ISDIR(inode->i_mode))
+ inode->opaque = true;
+ else
+ erofs_dbg("file %s: opaque xattr on non-dir",
+ inode->i_srcpath);
+ /* strip overlayfs xattrs */
+ list_del(&node->list);
+ free(node);
+ }
+ }
+}
+
int erofs_xattr_insert_name_prefix(const char *prefix)
{
struct ea_type_node *tnode;
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 23+ messages in thread* Re: [PATCH 08/13] erofs-utils: lib: add erofs_inode_tag_opaque() helper
2023-08-14 3:42 ` [PATCH 08/13] erofs-utils: lib: add erofs_inode_tag_opaque() helper Jingbo Xu
@ 2023-08-14 7:05 ` Gao Xiang
0 siblings, 0 replies; 23+ messages in thread
From: Gao Xiang @ 2023-08-14 7:05 UTC (permalink / raw)
To: Jingbo Xu, xiang, linux-erofs
On 2023/8/14 11:42, Jingbo Xu wrote:
> Add erofs_inode_tag_opaque() helper checking if it's an opaque directory.
>
> Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
> ---
> include/erofs/internal.h | 2 ++
> include/erofs/xattr.h | 22 ++++++++++++++++++++
> lib/tar.c | 5 -----
> lib/xattr.c | 44 ++++++++++++++++++++++++++++++++++++++++
> 4 files changed, 68 insertions(+), 5 deletions(-)
>
> diff --git a/include/erofs/internal.h b/include/erofs/internal.h
> index 892dc96..3631cc1 100644
> --- a/include/erofs/internal.h
> +++ b/include/erofs/internal.h
> @@ -219,6 +219,8 @@ struct erofs_inode {
> #endif
> erofs_off_t fragmentoff;
> unsigned int fragment_size;
> +
> + bool opaque;
could we move it upwards? for example, just under
bool with_tmpfile;
> };
>
> static inline erofs_off_t erofs_iloc(struct erofs_inode *inode)
> diff --git a/include/erofs/xattr.h b/include/erofs/xattr.h
> index 634daf9..21d669b 100644
> --- a/include/erofs/xattr.h
> +++ b/include/erofs/xattr.h
> @@ -73,6 +73,27 @@ static inline unsigned int xattrblock_offset(struct erofs_inode *vi,
> #ifndef XATTR_NAME_POSIX_ACL_DEFAULT
> #define XATTR_NAME_POSIX_ACL_DEFAULT "system.posix_acl_default"
> #endif
> +#ifndef OVL_XATTR_NAMESPACE
> +#define OVL_XATTR_NAMESPACE "overlay."
> +#endif
> +#ifndef OVL_XATTR_OPAQUE_POSTFIX
> +#define OVL_XATTR_OPAQUE_POSTFIX "opaque"
> +#endif
> +#ifndef OVL_XATTR_TRUSTED_PREFIX
> +#define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE
> +#endif
> +#ifndef OVL_XATTR_OPAQUE_SUFFIX
> +#define OVL_XATTR_OPAQUE_SUFFIX OVL_XATTR_NAMESPACE OVL_XATTR_OPAQUE_POSTFIX
> +#endif
> +#ifndef OVL_XATTR_OPAQUE_SUFFIX_LEN
> +#define OVL_XATTR_OPAQUE_SUFFIX_LEN (sizeof(OVL_XATTR_OPAQUE_SUFFIX) - 1)
> +#endif
> +#ifndef OVL_XATTR_OPAQUE
> +#define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX
> +#endif
> +#ifndef OVL_XATTR_OPAQUE_LEN
> +#define OVL_XATTR_OPAQUE_LEN (sizeof(OVL_XATTR_OPAQUE) - 1)
> +#endif
I'd like to avoid export these symbols in the headers, how about moving all
related logic into lib/xattr.c?
>
> int erofs_scan_file_xattrs(struct erofs_inode *inode);
> int erofs_prepare_xattr_ibody(struct erofs_inode *inode);
> @@ -86,6 +107,7 @@ int erofs_xattr_write_name_prefixes(struct erofs_sb_info *sbi, FILE *f);
> int erofs_setxattr(struct erofs_inode *inode, char *key,
> const void *value, size_t size);
> int erofs_read_xattrs_from_disk(struct erofs_inode *inode);
> +void erofs_inode_tag_opaque(struct erofs_inode *inode);
>
> #ifdef __cplusplus
> }
> diff --git a/lib/tar.c b/lib/tar.c
> index 42590d2..d7a58d2 100644
> --- a/lib/tar.c
> +++ b/lib/tar.c
> @@ -19,11 +19,6 @@
> #include "erofs/xattr.h"
> #include "erofs/blobchunk.h"
>
> -#define OVL_XATTR_NAMESPACE "overlay."
> -#define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE
> -#define OVL_XATTR_OPAQUE_POSTFIX "opaque"
> -#define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX
> -
> #define EROFS_WHITEOUT_DEV 0
>
> static char erofs_libbuf[16384];
> diff --git a/lib/xattr.c b/lib/xattr.c
> index 8d8f9f0..e9aff53 100644
> --- a/lib/xattr.c
> +++ b/lib/xattr.c
> @@ -1421,6 +1421,50 @@ int erofs_listxattr(struct erofs_inode *vi, char *buffer, size_t buffer_size)
> return shared_listxattr(vi, &it);
> }
>
> +static bool erofs_xattr_is_opaque(struct xattr_item *item)
> +{
> + struct ea_type_node *tnode;
> + unsigned int plen;
> + const char *prefix;
> +
> + if (item->prefix == EROFS_XATTR_INDEX_TRUSTED &&
> + !strncmp(item->kvbuf, OVL_XATTR_OPAQUE_SUFFIX,
> + OVL_XATTR_OPAQUE_SUFFIX_LEN))
> + return true;
> +
> + if (item->prefix & EROFS_XATTR_LONG_PREFIX) {
> + list_for_each_entry(tnode, &ea_name_prefixes, list) {
> + prefix = tnode->type.prefix;
> + plen = tnode->type.prefix_len;
> + if (tnode->index == item->prefix &&
> + plen + item->len[0] == OVL_XATTR_OPAQUE_LEN &&
> + !strncmp(prefix, OVL_XATTR_OPAQUE, plen) &&
> + !strncmp(item->kvbuf, OVL_XATTR_OPAQUE + plen,
> + item->len[0]))
> + return true;
These looks too complicated to me, how about just connect prefix and
suffix strings and do a string comparsion together?
Thanks,
Gao Xiang
^ permalink raw reply [flat|nested] 23+ messages in thread
* [PATCH 09/13] erofs-utils: lib: add erofs_insert_ihash() helper
2023-08-14 3:42 [PATCH 00/13] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (7 preceding siblings ...)
2023-08-14 3:42 ` [PATCH 08/13] erofs-utils: lib: add erofs_inode_tag_opaque() helper Jingbo Xu
@ 2023-08-14 3:42 ` Jingbo Xu
2023-08-14 3:42 ` [PATCH 10/13] erofs-utils: lib: add erofs_rebuild_dump_tree() helper Jingbo Xu
` (3 subsequent siblings)
12 siblings, 0 replies; 23+ messages in thread
From: Jingbo Xu @ 2023-08-14 3:42 UTC (permalink / raw)
To: xiang, linux-erofs
Add erofs_insert_ihash() helper inserting inode into inode hash table.
Also add prototypes of erofs_iget() and erofs_iget_by_nid() in the
header file.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
include/erofs/inode.h | 3 +++
lib/inode.c | 10 +++++++---
2 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/include/erofs/inode.h b/include/erofs/inode.h
index e8a5670..1c602a8 100644
--- a/include/erofs/inode.h
+++ b/include/erofs/inode.h
@@ -25,6 +25,9 @@ u32 erofs_new_encode_dev(dev_t dev);
unsigned char erofs_mode_to_ftype(umode_t mode);
unsigned char erofs_ftype_to_dtype(unsigned int filetype);
void erofs_inode_manager_init(void);
+void erofs_insert_ihash(struct erofs_inode *inode, dev_t dev, ino_t ino);
+struct erofs_inode *erofs_iget(dev_t dev, ino_t ino);
+struct erofs_inode *erofs_iget_by_nid(erofs_nid_t nid);
unsigned int erofs_iput(struct erofs_inode *inode);
erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
diff --git a/lib/inode.c b/lib/inode.c
index d54f84f..b967aab 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -75,6 +75,12 @@ void erofs_inode_manager_init(void)
init_list_head(&inode_hashtable[i]);
}
+void erofs_insert_ihash(struct erofs_inode *inode, dev_t dev, ino_t ino)
+{
+ list_add(&inode->i_hash,
+ &inode_hashtable[(ino ^ dev) % NR_INODE_HASHTABLE]);
+}
+
/* get the inode from the (source) inode # */
struct erofs_inode *erofs_iget(dev_t dev, ino_t ino)
{
@@ -976,9 +982,7 @@ static int erofs_fill_inode(struct erofs_inode *inode, struct stat *st,
inode->inode_isize = sizeof(struct erofs_inode_compact);
}
- list_add(&inode->i_hash,
- &inode_hashtable[(st->st_ino ^ st->st_dev) %
- NR_INODE_HASHTABLE]);
+ erofs_insert_ihash(inode, st->st_dev, st->st_ino);
return 0;
}
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 23+ messages in thread* [PATCH 10/13] erofs-utils: lib: add erofs_rebuild_dump_tree() helper
2023-08-14 3:42 [PATCH 00/13] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (8 preceding siblings ...)
2023-08-14 3:42 ` [PATCH 09/13] erofs-utils: lib: add erofs_insert_ihash() helper Jingbo Xu
@ 2023-08-14 3:42 ` Jingbo Xu
2023-08-14 3:42 ` [PATCH 11/13] erofs-utils: lib: add erofs_rebuild_get_dentry() helper Jingbo Xu
` (2 subsequent siblings)
12 siblings, 0 replies; 23+ messages in thread
From: Jingbo Xu @ 2023-08-14 3:42 UTC (permalink / raw)
To: xiang, linux-erofs
Enhance tarerofs_dump_tree() so that it could optionally skip whiteout
files when it's supposed to output image in merged view. Rename it to
erofs_rebuild_dump_tree().
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
include/erofs/inode.h | 2 +-
include/erofs/internal.h | 2 ++
lib/inode.c | 23 ++++++++++++++++++++---
lib/tar.c | 2 --
mkfs/main.c | 2 +-
5 files changed, 24 insertions(+), 7 deletions(-)
diff --git a/include/erofs/inode.h b/include/erofs/inode.h
index 1c602a8..f273e9e 100644
--- a/include/erofs/inode.h
+++ b/include/erofs/inode.h
@@ -32,7 +32,7 @@ unsigned int erofs_iput(struct erofs_inode *inode);
erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
const char *name);
-int tarerofs_dump_tree(struct erofs_inode *dir);
+int erofs_rebuild_dump_tree(struct erofs_inode *dir, bool merged_view);
int erofs_init_empty_dir(struct erofs_inode *dir);
struct erofs_inode *erofs_new_inode(void);
struct erofs_inode *erofs_mkfs_build_tree_from_path(const char *path);
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 3631cc1..179ecb3 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -416,6 +416,8 @@ static inline u32 erofs_crc32c(u32 crc, const u8 *in, size_t len)
return crc;
}
+#define EROFS_WHITEOUT_DEV 0
+
#ifdef __cplusplus
}
#endif
diff --git a/lib/inode.c b/lib/inode.c
index b967aab..7f59a58 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -1323,9 +1323,14 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)
return inode;
}
-int tarerofs_dump_tree(struct erofs_inode *dir)
+static bool erofs_inode_is_whiteout(struct erofs_inode *inode)
{
- struct erofs_dentry *d;
+ return S_ISCHR(inode->i_mode) && inode->u.i_rdev == EROFS_WHITEOUT_DEV;
+}
+
+int erofs_rebuild_dump_tree(struct erofs_inode *dir, bool merged_view)
+{
+ struct erofs_dentry *d, *n;
unsigned int nr_subdirs;
int ret;
@@ -1368,6 +1373,18 @@ int tarerofs_dump_tree(struct erofs_inode *dir)
return 0;
}
+ if (merged_view) {
+ list_for_each_entry_safe(d, n, &dir->i_subdirs, d_child) {
+ if (erofs_inode_is_whiteout(d->inode)) {
+ erofs_dbg("skip whiteout %s", d->inode->i_srcpath);
+ erofs_d_invalidate(d);
+ list_del(&d->d_child);
+ free(d);
+ continue;
+ }
+ }
+ }
+
nr_subdirs = 0;
list_for_each_entry(d, &dir->i_subdirs, d_child)
++nr_subdirs;
@@ -1391,7 +1408,7 @@ int tarerofs_dump_tree(struct erofs_inode *dir)
continue;
inode = erofs_igrab(d->inode);
- ret = tarerofs_dump_tree(inode);
+ ret = erofs_rebuild_dump_tree(inode, merged_view);
dir->i_nlink += (erofs_mode_to_ftype(inode->i_mode) == EROFS_FT_DIR);
erofs_iput(inode);
if (ret)
diff --git a/lib/tar.c b/lib/tar.c
index d7a58d2..3205f06 100644
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -19,8 +19,6 @@
#include "erofs/xattr.h"
#include "erofs/blobchunk.h"
-#define EROFS_WHITEOUT_DEV 0
-
static char erofs_libbuf[16384];
struct tar_header {
diff --git a/mkfs/main.c b/mkfs/main.c
index c03a7a8..269ccf4 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -946,7 +946,7 @@ int main(int argc, char **argv)
if (err < 0)
goto exit;
- err = tarerofs_dump_tree(root_inode);
+ err = erofs_rebuild_dump_tree(root_inode, false);
if (err < 0)
goto exit;
}
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 23+ messages in thread* [PATCH 11/13] erofs-utils: lib: add erofs_rebuild_get_dentry() helper
2023-08-14 3:42 [PATCH 00/13] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (9 preceding siblings ...)
2023-08-14 3:42 ` [PATCH 10/13] erofs-utils: lib: add erofs_rebuild_dump_tree() helper Jingbo Xu
@ 2023-08-14 3:42 ` Jingbo Xu
2023-08-14 3:42 ` [PATCH 12/13] erofs-utils: lib: add erofs_rebuild_load_tree() helper Jingbo Xu
2023-08-14 3:42 ` [PATCH 13/13] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
12 siblings, 0 replies; 23+ messages in thread
From: Jingbo Xu @ 2023-08-14 3:42 UTC (permalink / raw)
To: xiang, linux-erofs
Rename tarerofs_get_dentry() to erofs_rebuild_get_dentry().
Also make `whout` and 'opq' parameter optional when `aufs` is false.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
include/erofs/rebuild.h | 19 +++++++
lib/Makefile.am | 3 +-
lib/rebuild.c | 117 ++++++++++++++++++++++++++++++++++++++++
lib/tar.c | 109 ++-----------------------------------
4 files changed, 141 insertions(+), 107 deletions(-)
create mode 100644 include/erofs/rebuild.h
create mode 100644 lib/rebuild.c
diff --git a/include/erofs/rebuild.h b/include/erofs/rebuild.h
new file mode 100644
index 0000000..92873c9
--- /dev/null
+++ b/include/erofs/rebuild.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */
+#ifndef __EROFS_REBUILD_H
+#define __EROFS_REBUILD_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#include "internal.h"
+
+struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
+ char *path, bool aufs, bool *whout, bool *opq);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 7a5dc03..4bcc09b 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -24,6 +24,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
$(top_srcdir)/include/erofs/xattr.h \
$(top_srcdir)/include/erofs/compress_hints.h \
$(top_srcdir)/include/erofs/fragments.h \
+ $(top_srcdir)/include/erofs/rebuild.h \
$(top_srcdir)/lib/liberofs_private.h
noinst_HEADERS += compressor.h
@@ -31,7 +32,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
namei.c data.c compress.c compressor.c zmap.c decompress.c \
compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \
- block_list.c
+ block_list.c rebuild.c
liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include
if ENABLE_LZ4
diff --git a/lib/rebuild.c b/lib/rebuild.c
new file mode 100644
index 0000000..7aaa071
--- /dev/null
+++ b/lib/rebuild.c
@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0
+#define _GNU_SOURCE
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include "erofs/print.h"
+#include "erofs/inode.h"
+#include "erofs/rebuild.h"
+#include "erofs/internal.h"
+
+#ifdef HAVE_LINUX_AUFS_TYPE_H
+#include <linux/aufs_type.h>
+#else
+#define AUFS_WH_PFX ".wh."
+#define AUFS_DIROPQ_NAME AUFS_WH_PFX ".opq"
+#define AUFS_WH_DIROPQ AUFS_WH_PFX AUFS_DIROPQ_NAME
+#endif
+
+static struct erofs_dentry *erofs_rebuild_mkdir(struct erofs_inode *dir,
+ const char *s)
+{
+ struct erofs_inode *inode;
+ struct erofs_dentry *d;
+
+ inode = erofs_new_inode();
+ if (IS_ERR(inode))
+ return ERR_CAST(inode);
+
+ inode->i_mode = S_IFDIR | 0755;
+ inode->i_parent = dir;
+ inode->i_uid = getuid();
+ inode->i_gid = getgid();
+ inode->i_mtime = inode->sbi->build_time;
+ inode->i_mtime_nsec = inode->sbi->build_time_nsec;
+ erofs_init_empty_dir(inode);
+
+ d = erofs_d_alloc(dir, s);
+ if (!IS_ERR(d)) {
+ d->type = EROFS_FT_DIR;
+ d->inode = inode;
+ }
+ return d;
+}
+
+struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
+ char *path, bool aufs, bool *whout, bool *opq)
+{
+ struct erofs_dentry *d = NULL;
+ unsigned int len = strlen(path);
+ char *s = path;
+
+ if (aufs) {
+ *whout = false;
+ *opq = false;
+ }
+
+ while (s < path + len) {
+ char *slash = memchr(s, '/', path + len - s);
+ if (slash) {
+ if (s == slash) {
+ while (*++s == '/'); /* skip '//...' */
+ continue;
+ }
+ *slash = '\0';
+ }
+
+ if (!memcmp(s, ".", 2)) {
+ /* null */
+ } else if (!memcmp(s, "..", 3)) {
+ pwd = pwd->i_parent;
+ } else {
+ struct erofs_inode *inode = NULL;
+
+ if (aufs && !slash) {
+ if (!memcmp(s, AUFS_WH_DIROPQ, sizeof(AUFS_WH_DIROPQ))) {
+ *opq = true;
+ break;
+ }
+ if (!memcmp(s, AUFS_WH_PFX, sizeof(AUFS_WH_PFX) - 1)) {
+ s += sizeof(AUFS_WH_PFX) - 1;
+ *whout = true;
+ }
+ }
+
+ list_for_each_entry(d, &pwd->i_subdirs, d_child) {
+ if (!strcmp(d->name, s)) {
+ if (d->type != EROFS_FT_DIR && slash)
+ return ERR_PTR(-EIO);
+ inode = d->inode;
+ break;
+ }
+ }
+
+ if (inode) {
+ pwd = inode;
+ } else if (!slash) {
+ d = erofs_d_alloc(pwd, s);
+ if (IS_ERR(d))
+ return d;
+ d->type = EROFS_FT_UNKNOWN;
+ d->inode = pwd;
+ } else {
+ d = erofs_rebuild_mkdir(pwd, s);
+ if (IS_ERR(d))
+ return d;
+ pwd = d->inode;
+ }
+ }
+ if (slash) {
+ *slash = '/';
+ s = slash + 1;
+ } else {
+ break;
+ }
+ }
+ return d;
+}
diff --git a/lib/tar.c b/lib/tar.c
index 3205f06..9a45b1c 100644
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -3,13 +3,6 @@
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
-#ifdef HAVE_LINUX_AUFS_TYPE_H
-#include <linux/aufs_type.h>
-#else
-#define AUFS_WH_PFX ".wh."
-#define AUFS_DIROPQ_NAME AUFS_WH_PFX ".opq"
-#define AUFS_WH_DIROPQ AUFS_WH_PFX AUFS_DIROPQ_NAME
-#endif
#include "erofs/print.h"
#include "erofs/cache.h"
#include "erofs/inode.h"
@@ -18,6 +11,7 @@
#include "erofs/io.h"
#include "erofs/xattr.h"
#include "erofs/blobchunk.h"
+#include "erofs/rebuild.h"
static char erofs_libbuf[16384];
@@ -131,103 +125,6 @@ static long long tarerofs_parsenum(const char *ptr, int len)
return tarerofs_otoi(ptr, len);
}
-static struct erofs_dentry *tarerofs_mkdir(struct erofs_inode *dir, const char *s)
-{
- struct erofs_inode *inode;
- struct erofs_dentry *d;
-
- inode = erofs_new_inode();
- if (IS_ERR(inode))
- return ERR_CAST(inode);
-
- inode->i_mode = S_IFDIR | 0755;
- inode->i_parent = dir;
- inode->i_uid = getuid();
- inode->i_gid = getgid();
- inode->i_mtime = inode->sbi->build_time;
- inode->i_mtime_nsec = inode->sbi->build_time_nsec;
- erofs_init_empty_dir(inode);
-
- d = erofs_d_alloc(dir, s);
- if (!IS_ERR(d)) {
- d->type = EROFS_FT_DIR;
- d->inode = inode;
- }
- return d;
-}
-
-static struct erofs_dentry *tarerofs_get_dentry(struct erofs_inode *pwd, char *path,
- bool aufs, bool *whout, bool *opq)
-{
- struct erofs_dentry *d = NULL;
- unsigned int len = strlen(path);
- char *s = path;
-
- *whout = false;
- *opq = false;
-
- while (s < path + len) {
- char *slash = memchr(s, '/', path + len - s);
- if (slash) {
- if (s == slash) {
- while (*++s == '/'); /* skip '//...' */
- continue;
- }
- *slash = '\0';
- }
-
- if (!memcmp(s, ".", 2)) {
- /* null */
- } else if (!memcmp(s, "..", 3)) {
- pwd = pwd->i_parent;
- } else {
- struct erofs_inode *inode = NULL;
-
- if (aufs && !slash) {
- if (!memcmp(s, AUFS_WH_DIROPQ, sizeof(AUFS_WH_DIROPQ))) {
- *opq = true;
- break;
- }
- if (!memcmp(s, AUFS_WH_PFX, sizeof(AUFS_WH_PFX) - 1)) {
- s += sizeof(AUFS_WH_PFX) - 1;
- *whout = true;
- }
- }
-
- list_for_each_entry(d, &pwd->i_subdirs, d_child) {
- if (!strcmp(d->name, s)) {
- if (d->type != EROFS_FT_DIR && slash)
- return ERR_PTR(-EIO);
- inode = d->inode;
- break;
- }
- }
-
- if (inode) {
- pwd = inode;
- } else if (!slash) {
- d = erofs_d_alloc(pwd, s);
- if (IS_ERR(d))
- return d;
- d->type = EROFS_FT_UNKNOWN;
- d->inode = pwd;
- } else {
- d = tarerofs_mkdir(pwd, s);
- if (IS_ERR(d))
- return d;
- pwd = d->inode;
- }
- }
- if (slash) {
- *slash = '/';
- s = slash + 1;
- } else {
- break;
- }
- }
- return d;
-}
-
int tarerofs_parse_pax_header(int fd, struct erofs_pax_header *eh, u32 size)
{
char *buf, *p;
@@ -610,7 +507,7 @@ restart:
erofs_dbg("parsing %s (mode %05o)", eh.path, st.st_mode);
- d = tarerofs_get_dentry(root, eh.path, tar->aufs, &whout, &opq);
+ d = erofs_rebuild_get_dentry(root, eh.path, tar->aufs, &whout, &opq);
if (IS_ERR(d)) {
ret = PTR_ERR(d);
goto out;
@@ -643,7 +540,7 @@ restart:
}
d->inode = NULL;
- d2 = tarerofs_get_dentry(root, eh.link, tar->aufs, &dumb, &dumb);
+ d2 = erofs_rebuild_get_dentry(root, eh.link, tar->aufs, &dumb, &dumb);
if (IS_ERR(d2)) {
ret = PTR_ERR(d2);
goto out;
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 23+ messages in thread* [PATCH 12/13] erofs-utils: lib: add erofs_rebuild_load_tree() helper
2023-08-14 3:42 [PATCH 00/13] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (10 preceding siblings ...)
2023-08-14 3:42 ` [PATCH 11/13] erofs-utils: lib: add erofs_rebuild_get_dentry() helper Jingbo Xu
@ 2023-08-14 3:42 ` Jingbo Xu
2023-08-14 3:42 ` [PATCH 13/13] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
12 siblings, 0 replies; 23+ messages in thread
From: Jingbo Xu @ 2023-08-14 3:42 UTC (permalink / raw)
To: xiang, linux-erofs
Add erofs_rebuild_load_tree() helper loading inode tree from given erofs
image, and making it merged into a given inode tree in an overlayfs like
model.
Since we need to read the content of the symlink file from disk when
loading tree, add dependency on zlib_LIBS for mkfs.erofs.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
include/erofs/internal.h | 1 +
include/erofs/rebuild.h | 2 +
lib/rebuild.c | 270 +++++++++++++++++++++++++++++++++++++++
mkfs/Makefile.am | 3 +-
4 files changed, 275 insertions(+), 1 deletion(-)
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 179ecb3..513e860 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -102,6 +102,7 @@ struct erofs_sb_info {
int devfd;
u64 devsz;
+ dev_t dev;
unsigned int nblobs;
unsigned int blobfd[256];
};
diff --git a/include/erofs/rebuild.h b/include/erofs/rebuild.h
index 92873c9..3ac074c 100644
--- a/include/erofs/rebuild.h
+++ b/include/erofs/rebuild.h
@@ -12,6 +12,8 @@ extern "C"
struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
char *path, bool aufs, bool *whout, bool *opq);
+int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi);
+
#ifdef __cplusplus
}
#endif
diff --git a/lib/rebuild.c b/lib/rebuild.c
index 7aaa071..b7e08b2 100644
--- a/lib/rebuild.c
+++ b/lib/rebuild.c
@@ -3,9 +3,18 @@
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
+#include <sys/stat.h>
+#include <config.h>
+#if defined(HAVE_SYS_SYSMACROS_H)
+#include <sys/sysmacros.h>
+#endif
#include "erofs/print.h"
#include "erofs/inode.h"
#include "erofs/rebuild.h"
+#include "erofs/io.h"
+#include "erofs/dir.h"
+#include "erofs/xattr.h"
+#include "erofs/blobchunk.h"
#include "erofs/internal.h"
#ifdef HAVE_LINUX_AUFS_TYPE_H
@@ -115,3 +124,264 @@ struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
}
return d;
}
+
+static int erofs_rebuild_fill_inode_map(struct erofs_inode *inode,
+ struct erofs_inode *vi, dev_t dev)
+{
+ int ret;
+ struct erofs_sb_info *sbi = vi->sbi;
+ unsigned int count, unit, chunkbits, i;
+ unsigned int deviceid = dev;
+ erofs_off_t chunksize;
+ erofs_blk_t blkaddr;
+ struct erofs_inode_chunk_index *idx;
+
+ /* TODO: fill data map in other layouts */
+ if (vi->datalayout != EROFS_INODE_CHUNK_BASED &&
+ vi->datalayout != EROFS_INODE_FLAT_PLAIN) {
+ erofs_err("unsupported datalayout %d", vi->datalayout);
+ return -EOPNOTSUPP;
+ }
+
+ if (sbi->extra_devices) {
+ chunkbits = vi->u.chunkbits;
+ inode->u.chunkformat = vi->u.chunkformat;
+ } else {
+ chunkbits = ilog2(inode->i_size - 1) + 1;
+ if (chunkbits < sbi->blkszbits)
+ chunkbits = sbi->blkszbits;
+ if (chunkbits - sbi->blkszbits > EROFS_CHUNK_FORMAT_BLKBITS_MASK)
+ chunkbits = EROFS_CHUNK_FORMAT_BLKBITS_MASK + sbi->blkszbits;
+ inode->u.chunkformat = EROFS_CHUNK_FORMAT_INDEXES;
+ inode->u.chunkformat |= chunkbits - sbi->blkszbits;
+ }
+ chunksize = 1ULL << chunkbits;
+ count = DIV_ROUND_UP(inode->i_size, chunksize);
+
+ unit = sizeof(struct erofs_inode_chunk_index);
+ inode->extent_isize = count * unit;
+ idx = malloc(max(sizeof(*idx), sizeof(void *)));
+ if (!idx)
+ return -ENOMEM;
+ inode->chunkindexes = idx;
+
+ for (i = 0; i < count; i++) {
+ struct erofs_blobchunk *chunk;
+ struct erofs_map_blocks map = {
+ .index = UINT_MAX,
+ };
+
+ map.m_la = i << chunkbits;
+ ret = erofs_map_blocks(vi, &map, 0);
+ if (ret)
+ goto err;
+
+ blkaddr = erofs_blknr(sbi, map.m_pa);
+ chunk = erofs_get_unhashed_chunk(deviceid, blkaddr, 0);
+ if (IS_ERR(chunk)) {
+ ret = PTR_ERR(chunk);
+ goto err;
+ }
+ *(void **)idx++ = chunk;
+ erofs_dbg("\t%s: chunk %d (deviceid %u, blkaddr %u)",
+ inode->i_srcpath, i, deviceid, blkaddr);
+
+ }
+ inode->datalayout = EROFS_INODE_CHUNK_BASED;
+ return 0;
+err:
+ free(idx);
+ inode->chunkindexes = NULL;
+ return ret;
+}
+
+static int erofs_rebuild_fill_inode(struct erofs_inode *inode,
+ struct erofs_inode *vi, dev_t dev)
+{
+ int ret = 0;
+
+ inode->i_srcpath = strdup(vi->i_srcpath);
+ inode->i_mode = vi->i_mode;
+ inode->i_uid = vi->i_uid;
+ inode->i_gid = vi->i_gid;
+ inode->i_mtime = vi->i_mtime;
+ inode->i_ino[1] = vi->i_ino[0];
+ inode->i_nlink = 1;
+
+ switch (inode->i_mode & S_IFMT) {
+ case S_IFCHR:
+ case S_IFBLK:
+ case S_IFIFO:
+ case S_IFSOCK:
+ inode->u.i_rdev = erofs_new_encode_dev(vi->u.i_rdev);
+ inode->i_size = 0;
+ erofs_dbg("\tdev: %d %d", major(vi->u.i_rdev),
+ minor(vi->u.i_rdev));
+ break;
+ case S_IFDIR:
+ inode->i_size = 0;
+ ret = erofs_init_empty_dir(inode);
+ break;
+ case S_IFLNK:
+ inode->i_size = vi->i_size;
+ inode->i_link = malloc(inode->i_size + 1);
+ ret = erofs_pread(vi, inode->i_link, inode->i_size, 0);
+ erofs_dbg("\tsymlink: %s -> %s", inode->i_srcpath, inode->i_link);
+ break;
+ case S_IFREG:
+ inode->i_size = vi->i_size;
+ if (inode->i_size)
+ ret = erofs_rebuild_fill_inode_map(inode, vi, dev);
+ else
+ inode->u.i_blkaddr = NULL_ADDR;
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+ return ret;
+}
+
+struct erofs_rebuild_dir_context {
+ struct erofs_dir_context ctx;
+ struct erofs_inode *root;
+ dev_t dev;
+};
+
+static int erofs_rebuild_dirent_iter(struct erofs_dir_context *ctx)
+{
+ struct erofs_rebuild_dir_context *rctx = (void *)ctx;
+ struct erofs_inode *dir = ctx->dir;
+ struct erofs_sb_info *sbi = dir->sbi;
+ struct erofs_inode *inode, vi = {};
+ struct erofs_dentry *d;
+ char *path;
+ int ret;
+
+ if (ctx->dot_dotdot)
+ return 0;
+
+ vi.nid = ctx->de_nid;
+ vi.sbi = sbi;
+ ret = erofs_read_inode_from_disk(&vi);
+ if (ret) {
+ erofs_err("file %s/%.*s: read failed",
+ dir->i_srcpath, ctx->de_namelen, ctx->dname);
+ return ret;
+ }
+
+ ret = asprintf(&path, "%s/%.*s", dir->i_srcpath, ctx->de_namelen,
+ ctx->dname);
+ if (ret < 0) {
+ erofs_err("file %s/%.*s: failed to alloc path",
+ dir->i_srcpath, ctx->de_namelen, ctx->dname);
+ return ret;
+ }
+ vi.i_srcpath = path;
+
+ d = erofs_rebuild_get_dentry(rctx->root, path, false, NULL, NULL);
+ if (IS_ERR(d)) {
+ ret = PTR_ERR(d);
+ goto exit;
+ }
+
+ if (d->type != EROFS_FT_UNKNOWN) {
+ inode = d->inode;
+ DBG_BUGON((inode->i_mode & S_IFMT) != (vi.i_mode & S_IFMT));
+ if (!S_ISDIR(inode->i_mode) || inode->opaque) {
+ erofs_dbg("file %s: %s (%d) exists",
+ path, inode->i_srcpath,
+ erofs_mode_to_ftype(inode->i_mode));
+ free(path);
+ return 0;
+ }
+ erofs_dbg("dir %s: %s merging", path, inode->i_srcpath);
+ } else {
+ erofs_dbg("loading file: %s (%d) (nid %llu)",
+ path, erofs_mode_to_ftype(vi.i_mode), ctx->de_nid);
+ if (S_ISREG(vi.i_mode) && vi.i_nlink > 1 &&
+ (inode = erofs_iget(rctx->dev, vi.i_ino[0]))) {
+ /* hardlink file */
+ if (S_ISDIR(inode->i_mode)) {
+ erofs_dbg("hardlink directory not supported");
+ erofs_iput(inode);
+ return -EISDIR;
+ }
+ inode->i_nlink++;
+ erofs_dbg("\thardlink: %s -> %s", vi.i_srcpath, inode->i_srcpath);
+ } else {
+ inode = erofs_new_inode();
+ if (IS_ERR(inode)) {
+ ret = PTR_ERR(inode);
+ goto exit;
+ }
+
+ inode->i_parent = d->inode;
+ inode->dev = rctx->dev;
+ ret = erofs_rebuild_fill_inode(inode, &vi, rctx->dev);
+ if (ret)
+ goto exit_inode;
+
+ ret = erofs_read_xattrs_from_disk(&vi);
+ if (ret)
+ goto exit_inode;
+ list_splice_tail(&vi.i_xattrs, &inode->i_xattrs);
+ erofs_inode_tag_opaque(inode);
+
+ erofs_insert_ihash(inode, rctx->dev, inode->i_ino[1]);
+ }
+
+ d->inode = inode;
+ d->type = erofs_mode_to_ftype(inode->i_mode);
+ }
+
+ ret = 0;
+ if (S_ISDIR(vi.i_mode)) {
+ struct erofs_rebuild_dir_context nctx = *rctx;
+ nctx.ctx.dir = &vi;
+ ret = erofs_iterate_dir(&nctx.ctx, false);
+ }
+exit:
+ free(path);
+ return ret;
+exit_inode:
+ erofs_iput(inode);
+ goto exit;
+}
+
+int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi)
+{
+ struct erofs_inode inode = {};
+ struct erofs_rebuild_dir_context ctx;
+ int ret;
+
+ if (!sbi->devname) {
+ erofs_err("please open the device first");
+ return -EINVAL;
+ }
+
+ ret = erofs_read_superblock(sbi);
+ if (ret) {
+ erofs_err("failed to read superblock of img %s", sbi->devname);
+ return ret;
+ }
+
+ inode.nid = sbi->root_nid;
+ inode.sbi = sbi;
+ ret = erofs_read_inode_from_disk(&inode);
+ if (ret) {
+ erofs_err("failed to read root inode of img %s", sbi->devname);
+ return ret;
+ }
+ inode.i_srcpath = strdup("/");
+
+ ctx = (struct erofs_rebuild_dir_context) {
+ .ctx.dir = &inode,
+ .ctx.cb = erofs_rebuild_dirent_iter,
+ .root = root,
+ .dev = sbi->dev,
+ };
+ ret = erofs_iterate_dir(&ctx.ctx, false);
+ free(inode.i_srcpath);
+ return ret;
+}
diff --git a/mkfs/Makefile.am b/mkfs/Makefile.am
index 603c2f3..dd75485 100644
--- a/mkfs/Makefile.am
+++ b/mkfs/Makefile.am
@@ -6,4 +6,5 @@ AM_CPPFLAGS = ${libselinux_CFLAGS}
mkfs_erofs_SOURCES = main.c
mkfs_erofs_CFLAGS = -Wall -I$(top_srcdir)/include
mkfs_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \
- ${libuuid_LIBS} ${liblz4_LIBS} ${liblzma_LIBS} ${libdeflate_LIBS}
+ ${libuuid_LIBS} ${liblz4_LIBS} ${liblzma_LIBS} ${zlib_LIBS} \
+ ${libdeflate_LIBS}
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 23+ messages in thread* [PATCH 13/13] erofs-utils: mkfs: introduce rebuild mode
2023-08-14 3:42 [PATCH 00/13] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (11 preceding siblings ...)
2023-08-14 3:42 ` [PATCH 12/13] erofs-utils: lib: add erofs_rebuild_load_tree() helper Jingbo Xu
@ 2023-08-14 3:42 ` Jingbo Xu
12 siblings, 0 replies; 23+ messages in thread
From: Jingbo Xu @ 2023-08-14 3:42 UTC (permalink / raw)
To: xiang, linux-erofs
Introduce a new rebuild mode merging multiple erofs images generated
from either tarerfs index mode (--tar=i):
mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar
...
mkfs.erofs --tar=i --aufs layerN.erofs layerN.tar
mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs
or tarerofs non-index mode:
mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar
...
mkfs.erofs --tar=f -Enoinline_data --aufs layerN.erofs layerN.tar
mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs
When calling rebuild mode, the top most layer must be specified firstly
in the cmdline, while the bottom most layer i.e. layer0 the lastly.
The rebuild mode doesn't support flat inline datalayout yet.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
include/erofs/internal.h | 2 +
mkfs/main.c | 196 +++++++++++++++++++++++++++++++--------
2 files changed, 157 insertions(+), 41 deletions(-)
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 513e860..21c6751 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -105,6 +105,8 @@ struct erofs_sb_info {
dev_t dev;
unsigned int nblobs;
unsigned int blobfd[256];
+
+ struct list_head list;
};
/* make sure that any user of the erofs headers has atleast 64bit off_t type */
diff --git a/mkfs/main.c b/mkfs/main.c
index 269ccf4..cda2654 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -26,6 +26,7 @@
#include "erofs/compress_hints.h"
#include "erofs/blobchunk.h"
#include "erofs/fragments.h"
+#include "erofs/rebuild.h"
#include "../lib/liberofs_private.h"
#include "../lib/liberofs_uuid.h"
@@ -134,6 +135,10 @@ static void usage(void)
static unsigned int pclustersize_packed, pclustersize_max;
static struct erofs_tarfile erofstar;
static bool tar_mode;
+static bool rebuild_mode;
+
+static unsigned int rebuild_src_count;
+static LIST_HEAD(rebuild_src_list);
static int parse_extended_opts(const char *opts)
{
@@ -274,10 +279,23 @@ static int mkfs_parse_compress_algs(char *algs)
return 0;
}
+static void erofs_rebuild_cleanup(void)
+{
+ struct erofs_sb_info *src, *n;
+
+ list_for_each_entry_safe(src, n, &rebuild_src_list, list) {
+ list_del(&src->list);
+ dev_close(src);
+ free(src);
+ }
+ rebuild_src_count = 0;
+}
+
static int mkfs_parse_options_cfg(int argc, char *argv[])
{
char *endptr;
- int opt, i;
+ struct stat st;
+ int opt, i, err;
bool quiet = false;
while ((opt = getopt_long(argc, argv, "C:E:L:T:U:b:d:x:z:",
@@ -533,7 +551,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
} else {
erofstar.fd = STDIN_FILENO;
}
- }else {
+ } else {
cfg.c_src_path = realpath(argv[optind++], NULL);
if (!cfg.c_src_path) {
erofs_err("failed to parse source directory: %s",
@@ -541,11 +559,51 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
return -ENOENT;
}
- if (optind < argc) {
+ if (tar_mode) {
+ erofstar.fd = open(cfg.c_src_path, O_RDONLY);
+ if (erofstar.fd < 0) {
+ erofs_err("failed to open file: %s", cfg.c_src_path);
+ usage();
+ return -errno;
+ }
+ } else {
+ err = lstat(cfg.c_src_path, &st);
+ if (err)
+ return -errno;
+ if (S_ISDIR(st.st_mode))
+ erofs_set_fs_root(cfg.c_src_path);
+ else
+ rebuild_mode = true;
+ }
+
+ if (rebuild_mode) {
+ optind--; /* roll back to retrieve the first source */
+ while (optind < argc) {
+ struct erofs_sb_info *src;
+
+ src = malloc(sizeof(struct erofs_sb_info));
+ if (!src) {
+ erofs_rebuild_cleanup();
+ return -ENOMEM;
+ }
+
+ err = dev_open_ro(src, argv[optind++]);
+ if (err) {
+ free(src);
+ erofs_rebuild_cleanup();
+ return err;
+ }
+
+ /* extra device index starts from 1 */
+ src->dev = ++rebuild_src_count;
+ list_add_tail(&src->list, &rebuild_src_list);
+ }
+ } else if (optind < argc) {
erofs_err("unexpected argument: %s\n", argv[optind]);
return -EINVAL;
}
}
+
if (quiet) {
cfg.c_dbg_lvl = EROFS_ERR;
cfg.c_showprogress = false;
@@ -730,13 +788,63 @@ void erofs_show_progs(int argc, char *argv[])
printf("%s %s\n", basename(argv[0]), cfg.c_version);
}
+struct erofs_inode *erofs_rebuild_load_trees(void)
+{
+ struct erofs_sb_info *src;
+ struct erofs_inode *root;
+ unsigned int extra_devices = 0;
+ erofs_blk_t nblocks;
+ int ret, i = 0;
+
+ root = erofs_new_inode();
+ if (IS_ERR(root))
+ return root;
+ root->i_srcpath = strdup("/");
+ root->i_mode = S_IFDIR | 0777;
+ root->i_parent = root;
+ root->i_mtime = root->sbi->build_time;
+ root->i_mtime_nsec = root->sbi->build_time_nsec;
+ erofs_init_empty_dir(root);
+
+ list_for_each_entry(src, &rebuild_src_list, list) {
+ ret = erofs_rebuild_load_tree(root, src);
+ if (ret)
+ return ERR_PTR(ret);
+
+ if (src->extra_devices > 1) {
+ erofs_err("%s: unsupported number of extra devices",
+ src->devname, src->extra_devices);
+ return ERR_PTR(-EOPNOTSUPP);
+ }
+ extra_devices += src->extra_devices;
+ }
+
+ if (extra_devices && extra_devices != rebuild_src_count) {
+ erofs_err("unsupported mix of source images");
+ return ERR_PTR(-EOPNOTSUPP);
+ }
+
+ ret = erofs_mkfs_init_devices(&sbi, rebuild_src_count);
+ if (ret)
+ return ERR_PTR(ret);
+
+ list_for_each_entry(src, &rebuild_src_list, list) {
+ if (extra_devices)
+ nblocks = src->devs[0].blocks;
+ else
+ nblocks = src->primarydevice_blocks;
+ sbi.devs[i++].blocks = nblocks;
+ }
+
+ return root;
+}
+
int main(int argc, char **argv)
{
int err = 0;
struct erofs_buffer_head *sb_bh;
struct erofs_inode *root_inode, *packed_inode;
erofs_nid_t root_nid, packed_nid;
- struct stat st;
erofs_blk_t nblocks;
struct timeval t;
char uuid_str[37];
@@ -759,26 +867,6 @@ int main(int argc, char **argv)
return 1;
}
- if (!tar_mode) {
- err = lstat(cfg.c_src_path, &st);
- if (err)
- return 1;
- if (!S_ISDIR(st.st_mode)) {
- erofs_err("root of the filesystem is not a directory - %s",
- cfg.c_src_path);
- usage();
- return 1;
- }
- erofs_set_fs_root(cfg.c_src_path);
- } else if (cfg.c_src_path) {
- erofstar.fd = open(cfg.c_src_path, O_RDONLY);
- if (erofstar.fd < 0) {
- erofs_err("failed to open file: %s", cfg.c_src_path);
- usage();
- return 1;
- }
- }
-
if (cfg.c_unix_timestamp != -1) {
sbi.build_time = cfg.c_unix_timestamp;
sbi.build_time_nsec = 0;
@@ -845,6 +933,21 @@ int main(int argc, char **argv)
}
}
+ if (rebuild_mode) {
+ struct erofs_sb_info *src;
+
+ src = list_first_entry(&rebuild_src_list, struct erofs_sb_info, list);
+ if (!src)
+ goto exit;
+ err = erofs_read_superblock(src);
+ if (err) {
+ erofs_err("failed to read superblock of img %s", src->devname);
+ goto exit;
+ }
+ sbi.blkszbits = src->blkszbits;
+ erofs_dbg("sbi.blkszbits %d", sbi.blkszbits);
+ }
+
sb_bh = erofs_buffer_init();
if (IS_ERR(sb_bh)) {
err = PTR_ERR(sb_bh);
@@ -912,23 +1015,7 @@ int main(int argc, char **argv)
erofs_inode_manager_init();
- if (!tar_mode) {
- err = erofs_build_shared_xattrs_from_path(&sbi, cfg.c_src_path);
- if (err) {
- erofs_err("failed to build shared xattrs: %s",
- erofs_strerror(err));
- goto exit;
- }
-
- if (cfg.c_extra_ea_name_prefixes)
- erofs_xattr_write_name_prefixes(&sbi, packedfile);
-
- root_inode = erofs_mkfs_build_tree_from_path(cfg.c_src_path);
- if (IS_ERR(root_inode)) {
- err = PTR_ERR(root_inode);
- goto exit;
- }
- } else {
+ if (tar_mode) {
root_inode = erofs_new_inode();
if (IS_ERR(root_inode)) {
err = PTR_ERR(root_inode);
@@ -949,6 +1036,32 @@ int main(int argc, char **argv)
err = erofs_rebuild_dump_tree(root_inode, false);
if (err < 0)
goto exit;
+ } else if (rebuild_mode) {
+ root_inode = erofs_rebuild_load_trees();
+ if (IS_ERR(root_inode)) {
+ err = PTR_ERR(root_inode);
+ goto exit;
+ }
+
+ err = erofs_rebuild_dump_tree(root_inode, true);
+ if (err)
+ goto exit;
+ } else {
+ err = erofs_build_shared_xattrs_from_path(&sbi, cfg.c_src_path);
+ if (err) {
+ erofs_err("failed to build shared xattrs: %s",
+ erofs_strerror(err));
+ goto exit;
+ }
+
+ if (cfg.c_extra_ea_name_prefixes)
+ erofs_xattr_write_name_prefixes(&sbi, packedfile);
+
+ root_inode = erofs_mkfs_build_tree_from_path(cfg.c_src_path);
+ if (IS_ERR(root_inode)) {
+ err = PTR_ERR(root_inode);
+ goto exit;
+ }
}
root_nid = erofs_lookupnid(root_inode);
erofs_iput(root_inode);
@@ -1002,6 +1115,7 @@ exit:
z_erofs_fragments_exit();
erofs_packedfile_exit();
erofs_xattr_cleanup_name_prefixes();
+ erofs_rebuild_cleanup();
erofs_exit_configure();
if (err) {
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 23+ messages in thread