* [PATCH v3 01/11] erofs-utils: lib: fix erofs_init_devices
2023-08-22 9:24 [PATCH v3 00/11] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
@ 2023-08-22 9:24 ` Jingbo Xu
2023-08-22 9:26 ` Jingbo Xu
2023-08-22 9:24 ` [PATCH v3 02/11] erofs-utils: lib: scan devtable if extra_devices is not specified Jingbo Xu
` (9 subsequent siblings)
10 siblings, 1 reply; 14+ messages in thread
From: Jingbo Xu @ 2023-08-22 9:24 UTC (permalink / raw)
To: xiang, linux-erofs
Fix the missing le32_to_cpu() when parsing the device slot. Read and
cache the number of blocks of each device for later use.
Fixes: 0ce853a01123 ("erofs-utils: fuse: add multiple device support")
[1] https://github.com/erofs/erofsnightly/actions/runs/5921003885/job/16053013007
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
lib/super.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/lib/super.c b/lib/super.c
index 21dc51f..4fe81c3 100644
--- a/lib/super.c
+++ b/lib/super.c
@@ -60,8 +60,9 @@ static int erofs_init_devices(struct erofs_sb_info *sbi,
return ret;
}
- sbi->devs[i].mapped_blkaddr = dis.mapped_blkaddr;
- sbi->total_blocks += dis.blocks;
+ sbi->devs[i].mapped_blkaddr = le32_to_cpu(dis.mapped_blkaddr);
+ sbi->devs[i].blocks = le32_to_cpu(dis.blocks);
+ sbi->total_blocks += sbi->devs[i].blocks;
pos += EROFS_DEVT_SLOT_SIZE;
}
return 0;
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH v3 02/11] erofs-utils: lib: scan devtable if extra_devices is not specified
2023-08-22 9:24 [PATCH v3 00/11] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
2023-08-22 9:24 ` [PATCH v3 01/11] erofs-utils: lib: fix erofs_init_devices Jingbo Xu
@ 2023-08-22 9:24 ` Jingbo Xu
2023-08-23 3:15 ` Gao Xiang
2023-08-22 9:24 ` [PATCH v3 03/11] erofs-utils: lib: fix dirent type of whiteout in tarerofs Jingbo Xu
` (8 subsequent siblings)
10 siblings, 1 reply; 14+ messages in thread
From: Jingbo Xu @ 2023-08-22 9:24 UTC (permalink / raw)
To: xiang, linux-erofs
Scan the devtable (if any) automatically when reading superblock from
disk if sbi->extra_devices is not specified, and initialize
sbi->extra_devices with the number of on-disk device slots.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
lib/super.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lib/super.c b/lib/super.c
index 4fe81c3..fc709fc 100644
--- a/lib/super.c
+++ b/lib/super.c
@@ -37,7 +37,8 @@ static int erofs_init_devices(struct erofs_sb_info *sbi,
else
ondisk_extradevs = le16_to_cpu(dsb->extra_devices);
- if (ondisk_extradevs != sbi->extra_devices) {
+ if (sbi->extra_devices &&
+ ondisk_extradevs != sbi->extra_devices) {
erofs_err("extra devices don't match (ondisk %u, given %u)",
ondisk_extradevs, sbi->extra_devices);
return -EINVAL;
@@ -45,6 +46,7 @@ static int erofs_init_devices(struct erofs_sb_info *sbi,
if (!ondisk_extradevs)
return 0;
+ sbi->extra_devices = ondisk_extradevs;
sbi->device_id_mask = roundup_pow_of_two(ondisk_extradevs + 1) - 1;
sbi->devs = calloc(ondisk_extradevs, sizeof(*sbi->devs));
if (!sbi->devs)
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 14+ messages in thread* Re: [PATCH v3 02/11] erofs-utils: lib: scan devtable if extra_devices is not specified
2023-08-22 9:24 ` [PATCH v3 02/11] erofs-utils: lib: scan devtable if extra_devices is not specified Jingbo Xu
@ 2023-08-23 3:15 ` Gao Xiang
0 siblings, 0 replies; 14+ messages in thread
From: Gao Xiang @ 2023-08-23 3:15 UTC (permalink / raw)
To: Jingbo Xu, xiang, linux-erofs
On 2023/8/22 17:24, Jingbo Xu wrote:
> Scan the devtable (if any) automatically when reading superblock from
> disk if sbi->extra_devices is not specified, and initialize
> sbi->extra_devices with the number of on-disk device slots.
>
> Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
> ---
> lib/super.c | 4 +++-
> 1 file changed, 3 insertions(+), 1 deletion(-)
>
> diff --git a/lib/super.c b/lib/super.c
> index 4fe81c3..fc709fc 100644
> --- a/lib/super.c
> +++ b/lib/super.c
> @@ -37,7 +37,8 @@ static int erofs_init_devices(struct erofs_sb_info *sbi,
> else
> ondisk_extradevs = le16_to_cpu(dsb->extra_devices);
>
> - if (ondisk_extradevs != sbi->extra_devices) {
> + if (sbi->extra_devices &&
> + ondisk_extradevs != sbi->extra_devices) {
If you'd like to keep in sync with the kernel
erofs_scan_devices(), please point it out explicitly in the commit message.
Thanks,
Gao Xiang
> erofs_err("extra devices don't match (ondisk %u, given %u)",
> ondisk_extradevs, sbi->extra_devices);
> return -EINVAL;
> @@ -45,6 +46,7 @@ static int erofs_init_devices(struct erofs_sb_info *sbi,
> if (!ondisk_extradevs)
> return 0;
>
> + sbi->extra_devices = ondisk_extradevs;
> sbi->device_id_mask = roundup_pow_of_two(ondisk_extradevs + 1) - 1;
> sbi->devs = calloc(ondisk_extradevs, sizeof(*sbi->devs));
> if (!sbi->devs)
^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH v3 03/11] erofs-utils: lib: fix dirent type of whiteout in tarerofs
2023-08-22 9:24 [PATCH v3 00/11] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
2023-08-22 9:24 ` [PATCH v3 01/11] erofs-utils: lib: fix erofs_init_devices Jingbo Xu
2023-08-22 9:24 ` [PATCH v3 02/11] erofs-utils: lib: scan devtable if extra_devices is not specified Jingbo Xu
@ 2023-08-22 9:24 ` Jingbo Xu
2023-08-22 9:24 ` [PATCH v3 04/11] erofs-utils: lib: add list_splice_tail() helper Jingbo Xu
` (7 subsequent siblings)
10 siblings, 0 replies; 14+ messages in thread
From: Jingbo Xu @ 2023-08-22 9:24 UTC (permalink / raw)
To: xiang, linux-erofs
Set the correct dirent type for whiteout.
Fixes: 95d315fd7958 ("erofs-utils: introduce tarerofs")
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
lib/tar.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/tar.c b/lib/tar.c
index 42590d2..328ab98 100644
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -694,6 +694,7 @@ new_inode:
if (whout) {
inode->i_mode = (inode->i_mode & ~S_IFMT) | S_IFCHR;
inode->u.i_rdev = EROFS_WHITEOUT_DEV;
+ d->type = EROFS_FT_CHRDEV;
} else {
inode->i_mode = st.st_mode;
if (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode))
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH v3 04/11] erofs-utils: lib: add list_splice_tail() helper
2023-08-22 9:24 [PATCH v3 00/11] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (2 preceding siblings ...)
2023-08-22 9:24 ` [PATCH v3 03/11] erofs-utils: lib: fix dirent type of whiteout in tarerofs Jingbo Xu
@ 2023-08-22 9:24 ` Jingbo Xu
2023-08-22 9:24 ` [PATCH v3 05/11] erofs-utils: lib: make erofs_get_unhashed_chunk() global Jingbo Xu
` (6 subsequent siblings)
10 siblings, 0 replies; 14+ messages in thread
From: Jingbo Xu @ 2023-08-22 9:24 UTC (permalink / raw)
To: xiang, linux-erofs
Add list_splice_tail() helper.
Reviewed-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
include/erofs/list.h | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/include/erofs/list.h b/include/erofs/list.h
index 3f5da1a..d7a9fee 100644
--- a/include/erofs/list.h
+++ b/include/erofs/list.h
@@ -70,6 +70,26 @@ static inline int list_empty(struct list_head *head)
return head->next == head;
}
+static inline void __list_splice(struct list_head *list,
+ struct list_head *prev, struct list_head *next)
+{
+ struct list_head *first = list->next;
+ struct list_head *last = list->prev;
+
+ first->prev = prev;
+ prev->next = first;
+
+ last->next = next;
+ next->prev = last;
+}
+
+static inline void list_splice_tail(struct list_head *list,
+ struct list_head *head)
+{
+ if (!list_empty(list))
+ __list_splice(list, head->prev, head);
+}
+
#define list_entry(ptr, type, member) container_of(ptr, type, member)
#define list_first_entry(ptr, type, member) \
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH v3 05/11] erofs-utils: lib: make erofs_get_unhashed_chunk() global
2023-08-22 9:24 [PATCH v3 00/11] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (3 preceding siblings ...)
2023-08-22 9:24 ` [PATCH v3 04/11] erofs-utils: lib: add list_splice_tail() helper Jingbo Xu
@ 2023-08-22 9:24 ` Jingbo Xu
2023-08-22 9:24 ` [PATCH v3 06/11] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper Jingbo Xu
` (5 subsequent siblings)
10 siblings, 0 replies; 14+ messages in thread
From: Jingbo Xu @ 2023-08-22 9:24 UTC (permalink / raw)
To: xiang, linux-erofs
... so that it could be called from outside blobchunk.c later.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
include/erofs/blobchunk.h | 2 ++
lib/blobchunk.c | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/include/erofs/blobchunk.h b/include/erofs/blobchunk.h
index 010aee1..fb85d8e 100644
--- a/include/erofs/blobchunk.h
+++ b/include/erofs/blobchunk.h
@@ -14,6 +14,8 @@ extern "C"
#include "erofs/internal.h"
+struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
+ erofs_blk_t blkaddr, erofs_off_t sourceoffset);
int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, erofs_off_t off);
int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd);
int tarerofs_write_chunkes(struct erofs_inode *inode, erofs_off_t data_offset);
diff --git a/lib/blobchunk.c b/lib/blobchunk.c
index cada5bb..07f18bd 100644
--- a/lib/blobchunk.c
+++ b/lib/blobchunk.c
@@ -38,7 +38,7 @@ struct erofs_blobchunk erofs_holechunk = {
};
static LIST_HEAD(unhashed_blobchunks);
-static struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
+struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
erofs_blk_t blkaddr, erofs_off_t sourceoffset)
{
struct erofs_blobchunk *chunk;
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH v3 06/11] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
2023-08-22 9:24 [PATCH v3 00/11] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (4 preceding siblings ...)
2023-08-22 9:24 ` [PATCH v3 05/11] erofs-utils: lib: make erofs_get_unhashed_chunk() global Jingbo Xu
@ 2023-08-22 9:24 ` Jingbo Xu
2023-08-22 9:24 ` [PATCH v3 07/11] erofs-utils: lib: add erofs_insert_ihash() helper Jingbo Xu
` (4 subsequent siblings)
10 siblings, 0 replies; 14+ messages in thread
From: Jingbo Xu @ 2023-08-22 9:24 UTC (permalink / raw)
To: xiang, linux-erofs
Add erofs_read_xattrs_from_disk() helper reading extended attributes
from disk, add checking if it's an opaque directory.
Move all xattr name related macros to xattr.c and introduce
erofs_set_opaque_xattr() helper to hide all these details.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
include/erofs/internal.h | 1 +
include/erofs/xattr.h | 33 +-----------
lib/tar.c | 7 +--
lib/xattr.c | 112 +++++++++++++++++++++++++++++++++++++++
4 files changed, 116 insertions(+), 37 deletions(-)
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 3e73eef..455a73a 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -190,6 +190,7 @@ struct erofs_inode {
bool compressed_idata;
bool lazy_tailblock;
bool with_tmpfile;
+ bool opaque;
unsigned int xattr_isize;
unsigned int extent_isize;
diff --git a/include/erofs/xattr.h b/include/erofs/xattr.h
index 748442a..2c8f8dd 100644
--- a/include/erofs/xattr.h
+++ b/include/erofs/xattr.h
@@ -43,37 +43,6 @@ static inline unsigned int xattrblock_offset(struct erofs_inode *vi,
(_size - sizeof(struct erofs_xattr_ibody_header)) / \
sizeof(struct erofs_xattr_entry) + 1; })
-#ifndef XATTR_SYSTEM_PREFIX
-#define XATTR_SYSTEM_PREFIX "system."
-#endif
-#ifndef XATTR_SYSTEM_PREFIX_LEN
-#define XATTR_SYSTEM_PREFIX_LEN (sizeof(XATTR_SYSTEM_PREFIX) - 1)
-#endif
-#ifndef XATTR_USER_PREFIX
-#define XATTR_USER_PREFIX "user."
-#endif
-#ifndef XATTR_USER_PREFIX_LEN
-#define XATTR_USER_PREFIX_LEN (sizeof(XATTR_USER_PREFIX) - 1)
-#endif
-#ifndef XATTR_SECURITY_PREFIX
-#define XATTR_SECURITY_PREFIX "security."
-#endif
-#ifndef XATTR_SECURITY_PREFIX_LEN
-#define XATTR_SECURITY_PREFIX_LEN (sizeof(XATTR_SECURITY_PREFIX) - 1)
-#endif
-#ifndef XATTR_TRUSTED_PREFIX
-#define XATTR_TRUSTED_PREFIX "trusted."
-#endif
-#ifndef XATTR_TRUSTED_PREFIX_LEN
-#define XATTR_TRUSTED_PREFIX_LEN (sizeof(XATTR_TRUSTED_PREFIX) - 1)
-#endif
-#ifndef XATTR_NAME_POSIX_ACL_ACCESS
-#define XATTR_NAME_POSIX_ACL_ACCESS "system.posix_acl_access"
-#endif
-#ifndef XATTR_NAME_POSIX_ACL_DEFAULT
-#define XATTR_NAME_POSIX_ACL_DEFAULT "system.posix_acl_default"
-#endif
-
int erofs_scan_file_xattrs(struct erofs_inode *inode);
int erofs_prepare_xattr_ibody(struct erofs_inode *inode);
char *erofs_export_xattr_ibody(struct list_head *ixattrs, unsigned int size);
@@ -87,6 +56,8 @@ int erofs_xattr_prefixes_init(struct erofs_sb_info *sbi);
int erofs_setxattr(struct erofs_inode *inode, char *key,
const void *value, size_t size);
+int erofs_set_opaque_xattr(struct erofs_inode *inode);
+int erofs_read_xattrs_from_disk(struct erofs_inode *inode);
#ifdef __cplusplus
}
diff --git a/lib/tar.c b/lib/tar.c
index 328ab98..ce54d17 100644
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -19,11 +19,6 @@
#include "erofs/xattr.h"
#include "erofs/blobchunk.h"
-#define OVL_XATTR_NAMESPACE "overlay."
-#define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE
-#define OVL_XATTR_OPAQUE_POSTFIX "opaque"
-#define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX
-
#define EROFS_WHITEOUT_DEV 0
static char erofs_libbuf[16384];
@@ -633,7 +628,7 @@ restart:
} else if (opq) {
DBG_BUGON(d->type == EROFS_FT_UNKNOWN);
DBG_BUGON(!d->inode);
- ret = erofs_setxattr(d->inode, OVL_XATTR_OPAQUE, "y", 1);
+ ret = erofs_set_opaque_xattr(d->inode);
goto out;
} else if (th.typeflag == '1') { /* hard link cases */
struct erofs_dentry *d2;
diff --git a/lib/xattr.c b/lib/xattr.c
index 46a301a..a164cc3 100644
--- a/lib/xattr.c
+++ b/lib/xattr.c
@@ -20,6 +20,49 @@
#include "erofs/fragments.h"
#include "liberofs_private.h"
+#ifndef XATTR_SYSTEM_PREFIX
+#define XATTR_SYSTEM_PREFIX "system."
+#endif
+#ifndef XATTR_SYSTEM_PREFIX_LEN
+#define XATTR_SYSTEM_PREFIX_LEN (sizeof(XATTR_SYSTEM_PREFIX) - 1)
+#endif
+#ifndef XATTR_USER_PREFIX
+#define XATTR_USER_PREFIX "user."
+#endif
+#ifndef XATTR_USER_PREFIX_LEN
+#define XATTR_USER_PREFIX_LEN (sizeof(XATTR_USER_PREFIX) - 1)
+#endif
+#ifndef XATTR_SECURITY_PREFIX
+#define XATTR_SECURITY_PREFIX "security."
+#endif
+#ifndef XATTR_SECURITY_PREFIX_LEN
+#define XATTR_SECURITY_PREFIX_LEN (sizeof(XATTR_SECURITY_PREFIX) - 1)
+#endif
+#ifndef XATTR_TRUSTED_PREFIX
+#define XATTR_TRUSTED_PREFIX "trusted."
+#endif
+#ifndef XATTR_TRUSTED_PREFIX_LEN
+#define XATTR_TRUSTED_PREFIX_LEN (sizeof(XATTR_TRUSTED_PREFIX) - 1)
+#endif
+#ifndef XATTR_NAME_POSIX_ACL_ACCESS
+#define XATTR_NAME_POSIX_ACL_ACCESS "system.posix_acl_access"
+#endif
+#ifndef XATTR_NAME_POSIX_ACL_DEFAULT
+#define XATTR_NAME_POSIX_ACL_DEFAULT "system.posix_acl_default"
+#endif
+#ifndef OVL_XATTR_NAMESPACE
+#define OVL_XATTR_NAMESPACE "overlay."
+#endif
+#ifndef OVL_XATTR_OPAQUE_POSTFIX
+#define OVL_XATTR_OPAQUE_POSTFIX "opaque"
+#endif
+#ifndef OVL_XATTR_TRUSTED_PREFIX
+#define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE
+#endif
+#ifndef OVL_XATTR_OPAQUE
+#define OVL_XATTR_OPAQUE OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_OPAQUE_POSTFIX
+#endif
+
#define EA_HASHTABLE_BITS 16
struct xattr_item {
@@ -442,6 +485,11 @@ int erofs_setxattr(struct erofs_inode *inode, char *key,
return erofs_xattr_add(&inode->i_xattrs, item);
}
+int erofs_set_opaque_xattr(struct erofs_inode *inode)
+{
+ return erofs_setxattr(inode, OVL_XATTR_OPAQUE, "y", 1);
+}
+
#ifdef WITH_ANDROID
static int erofs_droid_xattr_set_caps(struct erofs_inode *inode)
{
@@ -500,6 +548,70 @@ int erofs_scan_file_xattrs(struct erofs_inode *inode)
return erofs_droid_xattr_set_caps(inode);
}
+int erofs_read_xattrs_from_disk(struct erofs_inode *inode)
+{
+ ssize_t kllen;
+ char *keylst, *key;
+ int ret;
+
+ init_list_head(&inode->i_xattrs);
+ kllen = erofs_listxattr(inode, NULL, 0);
+ if (kllen < 0)
+ return kllen;
+ if (kllen <= 1)
+ return 0;
+
+ keylst = malloc(kllen);
+ if (!keylst)
+ return -ENOMEM;
+
+ ret = erofs_listxattr(inode, keylst, kllen);
+ if (ret < 0)
+ goto out;
+
+ for (key = keylst; key < keylst + kllen; key += strlen(key) + 1) {
+ void *value = NULL;
+ size_t size = 0;
+
+ if (!strcmp(key, OVL_XATTR_OPAQUE)) {
+ if (!S_ISDIR(inode->i_mode)) {
+ erofs_dbg("file %s: opaque xattr on non-dir",
+ inode->i_srcpath);
+ ret = -EINVAL;
+ goto out;
+ }
+ inode->opaque = true;
+ }
+
+ ret = erofs_getxattr(inode, key, NULL, 0);
+ if (ret < 0)
+ goto out;
+ if (ret) {
+ size = ret;
+ value = malloc(size);
+ if (!value) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ ret = erofs_getxattr(inode, key, value, size);
+ if (ret < 0) {
+ free(value);
+ goto out;
+ }
+ DBG_BUGON(ret != size);
+ }
+
+ ret = erofs_setxattr(inode, key, value, size);
+ free(value);
+ if (ret)
+ break;
+ }
+out:
+ free(keylst);
+ return ret;
+}
+
int erofs_prepare_xattr_ibody(struct erofs_inode *inode)
{
int ret;
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH v3 07/11] erofs-utils: lib: add erofs_insert_ihash() helper
2023-08-22 9:24 [PATCH v3 00/11] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (5 preceding siblings ...)
2023-08-22 9:24 ` [PATCH v3 06/11] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper Jingbo Xu
@ 2023-08-22 9:24 ` Jingbo Xu
2023-08-22 9:24 ` [PATCH v3 08/11] erofs-utils: lib: add erofs_rebuild_dump_tree() helper Jingbo Xu
` (3 subsequent siblings)
10 siblings, 0 replies; 14+ messages in thread
From: Jingbo Xu @ 2023-08-22 9:24 UTC (permalink / raw)
To: xiang, linux-erofs
Add erofs_insert_ihash() helper inserting inode into inode hash table.
Also add prototypes of erofs_iget() and erofs_iget_by_nid() in the
header file.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
include/erofs/inode.h | 3 +++
lib/inode.c | 10 +++++++---
2 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/include/erofs/inode.h b/include/erofs/inode.h
index e8a5670..1c602a8 100644
--- a/include/erofs/inode.h
+++ b/include/erofs/inode.h
@@ -25,6 +25,9 @@ u32 erofs_new_encode_dev(dev_t dev);
unsigned char erofs_mode_to_ftype(umode_t mode);
unsigned char erofs_ftype_to_dtype(unsigned int filetype);
void erofs_inode_manager_init(void);
+void erofs_insert_ihash(struct erofs_inode *inode, dev_t dev, ino_t ino);
+struct erofs_inode *erofs_iget(dev_t dev, ino_t ino);
+struct erofs_inode *erofs_iget_by_nid(erofs_nid_t nid);
unsigned int erofs_iput(struct erofs_inode *inode);
erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
diff --git a/lib/inode.c b/lib/inode.c
index d54f84f..b967aab 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -75,6 +75,12 @@ void erofs_inode_manager_init(void)
init_list_head(&inode_hashtable[i]);
}
+void erofs_insert_ihash(struct erofs_inode *inode, dev_t dev, ino_t ino)
+{
+ list_add(&inode->i_hash,
+ &inode_hashtable[(ino ^ dev) % NR_INODE_HASHTABLE]);
+}
+
/* get the inode from the (source) inode # */
struct erofs_inode *erofs_iget(dev_t dev, ino_t ino)
{
@@ -976,9 +982,7 @@ static int erofs_fill_inode(struct erofs_inode *inode, struct stat *st,
inode->inode_isize = sizeof(struct erofs_inode_compact);
}
- list_add(&inode->i_hash,
- &inode_hashtable[(st->st_ino ^ st->st_dev) %
- NR_INODE_HASHTABLE]);
+ erofs_insert_ihash(inode, st->st_dev, st->st_ino);
return 0;
}
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH v3 08/11] erofs-utils: lib: add erofs_rebuild_dump_tree() helper
2023-08-22 9:24 [PATCH v3 00/11] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (6 preceding siblings ...)
2023-08-22 9:24 ` [PATCH v3 07/11] erofs-utils: lib: add erofs_insert_ihash() helper Jingbo Xu
@ 2023-08-22 9:24 ` Jingbo Xu
2023-08-22 9:24 ` [PATCH v3 09/11] erofs-utils: lib: add erofs_rebuild_get_dentry() helper Jingbo Xu
` (2 subsequent siblings)
10 siblings, 0 replies; 14+ messages in thread
From: Jingbo Xu @ 2023-08-22 9:24 UTC (permalink / raw)
To: xiang, linux-erofs
Enhance tarerofs_dump_tree() so that it could optionally skip specified
files, usually for whiteout files.
Rename it to erofs_rebuild_dump_tree().
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
include/erofs/inode.h | 2 +-
include/erofs/internal.h | 1 +
lib/inode.c | 15 +++++++++++----
mkfs/main.c | 2 +-
4 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/include/erofs/inode.h b/include/erofs/inode.h
index 1c602a8..fe9dda2 100644
--- a/include/erofs/inode.h
+++ b/include/erofs/inode.h
@@ -32,7 +32,7 @@ unsigned int erofs_iput(struct erofs_inode *inode);
erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
const char *name);
-int tarerofs_dump_tree(struct erofs_inode *dir);
+int erofs_rebuild_dump_tree(struct erofs_inode *dir);
int erofs_init_empty_dir(struct erofs_inode *dir);
struct erofs_inode *erofs_new_inode(void);
struct erofs_inode *erofs_mkfs_build_tree_from_path(const char *path);
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 455a73a..fa0a240 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -191,6 +191,7 @@ struct erofs_inode {
bool lazy_tailblock;
bool with_tmpfile;
bool opaque;
+ bool drop;
unsigned int xattr_isize;
unsigned int extent_isize;
diff --git a/lib/inode.c b/lib/inode.c
index b967aab..96012e4 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -1323,9 +1323,9 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)
return inode;
}
-int tarerofs_dump_tree(struct erofs_inode *dir)
+int erofs_rebuild_dump_tree(struct erofs_inode *dir)
{
- struct erofs_dentry *d;
+ struct erofs_dentry *d, *n;
unsigned int nr_subdirs;
int ret;
@@ -1369,8 +1369,15 @@ int tarerofs_dump_tree(struct erofs_inode *dir)
}
nr_subdirs = 0;
- list_for_each_entry(d, &dir->i_subdirs, d_child)
+ list_for_each_entry_safe(d, n, &dir->i_subdirs, d_child) {
+ if (d->inode->drop) {
+ list_del(&d->d_child);
+ erofs_d_invalidate(d);
+ free(d);
+ continue;
+ }
++nr_subdirs;
+ }
ret = erofs_prepare_dir_layout(dir, nr_subdirs);
if (ret)
@@ -1391,7 +1398,7 @@ int tarerofs_dump_tree(struct erofs_inode *dir)
continue;
inode = erofs_igrab(d->inode);
- ret = tarerofs_dump_tree(inode);
+ ret = erofs_rebuild_dump_tree(inode);
dir->i_nlink += (erofs_mode_to_ftype(inode->i_mode) == EROFS_FT_DIR);
erofs_iput(inode);
if (ret)
diff --git a/mkfs/main.c b/mkfs/main.c
index c03a7a8..628af59 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -946,7 +946,7 @@ int main(int argc, char **argv)
if (err < 0)
goto exit;
- err = tarerofs_dump_tree(root_inode);
+ err = erofs_rebuild_dump_tree(root_inode);
if (err < 0)
goto exit;
}
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH v3 09/11] erofs-utils: lib: add erofs_rebuild_get_dentry() helper
2023-08-22 9:24 [PATCH v3 00/11] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (7 preceding siblings ...)
2023-08-22 9:24 ` [PATCH v3 08/11] erofs-utils: lib: add erofs_rebuild_dump_tree() helper Jingbo Xu
@ 2023-08-22 9:24 ` Jingbo Xu
2023-08-22 9:24 ` [PATCH v3 10/11] erofs-utils: lib: add erofs_rebuild_load_tree() helper Jingbo Xu
2023-08-22 9:24 ` [PATCH v3 11/11] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
10 siblings, 0 replies; 14+ messages in thread
From: Jingbo Xu @ 2023-08-22 9:24 UTC (permalink / raw)
To: xiang, linux-erofs
Rename tarerofs_get_dentry() to erofs_rebuild_get_dentry().
Also make `whout` and 'opq' parameter optional when `aufs` is false.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
include/erofs/rebuild.h | 19 +++++++
lib/Makefile.am | 3 +-
lib/rebuild.c | 117 ++++++++++++++++++++++++++++++++++++++++
lib/tar.c | 109 ++-----------------------------------
4 files changed, 141 insertions(+), 107 deletions(-)
create mode 100644 include/erofs/rebuild.h
create mode 100644 lib/rebuild.c
diff --git a/include/erofs/rebuild.h b/include/erofs/rebuild.h
new file mode 100644
index 0000000..92873c9
--- /dev/null
+++ b/include/erofs/rebuild.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */
+#ifndef __EROFS_REBUILD_H
+#define __EROFS_REBUILD_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#include "internal.h"
+
+struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
+ char *path, bool aufs, bool *whout, bool *opq);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 7a5dc03..4bcc09b 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -24,6 +24,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
$(top_srcdir)/include/erofs/xattr.h \
$(top_srcdir)/include/erofs/compress_hints.h \
$(top_srcdir)/include/erofs/fragments.h \
+ $(top_srcdir)/include/erofs/rebuild.h \
$(top_srcdir)/lib/liberofs_private.h
noinst_HEADERS += compressor.h
@@ -31,7 +32,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
namei.c data.c compress.c compressor.c zmap.c decompress.c \
compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \
- block_list.c
+ block_list.c rebuild.c
liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include
if ENABLE_LZ4
diff --git a/lib/rebuild.c b/lib/rebuild.c
new file mode 100644
index 0000000..7aaa071
--- /dev/null
+++ b/lib/rebuild.c
@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0
+#define _GNU_SOURCE
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include "erofs/print.h"
+#include "erofs/inode.h"
+#include "erofs/rebuild.h"
+#include "erofs/internal.h"
+
+#ifdef HAVE_LINUX_AUFS_TYPE_H
+#include <linux/aufs_type.h>
+#else
+#define AUFS_WH_PFX ".wh."
+#define AUFS_DIROPQ_NAME AUFS_WH_PFX ".opq"
+#define AUFS_WH_DIROPQ AUFS_WH_PFX AUFS_DIROPQ_NAME
+#endif
+
+static struct erofs_dentry *erofs_rebuild_mkdir(struct erofs_inode *dir,
+ const char *s)
+{
+ struct erofs_inode *inode;
+ struct erofs_dentry *d;
+
+ inode = erofs_new_inode();
+ if (IS_ERR(inode))
+ return ERR_CAST(inode);
+
+ inode->i_mode = S_IFDIR | 0755;
+ inode->i_parent = dir;
+ inode->i_uid = getuid();
+ inode->i_gid = getgid();
+ inode->i_mtime = inode->sbi->build_time;
+ inode->i_mtime_nsec = inode->sbi->build_time_nsec;
+ erofs_init_empty_dir(inode);
+
+ d = erofs_d_alloc(dir, s);
+ if (!IS_ERR(d)) {
+ d->type = EROFS_FT_DIR;
+ d->inode = inode;
+ }
+ return d;
+}
+
+struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
+ char *path, bool aufs, bool *whout, bool *opq)
+{
+ struct erofs_dentry *d = NULL;
+ unsigned int len = strlen(path);
+ char *s = path;
+
+ if (aufs) {
+ *whout = false;
+ *opq = false;
+ }
+
+ while (s < path + len) {
+ char *slash = memchr(s, '/', path + len - s);
+ if (slash) {
+ if (s == slash) {
+ while (*++s == '/'); /* skip '//...' */
+ continue;
+ }
+ *slash = '\0';
+ }
+
+ if (!memcmp(s, ".", 2)) {
+ /* null */
+ } else if (!memcmp(s, "..", 3)) {
+ pwd = pwd->i_parent;
+ } else {
+ struct erofs_inode *inode = NULL;
+
+ if (aufs && !slash) {
+ if (!memcmp(s, AUFS_WH_DIROPQ, sizeof(AUFS_WH_DIROPQ))) {
+ *opq = true;
+ break;
+ }
+ if (!memcmp(s, AUFS_WH_PFX, sizeof(AUFS_WH_PFX) - 1)) {
+ s += sizeof(AUFS_WH_PFX) - 1;
+ *whout = true;
+ }
+ }
+
+ list_for_each_entry(d, &pwd->i_subdirs, d_child) {
+ if (!strcmp(d->name, s)) {
+ if (d->type != EROFS_FT_DIR && slash)
+ return ERR_PTR(-EIO);
+ inode = d->inode;
+ break;
+ }
+ }
+
+ if (inode) {
+ pwd = inode;
+ } else if (!slash) {
+ d = erofs_d_alloc(pwd, s);
+ if (IS_ERR(d))
+ return d;
+ d->type = EROFS_FT_UNKNOWN;
+ d->inode = pwd;
+ } else {
+ d = erofs_rebuild_mkdir(pwd, s);
+ if (IS_ERR(d))
+ return d;
+ pwd = d->inode;
+ }
+ }
+ if (slash) {
+ *slash = '/';
+ s = slash + 1;
+ } else {
+ break;
+ }
+ }
+ return d;
+}
diff --git a/lib/tar.c b/lib/tar.c
index ce54d17..cf3497f 100644
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -3,13 +3,6 @@
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
-#ifdef HAVE_LINUX_AUFS_TYPE_H
-#include <linux/aufs_type.h>
-#else
-#define AUFS_WH_PFX ".wh."
-#define AUFS_DIROPQ_NAME AUFS_WH_PFX ".opq"
-#define AUFS_WH_DIROPQ AUFS_WH_PFX AUFS_DIROPQ_NAME
-#endif
#include "erofs/print.h"
#include "erofs/cache.h"
#include "erofs/inode.h"
@@ -18,6 +11,7 @@
#include "erofs/io.h"
#include "erofs/xattr.h"
#include "erofs/blobchunk.h"
+#include "erofs/rebuild.h"
#define EROFS_WHITEOUT_DEV 0
@@ -133,103 +127,6 @@ static long long tarerofs_parsenum(const char *ptr, int len)
return tarerofs_otoi(ptr, len);
}
-static struct erofs_dentry *tarerofs_mkdir(struct erofs_inode *dir, const char *s)
-{
- struct erofs_inode *inode;
- struct erofs_dentry *d;
-
- inode = erofs_new_inode();
- if (IS_ERR(inode))
- return ERR_CAST(inode);
-
- inode->i_mode = S_IFDIR | 0755;
- inode->i_parent = dir;
- inode->i_uid = getuid();
- inode->i_gid = getgid();
- inode->i_mtime = inode->sbi->build_time;
- inode->i_mtime_nsec = inode->sbi->build_time_nsec;
- erofs_init_empty_dir(inode);
-
- d = erofs_d_alloc(dir, s);
- if (!IS_ERR(d)) {
- d->type = EROFS_FT_DIR;
- d->inode = inode;
- }
- return d;
-}
-
-static struct erofs_dentry *tarerofs_get_dentry(struct erofs_inode *pwd, char *path,
- bool aufs, bool *whout, bool *opq)
-{
- struct erofs_dentry *d = NULL;
- unsigned int len = strlen(path);
- char *s = path;
-
- *whout = false;
- *opq = false;
-
- while (s < path + len) {
- char *slash = memchr(s, '/', path + len - s);
- if (slash) {
- if (s == slash) {
- while (*++s == '/'); /* skip '//...' */
- continue;
- }
- *slash = '\0';
- }
-
- if (!memcmp(s, ".", 2)) {
- /* null */
- } else if (!memcmp(s, "..", 3)) {
- pwd = pwd->i_parent;
- } else {
- struct erofs_inode *inode = NULL;
-
- if (aufs && !slash) {
- if (!memcmp(s, AUFS_WH_DIROPQ, sizeof(AUFS_WH_DIROPQ))) {
- *opq = true;
- break;
- }
- if (!memcmp(s, AUFS_WH_PFX, sizeof(AUFS_WH_PFX) - 1)) {
- s += sizeof(AUFS_WH_PFX) - 1;
- *whout = true;
- }
- }
-
- list_for_each_entry(d, &pwd->i_subdirs, d_child) {
- if (!strcmp(d->name, s)) {
- if (d->type != EROFS_FT_DIR && slash)
- return ERR_PTR(-EIO);
- inode = d->inode;
- break;
- }
- }
-
- if (inode) {
- pwd = inode;
- } else if (!slash) {
- d = erofs_d_alloc(pwd, s);
- if (IS_ERR(d))
- return d;
- d->type = EROFS_FT_UNKNOWN;
- d->inode = pwd;
- } else {
- d = tarerofs_mkdir(pwd, s);
- if (IS_ERR(d))
- return d;
- pwd = d->inode;
- }
- }
- if (slash) {
- *slash = '/';
- s = slash + 1;
- } else {
- break;
- }
- }
- return d;
-}
-
int tarerofs_parse_pax_header(int fd, struct erofs_pax_header *eh, u32 size)
{
char *buf, *p;
@@ -612,7 +509,7 @@ restart:
erofs_dbg("parsing %s (mode %05o)", eh.path, st.st_mode);
- d = tarerofs_get_dentry(root, eh.path, tar->aufs, &whout, &opq);
+ d = erofs_rebuild_get_dentry(root, eh.path, tar->aufs, &whout, &opq);
if (IS_ERR(d)) {
ret = PTR_ERR(d);
goto out;
@@ -645,7 +542,7 @@ restart:
}
d->inode = NULL;
- d2 = tarerofs_get_dentry(root, eh.link, tar->aufs, &dumb, &dumb);
+ d2 = erofs_rebuild_get_dentry(root, eh.link, tar->aufs, &dumb, &dumb);
if (IS_ERR(d2)) {
ret = PTR_ERR(d2);
goto out;
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH v3 10/11] erofs-utils: lib: add erofs_rebuild_load_tree() helper
2023-08-22 9:24 [PATCH v3 00/11] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (8 preceding siblings ...)
2023-08-22 9:24 ` [PATCH v3 09/11] erofs-utils: lib: add erofs_rebuild_get_dentry() helper Jingbo Xu
@ 2023-08-22 9:24 ` Jingbo Xu
2023-08-22 9:24 ` [PATCH v3 11/11] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
10 siblings, 0 replies; 14+ messages in thread
From: Jingbo Xu @ 2023-08-22 9:24 UTC (permalink / raw)
To: xiang, linux-erofs
Add erofs_rebuild_load_tree() helper loading inode tree from given erofs
image, and making it merged into a given inode tree in an overlayfs like
model.
Since we need to read the content of the symlink file from disk when
loading tree, add dependency on zlib_LIBS for mkfs.erofs.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
include/erofs/internal.h | 3 +
include/erofs/rebuild.h | 2 +
lib/rebuild.c | 278 +++++++++++++++++++++++++++++++++++++++
lib/tar.c | 2 -
4 files changed, 283 insertions(+), 2 deletions(-)
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index fa0a240..2559892 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -108,6 +108,7 @@ struct erofs_sb_info {
int devfd;
u64 devsz;
+ dev_t dev;
unsigned int nblobs;
unsigned int blobfd[256];
};
@@ -424,6 +425,8 @@ static inline u32 erofs_crc32c(u32 crc, const u8 *in, size_t len)
return crc;
}
+#define EROFS_WHITEOUT_DEV 0
+
#ifdef __cplusplus
}
#endif
diff --git a/include/erofs/rebuild.h b/include/erofs/rebuild.h
index 92873c9..3ac074c 100644
--- a/include/erofs/rebuild.h
+++ b/include/erofs/rebuild.h
@@ -12,6 +12,8 @@ extern "C"
struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
char *path, bool aufs, bool *whout, bool *opq);
+int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi);
+
#ifdef __cplusplus
}
#endif
diff --git a/lib/rebuild.c b/lib/rebuild.c
index 7aaa071..80eb7e6 100644
--- a/lib/rebuild.c
+++ b/lib/rebuild.c
@@ -3,9 +3,18 @@
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
+#include <sys/stat.h>
+#include <config.h>
+#if defined(HAVE_SYS_SYSMACROS_H)
+#include <sys/sysmacros.h>
+#endif
#include "erofs/print.h"
#include "erofs/inode.h"
#include "erofs/rebuild.h"
+#include "erofs/io.h"
+#include "erofs/dir.h"
+#include "erofs/xattr.h"
+#include "erofs/blobchunk.h"
#include "erofs/internal.h"
#ifdef HAVE_LINUX_AUFS_TYPE_H
@@ -115,3 +124,272 @@ struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
}
return d;
}
+
+static int erofs_rebuild_fill_inode_map(struct erofs_inode *inode,
+ struct erofs_inode *vi)
+{
+ int ret;
+ struct erofs_sb_info *sbi = vi->sbi;
+ unsigned int count, unit, chunkbits, i;
+ unsigned int deviceid = inode->dev;
+ erofs_off_t chunksize;
+ erofs_blk_t blkaddr;
+ struct erofs_inode_chunk_index *idx;
+
+ /* TODO: fill data map in other layouts */
+ if (vi->datalayout != EROFS_INODE_CHUNK_BASED &&
+ vi->datalayout != EROFS_INODE_FLAT_PLAIN) {
+ erofs_err("unsupported datalayout %d", vi->datalayout);
+ return -EOPNOTSUPP;
+ }
+
+ if (sbi->extra_devices) {
+ chunkbits = vi->u.chunkbits;
+ inode->u.chunkformat = vi->u.chunkformat;
+ } else {
+ chunkbits = ilog2(inode->i_size - 1) + 1;
+ if (chunkbits < sbi->blkszbits)
+ chunkbits = sbi->blkszbits;
+ if (chunkbits - sbi->blkszbits > EROFS_CHUNK_FORMAT_BLKBITS_MASK)
+ chunkbits = EROFS_CHUNK_FORMAT_BLKBITS_MASK + sbi->blkszbits;
+ inode->u.chunkformat = EROFS_CHUNK_FORMAT_INDEXES;
+ inode->u.chunkformat |= chunkbits - sbi->blkszbits;
+ }
+ chunksize = 1ULL << chunkbits;
+ count = DIV_ROUND_UP(inode->i_size, chunksize);
+
+ unit = sizeof(struct erofs_inode_chunk_index);
+ inode->extent_isize = count * unit;
+ idx = malloc(max(sizeof(*idx), sizeof(void *)));
+ if (!idx)
+ return -ENOMEM;
+ inode->chunkindexes = idx;
+
+ for (i = 0; i < count; i++) {
+ struct erofs_blobchunk *chunk;
+ struct erofs_map_blocks map = {
+ .index = UINT_MAX,
+ };
+
+ map.m_la = i << chunkbits;
+ ret = erofs_map_blocks(vi, &map, 0);
+ if (ret)
+ goto err;
+
+ blkaddr = erofs_blknr(sbi, map.m_pa);
+ chunk = erofs_get_unhashed_chunk(deviceid, blkaddr, 0);
+ if (IS_ERR(chunk)) {
+ ret = PTR_ERR(chunk);
+ goto err;
+ }
+ *(void **)idx++ = chunk;
+ erofs_dbg("\t%s: chunk %d (deviceid %u, blkaddr %u)",
+ inode->i_srcpath, i, deviceid, blkaddr);
+
+ }
+ inode->datalayout = EROFS_INODE_CHUNK_BASED;
+ return 0;
+err:
+ free(idx);
+ inode->chunkindexes = NULL;
+ return ret;
+}
+
+static int erofs_rebuild_fill_inode(struct erofs_inode *inode,
+ struct erofs_inode *vi)
+{
+ int ret = 0;
+
+ inode->i_srcpath = strdup(vi->i_srcpath);
+ inode->i_mode = vi->i_mode;
+ inode->i_uid = vi->i_uid;
+ inode->i_gid = vi->i_gid;
+ inode->i_mtime = vi->i_mtime;
+ inode->i_ino[1] = vi->nid;
+ inode->i_nlink = 1;
+ inode->opaque = vi->opaque;
+ list_splice_tail(&vi->i_xattrs, &inode->i_xattrs);
+
+ switch (inode->i_mode & S_IFMT) {
+ case S_IFCHR:
+ case S_IFBLK:
+ case S_IFIFO:
+ case S_IFSOCK:
+ inode->u.i_rdev = erofs_new_encode_dev(vi->u.i_rdev);
+ inode->i_size = 0;
+ erofs_dbg("\tdev: %d %d", major(vi->u.i_rdev),
+ minor(vi->u.i_rdev));
+ break;
+ case S_IFDIR:
+ inode->i_size = 0;
+ ret = erofs_init_empty_dir(inode);
+ break;
+ case S_IFLNK:
+ inode->i_size = vi->i_size;
+ inode->i_link = malloc(inode->i_size + 1);
+ ret = erofs_pread(vi, inode->i_link, inode->i_size, 0);
+ erofs_dbg("\tsymlink: %s -> %s", inode->i_srcpath, inode->i_link);
+ break;
+ case S_IFREG:
+ inode->i_size = vi->i_size;
+ if (inode->i_size)
+ ret = erofs_rebuild_fill_inode_map(inode, vi);
+ else
+ inode->u.i_blkaddr = NULL_ADDR;
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+ return ret;
+}
+
+struct erofs_rebuild_dir_context {
+ struct erofs_dir_context ctx;
+ struct erofs_inode *root;
+ dev_t dev;
+};
+
+static int erofs_rebuild_parse_inode(struct erofs_rebuild_dir_context *rctx,
+ struct erofs_inode *vi)
+{
+ struct erofs_inode *inode;
+ struct erofs_dentry *d;
+ int ret;
+
+ d = erofs_rebuild_get_dentry(rctx->root, vi->i_srcpath,
+ false, NULL, NULL);
+ if (IS_ERR(d))
+ return PTR_ERR(d);
+
+ if (d->type != EROFS_FT_UNKNOWN) {
+ inode = d->inode;
+ /* drop whiteout if victim has appeared */
+ if (S_ISCHR(inode->i_mode) &&
+ inode->u.i_rdev == EROFS_WHITEOUT_DEV) {
+ inode->drop = true;
+ return 0;
+ }
+
+ DBG_BUGON((inode->i_mode & S_IFMT) != (vi->i_mode & S_IFMT));
+ if (!S_ISDIR(inode->i_mode) || inode->opaque) {
+ erofs_dbg("file %s: %s (%d) exists",
+ vi->i_srcpath, inode->i_srcpath,
+ erofs_mode_to_ftype(inode->i_mode));
+ return 0;
+ }
+ erofs_dbg("dir %s: %s merging", vi->i_srcpath, inode->i_srcpath);
+ } else {
+ erofs_dbg("loading file: %s (%d) (nid %llu)", vi->i_srcpath,
+ erofs_mode_to_ftype(vi->i_mode), vi->nid);
+ if (S_ISREG(vi->i_mode) && vi->i_nlink > 1 &&
+ (inode = erofs_iget(rctx->dev, vi->nid))) {
+ /* hardlink file */
+ if (S_ISDIR(inode->i_mode)) {
+ erofs_dbg("hardlink directory not supported");
+ ret = -EISDIR;
+ goto put_inode;
+ }
+ inode->i_nlink++;
+ erofs_dbg("\thardlink: %s -> %s",
+ vi->i_srcpath, inode->i_srcpath);
+ } else {
+ inode = erofs_new_inode();
+ if (IS_ERR(inode))
+ return PTR_ERR(inode);
+
+ ret = erofs_read_xattrs_from_disk(vi);
+ if (ret)
+ goto put_inode;
+
+ inode->i_parent = d->inode;
+ inode->dev = rctx->dev;
+ ret = erofs_rebuild_fill_inode(inode, vi);
+ if (ret)
+ goto put_inode;
+
+ erofs_insert_ihash(inode, rctx->dev, vi->nid);
+ }
+
+ d->inode = inode;
+ d->type = erofs_mode_to_ftype(inode->i_mode);
+ }
+
+ ret = 0;
+ if (S_ISDIR(vi->i_mode)) {
+ struct erofs_rebuild_dir_context nctx = *rctx;
+ nctx.ctx.dir = vi;
+ ret = erofs_iterate_dir(&nctx.ctx, false);
+ }
+ return ret;
+
+put_inode:
+ erofs_iput(inode);
+ return ret;
+}
+
+static int erofs_rebuild_dirent_iter(struct erofs_dir_context *ctx)
+{
+ struct erofs_rebuild_dir_context *rctx = (void *)ctx;
+ struct erofs_inode *dir = ctx->dir;
+ struct erofs_inode vi = {};
+ char *path;
+ int ret;
+
+ if (ctx->dot_dotdot)
+ return 0;
+
+ erofs_dbg("file %s/%.*s", dir->i_srcpath, ctx->de_namelen, ctx->dname);
+ vi.nid = ctx->de_nid;
+ vi.sbi = dir->sbi;
+ ret = erofs_read_inode_from_disk(&vi);
+ if (ret)
+ return ret;
+
+ ret = asprintf(&path, "%s/%.*s", dir->i_srcpath, ctx->de_namelen,
+ ctx->dname);
+ if (ret < 0)
+ return ret;
+ vi.i_srcpath = path;
+
+ ret = erofs_rebuild_parse_inode(rctx, &vi);
+ free(path);
+ return ret;
+}
+
+int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi)
+{
+ struct erofs_inode inode = {};
+ struct erofs_rebuild_dir_context ctx;
+ int ret;
+
+ if (!sbi->devname) {
+ erofs_err("please open the device first");
+ return -EINVAL;
+ }
+
+ ret = erofs_read_superblock(sbi);
+ if (ret) {
+ erofs_err("failed to read superblock of img %s", sbi->devname);
+ return ret;
+ }
+
+ inode.nid = sbi->root_nid;
+ inode.sbi = sbi;
+ ret = erofs_read_inode_from_disk(&inode);
+ if (ret) {
+ erofs_err("failed to read root inode of img %s", sbi->devname);
+ return ret;
+ }
+ inode.i_srcpath = strdup("/");
+
+ ctx = (struct erofs_rebuild_dir_context) {
+ .ctx.dir = &inode,
+ .ctx.cb = erofs_rebuild_dirent_iter,
+ .root = root,
+ .dev = sbi->dev,
+ };
+ ret = erofs_iterate_dir(&ctx.ctx, false);
+ free(inode.i_srcpath);
+ return ret;
+}
diff --git a/lib/tar.c b/lib/tar.c
index cf3497f..26830f4 100644
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -13,8 +13,6 @@
#include "erofs/blobchunk.h"
#include "erofs/rebuild.h"
-#define EROFS_WHITEOUT_DEV 0
-
static char erofs_libbuf[16384];
struct tar_header {
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH v3 11/11] erofs-utils: mkfs: introduce rebuild mode
2023-08-22 9:24 [PATCH v3 00/11] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (9 preceding siblings ...)
2023-08-22 9:24 ` [PATCH v3 10/11] erofs-utils: lib: add erofs_rebuild_load_tree() helper Jingbo Xu
@ 2023-08-22 9:24 ` Jingbo Xu
10 siblings, 0 replies; 14+ messages in thread
From: Jingbo Xu @ 2023-08-22 9:24 UTC (permalink / raw)
To: xiang, linux-erofs
Introduce a new rebuild mode merging multiple erofs images generated
from either tarerfs index mode (--tar=i):
mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar
...
mkfs.erofs --tar=i --aufs layerN.erofs layerN.tar
mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs
or tarerofs non-index mode (--tar=f):
mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar
...
mkfs.erofs --tar=f -Enoinline_data --aufs layerN.erofs layerN.tar
mkfs.erofs merge.erofs layerN.erofs ... layer0.erofs
When calling rebuild mode, the top most layer must be specified firstly
in the cmdline, while the bottom most layer i.e. layer0 the lastly.
The rebuild mode doesn't support flat inline datalayout yet.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
include/erofs/internal.h | 2 +
mkfs/main.c | 197 +++++++++++++++++++++++++++++++--------
2 files changed, 158 insertions(+), 41 deletions(-)
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 2559892..517352f 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -111,6 +111,8 @@ struct erofs_sb_info {
dev_t dev;
unsigned int nblobs;
unsigned int blobfd[256];
+
+ struct list_head list;
};
/* make sure that any user of the erofs headers has atleast 64bit off_t type */
diff --git a/mkfs/main.c b/mkfs/main.c
index 628af59..3463751 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -26,6 +26,7 @@
#include "erofs/compress_hints.h"
#include "erofs/blobchunk.h"
#include "erofs/fragments.h"
+#include "erofs/rebuild.h"
#include "../lib/liberofs_private.h"
#include "../lib/liberofs_uuid.h"
@@ -134,6 +135,10 @@ static void usage(void)
static unsigned int pclustersize_packed, pclustersize_max;
static struct erofs_tarfile erofstar;
static bool tar_mode;
+static bool rebuild_mode;
+
+static unsigned int rebuild_src_count;
+static LIST_HEAD(rebuild_src_list);
static int parse_extended_opts(const char *opts)
{
@@ -274,10 +279,24 @@ static int mkfs_parse_compress_algs(char *algs)
return 0;
}
+static void erofs_rebuild_cleanup(void)
+{
+ struct erofs_sb_info *src, *n;
+
+ list_for_each_entry_safe(src, n, &rebuild_src_list, list) {
+ list_del(&src->list);
+ erofs_put_super(src);
+ dev_close(src);
+ free(src);
+ }
+ rebuild_src_count = 0;
+}
+
static int mkfs_parse_options_cfg(int argc, char *argv[])
{
char *endptr;
- int opt, i;
+ struct stat st;
+ int opt, i, err;
bool quiet = false;
while ((opt = getopt_long(argc, argv, "C:E:L:T:U:b:d:x:z:",
@@ -533,7 +552,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
} else {
erofstar.fd = STDIN_FILENO;
}
- }else {
+ } else {
cfg.c_src_path = realpath(argv[optind++], NULL);
if (!cfg.c_src_path) {
erofs_err("failed to parse source directory: %s",
@@ -541,11 +560,51 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
return -ENOENT;
}
- if (optind < argc) {
+ if (tar_mode) {
+ erofstar.fd = open(cfg.c_src_path, O_RDONLY);
+ if (erofstar.fd < 0) {
+ erofs_err("failed to open file: %s", cfg.c_src_path);
+ usage();
+ return -errno;
+ }
+ } else {
+ err = lstat(cfg.c_src_path, &st);
+ if (err)
+ return -errno;
+ if (S_ISDIR(st.st_mode))
+ erofs_set_fs_root(cfg.c_src_path);
+ else
+ rebuild_mode = true;
+ }
+
+ if (rebuild_mode) {
+ optind--; /* roll back to retrieve the first source */
+ while (optind < argc) {
+ struct erofs_sb_info *src;
+
+ src = malloc(sizeof(struct erofs_sb_info));
+ if (!src) {
+ erofs_rebuild_cleanup();
+ return -ENOMEM;
+ }
+
+ err = dev_open_ro(src, argv[optind++]);
+ if (err) {
+ free(src);
+ erofs_rebuild_cleanup();
+ return err;
+ }
+
+ /* extra device index starts from 1 */
+ src->dev = ++rebuild_src_count;
+ list_add_tail(&src->list, &rebuild_src_list);
+ }
+ } else if (optind < argc) {
erofs_err("unexpected argument: %s\n", argv[optind]);
return -EINVAL;
}
}
+
if (quiet) {
cfg.c_dbg_lvl = EROFS_ERR;
cfg.c_showprogress = false;
@@ -730,13 +789,63 @@ void erofs_show_progs(int argc, char *argv[])
printf("%s %s\n", basename(argv[0]), cfg.c_version);
}
+struct erofs_inode *erofs_rebuild_load_trees(void)
+{
+ struct erofs_sb_info *src;
+ struct erofs_inode *root;
+ unsigned int extra_devices = 0;
+ erofs_blk_t nblocks;
+ int ret, i = 0;
+
+ root = erofs_new_inode();
+ if (IS_ERR(root))
+ return root;
+ root->i_srcpath = strdup("/");
+ root->i_mode = S_IFDIR | 0777;
+ root->i_parent = root;
+ root->i_mtime = root->sbi->build_time;
+ root->i_mtime_nsec = root->sbi->build_time_nsec;
+ erofs_init_empty_dir(root);
+
+ list_for_each_entry(src, &rebuild_src_list, list) {
+ ret = erofs_rebuild_load_tree(root, src);
+ if (ret)
+ return ERR_PTR(ret);
+
+ if (src->extra_devices > 1) {
+ erofs_err("%s: unsupported number of extra devices",
+ src->devname, src->extra_devices);
+ return ERR_PTR(-EOPNOTSUPP);
+ }
+ extra_devices += src->extra_devices;
+ }
+
+ if (extra_devices && extra_devices != rebuild_src_count) {
+ erofs_err("unsupported mix of source images");
+ return ERR_PTR(-EOPNOTSUPP);
+ }
+
+ ret = erofs_mkfs_init_devices(&sbi, rebuild_src_count);
+ if (ret)
+ return ERR_PTR(ret);
+
+ list_for_each_entry(src, &rebuild_src_list, list) {
+ if (extra_devices)
+ nblocks = src->devs[0].blocks;
+ else
+ nblocks = src->primarydevice_blocks;
+ sbi.devs[i++].blocks = nblocks;
+ }
+
+ return root;
+}
+
int main(int argc, char **argv)
{
int err = 0;
struct erofs_buffer_head *sb_bh;
struct erofs_inode *root_inode, *packed_inode;
erofs_nid_t root_nid, packed_nid;
- struct stat st;
erofs_blk_t nblocks;
struct timeval t;
char uuid_str[37];
@@ -759,26 +868,6 @@ int main(int argc, char **argv)
return 1;
}
- if (!tar_mode) {
- err = lstat(cfg.c_src_path, &st);
- if (err)
- return 1;
- if (!S_ISDIR(st.st_mode)) {
- erofs_err("root of the filesystem is not a directory - %s",
- cfg.c_src_path);
- usage();
- return 1;
- }
- erofs_set_fs_root(cfg.c_src_path);
- } else if (cfg.c_src_path) {
- erofstar.fd = open(cfg.c_src_path, O_RDONLY);
- if (erofstar.fd < 0) {
- erofs_err("failed to open file: %s", cfg.c_src_path);
- usage();
- return 1;
- }
- }
-
if (cfg.c_unix_timestamp != -1) {
sbi.build_time = cfg.c_unix_timestamp;
sbi.build_time_nsec = 0;
@@ -845,6 +934,21 @@ int main(int argc, char **argv)
}
}
+ if (rebuild_mode) {
+ struct erofs_sb_info *src;
+
+ src = list_first_entry(&rebuild_src_list, struct erofs_sb_info, list);
+ if (!src)
+ goto exit;
+ err = erofs_read_superblock(src);
+ if (err) {
+ erofs_err("failed to read superblock of img %s", src->devname);
+ goto exit;
+ }
+ sbi.blkszbits = src->blkszbits;
+ erofs_dbg("sbi.blkszbits %d", sbi.blkszbits);
+ }
+
sb_bh = erofs_buffer_init();
if (IS_ERR(sb_bh)) {
err = PTR_ERR(sb_bh);
@@ -912,23 +1016,7 @@ int main(int argc, char **argv)
erofs_inode_manager_init();
- if (!tar_mode) {
- err = erofs_build_shared_xattrs_from_path(&sbi, cfg.c_src_path);
- if (err) {
- erofs_err("failed to build shared xattrs: %s",
- erofs_strerror(err));
- goto exit;
- }
-
- if (cfg.c_extra_ea_name_prefixes)
- erofs_xattr_write_name_prefixes(&sbi, packedfile);
-
- root_inode = erofs_mkfs_build_tree_from_path(cfg.c_src_path);
- if (IS_ERR(root_inode)) {
- err = PTR_ERR(root_inode);
- goto exit;
- }
- } else {
+ if (tar_mode) {
root_inode = erofs_new_inode();
if (IS_ERR(root_inode)) {
err = PTR_ERR(root_inode);
@@ -949,6 +1037,32 @@ int main(int argc, char **argv)
err = erofs_rebuild_dump_tree(root_inode);
if (err < 0)
goto exit;
+ } else if (rebuild_mode) {
+ root_inode = erofs_rebuild_load_trees();
+ if (IS_ERR(root_inode)) {
+ err = PTR_ERR(root_inode);
+ goto exit;
+ }
+
+ err = erofs_rebuild_dump_tree(root_inode);
+ if (err)
+ goto exit;
+ } else {
+ err = erofs_build_shared_xattrs_from_path(&sbi, cfg.c_src_path);
+ if (err) {
+ erofs_err("failed to build shared xattrs: %s",
+ erofs_strerror(err));
+ goto exit;
+ }
+
+ if (cfg.c_extra_ea_name_prefixes)
+ erofs_xattr_write_name_prefixes(&sbi, packedfile);
+
+ root_inode = erofs_mkfs_build_tree_from_path(cfg.c_src_path);
+ if (IS_ERR(root_inode)) {
+ err = PTR_ERR(root_inode);
+ goto exit;
+ }
}
root_nid = erofs_lookupnid(root_inode);
erofs_iput(root_inode);
@@ -1002,6 +1116,7 @@ exit:
z_erofs_fragments_exit();
erofs_packedfile_exit();
erofs_xattr_cleanup_name_prefixes();
+ erofs_rebuild_cleanup();
erofs_exit_configure();
if (err) {
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 14+ messages in thread