* [PATCH v8 1/8] erofs-utils: lib: add list_splice_tail() helper
2023-09-13 12:02 [PATCH v8 0/8] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
@ 2023-09-13 12:02 ` Jingbo Xu
2023-09-13 12:02 ` [PATCH v8 2/8] erofs-utils: lib: make erofs_get_unhashed_chunk() global Jingbo Xu
` (7 subsequent siblings)
8 siblings, 0 replies; 13+ messages in thread
From: Jingbo Xu @ 2023-09-13 12:02 UTC (permalink / raw)
To: hsiangkao, linux-erofs
Add list_splice_tail() helper.
Reviewed-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
include/erofs/list.h | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/include/erofs/list.h b/include/erofs/list.h
index 3f5da1a..d7a9fee 100644
--- a/include/erofs/list.h
+++ b/include/erofs/list.h
@@ -70,6 +70,26 @@ static inline int list_empty(struct list_head *head)
return head->next == head;
}
+static inline void __list_splice(struct list_head *list,
+ struct list_head *prev, struct list_head *next)
+{
+ struct list_head *first = list->next;
+ struct list_head *last = list->prev;
+
+ first->prev = prev;
+ prev->next = first;
+
+ last->next = next;
+ next->prev = last;
+}
+
+static inline void list_splice_tail(struct list_head *list,
+ struct list_head *head)
+{
+ if (!list_empty(list))
+ __list_splice(list, head->prev, head);
+}
+
#define list_entry(ptr, type, member) container_of(ptr, type, member)
#define list_first_entry(ptr, type, member) \
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH v8 2/8] erofs-utils: lib: make erofs_get_unhashed_chunk() global
2023-09-13 12:02 [PATCH v8 0/8] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
2023-09-13 12:02 ` [PATCH v8 1/8] erofs-utils: lib: add list_splice_tail() helper Jingbo Xu
@ 2023-09-13 12:02 ` Jingbo Xu
2023-09-13 12:02 ` [PATCH v8 3/8] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper Jingbo Xu
` (6 subsequent siblings)
8 siblings, 0 replies; 13+ messages in thread
From: Jingbo Xu @ 2023-09-13 12:02 UTC (permalink / raw)
To: hsiangkao, linux-erofs
... so that it could be called from outside blobchunk.c later.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
include/erofs/blobchunk.h | 2 ++
lib/blobchunk.c | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/include/erofs/blobchunk.h b/include/erofs/blobchunk.h
index 010aee1..fb85d8e 100644
--- a/include/erofs/blobchunk.h
+++ b/include/erofs/blobchunk.h
@@ -14,6 +14,8 @@ extern "C"
#include "erofs/internal.h"
+struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
+ erofs_blk_t blkaddr, erofs_off_t sourceoffset);
int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, erofs_off_t off);
int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd);
int tarerofs_write_chunkes(struct erofs_inode *inode, erofs_off_t data_offset);
diff --git a/lib/blobchunk.c b/lib/blobchunk.c
index 86b29c1..71fb2ff 100644
--- a/lib/blobchunk.c
+++ b/lib/blobchunk.c
@@ -38,7 +38,7 @@ struct erofs_blobchunk erofs_holechunk = {
};
static LIST_HEAD(unhashed_blobchunks);
-static struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
+struct erofs_blobchunk *erofs_get_unhashed_chunk(unsigned int device_id,
erofs_blk_t blkaddr, erofs_off_t sourceoffset)
{
struct erofs_blobchunk *chunk;
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH v8 3/8] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper
2023-09-13 12:02 [PATCH v8 0/8] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
2023-09-13 12:02 ` [PATCH v8 1/8] erofs-utils: lib: add list_splice_tail() helper Jingbo Xu
2023-09-13 12:02 ` [PATCH v8 2/8] erofs-utils: lib: make erofs_get_unhashed_chunk() global Jingbo Xu
@ 2023-09-13 12:02 ` Jingbo Xu
2023-09-13 12:02 ` [PATCH v8 4/8] erofs-utils: lib: add erofs_insert_ihash() helper Jingbo Xu
` (5 subsequent siblings)
8 siblings, 0 replies; 13+ messages in thread
From: Jingbo Xu @ 2023-09-13 12:02 UTC (permalink / raw)
To: hsiangkao, linux-erofs
Add erofs_read_xattrs_from_disk() helper to reading extended
attributes from disk.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
include/erofs/internal.h | 1 +
include/erofs/xattr.h | 1 +
lib/xattr.c | 69 ++++++++++++++++++++++++++++++++++++++++
3 files changed, 71 insertions(+)
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index c36bb24..3b866c9 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -196,6 +196,7 @@ struct erofs_inode {
bool compressed_idata;
bool lazy_tailblock;
bool with_tmpfile;
+ bool opaque;
/* OVL: non-merge dir that may contain whiteout entries */
bool whiteouts;
diff --git a/include/erofs/xattr.h b/include/erofs/xattr.h
index 2ecb18e..0364f24 100644
--- a/include/erofs/xattr.h
+++ b/include/erofs/xattr.h
@@ -58,6 +58,7 @@ int erofs_setxattr(struct erofs_inode *inode, char *key,
const void *value, size_t size);
int erofs_set_opaque_xattr(struct erofs_inode *inode);
int erofs_set_origin_xattr(struct erofs_inode *inode);
+int erofs_read_xattrs_from_disk(struct erofs_inode *inode);
#ifdef __cplusplus
}
diff --git a/lib/xattr.c b/lib/xattr.c
index bf63a81..e3a1b44 100644
--- a/lib/xattr.c
+++ b/lib/xattr.c
@@ -566,6 +566,75 @@ int erofs_scan_file_xattrs(struct erofs_inode *inode)
return erofs_droid_xattr_set_caps(inode);
}
+int erofs_read_xattrs_from_disk(struct erofs_inode *inode)
+{
+ ssize_t kllen;
+ char *keylst, *key;
+ int ret;
+
+ init_list_head(&inode->i_xattrs);
+ kllen = erofs_listxattr(inode, NULL, 0);
+ if (kllen < 0)
+ return kllen;
+ if (kllen <= 1)
+ return 0;
+
+ keylst = malloc(kllen);
+ if (!keylst)
+ return -ENOMEM;
+
+ ret = erofs_listxattr(inode, keylst, kllen);
+ if (ret < 0)
+ goto out;
+
+ for (key = keylst; key < keylst + kllen; key += strlen(key) + 1) {
+ void *value = NULL;
+ size_t size = 0;
+
+ if (!strcmp(key, OVL_XATTR_OPAQUE)) {
+ if (!S_ISDIR(inode->i_mode)) {
+ erofs_dbg("file %s: opaque xattr on non-dir",
+ inode->i_srcpath);
+ ret = -EINVAL;
+ goto out;
+ }
+ inode->opaque = true;
+ }
+
+ ret = erofs_getxattr(inode, key, NULL, 0);
+ if (ret < 0)
+ goto out;
+ if (ret) {
+ size = ret;
+ value = malloc(size);
+ if (!value) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ ret = erofs_getxattr(inode, key, value, size);
+ if (ret < 0) {
+ free(value);
+ goto out;
+ }
+ DBG_BUGON(ret != size);
+ } else if (S_ISDIR(inode->i_mode) &&
+ !strcmp(key, OVL_XATTR_ORIGIN)) {
+ ret = 0;
+ inode->whiteouts = true;
+ continue;
+ }
+
+ ret = erofs_setxattr(inode, key, value, size);
+ free(value);
+ if (ret)
+ break;
+ }
+out:
+ free(keylst);
+ return ret;
+}
+
static inline unsigned int erofs_next_xattr_align(unsigned int pos,
struct xattr_item *item)
{
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH v8 4/8] erofs-utils: lib: add erofs_insert_ihash() helper
2023-09-13 12:02 [PATCH v8 0/8] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (2 preceding siblings ...)
2023-09-13 12:02 ` [PATCH v8 3/8] erofs-utils: lib: add erofs_read_xattrs_from_disk() helper Jingbo Xu
@ 2023-09-13 12:02 ` Jingbo Xu
2023-09-13 12:03 ` [PATCH v8 5/8] erofs-utils: lib: add erofs_rebuild_get_dentry() helper Jingbo Xu
` (4 subsequent siblings)
8 siblings, 0 replies; 13+ messages in thread
From: Jingbo Xu @ 2023-09-13 12:02 UTC (permalink / raw)
To: hsiangkao, linux-erofs
Add erofs_insert_ihash() helper inserting inode into inode hash table.
Also add prototypes of erofs_iget() and erofs_iget_by_nid() in the
header file.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
Reviewed-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
include/erofs/inode.h | 3 +++
lib/inode.c | 10 +++++++---
2 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/include/erofs/inode.h b/include/erofs/inode.h
index e8a5670..1c602a8 100644
--- a/include/erofs/inode.h
+++ b/include/erofs/inode.h
@@ -25,6 +25,9 @@ u32 erofs_new_encode_dev(dev_t dev);
unsigned char erofs_mode_to_ftype(umode_t mode);
unsigned char erofs_ftype_to_dtype(unsigned int filetype);
void erofs_inode_manager_init(void);
+void erofs_insert_ihash(struct erofs_inode *inode, dev_t dev, ino_t ino);
+struct erofs_inode *erofs_iget(dev_t dev, ino_t ino);
+struct erofs_inode *erofs_iget_by_nid(erofs_nid_t nid);
unsigned int erofs_iput(struct erofs_inode *inode);
erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
diff --git a/lib/inode.c b/lib/inode.c
index 92796c9..79e1795 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -75,6 +75,12 @@ void erofs_inode_manager_init(void)
init_list_head(&inode_hashtable[i]);
}
+void erofs_insert_ihash(struct erofs_inode *inode, dev_t dev, ino_t ino)
+{
+ list_add(&inode->i_hash,
+ &inode_hashtable[(ino ^ dev) % NR_INODE_HASHTABLE]);
+}
+
/* get the inode from the (source) inode # */
struct erofs_inode *erofs_iget(dev_t dev, ino_t ino)
{
@@ -976,9 +982,7 @@ static int erofs_fill_inode(struct erofs_inode *inode, struct stat *st,
inode->inode_isize = sizeof(struct erofs_inode_compact);
}
- list_add(&inode->i_hash,
- &inode_hashtable[(st->st_ino ^ st->st_dev) %
- NR_INODE_HASHTABLE]);
+ erofs_insert_ihash(inode, st->st_dev, st->st_ino);
return 0;
}
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH v8 5/8] erofs-utils: lib: add erofs_rebuild_get_dentry() helper
2023-09-13 12:02 [PATCH v8 0/8] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (3 preceding siblings ...)
2023-09-13 12:02 ` [PATCH v8 4/8] erofs-utils: lib: add erofs_insert_ihash() helper Jingbo Xu
@ 2023-09-13 12:03 ` Jingbo Xu
2023-09-13 12:03 ` [PATCH v8 6/8] erofs-utils: lib: add erofs_rebuild_load_tree() helper Jingbo Xu
` (3 subsequent siblings)
8 siblings, 0 replies; 13+ messages in thread
From: Jingbo Xu @ 2023-09-13 12:03 UTC (permalink / raw)
To: hsiangkao, linux-erofs
Rename tarerofs_get_dentry() to erofs_rebuild_get_dentry() and
move it into lib/rebuild.c.
No logic changes.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
include/erofs/rebuild.h | 19 +++++++
lib/Makefile.am | 3 +-
lib/rebuild.c | 116 ++++++++++++++++++++++++++++++++++++++++
lib/tar.c | 109 ++-----------------------------------
4 files changed, 140 insertions(+), 107 deletions(-)
create mode 100644 include/erofs/rebuild.h
create mode 100644 lib/rebuild.c
diff --git a/include/erofs/rebuild.h b/include/erofs/rebuild.h
new file mode 100644
index 0000000..92873c9
--- /dev/null
+++ b/include/erofs/rebuild.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */
+#ifndef __EROFS_REBUILD_H
+#define __EROFS_REBUILD_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#include "internal.h"
+
+struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
+ char *path, bool aufs, bool *whout, bool *opq);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 3e09516..8a45bd6 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -25,6 +25,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
$(top_srcdir)/include/erofs/compress_hints.h \
$(top_srcdir)/include/erofs/fragments.h \
$(top_srcdir)/include/erofs/xxhash.h \
+ $(top_srcdir)/include/erofs/rebuild.h \
$(top_srcdir)/lib/liberofs_private.h
noinst_HEADERS += compressor.h
@@ -32,7 +33,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
namei.c data.c compress.c compressor.c zmap.c decompress.c \
compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \
- block_list.c xxhash.c
+ block_list.c xxhash.c rebuild.c
liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include
if ENABLE_LZ4
diff --git a/lib/rebuild.c b/lib/rebuild.c
new file mode 100644
index 0000000..2398ca6
--- /dev/null
+++ b/lib/rebuild.c
@@ -0,0 +1,116 @@
+// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0
+#define _GNU_SOURCE
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include "erofs/print.h"
+#include "erofs/inode.h"
+#include "erofs/rebuild.h"
+#include "erofs/internal.h"
+
+#ifdef HAVE_LINUX_AUFS_TYPE_H
+#include <linux/aufs_type.h>
+#else
+#define AUFS_WH_PFX ".wh."
+#define AUFS_DIROPQ_NAME AUFS_WH_PFX ".opq"
+#define AUFS_WH_DIROPQ AUFS_WH_PFX AUFS_DIROPQ_NAME
+#endif
+
+static struct erofs_dentry *erofs_rebuild_mkdir(struct erofs_inode *dir,
+ const char *s)
+{
+ struct erofs_inode *inode;
+ struct erofs_dentry *d;
+
+ inode = erofs_new_inode();
+ if (IS_ERR(inode))
+ return ERR_CAST(inode);
+
+ inode->i_mode = S_IFDIR | 0755;
+ inode->i_parent = dir;
+ inode->i_uid = getuid();
+ inode->i_gid = getgid();
+ inode->i_mtime = inode->sbi->build_time;
+ inode->i_mtime_nsec = inode->sbi->build_time_nsec;
+ erofs_init_empty_dir(inode);
+
+ d = erofs_d_alloc(dir, s);
+ if (!IS_ERR(d)) {
+ d->type = EROFS_FT_DIR;
+ d->inode = inode;
+ }
+ return d;
+}
+
+struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
+ char *path, bool aufs, bool *whout, bool *opq)
+{
+ struct erofs_dentry *d = NULL;
+ unsigned int len = strlen(path);
+ char *s = path;
+
+ *whout = false;
+ *opq = false;
+
+ while (s < path + len) {
+ char *slash = memchr(s, '/', path + len - s);
+
+ if (slash) {
+ if (s == slash) {
+ while (*++s == '/'); /* skip '//...' */
+ continue;
+ }
+ *slash = '\0';
+ }
+
+ if (!memcmp(s, ".", 2)) {
+ /* null */
+ } else if (!memcmp(s, "..", 3)) {
+ pwd = pwd->i_parent;
+ } else {
+ struct erofs_inode *inode = NULL;
+
+ if (aufs && !slash) {
+ if (!memcmp(s, AUFS_WH_DIROPQ, sizeof(AUFS_WH_DIROPQ))) {
+ *opq = true;
+ break;
+ }
+ if (!memcmp(s, AUFS_WH_PFX, sizeof(AUFS_WH_PFX) - 1)) {
+ s += sizeof(AUFS_WH_PFX) - 1;
+ *whout = true;
+ }
+ }
+
+ list_for_each_entry(d, &pwd->i_subdirs, d_child) {
+ if (!strcmp(d->name, s)) {
+ if (d->type != EROFS_FT_DIR && slash)
+ return ERR_PTR(-EIO);
+ inode = d->inode;
+ break;
+ }
+ }
+
+ if (inode) {
+ pwd = inode;
+ } else if (!slash) {
+ d = erofs_d_alloc(pwd, s);
+ if (IS_ERR(d))
+ return d;
+ d->type = EROFS_FT_UNKNOWN;
+ d->inode = pwd;
+ } else {
+ d = erofs_rebuild_mkdir(pwd, s);
+ if (IS_ERR(d))
+ return d;
+ pwd = d->inode;
+ }
+ }
+ if (slash) {
+ *slash = '/';
+ s = slash + 1;
+ } else {
+ break;
+ }
+ }
+ return d;
+}
diff --git a/lib/tar.c b/lib/tar.c
index b58bfd5..0b96cae 100644
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -3,13 +3,6 @@
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
-#ifdef HAVE_LINUX_AUFS_TYPE_H
-#include <linux/aufs_type.h>
-#else
-#define AUFS_WH_PFX ".wh."
-#define AUFS_DIROPQ_NAME AUFS_WH_PFX ".opq"
-#define AUFS_WH_DIROPQ AUFS_WH_PFX AUFS_DIROPQ_NAME
-#endif
#include "erofs/print.h"
#include "erofs/cache.h"
#include "erofs/inode.h"
@@ -18,6 +11,7 @@
#include "erofs/io.h"
#include "erofs/xattr.h"
#include "erofs/blobchunk.h"
+#include "erofs/rebuild.h"
static char erofs_libbuf[16384];
@@ -131,103 +125,6 @@ static long long tarerofs_parsenum(const char *ptr, int len)
return tarerofs_otoi(ptr, len);
}
-static struct erofs_dentry *tarerofs_mkdir(struct erofs_inode *dir, const char *s)
-{
- struct erofs_inode *inode;
- struct erofs_dentry *d;
-
- inode = erofs_new_inode();
- if (IS_ERR(inode))
- return ERR_CAST(inode);
-
- inode->i_mode = S_IFDIR | 0755;
- inode->i_parent = dir;
- inode->i_uid = getuid();
- inode->i_gid = getgid();
- inode->i_mtime = inode->sbi->build_time;
- inode->i_mtime_nsec = inode->sbi->build_time_nsec;
- erofs_init_empty_dir(inode);
-
- d = erofs_d_alloc(dir, s);
- if (!IS_ERR(d)) {
- d->type = EROFS_FT_DIR;
- d->inode = inode;
- }
- return d;
-}
-
-static struct erofs_dentry *tarerofs_get_dentry(struct erofs_inode *pwd, char *path,
- bool aufs, bool *whout, bool *opq)
-{
- struct erofs_dentry *d = NULL;
- unsigned int len = strlen(path);
- char *s = path;
-
- *whout = false;
- *opq = false;
-
- while (s < path + len) {
- char *slash = memchr(s, '/', path + len - s);
- if (slash) {
- if (s == slash) {
- while (*++s == '/'); /* skip '//...' */
- continue;
- }
- *slash = '\0';
- }
-
- if (!memcmp(s, ".", 2)) {
- /* null */
- } else if (!memcmp(s, "..", 3)) {
- pwd = pwd->i_parent;
- } else {
- struct erofs_inode *inode = NULL;
-
- if (aufs && !slash) {
- if (!memcmp(s, AUFS_WH_DIROPQ, sizeof(AUFS_WH_DIROPQ))) {
- *opq = true;
- break;
- }
- if (!memcmp(s, AUFS_WH_PFX, sizeof(AUFS_WH_PFX) - 1)) {
- s += sizeof(AUFS_WH_PFX) - 1;
- *whout = true;
- }
- }
-
- list_for_each_entry(d, &pwd->i_subdirs, d_child) {
- if (!strcmp(d->name, s)) {
- if (d->type != EROFS_FT_DIR && slash)
- return ERR_PTR(-EIO);
- inode = d->inode;
- break;
- }
- }
-
- if (inode) {
- pwd = inode;
- } else if (!slash) {
- d = erofs_d_alloc(pwd, s);
- if (IS_ERR(d))
- return d;
- d->type = EROFS_FT_UNKNOWN;
- d->inode = pwd;
- } else {
- d = tarerofs_mkdir(pwd, s);
- if (IS_ERR(d))
- return d;
- pwd = d->inode;
- }
- }
- if (slash) {
- *slash = '/';
- s = slash + 1;
- } else {
- break;
- }
- }
- return d;
-}
-
struct tarerofs_xattr_item {
struct list_head list;
char *kv;
@@ -765,7 +662,7 @@ restart:
erofs_dbg("parsing %s (mode %05o)", eh.path, st.st_mode);
- d = tarerofs_get_dentry(root, eh.path, tar->aufs, &whout, &opq);
+ d = erofs_rebuild_get_dentry(root, eh.path, tar->aufs, &whout, &opq);
if (IS_ERR(d)) {
ret = PTR_ERR(d);
goto out;
@@ -798,7 +695,7 @@ restart:
}
d->inode = NULL;
- d2 = tarerofs_get_dentry(root, eh.link, tar->aufs, &dumb, &dumb);
+ d2 = erofs_rebuild_get_dentry(root, eh.link, tar->aufs, &dumb, &dumb);
if (IS_ERR(d2)) {
ret = PTR_ERR(d2);
goto out;
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH v8 6/8] erofs-utils: lib: add erofs_rebuild_load_tree() helper
2023-09-13 12:02 [PATCH v8 0/8] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (4 preceding siblings ...)
2023-09-13 12:03 ` [PATCH v8 5/8] erofs-utils: lib: add erofs_rebuild_get_dentry() helper Jingbo Xu
@ 2023-09-13 12:03 ` Jingbo Xu
2023-09-13 12:03 ` [PATCH v8 7/8] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (2 subsequent siblings)
8 siblings, 0 replies; 13+ messages in thread
From: Jingbo Xu @ 2023-09-13 12:03 UTC (permalink / raw)
To: hsiangkao, linux-erofs
Add erofs_rebuild_load_tree() helper to load the inode tree from a
given erofs image, and make it merged acting as an overlayfs-like
model.
Since the content of the symlink file needs to be read when loading
tree, let's add a dependency on zlib_LIBS for mkfs.erofs.
Also rename tarerofs_dump_tree() to erofs_rebuild_dump_tree() since
it is also called in the rebuild mode.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
include/erofs/inode.h | 2 +-
include/erofs/internal.h | 6 +-
include/erofs/rebuild.h | 2 +
lib/inode.c | 4 +-
lib/rebuild.c | 288 +++++++++++++++++++++++++++++++++++++++
mkfs/main.c | 2 +-
6 files changed, 298 insertions(+), 6 deletions(-)
diff --git a/include/erofs/inode.h b/include/erofs/inode.h
index 1c602a8..fe9dda2 100644
--- a/include/erofs/inode.h
+++ b/include/erofs/inode.h
@@ -32,7 +32,7 @@ unsigned int erofs_iput(struct erofs_inode *inode);
erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
const char *name);
-int tarerofs_dump_tree(struct erofs_inode *dir);
+int erofs_rebuild_dump_tree(struct erofs_inode *dir);
int erofs_init_empty_dir(struct erofs_inode *dir);
struct erofs_inode *erofs_new_inode(void);
struct erofs_inode *erofs_mkfs_build_tree_from_path(const char *path);
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 3b866c9..19b912b 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -109,6 +109,7 @@ struct erofs_sb_info {
int devfd;
u64 devsz;
+ dev_t dev;
unsigned int nblobs;
unsigned int blobfd[256];
@@ -155,8 +156,6 @@ struct erofs_inode {
union {
/* (erofsfuse) runtime flags */
unsigned int flags;
- /* (mkfs.erofs) device ID containing source file */
- u32 dev;
/* (mkfs.erofs) queued sub-directories blocking dump */
u32 subdirs_queued;
};
@@ -164,6 +163,9 @@ struct erofs_inode {
struct erofs_sb_info *sbi;
struct erofs_inode *i_parent;
+ /* (mkfs.erofs) device ID containing source file */
+ u32 dev;
+
umode_t i_mode;
erofs_off_t i_size;
diff --git a/include/erofs/rebuild.h b/include/erofs/rebuild.h
index 92873c9..3ac074c 100644
--- a/include/erofs/rebuild.h
+++ b/include/erofs/rebuild.h
@@ -12,6 +12,8 @@ extern "C"
struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
char *path, bool aufs, bool *whout, bool *opq);
+int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi);
+
#ifdef __cplusplus
}
#endif
diff --git a/lib/inode.c b/lib/inode.c
index 79e1795..93e6b23 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -1324,7 +1324,7 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)
return inode;
}
-int tarerofs_dump_tree(struct erofs_inode *dir)
+int erofs_rebuild_dump_tree(struct erofs_inode *dir)
{
struct erofs_dentry *d;
unsigned int nr_subdirs;
@@ -1395,7 +1395,7 @@ int tarerofs_dump_tree(struct erofs_inode *dir)
continue;
inode = erofs_igrab(d->inode);
- ret = tarerofs_dump_tree(inode);
+ ret = erofs_rebuild_dump_tree(inode);
dir->i_nlink += (erofs_mode_to_ftype(inode->i_mode) == EROFS_FT_DIR);
erofs_iput(inode);
if (ret)
diff --git a/lib/rebuild.c b/lib/rebuild.c
index 2398ca6..27a1df4 100644
--- a/lib/rebuild.c
+++ b/lib/rebuild.c
@@ -3,9 +3,18 @@
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
+#include <sys/stat.h>
+#include <config.h>
+#if defined(HAVE_SYS_SYSMACROS_H)
+#include <sys/sysmacros.h>
+#endif
#include "erofs/print.h"
#include "erofs/inode.h"
#include "erofs/rebuild.h"
+#include "erofs/io.h"
+#include "erofs/dir.h"
+#include "erofs/xattr.h"
+#include "erofs/blobchunk.h"
#include "erofs/internal.h"
#ifdef HAVE_LINUX_AUFS_TYPE_H
@@ -114,3 +123,282 @@ struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
}
return d;
}
+
+static int erofs_rebuild_fixup_inode_index(struct erofs_inode *inode)
+{
+ int ret;
+ unsigned int count, unit, chunkbits, i;
+ struct erofs_inode_chunk_index *idx;
+ erofs_off_t chunksize;
+ erofs_blk_t blkaddr;
+
+ /* TODO: fill data map in other layouts */
+ if (inode->datalayout != EROFS_INODE_CHUNK_BASED &&
+ inode->datalayout != EROFS_INODE_FLAT_PLAIN) {
+ erofs_err("%s: unsupported datalayout %d", inode->i_srcpath, inode->datalayout);
+ return -EOPNOTSUPP;
+ }
+
+ if (inode->sbi->extra_devices) {
+ chunkbits = inode->u.chunkbits;
+ if (chunkbits < sbi.blkszbits) {
+ erofs_err("%s: chunk size %u is too small to fit the target block size %u",
+ inode->i_srcpath, 1U << chunkbits, 1U << sbi.blkszbits);
+ return -EINVAL;
+ }
+ } else {
+ chunkbits = ilog2(inode->i_size - 1) + 1;
+ if (chunkbits < sbi.blkszbits)
+ chunkbits = sbi.blkszbits;
+ if (chunkbits - sbi.blkszbits > EROFS_CHUNK_FORMAT_BLKBITS_MASK)
+ chunkbits = EROFS_CHUNK_FORMAT_BLKBITS_MASK + sbi.blkszbits;
+ }
+ chunksize = 1ULL << chunkbits;
+ count = DIV_ROUND_UP(inode->i_size, chunksize);
+
+ unit = sizeof(struct erofs_inode_chunk_index);
+ inode->extent_isize = count * unit;
+ idx = malloc(max(sizeof(*idx), sizeof(void *)));
+ if (!idx)
+ return -ENOMEM;
+ inode->chunkindexes = idx;
+
+ for (i = 0; i < count; i++) {
+ struct erofs_blobchunk *chunk;
+ struct erofs_map_blocks map = {
+ .index = UINT_MAX,
+ };
+
+ map.m_la = i << chunkbits;
+ ret = erofs_map_blocks(inode, &map, 0);
+ if (ret)
+ goto err;
+
+ blkaddr = erofs_blknr(&sbi, map.m_pa);
+ chunk = erofs_get_unhashed_chunk(inode->dev, blkaddr, 0);
+ if (IS_ERR(chunk)) {
+ ret = PTR_ERR(chunk);
+ goto err;
+ }
+ *(void **)idx++ = chunk;
+
+ }
+ inode->datalayout = EROFS_INODE_CHUNK_BASED;
+ inode->u.chunkformat = EROFS_CHUNK_FORMAT_INDEXES;
+ inode->u.chunkformat |= chunkbits - sbi.blkszbits;
+ return 0;
+err:
+ free(idx);
+ inode->chunkindexes = NULL;
+ return ret;
+}
+
+static int erofs_rebuild_fill_inode(struct erofs_inode *inode)
+{
+ switch (inode->i_mode & S_IFMT) {
+ case S_IFCHR:
+ if (erofs_inode_is_whiteout(inode))
+ inode->i_parent->whiteouts = true;
+ /* fallthrough */
+ case S_IFBLK:
+ case S_IFIFO:
+ case S_IFSOCK:
+ inode->i_size = 0;
+ erofs_dbg("\tdev: %d %d", major(inode->u.i_rdev),
+ minor(inode->u.i_rdev));
+ inode->u.i_rdev = erofs_new_encode_dev(inode->u.i_rdev);
+ return 0;
+ case S_IFDIR:
+ return erofs_init_empty_dir(inode);
+ case S_IFLNK: {
+ int ret;
+
+ inode->i_link = malloc(inode->i_size + 1);
+ if (!inode->i_link)
+ return -ENOMEM;
+ ret = erofs_pread(inode, inode->i_link, inode->i_size, 0);
+ erofs_dbg("\tsymlink: %s -> %s", inode->i_srcpath, inode->i_link);
+ return ret;
+ }
+ case S_IFREG:
+ if (inode->i_size)
+ return erofs_rebuild_fixup_inode_index(inode);
+ return 0;
+ default:
+ break;
+ }
+ return -EINVAL;
+}
+
+/*
+ * @parent: parent directory in inode tree
+ * @ctx.dir: parent directory when itering erofs_iterate_dir()
+ */
+struct erofs_rebuild_dir_context {
+ struct erofs_dir_context ctx;
+ struct erofs_inode *parent;
+};
+
+static int erofs_rebuild_dirent_iter(struct erofs_dir_context *ctx)
+{
+ struct erofs_rebuild_dir_context *rctx = (void *)ctx;
+ struct erofs_inode *parent = rctx->parent;
+ struct erofs_inode *dir = ctx->dir;
+ struct erofs_inode *inode, *candidate;
+ struct erofs_inode src;
+ struct erofs_dentry *d;
+ char *path, *dname;
+ bool dumb;
+ int ret;
+
+ if (ctx->dot_dotdot)
+ return 0;
+
+ ret = asprintf(&path, "%s/%.*s", rctx->parent->i_srcpath,
+ ctx->de_namelen, ctx->dname);
+ if (ret < 0)
+ return ret;
+
+ erofs_dbg("parsing %s", path);
+ dname = path + strlen(parent->i_srcpath) + 1;
+
+ d = erofs_rebuild_get_dentry(parent, dname, false, &dumb, &dumb);
+ if (IS_ERR(d)) {
+ ret = PTR_ERR(d);
+ goto out;
+ }
+
+ ret = 0;
+ if (d->type != EROFS_FT_UNKNOWN) {
+ /*
+ * bail out if the file exists in the upper layers. (Note that
+ * extended attributes won't be merged too even for dirs.)
+ */
+ if (!S_ISDIR(d->inode->i_mode) || d->inode->opaque)
+ goto out;
+
+ /* merge directory entries */
+ src = (struct erofs_inode) {
+ .sbi = dir->sbi,
+ .nid = ctx->de_nid
+ };
+ ret = erofs_read_inode_from_disk(&src);
+ if (ret || !S_ISDIR(src.i_mode))
+ goto out;
+ parent = d->inode;
+ inode = dir = &src;
+ } else {
+ u64 nid;
+
+ DBG_BUGON(parent != d->inode);
+ inode = erofs_new_inode();
+ if (IS_ERR(inode)) {
+ ret = PTR_ERR(inode);
+ goto out;
+ }
+
+ /* reuse i_ino[0] to read nid in source fs */
+ nid = inode->i_ino[0];
+ inode->sbi = dir->sbi;
+ inode->nid = ctx->de_nid;
+ ret = erofs_read_inode_from_disk(inode);
+ if (ret)
+ goto out;
+
+ /* restore nid in new generated fs */
+ inode->i_ino[1] = inode->i_ino[0];
+ inode->i_ino[0] = nid;
+ inode->dev = inode->sbi->dev;
+
+ if (S_ISREG(inode->i_mode) && inode->i_nlink > 1 &&
+ (candidate = erofs_iget(inode->dev, ctx->de_nid))) {
+ /* hardlink file */
+ erofs_iput(inode);
+ inode = candidate;
+ if (S_ISDIR(inode->i_mode)) {
+ erofs_err("hardlink directory not supported");
+ ret = -EISDIR;
+ goto out;
+ }
+ inode->i_nlink++;
+ erofs_dbg("\thardlink: %s -> %s", path, inode->i_srcpath);
+ } else {
+ ret = erofs_read_xattrs_from_disk(inode);
+ if (ret) {
+ erofs_iput(inode);
+ goto out;
+ }
+
+ inode->i_parent = d->inode;
+ inode->i_srcpath = path;
+ path = NULL;
+ inode->i_ino[1] = inode->nid;
+ inode->i_nlink = 1;
+
+ ret = erofs_rebuild_fill_inode(inode);
+ if (ret) {
+ erofs_iput(inode);
+ goto out;
+ }
+
+ erofs_insert_ihash(inode, inode->dev, inode->i_ino[1]);
+ parent = dir = inode;
+ }
+
+ d->inode = inode;
+ d->type = erofs_mode_to_ftype(inode->i_mode);
+ }
+
+ if (S_ISDIR(inode->i_mode)) {
+ struct erofs_rebuild_dir_context nctx = *rctx;
+
+ nctx.parent = parent;
+ nctx.ctx.dir = dir;
+ ret = erofs_iterate_dir(&nctx.ctx, false);
+ if (ret)
+ goto out;
+ }
+
+ /* reset sbi, nid after subdirs are all loaded for the final dump */
+ inode->sbi = &sbi;
+ inode->nid = 0;
+out:
+ free(path);
+ return ret;
+}
+
+int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi)
+{
+ struct erofs_inode inode = {};
+ struct erofs_rebuild_dir_context ctx;
+ int ret;
+
+ if (!sbi->devname) {
+ erofs_err("failed to find a device for rebuilding");
+ return -EINVAL;
+ }
+
+ ret = erofs_read_superblock(sbi);
+ if (ret) {
+ erofs_err("failed to read superblock of %s", sbi->devname);
+ return ret;
+ }
+
+ inode.nid = sbi->root_nid;
+ inode.sbi = sbi;
+ ret = erofs_read_inode_from_disk(&inode);
+ if (ret) {
+ erofs_err("failed to read root inode of %s", sbi->devname);
+ return ret;
+ }
+ inode.i_srcpath = strdup("/");
+
+ ctx = (struct erofs_rebuild_dir_context) {
+ .ctx.dir = &inode,
+ .ctx.cb = erofs_rebuild_dirent_iter,
+ .parent = root,
+ };
+ ret = erofs_iterate_dir(&ctx.ctx, false);
+ free(inode.i_srcpath);
+ return ret;
+}
diff --git a/mkfs/main.c b/mkfs/main.c
index c7f4047..49b91c5 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -965,7 +965,7 @@ int main(int argc, char **argv)
if (err < 0)
goto exit;
- err = tarerofs_dump_tree(root_inode);
+ err = erofs_rebuild_dump_tree(root_inode);
if (err < 0)
goto exit;
}
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH v8 7/8] erofs-utils: mkfs: introduce rebuild mode
2023-09-13 12:02 [PATCH v8 0/8] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (5 preceding siblings ...)
2023-09-13 12:03 ` [PATCH v8 6/8] erofs-utils: lib: add erofs_rebuild_load_tree() helper Jingbo Xu
@ 2023-09-13 12:03 ` Jingbo Xu
2023-09-14 7:49 ` Gao Xiang
2023-09-13 12:03 ` [PATCH v8 8/8] erofs-utils: mkfs: add `--ovlfs-strip` option Jingbo Xu
2023-09-13 12:03 ` [PATCH v3] erofs-utils: lib: refactor extended attribute name prefixes Jingbo Xu
8 siblings, 1 reply; 13+ messages in thread
From: Jingbo Xu @ 2023-09-13 12:03 UTC (permalink / raw)
To: hsiangkao, linux-erofs
Introduce a new EXPERIMENTAL rebuild mode, which can be used to
generate a meta-only multidev manifest image with an overlayfs-like
merged tree from multiple specific EROFS images either of
tarerofs index mode (--tar=i):
mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar
...
mkfs.erofs --tar=i --aufs layerN.erofs layerN-1.tar
or mkfs.erofs uncompressed mode without inline data:
mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar
...
mkfs.erofs --tar=f -Enoinline_data --aufs layerN-1.erofs layerN-1.tar
To merge these layers, just type:
mkfs.erofs merged.erofs layer0.erofs ... layerN-1.erofs
It doesn't support compression and/or flat inline datalayout yet.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
mkfs/main.c | 219 ++++++++++++++++++++++++++++++++++++++++------------
1 file changed, 168 insertions(+), 51 deletions(-)
diff --git a/mkfs/main.c b/mkfs/main.c
index 49b91c5..ce2b0e2 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -26,6 +26,7 @@
#include "erofs/compress_hints.h"
#include "erofs/blobchunk.h"
#include "erofs/fragments.h"
+#include "erofs/rebuild.h"
#include "../lib/liberofs_private.h"
#include "../lib/liberofs_uuid.h"
@@ -83,8 +84,8 @@ static void print_available_compressors(FILE *f, const char *delim)
static void usage(void)
{
- fputs("usage: [options] FILE DIRECTORY\n\n"
- "Generate erofs image from DIRECTORY to FILE, and [options] are:\n"
+ fputs("usage: [options] FILE SOURCE(s)\n"
+ "Generate EROFS image (FILE) from DIRECTORY, TARBALL and/or EROFS images. And [options] are:\n"
" -b# set block size to # (# = page size by default)\n"
" -d# set output message level to # (maximum 9)\n"
" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
@@ -135,7 +136,10 @@ static unsigned int pclustersize_packed, pclustersize_max;
static struct erofs_tarfile erofstar = {
.global.xattrs = LIST_HEAD_INIT(erofstar.global.xattrs)
};
-static bool tar_mode;
+static bool tar_mode, rebuild_mode;
+
+static unsigned int rebuild_src_count;
+static LIST_HEAD(rebuild_src_list);
static int parse_extended_opts(const char *opts)
{
@@ -277,10 +281,23 @@ static int mkfs_parse_compress_algs(char *algs)
return 0;
}
+static void erofs_rebuild_cleanup(void)
+{
+ struct erofs_sb_info *src, *n;
+
+ list_for_each_entry_safe(src, n, &rebuild_src_list, list) {
+ list_del(&src->list);
+ erofs_put_super(src);
+ dev_close(src);
+ free(src);
+ }
+ rebuild_src_count = 0;
+}
+
static int mkfs_parse_options_cfg(int argc, char *argv[])
{
char *endptr;
- int opt, i;
+ int opt, i, err;
bool quiet = false;
while ((opt = getopt_long(argc, argv, "C:E:L:T:U:b:d:x:z:",
@@ -531,12 +548,14 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
if (optind >= argc) {
if (!tar_mode) {
- erofs_err("missing argument: DIRECTORY");
+ erofs_err("missing argument: SOURCE(s)");
return -EINVAL;
} else {
erofstar.fd = STDIN_FILENO;
}
- }else {
+ } else {
+ struct stat st;
+
cfg.c_src_path = realpath(argv[optind++], NULL);
if (!cfg.c_src_path) {
erofs_err("failed to parse source directory: %s",
@@ -544,7 +563,47 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
return -ENOENT;
}
- if (optind < argc) {
+ if (tar_mode) {
+ erofstar.fd = open(cfg.c_src_path, O_RDONLY);
+ if (erofstar.fd < 0) {
+ erofs_err("failed to open file: %s", cfg.c_src_path);
+ usage();
+ return -errno;
+ }
+ } else {
+ err = lstat(cfg.c_src_path, &st);
+ if (err)
+ return -errno;
+ if (S_ISDIR(st.st_mode))
+ erofs_set_fs_root(cfg.c_src_path);
+ else
+ rebuild_mode = true;
+ }
+
+ if (rebuild_mode) {
+ char *srcpath = cfg.c_src_path;
+
+ do {
+ struct erofs_sb_info *src;
+
+ src = calloc(1, sizeof(struct erofs_sb_info));
+ if (!src) {
+ erofs_rebuild_cleanup();
+ return -ENOMEM;
+ }
+
+ err = dev_open_ro(src, srcpath);
+ if (err) {
+ free(src);
+ erofs_rebuild_cleanup();
+ return err;
+ }
+
+ /* extra device index starts from 1 */
+ src->dev = ++rebuild_src_count;
+ list_add(&src->list, &rebuild_src_list);
+ } while (optind < argc && (srcpath = argv[optind++]));
+ } else if (optind < argc) {
erofs_err("unexpected argument: %s\n", argv[optind]);
return -EINVAL;
}
@@ -735,6 +794,61 @@ void erofs_show_progs(int argc, char *argv[])
if (cfg.c_dbg_lvl >= EROFS_WARN)
printf("%s %s\n", basename(argv[0]), cfg.c_version);
}
+static struct erofs_inode *erofs_alloc_root_inode(void)
+{
+ struct erofs_inode *root;
+
+ root = erofs_new_inode();
+ if (IS_ERR(root))
+ return root;
+ root->i_srcpath = strdup("/");
+ root->i_mode = S_IFDIR | 0777;
+ root->i_parent = root;
+ root->i_mtime = root->sbi->build_time;
+ root->i_mtime_nsec = root->sbi->build_time_nsec;
+ erofs_init_empty_dir(root);
+ return root;
+}
+
+static int erofs_rebuild_load_trees(struct erofs_inode *root)
+{
+ struct erofs_sb_info *src;
+ unsigned int extra_devices = 0;
+ erofs_blk_t nblocks;
+ int ret, i = 0;
+
+ list_for_each_entry(src, &rebuild_src_list, list) {
+ ret = erofs_rebuild_load_tree(root, src);
+ if (ret) {
+ erofs_err("failed to load %s", src->devname);
+ return ret;
+ }
+ if (src->extra_devices > 1) {
+ erofs_err("%s: unsupported number of extra devices",
+ src->devname, src->extra_devices);
+ return -EOPNOTSUPP;
+ }
+ extra_devices += src->extra_devices;
+ }
+
+ if (extra_devices && extra_devices != rebuild_src_count) {
+ erofs_err("unsupported mix of source images");
+ return -EOPNOTSUPP;
+ }
+
+ ret = erofs_mkfs_init_devices(&sbi, rebuild_src_count);
+ if (ret)
+ return ret;
+
+ list_for_each_entry(src, &rebuild_src_list, list) {
+ if (extra_devices)
+ nblocks = src->devs[0].blocks;
+ else
+ nblocks = src->primarydevice_blocks;
+ sbi.devs[i++].blocks = nblocks;
+ }
+ return 0;
+}
static void erofs_mkfs_showsummaries(erofs_blk_t nblocks)
{
@@ -761,7 +875,6 @@ int main(int argc, char **argv)
struct erofs_buffer_head *sb_bh;
struct erofs_inode *root_inode, *packed_inode;
erofs_nid_t root_nid, packed_nid;
- struct stat st;
erofs_blk_t nblocks;
struct timeval t;
FILE *packedfile = NULL;
@@ -783,26 +896,6 @@ int main(int argc, char **argv)
return 1;
}
- if (!tar_mode) {
- err = lstat(cfg.c_src_path, &st);
- if (err)
- return 1;
- if (!S_ISDIR(st.st_mode)) {
- erofs_err("root of the filesystem is not a directory - %s",
- cfg.c_src_path);
- usage();
- return 1;
- }
- erofs_set_fs_root(cfg.c_src_path);
- } else if (cfg.c_src_path) {
- erofstar.fd = open(cfg.c_src_path, O_RDONLY);
- if (erofstar.fd < 0) {
- erofs_err("failed to open file: %s", cfg.c_src_path);
- usage();
- return 1;
- }
- }
-
if (cfg.c_unix_timestamp != -1) {
sbi.build_time = cfg.c_unix_timestamp;
sbi.build_time_nsec = 0;
@@ -866,6 +959,22 @@ int main(int argc, char **argv)
}
}
+ if (rebuild_mode) {
+ struct erofs_sb_info *src;
+
+ erofs_warn("EXPERIMENTAL rebuild mode in use. Use at your own risk!");
+
+ src = list_first_entry(&rebuild_src_list, struct erofs_sb_info, list);
+ if (!src)
+ goto exit;
+ err = erofs_read_superblock(src);
+ if (err) {
+ erofs_err("failed to read superblock of %s", src->devname);
+ goto exit;
+ }
+ sbi.blkszbits = src->blkszbits;
+ }
+
sb_bh = erofs_buffer_init();
if (IS_ERR(sb_bh)) {
err = PTR_ERR(sb_bh);
@@ -931,7 +1040,35 @@ int main(int argc, char **argv)
erofs_inode_manager_init();
- if (!tar_mode) {
+ if (tar_mode) {
+ root_inode = erofs_alloc_root_inode();
+ if (IS_ERR(root_inode)) {
+ err = PTR_ERR(root_inode);
+ goto exit;
+ }
+
+ while (!(err = tarerofs_parse_tar(root_inode, &erofstar)));
+
+ if (err < 0)
+ goto exit;
+
+ err = erofs_rebuild_dump_tree(root_inode);
+ if (err < 0)
+ goto exit;
+ } else if (rebuild_mode) {
+ root_inode = erofs_alloc_root_inode();
+ if (IS_ERR(root_inode)) {
+ err = PTR_ERR(root_inode);
+ goto exit;
+ }
+
+ err = erofs_rebuild_load_trees(root_inode);
+ if (err)
+ goto exit;
+ err = erofs_rebuild_dump_tree(root_inode);
+ if (err)
+ goto exit;
+ } else {
err = erofs_build_shared_xattrs_from_path(&sbi, cfg.c_src_path);
if (err) {
erofs_err("failed to build shared xattrs: %s",
@@ -947,32 +1084,11 @@ int main(int argc, char **argv)
err = PTR_ERR(root_inode);
goto exit;
}
- } else {
- root_inode = erofs_new_inode();
- if (IS_ERR(root_inode)) {
- err = PTR_ERR(root_inode);
- goto exit;
- }
- root_inode->i_srcpath = strdup("/");
- root_inode->i_mode = S_IFDIR | 0777;
- root_inode->i_parent = root_inode;
- root_inode->i_mtime = sbi.build_time;
- root_inode->i_mtime_nsec = sbi.build_time_nsec;
- erofs_init_empty_dir(root_inode);
-
- while (!(err = tarerofs_parse_tar(root_inode, &erofstar)));
-
- if (err < 0)
- goto exit;
-
- err = erofs_rebuild_dump_tree(root_inode);
- if (err < 0)
- goto exit;
}
root_nid = erofs_lookupnid(root_inode);
erofs_iput(root_inode);
- if (erofstar.index_mode || cfg.c_chunkbits) {
+ if (erofstar.index_mode || cfg.c_chunkbits || sbi.extra_devices) {
if (erofstar.index_mode && !erofstar.mapfile)
sbi.devs[0].blocks =
BLK_ROUND_UP(&sbi, erofstar.offset);
@@ -1026,6 +1142,7 @@ exit:
z_erofs_fragments_exit();
erofs_packedfile_exit();
erofs_xattr_cleanup_name_prefixes();
+ erofs_rebuild_cleanup();
erofs_exit_configure();
if (err) {
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 13+ messages in thread* Re: [PATCH v8 7/8] erofs-utils: mkfs: introduce rebuild mode
2023-09-13 12:03 ` [PATCH v8 7/8] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
@ 2023-09-14 7:49 ` Gao Xiang
0 siblings, 0 replies; 13+ messages in thread
From: Gao Xiang @ 2023-09-14 7:49 UTC (permalink / raw)
To: Jingbo Xu, linux-erofs
On 2023/9/13 20:03, Jingbo Xu wrote:
> Introduce a new EXPERIMENTAL rebuild mode, which can be used to
> generate a meta-only multidev manifest image with an overlayfs-like
> merged tree from multiple specific EROFS images either of
>
> tarerofs index mode (--tar=i):
>
> mkfs.erofs --tar=i --aufs layer0.erofs layer0.tar
> ...
> mkfs.erofs --tar=i --aufs layerN.erofs layerN-1.tar
>
> or mkfs.erofs uncompressed mode without inline data:
>
> mkfs.erofs --tar=f -Enoinline_data --aufs layer0.erofs layer0.tar
> ...
> mkfs.erofs --tar=f -Enoinline_data --aufs layerN-1.erofs layerN-1.tar
>
> To merge these layers, just type:
> mkfs.erofs merged.erofs layer0.erofs ... layerN-1.erofs
>
> It doesn't support compression and/or flat inline datalayout yet.
>
> Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
> Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
As Jingbo reported, applying the following diff to fix the device table:
diff --git a/mkfs/main.c b/mkfs/main.c
index 389bb1b..c5dda66 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -590,10 +590,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
if (rebuild_mode) {
char *srcpath = cfg.c_src_path;
+ struct erofs_sb_info *src;
do {
- struct erofs_sb_info *src;
-
src = calloc(1, sizeof(struct erofs_sb_info));
if (!src) {
erofs_rebuild_cleanup();
@@ -823,7 +822,7 @@ static int erofs_rebuild_load_trees(struct erofs_inode *root)
struct erofs_sb_info *src;
unsigned int extra_devices = 0;
erofs_blk_t nblocks;
- int ret, i = 0;
+ int ret;
list_for_each_entry(src, &rebuild_src_list, list) {
ret = erofs_rebuild_load_tree(root, src);
@@ -853,7 +852,7 @@ static int erofs_rebuild_load_trees(struct erofs_inode *root)
nblocks = src->devs[0].blocks;
else
nblocks = src->primarydevice_blocks;
- sbi.devs[i++].blocks = nblocks;
+ sbi.devs[src->dev - 1].blocks = nblocks;
}
return 0;
}
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 13+ messages in thread
* [PATCH v8 8/8] erofs-utils: mkfs: add `--ovlfs-strip` option
2023-09-13 12:02 [PATCH v8 0/8] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (6 preceding siblings ...)
2023-09-13 12:03 ` [PATCH v8 7/8] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
@ 2023-09-13 12:03 ` Jingbo Xu
2023-09-13 13:47 ` Gao Xiang
2023-09-13 12:03 ` [PATCH v3] erofs-utils: lib: refactor extended attribute name prefixes Jingbo Xu
8 siblings, 1 reply; 13+ messages in thread
From: Jingbo Xu @ 2023-09-13 12:03 UTC (permalink / raw)
To: hsiangkao, linux-erofs
Add `--ovlfs-strip=[0|1]` option for tarfs and rebuild mode for now
in order to control whether some overlayfs related stuffs (e.g.
whiteout files, OVL_XATTR_OPAQUE and OVL_XATTR_ORIGIN xattrs) are
finally stripped.
This option is disabled by default for mkfs, that is, the overlayfs
related stuffs described above are kept in the image by default.
Specify `--ovlfs-strip=1` explicitly to strip these stuffs.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
include/erofs/config.h | 1 +
include/erofs/xattr.h | 1 +
lib/inode.c | 17 ++++++++++++++---
lib/xattr.c | 18 ++++++++++++++++++
mkfs/main.c | 8 ++++++++
5 files changed, 42 insertions(+), 3 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 5d3bfba..e342722 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -54,6 +54,7 @@ struct erofs_configure {
bool c_showprogress;
bool c_extra_ea_name_prefixes;
bool c_xattr_name_filter;
+ bool c_ovlfs_strip;
#ifdef HAVE_LIBSELINUX
struct selabel_handle *sehnd;
diff --git a/include/erofs/xattr.h b/include/erofs/xattr.h
index 0364f24..0f76037 100644
--- a/include/erofs/xattr.h
+++ b/include/erofs/xattr.h
@@ -57,6 +57,7 @@ int erofs_xattr_prefixes_init(struct erofs_sb_info *sbi);
int erofs_setxattr(struct erofs_inode *inode, char *key,
const void *value, size_t size);
int erofs_set_opaque_xattr(struct erofs_inode *inode);
+void erofs_clear_opaque_xattr(struct erofs_inode *inode);
int erofs_set_origin_xattr(struct erofs_inode *inode);
int erofs_read_xattrs_from_disk(struct erofs_inode *inode);
diff --git a/lib/inode.c b/lib/inode.c
index 93e6b23..37aa79e 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -1326,7 +1326,7 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)
int erofs_rebuild_dump_tree(struct erofs_inode *dir)
{
- struct erofs_dentry *d;
+ struct erofs_dentry *d, *n;
unsigned int nr_subdirs;
int ret;
@@ -1341,7 +1341,10 @@ int erofs_rebuild_dump_tree(struct erofs_inode *dir)
dir->inode_isize = sizeof(struct erofs_inode_compact);
}
- if (dir->whiteouts)
+ /* strip all unnecessary overlayfs xattrs when ovlfs_strip is enabled */
+ if (cfg.c_ovlfs_strip)
+ erofs_clear_opaque_xattr(dir);
+ else if (dir->whiteouts)
erofs_set_origin_xattr(dir);
ret = erofs_prepare_xattr_ibody(dir);
@@ -1373,8 +1376,16 @@ int erofs_rebuild_dump_tree(struct erofs_inode *dir)
}
nr_subdirs = 0;
- list_for_each_entry(d, &dir->i_subdirs, d_child)
+ list_for_each_entry_safe(d, n, &dir->i_subdirs, d_child) {
+ if (cfg.c_ovlfs_strip && erofs_inode_is_whiteout(d->inode)) {
+ erofs_dbg("remove whiteout %s", d->inode->i_srcpath);
+ list_del(&d->d_child);
+ erofs_d_invalidate(d);
+ free(d);
+ continue;
+ }
++nr_subdirs;
+ }
ret = erofs_prepare_dir_layout(dir, nr_subdirs);
if (ret)
diff --git a/lib/xattr.c b/lib/xattr.c
index e3a1b44..fffccb4 100644
--- a/lib/xattr.c
+++ b/lib/xattr.c
@@ -499,11 +499,29 @@ int erofs_setxattr(struct erofs_inode *inode, char *key,
return erofs_xattr_add(&inode->i_xattrs, item);
}
+static void erofs_clearxattr(struct erofs_inode *inode, const char *key)
+{
+ struct inode_xattr_node *node, *n;
+
+ list_for_each_entry_safe(node, n, &inode->i_xattrs, list) {
+ if (!strcmp(node->item->kvbuf, key)) {
+ list_del(&node->list);
+ put_xattritem(node->item);
+ free(node);
+ }
+ }
+}
+
int erofs_set_opaque_xattr(struct erofs_inode *inode)
{
return erofs_setxattr(inode, OVL_XATTR_OPAQUE, "y", 1);
}
+void erofs_clear_opaque_xattr(struct erofs_inode *inode)
+{
+ erofs_clearxattr(inode, OVL_XATTR_OPAQUE);
+}
+
int erofs_set_origin_xattr(struct erofs_inode *inode)
{
return erofs_setxattr(inode, OVL_XATTR_ORIGIN, NULL, 0);
diff --git a/mkfs/main.c b/mkfs/main.c
index ce2b0e2..389bb1b 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -64,6 +64,7 @@ static struct option long_options[] = {
{"fs-config-file", required_argument, NULL, 514},
{"block-list-file", required_argument, NULL, 515},
#endif
+ {"ovlfs-strip", optional_argument, NULL, 516},
{0, 0, 0, 0},
};
@@ -115,6 +116,7 @@ static void usage(void)
" --preserve-mtime keep per-file modification time strictly\n"
" --aufs replace aufs special files with overlayfs metadata\n"
" --tar=[fi] generate an image from tarball(s)\n"
+ " --ovlfs-strip=[01] strip overlayfs metadata in the target image (e.g. whiteouts)\n"
" --quiet quiet execution (do not write anything to standard output.)\n"
#ifndef NDEBUG
" --random-pclusterblks randomize pclusterblks for big pcluster (debugging only)\n"
@@ -516,6 +518,12 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
case 21:
erofstar.aufs = true;
break;
+ case 516:
+ if (!optarg || !strcmp(optarg, "1"))
+ cfg.c_ovlfs_strip = true;
+ else
+ cfg.c_ovlfs_strip = false;
+ break;
case 1:
usage();
exit(0);
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 13+ messages in thread* Re: [PATCH v8 8/8] erofs-utils: mkfs: add `--ovlfs-strip` option
2023-09-13 12:03 ` [PATCH v8 8/8] erofs-utils: mkfs: add `--ovlfs-strip` option Jingbo Xu
@ 2023-09-13 13:47 ` Gao Xiang
0 siblings, 0 replies; 13+ messages in thread
From: Gao Xiang @ 2023-09-13 13:47 UTC (permalink / raw)
To: Jingbo Xu; +Cc: hsiangkao, linux-erofs
On Wed, Sep 13, 2023 at 08:03:03PM +0800, Jingbo Xu wrote:
> Add `--ovlfs-strip=[0|1]` option for tarfs and rebuild mode for now
> in order to control whether some overlayfs related stuffs (e.g.
> whiteout files, OVL_XATTR_OPAQUE and OVL_XATTR_ORIGIN xattrs) are
> finally stripped.
>
> This option is disabled by default for mkfs, that is, the overlayfs
> related stuffs described above are kept in the image by default.
>
> Specify `--ovlfs-strip=1` explicitly to strip these stuffs.
>
> Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
> Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
> ---
...
> diff --git a/lib/xattr.c b/lib/xattr.c
> index e3a1b44..fffccb4 100644
> --- a/lib/xattr.c
> +++ b/lib/xattr.c
> @@ -499,11 +499,29 @@ int erofs_setxattr(struct erofs_inode *inode, char *key,
> return erofs_xattr_add(&inode->i_xattrs, item);
> }
>
> +static void erofs_clearxattr(struct erofs_inode *inode, const char *key)
I rename it as erofs_removexattr().
Thanks,
Gao Xiang
^ permalink raw reply [flat|nested] 13+ messages in thread
* [PATCH v3] erofs-utils: lib: refactor extended attribute name prefixes
2023-09-13 12:02 [PATCH v8 0/8] erofs-utils: mkfs: introduce rebuild mode Jingbo Xu
` (7 preceding siblings ...)
2023-09-13 12:03 ` [PATCH v8 8/8] erofs-utils: mkfs: add `--ovlfs-strip` option Jingbo Xu
@ 2023-09-13 12:03 ` Jingbo Xu
2023-09-13 12:13 ` Jingbo Xu
8 siblings, 1 reply; 13+ messages in thread
From: Jingbo Xu @ 2023-09-13 12:03 UTC (permalink / raw)
To: hsiangkao, linux-erofs
Previously, the extended attribute name in xattr_item was actually the
part with the matched prefix stripped, which makes it clumsy to strip
or delete a specific attribute from one file.
To fix this, make the complete attribute name stored in xattr_item.
One thing worth noting is that the attribute name in xattr_item has a
trailing '\0' for ease of name comparison, while the trailing '\0' will
get stripped when writing to on-disk erofs_xattr_entry.
Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com>
---
v3:
- introduce EROFS_XATTR_KSIZE() and EROFS_XATTR_KVSIZE() macro
- make the full long name prefix (instead of the infix in v2) stored in
ea_type_node; and thus compare with the full long name prefix in
get_xattritem() directly
- convert the data type of in-memory prefix index and prefix_len from
u8/u16 to unsigned int
---
lib/xattr.c | 379 ++++++++++++++++++++++------------------------------
1 file changed, 161 insertions(+), 218 deletions(-)
diff --git a/lib/xattr.c b/lib/xattr.c
index 54a6ae2..8119013 100644
--- a/lib/xattr.c
+++ b/lib/xattr.c
@@ -51,6 +51,12 @@
#ifndef XATTR_NAME_POSIX_ACL_DEFAULT
#define XATTR_NAME_POSIX_ACL_DEFAULT "system.posix_acl_default"
#endif
+#ifndef XATTR_NAME_SECURITY_SELINUX
+#define XATTR_NAME_SECURITY_SELINUX "security.selinux"
+#endif
+#ifndef XATTR_NAME_SECURITY_CAPABILITY
+#define XATTR_NAME_SECURITY_CAPABILITY "security.capability"
+#endif
#ifndef OVL_XATTR_NAMESPACE
#define OVL_XATTR_NAMESPACE "overlay."
#endif
@@ -72,12 +78,23 @@
#define EA_HASHTABLE_BITS 16
+/* extra one byte for the trailing `\0` of attribute name */
+#define EROFS_XATTR_KSIZE(kvlen) (kvlen[0] + 1)
+#define EROFS_XATTR_KVSIZE(kvlen) (EROFS_XATTR_KSIZE(kvlen) + kvlen[1])
+
+/*
+ * @base_index: the index of matched predefined short prefix
+ * @prefix: the index of matched long prefix if any;
+ * same as base_index otherwise
+ * @prefix_len: the length of matched long prefix if any;
+ * the length of matched predefined short prefix otherwise
+ */
struct xattr_item {
struct xattr_item *next_shared_xattr;
const char *kvbuf;
unsigned int hash[2], len[2], count;
int shared_xattr_id;
- u8 prefix;
+ unsigned int prefix, base_index, prefix_len;
struct hlist_node node;
};
@@ -93,7 +110,7 @@ static unsigned int shared_xattrs_count;
static struct xattr_prefix {
const char *prefix;
- u8 prefix_len;
+ unsigned int prefix_len;
} xattr_types[] = {
[EROFS_XATTR_INDEX_USER] = {
XATTR_USER_PREFIX,
@@ -116,11 +133,27 @@ static struct xattr_prefix {
struct ea_type_node {
struct list_head list;
struct xattr_prefix type;
- u8 index;
+ unsigned int index, base_index, base_len;
};
+
static LIST_HEAD(ea_name_prefixes);
static unsigned int ea_prefix_count;
+static bool match_prefix(const char *key, unsigned int *index,
+ unsigned int *len)
+{
+ struct xattr_prefix *p;
+
+ for (p = xattr_types; p < xattr_types + ARRAY_SIZE(xattr_types); ++p) {
+ if (p->prefix && !strncmp(p->prefix, key, p->prefix_len)) {
+ *len = p->prefix_len;
+ *index = p - xattr_types;
+ return true;
+ }
+ }
+ return false;
+}
+
static unsigned int BKDRHash(char *str, unsigned int len)
{
const unsigned int seed = 131313;
@@ -133,13 +166,12 @@ static unsigned int BKDRHash(char *str, unsigned int len)
return hash;
}
-static unsigned int xattr_item_hash(u8 prefix, char *buf,
- unsigned int len[2], unsigned int hash[2])
+static unsigned int xattr_item_hash(char *buf, unsigned int len[2],
+ unsigned int hash[2])
{
hash[0] = BKDRHash(buf, len[0]); /* key */
hash[1] = BKDRHash(buf + len[0], len[1]); /* value */
-
- return prefix ^ hash[0] ^ hash[1];
+ return hash[0] ^ hash[1];
}
static unsigned int put_xattritem(struct xattr_item *item)
@@ -150,17 +182,14 @@ static unsigned int put_xattritem(struct xattr_item *item)
return 0;
}
-static struct xattr_item *get_xattritem(u8 prefix, char *kvbuf,
- unsigned int len[2])
+static struct xattr_item *get_xattritem(char *kvbuf, unsigned int len[2])
{
struct xattr_item *item;
unsigned int hash[2], hkey;
- hkey = xattr_item_hash(prefix, kvbuf, len, hash);
-
+ hkey = xattr_item_hash(kvbuf, len, hash);
hash_for_each_possible(ea_hashtable, item, node, hkey) {
- if (prefix == item->prefix &&
- item->len[0] == len[0] && item->len[1] == len[1] &&
+ if (item->len[0] == len[0] && item->len[1] == len[1] &&
item->hash[0] == hash[0] && item->hash[1] == hash[1] &&
!memcmp(kvbuf, item->kvbuf, len[0] + len[1])) {
free(kvbuf);
@@ -174,6 +203,14 @@ static struct xattr_item *get_xattritem(u8 prefix, char *kvbuf,
free(kvbuf);
return ERR_PTR(-ENOMEM);
}
+
+ if (!match_prefix(kvbuf, &item->base_index, &item->prefix_len)) {
+ free(item);
+ free(kvbuf);
+ return ERR_PTR(-ENODATA);
+ }
+ DBG_BUGON(len[0] < item->prefix_len);
+
INIT_HLIST_NODE(&item->node);
item->count = 1;
item->kvbuf = kvbuf;
@@ -182,56 +219,37 @@ static struct xattr_item *get_xattritem(u8 prefix, char *kvbuf,
item->hash[0] = hash[0];
item->hash[1] = hash[1];
item->shared_xattr_id = -1;
- item->prefix = prefix;
- hash_add(ea_hashtable, &item->node, hkey);
- return item;
-}
+ item->prefix = item->base_index;
-static bool match_base_prefix(const char *key, u8 *index, u16 *len)
-{
- struct xattr_prefix *p;
+ if (cfg.c_extra_ea_name_prefixes) {
+ struct ea_type_node *tnode;
- for (p = xattr_types; p < xattr_types + ARRAY_SIZE(xattr_types); ++p) {
- if (p->prefix && !strncmp(p->prefix, key, p->prefix_len)) {
- *len = p->prefix_len;
- *index = p - xattr_types;
- return true;
+ list_for_each_entry(tnode, &ea_name_prefixes, list) {
+ if (item->base_index == tnode->base_index &&
+ !strncmp(tnode->type.prefix, kvbuf,
+ tnode->type.prefix_len)) {
+ item->prefix = tnode->index;
+ item->prefix_len = tnode->type.prefix_len;
+ break;
+ }
}
}
- return false;
-}
-static bool match_prefix(const char *key, u8 *index, u16 *len)
-{
- struct xattr_prefix *p;
- struct ea_type_node *tnode;
-
- list_for_each_entry(tnode, &ea_name_prefixes, list) {
- p = &tnode->type;
- if (p->prefix && !strncmp(p->prefix, key, p->prefix_len)) {
- *len = p->prefix_len;
- *index = tnode->index;
- return true;
- }
- }
- return match_base_prefix(key, index, len);
+ hash_add(ea_hashtable, &item->node, hkey);
+ return item;
}
static struct xattr_item *parse_one_xattr(const char *path, const char *key,
unsigned int keylen)
{
ssize_t ret;
- u8 prefix;
- u16 prefixlen;
unsigned int len[2];
char *kvbuf;
erofs_dbg("parse xattr [%s] of %s", path, key);
- if (!match_prefix(key, &prefix, &prefixlen))
- return ERR_PTR(-ENODATA);
-
- DBG_BUGON(keylen < prefixlen);
+ /* length of the key */
+ len[0] = keylen;
/* determine length of the value */
#ifdef HAVE_LGETXATTR
@@ -246,19 +264,18 @@ static struct xattr_item *parse_one_xattr(const char *path, const char *key,
len[1] = ret;
/* allocate key-value buffer */
- len[0] = keylen - prefixlen;
-
- kvbuf = malloc(len[0] + len[1]);
+ kvbuf = malloc(EROFS_XATTR_KVSIZE(len));
if (!kvbuf)
return ERR_PTR(-ENOMEM);
- memcpy(kvbuf, key + prefixlen, len[0]);
+ sprintf(kvbuf, "%s", key);
if (len[1]) {
/* copy value to buffer */
#ifdef HAVE_LGETXATTR
- ret = lgetxattr(path, key, kvbuf + len[0], len[1]);
+ ret = lgetxattr(path, key, kvbuf + EROFS_XATTR_KSIZE(len),
+ len[1]);
#elif defined(__APPLE__)
- ret = getxattr(path, key, kvbuf + len[0], len[1], 0,
- XATTR_NOFOLLOW);
+ ret = getxattr(path, key, kvbuf + EROFS_XATTR_KSIZE(len),
+ len[1], 0, XATTR_NOFOLLOW);
#else
free(kvbuf);
return ERR_PTR(-EOPNOTSUPP);
@@ -273,7 +290,7 @@ static struct xattr_item *parse_one_xattr(const char *path, const char *key,
len[1] = ret;
}
}
- return get_xattritem(prefix, kvbuf, len);
+ return get_xattritem(kvbuf, len);
}
static struct xattr_item *erofs_get_selabel_xattr(const char *srcpath,
@@ -308,16 +325,17 @@ static struct xattr_item *erofs_get_selabel_xattr(const char *srcpath,
return NULL;
}
- len[0] = sizeof("selinux") - 1;
+ len[0] = sizeof(XATTR_NAME_SECURITY_SELINUX) - 1;
len[1] = strlen(secontext);
- kvbuf = malloc(len[0] + len[1] + 1);
+ kvbuf = malloc(EROFS_XATTR_KVSIZE(len));
if (!kvbuf) {
freecon(secontext);
return ERR_PTR(-ENOMEM);
}
- sprintf(kvbuf, "selinux%s", secontext);
+ sprintf(kvbuf, "%s", XATTR_NAME_SECURITY_SELINUX);
+ memcpy(kvbuf + EROFS_XATTR_KSIZE(len), secontext, len[1]);
freecon(secontext);
- return get_xattritem(EROFS_XATTR_INDEX_SECURITY, kvbuf, len);
+ return get_xattritem(kvbuf, len);
}
#endif
return NULL;
@@ -466,24 +484,18 @@ int erofs_setxattr(struct erofs_inode *inode, char *key,
char *kvbuf;
unsigned int len[2];
struct xattr_item *item;
- u8 prefix;
- u16 prefixlen;
-
- if (!match_prefix(key, &prefix, &prefixlen))
- return -ENODATA;
+ len[0] = strlen(key);
len[1] = size;
- /* allocate key-value buffer */
- len[0] = strlen(key) - prefixlen;
- kvbuf = malloc(len[0] + len[1]);
+ kvbuf = malloc(EROFS_XATTR_KVSIZE(len));
if (!kvbuf)
return -ENOMEM;
- memcpy(kvbuf, key + prefixlen, len[0]);
- memcpy(kvbuf + len[0], value, size);
+ sprintf(kvbuf, "%s", key);
+ memcpy(kvbuf + EROFS_XATTR_KSIZE(len), value, size);
- item = get_xattritem(prefix, kvbuf, len);
+ item = get_xattritem(kvbuf, len);
if (IS_ERR(item))
return PTR_ERR(item);
DBG_BUGON(!item);
@@ -513,22 +525,22 @@ static int erofs_droid_xattr_set_caps(struct erofs_inode *inode)
if (!capabilities)
return 0;
- len[0] = sizeof("capability") - 1;
+ len[0] = sizeof(XATTR_NAME_SECURITY_CAPABILITY) - 1;
len[1] = sizeof(caps);
- kvbuf = malloc(len[0] + len[1]);
+ kvbuf = malloc(EROFS_XATTR_KVSIZE(len));
if (!kvbuf)
return -ENOMEM;
- memcpy(kvbuf, "capability", len[0]);
+ sprintf(kvbuf, "%s", XATTR_NAME_SECURITY_CAPABILITY);
caps.magic_etc = VFS_CAP_REVISION_2 | VFS_CAP_FLAGS_EFFECTIVE;
caps.data[0].permitted = (u32) capabilities;
caps.data[0].inheritable = 0;
caps.data[1].permitted = (u32) (capabilities >> 32);
caps.data[1].inheritable = 0;
- memcpy(kvbuf + len[0], &caps, len[1]);
+ memcpy(kvbuf + EROFS_XATTR_KSIZE(len), &caps, len[1]);
- item = get_xattritem(EROFS_XATTR_INDEX_SECURITY, kvbuf, len);
+ item = get_xattritem(kvbuf, len);
if (IS_ERR(item))
return PTR_ERR(item);
DBG_BUGON(!item);
@@ -558,6 +570,13 @@ int erofs_scan_file_xattrs(struct erofs_inode *inode)
return erofs_droid_xattr_set_caps(inode);
}
+static inline unsigned int erofs_next_xattr_align(unsigned int pos,
+ struct xattr_item *item)
+{
+ return EROFS_XATTR_ALIGN(pos + sizeof(struct erofs_xattr_entry) +
+ item->len[0] + item->len[1] - item->prefix_len);
+}
+
int erofs_prepare_xattr_ibody(struct erofs_inode *inode)
{
int ret;
@@ -572,14 +591,13 @@ int erofs_prepare_xattr_ibody(struct erofs_inode *inode)
/* get xattr ibody size */
ret = sizeof(struct erofs_xattr_ibody_header);
list_for_each_entry(node, ixattrs, list) {
- const struct xattr_item *item = node->item;
+ struct xattr_item *item = node->item;
if (item->shared_xattr_id >= 0) {
ret += sizeof(__le32);
continue;
}
- ret += sizeof(struct erofs_xattr_entry);
- ret = EROFS_XATTR_ALIGN(ret + item->len[0] + item->len[1]);
+ ret = erofs_next_xattr_align(ret, item);
}
inode->xattr_isize = ret;
return ret;
@@ -688,25 +706,9 @@ static int comp_shared_xattr_item(const void *a, const void *b)
return la > lb;
}
-static inline int erofs_xattr_index_by_prefix(const char *prefix, int *len)
-{
- if (!strncmp(prefix, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)){
- *len = XATTR_USER_PREFIX_LEN;
- return EROFS_XATTR_INDEX_USER;
- } else if (!strncmp(prefix, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN)) {
- *len = XATTR_TRUSTED_PREFIX_LEN;
- return EROFS_XATTR_INDEX_TRUSTED;
- } else if (!strncmp(prefix, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN)) {
- *len = XATTR_SECURITY_PREFIX_LEN;
- return EROFS_XATTR_INDEX_SECURITY;
- }
- return -ENODATA;
-}
-
int erofs_xattr_write_name_prefixes(struct erofs_sb_info *sbi, FILE *f)
{
struct ea_type_node *tnode;
- struct xattr_prefix *p;
off_t offset;
if (!ea_prefix_count)
@@ -732,16 +734,13 @@ int erofs_xattr_write_name_prefixes(struct erofs_sb_info *sbi, FILE *f)
u8 data[EROFS_NAME_LEN + 2 +
sizeof(struct erofs_xattr_long_prefix)];
} u;
- int ret, len;
+ int len, infix_len;
- p = &tnode->type;
- ret = erofs_xattr_index_by_prefix(p->prefix, &len);
- if (ret < 0)
- return ret;
- u.s.prefix.base_index = ret;
- memcpy(u.s.prefix.infix, p->prefix + len, p->prefix_len - len);
- len = sizeof(struct erofs_xattr_long_prefix) +
- p->prefix_len - len;
+ u.s.prefix.base_index = tnode->base_index;
+ infix_len = tnode->type.prefix_len - tnode->base_len;
+ memcpy(u.s.prefix.infix, tnode->type.prefix + tnode->base_len,
+ infix_len);
+ len = sizeof(struct erofs_xattr_long_prefix) + infix_len;
u.s.size = cpu_to_le16(len);
if (fwrite(&u.s, sizeof(__le16) + len, 1, f) != 1)
return -EIO;
@@ -754,11 +753,30 @@ int erofs_xattr_write_name_prefixes(struct erofs_sb_info *sbi, FILE *f)
return 0;
}
+static void erofs_write_xattr_entry(char *buf, struct xattr_item *item)
+{
+ struct erofs_xattr_entry entry = {
+ .e_name_index = item->prefix,
+ .e_name_len = item->len[0] - item->prefix_len,
+ .e_value_size = cpu_to_le16(item->len[1]),
+ };
+
+ memcpy(buf, &entry, sizeof(entry));
+ buf += sizeof(struct erofs_xattr_entry);
+ memcpy(buf, item->kvbuf + item->prefix_len,
+ item->len[0] - item->prefix_len);
+ buf += item->len[0] - item->prefix_len;
+ memcpy(buf, item->kvbuf + item->len[0] + 1, item->len[1]);
+
+ erofs_dbg("writing xattr %d %s (%d %s)", item->base_index, item->kvbuf,
+ item->prefix, item->kvbuf + item->prefix_len);
+}
+
int erofs_build_shared_xattrs_from_path(struct erofs_sb_info *sbi, const char *path)
{
int ret;
struct erofs_buffer_head *bh;
- struct xattr_item *n, **sorted_n;
+ struct xattr_item *item, *n, **sorted_n;
char *buf;
unsigned int p, i;
erofs_off_t off;
@@ -787,13 +805,11 @@ int erofs_build_shared_xattrs_from_path(struct erofs_sb_info *sbi, const char *p
i = 0;
while (shared_xattrs_list) {
- struct xattr_item *item = shared_xattrs_list;
-
+ item = shared_xattrs_list;
sorted_n[i++] = item;
shared_xattrs_list = item->next_shared_xattr;
- shared_xattrs_size += sizeof(struct erofs_xattr_entry);
- shared_xattrs_size = EROFS_XATTR_ALIGN(shared_xattrs_size +
- item->len[0] + item->len[1]);
+ shared_xattrs_size = erofs_next_xattr_align(shared_xattrs_size,
+ item);
}
DBG_BUGON(i != shared_xattrs_count);
sorted_n[i] = NULL;
@@ -820,20 +836,11 @@ int erofs_build_shared_xattrs_from_path(struct erofs_sb_info *sbi, const char *p
off %= erofs_blksiz(sbi);
p = 0;
for (i = 0; i < shared_xattrs_count; i++) {
- struct xattr_item *item = sorted_n[i];
- const struct erofs_xattr_entry entry = {
- .e_name_index = item->prefix,
- .e_name_len = item->len[0],
- .e_value_size = cpu_to_le16(item->len[1])
- };
-
+ item = sorted_n[i];
+ erofs_write_xattr_entry(buf + p, item);
item->next_shared_xattr = sorted_n[i + 1];
item->shared_xattr_id = (off + p) / sizeof(__le32);
-
- memcpy(buf + p, &entry, sizeof(entry));
- p += sizeof(struct erofs_xattr_entry);
- memcpy(buf + p, item->kvbuf, item->len[0] + item->len[1]);
- p = EROFS_XATTR_ALIGN(p + item->len[0] + item->len[1]);
+ p = erofs_next_xattr_align(p, item);
}
shared_xattrs_list = sorted_n[0];
free(sorted_n);
@@ -846,70 +853,12 @@ out:
return ret;
}
-static int erofs_xattr_filter_hashbit(struct xattr_item *item)
-{
- u8 prefix = item->prefix;
- const char *key = item->kvbuf;
- unsigned int len = item->len[0];
- char *name = NULL;
- uint32_t hashbit;
-
- if (prefix & EROFS_XATTR_LONG_PREFIX) {
- struct ea_type_node *tnode;
- u16 prefix_len;
- int ret;
-
- list_for_each_entry(tnode, &ea_name_prefixes, list) {
- if (tnode->index == item->prefix) {
- ret = asprintf(&name, "%s%.*s",
- tnode->type.prefix, len, key);
- if (ret < 0)
- return -ENOMEM;
- break;
- }
- }
- if (!name)
- return -ENOENT;
-
- if (!match_base_prefix(name, &prefix, &prefix_len)) {
- free(name);
- return -ENOENT;
- }
- key = name + prefix_len;
- len = strlen(key);
- }
-
- hashbit = xxh32(key, len, EROFS_XATTR_FILTER_SEED + prefix) &
- (EROFS_XATTR_FILTER_BITS - 1);
- if (name)
- free(name);
- return hashbit;
-}
-
-static u32 erofs_xattr_filter_map(struct list_head *ixattrs)
-{
- struct inode_xattr_node *node, *n;
- u32 name_filter;
- int hashbit;
-
- name_filter = 0;
- list_for_each_entry_safe(node, n, ixattrs, list) {
- hashbit = erofs_xattr_filter_hashbit(node->item);
- if (hashbit < 0) {
- erofs_warn("failed to generate xattr name filter: %s",
- strerror(-hashbit));
- return 0;
- }
- name_filter |= (1UL << hashbit);
- }
- return EROFS_XATTR_FILTER_DEFAULT & ~name_filter;
-}
-
char *erofs_export_xattr_ibody(struct erofs_inode *inode)
{
struct list_head *ixattrs = &inode->i_xattrs;
unsigned int size = inode->xattr_isize;
struct inode_xattr_node *node, *n;
+ struct xattr_item *item;
struct erofs_xattr_ibody_header *header;
LIST_HEAD(ilst);
unsigned int p;
@@ -922,16 +871,29 @@ char *erofs_export_xattr_ibody(struct erofs_inode *inode)
header->h_shared_count = 0;
if (cfg.c_xattr_name_filter) {
- header->h_name_filter =
- cpu_to_le32(erofs_xattr_filter_map(ixattrs));
+ u32 name_filter = 0;
+ int hashbit;
+ unsigned int base_len;
+
+ list_for_each_entry(node, ixattrs, list) {
+ item = node->item;
+ base_len = xattr_types[item->base_index].prefix_len;
+ hashbit = xxh32(item->kvbuf + base_len,
+ item->len[0] - base_len,
+ EROFS_XATTR_FILTER_SEED + item->base_index) &
+ (EROFS_XATTR_FILTER_BITS - 1);
+ name_filter |= (1UL << hashbit);
+ }
+ name_filter = EROFS_XATTR_FILTER_DEFAULT & ~name_filter;
+
+ header->h_name_filter = cpu_to_le32(name_filter);
if (header->h_name_filter)
erofs_sb_set_xattr_filter(inode->sbi);
}
p = sizeof(struct erofs_xattr_ibody_header);
list_for_each_entry_safe(node, n, ixattrs, list) {
- struct xattr_item *const item = node->item;
-
+ item = node->item;
list_del(&node->list);
/* move inline xattrs to the onstack list */
@@ -948,18 +910,9 @@ char *erofs_export_xattr_ibody(struct erofs_inode *inode)
}
list_for_each_entry_safe(node, n, &ilst, list) {
- struct xattr_item *const item = node->item;
- const struct erofs_xattr_entry entry = {
- .e_name_index = item->prefix,
- .e_name_len = item->len[0],
- .e_value_size = cpu_to_le16(item->len[1])
- };
-
- memcpy(buf + p, &entry, sizeof(entry));
- p += sizeof(struct erofs_xattr_entry);
- memcpy(buf + p, item->kvbuf, item->len[0] + item->len[1]);
- p = EROFS_XATTR_ALIGN(p + item->len[0] + item->len[1]);
-
+ item = node->item;
+ erofs_write_xattr_entry(buf + p, item);
+ p = erofs_next_xattr_align(p, item);
list_del(&node->list);
free(node);
put_xattritem(item);
@@ -1354,8 +1307,7 @@ int erofs_getxattr(struct erofs_inode *vi, const char *name, char *buffer,
size_t buffer_size)
{
int ret;
- u8 prefix;
- u16 prefixlen;
+ unsigned int prefix, prefixlen;
struct getxattr_iter it;
if (!name)
@@ -1365,7 +1317,7 @@ int erofs_getxattr(struct erofs_inode *vi, const char *name, char *buffer,
if (ret)
return ret;
- if (!match_base_prefix(name, &prefix, &prefixlen))
+ if (!match_prefix(name, &prefix, &prefixlen))
return -ENODATA;
it.it.sbi = vi->sbi;
@@ -1532,34 +1484,25 @@ int erofs_listxattr(struct erofs_inode *vi, char *buffer, size_t buffer_size)
int erofs_xattr_insert_name_prefix(const char *prefix)
{
struct ea_type_node *tnode;
- struct xattr_prefix *p;
- bool matched = false;
- char *s;
if (ea_prefix_count >= 0x80 || strlen(prefix) > UINT8_MAX)
return -EOVERFLOW;
- for (p = xattr_types; p < xattr_types + ARRAY_SIZE(xattr_types); ++p) {
- if (!strncmp(p->prefix, prefix, p->prefix_len)) {
- matched = true;
- break;
- }
- }
- if (!matched)
- return -ENODATA;
-
- s = strdup(prefix);
- if (!s)
+ tnode = calloc(1, sizeof(*tnode));
+ if (!tnode)
return -ENOMEM;
- tnode = malloc(sizeof(*tnode));
- if (!tnode) {
- free(s);
- return -ENOMEM;
+ if (!match_prefix(prefix, &tnode->base_index, &tnode->base_len)) {
+ free(tnode);
+ return -ENODATA;
}
- tnode->type.prefix = s;
tnode->type.prefix_len = strlen(prefix);
+ tnode->type.prefix = strdup(prefix);
+ if (!tnode->type.prefix) {
+ free(tnode);
+ return -ENOMEM;
+ }
tnode->index = EROFS_XATTR_LONG_PREFIX | ea_prefix_count;
ea_prefix_count++;
--
2.19.1.6.gb485710b
^ permalink raw reply related [flat|nested] 13+ messages in thread