Re: [PATCH v5] erofs-utils: introduce exclude dirs and files

[Gao Xiang <gaoxiang25@huawei.com>]

On Sun, Feb 16, 2020 at 09:32:19PM +0800, Li Guifu wrote:
> From: Li GuiFu <bluce.lee@aliyun.com>
> 
> Add excluded file feature "--exclude-path=" and '--exlude-regex=',

typo
                                                  ^ '--exclude-regex='

> which can be used to build EROFS image without some user specific
> files or dirs.
> Note that you may give multiple '--exclude-path' or '--exclude-regex'
> options.
> 
> Signed-off-by: Li Guifu <blucer.lee@foxmail.com>


please fix your email here.

Signed-off-by: Li GuiFu <bluce.lee@aliyun.com>


> Signed-off-by: Gao Xiang <gaoxiang25@huawei.com>
> ---
> change since v5:
>  - add regex free
>  - clean log print
> 
>  include/erofs/exclude.h |  28 +++++++++
>  lib/Makefile.am         |   2 +-
>  lib/exclude.c           | 133 ++++++++++++++++++++++++++++++++++++++++
>  lib/inode.c             |   5 ++
>  man/mkfs.erofs.1        |   8 +++
>  mkfs/main.c             |  36 +++++++++--
>  6 files changed, 205 insertions(+), 7 deletions(-)
>  create mode 100644 include/erofs/exclude.h
>  create mode 100644 lib/exclude.c
> 
> diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h
> new file mode 100644
> index 0000000..e158a14
> --- /dev/null
> +++ b/include/erofs/exclude.h
> @@ -0,0 +1,28 @@
> +/* SPDX-License-Identifier: GPL-2.0+ */
> +/*
> + * erofs-utils/include/erofs/exclude.h
> + *
> + * Created by Li Guifu <blucer.lee@foxmail.com>

same here.

Created by Li GuiFu <bluce.lee@aliyun.com>

> + */
> +#ifndef __EROFS_EXCLUDE_H
> +#define __EROFS_EXCLUDE_H
> +#include <sys/types.h>
> +#include <regex.h>
> +
> +struct erofs_exclude_rule {
> +	struct list_head list;
> +
> +	union {
> +		char *pattern;
> +		regex_t *reg;
> +	};

how about leave original pattern here. I mean

struct erofs_exclude_rule {
	struct list_head list;

	char *pattern;		/* save original pattern for exact or regex match */
	regex_t reg;
};

and define the following macros in "lib/exclude.c":

#define EXCLUDE_RULE_EXACT_SIZE	offsetof(struct erofs_exclude_rule, reg)
#define EXCLUDE_RULE_REGEX_SIZE	sizeof(struct erofs_exclude_rule)

> +};
> +
> +void erofs_exclude_set_root(const char *rootdir);
> +void erofs_cleanup_exclude_rules(void);
> +
> +int erofs_parse_exclude_path(const char *args, bool is_regex);
> +struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
> +						 const char *name);
> +#endif
> +
> diff --git a/lib/Makefile.am b/lib/Makefile.am
> index 1ff81f9..e4b51e6 100644
> --- a/lib/Makefile.am
> +++ b/lib/Makefile.am
> @@ -3,7 +3,7 @@
>  
>  noinst_LTLIBRARIES = liberofs.la
>  liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
> -		      compress.c compressor.c
> +		      compress.c compressor.c exclude.c
>  liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
>  if ENABLE_LZ4
>  liberofs_la_CFLAGS += ${LZ4_CFLAGS}
> diff --git a/lib/exclude.c b/lib/exclude.c
> new file mode 100644
> index 0000000..268a6f7
> --- /dev/null
> +++ b/lib/exclude.c
> @@ -0,0 +1,133 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * erofs-utils/lib/exclude.c
> + *
> + * Created by Li Guifu <blucer.lee@foxmail.com>

same here.

> + */
> +#include <string.h>
> +#include <stdlib.h>
> +#include "erofs/err.h"
> +#include "erofs/list.h"
> +#include "erofs/print.h"
> +#include "erofs/exclude.h"
> +
> +static LIST_HEAD(exclude_head);
> +static LIST_HEAD(regex_exclude_head);
> +
> +static unsigned int rpathlen;		/* root directory prefix length */
> +
> +void erofs_exclude_set_root(const char *rootdir)
> +{
> +	rpathlen = strlen(rootdir);
> +}
> +
> +static struct erofs_exclude_rule
> +*erofs_insert_exclude(const char *s, bool is_regex)


static struct erofs_exclude_rule *erofs_insert_exclude(const char *s,
						       bool is_regex)


> +{
> +	int ret = -ENOMEM;
> +	struct erofs_exclude_rule *r;
> +	struct list_head *h;
> +
> +	r = malloc(sizeof(*r));

	if (is_regex)
		r = malloc(EXCLUDE_RULE_REGEX_SIZE);
	else
		r = malloc(EXCLUDE_RULE_EXACT_SIZE);


> +	if (!r)
> +		return ERR_PTR(-ENOMEM);
> +
> +	/* exact match */
> +	if (is_regex) {
> +		r->reg = malloc(sizeof(regex_t));
> +		if (!r->reg)
> +			goto err_rule;
> +		ret = regcomp(r->reg, s, REG_EXTENDED|REG_NOSUB);
> +		if(ret) {
> +			char str[1024]; /* overflow safe */
> +
> +			regerror(ret, r->reg, str, 1024);
> +			erofs_err("invalid regex %s,because %s\n", s, str);
> +			goto err_rule;
> +		}
> +		h = &regex_exclude_head;
> +	} else {
> +		r->pattern = strdup(s);
> +
> +		if (!r->pattern)
> +			goto err_rule;
> +		h = &exclude_head;
> +	}

	r->pattern = strdup(s);
	if (is_regex) {
		ret = regcomp(r->reg, s, ...);

		...

		h = &regex_exclude_head;
	} else {
		h = &exclude_head;
	}

> +
> +	list_add_tail(&r->list, h);
> +	erofs_info("Insert exclude:[%s]\n", s);
> +	return r;
> +err_rule:
> +	free(r);
> +	return ERR_PTR(ret);
> +}
> +
> +void erofs_cleanup_exclude_rules(void)
> +{
> +	struct erofs_exclude_rule *r, *n;
> +
> +	list_for_each_entry_safe(r, n, &exclude_head, list) {
> +		list_del(&r->list);
> +		free(r->pattern);
> +		free(r);
> +	}
> +
> +	list_for_each_entry_safe(r, n, &regex_exclude_head, list) {
> +		list_del(&r->list);
> +		regfree(r->reg);
> +		free(r);
> +	}

some modification here.

> +}
> +
> +int erofs_parse_exclude_path(const char *args, bool is_regex)
> +{
> +	struct erofs_exclude_rule *r = erofs_insert_exclude(args, is_regex);
> +
> +	if (IS_ERR(r)) {
> +		erofs_cleanup_exclude_rules();
> +		return PTR_ERR(r);
> +	}
> +	return 0;
> +}
> +
> +struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
> +						 const char *name)
> +{
> +	char buf[PATH_MAX];
> +	const char *s;
> +	struct erofs_exclude_rule *r;
> +
> +	if (!dir) {
> +		/* no prefix */
> +		s = name;
> +	} else {
> +		sprintf(buf, "%s/%s", dir, name);
> +		s = buf;
> +	}
> +
> +	s += rpathlen;
> +	while (*s == '/')
> +		s++;
> +
> +	list_for_each_entry(r, &exclude_head, list) {
> +		if (!strcmp(r->pattern, s))
> +			return r;
> +	}
> +
> +	list_for_each_entry(r, &regex_exclude_head, list) {
> +		int ret = regexec(r->reg, s, (size_t) 0, NULL, 0);
> +
> +		if(!ret) {
> +			erofs_info("matched:%s\n", s);
> +			return r;
> +		} else if (ret != REG_NOMATCH) {
> +			char str[1024]; /* overflow safe */
> +
> +			regerror(ret, r->reg, str, 1024);
> +			erofs_err("invalid regex %s,because %s\n", s, str);
> +		}
> +	}
> +
> +	return NULL;
> +}
> +
> diff --git a/lib/inode.c b/lib/inode.c
> index bd0652b..7114023 100644
> --- a/lib/inode.c
> +++ b/lib/inode.c
> @@ -20,6 +20,7 @@
>  #include "erofs/io.h"
>  #include "erofs/compress.h"
>  #include "erofs/xattr.h"
> +#include "erofs/exclude.h"
>  
>  struct erofs_sb_info sbi;
>  
> @@ -877,6 +878,10 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
>  		    !strncmp(dp->d_name, "lost+found", strlen("lost+found")))
>  			continue;
>  
> +		/* skip if it's a exclude file */
> +		if (erofs_is_exclude_path(dir->i_srcpath, dp->d_name))
> +			continue;
> +
>  		d = erofs_d_alloc(dir, dp->d_name);
>  		if (IS_ERR(d)) {
>  			ret = PTR_ERR(d);
> diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
> index d6bf828..ab19927 100644
> --- a/man/mkfs.erofs.1
> +++ b/man/mkfs.erofs.1
> @@ -52,6 +52,14 @@ Forcely generate extended inodes (64-byte inodes) to output.
>  Set all files to the given UNIX timestamp. Reproducible builds requires setting
>  all to a specific one.
>  .TP
> +.BI "\-\-exclude-path=" path
> +Ignore file that matches the exact literal path.
> +You may give multiple `--exclude-path' options.
> +.TP
> +.BI "\-\-exclude-regex=" path
> +Ignore file that matches the exact literal path given by a regex expression
> +You may give multiple `--exclude-regex` options


+Ignore files that match what is given by a regex expression
+You may give multiple `--exclude-regex` options


> +.TP
>  .B \-\-help
>  Display this help and exit.
>  .SH AUTHOR
> diff --git a/mkfs/main.c b/mkfs/main.c
> index 817a6c1..d0c9869 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -21,6 +21,7 @@
>  #include "erofs/io.h"
>  #include "erofs/compress.h"
>  #include "erofs/xattr.h"
> +#include "erofs/exclude.h"
>  
>  #ifdef HAVE_LIBUUID
>  #include <uuid/uuid.h>
> @@ -30,6 +31,8 @@
>  
>  static struct option long_options[] = {
>  	{"help", no_argument, 0, 1},
> +	{"exclude-path", required_argument, NULL, 2},
> +	{"exclude-regex", required_argument, NULL, 3},
>  	{0, 0, 0, 0},
>  };
>  
> @@ -50,12 +53,14 @@ static void usage(void)
>  {
>  	fputs("usage: [options] FILE DIRECTORY\n\n"
>  	      "Generate erofs image from DIRECTORY to FILE, and [options] are:\n"
> -	      " -zX[,Y]   X=compressor (Y=compression level, optional)\n"
> -	      " -d#       set output message level to # (maximum 9)\n"
> -	      " -x#       set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
> -	      " -EX[,...] X=extended options\n"
> -	      " -T#       set a fixed UNIX timestamp # to all files\n"
> -	      " --help    display this help and exit\n"
> +	      " -zX[,Y]          X=compressor (Y=compression level, optional)\n"
> +	      " -d#              set output message level to # (maximum 9)\n"
> +	      " -x#              set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
> +	      " -EX[,...]        X=extended options\n"
> +	      " -T#              set a fixed UNIX timestamp # to all files\n"
> +	      " --exclude-path=X avoid including file X (X = exact literal path)\n"
> +	      " --exclude-regex=X avoid including file X (X = exact literal path of a regex)\n"

	      " --exclude-regex=X avoid including file X (X = regex path)\n"

Thanks,
Gao Xiang