Re: [LTP] [PATCH 4/7] docparse: Implement ARRAY_SIZE()

[Richard Palethorpe <rpalethorpe@suse.de>]

Hi,

Cyril Hrubis <chrubis@suse.cz> writes:

> Adds a special handlingn for ARRAY_SIZE() macro.
>
> If we stumble upon ARRAY_SIZE() in the tst_test structure we try to
> lookup the array and count its members.

Note that this is the type of thing Sparse should be able to handle
well. As it should evaluate this to a constant. Although there is the
risk it would be too clever and eliminate some expressions (for
example).

>
> Proper parsing of .test_variants also requires that we add -I switch to
> the docparse to be able to specify include paths on a command line since
> some variants are stuck in top level include while others are in
> testcases/kernel/syscalls/utils/.
>
> + tests
>
> Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
> ---
>  docparse/docparse.c               | 222 +++++++++++++++++++++++++++---
>  docparse/parse.sh                 |   2 +-
>  docparse/tests/array_size01.c     |   5 +
>  docparse/tests/array_size01.c.out |   4 +
>  docparse/tests/array_size02.c     |   5 +
>  docparse/tests/array_size02.c.out |   4 +
>  docparse/tests/array_size03.c     |  10 ++
>  docparse/tests/array_size03.c.out |   4 +
>  docparse/tests/array_size04.c     |   5 +
>  docparse/tests/array_size04.c.out |   4 +
>  docparse/tests/include.h          |   6 +
>  11 files changed, 248 insertions(+), 23 deletions(-)
>  create mode 100644 docparse/tests/array_size01.c
>  create mode 100644 docparse/tests/array_size01.c.out
>  create mode 100644 docparse/tests/array_size02.c
>  create mode 100644 docparse/tests/array_size02.c.out
>  create mode 100644 docparse/tests/array_size03.c
>  create mode 100644 docparse/tests/array_size03.c.out
>  create mode 100644 docparse/tests/array_size04.c
>  create mode 100644 docparse/tests/array_size04.c.out
>
> diff --git a/docparse/docparse.c b/docparse/docparse.c
> index cab01c384..9a1ed830c 100644
> --- a/docparse/docparse.c
> +++ b/docparse/docparse.c
> @@ -15,7 +15,12 @@
>  
>  #include "data_storage.h"
>  
> +#define INCLUDE_PATH_MAX 5
> +
>  static int verbose;
> +static char *cmdline_includepath[INCLUDE_PATH_MAX];
> +static unsigned int cmdline_includepaths;
> +static char *includepath;
>  
>  #define WARN(str) fprintf(stderr, "WARNING: " str "\n")
>  
> @@ -131,13 +136,14 @@ static void maybe_comment(FILE *f, struct data_node *doc)
>  	}
>  }
>  
> -static char *next_token(FILE *f, struct data_node *doc)
> +static char *next_token2(FILE *f, char *buf, size_t buf_len, struct data_node *doc)
>  {
>  	size_t i = 0;
> -	static char buf[4096];
>  	int c;
>  	int in_str = 0;
>  
> +	buf_len--;
> +
>  	for (;;) {
>  		c = fgetc(f);
>  
> @@ -150,7 +156,8 @@ static char *next_token(FILE *f, struct data_node *doc)
>  					goto exit;
>  			}
>  
> -			buf[i++] = c;
> +			if (i < buf_len)
> +				buf[i++] = c;
>  			continue;
>  		}
>  
> @@ -169,7 +176,8 @@ static char *next_token(FILE *f, struct data_node *doc)
>  				goto exit;
>  			}
>  
> -			buf[i++] = c;
> +			if (i < buf_len)
> +				buf[i++] = c;
>  			goto exit;
>  		case '0' ... '9':
>  		case 'a' ... 'z':
> @@ -178,7 +186,8 @@ static char *next_token(FILE *f, struct data_node *doc)
>  		case '_':
>  		case '-':
>  		case '#':
> -			buf[i++] = c;
> +			if (i < buf_len)
> +				buf[i++] = c;

I suspect you need to remove spaces after '#' (see below).

>  		break;
>  		case '/':
>  			maybe_comment(f, doc);
> @@ -203,11 +212,33 @@ exit:
>  	return buf;
>  }
>  
> -static FILE *open_include(const char *includepath, FILE *f)
> +static char *next_token(FILE *f, struct data_node *doc)
>  {
> -	char buf[256];
> +	static char buf[4096];
> +
> +	return next_token2(f, buf, sizeof(buf), doc);
> +}
> +
> +static FILE *open_file(const char *dir, const char *fname)
> +{
> +	FILE *f;
>  	char *path;
> +
> +	if (asprintf(&path, "%s/%s", dir, fname) < 0)
> +		return NULL;
> +
> +	f = fopen(path, "r");
> +
> +	free(path);
> +
> +	return f;
> +}
> +
> +static FILE *open_include(FILE *f)
> +{
> +	char buf[256], *fname;
>  	FILE *inc;
> +	unsigned int i;
>  
>  	if (!fscanf(f, "%s\n", buf))
>  		return NULL;
> @@ -215,24 +246,36 @@ static FILE *open_include(const char *includepath, FILE *f)
>  	if (buf[0] != '"')
>  		return NULL;
>  
> -	char *filename = buf + 1;
> +	fname = buf + 1;
>  
>  	if (!buf[0])
>  		return NULL;
>  
> -	filename[strlen(filename)-1] = 0;
> +	fname[strlen(fname)-1] = 0;
>  
> -	if (asprintf(&path, "%s/%s", includepath, filename) < 0)
> -		return NULL;
> +	inc = open_file(includepath, fname);
> +	if (inc) {
> +		if (verbose)
> +			fprintf(stderr, "INCLUDE %s/%s\n", includepath, fname);
>  
> -	inc = fopen(path, "r");
> +		return inc;
> +	}
>  
> -	if (inc && verbose)
> -		fprintf(stderr, "INCLUDE %s\n", path);
> +	for (i = 0; i < cmdline_includepaths; i++) {
> +		inc = open_file(cmdline_includepath[i], fname);
>  
> -	free(path);
> +		if (!inc)
> +			continue;
> +
> +		if (verbose) {
> +			fprintf(stderr, "INCLUDE %s/%s\n",
> +				cmdline_includepath[i], fname);
> +		}
>  
> -	return inc;
> +		return inc;
> +	}
> +
> +	return NULL;
>  }
>  
>  static void close_include(FILE *inc)
> @@ -299,6 +342,127 @@ static void try_apply_macro(char **res)
>  	*res = ret->data;
>  }
>  
> +static int parse_get_array_len(FILE *f)
> +{
> +	const char *token;
> +	int cnt = 0, depth = 0, prev_comma = 0;
> +
> +	if (!(token = next_token(f, NULL)))
> +		return 0;
> +
> +	if (strcmp(token, "{"))
> +		return 0;
> +
> +	for (;;) {
> +		if (!(token = next_token(f, NULL)))
> +			return 0;
> +
> +		if (!strcmp(token, "{"))
> +			depth++;
> +
> +		if (!strcmp(token, "}"))
> +			depth--;
> +		else
> +			prev_comma = 0;
> +
> +		if (!strcmp(token, ",") && !depth) {
> +			prev_comma = 1;
> +			cnt++;
> +		}
> +
> +		if (depth < 0)
> +			return cnt + !prev_comma;
> +	}
> +}
> +
> +static void look_for_array_size(FILE *f, const char *arr_id, struct data_node **res)
> +{
> +	const char *token;
> +	char buf[2][2048] = {};
> +	int cur_buf = 0;
> +	int prev_buf = 1;
> +
> +	for (;;) {
> +		if (!(token = next_token2(f, buf[cur_buf], 2048, NULL)))
> +			break;
> +
> +		if (!strcmp(token, "=") && !strcmp(buf[prev_buf], arr_id)) {
> +			int arr_len = parse_get_array_len(f);
> +
> +			if (verbose)
> +				fprintf(stderr, "ARRAY %s LENGTH = %i\n", arr_id, arr_len);
> +
> +			*res = data_node_int(arr_len);
> +
> +			break;
> +		}
> +
> +		if (strcmp(buf[cur_buf], "]") && strcmp(buf[cur_buf], "[")) {
> +			cur_buf = !cur_buf;
> +			prev_buf = !prev_buf;
> +		}
> +	}
> +}
> +
> +static int parse_array_size(FILE *f, struct data_node **res)
> +{
> +	const char *token;
> +	char *arr_id;
> +	long pos;
> +
> +	*res = NULL;
> +
> +	if (!(token = next_token(f, NULL)))
> +		return 1;
> +
> +	if (strcmp(token, "("))
> +		return 1;
> +
> +	if (!(token = next_token(f, NULL)))
> +		return 1;
> +
> +	arr_id = strdup(token);
> +
> +	if (verbose)
> +		fprintf(stderr, "COMPUTING ARRAY '%s' LENGHT\n", arr_id);
> +
> +	pos = ftell(f);
> +
> +	rewind(f);
> +
> +	look_for_array_size(f, arr_id, res);
> +
> +	if (!*res) {
> +		FILE *inc;
> +
> +		rewind(f);
> +
> +		for (;;) {
> +			if (!(token = next_token(f, NULL)))
> +				break;
> +
> +			if (!strcmp(token, "#include")) {

Are spaces removed after the '#'?

e.g.

#if X
#  include<Y>
#endif

> +				inc = open_include(f);
> +
> +				if (inc) {
> +					look_for_array_size(inc, arr_id, res);
> +					close_include(inc);
> +				}
> +			}
> +
> +			if (*res)
> +				break;
> +		}
> +	}
> +
> +	free(arr_id);
> +
> +	if (fseek(f, pos, SEEK_SET))
> +		return 1;
> +
> +	return 0;
> +}
> +
>  static int parse_test_struct(FILE *f, struct data_node *doc, struct data_node *node)
>  {
>  	char *token;
> @@ -344,11 +508,17 @@ static int parse_test_struct(FILE *f, struct data_node *doc, struct data_node *n
>  		if (!strcmp(token, "{")) {
>  			ret = data_node_array();
>  			parse_array(f, ret);
> +		} else if (!strcmp(token, "ARRAY_SIZE")) {
> +			if (parse_array_size(f, &ret))
> +				return 1;
>  		} else {
>  			try_apply_macro(&token);
>  			ret = data_node_string(token);
>  		}
>  
> +		if (!ret)
> +			continue;
> +
>  		const char *key = id;
>  		if (key[0] == '.')
>  			key++;
> @@ -448,12 +618,12 @@ static void parse_macro(FILE *f)
>  	hsearch(e, ENTER);
>  }
>  
> -static void parse_include_macros(const char *includepath, FILE *f)
> +static void parse_include_macros(FILE *f)
>  {
>  	FILE *inc;
>  	const char *token;
>  
> -	inc = open_include(includepath, f);
> +	inc = open_include(f);
>  	if (!inc)
>  		return;
>  
> @@ -477,7 +647,7 @@ static struct data_node *parse_file(const char *fname)
>  
>  	FILE *f = fopen(fname, "r");
>  
> -	const char *includepath = dirname(strdup(fname));
> +	includepath = dirname(strdup(fname));
>  
>  	struct data_node *res = data_node_hash();
>  	struct data_node *doc = data_node_array();
> @@ -491,7 +661,7 @@ static struct data_node *parse_file(const char *fname)
>  					parse_macro(f);
>  
>  				if (!strcmp(token, "#include"))
> -					parse_include_macros(includepath, f);
> +					parse_include_macros(f);
>  			}
>  
>  			state = 0;
> @@ -504,7 +674,6 @@ static struct data_node *parse_file(const char *fname)
>  		parse_test_struct(f, doc, res);
>  	}
>  
> -
>  	if (data_node_array_len(doc)) {
>  		data_node_hash_add(res, "doc", doc);
>  		found = 1;
> @@ -564,6 +733,7 @@ static void print_help(const char *prgname)
>  {
>  	printf("usage: %s [-vh] input.c\n\n", prgname);
>  	printf("-v sets verbose mode\n");
> +	printf("-I add include path\n");
>  	printf("-h prints this help\n\n");
>  	exit(0);
>  }
> @@ -574,11 +744,19 @@ int main(int argc, char *argv[])
>  	struct data_node *res;
>  	int opt;
>  
> -	while ((opt = getopt(argc, argv, "hv")) != -1) {
> +	while ((opt = getopt(argc, argv, "hI:v")) != -1) {
>  		switch (opt) {
>  		case 'h':
>  			print_help(argv[0]);
>  		break;
> +		case 'I':
> +			if (cmdline_includepaths >= INCLUDE_PATH_MAX) {
> +				fprintf(stderr, "Too much include paths!");
> +				exit(1);
> +			}
> +
> +			cmdline_includepath[cmdline_includepaths++] = optarg;
> +		break;
>  		case 'v':
>  			verbose = 1;
>  		break;
> diff --git a/docparse/parse.sh b/docparse/parse.sh
> index 79011bc10..2ace34fa0 100755
> --- a/docparse/parse.sh
> +++ b/docparse/parse.sh
> @@ -26,7 +26,7 @@ echo ' "tests": {'
>  first=1
>  
>  for test in `find testcases/ -name '*.c'`; do
> -	a=$($top_builddir/docparse/docparse "$test")
> +	a=$($top_builddir/docparse/docparse -Iinclude -Itestcases/kernel/syscalls/utils/ "$test")
>  	if [ -n "$a" ]; then
>  		if [ -z "$first" ]; then
>  			echo ','
> diff --git a/docparse/tests/array_size01.c b/docparse/tests/array_size01.c
> new file mode 100644
> index 000000000..8a0b9252b
> --- /dev/null
> +++ b/docparse/tests/array_size01.c
> @@ -0,0 +1,5 @@
> +static int variants = {1};

I can see how this might work, but looks odd.

> +
> +static struct tst_test test = {
> +	.test_variants = ARRAY_SIZE(variants),
> +};
> diff --git a/docparse/tests/array_size01.c.out b/docparse/tests/array_size01.c.out
> new file mode 100644
> index 000000000..ec364be12
> --- /dev/null
> +++ b/docparse/tests/array_size01.c.out
> @@ -0,0 +1,4 @@
> +  "array_size01": {
> +   "test_variants": 1,
> +   "fname": "array_size01.c"
> +  }
> \ No newline at end of file
> diff --git a/docparse/tests/array_size02.c b/docparse/tests/array_size02.c
> new file mode 100644
> index 000000000..5c7d4471e
> --- /dev/null
> +++ b/docparse/tests/array_size02.c
> @@ -0,0 +1,5 @@
> +static int variants = {{1}, {2}, {3}};

Should this be variants[]?

> +
> +static struct tst_test test = {
> +	.test_variants = ARRAY_SIZE(variants),
> +};
> diff --git a/docparse/tests/array_size02.c.out b/docparse/tests/array_size02.c.out
> new file mode 100644
> index 000000000..122686952
> --- /dev/null
> +++ b/docparse/tests/array_size02.c.out
> @@ -0,0 +1,4 @@
> +  "array_size02": {
> +   "test_variants": 3,
> +   "fname": "array_size02.c"
> +  }
> \ No newline at end of file
> diff --git a/docparse/tests/array_size03.c b/docparse/tests/array_size03.c
> new file mode 100644
> index 000000000..9058db813
> --- /dev/null
> +++ b/docparse/tests/array_size03.c
> @@ -0,0 +1,10 @@
> +static struct foo variants[] = {
> +#ifdef FOO
> +	{.bar = 11},
> +#endif
> +	{.bar = 10},
> +};
> +
> +static struct tst_test test = {
> +	.test_variants = ARRAY_SIZE(variants),
> +};
> diff --git a/docparse/tests/array_size03.c.out b/docparse/tests/array_size03.c.out
> new file mode 100644
> index 000000000..bb690c9f5
> --- /dev/null
> +++ b/docparse/tests/array_size03.c.out
> @@ -0,0 +1,4 @@
> +  "array_size03": {
> +   "test_variants": 2,
> +   "fname": "array_size03.c"
> +  }
> \ No newline at end of file
> diff --git a/docparse/tests/array_size04.c b/docparse/tests/array_size04.c
> new file mode 100644
> index 000000000..5f1d9986e
> --- /dev/null
> +++ b/docparse/tests/array_size04.c
> @@ -0,0 +1,5 @@
> +#include "include.h"
> +
> +static struct tst_test test = {
> +	.test_variants = ARRAY_SIZE(variants),
> +};
> diff --git a/docparse/tests/array_size04.c.out b/docparse/tests/array_size04.c.out
> new file mode 100644
> index 000000000..6b8d41792
> --- /dev/null
> +++ b/docparse/tests/array_size04.c.out
> @@ -0,0 +1,4 @@
> +  "array_size04": {
> +   "test_variants": 3,
> +   "fname": "array_size04.c"
> +  }
> \ No newline at end of file
> diff --git a/docparse/tests/include.h b/docparse/tests/include.h
> index efa11d24f..fbc69a561 100644
> --- a/docparse/tests/include.h
> +++ b/docparse/tests/include.h
> @@ -1 +1,7 @@
>  #define TEST_VARIANTS 10
> +
> +static struct variants[] = {
> +	{.bar = 10},
> +	{.bar = 11},
> +	{.bar = 12}
> +};
> -- 
> 2.32.0


-- 
Thank you,
Richard.

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp