[LTP] [PATCH 0/2] Two more metadata parser fixes

[LTP] [PATCH 0/2] Two more metadata parser fixes

[Cyril Hrubis]

This series fixes a metadata parser for the ".filesystems" entry of the
tst_test structure. First patch fixes the data storage to JSON
serialization and second patch changes the parser to convert structure
intialization with only designated initializers into a hash (object)
rather than an arrray.

@Andrea I suppose that the documentation generator needs a fix after
        this is applied.

Cyril Hrubis (2):
  metadata: data_storage: Fix hash to json object serialization
  metaparse: Map arrays with designated initializers to JSON objects

 metadata/data_storage.h |  28 +++++++--
 metadata/metaparse.c    | 134 +++++++++++++++++++++++++++++++++-------
 2 files changed, 135 insertions(+), 27 deletions(-)

-- 
2.49.0


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

[LTP] [PATCH 1/2] metadata: data_storage: Fix hash to json object serialization

[Cyril Hrubis]

We have to move the curly braces into the switch() that prints the JSON
values otherwise inner object will not have curly braces around them and
the resulting JSON is not valid.

The changes between the old generated metadata and new metadata are only
in the whitespaces:

diff -w ltp-prev.json ltp.json
7c7
<   "version": "20250530-41-g080ced081"
---
>   "version": "20250530-42-g73a97b583"

Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
---
 metadata/data_storage.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/metadata/data_storage.h b/metadata/data_storage.h
index 82d67b829..f837feec8 100644
--- a/metadata/data_storage.h
+++ b/metadata/data_storage.h
@@ -396,14 +396,16 @@ static inline void data_to_json_(struct data_node *self, FILE *f, unsigned int p
 		data_fprintf(f, padd, "null");
 	break;
 	case DATA_HASH:
+		data_fprintf(f, do_padd ? padd : 0, "{\n");
 		for (i = 0; i < self->hash.elems_used; i++) {
-			data_fprintf(f, padd, "\"%s\": ", self->hash.elems[i].id);
+			data_fprintf(f, padd+1, "\"%s\": ", self->hash.elems[i].id);
 			data_to_json_(self->hash.elems[i].node, f, padd+1, 0);
 			if (i < self->hash.elems_used - 1)
 				fprintf(f, ",\n");
 			else
 				fprintf(f, "\n");
 		}
+		data_fprintf(f, padd, "}");
 	break;
 	case DATA_ARRAY:
 		data_fprintf(f, do_padd ? padd : 0, "[\n");
@@ -421,9 +423,7 @@ static inline void data_to_json_(struct data_node *self, FILE *f, unsigned int p
 
 static inline void data_to_json(struct data_node *self, FILE *f, unsigned int padd)
 {
-	fprintf(f, "{\n");
-	data_to_json_(self, f, padd + 1, 1);
-	data_fprintf(f, padd, "}");
+	data_to_json_(self, f, padd, 0);
 }
 
 #endif /* DATA_STORAGE_H__ */
-- 
2.49.0


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

[LTP] [PATCH 2/2] metaparse: Map arrays with designated initializers to JSON objects

[Cyril Hrubis]

The parser now scans array declarations and if it finds out that
complete initialization is using designated initializers the result is
mapped into a JSON object instead of an arrray.

A diff snippet looks like:

--------------------------
    "filesystems": [
-    [
-     ".type=btrfs"
-    ],
-    [
-     ".type=bcachefs"
-    ],
-    [
-     ".type=xfs",
-     ".min_kver=4.16",
-     ".mkfs_ver=mkfs.xfs >= 1.5.0",
-     [
+    {
+     "type": "btrfs"
+    },
+    {
+     "type": "bcachefs"
+    },
+    {
+     "type": "xfs",
+     "min_kver": "4.16",
+     "mkfs_ver": "mkfs.xfs >= 1.5.0",
+     "mkfs_opts": [
       "-m",
       "reflink=1"
-     ],
-     ".mkfs_opts=(constchar*const[])"
-    ]
+     ]
+    }
    ],
--------------------------

Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
---
 metadata/data_storage.h |  20 ++++++
 metadata/metaparse.c    | 134 +++++++++++++++++++++++++++++++++-------
 2 files changed, 131 insertions(+), 23 deletions(-)

diff --git a/metadata/data_storage.h b/metadata/data_storage.h
index f837feec8..6ca5d7d90 100644
--- a/metadata/data_storage.h
+++ b/metadata/data_storage.h
@@ -10,6 +10,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
+#include <stdbool.h>
 
 enum data_type {
 	DATA_ARRAY,
@@ -228,6 +229,13 @@ static inline struct data_node *data_node_hash_get(struct data_node *self, const
 	return hash->elems[i].node;
 }
 
+static inline unsigned int data_node_hash_len(struct data_node *self)
+{
+	struct data_node_hash *hash = &self->hash;
+
+	return hash->elems_used;
+}
+
 static inline int data_node_array_add(struct data_node *self, struct data_node *payload)
 {
 	if (self->type != DATA_ARRAY)
@@ -285,6 +293,18 @@ static inline void data_print_padd(unsigned int i)
 		putchar(' ');
 }
 
+static inline bool data_node_is_empty(struct data_node *self)
+{
+	switch (self->type) {
+	case DATA_ARRAY:
+		return data_node_array_len(self) == 0;
+	case DATA_HASH:
+		return data_node_hash_len(self) == 0;
+	default:
+		return false;
+	}
+}
+
 static inline void data_node_print_(struct data_node *self, unsigned int padd)
 {
 	unsigned int i;
diff --git a/metadata/metaparse.c b/metadata/metaparse.c
index 9723a92c2..e9e9aee10 100644
--- a/metadata/metaparse.c
+++ b/metadata/metaparse.c
@@ -337,15 +337,20 @@ static void try_apply_macro(char **res)
 	*res = ret->data;
 }
 
-static void finalize_array_entry(char **entry, struct data_node *node)
+static void finalize_array_entry(char **val, char **id, struct data_node *node)
 {
-	if (!*entry)
+	if (!*val)
 		return;
 
-	data_node_array_add(node, data_node_string(*entry));
+	if (*id)
+		data_node_hash_add(node, *id+1, data_node_string(*val));
+	else
+		data_node_array_add(node, data_node_string(*val));
 
-	free(*entry);
-	*entry = NULL;
+	free(*id);
+	free(*val);
+	*id = NULL;
+	*val = NULL;
 }
 
 static void str_append(char **res, char *append)
@@ -369,24 +374,98 @@ err:
 	exit(1);
 }
 
-static int parse_array(FILE *f, struct data_node *node)
+static int array_is_hash(FILE *f)
 {
+	long pos = ftell(f);
+	int has_ids = 1;
+	int elem_seen = 0;
+	int in_id = 1;
 	char *token;
-	char *entry = NULL;
+
+	while ((token = next_token(f, NULL))) {
+
+		if (!strcmp(token, "}"))
+			goto ret;
+
+		elem_seen = 1;
+
+		if (!strcmp(token, "{")) {
+			if (in_id) {
+				has_ids = 0;
+				goto ret;
+			}
+
+			int level = 1;
+
+			for (;;) {
+				token = next_token(f, NULL);
+
+				if (!token)
+					goto ret;
+
+				if (!strcmp(token, "{"))
+					level++;
+
+				if (!strcmp(token, "}"))
+					level--;
+
+				if (!level)
+					break;
+			}
+		} else if (!strcmp(token, ",")) {
+			if (in_id) {
+				has_ids = 0;
+				goto ret;
+			}
+
+			in_id = 1;
+		} else if (!strcmp(token, "=")) {
+			in_id = 0;
+		}
+	}
+
+ret:
+	fseek(f, pos, SEEK_SET);
+	return elem_seen && has_ids;
+}
+
+static int parse_array(FILE *f, const char *arr_id, struct data_node **ret)
+{
+	char *token;
+	char *val = NULL, *id = NULL;
 	int parent_cnt = 0;
+	int is_hash = array_is_hash(f);
+
+	if (verbose)
+		fprintf(stderr, "PARSING ARRAY (%s) is_hash = %i\n", arr_id, is_hash);
+
+	if (is_hash)
+		*ret = data_node_hash();
+	else
+		*ret = data_node_array();
 
 	for (;;) {
 		if (!(token = next_token(f, NULL)))
 			return 1;
 
 		if (!strcmp(token, "{")) {
-			struct data_node *ret = data_node_array();
-			parse_array(f, ret);
+			struct data_node *sub_ret;
 
-			if (data_node_array_len(ret))
-				data_node_array_add(node, ret);
-			else
-				data_node_free(ret);
+			parse_array(f, id, &sub_ret);
+
+			if (data_node_is_empty(sub_ret)) {
+				data_node_free(sub_ret);
+			} else {
+				if (is_hash)
+					data_node_hash_add(*ret, id+1, sub_ret);
+				else
+					data_node_array_add(*ret, sub_ret);
+			}
+
+			free(id);
+			id = NULL;
+			free(val);
+			val = NULL;
 
 			continue;
 		}
@@ -394,23 +473,33 @@ static int parse_array(FILE *f, struct data_node *node)
 		if (!strcmp(token, "}")) {
 			struct data_node *arr_last;
 
-			finalize_array_entry(&entry, node);
-
+			finalize_array_entry(&val, &id, *ret);
 			/* Remove NULL terminating entry, if present. */
-			arr_last = data_node_array_last(node);
-			if (arr_last && arr_last->type == DATA_NULL)
-				data_node_array_last_rem(node);
+			if (!is_hash) {
+				arr_last = data_node_array_last(*ret);
+				if (arr_last && arr_last->type == DATA_NULL)
+					data_node_array_last_rem(*ret);
+			}
 
 			return 0;
 		}
 
+		if (is_hash && !strcmp(token, "=") && !id) {
+			id = val;
+			val = NULL;
+			continue;
+		}
+
 		if (!strcmp(token, ",") && parent_cnt <= 0) {
-			finalize_array_entry(&entry, node);
+			finalize_array_entry(&val, &id, *ret);
 			continue;
 		}
 
 		if (!strcmp(token, "NULL")) {
-			data_node_array_add(node, data_node_null());
+			if (is_hash)
+				data_node_hash_add(*ret, id, data_node_null());
+			else
+				data_node_array_add(*ret, data_node_null());
 			continue;
 		}
 
@@ -421,7 +510,7 @@ static int parse_array(FILE *f, struct data_node *node)
 			parent_cnt--;
 
 		try_apply_macro(&token);
-		str_append(&entry, token);
+		str_append(&val, token);
 	}
 
 	return 0;
@@ -605,8 +694,7 @@ static int parse_test_struct(FILE *f, struct data_node *doc, struct data_node *n
 		}
 
 		if (!strcmp(token, "{")) {
-			ret = data_node_array();
-			parse_array(f, ret);
+			parse_array(f, id, &ret);
 		} else if (!strcmp(token, "ARRAY_SIZE")) {
 			if (parse_array_size(f, &ret))
 				return 1;
-- 
2.49.0


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

Re: [LTP] [PATCH 0/2] Two more metadata parser fixes

[Andrea Cervesato via ltp]

Hi!

On 6/20/25 5:43 PM, Cyril Hrubis wrote:
> This series fixes a metadata parser for the ".filesystems" entry of the
> tst_test structure. First patch fixes the data storage to JSON
> serialization and second patch changes the parser to convert structure
> intialization with only designated initializers into a hash (object)
> rather than an arrray.
>
> @Andrea I suppose that the documentation generator needs a fix after
>          this is applied.
I will take a look once is merged. In general, I like the idea of this 
patch-set.
> Cyril Hrubis (2):
>    metadata: data_storage: Fix hash to json object serialization
>    metaparse: Map arrays with designated initializers to JSON objects
>
>   metadata/data_storage.h |  28 +++++++--
>   metadata/metaparse.c    | 134 +++++++++++++++++++++++++++++++++-------
>   2 files changed, 135 insertions(+), 27 deletions(-)
>
- Andrea

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

Re: [LTP] [PATCH 0/2] Two more metadata parser fixes

[Andrea Cervesato via ltp]

Hi!

Feel free to merge, I will fix kirk later on.

Acked-by: Andrea Cervesato <andrea.cervesato@suse.com>

- Andrea

On 6/20/25 5:43 PM, Cyril Hrubis wrote:
> This series fixes a metadata parser for the ".filesystems" entry of the
> tst_test structure. First patch fixes the data storage to JSON
> serialization and second patch changes the parser to convert structure
> intialization with only designated initializers into a hash (object)
> rather than an arrray.
>
> @Andrea I suppose that the documentation generator needs a fix after
>          this is applied.
>
> Cyril Hrubis (2):
>    metadata: data_storage: Fix hash to json object serialization
>    metaparse: Map arrays with designated initializers to JSON objects
>
>   metadata/data_storage.h |  28 +++++++--
>   metadata/metaparse.c    | 134 +++++++++++++++++++++++++++++++++-------
>   2 files changed, 135 insertions(+), 27 deletions(-)
>

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

Re: [LTP] [PATCH 0/2] Two more metadata parser fixes

[Cyril Hrubis]

Hi!
> Feel free to merge, I will fix kirk later on.

These changes also crash doc generator since the format of the metadata
has changed:

Traceback (most recent call last):
  File "/home/metan/Work/ltp/doc/.venv/lib/python3.12/site-packages/sphinx/events.py", line 97, in emit
    results.append(listener.handler(self.app, *args))
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/metan/Work/ltp/doc/conf.py", line 517, in generate_test_catalog
    text.extend(_generate_setup_table(conf))
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/metan/Work/ltp/doc/conf.py", line 407, in _generate_setup_table
    values.append(f'{value[0]}, {value[1]}')
                     ~~~~~^^^
KeyError: 0

-- 
Cyril Hrubis
chrubis@suse.cz

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

Re: [LTP] [PATCH 0/2] Two more metadata parser fixes

[Andrea Cervesato via ltp]

On 7/7/25 1:10 PM, Cyril Hrubis wrote:
> Hi!
>> Feel free to merge, I will fix kirk later on.
> These changes also crash doc generator since the format of the metadata
> has changed:
>
> Traceback (most recent call last):
>    File "/home/metan/Work/ltp/doc/.venv/lib/python3.12/site-packages/sphinx/events.py", line 97, in emit
>      results.append(listener.handler(self.app, *args))
>                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>    File "/home/metan/Work/ltp/doc/conf.py", line 517, in generate_test_catalog
>      text.extend(_generate_setup_table(conf))
>                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^
>    File "/home/metan/Work/ltp/doc/conf.py", line 407, in _generate_setup_table
>      values.append(f'{value[0]}, {value[1]}')
>                       ~~~~~^^^
> KeyError: 0
>
Sigh, we need a patch to include in this patch series then..

- Andrea


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

Re: [LTP] [PATCH 2/2] metaparse: Map arrays with designated initializers to JSON objects

[Andrea Cervesato via ltp]

Hi!

This patch is actually causing documentation build failure due to the 
way we are defining .hugepages as:

     .hugepages = {TST_NO_HUGEPAGES},

The shmget02 metadata (for instance) will have an empty hugepages 
dictionary:

   "shmget02": {
    "needs_tmpdir": "1",
    "needs_root": "1",
    "forks_child": "1",
    "save_restore": [
     [
      "/proc/sys/kernel/shmmax",
      "8192",
      "TCONF_MISSING|TBROK_RO"
     ]
    ],
    "doc": [
     "Test for ENOENT, EEXIST, EINVAL, EACCES, EPERM errors.",
     "",
     "- ENOENT - No segment exists for the given key and IPC_CREAT was 
not specified.",
     "- EEXIST - the segment exists and IPC_CREAT | IPC_EXCL is given.",
     "- EINVAL - A new segment was to be created and size is less than 
SHMMIN or",
     "  greater than SHMMAX. Or a segment for the given key exists, but 
size is",
     "  gran eater than the size of that segment.",
     "- EACCES - The user does not have permission to access the shared 
memory segment.",
     "- EPERM - The SHM_HUGETLB flag was specified, but the caller was not",
     "  privileged (did not have the CAP_IPC_LOCK capability) and is not 
a member",
     "  of the sysctl_hugetlb_shm_group group.",
     "- ENOMEM - The SHM_HUGETLB flag was specified, the caller was 
privileged but",
     "  not have enough hugepage memory space."
    ],
    "hugepages": {
    },
    "fname": "testcases/kernel/syscalls/ipc/shmget/shmget02.c"
   }

Which will cause the following error in the documentation build:

Traceback (most recent call last):
   File 
"/home/acer/Projects/ltp/doc/.venv/lib/python3.12/site-packages/sphinx/events.py", 
line 98, in emit
     results.append(listener.handler(self.app, *args))
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   File "/home/acer/Projects/ltp/doc/conf.py", line 518, in 
generate_test_catalog
     text.extend(_generate_setup_table(conf))
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
   File "/home/acer/Projects/ltp/doc/conf.py", line 407, in 
_generate_setup_table
     values.append(f'{value[0]}, {value[1]}')
                      ~~~~~^^^
KeyError: 0

- Andrea


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

Re: [LTP] [PATCH 2/2] metaparse: Map arrays with designated initializers to JSON objects

[Cyril Hrubis]

Hi!
> This patch is actually causing documentation build failure due to the 
> way we are defining .hugepages as:
> 
>      .hugepages = {TST_NO_HUGEPAGES},
> 
> The shmget02 metadata (for instance) will have an empty hugepages 
> dictionary:

Ah, right, I've missed a special case in the C structure parsing.

This on the top of the patchset fixes the problem:

diff --git a/metadata/metaparse.c b/metadata/metaparse.c
index e9e9aee10..36736ac06 100644
--- a/metadata/metaparse.c
+++ b/metadata/metaparse.c
@@ -379,13 +379,17 @@ static int array_is_hash(FILE *f)
        long pos = ftell(f);
        int has_ids = 1;
        int elem_seen = 0;
+       int comma_last = 0;
        int in_id = 1;
        char *token;

        while ((token = next_token(f, NULL))) {

-               if (!strcmp(token, "}"))
+               if (!strcmp(token, "}")) {
+                       if (in_id && !comma_last)
+                               has_ids = 0;
                        goto ret;
+               }

                elem_seen = 1;

@@ -419,8 +423,12 @@ static int array_is_hash(FILE *f)
                        }

                        in_id = 1;
+
+                       comma_last = 1;
                } else if (!strcmp(token, "=")) {
                        in_id = 0;
+               } else {
+                       comma_last = 0;
                }
        }


I will send a new version.

-- 
Cyril Hrubis
chrubis@suse.cz

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

Re: [LTP] [PATCH 0/2] Two more metadata parser fixes

[Andrea Cervesato via ltp]

Acked-by: Andrea Cervesato <andrea.cervesato@suse.com>

On 6/20/25 5:43 PM, Cyril Hrubis wrote:
> This series fixes a metadata parser for the ".filesystems" entry of the
> tst_test structure. First patch fixes the data storage to JSON
> serialization and second patch changes the parser to convert structure
> intialization with only designated initializers into a hash (object)
> rather than an arrray.
>
> @Andrea I suppose that the documentation generator needs a fix after
>          this is applied.
>
> Cyril Hrubis (2):
>    metadata: data_storage: Fix hash to json object serialization
>    metaparse: Map arrays with designated initializers to JSON objects
>
>   metadata/data_storage.h |  28 +++++++--
>   metadata/metaparse.c    | 134 +++++++++++++++++++++++++++++++++-------
>   2 files changed, 135 insertions(+), 27 deletions(-)
>

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp