[Qemu-devel] [PATCH v2 0/2] qapi: Allow modularization of QAPI schema files

[Qemu-devel] [PATCH v2 0/2] qapi: Allow modularization of QAPI schema files

[Lluís Vilanova]

Adds the "include(...)" primitive to the syntax of QAPI schema files, allowing
these to be modularized into multiple per-topic files in the future.

Signed-off-by: Lluís Vilanova <vilanova@ac.upc.edu>
---

Changes in v2:

* Change the scripts to use an explicit input file instead of standard input.
* Fix "tests/Makefile" to use the new argument.
* Get the input directory for the "include" directive from the input file
  dirname.


Lluís Vilanova (2):
      qapi: Use an explicit input file
      qapi: Add a primitive to include other files from a QAPI schema file


 Makefile                 |   24 ++++++++++++++++++------
 scripts/qapi-commands.py |   10 +++++++---
 scripts/qapi-types.py    |    9 ++++++---
 scripts/qapi-visit.py    |    9 ++++++---
 scripts/qapi.py          |   26 +++++++++++++++++++++++---
 tests/Makefile           |   12 +++++++++---
 6 files changed, 69 insertions(+), 21 deletions(-)


To: qemu-devel@nongnu.org
Cc: Luiz Capitulino <lcapitulino@redhat.com>
Cc: Markus Armbruster <armbru@redhat.com>

[Qemu-devel] [PATCH v2 1/2] qapi: Use an explicit input file

[Lluís Vilanova]

Use an explicit input file on the command-line instead of reading from standard input

Signed-off-by: Lluís Vilanova <vilanova@ac.upc.edu>Summary:
---
 Makefile                 |   24 ++++++++++++++++++------
 scripts/qapi-commands.py |   10 +++++++---
 scripts/qapi-types.py    |    9 ++++++---
 scripts/qapi-visit.py    |    9 ++++++---
 scripts/qapi.py          |    4 ++--
 tests/Makefile           |   12 +++++++++---
 6 files changed, 48 insertions(+), 20 deletions(-)

diff --git a/Makefile b/Makefile
index a9b6e39..50a8e38 100644
--- a/Makefile
+++ b/Makefile
@@ -212,23 +212,35 @@ qapi-py = $(SRC_PATH)/scripts/qapi.py $(SRC_PATH)/scripts/ordereddict.py
 
 qga/qapi-generated/qga-qapi-types.c qga/qapi-generated/qga-qapi-types.h :\
 $(SRC_PATH)/qga/qapi-schema.json $(SRC_PATH)/scripts/qapi-types.py $(qapi-py)
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py $(gen-out-type) -o qga/qapi-generated -p "qga-" < $<, "  GEN   $@")
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py \
+		$(gen-out-type) -i "$<" -o qga/qapi-generated -p "qga-", \
+		"  GEN   $@")
 qga/qapi-generated/qga-qapi-visit.c qga/qapi-generated/qga-qapi-visit.h :\
 $(SRC_PATH)/qga/qapi-schema.json $(SRC_PATH)/scripts/qapi-visit.py $(qapi-py)
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-visit.py $(gen-out-type) -o qga/qapi-generated -p "qga-" < $<, "  GEN   $@")
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-visit.py \
+		$(gen-out-type) -i "$<" -o qga/qapi-generated -p "qga-", \
+		"  GEN   $@")
 qga/qapi-generated/qga-qmp-commands.h qga/qapi-generated/qga-qmp-marshal.c :\
 $(SRC_PATH)/qga/qapi-schema.json $(SRC_PATH)/scripts/qapi-commands.py $(qapi-py)
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py $(gen-out-type) -o qga/qapi-generated -p "qga-" < $<, "  GEN   $@")
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py \
+		$(gen-out-type) -i "$<" -o qga/qapi-generated -p "qga-", \
+		"  GEN   $@")
 
 qapi-types.c qapi-types.h :\
 $(SRC_PATH)/qapi-schema.json $(SRC_PATH)/scripts/qapi-types.py $(qapi-py)
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py $(gen-out-type) -o "." -b < $<, "  GEN   $@")
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py \
+		$(gen-out-type) -i "$<" -o "." -b, \
+		"  GEN   $@")
 qapi-visit.c qapi-visit.h :\
 $(SRC_PATH)/qapi-schema.json $(SRC_PATH)/scripts/qapi-visit.py $(qapi-py)
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-visit.py $(gen-out-type) -o "." -b < $<, "  GEN   $@")
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-visit.py \
+		$(gen-out-type) -i "$<" -o "." -b, \
+		"  GEN   $@")
 qmp-commands.h qmp-marshal.c :\
 $(SRC_PATH)/qapi-schema.json $(SRC_PATH)/scripts/qapi-commands.py $(qapi-py)
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py $(gen-out-type) -m -o "." < $<, "  GEN   $@")
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py \
+		$(gen-out-type) -i "$<" -o "." -m, \
+		"  GEN   $@")
 
 QGALIB_GEN=$(addprefix qga/qapi-generated/, qga-qapi-types.h qga-qapi-visit.h qga-qmp-commands.h)
 $(qga-obj-y) qemu-ga.o: $(QGALIB_GEN)
diff --git a/scripts/qapi-commands.py b/scripts/qapi-commands.py
index b12b696..1657f21 100644
--- a/scripts/qapi-commands.py
+++ b/scripts/qapi-commands.py
@@ -389,13 +389,15 @@ def gen_command_def_prologue(prefix="", proxy=False):
 
 
 try:
-    opts, args = getopt.gnu_getopt(sys.argv[1:], "chp:o:m",
+    opts, args = getopt.gnu_getopt(sys.argv[1:], "chp:i:o:m",
                                    ["source", "header", "prefix=",
-                                    "output-dir=", "type=", "middle"])
+                                    "input-file=", "output-dir=",
+                                    "type=", "middle"])
 except getopt.GetoptError, err:
     print str(err)
     sys.exit(1)
 
+input_file = ""
 output_dir = ""
 prefix = ""
 dispatch_type = "sync"
@@ -409,6 +411,8 @@ do_h = False
 for o, a in opts:
     if o in ("-p", "--prefix"):
         prefix = a
+    elif o in ("-i", "--input-file"):
+        input_file = a
     elif o in ("-o", "--output-dir"):
         output_dir = a + "/"
     elif o in ("-t", "--type"):
@@ -440,7 +444,7 @@ except os.error, e:
     if e.errno != errno.EEXIST:
         raise
 
-exprs = parse_schema(sys.stdin)
+exprs = parse_schema(input_file)
 commands = filter(lambda expr: expr.has_key('command'), exprs)
 commands = filter(lambda expr: not expr.has_key('gen'), commands)
 
diff --git a/scripts/qapi-types.py b/scripts/qapi-types.py
index 4a1652b..7304543 100644
--- a/scripts/qapi-types.py
+++ b/scripts/qapi-types.py
@@ -282,14 +282,15 @@ void qapi_free_%(type)s(%(c_type)s obj)
 
 
 try:
-    opts, args = getopt.gnu_getopt(sys.argv[1:], "chbp:o:",
+    opts, args = getopt.gnu_getopt(sys.argv[1:], "chbp:i:o:",
                                    ["source", "header", "builtins",
-                                    "prefix=", "output-dir="])
+                                    "prefix=", "input-file=", "output-dir="])
 except getopt.GetoptError, err:
     print str(err)
     sys.exit(1)
 
 output_dir = ""
+input_file = ""
 prefix = ""
 c_file = 'qapi-types.c'
 h_file = 'qapi-types.h'
@@ -301,6 +302,8 @@ do_builtins = False
 for o, a in opts:
     if o in ("-p", "--prefix"):
         prefix = a
+    elif o in ("-i", "--input-file"):
+        input_file = a
     elif o in ("-o", "--output-dir"):
         output_dir = a + "/"
     elif o in ("-c", "--source"):
@@ -381,7 +384,7 @@ fdecl.write(mcgen('''
 ''',
                   guard=guardname(h_file)))
 
-exprs = parse_schema(sys.stdin)
+exprs = parse_schema(input_file)
 exprs = filter(lambda expr: not expr.has_key('gen'), exprs)
 
 fdecl.write(guardstart("QAPI_TYPES_BUILTIN_STRUCT_DECL"))
diff --git a/scripts/qapi-visit.py b/scripts/qapi-visit.py
index 65f1a54..856f969 100644
--- a/scripts/qapi-visit.py
+++ b/scripts/qapi-visit.py
@@ -383,13 +383,14 @@ void visit_type_%(name)s(Visitor *m, %(name)s * obj, const char *name, Error **e
                 name=name)
 
 try:
-    opts, args = getopt.gnu_getopt(sys.argv[1:], "chbp:o:",
+    opts, args = getopt.gnu_getopt(sys.argv[1:], "chbp:i:o:",
                                    ["source", "header", "builtins", "prefix=",
-                                    "output-dir="])
+                                    "input-file=", "output-dir="])
 except getopt.GetoptError, err:
     print str(err)
     sys.exit(1)
 
+input_file = ""
 output_dir = ""
 prefix = ""
 c_file = 'qapi-visit.c'
@@ -402,6 +403,8 @@ do_builtins = False
 for o, a in opts:
     if o in ("-p", "--prefix"):
         prefix = a
+    elif o in ("-i", "--input-file"):
+        input_file = a
     elif o in ("-o", "--output-dir"):
         output_dir = a + "/"
     elif o in ("-c", "--source"):
@@ -480,7 +483,7 @@ fdecl.write(mcgen('''
 ''',
                   prefix=prefix, guard=guardname(h_file)))
 
-exprs = parse_schema(sys.stdin)
+exprs = parse_schema(input_file)
 
 # to avoid header dependency hell, we always generate declarations
 # for built-in types in our header files and simply guard them
diff --git a/scripts/qapi.py b/scripts/qapi.py
index 9b3de4c..9fc6fb2 100644
--- a/scripts/qapi.py
+++ b/scripts/qapi.py
@@ -158,9 +158,9 @@ class QAPISchema:
             raise QAPISchemaError(self, 'Expected "{", "[" or string')
         return expr
 
-def parse_schema(fp):
+def parse_schema(input_path):
     try:
-        schema = QAPISchema(fp)
+        schema = QAPISchema(open(input_path, "r"))
     except QAPISchemaError, e:
         print >>sys.stderr, e
         exit(1)
diff --git a/tests/Makefile b/tests/Makefile
index fd36eee..cc227b3 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -174,13 +174,19 @@ tests/test-vmstate$(EXESUF): tests/test-vmstate.o \
 
 tests/test-qapi-types.c tests/test-qapi-types.h :\
 $(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-types.py
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py $(gen-out-type) -o tests -p "test-" < $<, "  GEN   $@")
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py \
+		$(gen-out-type) -i "$<" -o tests -p "test-", \
+		"  GEN   $@")
 tests/test-qapi-visit.c tests/test-qapi-visit.h :\
 $(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-visit.py
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-visit.py $(gen-out-type) -o tests -p "test-" < $<, "  GEN   $@")
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-visit.py \
+		$(gen-out-type) -i "$<" -o tests -p "test-", \
+		"  GEN   $@")
 tests/test-qmp-commands.h tests/test-qmp-marshal.c :\
 $(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-commands.py
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py $(gen-out-type) -o tests -p "test-" < $<, "  GEN   $@")
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py \
+		$(gen-out-type) -i "$<" -o tests -p "test-", \
+		"  GEN   $@")
 
 tests/test-string-output-visitor$(EXESUF): tests/test-string-output-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
 tests/test-string-input-visitor$(EXESUF): tests/test-string-input-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a

[Qemu-devel] [PATCH v2 2/2] qapi: Add a primitive to include other files from a QAPI schema file

[Lluís Vilanova]

Adds the "include(...)" primitive to the syntax of QAPI schema files.

Signed-off-by: Lluís Vilanova <vilanova@ac.upc.edu>
---
 scripts/qapi.py |   24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/scripts/qapi.py b/scripts/qapi.py
index 9fc6fb2..80fb594 100644
--- a/scripts/qapi.py
+++ b/scripts/qapi.py
@@ -11,9 +11,13 @@
 # This work is licensed under the terms of the GNU GPLv2.
 # See the COPYING.LIB file in the top-level directory.
 
+import os
+import re
 from ordereddict import OrderedDict
 import sys
 
+include_cre = re.compile("include\(\"([^\"]*)\"\)")
+
 builtin_types = [
     'str', 'int', 'number', 'bool',
     'int8', 'int16', 'int32', 'int64',
@@ -54,8 +58,9 @@ class QAPISchemaError(Exception):
 
 class QAPISchema:
 
-    def __init__(self, fp):
+    def __init__(self, fp, input_dir):
         self.fp = fp
+        self.input_dir = input_dir
         self.src = fp.read()
         if self.src == '' or self.src[-1] != '\n':
             self.src += '\n'
@@ -100,6 +105,20 @@ class QAPISchema:
                 if self.cursor == len(self.src):
                     self.tok = None
                     return
+            elif self.tok == 'i':
+                include_src = self.src[self.cursor-1:]
+                include_match = include_cre.match(include_src)
+                if include_match is not None:
+                    include_path = os.path.join(self.input_dir,
+                                                include_match.group(1))
+                    if not os.path.isfile(include_path):
+                        raise QAPISchemaError(
+                            self,
+                            'Non-existing included file "%s"' % include_path)
+                    include_schema = QAPISchema(open(include_path),
+                                                self.input_dir)
+                    self.exprs += include_schema.exprs
+                    self.cursor += include_match.span()[1] - 1
             elif not self.tok.isspace():
                 raise QAPISchemaError(self, 'Stray "%s"' % self.tok)
 
@@ -159,8 +178,9 @@ class QAPISchema:
         return expr
 
 def parse_schema(input_path):
+    input_dir = os.path.dirname(input_path)
     try:
-        schema = QAPISchema(open(input_path, "r"))
+        schema = QAPISchema(open(input_path, "r"), input_dir)
     except QAPISchemaError, e:
         print >>sys.stderr, e
         exit(1)

Re: [Qemu-devel] [PATCH v2 1/2] qapi: Use an explicit input file

[Eric Blake]

[-- Attachment #1: Type: text/plain, Size: 815 bytes --]

On 02/10/2014 12:58 PM, Lluís Vilanova wrote:
> Use an explicit input file on the command-line instead of reading from standard input
> 
> Signed-off-by: Lluís Vilanova <vilanova@ac.upc.edu>Summary:

Screwed up S-o-b line, but the maintainer can probably fix that without
needing a respin.

> ---
>  Makefile                 |   24 ++++++++++++++++++------
>  scripts/qapi-commands.py |   10 +++++++---
>  scripts/qapi-types.py    |    9 ++++++---
>  scripts/qapi-visit.py    |    9 ++++++---
>  scripts/qapi.py          |    4 ++--
>  tests/Makefile           |   12 +++++++++---
>  6 files changed, 48 insertions(+), 20 deletions(-)
> 

Reviewed-by: Eric Blake <eblake@redhat.com>

-- 
Eric Blake   eblake redhat com    +1-919-301-3266
Libvirt virtualization library http://libvirt.org


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 604 bytes --]

Re: [Qemu-devel] [PATCH v2 2/2] qapi: Add a primitive to include other files from a QAPI schema file

[Eric Blake]

[-- Attachment #1: Type: text/plain, Size: 1348 bytes --]

On 02/10/2014 12:58 PM, Lluís Vilanova wrote:
> Adds the "include(...)" primitive to the syntax of QAPI schema files.
> 
> Signed-off-by: Lluís Vilanova <vilanova@ac.upc.edu>
> ---
>  scripts/qapi.py |   24 ++++++++++++++++++++++--
>  1 file changed, 22 insertions(+), 2 deletions(-)
> 

> +            elif self.tok == 'i':
> +                include_src = self.src[self.cursor-1:]
> +                include_match = include_cre.match(include_src)
> +                if include_match is not None:
> +                    include_path = os.path.join(self.input_dir,
> +                                                include_match.group(1))
> +                    if not os.path.isfile(include_path):
> +                        raise QAPISchemaError(
> +                            self,
> +                            'Non-existing included file "%s"' % include_path)

Should you also check against loops, and gracefully fail if a file tries
to include itself, or if file A includes B while B includes A?

This changes the code generator, but not the documentation.  For this
patch to be complete, you need to edit somewhere in the docs tree,
probably docs/qapi-code-gen.txt, to describe how to use includes.

-- 
Eric Blake   eblake redhat com    +1-919-301-3266
Libvirt virtualization library http://libvirt.org


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 604 bytes --]

Re: [Qemu-devel] [PATCH v2 0/2] qapi: Allow modularization of QAPI schema files

[Markus Armbruster]

Lluís Vilanova <vilanova@ac.upc.edu> writes:

> Adds the "include(...)" primitive to the syntax of QAPI schema files, allowing
> these to be modularized into multiple per-topic files in the future.
>
> Signed-off-by: Lluís Vilanova <vilanova@ac.upc.edu>
> ---
>
> Changes in v2:
>
> * Change the scripts to use an explicit input file instead of standard input.
> * Fix "tests/Makefile" to use the new argument.
> * Get the input directory for the "include" directive from the input file
>   dirname.
>
>
> Lluís Vilanova (2):
>       qapi: Use an explicit input file
>       qapi: Add a primitive to include other files from a QAPI schema file
>
>
>  Makefile                 |   24 ++++++++++++++++++------
>  scripts/qapi-commands.py |   10 +++++++---
>  scripts/qapi-types.py    |    9 ++++++---
>  scripts/qapi-visit.py    |    9 ++++++---
>  scripts/qapi.py          |   26 +++++++++++++++++++++++---
>  tests/Makefile           |   12 +++++++++---
>  6 files changed, 69 insertions(+), 21 deletions(-)

Diffstat shows that you forgot to include the tests :)

Suggest to rebase on top of my "[PATCH v2 00/13] qapi: Test coverage &
clean up generated code", then add suitable tests to tests/qapi-schema/.
Make sure to cover reporting of errors in included files.