[Qemu-devel] [PATCH] Fix qapi code generation wrt parallel build

qemu-devel.nongnu.org archive mirror
 help / color / mirror / Atom feed

From: Avi Kivity <avi@redhat.com>
To: Anthony Liguori <anthony@codemonkey.ws>, qemu-devel@nongnu.org
Subject: [Qemu-devel] [PATCH] Fix qapi code generation wrt parallel build
Date: Tue, 27 Dec 2011 16:02:16 +0200	[thread overview]
Message-ID: <1324994536-3955-1-git-send-email-avi@redhat.com> (raw)

Make's multiple output syntax

  x.c x.h: x.template
       gen < x.template

actually invokes the command once for x.c and once for x.h (with differing $@
in each invocation).  During a parallel build, the two commands may be invoked
in parallel; this opens up a race, where the second invocation trashes a file
supposedly produced during the first, and now in use by a dependent command.

The various qapi code generators are susceptible to this; fix by making them
generate just one file per invocation.

Signed-off-by: Avi Kivity <avi@redhat.com>
---
 Makefile                 |   20 +++++++++++---------
 scripts/qapi-commands.py |   30 +++++++++++++++++++++++++++---
 scripts/qapi-types.py    |   27 ++++++++++++++++++++++++---
 scripts/qapi-visit.py    |   27 ++++++++++++++++++++++++---
 4 files changed, 86 insertions(+), 18 deletions(-)

diff --git a/Makefile b/Makefile
index 2c03055..5958289 100644
--- a/Makefile
+++ b/Makefile
@@ -172,35 +172,37 @@ qapi-dir := $(BUILD_DIR)/qapi-generated
 test-qmp-input-visitor.o test-qmp-output-visitor.o test-qmp-commands.o qemu-ga$(EXESUF): QEMU_CFLAGS += -I $(qapi-dir)
 qemu-ga$(EXESUF): LIBS = $(LIBS_QGA)
 
+gen-out-type = $(subst .,-,$@)
+
 $(qapi-dir)/test-qapi-types.c $(qapi-dir)/test-qapi-types.h :\
 $(SRC_PATH)/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-types.py
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py -o "$(qapi-dir)" -p "test-" < $<, "  GEN   $@")
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py $(gen-out-type) -o "$(qapi-dir)" -p "test-" < $<, "  GEN   $@")
 $(qapi-dir)/test-qapi-visit.c $(qapi-dir)/test-qapi-visit.h :\
 $(SRC_PATH)/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-visit.py
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-visit.py -o "$(qapi-dir)" -p "test-" < $<, "  GEN   $@")
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-visit.py $(gen-out-type) -o "$(qapi-dir)" -p "test-" < $<, "  GEN   $@")
 $(qapi-dir)/test-qmp-commands.h $(qapi-dir)/test-qmp-marshal.c :\
 $(SRC_PATH)/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-commands.py
-	    $(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py -o "$(qapi-dir)" -p "test-" < $<, "  GEN   $@")
+	    $(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py $(gen-out-type) -o "$(qapi-dir)" -p "test-" < $<, "  GEN   $@")
 
 $(qapi-dir)/qga-qapi-types.c $(qapi-dir)/qga-qapi-types.h :\
 $(SRC_PATH)/qapi-schema-guest.json $(SRC_PATH)/scripts/qapi-types.py
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py -o "$(qapi-dir)" -p "qga-" < $<, "  GEN   $@")
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py $(gen-out-type) -o "$(qapi-dir)" -p "qga-" < $<, "  GEN   $@")
 $(qapi-dir)/qga-qapi-visit.c $(qapi-dir)/qga-qapi-visit.h :\
 $(SRC_PATH)/qapi-schema-guest.json $(SRC_PATH)/scripts/qapi-visit.py
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-visit.py -o "$(qapi-dir)" -p "qga-" < $<, "  GEN   $@")
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-visit.py $(gen-out-type) -o "$(qapi-dir)" -p "qga-" < $<, "  GEN   $@")
 $(qapi-dir)/qga-qmp-commands.h $(qapi-dir)/qga-qmp-marshal.c :\
 $(SRC_PATH)/qapi-schema-guest.json $(SRC_PATH)/scripts/qapi-commands.py
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py -o "$(qapi-dir)" -p "qga-" < $<, "  GEN   $@")
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py $(gen-out-type) -o "$(qapi-dir)" -p "qga-" < $<, "  GEN   $@")
 
 qapi-types.c qapi-types.h :\
 $(SRC_PATH)/qapi-schema.json $(SRC_PATH)/scripts/qapi-types.py
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py -o "." < $<, "  GEN   $@")
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py $(gen-out-type) -o "." < $<, "  GEN   $@")
 qapi-visit.c qapi-visit.h :\
 $(SRC_PATH)/qapi-schema.json $(SRC_PATH)/scripts/qapi-visit.py
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-visit.py -o "."  < $<, "  GEN   $@")
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-visit.py $(gen-out-type) -o "."  < $<, "  GEN   $@")
 qmp-commands.h qmp-marshal.c :\
 $(SRC_PATH)/qapi-schema.json $(SRC_PATH)/scripts/qapi-commands.py
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py -m -o "." < $<, "  GEN   $@")
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py $(gen-out-type) -m -o "." < $<, "  GEN   $@")
 
 test-qmp-output-visitor.o: $(addprefix $(qapi-dir)/, test-qapi-types.c test-qapi-types.h test-qapi-visit.c test-qapi-visit.h) $(qapi-obj-y)
 test-qmp-output-visitor: test-qmp-output-visitor.o $(qobject-obj-y) $(qapi-obj-y) $(tools-obj-y) $(qapi-dir)/test-qapi-visit.o $(qapi-dir)/test-qapi-types.o
diff --git a/scripts/qapi-commands.py b/scripts/qapi-commands.py
index 54d1f5d..bd7b207 100644
--- a/scripts/qapi-commands.py
+++ b/scripts/qapi-commands.py
@@ -372,7 +372,9 @@ def gen_command_def_prologue(prefix="", proxy=False):
 
 
 try:
-    opts, args = getopt.gnu_getopt(sys.argv[1:], "p:o:m", ["prefix=", "output-dir=", "type=", "middle"])
+    opts, args = getopt.gnu_getopt(sys.argv[1:], "chp:o:m",
+                                   ["source", "header", "prefix=",
+                                    "output-dir=", "type=", "middle"])
 except getopt.GetoptError, err:
     print str(err)
     sys.exit(1)
@@ -384,6 +386,9 @@ c_file = 'qmp-marshal.c'
 h_file = 'qmp-commands.h'
 middle_mode = False
 
+do_c = False
+do_h = False
+
 for o, a in opts:
     if o in ("-p", "--prefix"):
         prefix = a
@@ -393,10 +398,29 @@ for o, a in opts:
         dispatch_type = a
     elif o in ("-m", "--middle"):
         middle_mode = True
+    elif o in ("-c", "--source"):
+        do_h = True
+    elif o in ("-h", "--header"):
+        do_c = True
+
+if not do_c and not do_h:
+    do_c = True
+    do_h = True
 
 c_file = output_dir + prefix + c_file
 h_file = output_dir + prefix + h_file
 
+def maybe_open(really, name, opt):
+    class Null(object):
+        def write(self, str):
+            pass
+        def read(self):
+            return ''
+    if really:
+        return open(name, opt)
+    else:
+        return Null()
+
 try:
     os.makedirs(output_dir)
 except os.error, e:
@@ -408,8 +432,8 @@ commands = filter(lambda expr: expr.has_key('command'), exprs)
 commands = filter(lambda expr: not expr.has_key('gen'), commands)
 
 if dispatch_type == "sync":
-    fdecl = open(h_file, 'w')
-    fdef = open(c_file, 'w')
+    fdecl = maybe_open(do_h, h_file, 'w')
+    fdef = maybe_open(do_c, c_file, 'w')
     ret = gen_command_decl_prologue(header=basename(h_file), guard=guardname(h_file), prefix=prefix)
     fdecl.write(ret)
     ret = gen_command_def_prologue(prefix=prefix)
diff --git a/scripts/qapi-types.py b/scripts/qapi-types.py
index 267cb49..ae644bc 100644
--- a/scripts/qapi-types.py
+++ b/scripts/qapi-types.py
@@ -163,7 +163,8 @@ void qapi_free_%(type)s(%(c_type)s obj)
 
 
 try:
-    opts, args = getopt.gnu_getopt(sys.argv[1:], "p:o:", ["prefix=", "output-dir="])
+    opts, args = getopt.gnu_getopt(sys.argv[1:], "chp:o:",
+                                   ["source", "header", "prefix=", "output-dir="])
 except getopt.GetoptError, err:
     print str(err)
     sys.exit(1)
@@ -173,11 +174,22 @@ prefix = ""
 c_file = 'qapi-types.c'
 h_file = 'qapi-types.h'
 
+do_c = False
+do_h = False
+
 for o, a in opts:
     if o in ("-p", "--prefix"):
         prefix = a
     elif o in ("-o", "--output-dir"):
         output_dir = a + "/"
+    elif o in ("-c", "--source"):
+        do_h = True
+    elif o in ("-h", "--header"):
+        do_c = True
+
+if not do_c and not do_h:
+    do_c = True
+    do_h = True
 
 c_file = output_dir + prefix + c_file
 h_file = output_dir + prefix + h_file
@@ -188,8 +200,17 @@ except os.error, e:
     if e.errno != errno.EEXIST:
         raise
 
-fdef = open(c_file, 'w')
-fdecl = open(h_file, 'w')
+def maybe_open(really, name, opt):
+    class Null(object):
+        def write(self, str):
+            pass
+        def read(self):
+            return ''
+    if really:
+        return open(name, opt)
+
+fdef = maybe_open(do_c, c_file, 'w')
+fdecl = maybe_open(do_h, h_file, 'w')
 
 fdef.write(mcgen('''
 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
diff --git a/scripts/qapi-visit.py b/scripts/qapi-visit.py
index 62de83d..e9d0584 100644
--- a/scripts/qapi-visit.py
+++ b/scripts/qapi-visit.py
@@ -139,7 +139,8 @@ void visit_type_%(name)s(Visitor *m, %(name)s * obj, const char *name, Error **e
                 name=name)
 
 try:
-    opts, args = getopt.gnu_getopt(sys.argv[1:], "p:o:", ["prefix=", "output-dir="])
+    opts, args = getopt.gnu_getopt(sys.argv[1:], "chp:o:",
+                                   ["source", "header", "prefix=", "output-dir="])
 except getopt.GetoptError, err:
     print str(err)
     sys.exit(1)
@@ -149,11 +150,22 @@ prefix = ""
 c_file = 'qapi-visit.c'
 h_file = 'qapi-visit.h'
 
+do_c = False
+do_h = False
+
 for o, a in opts:
     if o in ("-p", "--prefix"):
         prefix = a
     elif o in ("-o", "--output-dir"):
         output_dir = a + "/"
+    elif o in ("-c", "--source"):
+        do_h = True
+    elif o in ("-h", "--header"):
+        do_c = True
+
+if not do_c and not do_h:
+    do_c = True
+    do_h = True
 
 c_file = output_dir + prefix + c_file
 h_file = output_dir + prefix + h_file
@@ -164,8 +176,17 @@ except os.error, e:
     if e.errno != errno.EEXIST:
         raise
 
-fdef = open(c_file, 'w')
-fdecl = open(h_file, 'w')
+def maybe_open(really, name, opt):
+    class Null(object):
+        def write(self, str):
+            pass
+        def read(self):
+            return ''
+    if really:
+        return open(name, opt)
+
+fdef = maybe_open(do_c, c_file, 'w')
+fdecl = maybe_open(do_h, h_file, 'w')
 
 fdef.write(mcgen('''
 /* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT MODIFY */
-- 
1.7.7.1

next             reply	other threads:[~2011-12-27 14:02 UTC|newest]

Thread overview: 2+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2011-12-27 14:02 Avi Kivity [this message]
2011-12-27 16:37 ` [Qemu-devel] [PATCH] Fix qapi code generation wrt parallel build Anthony Liguori

find likely ancestor, descendant, or conflicting patches for this message:
( dfblob:2c03055 dfblob:5958289 dfblob:54d1f5d dfblob:bd7b207
dfblob:267cb49 dfblob:ae644bc dfblob:62de83d dfblob:e9d0584 )
 OR (
bs:"[Qemu-devel] [PATCH] Fix qapi code generation wrt parallel build" )
	(help)

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=1324994536-3955-1-git-send-email-avi@redhat.com \
    --to=avi@redhat.com \
    --cc=anthony@codemonkey.ws \
    --cc=qemu-devel@nongnu.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

Be sure your reply has a Subject: header at the top and a blank line before the message body.

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).