* [PATCH v2 01/38] [DO-NOT-MERGE] qapi: add debugging tools
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-22 23:43 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 02/38] qapi-gen: Separate arg-parsing from generation John Snow
` (37 subsequent siblings)
38 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
This adds some really childishly simple debugging tools. Maybe they're
interesting for someone else, too?
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/debug.py | 78 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 78 insertions(+)
create mode 100644 scripts/qapi/debug.py
diff --git a/scripts/qapi/debug.py b/scripts/qapi/debug.py
new file mode 100644
index 0000000000..bacf5ee180
--- /dev/null
+++ b/scripts/qapi/debug.py
@@ -0,0 +1,78 @@
+"""
+Small debugging facilities for mypy static analysis work.
+(C) 2020 John Snow, for Red Hat, Inc.
+"""
+
+import inspect
+import json
+from typing import Dict, List, Any
+from types import FrameType
+
+
+OBSERVED_TYPES: Dict[str, List[str]] = {}
+
+
+# You have no idea how long it took to find this return type...
+def caller_frame() -> FrameType:
+ """
+ Returns the stack frame of the caller's caller.
+ e.g. foo() -> caller() -> caller_frame() return's foo's stack frame.
+ """
+ stack = inspect.stack()
+ caller = stack[2].frame
+ if caller is None:
+ msg = "Python interpreter does not support stack frame inspection"
+ raise RuntimeError(msg)
+ return caller
+
+
+def _add_type_record(name: str, typestr: str) -> None:
+ seen = OBSERVED_TYPES.setdefault(name, [])
+ if typestr not in seen:
+ seen.append(typestr)
+
+
+def record_type(name: str, value: Any, dict_names: bool = False) -> None:
+ """
+ Record the type of a variable.
+
+ :param name: The name of the variable
+ :param value: The value of the variable
+ """
+ _add_type_record(name, str(type(value)))
+
+ try:
+ for key, subvalue in value.items():
+ subname = f"{name}.{key}" if dict_names else f"{name}.[dict_value]"
+ _add_type_record(subname, str(type(subvalue)))
+ return
+ except AttributeError:
+ # (Wasn't a dict or anything resembling one.)
+ pass
+
+ # str is iterable, but not in the way we want!
+ if isinstance(value, str):
+ return
+
+ try:
+ for elem in value:
+ _add_type_record(f"{name}.[list_elem]", str(type(elem)))
+ except TypeError:
+ # (Wasn't a list or anything else iterable.)
+ pass
+
+
+def show_types() -> None:
+ """
+ Print all of the currently known variable types to stdout.
+ """
+ print(json.dumps(OBSERVED_TYPES, indent=2))
+
+
+def record_locals(show: bool = False, dict_names: bool = False) -> None:
+ caller = caller_frame()
+ name = caller.f_code.co_name
+ for key, value in caller.f_locals.items():
+ record_type(f"{name}.{key}", value, dict_names=dict_names)
+ if show:
+ show_types()
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 01/38] [DO-NOT-MERGE] qapi: add debugging tools
2020-09-22 21:00 ` [PATCH v2 01/38] [DO-NOT-MERGE] qapi: add debugging tools John Snow
@ 2020-09-22 23:43 ` Cleber Rosa
2020-09-23 16:48 ` John Snow
0 siblings, 1 reply; 190+ messages in thread
From: Cleber Rosa @ 2020-09-22 23:43 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 3348 bytes --]
On Tue, Sep 22, 2020 at 05:00:24PM -0400, John Snow wrote:
> This adds some really childishly simple debugging tools. Maybe they're
> interesting for someone else, too?
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/debug.py | 78 +++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 78 insertions(+)
> create mode 100644 scripts/qapi/debug.py
>
> diff --git a/scripts/qapi/debug.py b/scripts/qapi/debug.py
> new file mode 100644
> index 0000000000..bacf5ee180
> --- /dev/null
> +++ b/scripts/qapi/debug.py
> @@ -0,0 +1,78 @@
> +"""
> +Small debugging facilities for mypy static analysis work.
> +(C) 2020 John Snow, for Red Hat, Inc.
> +"""
> +
> +import inspect
> +import json
> +from typing import Dict, List, Any
> +from types import FrameType
> +
> +
> +OBSERVED_TYPES: Dict[str, List[str]] = {}
> +
> +
> +# You have no idea how long it took to find this return type...
> +def caller_frame() -> FrameType:
> + """
> + Returns the stack frame of the caller's caller.
> + e.g. foo() -> caller() -> caller_frame() return's foo's stack frame.
> + """
> + stack = inspect.stack()
> + caller = stack[2].frame
> + if caller is None:
> + msg = "Python interpreter does not support stack frame inspection"
> + raise RuntimeError(msg)
> + return caller
> +
> +
> +def _add_type_record(name: str, typestr: str) -> None:
> + seen = OBSERVED_TYPES.setdefault(name, [])
> + if typestr not in seen:
> + seen.append(typestr)
> +
> +
> +def record_type(name: str, value: Any, dict_names: bool = False) -> None:
> + """
> + Record the type of a variable.
> +
> + :param name: The name of the variable
> + :param value: The value of the variable
> + """
> + _add_type_record(name, str(type(value)))
> +
> + try:
> + for key, subvalue in value.items():
> + subname = f"{name}.{key}" if dict_names else f"{name}.[dict_value]"
> + _add_type_record(subname, str(type(subvalue)))
> + return
> + except AttributeError:
> + # (Wasn't a dict or anything resembling one.)
> + pass
> +
> + # str is iterable, but not in the way we want!
> + if isinstance(value, str):
> + return
> +
> + try:
> + for elem in value:
> + _add_type_record(f"{name}.[list_elem]", str(type(elem)))
> + except TypeError:
> + # (Wasn't a list or anything else iterable.)
> + pass
> +
> +
> +def show_types() -> None:
> + """
> + Print all of the currently known variable types to stdout.
> + """
> + print(json.dumps(OBSERVED_TYPES, indent=2))
> +
Maybe the following will be cheaper (no json conversion):
pprint.pprint(OBSERVED_TYPES, indent=2)
Other than that, I'd vote for including this if there's a bit more
documentation on how to use it, or an example script. Maybe there
already is, and I did not get to it yet.
- Cleber.
> +
> +def record_locals(show: bool = False, dict_names: bool = False) -> None:
> + caller = caller_frame()
> + name = caller.f_code.co_name
> + for key, value in caller.f_locals.items():
> + record_type(f"{name}.{key}", value, dict_names=dict_names)
> + if show:
> + show_types()
> --
> 2.26.2
>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 01/38] [DO-NOT-MERGE] qapi: add debugging tools
2020-09-22 23:43 ` Cleber Rosa
@ 2020-09-23 16:48 ` John Snow
0 siblings, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-23 16:48 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
On 9/22/20 7:43 PM, Cleber Rosa wrote:
> On Tue, Sep 22, 2020 at 05:00:24PM -0400, John Snow wrote:
>> This adds some really childishly simple debugging tools. Maybe they're
>> interesting for someone else, too?
>>
>> Signed-off-by: John Snow <jsnow@redhat.com>
>> ---
>> scripts/qapi/debug.py | 78 +++++++++++++++++++++++++++++++++++++++++++
>> 1 file changed, 78 insertions(+)
>> create mode 100644 scripts/qapi/debug.py
>>
>> diff --git a/scripts/qapi/debug.py b/scripts/qapi/debug.py
>> new file mode 100644
>> index 0000000000..bacf5ee180
>> --- /dev/null
>> +++ b/scripts/qapi/debug.py
>> @@ -0,0 +1,78 @@
>> +"""
>> +Small debugging facilities for mypy static analysis work.
>> +(C) 2020 John Snow, for Red Hat, Inc.
>> +"""
>> +
>> +import inspect
>> +import json
>> +from typing import Dict, List, Any
>> +from types import FrameType
>> +
>> +
>> +OBSERVED_TYPES: Dict[str, List[str]] = {}
>> +
>> +
>> +# You have no idea how long it took to find this return type...
>> +def caller_frame() -> FrameType:
>> + """
>> + Returns the stack frame of the caller's caller.
>> + e.g. foo() -> caller() -> caller_frame() return's foo's stack frame.
>> + """
>> + stack = inspect.stack()
>> + caller = stack[2].frame
>> + if caller is None:
>> + msg = "Python interpreter does not support stack frame inspection"
>> + raise RuntimeError(msg)
>> + return caller
>> +
>> +
>> +def _add_type_record(name: str, typestr: str) -> None:
>> + seen = OBSERVED_TYPES.setdefault(name, [])
>> + if typestr not in seen:
>> + seen.append(typestr)
>> +
>> +
>> +def record_type(name: str, value: Any, dict_names: bool = False) -> None:
>> + """
>> + Record the type of a variable.
>> +
>> + :param name: The name of the variable
>> + :param value: The value of the variable
>> + """
>> + _add_type_record(name, str(type(value)))
>> +
>> + try:
>> + for key, subvalue in value.items():
>> + subname = f"{name}.{key}" if dict_names else f"{name}.[dict_value]"
>> + _add_type_record(subname, str(type(subvalue)))
>> + return
>> + except AttributeError:
>> + # (Wasn't a dict or anything resembling one.)
>> + pass
>> +
>> + # str is iterable, but not in the way we want!
>> + if isinstance(value, str):
>> + return
>> +
>> + try:
>> + for elem in value:
>> + _add_type_record(f"{name}.[list_elem]", str(type(elem)))
>> + except TypeError:
>> + # (Wasn't a list or anything else iterable.)
>> + pass
>> +
>> +
>> +def show_types() -> None:
>> + """
>> + Print all of the currently known variable types to stdout.
>> + """
>> + print(json.dumps(OBSERVED_TYPES, indent=2))
>> +
>
> Maybe the following will be cheaper (no json conversion):
>
> pprint.pprint(OBSERVED_TYPES, indent=2)
>
> Other than that, I'd vote for including this if there's a bit more
> documentation on how to use it, or an example script. Maybe there
> already is, and I did not get to it yet.
>
> - Cleber.
>
Nope, this is just a dumb script I did to observe types in flight.
There are apparently bigger, beefier tools that I don't know how to use
yet: https://github.com/dropbox/pyannotate
I just included my own little tool as a reference thing to be archived
on list, I have no desire to spruce it up. I'd rather spend my time
learning pyannotate.
--js
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 02/38] qapi-gen: Separate arg-parsing from generation
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
2020-09-22 21:00 ` [PATCH v2 01/38] [DO-NOT-MERGE] qapi: add debugging tools John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-22 21:19 ` Eduardo Habkost
2020-09-23 0:00 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 03/38] qapi: move generator entrypoint into module John Snow
` (36 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
This is a minor re-work of the entrypoint script. It isolates a
generate() method from the actual command-line mechanism.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi-gen.py | 87 ++++++++++++++++++++++++++++++++-------------
1 file changed, 63 insertions(+), 24 deletions(-)
diff --git a/scripts/qapi-gen.py b/scripts/qapi-gen.py
index 4b03f7d53b..59becba3e1 100644
--- a/scripts/qapi-gen.py
+++ b/scripts/qapi-gen.py
@@ -1,9 +1,13 @@
#!/usr/bin/env python3
-# QAPI generator
-#
+
# This work is licensed under the terms of the GNU GPL, version 2 or later.
# See the COPYING file in the top-level directory.
+"""
+QAPI Generator
+
+This script is the main entry point for generating C code from the QAPI schema.
+"""
import argparse
import re
@@ -11,21 +15,65 @@
from qapi.commands import gen_commands
from qapi.doc import gen_doc
+from qapi.error import QAPIError
from qapi.events import gen_events
from qapi.introspect import gen_introspect
-from qapi.schema import QAPIError, QAPISchema
+from qapi.schema import QAPISchema
from qapi.types import gen_types
from qapi.visit import gen_visit
-def main(argv):
+DEFAULT_OUTPUT_DIR = ''
+DEFAULT_PREFIX = ''
+
+
+def generate(schema_file: str,
+ output_dir: str,
+ prefix: str,
+ unmask: bool = False,
+ builtins: bool = False) -> None:
+ """
+ generate uses a given schema to produce C code in the target directory.
+
+ :param schema_file: The primary QAPI schema file.
+ :param output_dir: The output directory to store generated code.
+ :param prefix: Optional C-code prefix for symbol names.
+ :param unmask: Expose non-ABI names through introspection?
+ :param builtins: Generate code for built-in types?
+
+ :raise QAPIError: On failures.
+ """
+ match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', prefix)
+ if match and match.end() != len(prefix):
+ msg = "funny character '{:s}' in prefix '{:s}'".format(
+ prefix[match.end()], prefix)
+ raise QAPIError('', None, msg)
+
+ schema = QAPISchema(schema_file)
+ gen_types(schema, output_dir, prefix, builtins)
+ gen_visit(schema, output_dir, prefix, builtins)
+ gen_commands(schema, output_dir, prefix)
+ gen_events(schema, output_dir, prefix)
+ gen_introspect(schema, output_dir, prefix, unmask)
+ gen_doc(schema, output_dir, prefix)
+
+
+def main() -> int:
+ """
+ gapi-gen shell script entrypoint.
+ Expects arguments via sys.argv, see --help for details.
+
+ :return: int, 0 on success, 1 on failure.
+ """
parser = argparse.ArgumentParser(
description='Generate code from a QAPI schema')
parser.add_argument('-b', '--builtins', action='store_true',
help="generate code for built-in types")
- parser.add_argument('-o', '--output-dir', action='store', default='',
+ parser.add_argument('-o', '--output-dir', action='store',
+ default=DEFAULT_OUTPUT_DIR,
help="write output to directory OUTPUT_DIR")
- parser.add_argument('-p', '--prefix', action='store', default='',
+ parser.add_argument('-p', '--prefix', action='store',
+ default=DEFAULT_PREFIX,
help="prefix for symbols")
parser.add_argument('-u', '--unmask-non-abi-names', action='store_true',
dest='unmask',
@@ -33,26 +81,17 @@ def main(argv):
parser.add_argument('schema', action='store')
args = parser.parse_args()
- match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', args.prefix)
- if match.end() != len(args.prefix):
- print("%s: 'funny character '%s' in argument of --prefix"
- % (sys.argv[0], args.prefix[match.end()]),
- file=sys.stderr)
- sys.exit(1)
-
try:
- schema = QAPISchema(args.schema)
+ generate(args.schema,
+ output_dir=args.output_dir,
+ prefix=args.prefix,
+ unmask=args.unmask,
+ builtins=args.builtins)
except QAPIError as err:
- print(err, file=sys.stderr)
- exit(1)
-
- gen_types(schema, args.output_dir, args.prefix, args.builtins)
- gen_visit(schema, args.output_dir, args.prefix, args.builtins)
- gen_commands(schema, args.output_dir, args.prefix)
- gen_events(schema, args.output_dir, args.prefix)
- gen_introspect(schema, args.output_dir, args.prefix, args.unmask)
- gen_doc(schema, args.output_dir, args.prefix)
+ print(f"{sys.argv[0]}: {str(err)}", file=sys.stderr)
+ return 1
+ return 0
if __name__ == '__main__':
- main(sys.argv)
+ sys.exit(main())
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 02/38] qapi-gen: Separate arg-parsing from generation
2020-09-22 21:00 ` [PATCH v2 02/38] qapi-gen: Separate arg-parsing from generation John Snow
@ 2020-09-22 21:19 ` Eduardo Habkost
2020-09-23 0:00 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-22 21:19 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:25PM -0400, John Snow wrote:
> This is a minor re-work of the entrypoint script. It isolates a
> generate() method from the actual command-line mechanism.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 02/38] qapi-gen: Separate arg-parsing from generation
2020-09-22 21:00 ` [PATCH v2 02/38] qapi-gen: Separate arg-parsing from generation John Snow
2020-09-22 21:19 ` Eduardo Habkost
@ 2020-09-23 0:00 ` Cleber Rosa
2020-09-23 17:05 ` John Snow
2020-09-25 11:34 ` Markus Armbruster
1 sibling, 2 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 0:00 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 5682 bytes --]
On Tue, Sep 22, 2020 at 05:00:25PM -0400, John Snow wrote:
> This is a minor re-work of the entrypoint script. It isolates a
> generate() method from the actual command-line mechanism.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi-gen.py | 87 ++++++++++++++++++++++++++++++++-------------
> 1 file changed, 63 insertions(+), 24 deletions(-)
>
> diff --git a/scripts/qapi-gen.py b/scripts/qapi-gen.py
> index 4b03f7d53b..59becba3e1 100644
> --- a/scripts/qapi-gen.py
> +++ b/scripts/qapi-gen.py
> @@ -1,9 +1,13 @@
> #!/usr/bin/env python3
> -# QAPI generator
> -#
> +
> # This work is licensed under the terms of the GNU GPL, version 2 or later.
> # See the COPYING file in the top-level directory.
>
> +"""
> +QAPI Generator
> +
> +This script is the main entry point for generating C code from the QAPI schema.
> +"""
>
> import argparse
> import re
> @@ -11,21 +15,65 @@
>
> from qapi.commands import gen_commands
> from qapi.doc import gen_doc
> +from qapi.error import QAPIError
> from qapi.events import gen_events
> from qapi.introspect import gen_introspect
> -from qapi.schema import QAPIError, QAPISchema
> +from qapi.schema import QAPISchema
> from qapi.types import gen_types
> from qapi.visit import gen_visit
>
>
> -def main(argv):
> +DEFAULT_OUTPUT_DIR = ''
> +DEFAULT_PREFIX = ''
I did not understand the purpose of these. If they're used only as
the default value for the command line option parsing, I'd suggest
dropping them.
> +
> +
> +def generate(schema_file: str,
> + output_dir: str,
> + prefix: str,
> + unmask: bool = False,
> + builtins: bool = False) -> None:
> + """
> + generate uses a given schema to produce C code in the target directory.
> +
> + :param schema_file: The primary QAPI schema file.
> + :param output_dir: The output directory to store generated code.
> + :param prefix: Optional C-code prefix for symbol names.
> + :param unmask: Expose non-ABI names through introspection?
> + :param builtins: Generate code for built-in types?
> +
> + :raise QAPIError: On failures.
> + """
> + match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', prefix)
> + if match and match.end() != len(prefix):
Nice catch with the extra check here. Maybe worth mentioning and/or
splitting the change?
> + msg = "funny character '{:s}' in prefix '{:s}'".format(
> + prefix[match.end()], prefix)
> + raise QAPIError('', None, msg)
> +
> + schema = QAPISchema(schema_file)
> + gen_types(schema, output_dir, prefix, builtins)
> + gen_visit(schema, output_dir, prefix, builtins)
> + gen_commands(schema, output_dir, prefix)
> + gen_events(schema, output_dir, prefix)
> + gen_introspect(schema, output_dir, prefix, unmask)
> + gen_doc(schema, output_dir, prefix)
> +
> +
> +def main() -> int:
One extra Pythonic touch would be to use a bool here, and then:
sys.exit(0 if main() else 1)
But that's probably overkill.
> + """
> + gapi-gen shell script entrypoint.
> + Expects arguments via sys.argv, see --help for details.
> +
> + :return: int, 0 on success, 1 on failure.
> + """
> parser = argparse.ArgumentParser(
> description='Generate code from a QAPI schema')
> parser.add_argument('-b', '--builtins', action='store_true',
> help="generate code for built-in types")
> - parser.add_argument('-o', '--output-dir', action='store', default='',
> + parser.add_argument('-o', '--output-dir', action='store',
> + default=DEFAULT_OUTPUT_DIR,
> help="write output to directory OUTPUT_DIR")
> - parser.add_argument('-p', '--prefix', action='store', default='',
> + parser.add_argument('-p', '--prefix', action='store',
> + default=DEFAULT_PREFIX,
> help="prefix for symbols")
> parser.add_argument('-u', '--unmask-non-abi-names', action='store_true',
> dest='unmask',
> @@ -33,26 +81,17 @@ def main(argv):
> parser.add_argument('schema', action='store')
> args = parser.parse_args()
>
> - match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', args.prefix)
> - if match.end() != len(args.prefix):
> - print("%s: 'funny character '%s' in argument of --prefix"
> - % (sys.argv[0], args.prefix[match.end()]),
> - file=sys.stderr)
> - sys.exit(1)
> -
> try:
> - schema = QAPISchema(args.schema)
> + generate(args.schema,
> + output_dir=args.output_dir,
> + prefix=args.prefix,
> + unmask=args.unmask,
> + builtins=args.builtins)
> except QAPIError as err:
> - print(err, file=sys.stderr)
> - exit(1)
> -
Glad to see that this "quitter" is gone in favor of one and only
sys.exit().
- Cleber.
> - gen_types(schema, args.output_dir, args.prefix, args.builtins)
> - gen_visit(schema, args.output_dir, args.prefix, args.builtins)
> - gen_commands(schema, args.output_dir, args.prefix)
> - gen_events(schema, args.output_dir, args.prefix)
> - gen_introspect(schema, args.output_dir, args.prefix, args.unmask)
> - gen_doc(schema, args.output_dir, args.prefix)
> + print(f"{sys.argv[0]}: {str(err)}", file=sys.stderr)
> + return 1
> + return 0
>
>
> if __name__ == '__main__':
> - main(sys.argv)
> + sys.exit(main())
> --
> 2.26.2
>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 02/38] qapi-gen: Separate arg-parsing from generation
2020-09-23 0:00 ` Cleber Rosa
@ 2020-09-23 17:05 ` John Snow
2020-09-24 19:24 ` Cleber Rosa
2020-09-25 11:34 ` Markus Armbruster
1 sibling, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-23 17:05 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
On 9/22/20 8:00 PM, Cleber Rosa wrote:
> On Tue, Sep 22, 2020 at 05:00:25PM -0400, John Snow wrote:
>> This is a minor re-work of the entrypoint script. It isolates a
>> generate() method from the actual command-line mechanism.
>>
>> Signed-off-by: John Snow <jsnow@redhat.com>
>> ---
>> scripts/qapi-gen.py | 87 ++++++++++++++++++++++++++++++++-------------
>> 1 file changed, 63 insertions(+), 24 deletions(-)
>>
>> diff --git a/scripts/qapi-gen.py b/scripts/qapi-gen.py
>> index 4b03f7d53b..59becba3e1 100644
>> --- a/scripts/qapi-gen.py
>> +++ b/scripts/qapi-gen.py
>> @@ -1,9 +1,13 @@
>> #!/usr/bin/env python3
>> -# QAPI generator
>> -#
>> +
>> # This work is licensed under the terms of the GNU GPL, version 2 or later.
>> # See the COPYING file in the top-level directory.
>>
>> +"""
>> +QAPI Generator
>> +
>> +This script is the main entry point for generating C code from the QAPI schema.
>> +"""
>>
>> import argparse
>> import re
>> @@ -11,21 +15,65 @@
>>
>> from qapi.commands import gen_commands
>> from qapi.doc import gen_doc
>> +from qapi.error import QAPIError
>> from qapi.events import gen_events
>> from qapi.introspect import gen_introspect
>> -from qapi.schema import QAPIError, QAPISchema
>> +from qapi.schema import QAPISchema
>> from qapi.types import gen_types
>> from qapi.visit import gen_visit
>>
>>
>> -def main(argv):
>> +DEFAULT_OUTPUT_DIR = ''
>> +DEFAULT_PREFIX = ''
>
> I did not understand the purpose of these. If they're used only as
> the default value for the command line option parsing, I'd suggest
> dropping them.
>
The alternative is setting default='' inline below, which is fine, but
found them kind of buried; and looking a bit too much like a weird magic
constant. Aesthetically, I liked making them obvious.
You found them! My plan worked.
>> +
>> +
>> +def generate(schema_file: str,
>> + output_dir: str,
>> + prefix: str,
>> + unmask: bool = False,
>> + builtins: bool = False) -> None:
>> + """
>> + generate uses a given schema to produce C code in the target directory.
>> +
>> + :param schema_file: The primary QAPI schema file.
>> + :param output_dir: The output directory to store generated code.
>> + :param prefix: Optional C-code prefix for symbol names.
>> + :param unmask: Expose non-ABI names through introspection?
>> + :param builtins: Generate code for built-in types?
>> +
>> + :raise QAPIError: On failures.
>> + """
>> + match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', prefix)
>> + if match and match.end() != len(prefix):
>
> Nice catch with the extra check here. Maybe worth mentioning and/or
> splitting the change?
>
If you review all 125 patches, I will do this for you ;)
>> + msg = "funny character '{:s}' in prefix '{:s}'".format(
>> + prefix[match.end()], prefix)
>> + raise QAPIError('', None, msg)
>> +
>> + schema = QAPISchema(schema_file)
>> + gen_types(schema, output_dir, prefix, builtins)
>> + gen_visit(schema, output_dir, prefix, builtins)
>> + gen_commands(schema, output_dir, prefix)
>> + gen_events(schema, output_dir, prefix)
>> + gen_introspect(schema, output_dir, prefix, unmask)
>> + gen_doc(schema, output_dir, prefix)
>> +
>> +
>> +def main() -> int:
>
> One extra Pythonic touch would be to use a bool here, and then:
>
> sys.exit(0 if main() else 1)
>
> But that's probably overkill.
>
I think a function named main() is fair enough to return int -- we are
declaring that this is a shell script pretty explicitly, and it is
allowed to return a status code.
Shifting the knowledge of how shell codes work up one more layer is
probably not ... urgent.
>> + """
>> + gapi-gen shell script entrypoint.
>> + Expects arguments via sys.argv, see --help for details.
>> +
>> + :return: int, 0 on success, 1 on failure.
>> + """
>> parser = argparse.ArgumentParser(
>> description='Generate code from a QAPI schema')
>> parser.add_argument('-b', '--builtins', action='store_true',
>> help="generate code for built-in types")
>> - parser.add_argument('-o', '--output-dir', action='store', default='',
>> + parser.add_argument('-o', '--output-dir', action='store',
>> + default=DEFAULT_OUTPUT_DIR,
>> help="write output to directory OUTPUT_DIR")
>> - parser.add_argument('-p', '--prefix', action='store', default='',
>> + parser.add_argument('-p', '--prefix', action='store',
>> + default=DEFAULT_PREFIX,
>> help="prefix for symbols")
>> parser.add_argument('-u', '--unmask-non-abi-names', action='store_true',
>> dest='unmask',
>> @@ -33,26 +81,17 @@ def main(argv):
>> parser.add_argument('schema', action='store')
>> args = parser.parse_args()
>>
>> - match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', args.prefix)
>> - if match.end() != len(args.prefix):
>> - print("%s: 'funny character '%s' in argument of --prefix"
>> - % (sys.argv[0], args.prefix[match.end()]),
>> - file=sys.stderr)
>> - sys.exit(1)
>> -
>> try:
>> - schema = QAPISchema(args.schema)
>> + generate(args.schema,
>> + output_dir=args.output_dir,
>> + prefix=args.prefix,
>> + unmask=args.unmask,
>> + builtins=args.builtins)
>> except QAPIError as err:
>> - print(err, file=sys.stderr)
>> - exit(1)
>> -
>
> Glad to see that this "quitter" is gone in favor of one and only
> sys.exit().
>
> - Cleber.
>
>> - gen_types(schema, args.output_dir, args.prefix, args.builtins)
>> - gen_visit(schema, args.output_dir, args.prefix, args.builtins)
>> - gen_commands(schema, args.output_dir, args.prefix)
>> - gen_events(schema, args.output_dir, args.prefix)
>> - gen_introspect(schema, args.output_dir, args.prefix, args.unmask)
>> - gen_doc(schema, args.output_dir, args.prefix)
>> + print(f"{sys.argv[0]}: {str(err)}", file=sys.stderr)
>> + return 1
>> + return 0
>>
>>
>> if __name__ == '__main__':
>> - main(sys.argv)
>> + sys.exit(main())
>> --
>> 2.26.2
>>
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 02/38] qapi-gen: Separate arg-parsing from generation
2020-09-23 17:05 ` John Snow
@ 2020-09-24 19:24 ` Cleber Rosa
0 siblings, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-24 19:24 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 2372 bytes --]
On Wed, Sep 23, 2020 at 01:05:47PM -0400, John Snow wrote:
> On 9/22/20 8:00 PM, Cleber Rosa wrote:
> > On Tue, Sep 22, 2020 at 05:00:25PM -0400, John Snow wrote:
> > > This is a minor re-work of the entrypoint script. It isolates a
> > > generate() method from the actual command-line mechanism.
> > >
> > > Signed-off-by: John Snow <jsnow@redhat.com>
> > > ---
> > > scripts/qapi-gen.py | 87 ++++++++++++++++++++++++++++++++-------------
> > > 1 file changed, 63 insertions(+), 24 deletions(-)
> > >
> > > diff --git a/scripts/qapi-gen.py b/scripts/qapi-gen.py
> > > index 4b03f7d53b..59becba3e1 100644
> > > --- a/scripts/qapi-gen.py
> > > +++ b/scripts/qapi-gen.py
> > > @@ -1,9 +1,13 @@
> > > #!/usr/bin/env python3
> > > -# QAPI generator
> > > -#
> > > +
> > > # This work is licensed under the terms of the GNU GPL, version 2 or later.
> > > # See the COPYING file in the top-level directory.
> > > +"""
> > > +QAPI Generator
> > > +
> > > +This script is the main entry point for generating C code from the QAPI schema.
> > > +"""
> > > import argparse
> > > import re
> > > @@ -11,21 +15,65 @@
> > > from qapi.commands import gen_commands
> > > from qapi.doc import gen_doc
> > > +from qapi.error import QAPIError
> > > from qapi.events import gen_events
> > > from qapi.introspect import gen_introspect
> > > -from qapi.schema import QAPIError, QAPISchema
> > > +from qapi.schema import QAPISchema
> > > from qapi.types import gen_types
> > > from qapi.visit import gen_visit
> > > -def main(argv):
> > > +DEFAULT_OUTPUT_DIR = ''
> > > +DEFAULT_PREFIX = ''
> >
> > I did not understand the purpose of these. If they're used only as
> > the default value for the command line option parsing, I'd suggest
> > dropping them.
> >
>
> The alternative is setting default='' inline below, which is fine, but found
> them kind of buried; and looking a bit too much like a weird magic constant.
> Aesthetically, I liked making them obvious.
>
I differ in opinion and style, and I don't think you should adopt
mine. Just for the record, though, consider if would do the same for
dozens of command line options... IMO it'd be fine to declare them for
extra explicitness, but I see no reason for them to be module globals.
> You found them! My plan worked.
>
:)
- Cleber.
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 02/38] qapi-gen: Separate arg-parsing from generation
2020-09-23 0:00 ` Cleber Rosa
2020-09-23 17:05 ` John Snow
@ 2020-09-25 11:34 ` Markus Armbruster
2020-09-25 15:37 ` John Snow
1 sibling, 1 reply; 190+ messages in thread
From: Markus Armbruster @ 2020-09-25 11:34 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, John Snow, Michael Roth,
qemu-devel, Alex Bennée
Cleber Rosa <crosa@redhat.com> writes:
> On Tue, Sep 22, 2020 at 05:00:25PM -0400, John Snow wrote:
>> This is a minor re-work of the entrypoint script. It isolates a
>> generate() method from the actual command-line mechanism.
>>
>> Signed-off-by: John Snow <jsnow@redhat.com>
>> ---
>> scripts/qapi-gen.py | 87 ++++++++++++++++++++++++++++++++-------------
>> 1 file changed, 63 insertions(+), 24 deletions(-)
>>
>> diff --git a/scripts/qapi-gen.py b/scripts/qapi-gen.py
>> index 4b03f7d53b..59becba3e1 100644
>> --- a/scripts/qapi-gen.py
>> +++ b/scripts/qapi-gen.py
>> @@ -1,9 +1,13 @@
>> #!/usr/bin/env python3
>> -# QAPI generator
>> -#
>> +
>> # This work is licensed under the terms of the GNU GPL, version 2 or later.
>> # See the COPYING file in the top-level directory.
>>
>> +"""
>> +QAPI Generator
>> +
>> +This script is the main entry point for generating C code from the QAPI schema.
>> +"""
>>
>> import argparse
>> import re
>> @@ -11,21 +15,65 @@
>>
>> from qapi.commands import gen_commands
>> from qapi.doc import gen_doc
>> +from qapi.error import QAPIError
>> from qapi.events import gen_events
>> from qapi.introspect import gen_introspect
>> -from qapi.schema import QAPIError, QAPISchema
>> +from qapi.schema import QAPISchema
>> from qapi.types import gen_types
>> from qapi.visit import gen_visit
>>
>>
>> -def main(argv):
>> +DEFAULT_OUTPUT_DIR = ''
>> +DEFAULT_PREFIX = ''
>
> I did not understand the purpose of these. If they're used only as
> the default value for the command line option parsing, I'd suggest
> dropping them.
>
>> +
>> +
>> +def generate(schema_file: str,
>> + output_dir: str,
>> + prefix: str,
>> + unmask: bool = False,
>> + builtins: bool = False) -> None:
>> + """
>> + generate uses a given schema to produce C code in the target directory.
>> +
>> + :param schema_file: The primary QAPI schema file.
>> + :param output_dir: The output directory to store generated code.
>> + :param prefix: Optional C-code prefix for symbol names.
>> + :param unmask: Expose non-ABI names through introspection?
>> + :param builtins: Generate code for built-in types?
>> +
>> + :raise QAPIError: On failures.
>> + """
>> + match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', prefix)
>> + if match and match.end() != len(prefix):
>
> Nice catch with the extra check here. Maybe worth mentioning and/or
> splitting the change?
Please do not sneak additional checking into patches advertized as pure
refactoring. It makes me look for more sneakery with a microscope.
This re.match() cannot possibly fail. Three cases:
* First character is funny
The regexp matches the empty string. There's a reason the regexp ends
with '?'.
* Non-first character is funny
The regexp matches the non-funny prefix.
* No character is funny
The regexp matches the complete string.
Checking impossible conditions as if they were possible is confusing.
Please drop the additional check.
We can talk about checking this impossible condition with
assert(match)
if you believe it makes the code easier to understand (it does not
improve its behavior).
>> + msg = "funny character '{:s}' in prefix '{:s}'".format(
>> + prefix[match.end()], prefix)
>> + raise QAPIError('', None, msg)
>> +
>> + schema = QAPISchema(schema_file)
>> + gen_types(schema, output_dir, prefix, builtins)
>> + gen_visit(schema, output_dir, prefix, builtins)
>> + gen_commands(schema, output_dir, prefix)
>> + gen_events(schema, output_dir, prefix)
>> + gen_introspect(schema, output_dir, prefix, unmask)
>> + gen_doc(schema, output_dir, prefix)
>> +
>> +
>> +def main() -> int:
>
> One extra Pythonic touch would be to use a bool here, and then:
>
> sys.exit(0 if main() else 1)
>
> But that's probably overkill.
Yuck :)
>> + """
>> + gapi-gen shell script entrypoint.
>> + Expects arguments via sys.argv, see --help for details.
>> +
>> + :return: int, 0 on success, 1 on failure.
>> + """
>> parser = argparse.ArgumentParser(
>> description='Generate code from a QAPI schema')
>> parser.add_argument('-b', '--builtins', action='store_true',
>> help="generate code for built-in types")
>> - parser.add_argument('-o', '--output-dir', action='store', default='',
>> + parser.add_argument('-o', '--output-dir', action='store',
>> + default=DEFAULT_OUTPUT_DIR,
>> help="write output to directory OUTPUT_DIR")
>> - parser.add_argument('-p', '--prefix', action='store', default='',
>> + parser.add_argument('-p', '--prefix', action='store',
>> + default=DEFAULT_PREFIX,
>> help="prefix for symbols")
>> parser.add_argument('-u', '--unmask-non-abi-names', action='store_true',
>> dest='unmask',
>> @@ -33,26 +81,17 @@ def main(argv):
>> parser.add_argument('schema', action='store')
>> args = parser.parse_args()
>>
>> - match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', args.prefix)
>> - if match.end() != len(args.prefix):
>> - print("%s: 'funny character '%s' in argument of --prefix"
>> - % (sys.argv[0], args.prefix[match.end()]),
>> - file=sys.stderr)
>> - sys.exit(1)
>> -
>> try:
>> - schema = QAPISchema(args.schema)
>> + generate(args.schema,
>> + output_dir=args.output_dir,
>> + prefix=args.prefix,
>> + unmask=args.unmask,
>> + builtins=args.builtins)
>> except QAPIError as err:
>> - print(err, file=sys.stderr)
>> - exit(1)
>> -
>
> Glad to see that this "quitter" is gone in favor of one and only
> sys.exit().
I'm totally fine with having multiple sys.exit() in main(). They become
icky only when buried too deeply.
I'm fine with John's patch regardless.
[...]
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 02/38] qapi-gen: Separate arg-parsing from generation
2020-09-25 11:34 ` Markus Armbruster
@ 2020-09-25 15:37 ` John Snow
2020-09-28 11:45 ` Markus Armbruster
0 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-25 15:37 UTC (permalink / raw)
To: Markus Armbruster, Cleber Rosa
Cc: qemu-devel, Peter Maydell, Alex Bennée, Eduardo Habkost,
Michael Roth
On 9/25/20 7:34 AM, Markus Armbruster wrote:
> Cleber Rosa <crosa@redhat.com> writes:
>
>> On Tue, Sep 22, 2020 at 05:00:25PM -0400, John Snow wrote:
>>> This is a minor re-work of the entrypoint script. It isolates a
>>> generate() method from the actual command-line mechanism.
>>>
>>> Signed-off-by: John Snow <jsnow@redhat.com>
>>> ---
>>> scripts/qapi-gen.py | 87 ++++++++++++++++++++++++++++++++-------------
>>> 1 file changed, 63 insertions(+), 24 deletions(-)
>>>
>>> diff --git a/scripts/qapi-gen.py b/scripts/qapi-gen.py
>>> index 4b03f7d53b..59becba3e1 100644
>>> --- a/scripts/qapi-gen.py
>>> +++ b/scripts/qapi-gen.py
>>> @@ -1,9 +1,13 @@
>>> #!/usr/bin/env python3
>>> -# QAPI generator
>>> -#
>>> +
>>> # This work is licensed under the terms of the GNU GPL, version 2 or later.
>>> # See the COPYING file in the top-level directory.
>>>
>>> +"""
>>> +QAPI Generator
>>> +
>>> +This script is the main entry point for generating C code from the QAPI schema.
>>> +"""
>>>
>>> import argparse
>>> import re
>>> @@ -11,21 +15,65 @@
>>>
>>> from qapi.commands import gen_commands
>>> from qapi.doc import gen_doc
>>> +from qapi.error import QAPIError
>>> from qapi.events import gen_events
>>> from qapi.introspect import gen_introspect
>>> -from qapi.schema import QAPIError, QAPISchema
>>> +from qapi.schema import QAPISchema
>>> from qapi.types import gen_types
>>> from qapi.visit import gen_visit
>>>
>>>
>>> -def main(argv):
>>> +DEFAULT_OUTPUT_DIR = ''
>>> +DEFAULT_PREFIX = ''
>>
>> I did not understand the purpose of these. If they're used only as
>> the default value for the command line option parsing, I'd suggest
>> dropping them.
>>
>>> +
>>> +
>>> +def generate(schema_file: str,
>>> + output_dir: str,
>>> + prefix: str,
>>> + unmask: bool = False,
>>> + builtins: bool = False) -> None:
>>> + """
>>> + generate uses a given schema to produce C code in the target directory.
>>> +
>>> + :param schema_file: The primary QAPI schema file.
>>> + :param output_dir: The output directory to store generated code.
>>> + :param prefix: Optional C-code prefix for symbol names.
>>> + :param unmask: Expose non-ABI names through introspection?
>>> + :param builtins: Generate code for built-in types?
>>> +
>>> + :raise QAPIError: On failures.
>>> + """
>>> + match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', prefix)
>>> + if match and match.end() != len(prefix):
>>
>> Nice catch with the extra check here. Maybe worth mentioning and/or
>> splitting the change?
>
> Please do not sneak additional checking into patches advertized as pure
> refactoring. It makes me look for more sneakery with a microscope.
>
> This re.match() cannot possibly fail. Three cases:
>
> * First character is funny
>
> The regexp matches the empty string. There's a reason the regexp ends
> with '?'.
>
> * Non-first character is funny
>
> The regexp matches the non-funny prefix.
>
> * No character is funny
>
> The regexp matches the complete string.
>
> Checking impossible conditions as if they were possible is confusing.
> Please drop the additional check.
>
> We can talk about checking this impossible condition with
>
> assert(match)
>
> if you believe it makes the code easier to understand (it does not
> improve its behavior).
>
My use of strict_optional=False is what prevents this from exhibiting as
an error in mypy. An assert will help convince mypy that 'match' cannot
possibly be 'None'.
eh, well. I will fix this when I remove strict_optional, so I will just
remove this additional check for now to avoid adding another patch to
this series.
--js
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 02/38] qapi-gen: Separate arg-parsing from generation
2020-09-25 15:37 ` John Snow
@ 2020-09-28 11:45 ` Markus Armbruster
0 siblings, 0 replies; 190+ messages in thread
From: Markus Armbruster @ 2020-09-28 11:45 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, Michael Roth, qemu-devel,
Cleber Rosa, Alex Bennée
John Snow <jsnow@redhat.com> writes:
> On 9/25/20 7:34 AM, Markus Armbruster wrote:
>> Cleber Rosa <crosa@redhat.com> writes:
>>
>>> On Tue, Sep 22, 2020 at 05:00:25PM -0400, John Snow wrote:
>>>> This is a minor re-work of the entrypoint script. It isolates a
>>>> generate() method from the actual command-line mechanism.
>>>>
>>>> Signed-off-by: John Snow <jsnow@redhat.com>
[...]
>>>> +def generate(schema_file: str,
>>>> + output_dir: str,
>>>> + prefix: str,
>>>> + unmask: bool = False,
>>>> + builtins: bool = False) -> None:
>>>> + """
>>>> + generate uses a given schema to produce C code in the target directory.
>>>> +
>>>> + :param schema_file: The primary QAPI schema file.
>>>> + :param output_dir: The output directory to store generated code.
>>>> + :param prefix: Optional C-code prefix for symbol names.
>>>> + :param unmask: Expose non-ABI names through introspection?
>>>> + :param builtins: Generate code for built-in types?
>>>> +
>>>> + :raise QAPIError: On failures.
>>>> + """
>>>> + match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', prefix)
>>>> + if match and match.end() != len(prefix):
>>>
>>> Nice catch with the extra check here. Maybe worth mentioning and/or
>>> splitting the change?
>>
>> Please do not sneak additional checking into patches advertized as pure
>> refactoring. It makes me look for more sneakery with a microscope.
>>
>> This re.match() cannot possibly fail. Three cases:
>>
>> * First character is funny
>>
>> The regexp matches the empty string. There's a reason the regexp ends
>> with '?'.
>>
>> * Non-first character is funny
>>
>> The regexp matches the non-funny prefix.
>>
>> * No character is funny
>>
>> The regexp matches the complete string.
>>
>> Checking impossible conditions as if they were possible is confusing.
>> Please drop the additional check.
>>
>> We can talk about checking this impossible condition with
>>
>> assert(match)
>>
>> if you believe it makes the code easier to understand (it does not
>> improve its behavior).
>
> My use of strict_optional=False is what prevents this from exhibiting
> as an error in mypy. An assert will help convince mypy that 'match'
> cannot possibly be 'None'.
Adding assertions to help mypy along is okay.
> eh, well. I will fix this when I remove strict_optional, so I will
> just remove this additional check for now to avoid adding another
> patch to this series.
Makes sense to me.
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 03/38] qapi: move generator entrypoint into module
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
2020-09-22 21:00 ` [PATCH v2 01/38] [DO-NOT-MERGE] qapi: add debugging tools John Snow
2020-09-22 21:00 ` [PATCH v2 02/38] qapi-gen: Separate arg-parsing from generation John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-22 21:23 ` Eduardo Habkost
2020-09-23 0:29 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 04/38] qapi: Prefer explicit relative imports John Snow
` (35 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
As part of delinting and adding type hints to the QAPI generator, it's
helpful for the entrypoint to be part of the package, only leaving a
very tiny entrypoint shim outside of the module.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi-gen.py | 90 +++----------------------------------------
scripts/qapi/main.py | 91 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 97 insertions(+), 84 deletions(-)
create mode 100644 scripts/qapi/main.py
diff --git a/scripts/qapi-gen.py b/scripts/qapi-gen.py
index 59becba3e1..f3518d29a5 100644
--- a/scripts/qapi-gen.py
+++ b/scripts/qapi-gen.py
@@ -4,94 +4,16 @@
# See the COPYING file in the top-level directory.
"""
-QAPI Generator
+QAPI code generation execution shim.
-This script is the main entry point for generating C code from the QAPI schema.
+This standalone script exists primarily to facilitate the running of the QAPI
+code generator without needing to install the python module to the current
+execution environment.
"""
-import argparse
-import re
import sys
-from qapi.commands import gen_commands
-from qapi.doc import gen_doc
-from qapi.error import QAPIError
-from qapi.events import gen_events
-from qapi.introspect import gen_introspect
-from qapi.schema import QAPISchema
-from qapi.types import gen_types
-from qapi.visit import gen_visit
-
-
-DEFAULT_OUTPUT_DIR = ''
-DEFAULT_PREFIX = ''
-
-
-def generate(schema_file: str,
- output_dir: str,
- prefix: str,
- unmask: bool = False,
- builtins: bool = False) -> None:
- """
- generate uses a given schema to produce C code in the target directory.
-
- :param schema_file: The primary QAPI schema file.
- :param output_dir: The output directory to store generated code.
- :param prefix: Optional C-code prefix for symbol names.
- :param unmask: Expose non-ABI names through introspection?
- :param builtins: Generate code for built-in types?
-
- :raise QAPIError: On failures.
- """
- match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', prefix)
- if match and match.end() != len(prefix):
- msg = "funny character '{:s}' in prefix '{:s}'".format(
- prefix[match.end()], prefix)
- raise QAPIError('', None, msg)
-
- schema = QAPISchema(schema_file)
- gen_types(schema, output_dir, prefix, builtins)
- gen_visit(schema, output_dir, prefix, builtins)
- gen_commands(schema, output_dir, prefix)
- gen_events(schema, output_dir, prefix)
- gen_introspect(schema, output_dir, prefix, unmask)
- gen_doc(schema, output_dir, prefix)
-
-
-def main() -> int:
- """
- gapi-gen shell script entrypoint.
- Expects arguments via sys.argv, see --help for details.
-
- :return: int, 0 on success, 1 on failure.
- """
- parser = argparse.ArgumentParser(
- description='Generate code from a QAPI schema')
- parser.add_argument('-b', '--builtins', action='store_true',
- help="generate code for built-in types")
- parser.add_argument('-o', '--output-dir', action='store',
- default=DEFAULT_OUTPUT_DIR,
- help="write output to directory OUTPUT_DIR")
- parser.add_argument('-p', '--prefix', action='store',
- default=DEFAULT_PREFIX,
- help="prefix for symbols")
- parser.add_argument('-u', '--unmask-non-abi-names', action='store_true',
- dest='unmask',
- help="expose non-ABI names in introspection")
- parser.add_argument('schema', action='store')
- args = parser.parse_args()
-
- try:
- generate(args.schema,
- output_dir=args.output_dir,
- prefix=args.prefix,
- unmask=args.unmask,
- builtins=args.builtins)
- except QAPIError as err:
- print(f"{sys.argv[0]}: {str(err)}", file=sys.stderr)
- return 1
- return 0
-
+from qapi import main
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main.main())
diff --git a/scripts/qapi/main.py b/scripts/qapi/main.py
new file mode 100644
index 0000000000..18c246bbb4
--- /dev/null
+++ b/scripts/qapi/main.py
@@ -0,0 +1,91 @@
+# This work is licensed under the terms of the GNU GPL, version 2 or later.
+# See the COPYING file in the top-level directory.
+
+"""
+QAPI Generator
+
+This is the main entry point for generating C code from the QAPI schema.
+"""
+
+import argparse
+import re
+import sys
+
+from qapi.commands import gen_commands
+from qapi.doc import gen_doc
+from qapi.error import QAPIError
+from qapi.events import gen_events
+from qapi.introspect import gen_introspect
+from qapi.schema import QAPISchema
+from qapi.types import gen_types
+from qapi.visit import gen_visit
+
+
+DEFAULT_OUTPUT_DIR = ''
+DEFAULT_PREFIX = ''
+
+
+def generate(schema_file: str,
+ output_dir: str,
+ prefix: str,
+ unmask: bool = False,
+ builtins: bool = False) -> None:
+ """
+ generate uses a given schema to produce C code in the target directory.
+
+ :param schema_file: The primary QAPI schema file.
+ :param output_dir: The output directory to store generated code.
+ :param prefix: Optional C-code prefix for symbol names.
+ :param unmask: Expose non-ABI names through introspection?
+ :param builtins: Generate code for built-in types?
+
+ :raise QAPIError: On failures.
+ """
+ match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', prefix)
+ if match and match.end() != len(prefix):
+ msg = "funny character '{:s}' in prefix '{:s}'".format(
+ prefix[match.end()], prefix)
+ raise QAPIError('', None, msg)
+
+ schema = QAPISchema(schema_file)
+ gen_types(schema, output_dir, prefix, builtins)
+ gen_visit(schema, output_dir, prefix, builtins)
+ gen_commands(schema, output_dir, prefix)
+ gen_events(schema, output_dir, prefix)
+ gen_introspect(schema, output_dir, prefix, unmask)
+ gen_doc(schema, output_dir, prefix)
+
+
+def main() -> int:
+ """
+ gapi-gen shell script entrypoint.
+ Expects arguments via sys.argv, see --help for details.
+
+ :return: int, 0 on success, 1 on failure.
+ """
+ parser = argparse.ArgumentParser(
+ description='Generate code from a QAPI schema')
+ parser.add_argument('-b', '--builtins', action='store_true',
+ help="generate code for built-in types")
+ parser.add_argument('-o', '--output-dir', action='store',
+ default=DEFAULT_OUTPUT_DIR,
+ help="write output to directory OUTPUT_DIR")
+ parser.add_argument('-p', '--prefix', action='store',
+ default=DEFAULT_PREFIX,
+ help="prefix for symbols")
+ parser.add_argument('-u', '--unmask-non-abi-names', action='store_true',
+ dest='unmask',
+ help="expose non-ABI names in introspection")
+ parser.add_argument('schema', action='store')
+ args = parser.parse_args()
+
+ try:
+ generate(args.schema,
+ output_dir=args.output_dir,
+ prefix=args.prefix,
+ unmask=args.unmask,
+ builtins=args.builtins)
+ except QAPIError as err:
+ print(f"{sys.argv[0]}: {str(err)}", file=sys.stderr)
+ return 1
+ return 0
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 03/38] qapi: move generator entrypoint into module
2020-09-22 21:00 ` [PATCH v2 03/38] qapi: move generator entrypoint into module John Snow
@ 2020-09-22 21:23 ` Eduardo Habkost
2020-09-23 17:09 ` John Snow
2020-09-23 0:29 ` Cleber Rosa
1 sibling, 1 reply; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-22 21:23 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:26PM -0400, John Snow wrote:
> As part of delinting and adding type hints to the QAPI generator, it's
> helpful for the entrypoint to be part of the package, only leaving a
> very tiny entrypoint shim outside of the module.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi-gen.py | 90 +++----------------------------------------
> scripts/qapi/main.py | 91 ++++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 97 insertions(+), 84 deletions(-)
> create mode 100644 scripts/qapi/main.py
>
[...]
> diff --git a/scripts/qapi/main.py b/scripts/qapi/main.py
> new file mode 100644
> index 0000000000..18c246bbb4
> --- /dev/null
> +++ b/scripts/qapi/main.py
> @@ -0,0 +1,91 @@
> +# This work is licensed under the terms of the GNU GPL, version 2 or later.
> +# See the COPYING file in the top-level directory.
> +
> +"""
> +QAPI Generator
> +
> +This is the main entry point for generating C code from the QAPI schema.
> +"""
> +
> +import argparse
> +import re
> +import sys
> +
> +from qapi.commands import gen_commands
> +from qapi.doc import gen_doc
> +from qapi.error import QAPIError
> +from qapi.events import gen_events
> +from qapi.introspect import gen_introspect
> +from qapi.schema import QAPISchema
> +from qapi.types import gen_types
> +from qapi.visit import gen_visit
Do you plan to change this to use relative imports eventually?
In either case:
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 03/38] qapi: move generator entrypoint into module
2020-09-22 21:23 ` Eduardo Habkost
@ 2020-09-23 17:09 ` John Snow
0 siblings, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-23 17:09 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, Michael Roth, qemu-devel, Markus Armbruster,
Cleber Rosa, Alex Bennée
On 9/22/20 5:23 PM, Eduardo Habkost wrote:
>> +from qapi.commands import gen_commands
>> +from qapi.doc import gen_doc
>> +from qapi.error import QAPIError
>> +from qapi.events import gen_events
>> +from qapi.introspect import gen_introspect
>> +from qapi.schema import QAPISchema
>> +from qapi.types import gen_types
>> +from qapi.visit import gen_visit
> Do you plan to change this to use relative imports eventually?
Happens in 04/38.
--js
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 03/38] qapi: move generator entrypoint into module
2020-09-22 21:00 ` [PATCH v2 03/38] qapi: move generator entrypoint into module John Snow
2020-09-22 21:23 ` Eduardo Habkost
@ 2020-09-23 0:29 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 0:29 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 657 bytes --]
On Tue, Sep 22, 2020 at 05:00:26PM -0400, John Snow wrote:
> As part of delinting and adding type hints to the QAPI generator, it's
> helpful for the entrypoint to be part of the package, only leaving a
> very tiny entrypoint shim outside of the module.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi-gen.py | 90 +++----------------------------------------
> scripts/qapi/main.py | 91 ++++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 97 insertions(+), 84 deletions(-)
> create mode 100644 scripts/qapi/main.py
>
Reviewed-by: Cleber Rosa <crosa@redhat.com>
Tested-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 04/38] qapi: Prefer explicit relative imports
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (2 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 03/38] qapi: move generator entrypoint into module John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-22 21:25 ` Eduardo Habkost
2020-09-23 13:18 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 05/38] qapi: Remove wildcard includes John Snow
` (34 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
All of the QAPI include statements are changed to be package-aware, as
explicit relative imports.
A quirk of Python packages is that the name of the package exists only
*outside* of the package. This means that to a module inside of the qapi
folder, there is inherently no such thing as the "qapi" package. The
reason these imports work is because the "qapi" package exists in the
context of the caller -- the execution shim, where sys.path includes a
directory that has a 'qapi' folder in it.
When we write "from qapi import sibling", we are NOT referencing the folder
'qapi', but rather "any package named qapi in sys.path". If you should
so happen to have a 'qapi' package in your path, it will use *that*
package.
When we write "from .sibling import foo", we always reference explicitly
our sibling module; guaranteeing consistency in *where* we are importing
these modules from.
This can be useful when working with virtual environments and packages
in development mode. In development mode, a package is installed as a
series of symlinks that forwards to your same source files. The problem
arises because code quality checkers will follow "import qapi.x" to the
"installed" version instead of the sibling file and -- even though they
are the same file -- they have different module paths, and this causes
cyclic import problems, false positive type mismatch errors, and more.
It can also be useful when dealing with hierarchical packages, e.g. if
we allow qemu.core.qmp, qemu.qapi.parser, etc.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/commands.py | 4 ++--
scripts/qapi/doc.py | 2 +-
scripts/qapi/events.py | 8 ++++----
scripts/qapi/expr.py | 4 ++--
scripts/qapi/gen.py | 4 ++--
scripts/qapi/introspect.py | 8 ++++----
scripts/qapi/main.py | 16 ++++++++--------
scripts/qapi/parser.py | 4 ++--
scripts/qapi/schema.py | 8 ++++----
scripts/qapi/types.py | 6 +++---
scripts/qapi/visit.py | 6 +++---
11 files changed, 35 insertions(+), 35 deletions(-)
diff --git a/scripts/qapi/commands.py b/scripts/qapi/commands.py
index 3cf9e1110b..ce5926146a 100644
--- a/scripts/qapi/commands.py
+++ b/scripts/qapi/commands.py
@@ -13,8 +13,8 @@
See the COPYING file in the top-level directory.
"""
-from qapi.common import *
-from qapi.gen import QAPIGenCCode, QAPISchemaModularCVisitor, ifcontext
+from .common import *
+from .gen import QAPIGenCCode, QAPISchemaModularCVisitor, ifcontext
def gen_command_decl(name, arg_type, boxed, ret_type):
diff --git a/scripts/qapi/doc.py b/scripts/qapi/doc.py
index 92f584edcf..cbf7076ed9 100644
--- a/scripts/qapi/doc.py
+++ b/scripts/qapi/doc.py
@@ -5,7 +5,7 @@
"""This script produces the documentation of a qapi schema in texinfo format"""
import re
-from qapi.gen import QAPIGenDoc, QAPISchemaVisitor
+from .gen import QAPIGenDoc, QAPISchemaVisitor
MSG_FMT = """
diff --git a/scripts/qapi/events.py b/scripts/qapi/events.py
index b544af5a1c..0467272438 100644
--- a/scripts/qapi/events.py
+++ b/scripts/qapi/events.py
@@ -12,10 +12,10 @@
See the COPYING file in the top-level directory.
"""
-from qapi.common import *
-from qapi.gen import QAPISchemaModularCVisitor, ifcontext
-from qapi.schema import QAPISchemaEnumMember
-from qapi.types import gen_enum, gen_enum_lookup
+from .common import *
+from .gen import QAPISchemaModularCVisitor, ifcontext
+from .schema import QAPISchemaEnumMember
+from .types import gen_enum, gen_enum_lookup
def build_event_send_proto(name, arg_type, boxed):
diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py
index 2942520399..03b31ecfc1 100644
--- a/scripts/qapi/expr.py
+++ b/scripts/qapi/expr.py
@@ -16,8 +16,8 @@
import re
from collections import OrderedDict
-from qapi.common import c_name
-from qapi.error import QAPISemError
+from .common import c_name
+from .error import QAPISemError
# Names must be letters, numbers, -, and _. They must start with letter,
diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
index bf5552a4e7..8df19a0df0 100644
--- a/scripts/qapi/gen.py
+++ b/scripts/qapi/gen.py
@@ -17,8 +17,8 @@
import re
from contextlib import contextmanager
-from qapi.common import *
-from qapi.schema import QAPISchemaVisitor
+from .common import *
+from .schema import QAPISchemaVisitor
class QAPIGen:
diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py
index 23652be810..2a34cd1e8e 100644
--- a/scripts/qapi/introspect.py
+++ b/scripts/qapi/introspect.py
@@ -10,10 +10,10 @@
See the COPYING file in the top-level directory.
"""
-from qapi.common import *
-from qapi.gen import QAPISchemaMonolithicCVisitor
-from qapi.schema import (QAPISchemaArrayType, QAPISchemaBuiltinType,
- QAPISchemaType)
+from .common import *
+from .gen import QAPISchemaMonolithicCVisitor
+from .schema import (QAPISchemaArrayType, QAPISchemaBuiltinType,
+ QAPISchemaType)
def _make_tree(obj, ifcond, features, extra=None):
diff --git a/scripts/qapi/main.py b/scripts/qapi/main.py
index 18c246bbb4..3f8338ade8 100644
--- a/scripts/qapi/main.py
+++ b/scripts/qapi/main.py
@@ -11,14 +11,14 @@
import re
import sys
-from qapi.commands import gen_commands
-from qapi.doc import gen_doc
-from qapi.error import QAPIError
-from qapi.events import gen_events
-from qapi.introspect import gen_introspect
-from qapi.schema import QAPISchema
-from qapi.types import gen_types
-from qapi.visit import gen_visit
+from .commands import gen_commands
+from .doc import gen_doc
+from .error import QAPIError
+from .events import gen_events
+from .introspect import gen_introspect
+from .schema import QAPISchema
+from .types import gen_types
+from .visit import gen_visit
DEFAULT_OUTPUT_DIR = ''
diff --git a/scripts/qapi/parser.py b/scripts/qapi/parser.py
index 165925ca72..327cf05736 100644
--- a/scripts/qapi/parser.py
+++ b/scripts/qapi/parser.py
@@ -18,8 +18,8 @@
import re
from collections import OrderedDict
-from qapi.error import QAPIParseError, QAPISemError
-from qapi.source import QAPISourceInfo
+from .error import QAPIParseError, QAPISemError
+from .source import QAPISourceInfo
class QAPISchemaParser:
diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py
index 78309a00f0..a835ee6fde 100644
--- a/scripts/qapi/schema.py
+++ b/scripts/qapi/schema.py
@@ -18,10 +18,10 @@
import re
from collections import OrderedDict
-from qapi.common import c_name, pointer_suffix
-from qapi.error import QAPIError, QAPISemError
-from qapi.expr import check_exprs
-from qapi.parser import QAPISchemaParser
+from .common import c_name, pointer_suffix
+from .error import QAPIError, QAPISemError
+from .expr import check_exprs
+from .parser import QAPISchemaParser
class QAPISchemaEntity:
diff --git a/scripts/qapi/types.py b/scripts/qapi/types.py
index 3640f17cd6..ca9a5aacb3 100644
--- a/scripts/qapi/types.py
+++ b/scripts/qapi/types.py
@@ -13,9 +13,9 @@
# See the COPYING file in the top-level directory.
"""
-from qapi.common import *
-from qapi.gen import QAPISchemaModularCVisitor, ifcontext
-from qapi.schema import QAPISchemaEnumMember, QAPISchemaObjectType
+from .common import *
+from .gen import QAPISchemaModularCVisitor, ifcontext
+from .schema import QAPISchemaEnumMember, QAPISchemaObjectType
# variants must be emitted before their container; track what has already
diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py
index cdabc5fa28..7850f6e848 100644
--- a/scripts/qapi/visit.py
+++ b/scripts/qapi/visit.py
@@ -13,9 +13,9 @@
See the COPYING file in the top-level directory.
"""
-from qapi.common import *
-from qapi.gen import QAPISchemaModularCVisitor, ifcontext
-from qapi.schema import QAPISchemaObjectType
+from .common import *
+from .gen import QAPISchemaModularCVisitor, ifcontext
+from .schema import QAPISchemaObjectType
def gen_visit_decl(name, scalar=False):
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 04/38] qapi: Prefer explicit relative imports
2020-09-22 21:00 ` [PATCH v2 04/38] qapi: Prefer explicit relative imports John Snow
@ 2020-09-22 21:25 ` Eduardo Habkost
2020-09-23 13:18 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-22 21:25 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:27PM -0400, John Snow wrote:
> All of the QAPI include statements are changed to be package-aware, as
> explicit relative imports.
>
> A quirk of Python packages is that the name of the package exists only
> *outside* of the package. This means that to a module inside of the qapi
> folder, there is inherently no such thing as the "qapi" package. The
> reason these imports work is because the "qapi" package exists in the
> context of the caller -- the execution shim, where sys.path includes a
> directory that has a 'qapi' folder in it.
>
> When we write "from qapi import sibling", we are NOT referencing the folder
> 'qapi', but rather "any package named qapi in sys.path". If you should
> so happen to have a 'qapi' package in your path, it will use *that*
> package.
>
> When we write "from .sibling import foo", we always reference explicitly
> our sibling module; guaranteeing consistency in *where* we are importing
> these modules from.
>
> This can be useful when working with virtual environments and packages
> in development mode. In development mode, a package is installed as a
> series of symlinks that forwards to your same source files. The problem
> arises because code quality checkers will follow "import qapi.x" to the
> "installed" version instead of the sibling file and -- even though they
> are the same file -- they have different module paths, and this causes
> cyclic import problems, false positive type mismatch errors, and more.
>
> It can also be useful when dealing with hierarchical packages, e.g. if
> we allow qemu.core.qmp, qemu.qapi.parser, etc.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Here's the answer to the question I sent on patch 03/38. :)
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 04/38] qapi: Prefer explicit relative imports
2020-09-22 21:00 ` [PATCH v2 04/38] qapi: Prefer explicit relative imports John Snow
2020-09-22 21:25 ` Eduardo Habkost
@ 2020-09-23 13:18 ` Cleber Rosa
2020-09-23 17:12 ` John Snow
1 sibling, 1 reply; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 13:18 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 2439 bytes --]
On Tue, Sep 22, 2020 at 05:00:27PM -0400, John Snow wrote:
> All of the QAPI include statements are changed to be package-aware, as
> explicit relative imports.
>
> A quirk of Python packages is that the name of the package exists only
> *outside* of the package. This means that to a module inside of the qapi
> folder, there is inherently no such thing as the "qapi" package. The
> reason these imports work is because the "qapi" package exists in the
> context of the caller -- the execution shim, where sys.path includes a
> directory that has a 'qapi' folder in it.
>
> When we write "from qapi import sibling", we are NOT referencing the folder
> 'qapi', but rather "any package named qapi in sys.path". If you should
> so happen to have a 'qapi' package in your path, it will use *that*
> package.
>
> When we write "from .sibling import foo", we always reference explicitly
> our sibling module; guaranteeing consistency in *where* we are importing
> these modules from.
>
> This can be useful when working with virtual environments and packages
> in development mode. In development mode, a package is installed as a
> series of symlinks that forwards to your same source files. The problem
> arises because code quality checkers will follow "import qapi.x" to the
> "installed" version instead of the sibling file and -- even though they
> are the same file -- they have different module paths, and this causes
> cyclic import problems, false positive type mismatch errors, and more.
>
> It can also be useful when dealing with hierarchical packages, e.g. if
> we allow qemu.core.qmp, qemu.qapi.parser, etc.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/commands.py | 4 ++--
> scripts/qapi/doc.py | 2 +-
> scripts/qapi/events.py | 8 ++++----
> scripts/qapi/expr.py | 4 ++--
> scripts/qapi/gen.py | 4 ++--
> scripts/qapi/introspect.py | 8 ++++----
> scripts/qapi/main.py | 16 ++++++++--------
> scripts/qapi/parser.py | 4 ++--
> scripts/qapi/schema.py | 8 ++++----
> scripts/qapi/types.py | 6 +++---
> scripts/qapi/visit.py | 6 +++---
> 11 files changed, 35 insertions(+), 35 deletions(-)
>
Relative imports are a source of heated debates, but when properly
used in a self contained module like here, they are very posititive
IMO.
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 04/38] qapi: Prefer explicit relative imports
2020-09-23 13:18 ` Cleber Rosa
@ 2020-09-23 17:12 ` John Snow
2020-09-24 19:25 ` Cleber Rosa
0 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-23 17:12 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
On 9/23/20 9:18 AM, Cleber Rosa wrote:
> Relative imports are a source of heated debates, but when properly
> used in a self contained module like here, they are very posititive
> IMO.
Still? I know they were loathed pre-3.5, but in my subjective experience
they behave the nicest overall in the modern python dialect.
What are the downsides?
--js
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 04/38] qapi: Prefer explicit relative imports
2020-09-23 17:12 ` John Snow
@ 2020-09-24 19:25 ` Cleber Rosa
2020-09-24 22:17 ` Beraldo Leal
0 siblings, 1 reply; 190+ messages in thread
From: Cleber Rosa @ 2020-09-24 19:25 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée, Beraldo Leal
[-- Attachment #1: Type: text/plain, Size: 529 bytes --]
On Wed, Sep 23, 2020 at 01:12:09PM -0400, John Snow wrote:
> On 9/23/20 9:18 AM, Cleber Rosa wrote:
> > Relative imports are a source of heated debates, but when properly
> > used in a self contained module like here, they are very posititive
> > IMO.
>
> Still? I know they were loathed pre-3.5, but in my subjective experience
> they behave the nicest overall in the modern python dialect.
>
> What are the downsides?
>
> --js
I'll just invite Beraldo to the discussion and let the fun begin :).
- Cleber.
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 04/38] qapi: Prefer explicit relative imports
2020-09-24 19:25 ` Cleber Rosa
@ 2020-09-24 22:17 ` Beraldo Leal
2020-09-24 22:36 ` Cleber Rosa
0 siblings, 1 reply; 190+ messages in thread
From: Beraldo Leal @ 2020-09-24 22:17 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, John Snow,
Michael Roth, Markus Armbruster, Alex Bennée
On Thu, Sep 24, 2020 at 03:25:50PM -0400, Cleber Rosa wrote:
> On Wed, Sep 23, 2020 at 01:12:09PM -0400, John Snow wrote:
> > On 9/23/20 9:18 AM, Cleber Rosa wrote:
> > > Relative imports are a source of heated debates, but when properly
> > > used in a self contained module like here, they are very posititive
> > > IMO.
> >
> > Still? I know they were loathed pre-3.5, but in my subjective experience
> > they behave the nicest overall in the modern python dialect.
> >
> > What are the downsides?
> >
> > --js
>
> I'll just invite Beraldo to the discussion and let the fun begin :).
Nice try, Cleber! ;)
Well, relative imports are supported by Guido, so I'm not here to say
different. There are some use-cases.
I'm not fully aware of the qapi context and big picture here, but I
guess that depends on how you would like to use your package/scripts.
Some may say that one "downside" is that relative imports are not as
readable as absolute ones. But reading the 04/38 PATH description by
jsnow, yes, looks like using relative imports is one valid option here.
I prefer to use my scripts as packages inside venvs, and I use to have a
setup.py, with absolute imports whenever possible, and when in
development mode, make use of `python3 setup.py develop` which will
create the "links" for me.
--
Beraldo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 04/38] qapi: Prefer explicit relative imports
2020-09-24 22:17 ` Beraldo Leal
@ 2020-09-24 22:36 ` Cleber Rosa
0 siblings, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-24 22:36 UTC (permalink / raw)
To: Beraldo Leal
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, John Snow,
Michael Roth, Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 2162 bytes --]
On Thu, Sep 24, 2020 at 07:17:47PM -0300, Beraldo Leal wrote:
> On Thu, Sep 24, 2020 at 03:25:50PM -0400, Cleber Rosa wrote:
> > On Wed, Sep 23, 2020 at 01:12:09PM -0400, John Snow wrote:
> > > On 9/23/20 9:18 AM, Cleber Rosa wrote:
> > > > Relative imports are a source of heated debates, but when properly
> > > > used in a self contained module like here, they are very posititive
> > > > IMO.
> > >
> > > Still? I know they were loathed pre-3.5, but in my subjective experience
> > > they behave the nicest overall in the modern python dialect.
> > >
> > > What are the downsides?
> > >
> > > --js
> >
> > I'll just invite Beraldo to the discussion and let the fun begin :).
>
> Nice try, Cleber! ;)
>
C'mon... I was hoping for nothing less than an emacs .vs. vi kind of
discussion.
> Well, relative imports are supported by Guido, so I'm not here to say
> different. There are some use-cases.
>
> I'm not fully aware of the qapi context and big picture here, but I
> guess that depends on how you would like to use your package/scripts.
>
> Some may say that one "downside" is that relative imports are not as
> readable as absolute ones. But reading the 04/38 PATH description by
> jsnow, yes, looks like using relative imports is one valid option here.
>
> I prefer to use my scripts as packages inside venvs, and I use to have a
> setup.py, with absolute imports whenever possible, and when in
> development mode, make use of `python3 setup.py develop` which will
> create the "links" for me.
>
Now seriously, these are good point, thanks. John, I invited Beraldo
to give his take on the subject here because he started this issue
on Avocado land:
https://github.com/avocado-framework/avocado/issues/3525
So far on Avocado we've kept the relative imports on most places. We do
have some occurences of "triple upper level" imports that don't look
very nice IMO though:
https://github.com/avocado-framework/avocado/blob/master/avocado/utils/software_manager/backends/rpm.py#L5
> --
> Beraldo
>
Anyway, I think we're all in agreement with the approach taken here.
- Cleber.
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 05/38] qapi: Remove wildcard includes
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (3 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 04/38] qapi: Prefer explicit relative imports John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-22 21:37 ` Eduardo Habkost
2020-09-23 13:27 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 06/38] qapi: delint using flake8 John Snow
` (33 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Wildcard includes become hard to manage when refactoring and dealing
with circular dependencies with strictly typed mypy.
flake8 also flags each one as a warning, as it is not smart enough to
know which names exist in the imported file.
Remove them and include things explicitly by name instead.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/commands.py | 6 +++++-
scripts/qapi/events.py | 7 ++++++-
scripts/qapi/gen.py | 12 +++++++++---
scripts/qapi/introspect.py | 7 ++++++-
scripts/qapi/types.py | 8 +++++++-
scripts/qapi/visit.py | 10 +++++++++-
6 files changed, 42 insertions(+), 8 deletions(-)
diff --git a/scripts/qapi/commands.py b/scripts/qapi/commands.py
index ce5926146a..e1df0e341f 100644
--- a/scripts/qapi/commands.py
+++ b/scripts/qapi/commands.py
@@ -13,7 +13,11 @@
See the COPYING file in the top-level directory.
"""
-from .common import *
+from .common import (
+ build_params,
+ c_name,
+ mcgen,
+)
from .gen import QAPIGenCCode, QAPISchemaModularCVisitor, ifcontext
diff --git a/scripts/qapi/events.py b/scripts/qapi/events.py
index 0467272438..6b3afa14d7 100644
--- a/scripts/qapi/events.py
+++ b/scripts/qapi/events.py
@@ -12,7 +12,12 @@
See the COPYING file in the top-level directory.
"""
-from .common import *
+from .common import (
+ build_params,
+ c_enum_const,
+ c_name,
+ mcgen,
+)
from .gen import QAPISchemaModularCVisitor, ifcontext
from .schema import QAPISchemaEnumMember
from .types import gen_enum, gen_enum_lookup
diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
index 8df19a0df0..11472ba043 100644
--- a/scripts/qapi/gen.py
+++ b/scripts/qapi/gen.py
@@ -11,13 +11,19 @@
# This work is licensed under the terms of the GNU GPL, version 2.
# See the COPYING file in the top-level directory.
-
+from contextlib import contextmanager
import errno
import os
import re
-from contextlib import contextmanager
-from .common import *
+from .common import (
+ c_fname,
+ gen_endif,
+ gen_if,
+ guardend,
+ guardstart,
+ mcgen,
+)
from .schema import QAPISchemaVisitor
diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py
index 2a34cd1e8e..b036fcf9ce 100644
--- a/scripts/qapi/introspect.py
+++ b/scripts/qapi/introspect.py
@@ -10,7 +10,12 @@
See the COPYING file in the top-level directory.
"""
-from .common import *
+from .common import (
+ c_name,
+ gen_endif,
+ gen_if,
+ mcgen,
+)
from .gen import QAPISchemaMonolithicCVisitor
from .schema import (QAPISchemaArrayType, QAPISchemaBuiltinType,
QAPISchemaType)
diff --git a/scripts/qapi/types.py b/scripts/qapi/types.py
index ca9a5aacb3..53b47f9e58 100644
--- a/scripts/qapi/types.py
+++ b/scripts/qapi/types.py
@@ -13,7 +13,13 @@
# See the COPYING file in the top-level directory.
"""
-from .common import *
+from .common import (
+ c_enum_const,
+ c_name,
+ gen_endif,
+ gen_if,
+ mcgen,
+)
from .gen import QAPISchemaModularCVisitor, ifcontext
from .schema import QAPISchemaEnumMember, QAPISchemaObjectType
diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py
index 7850f6e848..ea277e7704 100644
--- a/scripts/qapi/visit.py
+++ b/scripts/qapi/visit.py
@@ -13,7 +13,15 @@
See the COPYING file in the top-level directory.
"""
-from .common import *
+from .common import (
+ c_enum_const,
+ c_name,
+ gen_endif,
+ gen_if,
+ mcgen,
+ pop_indent,
+ push_indent,
+)
from .gen import QAPISchemaModularCVisitor, ifcontext
from .schema import QAPISchemaObjectType
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 05/38] qapi: Remove wildcard includes
2020-09-22 21:00 ` [PATCH v2 05/38] qapi: Remove wildcard includes John Snow
@ 2020-09-22 21:37 ` Eduardo Habkost
2020-09-23 13:27 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-22 21:37 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:28PM -0400, John Snow wrote:
> Wildcard includes become hard to manage when refactoring and dealing
> with circular dependencies with strictly typed mypy.
>
> flake8 also flags each one as a warning, as it is not smart enough to
> know which names exist in the imported file.
>
> Remove them and include things explicitly by name instead.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 05/38] qapi: Remove wildcard includes
2020-09-22 21:00 ` [PATCH v2 05/38] qapi: Remove wildcard includes John Snow
2020-09-22 21:37 ` Eduardo Habkost
@ 2020-09-23 13:27 ` Cleber Rosa
2020-09-23 17:21 ` John Snow
1 sibling, 1 reply; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 13:27 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 4686 bytes --]
On Tue, Sep 22, 2020 at 05:00:28PM -0400, John Snow wrote:
> Wildcard includes become hard to manage when refactoring and dealing
> with circular dependencies with strictly typed mypy.
>
> flake8 also flags each one as a warning, as it is not smart enough to
> know which names exist in the imported file.
>
> Remove them and include things explicitly by name instead.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/commands.py | 6 +++++-
> scripts/qapi/events.py | 7 ++++++-
> scripts/qapi/gen.py | 12 +++++++++---
> scripts/qapi/introspect.py | 7 ++++++-
> scripts/qapi/types.py | 8 +++++++-
> scripts/qapi/visit.py | 10 +++++++++-
> 6 files changed, 42 insertions(+), 8 deletions(-)
>
> diff --git a/scripts/qapi/commands.py b/scripts/qapi/commands.py
> index ce5926146a..e1df0e341f 100644
> --- a/scripts/qapi/commands.py
> +++ b/scripts/qapi/commands.py
> @@ -13,7 +13,11 @@
> See the COPYING file in the top-level directory.
> """
>
> -from .common import *
> +from .common import (
> + build_params,
> + c_name,
> + mcgen,
> +)
> from .gen import QAPIGenCCode, QAPISchemaModularCVisitor, ifcontext
>
>
Is this import style being suggested or enforced by any tool? I've
been using isort with very good results (both as a check tool, and as
an emacs extension). For instance, the block about would look like:
from .common import build_params, c_name, mcgen
from .gen import QAPIGenCCode, QAPISchemaModularCVisitor, ifcontext
> diff --git a/scripts/qapi/events.py b/scripts/qapi/events.py
> index 0467272438..6b3afa14d7 100644
> --- a/scripts/qapi/events.py
> +++ b/scripts/qapi/events.py
> @@ -12,7 +12,12 @@
> See the COPYING file in the top-level directory.
> """
>
> -from .common import *
> +from .common import (
> + build_params,
> + c_enum_const,
> + c_name,
> + mcgen,
> +)
> from .gen import QAPISchemaModularCVisitor, ifcontext
> from .schema import QAPISchemaEnumMember
> from .types import gen_enum, gen_enum_lookup
> diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
> index 8df19a0df0..11472ba043 100644
> --- a/scripts/qapi/gen.py
> +++ b/scripts/qapi/gen.py
> @@ -11,13 +11,19 @@
> # This work is licensed under the terms of the GNU GPL, version 2.
> # See the COPYING file in the top-level directory.
>
> -
> +from contextlib import contextmanager
> import errno
> import os
> import re
> -from contextlib import contextmanager
>
> -from .common import *
> +from .common import (
> + c_fname,
> + gen_endif,
> + gen_if,
> + guardend,
> + guardstart,
> + mcgen,
> +)
> from .schema import QAPISchemaVisitor
>
>
> diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py
> index 2a34cd1e8e..b036fcf9ce 100644
> --- a/scripts/qapi/introspect.py
> +++ b/scripts/qapi/introspect.py
> @@ -10,7 +10,12 @@
> See the COPYING file in the top-level directory.
> """
>
> -from .common import *
> +from .common import (
> + c_name,
> + gen_endif,
> + gen_if,
> + mcgen,
> +)
> from .gen import QAPISchemaMonolithicCVisitor
> from .schema import (QAPISchemaArrayType, QAPISchemaBuiltinType,
> QAPISchemaType)
> diff --git a/scripts/qapi/types.py b/scripts/qapi/types.py
> index ca9a5aacb3..53b47f9e58 100644
> --- a/scripts/qapi/types.py
> +++ b/scripts/qapi/types.py
> @@ -13,7 +13,13 @@
> # See the COPYING file in the top-level directory.
> """
>
> -from .common import *
> +from .common import (
> + c_enum_const,
> + c_name,
> + gen_endif,
> + gen_if,
> + mcgen,
> +)
> from .gen import QAPISchemaModularCVisitor, ifcontext
> from .schema import QAPISchemaEnumMember, QAPISchemaObjectType
>
> diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py
> index 7850f6e848..ea277e7704 100644
> --- a/scripts/qapi/visit.py
> +++ b/scripts/qapi/visit.py
> @@ -13,7 +13,15 @@
> See the COPYING file in the top-level directory.
> """
>
> -from .common import *
> +from .common import (
> + c_enum_const,
> + c_name,
> + gen_endif,
> + gen_if,
> + mcgen,
> + pop_indent,
> + push_indent,
> +)
And here, isort will add the paranthesis (it does so based on space demands):
from .common import (c_enum_const, c_name, gen_endif, gen_if, mcgen,
pop_indent, push_indent)
from .gen import QAPISchemaModularCVisitor, ifcontext
from .schema import QAPISchemaObjectType
Other than those suggestions, it LGTM.
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 05/38] qapi: Remove wildcard includes
2020-09-23 13:27 ` Cleber Rosa
@ 2020-09-23 17:21 ` John Snow
2020-09-24 19:27 ` Cleber Rosa
2020-09-24 20:04 ` John Snow
0 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-23 17:21 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
On 9/23/20 9:27 AM, Cleber Rosa wrote:
> On Tue, Sep 22, 2020 at 05:00:28PM -0400, John Snow wrote:
>> Wildcard includes become hard to manage when refactoring and dealing
>> with circular dependencies with strictly typed mypy.
>>
>> flake8 also flags each one as a warning, as it is not smart enough to
>> know which names exist in the imported file.
>>
>> Remove them and include things explicitly by name instead.
>>
>> Signed-off-by: John Snow <jsnow@redhat.com>
>> ---
>> scripts/qapi/commands.py | 6 +++++-
>> scripts/qapi/events.py | 7 ++++++-
>> scripts/qapi/gen.py | 12 +++++++++---
>> scripts/qapi/introspect.py | 7 ++++++-
>> scripts/qapi/types.py | 8 +++++++-
>> scripts/qapi/visit.py | 10 +++++++++-
>> 6 files changed, 42 insertions(+), 8 deletions(-)
>>
>> diff --git a/scripts/qapi/commands.py b/scripts/qapi/commands.py
>> index ce5926146a..e1df0e341f 100644
>> --- a/scripts/qapi/commands.py
>> +++ b/scripts/qapi/commands.py
>> @@ -13,7 +13,11 @@
>> See the COPYING file in the top-level directory.
>> """
>>
>> -from .common import *
>> +from .common import (
>> + build_params,
>> + c_name,
>> + mcgen,
>> +)
>> from .gen import QAPIGenCCode, QAPISchemaModularCVisitor, ifcontext
>>
>>
>
> Is this import style being suggested or enforced by any tool? I've
> been using isort with very good results (both as a check tool, and as
> an emacs extension). For instance, the block about would look like:
>
> from .common import build_params, c_name, mcgen
> from .gen import QAPIGenCCode, QAPISchemaModularCVisitor, ifcontext
>
Not enforced by any tool, no. Just subjective preference for
git-friendly import lines. They conflict on rebase a lot less.
I have been using emacs sort-lines to order the names in a group.
>> diff --git a/scripts/qapi/events.py b/scripts/qapi/events.py
>> index 0467272438..6b3afa14d7 100644
>> --- a/scripts/qapi/events.py
>> +++ b/scripts/qapi/events.py
>> @@ -12,7 +12,12 @@
>> See the COPYING file in the top-level directory.
>> """
>>
>> -from .common import *
>> +from .common import (
>> + build_params,
>> + c_enum_const,
>> + c_name,
>> + mcgen,
>> +)
>> from .gen import QAPISchemaModularCVisitor, ifcontext
>> from .schema import QAPISchemaEnumMember
>> from .types import gen_enum, gen_enum_lookup
>> diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
>> index 8df19a0df0..11472ba043 100644
>> --- a/scripts/qapi/gen.py
>> +++ b/scripts/qapi/gen.py
>> @@ -11,13 +11,19 @@
>> # This work is licensed under the terms of the GNU GPL, version 2.
>> # See the COPYING file in the top-level directory.
>>
>> -
>> +from contextlib import contextmanager
>> import errno
>> import os
>> import re
>> -from contextlib import contextmanager
>>
>> -from .common import *
>> +from .common import (
>> + c_fname,
>> + gen_endif,
>> + gen_if,
>> + guardend,
>> + guardstart,
>> + mcgen,
>> +)
>> from .schema import QAPISchemaVisitor
>>
>>
>> diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py
>> index 2a34cd1e8e..b036fcf9ce 100644
>> --- a/scripts/qapi/introspect.py
>> +++ b/scripts/qapi/introspect.py
>> @@ -10,7 +10,12 @@
>> See the COPYING file in the top-level directory.
>> """
>>
>> -from .common import *
>> +from .common import (
>> + c_name,
>> + gen_endif,
>> + gen_if,
>> + mcgen,
>> +)
>> from .gen import QAPISchemaMonolithicCVisitor
>> from .schema import (QAPISchemaArrayType, QAPISchemaBuiltinType,
>> QAPISchemaType)
>> diff --git a/scripts/qapi/types.py b/scripts/qapi/types.py
>> index ca9a5aacb3..53b47f9e58 100644
>> --- a/scripts/qapi/types.py
>> +++ b/scripts/qapi/types.py
>> @@ -13,7 +13,13 @@
>> # See the COPYING file in the top-level directory.
>> """
>>
>> -from .common import *
>> +from .common import (
>> + c_enum_const,
>> + c_name,
>> + gen_endif,
>> + gen_if,
>> + mcgen,
>> +)
>> from .gen import QAPISchemaModularCVisitor, ifcontext
>> from .schema import QAPISchemaEnumMember, QAPISchemaObjectType
>>
>> diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py
>> index 7850f6e848..ea277e7704 100644
>> --- a/scripts/qapi/visit.py
>> +++ b/scripts/qapi/visit.py
>> @@ -13,7 +13,15 @@
>> See the COPYING file in the top-level directory.
>> """
>>
>> -from .common import *
>> +from .common import (
>> + c_enum_const,
>> + c_name,
>> + gen_endif,
>> + gen_if,
>> + mcgen,
>> + pop_indent,
>> + push_indent,
>> +)
>
> And here, isort will add the paranthesis (it does so based on space demands):
>
> from .common import (c_enum_const, c_name, gen_endif, gen_if, mcgen,
> pop_indent, push_indent)
> from .gen import QAPISchemaModularCVisitor, ifcontext
> from .schema import QAPISchemaObjectType
>
> Other than those suggestions, it LGTM.
>
OK. We can add a check that asserts that isort(file) == file to keep our
include regimes consistent. I'll look into the tool, but it will be
after this marathon of a series.
Here's a gitlab issue I made on my QEMU fork to help me keep track of
Python-related issues that I intend to use:
https://gitlab.com/jsnow/qemu/-/issues/6
> Reviewed-by: Cleber Rosa <crosa@redhat.com>
>
Thanks!
--js
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 05/38] qapi: Remove wildcard includes
2020-09-23 17:21 ` John Snow
@ 2020-09-24 19:27 ` Cleber Rosa
2020-09-24 20:04 ` John Snow
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-24 19:27 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 5996 bytes --]
On Wed, Sep 23, 2020 at 01:21:37PM -0400, John Snow wrote:
> On 9/23/20 9:27 AM, Cleber Rosa wrote:
> > On Tue, Sep 22, 2020 at 05:00:28PM -0400, John Snow wrote:
> > > Wildcard includes become hard to manage when refactoring and dealing
> > > with circular dependencies with strictly typed mypy.
> > >
> > > flake8 also flags each one as a warning, as it is not smart enough to
> > > know which names exist in the imported file.
> > >
> > > Remove them and include things explicitly by name instead.
> > >
> > > Signed-off-by: John Snow <jsnow@redhat.com>
> > > ---
> > > scripts/qapi/commands.py | 6 +++++-
> > > scripts/qapi/events.py | 7 ++++++-
> > > scripts/qapi/gen.py | 12 +++++++++---
> > > scripts/qapi/introspect.py | 7 ++++++-
> > > scripts/qapi/types.py | 8 +++++++-
> > > scripts/qapi/visit.py | 10 +++++++++-
> > > 6 files changed, 42 insertions(+), 8 deletions(-)
> > >
> > > diff --git a/scripts/qapi/commands.py b/scripts/qapi/commands.py
> > > index ce5926146a..e1df0e341f 100644
> > > --- a/scripts/qapi/commands.py
> > > +++ b/scripts/qapi/commands.py
> > > @@ -13,7 +13,11 @@
> > > See the COPYING file in the top-level directory.
> > > """
> > > -from .common import *
> > > +from .common import (
> > > + build_params,
> > > + c_name,
> > > + mcgen,
> > > +)
> > > from .gen import QAPIGenCCode, QAPISchemaModularCVisitor, ifcontext
> >
> > Is this import style being suggested or enforced by any tool? I've
> > been using isort with very good results (both as a check tool, and as
> > an emacs extension). For instance, the block about would look like:
> >
> > from .common import build_params, c_name, mcgen
> > from .gen import QAPIGenCCode, QAPISchemaModularCVisitor, ifcontext
> >
>
> Not enforced by any tool, no. Just subjective preference for git-friendly
> import lines. They conflict on rebase a lot less.
>
> I have been using emacs sort-lines to order the names in a group.
>
> > > diff --git a/scripts/qapi/events.py b/scripts/qapi/events.py
> > > index 0467272438..6b3afa14d7 100644
> > > --- a/scripts/qapi/events.py
> > > +++ b/scripts/qapi/events.py
> > > @@ -12,7 +12,12 @@
> > > See the COPYING file in the top-level directory.
> > > """
> > > -from .common import *
> > > +from .common import (
> > > + build_params,
> > > + c_enum_const,
> > > + c_name,
> > > + mcgen,
> > > +)
> > > from .gen import QAPISchemaModularCVisitor, ifcontext
> > > from .schema import QAPISchemaEnumMember
> > > from .types import gen_enum, gen_enum_lookup
> > > diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
> > > index 8df19a0df0..11472ba043 100644
> > > --- a/scripts/qapi/gen.py
> > > +++ b/scripts/qapi/gen.py
> > > @@ -11,13 +11,19 @@
> > > # This work is licensed under the terms of the GNU GPL, version 2.
> > > # See the COPYING file in the top-level directory.
> > > -
> > > +from contextlib import contextmanager
> > > import errno
> > > import os
> > > import re
> > > -from contextlib import contextmanager
> > > -from .common import *
> > > +from .common import (
> > > + c_fname,
> > > + gen_endif,
> > > + gen_if,
> > > + guardend,
> > > + guardstart,
> > > + mcgen,
> > > +)
> > > from .schema import QAPISchemaVisitor
> > > diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py
> > > index 2a34cd1e8e..b036fcf9ce 100644
> > > --- a/scripts/qapi/introspect.py
> > > +++ b/scripts/qapi/introspect.py
> > > @@ -10,7 +10,12 @@
> > > See the COPYING file in the top-level directory.
> > > """
> > > -from .common import *
> > > +from .common import (
> > > + c_name,
> > > + gen_endif,
> > > + gen_if,
> > > + mcgen,
> > > +)
> > > from .gen import QAPISchemaMonolithicCVisitor
> > > from .schema import (QAPISchemaArrayType, QAPISchemaBuiltinType,
> > > QAPISchemaType)
> > > diff --git a/scripts/qapi/types.py b/scripts/qapi/types.py
> > > index ca9a5aacb3..53b47f9e58 100644
> > > --- a/scripts/qapi/types.py
> > > +++ b/scripts/qapi/types.py
> > > @@ -13,7 +13,13 @@
> > > # See the COPYING file in the top-level directory.
> > > """
> > > -from .common import *
> > > +from .common import (
> > > + c_enum_const,
> > > + c_name,
> > > + gen_endif,
> > > + gen_if,
> > > + mcgen,
> > > +)
> > > from .gen import QAPISchemaModularCVisitor, ifcontext
> > > from .schema import QAPISchemaEnumMember, QAPISchemaObjectType
> > > diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py
> > > index 7850f6e848..ea277e7704 100644
> > > --- a/scripts/qapi/visit.py
> > > +++ b/scripts/qapi/visit.py
> > > @@ -13,7 +13,15 @@
> > > See the COPYING file in the top-level directory.
> > > """
> > > -from .common import *
> > > +from .common import (
> > > + c_enum_const,
> > > + c_name,
> > > + gen_endif,
> > > + gen_if,
> > > + mcgen,
> > > + pop_indent,
> > > + push_indent,
> > > +)
> >
> > And here, isort will add the paranthesis (it does so based on space demands):
> >
> > from .common import (c_enum_const, c_name, gen_endif, gen_if, mcgen,
> > pop_indent, push_indent)
> > from .gen import QAPISchemaModularCVisitor, ifcontext
> > from .schema import QAPISchemaObjectType
> >
> > Other than those suggestions, it LGTM.
> >
>
> OK. We can add a check that asserts that isort(file) == file to keep our
> include regimes consistent. I'll look into the tool, but it will be after
> this marathon of a series.
>
That goes without saying!
> Here's a gitlab issue I made on my QEMU fork to help me keep track of
> Python-related issues that I intend to use:
> https://gitlab.com/jsnow/qemu/-/issues/6
>
Nice!
- Cleber.
> > Reviewed-by: Cleber Rosa <crosa@redhat.com>
> >
>
> Thanks!
>
> --js
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 05/38] qapi: Remove wildcard includes
2020-09-23 17:21 ` John Snow
2020-09-24 19:27 ` Cleber Rosa
@ 2020-09-24 20:04 ` John Snow
1 sibling, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-24 20:04 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
On 9/23/20 1:21 PM, John Snow wrote:
> On 9/23/20 9:27 AM, Cleber Rosa wrote:
>> On Tue, Sep 22, 2020 at 05:00:28PM -0400, John Snow wrote:
>>> Wildcard includes become hard to manage when refactoring and dealing
>>> with circular dependencies with strictly typed mypy.
>>>
>>> flake8 also flags each one as a warning, as it is not smart enough to
>>> know which names exist in the imported file.
>>>
>>> Remove them and include things explicitly by name instead.
>>>
>>> Signed-off-by: John Snow <jsnow@redhat.com>
>>> ---
>>>  scripts/qapi/commands.py  | 6 +++++-
>>>  scripts/qapi/events.py    | 7 ++++++-
>>>  scripts/qapi/gen.py       | 12 +++++++++---
>>> Â scripts/qapi/introspect.py |Â 7 ++++++-
>>>  scripts/qapi/types.py     | 8 +++++++-
>>>  scripts/qapi/visit.py     | 10 +++++++++-
>>> Â 6 files changed, 42 insertions(+), 8 deletions(-)
>>>
>>> diff --git a/scripts/qapi/commands.py b/scripts/qapi/commands.py
>>> index ce5926146a..e1df0e341f 100644
>>> --- a/scripts/qapi/commands.py
>>> +++ b/scripts/qapi/commands.py
>>> @@ -13,7 +13,11 @@
>>> Â See the COPYING file in the top-level directory.
>>> Â """
>>> -from .common import *
>>> +from .common import (
>>> +Â Â Â build_params,
>>> +Â Â Â c_name,
>>> +Â Â Â mcgen,
>>> +)
>>> Â from .gen import QAPIGenCCode, QAPISchemaModularCVisitor, ifcontext
>>
>> Is this import style being suggested or enforced by any tool? I've
>> been using isort with very good results (both as a check tool, and as
>> an emacs extension). For instance, the block about would look like:
>>
>> Â Â Â from .common import build_params, c_name, mcgen
>> Â Â Â from .gen import QAPIGenCCode, QAPISchemaModularCVisitor, ifcontext
>>
>
> Not enforced by any tool, no. Just subjective preference for
> git-friendly import lines. They conflict on rebase a lot less.
>
> I have been using emacs sort-lines to order the names in a group.
>
>>> diff --git a/scripts/qapi/events.py b/scripts/qapi/events.py
>>> index 0467272438..6b3afa14d7 100644
>>> --- a/scripts/qapi/events.py
>>> +++ b/scripts/qapi/events.py
>>> @@ -12,7 +12,12 @@
>>> Â See the COPYING file in the top-level directory.
>>> Â """
>>> -from .common import *
>>> +from .common import (
>>> +Â Â Â build_params,
>>> +Â Â Â c_enum_const,
>>> +Â Â Â c_name,
>>> +Â Â Â mcgen,
>>> +)
>>> Â from .gen import QAPISchemaModularCVisitor, ifcontext
>>> Â from .schema import QAPISchemaEnumMember
>>> Â from .types import gen_enum, gen_enum_lookup
>>> diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
>>> index 8df19a0df0..11472ba043 100644
>>> --- a/scripts/qapi/gen.py
>>> +++ b/scripts/qapi/gen.py
>>> @@ -11,13 +11,19 @@
>>> Â # This work is licensed under the terms of the GNU GPL, version 2.
>>> Â # See the COPYING file in the top-level directory.
>>> -
>>> +from contextlib import contextmanager
>>> Â import errno
>>> Â import os
>>> Â import re
>>> -from contextlib import contextmanager
>>> -from .common import *
>>> +from .common import (
>>> +Â Â Â c_fname,
>>> +Â Â Â gen_endif,
>>> +Â Â Â gen_if,
>>> +Â Â Â guardend,
>>> +Â Â Â guardstart,
>>> +Â Â Â mcgen,
>>> +)
>>> Â from .schema import QAPISchemaVisitor
>>> diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py
>>> index 2a34cd1e8e..b036fcf9ce 100644
>>> --- a/scripts/qapi/introspect.py
>>> +++ b/scripts/qapi/introspect.py
>>> @@ -10,7 +10,12 @@
>>> Â See the COPYING file in the top-level directory.
>>> Â """
>>> -from .common import *
>>> +from .common import (
>>> +Â Â Â c_name,
>>> +Â Â Â gen_endif,
>>> +Â Â Â gen_if,
>>> +Â Â Â mcgen,
>>> +)
>>> Â from .gen import QAPISchemaMonolithicCVisitor
>>> Â from .schema import (QAPISchemaArrayType, QAPISchemaBuiltinType,
>>> Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â QAPISchemaType)
>>> diff --git a/scripts/qapi/types.py b/scripts/qapi/types.py
>>> index ca9a5aacb3..53b47f9e58 100644
>>> --- a/scripts/qapi/types.py
>>> +++ b/scripts/qapi/types.py
>>> @@ -13,7 +13,13 @@
>>> Â # See the COPYING file in the top-level directory.
>>> Â """
>>> -from .common import *
>>> +from .common import (
>>> +Â Â Â c_enum_const,
>>> +Â Â Â c_name,
>>> +Â Â Â gen_endif,
>>> +Â Â Â gen_if,
>>> +Â Â Â mcgen,
>>> +)
>>> Â from .gen import QAPISchemaModularCVisitor, ifcontext
>>> Â from .schema import QAPISchemaEnumMember, QAPISchemaObjectType
>>> diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py
>>> index 7850f6e848..ea277e7704 100644
>>> --- a/scripts/qapi/visit.py
>>> +++ b/scripts/qapi/visit.py
>>> @@ -13,7 +13,15 @@
>>> Â See the COPYING file in the top-level directory.
>>> Â """
>>> -from .common import *
>>> +from .common import (
>>> +Â Â Â c_enum_const,
>>> +Â Â Â c_name,
>>> +Â Â Â gen_endif,
>>> +Â Â Â gen_if,
>>> +Â Â Â mcgen,
>>> +Â Â Â pop_indent,
>>> +Â Â Â push_indent,
>>> +)
>>
>> And here, isort will add the paranthesis (it does so based on space
>> demands):
>>
>> Â Â Â from .common import (c_enum_const, c_name, gen_endif, gen_if, mcgen,
>> Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â pop_indent, push_indent)
>> Â Â Â from .gen import QAPISchemaModularCVisitor, ifcontext
>> Â Â Â from .schema import QAPISchemaObjectType
>>
>> Other than those suggestions, it LGTM.
>>
>
> OK. We can add a check that asserts that isort(file) == file to keep our
> include regimes consistent. I'll look into the tool, but it will be
> after this marathon of a series.
>
> Here's a gitlab issue I made on my QEMU fork to help me keep track of
> Python-related issues that I intend to use:
> https://gitlab.com/jsnow/qemu/-/issues/6
>
I've found that
`isort --force-sort-within-sections --force-grid-wrap 4 --multi-line 3
--trailing-comma`
is pretty close to what I was already doing, so I'll adopt this for the
respin on good faith that nobody will retract an R-B for new import
orderings.
force sort: I prefer to sort by module, so I intersperse "from x" and
"import x" style in one section. This keeps the module reference
absolutely consistent regardless of HOW we import from it.
force grid: 4 or more imports from a module will get wrapped using the
one-per-line style.
multi-line: '3' just refers to the one-per-line style of imports that I
already use.
trailing comma: A little buddy that hangs out with you.
>> Reviewed-by: Cleber Rosa <crosa@redhat.com>
>>
>
> Thanks!
>
> --js
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 06/38] qapi: delint using flake8
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (4 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 05/38] qapi: Remove wildcard includes John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-22 21:43 ` Eduardo Habkost
2020-09-23 13:37 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 07/38] qapi: add pylintrc John Snow
` (32 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Petty style guide fixes and line length enforcement. Not a big win, not
a big loss, but flake8 passes 100% on the qapi module, which gives us an
easy baseline to enforce hereafter.
A note on the flake8 exception: flake8 will warn on *any* bare except,
but pylint's is context-aware and will suppress the warning if you
re-raise the exception.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/.flake8 | 2 ++
scripts/qapi/commands.py | 3 ++-
scripts/qapi/schema.py | 8 +++++---
scripts/qapi/visit.py | 16 +++++++++++-----
4 files changed, 20 insertions(+), 9 deletions(-)
create mode 100644 scripts/qapi/.flake8
diff --git a/scripts/qapi/.flake8 b/scripts/qapi/.flake8
new file mode 100644
index 0000000000..6b158c68b8
--- /dev/null
+++ b/scripts/qapi/.flake8
@@ -0,0 +1,2 @@
+[flake8]
+extend-ignore = E722 # Prefer pylint's bare-except checks to flake8's
diff --git a/scripts/qapi/commands.py b/scripts/qapi/commands.py
index e1df0e341f..d9cda3b209 100644
--- a/scripts/qapi/commands.py
+++ b/scripts/qapi/commands.py
@@ -69,7 +69,8 @@ def gen_call(name, arg_type, boxed, ret_type):
def gen_marshal_output(ret_type):
return mcgen('''
-static void qmp_marshal_output_%(c_name)s(%(c_type)s ret_in, QObject **ret_out, Error **errp)
+static void qmp_marshal_output_%(c_name)s(%(c_type)s ret_in,
+ QObject **ret_out, Error **errp)
{
Visitor *v;
diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py
index a835ee6fde..b77e0e56b2 100644
--- a/scripts/qapi/schema.py
+++ b/scripts/qapi/schema.py
@@ -536,7 +536,7 @@ def set_defined_in(self, name):
v.set_defined_in(name)
def check(self, schema, seen):
- if not self.tag_member: # flat union
+ if not self.tag_member: # flat union
self.tag_member = seen.get(c_name(self._tag_name))
base = "'base'"
# Pointing to the base type when not implicit would be
@@ -821,7 +821,7 @@ def __init__(self, fname):
self._entity_dict = {}
self._module_dict = OrderedDict()
self._schema_dir = os.path.dirname(fname)
- self._make_module(None) # built-ins
+ self._make_module(None) # built-ins
self._make_module(fname)
self._predefining = True
self._def_predefineds()
@@ -965,7 +965,9 @@ def _make_implicit_object_type(self, name, info, ifcond, role, members):
# But it's not tight: the disjunction need not imply it. We
# may end up compiling useless wrapper types.
# TODO kill simple unions or implement the disjunction
- assert (ifcond or []) == typ._ifcond # pylint: disable=protected-access
+
+ # pylint: disable=protected-access
+ assert (ifcond or []) == typ._ifcond
else:
self._def_entity(QAPISchemaObjectType(
name, info, None, ifcond, None, None, members, None))
diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py
index ea277e7704..808410d6f1 100644
--- a/scripts/qapi/visit.py
+++ b/scripts/qapi/visit.py
@@ -31,7 +31,9 @@ def gen_visit_decl(name, scalar=False):
if not scalar:
c_type += '*'
return mcgen('''
-bool visit_type_%(c_name)s(Visitor *v, const char *name, %(c_type)sobj, Error **errp);
+
+bool visit_type_%(c_name)s(Visitor *v, const char *name,
+ %(c_type)sobj, Error **errp);
''',
c_name=c_name(name), c_type=c_type)
@@ -125,7 +127,8 @@ def gen_visit_object_members(name, base, members, variants):
def gen_visit_list(name, element_type):
return mcgen('''
-bool visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s **obj, Error **errp)
+bool visit_type_%(c_name)s(Visitor *v, const char *name,
+ %(c_name)s **obj, Error **errp)
{
bool ok = false;
%(c_name)s *tail;
@@ -158,7 +161,8 @@ def gen_visit_list(name, element_type):
def gen_visit_enum(name):
return mcgen('''
-bool visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s *obj, Error **errp)
+bool visit_type_%(c_name)s(Visitor *v, const char *name,
+ %(c_name)s *obj, Error **errp)
{
int value = *obj;
bool ok = visit_type_enum(v, name, &value, &%(c_name)s_lookup, errp);
@@ -172,7 +176,8 @@ def gen_visit_enum(name):
def gen_visit_alternate(name, variants):
ret = mcgen('''
-bool visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s **obj, Error **errp)
+bool visit_type_%(c_name)s
+ (Visitor *v, const char *name, %(c_name)s **obj, Error **errp)
{
bool ok = false;
@@ -247,7 +252,8 @@ def gen_visit_alternate(name, variants):
def gen_visit_object(name, base, members, variants):
return mcgen('''
-bool visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s **obj, Error **errp)
+bool visit_type_%(c_name)s
+ (Visitor *v, const char *name, %(c_name)s **obj, Error **errp)
{
bool ok = false;
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 06/38] qapi: delint using flake8
2020-09-22 21:00 ` [PATCH v2 06/38] qapi: delint using flake8 John Snow
@ 2020-09-22 21:43 ` Eduardo Habkost
2020-09-23 13:37 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-22 21:43 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:29PM -0400, John Snow wrote:
> Petty style guide fixes and line length enforcement. Not a big win, not
> a big loss, but flake8 passes 100% on the qapi module, which gives us an
> easy baseline to enforce hereafter.
>
> A note on the flake8 exception: flake8 will warn on *any* bare except,
> but pylint's is context-aware and will suppress the warning if you
> re-raise the exception.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 06/38] qapi: delint using flake8
2020-09-22 21:00 ` [PATCH v2 06/38] qapi: delint using flake8 John Snow
2020-09-22 21:43 ` Eduardo Habkost
@ 2020-09-23 13:37 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 13:37 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 524 bytes --]
On Tue, Sep 22, 2020 at 05:00:29PM -0400, John Snow wrote:
> Petty style guide fixes and line length enforcement. Not a big win, not
> a big loss, but flake8 passes 100% on the qapi module, which gives us an
> easy baseline to enforce hereafter.
>
> A note on the flake8 exception: flake8 will warn on *any* bare except,
> but pylint's is context-aware and will suppress the warning if you
> re-raise the exception.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 07/38] qapi: add pylintrc
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (5 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 06/38] qapi: delint using flake8 John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-22 21:54 ` Eduardo Habkost
2020-09-23 13:42 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 08/38] qapi/common.py: Remove python compatibility workaround John Snow
` (31 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Using `pylint --generate-rcfile > pylintrc`, generate a skeleton
pylintrc file. Sections that are not presently relevant (by the end of
this series) are removed leaving just the empty section as a search
engine / documentation hint to future authors.
Right now, quite a few modules are ignored as they are known to fail as
of this commit. modules will be removed from the known-bad list
throughout this and following series as they are repaired.
Note: Normally, pylintrc would go in the folder above the module, but as
that folder is shared by many things, it is going inside the module
folder (for now). Due to a bug in pylint 2.5.x, pylint does not
correctly recognize when it is being run from "inside" a package, and
must be run *outside* of the package.
Therefore, to run it, you must:
> pylint scripts/qapi/ --rcfile=scripts/qapi/pylintrc
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/pylintrc | 74 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 74 insertions(+)
create mode 100644 scripts/qapi/pylintrc
diff --git a/scripts/qapi/pylintrc b/scripts/qapi/pylintrc
new file mode 100644
index 0000000000..02dd22c863
--- /dev/null
+++ b/scripts/qapi/pylintrc
@@ -0,0 +1,74 @@
+[MASTER]
+
+# Add files or directories matching the regex patterns to the ignore list.
+# The regex matches against base names, not paths.
+ignore-patterns=common.py,
+ doc.py,
+ error.py,
+ expr.py,
+ gen.py,
+ parser.py,
+ schema.py,
+ source.py,
+ types.py,
+ visit.py,
+
+
+[MESSAGES CONTROL]
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=fixme,
+ missing-docstring,
+ too-many-arguments,
+ too-many-branches,
+ too-many-statements,
+ too-many-instance-attributes,
+
+[REPORTS]
+
+[REFACTORING]
+
+[MISCELLANEOUS]
+
+[LOGGING]
+
+[BASIC]
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+ j,
+ k,
+ ex,
+ Run,
+ _
+
+[VARIABLES]
+
+[STRING]
+
+[SPELLING]
+
+[FORMAT]
+
+[SIMILARITIES]
+
+# Ignore import statements themselves when computing similarities.
+ignore-imports=yes
+
+[TYPECHECK]
+
+[CLASSES]
+
+[IMPORTS]
+
+[DESIGN]
+
+[EXCEPTIONS]
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 07/38] qapi: add pylintrc
2020-09-22 21:00 ` [PATCH v2 07/38] qapi: add pylintrc John Snow
@ 2020-09-22 21:54 ` Eduardo Habkost
2020-09-23 13:42 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-22 21:54 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:30PM -0400, John Snow wrote:
> Using `pylint --generate-rcfile > pylintrc`, generate a skeleton
> pylintrc file. Sections that are not presently relevant (by the end of
> this series) are removed leaving just the empty section as a search
> engine / documentation hint to future authors.
>
> Right now, quite a few modules are ignored as they are known to fail as
> of this commit. modules will be removed from the known-bad list
> throughout this and following series as they are repaired.
>
> Note: Normally, pylintrc would go in the folder above the module, but as
> that folder is shared by many things, it is going inside the module
> folder (for now). Due to a bug in pylint 2.5.x, pylint does not
> correctly recognize when it is being run from "inside" a package, and
> must be run *outside* of the package.
>
> Therefore, to run it, you must:
>
> > pylint scripts/qapi/ --rcfile=scripts/qapi/pylintrc
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Tested-by: Eduardo Habkost <ehabkost@redhat.com>
This doesn't generate any warnings after this patch, but at the
end of your -pt6 branch I got some pylint warnings. I am
bisecting it to try to identify the patch where the warnings are
introduced.
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 07/38] qapi: add pylintrc
2020-09-22 21:00 ` [PATCH v2 07/38] qapi: add pylintrc John Snow
2020-09-22 21:54 ` Eduardo Habkost
@ 2020-09-23 13:42 ` Cleber Rosa
2020-09-23 17:23 ` John Snow
1 sibling, 1 reply; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 13:42 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 1469 bytes --]
On Tue, Sep 22, 2020 at 05:00:30PM -0400, John Snow wrote:
> Using `pylint --generate-rcfile > pylintrc`, generate a skeleton
> pylintrc file. Sections that are not presently relevant (by the end of
> this series) are removed leaving just the empty section as a search
> engine / documentation hint to future authors.
>
> Right now, quite a few modules are ignored as they are known to fail as
> of this commit. modules will be removed from the known-bad list
> throughout this and following series as they are repaired.
>
> Note: Normally, pylintrc would go in the folder above the module, but as
> that folder is shared by many things, it is going inside the module
> folder (for now). Due to a bug in pylint 2.5.x, pylint does not
> correctly recognize when it is being run from "inside" a package, and
> must be run *outside* of the package.
>
> Therefore, to run it, you must:
>
> > pylint scripts/qapi/ --rcfile=scripts/qapi/pylintrc
>
> Signed-off-by: John Snow <jsnow@redhat.com>
One concern I have here is that the pylint version is not defined.
Based on experience, different pylint will behave differently, because
among other things, it may introduce new checks.
I'd at the very least document the pylint version used in the commit
message, until a "requirements.txt"-like solution pinning a version is
given.
Other than that,
Reviewed-by: Cleber Rosa <crosa@redhat.com>
Tested-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 07/38] qapi: add pylintrc
2020-09-23 13:42 ` Cleber Rosa
@ 2020-09-23 17:23 ` John Snow
2020-09-24 19:29 ` Cleber Rosa
0 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-23 17:23 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
On 9/23/20 9:42 AM, Cleber Rosa wrote:
> On Tue, Sep 22, 2020 at 05:00:30PM -0400, John Snow wrote:
>> Using `pylint --generate-rcfile > pylintrc`, generate a skeleton
>> pylintrc file. Sections that are not presently relevant (by the end of
>> this series) are removed leaving just the empty section as a search
>> engine / documentation hint to future authors.
>>
>> Right now, quite a few modules are ignored as they are known to fail as
>> of this commit. modules will be removed from the known-bad list
>> throughout this and following series as they are repaired.
>>
>> Note: Normally, pylintrc would go in the folder above the module, but as
>> that folder is shared by many things, it is going inside the module
>> folder (for now). Due to a bug in pylint 2.5.x, pylint does not
>> correctly recognize when it is being run from "inside" a package, and
>> must be run *outside* of the package.
>>
>> Therefore, to run it, you must:
>>
>> > pylint scripts/qapi/ --rcfile=scripts/qapi/pylintrc
>>
>> Signed-off-by: John Snow <jsnow@redhat.com>
>
> One concern I have here is that the pylint version is not defined.
> Based on experience, different pylint will behave differently, because
> among other things, it may introduce new checks.
>
> I'd at the very least document the pylint version used in the commit
> message, until a "requirements.txt"-like solution pinning a version is
> given.
>
> Other than that,
>
> Reviewed-by: Cleber Rosa <crosa@redhat.com>
> Tested-by: Cleber Rosa <crosa@redhat.com>
>
Alright, I'll put it in the commit message itself instead of in the
cover letter.
The next step is to re-engage on that Makefile patch that I was working
on for ./python/qemu and introduce it here too, which will document the
pinned versions correctly.
--js
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 07/38] qapi: add pylintrc
2020-09-23 17:23 ` John Snow
@ 2020-09-24 19:29 ` Cleber Rosa
0 siblings, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-24 19:29 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 2177 bytes --]
On Wed, Sep 23, 2020 at 01:23:56PM -0400, John Snow wrote:
> On 9/23/20 9:42 AM, Cleber Rosa wrote:
> > On Tue, Sep 22, 2020 at 05:00:30PM -0400, John Snow wrote:
> > > Using `pylint --generate-rcfile > pylintrc`, generate a skeleton
> > > pylintrc file. Sections that are not presently relevant (by the end of
> > > this series) are removed leaving just the empty section as a search
> > > engine / documentation hint to future authors.
> > >
> > > Right now, quite a few modules are ignored as they are known to fail as
> > > of this commit. modules will be removed from the known-bad list
> > > throughout this and following series as they are repaired.
> > >
> > > Note: Normally, pylintrc would go in the folder above the module, but as
> > > that folder is shared by many things, it is going inside the module
> > > folder (for now). Due to a bug in pylint 2.5.x, pylint does not
> > > correctly recognize when it is being run from "inside" a package, and
> > > must be run *outside* of the package.
> > >
> > > Therefore, to run it, you must:
> > >
> > > > pylint scripts/qapi/ --rcfile=scripts/qapi/pylintrc
> > >
> > > Signed-off-by: John Snow <jsnow@redhat.com>
> >
> > One concern I have here is that the pylint version is not defined.
> > Based on experience, different pylint will behave differently, because
> > among other things, it may introduce new checks.
> >
> > I'd at the very least document the pylint version used in the commit
> > message, until a "requirements.txt"-like solution pinning a version is
> > given.
> >
> > Other than that,
> >
> > Reviewed-by: Cleber Rosa <crosa@redhat.com>
> > Tested-by: Cleber Rosa <crosa@redhat.com>
> >
>
> Alright, I'll put it in the commit message itself instead of in the cover
> letter.
>
I missed that info on the cover letter, so my apologies. But still, I
think it's a good idea to have that preserved in the repo history
indeed.
> The next step is to re-engage on that Makefile patch that I was working on
> for ./python/qemu and introduce it here too, which will document the pinned
> versions correctly.
>
> --js
Agreed!
- Cleber.
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 08/38] qapi/common.py: Remove python compatibility workaround
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (6 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 07/38] qapi: add pylintrc John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-22 22:05 ` Eduardo Habkost
2020-09-23 14:37 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 09/38] qapi/common.py: Add indent manager John Snow
` (30 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/common.py | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
index ba35abea47..cee63eb95c 100644
--- a/scripts/qapi/common.py
+++ b/scripts/qapi/common.py
@@ -119,10 +119,7 @@ def cgen(code, **kwds):
raw = code % kwds
if indent_level:
indent = genindent(indent_level)
- # re.subn() lacks flags support before Python 2.7, use re.compile()
- raw = re.subn(re.compile(r'^(?!(#|$))', re.MULTILINE),
- indent, raw)
- raw = raw[0]
+ raw = re.sub(r'^(?!(#|$))', indent, raw, flags=re.MULTILINE)
return re.sub(re.escape(eatspace) + r' *', '', raw)
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 08/38] qapi/common.py: Remove python compatibility workaround
2020-09-22 21:00 ` [PATCH v2 08/38] qapi/common.py: Remove python compatibility workaround John Snow
@ 2020-09-22 22:05 ` Eduardo Habkost
2020-09-23 14:37 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-22 22:05 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:31PM -0400, John Snow wrote:
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 08/38] qapi/common.py: Remove python compatibility workaround
2020-09-22 21:00 ` [PATCH v2 08/38] qapi/common.py: Remove python compatibility workaround John Snow
2020-09-22 22:05 ` Eduardo Habkost
@ 2020-09-23 14:37 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 14:37 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 150 bytes --]
On Tue, Sep 22, 2020 at 05:00:31PM -0400, John Snow wrote:
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 09/38] qapi/common.py: Add indent manager
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (7 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 08/38] qapi/common.py: Remove python compatibility workaround John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-22 22:22 ` Eduardo Habkost
2020-09-23 14:55 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 10/38] qapi/common.py: delint with pylint John Snow
` (29 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Code style tools really dislike the use of global keywords, because it
generally involves re-binding the name at runtime which can have strange
effects depending on when and how that global name is referenced in
other modules.
Make a little indent level manager instead.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/common.py | 51 +++++++++++++++++++++++++++++-------------
scripts/qapi/visit.py | 7 +++---
2 files changed, 38 insertions(+), 20 deletions(-)
diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
index cee63eb95c..e0c5871b10 100644
--- a/scripts/qapi/common.py
+++ b/scripts/qapi/common.py
@@ -93,33 +93,52 @@ def c_name(name, protect=True):
pointer_suffix = ' *' + eatspace
-def genindent(count):
- ret = ''
- for _ in range(count):
- ret += ' '
- return ret
+class Indentation:
+ """
+ Indentation level management.
+ :param initial: Initial number of spaces, default 0.
+ """
+ def __init__(self, initial: int = 0) -> None:
+ self._level = initial
-indent_level = 0
+ def __int__(self) -> int:
+ return self._level
+ def __repr__(self) -> str:
+ return "{}({:d})".format(type(self).__name__, self._level)
-def push_indent(indent_amount=4):
- global indent_level
- indent_level += indent_amount
+ def __str__(self) -> str:
+ """Return the current indentation as a string of spaces."""
+ return ' ' * self._level
+ def __bool__(self) -> bool:
+ """True when there is a non-zero indentation."""
+ return bool(self._level)
-def pop_indent(indent_amount=4):
- global indent_level
- indent_level -= indent_amount
+ def increase(self, amount: int = 4) -> int:
+ """Increase the indentation level by `amount`, default 4."""
+ self._level += amount
+ return self._level
+
+ def decrease(self, amount: int = 4) -> int:
+ """Decrease the indentation level by `amount`, default 4."""
+ if self._level < amount:
+ raise ArithmeticError(
+ f"Can't remove {amount:d} spaces from {self!r}")
+ self._level -= amount
+ return self._level
+
+
+indent = Indentation()
# Generate @code with @kwds interpolated.
-# Obey indent_level, and strip eatspace.
+# Obey indent, and strip eatspace.
def cgen(code, **kwds):
raw = code % kwds
- if indent_level:
- indent = genindent(indent_level)
- raw = re.sub(r'^(?!(#|$))', indent, raw, flags=re.MULTILINE)
+ if indent:
+ raw = re.sub(r'^(?!(#|$))', str(indent), raw, flags=re.MULTILINE)
return re.sub(re.escape(eatspace) + r' *', '', raw)
diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py
index 808410d6f1..4edaee33e3 100644
--- a/scripts/qapi/visit.py
+++ b/scripts/qapi/visit.py
@@ -19,8 +19,7 @@
gen_endif,
gen_if,
mcgen,
- pop_indent,
- push_indent,
+ indent,
)
from .gen import QAPISchemaModularCVisitor, ifcontext
from .schema import QAPISchemaObjectType
@@ -69,7 +68,7 @@ def gen_visit_object_members(name, base, members, variants):
if (visit_optional(v, "%(name)s", &obj->has_%(c_name)s)) {
''',
name=memb.name, c_name=c_name(memb.name))
- push_indent()
+ indent.increase()
ret += mcgen('''
if (!visit_type_%(c_type)s(v, "%(name)s", &obj->%(c_name)s, errp)) {
return false;
@@ -78,7 +77,7 @@ def gen_visit_object_members(name, base, members, variants):
c_type=memb.type.c_name(), name=memb.name,
c_name=c_name(memb.name))
if memb.optional:
- pop_indent()
+ indent.decrease()
ret += mcgen('''
}
''')
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 09/38] qapi/common.py: Add indent manager
2020-09-22 21:00 ` [PATCH v2 09/38] qapi/common.py: Add indent manager John Snow
@ 2020-09-22 22:22 ` Eduardo Habkost
2020-09-23 17:29 ` John Snow
2020-09-23 14:55 ` Cleber Rosa
1 sibling, 1 reply; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-22 22:22 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:32PM -0400, John Snow wrote:
> Code style tools really dislike the use of global keywords, because it
> generally involves re-binding the name at runtime which can have strange
> effects depending on when and how that global name is referenced in
> other modules.
>
> Make a little indent level manager instead.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/common.py | 51 +++++++++++++++++++++++++++++-------------
> scripts/qapi/visit.py | 7 +++---
> 2 files changed, 38 insertions(+), 20 deletions(-)
>
> diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
> index cee63eb95c..e0c5871b10 100644
> --- a/scripts/qapi/common.py
> +++ b/scripts/qapi/common.py
> @@ -93,33 +93,52 @@ def c_name(name, protect=True):
> pointer_suffix = ' *' + eatspace
>
>
> -def genindent(count):
> - ret = ''
> - for _ in range(count):
> - ret += ' '
> - return ret
> +class Indentation:
> + """
> + Indentation level management.
>
> + :param initial: Initial number of spaces, default 0.
> + """
> + def __init__(self, initial: int = 0) -> None:
> + self._level = initial
>
> -indent_level = 0
> + def __int__(self) -> int:
> + return self._level
>
> + def __repr__(self) -> str:
> + return "{}({:d})".format(type(self).__name__, self._level)
>
> -def push_indent(indent_amount=4):
> - global indent_level
> - indent_level += indent_amount
> + def __str__(self) -> str:
> + """Return the current indentation as a string of spaces."""
> + return ' ' * self._level
>
> + def __bool__(self) -> bool:
> + """True when there is a non-zero indentation."""
> + return bool(self._level)
>
> -def pop_indent(indent_amount=4):
> - global indent_level
> - indent_level -= indent_amount
> + def increase(self, amount: int = 4) -> int:
> + """Increase the indentation level by `amount`, default 4."""
> + self._level += amount
> + return self._level
> +
> + def decrease(self, amount: int = 4) -> int:
> + """Decrease the indentation level by `amount`, default 4."""
> + if self._level < amount:
> + raise ArithmeticError(
> + f"Can't remove {amount:d} spaces from {self!r}")
> + self._level -= amount
> + return self._level
> +
> +
> +indent = Indentation()
Personally, I would keep the push_indent(), pop_indent() API, and
introduce an indent() function, to follow the existing API style
of plain functions.
Something like:
indent_prefixes = []
def push_indent(amount=4):
"""Add `amount` spaces to indentation prefix"""
indent_prefixes.push(' '*amount)
def pop_indent():
"""Revert the most recent push_indent() call"""
indent_prefixes.pop()
def indent():
"""Return the current indentation prefix"""
return ''.join(indent_prefixes)
What you did is still an improvement, though, so:
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 09/38] qapi/common.py: Add indent manager
2020-09-22 22:22 ` Eduardo Habkost
@ 2020-09-23 17:29 ` John Snow
2020-09-25 11:51 ` Markus Armbruster
0 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-23 17:29 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, Michael Roth, qemu-devel, Markus Armbruster,
Cleber Rosa, Alex Bennée
On 9/22/20 6:22 PM, Eduardo Habkost wrote:
> On Tue, Sep 22, 2020 at 05:00:32PM -0400, John Snow wrote:
>> Code style tools really dislike the use of global keywords, because it
>> generally involves re-binding the name at runtime which can have strange
>> effects depending on when and how that global name is referenced in
>> other modules.
>>
>> Make a little indent level manager instead.
>>
>> Signed-off-by: John Snow <jsnow@redhat.com>
>> ---
>> scripts/qapi/common.py | 51 +++++++++++++++++++++++++++++-------------
>> scripts/qapi/visit.py | 7 +++---
>> 2 files changed, 38 insertions(+), 20 deletions(-)
>>
>> diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
>> index cee63eb95c..e0c5871b10 100644
>> --- a/scripts/qapi/common.py
>> +++ b/scripts/qapi/common.py
>> @@ -93,33 +93,52 @@ def c_name(name, protect=True):
>> pointer_suffix = ' *' + eatspace
>>
>>
>> -def genindent(count):
>> - ret = ''
>> - for _ in range(count):
>> - ret += ' '
>> - return ret
>> +class Indentation:
>> + """
>> + Indentation level management.
>>
>> + :param initial: Initial number of spaces, default 0.
>> + """
>> + def __init__(self, initial: int = 0) -> None:
>> + self._level = initial
>>
>> -indent_level = 0
>> + def __int__(self) -> int:
>> + return self._level
>>
>> + def __repr__(self) -> str:
>> + return "{}({:d})".format(type(self).__name__, self._level)
>>
>> -def push_indent(indent_amount=4):
>> - global indent_level
>> - indent_level += indent_amount
>> + def __str__(self) -> str:
>> + """Return the current indentation as a string of spaces."""
>> + return ' ' * self._level
>>
>> + def __bool__(self) -> bool:
>> + """True when there is a non-zero indentation."""
>> + return bool(self._level)
>>
>> -def pop_indent(indent_amount=4):
>> - global indent_level
>> - indent_level -= indent_amount
>> + def increase(self, amount: int = 4) -> int:
>> + """Increase the indentation level by `amount`, default 4."""
>> + self._level += amount
>> + return self._level
>> +
>> + def decrease(self, amount: int = 4) -> int:
>> + """Decrease the indentation level by `amount`, default 4."""
>> + if self._level < amount:
>> + raise ArithmeticError(
>> + f"Can't remove {amount:d} spaces from {self!r}")
>> + self._level -= amount
>> + return self._level
>> +
>> +
>> +indent = Indentation()
>
> Personally, I would keep the push_indent(), pop_indent() API, and
> introduce an indent() function, to follow the existing API style
> of plain functions.
>
> Something like:
>
> indent_prefixes = []
> def push_indent(amount=4):
> """Add `amount` spaces to indentation prefix"""
> indent_prefixes.push(' '*amount)
> def pop_indent():
> """Revert the most recent push_indent() call"""
> indent_prefixes.pop()
> def indent():
> """Return the current indentation prefix"""
> return ''.join(indent_prefixes)
>
> What you did is still an improvement, though, so:
>
> Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
>
Yeah, there's only one user right now, so ... I just kinda wanted to get
rid of the global usage. Maybe if we make the code generator fancier
we'll find out what form is best.
--js
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 09/38] qapi/common.py: Add indent manager
2020-09-23 17:29 ` John Snow
@ 2020-09-25 11:51 ` Markus Armbruster
2020-09-25 13:13 ` Eduardo Habkost
2020-09-25 14:42 ` John Snow
0 siblings, 2 replies; 190+ messages in thread
From: Markus Armbruster @ 2020-09-25 11:51 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Cleber Rosa, Alex Bennée
John Snow <jsnow@redhat.com> writes:
> On 9/22/20 6:22 PM, Eduardo Habkost wrote:
>> On Tue, Sep 22, 2020 at 05:00:32PM -0400, John Snow wrote:
>>> Code style tools really dislike the use of global keywords, because it
>>> generally involves re-binding the name at runtime which can have strange
>>> effects depending on when and how that global name is referenced in
>>> other modules.
>>>
>>> Make a little indent level manager instead.
>>>
>>> Signed-off-by: John Snow <jsnow@redhat.com>
>>> ---
>>> scripts/qapi/common.py | 51 +++++++++++++++++++++++++++++-------------
>>> scripts/qapi/visit.py | 7 +++---
>>> 2 files changed, 38 insertions(+), 20 deletions(-)
>>>
>>> diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
>>> index cee63eb95c..e0c5871b10 100644
>>> --- a/scripts/qapi/common.py
>>> +++ b/scripts/qapi/common.py
>>> @@ -93,33 +93,52 @@ def c_name(name, protect=True):
>>> pointer_suffix = ' *' + eatspace
>>>
>>> -def genindent(count):
>>> - ret = ''
>>> - for _ in range(count):
>>> - ret += ' '
>>> - return ret
>>> +class Indentation:
>>> + """
>>> + Indentation level management.
>>> + :param initial: Initial number of spaces, default 0.
>>> + """
>>> + def __init__(self, initial: int = 0) -> None:
>>> + self._level = initial
>>> -indent_level = 0
>>> + def __int__(self) -> int:
>>> + return self._level
>>> + def __repr__(self) -> str:
>>> + return "{}({:d})".format(type(self).__name__, self._level)
>>> -def push_indent(indent_amount=4):
>>> - global indent_level
>>> - indent_level += indent_amount
>>> + def __str__(self) -> str:
>>> + """Return the current indentation as a string of spaces."""
>>> + return ' ' * self._level
>>> + def __bool__(self) -> bool:
>>> + """True when there is a non-zero indentation."""
>>> + return bool(self._level)
>>> -def pop_indent(indent_amount=4):
>>> - global indent_level
>>> - indent_level -= indent_amount
>>> + def increase(self, amount: int = 4) -> int:
>>> + """Increase the indentation level by `amount`, default 4."""
>>> + self._level += amount
>>> + return self._level
>>> +
>>> + def decrease(self, amount: int = 4) -> int:
>>> + """Decrease the indentation level by `amount`, default 4."""
>>> + if self._level < amount:
>>> + raise ArithmeticError(
>>> + f"Can't remove {amount:d} spaces from {self!r}")
>>> + self._level -= amount
>>> + return self._level
>>> +
>>> +
>>> +indent = Indentation()
>> Personally, I would keep the push_indent(), pop_indent() API, and
>> introduce an indent() function, to follow the existing API style
>> of plain functions.
>> Something like:
>> indent_prefixes = []
>> def push_indent(amount=4):
>> """Add `amount` spaces to indentation prefix"""
>> indent_prefixes.push(' '*amount)
>> def pop_indent():
>> """Revert the most recent push_indent() call"""
>> indent_prefixes.pop()
>> def indent():
>> """Return the current indentation prefix"""
>> return ''.join(indent_prefixes)
>> What you did is still an improvement, though, so:
>> Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
>>
>
> Yeah, there's only one user right now, so ... I just kinda wanted to
> get rid of the global usage. Maybe if we make the code generator
> fancier we'll find out what form is best.
You don't get rid of the global variable, you just change it from
integer to a class. A class can be handier when generating multiple
things interleaved, because you can have one class instance per thing.
Note that we already have a class instance per thing we generate:
instances of subtypes of QAPIGen. The thought of moving the indentation
machinery into QAPIGen or or maybe QAPIGenCCode crossed my mind many
moons ago, but I had bigger fish to fry, and then I forgot :)
John, I suggest you don't try to make this pretty just yet. Do what
needs to be done for the type hint job. We can make it pretty later.
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 09/38] qapi/common.py: Add indent manager
2020-09-25 11:51 ` Markus Armbruster
@ 2020-09-25 13:13 ` Eduardo Habkost
2020-09-25 13:47 ` Markus Armbruster
2020-09-25 14:42 ` John Snow
1 sibling, 1 reply; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-25 13:13 UTC (permalink / raw)
To: Markus Armbruster
Cc: Peter Maydell, John Snow, Michael Roth, qemu-devel, Cleber Rosa,
Alex Bennée
On Fri, Sep 25, 2020 at 01:51:54PM +0200, Markus Armbruster wrote:
> John Snow <jsnow@redhat.com> writes:
> > On 9/22/20 6:22 PM, Eduardo Habkost wrote:
[...]
> > Yeah, there's only one user right now, so ... I just kinda wanted to
> > get rid of the global usage. Maybe if we make the code generator
> > fancier we'll find out what form is best.
>
> You don't get rid of the global variable, you just change it from
> integer to a class. A class can be handier when generating multiple
> things interleaved, because you can have one class instance per thing.
To be fair, John doesn't claim to be getting rid of a global
variable. He's getting rid of usage of the 'global' keyword to
make linters happier. There are ways to do that without changing
the code too much, though.
>
> Note that we already have a class instance per thing we generate:
> instances of subtypes of QAPIGen. The thought of moving the indentation
> machinery into QAPIGen or or maybe QAPIGenCCode crossed my mind many
> moons ago, but I had bigger fish to fry, and then I forgot :)
>
> John, I suggest you don't try to make this pretty just yet. Do what
> needs to be done for the type hint job. We can make it pretty later.
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 09/38] qapi/common.py: Add indent manager
2020-09-25 13:13 ` Eduardo Habkost
@ 2020-09-25 13:47 ` Markus Armbruster
0 siblings, 0 replies; 190+ messages in thread
From: Markus Armbruster @ 2020-09-25 13:47 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, John Snow, Michael Roth, qemu-devel, Cleber Rosa,
Alex Bennée
Eduardo Habkost <ehabkost@redhat.com> writes:
> On Fri, Sep 25, 2020 at 01:51:54PM +0200, Markus Armbruster wrote:
>> John Snow <jsnow@redhat.com> writes:
>> > On 9/22/20 6:22 PM, Eduardo Habkost wrote:
> [...]
>> > Yeah, there's only one user right now, so ... I just kinda wanted to
>> > get rid of the global usage. Maybe if we make the code generator
>> > fancier we'll find out what form is best.
>>
>> You don't get rid of the global variable, you just change it from
>> integer to a class. A class can be handier when generating multiple
>> things interleaved, because you can have one class instance per thing.
>
> To be fair, John doesn't claim to be getting rid of a global
> variable. He's getting rid of usage of the 'global' keyword to
> make linters happier.
True.
> There are ways to do that without changing
> the code too much, though.
Let's do something easy and cheap, because ...
>> Note that we already have a class instance per thing we generate:
>> instances of subtypes of QAPIGen. The thought of moving the indentation
>> machinery into QAPIGen or or maybe QAPIGenCCode crossed my mind many
>> moons ago, but I had bigger fish to fry, and then I forgot :)
... we'll probably want to move this stuff into QAPIGen later, and
that's when we should make it pretty.
>> John, I suggest you don't try to make this pretty just yet. Do what
>> needs to be done for the type hint job. We can make it pretty later.
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 09/38] qapi/common.py: Add indent manager
2020-09-25 11:51 ` Markus Armbruster
2020-09-25 13:13 ` Eduardo Habkost
@ 2020-09-25 14:42 ` John Snow
1 sibling, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-25 14:42 UTC (permalink / raw)
To: Markus Armbruster
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Cleber Rosa, Alex Bennée
On 9/25/20 7:51 AM, Markus Armbruster wrote:
> John Snow <jsnow@redhat.com> writes:
>
>> On 9/22/20 6:22 PM, Eduardo Habkost wrote:
>>> On Tue, Sep 22, 2020 at 05:00:32PM -0400, John Snow wrote:
>>>> Code style tools really dislike the use of global keywords, because it
>>>> generally involves re-binding the name at runtime which can have strange
>>>> effects depending on when and how that global name is referenced in
>>>> other modules.
>>>>
>>>> Make a little indent level manager instead.
>>>>
>>>> Signed-off-by: John Snow <jsnow@redhat.com>
>>>> ---
>>>> scripts/qapi/common.py | 51 +++++++++++++++++++++++++++++-------------
>>>> scripts/qapi/visit.py | 7 +++---
>>>> 2 files changed, 38 insertions(+), 20 deletions(-)
>>>>
>>>> diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
>>>> index cee63eb95c..e0c5871b10 100644
>>>> --- a/scripts/qapi/common.py
>>>> +++ b/scripts/qapi/common.py
>>>> @@ -93,33 +93,52 @@ def c_name(name, protect=True):
>>>> pointer_suffix = ' *' + eatspace
>>>>
>>>> -def genindent(count):
>>>> - ret = ''
>>>> - for _ in range(count):
>>>> - ret += ' '
>>>> - return ret
>>>> +class Indentation:
>>>> + """
>>>> + Indentation level management.
>>>> + :param initial: Initial number of spaces, default 0.
>>>> + """
>>>> + def __init__(self, initial: int = 0) -> None:
>>>> + self._level = initial
>>>> -indent_level = 0
>>>> + def __int__(self) -> int:
>>>> + return self._level
>>>> + def __repr__(self) -> str:
>>>> + return "{}({:d})".format(type(self).__name__, self._level)
>>>> -def push_indent(indent_amount=4):
>>>> - global indent_level
>>>> - indent_level += indent_amount
>>>> + def __str__(self) -> str:
>>>> + """Return the current indentation as a string of spaces."""
>>>> + return ' ' * self._level
>>>> + def __bool__(self) -> bool:
>>>> + """True when there is a non-zero indentation."""
>>>> + return bool(self._level)
>>>> -def pop_indent(indent_amount=4):
>>>> - global indent_level
>>>> - indent_level -= indent_amount
>>>> + def increase(self, amount: int = 4) -> int:
>>>> + """Increase the indentation level by `amount`, default 4."""
>>>> + self._level += amount
>>>> + return self._level
>>>> +
>>>> + def decrease(self, amount: int = 4) -> int:
>>>> + """Decrease the indentation level by `amount`, default 4."""
>>>> + if self._level < amount:
>>>> + raise ArithmeticError(
>>>> + f"Can't remove {amount:d} spaces from {self!r}")
>>>> + self._level -= amount
>>>> + return self._level
>>>> +
>>>> +
>>>> +indent = Indentation()
>>> Personally, I would keep the push_indent(), pop_indent() API, and
>>> introduce an indent() function, to follow the existing API style
>>> of plain functions.
>>> Something like:
>>> indent_prefixes = []
>>> def push_indent(amount=4):
>>> """Add `amount` spaces to indentation prefix"""
>>> indent_prefixes.push(' '*amount)
>>> def pop_indent():
>>> """Revert the most recent push_indent() call"""
>>> indent_prefixes.pop()
>>> def indent():
>>> """Return the current indentation prefix"""
>>> return ''.join(indent_prefixes)
>>> What you did is still an improvement, though, so:
>>> Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
>>>
>>
>> Yeah, there's only one user right now, so ... I just kinda wanted to
>> get rid of the global usage. Maybe if we make the code generator
>> fancier we'll find out what form is best.
>
> You don't get rid of the global variable, you just change it from
> integer to a class. A class can be handier when generating multiple
> things interleaved, because you can have one class instance per thing.
>
The 'global' keyword, not 'global scoped'. The key thing was to remove
re-binding of the global name, which is now true. We bind to this object
once and then modify object state.
The global keyword allows us to re-bind the variable at the global scope
which is the pattern to avoid. Having globally scoped things you don't
rebind is perfectly fine.
> Note that we already have a class instance per thing we generate:
> instances of subtypes of QAPIGen. The thought of moving the indentation
> machinery into QAPIGen or or maybe QAPIGenCCode crossed my mind many
> moons ago, but I had bigger fish to fry, and then I forgot :)
>
> John, I suggest you don't try to make this pretty just yet. Do what
> needs to be done for the type hint job. We can make it pretty later.
>
Mmhmm. I'm fine with with we have here for now. We can try to make
things pretty later. We have the rust support series to consider, too,
so I am not going to put the cart before the horse.
--js
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 09/38] qapi/common.py: Add indent manager
2020-09-22 21:00 ` [PATCH v2 09/38] qapi/common.py: Add indent manager John Snow
2020-09-22 22:22 ` Eduardo Habkost
@ 2020-09-23 14:55 ` Cleber Rosa
2020-09-23 17:30 ` John Snow
1 sibling, 1 reply; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 14:55 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 2311 bytes --]
On Tue, Sep 22, 2020 at 05:00:32PM -0400, John Snow wrote:
> Code style tools really dislike the use of global keywords, because it
> generally involves re-binding the name at runtime which can have strange
> effects depending on when and how that global name is referenced in
> other modules.
>
> Make a little indent level manager instead.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/common.py | 51 +++++++++++++++++++++++++++++-------------
> scripts/qapi/visit.py | 7 +++---
> 2 files changed, 38 insertions(+), 20 deletions(-)
>
> diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
> index cee63eb95c..e0c5871b10 100644
> --- a/scripts/qapi/common.py
> +++ b/scripts/qapi/common.py
> @@ -93,33 +93,52 @@ def c_name(name, protect=True):
> pointer_suffix = ' *' + eatspace
>
>
> -def genindent(count):
> - ret = ''
> - for _ in range(count):
> - ret += ' '
> - return ret
> +class Indentation:
> + """
> + Indentation level management.
>
> + :param initial: Initial number of spaces, default 0.
> + """
> + def __init__(self, initial: int = 0) -> None:
> + self._level = initial
>
> -indent_level = 0
> + def __int__(self) -> int:
> + return self._level
>
> + def __repr__(self) -> str:
> + return "{}({:d})".format(type(self).__name__, self._level)
>
> -def push_indent(indent_amount=4):
> - global indent_level
> - indent_level += indent_amount
> + def __str__(self) -> str:
> + """Return the current indentation as a string of spaces."""
> + return ' ' * self._level
>
> + def __bool__(self) -> bool:
> + """True when there is a non-zero indentation."""
> + return bool(self._level)
>
> -def pop_indent(indent_amount=4):
> - global indent_level
> - indent_level -= indent_amount
> + def increase(self, amount: int = 4) -> int:
> + """Increase the indentation level by `amount`, default 4."""
> + self._level += amount
> + return self._level
> +
Do you have a use case for returning the level? If not, I'd go
without it, and add a "level" property instead, as it'd serve more
cases.
Other than that,
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 09/38] qapi/common.py: Add indent manager
2020-09-23 14:55 ` Cleber Rosa
@ 2020-09-23 17:30 ` John Snow
2020-09-25 11:55 ` Markus Armbruster
0 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-23 17:30 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
On 9/23/20 10:55 AM, Cleber Rosa wrote:
> Do you have a use case for returning the level? If not, I'd go
> without it, and add a "level" property instead, as it'd serve more
> cases.
__int__ is doing that lifting. I can remove the return.
--js
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 09/38] qapi/common.py: Add indent manager
2020-09-23 17:30 ` John Snow
@ 2020-09-25 11:55 ` Markus Armbruster
2020-09-25 15:41 ` John Snow
0 siblings, 1 reply; 190+ messages in thread
From: Markus Armbruster @ 2020-09-25 11:55 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Cleber Rosa, Alex Bennée
John Snow <jsnow@redhat.com> writes:
> On 9/23/20 10:55 AM, Cleber Rosa wrote:
>> Do you have a use case for returning the level? If not, I'd go
>> without it, and add a "level" property instead, as it'd serve more
>> cases.
>
> __int__ is doing that lifting. I can remove the return.
I like my functions to return something useful.
Use your judgement.
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 09/38] qapi/common.py: Add indent manager
2020-09-25 11:55 ` Markus Armbruster
@ 2020-09-25 15:41 ` John Snow
0 siblings, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-25 15:41 UTC (permalink / raw)
To: Markus Armbruster
Cc: Peter Maydell, Eduardo Habkost, Michael Roth, qemu-devel,
Cleber Rosa, Alex Bennée
On 9/25/20 7:55 AM, Markus Armbruster wrote:
> John Snow <jsnow@redhat.com> writes:
>
>> On 9/23/20 10:55 AM, Cleber Rosa wrote:
>>> Do you have a use case for returning the level? If not, I'd go
>>> without it, and add a "level" property instead, as it'd serve more
>>> cases.
>>
>> __int__ is doing that lifting. I can remove the return.
>
> I like my functions to return something useful.
>
> Use your judgement.
>
>
Eh, Cleber had a point. Nothing uses it.
(AKA: I already made the edit ...)
Like you say, we'll figure out the truly beautiful way to do code
generation when we tackle this all together, holistically. What I've got
works for now (and isn't terribly complex), let's just roll with it
while we fry the bigger fish.
--js
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 10/38] qapi/common.py: delint with pylint
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (8 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 09/38] qapi/common.py: Add indent manager John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-22 21:46 ` Eduardo Habkost
2020-09-23 16:01 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 11/38] qapi/common.py: Replace one-letter 'c' variable John Snow
` (28 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
At this point, that just means using a consistent strategy for constant names.
constants get UPPER_CASE and names not used externally get a leading underscore.
As a preference, while renaming constants to be UPPERCASE, move them to
the head of the file. Generally, it's nice to be able to audit the code
that runs on import in one central place.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/common.py | 18 ++++++++----------
scripts/qapi/schema.py | 14 +++++++-------
2 files changed, 15 insertions(+), 17 deletions(-)
diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
index e0c5871b10..bddfb5a9e5 100644
--- a/scripts/qapi/common.py
+++ b/scripts/qapi/common.py
@@ -14,6 +14,11 @@
import re
+EATSPACE = '\033EATSPACE.'
+POINTER_SUFFIX = ' *' + EATSPACE
+_C_NAME_TRANS = str.maketrans('.-', '__')
+
+
# ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
# ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
# ENUM24_Name -> ENUM24_NAME
@@ -42,9 +47,6 @@ def c_enum_const(type_name, const_name, prefix=None):
return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
-c_name_trans = str.maketrans('.-', '__')
-
-
# Map @name to a valid C identifier.
# If @protect, avoid returning certain ticklish identifiers (like
# C keywords) by prepending 'q_'.
@@ -82,17 +84,13 @@ def c_name(name, protect=True):
'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
# namespace pollution:
polluted_words = set(['unix', 'errno', 'mips', 'sparc', 'i386'])
- name = name.translate(c_name_trans)
+ name = name.translate(_C_NAME_TRANS)
if protect and (name in c89_words | c99_words | c11_words | gcc_words
| cpp_words | polluted_words):
return 'q_' + name
return name
-eatspace = '\033EATSPACE.'
-pointer_suffix = ' *' + eatspace
-
-
class Indentation:
"""
Indentation level management.
@@ -134,12 +132,12 @@ def decrease(self, amount: int = 4) -> int:
# Generate @code with @kwds interpolated.
-# Obey indent, and strip eatspace.
+# Obey indent, and strip EATSPACE.
def cgen(code, **kwds):
raw = code % kwds
if indent:
raw = re.sub(r'^(?!(#|$))', str(indent), raw, flags=re.MULTILINE)
- return re.sub(re.escape(eatspace) + r' *', '', raw)
+ return re.sub(re.escape(EATSPACE) + r' *', '', raw)
def mcgen(code, **kwds):
diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py
index b77e0e56b2..b4921b46c9 100644
--- a/scripts/qapi/schema.py
+++ b/scripts/qapi/schema.py
@@ -18,7 +18,7 @@
import re
from collections import OrderedDict
-from .common import c_name, pointer_suffix
+from .common import c_name, POINTER_SUFFIX
from .error import QAPIError, QAPISemError
from .expr import check_exprs
from .parser import QAPISchemaParser
@@ -309,7 +309,7 @@ def is_implicit(self):
return True
def c_type(self):
- return c_name(self.name) + pointer_suffix
+ return c_name(self.name) + POINTER_SUFFIX
def json_type(self):
return 'array'
@@ -430,7 +430,7 @@ def c_name(self):
def c_type(self):
assert not self.is_implicit()
- return c_name(self.name) + pointer_suffix
+ return c_name(self.name) + POINTER_SUFFIX
def c_unboxed_type(self):
return c_name(self.name)
@@ -504,7 +504,7 @@ def connect_doc(self, doc=None):
v.connect_doc(doc)
def c_type(self):
- return c_name(self.name) + pointer_suffix
+ return c_name(self.name) + POINTER_SUFFIX
def json_type(self):
return 'value'
@@ -896,7 +896,7 @@ def _def_builtin_type(self, name, json_type, c_type):
self._make_array_type(name, None)
def _def_predefineds(self):
- for t in [('str', 'string', 'char' + pointer_suffix),
+ for t in [('str', 'string', 'char' + POINTER_SUFFIX),
('number', 'number', 'double'),
('int', 'int', 'int64_t'),
('int8', 'int', 'int8_t'),
@@ -909,8 +909,8 @@ def _def_predefineds(self):
('uint64', 'int', 'uint64_t'),
('size', 'int', 'uint64_t'),
('bool', 'boolean', 'bool'),
- ('any', 'value', 'QObject' + pointer_suffix),
- ('null', 'null', 'QNull' + pointer_suffix)]:
+ ('any', 'value', 'QObject' + POINTER_SUFFIX),
+ ('null', 'null', 'QNull' + POINTER_SUFFIX)]:
self._def_builtin_type(*t)
self.the_empty_object_type = QAPISchemaObjectType(
'q_empty', None, None, None, None, None, [], None)
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 10/38] qapi/common.py: delint with pylint
2020-09-22 21:00 ` [PATCH v2 10/38] qapi/common.py: delint with pylint John Snow
@ 2020-09-22 21:46 ` Eduardo Habkost
2020-09-23 16:01 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-22 21:46 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:33PM -0400, John Snow wrote:
> At this point, that just means using a consistent strategy for constant names.
> constants get UPPER_CASE and names not used externally get a leading underscore.
>
> As a preference, while renaming constants to be UPPERCASE, move them to
> the head of the file. Generally, it's nice to be able to audit the code
> that runs on import in one central place.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 10/38] qapi/common.py: delint with pylint
2020-09-22 21:00 ` [PATCH v2 10/38] qapi/common.py: delint with pylint John Snow
2020-09-22 21:46 ` Eduardo Habkost
@ 2020-09-23 16:01 ` Cleber Rosa
2020-09-23 17:37 ` John Snow
1 sibling, 1 reply; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 16:01 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 1497 bytes --]
On Tue, Sep 22, 2020 at 05:00:33PM -0400, John Snow wrote:
> At this point, that just means using a consistent strategy for constant names.
> constants get UPPER_CASE and names not used externally get a leading underscore.
>
> As a preference, while renaming constants to be UPPERCASE, move them to
> the head of the file. Generally, it's nice to be able to audit the code
> that runs on import in one central place.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/common.py | 18 ++++++++----------
> scripts/qapi/schema.py | 14 +++++++-------
> 2 files changed, 15 insertions(+), 17 deletions(-)
>
> diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
> index e0c5871b10..bddfb5a9e5 100644
> --- a/scripts/qapi/common.py
> +++ b/scripts/qapi/common.py
> @@ -14,6 +14,11 @@
> import re
>
>
> +EATSPACE = '\033EATSPACE.'
> +POINTER_SUFFIX = ' *' + EATSPACE
> +_C_NAME_TRANS = str.maketrans('.-', '__')
IMO _C_NAME_TRANS is solely the concern of the c_name() function, and
should not be a global. If you're concerned with speed (which I don't
think you should) you could still do:
def c_name(name, protect=True,
name_translation=str.maketrans('.-', '__')):
...
name = name.translate(name_translation)
Keeping in mind that you're adding a mutable type to a function
argument *on purpose*. I'd really favor having that statement within
the only function that uses it, though.
- Cleber.
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 10/38] qapi/common.py: delint with pylint
2020-09-23 16:01 ` Cleber Rosa
@ 2020-09-23 17:37 ` John Snow
2020-09-24 19:30 ` Cleber Rosa
0 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-23 17:37 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
On 9/23/20 12:01 PM, Cleber Rosa wrote:
> On Tue, Sep 22, 2020 at 05:00:33PM -0400, John Snow wrote:
>> At this point, that just means using a consistent strategy for constant names.
>> constants get UPPER_CASE and names not used externally get a leading underscore.
>>
>> As a preference, while renaming constants to be UPPERCASE, move them to
>> the head of the file. Generally, it's nice to be able to audit the code
>> that runs on import in one central place.
>>
>> Signed-off-by: John Snow <jsnow@redhat.com>
>> ---
>> scripts/qapi/common.py | 18 ++++++++----------
>> scripts/qapi/schema.py | 14 +++++++-------
>> 2 files changed, 15 insertions(+), 17 deletions(-)
>>
>> diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
>> index e0c5871b10..bddfb5a9e5 100644
>> --- a/scripts/qapi/common.py
>> +++ b/scripts/qapi/common.py
>> @@ -14,6 +14,11 @@
>> import re
>>
>>
>> +EATSPACE = '\033EATSPACE.'
>> +POINTER_SUFFIX = ' *' + EATSPACE
>> +_C_NAME_TRANS = str.maketrans('.-', '__')
>
> IMO _C_NAME_TRANS is solely the concern of the c_name() function, and
> should not be a global. If you're concerned with speed (which I don't
> think you should) you could still do:
>
It's true, but that's why it's marked internal here with the leading
underscore -- it will not be exported. It was also *already* defined at
the module level, I didn't hoist it.
I think what is written here is already the simplest thing that works.
> def c_name(name, protect=True,
> name_translation=str.maketrans('.-', '__')):
> ...
> name = name.translate(name_translation)
>
> Keeping in mind that you're adding a mutable type to a function
> argument *on purpose*. I'd really favor having that statement within
> the only function that uses it, though.
>
That complicates the signature, so I think we shouldn't.
--js
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 10/38] qapi/common.py: delint with pylint
2020-09-23 17:37 ` John Snow
@ 2020-09-24 19:30 ` Cleber Rosa
0 siblings, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-24 19:30 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 2218 bytes --]
On Wed, Sep 23, 2020 at 01:37:06PM -0400, John Snow wrote:
> On 9/23/20 12:01 PM, Cleber Rosa wrote:
> > On Tue, Sep 22, 2020 at 05:00:33PM -0400, John Snow wrote:
> > > At this point, that just means using a consistent strategy for constant names.
> > > constants get UPPER_CASE and names not used externally get a leading underscore.
> > >
> > > As a preference, while renaming constants to be UPPERCASE, move them to
> > > the head of the file. Generally, it's nice to be able to audit the code
> > > that runs on import in one central place.
> > >
> > > Signed-off-by: John Snow <jsnow@redhat.com>
> > > ---
> > > scripts/qapi/common.py | 18 ++++++++----------
> > > scripts/qapi/schema.py | 14 +++++++-------
> > > 2 files changed, 15 insertions(+), 17 deletions(-)
> > >
> > > diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
> > > index e0c5871b10..bddfb5a9e5 100644
> > > --- a/scripts/qapi/common.py
> > > +++ b/scripts/qapi/common.py
> > > @@ -14,6 +14,11 @@
> > > import re
> > > +EATSPACE = '\033EATSPACE.'
> > > +POINTER_SUFFIX = ' *' + EATSPACE
> > > +_C_NAME_TRANS = str.maketrans('.-', '__')
> >
> > IMO _C_NAME_TRANS is solely the concern of the c_name() function, and
> > should not be a global. If you're concerned with speed (which I don't
> > think you should) you could still do:
> >
>
> It's true, but that's why it's marked internal here with the leading
> underscore -- it will not be exported. It was also *already* defined at the
> module level, I didn't hoist it.
>
> I think what is written here is already the simplest thing that works.
>
> > def c_name(name, protect=True,
> > name_translation=str.maketrans('.-', '__')):
> > ...
> > name = name.translate(name_translation)
> >
> > Keeping in mind that you're adding a mutable type to a function
> > argument *on purpose*. I'd really favor having that statement within
> > the only function that uses it, though.
> >
>
> That complicates the signature, so I think we shouldn't.
>
> --js
>
>
Not a too strong opinion about this to block it. So if I hadn't
already:
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 11/38] qapi/common.py: Replace one-letter 'c' variable
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (9 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 10/38] qapi/common.py: delint with pylint John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-22 22:24 ` Eduardo Habkost
2020-09-23 16:15 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 12/38] qapi/common.py: check with pylint John Snow
` (27 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/common.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
index bddfb5a9e5..682e74fe65 100644
--- a/scripts/qapi/common.py
+++ b/scripts/qapi/common.py
@@ -30,14 +30,14 @@ def camel_to_upper(value):
new_name = ''
length = len(c_fun_str)
for i in range(length):
- c = c_fun_str[i]
- # When c is upper and no '_' appears before, do more checks
- if c.isupper() and (i > 0) and c_fun_str[i - 1] != '_':
+ char = c_fun_str[i]
+ # When char is upper case and no '_' appears before, do more checks
+ if char.isupper() and (i > 0) and c_fun_str[i - 1] != '_':
if i < length - 1 and c_fun_str[i + 1].islower():
new_name += '_'
elif c_fun_str[i - 1].isdigit():
new_name += '_'
- new_name += c
+ new_name += char
return new_name.lstrip('_').upper()
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 11/38] qapi/common.py: Replace one-letter 'c' variable
2020-09-22 21:00 ` [PATCH v2 11/38] qapi/common.py: Replace one-letter 'c' variable John Snow
@ 2020-09-22 22:24 ` Eduardo Habkost
2020-09-23 16:15 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-22 22:24 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:34PM -0400, John Snow wrote:
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 11/38] qapi/common.py: Replace one-letter 'c' variable
2020-09-22 21:00 ` [PATCH v2 11/38] qapi/common.py: Replace one-letter 'c' variable John Snow
2020-09-22 22:24 ` Eduardo Habkost
@ 2020-09-23 16:15 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 16:15 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 150 bytes --]
On Tue, Sep 22, 2020 at 05:00:34PM -0400, John Snow wrote:
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 12/38] qapi/common.py: check with pylint
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (10 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 11/38] qapi/common.py: Replace one-letter 'c' variable John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-22 22:26 ` Eduardo Habkost
2020-09-23 16:18 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 13/38] qapi/common.py: add type hint annotations John Snow
` (26 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Remove qapi/common.py from the pylintrc ignore list.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/pylintrc | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/scripts/qapi/pylintrc b/scripts/qapi/pylintrc
index 02dd22c863..6151427a51 100644
--- a/scripts/qapi/pylintrc
+++ b/scripts/qapi/pylintrc
@@ -2,8 +2,7 @@
# Add files or directories matching the regex patterns to the ignore list.
# The regex matches against base names, not paths.
-ignore-patterns=common.py,
- doc.py,
+ignore-patterns=doc.py,
error.py,
expr.py,
gen.py,
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 12/38] qapi/common.py: check with pylint
2020-09-22 21:00 ` [PATCH v2 12/38] qapi/common.py: check with pylint John Snow
@ 2020-09-22 22:26 ` Eduardo Habkost
2020-09-23 16:18 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-22 22:26 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:35PM -0400, John Snow wrote:
> Remove qapi/common.py from the pylintrc ignore list.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
No pylint warnings detected.
Tested-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 12/38] qapi/common.py: check with pylint
2020-09-22 21:00 ` [PATCH v2 12/38] qapi/common.py: check with pylint John Snow
2020-09-22 22:26 ` Eduardo Habkost
@ 2020-09-23 16:18 ` Cleber Rosa
2020-09-23 16:30 ` John Snow
1 sibling, 1 reply; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 16:18 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 568 bytes --]
On Tue, Sep 22, 2020 at 05:00:35PM -0400, John Snow wrote:
> Remove qapi/common.py from the pylintrc ignore list.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
$ pylint scripts/qapi/ --rcfile=scripts/qapi/pylintrc
--------------------------------------------------------------------
Your code has been rated at 10.00/10 (previous run: 10.00/10, +0.00)
I still think we should record (and maybe I'm missing it) the pylint
version used though. Anyway:
Reviewed-by: Cleber Rosa <crosa@redhat.com>
Tested-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 13/38] qapi/common.py: add type hint annotations
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (11 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 12/38] qapi/common.py: check with pylint John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-22 22:44 ` Eduardo Habkost
2020-09-23 19:28 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 14/38] qapi/common.py: Convert comments into docstrings, and elaborate John Snow
` (25 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Annotations do not change runtime behavior.
This commit *only* adds annotations.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/common.py | 27 ++++++++++++++++-----------
1 file changed, 16 insertions(+), 11 deletions(-)
diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
index 682e74fe65..0ce4a107e6 100644
--- a/scripts/qapi/common.py
+++ b/scripts/qapi/common.py
@@ -12,6 +12,7 @@
# See the COPYING file in the top-level directory.
import re
+from typing import Optional, Sequence
EATSPACE = '\033EATSPACE.'
@@ -22,7 +23,7 @@
# ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
# ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
# ENUM24_Name -> ENUM24_NAME
-def camel_to_upper(value):
+def camel_to_upper(value: str) -> str:
c_fun_str = c_name(value, False)
if value.isupper():
return c_fun_str
@@ -41,7 +42,9 @@ def camel_to_upper(value):
return new_name.lstrip('_').upper()
-def c_enum_const(type_name, const_name, prefix=None):
+def c_enum_const(type_name: str,
+ const_name: str,
+ prefix: Optional[str] = None) -> str:
if prefix is not None:
type_name = prefix
return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
@@ -56,7 +59,7 @@ def c_enum_const(type_name, const_name, prefix=None):
# into substrings of a generated C function name.
# '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
# protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
-def c_name(name, protect=True):
+def c_name(name: str, protect: bool = True) -> str:
# ANSI X3J11/88-090, 3.1.1
c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
'default', 'do', 'double', 'else', 'enum', 'extern',
@@ -133,24 +136,24 @@ def decrease(self, amount: int = 4) -> int:
# Generate @code with @kwds interpolated.
# Obey indent, and strip EATSPACE.
-def cgen(code, **kwds):
+def cgen(code: str, **kwds: object) -> str:
raw = code % kwds
if indent:
raw = re.sub(r'^(?!(#|$))', str(indent), raw, flags=re.MULTILINE)
return re.sub(re.escape(EATSPACE) + r' *', '', raw)
-def mcgen(code, **kwds):
+def mcgen(code: str, **kwds: object) -> str:
if code[0] == '\n':
code = code[1:]
return cgen(code, **kwds)
-def c_fname(filename):
+def c_fname(filename: str) -> str:
return re.sub(r'[^A-Za-z0-9_]', '_', filename)
-def guardstart(name):
+def guardstart(name: str) -> str:
return mcgen('''
#ifndef %(name)s
#define %(name)s
@@ -159,7 +162,7 @@ def guardstart(name):
name=c_fname(name).upper())
-def guardend(name):
+def guardend(name: str) -> str:
return mcgen('''
#endif /* %(name)s */
@@ -167,7 +170,7 @@ def guardend(name):
name=c_fname(name).upper())
-def gen_if(ifcond):
+def gen_if(ifcond: Sequence[str]) -> str:
ret = ''
for ifc in ifcond:
ret += mcgen('''
@@ -176,7 +179,7 @@ def gen_if(ifcond):
return ret
-def gen_endif(ifcond):
+def gen_endif(ifcond: Sequence[str]) -> str:
ret = ''
for ifc in reversed(ifcond):
ret += mcgen('''
@@ -185,7 +188,9 @@ def gen_endif(ifcond):
return ret
-def build_params(arg_type, boxed, extra=None):
+def build_params(arg_type,
+ boxed: bool,
+ extra: Optional[str] = None) -> str:
ret = ''
sep = ''
if boxed:
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 13/38] qapi/common.py: add type hint annotations
2020-09-22 21:00 ` [PATCH v2 13/38] qapi/common.py: add type hint annotations John Snow
@ 2020-09-22 22:44 ` Eduardo Habkost
2020-09-23 17:57 ` John Snow
2020-09-23 19:28 ` Cleber Rosa
1 sibling, 1 reply; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-22 22:44 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:36PM -0400, John Snow wrote:
> Annotations do not change runtime behavior.
> This commit *only* adds annotations.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/common.py | 27 ++++++++++++++++-----------
> 1 file changed, 16 insertions(+), 11 deletions(-)
>
> diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
> index 682e74fe65..0ce4a107e6 100644
> --- a/scripts/qapi/common.py
> +++ b/scripts/qapi/common.py
> @@ -12,6 +12,7 @@
[...]
> @@ -176,7 +179,7 @@ def gen_if(ifcond):
> return ret
>
>
> -def gen_endif(ifcond):
> +def gen_endif(ifcond: Sequence[str]) -> str:
Does this need to require a Sequence? It looks like it could be
Iterable.
I don't think this should block the patch, though, so:
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
> ret = ''
> for ifc in reversed(ifcond):
> ret += mcgen('''
> @@ -185,7 +188,9 @@ def gen_endif(ifcond):
> return ret
>
>
> -def build_params(arg_type, boxed, extra=None):
> +def build_params(arg_type,
> + boxed: bool,
> + extra: Optional[str] = None) -> str:
> ret = ''
> sep = ''
> if boxed:
> --
> 2.26.2
>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 13/38] qapi/common.py: add type hint annotations
2020-09-22 22:44 ` Eduardo Habkost
@ 2020-09-23 17:57 ` John Snow
2020-09-23 18:19 ` Eduardo Habkost
0 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-23 17:57 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, Michael Roth, qemu-devel, Markus Armbruster,
Cleber Rosa, Alex Bennée
On 9/22/20 6:44 PM, Eduardo Habkost wrote:
> Does this need to require a Sequence? It looks like it could be
> Iterable.
>
> I don't think this should block the patch, though, so:
>
> Reviewed-by: Eduardo Habkost<ehabkost@redhat.com>
gen_if can take an Iterator, gen_endif needs a Sequence because it uses
reversed().
I have not been very fastidious about choosing the MOST possibly
abstracted type, but you are right that we SHOULD. I was hoping to find
all occurrences of List and try slackening them to
Collection/Sequence/Iterable etc, but as a follow-up.
In this case, since gen_endif actually does want the stricter type, I'm
deciding to leave them as Sequence for signature parity.
--js
(P.S. in case it comes up in future review, there is a bug that prevents
us from using Collection in 3.6:
https://github.com/PyCQA/pylint/issues/2377 )
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 13/38] qapi/common.py: add type hint annotations
2020-09-23 17:57 ` John Snow
@ 2020-09-23 18:19 ` Eduardo Habkost
0 siblings, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 18:19 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Michael Roth, qemu-devel, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Wed, Sep 23, 2020 at 01:57:25PM -0400, John Snow wrote:
> On 9/22/20 6:44 PM, Eduardo Habkost wrote:
> > Does this need to require a Sequence? It looks like it could be
> > Iterable.
> >
> > I don't think this should block the patch, though, so:
> >
> > Reviewed-by: Eduardo Habkost<ehabkost@redhat.com>
>
> gen_if can take an Iterator, gen_endif needs a Sequence because it uses
> reversed().
>
> I have not been very fastidious about choosing the MOST possibly abstracted
> type, but you are right that we SHOULD. I was hoping to find all occurrences
> of List and try slackening them to Collection/Sequence/Iterable etc, but as
> a follow-up.
>
> In this case, since gen_endif actually does want the stricter type, I'm
> deciding to leave them as Sequence for signature parity.
I hadn't noticed reversed() requires a Sequence. I agree using
the same type on both is better. Thanks!
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 13/38] qapi/common.py: add type hint annotations
2020-09-22 21:00 ` [PATCH v2 13/38] qapi/common.py: add type hint annotations John Snow
2020-09-22 22:44 ` Eduardo Habkost
@ 2020-09-23 19:28 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 19:28 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 245 bytes --]
On Tue, Sep 22, 2020 at 05:00:36PM -0400, John Snow wrote:
> Annotations do not change runtime behavior.
> This commit *only* adds annotations.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 14/38] qapi/common.py: Convert comments into docstrings, and elaborate
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (12 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 13/38] qapi/common.py: add type hint annotations John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 14:22 ` Eduardo Habkost
2020-09-23 19:38 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 15/38] qapi/common.py: move build_params into gen.py John Snow
` (24 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
As docstrings, they'll show up in documentation and IDE help.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/common.py | 51 ++++++++++++++++++++++++++++++------------
1 file changed, 37 insertions(+), 14 deletions(-)
diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
index 0ce4a107e6..730283722a 100644
--- a/scripts/qapi/common.py
+++ b/scripts/qapi/common.py
@@ -20,10 +20,18 @@
_C_NAME_TRANS = str.maketrans('.-', '__')
-# ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
-# ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
-# ENUM24_Name -> ENUM24_NAME
def camel_to_upper(value: str) -> str:
+ """
+ Converts CamelCase to CAMEL_CASE.
+
+ Examples:
+ ENUMName -> ENUM_NAME
+ EnumName1 -> ENUM_NAME1
+ ENUM_NAME -> ENUM_NAME
+ ENUM_NAME1 -> ENUM_NAME1
+ ENUM_Name2 -> ENUM_NAME2
+ ENUM24_Name -> ENUM24_NAME
+ """
c_fun_str = c_name(value, False)
if value.isupper():
return c_fun_str
@@ -45,21 +53,33 @@ def camel_to_upper(value: str) -> str:
def c_enum_const(type_name: str,
const_name: str,
prefix: Optional[str] = None) -> str:
+ """
+ Generate a C enumeration constant name.
+
+ :param type_name: The name of the enumeration.
+ :param const_name: The name of this constant.
+ :param prefix: Optional, prefix that overrides the type_name.
+ """
if prefix is not None:
type_name = prefix
return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
-# Map @name to a valid C identifier.
-# If @protect, avoid returning certain ticklish identifiers (like
-# C keywords) by prepending 'q_'.
-#
-# Used for converting 'name' from a 'name':'type' qapi definition
-# into a generated struct member, as well as converting type names
-# into substrings of a generated C function name.
-# '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
-# protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
def c_name(name: str, protect: bool = True) -> str:
+ """
+ Map `name` to a valid C identifier.
+
+ Used for converting 'name' from a 'name':'type' qapi definition
+ into a generated struct member, as well as converting type names
+ into substrings of a generated C function name.
+
+ '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
+ protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
+
+ :param name: The name to map.
+ :param protect: If true, avoid returning certain ticklish identifiers
+ (like C keywords) by prepending ``q_``.
+ """
# ANSI X3J11/88-090, 3.1.1
c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
'default', 'do', 'double', 'else', 'enum', 'extern',
@@ -134,9 +154,12 @@ def decrease(self, amount: int = 4) -> int:
indent = Indentation()
-# Generate @code with @kwds interpolated.
-# Obey indent, and strip EATSPACE.
def cgen(code: str, **kwds: object) -> str:
+ """
+ Generate `code` with `kwds` interpolated.
+
+ Obey `indent`, and strip `EATSPACE`.
+ """
raw = code % kwds
if indent:
raw = re.sub(r'^(?!(#|$))', str(indent), raw, flags=re.MULTILINE)
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 14/38] qapi/common.py: Convert comments into docstrings, and elaborate
2020-09-22 21:00 ` [PATCH v2 14/38] qapi/common.py: Convert comments into docstrings, and elaborate John Snow
@ 2020-09-23 14:22 ` Eduardo Habkost
2020-09-23 19:38 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 14:22 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:37PM -0400, John Snow wrote:
> As docstrings, they'll show up in documentation and IDE help.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 14/38] qapi/common.py: Convert comments into docstrings, and elaborate
2020-09-22 21:00 ` [PATCH v2 14/38] qapi/common.py: Convert comments into docstrings, and elaborate John Snow
2020-09-23 14:22 ` Eduardo Habkost
@ 2020-09-23 19:38 ` Cleber Rosa
2020-09-23 21:18 ` John Snow
1 sibling, 1 reply; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 19:38 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 3772 bytes --]
On Tue, Sep 22, 2020 at 05:00:37PM -0400, John Snow wrote:
> As docstrings, they'll show up in documentation and IDE help.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/common.py | 51 ++++++++++++++++++++++++++++++------------
> 1 file changed, 37 insertions(+), 14 deletions(-)
>
> diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
> index 0ce4a107e6..730283722a 100644
> --- a/scripts/qapi/common.py
> +++ b/scripts/qapi/common.py
> @@ -20,10 +20,18 @@
> _C_NAME_TRANS = str.maketrans('.-', '__')
>
>
> -# ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
> -# ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
> -# ENUM24_Name -> ENUM24_NAME
> def camel_to_upper(value: str) -> str:
> + """
> + Converts CamelCase to CAMEL_CASE.
> +
> + Examples:
> + ENUMName -> ENUM_NAME
> + EnumName1 -> ENUM_NAME1
> + ENUM_NAME -> ENUM_NAME
> + ENUM_NAME1 -> ENUM_NAME1
> + ENUM_Name2 -> ENUM_NAME2
> + ENUM24_Name -> ENUM24_NAME
> + """
> c_fun_str = c_name(value, False)
> if value.isupper():
> return c_fun_str
> @@ -45,21 +53,33 @@ def camel_to_upper(value: str) -> str:
> def c_enum_const(type_name: str,
> const_name: str,
> prefix: Optional[str] = None) -> str:
> + """
> + Generate a C enumeration constant name.
> +
> + :param type_name: The name of the enumeration.
> + :param const_name: The name of this constant.
> + :param prefix: Optional, prefix that overrides the type_name.
> + """
> if prefix is not None:
> type_name = prefix
> return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
>
>
> -# Map @name to a valid C identifier.
> -# If @protect, avoid returning certain ticklish identifiers (like
> -# C keywords) by prepending 'q_'.
> -#
> -# Used for converting 'name' from a 'name':'type' qapi definition
> -# into a generated struct member, as well as converting type names
> -# into substrings of a generated C function name.
> -# '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
> -# protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
> def c_name(name: str, protect: bool = True) -> str:
> + """
> + Map `name` to a valid C identifier.
> +
> + Used for converting 'name' from a 'name':'type' qapi definition
> + into a generated struct member, as well as converting type names
> + into substrings of a generated C function name.
> +
> + '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
> + protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
> +
> + :param name: The name to map.
> + :param protect: If true, avoid returning certain ticklish identifiers
> + (like C keywords) by prepending ``q_``.
> + """
> # ANSI X3J11/88-090, 3.1.1
> c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
> 'default', 'do', 'double', 'else', 'enum', 'extern',
> @@ -134,9 +154,12 @@ def decrease(self, amount: int = 4) -> int:
> indent = Indentation()
>
>
> -# Generate @code with @kwds interpolated.
> -# Obey indent, and strip EATSPACE.
> def cgen(code: str, **kwds: object) -> str:
> + """
> + Generate `code` with `kwds` interpolated.
> +
> + Obey `indent`, and strip `EATSPACE`.
> + """
This probably won't help on IDEs (never checked any), but sphinx will
let you do:
"""
Generate `code` with `kwds` interpolated.
Obey `indent`, and strip :data:`EATSPACE`.
"""
I'm not sure that a maximum level of docstring "sphinxzation" is the
goal here, though.
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 14/38] qapi/common.py: Convert comments into docstrings, and elaborate
2020-09-23 19:38 ` Cleber Rosa
@ 2020-09-23 21:18 ` John Snow
2020-09-25 17:02 ` Cleber Rosa
0 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-23 21:18 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
On 9/23/20 3:38 PM, Cleber Rosa wrote:
> On Tue, Sep 22, 2020 at 05:00:37PM -0400, John Snow wrote:
>> As docstrings, they'll show up in documentation and IDE help.
>>
>> Signed-off-by: John Snow <jsnow@redhat.com>
>> ---
>> scripts/qapi/common.py | 51 ++++++++++++++++++++++++++++++------------
>> 1 file changed, 37 insertions(+), 14 deletions(-)
>>
>> diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
>> index 0ce4a107e6..730283722a 100644
>> --- a/scripts/qapi/common.py
>> +++ b/scripts/qapi/common.py
>> @@ -20,10 +20,18 @@
>> _C_NAME_TRANS = str.maketrans('.-', '__')
>>
>>
>> -# ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
>> -# ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
>> -# ENUM24_Name -> ENUM24_NAME
>> def camel_to_upper(value: str) -> str:
>> + """
>> + Converts CamelCase to CAMEL_CASE.
>> +
>> + Examples:
>> + ENUMName -> ENUM_NAME
>> + EnumName1 -> ENUM_NAME1
>> + ENUM_NAME -> ENUM_NAME
>> + ENUM_NAME1 -> ENUM_NAME1
>> + ENUM_Name2 -> ENUM_NAME2
>> + ENUM24_Name -> ENUM24_NAME
>> + """
>> c_fun_str = c_name(value, False)
>> if value.isupper():
>> return c_fun_str
>> @@ -45,21 +53,33 @@ def camel_to_upper(value: str) -> str:
>> def c_enum_const(type_name: str,
>> const_name: str,
>> prefix: Optional[str] = None) -> str:
>> + """
>> + Generate a C enumeration constant name.
>> +
>> + :param type_name: The name of the enumeration.
>> + :param const_name: The name of this constant.
>> + :param prefix: Optional, prefix that overrides the type_name.
>> + """
>> if prefix is not None:
>> type_name = prefix
>> return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
>>
>>
>> -# Map @name to a valid C identifier.
>> -# If @protect, avoid returning certain ticklish identifiers (like
>> -# C keywords) by prepending 'q_'.
>> -#
>> -# Used for converting 'name' from a 'name':'type' qapi definition
>> -# into a generated struct member, as well as converting type names
>> -# into substrings of a generated C function name.
>> -# '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
>> -# protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
>> def c_name(name: str, protect: bool = True) -> str:
>> + """
>> + Map `name` to a valid C identifier.
>> +
>> + Used for converting 'name' from a 'name':'type' qapi definition
>> + into a generated struct member, as well as converting type names
>> + into substrings of a generated C function name.
>> +
>> + '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
>> + protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
>> +
>> + :param name: The name to map.
>> + :param protect: If true, avoid returning certain ticklish identifiers
>> + (like C keywords) by prepending ``q_``.
>> + """
>> # ANSI X3J11/88-090, 3.1.1
>> c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
>> 'default', 'do', 'double', 'else', 'enum', 'extern',
>> @@ -134,9 +154,12 @@ def decrease(self, amount: int = 4) -> int:
>> indent = Indentation()
>>
>>
>> -# Generate @code with @kwds interpolated.
>> -# Obey indent, and strip EATSPACE.
>> def cgen(code: str, **kwds: object) -> str:
>> + """
>> + Generate `code` with `kwds` interpolated.
>> +
>> + Obey `indent`, and strip `EATSPACE`.
>> + """
>
> This probably won't help on IDEs (never checked any), but sphinx will
> let you do:
>
> """
> Generate `code` with `kwds` interpolated.
>
> Obey `indent`, and strip :data:`EATSPACE`.
> """
>
> I'm not sure that a maximum level of docstring "sphinxzation" is the
> goal here, though.
>
> Reviewed-by: Cleber Rosa <crosa@redhat.com>
>
It isn't yet, but I intend to address that when I remove
missing-docstring from pylint exemptions. Do I need :data: if I set the
default role to 'any'?
I'll probably try to enable sphinx at that time (and put the docs in a
devel/python manual?) and worry about the formatting at that point.
--js
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 14/38] qapi/common.py: Convert comments into docstrings, and elaborate
2020-09-23 21:18 ` John Snow
@ 2020-09-25 17:02 ` Cleber Rosa
2020-09-25 17:13 ` John Snow
0 siblings, 1 reply; 190+ messages in thread
From: Cleber Rosa @ 2020-09-25 17:02 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 4791 bytes --]
On Wed, Sep 23, 2020 at 05:18:54PM -0400, John Snow wrote:
> On 9/23/20 3:38 PM, Cleber Rosa wrote:
> > On Tue, Sep 22, 2020 at 05:00:37PM -0400, John Snow wrote:
> > > As docstrings, they'll show up in documentation and IDE help.
> > >
> > > Signed-off-by: John Snow <jsnow@redhat.com>
> > > ---
> > > scripts/qapi/common.py | 51 ++++++++++++++++++++++++++++++------------
> > > 1 file changed, 37 insertions(+), 14 deletions(-)
> > >
> > > diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
> > > index 0ce4a107e6..730283722a 100644
> > > --- a/scripts/qapi/common.py
> > > +++ b/scripts/qapi/common.py
> > > @@ -20,10 +20,18 @@
> > > _C_NAME_TRANS = str.maketrans('.-', '__')
> > > -# ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
> > > -# ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
> > > -# ENUM24_Name -> ENUM24_NAME
> > > def camel_to_upper(value: str) -> str:
> > > + """
> > > + Converts CamelCase to CAMEL_CASE.
> > > +
> > > + Examples:
> > > + ENUMName -> ENUM_NAME
> > > + EnumName1 -> ENUM_NAME1
> > > + ENUM_NAME -> ENUM_NAME
> > > + ENUM_NAME1 -> ENUM_NAME1
> > > + ENUM_Name2 -> ENUM_NAME2
> > > + ENUM24_Name -> ENUM24_NAME
> > > + """
> > > c_fun_str = c_name(value, False)
> > > if value.isupper():
> > > return c_fun_str
> > > @@ -45,21 +53,33 @@ def camel_to_upper(value: str) -> str:
> > > def c_enum_const(type_name: str,
> > > const_name: str,
> > > prefix: Optional[str] = None) -> str:
> > > + """
> > > + Generate a C enumeration constant name.
> > > +
> > > + :param type_name: The name of the enumeration.
> > > + :param const_name: The name of this constant.
> > > + :param prefix: Optional, prefix that overrides the type_name.
> > > + """
> > > if prefix is not None:
> > > type_name = prefix
> > > return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
> > > -# Map @name to a valid C identifier.
> > > -# If @protect, avoid returning certain ticklish identifiers (like
> > > -# C keywords) by prepending 'q_'.
> > > -#
> > > -# Used for converting 'name' from a 'name':'type' qapi definition
> > > -# into a generated struct member, as well as converting type names
> > > -# into substrings of a generated C function name.
> > > -# '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
> > > -# protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
> > > def c_name(name: str, protect: bool = True) -> str:
> > > + """
> > > + Map `name` to a valid C identifier.
> > > +
> > > + Used for converting 'name' from a 'name':'type' qapi definition
> > > + into a generated struct member, as well as converting type names
> > > + into substrings of a generated C function name.
> > > +
> > > + '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
> > > + protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
> > > +
> > > + :param name: The name to map.
> > > + :param protect: If true, avoid returning certain ticklish identifiers
> > > + (like C keywords) by prepending ``q_``.
> > > + """
> > > # ANSI X3J11/88-090, 3.1.1
> > > c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
> > > 'default', 'do', 'double', 'else', 'enum', 'extern',
> > > @@ -134,9 +154,12 @@ def decrease(self, amount: int = 4) -> int:
> > > indent = Indentation()
> > > -# Generate @code with @kwds interpolated.
> > > -# Obey indent, and strip EATSPACE.
> > > def cgen(code: str, **kwds: object) -> str:
> > > + """
> > > + Generate `code` with `kwds` interpolated.
> > > +
> > > + Obey `indent`, and strip `EATSPACE`.
> > > + """
> >
> > This probably won't help on IDEs (never checked any), but sphinx will
> > let you do:
> >
> > """
> > Generate `code` with `kwds` interpolated.
> >
> > Obey `indent`, and strip :data:`EATSPACE`.
> > """
> >
> > I'm not sure that a maximum level of docstring "sphinxzation" is the
> > goal here, though.
> >
> > Reviewed-by: Cleber Rosa <crosa@redhat.com>
> >
>
> It isn't yet, but I intend to address that when I remove missing-docstring
> from pylint exemptions. Do I need :data: if I set the default role to 'any'?
>
That's a good question. According to the docs "any" will do its best,
so it's probably a good fallback. I do still favor using the correct
role from the start if I can help it.
> I'll probably try to enable sphinx at that time (and put the docs in a
> devel/python manual?) and worry about the formatting at that point.
>
> --js
Nice!
- Cleber.
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 14/38] qapi/common.py: Convert comments into docstrings, and elaborate
2020-09-25 17:02 ` Cleber Rosa
@ 2020-09-25 17:13 ` John Snow
0 siblings, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-25 17:13 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
On 9/25/20 1:02 PM, Cleber Rosa wrote:
> On Wed, Sep 23, 2020 at 05:18:54PM -0400, John Snow wrote:
>> On 9/23/20 3:38 PM, Cleber Rosa wrote:
>>> On Tue, Sep 22, 2020 at 05:00:37PM -0400, John Snow wrote:
>>>> As docstrings, they'll show up in documentation and IDE help.
>>>>
>>>> Signed-off-by: John Snow <jsnow@redhat.com>
>>>> ---
>>>> scripts/qapi/common.py | 51 ++++++++++++++++++++++++++++++------------
>>>> 1 file changed, 37 insertions(+), 14 deletions(-)
>>>>
>>>> diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
>>>> index 0ce4a107e6..730283722a 100644
>>>> --- a/scripts/qapi/common.py
>>>> +++ b/scripts/qapi/common.py
>>>> @@ -20,10 +20,18 @@
>>>> _C_NAME_TRANS = str.maketrans('.-', '__')
>>>> -# ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
>>>> -# ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
>>>> -# ENUM24_Name -> ENUM24_NAME
>>>> def camel_to_upper(value: str) -> str:
>>>> + """
>>>> + Converts CamelCase to CAMEL_CASE.
>>>> +
>>>> + Examples:
>>>> + ENUMName -> ENUM_NAME
>>>> + EnumName1 -> ENUM_NAME1
>>>> + ENUM_NAME -> ENUM_NAME
>>>> + ENUM_NAME1 -> ENUM_NAME1
>>>> + ENUM_Name2 -> ENUM_NAME2
>>>> + ENUM24_Name -> ENUM24_NAME
>>>> + """
>>>> c_fun_str = c_name(value, False)
>>>> if value.isupper():
>>>> return c_fun_str
>>>> @@ -45,21 +53,33 @@ def camel_to_upper(value: str) -> str:
>>>> def c_enum_const(type_name: str,
>>>> const_name: str,
>>>> prefix: Optional[str] = None) -> str:
>>>> + """
>>>> + Generate a C enumeration constant name.
>>>> +
>>>> + :param type_name: The name of the enumeration.
>>>> + :param const_name: The name of this constant.
>>>> + :param prefix: Optional, prefix that overrides the type_name.
>>>> + """
>>>> if prefix is not None:
>>>> type_name = prefix
>>>> return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
>>>> -# Map @name to a valid C identifier.
>>>> -# If @protect, avoid returning certain ticklish identifiers (like
>>>> -# C keywords) by prepending 'q_'.
>>>> -#
>>>> -# Used for converting 'name' from a 'name':'type' qapi definition
>>>> -# into a generated struct member, as well as converting type names
>>>> -# into substrings of a generated C function name.
>>>> -# '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
>>>> -# protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
>>>> def c_name(name: str, protect: bool = True) -> str:
>>>> + """
>>>> + Map `name` to a valid C identifier.
>>>> +
>>>> + Used for converting 'name' from a 'name':'type' qapi definition
>>>> + into a generated struct member, as well as converting type names
>>>> + into substrings of a generated C function name.
>>>> +
>>>> + '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
>>>> + protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
>>>> +
>>>> + :param name: The name to map.
>>>> + :param protect: If true, avoid returning certain ticklish identifiers
>>>> + (like C keywords) by prepending ``q_``.
>>>> + """
>>>> # ANSI X3J11/88-090, 3.1.1
>>>> c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
>>>> 'default', 'do', 'double', 'else', 'enum', 'extern',
>>>> @@ -134,9 +154,12 @@ def decrease(self, amount: int = 4) -> int:
>>>> indent = Indentation()
>>>> -# Generate @code with @kwds interpolated.
>>>> -# Obey indent, and strip EATSPACE.
>>>> def cgen(code: str, **kwds: object) -> str:
>>>> + """
>>>> + Generate `code` with `kwds` interpolated.
>>>> +
>>>> + Obey `indent`, and strip `EATSPACE`.
>>>> + """
>>>
>>> This probably won't help on IDEs (never checked any), but sphinx will
>>> let you do:
>>>
>>> """
>>> Generate `code` with `kwds` interpolated.
>>>
>>> Obey `indent`, and strip :data:`EATSPACE`.
>>> """
>>>
>>> I'm not sure that a maximum level of docstring "sphinxzation" is the
>>> goal here, though.
>>>
>>> Reviewed-by: Cleber Rosa <crosa@redhat.com>
>>>
>>
>> It isn't yet, but I intend to address that when I remove missing-docstring
>> from pylint exemptions. Do I need :data: if I set the default role to 'any'?
>>
>
> That's a good question. According to the docs "any" will do its best,
> so it's probably a good fallback. I do still favor using the correct
> role from the start if I can help it.
>
>> I'll probably try to enable sphinx at that time (and put the docs in a
>> devel/python manual?) and worry about the formatting at that point.
>>
>> --js
>
> Nice!
>
> - Cleber.
>
As of v3, I started toying with this, as you can see. It is a goal of
mine to hit full doc coverage in this package, eventually.
What I learned: you can reference data members, but only if they have a
comment. Otherwise, they are skipped. Sphinx does not appear to offer an
"undocumented data member" option the same way it does for "undocumented
member".
Using the "Any" role is nice, and I prefer it. If it finds that a target
is ambiguous (2+ references), it will throw an error and the sphinx
build will fail. This is good enough for me: there's no reason to
clutter the docstrings with Sphinxese if we don't have to.
I would like to try and keep these readable to humans who are just in
emacs/vim editing code, too.
--js
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 15/38] qapi/common.py: move build_params into gen.py
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (13 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 14/38] qapi/common.py: Convert comments into docstrings, and elaborate John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 14:26 ` Eduardo Habkost
2020-09-23 20:04 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 16/38] qapi: establish mypy type-checking baseline John Snow
` (23 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Including it in common.py creates a circular import dependency; schema
relies on common, but common.build_params requires a type annotation
from schema. To type this properly, it needs to be moved outside the
cycle.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/commands.py | 8 ++++++--
scripts/qapi/common.py | 23 -----------------------
scripts/qapi/events.py | 7 +++++--
scripts/qapi/gen.py | 34 ++++++++++++++++++++++++++++++++--
4 files changed, 43 insertions(+), 29 deletions(-)
diff --git a/scripts/qapi/commands.py b/scripts/qapi/commands.py
index d9cda3b209..9b3f24b8ed 100644
--- a/scripts/qapi/commands.py
+++ b/scripts/qapi/commands.py
@@ -14,11 +14,15 @@
"""
from .common import (
- build_params,
c_name,
mcgen,
)
-from .gen import QAPIGenCCode, QAPISchemaModularCVisitor, ifcontext
+from .gen import (
+ QAPIGenCCode,
+ QAPISchemaModularCVisitor,
+ build_params,
+ ifcontext,
+)
def gen_command_decl(name, arg_type, boxed, ret_type):
diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py
index 730283722a..50c3d36e27 100644
--- a/scripts/qapi/common.py
+++ b/scripts/qapi/common.py
@@ -209,26 +209,3 @@ def gen_endif(ifcond: Sequence[str]) -> str:
#endif /* %(cond)s */
''', cond=ifc)
return ret
-
-
-def build_params(arg_type,
- boxed: bool,
- extra: Optional[str] = None) -> str:
- ret = ''
- sep = ''
- if boxed:
- assert arg_type
- ret += '%s arg' % arg_type.c_param_type()
- sep = ', '
- elif arg_type:
- assert not arg_type.variants
- for memb in arg_type.members:
- ret += sep
- sep = ', '
- if memb.optional:
- ret += 'bool has_%s, ' % c_name(memb.name)
- ret += '%s %s' % (memb.type.c_param_type(),
- c_name(memb.name))
- if extra:
- ret += sep + extra
- return ret if ret else 'void'
diff --git a/scripts/qapi/events.py b/scripts/qapi/events.py
index 6b3afa14d7..69a0ac5471 100644
--- a/scripts/qapi/events.py
+++ b/scripts/qapi/events.py
@@ -13,12 +13,15 @@
"""
from .common import (
- build_params,
c_enum_const,
c_name,
mcgen,
)
-from .gen import QAPISchemaModularCVisitor, ifcontext
+from .gen import (
+ QAPISchemaModularCVisitor,
+ build_params,
+ ifcontext,
+)
from .schema import QAPISchemaEnumMember
from .types import gen_enum, gen_enum_lookup
diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
index 11472ba043..9898d513ae 100644
--- a/scripts/qapi/gen.py
+++ b/scripts/qapi/gen.py
@@ -2,9 +2,11 @@
#
# QAPI code generation
#
-# Copyright (c) 2018-2019 Red Hat Inc.
+# Copyright IBM, Corp. 2011
+# Copyright (c) 2013-2019 Red Hat Inc.
#
# Authors:
+# Anthony Liguori <aliguori@us.ibm.com>
# Markus Armbruster <armbru@redhat.com>
# Marc-André Lureau <marcandre.lureau@redhat.com>
#
@@ -15,16 +17,21 @@
import errno
import os
import re
+from typing import Optional
from .common import (
c_fname,
+ c_name,
gen_endif,
gen_if,
guardend,
guardstart,
mcgen,
)
-from .schema import QAPISchemaVisitor
+from .schema import (
+ QAPISchemaObjectType,
+ QAPISchemaVisitor,
+)
class QAPIGen:
@@ -90,6 +97,29 @@ def _wrap_ifcond(ifcond, before, after):
return out
+def build_params(arg_type: Optional[QAPISchemaObjectType],
+ boxed: bool,
+ extra: Optional[str] = None) -> str:
+ ret = ''
+ sep = ''
+ if boxed:
+ assert arg_type
+ ret += '%s arg' % arg_type.c_param_type()
+ sep = ', '
+ elif arg_type:
+ assert not arg_type.variants
+ for memb in arg_type.members:
+ ret += sep
+ sep = ', '
+ if memb.optional:
+ ret += 'bool has_%s, ' % c_name(memb.name)
+ ret += '%s %s' % (memb.type.c_param_type(),
+ c_name(memb.name))
+ if extra:
+ ret += sep + extra
+ return ret if ret else 'void'
+
+
class QAPIGenCCode(QAPIGen):
def __init__(self, fname):
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 15/38] qapi/common.py: move build_params into gen.py
2020-09-22 21:00 ` [PATCH v2 15/38] qapi/common.py: move build_params into gen.py John Snow
@ 2020-09-23 14:26 ` Eduardo Habkost
2020-09-23 20:04 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 14:26 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:38PM -0400, John Snow wrote:
> Including it in common.py creates a circular import dependency; schema
> relies on common, but common.build_params requires a type annotation
> from schema. To type this properly, it needs to be moved outside the
> cycle.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 15/38] qapi/common.py: move build_params into gen.py
2020-09-22 21:00 ` [PATCH v2 15/38] qapi/common.py: move build_params into gen.py John Snow
2020-09-23 14:26 ` Eduardo Habkost
@ 2020-09-23 20:04 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 20:04 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 386 bytes --]
On Tue, Sep 22, 2020 at 05:00:38PM -0400, John Snow wrote:
> Including it in common.py creates a circular import dependency; schema
> relies on common, but common.build_params requires a type annotation
> from schema. To type this properly, it needs to be moved outside the
> cycle.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 16/38] qapi: establish mypy type-checking baseline
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (14 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 15/38] qapi/common.py: move build_params into gen.py John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 14:29 ` Eduardo Habkost
2020-09-23 20:11 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 17/38] qapi/events.py: add type hint annotations John Snow
` (22 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Fix two very minor issues, and then establish a mypy type-checking
baseline.
Like pylint, this should be run from the folder above:
> mypy --config-file=qapi/mypy.ini qapi/
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/doc.py | 3 +-
scripts/qapi/mypy.ini | 65 ++++++++++++++++++++++++++++++++++++++++++
scripts/qapi/schema.py | 3 +-
3 files changed, 69 insertions(+), 2 deletions(-)
create mode 100644 scripts/qapi/mypy.ini
diff --git a/scripts/qapi/doc.py b/scripts/qapi/doc.py
index cbf7076ed9..70f7cdfaa6 100644
--- a/scripts/qapi/doc.py
+++ b/scripts/qapi/doc.py
@@ -5,7 +5,8 @@
"""This script produces the documentation of a qapi schema in texinfo format"""
import re
-from .gen import QAPIGenDoc, QAPISchemaVisitor
+from .gen import QAPIGenDoc
+from .schema import QAPISchemaVisitor
MSG_FMT = """
diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini
new file mode 100644
index 0000000000..a0f2365a53
--- /dev/null
+++ b/scripts/qapi/mypy.ini
@@ -0,0 +1,65 @@
+[mypy]
+strict = True
+strict_optional = False
+disallow_untyped_calls = False
+python_version = 3.6
+
+[mypy-qapi.commands]
+disallow_untyped_defs = False
+disallow_incomplete_defs = False
+check_untyped_defs = False
+
+[mypy-qapi.doc]
+disallow_subclassing_any = False
+disallow_untyped_defs = False
+disallow_incomplete_defs = False
+
+[mypy-qapi.error]
+disallow_untyped_defs = False
+disallow_incomplete_defs = False
+check_untyped_defs = False
+
+[mypy-qapi.events]
+disallow_untyped_defs = False
+disallow_incomplete_defs = False
+check_untyped_defs = False
+
+[mypy-qapi.expr]
+disallow_untyped_defs = False
+disallow_incomplete_defs = False
+check_untyped_defs = False
+
+[mypy-qapi.gen]
+disallow_untyped_defs = False
+disallow_incomplete_defs = False
+check_untyped_defs = False
+
+[mypy-qapi.introspect]
+disallow_untyped_defs = False
+disallow_incomplete_defs = False
+check_untyped_defs = False
+
+[mypy-qapi.parser]
+disallow_untyped_defs = False
+disallow_incomplete_defs = False
+check_untyped_defs = False
+
+[mypy-qapi.schema]
+disallow_untyped_defs = False
+disallow_incomplete_defs = False
+check_untyped_defs = False
+
+[mypy-qapi.source]
+disallow_untyped_defs = False
+disallow_incomplete_defs = False
+check_untyped_defs = False
+
+[mypy-qapi.types]
+disallow_untyped_defs = False
+disallow_incomplete_defs = False
+check_untyped_defs = False
+
+[mypy-qapi.visit]
+disallow_untyped_defs = False
+disallow_incomplete_defs = False
+check_untyped_defs = False
diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py
index b4921b46c9..bb0cd717f1 100644
--- a/scripts/qapi/schema.py
+++ b/scripts/qapi/schema.py
@@ -17,6 +17,7 @@
import os
import re
from collections import OrderedDict
+from typing import Optional
from .common import c_name, POINTER_SUFFIX
from .error import QAPIError, QAPISemError
@@ -25,7 +26,7 @@
class QAPISchemaEntity:
- meta = None
+ meta: Optional[str] = None
def __init__(self, name, info, doc, ifcond=None, features=None):
assert name is None or isinstance(name, str)
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 16/38] qapi: establish mypy type-checking baseline
2020-09-22 21:00 ` [PATCH v2 16/38] qapi: establish mypy type-checking baseline John Snow
@ 2020-09-23 14:29 ` Eduardo Habkost
2020-09-23 20:11 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 14:29 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:39PM -0400, John Snow wrote:
> Fix two very minor issues, and then establish a mypy type-checking
> baseline.
>
> Like pylint, this should be run from the folder above:
>
> > mypy --config-file=qapi/mypy.ini qapi/
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Works with mypy==0.770.
Tested-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 16/38] qapi: establish mypy type-checking baseline
2020-09-22 21:00 ` [PATCH v2 16/38] qapi: establish mypy type-checking baseline John Snow
2020-09-23 14:29 ` Eduardo Habkost
@ 2020-09-23 20:11 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 20:11 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 400 bytes --]
On Tue, Sep 22, 2020 at 05:00:39PM -0400, John Snow wrote:
> Fix two very minor issues, and then establish a mypy type-checking
> baseline.
>
> Like pylint, this should be run from the folder above:
>
> > mypy --config-file=qapi/mypy.ini qapi/
Like with pylint, let's at least document the version of mypy used.
For example, with 0.740, this fails. With 0.782, it passes.
- Cleber.
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 17/38] qapi/events.py: add type hint annotations
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (15 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 16/38] qapi: establish mypy type-checking baseline John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 14:31 ` Eduardo Habkost
2020-09-23 20:18 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 18/38] qapi/events.py: Move comments into docstrings John Snow
` (21 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Annotations do not change runtime behavior.
This commit *only* adds annotations.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/events.py | 46 ++++++++++++++++++++++++++++++++----------
scripts/qapi/mypy.ini | 5 -----
2 files changed, 35 insertions(+), 16 deletions(-)
diff --git a/scripts/qapi/events.py b/scripts/qapi/events.py
index 69a0ac5471..00a9513c16 100644
--- a/scripts/qapi/events.py
+++ b/scripts/qapi/events.py
@@ -12,6 +12,8 @@
See the COPYING file in the top-level directory.
"""
+from typing import List
+
from .common import (
c_enum_const,
c_name,
@@ -22,17 +24,27 @@
build_params,
ifcontext,
)
-from .schema import QAPISchemaEnumMember
+from .schema import (
+ QAPISchema,
+ QAPISchemaEnumMember,
+ QAPISchemaFeature,
+ QAPISchemaObjectType,
+)
+from .source import QAPISourceInfo
from .types import gen_enum, gen_enum_lookup
-def build_event_send_proto(name, arg_type, boxed):
+def build_event_send_proto(name: str,
+ arg_type: QAPISchemaObjectType,
+ boxed: bool) -> str:
return 'void qapi_event_send_%(c_name)s(%(param)s)' % {
'c_name': c_name(name.lower()),
'param': build_params(arg_type, boxed)}
-def gen_event_send_decl(name, arg_type, boxed):
+def gen_event_send_decl(name: str,
+ arg_type: QAPISchemaObjectType,
+ boxed: bool) -> str:
return mcgen('''
%(proto)s;
@@ -41,7 +53,7 @@ def gen_event_send_decl(name, arg_type, boxed):
# Declare and initialize an object 'qapi' using parameters from build_params()
-def gen_param_var(typ):
+def gen_param_var(typ: QAPISchemaObjectType) -> str:
assert not typ.variants
ret = mcgen('''
%(c_name)s param = {
@@ -69,7 +81,11 @@ def gen_param_var(typ):
return ret
-def gen_event_send(name, arg_type, boxed, event_enum_name, event_emit):
+def gen_event_send(name: str,
+ arg_type: QAPISchemaObjectType,
+ boxed: bool,
+ event_enum_name: str,
+ event_emit: str) -> str:
# FIXME: Our declaration of local variables (and of 'errp' in the
# parameter list) can collide with exploded members of the event's
# data type passed in as parameters. If this collision ever hits in
@@ -145,15 +161,15 @@ def gen_event_send(name, arg_type, boxed, event_enum_name, event_emit):
class QAPISchemaGenEventVisitor(QAPISchemaModularCVisitor):
- def __init__(self, prefix):
+ def __init__(self, prefix: str):
super().__init__(
prefix, 'qapi-events',
' * Schema-defined QAPI/QMP events', None, __doc__)
self._event_enum_name = c_name(prefix + 'QAPIEvent', protect=False)
- self._event_enum_members = []
+ self._event_enum_members: List[QAPISchemaEnumMember] = []
self._event_emit_name = c_name(prefix + 'qapi_event_emit')
- def _begin_user_module(self, name):
+ def _begin_user_module(self, name: str) -> None:
events = self._module_basename('qapi-events', name)
types = self._module_basename('qapi-types', name)
visit = self._module_basename('qapi-visit', name)
@@ -176,7 +192,7 @@ def _begin_user_module(self, name):
''',
types=types))
- def visit_end(self):
+ def visit_end(self) -> None:
self._add_system_module('emit', ' * QAPI Events emission')
self._genc.preamble_add(mcgen('''
#include "qemu/osdep.h"
@@ -197,7 +213,13 @@ def visit_end(self):
event_emit=self._event_emit_name,
event_enum=self._event_enum_name))
- def visit_event(self, name, info, ifcond, features, arg_type, boxed):
+ def visit_event(self,
+ name: str,
+ info: QAPISourceInfo,
+ ifcond: List[str],
+ features: List[QAPISchemaFeature],
+ arg_type: QAPISchemaObjectType,
+ boxed: bool) -> None:
with ifcontext(ifcond, self._genh, self._genc):
self._genh.add(gen_event_send_decl(name, arg_type, boxed))
self._genc.add(gen_event_send(name, arg_type, boxed,
@@ -208,7 +230,9 @@ def visit_event(self, name, info, ifcond, features, arg_type, boxed):
self._event_enum_members.append(QAPISchemaEnumMember(name, None))
-def gen_events(schema, output_dir, prefix):
+def gen_events(schema: QAPISchema,
+ output_dir: str,
+ prefix: str) -> None:
vis = QAPISchemaGenEventVisitor(prefix)
schema.visit(vis)
vis.write(output_dir)
diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini
index a0f2365a53..b668776d94 100644
--- a/scripts/qapi/mypy.ini
+++ b/scripts/qapi/mypy.ini
@@ -19,11 +19,6 @@ disallow_untyped_defs = False
disallow_incomplete_defs = False
check_untyped_defs = False
-[mypy-qapi.events]
-disallow_untyped_defs = False
-disallow_incomplete_defs = False
-check_untyped_defs = False
-
[mypy-qapi.expr]
disallow_untyped_defs = False
disallow_incomplete_defs = False
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 17/38] qapi/events.py: add type hint annotations
2020-09-22 21:00 ` [PATCH v2 17/38] qapi/events.py: add type hint annotations John Snow
@ 2020-09-23 14:31 ` Eduardo Habkost
2020-09-23 20:18 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 14:31 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:40PM -0400, John Snow wrote:
> Annotations do not change runtime behavior.
> This commit *only* adds annotations.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 17/38] qapi/events.py: add type hint annotations
2020-09-22 21:00 ` [PATCH v2 17/38] qapi/events.py: add type hint annotations John Snow
2020-09-23 14:31 ` Eduardo Habkost
@ 2020-09-23 20:18 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 20:18 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 245 bytes --]
On Tue, Sep 22, 2020 at 05:00:40PM -0400, John Snow wrote:
> Annotations do not change runtime behavior.
> This commit *only* adds annotations.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 18/38] qapi/events.py: Move comments into docstrings
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (16 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 17/38] qapi/events.py: add type hint annotations John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 14:48 ` Eduardo Habkost
2020-09-23 20:19 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 19/38] qapi/commands.py: Don't re-bind to variable of different type John Snow
` (20 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/events.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/scripts/qapi/events.py b/scripts/qapi/events.py
index 00a9513c16..e859fd7a52 100644
--- a/scripts/qapi/events.py
+++ b/scripts/qapi/events.py
@@ -52,8 +52,10 @@ def gen_event_send_decl(name: str,
proto=build_event_send_proto(name, arg_type, boxed))
-# Declare and initialize an object 'qapi' using parameters from build_params()
def gen_param_var(typ: QAPISchemaObjectType) -> str:
+ """
+ Declare and initialize a qapi object, using parameters from `build_params`.
+ """
assert not typ.variants
ret = mcgen('''
%(c_name)s param = {
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 18/38] qapi/events.py: Move comments into docstrings
2020-09-22 21:00 ` [PATCH v2 18/38] qapi/events.py: Move comments into docstrings John Snow
@ 2020-09-23 14:48 ` Eduardo Habkost
2020-09-23 18:21 ` John Snow
2020-09-25 12:19 ` Markus Armbruster
2020-09-23 20:19 ` Cleber Rosa
1 sibling, 2 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 14:48 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:41PM -0400, John Snow wrote:
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/events.py | 4 +++-
> 1 file changed, 3 insertions(+), 1 deletion(-)
>
> diff --git a/scripts/qapi/events.py b/scripts/qapi/events.py
> index 00a9513c16..e859fd7a52 100644
> --- a/scripts/qapi/events.py
> +++ b/scripts/qapi/events.py
> @@ -52,8 +52,10 @@ def gen_event_send_decl(name: str,
> proto=build_event_send_proto(name, arg_type, boxed))
>
>
> -# Declare and initialize an object 'qapi' using parameters from build_params()
> def gen_param_var(typ: QAPISchemaObjectType) -> str:
> + """
> + Declare and initialize a qapi object, using parameters from `build_params`.
The mention of "object 'qapi'" is gone, and this seems correct
because there's no object called 'qapi' anywhere in this
function. So:
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
Comments/questions for follow up patches, because it's beyond the
scope of this series:
- Was the doc string supposed to say "an object 'param'" instead
of "an object 'qapi'", as that's the name of the variable it
declares?
- The "using parameters from build_params()" part was confusing to
me. I thought it meant gen_param_var() would call build_params().
I took a while to notice it actually meant "using the C
function parameters that were previously declared using
build_params() at build_event_send_proto()". I don't know
how we could make it clearer.
> + """
> assert not typ.variants
> ret = mcgen('''
> %(c_name)s param = {
> --
> 2.26.2
>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 18/38] qapi/events.py: Move comments into docstrings
2020-09-23 14:48 ` Eduardo Habkost
@ 2020-09-23 18:21 ` John Snow
2020-09-25 12:19 ` Markus Armbruster
1 sibling, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-23 18:21 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On 9/23/20 10:48 AM, Eduardo Habkost wrote:
> On Tue, Sep 22, 2020 at 05:00:41PM -0400, John Snow wrote:
>> Signed-off-by: John Snow <jsnow@redhat.com>
>> ---
>> scripts/qapi/events.py | 4 +++-
>> 1 file changed, 3 insertions(+), 1 deletion(-)
>>
>> diff --git a/scripts/qapi/events.py b/scripts/qapi/events.py
>> index 00a9513c16..e859fd7a52 100644
>> --- a/scripts/qapi/events.py
>> +++ b/scripts/qapi/events.py
>> @@ -52,8 +52,10 @@ def gen_event_send_decl(name: str,
>> proto=build_event_send_proto(name, arg_type, boxed))
>>
>>
>> -# Declare and initialize an object 'qapi' using parameters from build_params()
>> def gen_param_var(typ: QAPISchemaObjectType) -> str:
>> + """
>> + Declare and initialize a qapi object, using parameters from `build_params`.
>
> The mention of "object 'qapi'" is gone, and this seems correct
> because there's no object called 'qapi' anywhere in this
> function. So:
> > Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
>
> Comments/questions for follow up patches, because it's beyond the
> scope of this series:
>
> - Was the doc string supposed to say "an object 'param'" instead
> of "an object 'qapi'", as that's the name of the variable it
> declares?
> - The "using parameters from build_params()" part was confusing to
> me. I thought it meant gen_param_var() would call build_params().
> I took a while to notice it actually meant "using the C
> function parameters that were previously declared using
> build_params() at build_event_send_proto()". I don't know
> how we could make it clearer.
>
Good questions for Markus.
(By the way, Markus: I do intend to remove the "missing-docstring"
warning from the exceptions, and at such time we can have a party
writing docstrings for everything.
I intend to devote an entire series to doing this during the release
window.)
--js
>> + """
>> assert not typ.variants
>> ret = mcgen('''
>> %(c_name)s param = {
>> --
>> 2.26.2
>>
>
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 18/38] qapi/events.py: Move comments into docstrings
2020-09-23 14:48 ` Eduardo Habkost
2020-09-23 18:21 ` John Snow
@ 2020-09-25 12:19 ` Markus Armbruster
2020-09-25 15:55 ` John Snow
1 sibling, 1 reply; 190+ messages in thread
From: Markus Armbruster @ 2020-09-25 12:19 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, John Snow, qemu-devel, Michael Roth, Cleber Rosa,
Alex Bennée
Eduardo Habkost <ehabkost@redhat.com> writes:
> On Tue, Sep 22, 2020 at 05:00:41PM -0400, John Snow wrote:
>> Signed-off-by: John Snow <jsnow@redhat.com>
>> ---
>> scripts/qapi/events.py | 4 +++-
>> 1 file changed, 3 insertions(+), 1 deletion(-)
>>
>> diff --git a/scripts/qapi/events.py b/scripts/qapi/events.py
>> index 00a9513c16..e859fd7a52 100644
>> --- a/scripts/qapi/events.py
>> +++ b/scripts/qapi/events.py
>> @@ -52,8 +52,10 @@ def gen_event_send_decl(name: str,
>> proto=build_event_send_proto(name, arg_type, boxed))
>>
>>
>> -# Declare and initialize an object 'qapi' using parameters from build_params()
>> def gen_param_var(typ: QAPISchemaObjectType) -> str:
>> + """
>> + Declare and initialize a qapi object, using parameters from `build_params`.
>
> The mention of "object 'qapi'" is gone, and this seems correct
> because there's no object called 'qapi' anywhere in this
> function. So:
>
> Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
>
> Comments/questions for follow up patches, because it's beyond the
> scope of this series:
>
> - Was the doc string supposed to say "an object 'param'" instead
> of "an object 'qapi'", as that's the name of the variable it
> declares?
Checking history... yes. The variable was renamed from @qapi to @param
during review.
> - The "using parameters from build_params()" part was confusing to
> me. I thought it meant gen_param_var() would call build_params().
> I took a while to notice it actually meant "using the C
> function parameters that were previously declared using
> build_params() at build_event_send_proto()". I don't know
> how we could make it clearer.
What about:
Generate a QAPI struct variable holding the event parameters,
initialized with the function's arguments.
>> + """
>> assert not typ.variants
>> ret = mcgen('''
>> %(c_name)s param = {
>> --
>> 2.26.2
>>
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 18/38] qapi/events.py: Move comments into docstrings
2020-09-25 12:19 ` Markus Armbruster
@ 2020-09-25 15:55 ` John Snow
2020-09-28 11:49 ` Markus Armbruster
0 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-25 15:55 UTC (permalink / raw)
To: Markus Armbruster, Eduardo Habkost
Cc: Peter Maydell, Cleber Rosa, Alex Bennée, qemu-devel,
Michael Roth
On 9/25/20 8:19 AM, Markus Armbruster wrote:
> What about:
>
> Generate a QAPI struct variable holding the event parameters,
> initialized with the function's arguments.
Line length and style-guide limitations; docstrings need a one-liner
summary.
(Consistency is the hobgoblin, blah blah blah.)
I am writing:
"""
Generate a QAPI struct variable with an initializer.
The QAPI struct describes the event parameters, and the initializer
references the function arguments defined in `gen_event_send`.
"""
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 18/38] qapi/events.py: Move comments into docstrings
2020-09-25 15:55 ` John Snow
@ 2020-09-28 11:49 ` Markus Armbruster
2020-09-28 15:04 ` John Snow
0 siblings, 1 reply; 190+ messages in thread
From: Markus Armbruster @ 2020-09-28 11:49 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Cleber Rosa, Alex Bennée
John Snow <jsnow@redhat.com> writes:
> On 9/25/20 8:19 AM, Markus Armbruster wrote:
>> What about:
>> Generate a QAPI struct variable holding the event parameters,
>> initialized with the function's arguments.
>
> Line length and style-guide limitations; docstrings need a one-liner
> summary.
They do!
> (Consistency is the hobgoblin, blah blah blah.)
>
> I am writing:
>
> """
> Generate a QAPI struct variable with an initializer.
>
> The QAPI struct describes the event parameters, and the initializer
> references the function arguments defined in `gen_event_send`.
> """
Better.
My second try:
Generate a struct variable holding the event parameters.
Initialize it with the function arguments defined in in
`gen_event_send`.
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 18/38] qapi/events.py: Move comments into docstrings
2020-09-28 11:49 ` Markus Armbruster
@ 2020-09-28 15:04 ` John Snow
0 siblings, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-28 15:04 UTC (permalink / raw)
To: Markus Armbruster
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Cleber Rosa, Alex Bennée
On 9/28/20 7:49 AM, Markus Armbruster wrote:
> John Snow <jsnow@redhat.com> writes:
>
>> On 9/25/20 8:19 AM, Markus Armbruster wrote:
>>> What about:
>>> Generate a QAPI struct variable holding the event parameters,
>>> initialized with the function's arguments.
>>
>> Line length and style-guide limitations; docstrings need a one-liner
>> summary.
>
> They do!
>
>> (Consistency is the hobgoblin, blah blah blah.)
>>
>> I am writing:
>>
>> """
>> Generate a QAPI struct variable with an initializer.
>>
>> The QAPI struct describes the event parameters, and the initializer
>> references the function arguments defined in `gen_event_send`.
>> """
>
> Better.
>
> My second try:
>
> Generate a struct variable holding the event parameters.
>
> Initialize it with the function arguments defined in in
> `gen_event_send`.
>
You're the boss!
--js
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 18/38] qapi/events.py: Move comments into docstrings
2020-09-22 21:00 ` [PATCH v2 18/38] qapi/events.py: Move comments into docstrings John Snow
2020-09-23 14:48 ` Eduardo Habkost
@ 2020-09-23 20:19 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 20:19 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 150 bytes --]
On Tue, Sep 22, 2020 at 05:00:41PM -0400, John Snow wrote:
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 19/38] qapi/commands.py: Don't re-bind to variable of different type
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (17 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 18/38] qapi/events.py: Move comments into docstrings John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 14:48 ` Eduardo Habkost
2020-09-23 20:21 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 20/38] qapi/commands.py: add notational type hints John Snow
` (19 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Mypy isn't a fan of rebinding a variable with a new data type.
It's easy enough to avoid.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/commands.py | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/scripts/qapi/commands.py b/scripts/qapi/commands.py
index 9b3f24b8ed..389dd879a6 100644
--- a/scripts/qapi/commands.py
+++ b/scripts/qapi/commands.py
@@ -198,14 +198,12 @@ def gen_register_command(name, success_response, allow_oob, allow_preconfig):
if not options:
options = ['QCO_NO_OPTIONS']
- options = " | ".join(options)
-
ret = mcgen('''
qmp_register_command(cmds, "%(name)s",
qmp_marshal_%(c_name)s, %(opts)s);
''',
name=name, c_name=c_name(name),
- opts=options)
+ opts=" | ".join(options))
return ret
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 19/38] qapi/commands.py: Don't re-bind to variable of different type
2020-09-22 21:00 ` [PATCH v2 19/38] qapi/commands.py: Don't re-bind to variable of different type John Snow
@ 2020-09-23 14:48 ` Eduardo Habkost
2020-09-23 20:21 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 14:48 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:42PM -0400, John Snow wrote:
> Mypy isn't a fan of rebinding a variable with a new data type.
> It's easy enough to avoid.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 19/38] qapi/commands.py: Don't re-bind to variable of different type
2020-09-22 21:00 ` [PATCH v2 19/38] qapi/commands.py: Don't re-bind to variable of different type John Snow
2020-09-23 14:48 ` Eduardo Habkost
@ 2020-09-23 20:21 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 20:21 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 254 bytes --]
On Tue, Sep 22, 2020 at 05:00:42PM -0400, John Snow wrote:
> Mypy isn't a fan of rebinding a variable with a new data type.
> It's easy enough to avoid.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 20/38] qapi/commands.py: add notational type hints
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (18 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 19/38] qapi/commands.py: Don't re-bind to variable of different type John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 14:50 ` Eduardo Habkost
2020-09-23 22:17 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 21/38] qapi/commands.py: enable checking with mypy John Snow
` (18 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/commands.py | 66 +++++++++++++++++++++++++++++-----------
1 file changed, 49 insertions(+), 17 deletions(-)
diff --git a/scripts/qapi/commands.py b/scripts/qapi/commands.py
index 389dd879a6..80f22d562d 100644
--- a/scripts/qapi/commands.py
+++ b/scripts/qapi/commands.py
@@ -13,19 +13,32 @@
See the COPYING file in the top-level directory.
"""
+from typing import Dict, Optional, List, Set
+
from .common import (
c_name,
mcgen,
)
from .gen import (
+ QAPIGenC,
QAPIGenCCode,
QAPISchemaModularCVisitor,
build_params,
ifcontext,
)
+from .schema import (
+ QAPISchema,
+ QAPISchemaFeature,
+ QAPISchemaObjectType,
+ QAPISchemaType,
+)
+from .source import QAPISourceInfo
-def gen_command_decl(name, arg_type, boxed, ret_type):
+def gen_command_decl(name: str,
+ arg_type: Optional[QAPISchemaObjectType],
+ boxed: bool,
+ ret_type: Optional[QAPISchemaType]) -> str:
return mcgen('''
%(c_type)s qmp_%(c_name)s(%(params)s);
''',
@@ -34,7 +47,10 @@ def gen_command_decl(name, arg_type, boxed, ret_type):
params=build_params(arg_type, boxed, 'Error **errp'))
-def gen_call(name, arg_type, boxed, ret_type):
+def gen_call(name: str,
+ arg_type: Optional[QAPISchemaObjectType],
+ boxed: bool,
+ ret_type: Optional[QAPISchemaType]) -> str:
ret = ''
argstr = ''
@@ -70,7 +86,7 @@ def gen_call(name, arg_type, boxed, ret_type):
return ret
-def gen_marshal_output(ret_type):
+def gen_marshal_output(ret_type: QAPISchemaType) -> str:
return mcgen('''
static void qmp_marshal_output_%(c_name)s(%(c_type)s ret_in,
@@ -91,19 +107,22 @@ def gen_marshal_output(ret_type):
c_type=ret_type.c_type(), c_name=ret_type.c_name())
-def build_marshal_proto(name):
+def build_marshal_proto(name: str) -> str:
return ('void qmp_marshal_%s(QDict *args, QObject **ret, Error **errp)'
% c_name(name))
-def gen_marshal_decl(name):
+def gen_marshal_decl(name: str) -> str:
return mcgen('''
%(proto)s;
''',
proto=build_marshal_proto(name))
-def gen_marshal(name, arg_type, boxed, ret_type):
+def gen_marshal(name: str,
+ arg_type: Optional[QAPISchemaObjectType],
+ boxed: bool,
+ ret_type: Optional[QAPISchemaType]) -> str:
have_args = boxed or (arg_type and not arg_type.is_empty())
ret = mcgen('''
@@ -185,7 +204,10 @@ def gen_marshal(name, arg_type, boxed, ret_type):
return ret
-def gen_register_command(name, success_response, allow_oob, allow_preconfig):
+def gen_register_command(name: str,
+ success_response: bool,
+ allow_oob: bool,
+ allow_preconfig: bool) -> str:
options = []
if not success_response:
@@ -207,7 +229,7 @@ def gen_register_command(name, success_response, allow_oob, allow_preconfig):
return ret
-def gen_registry(registry, prefix):
+def gen_registry(registry: str, prefix: str) -> str:
ret = mcgen('''
void %(c_prefix)sqmp_init_marshal(QmpCommandList *cmds)
@@ -224,15 +246,14 @@ def gen_registry(registry, prefix):
class QAPISchemaGenCommandVisitor(QAPISchemaModularCVisitor):
-
- def __init__(self, prefix):
+ def __init__(self, prefix: str):
super().__init__(
prefix, 'qapi-commands',
' * Schema-defined QAPI/QMP commands', None, __doc__)
self._regy = QAPIGenCCode(None)
- self._visited_ret_types = {}
+ self._visited_ret_types: Dict[QAPIGenC, Set[QAPISchemaType]] = {}
- def _begin_user_module(self, name):
+ def _begin_user_module(self, name: str) -> None:
self._visited_ret_types[self._genc] = set()
commands = self._module_basename('qapi-commands', name)
types = self._module_basename('qapi-types', name)
@@ -256,7 +277,7 @@ def _begin_user_module(self, name):
''',
types=types))
- def visit_end(self):
+ def visit_end(self) -> None:
self._add_system_module('init', ' * QAPI Commands initialization')
self._genh.add(mcgen('''
#include "qapi/qmp/dispatch.h"
@@ -272,9 +293,18 @@ def visit_end(self):
prefix=self._prefix))
self._genc.add(gen_registry(self._regy.get_content(), self._prefix))
- def visit_command(self, name, info, ifcond, features,
- arg_type, ret_type, gen, success_response, boxed,
- allow_oob, allow_preconfig):
+ def visit_command(self,
+ name: str,
+ info: QAPISourceInfo,
+ ifcond: List[str],
+ features: List[QAPISchemaFeature],
+ arg_type: Optional[QAPISchemaObjectType],
+ ret_type: Optional[QAPISchemaType],
+ gen: bool,
+ success_response: bool,
+ boxed: bool,
+ allow_oob: bool,
+ allow_preconfig: bool) -> None:
if not gen:
return
# FIXME: If T is a user-defined type, the user is responsible
@@ -295,7 +325,9 @@ def visit_command(self, name, info, ifcond, features,
allow_oob, allow_preconfig))
-def gen_commands(schema, output_dir, prefix):
+def gen_commands(schema: QAPISchema,
+ output_dir: str,
+ prefix: str) -> None:
vis = QAPISchemaGenCommandVisitor(prefix)
schema.visit(vis)
vis.write(output_dir)
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 20/38] qapi/commands.py: add notational type hints
2020-09-22 21:00 ` [PATCH v2 20/38] qapi/commands.py: add notational type hints John Snow
@ 2020-09-23 14:50 ` Eduardo Habkost
2020-09-23 18:23 ` John Snow
2020-09-23 22:17 ` Cleber Rosa
1 sibling, 1 reply; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 14:50 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:43PM -0400, John Snow wrote:
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 20/38] qapi/commands.py: add notational type hints
2020-09-23 14:50 ` Eduardo Habkost
@ 2020-09-23 18:23 ` John Snow
0 siblings, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-23 18:23 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On 9/23/20 10:50 AM, Eduardo Habkost wrote:
> On Tue, Sep 22, 2020 at 05:00:43PM -0400, John Snow wrote:
>> Signed-off-by: John Snow <jsnow@redhat.com>
>
> Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
>
Oops, I didn't use my consistent type hint commit message for this one.
Changing that to avoid my invented word "notational".
--js
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 20/38] qapi/commands.py: add notational type hints
2020-09-22 21:00 ` [PATCH v2 20/38] qapi/commands.py: add notational type hints John Snow
2020-09-23 14:50 ` Eduardo Habkost
@ 2020-09-23 22:17 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 22:17 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 150 bytes --]
On Tue, Sep 22, 2020 at 05:00:43PM -0400, John Snow wrote:
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 21/38] qapi/commands.py: enable checking with mypy
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (19 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 20/38] qapi/commands.py: add notational type hints John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 14:51 ` Eduardo Habkost
2020-09-23 22:21 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 22/38] qapi/source.py: add type hint annotations John Snow
` (17 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/mypy.ini | 5 -----
1 file changed, 5 deletions(-)
diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini
index b668776d94..9da1dccef4 100644
--- a/scripts/qapi/mypy.ini
+++ b/scripts/qapi/mypy.ini
@@ -4,11 +4,6 @@ strict_optional = False
disallow_untyped_calls = False
python_version = 3.6
-[mypy-qapi.commands]
-disallow_untyped_defs = False
-disallow_incomplete_defs = False
-check_untyped_defs = False
-
[mypy-qapi.doc]
disallow_subclassing_any = False
disallow_untyped_defs = False
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 21/38] qapi/commands.py: enable checking with mypy
2020-09-22 21:00 ` [PATCH v2 21/38] qapi/commands.py: enable checking with mypy John Snow
@ 2020-09-23 14:51 ` Eduardo Habkost
2020-09-23 22:21 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 14:51 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:44PM -0400, John Snow wrote:
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 21/38] qapi/commands.py: enable checking with mypy
2020-09-22 21:00 ` [PATCH v2 21/38] qapi/commands.py: enable checking with mypy John Snow
2020-09-23 14:51 ` Eduardo Habkost
@ 2020-09-23 22:21 ` Cleber Rosa
2020-09-23 23:50 ` John Snow
1 sibling, 1 reply; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 22:21 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 974 bytes --]
On Tue, Sep 22, 2020 at 05:00:44PM -0400, John Snow wrote:
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/mypy.ini | 5 -----
> 1 file changed, 5 deletions(-)
>
> diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini
> index b668776d94..9da1dccef4 100644
> --- a/scripts/qapi/mypy.ini
> +++ b/scripts/qapi/mypy.ini
> @@ -4,11 +4,6 @@ strict_optional = False
> disallow_untyped_calls = False
> python_version = 3.6
>
> -[mypy-qapi.commands]
> -disallow_untyped_defs = False
> -disallow_incomplete_defs = False
> -check_untyped_defs = False
> -
> [mypy-qapi.doc]
> disallow_subclassing_any = False
> disallow_untyped_defs = False
> --
> 2.26.2
>
IMO, this increase of strictness for "commands" would make more sense
to be squashed together with the previous changes on "commands.py".
Not only here, but for the other patches for the other modules too.
Anyway,
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 21/38] qapi/commands.py: enable checking with mypy
2020-09-23 22:21 ` Cleber Rosa
@ 2020-09-23 23:50 ` John Snow
0 siblings, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-23 23:50 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
On 9/23/20 6:21 PM, Cleber Rosa wrote:
> On Tue, Sep 22, 2020 at 05:00:44PM -0400, John Snow wrote:
>> Signed-off-by: John Snow <jsnow@redhat.com>
>> ---
>> scripts/qapi/mypy.ini | 5 -----
>> 1 file changed, 5 deletions(-)
>>
>> diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini
>> index b668776d94..9da1dccef4 100644
>> --- a/scripts/qapi/mypy.ini
>> +++ b/scripts/qapi/mypy.ini
>> @@ -4,11 +4,6 @@ strict_optional = False
>> disallow_untyped_calls = False
>> python_version = 3.6
>>
>> -[mypy-qapi.commands]
>> -disallow_untyped_defs = False
>> -disallow_incomplete_defs = False
>> -check_untyped_defs = False
>> -
>> [mypy-qapi.doc]
>> disallow_subclassing_any = False
>> disallow_untyped_defs = False
>> --
>> 2.26.2
>>
>
> IMO, this increase of strictness for "commands" would make more sense
> to be squashed together with the previous changes on "commands.py".
> Not only here, but for the other patches for the other modules too.
>
> Anyway,
>
> Reviewed-by: Cleber Rosa <crosa@redhat.com>
>
Admittedly, the only reason I *didn't* is because these patches have
been reordered a *lot* and in some cases, it helped me to have distinct
"This patch is last and enables the checks!" commits.
(I am hedging my bets that more re-ordering is in my future.)
I will squash the "Enable such-and-such" commits with whatever fixed the
last error for final inclusion, but I might keep them separate for now
just for my own convenience.
Sorry for the volume.
--js
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 22/38] qapi/source.py: add type hint annotations
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (20 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 21/38] qapi/commands.py: enable checking with mypy John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 14:52 ` Eduardo Habkost
2020-09-23 22:36 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 23/38] qapi/source.py: delint with pylint John Snow
` (16 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Annotations do not change runtime behavior.
This commit *only* adds annotations.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/mypy.ini | 5 -----
scripts/qapi/source.py | 31 ++++++++++++++++++-------------
2 files changed, 18 insertions(+), 18 deletions(-)
diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini
index 9da1dccef4..43c8bd1973 100644
--- a/scripts/qapi/mypy.ini
+++ b/scripts/qapi/mypy.ini
@@ -39,11 +39,6 @@ disallow_untyped_defs = False
disallow_incomplete_defs = False
check_untyped_defs = False
-[mypy-qapi.source]
-disallow_untyped_defs = False
-disallow_incomplete_defs = False
-check_untyped_defs = False
-
[mypy-qapi.types]
disallow_untyped_defs = False
disallow_incomplete_defs = False
diff --git a/scripts/qapi/source.py b/scripts/qapi/source.py
index e97b9a8e15..1cc6a5b82d 100644
--- a/scripts/qapi/source.py
+++ b/scripts/qapi/source.py
@@ -11,37 +11,42 @@
import copy
import sys
+from typing import List, Optional, TypeVar
class QAPISchemaPragma:
- def __init__(self):
+ def __init__(self) -> None:
# Are documentation comments required?
self.doc_required = False
# Whitelist of commands allowed to return a non-dictionary
- self.returns_whitelist = []
+ self.returns_whitelist: List[str] = []
# Whitelist of entities allowed to violate case conventions
- self.name_case_whitelist = []
+ self.name_case_whitelist: List[str] = []
class QAPISourceInfo:
- def __init__(self, fname, line, parent):
+ T = TypeVar('T', bound='QAPISourceInfo')
+
+ def __init__(self: T, fname: str, line: int, parent: Optional[T]):
self.fname = fname
self.line = line
self.parent = parent
- self.pragma = parent.pragma if parent else QAPISchemaPragma()
- self.defn_meta = None
- self.defn_name = None
+ self.pragma: QAPISchemaPragma = (
+ parent.pragma if parent else QAPISchemaPragma()
+ )
+ self.defn_meta: Optional[str] = None
+ self.defn_name: Optional[str] = None
- def set_defn(self, meta, name):
+ def set_defn(self, meta: str, name: str) -> None:
self.defn_meta = meta
self.defn_name = name
- def next_line(self):
+ def next_line(self: T) -> T:
info = copy.copy(self)
info.line += 1
return info
- def loc(self):
+ def loc(self) -> str:
if self.fname is None:
return sys.argv[0]
ret = self.fname
@@ -49,13 +54,13 @@ def loc(self):
ret += ':%d' % self.line
return ret
- def in_defn(self):
+ def in_defn(self) -> str:
if self.defn_name:
return "%s: In %s '%s':\n" % (self.fname,
self.defn_meta, self.defn_name)
return ''
- def include_path(self):
+ def include_path(self) -> str:
ret = ''
parent = self.parent
while parent:
@@ -63,5 +68,5 @@ def include_path(self):
parent = parent.parent
return ret
- def __str__(self):
+ def __str__(self) -> str:
return self.include_path() + self.in_defn() + self.loc()
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 22/38] qapi/source.py: add type hint annotations
2020-09-22 21:00 ` [PATCH v2 22/38] qapi/source.py: add type hint annotations John Snow
@ 2020-09-23 14:52 ` Eduardo Habkost
2020-09-23 22:36 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 14:52 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:45PM -0400, John Snow wrote:
> Annotations do not change runtime behavior.
> This commit *only* adds annotations.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 22/38] qapi/source.py: add type hint annotations
2020-09-22 21:00 ` [PATCH v2 22/38] qapi/source.py: add type hint annotations John Snow
2020-09-23 14:52 ` Eduardo Habkost
@ 2020-09-23 22:36 ` Cleber Rosa
2020-09-23 23:55 ` John Snow
1 sibling, 1 reply; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 22:36 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 2210 bytes --]
On Tue, Sep 22, 2020 at 05:00:45PM -0400, John Snow wrote:
> Annotations do not change runtime behavior.
> This commit *only* adds annotations.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/mypy.ini | 5 -----
> scripts/qapi/source.py | 31 ++++++++++++++++++-------------
> 2 files changed, 18 insertions(+), 18 deletions(-)
>
> diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini
> index 9da1dccef4..43c8bd1973 100644
> --- a/scripts/qapi/mypy.ini
> +++ b/scripts/qapi/mypy.ini
> @@ -39,11 +39,6 @@ disallow_untyped_defs = False
> disallow_incomplete_defs = False
> check_untyped_defs = False
>
> -[mypy-qapi.source]
> -disallow_untyped_defs = False
> -disallow_incomplete_defs = False
> -check_untyped_defs = False
> -
This is what I meant in my comment in the previous patch. It looks
like a mix of commit grannurality styles. Not a blocker though.
> [mypy-qapi.types]
> disallow_untyped_defs = False
> disallow_incomplete_defs = False
> diff --git a/scripts/qapi/source.py b/scripts/qapi/source.py
> index e97b9a8e15..1cc6a5b82d 100644
> --- a/scripts/qapi/source.py
> +++ b/scripts/qapi/source.py
> @@ -11,37 +11,42 @@
>
> import copy
> import sys
> +from typing import List, Optional, TypeVar
>
>
> class QAPISchemaPragma:
> - def __init__(self):
> + def __init__(self) -> None:
I don't follow the reason for typing this...
> # Are documentation comments required?
> self.doc_required = False
> # Whitelist of commands allowed to return a non-dictionary
> - self.returns_whitelist = []
> + self.returns_whitelist: List[str] = []
> # Whitelist of entities allowed to violate case conventions
> - self.name_case_whitelist = []
> + self.name_case_whitelist: List[str] = []
>
>
> class QAPISourceInfo:
> - def __init__(self, fname, line, parent):
> + T = TypeVar('T', bound='QAPISourceInfo')
> +
> + def __init__(self: T, fname: str, line: int, parent: Optional[T]):
And not this... to tune my review approach, should I assume that this
series intends to add complete type hints or not?
- Cleber.
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 22/38] qapi/source.py: add type hint annotations
2020-09-23 22:36 ` Cleber Rosa
@ 2020-09-23 23:55 ` John Snow
2020-09-25 12:22 ` Markus Armbruster
2020-09-25 17:05 ` Cleber Rosa
0 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-23 23:55 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
On 9/23/20 6:36 PM, Cleber Rosa wrote:
> On Tue, Sep 22, 2020 at 05:00:45PM -0400, John Snow wrote:
>> Annotations do not change runtime behavior.
>> This commit *only* adds annotations.
>>
>> Signed-off-by: John Snow <jsnow@redhat.com>
>> ---
>> scripts/qapi/mypy.ini | 5 -----
>> scripts/qapi/source.py | 31 ++++++++++++++++++-------------
>> 2 files changed, 18 insertions(+), 18 deletions(-)
>>
>> diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini
>> index 9da1dccef4..43c8bd1973 100644
>> --- a/scripts/qapi/mypy.ini
>> +++ b/scripts/qapi/mypy.ini
>> @@ -39,11 +39,6 @@ disallow_untyped_defs = False
>> disallow_incomplete_defs = False
>> check_untyped_defs = False
>>
>> -[mypy-qapi.source]
>> -disallow_untyped_defs = False
>> -disallow_incomplete_defs = False
>> -check_untyped_defs = False
>> -
>
> This is what I meant in my comment in the previous patch. It looks
> like a mix of commit grannurality styles. Not a blocker though.
>
Yep. Just how the chips fell. Some files were just very quick to cleanup
and I didn't have to refactor them much when I split things out, so the
enablements got rolled in.
I will, once reviews are in (and there is a commitment to merge), try to
squash things where it seems appropriate.
>> [mypy-qapi.types]
>> disallow_untyped_defs = False
>> disallow_incomplete_defs = False
>> diff --git a/scripts/qapi/source.py b/scripts/qapi/source.py
>> index e97b9a8e15..1cc6a5b82d 100644
>> --- a/scripts/qapi/source.py
>> +++ b/scripts/qapi/source.py
>> @@ -11,37 +11,42 @@
>>
>> import copy
>> import sys
>> +from typing import List, Optional, TypeVar
>>
>>
>> class QAPISchemaPragma:
>> - def __init__(self):
>> + def __init__(self) -> None:
>
> I don't follow the reason for typing this...
>
>> # Are documentation comments required?
>> self.doc_required = False
>> # Whitelist of commands allowed to return a non-dictionary
>> - self.returns_whitelist = []
>> + self.returns_whitelist: List[str] = []
>> # Whitelist of entities allowed to violate case conventions
>> - self.name_case_whitelist = []
>> + self.name_case_whitelist: List[str] = []
>>
>>
>> class QAPISourceInfo:
>> - def __init__(self, fname, line, parent):
>> + T = TypeVar('T', bound='QAPISourceInfo')
>> +
>> + def __init__(self: T, fname: str, line: int, parent: Optional[T]):
>
> And not this... to tune my review approach, should I assume that this
> series intends to add complete type hints or not?
>
This is a mypy quirk you've discovered that I've simply forgotten about.
When __init__ has *no* arguments, you need to annotate its return to
explain to mypy that you have fully typed that method. It's a sentinel
that says "Please type check this class!"
When __init__ has some arguments, you only need to type those arguments
and not the return type. The sentinel is not needed.
__init__ *never* returns anything, so I opted to omit this useless
annotation whenever it was possible to do so.
> - Cleber.
>
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 22/38] qapi/source.py: add type hint annotations
2020-09-23 23:55 ` John Snow
@ 2020-09-25 12:22 ` Markus Armbruster
2020-09-25 16:20 ` John Snow
2020-09-25 17:05 ` Cleber Rosa
1 sibling, 1 reply; 190+ messages in thread
From: Markus Armbruster @ 2020-09-25 12:22 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Cleber Rosa, Alex Bennée
John Snow <jsnow@redhat.com> writes:
> On 9/23/20 6:36 PM, Cleber Rosa wrote:
>> On Tue, Sep 22, 2020 at 05:00:45PM -0400, John Snow wrote:
>>> Annotations do not change runtime behavior.
>>> This commit *only* adds annotations.
>>>
>>> Signed-off-by: John Snow <jsnow@redhat.com>
[...]
>>> diff --git a/scripts/qapi/source.py b/scripts/qapi/source.py
>>> index e97b9a8e15..1cc6a5b82d 100644
>>> --- a/scripts/qapi/source.py
>>> +++ b/scripts/qapi/source.py
>>> @@ -11,37 +11,42 @@
>>> import copy
>>> import sys
>>> +from typing import List, Optional, TypeVar
>>>
>>> class QAPISchemaPragma:
>>> - def __init__(self):
>>> + def __init__(self) -> None:
>> I don't follow the reason for typing this...
>>
>>> # Are documentation comments required?
>>> self.doc_required = False
>>> # Whitelist of commands allowed to return a non-dictionary
>>> - self.returns_whitelist = []
>>> + self.returns_whitelist: List[str] = []
>>> # Whitelist of entities allowed to violate case conventions
>>> - self.name_case_whitelist = []
>>> + self.name_case_whitelist: List[str] = []
>>>
>>> class QAPISourceInfo:
>>> - def __init__(self, fname, line, parent):
>>> + T = TypeVar('T', bound='QAPISourceInfo')
>>> +
>>> + def __init__(self: T, fname: str, line: int, parent: Optional[T]):
>> And not this... to tune my review approach, should I assume that
>> this
>> series intends to add complete type hints or not?
>>
>
> This is a mypy quirk you've discovered that I've simply forgotten about.
>
> When __init__ has *no* arguments, you need to annotate its return to
> explain to mypy that you have fully typed that method. It's a sentinel
> that says "Please type check this class!"
>
> When __init__ has some arguments, you only need to type those
> arguments and not the return type. The sentinel is not needed.
>
> __init__ *never* returns anything, so I opted to omit this useless
> annotation whenever it was possible to do so.
Worth capturing in a comment and/or commit message?
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 22/38] qapi/source.py: add type hint annotations
2020-09-23 23:55 ` John Snow
2020-09-25 12:22 ` Markus Armbruster
@ 2020-09-25 17:05 ` Cleber Rosa
2020-09-25 17:20 ` John Snow
1 sibling, 1 reply; 190+ messages in thread
From: Cleber Rosa @ 2020-09-25 17:05 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 3219 bytes --]
On Wed, Sep 23, 2020 at 07:55:50PM -0400, John Snow wrote:
> On 9/23/20 6:36 PM, Cleber Rosa wrote:
> > On Tue, Sep 22, 2020 at 05:00:45PM -0400, John Snow wrote:
> > > Annotations do not change runtime behavior.
> > > This commit *only* adds annotations.
> > >
> > > Signed-off-by: John Snow <jsnow@redhat.com>
> > > ---
> > > scripts/qapi/mypy.ini | 5 -----
> > > scripts/qapi/source.py | 31 ++++++++++++++++++-------------
> > > 2 files changed, 18 insertions(+), 18 deletions(-)
> > >
> > > diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini
> > > index 9da1dccef4..43c8bd1973 100644
> > > --- a/scripts/qapi/mypy.ini
> > > +++ b/scripts/qapi/mypy.ini
> > > @@ -39,11 +39,6 @@ disallow_untyped_defs = False
> > > disallow_incomplete_defs = False
> > > check_untyped_defs = False
> > > -[mypy-qapi.source]
> > > -disallow_untyped_defs = False
> > > -disallow_incomplete_defs = False
> > > -check_untyped_defs = False
> > > -
> >
> > This is what I meant in my comment in the previous patch. It looks
> > like a mix of commit grannurality styles. Not a blocker though.
> >
>
> Yep. Just how the chips fell. Some files were just very quick to cleanup and
> I didn't have to refactor them much when I split things out, so the
> enablements got rolled in.
>
> I will, once reviews are in (and there is a commitment to merge), try to
> squash things where it seems appropriate.
>
> > > [mypy-qapi.types]
> > > disallow_untyped_defs = False
> > > disallow_incomplete_defs = False
> > > diff --git a/scripts/qapi/source.py b/scripts/qapi/source.py
> > > index e97b9a8e15..1cc6a5b82d 100644
> > > --- a/scripts/qapi/source.py
> > > +++ b/scripts/qapi/source.py
> > > @@ -11,37 +11,42 @@
> > > import copy
> > > import sys
> > > +from typing import List, Optional, TypeVar
> > > class QAPISchemaPragma:
> > > - def __init__(self):
> > > + def __init__(self) -> None:
> >
> > I don't follow the reason for typing this...
> >
> > > # Are documentation comments required?
> > > self.doc_required = False
> > > # Whitelist of commands allowed to return a non-dictionary
> > > - self.returns_whitelist = []
> > > + self.returns_whitelist: List[str] = []
> > > # Whitelist of entities allowed to violate case conventions
> > > - self.name_case_whitelist = []
> > > + self.name_case_whitelist: List[str] = []
> > > class QAPISourceInfo:
> > > - def __init__(self, fname, line, parent):
> > > + T = TypeVar('T', bound='QAPISourceInfo')
> > > +
> > > + def __init__(self: T, fname: str, line: int, parent: Optional[T]):
> >
> > And not this... to tune my review approach, should I assume that this
> > series intends to add complete type hints or not?
> >
>
> This is a mypy quirk you've discovered that I've simply forgotten about.
>
> When __init__ has *no* arguments, you need to annotate its return to explain
> to mypy that you have fully typed that method. It's a sentinel that says
> "Please type check this class!"
>
Ouch. Is this a permanent quirk or a known bug that will eventually
be addressed?
- Cleber.
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 22/38] qapi/source.py: add type hint annotations
2020-09-25 17:05 ` Cleber Rosa
@ 2020-09-25 17:20 ` John Snow
0 siblings, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-25 17:20 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
On 9/25/20 1:05 PM, Cleber Rosa wrote:
> On Wed, Sep 23, 2020 at 07:55:50PM -0400, John Snow wrote:
>> On 9/23/20 6:36 PM, Cleber Rosa wrote:
>>> On Tue, Sep 22, 2020 at 05:00:45PM -0400, John Snow wrote:
>>>> Annotations do not change runtime behavior.
>>>> This commit *only* adds annotations.
>>>>
>>>> Signed-off-by: John Snow <jsnow@redhat.com>
>>>> ---
>>>> scripts/qapi/mypy.ini | 5 -----
>>>> scripts/qapi/source.py | 31 ++++++++++++++++++-------------
>>>> 2 files changed, 18 insertions(+), 18 deletions(-)
>>>>
>>>> diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini
>>>> index 9da1dccef4..43c8bd1973 100644
>>>> --- a/scripts/qapi/mypy.ini
>>>> +++ b/scripts/qapi/mypy.ini
>>>> @@ -39,11 +39,6 @@ disallow_untyped_defs = False
>>>> disallow_incomplete_defs = False
>>>> check_untyped_defs = False
>>>> -[mypy-qapi.source]
>>>> -disallow_untyped_defs = False
>>>> -disallow_incomplete_defs = False
>>>> -check_untyped_defs = False
>>>> -
>>>
>>> This is what I meant in my comment in the previous patch. It looks
>>> like a mix of commit grannurality styles. Not a blocker though.
>>>
>>
>> Yep. Just how the chips fell. Some files were just very quick to cleanup and
>> I didn't have to refactor them much when I split things out, so the
>> enablements got rolled in.
>>
>> I will, once reviews are in (and there is a commitment to merge), try to
>> squash things where it seems appropriate.
>>
>>>> [mypy-qapi.types]
>>>> disallow_untyped_defs = False
>>>> disallow_incomplete_defs = False
>>>> diff --git a/scripts/qapi/source.py b/scripts/qapi/source.py
>>>> index e97b9a8e15..1cc6a5b82d 100644
>>>> --- a/scripts/qapi/source.py
>>>> +++ b/scripts/qapi/source.py
>>>> @@ -11,37 +11,42 @@
>>>> import copy
>>>> import sys
>>>> +from typing import List, Optional, TypeVar
>>>> class QAPISchemaPragma:
>>>> - def __init__(self):
>>>> + def __init__(self) -> None:
>>>
>>> I don't follow the reason for typing this...
>>>
>>>> # Are documentation comments required?
>>>> self.doc_required = False
>>>> # Whitelist of commands allowed to return a non-dictionary
>>>> - self.returns_whitelist = []
>>>> + self.returns_whitelist: List[str] = []
>>>> # Whitelist of entities allowed to violate case conventions
>>>> - self.name_case_whitelist = []
>>>> + self.name_case_whitelist: List[str] = []
>>>> class QAPISourceInfo:
>>>> - def __init__(self, fname, line, parent):
>>>> + T = TypeVar('T', bound='QAPISourceInfo')
>>>> +
>>>> + def __init__(self: T, fname: str, line: int, parent: Optional[T]):
>>>
>>> And not this... to tune my review approach, should I assume that this
>>> series intends to add complete type hints or not?
>>>
>>
>> This is a mypy quirk you've discovered that I've simply forgotten about.
>>
>> When __init__ has *no* arguments, you need to annotate its return to explain
>> to mypy that you have fully typed that method. It's a sentinel that says
>> "Please type check this class!"
>>
>
> Ouch. Is this a permanent quirk or a known bug that will eventually
> be addressed?
Permanent, it is a feature.
mypy intentionally supports gradual typing as a paradigm: it allows you
to intermix "typed" and "untyped" functions.
```
def __init__(self):
pass
```
Happens to pass as both untyped and fully typed. In order to distinguish
it in this one case, you must add the return annotation as a declaration
of intent.
However, when using '--strict' mode, you are declaring your intent to
mypy that everything MUST be strictly typed, so perhaps in this case it
would be possible to omit the annotation for __init__.
So maybe someday this will change; but given how uncommon it is to write
no-argument init methods, I am hardly bothered by it. Mypy will remind
you if you forget.
>
> - Cleber.
>
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 23/38] qapi/source.py: delint with pylint
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (21 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 22/38] qapi/source.py: add type hint annotations John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 14:59 ` Eduardo Habkost
2020-09-23 22:41 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 24/38] qapi/gen.py: Fix edge-case of _is_user_module John Snow
` (15 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Shush an error and leave a hint for future cleanups when we're allowed
to use Python 3.7+.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/pylintrc | 1 -
scripts/qapi/source.py | 3 +++
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/scripts/qapi/pylintrc b/scripts/qapi/pylintrc
index 6151427a51..7438806096 100644
--- a/scripts/qapi/pylintrc
+++ b/scripts/qapi/pylintrc
@@ -8,7 +8,6 @@ ignore-patterns=doc.py,
gen.py,
parser.py,
schema.py,
- source.py,
types.py,
visit.py,
diff --git a/scripts/qapi/source.py b/scripts/qapi/source.py
index 1cc6a5b82d..ba991d798f 100644
--- a/scripts/qapi/source.py
+++ b/scripts/qapi/source.py
@@ -15,6 +15,9 @@
class QAPISchemaPragma:
+ # Replace with @dataclass in Python 3.7+
+ # pylint: disable=too-few-public-methods
+
def __init__(self) -> None:
# Are documentation comments required?
self.doc_required = False
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 23/38] qapi/source.py: delint with pylint
2020-09-22 21:00 ` [PATCH v2 23/38] qapi/source.py: delint with pylint John Snow
@ 2020-09-23 14:59 ` Eduardo Habkost
2020-09-23 22:41 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 14:59 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:46PM -0400, John Snow wrote:
> Shush an error and leave a hint for future cleanups when we're allowed
> to use Python 3.7+.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 23/38] qapi/source.py: delint with pylint
2020-09-22 21:00 ` [PATCH v2 23/38] qapi/source.py: delint with pylint John Snow
2020-09-23 14:59 ` Eduardo Habkost
@ 2020-09-23 22:41 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 22:41 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 298 bytes --]
On Tue, Sep 22, 2020 at 05:00:46PM -0400, John Snow wrote:
> Shush an error and leave a hint for future cleanups when we're allowed
> to use Python 3.7+.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Cleber Rosa <crosa@redhat.com>
Tested-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 24/38] qapi/gen.py: Fix edge-case of _is_user_module
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (22 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 23/38] qapi/source.py: delint with pylint John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 15:17 ` Eduardo Habkost
2020-09-23 23:08 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 25/38] qapi/gen.py: add type hint annotations John Snow
` (14 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
The edge case is that if the name is '', this expression returns a
string instead of a bool, which violates our declared type.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/gen.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
index 9898d513ae..cb2b2655c3 100644
--- a/scripts/qapi/gen.py
+++ b/scripts/qapi/gen.py
@@ -251,7 +251,7 @@ def __init__(self, prefix, what, user_blurb, builtin_blurb, pydoc):
@staticmethod
def _is_user_module(name):
- return name and not name.startswith('./')
+ return name is not None and not name.startswith('./')
@staticmethod
def _is_builtin_module(name):
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 24/38] qapi/gen.py: Fix edge-case of _is_user_module
2020-09-22 21:00 ` [PATCH v2 24/38] qapi/gen.py: Fix edge-case of _is_user_module John Snow
@ 2020-09-23 15:17 ` Eduardo Habkost
2020-09-23 18:29 ` John Snow
2020-09-25 13:00 ` Markus Armbruster
2020-09-23 23:08 ` Cleber Rosa
1 sibling, 2 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 15:17 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:47PM -0400, John Snow wrote:
> The edge case is that if the name is '', this expression returns a
> string instead of a bool, which violates our declared type.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/gen.py | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
> index 9898d513ae..cb2b2655c3 100644
> --- a/scripts/qapi/gen.py
> +++ b/scripts/qapi/gen.py
> @@ -251,7 +251,7 @@ def __init__(self, prefix, what, user_blurb, builtin_blurb, pydoc):
>
> @staticmethod
> def _is_user_module(name):
> - return name and not name.startswith('./')
> + return name is not None and not name.startswith('./')
This changes behavior if name=='', and I guess this is OK, but
I'm not sure. I miss documentation on `visit_module()`,
`visit_include()`, and `_is_user_module()`. I don't know what
`name` means here, and what is a "user module".
>
> @staticmethod
> def _is_builtin_module(name):
> --
> 2.26.2
>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 24/38] qapi/gen.py: Fix edge-case of _is_user_module
2020-09-23 15:17 ` Eduardo Habkost
@ 2020-09-23 18:29 ` John Snow
2020-09-23 18:33 ` Eduardo Habkost
2020-09-25 13:00 ` Markus Armbruster
1 sibling, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-23 18:29 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, Michael Roth, qemu-devel, Markus Armbruster,
Cleber Rosa, Alex Bennée
On 9/23/20 11:17 AM, Eduardo Habkost wrote:
> This changes behavior if name=='', and I guess this is OK, but
> I'm not sure. I miss documentation on `visit_module()`,
> `visit_include()`, and `_is_user_module()`. I don't know what
> `name` means here, and what is a "user module".
>
Good spot, I missed that.
I can probably do: bool(name and not name.startswith('./'))
to convert explicitly the empty string to false. I will allow Markus the
chance to explain the module stuff.
--js
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 24/38] qapi/gen.py: Fix edge-case of _is_user_module
2020-09-23 18:29 ` John Snow
@ 2020-09-23 18:33 ` Eduardo Habkost
2020-09-23 23:10 ` Cleber Rosa
0 siblings, 1 reply; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 18:33 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Michael Roth, qemu-devel, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Wed, Sep 23, 2020 at 02:29:28PM -0400, John Snow wrote:
> On 9/23/20 11:17 AM, Eduardo Habkost wrote:
> > This changes behavior if name=='', and I guess this is OK, but
> > I'm not sure. I miss documentation on `visit_module()`,
> > `visit_include()`, and `_is_user_module()`. I don't know what
> > `name` means here, and what is a "user module".
> >
>
> Good spot, I missed that.
>
> I can probably do: bool(name and not name.startswith('./'))
>
> to convert explicitly the empty string to false. I will allow Markus the
> chance to explain the module stuff.
Sound good to me. If the current behavior needs to be changed,
it can be fixed later.
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 24/38] qapi/gen.py: Fix edge-case of _is_user_module
2020-09-23 18:33 ` Eduardo Habkost
@ 2020-09-23 23:10 ` Cleber Rosa
2020-09-23 23:13 ` Cleber Rosa
0 siblings, 1 reply; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 23:10 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, Michael Roth, John Snow, qemu-devel,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 655 bytes --]
On Wed, Sep 23, 2020 at 02:33:35PM -0400, Eduardo Habkost wrote:
> On Wed, Sep 23, 2020 at 02:29:28PM -0400, John Snow wrote:
> > On 9/23/20 11:17 AM, Eduardo Habkost wrote:
> > > This changes behavior if name=='', and I guess this is OK, but
> > > I'm not sure. I miss documentation on `visit_module()`,
> > > `visit_include()`, and `_is_user_module()`. I don't know what
> > > `name` means here, and what is a "user module".
> > >
> >
> > Good spot, I missed that.
> >
Nice catch indeed.
> > I can probably do: bool(name and not name.startswith('./'))
> >
In this case I like my previous suggestion even better. :)
- Cleber.
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 24/38] qapi/gen.py: Fix edge-case of _is_user_module
2020-09-23 23:10 ` Cleber Rosa
@ 2020-09-23 23:13 ` Cleber Rosa
0 siblings, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 23:13 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, Michael Roth, John Snow, qemu-devel,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 842 bytes --]
On Wed, Sep 23, 2020 at 07:10:35PM -0400, Cleber Rosa wrote:
> On Wed, Sep 23, 2020 at 02:33:35PM -0400, Eduardo Habkost wrote:
> > On Wed, Sep 23, 2020 at 02:29:28PM -0400, John Snow wrote:
> > > On 9/23/20 11:17 AM, Eduardo Habkost wrote:
> > > > This changes behavior if name=='', and I guess this is OK, but
> > > > I'm not sure. I miss documentation on `visit_module()`,
> > > > `visit_include()`, and `_is_user_module()`. I don't know what
> > > > `name` means here, and what is a "user module".
> > > >
> > >
> > > Good spot, I missed that.
> > >
>
> Nice catch indeed.
>
> > > I can probably do: bool(name and not name.startswith('./'))
> > >
>
> In this case I like my previous suggestion even better. :)
>
> - Cleber.
Never mind me... I noticed that this actually gets called with None.
- Cleber.
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 24/38] qapi/gen.py: Fix edge-case of _is_user_module
2020-09-23 15:17 ` Eduardo Habkost
2020-09-23 18:29 ` John Snow
@ 2020-09-25 13:00 ` Markus Armbruster
2020-09-25 15:15 ` Eduardo Habkost
2020-09-25 16:29 ` John Snow
1 sibling, 2 replies; 190+ messages in thread
From: Markus Armbruster @ 2020-09-25 13:00 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, John Snow, qemu-devel, Michael Roth, Cleber Rosa,
Alex Bennée
Eduardo Habkost <ehabkost@redhat.com> writes:
> On Tue, Sep 22, 2020 at 05:00:47PM -0400, John Snow wrote:
>> The edge case is that if the name is '', this expression returns a
>> string instead of a bool, which violates our declared type.
>>
>> Signed-off-by: John Snow <jsnow@redhat.com>
>> ---
>> scripts/qapi/gen.py | 2 +-
>> 1 file changed, 1 insertion(+), 1 deletion(-)
>>
>> diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
>> index 9898d513ae..cb2b2655c3 100644
>> --- a/scripts/qapi/gen.py
>> +++ b/scripts/qapi/gen.py
>> @@ -251,7 +251,7 @@ def __init__(self, prefix, what, user_blurb, builtin_blurb, pydoc):
>>
>> @staticmethod
>> def _is_user_module(name):
>> - return name and not name.startswith('./')
>> + return name is not None and not name.startswith('./')
>
> This changes behavior if name=='', and I guess this is OK, but
> I'm not sure.
@name is either
(1) A module pathname relative to the main module
This is a module defined by the user.
(2) system module name, starting with './'
This is a named system module. We currently have two: './init' in
commands.py, and and './emit' in events.py.
(3) None
This is the (nameless) system module for built-in stuff. It
predates (2). Using './builtin' would probably be better now.
Note that (1) and (2) are disjoint: relative pathnames do not begin with
'./'.
name='' is not possible, because '' is not a valid pathname.
> I miss documentation on `visit_module()`,
> `visit_include()`, and `_is_user_module()`. I don't know what
> `name` means here, and what is a "user module".
Valid complaints! The code is subtle in places, without helping its
readers along with comments or doc strings.
>
>>
>> @staticmethod
>> def _is_builtin_module(name):
>> --
>> 2.26.2
>>
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 24/38] qapi/gen.py: Fix edge-case of _is_user_module
2020-09-25 13:00 ` Markus Armbruster
@ 2020-09-25 15:15 ` Eduardo Habkost
2020-09-25 15:50 ` Eduardo Habkost
2020-09-25 16:29 ` John Snow
1 sibling, 1 reply; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-25 15:15 UTC (permalink / raw)
To: Markus Armbruster
Cc: Peter Maydell, John Snow, qemu-devel, Michael Roth, Cleber Rosa,
Alex Bennée
On Fri, Sep 25, 2020 at 03:00:51PM +0200, Markus Armbruster wrote:
> Eduardo Habkost <ehabkost@redhat.com> writes:
>
> > On Tue, Sep 22, 2020 at 05:00:47PM -0400, John Snow wrote:
> >> The edge case is that if the name is '', this expression returns a
> >> string instead of a bool, which violates our declared type.
> >>
> >> Signed-off-by: John Snow <jsnow@redhat.com>
> >> ---
> >> scripts/qapi/gen.py | 2 +-
> >> 1 file changed, 1 insertion(+), 1 deletion(-)
> >>
> >> diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
> >> index 9898d513ae..cb2b2655c3 100644
> >> --- a/scripts/qapi/gen.py
> >> +++ b/scripts/qapi/gen.py
> >> @@ -251,7 +251,7 @@ def __init__(self, prefix, what, user_blurb, builtin_blurb, pydoc):
> >>
> >> @staticmethod
> >> def _is_user_module(name):
> >> - return name and not name.startswith('./')
> >> + return name is not None and not name.startswith('./')
> >
> > This changes behavior if name=='', and I guess this is OK, but
> > I'm not sure.
>
> @name is either
>
> (1) A module pathname relative to the main module
>
> This is a module defined by the user.
>
> (2) system module name, starting with './'
>
> This is a named system module. We currently have two: './init' in
> commands.py, and and './emit' in events.py.
>
> (3) None
>
> This is the (nameless) system module for built-in stuff. It
> predates (2). Using './builtin' would probably be better now.
>
> Note that (1) and (2) are disjoint: relative pathnames do not begin with
> './'.
>
> name='' is not possible, because '' is not a valid pathname.
Thanks! So, the './' prefix is just internal state and never
visible to the outside, correct? I would use a separate bool
instead of trying to encode additional state inside the string.
>
> > I miss documentation on `visit_module()`,
> > `visit_include()`, and `_is_user_module()`. I don't know what
> > `name` means here, and what is a "user module".
>
> Valid complaints! The code is subtle in places, without helping its
> readers along with comments or doc strings.
>
> >
> >>
> >> @staticmethod
> >> def _is_builtin_module(name):
> >> --
> >> 2.26.2
> >>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 24/38] qapi/gen.py: Fix edge-case of _is_user_module
2020-09-25 15:15 ` Eduardo Habkost
@ 2020-09-25 15:50 ` Eduardo Habkost
2020-09-28 12:04 ` Markus Armbruster
0 siblings, 1 reply; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-25 15:50 UTC (permalink / raw)
To: Markus Armbruster
Cc: Peter Maydell, John Snow, qemu-devel, Michael Roth, Cleber Rosa,
Alex Bennée
On Fri, Sep 25, 2020 at 11:15:28AM -0400, Eduardo Habkost wrote:
> On Fri, Sep 25, 2020 at 03:00:51PM +0200, Markus Armbruster wrote:
> > Eduardo Habkost <ehabkost@redhat.com> writes:
> >
> > > On Tue, Sep 22, 2020 at 05:00:47PM -0400, John Snow wrote:
> > >> The edge case is that if the name is '', this expression returns a
> > >> string instead of a bool, which violates our declared type.
> > >>
> > >> Signed-off-by: John Snow <jsnow@redhat.com>
> > >> ---
> > >> scripts/qapi/gen.py | 2 +-
> > >> 1 file changed, 1 insertion(+), 1 deletion(-)
> > >>
> > >> diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
> > >> index 9898d513ae..cb2b2655c3 100644
> > >> --- a/scripts/qapi/gen.py
> > >> +++ b/scripts/qapi/gen.py
> > >> @@ -251,7 +251,7 @@ def __init__(self, prefix, what, user_blurb, builtin_blurb, pydoc):
> > >>
> > >> @staticmethod
> > >> def _is_user_module(name):
> > >> - return name and not name.startswith('./')
> > >> + return name is not None and not name.startswith('./')
> > >
> > > This changes behavior if name=='', and I guess this is OK, but
> > > I'm not sure.
> >
> > @name is either
> >
> > (1) A module pathname relative to the main module
> >
> > This is a module defined by the user.
> >
> > (2) system module name, starting with './'
> >
> > This is a named system module. We currently have two: './init' in
> > commands.py, and and './emit' in events.py.
> >
> > (3) None
> >
> > This is the (nameless) system module for built-in stuff. It
> > predates (2). Using './builtin' would probably be better now.
> >
> > Note that (1) and (2) are disjoint: relative pathnames do not begin with
> > './'.
> >
> > name='' is not possible, because '' is not a valid pathname.
>
> Thanks! So, the './' prefix is just internal state and never
> visible to the outside, correct? I would use a separate bool
> instead of trying to encode additional state inside the string.
I've found only one place where the './' prefix might be leaking,
and I don't know if it's intentional or not:
Is the name argument to visit_include() supposed to be always
(1), or are './' pathnames allowed too?
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 24/38] qapi/gen.py: Fix edge-case of _is_user_module
2020-09-25 15:50 ` Eduardo Habkost
@ 2020-09-28 12:04 ` Markus Armbruster
0 siblings, 0 replies; 190+ messages in thread
From: Markus Armbruster @ 2020-09-28 12:04 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, John Snow, qemu-devel, Michael Roth, Cleber Rosa,
Alex Bennée
Eduardo Habkost <ehabkost@redhat.com> writes:
> On Fri, Sep 25, 2020 at 11:15:28AM -0400, Eduardo Habkost wrote:
>> On Fri, Sep 25, 2020 at 03:00:51PM +0200, Markus Armbruster wrote:
>> > Eduardo Habkost <ehabkost@redhat.com> writes:
>> >
>> > > On Tue, Sep 22, 2020 at 05:00:47PM -0400, John Snow wrote:
>> > >> The edge case is that if the name is '', this expression returns a
>> > >> string instead of a bool, which violates our declared type.
>> > >>
>> > >> Signed-off-by: John Snow <jsnow@redhat.com>
>> > >> ---
>> > >> scripts/qapi/gen.py | 2 +-
>> > >> 1 file changed, 1 insertion(+), 1 deletion(-)
>> > >>
>> > >> diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
>> > >> index 9898d513ae..cb2b2655c3 100644
>> > >> --- a/scripts/qapi/gen.py
>> > >> +++ b/scripts/qapi/gen.py
>> > >> @@ -251,7 +251,7 @@ def __init__(self, prefix, what, user_blurb, builtin_blurb, pydoc):
>> > >>
>> > >> @staticmethod
>> > >> def _is_user_module(name):
>> > >> - return name and not name.startswith('./')
>> > >> + return name is not None and not name.startswith('./')
>> > >
>> > > This changes behavior if name=='', and I guess this is OK, but
>> > > I'm not sure.
>> >
>> > @name is either
>> >
>> > (1) A module pathname relative to the main module
>> >
>> > This is a module defined by the user.
>> >
>> > (2) system module name, starting with './'
>> >
>> > This is a named system module. We currently have two: './init' in
>> > commands.py, and and './emit' in events.py.
>> >
>> > (3) None
>> >
>> > This is the (nameless) system module for built-in stuff. It
>> > predates (2). Using './builtin' would probably be better now.
>> >
>> > Note that (1) and (2) are disjoint: relative pathnames do not begin with
>> > './'.
>> >
>> > name='' is not possible, because '' is not a valid pathname.
>>
>> Thanks! So, the './' prefix is just internal state and never
>> visible to the outside, correct?
Yes.
>> I would use a separate bool
>> instead of trying to encode additional state inside the string.
>
> I've found only one place where the './' prefix might be leaking,
> and I don't know if it's intentional or not:
>
> Is the name argument to visit_include() supposed to be always
> (1), or are './' pathnames allowed too?
Always (1).
visit_include() gets passed a module name:
class QAPISchemaInclude(QAPISchemaEntity):
[...]
def visit(self, visitor):
super().visit(visitor)
visitor.visit_include(self._sub_module.name, self.info)
Module names are relative to the main module's directory:
def _module_name(self, fname):
if fname is None:
return None
return os.path.relpath(fname, self._schema_dir)
os,path.relpath() normalizes away './':
$ python
Python 3.8.5 (default, Aug 12 2020, 00:00:00)
[GCC 10.2.1 20200723 (Red Hat 10.2.1-1)] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> os.path.relpath('./sub.json', '')
'sub.json'
QAPISchema._make_module() uses ._module_name() as it should:
def _make_module(self, fname):
name = self._module_name(fname)
if name not in self._module_dict:
self._module_dict[name] = QAPISchemaModule(name)
return self._module_dict[name]
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 24/38] qapi/gen.py: Fix edge-case of _is_user_module
2020-09-25 13:00 ` Markus Armbruster
2020-09-25 15:15 ` Eduardo Habkost
@ 2020-09-25 16:29 ` John Snow
1 sibling, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-25 16:29 UTC (permalink / raw)
To: Markus Armbruster, Eduardo Habkost
Cc: Peter Maydell, Cleber Rosa, Alex Bennée, qemu-devel,
Michael Roth
On 9/25/20 9:00 AM, Markus Armbruster wrote:
> Eduardo Habkost <ehabkost@redhat.com> writes:
>
>> On Tue, Sep 22, 2020 at 05:00:47PM -0400, John Snow wrote:
>>> The edge case is that if the name is '', this expression returns a
>>> string instead of a bool, which violates our declared type.
>>>
>>> Signed-off-by: John Snow <jsnow@redhat.com>
>>> ---
>>> scripts/qapi/gen.py | 2 +-
>>> 1 file changed, 1 insertion(+), 1 deletion(-)
>>>
>>> diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
>>> index 9898d513ae..cb2b2655c3 100644
>>> --- a/scripts/qapi/gen.py
>>> +++ b/scripts/qapi/gen.py
>>> @@ -251,7 +251,7 @@ def __init__(self, prefix, what, user_blurb, builtin_blurb, pydoc):
>>>
>>> @staticmethod
>>> def _is_user_module(name):
>>> - return name and not name.startswith('./')
>>> + return name is not None and not name.startswith('./')
>>
>> This changes behavior if name=='', and I guess this is OK, but
>> I'm not sure.
>
> @name is either
>
> (1) A module pathname relative to the main module
>
> This is a module defined by the user.
>
> (2) system module name, starting with './'
>
> This is a named system module. We currently have two: './init' in
> commands.py, and and './emit' in events.py.
>
> (3) None
>
> This is the (nameless) system module for built-in stuff. It
> predates (2). Using './builtin' would probably be better now.
>
Yes please! This would help simplify Optional[str] to str in many
places, and removes doubt as to what "None" might imply.
Let's queue that idea up as a cleanup for after this typing series.
> Note that (1) and (2) are disjoint: relative pathnames do not begin with
> './'.
>
> name='' is not possible, because '' is not a valid pathname.
>
>> I miss documentation on `visit_module()`,
>> `visit_include()`, and `_is_user_module()`. I don't know what
>> `name` means here, and what is a "user module".
>
> Valid complaints! The code is subtle in places, without helping its
> readers along with comments or doc strings.
>
>>
>>>
>>> @staticmethod
>>> def _is_builtin_module(name):
>>> --
>>> 2.26.2
>>>
For now, I've done the simpler thing and wrapped the return in
bool(...), but we will be able to do much more cleanups if we eliminate
the possibility of "None" module names later. I'll get it all at once.
I'm adding it to my python TODO: https://gitlab.com/jsnow/qemu/-/issues/8
--js
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 24/38] qapi/gen.py: Fix edge-case of _is_user_module
2020-09-22 21:00 ` [PATCH v2 24/38] qapi/gen.py: Fix edge-case of _is_user_module John Snow
2020-09-23 15:17 ` Eduardo Habkost
@ 2020-09-23 23:08 ` Cleber Rosa
2020-09-23 23:13 ` Cleber Rosa
2020-09-23 23:57 ` John Snow
1 sibling, 2 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 23:08 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 989 bytes --]
On Tue, Sep 22, 2020 at 05:00:47PM -0400, John Snow wrote:
> The edge case is that if the name is '', this expression returns a
> string instead of a bool, which violates our declared type.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/gen.py | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
> index 9898d513ae..cb2b2655c3 100644
> --- a/scripts/qapi/gen.py
> +++ b/scripts/qapi/gen.py
> @@ -251,7 +251,7 @@ def __init__(self, prefix, what, user_blurb, builtin_blurb, pydoc):
>
> @staticmethod
> def _is_user_module(name):
> - return name and not name.startswith('./')
> + return name is not None and not name.startswith('./')
Another possibility here that handles the empty string situation and
will never return anything but a bool:
return all([name, not name.startswith('./')])
Either way,
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 24/38] qapi/gen.py: Fix edge-case of _is_user_module
2020-09-23 23:08 ` Cleber Rosa
@ 2020-09-23 23:13 ` Cleber Rosa
2020-09-23 23:57 ` John Snow
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 23:13 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 1125 bytes --]
On Wed, Sep 23, 2020 at 07:08:50PM -0400, Cleber Rosa wrote:
> On Tue, Sep 22, 2020 at 05:00:47PM -0400, John Snow wrote:
> > The edge case is that if the name is '', this expression returns a
> > string instead of a bool, which violates our declared type.
> >
> > Signed-off-by: John Snow <jsnow@redhat.com>
> > ---
> > scripts/qapi/gen.py | 2 +-
> > 1 file changed, 1 insertion(+), 1 deletion(-)
> >
> > diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
> > index 9898d513ae..cb2b2655c3 100644
> > --- a/scripts/qapi/gen.py
> > +++ b/scripts/qapi/gen.py
> > @@ -251,7 +251,7 @@ def __init__(self, prefix, what, user_blurb, builtin_blurb, pydoc):
> >
> > @staticmethod
> > def _is_user_module(name):
> > - return name and not name.startswith('./')
> > + return name is not None and not name.startswith('./')
>
> Another possibility here that handles the empty string situation and
> will never return anything but a bool:
>
> return all([name, not name.startswith('./')])
>
Never mind me... I noticed that this actually gets called with None.
- Cleber.
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 24/38] qapi/gen.py: Fix edge-case of _is_user_module
2020-09-23 23:08 ` Cleber Rosa
2020-09-23 23:13 ` Cleber Rosa
@ 2020-09-23 23:57 ` John Snow
1 sibling, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-23 23:57 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
On 9/23/20 7:08 PM, Cleber Rosa wrote:
> On Tue, Sep 22, 2020 at 05:00:47PM -0400, John Snow wrote:
>> The edge case is that if the name is '', this expression returns a
>> string instead of a bool, which violates our declared type.
>>
>> Signed-off-by: John Snow <jsnow@redhat.com>
>> ---
>> scripts/qapi/gen.py | 2 +-
>> 1 file changed, 1 insertion(+), 1 deletion(-)
>>
>> diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
>> index 9898d513ae..cb2b2655c3 100644
>> --- a/scripts/qapi/gen.py
>> +++ b/scripts/qapi/gen.py
>> @@ -251,7 +251,7 @@ def __init__(self, prefix, what, user_blurb, builtin_blurb, pydoc):
>>
>> @staticmethod
>> def _is_user_module(name):
>> - return name and not name.startswith('./')
>> + return name is not None and not name.startswith('./')
>
> Another possibility here that handles the empty string situation and
> will never return anything but a bool:
>
> return all([name, not name.startswith('./')])
>
> Either way,
>
> Reviewed-by: Cleber Rosa <crosa@redhat.com>
>
I wound up changing this per ehabkost's review.
--js
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 25/38] qapi/gen.py: add type hint annotations
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (23 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 24/38] qapi/gen.py: Fix edge-case of _is_user_module John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 15:18 ` Eduardo Habkost
2020-09-23 23:51 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 26/38] qapi/gen.py: Enable checking with mypy John Snow
` (13 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Annotations do not change runtime behavior.
This commit *only* adds annotations.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/gen.py | 102 +++++++++++++++++++++++---------------------
1 file changed, 53 insertions(+), 49 deletions(-)
diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
index cb2b2655c3..df8cf8271c 100644
--- a/scripts/qapi/gen.py
+++ b/scripts/qapi/gen.py
@@ -17,7 +17,7 @@
import errno
import os
import re
-from typing import Optional
+from typing import Dict, Generator, List, Optional, Tuple
from .common import (
c_fname,
@@ -32,31 +32,31 @@
QAPISchemaObjectType,
QAPISchemaVisitor,
)
+from .source import QAPISourceInfo
class QAPIGen:
-
- def __init__(self, fname):
+ def __init__(self, fname: Optional[str]):
self.fname = fname
self._preamble = ''
self._body = ''
- def preamble_add(self, text):
+ def preamble_add(self, text: str) -> None:
self._preamble += text
- def add(self, text):
+ def add(self, text: str) -> None:
self._body += text
- def get_content(self):
+ def get_content(self) -> str:
return self._top() + self._preamble + self._body + self._bottom()
- def _top(self):
+ def _top(self) -> str:
return ''
- def _bottom(self):
+ def _bottom(self) -> str:
return ''
- def write(self, output_dir):
+ def write(self, output_dir: str) -> None:
# Include paths starting with ../ are used to reuse modules of the main
# schema in specialised schemas. Don't overwrite the files that are
# already generated for the main schema.
@@ -81,7 +81,7 @@ def write(self, output_dir):
f.close()
-def _wrap_ifcond(ifcond, before, after):
+def _wrap_ifcond(ifcond: List[str], before: str, after: str) -> str:
if before == after:
return after # suppress empty #if ... #endif
@@ -121,40 +121,38 @@ def build_params(arg_type: Optional[QAPISchemaObjectType],
class QAPIGenCCode(QAPIGen):
-
- def __init__(self, fname):
+ def __init__(self, fname: Optional[str]):
super().__init__(fname)
- self._start_if = None
+ self._start_if: Optional[Tuple[List[str], str, str]] = None
- def start_if(self, ifcond):
+ def start_if(self, ifcond: List[str]) -> None:
assert self._start_if is None
self._start_if = (ifcond, self._body, self._preamble)
- def end_if(self):
+ def end_if(self) -> None:
assert self._start_if
self._wrap_ifcond()
self._start_if = None
- def _wrap_ifcond(self):
+ def _wrap_ifcond(self) -> None:
self._body = _wrap_ifcond(self._start_if[0],
self._start_if[1], self._body)
self._preamble = _wrap_ifcond(self._start_if[0],
self._start_if[2], self._preamble)
- def get_content(self):
+ def get_content(self) -> str:
assert self._start_if is None
return super().get_content()
class QAPIGenC(QAPIGenCCode):
-
- def __init__(self, fname, blurb, pydoc):
+ def __init__(self, fname: str, blurb: str, pydoc: str):
super().__init__(fname)
self._blurb = blurb
self._copyright = '\n * '.join(re.findall(r'^Copyright .*', pydoc,
re.MULTILINE))
- def _top(self):
+ def _top(self) -> str:
return mcgen('''
/* AUTOMATICALLY GENERATED, DO NOT MODIFY */
@@ -170,7 +168,7 @@ def _top(self):
''',
blurb=self._blurb, copyright=self._copyright)
- def _bottom(self):
+ def _bottom(self) -> str:
return mcgen('''
/* Dummy declaration to prevent empty .o file */
@@ -180,16 +178,16 @@ def _bottom(self):
class QAPIGenH(QAPIGenC):
-
- def _top(self):
+ def _top(self) -> str:
return super()._top() + guardstart(self.fname)
- def _bottom(self):
+ def _bottom(self) -> str:
return guardend(self.fname)
@contextmanager
-def ifcontext(ifcond, *args):
+def ifcontext(ifcond: List[str],
+ *args: QAPIGenCCode) -> Generator[None, None, None]:
"""A 'with' statement context manager to wrap with start_if()/end_if()
*args: any number of QAPIGenCCode
@@ -215,15 +213,17 @@ def ifcontext(ifcond, *args):
class QAPIGenDoc(QAPIGen):
-
- def _top(self):
+ def _top(self) -> str:
return (super()._top()
+ '@c AUTOMATICALLY GENERATED, DO NOT MODIFY\n\n')
class QAPISchemaMonolithicCVisitor(QAPISchemaVisitor):
-
- def __init__(self, prefix, what, blurb, pydoc):
+ def __init__(self,
+ prefix: str,
+ what: str,
+ blurb: str,
+ pydoc: str):
self._prefix = prefix
self._what = what
self._genc = QAPIGenC(self._prefix + self._what + '.c',
@@ -231,38 +231,42 @@ def __init__(self, prefix, what, blurb, pydoc):
self._genh = QAPIGenH(self._prefix + self._what + '.h',
blurb, pydoc)
- def write(self, output_dir):
+ def write(self, output_dir: str) -> None:
self._genc.write(output_dir)
self._genh.write(output_dir)
class QAPISchemaModularCVisitor(QAPISchemaVisitor):
-
- def __init__(self, prefix, what, user_blurb, builtin_blurb, pydoc):
+ def __init__(self,
+ prefix: str,
+ what: str,
+ user_blurb: str,
+ builtin_blurb: Optional[str],
+ pydoc: str):
self._prefix = prefix
self._what = what
self._user_blurb = user_blurb
self._builtin_blurb = builtin_blurb
self._pydoc = pydoc
- self._genc = None
- self._genh = None
- self._module = {}
- self._main_module = None
+ self._genc: Optional[QAPIGenC] = None
+ self._genh: Optional[QAPIGenH] = None
+ self._module: Dict[Optional[str], Tuple[QAPIGenC, QAPIGenH]] = {}
+ self._main_module: Optional[str] = None
@staticmethod
- def _is_user_module(name):
+ def _is_user_module(name: Optional[str]) -> bool:
return name is not None and not name.startswith('./')
@staticmethod
- def _is_builtin_module(name):
+ def _is_builtin_module(name: Optional[str]) -> bool:
return not name
- def _module_dirname(self, what, name):
+ def _module_dirname(self, what: str, name: Optional[str]) -> str:
if self._is_user_module(name):
return os.path.dirname(name)
return ''
- def _module_basename(self, what, name):
+ def _module_basename(self, what: str, name: Optional[str]) -> str:
ret = '' if self._is_builtin_module(name) else self._prefix
if self._is_user_module(name):
basename = os.path.basename(name)
@@ -274,27 +278,27 @@ def _module_basename(self, what, name):
ret += re.sub(r'-', '-' + name + '-', what)
return ret
- def _module_filename(self, what, name):
+ def _module_filename(self, what: str, name: Optional[str]) -> str:
return os.path.join(self._module_dirname(what, name),
self._module_basename(what, name))
- def _add_module(self, name, blurb):
+ def _add_module(self, name: Optional[str], blurb: str) -> None:
basename = self._module_filename(self._what, name)
genc = QAPIGenC(basename + '.c', blurb, self._pydoc)
genh = QAPIGenH(basename + '.h', blurb, self._pydoc)
self._module[name] = (genc, genh)
self._genc, self._genh = self._module[name]
- def _add_user_module(self, name, blurb):
+ def _add_user_module(self, name: str, blurb: str) -> None:
assert self._is_user_module(name)
if self._main_module is None:
self._main_module = name
self._add_module(name, blurb)
- def _add_system_module(self, name, blurb):
+ def _add_system_module(self, name: Optional[str], blurb: str) -> None:
self._add_module(name and './' + name, blurb)
- def write(self, output_dir, opt_builtins=False):
+ def write(self, output_dir: str, opt_builtins: bool = False) -> None:
for name in self._module:
if self._is_builtin_module(name) and not opt_builtins:
continue
@@ -302,13 +306,13 @@ def write(self, output_dir, opt_builtins=False):
genc.write(output_dir)
genh.write(output_dir)
- def _begin_system_module(self, name):
+ def _begin_system_module(self, name: None) -> None:
pass
- def _begin_user_module(self, name):
+ def _begin_user_module(self, name: str) -> None:
pass
- def visit_module(self, name):
+ def visit_module(self, name: Optional[str]) -> None:
if name is None:
if self._builtin_blurb:
self._add_system_module(None, self._builtin_blurb)
@@ -322,7 +326,7 @@ def visit_module(self, name):
self._add_user_module(name, self._user_blurb)
self._begin_user_module(name)
- def visit_include(self, name, info):
+ def visit_include(self, name: str, info: QAPISourceInfo) -> None:
relname = os.path.relpath(self._module_filename(self._what, name),
os.path.dirname(self._genh.fname))
self._genh.preamble_add(mcgen('''
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 25/38] qapi/gen.py: add type hint annotations
2020-09-22 21:00 ` [PATCH v2 25/38] qapi/gen.py: add type hint annotations John Snow
@ 2020-09-23 15:18 ` Eduardo Habkost
2020-09-23 23:51 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 15:18 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:48PM -0400, John Snow wrote:
> Annotations do not change runtime behavior.
> This commit *only* adds annotations.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 25/38] qapi/gen.py: add type hint annotations
2020-09-22 21:00 ` [PATCH v2 25/38] qapi/gen.py: add type hint annotations John Snow
2020-09-23 15:18 ` Eduardo Habkost
@ 2020-09-23 23:51 ` Cleber Rosa
2020-09-24 0:29 ` John Snow
1 sibling, 1 reply; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 23:51 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 4843 bytes --]
On Tue, Sep 22, 2020 at 05:00:48PM -0400, John Snow wrote:
> Annotations do not change runtime behavior.
> This commit *only* adds annotations.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/gen.py | 102 +++++++++++++++++++++++---------------------
> 1 file changed, 53 insertions(+), 49 deletions(-)
>
> diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
> index cb2b2655c3..df8cf8271c 100644
> --- a/scripts/qapi/gen.py
> +++ b/scripts/qapi/gen.py
> @@ -17,7 +17,7 @@
> import errno
> import os
> import re
> -from typing import Optional
> +from typing import Dict, Generator, List, Optional, Tuple
>
> from .common import (
> c_fname,
> @@ -32,31 +32,31 @@
> QAPISchemaObjectType,
> QAPISchemaVisitor,
> )
> +from .source import QAPISourceInfo
>
>
> class QAPIGen:
> -
> - def __init__(self, fname):
> + def __init__(self, fname: Optional[str]):
> self.fname = fname
> self._preamble = ''
> self._body = ''
>
> - def preamble_add(self, text):
> + def preamble_add(self, text: str) -> None:
> self._preamble += text
>
> - def add(self, text):
> + def add(self, text: str) -> None:
> self._body += text
>
> - def get_content(self):
> + def get_content(self) -> str:
> return self._top() + self._preamble + self._body + self._bottom()
>
> - def _top(self):
> + def _top(self) -> str:
> return ''
>
> - def _bottom(self):
> + def _bottom(self) -> str:
> return ''
>
> - def write(self, output_dir):
> + def write(self, output_dir: str) -> None:
> # Include paths starting with ../ are used to reuse modules of the main
> # schema in specialised schemas. Don't overwrite the files that are
> # already generated for the main schema.
> @@ -81,7 +81,7 @@ def write(self, output_dir):
> f.close()
>
>
> -def _wrap_ifcond(ifcond, before, after):
> +def _wrap_ifcond(ifcond: List[str], before: str, after: str) -> str:
> if before == after:
> return after # suppress empty #if ... #endif
>
> @@ -121,40 +121,38 @@ def build_params(arg_type: Optional[QAPISchemaObjectType],
>
>
> class QAPIGenCCode(QAPIGen):
> -
> - def __init__(self, fname):
> + def __init__(self, fname: Optional[str]):
> super().__init__(fname)
> - self._start_if = None
> + self._start_if: Optional[Tuple[List[str], str, str]] = None
>
> - def start_if(self, ifcond):
> + def start_if(self, ifcond: List[str]) -> None:
> assert self._start_if is None
> self._start_if = (ifcond, self._body, self._preamble)
>
> - def end_if(self):
> + def end_if(self) -> None:
> assert self._start_if
> self._wrap_ifcond()
> self._start_if = None
>
> - def _wrap_ifcond(self):
> + def _wrap_ifcond(self) -> None:
> self._body = _wrap_ifcond(self._start_if[0],
> self._start_if[1], self._body)
> self._preamble = _wrap_ifcond(self._start_if[0],
> self._start_if[2], self._preamble)
>
> - def get_content(self):
> + def get_content(self) -> str:
> assert self._start_if is None
> return super().get_content()
>
>
> class QAPIGenC(QAPIGenCCode):
> -
> - def __init__(self, fname, blurb, pydoc):
> + def __init__(self, fname: str, blurb: str, pydoc: str):
> super().__init__(fname)
> self._blurb = blurb
> self._copyright = '\n * '.join(re.findall(r'^Copyright .*', pydoc,
> re.MULTILINE))
>
> - def _top(self):
> + def _top(self) -> str:
> return mcgen('''
> /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
>
> @@ -170,7 +168,7 @@ def _top(self):
> ''',
> blurb=self._blurb, copyright=self._copyright)
>
> - def _bottom(self):
> + def _bottom(self) -> str:
> return mcgen('''
>
> /* Dummy declaration to prevent empty .o file */
> @@ -180,16 +178,16 @@ def _bottom(self):
>
>
> class QAPIGenH(QAPIGenC):
> -
> - def _top(self):
> + def _top(self) -> str:
> return super()._top() + guardstart(self.fname)
>
> - def _bottom(self):
> + def _bottom(self) -> str:
> return guardend(self.fname)
>
>
> @contextmanager
> -def ifcontext(ifcond, *args):
> +def ifcontext(ifcond: List[str],
> + *args: QAPIGenCCode) -> Generator[None, None, None]:
IIUC, this could simply be "Iterator[None]" instead of
"Generator[None, None, None]".
Anyway,
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 25/38] qapi/gen.py: add type hint annotations
2020-09-23 23:51 ` Cleber Rosa
@ 2020-09-24 0:29 ` John Snow
2020-09-24 1:29 ` Eduardo Habkost
0 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-24 0:29 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
On 9/23/20 7:51 PM, Cleber Rosa wrote:
> On Tue, Sep 22, 2020 at 05:00:48PM -0400, John Snow wrote:
>> Annotations do not change runtime behavior.
>> This commit *only* adds annotations.
>>
>> Signed-off-by: John Snow <jsnow@redhat.com>
>> ---
>> scripts/qapi/gen.py | 102 +++++++++++++++++++++++---------------------
>> 1 file changed, 53 insertions(+), 49 deletions(-)
>>
>> diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
>> index cb2b2655c3..df8cf8271c 100644
>> --- a/scripts/qapi/gen.py
>> +++ b/scripts/qapi/gen.py
>> @@ -17,7 +17,7 @@
>> import errno
>> import os
>> import re
>> -from typing import Optional
>> +from typing import Dict, Generator, List, Optional, Tuple
>>
>> from .common import (
>> c_fname,
>> @@ -32,31 +32,31 @@
>> QAPISchemaObjectType,
>> QAPISchemaVisitor,
>> )
>> +from .source import QAPISourceInfo
>>
>>
>> class QAPIGen:
>> -
>> - def __init__(self, fname):
>> + def __init__(self, fname: Optional[str]):
>> self.fname = fname
>> self._preamble = ''
>> self._body = ''
>>
>> - def preamble_add(self, text):
>> + def preamble_add(self, text: str) -> None:
>> self._preamble += text
>>
>> - def add(self, text):
>> + def add(self, text: str) -> None:
>> self._body += text
>>
>> - def get_content(self):
>> + def get_content(self) -> str:
>> return self._top() + self._preamble + self._body + self._bottom()
>>
>> - def _top(self):
>> + def _top(self) -> str:
>> return ''
>>
>> - def _bottom(self):
>> + def _bottom(self) -> str:
>> return ''
>>
>> - def write(self, output_dir):
>> + def write(self, output_dir: str) -> None:
>> # Include paths starting with ../ are used to reuse modules of the main
>> # schema in specialised schemas. Don't overwrite the files that are
>> # already generated for the main schema.
>> @@ -81,7 +81,7 @@ def write(self, output_dir):
>> f.close()
>>
>>
>> -def _wrap_ifcond(ifcond, before, after):
>> +def _wrap_ifcond(ifcond: List[str], before: str, after: str) -> str:
>> if before == after:
>> return after # suppress empty #if ... #endif
>>
>> @@ -121,40 +121,38 @@ def build_params(arg_type: Optional[QAPISchemaObjectType],
>>
>>
>> class QAPIGenCCode(QAPIGen):
>> -
>> - def __init__(self, fname):
>> + def __init__(self, fname: Optional[str]):
>> super().__init__(fname)
>> - self._start_if = None
>> + self._start_if: Optional[Tuple[List[str], str, str]] = None
>>
>> - def start_if(self, ifcond):
>> + def start_if(self, ifcond: List[str]) -> None:
>> assert self._start_if is None
>> self._start_if = (ifcond, self._body, self._preamble)
>>
>> - def end_if(self):
>> + def end_if(self) -> None:
>> assert self._start_if
>> self._wrap_ifcond()
>> self._start_if = None
>>
>> - def _wrap_ifcond(self):
>> + def _wrap_ifcond(self) -> None:
>> self._body = _wrap_ifcond(self._start_if[0],
>> self._start_if[1], self._body)
>> self._preamble = _wrap_ifcond(self._start_if[0],
>> self._start_if[2], self._preamble)
>>
>> - def get_content(self):
>> + def get_content(self) -> str:
>> assert self._start_if is None
>> return super().get_content()
>>
>>
>> class QAPIGenC(QAPIGenCCode):
>> -
>> - def __init__(self, fname, blurb, pydoc):
>> + def __init__(self, fname: str, blurb: str, pydoc: str):
>> super().__init__(fname)
>> self._blurb = blurb
>> self._copyright = '\n * '.join(re.findall(r'^Copyright .*', pydoc,
>> re.MULTILINE))
>>
>> - def _top(self):
>> + def _top(self) -> str:
>> return mcgen('''
>> /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
>>
>> @@ -170,7 +168,7 @@ def _top(self):
>> ''',
>> blurb=self._blurb, copyright=self._copyright)
>>
>> - def _bottom(self):
>> + def _bottom(self) -> str:
>> return mcgen('''
>>
>> /* Dummy declaration to prevent empty .o file */
>> @@ -180,16 +178,16 @@ def _bottom(self):
>>
>>
>> class QAPIGenH(QAPIGenC):
>> -
>> - def _top(self):
>> + def _top(self) -> str:
>> return super()._top() + guardstart(self.fname)
>>
>> - def _bottom(self):
>> + def _bottom(self) -> str:
>> return guardend(self.fname)
>>
>>
>> @contextmanager
>> -def ifcontext(ifcond, *args):
>> +def ifcontext(ifcond: List[str],
>> + *args: QAPIGenCCode) -> Generator[None, None, None]:
>
> IIUC, this could simply be "Iterator[None]" instead of
> "Generator[None, None, None]".
>
> Anyway,
>
> Reviewed-by: Cleber Rosa <crosa@redhat.com>
>
Oh, you're right! Let's do that instead.
Reference:
https://mypy.readthedocs.io/en/stable/kinds_of_types.html#generators
Eduardo, I am making this change and keeping your R-B.
--js
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 25/38] qapi/gen.py: add type hint annotations
2020-09-24 0:29 ` John Snow
@ 2020-09-24 1:29 ` Eduardo Habkost
0 siblings, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-24 1:29 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Markus Armbruster, qemu-devel, Michael Roth,
Cleber Rosa, Alex Bennée
On Wed, Sep 23, 2020 at 08:29:17PM -0400, John Snow wrote:
> On 9/23/20 7:51 PM, Cleber Rosa wrote:
> > On Tue, Sep 22, 2020 at 05:00:48PM -0400, John Snow wrote:
> > > Annotations do not change runtime behavior.
> > > This commit *only* adds annotations.
> > >
> > > Signed-off-by: John Snow <jsnow@redhat.com>
> > > ---
> > > scripts/qapi/gen.py | 102 +++++++++++++++++++++++---------------------
> > > 1 file changed, 53 insertions(+), 49 deletions(-)
> > >
> > > diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
> > > index cb2b2655c3..df8cf8271c 100644
> > > --- a/scripts/qapi/gen.py
> > > +++ b/scripts/qapi/gen.py
> > > @@ -17,7 +17,7 @@
> > > import errno
> > > import os
> > > import re
> > > -from typing import Optional
> > > +from typing import Dict, Generator, List, Optional, Tuple
> > > from .common import (
> > > c_fname,
> > > @@ -32,31 +32,31 @@
> > > QAPISchemaObjectType,
> > > QAPISchemaVisitor,
> > > )
> > > +from .source import QAPISourceInfo
> > > class QAPIGen:
> > > -
> > > - def __init__(self, fname):
> > > + def __init__(self, fname: Optional[str]):
> > > self.fname = fname
> > > self._preamble = ''
> > > self._body = ''
> > > - def preamble_add(self, text):
> > > + def preamble_add(self, text: str) -> None:
> > > self._preamble += text
> > > - def add(self, text):
> > > + def add(self, text: str) -> None:
> > > self._body += text
> > > - def get_content(self):
> > > + def get_content(self) -> str:
> > > return self._top() + self._preamble + self._body + self._bottom()
> > > - def _top(self):
> > > + def _top(self) -> str:
> > > return ''
> > > - def _bottom(self):
> > > + def _bottom(self) -> str:
> > > return ''
> > > - def write(self, output_dir):
> > > + def write(self, output_dir: str) -> None:
> > > # Include paths starting with ../ are used to reuse modules of the main
> > > # schema in specialised schemas. Don't overwrite the files that are
> > > # already generated for the main schema.
> > > @@ -81,7 +81,7 @@ def write(self, output_dir):
> > > f.close()
> > > -def _wrap_ifcond(ifcond, before, after):
> > > +def _wrap_ifcond(ifcond: List[str], before: str, after: str) -> str:
> > > if before == after:
> > > return after # suppress empty #if ... #endif
> > > @@ -121,40 +121,38 @@ def build_params(arg_type: Optional[QAPISchemaObjectType],
> > > class QAPIGenCCode(QAPIGen):
> > > -
> > > - def __init__(self, fname):
> > > + def __init__(self, fname: Optional[str]):
> > > super().__init__(fname)
> > > - self._start_if = None
> > > + self._start_if: Optional[Tuple[List[str], str, str]] = None
> > > - def start_if(self, ifcond):
> > > + def start_if(self, ifcond: List[str]) -> None:
> > > assert self._start_if is None
> > > self._start_if = (ifcond, self._body, self._preamble)
> > > - def end_if(self):
> > > + def end_if(self) -> None:
> > > assert self._start_if
> > > self._wrap_ifcond()
> > > self._start_if = None
> > > - def _wrap_ifcond(self):
> > > + def _wrap_ifcond(self) -> None:
> > > self._body = _wrap_ifcond(self._start_if[0],
> > > self._start_if[1], self._body)
> > > self._preamble = _wrap_ifcond(self._start_if[0],
> > > self._start_if[2], self._preamble)
> > > - def get_content(self):
> > > + def get_content(self) -> str:
> > > assert self._start_if is None
> > > return super().get_content()
> > > class QAPIGenC(QAPIGenCCode):
> > > -
> > > - def __init__(self, fname, blurb, pydoc):
> > > + def __init__(self, fname: str, blurb: str, pydoc: str):
> > > super().__init__(fname)
> > > self._blurb = blurb
> > > self._copyright = '\n * '.join(re.findall(r'^Copyright .*', pydoc,
> > > re.MULTILINE))
> > > - def _top(self):
> > > + def _top(self) -> str:
> > > return mcgen('''
> > > /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
> > > @@ -170,7 +168,7 @@ def _top(self):
> > > ''',
> > > blurb=self._blurb, copyright=self._copyright)
> > > - def _bottom(self):
> > > + def _bottom(self) -> str:
> > > return mcgen('''
> > > /* Dummy declaration to prevent empty .o file */
> > > @@ -180,16 +178,16 @@ def _bottom(self):
> > > class QAPIGenH(QAPIGenC):
> > > -
> > > - def _top(self):
> > > + def _top(self) -> str:
> > > return super()._top() + guardstart(self.fname)
> > > - def _bottom(self):
> > > + def _bottom(self) -> str:
> > > return guardend(self.fname)
> > > @contextmanager
> > > -def ifcontext(ifcond, *args):
> > > +def ifcontext(ifcond: List[str],
> > > + *args: QAPIGenCCode) -> Generator[None, None, None]:
> >
> > IIUC, this could simply be "Iterator[None]" instead of
> > "Generator[None, None, None]".
> >
> > Anyway,
> >
> > Reviewed-by: Cleber Rosa <crosa@redhat.com>
> >
>
> Oh, you're right! Let's do that instead.
>
> Reference:
> https://mypy.readthedocs.io/en/stable/kinds_of_types.html#generators
>
> Eduardo, I am making this change and keeping your R-B.
Agreed.
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 26/38] qapi/gen.py: Enable checking with mypy
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (24 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 25/38] qapi/gen.py: add type hint annotations John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 15:18 ` Eduardo Habkost
2020-09-23 23:59 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 27/38] qapi/gen.py: Remove unused parameter John Snow
` (12 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/mypy.ini | 5 -----
1 file changed, 5 deletions(-)
diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini
index 43c8bd1973..dbfeda748c 100644
--- a/scripts/qapi/mypy.ini
+++ b/scripts/qapi/mypy.ini
@@ -19,11 +19,6 @@ disallow_untyped_defs = False
disallow_incomplete_defs = False
check_untyped_defs = False
-[mypy-qapi.gen]
-disallow_untyped_defs = False
-disallow_incomplete_defs = False
-check_untyped_defs = False
-
[mypy-qapi.introspect]
disallow_untyped_defs = False
disallow_incomplete_defs = False
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 26/38] qapi/gen.py: Enable checking with mypy
2020-09-22 21:00 ` [PATCH v2 26/38] qapi/gen.py: Enable checking with mypy John Snow
@ 2020-09-23 15:18 ` Eduardo Habkost
2020-09-23 23:59 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 15:18 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:49PM -0400, John Snow wrote:
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 26/38] qapi/gen.py: Enable checking with mypy
2020-09-22 21:00 ` [PATCH v2 26/38] qapi/gen.py: Enable checking with mypy John Snow
2020-09-23 15:18 ` Eduardo Habkost
@ 2020-09-23 23:59 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-23 23:59 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 939 bytes --]
On Tue, Sep 22, 2020 at 05:00:49PM -0400, John Snow wrote:
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/mypy.ini | 5 -----
> 1 file changed, 5 deletions(-)
>
> diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini
> index 43c8bd1973..dbfeda748c 100644
> --- a/scripts/qapi/mypy.ini
> +++ b/scripts/qapi/mypy.ini
> @@ -19,11 +19,6 @@ disallow_untyped_defs = False
> disallow_incomplete_defs = False
> check_untyped_defs = False
>
> -[mypy-qapi.gen]
> -disallow_untyped_defs = False
> -disallow_incomplete_defs = False
> -check_untyped_defs = False
> -
> [mypy-qapi.introspect]
> disallow_untyped_defs = False
> disallow_incomplete_defs = False
> --
> 2.26.2
>
Like I mentioned before, I'd squash this together with the actual
changes that enable this increased coverage.
Anyway,
Reviewed-by: Cleber Rosa <crosa@redhat.com>
Tested-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 27/38] qapi/gen.py: Remove unused parameter
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (25 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 26/38] qapi/gen.py: Enable checking with mypy John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 15:19 ` Eduardo Habkost
2020-09-24 0:00 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 28/38] qapi/gen.py: update write() to be more idiomatic John Snow
` (11 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
module_basename doesn't use the 'what' argument, so remove it.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/gen.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
index df8cf8271c..ba32f776e6 100644
--- a/scripts/qapi/gen.py
+++ b/scripts/qapi/gen.py
@@ -261,7 +261,7 @@ def _is_user_module(name: Optional[str]) -> bool:
def _is_builtin_module(name: Optional[str]) -> bool:
return not name
- def _module_dirname(self, what: str, name: Optional[str]) -> str:
+ def _module_dirname(self, name: Optional[str]) -> str:
if self._is_user_module(name):
return os.path.dirname(name)
return ''
@@ -279,7 +279,7 @@ def _module_basename(self, what: str, name: Optional[str]) -> str:
return ret
def _module_filename(self, what: str, name: Optional[str]) -> str:
- return os.path.join(self._module_dirname(what, name),
+ return os.path.join(self._module_dirname(name),
self._module_basename(what, name))
def _add_module(self, name: Optional[str], blurb: str) -> None:
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 27/38] qapi/gen.py: Remove unused parameter
2020-09-22 21:00 ` [PATCH v2 27/38] qapi/gen.py: Remove unused parameter John Snow
@ 2020-09-23 15:19 ` Eduardo Habkost
2020-09-24 0:00 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 15:19 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:50PM -0400, John Snow wrote:
> module_basename doesn't use the 'what' argument, so remove it.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 27/38] qapi/gen.py: Remove unused parameter
2020-09-22 21:00 ` [PATCH v2 27/38] qapi/gen.py: Remove unused parameter John Snow
2020-09-23 15:19 ` Eduardo Habkost
@ 2020-09-24 0:00 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-24 0:00 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 224 bytes --]
On Tue, Sep 22, 2020 at 05:00:50PM -0400, John Snow wrote:
> module_basename doesn't use the 'what' argument, so remove it.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 28/38] qapi/gen.py: update write() to be more idiomatic
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (26 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 27/38] qapi/gen.py: Remove unused parameter John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 15:26 ` Eduardo Habkost
2020-09-24 16:01 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 29/38] qapi/gen.py: delint with pylint John Snow
` (10 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Make the file handling here just a tiny bit more idiomatic.
(I realize this is heavily subjective.)
Use exist_ok=True for os.makedirs and remove the exception,
use fdopen() to wrap the file descriptor in a File-like object,
and use a context manager for managing the file pointer.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/gen.py | 24 ++++++++++--------------
1 file changed, 10 insertions(+), 14 deletions(-)
diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
index ba32f776e6..cf340e66d4 100644
--- a/scripts/qapi/gen.py
+++ b/scripts/qapi/gen.py
@@ -14,7 +14,6 @@
# See the COPYING file in the top-level directory.
from contextlib import contextmanager
-import errno
import os
import re
from typing import Dict, Generator, List, Optional, Tuple
@@ -64,21 +63,18 @@ def write(self, output_dir: str) -> None:
return
pathname = os.path.join(output_dir, self.fname)
odir = os.path.dirname(pathname)
+
if odir:
- try:
- os.makedirs(odir)
- except os.error as e:
- if e.errno != errno.EEXIST:
- raise
+ os.makedirs(odir, exist_ok=True)
+
fd = os.open(pathname, os.O_RDWR | os.O_CREAT, 0o666)
- f = open(fd, 'r+', encoding='utf-8')
- text = self.get_content()
- oldtext = f.read(len(text) + 1)
- if text != oldtext:
- f.seek(0)
- f.truncate(0)
- f.write(text)
- f.close()
+ with os.fdopen(fd, 'r+', encoding='utf-8') as fp:
+ text = self.get_content()
+ oldtext = fp.read(len(text) + 1)
+ if text != oldtext:
+ fp.seek(0)
+ fp.truncate(0)
+ fp.write(text)
def _wrap_ifcond(ifcond: List[str], before: str, after: str) -> str:
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 28/38] qapi/gen.py: update write() to be more idiomatic
2020-09-22 21:00 ` [PATCH v2 28/38] qapi/gen.py: update write() to be more idiomatic John Snow
@ 2020-09-23 15:26 ` Eduardo Habkost
2020-09-23 18:37 ` John Snow
2020-09-24 16:01 ` Cleber Rosa
1 sibling, 1 reply; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 15:26 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:51PM -0400, John Snow wrote:
> Make the file handling here just a tiny bit more idiomatic.
> (I realize this is heavily subjective.)
>
> Use exist_ok=True for os.makedirs and remove the exception,
> use fdopen() to wrap the file descriptor in a File-like object,
> and use a context manager for managing the file pointer.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
I really miss a comment below explaining why we use
open(os.open(pathname, ...), ...) instead of open(pathname, ...).
> ---
> scripts/qapi/gen.py | 24 ++++++++++--------------
> 1 file changed, 10 insertions(+), 14 deletions(-)
>
> diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
> index ba32f776e6..cf340e66d4 100644
> --- a/scripts/qapi/gen.py
> +++ b/scripts/qapi/gen.py
> @@ -14,7 +14,6 @@
> # See the COPYING file in the top-level directory.
>
> from contextlib import contextmanager
> -import errno
> import os
> import re
> from typing import Dict, Generator, List, Optional, Tuple
> @@ -64,21 +63,18 @@ def write(self, output_dir: str) -> None:
> return
> pathname = os.path.join(output_dir, self.fname)
> odir = os.path.dirname(pathname)
> +
> if odir:
> - try:
> - os.makedirs(odir)
> - except os.error as e:
> - if e.errno != errno.EEXIST:
> - raise
> + os.makedirs(odir, exist_ok=True)
> +
> fd = os.open(pathname, os.O_RDWR | os.O_CREAT, 0o666)
> - f = open(fd, 'r+', encoding='utf-8')
> - text = self.get_content()
> - oldtext = f.read(len(text) + 1)
> - if text != oldtext:
> - f.seek(0)
> - f.truncate(0)
> - f.write(text)
> - f.close()
> + with os.fdopen(fd, 'r+', encoding='utf-8') as fp:
> + text = self.get_content()
> + oldtext = fp.read(len(text) + 1)
> + if text != oldtext:
> + fp.seek(0)
> + fp.truncate(0)
> + fp.write(text)
>
>
> def _wrap_ifcond(ifcond: List[str], before: str, after: str) -> str:
> --
> 2.26.2
>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 28/38] qapi/gen.py: update write() to be more idiomatic
2020-09-23 15:26 ` Eduardo Habkost
@ 2020-09-23 18:37 ` John Snow
2020-09-24 15:59 ` Cleber Rosa
0 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-23 18:37 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On 9/23/20 11:26 AM, Eduardo Habkost wrote:
> On Tue, Sep 22, 2020 at 05:00:51PM -0400, John Snow wrote:
>> Make the file handling here just a tiny bit more idiomatic.
>> (I realize this is heavily subjective.)
>>
>> Use exist_ok=True for os.makedirs and remove the exception,
>> use fdopen() to wrap the file descriptor in a File-like object,
>> and use a context manager for managing the file pointer.
>>
>> Signed-off-by: John Snow <jsnow@redhat.com>
>
> Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
>
> I really miss a comment below explaining why we use
> open(os.open(pathname, ...), ...) instead of open(pathname, ...).
>
Not known to me. It was introduced in 907b846653 as part of an effort to
reduce rebuild times. Maybe this avoids a modification time change if
the file already exists?
Markus?
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 28/38] qapi/gen.py: update write() to be more idiomatic
2020-09-23 18:37 ` John Snow
@ 2020-09-24 15:59 ` Cleber Rosa
2020-09-25 13:15 ` Markus Armbruster
0 siblings, 1 reply; 190+ messages in thread
From: Cleber Rosa @ 2020-09-24 15:59 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, Markus Armbruster, qemu-devel,
Michael Roth, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 1194 bytes --]
On Wed, Sep 23, 2020 at 02:37:27PM -0400, John Snow wrote:
> On 9/23/20 11:26 AM, Eduardo Habkost wrote:
> > On Tue, Sep 22, 2020 at 05:00:51PM -0400, John Snow wrote:
> > > Make the file handling here just a tiny bit more idiomatic.
> > > (I realize this is heavily subjective.)
> > >
> > > Use exist_ok=True for os.makedirs and remove the exception,
> > > use fdopen() to wrap the file descriptor in a File-like object,
> > > and use a context manager for managing the file pointer.
> > >
> > > Signed-off-by: John Snow <jsnow@redhat.com>
> >
> > Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
> >
> > I really miss a comment below explaining why we use
> > open(os.open(pathname, ...), ...) instead of open(pathname, ...).
> >
>
> Not known to me. It was introduced in 907b846653 as part of an effort to
> reduce rebuild times. Maybe this avoids a modification time change if the
> file already exists?
>
> Markus?
AFACIT the change on 907b846653 is effective because of the "is new
text different from old text?" conditional. I can not see how the
separate/duplicate open/fdopen would contribute to that.
But, let's hear from Markus.
- Cleber.
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 28/38] qapi/gen.py: update write() to be more idiomatic
2020-09-24 15:59 ` Cleber Rosa
@ 2020-09-25 13:15 ` Markus Armbruster
2020-09-25 13:24 ` Daniel P. Berrangé
2020-09-25 13:26 ` Eduardo Habkost
0 siblings, 2 replies; 190+ messages in thread
From: Markus Armbruster @ 2020-09-25 13:15 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, John Snow, Michael Roth,
qemu-devel, Alex Bennée
Cleber Rosa <crosa@redhat.com> writes:
> On Wed, Sep 23, 2020 at 02:37:27PM -0400, John Snow wrote:
>> On 9/23/20 11:26 AM, Eduardo Habkost wrote:
>> > On Tue, Sep 22, 2020 at 05:00:51PM -0400, John Snow wrote:
>> > > Make the file handling here just a tiny bit more idiomatic.
>> > > (I realize this is heavily subjective.)
>> > >
>> > > Use exist_ok=True for os.makedirs and remove the exception,
>> > > use fdopen() to wrap the file descriptor in a File-like object,
>> > > and use a context manager for managing the file pointer.
>> > >
>> > > Signed-off-by: John Snow <jsnow@redhat.com>
>> >
>> > Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
>> >
>> > I really miss a comment below explaining why we use
>> > open(os.open(pathname, ...), ...) instead of open(pathname, ...).
This code:
fd = os.open(pathname, os.O_RDWR | os.O_CREAT, 0o666)
f = open(fd, 'r+', encoding='utf-8')
>> Not known to me. It was introduced in 907b846653 as part of an effort to
>> reduce rebuild times. Maybe this avoids a modification time change if the
>> file already exists?
>>
>> Markus?
>
> AFACIT the change on 907b846653 is effective because of the "is new
> text different from old text?" conditional. I can not see how the
> separate/duplicate open/fdopen would contribute to that.
>
> But, let's hear from Markus.
This was my best attempt to open the file read/write, creating it if it
doesn't exist.
Plain
f = open(pathname, "r+", encoding='utf-8')
fails instead of creates, and
f = open(pathname, "w+", encoding='utf-8')
truncates.
If you know a better way, tell me!
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 28/38] qapi/gen.py: update write() to be more idiomatic
2020-09-25 13:15 ` Markus Armbruster
@ 2020-09-25 13:24 ` Daniel P. Berrangé
2020-09-25 13:34 ` Eric Blake
2020-09-25 13:52 ` Markus Armbruster
2020-09-25 13:26 ` Eduardo Habkost
1 sibling, 2 replies; 190+ messages in thread
From: Daniel P. Berrangé @ 2020-09-25 13:24 UTC (permalink / raw)
To: Markus Armbruster
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
qemu-devel, Cleber Rosa, John Snow
On Fri, Sep 25, 2020 at 03:15:57PM +0200, Markus Armbruster wrote:
> Cleber Rosa <crosa@redhat.com> writes:
>
> > On Wed, Sep 23, 2020 at 02:37:27PM -0400, John Snow wrote:
> >> On 9/23/20 11:26 AM, Eduardo Habkost wrote:
> >> > On Tue, Sep 22, 2020 at 05:00:51PM -0400, John Snow wrote:
> >> > > Make the file handling here just a tiny bit more idiomatic.
> >> > > (I realize this is heavily subjective.)
> >> > >
> >> > > Use exist_ok=True for os.makedirs and remove the exception,
> >> > > use fdopen() to wrap the file descriptor in a File-like object,
> >> > > and use a context manager for managing the file pointer.
> >> > >
> >> > > Signed-off-by: John Snow <jsnow@redhat.com>
> >> >
> >> > Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
> >> >
> >> > I really miss a comment below explaining why we use
> >> > open(os.open(pathname, ...), ...) instead of open(pathname, ...).
>
> This code:
>
> fd = os.open(pathname, os.O_RDWR | os.O_CREAT, 0o666)
> f = open(fd, 'r+', encoding='utf-8')
>
> >> Not known to me. It was introduced in 907b846653 as part of an effort to
> >> reduce rebuild times. Maybe this avoids a modification time change if the
> >> file already exists?
> >>
> >> Markus?
> >
> > AFACIT the change on 907b846653 is effective because of the "is new
> > text different from old text?" conditional. I can not see how the
> > separate/duplicate open/fdopen would contribute to that.
> >
> > But, let's hear from Markus.
>
> This was my best attempt to open the file read/write, creating it if it
> doesn't exist.
>
> Plain
>
> f = open(pathname, "r+", encoding='utf-8')
>
> fails instead of creates, and
>
> f = open(pathname, "w+", encoding='utf-8')
>
> truncates.
>
> If you know a better way, tell me!
IIUC, you need "a+" as the mode, rather than "w+"
Regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 28/38] qapi/gen.py: update write() to be more idiomatic
2020-09-25 13:24 ` Daniel P. Berrangé
@ 2020-09-25 13:34 ` Eric Blake
2020-09-25 13:52 ` Markus Armbruster
1 sibling, 0 replies; 190+ messages in thread
From: Eric Blake @ 2020-09-25 13:34 UTC (permalink / raw)
To: Daniel P. Berrangé, Markus Armbruster
Cc: Peter Maydell, Eduardo Habkost, John Snow, qemu-devel,
Michael Roth, Cleber Rosa, Alex Bennée
On 9/25/20 8:24 AM, Daniel P. Berrangé wrote:
>> This code:
>>
>> fd = os.open(pathname, os.O_RDWR | os.O_CREAT, 0o666)
>> f = open(fd, 'r+', encoding='utf-8')
>>
>> This was my best attempt to open the file read/write, creating it if it
>> doesn't exist.
>>
>> Plain
>>
>> f = open(pathname, "r+", encoding='utf-8')
>>
>> fails instead of creates, and
Checking what POSIX says for fopen():
https://pubs.opengroup.org/onlinepubs/9699919799/functions/fopen.html
Yep, "r+" does not use O_CREAT.
>>
>> f = open(pathname, "w+", encoding='utf-8')
>>
>> truncates.
Yep, "w+" uses O_TRUNC.
>>
>> If you know a better way, tell me!
>
> IIUC, you need "a+" as the mode, rather than "w+"
That uses O_APPEND. Which is fine if you are only appending and not
touching existing contents, but not what you want if you are doing
random access.
Yeah, the fopen() interface is rather puny, in that it does not express
as many modes as open() supports.
--
Eric Blake, Principal Software Engineer
Red Hat, Inc. +1-919-301-3226
Virtualization: qemu.org | libvirt.org
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 28/38] qapi/gen.py: update write() to be more idiomatic
2020-09-25 13:24 ` Daniel P. Berrangé
2020-09-25 13:34 ` Eric Blake
@ 2020-09-25 13:52 ` Markus Armbruster
2020-09-25 15:47 ` Eric Blake
1 sibling, 1 reply; 190+ messages in thread
From: Markus Armbruster @ 2020-09-25 13:52 UTC (permalink / raw)
To: Daniel P. Berrangé
Cc: Peter Maydell, Eduardo Habkost, John Snow, qemu-devel,
Michael Roth, Cleber Rosa, Alex Bennée
Daniel P. Berrangé <berrange@redhat.com> writes:
> On Fri, Sep 25, 2020 at 03:15:57PM +0200, Markus Armbruster wrote:
>> Cleber Rosa <crosa@redhat.com> writes:
>>
>> > On Wed, Sep 23, 2020 at 02:37:27PM -0400, John Snow wrote:
>> >> On 9/23/20 11:26 AM, Eduardo Habkost wrote:
>> >> > On Tue, Sep 22, 2020 at 05:00:51PM -0400, John Snow wrote:
>> >> > > Make the file handling here just a tiny bit more idiomatic.
>> >> > > (I realize this is heavily subjective.)
>> >> > >
>> >> > > Use exist_ok=True for os.makedirs and remove the exception,
>> >> > > use fdopen() to wrap the file descriptor in a File-like object,
>> >> > > and use a context manager for managing the file pointer.
>> >> > >
>> >> > > Signed-off-by: John Snow <jsnow@redhat.com>
>> >> >
>> >> > Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
>> >> >
>> >> > I really miss a comment below explaining why we use
>> >> > open(os.open(pathname, ...), ...) instead of open(pathname, ...).
>>
>> This code:
>>
>> fd = os.open(pathname, os.O_RDWR | os.O_CREAT, 0o666)
>> f = open(fd, 'r+', encoding='utf-8')
>>
>> >> Not known to me. It was introduced in 907b846653 as part of an effort to
>> >> reduce rebuild times. Maybe this avoids a modification time change if the
>> >> file already exists?
>> >>
>> >> Markus?
>> >
>> > AFACIT the change on 907b846653 is effective because of the "is new
>> > text different from old text?" conditional. I can not see how the
>> > separate/duplicate open/fdopen would contribute to that.
>> >
>> > But, let's hear from Markus.
>>
>> This was my best attempt to open the file read/write, creating it if it
>> doesn't exist.
>>
>> Plain
>>
>> f = open(pathname, "r+", encoding='utf-8')
>>
>> fails instead of creates, and
>>
>> f = open(pathname, "w+", encoding='utf-8')
>>
>> truncates.
>>
>> If you know a better way, tell me!
>
> IIUC, you need "a+" as the mode, rather than "w+"
Sure this lets me do
f.seek(0)
f.truncate(0)
f.write(text)
to overwrite the old contents on all systems?
Documentation cautions:
[...] 'a' for appending (which on some Unix systems, means that all
writes append to the end of the file regardless of the current seek
position).
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 28/38] qapi/gen.py: update write() to be more idiomatic
2020-09-25 13:52 ` Markus Armbruster
@ 2020-09-25 15:47 ` Eric Blake
2020-09-28 12:09 ` Markus Armbruster
0 siblings, 1 reply; 190+ messages in thread
From: Eric Blake @ 2020-09-25 15:47 UTC (permalink / raw)
To: Markus Armbruster, Daniel P. Berrangé
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
qemu-devel, Cleber Rosa, John Snow
On 9/25/20 8:52 AM, Markus Armbruster wrote:
>>> This was my best attempt to open the file read/write, creating it if it
>>> doesn't exist.
>>>
>>> Plain
>>>
>>> f = open(pathname, "r+", encoding='utf-8')
>>>
>>> fails instead of creates, and
>>>
>>> f = open(pathname, "w+", encoding='utf-8')
>>>
>>> truncates.
>>>
>>> If you know a better way, tell me!
>>
>> IIUC, you need "a+" as the mode, rather than "w+"
>
> Sure this lets me do
>
> f.seek(0)
> f.truncate(0)
> f.write(text)
>
> to overwrite the old contents on all systems?
As long as you do a single pass over the output (you issue a stream of
f.write() after the truncate, but never a seek), then this will work.
>
> Documentation cautions:
>
> [...] 'a' for appending (which on some Unix systems, means that all
> writes append to the end of the file regardless of the current seek
> position).
Yes, that means that random access is impossible on such a stream. But
not all file creation patterns require random access.
--
Eric Blake, Principal Software Engineer
Red Hat, Inc. +1-919-301-3226
Virtualization: qemu.org | libvirt.org
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 28/38] qapi/gen.py: update write() to be more idiomatic
2020-09-25 15:47 ` Eric Blake
@ 2020-09-28 12:09 ` Markus Armbruster
2020-09-28 14:08 ` John Snow
0 siblings, 1 reply; 190+ messages in thread
From: Markus Armbruster @ 2020-09-28 12:09 UTC (permalink / raw)
To: Eric Blake
Cc: Peter Maydell, Daniel P. Berrangé, Eduardo Habkost,
John Snow, qemu-devel, Michael Roth, Cleber Rosa,
Alex Bennée
Eric Blake <eblake@redhat.com> writes:
> On 9/25/20 8:52 AM, Markus Armbruster wrote:
>
>>>> This was my best attempt to open the file read/write, creating it if it
>>>> doesn't exist.
>>>>
>>>> Plain
>>>>
>>>> f = open(pathname, "r+", encoding='utf-8')
>>>>
>>>> fails instead of creates, and
>>>>
>>>> f = open(pathname, "w+", encoding='utf-8')
>>>>
>>>> truncates.
>>>>
>>>> If you know a better way, tell me!
>>>
>>> IIUC, you need "a+" as the mode, rather than "w+"
>> Sure this lets me do
>> f.seek(0)
>> f.truncate(0)
>> f.write(text)
>> to overwrite the old contents on all systems?
>
> As long as you do a single pass over the output (you issue a stream of
> f.write() after the truncate, but never a seek), then this will work.
Well, I do seek(), right before the truncate.
>> Documentation cautions:
>> [...] 'a' for appending (which on some Unix systems, means that
>> all
>> writes append to the end of the file regardless of the current seek
>> position).
>
> Yes, that means that random access is impossible on such a stream.
> But not all file creation patterns require random access.
To be honest, I still prefer the code I wrote, because there the reader
only wonders why I didn't just open(), while here we get to argue about
subtleties of mode "a+".
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 28/38] qapi/gen.py: update write() to be more idiomatic
2020-09-28 12:09 ` Markus Armbruster
@ 2020-09-28 14:08 ` John Snow
0 siblings, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-28 14:08 UTC (permalink / raw)
To: Markus Armbruster, Eric Blake
Cc: Peter Maydell, Daniel P. Berrangé, Eduardo Habkost,
qemu-devel, Michael Roth, Cleber Rosa, Alex Bennée
On 9/28/20 8:09 AM, Markus Armbruster wrote:
> Eric Blake <eblake@redhat.com> writes:
>
>> On 9/25/20 8:52 AM, Markus Armbruster wrote:
>>
>>>>> This was my best attempt to open the file read/write, creating it if it
>>>>> doesn't exist.
>>>>>
>>>>> Plain
>>>>>
>>>>> f = open(pathname, "r+", encoding='utf-8')
>>>>>
>>>>> fails instead of creates, and
>>>>>
>>>>> f = open(pathname, "w+", encoding='utf-8')
>>>>>
>>>>> truncates.
>>>>>
>>>>> If you know a better way, tell me!
>>>>
>>>> IIUC, you need "a+" as the mode, rather than "w+"
>>> Sure this lets me do
>>> f.seek(0)
>>> f.truncate(0)
>>> f.write(text)
>>> to overwrite the old contents on all systems?
>>
>> As long as you do a single pass over the output (you issue a stream of
>> f.write() after the truncate, but never a seek), then this will work.
>
> Well, I do seek(), right before the truncate.
>
>>> Documentation cautions:
>>> [...] 'a' for appending (which on some Unix systems, means that
>>> all
>>> writes append to the end of the file regardless of the current seek
>>> position).
>>
>> Yes, that means that random access is impossible on such a stream.
>> But not all file creation patterns require random access.
>
> To be honest, I still prefer the code I wrote, because there the reader
> only wonders why I didn't just open(), while here we get to argue about
> subtleties of mode "a+".
>
I kept your os.open, I agree with you here.
(I still rewrote to use the context managers, though.)
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 28/38] qapi/gen.py: update write() to be more idiomatic
2020-09-25 13:15 ` Markus Armbruster
2020-09-25 13:24 ` Daniel P. Berrangé
@ 2020-09-25 13:26 ` Eduardo Habkost
2020-09-25 16:33 ` John Snow
1 sibling, 1 reply; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-25 13:26 UTC (permalink / raw)
To: Markus Armbruster
Cc: Peter Maydell, John Snow, Michael Roth, qemu-devel, Cleber Rosa,
Alex Bennée
On Fri, Sep 25, 2020 at 03:15:57PM +0200, Markus Armbruster wrote:
> Cleber Rosa <crosa@redhat.com> writes:
>
> > On Wed, Sep 23, 2020 at 02:37:27PM -0400, John Snow wrote:
> >> On 9/23/20 11:26 AM, Eduardo Habkost wrote:
> >> > On Tue, Sep 22, 2020 at 05:00:51PM -0400, John Snow wrote:
> >> > > Make the file handling here just a tiny bit more idiomatic.
> >> > > (I realize this is heavily subjective.)
> >> > >
> >> > > Use exist_ok=True for os.makedirs and remove the exception,
> >> > > use fdopen() to wrap the file descriptor in a File-like object,
> >> > > and use a context manager for managing the file pointer.
> >> > >
> >> > > Signed-off-by: John Snow <jsnow@redhat.com>
> >> >
> >> > Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
> >> >
> >> > I really miss a comment below explaining why we use
> >> > open(os.open(pathname, ...), ...) instead of open(pathname, ...).
>
> This code:
>
> fd = os.open(pathname, os.O_RDWR | os.O_CREAT, 0o666)
> f = open(fd, 'r+', encoding='utf-8')
>
> >> Not known to me. It was introduced in 907b846653 as part of an effort to
> >> reduce rebuild times. Maybe this avoids a modification time change if the
> >> file already exists?
> >>
> >> Markus?
> >
> > AFACIT the change on 907b846653 is effective because of the "is new
> > text different from old text?" conditional. I can not see how the
> > separate/duplicate open/fdopen would contribute to that.
> >
> > But, let's hear from Markus.
>
> This was my best attempt to open the file read/write, creating it if it
> doesn't exist.
>
> Plain
>
> f = open(pathname, "r+", encoding='utf-8')
>
> fails instead of creates, and
>
> f = open(pathname, "w+", encoding='utf-8')
>
> truncates.
>
> If you know a better way, tell me!
Thanks for the explanation!
Yeah, it looks like there's no combination of open() flags that
would get translated to O_RDWR|O_CREAT.
Using os.open() like you did seems more straightforward than
catching FileNotFoundError.
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 28/38] qapi/gen.py: update write() to be more idiomatic
2020-09-25 13:26 ` Eduardo Habkost
@ 2020-09-25 16:33 ` John Snow
0 siblings, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-25 16:33 UTC (permalink / raw)
To: Eduardo Habkost, Markus Armbruster
Cc: Michael Roth, Peter Maydell, Alex Bennée, qemu-devel,
Cleber Rosa
On 9/25/20 9:26 AM, Eduardo Habkost wrote:
> On Fri, Sep 25, 2020 at 03:15:57PM +0200, Markus Armbruster wrote:
>> Cleber Rosa <crosa@redhat.com> writes:
>>
>>> On Wed, Sep 23, 2020 at 02:37:27PM -0400, John Snow wrote:
>>>> On 9/23/20 11:26 AM, Eduardo Habkost wrote:
>>>>> On Tue, Sep 22, 2020 at 05:00:51PM -0400, John Snow wrote:
>>>>>> Make the file handling here just a tiny bit more idiomatic.
>>>>>> (I realize this is heavily subjective.)
>>>>>>
>>>>>> Use exist_ok=True for os.makedirs and remove the exception,
>>>>>> use fdopen() to wrap the file descriptor in a File-like object,
>>>>>> and use a context manager for managing the file pointer.
>>>>>>
>>>>>> Signed-off-by: John Snow <jsnow@redhat.com>
>>>>>
>>>>> Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
>>>>>
>>>>> I really miss a comment below explaining why we use
>>>>> open(os.open(pathname, ...), ...) instead of open(pathname, ...).
>>
>> This code:
>>
>> fd = os.open(pathname, os.O_RDWR | os.O_CREAT, 0o666)
>> f = open(fd, 'r+', encoding='utf-8')
>>
>>>> Not known to me. It was introduced in 907b846653 as part of an effort to
>>>> reduce rebuild times. Maybe this avoids a modification time change if the
>>>> file already exists?
>>>>
>>>> Markus?
>>>
>>> AFACIT the change on 907b846653 is effective because of the "is new
>>> text different from old text?" conditional. I can not see how the
>>> separate/duplicate open/fdopen would contribute to that.
>>>
>>> But, let's hear from Markus.
>>
>> This was my best attempt to open the file read/write, creating it if it
>> doesn't exist.
>>
>> Plain
>>
>> f = open(pathname, "r+", encoding='utf-8')
>>
>> fails instead of creates, and
>>
>> f = open(pathname, "w+", encoding='utf-8')
>>
>> truncates.
>>
>> If you know a better way, tell me!
>
> Thanks for the explanation!
>
> Yeah, it looks like there's no combination of open() flags that
> would get translated to O_RDWR|O_CREAT.
>
> Using os.open() like you did seems more straightforward than
> catching FileNotFoundError.
>
OK. Using fdopen works for us, so let's stick with it. I will add a
comment explaining our reasoning.
Thanks!
--js
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 28/38] qapi/gen.py: update write() to be more idiomatic
2020-09-22 21:00 ` [PATCH v2 28/38] qapi/gen.py: update write() to be more idiomatic John Snow
2020-09-23 15:26 ` Eduardo Habkost
@ 2020-09-24 16:01 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-24 16:01 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 458 bytes --]
On Tue, Sep 22, 2020 at 05:00:51PM -0400, John Snow wrote:
> Make the file handling here just a tiny bit more idiomatic.
> (I realize this is heavily subjective.)
>
> Use exist_ok=True for os.makedirs and remove the exception,
> use fdopen() to wrap the file descriptor in a File-like object,
> and use a context manager for managing the file pointer.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 29/38] qapi/gen.py: delint with pylint
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (27 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 28/38] qapi/gen.py: update write() to be more idiomatic John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 15:44 ` Eduardo Habkost
2020-09-24 18:44 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 30/38] qapi/introspect.py: Add a typed 'extra' structure John Snow
` (9 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
'fp' and 'fd' are self-evident in context, add them to the list of OK
names.
_top and _bottom also need to stay class methods because some users
override the method and need to use `self`. Tell pylint to shush.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/gen.py | 2 ++
scripts/qapi/pylintrc | 5 +++--
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
index cf340e66d4..ed498397ad 100644
--- a/scripts/qapi/gen.py
+++ b/scripts/qapi/gen.py
@@ -50,9 +50,11 @@ def get_content(self) -> str:
return self._top() + self._preamble + self._body + self._bottom()
def _top(self) -> str:
+ # pylint: disable=no-self-use
return ''
def _bottom(self) -> str:
+ # pylint: disable=no-self-use
return ''
def write(self, output_dir: str) -> None:
diff --git a/scripts/qapi/pylintrc b/scripts/qapi/pylintrc
index 7438806096..de132d03cf 100644
--- a/scripts/qapi/pylintrc
+++ b/scripts/qapi/pylintrc
@@ -5,7 +5,6 @@
ignore-patterns=doc.py,
error.py,
expr.py,
- gen.py,
parser.py,
schema.py,
types.py,
@@ -46,7 +45,9 @@ good-names=i,
k,
ex,
Run,
- _
+ _,
+ fp, # fp = open(...)
+ fd, # fd = os.open(...)
[VARIABLES]
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 29/38] qapi/gen.py: delint with pylint
2020-09-22 21:00 ` [PATCH v2 29/38] qapi/gen.py: delint with pylint John Snow
@ 2020-09-23 15:44 ` Eduardo Habkost
2020-09-23 18:38 ` John Snow
2020-09-24 18:44 ` Cleber Rosa
1 sibling, 1 reply; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 15:44 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:52PM -0400, John Snow wrote:
> 'fp' and 'fd' are self-evident in context, add them to the list of OK
> names.
>
> _top and _bottom also need to stay class methods because some users
> override the method and need to use `self`. Tell pylint to shush.
Do you mean "stay instance methods"?
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/gen.py | 2 ++
> scripts/qapi/pylintrc | 5 +++--
> 2 files changed, 5 insertions(+), 2 deletions(-)
>
> diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
> index cf340e66d4..ed498397ad 100644
> --- a/scripts/qapi/gen.py
> +++ b/scripts/qapi/gen.py
> @@ -50,9 +50,11 @@ def get_content(self) -> str:
> return self._top() + self._preamble + self._body + self._bottom()
>
> def _top(self) -> str:
> + # pylint: disable=no-self-use
> return ''
>
> def _bottom(self) -> str:
> + # pylint: disable=no-self-use
> return ''
>
> def write(self, output_dir: str) -> None:
> diff --git a/scripts/qapi/pylintrc b/scripts/qapi/pylintrc
> index 7438806096..de132d03cf 100644
> --- a/scripts/qapi/pylintrc
> +++ b/scripts/qapi/pylintrc
> @@ -5,7 +5,6 @@
> ignore-patterns=doc.py,
> error.py,
> expr.py,
> - gen.py,
> parser.py,
> schema.py,
> types.py,
> @@ -46,7 +45,9 @@ good-names=i,
> k,
> ex,
> Run,
> - _
> + _,
> + fp, # fp = open(...)
> + fd, # fd = os.open(...)
>
> [VARIABLES]
>
> --
> 2.26.2
>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 29/38] qapi/gen.py: delint with pylint
2020-09-23 15:44 ` Eduardo Habkost
@ 2020-09-23 18:38 ` John Snow
0 siblings, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-23 18:38 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On 9/23/20 11:44 AM, Eduardo Habkost wrote:
> On Tue, Sep 22, 2020 at 05:00:52PM -0400, John Snow wrote:
>> 'fp' and 'fd' are self-evident in context, add them to the list of OK
>> names.
>>
>> _top and _bottom also need to stay class methods because some users
>> override the method and need to use `self`. Tell pylint to shush.
>
> Do you mean "stay instance methods"?
>
Yep. Fixed, thank you.
> Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
>
Thanks!
>>
>> Signed-off-by: John Snow <jsnow@redhat.com>
>> ---
>> scripts/qapi/gen.py | 2 ++
>> scripts/qapi/pylintrc | 5 +++--
>> 2 files changed, 5 insertions(+), 2 deletions(-)
>>
>> diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
>> index cf340e66d4..ed498397ad 100644
>> --- a/scripts/qapi/gen.py
>> +++ b/scripts/qapi/gen.py
>> @@ -50,9 +50,11 @@ def get_content(self) -> str:
>> return self._top() + self._preamble + self._body + self._bottom()
>>
>> def _top(self) -> str:
>> + # pylint: disable=no-self-use
>> return ''
>>
>> def _bottom(self) -> str:
>> + # pylint: disable=no-self-use
>> return ''
>>
>> def write(self, output_dir: str) -> None:
>> diff --git a/scripts/qapi/pylintrc b/scripts/qapi/pylintrc
>> index 7438806096..de132d03cf 100644
>> --- a/scripts/qapi/pylintrc
>> +++ b/scripts/qapi/pylintrc
>> @@ -5,7 +5,6 @@
>> ignore-patterns=doc.py,
>> error.py,
>> expr.py,
>> - gen.py,
>> parser.py,
>> schema.py,
>> types.py,
>> @@ -46,7 +45,9 @@ good-names=i,
>> k,
>> ex,
>> Run,
>> - _
>> + _,
>> + fp, # fp = open(...)
>> + fd, # fd = os.open(...)
>>
>> [VARIABLES]
>>
>> --
>> 2.26.2
>>
>
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 29/38] qapi/gen.py: delint with pylint
2020-09-22 21:00 ` [PATCH v2 29/38] qapi/gen.py: delint with pylint John Snow
2020-09-23 15:44 ` Eduardo Habkost
@ 2020-09-24 18:44 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-24 18:44 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 438 bytes --]
On Tue, Sep 22, 2020 at 05:00:52PM -0400, John Snow wrote:
> 'fp' and 'fd' are self-evident in context, add them to the list of OK
> names.
>
> _top and _bottom also need to stay class methods because some users
> override the method and need to use `self`. Tell pylint to shush.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
With the commit message change caught by Eduardo:
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 30/38] qapi/introspect.py: Add a typed 'extra' structure
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (28 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 29/38] qapi/gen.py: delint with pylint John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 16:13 ` Eduardo Habkost
2020-09-22 21:00 ` [PATCH v2 31/38] qapi/introspect.py: add _gen_features helper John Snow
` (8 subsequent siblings)
38 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Typing arbitrarily shaped dicts with mypy is difficult prior to Python
3.8; using explicit structures is nicer.
Since we always define an Extra type now, the return type of _make_tree
simplifies and always returns the tuple.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/introspect.py | 31 +++++++++++++++++++------------
1 file changed, 19 insertions(+), 12 deletions(-)
diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py
index b036fcf9ce..41ca8afc67 100644
--- a/scripts/qapi/introspect.py
+++ b/scripts/qapi/introspect.py
@@ -10,6 +10,8 @@
See the COPYING file in the top-level directory.
"""
+from typing import (NamedTuple, Optional, Sequence)
+
from .common import (
c_name,
gen_endif,
@@ -21,16 +23,21 @@
QAPISchemaType)
-def _make_tree(obj, ifcond, features, extra=None):
- if extra is None:
- extra = {}
- if ifcond:
- extra['if'] = ifcond
+class Extra(NamedTuple):
+ """
+ Extra contains data that isn't intended for output by introspection.
+ """
+ comment: Optional[str] = None
+ ifcond: Sequence[str] = tuple()
+
+
+def _make_tree(obj, ifcond, features,
+ extra: Optional[Extra] = None):
+ comment = extra.comment if extra else None
+ extra = Extra(comment, ifcond)
if features:
- obj['features'] = [(f.name, {'if': f.ifcond}) for f in features]
- if extra:
- return (obj, extra)
- return obj
+ obj['features'] = [(f.name, Extra(None, f.ifcond)) for f in features]
+ return (obj, extra)
def _tree_to_qlit(obj, level=0, suppress_first_indent=False):
@@ -40,8 +47,8 @@ def indent(level):
if isinstance(obj, tuple):
ifobj, extra = obj
- ifcond = extra.get('if')
- comment = extra.get('comment')
+ ifcond = extra.ifcond
+ comment = extra.comment
ret = ''
if comment:
ret += indent(level) + '/* %s */\n' % comment
@@ -168,7 +175,7 @@ def _gen_tree(self, name, mtype, obj, ifcond, features):
if not self._unmask:
# Output a comment to make it easy to map masked names
# back to the source when reading the generated output.
- extra = {'comment': '"%s" = %s' % (self._name(name), name)}
+ extra = Extra(comment=f'"{self._name(name)}" = {name}')
name = self._name(name)
obj['name'] = name
obj['meta-type'] = mtype
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 30/38] qapi/introspect.py: Add a typed 'extra' structure
2020-09-22 21:00 ` [PATCH v2 30/38] qapi/introspect.py: Add a typed 'extra' structure John Snow
@ 2020-09-23 16:13 ` Eduardo Habkost
2020-09-23 21:34 ` John Snow
0 siblings, 1 reply; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 16:13 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:53PM -0400, John Snow wrote:
> Typing arbitrarily shaped dicts with mypy is difficult prior to Python
> 3.8; using explicit structures is nicer.
>
> Since we always define an Extra type now, the return type of _make_tree
> simplifies and always returns the tuple.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/introspect.py | 31 +++++++++++++++++++------------
> 1 file changed, 19 insertions(+), 12 deletions(-)
>
Here I'm confused by both the original code and the new code.
I will try to review as a refactoring of existing code, but I'll
have suggestions for follow ups:
> diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py
> index b036fcf9ce..41ca8afc67 100644
> --- a/scripts/qapi/introspect.py
> +++ b/scripts/qapi/introspect.py
> @@ -10,6 +10,8 @@
> See the COPYING file in the top-level directory.
> """
>
> +from typing import (NamedTuple, Optional, Sequence)
> +
> from .common import (
> c_name,
> gen_endif,
> @@ -21,16 +23,21 @@
> QAPISchemaType)
>
>
> -def _make_tree(obj, ifcond, features, extra=None):
> - if extra is None:
> - extra = {}
> - if ifcond:
> - extra['if'] = ifcond
> +class Extra(NamedTuple):
> + """
> + Extra contains data that isn't intended for output by introspection.
> + """
> + comment: Optional[str] = None
> + ifcond: Sequence[str] = tuple()
> +
> +
> +def _make_tree(obj, ifcond, features,
> + extra: Optional[Extra] = None):
> + comment = extra.comment if extra else None
> + extra = Extra(comment, ifcond)
Here we have one big difference: now `extra` is being recreated,
and all fields except `extra.comment` are being ignored. On the
original version, all fields in `extra` were being kept. This
makes the existence of the `extra` argument pointless.
If you are going through the trouble of changing the type of the
4rd argument to _make_tree(), this seems more obvious:
diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py
index 41ca8afc672..c62af94c9ad 100644
--- a/scripts/qapi/introspect.py
+++ b/scripts/qapi/introspect.py
@@ -32,8 +32,7 @@ class Extra(NamedTuple):
def _make_tree(obj, ifcond, features,
- extra: Optional[Extra] = None):
- comment = extra.comment if extra else None
+ comment: Optional[str] = None):
extra = Extra(comment, ifcond)
if features:
obj['features'] = [(f.name, Extra(None, f.ifcond)) for f in features]
@@ -170,16 +169,16 @@ const QLitObject %(c_name)s = %(c_string)s;
return self._name(typ.name)
def _gen_tree(self, name, mtype, obj, ifcond, features):
- extra = None
+ comment = None
if mtype not in ('command', 'event', 'builtin', 'array'):
if not self._unmask:
# Output a comment to make it easy to map masked names
# back to the source when reading the generated output.
- extra = Extra(comment=f'"{self._name(name)}" = {name}')
+ comment = f'"{self._name(name)}" = {name}'
name = self._name(name)
obj['name'] = name
obj['meta-type'] = mtype
- self._trees.append(_make_tree(obj, ifcond, features, extra))
+ self._trees.append(_make_tree(obj, ifcond, features, comment))
def _gen_member(self, member):
obj = {'name': member.name, 'type': self._use_type(member.type)}
I understand you're trying to just make minimal refactoring, and I
don't think this should block your cleanup series. So:
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
> if features:
> - obj['features'] = [(f.name, {'if': f.ifcond}) for f in features]
> - if extra:
> - return (obj, extra)
> - return obj
> + obj['features'] = [(f.name, Extra(None, f.ifcond)) for f in features]
> + return (obj, extra)
>
>
> def _tree_to_qlit(obj, level=0, suppress_first_indent=False):
> @@ -40,8 +47,8 @@ def indent(level):
>
> if isinstance(obj, tuple):
> ifobj, extra = obj
> - ifcond = extra.get('if')
> - comment = extra.get('comment')
> + ifcond = extra.ifcond
> + comment = extra.comment
> ret = ''
> if comment:
> ret += indent(level) + '/* %s */\n' % comment
> @@ -168,7 +175,7 @@ def _gen_tree(self, name, mtype, obj, ifcond, features):
> if not self._unmask:
> # Output a comment to make it easy to map masked names
> # back to the source when reading the generated output.
> - extra = {'comment': '"%s" = %s' % (self._name(name), name)}
> + extra = Extra(comment=f'"{self._name(name)}" = {name}')
> name = self._name(name)
> obj['name'] = name
> obj['meta-type'] = mtype
> --
> 2.26.2
>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 30/38] qapi/introspect.py: Add a typed 'extra' structure
2020-09-23 16:13 ` Eduardo Habkost
@ 2020-09-23 21:34 ` John Snow
0 siblings, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-23 21:34 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, Michael Roth, qemu-devel, Markus Armbruster,
Cleber Rosa, Alex Bennée
On 9/23/20 12:13 PM, Eduardo Habkost wrote:
> On Tue, Sep 22, 2020 at 05:00:53PM -0400, John Snow wrote:
>> Typing arbitrarily shaped dicts with mypy is difficult prior to Python
>> 3.8; using explicit structures is nicer.
>>
>> Since we always define an Extra type now, the return type of _make_tree
>> simplifies and always returns the tuple.
>>
>> Signed-off-by: John Snow <jsnow@redhat.com>
>> ---
>> scripts/qapi/introspect.py | 31 +++++++++++++++++++------------
>> 1 file changed, 19 insertions(+), 12 deletions(-)
>>
>
> Here I'm confused by both the original code and the new code.
>
> I will try to review as a refactoring of existing code, but I'll
> have suggestions for follow ups:
>
>> diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py
>> index b036fcf9ce..41ca8afc67 100644
>> --- a/scripts/qapi/introspect.py
>> +++ b/scripts/qapi/introspect.py
>> @@ -10,6 +10,8 @@
>> See the COPYING file in the top-level directory.
>> """
>>
>> +from typing import (NamedTuple, Optional, Sequence)
>> +
>> from .common import (
>> c_name,
>> gen_endif,
>> @@ -21,16 +23,21 @@
>> QAPISchemaType)
>>
>>
>> -def _make_tree(obj, ifcond, features, extra=None):
>> - if extra is None:
>> - extra = {}
>> - if ifcond:
>> - extra['if'] = ifcond
>> +class Extra(NamedTuple):
>> + """
>> + Extra contains data that isn't intended for output by introspection.
>> + """
>> + comment: Optional[str] = None
>> + ifcond: Sequence[str] = tuple()
>> +
>> +
>> +def _make_tree(obj, ifcond, features,
>> + extra: Optional[Extra] = None):
>> + comment = extra.comment if extra else None
>> + extra = Extra(comment, ifcond)
>
> Here we have one big difference: now `extra` is being recreated,
> and all fields except `extra.comment` are being ignored. On the
> original version, all fields in `extra` were being kept. This
> makes the existence of the `extra` argument pointless.
>
Yup, oops.
> If you are going through the trouble of changing the type of the
> 4rd argument to _make_tree(), this seems more obvious:
>
Yes, agree. I came up with something similar after talking to you this
morning.
> diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py
> index 41ca8afc672..c62af94c9ad 100644
> --- a/scripts/qapi/introspect.py
> +++ b/scripts/qapi/introspect.py
> @@ -32,8 +32,7 @@ class Extra(NamedTuple):
>
>
> def _make_tree(obj, ifcond, features,
> - extra: Optional[Extra] = None):
> - comment = extra.comment if extra else None
> + comment: Optional[str] = None):
> extra = Extra(comment, ifcond)
> if features:
> obj['features'] = [(f.name, Extra(None, f.ifcond)) for f in features]
> @@ -170,16 +169,16 @@ const QLitObject %(c_name)s = %(c_string)s;
> return self._name(typ.name)
>
> def _gen_tree(self, name, mtype, obj, ifcond, features):
> - extra = None
> + comment = None
> if mtype not in ('command', 'event', 'builtin', 'array'):
> if not self._unmask:
> # Output a comment to make it easy to map masked names
> # back to the source when reading the generated output.
> - extra = Extra(comment=f'"{self._name(name)}" = {name}')
> + comment = f'"{self._name(name)}" = {name}'
> name = self._name(name)
> obj['name'] = name
> obj['meta-type'] = mtype
> - self._trees.append(_make_tree(obj, ifcond, features, extra))
> + self._trees.append(_make_tree(obj, ifcond, features, comment))
>
> def _gen_member(self, member):
> obj = {'name': member.name, 'type': self._use_type(member.type)}
>
> I understand you're trying to just make minimal refactoring, and I
> don't think this should block your cleanup series. So:
>
> Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
>
I appreciate the benefit-of-the-doubt, but I think this change is worth
making while we're here.
>
>> if features:
>> - obj['features'] = [(f.name, {'if': f.ifcond}) for f in features]
>> - if extra:
>> - return (obj, extra)
>> - return obj
>> + obj['features'] = [(f.name, Extra(None, f.ifcond)) for f in features]
>> + return (obj, extra)
>>
>>
>> def _tree_to_qlit(obj, level=0, suppress_first_indent=False):
>> @@ -40,8 +47,8 @@ def indent(level):
>>
>> if isinstance(obj, tuple):
>> ifobj, extra = obj
>> - ifcond = extra.get('if')
>> - comment = extra.get('comment')
>> + ifcond = extra.ifcond
>> + comment = extra.comment
>> ret = ''
>> if comment:
>> ret += indent(level) + '/* %s */\n' % comment
>> @@ -168,7 +175,7 @@ def _gen_tree(self, name, mtype, obj, ifcond, features):
>> if not self._unmask:
>> # Output a comment to make it easy to map masked names
>> # back to the source when reading the generated output.
>> - extra = {'comment': '"%s" = %s' % (self._name(name), name)}
>> + extra = Extra(comment=f'"{self._name(name)}" = {name}')
>> name = self._name(name)
>> obj['name'] = name
>> obj['meta-type'] = mtype
>> --
>> 2.26.2
>>
>
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 31/38] qapi/introspect.py: add _gen_features helper
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (29 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 30/38] qapi/introspect.py: Add a typed 'extra' structure John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 16:35 ` Eduardo Habkost
2020-09-22 21:00 ` [PATCH v2 32/38] qapi/introspect.py: create a typed 'Node' data structure John Snow
` (7 subsequent siblings)
38 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
_make_tree doesn't know if it is receiving an object or some other type;
adding features information should arguably be performed by the caller.
This will help us refactor _make_tree more gracefully in the next patch.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/introspect.py | 22 ++++++++++++++--------
1 file changed, 14 insertions(+), 8 deletions(-)
diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py
index 41ca8afc67..e1edd0b179 100644
--- a/scripts/qapi/introspect.py
+++ b/scripts/qapi/introspect.py
@@ -10,7 +10,7 @@
See the COPYING file in the top-level directory.
"""
-from typing import (NamedTuple, Optional, Sequence)
+from typing import (NamedTuple, List, Optional, Sequence)
from .common import (
c_name,
@@ -20,7 +20,7 @@
)
from .gen import QAPISchemaMonolithicCVisitor
from .schema import (QAPISchemaArrayType, QAPISchemaBuiltinType,
- QAPISchemaType)
+ QAPISchemaFeature, QAPISchemaType)
class Extra(NamedTuple):
@@ -31,12 +31,10 @@ class Extra(NamedTuple):
ifcond: Sequence[str] = tuple()
-def _make_tree(obj, ifcond, features,
+def _make_tree(obj, ifcond,
extra: Optional[Extra] = None):
comment = extra.comment if extra else None
extra = Extra(comment, ifcond)
- if features:
- obj['features'] = [(f.name, Extra(None, f.ifcond)) for f in features]
return (obj, extra)
@@ -169,6 +167,10 @@ def _use_type(self, typ):
return '[' + self._use_type(typ.element_type) + ']'
return self._name(typ.name)
+ @classmethod
+ def _gen_features(cls, features: List[QAPISchemaFeature]):
+ return [_make_tree(f.name, f.ifcond) for f in features]
+
def _gen_tree(self, name, mtype, obj, ifcond, features):
extra = None
if mtype not in ('command', 'event', 'builtin', 'array'):
@@ -179,13 +181,17 @@ def _gen_tree(self, name, mtype, obj, ifcond, features):
name = self._name(name)
obj['name'] = name
obj['meta-type'] = mtype
- self._trees.append(_make_tree(obj, ifcond, features, extra))
+ if features:
+ obj['features'] = self._gen_features(features)
+ self._trees.append(_make_tree(obj, ifcond, extra))
def _gen_member(self, member):
obj = {'name': member.name, 'type': self._use_type(member.type)}
if member.optional:
obj['default'] = None
- return _make_tree(obj, member.ifcond, member.features)
+ if member.features:
+ obj['features'] = self._gen_features(member.features)
+ return _make_tree(obj, member.ifcond)
def _gen_variants(self, tag_name, variants):
return {'tag': tag_name,
@@ -193,7 +199,7 @@ def _gen_variants(self, tag_name, variants):
def _gen_variant(self, variant):
obj = {'case': variant.name, 'type': self._use_type(variant.type)}
- return _make_tree(obj, variant.ifcond, None)
+ return _make_tree(obj, variant.ifcond)
def visit_builtin_type(self, name, info, json_type):
self._gen_tree(name, 'builtin', {'json-type': json_type}, [], None)
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 31/38] qapi/introspect.py: add _gen_features helper
2020-09-22 21:00 ` [PATCH v2 31/38] qapi/introspect.py: add _gen_features helper John Snow
@ 2020-09-23 16:35 ` Eduardo Habkost
2020-09-23 21:43 ` John Snow
0 siblings, 1 reply; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 16:35 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:54PM -0400, John Snow wrote:
> _make_tree doesn't know if it is receiving an object or some other type;
> adding features information should arguably be performed by the caller.
>
> This will help us refactor _make_tree more gracefully in the next patch.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/introspect.py | 22 ++++++++++++++--------
> 1 file changed, 14 insertions(+), 8 deletions(-)
>
> diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py
> index 41ca8afc67..e1edd0b179 100644
> --- a/scripts/qapi/introspect.py
> +++ b/scripts/qapi/introspect.py
> @@ -10,7 +10,7 @@
> See the COPYING file in the top-level directory.
> """
>
> -from typing import (NamedTuple, Optional, Sequence)
> +from typing import (NamedTuple, List, Optional, Sequence)
>
> from .common import (
> c_name,
> @@ -20,7 +20,7 @@
> )
> from .gen import QAPISchemaMonolithicCVisitor
> from .schema import (QAPISchemaArrayType, QAPISchemaBuiltinType,
> - QAPISchemaType)
> + QAPISchemaFeature, QAPISchemaType)
>
>
> class Extra(NamedTuple):
> @@ -31,12 +31,10 @@ class Extra(NamedTuple):
> ifcond: Sequence[str] = tuple()
>
>
> -def _make_tree(obj, ifcond, features,
> +def _make_tree(obj, ifcond,
> extra: Optional[Extra] = None):
> comment = extra.comment if extra else None
> extra = Extra(comment, ifcond)
I believe these two lines above should be removed, as suggested
in patch 30, but let's ignore that for now.
> - if features:
> - obj['features'] = [(f.name, Extra(None, f.ifcond)) for f in features]
I can't say I understand completely why moving these two lines
outside _make_tree() is useful, but if it makes the cleanup work
you did easier, I trust this is the right thing to do. The
changes look correct.
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
> return (obj, extra)
>
>
> @@ -169,6 +167,10 @@ def _use_type(self, typ):
> return '[' + self._use_type(typ.element_type) + ']'
> return self._name(typ.name)
>
> + @classmethod
> + def _gen_features(cls, features: List[QAPISchemaFeature]):
> + return [_make_tree(f.name, f.ifcond) for f in features]
> +
> def _gen_tree(self, name, mtype, obj, ifcond, features):
> extra = None
> if mtype not in ('command', 'event', 'builtin', 'array'):
> @@ -179,13 +181,17 @@ def _gen_tree(self, name, mtype, obj, ifcond, features):
> name = self._name(name)
> obj['name'] = name
> obj['meta-type'] = mtype
> - self._trees.append(_make_tree(obj, ifcond, features, extra))
> + if features:
> + obj['features'] = self._gen_features(features)
> + self._trees.append(_make_tree(obj, ifcond, extra))
>
> def _gen_member(self, member):
> obj = {'name': member.name, 'type': self._use_type(member.type)}
> if member.optional:
> obj['default'] = None
> - return _make_tree(obj, member.ifcond, member.features)
> + if member.features:
> + obj['features'] = self._gen_features(member.features)
> + return _make_tree(obj, member.ifcond)
>
> def _gen_variants(self, tag_name, variants):
> return {'tag': tag_name,
> @@ -193,7 +199,7 @@ def _gen_variants(self, tag_name, variants):
>
> def _gen_variant(self, variant):
> obj = {'case': variant.name, 'type': self._use_type(variant.type)}
> - return _make_tree(obj, variant.ifcond, None)
> + return _make_tree(obj, variant.ifcond)
>
> def visit_builtin_type(self, name, info, json_type):
> self._gen_tree(name, 'builtin', {'json-type': json_type}, [], None)
> --
> 2.26.2
>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 31/38] qapi/introspect.py: add _gen_features helper
2020-09-23 16:35 ` Eduardo Habkost
@ 2020-09-23 21:43 ` John Snow
2020-09-23 21:54 ` Eduardo Habkost
0 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-23 21:43 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, Michael Roth, qemu-devel, Markus Armbruster,
Cleber Rosa, Alex Bennée
On 9/23/20 12:35 PM, Eduardo Habkost wrote:
> I believe these two lines above should be removed, as suggested
> in patch 30, but let's ignore that for now.
>
Yup, headed there.
>> - if features:
>> - obj['features'] = [(f.name, Extra(None, f.ifcond)) for f in features]
> I can't say I understand completely why moving these two lines
> outside _make_tree() is useful, but if it makes the cleanup work
> you did easier, I trust this is the right thing to do. The
> changes look correct.
The basic premise is:
Why pass information you want to add to obj['features'] to a function to
make that assignment, when you could just perform that assignment yourself?
Otherwise, _make_tree, which accepts any arbitrary object (not just
dicts!) has to interrogate its arguments to make sure you gave it a dict
when you give it a features argument.
Type-wise, it's cleaner to perform this transformation when we KNOW we
have an object than it is to defer to a more abstracted function and
assert/downcast back to more specific types.
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 31/38] qapi/introspect.py: add _gen_features helper
2020-09-23 21:43 ` John Snow
@ 2020-09-23 21:54 ` Eduardo Habkost
0 siblings, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 21:54 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Michael Roth, qemu-devel, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Wed, Sep 23, 2020 at 05:43:45PM -0400, John Snow wrote:
> On 9/23/20 12:35 PM, Eduardo Habkost wrote:
> > I believe these two lines above should be removed, as suggested
> > in patch 30, but let's ignore that for now.
> >
>
> Yup, headed there.
>
> > > - if features:
> > > - obj['features'] = [(f.name, Extra(None, f.ifcond)) for f in features]
> > I can't say I understand completely why moving these two lines
> > outside _make_tree() is useful, but if it makes the cleanup work
> > you did easier, I trust this is the right thing to do. The
> > changes look correct.
>
> The basic premise is:
>
> Why pass information you want to add to obj['features'] to a function to
> make that assignment, when you could just perform that assignment yourself?
>
> Otherwise, _make_tree, which accepts any arbitrary object (not just dicts!)
> has to interrogate its arguments to make sure you gave it a dict when you
> give it a features argument.
Oh, I was not aware that obj could be not a dictionary. In this
case, it makes lots of sense to move the magic outside the
function.
>
> Type-wise, it's cleaner to perform this transformation when we KNOW we have
> an object than it is to defer to a more abstracted function and
> assert/downcast back to more specific types.
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 32/38] qapi/introspect.py: create a typed 'Node' data structure
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (30 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 31/38] qapi/introspect.py: add _gen_features helper John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 18:41 ` Eduardo Habkost
2020-09-22 21:00 ` [PATCH v2 33/38] qapi/introspect.py: add type hint annotations John Snow
` (6 subsequent siblings)
38 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Replacing the un-typed tuple, add a typed Node that we can add typed
metadata to.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/introspect.py | 53 ++++++++++++++++++++------------------
1 file changed, 28 insertions(+), 25 deletions(-)
diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py
index e1edd0b179..e0f5007ab7 100644
--- a/scripts/qapi/introspect.py
+++ b/scripts/qapi/introspect.py
@@ -31,11 +31,18 @@ class Extra(NamedTuple):
ifcond: Sequence[str] = tuple()
-def _make_tree(obj, ifcond,
- extra: Optional[Extra] = None):
- comment = extra.comment if extra else None
- extra = Extra(comment, ifcond)
- return (obj, extra)
+class Node:
+ """
+ Node generally contains a SchemaInfo-like type (as a dict),
+ But it also used to wrap comments/ifconds around leaf value types.
+ """
+ # Remove after 3.7 adds @dataclass:
+ # pylint: disable=too-few-public-methods
+ def __init__(self, data, ifcond: List[str],
+ extra: Optional[Extra] = None):
+ self.data = data
+ comment = extra.comment if extra else None
+ self.extra = Extra(comment, ifcond)
def _tree_to_qlit(obj, level=0, suppress_first_indent=False):
@@ -43,18 +50,15 @@ def _tree_to_qlit(obj, level=0, suppress_first_indent=False):
def indent(level):
return level * 4 * ' '
- if isinstance(obj, tuple):
- ifobj, extra = obj
- ifcond = extra.ifcond
- comment = extra.comment
+ if isinstance(obj, Node):
ret = ''
- if comment:
- ret += indent(level) + '/* %s */\n' % comment
- if ifcond:
- ret += gen_if(ifcond)
- ret += _tree_to_qlit(ifobj, level)
- if ifcond:
- ret += '\n' + gen_endif(ifcond)
+ if obj.extra.comment:
+ ret += indent(level) + '/* %s */\n' % obj.extra.comment
+ if obj.extra.ifcond:
+ ret += gen_if(obj.extra.ifcond)
+ ret += _tree_to_qlit(obj.data, level)
+ if obj.extra.ifcond:
+ ret += '\n' + gen_endif(obj.extra.ifcond)
return ret
ret = ''
@@ -169,7 +173,7 @@ def _use_type(self, typ):
@classmethod
def _gen_features(cls, features: List[QAPISchemaFeature]):
- return [_make_tree(f.name, f.ifcond) for f in features]
+ return [Node(f.name, f.ifcond) for f in features]
def _gen_tree(self, name, mtype, obj, ifcond, features):
extra = None
@@ -183,7 +187,7 @@ def _gen_tree(self, name, mtype, obj, ifcond, features):
obj['meta-type'] = mtype
if features:
obj['features'] = self._gen_features(features)
- self._trees.append(_make_tree(obj, ifcond, extra))
+ self._trees.append(Node(obj, ifcond, extra))
def _gen_member(self, member):
obj = {'name': member.name, 'type': self._use_type(member.type)}
@@ -191,7 +195,7 @@ def _gen_member(self, member):
obj['default'] = None
if member.features:
obj['features'] = self._gen_features(member.features)
- return _make_tree(obj, member.ifcond)
+ return Node(obj, member.ifcond)
def _gen_variants(self, tag_name, variants):
return {'tag': tag_name,
@@ -199,15 +203,14 @@ def _gen_variants(self, tag_name, variants):
def _gen_variant(self, variant):
obj = {'case': variant.name, 'type': self._use_type(variant.type)}
- return _make_tree(obj, variant.ifcond)
+ return Node(obj, variant.ifcond)
def visit_builtin_type(self, name, info, json_type):
self._gen_tree(name, 'builtin', {'json-type': json_type}, [], None)
def visit_enum_type(self, name, info, ifcond, features, members, prefix):
self._gen_tree(name, 'enum',
- {'values': [_make_tree(m.name, m.ifcond, None)
- for m in members]},
+ {'values': [Node(m.name, m.ifcond) for m in members]},
ifcond, features)
def visit_array_type(self, name, info, ifcond, element_type):
@@ -227,9 +230,9 @@ def visit_object_type_flat(self, name, info, ifcond, features,
def visit_alternate_type(self, name, info, ifcond, features, variants):
self._gen_tree(name, 'alternate',
{'members': [
- _make_tree({'type': self._use_type(m.type)},
- m.ifcond, None)
- for m in variants.variants]},
+ Node({'type': self._use_type(m.type)}, m.ifcond)
+ for m in variants.variants
+ ]},
ifcond, features)
def visit_command(self, name, info, ifcond, features,
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 32/38] qapi/introspect.py: create a typed 'Node' data structure
2020-09-22 21:00 ` [PATCH v2 32/38] qapi/introspect.py: create a typed 'Node' data structure John Snow
@ 2020-09-23 18:41 ` Eduardo Habkost
2020-09-23 21:48 ` John Snow
2020-09-23 22:44 ` John Snow
0 siblings, 2 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 18:41 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:55PM -0400, John Snow wrote:
> Replacing the un-typed tuple, add a typed Node that we can add typed
> metadata to.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
This is the most complex patch so far, and it's very hard to
understand what it does without type annotations.
Have you consider adding type annotations to the code before
patch 30/38 (even if using `object` in some parts), so we can
make this easier to review?
In case it's useful, below is an attempt to add type annotations
to the old code. It can be applied after patch 29/38. It reuses
portions of patch 33/38.
Signed-off-by: John Snow <jsnow@redhat.com>
Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
---
scripts/qapi/introspect.py | 138 ++++++++++++++++++++++++++-----------
scripts/qapi/mypy.ini | 5 --
scripts/qapi/schema.py | 2 +-
3 files changed, 100 insertions(+), 45 deletions(-)
diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py
index b036fcf9ce7..4eaebdef58b 100644
--- a/scripts/qapi/introspect.py
+++ b/scripts/qapi/introspect.py
@@ -10,6 +10,17 @@ This work is licensed under the terms of the GNU GPL, version 2.
See the COPYING file in the top-level directory.
"""
+from typing import (
+ Dict,
+ Generic,
+ List,
+ NamedTuple,
+ Optional,
+ Sequence,
+ TypeVar,
+ Tuple
+)
+
from .common import (
c_name,
gen_endif,
@@ -17,25 +28,48 @@ from .common import (
mcgen,
)
from .gen import QAPISchemaMonolithicCVisitor
-from .schema import (QAPISchemaArrayType, QAPISchemaBuiltinType,
- QAPISchemaType)
-
-
-def _make_tree(obj, ifcond, features, extra=None):
+from .schema import (
+ QAPISchema,
+ QAPISchemaArrayType,
+ QAPISchemaBuiltinType,
+ QAPISchemaEntity,
+ QAPISchemaEnumMember,
+ QAPISchemaFeature,
+ QAPISchemaObjectType,
+ QAPISchemaObjectTypeMember,
+ QAPISchemaType,
+ QAPISchemaVariant,
+ QAPISchemaVariants,
+)
+from .source import QAPISourceInfo
+
+T = TypeVar('T')
+# this should actually be: Union[str, list, dict, bool, 'AnnotatedNode']
+# but mypy doesn't support recursive types
+TreeNode = object
+TreeDict = Dict[str, TreeNode]
+Extra = Dict[str, object]
+AnnotatedNode = Tuple[T, Extra]
+
+def _make_tree(obj: TreeDict, ifcond: List[str],
+ features: List[QAPISchemaFeature],
+ extra: Optional[Extra] = None) -> TreeNode:
if extra is None:
extra = {}
if ifcond:
extra['if'] = ifcond
if features:
- obj['features'] = [(f.name, {'if': f.ifcond}) for f in features]
+ obj['features'] = ([(f.name, {'if': f.ifcond}) for f in features])
if extra:
return (obj, extra)
return obj
-def _tree_to_qlit(obj, level=0, suppress_first_indent=False):
+def _tree_to_qlit(obj: TreeNode,
+ level: int = 0,
+ suppress_first_indent : bool = False) -> str:
- def indent(level):
+ def indent(level: int) -> str:
return level * 4 * ' '
if isinstance(obj, tuple):
@@ -85,21 +119,20 @@ def _tree_to_qlit(obj, level=0, suppress_first_indent=False):
return ret
-def to_c_string(string):
+def to_c_string(string: str) -> str:
return '"' + string.replace('\\', r'\\').replace('"', r'\"') + '"'
class QAPISchemaGenIntrospectVisitor(QAPISchemaMonolithicCVisitor):
-
- def __init__(self, prefix, unmask):
+ def __init__(self, prefix: str, unmask: bool):
super().__init__(
prefix, 'qapi-introspect',
' * QAPI/QMP schema introspection', __doc__)
self._unmask = unmask
- self._schema = None
- self._trees = []
- self._used_types = []
- self._name_map = {}
+ self._schema: Optional[QAPISchema] = None
+ self._trees: List[TreeNode] = []
+ self._used_types: List[QAPISchemaType] = []
+ self._name_map: Dict[str, str] = {}
self._genc.add(mcgen('''
#include "qemu/osdep.h"
#include "%(prefix)sqapi-introspect.h"
@@ -107,10 +140,10 @@ class QAPISchemaGenIntrospectVisitor(QAPISchemaMonolithicCVisitor):
''',
prefix=prefix))
- def visit_begin(self, schema):
+ def visit_begin(self, schema: QAPISchema) -> None:
self._schema = schema
- def visit_end(self):
+ def visit_end(self) -> None:
# visit the types that are actually used
for typ in self._used_types:
typ.visit(self)
@@ -132,18 +165,18 @@ const QLitObject %(c_name)s = %(c_string)s;
self._used_types = []
self._name_map = {}
- def visit_needed(self, entity):
+ def visit_needed(self, entity: QAPISchemaEntity) -> bool:
# Ignore types on first pass; visit_end() will pick up used types
return not isinstance(entity, QAPISchemaType)
- def _name(self, name):
+ def _name(self, name: str) -> str:
if self._unmask:
return name
if name not in self._name_map:
self._name_map[name] = '%d' % len(self._name_map)
return self._name_map[name]
- def _use_type(self, typ):
+ def _use_type(self, typ: QAPISchemaType) -> str:
# Map the various integer types to plain int
if typ.json_type() == 'int':
typ = self._schema.lookup_type('int')
@@ -162,8 +195,10 @@ const QLitObject %(c_name)s = %(c_string)s;
return '[' + self._use_type(typ.element_type) + ']'
return self._name(typ.name)
- def _gen_tree(self, name, mtype, obj, ifcond, features):
- extra = None
+ def _gen_tree(self, name: str, mtype: str, obj: TreeDict,
+ ifcond: List[str],
+ features: Optional[List[QAPISchemaFeature]]) -> None:
+ extra: Extra = None
if mtype not in ('command', 'event', 'builtin', 'array'):
if not self._unmask:
# Output a comment to make it easy to map masked names
@@ -174,44 +209,60 @@ const QLitObject %(c_name)s = %(c_string)s;
obj['meta-type'] = mtype
self._trees.append(_make_tree(obj, ifcond, features, extra))
- def _gen_member(self, member):
+ def _gen_member(self,
+ member: QAPISchemaObjectTypeMember) -> TreeNode:
obj = {'name': member.name, 'type': self._use_type(member.type)}
if member.optional:
obj['default'] = None
return _make_tree(obj, member.ifcond, member.features)
- def _gen_variants(self, tag_name, variants):
+ def _gen_variants(self, tag_name: str,
+ variants: List[QAPISchemaVariant]) -> TreeDict:
return {'tag': tag_name,
'variants': [self._gen_variant(v) for v in variants]}
- def _gen_variant(self, variant):
+ def _gen_variant(self, variant: QAPISchemaVariant) -> TreeNode:
obj = {'case': variant.name, 'type': self._use_type(variant.type)}
return _make_tree(obj, variant.ifcond, None)
- def visit_builtin_type(self, name, info, json_type):
+ def visit_builtin_type(self, name: str,
+ info: Optional[QAPISourceInfo],
+ json_type: str) -> None:
self._gen_tree(name, 'builtin', {'json-type': json_type}, [], None)
- def visit_enum_type(self, name, info, ifcond, features, members, prefix):
+ def visit_enum_type(self, name: str, info: QAPISourceInfo,
+ ifcond: List[str], features: List[QAPISchemaFeature],
+ members: List[QAPISchemaEnumMember],
+ prefix: Optional[str]) -> None:
self._gen_tree(name, 'enum',
{'values': [_make_tree(m.name, m.ifcond, None)
for m in members]},
ifcond, features)
- def visit_array_type(self, name, info, ifcond, element_type):
+ def visit_array_type(self, name: str, info: Optional[QAPISourceInfo],
+ ifcond: List[str],
+ element_type: QAPISchemaType) -> None:
element = self._use_type(element_type)
self._gen_tree('[' + element + ']', 'array', {'element-type': element},
ifcond, None)
- def visit_object_type_flat(self, name, info, ifcond, features,
- members, variants):
- obj = {'members': [self._gen_member(m) for m in members]}
+ def visit_object_type_flat(self, name: str, info: Optional[QAPISourceInfo],
+ ifcond: List[str],
+ features: List[QAPISchemaFeature],
+ members: Sequence[QAPISchemaObjectTypeMember],
+ variants: Optional[QAPISchemaVariants]) -> None:
+ obj: TreeDict = {'members': [self._gen_member(m) for m in members]}
if variants:
obj.update(self._gen_variants(variants.tag_member.name,
variants.variants))
self._gen_tree(name, 'object', obj, ifcond, features)
- def visit_alternate_type(self, name, info, ifcond, features, variants):
+ def visit_alternate_type(self, name: str, info: QAPISourceInfo,
+ ifcond: List[str],
+ features: List[QAPISchemaFeature],
+ variants: QAPISchemaVariants) -> None:
+
self._gen_tree(name, 'alternate',
{'members': [
_make_tree({'type': self._use_type(m.type)},
@@ -219,24 +270,33 @@ const QLitObject %(c_name)s = %(c_string)s;
for m in variants.variants]},
ifcond, features)
- def visit_command(self, name, info, ifcond, features,
- arg_type, ret_type, gen, success_response, boxed,
- allow_oob, allow_preconfig):
+ def visit_command(self, name: str, info: QAPISourceInfo, ifcond: List[str],
+ features: List[QAPISchemaFeature],
+ arg_type: QAPISchemaObjectType,
+ ret_type: Optional[QAPISchemaType], gen: bool,
+ success_response: bool, boxed: bool, allow_oob: bool,
+ allow_preconfig: bool) -> None:
+
arg_type = arg_type or self._schema.the_empty_object_type
ret_type = ret_type or self._schema.the_empty_object_type
- obj = {'arg-type': self._use_type(arg_type),
- 'ret-type': self._use_type(ret_type)}
+ obj: TreeDict = {
+ 'arg-type': self._use_type(arg_type),
+ 'ret-type': self._use_type(ret_type)
+ }
if allow_oob:
obj['allow-oob'] = allow_oob
self._gen_tree(name, 'command', obj, ifcond, features)
- def visit_event(self, name, info, ifcond, features, arg_type, boxed):
+ def visit_event(self, name: str, info: QAPISourceInfo,
+ ifcond: List[str], features: List[QAPISchemaFeature],
+ arg_type: QAPISchemaObjectType, boxed: bool) -> None:
arg_type = arg_type or self._schema.the_empty_object_type
self._gen_tree(name, 'event', {'arg-type': self._use_type(arg_type)},
ifcond, features)
-def gen_introspect(schema, output_dir, prefix, opt_unmask):
+def gen_introspect(schema: QAPISchema, output_dir: str, prefix: str,
+ opt_unmask: bool) -> None:
vis = QAPISchemaGenIntrospectVisitor(prefix, opt_unmask)
schema.visit(vis)
vis.write(output_dir)
diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini
index dbfeda748cc..9ce8b56f225 100644
--- a/scripts/qapi/mypy.ini
+++ b/scripts/qapi/mypy.ini
@@ -19,11 +19,6 @@ disallow_untyped_defs = False
disallow_incomplete_defs = False
check_untyped_defs = False
-[mypy-qapi.introspect]
-disallow_untyped_defs = False
-disallow_incomplete_defs = False
-check_untyped_defs = False
-
[mypy-qapi.parser]
disallow_untyped_defs = False
disallow_incomplete_defs = False
diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py
index bb0cd717f1a..3023bab44b6 100644
--- a/scripts/qapi/schema.py
+++ b/scripts/qapi/schema.py
@@ -28,7 +28,7 @@ from .parser import QAPISchemaParser
class QAPISchemaEntity:
meta: Optional[str] = None
- def __init__(self, name, info, doc, ifcond=None, features=None):
+ def __init__(self, name: str, info, doc, ifcond=None, features=None):
assert name is None or isinstance(name, str)
for f in features or []:
assert isinstance(f, QAPISchemaFeature)
--
2.26.2
--
Eduardo
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 32/38] qapi/introspect.py: create a typed 'Node' data structure
2020-09-23 18:41 ` Eduardo Habkost
@ 2020-09-23 21:48 ` John Snow
2020-09-23 22:44 ` John Snow
1 sibling, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-23 21:48 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, Michael Roth, qemu-devel, Markus Armbruster,
Cleber Rosa, Alex Bennée
On 9/23/20 2:41 PM, Eduardo Habkost wrote:
> On Tue, Sep 22, 2020 at 05:00:55PM -0400, John Snow wrote:
>> Replacing the un-typed tuple, add a typed Node that we can add typed
>> metadata to.
>>
>> Signed-off-by: John Snow <jsnow@redhat.com>
>
> This is the most complex patch so far, and it's very hard to
> understand what it does without type annotations.
>
Mmm, sorry about that.
> Have you consider adding type annotations to the code before
> patch 30/38 (even if using `object` in some parts), so we can
> make this easier to review?
>
If you feel like it isn't just noise to learn types we're about to
destroy with something else, I'll try.
> In case it's useful, below is an attempt to add type annotations
> to the old code. It can be applied after patch 29/38. It reuses
> portions of patch 33/38.
>
OK. End result is I will squash the Extra and Node types together, but I
need to figure out how to make it look nice for the review history.
Gimmie a few to make a mess and I'll try to put it back together afterward.
> Signed-off-by: John Snow <jsnow@redhat.com>
> Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
> ---
> scripts/qapi/introspect.py | 138 ++++++++++++++++++++++++++-----------
> scripts/qapi/mypy.ini | 5 --
> scripts/qapi/schema.py | 2 +-
> 3 files changed, 100 insertions(+), 45 deletions(-)
>
> diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py
> index b036fcf9ce7..4eaebdef58b 100644
> --- a/scripts/qapi/introspect.py
> +++ b/scripts/qapi/introspect.py
> @@ -10,6 +10,17 @@ This work is licensed under the terms of the GNU GPL, version 2.
> See the COPYING file in the top-level directory.
> """
>
> +from typing import (
> + Dict,
> + Generic,
> + List,
> + NamedTuple,
> + Optional,
> + Sequence,
> + TypeVar,
> + Tuple
> +)
> +
> from .common import (
> c_name,
> gen_endif,
> @@ -17,25 +28,48 @@ from .common import (
> mcgen,
> )
> from .gen import QAPISchemaMonolithicCVisitor
> -from .schema import (QAPISchemaArrayType, QAPISchemaBuiltinType,
> - QAPISchemaType)
> -
> -
> -def _make_tree(obj, ifcond, features, extra=None):
> +from .schema import (
> + QAPISchema,
> + QAPISchemaArrayType,
> + QAPISchemaBuiltinType,
> + QAPISchemaEntity,
> + QAPISchemaEnumMember,
> + QAPISchemaFeature,
> + QAPISchemaObjectType,
> + QAPISchemaObjectTypeMember,
> + QAPISchemaType,
> + QAPISchemaVariant,
> + QAPISchemaVariants,
> +)
> +from .source import QAPISourceInfo
> +
> +T = TypeVar('T')
> +# this should actually be: Union[str, list, dict, bool, 'AnnotatedNode']
> +# but mypy doesn't support recursive types
> +TreeNode = object
> +TreeDict = Dict[str, TreeNode]
> +Extra = Dict[str, object]
> +AnnotatedNode = Tuple[T, Extra]
> +
> +def _make_tree(obj: TreeDict, ifcond: List[str],
> + features: List[QAPISchemaFeature],
> + extra: Optional[Extra] = None) -> TreeNode:
> if extra is None:
> extra = {}
> if ifcond:
> extra['if'] = ifcond
> if features:
> - obj['features'] = [(f.name, {'if': f.ifcond}) for f in features]
> + obj['features'] = ([(f.name, {'if': f.ifcond}) for f in features])
> if extra:
> return (obj, extra)
> return obj
>
>
> -def _tree_to_qlit(obj, level=0, suppress_first_indent=False):
> +def _tree_to_qlit(obj: TreeNode,
> + level: int = 0,
> + suppress_first_indent : bool = False) -> str:
>
> - def indent(level):
> + def indent(level: int) -> str:
> return level * 4 * ' '
>
> if isinstance(obj, tuple):
> @@ -85,21 +119,20 @@ def _tree_to_qlit(obj, level=0, suppress_first_indent=False):
> return ret
>
>
> -def to_c_string(string):
> +def to_c_string(string: str) -> str:
> return '"' + string.replace('\\', r'\\').replace('"', r'\"') + '"'
>
>
> class QAPISchemaGenIntrospectVisitor(QAPISchemaMonolithicCVisitor):
> -
> - def __init__(self, prefix, unmask):
> + def __init__(self, prefix: str, unmask: bool):
> super().__init__(
> prefix, 'qapi-introspect',
> ' * QAPI/QMP schema introspection', __doc__)
> self._unmask = unmask
> - self._schema = None
> - self._trees = []
> - self._used_types = []
> - self._name_map = {}
> + self._schema: Optional[QAPISchema] = None
> + self._trees: List[TreeNode] = []
> + self._used_types: List[QAPISchemaType] = []
> + self._name_map: Dict[str, str] = {}
> self._genc.add(mcgen('''
> #include "qemu/osdep.h"
> #include "%(prefix)sqapi-introspect.h"
> @@ -107,10 +140,10 @@ class QAPISchemaGenIntrospectVisitor(QAPISchemaMonolithicCVisitor):
> ''',
> prefix=prefix))
>
> - def visit_begin(self, schema):
> + def visit_begin(self, schema: QAPISchema) -> None:
> self._schema = schema
>
> - def visit_end(self):
> + def visit_end(self) -> None:
> # visit the types that are actually used
> for typ in self._used_types:
> typ.visit(self)
> @@ -132,18 +165,18 @@ const QLitObject %(c_name)s = %(c_string)s;
> self._used_types = []
> self._name_map = {}
>
> - def visit_needed(self, entity):
> + def visit_needed(self, entity: QAPISchemaEntity) -> bool:
> # Ignore types on first pass; visit_end() will pick up used types
> return not isinstance(entity, QAPISchemaType)
>
> - def _name(self, name):
> + def _name(self, name: str) -> str:
> if self._unmask:
> return name
> if name not in self._name_map:
> self._name_map[name] = '%d' % len(self._name_map)
> return self._name_map[name]
>
> - def _use_type(self, typ):
> + def _use_type(self, typ: QAPISchemaType) -> str:
> # Map the various integer types to plain int
> if typ.json_type() == 'int':
> typ = self._schema.lookup_type('int')
> @@ -162,8 +195,10 @@ const QLitObject %(c_name)s = %(c_string)s;
> return '[' + self._use_type(typ.element_type) + ']'
> return self._name(typ.name)
>
> - def _gen_tree(self, name, mtype, obj, ifcond, features):
> - extra = None
> + def _gen_tree(self, name: str, mtype: str, obj: TreeDict,
> + ifcond: List[str],
> + features: Optional[List[QAPISchemaFeature]]) -> None:
> + extra: Extra = None
> if mtype not in ('command', 'event', 'builtin', 'array'):
> if not self._unmask:
> # Output a comment to make it easy to map masked names
> @@ -174,44 +209,60 @@ const QLitObject %(c_name)s = %(c_string)s;
> obj['meta-type'] = mtype
> self._trees.append(_make_tree(obj, ifcond, features, extra))
>
> - def _gen_member(self, member):
> + def _gen_member(self,
> + member: QAPISchemaObjectTypeMember) -> TreeNode:
> obj = {'name': member.name, 'type': self._use_type(member.type)}
> if member.optional:
> obj['default'] = None
> return _make_tree(obj, member.ifcond, member.features)
>
> - def _gen_variants(self, tag_name, variants):
> + def _gen_variants(self, tag_name: str,
> + variants: List[QAPISchemaVariant]) -> TreeDict:
> return {'tag': tag_name,
> 'variants': [self._gen_variant(v) for v in variants]}
>
> - def _gen_variant(self, variant):
> + def _gen_variant(self, variant: QAPISchemaVariant) -> TreeNode:
> obj = {'case': variant.name, 'type': self._use_type(variant.type)}
> return _make_tree(obj, variant.ifcond, None)
>
> - def visit_builtin_type(self, name, info, json_type):
> + def visit_builtin_type(self, name: str,
> + info: Optional[QAPISourceInfo],
> + json_type: str) -> None:
> self._gen_tree(name, 'builtin', {'json-type': json_type}, [], None)
>
> - def visit_enum_type(self, name, info, ifcond, features, members, prefix):
> + def visit_enum_type(self, name: str, info: QAPISourceInfo,
> + ifcond: List[str], features: List[QAPISchemaFeature],
> + members: List[QAPISchemaEnumMember],
> + prefix: Optional[str]) -> None:
> self._gen_tree(name, 'enum',
> {'values': [_make_tree(m.name, m.ifcond, None)
> for m in members]},
> ifcond, features)
>
> - def visit_array_type(self, name, info, ifcond, element_type):
> + def visit_array_type(self, name: str, info: Optional[QAPISourceInfo],
> + ifcond: List[str],
> + element_type: QAPISchemaType) -> None:
> element = self._use_type(element_type)
> self._gen_tree('[' + element + ']', 'array', {'element-type': element},
> ifcond, None)
>
> - def visit_object_type_flat(self, name, info, ifcond, features,
> - members, variants):
> - obj = {'members': [self._gen_member(m) for m in members]}
> + def visit_object_type_flat(self, name: str, info: Optional[QAPISourceInfo],
> + ifcond: List[str],
> + features: List[QAPISchemaFeature],
> + members: Sequence[QAPISchemaObjectTypeMember],
> + variants: Optional[QAPISchemaVariants]) -> None:
> + obj: TreeDict = {'members': [self._gen_member(m) for m in members]}
> if variants:
> obj.update(self._gen_variants(variants.tag_member.name,
> variants.variants))
>
> self._gen_tree(name, 'object', obj, ifcond, features)
>
> - def visit_alternate_type(self, name, info, ifcond, features, variants):
> + def visit_alternate_type(self, name: str, info: QAPISourceInfo,
> + ifcond: List[str],
> + features: List[QAPISchemaFeature],
> + variants: QAPISchemaVariants) -> None:
> +
> self._gen_tree(name, 'alternate',
> {'members': [
> _make_tree({'type': self._use_type(m.type)},
> @@ -219,24 +270,33 @@ const QLitObject %(c_name)s = %(c_string)s;
> for m in variants.variants]},
> ifcond, features)
>
> - def visit_command(self, name, info, ifcond, features,
> - arg_type, ret_type, gen, success_response, boxed,
> - allow_oob, allow_preconfig):
> + def visit_command(self, name: str, info: QAPISourceInfo, ifcond: List[str],
> + features: List[QAPISchemaFeature],
> + arg_type: QAPISchemaObjectType,
> + ret_type: Optional[QAPISchemaType], gen: bool,
> + success_response: bool, boxed: bool, allow_oob: bool,
> + allow_preconfig: bool) -> None:
> +
> arg_type = arg_type or self._schema.the_empty_object_type
> ret_type = ret_type or self._schema.the_empty_object_type
> - obj = {'arg-type': self._use_type(arg_type),
> - 'ret-type': self._use_type(ret_type)}
> + obj: TreeDict = {
> + 'arg-type': self._use_type(arg_type),
> + 'ret-type': self._use_type(ret_type)
> + }
> if allow_oob:
> obj['allow-oob'] = allow_oob
> self._gen_tree(name, 'command', obj, ifcond, features)
>
> - def visit_event(self, name, info, ifcond, features, arg_type, boxed):
> + def visit_event(self, name: str, info: QAPISourceInfo,
> + ifcond: List[str], features: List[QAPISchemaFeature],
> + arg_type: QAPISchemaObjectType, boxed: bool) -> None:
> arg_type = arg_type or self._schema.the_empty_object_type
> self._gen_tree(name, 'event', {'arg-type': self._use_type(arg_type)},
> ifcond, features)
>
>
> -def gen_introspect(schema, output_dir, prefix, opt_unmask):
> +def gen_introspect(schema: QAPISchema, output_dir: str, prefix: str,
> + opt_unmask: bool) -> None:
> vis = QAPISchemaGenIntrospectVisitor(prefix, opt_unmask)
> schema.visit(vis)
> vis.write(output_dir)
> diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini
> index dbfeda748cc..9ce8b56f225 100644
> --- a/scripts/qapi/mypy.ini
> +++ b/scripts/qapi/mypy.ini
> @@ -19,11 +19,6 @@ disallow_untyped_defs = False
> disallow_incomplete_defs = False
> check_untyped_defs = False
>
> -[mypy-qapi.introspect]
> -disallow_untyped_defs = False
> -disallow_incomplete_defs = False
> -check_untyped_defs = False
> -
> [mypy-qapi.parser]
> disallow_untyped_defs = False
> disallow_incomplete_defs = False
> diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py
> index bb0cd717f1a..3023bab44b6 100644
> --- a/scripts/qapi/schema.py
> +++ b/scripts/qapi/schema.py
> @@ -28,7 +28,7 @@ from .parser import QAPISchemaParser
> class QAPISchemaEntity:
> meta: Optional[str] = None
>
> - def __init__(self, name, info, doc, ifcond=None, features=None):
> + def __init__(self, name: str, info, doc, ifcond=None, features=None):
> assert name is None or isinstance(name, str)
> for f in features or []:
> assert isinstance(f, QAPISchemaFeature)
>
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 32/38] qapi/introspect.py: create a typed 'Node' data structure
2020-09-23 18:41 ` Eduardo Habkost
2020-09-23 21:48 ` John Snow
@ 2020-09-23 22:44 ` John Snow
1 sibling, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-23 22:44 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, Michael Roth, qemu-devel, Markus Armbruster,
Cleber Rosa, Alex Bennée
On 9/23/20 2:41 PM, Eduardo Habkost wrote:
> On Tue, Sep 22, 2020 at 05:00:55PM -0400, John Snow wrote:
>> Replacing the un-typed tuple, add a typed Node that we can add typed
>> metadata to.
>>
>> Signed-off-by: John Snow <jsnow@redhat.com>
>
> This is the most complex patch so far, and it's very hard to
> understand what it does without type annotations.
>
> Have you consider adding type annotations to the code before
> patch 30/38 (even if using `object` in some parts), so we can
> make this easier to review?
>
> In case it's useful, below is an attempt to add type annotations
> to the old code. It can be applied after patch 29/38. It reuses
> portions of patch 33/38.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
> ---
> scripts/qapi/introspect.py | 138 ++++++++++++++++++++++++++-----------
> scripts/qapi/mypy.ini | 5 --
> scripts/qapi/schema.py | 2 +-
> 3 files changed, 100 insertions(+), 45 deletions(-)
>
> diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py
> index b036fcf9ce7..4eaebdef58b 100644
> --- a/scripts/qapi/introspect.py
> +++ b/scripts/qapi/introspect.py
> @@ -10,6 +10,17 @@ This work is licensed under the terms of the GNU GPL, version 2.
> See the COPYING file in the top-level directory.
> """
>
> +from typing import (
> + Dict,
> + Generic,
> + List,
> + NamedTuple,
> + Optional,
> + Sequence,
> + TypeVar,
> + Tuple
> +)
> +
> from .common import (
> c_name,
> gen_endif,
> @@ -17,25 +28,48 @@ from .common import (
> mcgen,
> )
> from .gen import QAPISchemaMonolithicCVisitor
> -from .schema import (QAPISchemaArrayType, QAPISchemaBuiltinType,
> - QAPISchemaType)
> -
> -
> -def _make_tree(obj, ifcond, features, extra=None):
> +from .schema import (
> + QAPISchema,
> + QAPISchemaArrayType,
> + QAPISchemaBuiltinType,
> + QAPISchemaEntity,
> + QAPISchemaEnumMember,
> + QAPISchemaFeature,
> + QAPISchemaObjectType,
> + QAPISchemaObjectTypeMember,
> + QAPISchemaType,
> + QAPISchemaVariant,
> + QAPISchemaVariants,
> +)
> +from .source import QAPISourceInfo
> +
> +T = TypeVar('T')
> +# this should actually be: Union[str, list, dict, bool, 'AnnotatedNode']
> +# but mypy doesn't support recursive types
> +TreeNode = object
> +TreeDict = Dict[str, TreeNode]
> +Extra = Dict[str, object]
> +AnnotatedNode = Tuple[T, Extra]
> +
> +def _make_tree(obj: TreeDict, ifcond: List[str],
^^^^^^^^
Technically untrue! obj may be a SchemaInfo-like dict, or a string.
(And I will tell you why mypy didn't catch this!)
> + features: List[QAPISchemaFeature],
> + extra: Optional[Extra] = None) -> TreeNode:
> if extra is None:
> extra = {}
> if ifcond:
> extra['if'] = ifcond
> if features:
> - obj['features'] = [(f.name, {'if': f.ifcond}) for f in features]
> + obj['features'] = ([(f.name, {'if': f.ifcond}) for f in features])
> if extra:
> return (obj, extra)
> return obj
>
>
> -def _tree_to_qlit(obj, level=0, suppress_first_indent=False):
> +def _tree_to_qlit(obj: TreeNode,
> + level: int = 0,
> + suppress_first_indent : bool = False) -> str:
>
> - def indent(level):
> + def indent(level: int) -> str:
> return level * 4 * ' '
>
> if isinstance(obj, tuple):
> @@ -85,21 +119,20 @@ def _tree_to_qlit(obj, level=0, suppress_first_indent=False):
> return ret
>
>
> -def to_c_string(string):
> +def to_c_string(string: str) -> str:
> return '"' + string.replace('\\', r'\\').replace('"', r'\"') + '"'
>
>
> class QAPISchemaGenIntrospectVisitor(QAPISchemaMonolithicCVisitor):
> -
> - def __init__(self, prefix, unmask):
> + def __init__(self, prefix: str, unmask: bool):
> super().__init__(
> prefix, 'qapi-introspect',
> ' * QAPI/QMP schema introspection', __doc__)
> self._unmask = unmask
> - self._schema = None
> - self._trees = []
> - self._used_types = []
> - self._name_map = {}
> + self._schema: Optional[QAPISchema] = None
> + self._trees: List[TreeNode] = []
> + self._used_types: List[QAPISchemaType] = []
> + self._name_map: Dict[str, str] = {}
> self._genc.add(mcgen('''
> #include "qemu/osdep.h"
> #include "%(prefix)sqapi-introspect.h"
> @@ -107,10 +140,10 @@ class QAPISchemaGenIntrospectVisitor(QAPISchemaMonolithicCVisitor):
> ''',
> prefix=prefix))
>
> - def visit_begin(self, schema):
> + def visit_begin(self, schema: QAPISchema) -> None:
> self._schema = schema
>
> - def visit_end(self):
> + def visit_end(self) -> None:
> # visit the types that are actually used
> for typ in self._used_types:
> typ.visit(self)
> @@ -132,18 +165,18 @@ const QLitObject %(c_name)s = %(c_string)s;
> self._used_types = []
> self._name_map = {}
>
> - def visit_needed(self, entity):
> + def visit_needed(self, entity: QAPISchemaEntity) -> bool:
> # Ignore types on first pass; visit_end() will pick up used types
> return not isinstance(entity, QAPISchemaType)
>
> - def _name(self, name):
> + def _name(self, name: str) -> str:
> if self._unmask:
> return name
> if name not in self._name_map:
> self._name_map[name] = '%d' % len(self._name_map)
> return self._name_map[name]
>
> - def _use_type(self, typ):
> + def _use_type(self, typ: QAPISchemaType) -> str:
> # Map the various integer types to plain int
> if typ.json_type() == 'int':
> typ = self._schema.lookup_type('int')
> @@ -162,8 +195,10 @@ const QLitObject %(c_name)s = %(c_string)s;
> return '[' + self._use_type(typ.element_type) + ']'
> return self._name(typ.name)
>
> - def _gen_tree(self, name, mtype, obj, ifcond, features):
> - extra = None
> + def _gen_tree(self, name: str, mtype: str, obj: TreeDict,
> + ifcond: List[str],
> + features: Optional[List[QAPISchemaFeature]]) -> None:
> + extra: Extra = None
> if mtype not in ('command', 'event', 'builtin', 'array'):
> if not self._unmask:
> # Output a comment to make it easy to map masked names
> @@ -174,44 +209,60 @@ const QLitObject %(c_name)s = %(c_string)s;
> obj['meta-type'] = mtype
> self._trees.append(_make_tree(obj, ifcond, features, extra))
>
> - def _gen_member(self, member):
> + def _gen_member(self,
> + member: QAPISchemaObjectTypeMember) -> TreeNode:
> obj = {'name': member.name, 'type': self._use_type(member.type)}
> if member.optional:
> obj['default'] = None
> return _make_tree(obj, member.ifcond, member.features)
>
> - def _gen_variants(self, tag_name, variants):
> + def _gen_variants(self, tag_name: str,
> + variants: List[QAPISchemaVariant]) -> TreeDict:
> return {'tag': tag_name,
> 'variants': [self._gen_variant(v) for v in variants]}
>
> - def _gen_variant(self, variant):
> + def _gen_variant(self, variant: QAPISchemaVariant) -> TreeNode:
> obj = {'case': variant.name, 'type': self._use_type(variant.type)}
> return _make_tree(obj, variant.ifcond, None)
>
> - def visit_builtin_type(self, name, info, json_type):
> + def visit_builtin_type(self, name: str,
> + info: Optional[QAPISourceInfo],
> + json_type: str) -> None:
> self._gen_tree(name, 'builtin', {'json-type': json_type}, [], None)
>
> - def visit_enum_type(self, name, info, ifcond, features, members, prefix):
> + def visit_enum_type(self, name: str, info: QAPISourceInfo,
> + ifcond: List[str], features: List[QAPISchemaFeature],
> + members: List[QAPISchemaEnumMember],
> + prefix: Optional[str]) -> None:
> self._gen_tree(name, 'enum',
> {'values': [_make_tree(m.name, m.ifcond, None)
> for m in members]},
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Over here, Members is List[QAPISchemaEnumMember], which is true enough,
but we don't yet have a typing for the 'str' field. mypy fills in with
the "Any" type, which is *assumed* to fill the type we want.
Nope! It's a string. Once you add the type annotations for schema.py,
this would fail.
> ifcond, features)
>
> - def visit_array_type(self, name, info, ifcond, element_type):
> + def visit_array_type(self, name: str, info: Optional[QAPISourceInfo],
> + ifcond: List[str],
> + element_type: QAPISchemaType) -> None:
> element = self._use_type(element_type)
> self._gen_tree('[' + element + ']', 'array', {'element-type': element},
> ifcond, None)
>
> - def visit_object_type_flat(self, name, info, ifcond, features,
> - members, variants):
> - obj = {'members': [self._gen_member(m) for m in members]}
> + def visit_object_type_flat(self, name: str, info: Optional[QAPISourceInfo],
> + ifcond: List[str],
> + features: List[QAPISchemaFeature],
> + members: Sequence[QAPISchemaObjectTypeMember],
> + variants: Optional[QAPISchemaVariants]) -> None:
> + obj: TreeDict = {'members': [self._gen_member(m) for m in members]}
> if variants:
> obj.update(self._gen_variants(variants.tag_member.name,
> variants.variants))
>
> self._gen_tree(name, 'object', obj, ifcond, features)
>
> - def visit_alternate_type(self, name, info, ifcond, features, variants):
> + def visit_alternate_type(self, name: str, info: QAPISourceInfo,
> + ifcond: List[str],
> + features: List[QAPISchemaFeature],
> + variants: QAPISchemaVariants) -> None:
> +
> self._gen_tree(name, 'alternate',
> {'members': [
> _make_tree({'type': self._use_type(m.type)},
> @@ -219,24 +270,33 @@ const QLitObject %(c_name)s = %(c_string)s;
> for m in variants.variants]},
> ifcond, features)
>
> - def visit_command(self, name, info, ifcond, features,
> - arg_type, ret_type, gen, success_response, boxed,
> - allow_oob, allow_preconfig):
> + def visit_command(self, name: str, info: QAPISourceInfo, ifcond: List[str],
> + features: List[QAPISchemaFeature],
> + arg_type: QAPISchemaObjectType,
> + ret_type: Optional[QAPISchemaType], gen: bool,
> + success_response: bool, boxed: bool, allow_oob: bool,
> + allow_preconfig: bool) -> None:
> +
> arg_type = arg_type or self._schema.the_empty_object_type
> ret_type = ret_type or self._schema.the_empty_object_type
> - obj = {'arg-type': self._use_type(arg_type),
> - 'ret-type': self._use_type(ret_type)}
> + obj: TreeDict = {
> + 'arg-type': self._use_type(arg_type),
> + 'ret-type': self._use_type(ret_type)
> + }
> if allow_oob:
> obj['allow-oob'] = allow_oob
> self._gen_tree(name, 'command', obj, ifcond, features)
>
> - def visit_event(self, name, info, ifcond, features, arg_type, boxed):
> + def visit_event(self, name: str, info: QAPISourceInfo,
> + ifcond: List[str], features: List[QAPISchemaFeature],
> + arg_type: QAPISchemaObjectType, boxed: bool) -> None:
> arg_type = arg_type or self._schema.the_empty_object_type
> self._gen_tree(name, 'event', {'arg-type': self._use_type(arg_type)},
> ifcond, features)
>
>
> -def gen_introspect(schema, output_dir, prefix, opt_unmask):
> +def gen_introspect(schema: QAPISchema, output_dir: str, prefix: str,
> + opt_unmask: bool) -> None:
> vis = QAPISchemaGenIntrospectVisitor(prefix, opt_unmask)
> schema.visit(vis)
> vis.write(output_dir)
> diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini
> index dbfeda748cc..9ce8b56f225 100644
> --- a/scripts/qapi/mypy.ini
> +++ b/scripts/qapi/mypy.ini
> @@ -19,11 +19,6 @@ disallow_untyped_defs = False
> disallow_incomplete_defs = False
> check_untyped_defs = False
>
> -[mypy-qapi.introspect]
> -disallow_untyped_defs = False
> -disallow_incomplete_defs = False
> -check_untyped_defs = False
> -
> [mypy-qapi.parser]
> disallow_untyped_defs = False
> disallow_incomplete_defs = False
> diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py
> index bb0cd717f1a..3023bab44b6 100644
> --- a/scripts/qapi/schema.py
> +++ b/scripts/qapi/schema.py
> @@ -28,7 +28,7 @@ from .parser import QAPISchemaParser
> class QAPISchemaEntity:
> meta: Optional[str] = None
>
> - def __init__(self, name, info, doc, ifcond=None, features=None):
> + def __init__(self, name: str, info, doc, ifcond=None, features=None):
> assert name is None or isinstance(name, str)
> for f in features or []:
> assert isinstance(f, QAPISchemaFeature)
>
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 33/38] qapi/introspect.py: add type hint annotations
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (31 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 32/38] qapi/introspect.py: create a typed 'Node' data structure John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-22 21:00 ` [PATCH v2 34/38] qapi/types.py: " John Snow
` (5 subsequent siblings)
38 siblings, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Annotations do not change runtime behavior.
This commit *only* adds annotations.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/introspect.py | 135 ++++++++++++++++++++++++++-----------
scripts/qapi/mypy.ini | 5 --
scripts/qapi/schema.py | 2 +-
3 files changed, 96 insertions(+), 46 deletions(-)
diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py
index e0f5007ab7..7c0444170d 100644
--- a/scripts/qapi/introspect.py
+++ b/scripts/qapi/introspect.py
@@ -10,7 +10,15 @@
See the COPYING file in the top-level directory.
"""
-from typing import (NamedTuple, List, Optional, Sequence)
+from typing import (
+ Dict,
+ Generic,
+ List,
+ NamedTuple,
+ Optional,
+ Sequence,
+ TypeVar,
+)
from .common import (
c_name,
@@ -19,8 +27,24 @@
mcgen,
)
from .gen import QAPISchemaMonolithicCVisitor
-from .schema import (QAPISchemaArrayType, QAPISchemaBuiltinType,
- QAPISchemaFeature, QAPISchemaType)
+from .schema import (
+ QAPISchema,
+ QAPISchemaArrayType,
+ QAPISchemaBuiltinType,
+ QAPISchemaEntity,
+ QAPISchemaEnumMember,
+ QAPISchemaFeature,
+ QAPISchemaObjectType,
+ QAPISchemaObjectTypeMember,
+ QAPISchemaType,
+ QAPISchemaVariant,
+ QAPISchemaVariants,
+)
+from .source import QAPISourceInfo
+
+
+_NodeType = TypeVar('_NodeType')
+_DObject = Dict[str, object]
class Extra(NamedTuple):
@@ -31,23 +55,24 @@ class Extra(NamedTuple):
ifcond: Sequence[str] = tuple()
-class Node:
+class Node(Generic[_NodeType]):
"""
Node generally contains a SchemaInfo-like type (as a dict),
But it also used to wrap comments/ifconds around leaf value types.
"""
# Remove after 3.7 adds @dataclass:
# pylint: disable=too-few-public-methods
- def __init__(self, data, ifcond: List[str],
+ def __init__(self, data: _NodeType, ifcond: List[str],
extra: Optional[Extra] = None):
self.data = data
comment = extra.comment if extra else None
self.extra = Extra(comment, ifcond)
-def _tree_to_qlit(obj, level=0, suppress_first_indent=False):
+def _tree_to_qlit(obj: object, level: int = 0,
+ suppress_first_indent: bool = False) -> str:
- def indent(level):
+ def indent(level: int) -> str:
return level * 4 * ' '
if isinstance(obj, Node):
@@ -94,21 +119,20 @@ def indent(level):
return ret
-def to_c_string(string):
+def to_c_string(string: str) -> str:
return '"' + string.replace('\\', r'\\').replace('"', r'\"') + '"'
class QAPISchemaGenIntrospectVisitor(QAPISchemaMonolithicCVisitor):
-
- def __init__(self, prefix, unmask):
+ def __init__(self, prefix: str, unmask: bool):
super().__init__(
prefix, 'qapi-introspect',
' * QAPI/QMP schema introspection', __doc__)
self._unmask = unmask
- self._schema = None
- self._trees = []
- self._used_types = []
- self._name_map = {}
+ self._schema: Optional[QAPISchema] = None
+ self._trees: List[Node[_DObject]] = []
+ self._used_types: List[QAPISchemaType] = []
+ self._name_map: Dict[str, str] = {}
self._genc.add(mcgen('''
#include "qemu/osdep.h"
#include "%(prefix)sqapi-introspect.h"
@@ -116,10 +140,10 @@ def __init__(self, prefix, unmask):
''',
prefix=prefix))
- def visit_begin(self, schema):
+ def visit_begin(self, schema: QAPISchema) -> None:
self._schema = schema
- def visit_end(self):
+ def visit_end(self) -> None:
# visit the types that are actually used
for typ in self._used_types:
typ.visit(self)
@@ -141,18 +165,18 @@ def visit_end(self):
self._used_types = []
self._name_map = {}
- def visit_needed(self, entity):
+ def visit_needed(self, entity: QAPISchemaEntity) -> bool:
# Ignore types on first pass; visit_end() will pick up used types
return not isinstance(entity, QAPISchemaType)
- def _name(self, name):
+ def _name(self, name: str) -> str:
if self._unmask:
return name
if name not in self._name_map:
self._name_map[name] = '%d' % len(self._name_map)
return self._name_map[name]
- def _use_type(self, typ):
+ def _use_type(self, typ: QAPISchemaType) -> str:
# Map the various integer types to plain int
if typ.json_type() == 'int':
typ = self._schema.lookup_type('int')
@@ -172,10 +196,13 @@ def _use_type(self, typ):
return self._name(typ.name)
@classmethod
- def _gen_features(cls, features: List[QAPISchemaFeature]):
+ def _gen_features(cls,
+ features: List[QAPISchemaFeature]) -> List[Node[str]]:
return [Node(f.name, f.ifcond) for f in features]
- def _gen_tree(self, name, mtype, obj, ifcond, features):
+ def _gen_tree(self, name: str, mtype: str, obj: _DObject,
+ ifcond: List[str],
+ features: Optional[List[QAPISchemaFeature]]) -> None:
extra = None
if mtype not in ('command', 'event', 'builtin', 'array'):
if not self._unmask:
@@ -189,45 +216,65 @@ def _gen_tree(self, name, mtype, obj, ifcond, features):
obj['features'] = self._gen_features(features)
self._trees.append(Node(obj, ifcond, extra))
- def _gen_member(self, member):
- obj = {'name': member.name, 'type': self._use_type(member.type)}
+ def _gen_member(self,
+ member: QAPISchemaObjectTypeMember) -> Node[_DObject]:
+ obj: _DObject = {
+ 'name': member.name,
+ 'type': self._use_type(member.type)
+ }
if member.optional:
obj['default'] = None
if member.features:
obj['features'] = self._gen_features(member.features)
return Node(obj, member.ifcond)
- def _gen_variants(self, tag_name, variants):
+ def _gen_variants(self, tag_name: str,
+ variants: List[QAPISchemaVariant]) -> _DObject:
return {'tag': tag_name,
'variants': [self._gen_variant(v) for v in variants]}
- def _gen_variant(self, variant):
- obj = {'case': variant.name, 'type': self._use_type(variant.type)}
+ def _gen_variant(self, variant: QAPISchemaVariant) -> Node[_DObject]:
+ obj: _DObject = {
+ 'case': variant.name,
+ 'type': self._use_type(variant.type)
+ }
return Node(obj, variant.ifcond)
- def visit_builtin_type(self, name, info, json_type):
+ def visit_builtin_type(self, name: str, info: Optional[QAPISourceInfo],
+ json_type: str) -> None:
self._gen_tree(name, 'builtin', {'json-type': json_type}, [], None)
- def visit_enum_type(self, name, info, ifcond, features, members, prefix):
+ def visit_enum_type(self, name: str, info: QAPISourceInfo,
+ ifcond: List[str], features: List[QAPISchemaFeature],
+ members: List[QAPISchemaEnumMember],
+ prefix: Optional[str]) -> None:
self._gen_tree(name, 'enum',
{'values': [Node(m.name, m.ifcond) for m in members]},
ifcond, features)
- def visit_array_type(self, name, info, ifcond, element_type):
+ def visit_array_type(self, name: str, info: Optional[QAPISourceInfo],
+ ifcond: List[str],
+ element_type: QAPISchemaType) -> None:
element = self._use_type(element_type)
self._gen_tree('[' + element + ']', 'array', {'element-type': element},
ifcond, None)
- def visit_object_type_flat(self, name, info, ifcond, features,
- members, variants):
- obj = {'members': [self._gen_member(m) for m in members]}
+ def visit_object_type_flat(self, name: str, info: Optional[QAPISourceInfo],
+ ifcond: List[str],
+ features: List[QAPISchemaFeature],
+ members: Sequence[QAPISchemaObjectTypeMember],
+ variants: Optional[QAPISchemaVariants]) -> None:
+ obj: _DObject = {'members': [self._gen_member(m) for m in members]}
if variants:
obj.update(self._gen_variants(variants.tag_member.name,
variants.variants))
self._gen_tree(name, 'object', obj, ifcond, features)
- def visit_alternate_type(self, name, info, ifcond, features, variants):
+ def visit_alternate_type(self, name: str, info: QAPISourceInfo,
+ ifcond: List[str],
+ features: List[QAPISchemaFeature],
+ variants: QAPISchemaVariants) -> None:
self._gen_tree(name, 'alternate',
{'members': [
Node({'type': self._use_type(m.type)}, m.ifcond)
@@ -235,24 +282,32 @@ def visit_alternate_type(self, name, info, ifcond, features, variants):
]},
ifcond, features)
- def visit_command(self, name, info, ifcond, features,
- arg_type, ret_type, gen, success_response, boxed,
- allow_oob, allow_preconfig):
+ def visit_command(self, name: str, info: QAPISourceInfo, ifcond: List[str],
+ features: List[QAPISchemaFeature],
+ arg_type: QAPISchemaObjectType,
+ ret_type: Optional[QAPISchemaType], gen: bool,
+ success_response: bool, boxed: bool, allow_oob: bool,
+ allow_preconfig: bool) -> None:
arg_type = arg_type or self._schema.the_empty_object_type
ret_type = ret_type or self._schema.the_empty_object_type
- obj = {'arg-type': self._use_type(arg_type),
- 'ret-type': self._use_type(ret_type)}
+ obj: _DObject = {
+ 'arg-type': self._use_type(arg_type),
+ 'ret-type': self._use_type(ret_type)
+ }
if allow_oob:
obj['allow-oob'] = allow_oob
self._gen_tree(name, 'command', obj, ifcond, features)
- def visit_event(self, name, info, ifcond, features, arg_type, boxed):
+ def visit_event(self, name: str, info: QAPISourceInfo,
+ ifcond: List[str], features: List[QAPISchemaFeature],
+ arg_type: QAPISchemaObjectType, boxed: bool) -> None:
arg_type = arg_type or self._schema.the_empty_object_type
self._gen_tree(name, 'event', {'arg-type': self._use_type(arg_type)},
ifcond, features)
-def gen_introspect(schema, output_dir, prefix, opt_unmask):
+def gen_introspect(schema: QAPISchema, output_dir: str, prefix: str,
+ opt_unmask: bool) -> None:
vis = QAPISchemaGenIntrospectVisitor(prefix, opt_unmask)
schema.visit(vis)
vis.write(output_dir)
diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini
index dbfeda748c..9ce8b56f22 100644
--- a/scripts/qapi/mypy.ini
+++ b/scripts/qapi/mypy.ini
@@ -19,11 +19,6 @@ disallow_untyped_defs = False
disallow_incomplete_defs = False
check_untyped_defs = False
-[mypy-qapi.introspect]
-disallow_untyped_defs = False
-disallow_incomplete_defs = False
-check_untyped_defs = False
-
[mypy-qapi.parser]
disallow_untyped_defs = False
disallow_incomplete_defs = False
diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py
index bb0cd717f1..3023bab44b 100644
--- a/scripts/qapi/schema.py
+++ b/scripts/qapi/schema.py
@@ -28,7 +28,7 @@
class QAPISchemaEntity:
meta: Optional[str] = None
- def __init__(self, name, info, doc, ifcond=None, features=None):
+ def __init__(self, name: str, info, doc, ifcond=None, features=None):
assert name is None or isinstance(name, str)
for f in features or []:
assert isinstance(f, QAPISchemaFeature)
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* [PATCH v2 34/38] qapi/types.py: add type hint annotations
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (32 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 33/38] qapi/introspect.py: add type hint annotations John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 19:11 ` Eduardo Habkost
2020-09-24 18:50 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 35/38] qapi/types.py: remove one-letter variables John Snow
` (4 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Annotations do not change runtime behavior.
This commit *only* adds annotations.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/mypy.ini | 5 ---
scripts/qapi/types.py | 86 ++++++++++++++++++++++++++++++++-----------
2 files changed, 64 insertions(+), 27 deletions(-)
diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini
index 9ce8b56f22..3babc380a8 100644
--- a/scripts/qapi/mypy.ini
+++ b/scripts/qapi/mypy.ini
@@ -29,11 +29,6 @@ disallow_untyped_defs = False
disallow_incomplete_defs = False
check_untyped_defs = False
-[mypy-qapi.types]
-disallow_untyped_defs = False
-disallow_incomplete_defs = False
-check_untyped_defs = False
-
[mypy-qapi.visit]
disallow_untyped_defs = False
disallow_incomplete_defs = False
diff --git a/scripts/qapi/types.py b/scripts/qapi/types.py
index 53b47f9e58..5533c5a126 100644
--- a/scripts/qapi/types.py
+++ b/scripts/qapi/types.py
@@ -13,6 +13,8 @@
# See the COPYING file in the top-level directory.
"""
+from typing import Optional, List
+
from .common import (
c_enum_const,
c_name,
@@ -21,7 +23,16 @@
mcgen,
)
from .gen import QAPISchemaModularCVisitor, ifcontext
-from .schema import QAPISchemaEnumMember, QAPISchemaObjectType
+from .source import QAPISourceInfo
+from .schema import (
+ QAPISchema,
+ QAPISchemaEnumMember,
+ QAPISchemaFeature,
+ QAPISchemaObjectType,
+ QAPISchemaObjectTypeMember,
+ QAPISchemaType,
+ QAPISchemaVariants,
+)
# variants must be emitted before their container; track what has already
@@ -29,7 +40,9 @@
objects_seen = set()
-def gen_enum_lookup(name, members, prefix=None):
+def gen_enum_lookup(name: str,
+ members: List[QAPISchemaEnumMember],
+ prefix: Optional[str] = None) -> str:
ret = mcgen('''
const QEnumLookup %(c_name)s_lookup = {
@@ -54,7 +67,9 @@ def gen_enum_lookup(name, members, prefix=None):
return ret
-def gen_enum(name, members, prefix=None):
+def gen_enum(name: str,
+ members: List[QAPISchemaEnumMember],
+ prefix: Optional[str] = None) -> str:
# append automatically generated _MAX value
enum_members = members + [QAPISchemaEnumMember('_MAX', None)]
@@ -88,7 +103,7 @@ def gen_enum(name, members, prefix=None):
return ret
-def gen_fwd_object_or_array(name):
+def gen_fwd_object_or_array(name: str) -> str:
return mcgen('''
typedef struct %(c_name)s %(c_name)s;
@@ -96,7 +111,7 @@ def gen_fwd_object_or_array(name):
c_name=c_name(name))
-def gen_array(name, element_type):
+def gen_array(name: str, element_type: QAPISchemaType) -> str:
return mcgen('''
struct %(c_name)s {
@@ -107,7 +122,7 @@ def gen_array(name, element_type):
c_name=c_name(name), c_type=element_type.c_type())
-def gen_struct_members(members):
+def gen_struct_members(members: List[QAPISchemaObjectTypeMember]) -> str:
ret = ''
for memb in members:
ret += gen_if(memb.ifcond)
@@ -124,7 +139,10 @@ def gen_struct_members(members):
return ret
-def gen_object(name, ifcond, base, members, variants):
+def gen_object(name: str, ifcond: List[str],
+ base: Optional[QAPISchemaObjectType],
+ members: List[QAPISchemaObjectTypeMember],
+ variants: Optional[QAPISchemaVariants]) -> str:
if name in objects_seen:
return ''
objects_seen.add(name)
@@ -178,7 +196,7 @@ def gen_object(name, ifcond, base, members, variants):
return ret
-def gen_upcast(name, base):
+def gen_upcast(name: str, base: QAPISchemaObjectType) -> str:
# C makes const-correctness ugly. We have to cast away const to let
# this function work for both const and non-const obj.
return mcgen('''
@@ -191,7 +209,7 @@ def gen_upcast(name, base):
c_name=c_name(name), base=base.c_name())
-def gen_variants(variants):
+def gen_variants(variants: QAPISchemaVariants) -> str:
ret = mcgen('''
union { /* union tag is @%(c_name)s */
''',
@@ -215,7 +233,7 @@ def gen_variants(variants):
return ret
-def gen_type_cleanup_decl(name):
+def gen_type_cleanup_decl(name: str) -> str:
ret = mcgen('''
void qapi_free_%(c_name)s(%(c_name)s *obj);
@@ -225,7 +243,7 @@ def gen_type_cleanup_decl(name):
return ret
-def gen_type_cleanup(name):
+def gen_type_cleanup(name: str) -> str:
ret = mcgen('''
void qapi_free_%(c_name)s(%(c_name)s *obj)
@@ -247,12 +265,12 @@ def gen_type_cleanup(name):
class QAPISchemaGenTypeVisitor(QAPISchemaModularCVisitor):
- def __init__(self, prefix):
+ def __init__(self, prefix: str):
super().__init__(
prefix, 'qapi-types', ' * Schema-defined QAPI types',
' * Built-in QAPI types', __doc__)
- def _begin_system_module(self, name):
+ def _begin_system_module(self, name: None) -> None:
self._genc.preamble_add(mcgen('''
#include "qemu/osdep.h"
#include "qapi/dealloc-visitor.h"
@@ -263,7 +281,7 @@ def _begin_system_module(self, name):
#include "qapi/util.h"
'''))
- def _begin_user_module(self, name):
+ def _begin_user_module(self, name: str) -> None:
types = self._module_basename('qapi-types', name)
visit = self._module_basename('qapi-visit', name)
self._genc.preamble_add(mcgen('''
@@ -277,27 +295,43 @@ def _begin_user_module(self, name):
#include "qapi/qapi-builtin-types.h"
'''))
- def visit_begin(self, schema):
+ def visit_begin(self, schema: QAPISchema) -> None:
# gen_object() is recursive, ensure it doesn't visit the empty type
objects_seen.add(schema.the_empty_object_type.name)
- def _gen_type_cleanup(self, name):
+ def _gen_type_cleanup(self, name: str) -> None:
self._genh.add(gen_type_cleanup_decl(name))
self._genc.add(gen_type_cleanup(name))
- def visit_enum_type(self, name, info, ifcond, features, members, prefix):
+ def visit_enum_type(self,
+ name: str,
+ info: Optional[QAPISourceInfo],
+ ifcond: List[str],
+ features: List[QAPISchemaFeature],
+ members: List[QAPISchemaEnumMember],
+ prefix: Optional[str]) -> None:
with ifcontext(ifcond, self._genh, self._genc):
self._genh.preamble_add(gen_enum(name, members, prefix))
self._genc.add(gen_enum_lookup(name, members, prefix))
- def visit_array_type(self, name, info, ifcond, element_type):
+ def visit_array_type(self,
+ name: str,
+ info: Optional[QAPISourceInfo],
+ ifcond: List[str],
+ element_type: QAPISchemaType) -> None:
with ifcontext(ifcond, self._genh, self._genc):
self._genh.preamble_add(gen_fwd_object_or_array(name))
self._genh.add(gen_array(name, element_type))
self._gen_type_cleanup(name)
- def visit_object_type(self, name, info, ifcond, features,
- base, members, variants):
+ def visit_object_type(self,
+ name: str,
+ info: Optional[QAPISourceInfo],
+ ifcond: List[str],
+ features: List[QAPISchemaFeature],
+ base: Optional[QAPISchemaObjectType],
+ members: List[QAPISchemaObjectTypeMember],
+ variants: Optional[QAPISchemaVariants]) -> None:
# Nothing to do for the special empty builtin
if name == 'q_empty':
return
@@ -313,7 +347,12 @@ def visit_object_type(self, name, info, ifcond, features,
# implicit types won't be directly allocated/freed
self._gen_type_cleanup(name)
- def visit_alternate_type(self, name, info, ifcond, features, variants):
+ def visit_alternate_type(self,
+ name: str,
+ info: QAPISourceInfo,
+ ifcond: List[str],
+ features: List[QAPISchemaFeature],
+ variants: QAPISchemaVariants) -> None:
with ifcontext(ifcond, self._genh):
self._genh.preamble_add(gen_fwd_object_or_array(name))
self._genh.add(gen_object(name, ifcond, None,
@@ -322,7 +361,10 @@ def visit_alternate_type(self, name, info, ifcond, features, variants):
self._gen_type_cleanup(name)
-def gen_types(schema, output_dir, prefix, opt_builtins):
+def gen_types(schema: QAPISchema,
+ output_dir: str,
+ prefix: str,
+ opt_builtins: bool) -> None:
vis = QAPISchemaGenTypeVisitor(prefix)
schema.visit(vis)
vis.write(output_dir, opt_builtins)
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 34/38] qapi/types.py: add type hint annotations
2020-09-22 21:00 ` [PATCH v2 34/38] qapi/types.py: " John Snow
@ 2020-09-23 19:11 ` Eduardo Habkost
2020-09-24 18:50 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 19:11 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:57PM -0400, John Snow wrote:
> Annotations do not change runtime behavior.
> This commit *only* adds annotations.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 34/38] qapi/types.py: add type hint annotations
2020-09-22 21:00 ` [PATCH v2 34/38] qapi/types.py: " John Snow
2020-09-23 19:11 ` Eduardo Habkost
@ 2020-09-24 18:50 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-24 18:50 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 245 bytes --]
On Tue, Sep 22, 2020 at 05:00:57PM -0400, John Snow wrote:
> Annotations do not change runtime behavior.
> This commit *only* adds annotations.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 35/38] qapi/types.py: remove one-letter variables
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (33 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 34/38] qapi/types.py: " John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 19:14 ` Eduardo Habkost
2020-09-24 18:53 ` Cleber Rosa
2020-09-22 21:00 ` [PATCH v2 36/38] qapi/visit.py: assert tag_member contains a QAPISchemaEnumType John Snow
` (3 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
"John, if pylint told you to jump off a bridge, would you?"
Hey, if it looked like fun, I might.
Now that this file is clean, enable pylint checks on this file.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/pylintrc | 1 -
scripts/qapi/types.py | 29 +++++++++++++++--------------
2 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/scripts/qapi/pylintrc b/scripts/qapi/pylintrc
index de132d03cf..cebaf600f9 100644
--- a/scripts/qapi/pylintrc
+++ b/scripts/qapi/pylintrc
@@ -7,7 +7,6 @@ ignore-patterns=doc.py,
expr.py,
parser.py,
schema.py,
- types.py,
visit.py,
diff --git a/scripts/qapi/types.py b/scripts/qapi/types.py
index 5533c5a126..78c2a5a3e9 100644
--- a/scripts/qapi/types.py
+++ b/scripts/qapi/types.py
@@ -49,14 +49,14 @@ def gen_enum_lookup(name: str,
.array = (const char *const[]) {
''',
c_name=c_name(name))
- for m in members:
- ret += gen_if(m.ifcond)
- index = c_enum_const(name, m.name, prefix)
+ for member in members:
+ ret += gen_if(member.ifcond)
+ index = c_enum_const(name, member.name, prefix)
ret += mcgen('''
[%(index)s] = "%(name)s",
''',
- index=index, name=m.name)
- ret += gen_endif(m.ifcond)
+ index=index, name=member.name)
+ ret += gen_endif(member.ifcond)
ret += mcgen('''
},
@@ -79,13 +79,13 @@ def gen_enum(name: str,
''',
c_name=c_name(name))
- for m in enum_members:
- ret += gen_if(m.ifcond)
+ for member in enum_members:
+ ret += gen_if(member.ifcond)
ret += mcgen('''
%(c_enum)s,
''',
- c_enum=c_enum_const(name, m.name, prefix))
- ret += gen_endif(m.ifcond)
+ c_enum=c_enum_const(name, member.name, prefix))
+ ret += gen_endif(member.ifcond)
ret += mcgen('''
} %(c_name)s;
@@ -148,11 +148,12 @@ def gen_object(name: str, ifcond: List[str],
objects_seen.add(name)
ret = ''
- if variants:
- for v in variants.variants:
- if isinstance(v.type, QAPISchemaObjectType):
- ret += gen_object(v.type.name, v.type.ifcond, v.type.base,
- v.type.local_members, v.type.variants)
+ for variant in variants.variants if variants else ():
+ obj = variant.type
+ if not isinstance(obj, QAPISchemaObjectType):
+ continue
+ ret += gen_object(obj.name, obj.ifcond, obj.base,
+ obj.local_members, obj.variants)
ret += mcgen('''
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 35/38] qapi/types.py: remove one-letter variables
2020-09-22 21:00 ` [PATCH v2 35/38] qapi/types.py: remove one-letter variables John Snow
@ 2020-09-23 19:14 ` Eduardo Habkost
2020-09-23 22:11 ` John Snow
2020-09-24 18:53 ` Cleber Rosa
1 sibling, 1 reply; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 19:14 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:58PM -0400, John Snow wrote:
> "John, if pylint told you to jump off a bridge, would you?"
> Hey, if it looked like fun, I might.
>
> Now that this file is clean, enable pylint checks on this file.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
[...]
> @@ -148,11 +148,12 @@ def gen_object(name: str, ifcond: List[str],
> objects_seen.add(name)
>
> ret = ''
> - if variants:
> - for v in variants.variants:
> - if isinstance(v.type, QAPISchemaObjectType):
> - ret += gen_object(v.type.name, v.type.ifcond, v.type.base,
> - v.type.local_members, v.type.variants)
> + for variant in variants.variants if variants else ():
I'm not sure I like this weird expression, but I believe asking
for a 120-patch cleanup series to be respun because of a tiny
style issue would be counterproductive, so:
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
> + obj = variant.type
> + if not isinstance(obj, QAPISchemaObjectType):
> + continue
> + ret += gen_object(obj.name, obj.ifcond, obj.base,
> + obj.local_members, obj.variants)
>
> ret += mcgen('''
>
> --
> 2.26.2
>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread* Re: [PATCH v2 35/38] qapi/types.py: remove one-letter variables
2020-09-23 19:14 ` Eduardo Habkost
@ 2020-09-23 22:11 ` John Snow
2020-09-24 20:54 ` Eduardo Habkost
0 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-23 22:11 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On 9/23/20 3:14 PM, Eduardo Habkost wrote:
> I'm not sure I like this weird expression, but I believe asking
> for a 120-patch cleanup series to be respun because of a tiny
> style issue would be counterproductive, so:
>
> Reviewed-by: Eduardo Habkost<ehabkost@redhat.com>
I was trying to reduce the indent level to accommodate the longer names,
but python ternaries *are* pretty weird.
It'd be nice to enforce always having a variants object instead (even if
it's empty!) and then add __bool__ and __iter__ methods to
QAPISchemaVariants such that you could always do:
"if variants"
or
"for variant in variants"
but we're not there just yet... should I just put it back the way it
was, with the deep nesting?
--js
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 35/38] qapi/types.py: remove one-letter variables
2020-09-23 22:11 ` John Snow
@ 2020-09-24 20:54 ` Eduardo Habkost
0 siblings, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-24 20:54 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Wed, Sep 23, 2020 at 06:11:01PM -0400, John Snow wrote:
> On 9/23/20 3:14 PM, Eduardo Habkost wrote:
> > I'm not sure I like this weird expression, but I believe asking
> > for a 120-patch cleanup series to be respun because of a tiny
> > style issue would be counterproductive, so:
> >
> > Reviewed-by: Eduardo Habkost<ehabkost@redhat.com>
>
> I was trying to reduce the indent level to accommodate the longer names, but
> python ternaries *are* pretty weird.
>
> It'd be nice to enforce always having a variants object instead (even if
> it's empty!) and then add __bool__ and __iter__ methods to
> QAPISchemaVariants such that you could always do:
>
> "if variants"
>
> or
>
> "for variant in variants"
>
> but we're not there just yet... should I just put it back the way it was,
> with the deep nesting?
I don't have a strong opinion. I got used to it after seeing the
same pattern being used a few times.
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 35/38] qapi/types.py: remove one-letter variables
2020-09-22 21:00 ` [PATCH v2 35/38] qapi/types.py: remove one-letter variables John Snow
2020-09-23 19:14 ` Eduardo Habkost
@ 2020-09-24 18:53 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-24 18:53 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 339 bytes --]
On Tue, Sep 22, 2020 at 05:00:58PM -0400, John Snow wrote:
> "John, if pylint told you to jump off a bridge, would you?"
> Hey, if it looked like fun, I might.
>
:D
> Now that this file is clean, enable pylint checks on this file.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 36/38] qapi/visit.py: assert tag_member contains a QAPISchemaEnumType
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (34 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 35/38] qapi/types.py: remove one-letter variables John Snow
@ 2020-09-22 21:00 ` John Snow
2020-09-23 19:15 ` Eduardo Habkost
2020-09-24 19:10 ` Cleber Rosa
2020-09-22 21:01 ` [PATCH v2 37/38] qapi/visit.py: remove unused parameters from gen_visit_object John Snow
` (2 subsequent siblings)
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:00 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/visit.py | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py
index 4edaee33e3..180c140180 100644
--- a/scripts/qapi/visit.py
+++ b/scripts/qapi/visit.py
@@ -22,7 +22,10 @@
indent,
)
from .gen import QAPISchemaModularCVisitor, ifcontext
-from .schema import QAPISchemaObjectType
+from .schema import (
+ QAPISchemaEnumType,
+ QAPISchemaObjectType,
+)
def gen_visit_decl(name, scalar=False):
@@ -84,15 +87,17 @@ def gen_visit_object_members(name, base, members, variants):
ret += gen_endif(memb.ifcond)
if variants:
+ tag_member = variants.tag_member
+ assert isinstance(tag_member.type, QAPISchemaEnumType)
+
ret += mcgen('''
switch (obj->%(c_name)s) {
''',
- c_name=c_name(variants.tag_member.name))
+ c_name=c_name(tag_member.name))
for var in variants.variants:
- case_str = c_enum_const(variants.tag_member.type.name,
- var.name,
- variants.tag_member.type.prefix)
+ case_str = c_enum_const(tag_member.type.name, var.name,
+ tag_member.type.prefix)
ret += gen_if(var.ifcond)
if var.type.name == 'q_empty':
# valid variant and nothing to do
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 36/38] qapi/visit.py: assert tag_member contains a QAPISchemaEnumType
2020-09-22 21:00 ` [PATCH v2 36/38] qapi/visit.py: assert tag_member contains a QAPISchemaEnumType John Snow
@ 2020-09-23 19:15 ` Eduardo Habkost
2020-09-23 22:13 ` John Snow
2020-09-24 19:10 ` Cleber Rosa
1 sibling, 1 reply; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 19:15 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:00:59PM -0400, John Snow wrote:
> Signed-off-by: John Snow <jsnow@redhat.com>
This for making mypy happy, correct? An explanation in the commit
message would be nice.
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 36/38] qapi/visit.py: assert tag_member contains a QAPISchemaEnumType
2020-09-23 19:15 ` Eduardo Habkost
@ 2020-09-23 22:13 ` John Snow
2020-09-24 19:12 ` Cleber Rosa
0 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-23 22:13 UTC (permalink / raw)
To: Eduardo Habkost
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On 9/23/20 3:15 PM, Eduardo Habkost wrote:
> On Tue, Sep 22, 2020 at 05:00:59PM -0400, John Snow wrote:
>> Signed-off-by: John Snow <jsnow@redhat.com>
>
> This for making mypy happy, correct? An explanation in the commit
> message would be nice.
>
> Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
>
Yes, it's for mypy -- but it's a runtime visible change. Technically our
type system isn't mature enough to express this constraint natively, so
it's being carried around as developer knowledge.
This formalizes that knowledge, albeit in a very crude way.
I've amended the commit msg.
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 36/38] qapi/visit.py: assert tag_member contains a QAPISchemaEnumType
2020-09-23 22:13 ` John Snow
@ 2020-09-24 19:12 ` Cleber Rosa
0 siblings, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-24 19:12 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, Markus Armbruster, qemu-devel,
Michael Roth, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 800 bytes --]
On Wed, Sep 23, 2020 at 06:13:30PM -0400, John Snow wrote:
> On 9/23/20 3:15 PM, Eduardo Habkost wrote:
> > On Tue, Sep 22, 2020 at 05:00:59PM -0400, John Snow wrote:
> > > Signed-off-by: John Snow <jsnow@redhat.com>
> >
> > This for making mypy happy, correct? An explanation in the commit
> > message would be nice.
> >
> > Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
> >
>
> Yes, it's for mypy -- but it's a runtime visible change. Technically our
> type system isn't mature enough to express this constraint natively, so it's
> being carried around as developer knowledge.
>
> This formalizes that knowledge, albeit in a very crude way.
>
> I've amended the commit msg.
OK, this answers my previous question about why it was handled as an
assert.
- Cleber.
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 36/38] qapi/visit.py: assert tag_member contains a QAPISchemaEnumType
2020-09-22 21:00 ` [PATCH v2 36/38] qapi/visit.py: assert tag_member contains a QAPISchemaEnumType John Snow
2020-09-23 19:15 ` Eduardo Habkost
@ 2020-09-24 19:10 ` Cleber Rosa
2020-09-24 19:36 ` John Snow
1 sibling, 1 reply; 190+ messages in thread
From: Cleber Rosa @ 2020-09-24 19:10 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 1092 bytes --]
On Tue, Sep 22, 2020 at 05:00:59PM -0400, John Snow wrote:
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
> scripts/qapi/visit.py | 15 ++++++++++-----
> 1 file changed, 10 insertions(+), 5 deletions(-)
>
> diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py
> index 4edaee33e3..180c140180 100644
> --- a/scripts/qapi/visit.py
> +++ b/scripts/qapi/visit.py
> @@ -22,7 +22,10 @@
> indent,
> )
> from .gen import QAPISchemaModularCVisitor, ifcontext
> -from .schema import QAPISchemaObjectType
> +from .schema import (
> + QAPISchemaEnumType,
> + QAPISchemaObjectType,
> +)
>
>
> def gen_visit_decl(name, scalar=False):
> @@ -84,15 +87,17 @@ def gen_visit_object_members(name, base, members, variants):
> ret += gen_endif(memb.ifcond)
>
> if variants:
> + tag_member = variants.tag_member
> + assert isinstance(tag_member.type, QAPISchemaEnumType)
> +
I'd be interested in knowing why this wasn't left to be handled by the
type checking only. Anyway,
Reviewed-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 36/38] qapi/visit.py: assert tag_member contains a QAPISchemaEnumType
2020-09-24 19:10 ` Cleber Rosa
@ 2020-09-24 19:36 ` John Snow
2020-09-24 23:52 ` Cleber Rosa
0 siblings, 1 reply; 190+ messages in thread
From: John Snow @ 2020-09-24 19:36 UTC (permalink / raw)
To: Cleber Rosa
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
On 9/24/20 3:10 PM, Cleber Rosa wrote:
> On Tue, Sep 22, 2020 at 05:00:59PM -0400, John Snow wrote:
>> Signed-off-by: John Snow <jsnow@redhat.com>
>> ---
>> scripts/qapi/visit.py | 15 ++++++++++-----
>> 1 file changed, 10 insertions(+), 5 deletions(-)
>>
>> diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py
>> index 4edaee33e3..180c140180 100644
>> --- a/scripts/qapi/visit.py
>> +++ b/scripts/qapi/visit.py
>> @@ -22,7 +22,10 @@
>> indent,
>> )
>> from .gen import QAPISchemaModularCVisitor, ifcontext
>> -from .schema import QAPISchemaObjectType
>> +from .schema import (
>> + QAPISchemaEnumType,
>> + QAPISchemaObjectType,
>> +)
>>
>>
>> def gen_visit_decl(name, scalar=False):
>> @@ -84,15 +87,17 @@ def gen_visit_object_members(name, base, members, variants):
>> ret += gen_endif(memb.ifcond)
>>
>> if variants:
>> + tag_member = variants.tag_member
>> + assert isinstance(tag_member.type, QAPISchemaEnumType)
>> +
>
> I'd be interested in knowing why this wasn't left to be handled by the
> type checking only. Anyway,
>
QAPISchemaVariants is a container type that is used to house a number of
QAPISchemaVariant types; but it (can) also take a tag_member to identify
one of the fields in the variants that can be used to differentiate them.
Now, we assert that tag_member must be a QAPISchemaObjectTypeMember.
QAPISchemaVariant is also a QAPISchemaObjectTypeMember.
a QAPISchemaObjectTypeMember is a QAPISchemaMember. a QAPISchemaMember
describes one 'member' of either an enum, a features list, or an object
member.
Now, the QAPISchemaObjectTypeMember (and not the QAPISchemaMember!)
serves as a container for a QAPISchemaType -- this is a wrapper type,
effectively. That contained type can be *anything*, because object
members can be *anything*.
Oops, but if we zoom back out, we are only able to constrain tag_member
to being a QAPISchemaObjectTypeMember, we have no expressive power over
its contained type.
That's why you need the assertion here; because of a loss of specificity
when we declare tag_member.
"Wow, John, it sounds like you should use a Generic type to be able to
describe the inner type of a QAPISchemaObjectTypeMember?"
Uh, yup, you're right! I should. But it's complicated, because
QAPISchemaMember does not have a contained type. Further, the contained
type of a Member may or may not be known at construction time right now.
It's fixable, and probably involves adding something like a "string
constant" dummy type to allow QAPISchemaMember to have a contained type.
"Hey, all of that sounds very messy. What if we just added in a few
assertions for right now while we get the preliminary types in, and then
we can make adjustments based on what makes the code prettier?"
Sounds like a plan, hypothetical quote person.
> Reviewed-by: Cleber Rosa <crosa@redhat.com>
>
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 36/38] qapi/visit.py: assert tag_member contains a QAPISchemaEnumType
2020-09-24 19:36 ` John Snow
@ 2020-09-24 23:52 ` Cleber Rosa
0 siblings, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-24 23:52 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 3302 bytes --]
On Thu, Sep 24, 2020 at 03:36:23PM -0400, John Snow wrote:
> On 9/24/20 3:10 PM, Cleber Rosa wrote:
> > On Tue, Sep 22, 2020 at 05:00:59PM -0400, John Snow wrote:
> > > Signed-off-by: John Snow <jsnow@redhat.com>
> > > ---
> > > scripts/qapi/visit.py | 15 ++++++++++-----
> > > 1 file changed, 10 insertions(+), 5 deletions(-)
> > >
> > > diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py
> > > index 4edaee33e3..180c140180 100644
> > > --- a/scripts/qapi/visit.py
> > > +++ b/scripts/qapi/visit.py
> > > @@ -22,7 +22,10 @@
> > > indent,
> > > )
> > > from .gen import QAPISchemaModularCVisitor, ifcontext
> > > -from .schema import QAPISchemaObjectType
> > > +from .schema import (
> > > + QAPISchemaEnumType,
> > > + QAPISchemaObjectType,
> > > +)
> > > def gen_visit_decl(name, scalar=False):
> > > @@ -84,15 +87,17 @@ def gen_visit_object_members(name, base, members, variants):
> > > ret += gen_endif(memb.ifcond)
> > > if variants:
> > > + tag_member = variants.tag_member
> > > + assert isinstance(tag_member.type, QAPISchemaEnumType)
> > > +
> >
> > I'd be interested in knowing why this wasn't left to be handled by the
> > type checking only. Anyway,
> >
>
> QAPISchemaVariants is a container type that is used to house a number of
> QAPISchemaVariant types; but it (can) also take a tag_member to identify one
> of the fields in the variants that can be used to differentiate them.
>
> Now, we assert that tag_member must be a QAPISchemaObjectTypeMember.
> QAPISchemaVariant is also a QAPISchemaObjectTypeMember.
>
> a QAPISchemaObjectTypeMember is a QAPISchemaMember. a QAPISchemaMember
> describes one 'member' of either an enum, a features list, or an object
> member.
>
> Now, the QAPISchemaObjectTypeMember (and not the QAPISchemaMember!) serves
> as a container for a QAPISchemaType -- this is a wrapper type, effectively.
> That contained type can be *anything*, because object members can be
> *anything*.
>
> Oops, but if we zoom back out, we are only able to constrain tag_member to
> being a QAPISchemaObjectTypeMember, we have no expressive power over its
> contained type.
>
> That's why you need the assertion here; because of a loss of specificity
> when we declare tag_member.
>
>
> "Wow, John, it sounds like you should use a Generic type to be able to
> describe the inner type of a QAPISchemaObjectTypeMember?"
>
> Uh, yup, you're right! I should. But it's complicated, because
> QAPISchemaMember does not have a contained type. Further, the contained type
> of a Member may or may not be known at construction time right now.
>
> It's fixable, and probably involves adding something like a "string
> constant" dummy type to allow QAPISchemaMember to have a contained type.
>
> "Hey, all of that sounds very messy. What if we just added in a few
> assertions for right now while we get the preliminary types in, and then we
> can make adjustments based on what makes the code prettier?"
>
> Sounds like a plan, hypothetical quote person.
>
> > Reviewed-by: Cleber Rosa <crosa@redhat.com>
> >
I did not attempt to learn the type names by heart (mental sanity
first) but I get the big picture. Thanks!
- Cleber.
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 37/38] qapi/visit.py: remove unused parameters from gen_visit_object
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (35 preceding siblings ...)
2020-09-22 21:00 ` [PATCH v2 36/38] qapi/visit.py: assert tag_member contains a QAPISchemaEnumType John Snow
@ 2020-09-22 21:01 ` John Snow
2020-09-23 19:16 ` Eduardo Habkost
2020-09-24 19:13 ` Cleber Rosa
2020-09-22 21:01 ` [PATCH v2 38/38] qapi/visit.py: add type hint annotations John Snow
2020-09-24 20:37 ` [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:01 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
And this fixes the pylint report for this file, so make sure we check
this in the future, too.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/pylintrc | 1 -
scripts/qapi/visit.py | 4 ++--
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/scripts/qapi/pylintrc b/scripts/qapi/pylintrc
index cebaf600f9..581755351b 100644
--- a/scripts/qapi/pylintrc
+++ b/scripts/qapi/pylintrc
@@ -7,7 +7,6 @@ ignore-patterns=doc.py,
expr.py,
parser.py,
schema.py,
- visit.py,
[MESSAGES CONTROL]
diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py
index 180c140180..ec9d36e72f 100644
--- a/scripts/qapi/visit.py
+++ b/scripts/qapi/visit.py
@@ -253,7 +253,7 @@ def gen_visit_alternate(name, variants):
return ret
-def gen_visit_object(name, base, members, variants):
+def gen_visit_object(name):
return mcgen('''
bool visit_type_%(c_name)s
@@ -346,7 +346,7 @@ def visit_object_type(self, name, info, ifcond, features,
if not name.startswith('q_'):
# only explicit types need an allocating visit
self._genh.add(gen_visit_decl(name))
- self._genc.add(gen_visit_object(name, base, members, variants))
+ self._genc.add(gen_visit_object(name))
def visit_alternate_type(self, name, info, ifcond, features, variants):
with ifcontext(ifcond, self._genh, self._genc):
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 37/38] qapi/visit.py: remove unused parameters from gen_visit_object
2020-09-22 21:01 ` [PATCH v2 37/38] qapi/visit.py: remove unused parameters from gen_visit_object John Snow
@ 2020-09-23 19:16 ` Eduardo Habkost
2020-09-24 19:13 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 19:16 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:01:00PM -0400, John Snow wrote:
> And this fixes the pylint report for this file, so make sure we check
> this in the future, too.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 37/38] qapi/visit.py: remove unused parameters from gen_visit_object
2020-09-22 21:01 ` [PATCH v2 37/38] qapi/visit.py: remove unused parameters from gen_visit_object John Snow
2020-09-23 19:16 ` Eduardo Habkost
@ 2020-09-24 19:13 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-24 19:13 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 332 bytes --]
On Tue, Sep 22, 2020 at 05:01:00PM -0400, John Snow wrote:
> And this fixes the pylint report for this file, so make sure we check
> this in the future, too.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
LGTM, and pylint is happy.
Reviewed-by: Cleber Rosa <crosa@redhat.com>
Tested-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* [PATCH v2 38/38] qapi/visit.py: add type hint annotations
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (36 preceding siblings ...)
2020-09-22 21:01 ` [PATCH v2 37/38] qapi/visit.py: remove unused parameters from gen_visit_object John Snow
@ 2020-09-22 21:01 ` John Snow
2020-09-23 19:17 ` Eduardo Habkost
2020-09-24 19:15 ` Cleber Rosa
2020-09-24 20:37 ` [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
38 siblings, 2 replies; 190+ messages in thread
From: John Snow @ 2020-09-22 21:01 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Eduardo Habkost, Alex Bennée, Michael Roth,
Cleber Rosa, John Snow
Annotations do not change runtime behavior.
This commit *only* adds annotations.
Signed-off-by: John Snow <jsnow@redhat.com>
---
scripts/qapi/mypy.ini | 5 ----
scripts/qapi/visit.py | 68 +++++++++++++++++++++++++++++++++----------
2 files changed, 52 insertions(+), 21 deletions(-)
diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini
index 3babc380a8..5ab3433c5f 100644
--- a/scripts/qapi/mypy.ini
+++ b/scripts/qapi/mypy.ini
@@ -28,8 +28,3 @@ check_untyped_defs = False
disallow_untyped_defs = False
disallow_incomplete_defs = False
check_untyped_defs = False
-
-[mypy-qapi.visit]
-disallow_untyped_defs = False
-disallow_incomplete_defs = False
-check_untyped_defs = False
diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py
index ec9d36e72f..3bd06661c4 100644
--- a/scripts/qapi/visit.py
+++ b/scripts/qapi/visit.py
@@ -13,6 +13,8 @@
See the COPYING file in the top-level directory.
"""
+from typing import List, Optional
+
from .common import (
c_enum_const,
c_name,
@@ -23,12 +25,19 @@
)
from .gen import QAPISchemaModularCVisitor, ifcontext
from .schema import (
+ QAPISchema,
+ QAPISchemaEnumMember,
QAPISchemaEnumType,
+ QAPISchemaFeature,
QAPISchemaObjectType,
+ QAPISchemaObjectTypeMember,
+ QAPISchemaType,
+ QAPISchemaVariants,
)
+from .source import QAPISourceInfo
-def gen_visit_decl(name, scalar=False):
+def gen_visit_decl(name: str, scalar: bool = False) -> str:
c_type = c_name(name) + ' *'
if not scalar:
c_type += '*'
@@ -40,7 +49,7 @@ def gen_visit_decl(name, scalar=False):
c_name=c_name(name), c_type=c_type)
-def gen_visit_members_decl(name):
+def gen_visit_members_decl(name: str) -> str:
return mcgen('''
bool visit_type_%(c_name)s_members(Visitor *v, %(c_name)s *obj, Error **errp);
@@ -48,7 +57,10 @@ def gen_visit_members_decl(name):
c_name=c_name(name))
-def gen_visit_object_members(name, base, members, variants):
+def gen_visit_object_members(name: str,
+ base: Optional[QAPISchemaObjectType],
+ members: List[QAPISchemaObjectTypeMember],
+ variants: Optional[QAPISchemaVariants]) -> str:
ret = mcgen('''
bool visit_type_%(c_name)s_members(Visitor *v, %(c_name)s *obj, Error **errp)
@@ -128,7 +140,7 @@ def gen_visit_object_members(name, base, members, variants):
return ret
-def gen_visit_list(name, element_type):
+def gen_visit_list(name: str, element_type: QAPISchemaType) -> str:
return mcgen('''
bool visit_type_%(c_name)s(Visitor *v, const char *name,
@@ -162,7 +174,7 @@ def gen_visit_list(name, element_type):
c_name=c_name(name), c_elt_type=element_type.c_name())
-def gen_visit_enum(name):
+def gen_visit_enum(name: str) -> str:
return mcgen('''
bool visit_type_%(c_name)s(Visitor *v, const char *name,
@@ -177,7 +189,7 @@ def gen_visit_enum(name):
c_name=c_name(name))
-def gen_visit_alternate(name, variants):
+def gen_visit_alternate(name: str, variants: QAPISchemaVariants) -> str:
ret = mcgen('''
bool visit_type_%(c_name)s
@@ -253,7 +265,7 @@ def gen_visit_alternate(name, variants):
return ret
-def gen_visit_object(name):
+def gen_visit_object(name: str) -> str:
return mcgen('''
bool visit_type_%(c_name)s
@@ -288,12 +300,12 @@ def gen_visit_object(name):
class QAPISchemaGenVisitVisitor(QAPISchemaModularCVisitor):
- def __init__(self, prefix):
+ def __init__(self, prefix: str):
super().__init__(
prefix, 'qapi-visit', ' * Schema-defined QAPI visitors',
' * Built-in QAPI visitors', __doc__)
- def _begin_system_module(self, name):
+ def _begin_system_module(self, name: None) -> None:
self._genc.preamble_add(mcgen('''
#include "qemu/osdep.h"
#include "qapi/error.h"
@@ -305,7 +317,7 @@ def _begin_system_module(self, name):
'''))
- def _begin_user_module(self, name):
+ def _begin_user_module(self, name: str) -> None:
types = self._module_basename('qapi-types', name)
visit = self._module_basename('qapi-visit', name)
self._genc.preamble_add(mcgen('''
@@ -322,18 +334,34 @@ def _begin_user_module(self, name):
''',
types=types))
- def visit_enum_type(self, name, info, ifcond, features, members, prefix):
+ def visit_enum_type(self,
+ name: str,
+ info: QAPISourceInfo,
+ ifcond: List[str],
+ features: List[QAPISchemaFeature],
+ members: List[QAPISchemaEnumMember],
+ prefix: Optional[str]) -> None:
with ifcontext(ifcond, self._genh, self._genc):
self._genh.add(gen_visit_decl(name, scalar=True))
self._genc.add(gen_visit_enum(name))
- def visit_array_type(self, name, info, ifcond, element_type):
+ def visit_array_type(self,
+ name: str,
+ info: Optional[QAPISourceInfo],
+ ifcond: List[str],
+ element_type: QAPISchemaType) -> None:
with ifcontext(ifcond, self._genh, self._genc):
self._genh.add(gen_visit_decl(name))
self._genc.add(gen_visit_list(name, element_type))
- def visit_object_type(self, name, info, ifcond, features,
- base, members, variants):
+ def visit_object_type(self,
+ name: str,
+ info: Optional[QAPISourceInfo],
+ ifcond: List[str],
+ features: List[QAPISchemaFeature],
+ base: Optional[QAPISchemaObjectType],
+ members: List[QAPISchemaObjectTypeMember],
+ variants: Optional[QAPISchemaVariants]) -> None:
# Nothing to do for the special empty builtin
if name == 'q_empty':
return
@@ -348,13 +376,21 @@ def visit_object_type(self, name, info, ifcond, features,
self._genh.add(gen_visit_decl(name))
self._genc.add(gen_visit_object(name))
- def visit_alternate_type(self, name, info, ifcond, features, variants):
+ def visit_alternate_type(self,
+ name: str,
+ info: QAPISourceInfo,
+ ifcond: List[str],
+ features: List[QAPISchemaFeature],
+ variants: QAPISchemaVariants) -> None:
with ifcontext(ifcond, self._genh, self._genc):
self._genh.add(gen_visit_decl(name))
self._genc.add(gen_visit_alternate(name, variants))
-def gen_visit(schema, output_dir, prefix, opt_builtins):
+def gen_visit(schema: QAPISchema,
+ output_dir: str,
+ prefix: str,
+ opt_builtins: bool) -> None:
vis = QAPISchemaGenVisitVisitor(prefix)
schema.visit(vis)
vis.write(output_dir, opt_builtins)
--
2.26.2
^ permalink raw reply related [flat|nested] 190+ messages in thread* Re: [PATCH v2 38/38] qapi/visit.py: add type hint annotations
2020-09-22 21:01 ` [PATCH v2 38/38] qapi/visit.py: add type hint annotations John Snow
@ 2020-09-23 19:17 ` Eduardo Habkost
2020-09-24 19:15 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Eduardo Habkost @ 2020-09-23 19:17 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, qemu-devel, Michael Roth, Markus Armbruster,
Cleber Rosa, Alex Bennée
On Tue, Sep 22, 2020 at 05:01:01PM -0400, John Snow wrote:
> Annotations do not change runtime behavior.
> This commit *only* adds annotations.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eduardo Habkost <ehabkost@redhat.com>
--
Eduardo
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 38/38] qapi/visit.py: add type hint annotations
2020-09-22 21:01 ` [PATCH v2 38/38] qapi/visit.py: add type hint annotations John Snow
2020-09-23 19:17 ` Eduardo Habkost
@ 2020-09-24 19:15 ` Cleber Rosa
1 sibling, 0 replies; 190+ messages in thread
From: Cleber Rosa @ 2020-09-24 19:15 UTC (permalink / raw)
To: John Snow
Cc: Peter Maydell, Eduardo Habkost, qemu-devel, Michael Roth,
Markus Armbruster, Alex Bennée
[-- Attachment #1: Type: text/plain, Size: 288 bytes --]
On Tue, Sep 22, 2020 at 05:01:01PM -0400, John Snow wrote:
> Annotations do not change runtime behavior.
> This commit *only* adds annotations.
>
> Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Cleber Rosa <crosa@redhat.com>
Tested-by: Cleber Rosa <crosa@redhat.com>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply [flat|nested] 190+ messages in thread
* Re: [PATCH v2 00/38] qapi: static typing conversion, pt1
2020-09-22 21:00 [PATCH v2 00/38] qapi: static typing conversion, pt1 John Snow
` (37 preceding siblings ...)
2020-09-22 21:01 ` [PATCH v2 38/38] qapi/visit.py: add type hint annotations John Snow
@ 2020-09-24 20:37 ` John Snow
38 siblings, 0 replies; 190+ messages in thread
From: John Snow @ 2020-09-24 20:37 UTC (permalink / raw)
To: Markus Armbruster, qemu-devel
Cc: Peter Maydell, Cleber Rosa, Alex Bennée, Michael Roth,
Eduardo Habkost
On 9/22/20 5:00 PM, John Snow wrote:
> Hi, this series adds static type hints to the QAPI module.
> This is part one!
>
Hi, I will be sending out a v3 shortly.
The status of this series is presently as follows:
+ [01] do-not-merge-qapi-add #
+ [02] qapi-gen-separate-arg-parsing # [SOB] JS [RB] EH
+ [03] qapi-move-generator-entrypoint # [SOB] JS [RB] CR,EH [TB] CR
+ [04] qapi-prefer-explicit-relative # [SOB] JS [RB] CR,EH
+ [05] qapi-remove-wildcard-includes # [SOB] JS [RB] CR,EH
+ [06] qapi-enforce-import-order # [SOB] JS
+ [07] qapi-delint-using-flake8 # [SOB] JS [RB] CR,EH
+ [08] qapi-add-pylintrc # [SOB] JS [TB] CR,EH [RB] CR
+ [09] qapi-common-py-remove-python # [SOB] JS [RB] CR,EH
+ [10] qapi-common-add-indent-manager # [SOB] JS [RB] CR,EH
+ [11] qapi-common-py-delint-with # [SOB] JS [RB] CR,EH
+ [12] replace-c-by-char # [SOB] JS [RB] CR,EH
+ [13] qapi-common-py-check-with # [SOB] JS [RB] CR [TB] CR,EH
+ [14] qapi-common-py-add-notational # [SOB] JS [RB] CR,EH
+ [15] qapi-common-move-comments-into # [SOB] JS [RB] CR,EH
+ [16] qapi-split-build_params-into # [SOB] JS [RB] CR,EH
+ [17] qapi-establish-mypy-type # [SOB] JS [TB] EH
+ [18] qapi-events-py-add-notational # [SOB] JS [RB] CR,EH
+ [19] qapi-events-move-comments-into # [SOB] JS [RB] CR,EH
+ [20] qapi-commands-py-don-t-re-bind # [SOB] JS [RB] CR,EH
+ [21] qapi-commands-py-add # [SOB] JS [RB] CR,EH
+ [22] qapi-commands-py-enable # [SOB] JS [RB] CR,EH
+ [23] qapi-source-py-add-notational # [SOB] JS [RB] EH
+ [24] qapi-source-py-delint-with # [SOB] JS [RB] CR,EH [TB] CR
+ [25] qapi-gen-py-fix-edge-case-of #
+ [26] qapi-gen-py-add-notational # [SOB] JS [RB] CR,EH
+ [27] qapi-gen-py-enable-checking # [SOB] JS [RB] CR,EH [TB] CR
+ [28] qapi-gen-py-remove-unused # [SOB] JS [RB] CR,EH
+ [29] qapi-gen-py-update-write-to-be # [SOB] JS [RB] CR,EH
+ [30] qapi-gen-py-delint-with-pylint # [SOB] JS [RB] CR,EH
+ [31] qapi-introspect-py-assert-obj #
+ [32] qapi-introspect-py-create-a # [SOB] EH,JS
+ [33] qapi-introspect-py-add #
+ [34] qapi-introspect-py-unify #
+ [35] qapi-introspect-py-replace #
+ [36] qapi-introspect-py-create-a-0 #
+ [37] qapi-types-py-add-type-hint # [SOB] JS [RB] CR,EH
+ [38] qapi-types-py-remove-one # [SOB] JS [RB] CR,EH
+ [39] qapi-visit-py-assert # [SOB] JS [RB] CR,EH
+ [40] qapi-visit-py-remove-unused # [SOB] JS [RB] CR,EH [TB] CR
> [41] qapi-visit-py-add-notational # [SOB] JS [RB] CR,EH [TB] CR
Thank you for your help so far, everyone!
--js
^ permalink raw reply [flat|nested] 190+ messages in thread