Re: [PATCH v2 4/4] qapi: expose all schema features to code

[Markus Armbruster <armbru@redhat.com>]

Daniel P. Berrangé <berrange@redhat.com> writes:

> This replaces use of the constants from the QapiSpecialFeatures
> enum, with constants from the auto-generate QapiFeatures enum
> in qapi-features.h
>
> The 'deprecated' and 'unstable' features still have a little bit of
> special handling, being force defined to be the 1st + 2nd features
> in the enum, regardless of whether they're used in the schema. This
> retains compatibility with common code that references the features
> via the QapiSpecialFeatures constants.
>
> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com>
> ---
>  meson.build              |  1 +
>  scripts/qapi/commands.py |  1 +
>  scripts/qapi/features.py | 62 ++++++++++++++++++++++++++++++++++++++++
>  scripts/qapi/gen.py      |  4 +--
>  scripts/qapi/main.py     |  2 ++
>  scripts/qapi/schema.py   | 19 +++++++++++-
>  scripts/qapi/types.py    |  6 ++--
>  scripts/qapi/visit.py    |  3 +-
>  8 files changed, 92 insertions(+), 6 deletions(-)
>  create mode 100644 scripts/qapi/features.py
>
> diff --git a/meson.build b/meson.build
> index d26690ce20..b9d58be66f 100644
> --- a/meson.build
> +++ b/meson.build
> @@ -3332,6 +3332,7 @@ qapi_gen_depends = [ meson.current_source_dir() / 'scripts/qapi/__init__.py',
>                       meson.current_source_dir() / 'scripts/qapi/schema.py',
>                       meson.current_source_dir() / 'scripts/qapi/source.py',
>                       meson.current_source_dir() / 'scripts/qapi/types.py',
> +                     meson.current_source_dir() / 'scripts/qapi/features.py',
>                       meson.current_source_dir() / 'scripts/qapi/visit.py',
>                       meson.current_source_dir() / 'scripts/qapi-gen.py'
>  ]
> diff --git a/scripts/qapi/commands.py b/scripts/qapi/commands.py
> index d629d2d97e..bf88bfc442 100644
> --- a/scripts/qapi/commands.py
> +++ b/scripts/qapi/commands.py
> @@ -355,6 +355,7 @@ def visit_begin(self, schema: QAPISchema) -> None:
>  #include "qemu/osdep.h"
>  #include "%(prefix)sqapi-commands.h"
>  #include "%(prefix)sqapi-init-commands.h"
> +#include "%(prefix)sqapi-features.h"
>  
>  void %(c_prefix)sqmp_init_marshal(QmpCommandList *cmds)
>  {
> diff --git a/scripts/qapi/features.py b/scripts/qapi/features.py
> new file mode 100644
> index 0000000000..dc10c7cea0
> --- /dev/null
> +++ b/scripts/qapi/features.py
> @@ -0,0 +1,62 @@
> +"""
> +QAPI types generator

QAPI features generator

> +
> +Copyright 2024 Red Hat
> +
> +This work is licensed under the terms of the GNU GPL, version 2.
> +# See the COPYING file in the top-level directory.
> +"""
> +
> +from typing import List, Optional

pylint gripes

    scripts/qapi/features.py:10:0: W0611: Unused List imported from typing (unused-
    import)
    scripts/qapi/features.py:10:0: W0611: Unused Optional imported from typing (unused-import)

> +
> +from .common import c_enum_const, mcgen, c_name

    scripts/qapi/features.py:12:0: W0611: Unused mcgen imported from common (unused-import)

> +from .gen import QAPISchemaMonolithicCVisitor
> +from .schema import (
> +    QAPISchema,
> +    QAPISchemaFeature,
> +)
> +from .source import QAPISourceInfo

    scripts/qapi/features.py:18:0: W0611: Unused QAPISourceInfo imported from source (unused-import)

> +
> +
> +class QAPISchemaGenFeatureVisitor(QAPISchemaMonolithicCVisitor):
> +
> +    def __init__(self, prefix: str):
> +        super().__init__(
> +            prefix, 'qapi-features',
> +            ' * Schema-defined QAPI features',
> +            __doc__)
> +
> +        self.features = {}

mypy gripes

    scripts/qapi/features.py:29: error: Need type annotation for "features" (hint: "features: Dict[<type>, <type>] = ...")  [var-annotated]

Elsewhere, we avoid rummaging in QAPISchema's innards by defining
suitable visit.  If that's too much trouble for you now, I can take this
as is an clean up on top.

> +
> +    def visit_begin(self, schema: QAPISchema):

mypy gripes

    scripts/qapi/features.py:31: error: Function is missing a return type annotation  [no-untyped-def]

> +        self.features = schema._feature_dict

pylint gripes

    scripts/qapi/features.py:32:24: W0212: Access to a protected member _feature_dict of a client class (protected-access)

> +
> +    def visit_end(self) -> None:
> +        features = [
> +            self.features[f]
> +            for f in QAPISchemaFeature.SPECIAL_NAMES
> +        ]
> +
> +        features.extend(
> +            sorted(
> +                filter(lambda f: not f.is_special(),
> +                       self.features.values()),
> +                key=lambda f: f.name)
> +        )

@features is schema._feature_dict.values() sorted by name in a certain
way, namely first the .SPECIAL_NAMES in order, then all the others in
alphabetical order.

Why you do this is not immediately obvious.  I guess it's to make the
generated enum a compatible extension of enum QapiSpecialFeature.  That
one exists for use by schema-independent support code such
compat_policy_input_ok() and qobject_output_policy_skip().

I further guess you sort the non-special features just to make the
generated code easier for humans to navigate.

Correct?

> +
> +        self._genh.add("typedef enum {\n")
> +        for f in features:
> +            self._genh.add(f"    {c_enum_const('qapi_feature', f.name)}")
> +            if f.name in QAPISchemaFeature.SPECIAL_NAMES:
> +                self._genh.add(f" = {c_enum_const('qapi', f.name)},\n" )

pycodestyle gripes

    scripts/qapi/features.py:51:71: E202 whitespace before ')'

> +            else:
> +                self._genh.add(",\n")
> +
> +        self._genh.add("} " + c_name('QapiFeature') + ";\n")
> +

pycodestyle gripes

    scripts/qapi/features.py:57:1: E302 expected 2 blank lines, found 1

This part generates a C enum.  It's similar to gen_enum() from types.py,
except we work with a list of QAPISchemaFeature here, and a list of
QAPISchemaEnumMember there.

To reuse gen_enum() here, we'd have to make up a member list, like we do
in events.py for enum QAPIEvent.

> +def gen_features(schema: QAPISchema,
> +                 output_dir: str,
> +                 prefix: str) -> None:
> +    vis = QAPISchemaGenFeatureVisitor(prefix)
> +    schema.visit(vis)
> +    vis.write(output_dir)
> diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
> index aba1a982f6..6ef603941c 100644
> --- a/scripts/qapi/gen.py
> +++ b/scripts/qapi/gen.py
> @@ -42,8 +42,8 @@
>  
>  
>  def gen_features(features: Sequence[QAPISchemaFeature]) -> str:
> -    features = [f"1u << {c_enum_const('qapi', feat.name)}"
> -                        for feat in features if feat.is_special()]
> +    features = [f"1u << {c_enum_const('qapi_feature', feat.name)}"
> +                        for feat in features]
>      return ' | '.join(features) or '0'
>  
>  
> diff --git a/scripts/qapi/main.py b/scripts/qapi/main.py
> index 316736b6a2..2b9a2c0c02 100644
> --- a/scripts/qapi/main.py
> +++ b/scripts/qapi/main.py
> @@ -18,6 +18,7 @@
>  from .introspect import gen_introspect
>  from .schema import QAPISchema
>  from .types import gen_types
> +from .features import gen_features
>  from .visit import gen_visit
>  
>  
> @@ -49,6 +50,7 @@ def generate(schema_file: str,
>  
>      schema = QAPISchema(schema_file)
>      gen_types(schema, output_dir, prefix, builtins)
> +    gen_features(schema, output_dir, prefix)
>      gen_visit(schema, output_dir, prefix, builtins)
>      gen_commands(schema, output_dir, prefix, gen_tracing)
>      gen_events(schema, output_dir, prefix)
> diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py
> index e97c978d38..5e14b1829b 100644
> --- a/scripts/qapi/schema.py
> +++ b/scripts/qapi/schema.py
> @@ -933,8 +933,11 @@ def connect_doc(self, doc: Optional[QAPIDoc]) -> None:
>  class QAPISchemaFeature(QAPISchemaMember):
>      role = 'feature'
>  
> +    # Features which are standardized across all schemas
> +    SPECIAL_NAMES = ['deprecated', 'unstable']
> +
>      def is_special(self) -> bool:
> -        return self.name in ('deprecated', 'unstable')
> +        return self.name in QAPISchemaFeature.SPECIAL_NAMES
>  
>  
>  class QAPISchemaObjectTypeMember(QAPISchemaMember):
> @@ -1138,6 +1141,11 @@ def __init__(self, fname: str):
>          self._entity_list: List[QAPISchemaEntity] = []
>          self._entity_dict: Dict[str, QAPISchemaDefinition] = {}
>          self._module_dict: Dict[str, QAPISchemaModule] = OrderedDict()
> +        self._feature_dict: Dict[str, QAPISchemaFeature] = {}
> +
> +        for f in QAPISchemaFeature.SPECIAL_NAMES:
> +            self._feature_dict[f] = QAPISchemaFeature(f, "special feature")

mypy gripes

    scripts/qapi/schema.py:1147: error: Argument 2 to "QAPISchemaFeature" has incompatible type "str"; expected "Optional[QAPISourceInfo]"  [arg-type]

We commonly use None as info value for built-in stuff, and that's why
it's Optional[QAPISourceInfo], not just QAPISourceInfo.

But do we really need to make up some QAPISchemaFeature?  Hmm.  The
appended patch dumbs down ._feature_dict to a set.

> +
>          self._schema_dir = os.path.dirname(fname)
>          self._make_module(QAPISchemaModule.BUILTIN_MODULE_NAME)
>          self._make_module(fname)
> @@ -1258,6 +1266,15 @@ def _make_features(
>      ) -> List[QAPISchemaFeature]:
>          if features is None:
>              return []
> +
> +        for f in features:
> +            feat = QAPISchemaFeature(f['name'], info)
> +            if feat.name not in self._feature_dict:
> +                if len(self._feature_dict) == 64:
> +                    raise Exception("Maximum of 64 schema features is permitted")

The limit is an implementation restriction.  Okay, we can lift it when
it bites us.

However, the reporting is less than nice:

    $ python scripts/qapi-gen.py -o $$ tests/qapi-schema/features-too-many.json 
    Traceback (most recent call last):
      File "/work/armbru/qemu/scripts/qapi-gen.py", line 19, in <module>
        sys.exit(main.main())
                 ^^^^^^^^^^^
      File "/work/armbru/qemu/scripts/qapi/main.py", line 96, in main
        generate(args.schema,
      File "/work/armbru/qemu/scripts/qapi/main.py", line 51, in generate
        schema = QAPISchema(schema_file)
                 ^^^^^^^^^^^^^^^^^^^^^^^
      File "/work/armbru/qemu/scripts/qapi/schema.py", line 1155, in __init__
        self._def_exprs(exprs)
      File "/work/armbru/qemu/scripts/qapi/schema.py", line 1482, in _def_exprs
        self._def_struct_type(expr)
      File "/work/armbru/qemu/scripts/qapi/schema.py", line 1377, in _def_struct_type
        features = self._make_features(expr.get('features'), info)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/work/armbru/qemu/scripts/qapi/schema.py", line 1274, in _make_features
        raise Exception("Maximum of 64 schema features is permitted")
    Exception: Maximum of 64 schema features is permitted



> +
> +                self._feature_dict[feat.name] = feat
> +
>          return [QAPISchemaFeature(f['name'], info,
>                                    QAPISchemaIfCond(f.get('if')))
>                  for f in features]
> diff --git a/scripts/qapi/types.py b/scripts/qapi/types.py
> index ade6b7a3d7..7618d3eb6f 100644
> --- a/scripts/qapi/types.py
> +++ b/scripts/qapi/types.py
> @@ -308,11 +308,13 @@ def _begin_user_module(self, name: str) -> None:
>  #include "qapi/dealloc-visitor.h"
>  #include "%(types)s.h"
>  #include "%(visit)s.h"
> +#include "%(prefix)sqapi-features.h"
>  ''',
> -                                      types=types, visit=visit))
> +                                      types=types, visit=visit, prefix=self._prefix))

Long line, easy to break:

                                         types=types, visit=visit,
                                         prefix=self._prefix))

>          self._genh.preamble_add(mcgen('''
>  #include "qapi/qapi-builtin-types.h"
> -'''))
> +''',
> +                                      prefix=self._prefix))
>  
>      def visit_begin(self, schema: QAPISchema) -> None:
>          # gen_object() is recursive, ensure it doesn't visit the empty type
> diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py
> index 8dbf4ef1c3..2d678c281d 100644
> --- a/scripts/qapi/visit.py
> +++ b/scripts/qapi/visit.py
> @@ -360,8 +360,9 @@ def _begin_user_module(self, name: str) -> None:
>  #include "qemu/osdep.h"
>  #include "qapi/error.h"
>  #include "%(visit)s.h"
> +#include "%(prefix)sqapi-features.h"
>  ''',
> -                                      visit=visit))
> +                                      visit=visit, prefix=self._prefix))
>          self._genh.preamble_add(mcgen('''
>  #include "qapi/qapi-builtin-visit.h"
>  #include "%(types)s.h"


diff --git a/scripts/qapi/features.py b/scripts/qapi/features.py
index dc10c7cea0..3662c1e568 100644
--- a/scripts/qapi/features.py
+++ b/scripts/qapi/features.py
@@ -7,7 +7,7 @@
 # See the COPYING file in the top-level directory.
 """
 
-from typing import List, Optional
+from typing import List, Optional, Set
 
 from .common import c_enum_const, mcgen, c_name
 from .gen import QAPISchemaMonolithicCVisitor
@@ -26,29 +26,25 @@ def __init__(self, prefix: str):
             ' * Schema-defined QAPI features',
             __doc__)
 
-        self.features = {}
+        self.features: Set['str']
 
-    def visit_begin(self, schema: QAPISchema):
-        self.features = schema._feature_dict
+    def visit_begin(self, schema: QAPISchema) -> None:
+        self.features = schema._features
 
     def visit_end(self) -> None:
-        features = [
-            self.features[f]
-            for f in QAPISchemaFeature.SPECIAL_NAMES
-        ]
+        features = QAPISchemaFeature.SPECIAL_NAMES.copy()
 
         features.extend(
             sorted(
-                filter(lambda f: not f.is_special(),
-                       self.features.values()),
-                key=lambda f: f.name)
+                filter(lambda f: f not in QAPISchemaFeature.SPECIAL_NAMES,
+                       self.features))
         )
 
         self._genh.add("typedef enum {\n")
         for f in features:
-            self._genh.add(f"    {c_enum_const('qapi_feature', f.name)}")
-            if f.name in QAPISchemaFeature.SPECIAL_NAMES:
-                self._genh.add(f" = {c_enum_const('qapi', f.name)},\n" )
+            self._genh.add(f"    {c_enum_const('qapi_feature', f)}")
+            if f in QAPISchemaFeature.SPECIAL_NAMES:
+                self._genh.add(f" = {c_enum_const('qapi', f)},\n" )
             else:
                 self._genh.add(",\n")
 
diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py
index 6ef603941c..3fb84d36c2 100644
--- a/scripts/qapi/gen.py
+++ b/scripts/qapi/gen.py
@@ -42,9 +42,9 @@
 
 
 def gen_features(features: Sequence[QAPISchemaFeature]) -> str:
-    features = [f"1u << {c_enum_const('qapi_feature', feat.name)}"
+    feats = [f"1u << {c_enum_const('qapi_feature', feat.name)}"
                         for feat in features]
-    return ' | '.join(features) or '0'
+    return ' | '.join(feats) or '0'
 
 
 class QAPIGen:
diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py
index 5e14b1829b..dd71cbc8b7 100644
--- a/scripts/qapi/schema.py
+++ b/scripts/qapi/schema.py
@@ -28,6 +28,7 @@
     Dict,
     List,
     Optional,
+    Set,
     Union,
     cast,
 )
@@ -1141,10 +1142,7 @@ def __init__(self, fname: str):
         self._entity_list: List[QAPISchemaEntity] = []
         self._entity_dict: Dict[str, QAPISchemaDefinition] = {}
         self._module_dict: Dict[str, QAPISchemaModule] = OrderedDict()
-        self._feature_dict: Dict[str, QAPISchemaFeature] = {}
-
-        for f in QAPISchemaFeature.SPECIAL_NAMES:
-            self._feature_dict[f] = QAPISchemaFeature(f, "special feature")
+        self._features: Set[str] = set(QAPISchemaFeature.SPECIAL_NAMES)
 
         self._schema_dir = os.path.dirname(fname)
         self._make_module(QAPISchemaModule.BUILTIN_MODULE_NAME)
@@ -1269,11 +1267,11 @@ def _make_features(
 
         for f in features:
             feat = QAPISchemaFeature(f['name'], info)
-            if feat.name not in self._feature_dict:
-                if len(self._feature_dict) == 64:
+            if feat.name not in self._features:
+                if len(self._features) == 64:
                     raise Exception("Maximum of 64 schema features is permitted")
 
-                self._feature_dict[feat.name] = feat
+                self._features.add(feat.name)
 
         return [QAPISchemaFeature(f['name'], info,
                                   QAPISchemaIfCond(f.get('if')))