Re: [PULL v2 12/17] qapi: Rewrite parsing of doc comment section symbols and tags

[Igor Mammedov <imammedo@redhat.com>]

On Wed, 10 May 2023 10:12:22 +0200
Markus Armbruster <armbru@redhat.com> wrote:

> To recognize a line starting with a section symbol and or tag, we
> first split it at the first space, then examine the part left of the
> space.  We can just as well examine the unsplit line, so do that.

this makes build fail on RHEL8.9 (Python 3.6.8)
with:
 configure --target-list=x86_64-softmmu --disable-docs
 make -j32
 ...
 /usr/bin/python3 /builds/qemu/scripts/qapi-gen.py -o qapi -b ../../builds/qemu/qapi/qapi-schema.json
Traceback (most recent call last):                                               
  File "/builds/qemu/scripts/qapi-gen.py", line 16, in <module>                  
    from qapi import main                                                        
  File "/builds/qemu/scripts/qapi/main.py", line 14, in <module>                 
    from .commands import gen_commands                                           
  File "/builds/qemu/scripts/qapi/commands.py", line 24, in <module>             
    from .gen import (                                                           
  File "/builds/qemu/scripts/qapi/gen.py", line 32, in <module>                  
    from .schema import (                                                        
  File "/builds/qemu/scripts/qapi/schema.py", line 31, in <module>               
    from .expr import check_exprs                                                
  File "/builds/qemu/scripts/qapi/expr.py", line 46, in <module>                 
    from .parser import QAPIExpression                                           
  File "/builds/qemu/scripts/qapi/parser.py", line 449, in <module>              
    class QAPIDoc:                                                               
  File "/builds/qemu/scripts/qapi/parser.py", line 563, in QAPIDoc               
    def _match_at_name_colon(string: str) -> re.Match:                           
AttributeError: module 're' has no attribute 'Match'   



> Signed-off-by: Markus Armbruster <armbru@redhat.com>
> Message-Id: <20230428105429.1687850-13-armbru@redhat.com>
> Reviewed-by: Juan Quintela <quintela@redhat.com>
> [Work around lack of walrus operator in Python 3.7 and older]
> ---
>  scripts/qapi/parser.py | 55 +++++++++++++++++++++---------------------
>  1 file changed, 27 insertions(+), 28 deletions(-)
> 
> diff --git a/scripts/qapi/parser.py b/scripts/qapi/parser.py
> index ddc14ceaba..a4ff9b6dbf 100644
> --- a/scripts/qapi/parser.py
> +++ b/scripts/qapi/parser.py
> @@ -560,12 +560,12 @@ def end_comment(self) -> None:
>          self._switch_section(QAPIDoc.NullSection(self._parser))
>  
>      @staticmethod
> -    def _is_section_tag(name: str) -> bool:
> -        return name in ('Returns:', 'Since:',
> -                        # those are often singular or plural
> -                        'Note:', 'Notes:',
> -                        'Example:', 'Examples:',
> -                        'TODO:')
> +    def _match_at_name_colon(string: str) -> re.Match:
> +        return re.match(r'@([^:]*): *', string)
> +
> +    @staticmethod
> +    def _match_section_tag(string: str) -> re.Match:
> +        return re.match(r'(Returns|Since|Notes?|Examples?|TODO): *', string)
>  
>      def _append_body_line(self, line: str) -> None:
>          """
> @@ -581,7 +581,6 @@ def _append_body_line(self, line: str) -> None:
>  
>          Else, append the line to the current section.
>          """
> -        name = line.split(' ', 1)[0]
>          # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
>          # recognized, and get silently treated as ordinary text
>          if not self.symbol and not self.body.text and line.startswith('@'):
> @@ -595,12 +594,12 @@ def _append_body_line(self, line: str) -> None:
>                      self._parser, "name required after '@'")
>          elif self.symbol:
>              # This is a definition documentation block
> -            if name.startswith('@') and name.endswith(':'):
> +            if self._match_at_name_colon(line):
>                  self._append_line = self._append_args_line
>                  self._append_args_line(line)
>              elif line == 'Features:':
>                  self._append_line = self._append_features_line
> -            elif self._is_section_tag(name):
> +            elif self._match_section_tag(line):
>                  self._append_line = self._append_various_line
>                  self._append_various_line(line)
>              else:
> @@ -621,16 +620,16 @@ def _append_args_line(self, line: str) -> None:
>          Else, append the line to the current section.
>  
>          """
> -        name = line.split(' ', 1)[0]
> -
> -        if name.startswith('@') and name.endswith(':'):
> +        match = self._match_at_name_colon(line)
> +        if match:
>              # If line is "@arg:   first line of description", find
>              # the index of 'f', which is the indent we expect for any
>              # following lines.  We then remove the leading "@arg:"
>              # from line and replace it with spaces so that 'f' has the
>              # same index as it did in the original line and can be
>              # handled the same way we will handle following lines.
> -            indent = must_match(r'@\S*:\s*', line).end()
> +            name = match.group(1)
> +            indent = match.end()
>              line = line[indent:]
>              if not line:
>                  # Line was just the "@arg:" header
> @@ -638,8 +637,8 @@ def _append_args_line(self, line: str) -> None:
>                  indent = -1
>              else:
>                  line = ' ' * indent + line
> -            self._start_args_section(name[1:-1], indent)
> -        elif self._is_section_tag(name):
> +            self._start_args_section(name, indent)
> +        elif self._match_section_tag(line):
>              self._append_line = self._append_various_line
>              self._append_various_line(line)
>              return
> @@ -656,16 +655,16 @@ def _append_args_line(self, line: str) -> None:
>          self._append_freeform(line)
>  
>      def _append_features_line(self, line: str) -> None:
> -        name = line.split(' ', 1)[0]
> -
> -        if name.startswith('@') and name.endswith(':'):
> +        match = self._match_at_name_colon(line)
> +        if match:
>              # If line is "@arg:   first line of description", find
>              # the index of 'f', which is the indent we expect for any
>              # following lines.  We then remove the leading "@arg:"
>              # from line and replace it with spaces so that 'f' has the
>              # same index as it did in the original line and can be
>              # handled the same way we will handle following lines.
> -            indent = must_match(r'@\S*:\s*', line).end()
> +            name = match.group(1)
> +            indent = match.end()
>              line = line[indent:]
>              if not line:
>                  # Line was just the "@arg:" header
> @@ -673,8 +672,8 @@ def _append_features_line(self, line: str) -> None:
>                  indent = -1
>              else:
>                  line = ' ' * indent + line
> -            self._start_features_section(name[1:-1], indent)
> -        elif self._is_section_tag(name):
> +            self._start_features_section(name, indent)
> +        elif self._match_section_tag(line):
>              self._append_line = self._append_various_line
>              self._append_various_line(line)
>              return
> @@ -698,13 +697,13 @@ def _append_various_line(self, line: str) -> None:
>  
>          Else, append the line to the current section.
>          """
> -        name = line.split(' ', 1)[0]
> -
> -        if name.startswith('@') and name.endswith(':'):
> +        match = self._match_at_name_colon(line)
> +        if match:
>              raise QAPIParseError(self._parser,
> -                                 "'%s' can't follow '%s' section"
> -                                 % (name, self.sections[0].name))
> -        if self._is_section_tag(name):
> +                                 "'@%s:' can't follow '%s' section"
> +                                 % (match.group(1), self.sections[0].name))
> +        match = self._match_section_tag(line)
> +        if match:
>              # If line is "Section:   first line of description", find
>              # the index of 'f', which is the indent we expect for any
>              # following lines.  We then remove the leading "Section:"
> @@ -719,7 +718,7 @@ def _append_various_line(self, line: str) -> None:
>                  indent = 0
>              else:
>                  line = ' ' * indent + line
> -            self._start_section(name[:-1], indent)
> +            self._start_section(match.group(1), indent)
>  
>          self._append_freeform(line)
>