Re: BBHandler/ConfHandler: Improve multiline comment handling

[Martin Jansa <martin.jansa@gmail.com>]

[-- Attachment #1: Type: text/plain, Size: 10252 bytes --]

On Fri, Dec 14, 2012 at 01:53:32PM +0000, Richard Purdie wrote:
> Faced with an expression like:
> 
>  # Some comment \
>  FOO = "bar"
> 
> what should bitbake do? Technically, the \ character means its multiline and
> currently the code treats this as a continuation of the comment. This can
> surprise some people and is not intuitive.
> 
> This patch makes bitbake simply error and asks the user to be clearer
> about what they mean.

I have at least 2 bb files with '# foo \', but error message is not nice:

$ bitbake -k azy-native azy
Pseudo may be out of date, rebuilding pseudo before the main build
ERROR: Unable to parse /OE/shr-core/meta-jama/recipes-jama/packagegroups/packagegroup-jama.bb                                                 | ETA:  00:00:58
Traceback (most recent call last):
  File "/OE/shr-core/bitbake/lib/bb/cache.py", line 674, in load_bbfile(bbfile='/OE/shr-core/meta-jama/recipes-jama/packagegroups/packagegroup-jama.bb', appends=[], config=<bb.data_smart.DataSmart object at 0x7fe632927d50>):
                     data.setVar('__BBAPPEND', " ".join(appends), bb_data)
    >            bb_data = parse.handle(bbfile, bb_data)
                 if chdir_back:
  File "/OE/shr-core/bitbake/lib/bb/parse/__init__.py", line 90, in handle(fn='/OE/shr-core/meta-jama/recipes-jama/packagegroups/packagegroup-jama.bb', data=<bb.data_smart.DataSmart object at 0xe287150>, include=0):
             if h['supports'](fn, data):
    >            return h['handle'](fn, data, include)
         raise ParseError("not a BitBake file", fn)
  File "/OE/shr-core/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 150, in handle(fn='/OE/shr-core/meta-jama/recipes-jama/packagegroups/packagegroup-jama.bb', d=<bb.data_smart.DataSmart object at 0xe287150>, include=0):
         # actual loading
    >    statements = get_statements(fn, abs_fn, base_name)

  File "/OE/shr-core/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 106, in get_statements(filename='/OE/shr-core/meta-jama/recipes-jama/packagegroups/packagegroup-jama.bb', absolute_filename='/OE/shr-core/meta-jama/recipes-jama/packagegroups/packagegroup-jama.bb', base_name='packagegroup-jama.bb'):
                 s = s.rstrip()
    >            feeder(lineno, s, filename, base_name, statements)
             if __inpython__:
  File "/OE/shr-core/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 205, in feeder(lineno=58, s='', fn='/OE/shr-core/meta-jama/recipes-jama/packagegroups/packagegroup-jama.bb', root='packagegroup-jama.bb', statements=[<bb.parse.ast.DataNode object at 0xd4c4850>, <bb.parse.ast.DataNode object at 0x31f2b10>, <bb.parse.ast.DataNode object at 0x9421a90>, <bb.parse.ast.InheritNode object at 0xc99f250>, <bb.parse.ast.DataNode object at 0xc99f410>, <bb.parse.ast.DataNode object at 0xc99f390>, <bb.parse.ast.DataNode object at 0xc99f790>, <bb.parse.ast.DataNode object at 0xc99f890>, <bb.parse.ast.DataNode object at 0xc99f6d0>, <bb.parse.ast.DataNode object at 0xc99f8d0>, <bb.parse.ast.DataNode object at 0xc99f750>]):

    >    if len(__residue__) != 0 and __residue__[0][0] == "#" and s[0] != "#":
             bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
IndexError: string index out of range

ERROR: There is a confusing multiline, partially commented expression on line 25 of file /OE/shr-core/meta-handheld/recipes-kernel/linux/linux-handhelds-2.6_2.6.21-hh20.bb (SRC_URI[rppatch35.md5sum] = "8ab51e8ff728f4155db64b9bb6ea6d71").
Please clarify whether this is all a comment or should be parsed.
ERROR: Command execution failed: Exited with 1

Summary: There were 3 ERROR messages shown, returning a non-zero exit code.

After removing that line from linux-handhelds-2.6 and fixing such lines in packagegroup-jama.bb, new recipe also with # foo \ is shown:

$ bitbake -k azy-native azy
Pseudo may be out of date, rebuilding pseudo before the main build
Loading cache: 100% |#########################################################################################################################| ETA:  00:00:00
Loaded 35 entries from dependency cache.
ERROR: Unable to parse /OE/shr-core/meta-jama/recipes-jama/packagegroups/packagegroup-jama-shr.bb                                             | ETA:  --:--:--
Traceback (most recent call last):
  File "/OE/shr-core/bitbake/lib/bb/cache.py", line 674, in load_bbfile(bbfile='/OE/shr-core/meta-jama/recipes-jama/packagegroups/packagegroup-jama-shr.bb', appends=[], config=<bb.data_smart.DataSmart object at 0x7f7cca917d50>):
                     data.setVar('__BBAPPEND', " ".join(appends), bb_data)
    >            bb_data = parse.handle(bbfile, bb_data)
                 if chdir_back:
  File "/OE/shr-core/bitbake/lib/bb/parse/__init__.py", line 90, in handle(fn='/OE/shr-core/meta-jama/recipes-jama/packagegroups/packagegroup-jama-shr.bb', data=<bb.data_smart.DataSmart object at 0x7f7cca8647d0>, include=0):
             if h['supports'](fn, data):
    >            return h['handle'](fn, data, include)
         raise ParseError("not a BitBake file", fn)
  File "/OE/shr-core/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 150, in handle(fn='/OE/shr-core/meta-jama/recipes-jama/packagegroups/packagegroup-jama-shr.bb', d=<bb.data_smart.DataSmart object at 0x7f7cca8647d0>, include=0):
         # actual loading
    >    statements = get_statements(fn, abs_fn, base_name)
     
  File "/OE/shr-core/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 106, in get_statements(filename='/OE/shr-core/meta-jama/recipes-jama/packagegroups/packagegroup-jama-shr.bb', absolute_filename='/OE/shr-core/meta-jama/recipes-jama/packagegroups/packagegroup-jama-shr.bb', base_name='packagegroup-jama-shr.bb'):
                 s = s.rstrip()
    >            feeder(lineno, s, filename, base_name, statements)
             if __inpython__:
  File "/OE/shr-core/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 205, in feeder(lineno=30, s='', fn='/OE/shr-core/meta-jama/recipes-jama/packagegroups/packagegroup-jama-shr.bb', root='packagegroup-jama-shr.bb', statements=[<bb.parse.ast.DataNode object at 0xc0b8910>, <bb.parse.ast.DataNode object at 0xc0b8d10>, <bb.parse.ast.DataNode object at 0xc0b8890>, <bb.parse.ast.InheritNode object at 0xc0b8a50>, <bb.parse.ast.DataNode object at 0xc0b8fd0>, <bb.parse.ast.DataNode object at 0xc0b8810>, <bb.parse.ast.DataNode object at 0xc0b8790>, <bb.parse.ast.DataNode object at 0xc0b8710>, <bb.parse.ast.DataNode object at 0xc0b8610>, <bb.parse.ast.DataNode object at 0xc0b8690>, <bb.parse.ast.DataNode object at 0xc0b8510>]):
     
    >    if len(__residue__) != 0 and __residue__[0][0] == "#" and s[0] != "#":
             bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
IndexError: string index out of range

ERROR: Command execution failed: Exited with 1

Summary: There were 2 ERROR messages shown, returning a non-zero exit code.

And then the same for meta-openembedded/meta-oe/recipes-support/cpufrequtils/cpufrequtils_008.bb ....

Cheers,

> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
> diff --git a/bitbake/lib/bb/parse/parse_py/BBHandler.py b/bitbake/lib/bb/parse/parse_py/BBHandler.py
> index 2ee8ebd..58049bd 100644
> --- a/bitbake/lib/bb/parse/parse_py/BBHandler.py
> +++ b/bitbake/lib/bb/parse/parse_py/BBHandler.py
> @@ -200,7 +200,10 @@ def feeder(lineno, s, fn, root, statements):
>  
>      if s and s[0] == '#':
>          if len(__residue__) != 0 and __residue__[0][0] != "#":
> -            bb.error("There is a comment on line %s of file %s (%s) which is in the middle of a multiline expression.\nBitbake used to ignore these but no longer does so, please fix your metadata as errors are likely as a result of this change." % (lineno, fn, s))
> +            bb.fatal("There is a comment on line %s of file %s (%s) which is in the middle of a multiline expression.\nBitbake used to ignore these but no longer does so, please fix your metadata as errors are likely as a result of this change." % (lineno, fn, s))
> +
> +    if len(__residue__) != 0 and __residue__[0][0] == "#" and s[0] != "#":
> +        bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
>  
>      if s and s[-1] == '\\':
>          __residue__.append(s[:-1])
> diff --git a/bitbake/lib/bb/parse/parse_py/ConfHandler.py b/bitbake/lib/bb/parse/parse_py/ConfHandler.py
> index dbc6776..9b09c9f 100644
> --- a/bitbake/lib/bb/parse/parse_py/ConfHandler.py
> +++ b/bitbake/lib/bb/parse/parse_py/ConfHandler.py
> @@ -98,15 +98,22 @@ def handle(fn, data, include):
>      while True:
>          lineno = lineno + 1
>          s = f.readline()
> -        if not s: break
> +        if not s:
> +            break
>          w = s.strip()
> -        if not w: continue          # skip empty lines
> +        # skip empty lines
> +        if not w:
> +            continue
>          s = s.rstrip()
> -        if s[0] == '#': continue    # skip comments
>          while s[-1] == '\\':
>              s2 = f.readline().strip()
>              lineno = lineno + 1
> +            if s2 and s[0] == "#" and s2[0] != "#":
> +                bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
>              s = s[:-1] + s2
> +        # skip comments
> +        if s[0] == '#':
> +            continue
>          feeder(lineno, s, fn, statements)
>  
>      # DONE WITH PARSING... time to evaluate
> 
> 
> 
> _______________________________________________
> bitbake-devel mailing list
> bitbake-devel@lists.openembedded.org
> http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/bitbake-devel

-- 
Martin 'JaMa' Jansa     jabber: Martin.Jansa@gmail.com

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 205 bytes --]