Re: [PATCH] lib/bb/build: handle incomplete message fragments in log FIFO

[Robert Yang <liezhi.yang@windriver.com>]

I tested this patch, didn't see the "Unrecognised command" any more.

// Robert

On 07/26/2016 06:32 AM, Ross Burton wrote:
> It's possible that the logging FIFO doesn't do a complete read (or the sender a
> complete write) with the result that an incomplete message is read in bitbake.
> This used to result in silently truncated lines but since 42d727 now also
> results in a warning as the start of the rest of the message isn't a valid
> logging command.
>
> Solve this by storing incoming bytes in a bytearray() across reads, and parsing
> complete messages from that.
>
> [ YOCTO #9999 ]
>
> Signed-off-by: Ross Burton <ross.burton@intel.com>
> ---
>   bitbake/lib/bb/build.py | 67 ++++++++++++++++++++++++++-----------------------
>   1 file changed, 36 insertions(+), 31 deletions(-)
>
> diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
> index a4deb00..c632a27 100644
> --- a/bitbake/lib/bb/build.py
> +++ b/bitbake/lib/bb/build.py
> @@ -385,39 +385,44 @@ exit $ret
>           else:
>               bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress))
>
> +    fifobuffer = bytearray()
>       def readfifo(data):
> -        lines = data.split(b'\0')
> -        for line in lines:
> -            # Just skip empty commands
> -            if not line:
> -                continue
> -            splitval = line.split(b' ', 1)
> -            cmd = splitval[0].decode("utf-8")
> -            if len(splitval) > 1:
> -                value = splitval[1].decode("utf-8")
> -            else:
> -                value = ''
> -            if cmd == 'bbplain':
> -                bb.plain(value)
> -            elif cmd == 'bbnote':
> -                bb.note(value)
> -            elif cmd == 'bbwarn':
> -                bb.warn(value)
> -            elif cmd == 'bberror':
> -                bb.error(value)
> -            elif cmd == 'bbfatal':
> -                # The caller will call exit themselves, so bb.error() is
> -                # what we want here rather than bb.fatal()
> -                bb.error(value)
> -            elif cmd == 'bbfatal_log':
> -                bb.error(value, forcelog=True)
> -            elif cmd == 'bbdebug':
> -                splitval = value.split(' ', 1)
> -                level = int(splitval[0])
> -                value = splitval[1]
> -                bb.debug(level, value)
> +        nonlocal fifobuffer
> +        fifobuffer.extend(data)
> +        while fifobuffer:
> +            message, token, nextmsg = fifobuffer.partition(b"\00")
> +            if token:
> +                splitval = message.split(b' ', 1)
> +                cmd = splitval[0].decode("utf-8")
> +                if len(splitval) > 1:
> +                    value = splitval[1].decode("utf-8")
> +                else:
> +                    value = ''
> +                if cmd == 'bbplain':
> +                    bb.plain(value)
> +                elif cmd == 'bbnote':
> +                    bb.note(value)
> +                elif cmd == 'bbwarn':
> +                    bb.warn(value)
> +                elif cmd == 'bberror':
> +                    bb.error(value)
> +                elif cmd == 'bbfatal':
> +                    # The caller will call exit themselves, so bb.error() is
> +                    # what we want here rather than bb.fatal()
> +                    bb.error(value)
> +                elif cmd == 'bbfatal_log':
> +                    bb.error(value, forcelog=True)
> +                elif cmd == 'bbdebug':
> +                    splitval = value.split(' ', 1)
> +                    level = int(splitval[0])
> +                    value = splitval[1]
> +                    bb.debug(level, value)
> +                else:
> +                    bb.warn("Unrecognised command '%s' on FIFO" % cmd)
> +                fifobuffer = nextmsg
>               else:
> -                bb.warn("Unrecognised command '%s' on FIFO" % cmd)
> +                break
> +
>       tempdir = d.getVar('T', True)
>       fifopath = os.path.join(tempdir, 'fifo.%s' % os.getpid())
>       if os.path.exists(fifopath):
>