* [PATCH] lib/bb/build: handle incomplete message fragments in log FIFO
@ 2016-07-25 22:32 Ross Burton
2016-07-26 5:44 ` Robert Yang
0 siblings, 1 reply; 2+ messages in thread
From: Ross Burton @ 2016-07-25 22:32 UTC (permalink / raw)
To: bitbake-devel
It's possible that the logging FIFO doesn't do a complete read (or the sender a
complete write) with the result that an incomplete message is read in bitbake.
This used to result in silently truncated lines but since 42d727 now also
results in a warning as the start of the rest of the message isn't a valid
logging command.
Solve this by storing incoming bytes in a bytearray() across reads, and parsing
complete messages from that.
[ YOCTO #9999 ]
Signed-off-by: Ross Burton <ross.burton@intel.com>
---
bitbake/lib/bb/build.py | 67 ++++++++++++++++++++++++++-----------------------
1 file changed, 36 insertions(+), 31 deletions(-)
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
index a4deb00..c632a27 100644
--- a/bitbake/lib/bb/build.py
+++ b/bitbake/lib/bb/build.py
@@ -385,39 +385,44 @@ exit $ret
else:
bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress))
+ fifobuffer = bytearray()
def readfifo(data):
- lines = data.split(b'\0')
- for line in lines:
- # Just skip empty commands
- if not line:
- continue
- splitval = line.split(b' ', 1)
- cmd = splitval[0].decode("utf-8")
- if len(splitval) > 1:
- value = splitval[1].decode("utf-8")
- else:
- value = ''
- if cmd == 'bbplain':
- bb.plain(value)
- elif cmd == 'bbnote':
- bb.note(value)
- elif cmd == 'bbwarn':
- bb.warn(value)
- elif cmd == 'bberror':
- bb.error(value)
- elif cmd == 'bbfatal':
- # The caller will call exit themselves, so bb.error() is
- # what we want here rather than bb.fatal()
- bb.error(value)
- elif cmd == 'bbfatal_log':
- bb.error(value, forcelog=True)
- elif cmd == 'bbdebug':
- splitval = value.split(' ', 1)
- level = int(splitval[0])
- value = splitval[1]
- bb.debug(level, value)
+ nonlocal fifobuffer
+ fifobuffer.extend(data)
+ while fifobuffer:
+ message, token, nextmsg = fifobuffer.partition(b"\00")
+ if token:
+ splitval = message.split(b' ', 1)
+ cmd = splitval[0].decode("utf-8")
+ if len(splitval) > 1:
+ value = splitval[1].decode("utf-8")
+ else:
+ value = ''
+ if cmd == 'bbplain':
+ bb.plain(value)
+ elif cmd == 'bbnote':
+ bb.note(value)
+ elif cmd == 'bbwarn':
+ bb.warn(value)
+ elif cmd == 'bberror':
+ bb.error(value)
+ elif cmd == 'bbfatal':
+ # The caller will call exit themselves, so bb.error() is
+ # what we want here rather than bb.fatal()
+ bb.error(value)
+ elif cmd == 'bbfatal_log':
+ bb.error(value, forcelog=True)
+ elif cmd == 'bbdebug':
+ splitval = value.split(' ', 1)
+ level = int(splitval[0])
+ value = splitval[1]
+ bb.debug(level, value)
+ else:
+ bb.warn("Unrecognised command '%s' on FIFO" % cmd)
+ fifobuffer = nextmsg
else:
- bb.warn("Unrecognised command '%s' on FIFO" % cmd)
+ break
+
tempdir = d.getVar('T', True)
fifopath = os.path.join(tempdir, 'fifo.%s' % os.getpid())
if os.path.exists(fifopath):
--
2.8.1
^ permalink raw reply related [flat|nested] 2+ messages in thread* Re: [PATCH] lib/bb/build: handle incomplete message fragments in log FIFO
2016-07-25 22:32 [PATCH] lib/bb/build: handle incomplete message fragments in log FIFO Ross Burton
@ 2016-07-26 5:44 ` Robert Yang
0 siblings, 0 replies; 2+ messages in thread
From: Robert Yang @ 2016-07-26 5:44 UTC (permalink / raw)
To: Ross Burton, bitbake-devel
I tested this patch, didn't see the "Unrecognised command" any more.
// Robert
On 07/26/2016 06:32 AM, Ross Burton wrote:
> It's possible that the logging FIFO doesn't do a complete read (or the sender a
> complete write) with the result that an incomplete message is read in bitbake.
> This used to result in silently truncated lines but since 42d727 now also
> results in a warning as the start of the rest of the message isn't a valid
> logging command.
>
> Solve this by storing incoming bytes in a bytearray() across reads, and parsing
> complete messages from that.
>
> [ YOCTO #9999 ]
>
> Signed-off-by: Ross Burton <ross.burton@intel.com>
> ---
> bitbake/lib/bb/build.py | 67 ++++++++++++++++++++++++++-----------------------
> 1 file changed, 36 insertions(+), 31 deletions(-)
>
> diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
> index a4deb00..c632a27 100644
> --- a/bitbake/lib/bb/build.py
> +++ b/bitbake/lib/bb/build.py
> @@ -385,39 +385,44 @@ exit $ret
> else:
> bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress))
>
> + fifobuffer = bytearray()
> def readfifo(data):
> - lines = data.split(b'\0')
> - for line in lines:
> - # Just skip empty commands
> - if not line:
> - continue
> - splitval = line.split(b' ', 1)
> - cmd = splitval[0].decode("utf-8")
> - if len(splitval) > 1:
> - value = splitval[1].decode("utf-8")
> - else:
> - value = ''
> - if cmd == 'bbplain':
> - bb.plain(value)
> - elif cmd == 'bbnote':
> - bb.note(value)
> - elif cmd == 'bbwarn':
> - bb.warn(value)
> - elif cmd == 'bberror':
> - bb.error(value)
> - elif cmd == 'bbfatal':
> - # The caller will call exit themselves, so bb.error() is
> - # what we want here rather than bb.fatal()
> - bb.error(value)
> - elif cmd == 'bbfatal_log':
> - bb.error(value, forcelog=True)
> - elif cmd == 'bbdebug':
> - splitval = value.split(' ', 1)
> - level = int(splitval[0])
> - value = splitval[1]
> - bb.debug(level, value)
> + nonlocal fifobuffer
> + fifobuffer.extend(data)
> + while fifobuffer:
> + message, token, nextmsg = fifobuffer.partition(b"\00")
> + if token:
> + splitval = message.split(b' ', 1)
> + cmd = splitval[0].decode("utf-8")
> + if len(splitval) > 1:
> + value = splitval[1].decode("utf-8")
> + else:
> + value = ''
> + if cmd == 'bbplain':
> + bb.plain(value)
> + elif cmd == 'bbnote':
> + bb.note(value)
> + elif cmd == 'bbwarn':
> + bb.warn(value)
> + elif cmd == 'bberror':
> + bb.error(value)
> + elif cmd == 'bbfatal':
> + # The caller will call exit themselves, so bb.error() is
> + # what we want here rather than bb.fatal()
> + bb.error(value)
> + elif cmd == 'bbfatal_log':
> + bb.error(value, forcelog=True)
> + elif cmd == 'bbdebug':
> + splitval = value.split(' ', 1)
> + level = int(splitval[0])
> + value = splitval[1]
> + bb.debug(level, value)
> + else:
> + bb.warn("Unrecognised command '%s' on FIFO" % cmd)
> + fifobuffer = nextmsg
> else:
> - bb.warn("Unrecognised command '%s' on FIFO" % cmd)
> + break
> +
> tempdir = d.getVar('T', True)
> fifopath = os.path.join(tempdir, 'fifo.%s' % os.getpid())
> if os.path.exists(fifopath):
>
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2016-07-26 5:44 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2016-07-25 22:32 [PATCH] lib/bb/build: handle incomplete message fragments in log FIFO Ross Burton
2016-07-26 5:44 ` Robert Yang
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.