* [PATCH 1/5] cooker: move EventLogWriteHandler to the top module level
2016-06-20 11:00 [PATCH 0/5] Fix writing events to file Ed Bartosh
@ 2016-06-20 11:00 ` Ed Bartosh
2016-06-20 11:00 ` [PATCH 2/5] cooker: encode event objects to base64 Ed Bartosh
` (3 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Ed Bartosh @ 2016-06-20 11:00 UTC (permalink / raw)
To: bitbake-devel
EventLogWriteHandler object was created and used in
BBCooker.initConfigurationData.
This causes creation of multiple EventLogWriteHandler objects
and results in duplicated entries in the output event file
as BBCooker.initConfigurationData is called multiple times.
Added eventlogfile parameter to EventLogWriteHandler to avoid using
global variable DEFAULT_EVENTFILE.
Moved EventLogWriteHandler to the module level.
Created EventLogWriteHandler object in BBCooker.__init__ to ensure that only
one handler object is created.
Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
---
bitbake/lib/bb/cooker.py | 126 ++++++++++++++++++++++++-----------------------
1 file changed, 64 insertions(+), 62 deletions(-)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 60ee5d7..b2cf0cd 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -42,6 +42,8 @@ import subprocess
import errno
import prserv.serv
import pyinotify
+import json
+import pickle
logger = logging.getLogger("BitBake")
collectlog = logging.getLogger("BitBake.Collection")
@@ -114,6 +116,63 @@ class CookerFeatures(object):
return next(self._features)
+class EventLogWriteHandler:
+
+ def __init__(self, cooker, eventfile):
+ # set our handler's event processor
+ self.event = self.EventWriter(cooker, eventfile)
+
+ class EventWriter:
+ def __init__(self, cooker, eventfile):
+ self.file_inited = None
+ self.cooker = cooker
+ self.eventfile = eventfile
+ self.event_queue = []
+
+ def init_file(self):
+ try:
+ # delete the old log
+ os.remove(self.eventfile)
+ except:
+ pass
+
+ # write current configuration data
+ with open(eventfile, "w") as f:
+ f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
+
+ def write_event(self, event):
+ with open(self.eventfile, "a") as f:
+ try:
+ f.write("%s\n" % json.dumps({"class":event.__module__ + "." + event.__class__.__name__, "vars":json.dumps(pickle.dumps(event)) }))
+ except Exception as e:
+ import traceback
+ print(e, traceback.format_exc())
+
+
+ def send(self, event):
+ event_class = event.__module__ + "." + event.__class__.__name__
+
+ # init on bb.event.BuildStarted
+ if self.file_inited is None:
+ if event_class == "bb.event.BuildStarted":
+ self.init_file()
+ self.file_inited = True
+
+ # write pending events
+ for e in self.event_queue:
+ self.write_event(e)
+
+ # also write the current event
+ self.write_event(event)
+
+ else:
+ # queue all events until the file is inited
+ self.event_queue.append(event)
+
+ else:
+ # we have the file, just write the event
+ self.write_event(event)
+
#============================================================================#
# BBCooker
#============================================================================#
@@ -151,6 +210,11 @@ class BBCooker:
self.initConfigurationData()
+ # we log all events to a file if so directed
+ if self.configuration.writeeventlog:
+ # register the log file writer as UI Handler
+ bb.event.register_UIHhandler(EventLogWriteHandler(self, self.configuration.writeeventlog))
+
self.inotify_modified_files = []
def _process_inotify_updates(server, notifier_list, abort):
@@ -301,68 +365,6 @@ class BBCooker:
if consolelog:
self.data.setVar("BB_CONSOLELOG", consolelog)
- # we log all events to a file if so directed
- if self.configuration.writeeventlog:
- import json, pickle
- DEFAULT_EVENTFILE = self.configuration.writeeventlog
- class EventLogWriteHandler():
-
- class EventWriter():
- def __init__(self, cooker):
- self.file_inited = None
- self.cooker = cooker
- self.event_queue = []
-
- def init_file(self):
- try:
- # delete the old log
- os.remove(DEFAULT_EVENTFILE)
- except:
- pass
-
- # write current configuration data
- with open(DEFAULT_EVENTFILE, "w") as f:
- f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
-
- def write_event(self, event):
- with open(DEFAULT_EVENTFILE, "a") as f:
- try:
- f.write("%s\n" % json.dumps({"class":event.__module__ + "." + event.__class__.__name__, "vars":json.dumps(pickle.dumps(event)) }))
- except Exception as e:
- import traceback
- print(e, traceback.format_exc())
-
-
- def send(self, event):
- event_class = event.__module__ + "." + event.__class__.__name__
-
- # init on bb.event.BuildStarted
- if self.file_inited is None:
- if event_class == "bb.event.BuildStarted":
- self.init_file()
- self.file_inited = True
-
- # write pending events
- for e in self.event_queue:
- self.write_event(e)
-
- # also write the current event
- self.write_event(event)
-
- else:
- # queue all events until the file is inited
- self.event_queue.append(event)
-
- else:
- # we have the file, just write the event
- self.write_event(event)
-
- # set our handler's event processor
- event = EventWriter(self) # self is the cooker here
-
- # register the log file writer as UI Handler
- bb.event.register_UIHhandler(EventLogWriteHandler())
-
#
# Copy of the data store which has been expanded.
# Used for firing events and accessing variables where expansion needs to be accounted for
--
2.1.4
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 2/5] cooker: encode event objects to base64
2016-06-20 11:00 [PATCH 0/5] Fix writing events to file Ed Bartosh
2016-06-20 11:00 ` [PATCH 1/5] cooker: move EventLogWriteHandler to the top module level Ed Bartosh
@ 2016-06-20 11:00 ` Ed Bartosh
2016-06-20 11:00 ` [PATCH 3/5] cooker: don't remove event file Ed Bartosh
` (2 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Ed Bartosh @ 2016-06-20 11:00 UTC (permalink / raw)
To: bitbake-devel
pickle converts python objects into the binary form that can't be
decoded to text and therefore can't be converted to JSON format.
Attempt to convert event objects raises this error:
TypeError:
b'\x80\x03cbb.runqueue\nrunQueueTaskSkipped\nq\x00)...
is not JSON serializable
Encoded pickled event objects to base64 to be able to convert data
structure to JSON.
[YOCTO #9803]
Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
---
bitbake/lib/bb/cooker.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index b2cf0cd..2de6b3e 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -44,6 +44,7 @@ import prserv.serv
import pyinotify
import json
import pickle
+import codecs
logger = logging.getLogger("BitBake")
collectlog = logging.getLogger("BitBake.Collection")
@@ -143,7 +144,9 @@ class EventLogWriteHandler:
def write_event(self, event):
with open(self.eventfile, "a") as f:
try:
- f.write("%s\n" % json.dumps({"class":event.__module__ + "." + event.__class__.__name__, "vars":json.dumps(pickle.dumps(event)) }))
+ str_event = codecs.encode(pickle.dumps(event), 'base64').decode('utf-8')
+ f.write("%s\n" % json.dumps({"class": event.__module__ + "." + event.__class__.__name__,
+ "vars": str_event}))
except Exception as e:
import traceback
print(e, traceback.format_exc())
--
2.1.4
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 3/5] cooker: don't remove event file
2016-06-20 11:00 [PATCH 0/5] Fix writing events to file Ed Bartosh
2016-06-20 11:00 ` [PATCH 1/5] cooker: move EventLogWriteHandler to the top module level Ed Bartosh
2016-06-20 11:00 ` [PATCH 2/5] cooker: encode event objects to base64 Ed Bartosh
@ 2016-06-20 11:00 ` Ed Bartosh
2016-06-20 11:00 ` [PATCH 4/5] cooker: replace EventLogWriteHandler with namedtuple Ed Bartosh
2016-06-20 11:00 ` [PATCH 5/5] cooker: clean up EvertWriter Ed Bartosh
4 siblings, 0 replies; 6+ messages in thread
From: Ed Bartosh @ 2016-06-20 11:00 UTC (permalink / raw)
To: bitbake-devel
There is no need to remove output file as it gets rewritten by
open(self.eventfile, 'w') anyway.
Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
---
bitbake/lib/bb/cooker.py | 6 ------
1 file changed, 6 deletions(-)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 2de6b3e..ca53ff4 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -131,12 +131,6 @@ class EventLogWriteHandler:
self.event_queue = []
def init_file(self):
- try:
- # delete the old log
- os.remove(self.eventfile)
- except:
- pass
-
# write current configuration data
with open(eventfile, "w") as f:
f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
--
2.1.4
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 4/5] cooker: replace EventLogWriteHandler with namedtuple
2016-06-20 11:00 [PATCH 0/5] Fix writing events to file Ed Bartosh
` (2 preceding siblings ...)
2016-06-20 11:00 ` [PATCH 3/5] cooker: don't remove event file Ed Bartosh
@ 2016-06-20 11:00 ` Ed Bartosh
2016-06-20 11:00 ` [PATCH 5/5] cooker: clean up EvertWriter Ed Bartosh
4 siblings, 0 replies; 6+ messages in thread
From: Ed Bartosh @ 2016-06-20 11:00 UTC (permalink / raw)
To: bitbake-devel
class EventLogWriteHandler is a simple wrapper class with only one
class member. Replacing it with namedtuple makes code less nested and more
readable.
Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
---
bitbake/lib/bb/cooker.py | 86 +++++++++++++++++++++++-------------------------
1 file changed, 41 insertions(+), 45 deletions(-)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index ca53ff4..3e4f8d8 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -33,7 +33,7 @@ import threading
from io import StringIO
from contextlib import closing
from functools import wraps
-from collections import defaultdict
+from collections import defaultdict, namedtuple
import bb, bb.exceptions, bb.command
from bb import utils, data, parse, event, cache, providers, taskdata, runqueue, build
import queue
@@ -117,58 +117,52 @@ class CookerFeatures(object):
return next(self._features)
-class EventLogWriteHandler:
-
+class EventWriter:
def __init__(self, cooker, eventfile):
- # set our handler's event processor
- self.event = self.EventWriter(cooker, eventfile)
-
- class EventWriter:
- def __init__(self, cooker, eventfile):
- self.file_inited = None
- self.cooker = cooker
- self.eventfile = eventfile
- self.event_queue = []
-
- def init_file(self):
- # write current configuration data
- with open(eventfile, "w") as f:
- f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
-
- def write_event(self, event):
- with open(self.eventfile, "a") as f:
- try:
- str_event = codecs.encode(pickle.dumps(event), 'base64').decode('utf-8')
- f.write("%s\n" % json.dumps({"class": event.__module__ + "." + event.__class__.__name__,
- "vars": str_event}))
- except Exception as e:
- import traceback
- print(e, traceback.format_exc())
+ self.file_inited = None
+ self.cooker = cooker
+ self.eventfile = eventfile
+ self.event_queue = []
+ def init_file(self):
+ # write current configuration data
+ with open(eventfile, "w") as f:
+ f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
+
+ def write_event(self, event):
+ with open(self.eventfile, "a") as f:
+ try:
+ str_event = codecs.encode(pickle.dumps(event), 'base64').decode('utf-8')
+ f.write("%s\n" % json.dumps({"class": event.__module__ + "." + event.__class__.__name__,
+ "vars": str_event}))
+ except Exception as e:
+ import traceback
+ print(e, traceback.format_exc())
- def send(self, event):
- event_class = event.__module__ + "." + event.__class__.__name__
- # init on bb.event.BuildStarted
- if self.file_inited is None:
- if event_class == "bb.event.BuildStarted":
- self.init_file()
- self.file_inited = True
+ def send(self, event):
+ event_class = event.__module__ + "." + event.__class__.__name__
- # write pending events
- for e in self.event_queue:
- self.write_event(e)
+ # init on bb.event.BuildStarted
+ if self.file_inited is None:
+ if event_class == "bb.event.BuildStarted":
+ self.init_file()
+ self.file_inited = True
- # also write the current event
- self.write_event(event)
+ # write pending events
+ for e in self.event_queue:
+ self.write_event(e)
- else:
- # queue all events until the file is inited
- self.event_queue.append(event)
+ # also write the current event
+ self.write_event(event)
else:
- # we have the file, just write the event
- self.write_event(event)
+ # queue all events until the file is inited
+ self.event_queue.append(event)
+
+ else:
+ # we have the file, just write the event
+ self.write_event(event)
#============================================================================#
# BBCooker
@@ -210,7 +204,9 @@ class BBCooker:
# we log all events to a file if so directed
if self.configuration.writeeventlog:
# register the log file writer as UI Handler
- bb.event.register_UIHhandler(EventLogWriteHandler(self, self.configuration.writeeventlog))
+ writer = EventWriter(self, self.configuration.writeeventlog)
+ EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event'])
+ bb.event.register_UIHhandler(EventLogWriteHandler(writer))
self.inotify_modified_files = []
--
2.1.4
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 5/5] cooker: clean up EvertWriter
2016-06-20 11:00 [PATCH 0/5] Fix writing events to file Ed Bartosh
` (3 preceding siblings ...)
2016-06-20 11:00 ` [PATCH 4/5] cooker: replace EventLogWriteHandler with namedtuple Ed Bartosh
@ 2016-06-20 11:00 ` Ed Bartosh
4 siblings, 0 replies; 6+ messages in thread
From: Ed Bartosh @ 2016-06-20 11:00 UTC (permalink / raw)
To: bitbake-devel
Restructured EventWriter code to make it more readable:
- got rid of init_file method as it's called only once
- renamed exception variable e -> err
- renamed event variable e -> evt
- simplified main 'if' structure of send method
Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
---
bitbake/lib/bb/cooker.py | 33 +++++++++++++--------------------
1 file changed, 13 insertions(+), 20 deletions(-)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 3e4f8d8..9bd3460 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -124,46 +124,39 @@ class EventWriter:
self.eventfile = eventfile
self.event_queue = []
- def init_file(self):
- # write current configuration data
- with open(eventfile, "w") as f:
- f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
-
def write_event(self, event):
with open(self.eventfile, "a") as f:
try:
str_event = codecs.encode(pickle.dumps(event), 'base64').decode('utf-8')
f.write("%s\n" % json.dumps({"class": event.__module__ + "." + event.__class__.__name__,
"vars": str_event}))
- except Exception as e:
+ except Exception as err:
import traceback
- print(e, traceback.format_exc())
-
+ print(err, traceback.format_exc())
def send(self, event):
- event_class = event.__module__ + "." + event.__class__.__name__
+ if self.file_inited:
+ # we have the file, just write the event
+ self.write_event(event)
+ else:
+ # init on bb.event.BuildStarted
+ name = "%s.%s" % (event.__module__, event.__class__.__name__)
+ if name == "bb.event.BuildStarted":
+ with open(self.eventfile, "w") as f:
+ f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
- # init on bb.event.BuildStarted
- if self.file_inited is None:
- if event_class == "bb.event.BuildStarted":
- self.init_file()
self.file_inited = True
# write pending events
- for e in self.event_queue:
- self.write_event(e)
+ for evt in self.event_queue:
+ self.write_event(evt)
# also write the current event
self.write_event(event)
-
else:
# queue all events until the file is inited
self.event_queue.append(event)
- else:
- # we have the file, just write the event
- self.write_event(event)
-
#============================================================================#
# BBCooker
#============================================================================#
--
2.1.4
^ permalink raw reply related [flat|nested] 6+ messages in thread