* [PATCH 1/7] bitbake: cooker, xmlrpc, servers: implement CookerFeatures
2013-09-18 12:15 [PATCH 0/7] Adding Bitbake support for WebHob Alex DAMIAN
@ 2013-09-18 12:15 ` Alex DAMIAN
2013-09-18 12:15 ` [PATCH 2/7] bitbake: cooker, runqueue: send the task dependency tree Alex DAMIAN
` (5 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Alex DAMIAN @ 2013-09-18 12:15 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
Implementing feature set selection that allows a client
to enable specific features in the server at connection time.
Only enabling of features is supported, as there is
no way to safely remove data loaded into the cooker.
Once enabled, a feature will remain enabled for the
life of the cooker.
Client-server connection now supports specifying the feature
set required by the client. This is implemented in the Process
server using a managed proxy list, so the server cooker
will now load dynamically needed features based on what client
connects to it.
In the XMLRPC server the feature set is requested by
using a parameter for registerUIHandler function.
This allows observer-only clients to also specify features
for the server.
The server code configuration now is completly separated
from the client code. All hardcoding of client knowledge is
removed from the server.
The extra_caches is removed as the client can now specify
the caches it needs using the feature. The UI modules
now need to specify the desired featureSet. HOB is modified
to conform to the featureSet specification.
The only feature available is CookerFeatures.HOB_EXTRA_CACHES
which forces loading the bb.cache_extra:HobRecipeInfo class.
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/bin/bitbake | 25 +++++--------------------
bitbake/lib/bb/cooker.py | 34 ++++++++++++++++++++++++++++++++--
bitbake/lib/bb/cookerdata.py | 1 -
bitbake/lib/bb/server/__init__.py | 2 +-
bitbake/lib/bb/server/process.py | 23 ++++++++++++++++++-----
bitbake/lib/bb/server/xmlrpc.py | 29 ++++++++++++++++++++++-------
bitbake/lib/bb/ui/hob.py | 2 +-
bitbake/lib/bb/ui/knotty.py | 3 +++
bitbake/lib/bb/ui/uievent.py | 7 +++++--
9 files changed, 87 insertions(+), 39 deletions(-)
diff --git a/bitbake/bin/bitbake b/bitbake/bin/bitbake
index 60c4b96..b4506ef 100755
--- a/bitbake/bin/bitbake
+++ b/bitbake/bin/bitbake
@@ -75,18 +75,6 @@ def get_ui(config):
sys.exit("FATAL: Invalid user interface '%s' specified.\n"
"Valid interfaces: depexp, goggle, ncurses, hob, knotty [default]." % interface)
-def gather_extra_cache_data():
- extra = []
- interfaces = ['depexp', 'goggle', 'ncurses', 'hob', 'knotty']
- for i in interfaces:
- try:
- ui = __import__("bb.ui." + i, fromlist = [i])
- if hasattr(ui, "extraCaches"):
- extra = extra + ui.extraCaches
- del ui
- except:
- pass
- return extra
# Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring others"""
warnlog = logging.getLogger("BitBake.Warnings")
@@ -302,25 +290,22 @@ def main():
# Clear away any spurious environment variables while we stoke up the cooker
cleanedvars = bb.utils.clean_environment()
- # Collect all the caches we need
- if configParams.server_only:
- configuration.extra_caches = gather_extra_cache_data()
- else:
- configuration.extra_caches = getattr(ui_module, "extraCaches", [])
-
if not configParams.remote_server:
# we start a server with a given configuration
server = start_server(servermodule, configParams, configuration)
else:
- # we start a stub server that is actually a XMLRPClient to
+ # we start a stub server that is actually a XMLRPClient that connects to a real server
server = servermodule.BitBakeXMLRPCClient(configParams.observe_only)
server.saveConnectionDetails(configParams.remote_server)
logger.removeHandler(handler)
if not configParams.server_only:
+ # Collect the feature set for the UI
+ featureset = getattr(ui_module, "featureSet", [])
+
# Setup a connection to the server (cooker)
- server_connection = server.establishConnection()
+ server_connection = server.establishConnection(featureset)
# Restore the environment in case the UI needs it
for k in cleanedvars:
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 1d5e5f6..8776e18 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -79,6 +79,29 @@ class SkippedPackage:
elif reason:
self.skipreason = reason
+
+class CookerFeatures(object):
+ _feature_list = [HOB_EXTRA_CACHES] = range(1)
+
+ def __init__(self):
+ self._features=set()
+
+ def setFeature(self, f):
+ # validate we got a request for a feature we support
+ if f not in CookerFeatures._feature_list:
+ return
+ self._features.add(f)
+
+ def __contains__(self, f):
+ return f in self._features
+
+ def __iter__(self):
+ return self._features.__iter__()
+
+ def next(self):
+ return self._features.next()
+
+
#============================================================================#
# BBCooker
#============================================================================#
@@ -90,6 +113,7 @@ class BBCooker:
def __init__(self, configuration):
self.recipecache = None
self.skiplist = {}
+ self.featureset = CookerFeatures()
self.configuration = configuration
@@ -122,7 +146,13 @@ class BBCooker:
self.state = state.initial
self.caches_array = []
- caches_name_array = ['bb.cache:CoreRecipeInfo'] + self.configuration.extra_caches
+
+ all_extra_cache_names = []
+ # We hardcode all known cache types in a single place, here.
+ if CookerFeatures.HOB_EXTRA_CACHES in self.featureset:
+ all_extra_cache_names.append("bb.cache_extra:HobRecipeInfo")
+
+ caches_name_array = ['bb.cache:CoreRecipeInfo'] + all_extra_cache_names
# At least CoreRecipeInfo will be loaded, so caches_array will never be empty!
# This is the entry point, no further check needed!
@@ -130,7 +160,7 @@ class BBCooker:
try:
module_name, cache_name = var.split(':')
module = __import__(module_name, fromlist=(cache_name,))
- self.caches_array.append(getattr(module, cache_name))
+ self.caches_array.append(getattr(module, cache_name))
except ImportError as exc:
logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc))
sys.exit("FATAL: Failed to import extra cache class '%s'." % cache_name)
diff --git a/bitbake/lib/bb/cookerdata.py b/bitbake/lib/bb/cookerdata.py
index 0b278b1..e640ed0 100644
--- a/bitbake/lib/bb/cookerdata.py
+++ b/bitbake/lib/bb/cookerdata.py
@@ -127,7 +127,6 @@ class CookerConfiguration(object):
self.dump_signatures = False
self.dry_run = False
self.tracking = False
- self.extra_caches = []
self.env = {}
diff --git a/bitbake/lib/bb/server/__init__.py b/bitbake/lib/bb/server/__init__.py
index 2e1c619..da5e480 100644
--- a/bitbake/lib/bb/server/__init__.py
+++ b/bitbake/lib/bb/server/__init__.py
@@ -89,7 +89,7 @@ class BitBakeBaseServer(object):
def detach(self):
return
- def establishConnection(self):
+ def establishConnection(self, featureset):
raise "Must redefine the %s.establishConnection()" % self.__class__.__name__
def endSession(self):
diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py
index e45e0c2..aa07202 100644
--- a/bitbake/lib/bb/server/process.py
+++ b/bitbake/lib/bb/server/process.py
@@ -31,7 +31,7 @@ import sys
import time
import select
from Queue import Empty
-from multiprocessing import Event, Process, util, Queue, Pipe, queues
+from multiprocessing import Event, Process, util, Queue, Pipe, queues, Manager
from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer
@@ -78,12 +78,13 @@ class ProcessServer(Process, BaseImplServer):
profile_filename = "profile.log"
profile_processed_filename = "profile.log.processed"
- def __init__(self, command_channel, event_queue):
+ def __init__(self, command_channel, event_queue, featurelist):
BaseImplServer.__init__(self)
- Process.__init__(self)
+ Process.__init__(self, args=(featurelist))
self.command_channel = command_channel
self.event_queue = event_queue
self.event = EventAdapter(event_queue)
+ self.featurelist = featurelist
self.quit = False
self.keep_running = Event()
@@ -94,6 +95,14 @@ class ProcessServer(Process, BaseImplServer):
for event in bb.event.ui_queue:
self.event_queue.put(event)
self.event_handle.value = bb.event.register_UIHhandler(self)
+
+ # process any feature changes based on what UI requested
+ original_featureset = list(self.cooker.featureset)
+ while len(self.featurelist)> 0:
+ self.cooker.featureset.setFeature(self.featurelist.pop())
+ if (original_featureset != list(self.cooker.featureset)):
+ self.cooker.reset()
+
bb.cooker.server_main(self.cooker, self.main)
def main(self):
@@ -198,13 +207,17 @@ class BitBakeServer(BitBakeBaseServer):
#
self.ui_channel, self.server_channel = Pipe()
self.event_queue = ProcessEventQueue(0)
- self.serverImpl = ProcessServer(self.server_channel, self.event_queue)
+ manager = Manager()
+ self.featurelist = manager.list()
+ self.serverImpl = ProcessServer(self.server_channel, self.event_queue, self.featurelist)
def detach(self):
self.serverImpl.start()
return
- def establishConnection(self):
+ def establishConnection(self, featureset):
+ for f in featureset:
+ self.featurelist.append(f)
self.connection = BitBakeProcessServerConnection(self.serverImpl, self.ui_channel, self.event_queue)
signal.signal(signal.SIGTERM, lambda i, s: self.connection.terminate())
return self.connection
diff --git a/bitbake/lib/bb/server/xmlrpc.py b/bitbake/lib/bb/server/xmlrpc.py
index 8326623..389327a 100644
--- a/bitbake/lib/bb/server/xmlrpc.py
+++ b/bitbake/lib/bb/server/xmlrpc.py
@@ -89,12 +89,23 @@ class BitBakeServerCommands():
self.server = server
self.has_client = False
- def registerEventHandler(self, host, port):
+ def registerEventHandler(self, host, port, featureset = []):
"""
Register a remote UI Event Handler
"""
s, t = _create_server(host, port)
+ # we don't allow connections if the cooker is running
+ if (self.cooker.state in [bb.cooker.state.parsing, bb.cooker.state.running]):
+ return None
+
+ original_featureset = list(self.cooker.featureset)
+ for f in featureset:
+ self.cooker.featureset.setFeature(f)
+
+ if (original_featureset != list(self.cooker.featureset)):
+ self.cooker.reset()
+
self.event_handle = bb.event.register_UIHhandler(s)
return self.event_handle
@@ -263,11 +274,12 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
self.connection_token = token
class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
- def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False):
+ def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False, featureset = []):
self.connection, self.transport = _create_server(serverImpl.host, serverImpl.port)
self.clientinfo = clientinfo
self.serverImpl = serverImpl
self.observer_only = observer_only
+ self.featureset = featureset
def connect(self):
if not self.observer_only:
@@ -277,7 +289,8 @@ class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
if token is None:
return None
self.transport.set_connection_token(token)
- self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo)
+
+ self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo, self.featureset)
for event in bb.event.ui_queue:
self.events.queue_event(event)
return self
@@ -301,14 +314,15 @@ class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
class BitBakeServer(BitBakeBaseServer):
def initServer(self, interface = ("localhost", 0)):
+ self.interface = interface
self.serverImpl = XMLRPCServer(interface)
def detach(self):
daemonize.createDaemon(self.serverImpl.serve_forever, "bitbake-cookerdaemon.log")
del self.cooker
- def establishConnection(self):
- self.connection = BitBakeXMLRPCServerConnection(self.serverImpl)
+ def establishConnection(self, featureset):
+ self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, self.interface, False, featureset)
return self.connection.connect()
def set_connection_token(self, token):
@@ -318,12 +332,13 @@ class BitBakeXMLRPCClient(BitBakeBaseServer):
def __init__(self, observer_only = False):
self.observer_only = observer_only
+ # if we need extra caches, just tell the server to load them all
pass
def saveConnectionDetails(self, remote):
self.remote = remote
- def establishConnection(self):
+ def establishConnection(self, featureset):
# The format of "remote" must be "server:port"
try:
[host, port] = self.remote.split(":")
@@ -340,7 +355,7 @@ class BitBakeXMLRPCClient(BitBakeBaseServer):
except:
return None
self.serverImpl = XMLRPCProxyServer(host, port)
- self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only)
+ self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only, featureset)
return self.connection.connect()
def endSession(self):
diff --git a/bitbake/lib/bb/ui/hob.py b/bitbake/lib/bb/ui/hob.py
index bc5ee14..154a3b3 100755
--- a/bitbake/lib/bb/ui/hob.py
+++ b/bitbake/lib/bb/ui/hob.py
@@ -46,7 +46,7 @@ from bb.ui.crumbs.hoblistmodel import RecipeListModel, PackageListModel
from bb.ui.crumbs.hobeventhandler import HobHandler
from bb.ui.crumbs.builder import Builder
-extraCaches = ['bb.cache_extra:HobRecipeInfo']
+featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES]
def event_handle_idle_func(eventHandler, hobHandler):
# Consume as many messages as we can in the time available to us
diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py
index 35590a2..0211b50 100644
--- a/bitbake/lib/bb/ui/knotty.py
+++ b/bitbake/lib/bb/ui/knotty.py
@@ -472,6 +472,9 @@ def main(server, eventHandler, params, tf = TerminalFilter):
event.taskid, event.taskstring, event.exitcode)
continue
+ if isinstance(event, bb.event.DepTreeGenerated):
+ continue
+
# ignore
if isinstance(event, (bb.event.BuildBase,
bb.event.StampUpdate,
diff --git a/bitbake/lib/bb/ui/uievent.py b/bitbake/lib/bb/ui/uievent.py
index 038029f..2133b44 100644
--- a/bitbake/lib/bb/ui/uievent.py
+++ b/bitbake/lib/bb/ui/uievent.py
@@ -28,7 +28,7 @@ import socket, threading, pickle
from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
class BBUIEventQueue:
- def __init__(self, BBServer, clientinfo=("localhost, 0")):
+ def __init__(self, BBServer, clientinfo=("localhost, 0"), featureset=[]):
self.eventQueue = []
self.eventQueueLock = threading.Lock()
@@ -44,7 +44,10 @@ class BBUIEventQueue:
server.register_function( self.send_event, "event.sendpickle" )
server.socket.settimeout(1)
- self.EventHandle = self.BBServer.registerEventHandler(self.host, self.port)
+ self.EventHandle = self.BBServer.registerEventHandler(self.host, self.port, featureset)
+
+ if (self.EventHandle == None):
+ bb.fatal("Could not register UI event handler")
self.server = server
--
1.8.1.2
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 4/7] bitbake: cooker: get extra information from recipe cache
2013-09-18 12:15 [PATCH 0/7] Adding Bitbake support for WebHob Alex DAMIAN
` (2 preceding siblings ...)
2013-09-18 12:15 ` [PATCH 3/7] bitbake: build, runqueue: adds info to the *runQueue* events Alex DAMIAN
@ 2013-09-18 12:15 ` Alex DAMIAN
2013-09-18 12:15 ` [PATCH 5/7] bitbake: cooker, command: add a command to return global data Alex DAMIAN
` (2 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Alex DAMIAN @ 2013-09-18 12:15 UTC (permalink / raw)
To: bitbake-devel, richard.purdie; +Cc: Alexandru DAMIAN
From: Alexandru DAMIAN <alexandru.damian@intel.com>
The loaded cache modules may add extra attributes to
the recipecache, that will be populated by the cache
classes required by the UI. These attributes
will be used by the UI to display relevant information.
Adds cachefields cache class field to specify
for each cache class which attributes will be set
in the recipecache.
Adds code to automatically expand depends tree with the
fields exported by the extra cache class.
Fixes a cache field name in the HOB UI.
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
bitbake/lib/bb/cache_extra.py | 6 ++++
bitbake/lib/bb/cooker.py | 50 ++++++++++++++++++--------------
bitbake/lib/bb/ui/crumbs/hoblistmodel.py | 2 +-
3 files changed, 36 insertions(+), 22 deletions(-)
diff --git a/bitbake/lib/bb/cache_extra.py b/bitbake/lib/bb/cache_extra.py
index 9e38a43..83f4959 100644
--- a/bitbake/lib/bb/cache_extra.py
+++ b/bitbake/lib/bb/cache_extra.py
@@ -35,6 +35,12 @@ class HobRecipeInfo(RecipeInfoCommon):
# such as (bb_cache.dat, bb_extracache_hob.dat)
cachefile = "bb_extracache_" + classname +".dat"
+ # override this member with the list of extra cache fields
+ # that this class will provide
+ cachefields = ['summary', 'license', 'section',
+ 'description', 'homepage', 'bugtracker',
+ 'prevision', 'files_info']
+
def __init__(self, filename, metadata):
self.summary = self.getvar('SUMMARY', metadata)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index dd45ae8..fafa518 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -481,6 +481,19 @@ class BBCooker:
depend_tree["pn"][pn] = {}
depend_tree["pn"][pn]["filename"] = fn
depend_tree["pn"][pn]["version"] = version
+
+ # if we have extra caches, list all attributes they bring in
+ extra_info = []
+ for cache_class in self.caches_array:
+ if type(cache_class) is type and issubclass(cache_class, bb.cache.RecipeInfoCommon) and hasattr(cache_class, 'cachefields'):
+ cachefields = getattr(cache_class, 'cachefields', [])
+ extra_info = extra_info + cachefields
+
+ # for all attributes stored, add them to the dependency tree
+ for ei in extra_info:
+ depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn]
+
+
for dep in rq.rqdata.runq_depends[task]:
depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]]
deppn = self.recipecache.pkg_fn[depfn]
@@ -543,35 +556,30 @@ class BBCooker:
depend_tree["rdepends-pkg"] = {}
depend_tree["rrecs-pkg"] = {}
+ # if we have extra caches, list all attributes they bring in
+ extra_info = []
+ for cache_class in self.caches_array:
+ if type(cache_class) is type and issubclass(cache_class, bb.cache.RecipeInfoCommon) and hasattr(cache_class, 'cachefields'):
+ cachefields = getattr(cache_class, 'cachefields', [])
+ extra_info = extra_info + cachefields
+
for task in xrange(len(tasks_fnid)):
fnid = tasks_fnid[task]
fn = taskdata.fn_index[fnid]
pn = self.recipecache.pkg_fn[fn]
- version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn]
- summary = self.recipecache.summary[fn]
- lic = self.recipecache.license[fn]
- section = self.recipecache.section[fn]
- description = self.recipecache.description[fn]
- homepage = self.recipecache.homepage[fn]
- bugtracker = self.recipecache.bugtracker[fn]
- files_info = self.recipecache.files_info[fn]
- rdepends = self.recipecache.rundeps[fn]
- rrecs = self.recipecache.runrecs[fn]
- prevision = self.recipecache.prevision[fn]
- inherits = self.recipecache.inherits.get(fn, None)
+
if pn not in depend_tree["pn"]:
depend_tree["pn"][pn] = {}
depend_tree["pn"][pn]["filename"] = fn
+ version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn]
depend_tree["pn"][pn]["version"] = version
- depend_tree["pn"][pn]["summary"] = summary
- depend_tree["pn"][pn]["license"] = lic
- depend_tree["pn"][pn]["section"] = section
- depend_tree["pn"][pn]["description"] = description
- depend_tree["pn"][pn]["inherits"] = inherits
- depend_tree["pn"][pn]["homepage"] = homepage
- depend_tree["pn"][pn]["bugtracker"] = bugtracker
- depend_tree["pn"][pn]["files_info"] = files_info
- depend_tree["pn"][pn]["revision"] = prevision
+ rdepends = self.recipecache.rundeps[fn]
+ rrecs = self.recipecache.runrecs[fn]
+ depend_tree["pn"][pn]["inherits"] = self.recipecache.inherits.get(fn, None)
+
+ # for all extra attributes stored, add them to the dependency tree
+ for ei in extra_info:
+ depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn]
if fnid not in seen_fnids:
seen_fnids.append(fnid)
diff --git a/bitbake/lib/bb/ui/crumbs/hoblistmodel.py b/bitbake/lib/bb/ui/crumbs/hoblistmodel.py
index 74d335c..b4d2a62 100644
--- a/bitbake/lib/bb/ui/crumbs/hoblistmodel.py
+++ b/bitbake/lib/bb/ui/crumbs/hoblistmodel.py
@@ -690,7 +690,7 @@ class RecipeListModel(gtk.ListStore):
inherits = event_model["pn"][item]["inherits"]
summary = event_model["pn"][item]["summary"]
version = event_model["pn"][item]["version"]
- revision = event_model["pn"][item]["revision"]
+ revision = event_model["pn"][item]["prevision"]
homepage = event_model["pn"][item]["homepage"]
bugtracker = event_model["pn"][item]["bugtracker"]
filename = event_model["pn"][item]["filename"]
--
1.8.1.2
^ permalink raw reply related [flat|nested] 8+ messages in thread