* [PATCH 01/13] Fix default function parameter assignment to a list
2015-08-17 11:12 [PATCH 00/13] Fixes and refactoring Paul Eggleton
@ 2015-08-17 11:12 ` Paul Eggleton
2015-08-17 11:12 ` [PATCH 02/13] command: intercept SystemExit to avoid trashing the server Paul Eggleton
` (12 subsequent siblings)
13 siblings, 0 replies; 17+ messages in thread
From: Paul Eggleton @ 2015-08-17 11:12 UTC (permalink / raw)
To: bitbake-devel
With python you should not assign a list as the default value of a
function parameter - because a list is mutable, the result will be that
the first time a value is passed it will actually modify the default.
Reference:
http://docs.python-guide.org/en/latest/writing/gotchas/#mutable-default-arguments
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
lib/bb/cooker.py | 15 +++++++++------
lib/bb/event.py | 4 ++--
lib/bb/fetch2/__init__.py | 23 +++++++++++++----------
lib/bb/msg.py | 7 +++++--
lib/bb/process.py | 5 ++++-
lib/bb/server/process.py | 4 +++-
lib/bb/server/xmlrpc.py | 7 +++++--
lib/bb/taskdata.py | 8 +++++---
lib/bb/ui/crumbs/hobeventhandler.py | 12 +++++++++---
lib/toaster/toastergui/views.py | 4 +++-
10 files changed, 58 insertions(+), 31 deletions(-)
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 0f99342..84bf46b 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -114,12 +114,13 @@ class BBCooker:
Manages one bitbake build run
"""
- def __init__(self, configuration, featureSet = []):
+ def __init__(self, configuration, featureSet=None):
self.recipecache = None
self.skiplist = {}
self.featureset = CookerFeatures()
- for f in featureSet:
- self.featureset.setFeature(f)
+ if featureSet:
+ for f in featureSet:
+ self.featureset.setFeature(f)
self.configuration = configuration
@@ -567,12 +568,14 @@ class BBCooker:
logger.plain("%-35s %25s %25s", p, lateststr, prefstr)
- def showEnvironment(self, buildfile = None, pkgs_to_build = []):
+ def showEnvironment(self, buildfile=None, pkgs_to_build=None):
"""
Show the outer or per-recipe environment
"""
fn = None
envdata = None
+ if not pkgs_to_build:
+ pkgs_to_build = []
if buildfile:
# Parse the configuration here. We need to do it explicitly here since
@@ -1037,13 +1040,13 @@ class BBCooker:
return pkg_list
- def generateTargetsTree(self, klass=None, pkgs=[]):
+ def generateTargetsTree(self, klass=None, pkgs=None):
"""
Generate a dependency tree of buildable targets
Generate an event with the result
"""
# if the caller hasn't specified a pkgs list default to universe
- if not len(pkgs):
+ if not pkgs:
pkgs = ['universe']
# if inherited_class passed ensure all recipes which inherit the
# specified class are included in pkgs
diff --git a/lib/bb/event.py b/lib/bb/event.py
index 61a7f4a..0e18110 100644
--- a/lib/bb/event.py
+++ b/lib/bb/event.py
@@ -176,7 +176,7 @@ def fire_from_worker(event, d):
fire_ui_handlers(event, d)
noop = lambda _: None
-def register(name, handler, mask=[]):
+def register(name, handler, mask=None):
"""Register an Event handler"""
# already registered
@@ -389,7 +389,7 @@ class DiskFull(Event):
class NoProvider(Event):
"""No Provider for an Event"""
- def __init__(self, item, runtime=False, dependees=None, reasons=[], close_matches=[]):
+ def __init__(self, item, runtime=False, dependees=None, reasons=None, close_matches=None):
Event.__init__(self)
self._item = item
self._runtime = runtime
diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index 7b4d130..ec0c31a 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -777,7 +777,7 @@ def localpath(url, d):
fetcher = bb.fetch2.Fetch([url], d)
return fetcher.localpath(url)
-def runfetchcmd(cmd, d, quiet = False, cleanup = []):
+def runfetchcmd(cmd, d, quiet=False, cleanup=None):
"""
Run cmd returning the command output
Raise an error if interrupted or cmd fails
@@ -802,6 +802,9 @@ def runfetchcmd(cmd, d, quiet = False, cleanup = []):
'SSH_AUTH_SOCK', 'SSH_AGENT_PID',
'SOCKS5_USER', 'SOCKS5_PASSWD']
+ if not cleanup:
+ cleanup = []
+
for var in exportvars:
val = d.getVar(var, True)
if val:
@@ -1267,7 +1270,7 @@ class FetchData(object):
class FetchMethod(object):
"""Base class for 'fetch'ing data"""
- def __init__(self, urls = []):
+ def __init__(self, urls=None):
self.urls = []
def supports(self, urldata, d):
@@ -1552,11 +1555,11 @@ class Fetch(object):
return local
- def download(self, urls = []):
+ def download(self, urls=None):
"""
Fetch all urls
"""
- if len(urls) == 0:
+ if not urls:
urls = self.urls
network = self.d.getVar("BB_NO_NETWORK", True)
@@ -1634,12 +1637,12 @@ class Fetch(object):
finally:
bb.utils.unlockfile(lf)
- def checkstatus(self, urls = []):
+ def checkstatus(self, urls=None):
"""
Check all urls exist upstream
"""
- if len(urls) == 0:
+ if not urls:
urls = self.urls
for u in urls:
@@ -1662,12 +1665,12 @@ class Fetch(object):
if not ret:
raise FetchError("URL %s doesn't work" % u, u)
- def unpack(self, root, urls = []):
+ def unpack(self, root, urls=None):
"""
Check all urls exist upstream
"""
- if len(urls) == 0:
+ if not urls:
urls = self.urls
for u in urls:
@@ -1685,12 +1688,12 @@ class Fetch(object):
if ud.lockfile:
bb.utils.unlockfile(lf)
- def clean(self, urls = []):
+ def clean(self, urls=None):
"""
Clean files that the fetcher gets or places
"""
- if len(urls) == 0:
+ if not urls:
urls = self.urls
for url in urls:
diff --git a/lib/bb/msg.py b/lib/bb/msg.py
index d79768d..786b5ae 100644
--- a/lib/bb/msg.py
+++ b/lib/bb/msg.py
@@ -150,7 +150,7 @@ loggerDefaultVerbose = False
loggerVerboseLogs = False
loggerDefaultDomains = []
-def init_msgconfig(verbose, debug, debug_domains = []):
+def init_msgconfig(verbose, debug, debug_domains=None):
"""
Set default verbosity and debug levels config the logger
"""
@@ -158,7 +158,10 @@ def init_msgconfig(verbose, debug, debug_domains = []):
bb.msg.loggerDefaultVerbose = verbose
if verbose:
bb.msg.loggerVerboseLogs = True
- bb.msg.loggerDefaultDomains = debug_domains
+ if debug_domains:
+ bb.msg.loggerDefaultDomains = debug_domains
+ else:
+ bb.msg.loggerDefaultDomains = []
def constructLogOptions():
debug = loggerDefaultDebugLevel
diff --git a/lib/bb/process.py b/lib/bb/process.py
index 7c79785..1c07f2d 100644
--- a/lib/bb/process.py
+++ b/lib/bb/process.py
@@ -128,10 +128,13 @@ def _logged_communicate(pipe, log, input, extrafiles):
pipe.stderr.close()
return ''.join(outdata), ''.join(errdata)
-def run(cmd, input=None, log=None, extrafiles=[], **options):
+def run(cmd, input=None, log=None, extrafiles=None, **options):
"""Convenience function to run a command and return its output, raising an
exception when the command fails"""
+ if not extrafiles:
+ extrafiles = []
+
if isinstance(cmd, basestring) and not "shell" in options:
options["shell"] = True
diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py
index ef3ee57..3198635 100644
--- a/lib/bb/server/process.py
+++ b/lib/bb/server/process.py
@@ -124,8 +124,10 @@ class ProcessServer(Process, BaseImplServer):
self.command_channel.close()
self.cooker.shutdown(True)
- def idle_commands(self, delay, fds = []):
+ def idle_commands(self, delay, fds=None):
nextsleep = delay
+ if not fds:
+ fds = []
for function, data in self._idlefuns.items():
try:
diff --git a/lib/bb/server/xmlrpc.py b/lib/bb/server/xmlrpc.py
index 75ec855..f1a2067 100644
--- a/lib/bb/server/xmlrpc.py
+++ b/lib/bb/server/xmlrpc.py
@@ -281,12 +281,15 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
self.connection_token = token
class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
- def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False, featureset = []):
+ def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False, featureset = None):
self.connection, self.transport = _create_server(serverImpl.host, serverImpl.port)
self.clientinfo = clientinfo
self.serverImpl = serverImpl
self.observer_only = observer_only
- self.featureset = featureset
+ if featureset:
+ self.featureset = featureset
+ else:
+ self.featureset = []
def connect(self, token = None):
if token is None:
diff --git a/lib/bb/taskdata.py b/lib/bb/taskdata.py
index ca58e17..5fab704 100644
--- a/lib/bb/taskdata.py
+++ b/lib/bb/taskdata.py
@@ -514,7 +514,7 @@ class TaskData:
self.add_runtime_target(fn, item)
self.add_tasks(fn, dataCache)
- def fail_fnid(self, fnid, missing_list = []):
+ def fail_fnid(self, fnid, missing_list=None):
"""
Mark a file as failed (unbuildable)
Remove any references from build and runtime provider lists
@@ -523,6 +523,8 @@ class TaskData:
"""
if fnid in self.failed_fnids:
return
+ if not missing_list:
+ missing_list = []
logger.debug(1, "File '%s' is unbuildable, removing...", self.fn_index[fnid])
self.failed_fnids.append(fnid)
for target in self.build_targets:
@@ -536,7 +538,7 @@ class TaskData:
if len(self.run_targets[target]) == 0:
self.remove_runtarget(target, missing_list)
- def remove_buildtarget(self, targetid, missing_list = []):
+ def remove_buildtarget(self, targetid, missing_list=None):
"""
Mark a build target as failed (unbuildable)
Trigger removal of any files that have this as a dependency
@@ -561,7 +563,7 @@ class TaskData:
logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list)
raise bb.providers.NoProvider(target)
- def remove_runtarget(self, targetid, missing_list = []):
+ def remove_runtarget(self, targetid, missing_list=None):
"""
Mark a run target as failed (unbuildable)
Trigger removal of any files that have this as a dependency
diff --git a/lib/bb/ui/crumbs/hobeventhandler.py b/lib/bb/ui/crumbs/hobeventhandler.py
index 43edb70..b71fb33 100644
--- a/lib/bb/ui/crumbs/hobeventhandler.py
+++ b/lib/bb/ui/crumbs/hobeventhandler.py
@@ -440,11 +440,17 @@ class HobHandler(gobject.GObject):
self.commands_async.append(self.SUB_BUILD_RECIPES)
self.run_next_command(self.GENERATE_PACKAGES)
- def generate_image(self, image, base_image, image_packages=[], toolchain_packages=[], default_task="build"):
+ def generate_image(self, image, base_image, image_packages=None, toolchain_packages=None, default_task="build"):
self.image = image
self.base_image = base_image
- self.package_queue = image_packages
- self.toolchain_packages = toolchain_packages
+ if image_packages:
+ self.package_queue = image_packages
+ else:
+ self.package_queue = []
+ if toolchain_packages:
+ self.toolchain_packages = toolchain_packages
+ else:
+ self.toolchain_packages = []
self.default_task = default_task
self.runCommand(["setPrePostConfFiles", "conf/.hob.conf", ""])
self.commands_async.append(self.SUB_PARSE_CONFIG)
diff --git a/lib/toaster/toastergui/views.py b/lib/toaster/toastergui/views.py
index 889b6c6..d29ddd6 100755
--- a/lib/toaster/toastergui/views.py
+++ b/lib/toaster/toastergui/views.py
@@ -522,7 +522,9 @@ def builddashboard( request, build_id ):
-def generateCoveredList2( revlist = [] ):
+def generateCoveredList2( revlist = None ):
+ if not revlist:
+ revlist = []
covered_list = [ x for x in revlist if x.outcome == Task.OUTCOME_COVERED ]
while len(covered_list):
revlist = [ x for x in revlist if x.outcome != Task.OUTCOME_COVERED ]
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 02/13] command: intercept SystemExit to avoid trashing the server
2015-08-17 11:12 [PATCH 00/13] Fixes and refactoring Paul Eggleton
2015-08-17 11:12 ` [PATCH 01/13] Fix default function parameter assignment to a list Paul Eggleton
@ 2015-08-17 11:12 ` Paul Eggleton
2015-08-17 11:12 ` [PATCH 03/13] cooker: further limit inotify watches Paul Eggleton
` (11 subsequent siblings)
13 siblings, 0 replies; 17+ messages in thread
From: Paul Eggleton @ 2015-08-17 11:12 UTC (permalink / raw)
To: bitbake-devel
If sys.exit() is called within a command run over XMLRPC, the XMLRPC
server is effectively trashed (apparently listening but no longer able
to respond to commands). We need to intercept the SystemExit exception
and deal with it as we would any other exception.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
lib/bb/command.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/bb/command.py b/lib/bb/command.py
index 24ff341..a7cac97 100644
--- a/lib/bb/command.py
+++ b/lib/bb/command.py
@@ -71,7 +71,7 @@ class Command:
result = command_method(self, commandline)
except CommandError as exc:
return None, exc.args[0]
- except Exception:
+ except (Exception, SystemExit):
import traceback
return None, traceback.format_exc()
else:
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 03/13] cooker: further limit inotify watches
2015-08-17 11:12 [PATCH 00/13] Fixes and refactoring Paul Eggleton
2015-08-17 11:12 ` [PATCH 01/13] Fix default function parameter assignment to a list Paul Eggleton
2015-08-17 11:12 ` [PATCH 02/13] command: intercept SystemExit to avoid trashing the server Paul Eggleton
@ 2015-08-17 11:12 ` Paul Eggleton
2015-08-17 11:12 ` [PATCH 04/13] command: ensure sync commands that read configuration see updates Paul Eggleton
` (10 subsequent siblings)
13 siblings, 0 replies; 17+ messages in thread
From: Paul Eggleton @ 2015-08-17 11:12 UTC (permalink / raw)
To: bitbake-devel
Unfortunately we were acting on inotify notifications about any files
changing within the watched directories, not just the ones we actually
care about. In OE the build directory is in BBPATH and hence it gets
watched, and we write things like bitbake.lock and
bitbake-cookerdaemon.log to that directory, hence effectively
notifications were being tripped on every bitbake invocation. To avoid
this, record which file/subdirectory we're interested in against each
watched directory so we can ignore any events for files/subdirectories
we don't care about.
Additionally, if we move up to the parent dir, ensure we haven't already
seen it before adding a watch on it (we were previously calling
watcher.add_watch() on the same directory multiple times before in a
typical OE configuration).
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
lib/bb/cooker.py | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 84bf46b..f0f9c66 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -126,12 +126,14 @@ class BBCooker:
self.configwatcher = pyinotify.WatchManager()
self.configwatcher.bbseen = []
+ self.configwatcher.bbwatchedfiles = []
self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
self.watcher = pyinotify.WatchManager()
self.watcher.bbseen = []
+ self.watcher.bbwatchedfiles = []
self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
@@ -185,6 +187,8 @@ class BBCooker:
signal.signal(signal.SIGHUP, self.sigterm_exception)
def config_notifications(self, event):
+ if not event.pathname in self.configwatcher.bbwatchedfiles:
+ return
if not event.path in self.inotify_modified_files:
self.inotify_modified_files.append(event.path)
self.baseconfig_valid = False
@@ -198,20 +202,27 @@ class BBCooker:
if not watcher:
watcher = self.watcher
for i in deps:
+ watcher.bbwatchedfiles.append(i[0])
f = os.path.dirname(i[0])
if f in watcher.bbseen:
continue
watcher.bbseen.append(f)
+ watchtarget = None
while True:
# We try and add watches for files that don't exist but if they did, would influence
# the parser. The parent directory of these files may not exist, in which case we need
# to watch any parent that does exist for changes.
try:
watcher.add_watch(f, self.watchmask, quiet=False)
+ if watchtarget:
+ watcher.bbwatchedfiles.append(watchtarget)
break
except pyinotify.WatchManagerError as e:
if 'ENOENT' in str(e):
+ watchtarget = f
f = os.path.dirname(f)
+ if f in watcher.bbseen:
+ break
watcher.bbseen.append(f)
continue
if 'ENOSPC' in str(e):
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 04/13] command: ensure sync commands that read configuration see updates
2015-08-17 11:12 [PATCH 00/13] Fixes and refactoring Paul Eggleton
` (2 preceding siblings ...)
2015-08-17 11:12 ` [PATCH 03/13] cooker: further limit inotify watches Paul Eggleton
@ 2015-08-17 11:12 ` Paul Eggleton
2015-08-17 11:12 ` [PATCH 05/13] cooker: ensure prefile/postfile can work in memory resident mode Paul Eggleton
` (9 subsequent siblings)
13 siblings, 0 replies; 17+ messages in thread
From: Paul Eggleton @ 2015-08-17 11:12 UTC (permalink / raw)
To: bitbake-devel
From: Richard Purdie <richard.purdie@linuxfoundation.org>
Add a means of ensuring that synchronous commands that read the results
of the configuration trigger a reparse of the configuration if any
underlying files have changed.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
lib/bb/command.py | 13 ++++++++++++-
lib/bb/cooker.py | 27 ++++++++++++++++-----------
2 files changed, 28 insertions(+), 12 deletions(-)
diff --git a/lib/bb/command.py b/lib/bb/command.py
index a7cac97..398c1d6 100644
--- a/lib/bb/command.py
+++ b/lib/bb/command.py
@@ -68,6 +68,8 @@ class Command:
if not hasattr(command_method, 'readonly') or False == getattr(command_method, 'readonly'):
return None, "Not able to execute not readonly commands in readonly mode"
try:
+ if getattr(command_method, 'needconfig', False):
+ self.cooker.updateCacheSync()
result = command_method(self, commandline)
except CommandError as exc:
return None, exc.args[0]
@@ -204,6 +206,7 @@ class CommandsSync:
postfiles = params[1].split()
command.cooker.configuration.prefile = prefiles
command.cooker.configuration.postfile = postfiles
+ setPrePostConfFiles.needconfig = False
def getCpuCount(self, command, params):
"""
@@ -211,10 +214,12 @@ class CommandsSync:
"""
return bb.utils.cpu_count()
getCpuCount.readonly = True
+ getCpuCount.needconfig = False
def matchFile(self, command, params):
fMatch = params[0]
return command.cooker.matchFile(fMatch)
+ matchFile.needconfig = False
def generateNewImage(self, command, params):
image = params[0]
@@ -228,6 +233,7 @@ class CommandsSync:
def ensureDir(self, command, params):
directory = params[0]
bb.utils.mkdirhier(directory)
+ ensureDir.needconfig = False
def setVarFile(self, command, params):
"""
@@ -238,6 +244,7 @@ class CommandsSync:
default_file = params[2]
op = params[3]
command.cooker.modifyConfigurationVar(var, val, default_file, op)
+ setVarFile.needconfig = False
def removeVarFile(self, command, params):
"""
@@ -245,6 +252,7 @@ class CommandsSync:
"""
var = params[0]
command.cooker.removeConfigurationVar(var)
+ removeVarFile.needconfig = False
def createConfigFile(self, command, params):
"""
@@ -252,6 +260,7 @@ class CommandsSync:
"""
name = params[0]
command.cooker.createConfigFile(name)
+ createConfigFile.needconfig = False
def setEventMask(self, command, params):
handlerNum = params[0]
@@ -259,6 +268,7 @@ class CommandsSync:
debug_domains = params[2]
mask = params[3]
return bb.event.set_UIHmask(handlerNum, llevel, debug_domains, mask)
+ setEventMask.needconfig = False
def setFeatures(self, command, params):
"""
@@ -266,7 +276,7 @@ class CommandsSync:
"""
features = params[0]
command.cooker.setFeatures(features)
-
+ setFeatures.needconfig = False
# although we change the internal state of the cooker, this is transparent since
# we always take and leave the cooker in state.initial
setFeatures.readonly = True
@@ -275,6 +285,7 @@ class CommandsSync:
options = params[0]
environment = params[1]
command.cooker.updateConfigOpts(options, environment)
+ updateConfig.needconfig = False
class CommandsAsync:
"""
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index f0f9c66..fb38289 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -1445,6 +1445,21 @@ class BBCooker:
if timestamp:
return timestr
+ def updateCacheSync(self):
+ if self.state == state.running:
+ return
+
+ # reload files for which we got notifications
+ for p in self.inotify_modified_files:
+ bb.parse.update_cache(p)
+ self.inotify_modified_files = []
+
+ if not self.baseconfig_valid:
+ logger.debug(1, "Reloading base configuration data")
+ self.initConfigurationData()
+ self.baseconfig_valid = True
+ self.parsecache_valid = False
+
# This is called for all async commands when self.state != running
def updateCache(self):
if self.state == state.running:
@@ -1456,17 +1471,7 @@ class BBCooker:
raise bb.BBHandledException()
if self.state != state.parsing:
-
- # reload files for which we got notifications
- for p in self.inotify_modified_files:
- bb.parse.update_cache(p)
- self.inotify_modified_files = []
-
- if not self.baseconfig_valid:
- logger.debug(1, "Reloading base configuration data")
- self.initConfigurationData()
- self.baseconfig_valid = True
- self.parsecache_valid = False
+ self.updateCacheSync()
if self.state != state.parsing and not self.parsecache_valid:
self.parseConfiguration ()
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 05/13] cooker: ensure prefile/postfile can work in memory resident mode
2015-08-17 11:12 [PATCH 00/13] Fixes and refactoring Paul Eggleton
` (3 preceding siblings ...)
2015-08-17 11:12 ` [PATCH 04/13] command: ensure sync commands that read configuration see updates Paul Eggleton
@ 2015-08-17 11:12 ` Paul Eggleton
2015-08-17 11:12 ` [PATCH 06/13] Fix -m handling if cannot connect to server Paul Eggleton
` (8 subsequent siblings)
13 siblings, 0 replies; 17+ messages in thread
From: Paul Eggleton @ 2015-08-17 11:12 UTC (permalink / raw)
To: bitbake-devel
The prefile/postfile options weren't working in memory resident mode
because they weren't being passed through to the server, so ensure that
they do get passed through and that the server is reset when the values
come through.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
lib/bb/cooker.py | 4 +++-
lib/bb/cookerdata.py | 3 ++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index fb38289..a2b0bb5 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -527,9 +527,11 @@ class BBCooker:
self.handleCollections( self.data.getVar("BBFILE_COLLECTIONS", True) )
def updateConfigOpts(self, options, environment):
+ clean = True
for o in options:
+ if o in ['prefile', 'postfile']:
+ clean = False
setattr(self.configuration, o, options[o])
- clean = True
for k in bb.utils.approved_variables():
if k in environment and k not in self.configuration.env:
logger.debug(1, "Updating environment variable %s to %s" % (k, environment[k]))
diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py
index 57fc6bb..b20040c 100644
--- a/lib/bb/cookerdata.py
+++ b/lib/bb/cookerdata.py
@@ -73,7 +73,8 @@ class ConfigParameters(object):
options = {}
for o in ["abort", "tryaltconfigs", "force", "invalidate_stamp",
"verbose", "debug", "dry_run", "dump_signatures",
- "debug_domains", "extra_assume_provided", "profile"]:
+ "debug_domains", "extra_assume_provided", "profile",
+ "prefile", "postfile"]:
options[o] = getattr(self.options, o)
ret, error = server.runCommand(["updateConfig", options, environment])
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 06/13] Fix -m handling if cannot connect to server
2015-08-17 11:12 [PATCH 00/13] Fixes and refactoring Paul Eggleton
` (4 preceding siblings ...)
2015-08-17 11:12 ` [PATCH 05/13] cooker: ensure prefile/postfile can work in memory resident mode Paul Eggleton
@ 2015-08-17 11:12 ` Paul Eggleton
2015-08-17 11:12 ` [PATCH 07/13] lib/bb/main: consolidate UI/server extension listing and loading Paul Eggleton
` (7 subsequent siblings)
13 siblings, 0 replies; 17+ messages in thread
From: Paul Eggleton @ 2015-08-17 11:12 UTC (permalink / raw)
To: bitbake-devel
If we can't connect to the server we should error out, because it might
not be that the server is actually dead - it might just be unable to
execute commands.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
lib/bb/main.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/lib/bb/main.py b/lib/bb/main.py
index f08a8ef..4d77408 100755
--- a/lib/bb/main.py
+++ b/lib/bb/main.py
@@ -358,8 +358,6 @@ def bitbake_main(configParams, configuration):
try:
server_connection = server.establishConnection(featureset)
except Exception as e:
- if configParams.kill_server:
- return 0
bb.fatal("Could not connect to server %s: %s" % (configParams.remote_server, str(e)))
# Restore the environment in case the UI needs it
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 07/13] lib/bb/main: consolidate UI/server extension listing and loading
2015-08-17 11:12 [PATCH 00/13] Fixes and refactoring Paul Eggleton
` (5 preceding siblings ...)
2015-08-17 11:12 ` [PATCH 06/13] Fix -m handling if cannot connect to server Paul Eggleton
@ 2015-08-17 11:12 ` Paul Eggleton
2015-08-18 9:32 ` Burton, Ross
2015-08-17 11:12 ` [PATCH 08/13] cooker: drop appliedappendlist Paul Eggleton
` (6 subsequent siblings)
13 siblings, 1 reply; 17+ messages in thread
From: Paul Eggleton @ 2015-08-17 11:12 UTC (permalink / raw)
To: bitbake-devel
Provide us with a means of showing the list of UIs / server choices for
the command line help, and do the processing in one place for both.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
lib/bb/main.py | 78 ++++++++++++++++++++++++++++++++++++----------------------
1 file changed, 48 insertions(+), 30 deletions(-)
diff --git a/lib/bb/main.py b/lib/bb/main.py
index 4d77408..2b720aa 100755
--- a/lib/bb/main.py
+++ b/lib/bb/main.py
@@ -41,22 +41,45 @@ logger = logging.getLogger("BitBake")
class BBMainException(Exception):
pass
-def get_ui(config):
- if not config.ui:
- # modify 'ui' attribute because it is also read by cooker
- config.ui = os.environ.get('BITBAKE_UI', 'knotty')
-
- interface = config.ui
-
+def list_extension_modules(pkg, checkattr):
+ """
+ Lists extension modules in a specific Python package
+ (e.g. UIs, servers)
+ Parameters:
+ pkg: previously imported Python package to list
+ checkattr: attribute to look for in module to determine if it's valid
+ as the type of extension you are looking for
+ """
+ import pkgutil
+ pkgdir = os.path.dirname(pkg.__file__)
+
+ modules = []
+ for _, modulename, _ in pkgutil.iter_modules([pkgdir]):
+ if os.path.isdir(os.path.join(pkgdir, modulename)):
+ # ignore directories
+ continue
+ try:
+ module = __import__(pkg.__name__, fromlist=[modulename])
+ except ImportError:
+ # If we can't import it, it's not valid
+ continue
+ module_if = getattr(module, modulename)
+ if getattr(module_if, 'hidden_extension', False):
+ continue
+ if not checkattr or hasattr(module_if, checkattr):
+ modules.append(modulename)
+ return modules
+
+def import_extension_module(pkg, modulename):
try:
# Dynamically load the UI based on the ui name. Although we
# suggest a fixed set this allows you to have flexibility in which
# ones are available.
- module = __import__("bb.ui", fromlist = [interface])
- return getattr(module, interface)
+ module = __import__(pkg.__name__, fromlist = [modulename])
+ return getattr(module, modulename)
except AttributeError:
- raise BBMainException("FATAL: Invalid user interface '%s' specified.\n"
- "Valid interfaces: depexp, goggle, ncurses, hob, knotty [default]." % interface)
+ raise BBMainException("FATAL: Unable to import extension module %s from %s" % (modulename, pkg.__name__))
+
# Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring others"""
@@ -146,11 +169,19 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
parser.add_option("-P", "--profile", help = "Profile the command and save reports.",
action = "store_true", dest = "profile", default = False)
- parser.add_option("-u", "--ui", help = "The user interface to use (e.g. knotty, hob, depexp).",
- action = "store", dest = "ui")
+ env_ui = os.environ.get('BITBAKE_UI', None)
+ valid_uis = list_extension_modules(bb.ui, 'main')
+ default_ui = env_ui or 'knotty'
+ if env_ui and not env_ui in valid_uis:
+ raise BBMainException('Invalid UI "%s" specified in BITBAKE_UI environment variable - valid choices: %s' % (env_ui, ', '.join(valid_uis)))
+ elif not default_ui in valid_uis:
+ raise BBMainException('Default UI "%s" could not be found')
+ parser.add_option("-u", "--ui", help = "The user interface to use (%s)." % ', '.join(valid_uis),
+ action="store", dest="ui", type="choice", choices=valid_uis, default=default_ui)
- parser.add_option("-t", "--servertype", help = "Choose which server to use, process or xmlrpc.",
- action = "store", dest = "servertype")
+ valid_server_types = list_extension_modules(bb.server, 'BitBakeServer')
+ parser.add_option("-t", "--servertype", help = "Choose which server type to use (%s)." % ', '.join(valid_server_types),
+ action = "store", dest = "servertype", default = "process")
parser.add_option("", "--token", help = "Specify the connection token to be used when connecting to a remote server.",
action = "store", dest = "xmlrpctoken")
@@ -279,21 +310,8 @@ def bitbake_main(configParams, configuration):
configuration.setConfigParameters(configParams)
- ui_module = get_ui(configParams)
-
- # Server type can be xmlrpc or process currently, if nothing is specified,
- # the default server is process
- if configParams.servertype:
- server_type = configParams.servertype
- else:
- server_type = 'process'
-
- try:
- module = __import__("bb.server", fromlist = [server_type])
- servermodule = getattr(module, server_type)
- except AttributeError:
- raise BBMainException("FATAL: Invalid server type '%s' specified.\n"
- "Valid interfaces: xmlrpc, process [default]." % server_type)
+ ui_module = import_extension_module(bb.ui, configParams.ui)
+ servermodule = import_extension_module(bb.server, configParams.servertype)
if configParams.server_only:
if configParams.servertype != "xmlrpc":
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* Re: [PATCH 07/13] lib/bb/main: consolidate UI/server extension listing and loading
2015-08-17 11:12 ` [PATCH 07/13] lib/bb/main: consolidate UI/server extension listing and loading Paul Eggleton
@ 2015-08-18 9:32 ` Burton, Ross
2015-08-18 10:29 ` [PATCH v2] " Paul Eggleton
0 siblings, 1 reply; 17+ messages in thread
From: Burton, Ross @ 2015-08-18 9:32 UTC (permalink / raw)
To: Paul Eggleton; +Cc: bitbake-devel
[-- Attachment #1: Type: text/plain, Size: 6337 bytes --]
It looks like this is responsible for some autobuilder failures where
bitbake decides to use hob instead of knotty2 and aborts on start up with:
FATAL: Hob requires Gtk+ 2.20.0 or higher, PyGtk 2.21.0 or higher (No
module named gobject).
Ross
On 17 August 2015 at 12:12, Paul Eggleton <paul.eggleton@linux.intel.com>
wrote:
> Provide us with a means of showing the list of UIs / server choices for
> the command line help, and do the processing in one place for both.
>
> Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
> ---
> lib/bb/main.py | 78
> ++++++++++++++++++++++++++++++++++++----------------------
> 1 file changed, 48 insertions(+), 30 deletions(-)
>
> diff --git a/lib/bb/main.py b/lib/bb/main.py
> index 4d77408..2b720aa 100755
> --- a/lib/bb/main.py
> +++ b/lib/bb/main.py
> @@ -41,22 +41,45 @@ logger = logging.getLogger("BitBake")
> class BBMainException(Exception):
> pass
>
> -def get_ui(config):
> - if not config.ui:
> - # modify 'ui' attribute because it is also read by cooker
> - config.ui = os.environ.get('BITBAKE_UI', 'knotty')
> -
> - interface = config.ui
> -
> +def list_extension_modules(pkg, checkattr):
> + """
> + Lists extension modules in a specific Python package
> + (e.g. UIs, servers)
> + Parameters:
> + pkg: previously imported Python package to list
> + checkattr: attribute to look for in module to determine if it's
> valid
> + as the type of extension you are looking for
> + """
> + import pkgutil
> + pkgdir = os.path.dirname(pkg.__file__)
> +
> + modules = []
> + for _, modulename, _ in pkgutil.iter_modules([pkgdir]):
> + if os.path.isdir(os.path.join(pkgdir, modulename)):
> + # ignore directories
> + continue
> + try:
> + module = __import__(pkg.__name__, fromlist=[modulename])
> + except ImportError:
> + # If we can't import it, it's not valid
> + continue
> + module_if = getattr(module, modulename)
> + if getattr(module_if, 'hidden_extension', False):
> + continue
> + if not checkattr or hasattr(module_if, checkattr):
> + modules.append(modulename)
> + return modules
> +
> +def import_extension_module(pkg, modulename):
> try:
> # Dynamically load the UI based on the ui name. Although we
> # suggest a fixed set this allows you to have flexibility in which
> # ones are available.
> - module = __import__("bb.ui", fromlist = [interface])
> - return getattr(module, interface)
> + module = __import__(pkg.__name__, fromlist = [modulename])
> + return getattr(module, modulename)
> except AttributeError:
> - raise BBMainException("FATAL: Invalid user interface '%s'
> specified.\n"
> - "Valid interfaces: depexp, goggle, ncurses, hob, knotty
> [default]." % interface)
> + raise BBMainException("FATAL: Unable to import extension module
> %s from %s" % (modulename, pkg.__name__))
> +
>
>
> # Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring
> others"""
> @@ -146,11 +169,19 @@ class
> BitBakeConfigParameters(cookerdata.ConfigParameters):
> parser.add_option("-P", "--profile", help = "Profile the command
> and save reports.",
> action = "store_true", dest = "profile", default =
> False)
>
> - parser.add_option("-u", "--ui", help = "The user interface to use
> (e.g. knotty, hob, depexp).",
> - action = "store", dest = "ui")
> + env_ui = os.environ.get('BITBAKE_UI', None)
> + valid_uis = list_extension_modules(bb.ui, 'main')
> + default_ui = env_ui or 'knotty'
> + if env_ui and not env_ui in valid_uis:
> + raise BBMainException('Invalid UI "%s" specified in
> BITBAKE_UI environment variable - valid choices: %s' % (env_ui, ',
> '.join(valid_uis)))
> + elif not default_ui in valid_uis:
> + raise BBMainException('Default UI "%s" could not be found')
> + parser.add_option("-u", "--ui", help = "The user interface to use
> (%s)." % ', '.join(valid_uis),
> + action="store", dest="ui", type="choice",
> choices=valid_uis, default=default_ui)
>
> - parser.add_option("-t", "--servertype", help = "Choose which
> server to use, process or xmlrpc.",
> - action = "store", dest = "servertype")
> + valid_server_types = list_extension_modules(bb.server,
> 'BitBakeServer')
> + parser.add_option("-t", "--servertype", help = "Choose which
> server type to use (%s)." % ', '.join(valid_server_types),
> + action = "store", dest = "servertype", default =
> "process")
>
> parser.add_option("", "--token", help = "Specify the connection
> token to be used when connecting to a remote server.",
> action = "store", dest = "xmlrpctoken")
> @@ -279,21 +310,8 @@ def bitbake_main(configParams, configuration):
>
> configuration.setConfigParameters(configParams)
>
> - ui_module = get_ui(configParams)
> -
> - # Server type can be xmlrpc or process currently, if nothing is
> specified,
> - # the default server is process
> - if configParams.servertype:
> - server_type = configParams.servertype
> - else:
> - server_type = 'process'
> -
> - try:
> - module = __import__("bb.server", fromlist = [server_type])
> - servermodule = getattr(module, server_type)
> - except AttributeError:
> - raise BBMainException("FATAL: Invalid server type '%s'
> specified.\n"
> - "Valid interfaces: xmlrpc, process
> [default]." % server_type)
> + ui_module = import_extension_module(bb.ui, configParams.ui)
> + servermodule = import_extension_module(bb.server,
> configParams.servertype)
>
> if configParams.server_only:
> if configParams.servertype != "xmlrpc":
> --
> 2.1.0
>
> --
> _______________________________________________
> bitbake-devel mailing list
> bitbake-devel@lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/bitbake-devel
>
[-- Attachment #2: Type: text/html, Size: 8104 bytes --]
^ permalink raw reply [flat|nested] 17+ messages in thread* [PATCH v2] lib/bb/main: consolidate UI/server extension listing and loading
2015-08-18 9:32 ` Burton, Ross
@ 2015-08-18 10:29 ` Paul Eggleton
0 siblings, 0 replies; 17+ messages in thread
From: Paul Eggleton @ 2015-08-18 10:29 UTC (permalink / raw)
To: bitbake-devel
Provide us with a means of showing the list of UIs / server choices for
the command line help, and do the processing in one place for both.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
Changes since v1:
- Handle SystemExit when trying to import the ui module i.e.
don't blow up when we try to import the hob module and PyGTK+
isn't installed
- Format the options a little bit more nicely in the help text
(also pushed to the paule/tinfoil2-pre branch)
lib/bb/main.py | 85 +++++++++++++++++++++++++++++++++++++---------------------
1 file changed, 54 insertions(+), 31 deletions(-)
diff --git a/lib/bb/main.py b/lib/bb/main.py
index 4d77408..c98cf44 100755
--- a/lib/bb/main.py
+++ b/lib/bb/main.py
@@ -41,22 +41,44 @@ logger = logging.getLogger("BitBake")
class BBMainException(Exception):
pass
-def get_ui(config):
- if not config.ui:
- # modify 'ui' attribute because it is also read by cooker
- config.ui = os.environ.get('BITBAKE_UI', 'knotty')
-
- interface = config.ui
-
+def list_extension_modules(pkg, checkattr):
+ """
+ Lists extension modules in a specific Python package
+ (e.g. UIs, servers)
+ Parameters:
+ pkg: previously imported Python package to list
+ checkattr: attribute to look for in module to determine if it's valid
+ as the type of extension you are looking for
+ """
+ import pkgutil
+ pkgdir = os.path.dirname(pkg.__file__)
+
+ modules = []
+ for _, modulename, _ in pkgutil.iter_modules([pkgdir]):
+ if os.path.isdir(os.path.join(pkgdir, modulename)):
+ # ignore directories
+ continue
+ try:
+ module = __import__(pkg.__name__, fromlist=[modulename])
+ except (ImportError, SystemExit):
+ # If we can't import it, it's not valid
+ continue
+ module_if = getattr(module, modulename)
+ if getattr(module_if, 'hidden_extension', False):
+ continue
+ if not checkattr or hasattr(module_if, checkattr):
+ modules.append(modulename)
+ return modules
+
+def import_extension_module(pkg, modulename):
try:
# Dynamically load the UI based on the ui name. Although we
# suggest a fixed set this allows you to have flexibility in which
# ones are available.
- module = __import__("bb.ui", fromlist = [interface])
- return getattr(module, interface)
+ module = __import__(pkg.__name__, fromlist = [modulename])
+ return getattr(module, modulename)
except AttributeError:
- raise BBMainException("FATAL: Invalid user interface '%s' specified.\n"
- "Valid interfaces: depexp, goggle, ncurses, hob, knotty [default]." % interface)
+ raise BBMainException("FATAL: Unable to import extension module %s from %s" % (modulename, pkg.__name__))
# Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring others"""
@@ -146,11 +168,25 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
parser.add_option("-P", "--profile", help = "Profile the command and save reports.",
action = "store_true", dest = "profile", default = False)
- parser.add_option("-u", "--ui", help = "The user interface to use (e.g. knotty, hob, depexp).",
- action = "store", dest = "ui")
-
- parser.add_option("-t", "--servertype", help = "Choose which server to use, process or xmlrpc.",
- action = "store", dest = "servertype")
+ def present_options(optionlist):
+ if len(optionlist) > 1:
+ return ' or '.join([', '.join(optionlist[:-1]), optionlist[-1]])
+ else:
+ return optionlist[0]
+
+ env_ui = os.environ.get('BITBAKE_UI', None)
+ valid_uis = list_extension_modules(bb.ui, 'main')
+ default_ui = env_ui or 'knotty'
+ if env_ui and not env_ui in valid_uis:
+ raise BBMainException('Invalid UI "%s" specified in BITBAKE_UI environment variable - valid choices: %s' % (env_ui, present_options(valid_uis)))
+ elif not default_ui in valid_uis:
+ raise BBMainException('Default UI "%s" could not be found')
+ parser.add_option("-u", "--ui", help = "The user interface to use (%s - default %%default)." % present_options(valid_uis),
+ action="store", dest="ui", type="choice", choices=valid_uis, default=default_ui)
+
+ valid_server_types = list_extension_modules(bb.server, 'BitBakeServer')
+ parser.add_option("-t", "--servertype", help = "Choose which server type to use (%s - default %%default)." % present_options(valid_server_types),
+ action = "store", dest = "servertype", default = "process")
parser.add_option("", "--token", help = "Specify the connection token to be used when connecting to a remote server.",
action = "store", dest = "xmlrpctoken")
@@ -279,21 +315,8 @@ def bitbake_main(configParams, configuration):
configuration.setConfigParameters(configParams)
- ui_module = get_ui(configParams)
-
- # Server type can be xmlrpc or process currently, if nothing is specified,
- # the default server is process
- if configParams.servertype:
- server_type = configParams.servertype
- else:
- server_type = 'process'
-
- try:
- module = __import__("bb.server", fromlist = [server_type])
- servermodule = getattr(module, server_type)
- except AttributeError:
- raise BBMainException("FATAL: Invalid server type '%s' specified.\n"
- "Valid interfaces: xmlrpc, process [default]." % server_type)
+ ui_module = import_extension_module(bb.ui, configParams.ui)
+ servermodule = import_extension_module(bb.server, configParams.servertype)
if configParams.server_only:
if configParams.servertype != "xmlrpc":
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread
* [PATCH 08/13] cooker: drop appliedappendlist
2015-08-17 11:12 [PATCH 00/13] Fixes and refactoring Paul Eggleton
` (6 preceding siblings ...)
2015-08-17 11:12 ` [PATCH 07/13] lib/bb/main: consolidate UI/server extension listing and loading Paul Eggleton
@ 2015-08-17 11:12 ` Paul Eggleton
2015-08-17 11:12 ` [PATCH 09/13] cooker: drop appendlist Paul Eggleton
` (5 subsequent siblings)
13 siblings, 0 replies; 17+ messages in thread
From: Paul Eggleton @ 2015-08-17 11:12 UTC (permalink / raw)
To: bitbake-devel
Whilst collecting this list on the fly may be quicker, doing so within a
function that's meant to *query* the list of bbappends is poor practice.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
lib/bb/cooker.py | 28 ++++++++++++++++++----------
1 file changed, 18 insertions(+), 10 deletions(-)
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index a2b0bb5..5452deb 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -919,15 +919,25 @@ class BBCooker:
print("}", file=tdepends_file)
logger.info("Task dependencies saved to 'task-depends.dot'")
- def show_appends_with_no_recipes( self ):
- appends_without_recipes = [self.collection.appendlist[recipe]
- for recipe in self.collection.appendlist
- if recipe not in self.collection.appliedappendlist]
+ def show_appends_with_no_recipes(self):
+ # Determine which bbappends haven't been applied
+
+ # First get list of recipes, including skipped
+ recipefns = self.recipecache.pkg_fn.keys()
+ recipefns.extend(self.skiplist.keys())
+
+ # Work out list of bbappends that have been applied
+ applied_appends = []
+ for fn in recipefns:
+ applied_appends.extend(self.collection.get_file_appends(fn))
+
+ appends_without_recipes = []
+ for _, appendfn in self.collection.bbappends:
+ if not appendfn in applied_appends:
+ appends_without_recipes.append(appendfn)
+
if appends_without_recipes:
- appendlines = (' %s' % append
- for appends in appends_without_recipes
- for append in appends)
- msg = 'No recipes available for:\n%s' % '\n'.join(appendlines)
+ msg = 'No recipes available for:\n %s' % '\n '.join(appends_without_recipes)
warn_only = self.data.getVar("BB_DANGLINGAPPENDS_WARNONLY", \
False) or "no"
if warn_only.lower() in ("1", "yes", "true"):
@@ -1646,7 +1656,6 @@ class CookerCollectFiles(object):
def __init__(self, priorities):
self.appendlist = {}
self.bbappends = []
- self.appliedappendlist = []
self.bbfile_config_priorities = priorities
def calc_bbfile_priority( self, filename, matched = None ):
@@ -1769,7 +1778,6 @@ class CookerCollectFiles(object):
for b in self.bbappends:
(bbappend, filename) = b
if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])):
- self.appliedappendlist.append(bbappend)
filelist.append(filename)
return filelist
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 09/13] cooker: drop appendlist
2015-08-17 11:12 [PATCH 00/13] Fixes and refactoring Paul Eggleton
` (7 preceding siblings ...)
2015-08-17 11:12 ` [PATCH 08/13] cooker: drop appliedappendlist Paul Eggleton
@ 2015-08-17 11:12 ` Paul Eggleton
2015-08-17 11:12 ` [PATCH 10/13] bitbake-layers: Convert flatten to use collections.bbappends Paul Eggleton
` (4 subsequent siblings)
13 siblings, 0 replies; 17+ messages in thread
From: Paul Eggleton @ 2015-08-17 11:12 UTC (permalink / raw)
To: bitbake-devel
Now we have the bbappends list and all users have been converted over to
use it, we don't need this anymore.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
lib/bb/cooker.py | 5 -----
1 file changed, 5 deletions(-)
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 5452deb..6dc9f19 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -1654,7 +1654,6 @@ class CookerExit(bb.event.Event):
class CookerCollectFiles(object):
def __init__(self, priorities):
- self.appendlist = {}
self.bbappends = []
self.bbfile_config_priorities = priorities
@@ -1750,10 +1749,6 @@ class CookerCollectFiles(object):
for f in bbappend:
base = os.path.basename(f).replace('.bbappend', '.bb')
self.bbappends.append((base, f))
- if not base in self.appendlist:
- self.appendlist[base] = []
- if f not in self.appendlist[base]:
- self.appendlist[base].append(f)
# Find overlayed recipes
# bbfiles will be in priority order which makes this easy
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 10/13] bitbake-layers: Convert flatten to use collections.bbappends
2015-08-17 11:12 [PATCH 00/13] Fixes and refactoring Paul Eggleton
` (8 preceding siblings ...)
2015-08-17 11:12 ` [PATCH 09/13] cooker: drop appendlist Paul Eggleton
@ 2015-08-17 11:12 ` Paul Eggleton
2015-08-17 11:12 ` [PATCH 11/13] bitbake-layers: refactor show-appends to stop using cooker bbappends list Paul Eggleton
` (3 subsequent siblings)
13 siblings, 0 replies; 17+ messages in thread
From: Paul Eggleton @ 2015-08-17 11:12 UTC (permalink / raw)
To: bitbake-devel
From: Richard Purdie <richard.purdie@linuxfoundation.org>
flatten support currently looks broken since it doesn't appear to
deal with handling "%" support in bbappend file names.
This patch converts it to use collections.get_file_appends() which
correctly handles "%" support.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
bin/bitbake-layers | 44 ++++++++++++++++++++------------------------
1 file changed, 20 insertions(+), 24 deletions(-)
diff --git a/bin/bitbake-layers b/bin/bitbake-layers
index 5116e59..62b51b0 100755
--- a/bin/bitbake-layers
+++ b/bin/bitbake-layers
@@ -595,7 +595,7 @@ build results (as the layer priority order has effectively changed).
return layerdir
return None
- appended_recipes = []
+ applied_appends = []
for layer in layers:
overlayed = []
for f in self.bbhandler.cooker.collection.overlayed.iterkeys():
@@ -623,31 +623,27 @@ build results (as the layer priority order has effectively changed).
logger.warn('Overwriting file %s', fdest)
bb.utils.copyfile(f1full, fdest)
if ext == '.bb':
- if f1 in self.bbhandler.cooker.collection.appendlist:
- appends = self.bbhandler.cooker.collection.appendlist[f1]
- if appends:
- logger.plain(' Applying appends to %s' % fdest )
- for appendname in appends:
- if layer_path_match(appendname):
- self.apply_append(appendname, fdest)
- appended_recipes.append(f1)
+ for append in self.bbhandler.cooker.collection.get_file_appends(f1full):
+ if layer_path_match(append):
+ logger.plain(' Applying append %s to %s' % (append, fdest))
+ self.apply_append(append, fdest)
+ applied_appends.append(append)
# Take care of when some layers are excluded and yet we have included bbappends for those recipes
- for recipename in self.bbhandler.cooker.collection.appendlist.iterkeys():
- if recipename not in appended_recipes:
- appends = self.bbhandler.cooker.collection.appendlist[recipename]
+ for b in self.bbhandler.cooker.collection.bbappends:
+ (recipename, appendname) = b
+ if appendname not in applied_appends:
first_append = None
- for appendname in appends:
- layer = layer_path_match(appendname)
- if layer:
- if first_append:
- self.apply_append(appendname, first_append)
- else:
- fdest = appendname[len(layer):]
- fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
- bb.utils.mkdirhier(os.path.dirname(fdest))
- bb.utils.copyfile(appendname, fdest)
- first_append = fdest
+ layer = layer_path_match(appendname)
+ if layer:
+ if first_append:
+ self.apply_append(appendname, first_append)
+ else:
+ fdest = appendname[len(layer):]
+ fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
+ bb.utils.mkdirhier(os.path.dirname(fdest))
+ bb.utils.copyfile(appendname, fdest)
+ first_append = fdest
# Get the regex for the first layer in our list (which is where the conf/layer.conf file will
# have come from)
@@ -723,7 +719,7 @@ build results (as the layer priority order has effectively changed).
Lists recipes with the bbappends that apply to them as subitems.
"""
self.init_bbhandler()
- if not self.bbhandler.cooker.collection.appendlist:
+ if not self.bbhandler.cooker.collection.bbappends:
logger.plain('No append files found')
return 0
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 11/13] bitbake-layers: refactor show-appends to stop using cooker bbappends list
2015-08-17 11:12 [PATCH 00/13] Fixes and refactoring Paul Eggleton
` (9 preceding siblings ...)
2015-08-17 11:12 ` [PATCH 10/13] bitbake-layers: Convert flatten to use collections.bbappends Paul Eggleton
@ 2015-08-17 11:12 ` Paul Eggleton
2015-08-17 11:12 ` [PATCH 12/13] bitbake-layers: use "with open" consistently Paul Eggleton
` (2 subsequent siblings)
13 siblings, 0 replies; 17+ messages in thread
From: Paul Eggleton @ 2015-08-17 11:12 UTC (permalink / raw)
To: bitbake-devel
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
bin/bitbake-layers | 21 +++++++++++++--------
1 file changed, 13 insertions(+), 8 deletions(-)
diff --git a/bin/bitbake-layers b/bin/bitbake-layers
index 62b51b0..8cf7196 100755
--- a/bin/bitbake-layers
+++ b/bin/bitbake-layers
@@ -719,18 +719,21 @@ build results (as the layer priority order has effectively changed).
Lists recipes with the bbappends that apply to them as subitems.
"""
self.init_bbhandler()
- if not self.bbhandler.cooker.collection.bbappends:
- logger.plain('No append files found')
- return 0
logger.plain('=== Appended recipes ===')
pnlist = list(self.bbhandler.cooker_data.pkg_pn.keys())
pnlist.sort()
+ appends = False
for pn in pnlist:
- self.show_appends_for_pn(pn)
+ if self.show_appends_for_pn(pn):
+ appends = True
+
+ if self.show_appends_for_skipped():
+ appends = True
- self.show_appends_for_skipped()
+ if not appends:
+ logger.plain('No append files found')
def show_appends_for_pn(self, pn):
filenames = self.bbhandler.cooker_data.pkg_pn[pn]
@@ -741,12 +744,12 @@ Lists recipes with the bbappends that apply to them as subitems.
self.bbhandler.cooker_data.pkg_pn)
best_filename = os.path.basename(best[3])
- self.show_appends_output(filenames, best_filename)
+ return self.show_appends_output(filenames, best_filename)
def show_appends_for_skipped(self):
filenames = [os.path.basename(f)
for f in self.bbhandler.cooker.skiplist.iterkeys()]
- self.show_appends_output(filenames, None, " (skipped)")
+ return self.show_appends_output(filenames, None, " (skipped)")
def show_appends_output(self, filenames, best_filename, name_suffix = ''):
appended, missing = self.get_appends_for_files(filenames)
@@ -760,7 +763,9 @@ Lists recipes with the bbappends that apply to them as subitems.
if best_filename in missing:
logger.warn('%s: missing append for preferred version',
best_filename)
-
+ return True
+ else:
+ return False
def get_appends_for_files(self, filenames):
appended, notappended = [], []
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 12/13] bitbake-layers: use "with open" consistently
2015-08-17 11:12 [PATCH 00/13] Fixes and refactoring Paul Eggleton
` (10 preceding siblings ...)
2015-08-17 11:12 ` [PATCH 11/13] bitbake-layers: refactor show-appends to stop using cooker bbappends list Paul Eggleton
@ 2015-08-17 11:12 ` Paul Eggleton
2015-08-17 11:12 ` [PATCH 13/13] lib/bb/parse: properly handle OSError when updating mtime cache Paul Eggleton
2015-08-17 17:14 ` [PATCH 00/13] Fixes and refactoring Christopher Larson
13 siblings, 0 replies; 17+ messages in thread
From: Paul Eggleton @ 2015-08-17 11:12 UTC (permalink / raw)
To: bitbake-devel
It's best practice to use "with open..." rather than open & close where
practical.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
bin/bitbake-layers | 62 +++++++++++++++++++++++++-----------------------------
1 file changed, 29 insertions(+), 33 deletions(-)
diff --git a/bin/bitbake-layers b/bin/bitbake-layers
index 8cf7196..5518c63 100755
--- a/bin/bitbake-layers
+++ b/bin/bitbake-layers
@@ -705,13 +705,11 @@ build results (as the layer priority order has effectively changed).
return os.path.basename(layerdir.rstrip(os.sep))
def apply_append(self, appendname, recipename):
- appendfile = open(appendname, 'r')
- recipefile = open(recipename, 'a')
- recipefile.write('\n')
- recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname))
- recipefile.writelines(appendfile.readlines())
- recipefile.close()
- appendfile.close()
+ with open(appendname, 'r') as appendfile:
+ with open(recipename, 'a') as recipefile:
+ recipefile.write('\n')
+ recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname))
+ recipefile.writelines(appendfile.readlines())
def do_show_appends(self, args):
"""list bbappend files and recipe files they apply to
@@ -879,20 +877,19 @@ NOTE: .bbappend files can impact the dependencies.
# The 'require/include xxx' in the bb file
pv_re = re.compile(r"\${PV}")
- fnfile = open(f, 'r')
- line = fnfile.readline()
- while line:
- m, keyword = self.match_require_include(line)
- # Found the 'require/include xxxx'
- if m:
- needed_file = m.group(1)
- # Replace the ${PV} with the real PV
- if pv_re.search(needed_file) and f in self.bbhandler.cooker_data.pkg_pepvpr:
- pv = self.bbhandler.cooker_data.pkg_pepvpr[f][1]
- needed_file = re.sub(r"\${PV}", pv, needed_file)
- self.print_cross_files(bbpath, keyword, layername, f, needed_file, args.filenames, ignore_layers)
+ with open(f, 'r') as fnfile:
line = fnfile.readline()
- fnfile.close()
+ while line:
+ m, keyword = self.match_require_include(line)
+ # Found the 'require/include xxxx'
+ if m:
+ needed_file = m.group(1)
+ # Replace the ${PV} with the real PV
+ if pv_re.search(needed_file) and f in self.bbhandler.cooker_data.pkg_pepvpr:
+ pv = self.bbhandler.cooker_data.pkg_pepvpr[f][1]
+ needed_file = re.sub(r"\${PV}", pv, needed_file)
+ self.print_cross_files(bbpath, keyword, layername, f, needed_file, args.filenames, ignore_layers)
+ line = fnfile.readline()
# The "require/include xxx" in conf/machine/*.conf, .inc and .bbclass
conf_re = re.compile(".*/conf/machine/[^\/]*\.conf$")
@@ -906,20 +903,19 @@ NOTE: .bbappend files can impact the dependencies.
f = os.path.join(dirpath, name)
s = conf_re.match(f) or inc_re.match(f) or bbclass_re.match(f)
if s:
- ffile = open(f, 'r')
- line = ffile.readline()
- while line:
- m, keyword = self.match_require_include(line)
- # Only bbclass has the "inherit xxx" here.
- bbclass=""
- if not m and f.endswith(".bbclass"):
- m, keyword = self.match_inherit(line)
- bbclass=".bbclass"
- # Find a 'require/include xxxx'
- if m:
- self.print_cross_files(bbpath, keyword, layername, f, m.group(1) + bbclass, args.filenames, ignore_layers)
+ with open(f, 'r') as ffile:
line = ffile.readline()
- ffile.close()
+ while line:
+ m, keyword = self.match_require_include(line)
+ # Only bbclass has the "inherit xxx" here.
+ bbclass=""
+ if not m and f.endswith(".bbclass"):
+ m, keyword = self.match_inherit(line)
+ bbclass=".bbclass"
+ # Find a 'require/include xxxx'
+ if m:
+ self.print_cross_files(bbpath, keyword, layername, f, m.group(1) + bbclass, args.filenames, ignore_layers)
+ line = ffile.readline()
def print_cross_files(self, bbpath, keyword, layername, f, needed_filename, show_filenames, ignore_layers):
"""Print the depends that crosses a layer boundary"""
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 13/13] lib/bb/parse: properly handle OSError when updating mtime cache
2015-08-17 11:12 [PATCH 00/13] Fixes and refactoring Paul Eggleton
` (11 preceding siblings ...)
2015-08-17 11:12 ` [PATCH 12/13] bitbake-layers: use "with open" consistently Paul Eggleton
@ 2015-08-17 11:12 ` Paul Eggleton
2015-08-17 17:14 ` [PATCH 00/13] Fixes and refactoring Christopher Larson
13 siblings, 0 replies; 17+ messages in thread
From: Paul Eggleton @ 2015-08-17 11:12 UTC (permalink / raw)
To: bitbake-devel
If a file no longer exists, drop it from the cache silently instead of
generating a traceback. This was visible in some cases when a recipe was
deleted when bitbake was resident in memory.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
lib/bb/parse/__init__.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/lib/bb/parse/__init__.py b/lib/bb/parse/__init__.py
index 1becaa4..67ec71f 100644
--- a/lib/bb/parse/__init__.py
+++ b/lib/bb/parse/__init__.py
@@ -71,7 +71,12 @@ def cached_mtime_noerror(f):
return __mtime_cache[f]
def update_mtime(f):
- __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
+ try:
+ __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
+ except OSError:
+ if f in __mtime_cache:
+ del __mtime_cache[f]
+ return 0
return __mtime_cache[f]
def update_cache(f):
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* Re: [PATCH 00/13] Fixes and refactoring
2015-08-17 11:12 [PATCH 00/13] Fixes and refactoring Paul Eggleton
` (12 preceding siblings ...)
2015-08-17 11:12 ` [PATCH 13/13] lib/bb/parse: properly handle OSError when updating mtime cache Paul Eggleton
@ 2015-08-17 17:14 ` Christopher Larson
13 siblings, 0 replies; 17+ messages in thread
From: Christopher Larson @ 2015-08-17 17:14 UTC (permalink / raw)
To: Paul Eggleton; +Cc: bitbake-devel@lists.openembedded.org
[-- Attachment #1: Type: text/plain, Size: 1891 bytes --]
On Mon, Aug 17, 2015 at 4:12 AM, Paul Eggleton <
paul.eggleton@linux.intel.com> wrote:
> Here are a bunch of bugfixes and refactoring patches that I was going to
> send as part of reworking tinfoil. Unfortunately that reworking isn't
> complete and I don't have time to finish it right now; here is what I do
> have that can stand on its own. Most of it relates to memory resident
> mode or tidying up code that calls into cooker.
>
>
> The following changes since commit
> 04e896d7fb170271fb09dae5c2a42acb4b68f513:
>
> toastergui: Added IDs to elements used in testing (2015-08-17 08:47:51
> +0100)
>
> are available in the git repository at:
>
> git://git.yoctoproject.org/poky-contrib paule/tinfoil2-pre
>
> http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/tinfoil2-pre
>
> Paul Eggleton (11):
> Fix default function parameter assignment to a list
> command: intercept SystemExit to avoid trashing the server
> cooker: further limit inotify watches
> cooker: ensure prefile/postfile can work in memory resident mode
> Fix -m handling if cannot connect to server
> lib/bb/main: consolidate UI/server extension listing and loading
> cooker: drop appliedappendlist
> cooker: drop appendlist
> bitbake-layers: refactor show-appends to stop using cooker bbappends list
> bitbake-layers: use "with open" consistently
> lib/bb/parse: properly handle OSError when updating mtime cache
>
> Richard Purdie (2):
> command: ensure sync commands that read configuration see updates
> bitbake-layers: Convert flatten to use collections.bbappends
>
For what it's worth, these all look sane to me.
Acked-by: Christopher Larson <chris_larson@mentor.com>
--
Christopher Larson
clarson at kergoth dot com
Founder - BitBake, OpenEmbedded, OpenZaurus
Maintainer - Tslib
Senior Software Engineer, Mentor Graphics
[-- Attachment #2: Type: text/html, Size: 2712 bytes --]
^ permalink raw reply [flat|nested] 17+ messages in thread