* [PATCH 1/8] cache: Use configuration's hash value to validate cache
2012-01-11 3:03 [PATCH 0/8 v2][PULL] Hob: bitbake related changes Dongxiao Xu
@ 2012-01-11 3:03 ` Dongxiao Xu
2012-01-11 3:03 ` [PATCH 2/8] command.py: add resolve option for generateTargetsTree API Dongxiao Xu
` (6 subsequent siblings)
7 siblings, 0 replies; 16+ messages in thread
From: Dongxiao Xu @ 2012-01-11 3:03 UTC (permalink / raw)
To: bitbake-devel
Previously we use the file time stamp to judge if a cache is valid.
Here this commit introduce a new method, which calculates the total
hash value for a certain configuration's key/value paris, and tag
it into cache filename, for example, bb_cache.dat.xxxyyyzzz.
This mechanism also ensures the cache's correctness if user
dynamically setting variables from some frontend GUI, like HOB.
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
---
lib/bb/cache.py | 32 ++++++++++++--------------------
lib/bb/cooker.py | 4 +++-
lib/bb/data_smart.py | 21 +++++++++++++++++++++
3 files changed, 36 insertions(+), 21 deletions(-)
diff --git a/lib/bb/cache.py b/lib/bb/cache.py
index 6b7fa6f..955b6df 100644
--- a/lib/bb/cache.py
+++ b/lib/bb/cache.py
@@ -42,10 +42,10 @@ except ImportError:
logger.info("Importing cPickle failed. "
"Falling back to a very slow implementation.")
-__cache_version__ = "142"
+__cache_version__ = "143"
-def getCacheFile(path, filename):
- return os.path.join(path, filename)
+def getCacheFile(path, filename, data_hash):
+ return os.path.join(path, filename + "." + data_hash)
# RecipeInfoCommon defines common data retrieving methods
# from meta data for caches. CoreRecipeInfo as well as other
@@ -254,7 +254,7 @@ class Cache(object):
BitBake Cache implementation
"""
- def __init__(self, data, caches_array):
+ def __init__(self, data, data_hash, caches_array):
# Pass caches_array information into Cache Constructor
# It will be used in later for deciding whether we
# need extra cache file dump/load support
@@ -266,6 +266,7 @@ class Cache(object):
self.data = None
self.data_fn = None
self.cacheclean = True
+ self.data_hash = data_hash
if self.cachedir in [None, '']:
self.has_cache = False
@@ -274,26 +275,17 @@ class Cache(object):
return
self.has_cache = True
- self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat")
+ self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash)
logger.debug(1, "Using cache in '%s'", self.cachedir)
bb.utils.mkdirhier(self.cachedir)
- # If any of configuration.data's dependencies are newer than the
- # cache there isn't even any point in loading it...
- newest_mtime = 0
- deps = data.getVar("__base_depends")
-
- old_mtimes = [old_mtime for _, old_mtime in deps]
- old_mtimes.append(newest_mtime)
- newest_mtime = max(old_mtimes)
-
cache_ok = True
if self.caches_array:
for cache_class in self.caches_array:
if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
- cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
- cache_ok = cache_ok and (bb.parse.cached_mtime_noerror(cachefile) >= newest_mtime)
+ cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
+ cache_ok = cache_ok and os.path.exists(cachefile)
cache_class.init_cacheData(self)
if cache_ok:
self.load_cachefile()
@@ -327,7 +319,7 @@ class Cache(object):
# Calculate the correct cachesize of all those cache files
for cache_class in self.caches_array:
if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
- cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
+ cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
with open(cachefile, "rb") as cachefile:
cachesize += os.fstat(cachefile.fileno()).st_size
@@ -335,7 +327,7 @@ class Cache(object):
for cache_class in self.caches_array:
if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
- cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
+ cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
with open(cachefile, "rb") as cachefile:
pickled = pickle.Unpickler(cachefile)
while cachefile:
@@ -588,7 +580,7 @@ class Cache(object):
for cache_class in self.caches_array:
if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
cache_class_name = cache_class.__name__
- cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
+ cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
file_dict[cache_class_name] = open(cachefile, "wb")
pickler_dict[cache_class_name] = pickle.Pickler(file_dict[cache_class_name], pickle.HIGHEST_PROTOCOL)
@@ -693,7 +685,7 @@ def init(cooker):
Files causing parsing errors are evicted from the cache.
"""
- return Cache(cooker.configuration.data)
+ return Cache(cooker.configuration.data, cooker.configuration.data_hash)
class CacheData(object):
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 2032718..9f5db92 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -857,6 +857,7 @@ class BBCooker:
bb.parse.init_parser(data)
bb.event.fire(bb.event.ConfigParsed(), data)
self.configuration.data = data
+ self.configuration.data_hash = data.get_hash()
def handleCollections( self, collections ):
"""Handle collections"""
@@ -1428,6 +1429,7 @@ class CookerParser(object):
self.filelist = filelist
self.cooker = cooker
self.cfgdata = cooker.configuration.data
+ self.cfghash = cooker.configuration.data_hash
# Accounting statistics
self.parsed = 0
@@ -1443,7 +1445,7 @@ class CookerParser(object):
self.num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or
multiprocessing.cpu_count())
- self.bb_cache = bb.cache.Cache(self.cfgdata, cooker.caches_array)
+ self.bb_cache = bb.cache.Cache(self.cfgdata, self.cfghash, cooker.caches_array)
self.fromcache = []
self.willparse = []
for filename in self.filelist:
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index ea13478..9864034 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -31,6 +31,7 @@ BitBake build tools.
import copy, re
from collections import MutableMapping
import logging
+import hashlib
import bb, bb.codeparser
from bb import utils
from bb.COW import COWDictBase
@@ -459,3 +460,23 @@ class DataSmart(MutableMapping):
def __delitem__(self, var):
self.delVar(var)
+
+ def get_hash(self):
+ data = ""
+ keys = iter(self)
+ for key in keys:
+ if key == "TIME":
+ continue
+ if key == "__depends":
+ deps = list(self.getVar(key, False))
+ deps.sort()
+ value = [deps[i][0] for i in range(len(deps))]
+ elif key == "PATH":
+ path = list(set(self.getVar(key, False).split(':')))
+ path.sort()
+ value = " ".join(path)
+ else:
+ value = self.getVar(key, False) or ""
+ data = data + key + ': ' + str(value) + '\n'
+
+ return hashlib.md5(data).hexdigest()
--
1.7.0.4
^ permalink raw reply related [flat|nested] 16+ messages in thread* [PATCH 2/8] command.py: add resolve option for generateTargetsTree API
2012-01-11 3:03 [PATCH 0/8 v2][PULL] Hob: bitbake related changes Dongxiao Xu
2012-01-11 3:03 ` [PATCH 1/8] cache: Use configuration's hash value to validate cache Dongxiao Xu
@ 2012-01-11 3:03 ` Dongxiao Xu
2012-01-11 3:03 ` [PATCH 3/8] event.py: Add a new event PackageInfo Dongxiao Xu
` (5 subsequent siblings)
7 siblings, 0 replies; 16+ messages in thread
From: Dongxiao Xu @ 2012-01-11 3:03 UTC (permalink / raw)
To: bitbake-devel
Currently we have generateTargetsTree API, which is used to get
dependency information. However in that tree, there will be
"virtual/xxx" in depends fields. Therefore we add the resolve option
to replace it with its real providers.
Besides, for packages that provided by multiple recipes, we will find
their preverred provider.
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
---
lib/bb/command.py | 11 ++++++++-
lib/bb/cooker.py | 58 ++++++++++++++++++++++++++++++++++++++++++----------
2 files changed, 56 insertions(+), 13 deletions(-)
diff --git a/lib/bb/command.py b/lib/bb/command.py
index 2a3a3af..05555c5 100644
--- a/lib/bb/command.py
+++ b/lib/bb/command.py
@@ -230,14 +230,21 @@ class CommandsAsync:
included in the package list.
If pkg_list provided use that list (plus any extras brought in by
klass) rather than generating a tree for all packages.
+
+ Add a new option "resolve" to indicate if we need to resolve the
+ replacement for "virtual/xxx" like pn.
"""
klass = params[0]
- if len(params) > 1:
+ resolve = False
+ if len(params) > 2:
+ pkg_list = params[1]
+ resolve = params[2]
+ elif len(params) > 1:
pkg_list = params[1]
else:
pkg_list = []
- command.cooker.generateTargetsTree(klass, pkg_list)
+ command.cooker.generateTargetsTree(klass, pkg_list, resolve)
command.finishAsyncCommand()
generateTargetsTree.needcache = True
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 9f5db92..eefc301 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -439,7 +439,20 @@ class BBCooker:
return depend_tree
- def generatePkgDepTreeData(self, pkgs_to_build, task):
+ def append_package(self, taskdata, depend_tree_package, package):
+ if package not in depend_tree_package:
+ targetid = taskdata.getrun_id(package)
+ if targetid in taskdata.run_targets and taskdata.run_targets[targetid]:
+ fnid = taskdata.run_targets[targetid][0]
+ fn = taskdata.fn_index[fnid]
+ pn = self.status.pkg_fn[fn]
+ version = "%s:%s-%s" % self.status.pkg_pepvpr[fn]
+ depend_tree_package[package] = {}
+ depend_tree_package[package]["pn"] = pn
+ depend_tree_package[package]["filename"] = fn
+ depend_tree_package[package]["version"] = version
+
+ def generatePkgDepTreeData(self, pkgs_to_build, task, resolve=False):
"""
Create a dependency tree of pkgs_to_build, returning the data.
"""
@@ -456,6 +469,7 @@ class BBCooker:
depend_tree["rdepends-pn"] = {}
depend_tree["packages"] = {}
depend_tree["rdepends-pkg"] = {}
+ depend_tree["rrecs-pkg"] = {}
for task in xrange(len(tasks_fnid)):
fnid = tasks_fnid[task]
@@ -465,6 +479,8 @@ class BBCooker:
summary = self.status.summary[fn]
lic = self.status.license[fn]
section = self.status.section[fn]
+ rdepends = self.status.rundeps[fn]
+ rrecs = self.status.runrecs[fn]
if pn not in depend_tree["pn"]:
depend_tree["pn"][pn] = {}
depend_tree["pn"][pn]["filename"] = fn
@@ -472,6 +488,7 @@ class BBCooker:
depend_tree["pn"][pn]["summary"] = summary
depend_tree["pn"][pn]["license"] = lic
depend_tree["pn"][pn]["section"] = section
+ depend_tree["pn"][pn]["packages"] = rdepends.keys()
if fnid not in seen_fnids:
seen_fnids.append(fnid)
@@ -479,25 +496,44 @@ class BBCooker:
depend_tree["depends"][pn] = []
for dep in taskdata.depids[fnid]:
- depend_tree["depends"][pn].append(taskdata.build_names_index[dep])
+ if resolve:
+ item = taskdata.build_names_index[dep]
+ pn_provider = ""
+ targetid = taskdata.getbuild_id(item)
+ if targetid in taskdata.build_targets and taskdata.build_targets[targetid]:
+ fnid = taskdata.build_targets[targetid][0]
+ fn_provider = taskdata.fn_index[fnid]
+ pn_provider = self.status.pkg_fn[fn_provider]
+ else:
+ pn_provider = item
+ depend_tree["depends"][pn].append(pn_provider)
+ else:
+ depend_tree["depends"][pn].append(taskdata.build_names_index[dep])
depend_tree["rdepends-pn"][pn] = []
for rdep in taskdata.rdepids[fnid]:
depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
- rdepends = self.status.rundeps[fn]
for package in rdepends:
depend_tree["rdepends-pkg"][package] = []
for rdepend in rdepends[package]:
depend_tree["rdepends-pkg"][package].append(rdepend)
- packages.append(package)
+ if resolve:
+ self.append_package(taskdata, depend_tree["packages"], rdepend)
+ if not package in packages:
+ packages.append(package)
+
+ for package in rrecs:
+ depend_tree["rrecs-pkg"][package] = []
+ for rrec in rrecs[package]:
+ depend_tree["rrecs-pkg"][package].append(rrec)
+ if resolve:
+ self.append_package(taskdata, depend_tree["packages"], rrec)
+ if not package in packages:
+ packages.append(package)
for package in packages:
- if package not in depend_tree["packages"]:
- depend_tree["packages"][package] = {}
- depend_tree["packages"][package]["pn"] = pn
- depend_tree["packages"][package]["filename"] = fn
- depend_tree["packages"][package]["version"] = version
+ self.append_package(taskdata, depend_tree["packages"], package)
return depend_tree
@@ -743,7 +779,7 @@ class BBCooker:
return pkg_list
- def generateTargetsTree(self, klass=None, pkgs=[]):
+ def generateTargetsTree(self, klass=None, pkgs=[], resolve=False):
"""
Generate a dependency tree of buildable targets
Generate an event with the result
@@ -758,7 +794,7 @@ class BBCooker:
pkgs = pkgs + extra_pkgs
# generate a dependency tree for all our packages
- tree = self.generatePkgDepTreeData(pkgs, 'build')
+ tree = self.generatePkgDepTreeData(pkgs, 'build', resolve)
bb.event.fire(bb.event.TargetsTreeGenerated(tree), self.configuration.data)
def buildWorldTargetList(self):
--
1.7.0.4
^ permalink raw reply related [flat|nested] 16+ messages in thread* [PATCH 3/8] event.py: Add a new event PackageInfo
2012-01-11 3:03 [PATCH 0/8 v2][PULL] Hob: bitbake related changes Dongxiao Xu
2012-01-11 3:03 ` [PATCH 1/8] cache: Use configuration's hash value to validate cache Dongxiao Xu
2012-01-11 3:03 ` [PATCH 2/8] command.py: add resolve option for generateTargetsTree API Dongxiao Xu
@ 2012-01-11 3:03 ` Dongxiao Xu
2012-01-11 3:03 ` [PATCH 4/8] Bitbake: change for adding progress bar in Hob2 Dongxiao Xu
` (4 subsequent siblings)
7 siblings, 0 replies; 16+ messages in thread
From: Dongxiao Xu @ 2012-01-11 3:03 UTC (permalink / raw)
To: bitbake-devel
This event is to pass package information.
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
---
lib/bb/event.py | 8 ++++++++
1 files changed, 8 insertions(+), 0 deletions(-)
diff --git a/lib/bb/event.py b/lib/bb/event.py
index 8d7f941..57bcfb5 100644
--- a/lib/bb/event.py
+++ b/lib/bb/event.py
@@ -467,3 +467,11 @@ class LogHandler(logging.Handler):
def filter(self, record):
record.taskpid = worker_pid
return True
+
+class PackageInfo(Event):
+ """
+ Package information for GUI
+ """
+ def __init__(self, pkginfolist):
+ Event.__init__(self)
+ self._pkginfolist = pkginfolist
--
1.7.0.4
^ permalink raw reply related [flat|nested] 16+ messages in thread* [PATCH 4/8] Bitbake: change for adding progress bar in Hob2.
2012-01-11 3:03 [PATCH 0/8 v2][PULL] Hob: bitbake related changes Dongxiao Xu
` (2 preceding siblings ...)
2012-01-11 3:03 ` [PATCH 3/8] event.py: Add a new event PackageInfo Dongxiao Xu
@ 2012-01-11 3:03 ` Dongxiao Xu
2012-01-11 3:03 ` [PATCH 5/8] bitbake: add -B option to bind with interface Dongxiao Xu
` (3 subsequent siblings)
7 siblings, 0 replies; 16+ messages in thread
From: Dongxiao Xu @ 2012-01-11 3:03 UTC (permalink / raw)
To: bitbake-devel
From: Shane Wang <shane.wang@intel.com>
The changes include:
- Clean some events in event.py
- Fire essential events for Hob2 to handle with more information.
- knotty changes
Signed-off-by: Shane Wang <shane.wang@intel.com>
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
---
lib/bb/cache.py | 2 +-
lib/bb/cooker.py | 12 +++++--
lib/bb/event.py | 83 ++++++++++++++++++++++++++++++++++++--------------
lib/bb/ui/knotty.py | 5 ++-
4 files changed, 73 insertions(+), 29 deletions(-)
diff --git a/lib/bb/cache.py b/lib/bb/cache.py
index 955b6df..c268fe6 100644
--- a/lib/bb/cache.py
+++ b/lib/bb/cache.py
@@ -345,7 +345,7 @@ class Cache(object):
current_percent = 100 * current_progress / cachesize
if current_percent > previous_percent:
previous_percent = current_percent
- bb.event.fire(bb.event.CacheLoadProgress(current_progress),
+ bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize),
self.data)
previous_progress += current_progress
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index eefc301..b02fdd7 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -344,6 +344,7 @@ class BBCooker:
"""
Prepare a runqueue and taskdata object for iteration over pkgs_to_build
"""
+ bb.event.fire(bb.event.TreeDataPreparationStarted(), self.configuration.data)
# Need files parsed
self.updateCache()
# If we are told to do the None task then query the default task
@@ -360,11 +361,14 @@ class BBCooker:
taskdata = bb.taskdata.TaskData(False, skiplist=self.skiplist)
runlist = []
+ current = 0
for k in pkgs_to_build:
taskdata.add_provider(localdata, self.status, k)
runlist.append([k, "do_%s" % task])
+ current += 1
+ bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(pkgs_to_build)), self.configuration.data)
taskdata.add_unresolved(localdata, self.status)
-
+ bb.event.fire(bb.event.TreeDataPreparationCompleted(len(pkgs_to_build)), self.configuration.data)
return runlist, taskdata
def generateTaskDepTreeData(self, pkgs_to_build, task):
@@ -1105,7 +1109,7 @@ class BBCooker:
return False
if not retval:
- bb.event.fire(bb.event.BuildCompleted(buildname, item, failures), self.configuration.event_data)
+ bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, item, failures), self.configuration.event_data)
self.command.finishAsyncCommand()
return False
if retval is True:
@@ -1146,7 +1150,7 @@ class BBCooker:
return False
if not retval:
- bb.event.fire(bb.event.BuildCompleted(buildname, targets, failures), self.configuration.data)
+ bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, targets, failures), self.configuration.data)
self.command.finishAsyncCommand()
return False
if retval is True:
@@ -1562,7 +1566,7 @@ class CookerParser(object):
if parsed:
self.parsed += 1
if self.parsed % self.progress_chunk == 0:
- bb.event.fire(bb.event.ParseProgress(self.parsed),
+ bb.event.fire(bb.event.ParseProgress(self.parsed, self.toparse),
self.cfgdata)
else:
self.cached += 1
diff --git a/lib/bb/event.py b/lib/bb/event.py
index 57bcfb5..5e8f3db 100644
--- a/lib/bb/event.py
+++ b/lib/bb/event.py
@@ -204,6 +204,27 @@ def getName(e):
else:
return e.__name__
+class OperationStarted(Event):
+ """An operation has begun"""
+ def __init__(self, msg = "Operation Started"):
+ Event.__init__(self)
+ self.msg = msg
+
+class OperationCompleted(Event):
+ """An operation has completed"""
+ def __init__(self, total, msg = "Operation Completed"):
+ Event.__init__(self)
+ self.total = total
+ self.msg = msg
+
+class OperationProgress(Event):
+ """An operation is in progress"""
+ def __init__(self, current, total, msg = "Operation in Progress"):
+ Event.__init__(self)
+ self.current = current
+ self.total = total
+ self.msg = msg + ": %s/%s (%.0f%%)" % (current, total, (current*1.0/total)*100);
+
class ConfigParsed(Event):
"""Configuration Parsing Complete"""
@@ -276,14 +297,20 @@ class BuildBase(Event):
-class BuildStarted(BuildBase):
+class BuildStarted(BuildBase, OperationStarted):
"""bbmake build run started"""
+ def __init__(self, n, p, failures = 0):
+ OperationStarted.__init__(self, "Building Started")
+ BuildBase.__init__(self, n, p, failures)
-
-class BuildCompleted(BuildBase):
+class BuildCompleted(BuildBase, OperationCompleted):
"""bbmake build run completed"""
-
-
+ def __init__(self, total, n, p, failures = 0):
+ if not failures:
+ OperationCompleted.__init__(self, total, "Building Succeeded")
+ else:
+ OperationCompleted.__init__(self, total, "Building Failed")
+ BuildBase.__init__(self, n, p, failures)
class NoProvider(Event):
@@ -329,17 +356,16 @@ class MultipleProviders(Event):
"""
return self._candidates
-class ParseStarted(Event):
+class ParseStarted(OperationStarted):
"""Recipe parsing for the runqueue has begun"""
def __init__(self, total):
- Event.__init__(self)
+ OperationStarted.__init__(self, "Recipe parsing Started")
self.total = total
-class ParseCompleted(Event):
+class ParseCompleted(OperationCompleted):
"""Recipe parsing for the runqueue has completed"""
-
def __init__(self, cached, parsed, skipped, masked, virtuals, errors, total):
- Event.__init__(self)
+ OperationCompleted.__init__(self, total, "Recipe parsing Completed")
self.cached = cached
self.parsed = parsed
self.skipped = skipped
@@ -347,33 +373,44 @@ class ParseCompleted(Event):
self.masked = masked
self.errors = errors
self.sofar = cached + parsed
- self.total = total
-class ParseProgress(Event):
+class ParseProgress(OperationProgress):
"""Recipe parsing progress"""
+ def __init__(self, current, total):
+ OperationProgress.__init__(self, current, total, "Recipe parsing")
- def __init__(self, current):
- self.current = current
-class CacheLoadStarted(Event):
+class CacheLoadStarted(OperationStarted):
"""Loading of the dependency cache has begun"""
def __init__(self, total):
- Event.__init__(self)
+ OperationStarted.__init__(self, "Loading cache Started")
self.total = total
-class CacheLoadProgress(Event):
+class CacheLoadProgress(OperationProgress):
"""Cache loading progress"""
- def __init__(self, current):
- Event.__init__(self)
- self.current = current
+ def __init__(self, current, total):
+ OperationProgress.__init__(self, current, total, "Loading cache")
-class CacheLoadCompleted(Event):
+class CacheLoadCompleted(OperationCompleted):
"""Cache loading is complete"""
def __init__(self, total, num_entries):
- Event.__init__(self)
- self.total = total
+ OperationCompleted.__init__(self, total, "Loading cache Completed")
self.num_entries = num_entries
+class TreeDataPreparationStarted(OperationStarted):
+ """Tree data preparation started"""
+ def __init__(self):
+ OperationStarted.__init__(self, "Preparing tree data Started")
+
+class TreeDataPreparationProgress(OperationProgress):
+ """Tree data preparation is in progress"""
+ def __init__(self, current, total):
+ OperationProgress.__init__(self, current, total, "Preparing tree data")
+
+class TreeDataPreparationCompleted(OperationCompleted):
+ """Tree data preparation completed"""
+ def __init__(self, total):
+ OperationCompleted.__init__(self, total, "Preparing tree data Completed")
class DepTreeGenerated(Event):
"""
diff --git a/lib/bb/ui/knotty.py b/lib/bb/ui/knotty.py
index 0340619..b4d901f 100644
--- a/lib/bb/ui/knotty.py
+++ b/lib/bb/ui/knotty.py
@@ -251,7 +251,10 @@ def main(server, eventHandler):
bb.event.RecipeParsed,
bb.event.RecipePreFinalise,
bb.runqueue.runQueueEvent,
- bb.runqueue.runQueueExitWait)):
+ bb.runqueue.runQueueExitWait,
+ bb.event.OperationStarted,
+ bb.event.OperationCompleted,
+ bb.event.OperationProgress)):
continue
logger.error("Unknown event: %s", event)
--
1.7.0.4
^ permalink raw reply related [flat|nested] 16+ messages in thread* [PATCH 5/8] bitbake: add -B option to bind with interface
2012-01-11 3:03 [PATCH 0/8 v2][PULL] Hob: bitbake related changes Dongxiao Xu
` (3 preceding siblings ...)
2012-01-11 3:03 ` [PATCH 4/8] Bitbake: change for adding progress bar in Hob2 Dongxiao Xu
@ 2012-01-11 3:03 ` Dongxiao Xu
2012-01-11 11:38 ` Richard Purdie
2012-01-11 3:03 ` [PATCH 6/8] bitbake: Add client socket info for BitBakeServerConnection Dongxiao Xu
` (2 subsequent siblings)
7 siblings, 1 reply; 16+ messages in thread
From: Dongxiao Xu @ 2012-01-11 3:03 UTC (permalink / raw)
To: bitbake-devel
When start bitbake as a server only process, we need to assign certain
interface to it.
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
---
bin/bitbake | 15 ++++++++++-----
lib/bb/server/xmlrpc.py | 6 +++---
2 files changed, 13 insertions(+), 8 deletions(-)
diff --git a/bin/bitbake b/bin/bitbake
index c2e6822..6e661ce 100755
--- a/bin/bitbake
+++ b/bin/bitbake
@@ -168,6 +168,8 @@ Default BBFILES are the .bb files in the current directory.""")
parser.add_option("", "--server-only", help = "Run bitbake without UI, the frontend can connect with bitbake server itself",
action = "store_true", dest = "server_only", default = False)
+ parser.add_option("-B", "--bind", help = "The name/address for the bitbake server to bind to",
+ action = "store", dest = "bind", default = False)
options, args = parser.parse_args(sys.argv)
configuration = BBConfiguration(options)
@@ -189,9 +191,6 @@ Default BBFILES are the .bb files in the current directory.""")
sys.exit("FATAL: Invalid server type '%s' specified.\n"
"Valid interfaces: xmlrpc, process [default], none." % servertype)
- if configuration.server_only and configuration.servertype != "xmlrpc":
- sys.exit("FATAL: If '--server-only' is defined, we must set the servertype as 'xmlrpc'.\n")
-
# Save a logfile for cooker into the current working directory. When the
# server is daemonized this logfile will be truncated.
cooker_logfile = os.path.join(os.getcwd(), "cooker.log")
@@ -211,9 +210,15 @@ Default BBFILES are the .bb files in the current directory.""")
# of the UIs (e.g. for DISPLAY, etc.)
bb.utils.clean_environment()
- server = server.BitBakeServer()
+ if configuration.server_only:
+ if configuration.servertype != "xmlrpc":
+ sys.exit("FATAL: If '--server-only' is defined, we must set the servertype as 'xmlrpc'.\n")
+ server = server.BitBakeServer()
+ server.initServer((configuration.bind, 0))
+ else:
+ server = server.BitBakeServer()
+ server.initServer()
- server.initServer()
idle = server.getServerIdleCB()
cooker = bb.cooker.BBCooker(configuration, idle, initialenv)
diff --git a/lib/bb/server/xmlrpc.py b/lib/bb/server/xmlrpc.py
index b5980c6..c53cee4 100644
--- a/lib/bb/server/xmlrpc.py
+++ b/lib/bb/server/xmlrpc.py
@@ -163,7 +163,7 @@ class BitBakeXMLRPCServer(SimpleXMLRPCServer):
# remove this when you're done with debugging
# allow_reuse_address = True
- def __init__(self, interface = ("localhost", 0)):
+ def __init__(self, interface):
"""
Constructor
"""
@@ -267,8 +267,8 @@ class BitBakeServerConnection():
pass
class BitBakeServer(object):
- def initServer(self):
- self.server = BitBakeXMLRPCServer()
+ def initServer(self, interface = ("localhost", 0)):
+ self.server = BitBakeXMLRPCServer(interface)
def addcooker(self, cooker):
self.cooker = cooker
--
1.7.0.4
^ permalink raw reply related [flat|nested] 16+ messages in thread* Re: [PATCH 5/8] bitbake: add -B option to bind with interface
2012-01-11 3:03 ` [PATCH 5/8] bitbake: add -B option to bind with interface Dongxiao Xu
@ 2012-01-11 11:38 ` Richard Purdie
2012-01-11 13:30 ` Xu, Dongxiao
0 siblings, 1 reply; 16+ messages in thread
From: Richard Purdie @ 2012-01-11 11:38 UTC (permalink / raw)
To: Dongxiao Xu; +Cc: bitbake-devel
On Wed, 2012-01-11 at 11:03 +0800, Dongxiao Xu wrote:
> When start bitbake as a server only process, we need to assign certain
> interface to it.
>
> Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
> ---
> bin/bitbake | 15 ++++++++++-----
> lib/bb/server/xmlrpc.py | 6 +++---
> 2 files changed, 13 insertions(+), 8 deletions(-)
>
> diff --git a/bin/bitbake b/bin/bitbake
> index c2e6822..6e661ce 100755
> --- a/bin/bitbake
> +++ b/bin/bitbake
> @@ -168,6 +168,8 @@ Default BBFILES are the .bb files in the current directory.""")
> parser.add_option("", "--server-only", help = "Run bitbake without UI, the frontend can connect with bitbake server itself",
> action = "store_true", dest = "server_only", default = False)
>
> + parser.add_option("-B", "--bind", help = "The name/address for the bitbake server to bind to",
> + action = "store", dest = "bind", default = False)
> options, args = parser.parse_args(sys.argv)
>
> configuration = BBConfiguration(options)
> @@ -189,9 +191,6 @@ Default BBFILES are the .bb files in the current directory.""")
> sys.exit("FATAL: Invalid server type '%s' specified.\n"
> "Valid interfaces: xmlrpc, process [default], none." % servertype)
>
> - if configuration.server_only and configuration.servertype != "xmlrpc":
> - sys.exit("FATAL: If '--server-only' is defined, we must set the servertype as 'xmlrpc'.\n")
> -
> # Save a logfile for cooker into the current working directory. When the
> # server is daemonized this logfile will be truncated.
> cooker_logfile = os.path.join(os.getcwd(), "cooker.log")
> @@ -211,9 +210,15 @@ Default BBFILES are the .bb files in the current directory.""")
> # of the UIs (e.g. for DISPLAY, etc.)
> bb.utils.clean_environment()
>
> - server = server.BitBakeServer()
> + if configuration.server_only:
> + if configuration.servertype != "xmlrpc":
> + sys.exit("FATAL: If '--server-only' is defined, we must set the servertype as 'xmlrpc'.\n")
> + server = server.BitBakeServer()
> + server.initServer((configuration.bind, 0))
> + else:
> + server = server.BitBakeServer()
> + server.initServer()
This looks like it will silently ignore configuration.bind in the non
server-only case. Don't you need something like:
+ if configuration.server_only:
+ if configuration.servertype != "xmlrpc":
+ sys.exit("FATAL: If '--server-only' is defined, we must set the servertype as 'xmlrpc'.\n")
+ server = server.BitBakeServer()
+ server.initServer((configuration.bind, 0))
+ elif configuration.bind:
+ server = server.BitBakeServer()
+ server.initServer((configuration.bind, 0))
+ else:
+ server = server.BitBakeServer()
+ server.initServer()
?
Cheers,
Richard
^ permalink raw reply [flat|nested] 16+ messages in thread* Re: [PATCH 5/8] bitbake: add -B option to bind with interface
2012-01-11 11:38 ` Richard Purdie
@ 2012-01-11 13:30 ` Xu, Dongxiao
2012-01-11 14:29 ` Richard Purdie
0 siblings, 1 reply; 16+ messages in thread
From: Xu, Dongxiao @ 2012-01-11 13:30 UTC (permalink / raw)
To: Richard Purdie; +Cc: bitbake-devel@lists.openembedded.org
Hi Richard,
> -----Original Message-----
> From: Richard Purdie [mailto:richard.purdie@linuxfoundation.org]
> Sent: Wednesday, January 11, 2012 7:38 PM
> To: Xu, Dongxiao
> Cc: bitbake-devel@lists.openembedded.org
> Subject: Re: [bitbake-devel] [PATCH 5/8] bitbake: add -B option to bind with
> interface
>
> On Wed, 2012-01-11 at 11:03 +0800, Dongxiao Xu wrote:
> > When start bitbake as a server only process, we need to assign certain
> > interface to it.
> >
> > Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
> > ---
> > bin/bitbake | 15 ++++++++++-----
> > lib/bb/server/xmlrpc.py | 6 +++---
> > 2 files changed, 13 insertions(+), 8 deletions(-)
> >
> > diff --git a/bin/bitbake b/bin/bitbake index c2e6822..6e661ce 100755
> > --- a/bin/bitbake
> > +++ b/bin/bitbake
> > @@ -168,6 +168,8 @@ Default BBFILES are the .bb files in the current
> directory.""")
> > parser.add_option("", "--server-only", help = "Run bitbake without UI,
> the frontend can connect with bitbake server itself",
> > action = "store_true", dest = "server_only", default =
> > False)
> >
> > + parser.add_option("-B", "--bind", help = "The name/address for the
> bitbake server to bind to",
> > + action = "store", dest = "bind", default = False)
> > options, args = parser.parse_args(sys.argv)
> >
> > configuration = BBConfiguration(options) @@ -189,9 +191,6 @@
> > Default BBFILES are the .bb files in the current directory.""")
> > sys.exit("FATAL: Invalid server type '%s' specified.\n"
> > "Valid interfaces: xmlrpc, process [default], none."
> > % servertype)
> >
> > - if configuration.server_only and configuration.servertype != "xmlrpc":
> > - sys.exit("FATAL: If '--server-only' is defined, we must set the
> servertype as 'xmlrpc'.\n")
> > -
> > # Save a logfile for cooker into the current working directory. When the
> > # server is daemonized this logfile will be truncated.
> > cooker_logfile = os.path.join(os.getcwd(), "cooker.log") @@
> > -211,9 +210,15 @@ Default BBFILES are the .bb files in the current
> directory.""")
> > # of the UIs (e.g. for DISPLAY, etc.)
> > bb.utils.clean_environment()
> >
> > - server = server.BitBakeServer()
> > + if configuration.server_only:
> > + if configuration.servertype != "xmlrpc":
> > + sys.exit("FATAL: If '--server-only' is defined, we must set the
> servertype as 'xmlrpc'.\n")
> > + server = server.BitBakeServer()
> > + server.initServer((configuration.bind, 0))
> > + else:
> > + server = server.BitBakeServer()
> > + server.initServer()
>
> This looks like it will silently ignore configuration.bind in the non server-only case.
> Don't you need something like:
>
>
> + if configuration.server_only:
> + if configuration.servertype != "xmlrpc":
> + sys.exit("FATAL: If '--server-only' is defined, we must set the
> servertype as 'xmlrpc'.\n")
> + server = server.BitBakeServer()
> + server.initServer((configuration.bind, 0))
> + elif configuration.bind:
> + server = server.BitBakeServer()
> + server.initServer((configuration.bind, 0))
> + else:
> + server = server.BitBakeServer()
> + server.initServer()
>
> ?
I had a thought of this code piece, we may need the following code. For --server-only option, it requires --bind, while not vice versa.
@@ -189,8 +191,14 @@ Default BBFILES are the .bb files in the current directory.""")
sys.exit("FATAL: Invalid server type '%s' specified.\n"
"Valid interfaces: xmlrpc, process [default], none." % servertype)
- if configuration.server_only and configuration.servertype != "xmlrpc":
- sys.exit("FATAL: If '--server-only' is defined, we must set the servertype as 'xmlrpc'.\n")
+ if configuration.server_only:
+ if configuration.servertype != "xmlrpc":
+ sys.exit("FATAL: If '--server-only' is defined, we must set the servertype as 'xmlrpc'.\n")
+ if not configuration.bind:
+ sys.exit("FATAL: The '--server-only' option requires a name/address to bind to with the -B option.\n")
+
+ if configuration.bind and configuration.servertype != "xmlrpc":
+ sys.exit("FATAL: If '-B' or '--bind' is defined, we must set the servertype as 'xmlrpc'.\n")
# Save a logfile for cooker into the current working directory. When the
# server is daemonized this logfile will be truncated.
@@ -212,8 +220,11 @@ Default BBFILES are the .bb files in the current directory.""")
bb.utils.clean_environment()
server = server.BitBakeServer()
+ if configuration.bind:
+ server.initServer((configuration.bind, 0))
+ else:
+ server.initServer()
- server.initServer()
idle = server.getServerIdleCB()
cooker = bb.cooker.BBCooker(configuration, idle, initialenv)
>
> Cheers,
>
> Richard
>
^ permalink raw reply [flat|nested] 16+ messages in thread* Re: [PATCH 5/8] bitbake: add -B option to bind with interface
2012-01-11 13:30 ` Xu, Dongxiao
@ 2012-01-11 14:29 ` Richard Purdie
0 siblings, 0 replies; 16+ messages in thread
From: Richard Purdie @ 2012-01-11 14:29 UTC (permalink / raw)
To: Xu, Dongxiao; +Cc: bitbake-devel@lists.openembedded.org
On Wed, 2012-01-11 at 13:30 +0000, Xu, Dongxiao wrote:
> > > # Save a logfile for cooker into the current working directory. When the
> > > # server is daemonized this logfile will be truncated.
> > > cooker_logfile = os.path.join(os.getcwd(), "cooker.log") @@
> > > -211,9 +210,15 @@ Default BBFILES are the .bb files in the current
> > directory.""")
> > > # of the UIs (e.g. for DISPLAY, etc.)
> > > bb.utils.clean_environment()
> > >
> > > - server = server.BitBakeServer()
> > > + if configuration.server_only:
> > > + if configuration.servertype != "xmlrpc":
> > > + sys.exit("FATAL: If '--server-only' is defined, we must set the
> > servertype as 'xmlrpc'.\n")
> > > + server = server.BitBakeServer()
> > > + server.initServer((configuration.bind, 0))
> > > + else:
> > > + server = server.BitBakeServer()
> > > + server.initServer()
> >
> > This looks like it will silently ignore configuration.bind in the non server-only case.
> > Don't you need something like:
> >
> >
> > + if configuration.server_only:
> > + if configuration.servertype != "xmlrpc":
> > + sys.exit("FATAL: If '--server-only' is defined, we must set the
> > servertype as 'xmlrpc'.\n")
> > + server = server.BitBakeServer()
> > + server.initServer((configuration.bind, 0))
> > + elif configuration.bind:
> > + server = server.BitBakeServer()
> > + server.initServer((configuration.bind, 0))
> > + else:
> > + server = server.BitBakeServer()
> > + server.initServer()
> >
> > ?
>
> I had a thought of this code piece, we may need the following code. For --server-only option, it requires --bind, while not vice versa.
>
> @@ -189,8 +191,14 @@ Default BBFILES are the .bb files in the current directory.""")
> sys.exit("FATAL: Invalid server type '%s' specified.\n"
> "Valid interfaces: xmlrpc, process [default], none." % servertype)
>
> - if configuration.server_only and configuration.servertype != "xmlrpc":
> - sys.exit("FATAL: If '--server-only' is defined, we must set the servertype as 'xmlrpc'.\n")
> + if configuration.server_only:
> + if configuration.servertype != "xmlrpc":
> + sys.exit("FATAL: If '--server-only' is defined, we must set the servertype as 'xmlrpc'.\n")
> + if not configuration.bind:
> + sys.exit("FATAL: The '--server-only' option requires a name/address to bind to with the -B option.\n")
> +
> + if configuration.bind and configuration.servertype != "xmlrpc":
> + sys.exit("FATAL: If '-B' or '--bind' is defined, we must set the servertype as 'xmlrpc'.\n")
That looks reasonable to me.
Cheers,
Richard
^ permalink raw reply [flat|nested] 16+ messages in thread
* [PATCH 6/8] bitbake: Add client socket info for BitBakeServerConnection
2012-01-11 3:03 [PATCH 0/8 v2][PULL] Hob: bitbake related changes Dongxiao Xu
` (4 preceding siblings ...)
2012-01-11 3:03 ` [PATCH 5/8] bitbake: add -B option to bind with interface Dongxiao Xu
@ 2012-01-11 3:03 ` Dongxiao Xu
2012-01-11 3:03 ` [PATCH 7/8] command.py: add new command to get the CPU info Dongxiao Xu
2012-01-11 3:03 ` [PATCH 8/8] runqueue: fire sceneQueueTaskStarted event when a setscene queue starts Dongxiao Xu
7 siblings, 0 replies; 16+ messages in thread
From: Dongxiao Xu @ 2012-01-11 3:03 UTC (permalink / raw)
To: bitbake-devel
In server/client split model, the client will bind to a specific address
and port. We need to pass the values to BitBakeServerConnection().
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
---
lib/bb/server/xmlrpc.py | 4 ++--
lib/bb/ui/uievent.py | 7 ++++---
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/lib/bb/server/xmlrpc.py b/lib/bb/server/xmlrpc.py
index c53cee4..eff8009 100644
--- a/lib/bb/server/xmlrpc.py
+++ b/lib/bb/server/xmlrpc.py
@@ -247,9 +247,9 @@ class BitbakeServerInfo():
self.port = port
class BitBakeServerConnection():
- def __init__(self, serverinfo):
+ def __init__(self, serverinfo, clientinfo=("localhost", 0)):
self.connection = _create_server(serverinfo.host, serverinfo.port)
- self.events = uievent.BBUIEventQueue(self.connection)
+ self.events = uievent.BBUIEventQueue(self.connection, clientinfo)
for event in bb.event.ui_queue:
self.events.queue_event(event)
diff --git a/lib/bb/ui/uievent.py b/lib/bb/ui/uievent.py
index 0e73817..28817a2 100644
--- a/lib/bb/ui/uievent.py
+++ b/lib/bb/ui/uievent.py
@@ -28,13 +28,14 @@ import socket, threading, pickle
from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
class BBUIEventQueue:
- def __init__(self, BBServer):
+ def __init__(self, BBServer, clientinfo=("localhost, 0")):
self.eventQueue = []
self.eventQueueLock = threading.Lock()
self.eventQueueNotify = threading.Event()
self.BBServer = BBServer
+ self.clientinfo = clientinfo
self.t = threading.Thread()
self.t.setDaemon(True)
@@ -72,7 +73,7 @@ class BBUIEventQueue:
def startCallbackHandler(self):
- server = UIXMLRPCServer()
+ server = UIXMLRPCServer(self.clientinfo)
self.host, self.port = server.socket.getsockname()
server.register_function( self.system_quit, "event.quit" )
@@ -98,7 +99,7 @@ class BBUIEventQueue:
class UIXMLRPCServer (SimpleXMLRPCServer):
- def __init__( self, interface = ("localhost", 0) ):
+ def __init__( self, interface ):
self.quit = False
SimpleXMLRPCServer.__init__( self,
interface,
--
1.7.0.4
^ permalink raw reply related [flat|nested] 16+ messages in thread* [PATCH 7/8] command.py: add new command to get the CPU info
2012-01-11 3:03 [PATCH 0/8 v2][PULL] Hob: bitbake related changes Dongxiao Xu
` (5 preceding siblings ...)
2012-01-11 3:03 ` [PATCH 6/8] bitbake: Add client socket info for BitBakeServerConnection Dongxiao Xu
@ 2012-01-11 3:03 ` Dongxiao Xu
2012-01-11 11:34 ` Richard Purdie
2012-01-11 3:03 ` [PATCH 8/8] runqueue: fire sceneQueueTaskStarted event when a setscene queue starts Dongxiao Xu
7 siblings, 1 reply; 16+ messages in thread
From: Dongxiao Xu @ 2012-01-11 3:03 UTC (permalink / raw)
To: bitbake-devel
Add new API in command.py to get the CPU core and threads information
in order to set the appropriate BB_NUMBER_THREADS and PARALLEL_MAKE
variables.
Signed-off-by: Shane Wang <shane.wang@intel.com>
---
bitbake/lib/bb/helper.py | 39 +++++++++++++++++++++++++++++++++++++++
lib/bb/command.py | 12 ++++++++++++
2 files changed, 51 insertions(+), 0 deletions(-)
create mode 100644 bitbake/lib/bb/helper.py
diff --git a/bitbake/lib/bb/helper.py b/bitbake/lib/bb/helper.py
new file mode 100644
index 0000000..291158b
--- /dev/null
+++ b/bitbake/lib/bb/helper.py
@@ -0,0 +1,39 @@
+#
+# Helper for BitBake Graphical GTK User Interface
+#
+# Copyright (C) 2011 Intel Corporation
+#
+# Authored by Shane Wang <shane.wang@intel.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+
+class CpuInfo(object):
+
+ coefficient = 4
+
+ @classmethod
+ def getNumOfCpus(cls):
+ pfile = os.popen('cat /proc/cpuinfo | grep cpu\ cores')
+ num = len(pfile.readlines())
+ return num
+
+ @classmethod
+ def getNumOfCpuCores(cls):
+ pfile = os.popen('cat /proc/cpuinfo | grep cpu\ cores | cut -d: -f2')
+ contents = pfile.readlines()
+ num = int(contents[0])
+ return num
+
diff --git a/lib/bb/command.py b/lib/bb/command.py
index 05555c5..eaf8236 100644
--- a/lib/bb/command.py
+++ b/lib/bb/command.py
@@ -30,6 +30,7 @@ Commands are queued in a CommandQueue
import bb.event
import bb.cooker
+import bb.helper
class CommandCompleted(bb.event.Event):
pass
@@ -173,6 +174,17 @@ class CommandsSync:
"""
command.cooker.reset()
+ def getDefaultNumOfThreads(self, command, params):
+ """
+ Get the default number of threads on the server = number of CPUs
+ """
+ return bb.helper.CpuInfo.getNumOfCpus()
+
+ def getMaxNumOfThreads(self, command, params):
+ """
+ Get the max number of threads that the server can tolerate
+ """
+ return bb.helper.CpuInfo.getNumOfCpus() * bb.helper.CpuInfo.coefficient
class CommandsAsync:
"""
--
1.7.0.4
^ permalink raw reply related [flat|nested] 16+ messages in thread* Re: [PATCH 7/8] command.py: add new command to get the CPU info
2012-01-11 3:03 ` [PATCH 7/8] command.py: add new command to get the CPU info Dongxiao Xu
@ 2012-01-11 11:34 ` Richard Purdie
2012-01-11 11:52 ` Wang, Shane
0 siblings, 1 reply; 16+ messages in thread
From: Richard Purdie @ 2012-01-11 11:34 UTC (permalink / raw)
To: Dongxiao Xu; +Cc: bitbake-devel
On Wed, 2012-01-11 at 11:03 +0800, Dongxiao Xu wrote:
> Add new API in command.py to get the CPU core and threads information
> in order to set the appropriate BB_NUMBER_THREADS and PARALLEL_MAKE
> variables.
>
> Signed-off-by: Shane Wang <shane.wang@intel.com>
> ---
> bitbake/lib/bb/helper.py | 39 +++++++++++++++++++++++++++++++++++++++
> lib/bb/command.py | 12 ++++++++++++
> 2 files changed, 51 insertions(+), 0 deletions(-)
> create mode 100644 bitbake/lib/bb/helper.py
>
> diff --git a/bitbake/lib/bb/helper.py b/bitbake/lib/bb/helper.py
> new file mode 100644
> index 0000000..291158b
> --- /dev/null
> +++ b/bitbake/lib/bb/helper.py
> @@ -0,0 +1,39 @@
> +#
> +# Helper for BitBake Graphical GTK User Interface
> +#
> +# Copyright (C) 2011 Intel Corporation
> +#
> +# Authored by Shane Wang <shane.wang@intel.com>
> +#
> +# This program is free software; you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License version 2 as
> +# published by the Free Software Foundation.
> +#
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> +# GNU General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License along
> +# with this program; if not, write to the Free Software Foundation, Inc.,
> +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> +
> +import os
> +
> +class CpuInfo(object):
> +
> + coefficient = 4
This is setting off alarm bells...
> + @classmethod
> + def getNumOfCpus(cls):
> + pfile = os.popen('cat /proc/cpuinfo | grep cpu\ cores')
> + num = len(pfile.readlines())
> + return num
You can get this number with something like:
import multiprocessing
multiprocessing.cpu_count()
> + @classmethod
> + def getNumOfCpuCores(cls):
> + pfile = os.popen('cat /proc/cpuinfo | grep cpu\ cores | cut -d: -f2')
> + contents = pfile.readlines()
> + num = int(contents[0])
> + return num
I'm curious what you're using the number of cores to do? It doesn't seem
used by your code?
> diff --git a/lib/bb/command.py b/lib/bb/command.py
> index 05555c5..eaf8236 100644
> --- a/lib/bb/command.py
> +++ b/lib/bb/command.py
> @@ -30,6 +30,7 @@ Commands are queued in a CommandQueue
>
> import bb.event
> import bb.cooker
> +import bb.helper
>
> class CommandCompleted(bb.event.Event):
> pass
> @@ -173,6 +174,17 @@ class CommandsSync:
> """
> command.cooker.reset()
>
> + def getDefaultNumOfThreads(self, command, params):
> + """
> + Get the default number of threads on the server = number of CPUs
> + """
> + return bb.helper.CpuInfo.getNumOfCpus()
> +
> + def getMaxNumOfThreads(self, command, params):
> + """
> + Get the max number of threads that the server can tolerate
> + """
> + return bb.helper.CpuInfo.getNumOfCpus() * bb.helper.CpuInfo.coefficient
>
I can understand needing to query the number of cpus but this last
function seems rather arbitrary. If you want to do bounds checking, I'd
suggest just adding the factor of 4 into the UI. I'm still not convinced
we should be setting any value for this though, or doing any bounds
checking on the value.
My only other comment would be to use bb.utils instead of creating a
bb.helper.
Cheers,
Richard
^ permalink raw reply [flat|nested] 16+ messages in thread* Re: [PATCH 7/8] command.py: add new command to get the CPU info
2012-01-11 11:34 ` Richard Purdie
@ 2012-01-11 11:52 ` Wang, Shane
2012-01-11 12:02 ` Wang, Shane
2012-01-11 13:24 ` Richard Purdie
0 siblings, 2 replies; 16+ messages in thread
From: Wang, Shane @ 2012-01-11 11:52 UTC (permalink / raw)
To: Richard Purdie, Xu, Dongxiao; +Cc: bitbake-devel@lists.openembedded.org
Richard Purdie wrote on 2012-01-11:
> On Wed, 2012-01-11 at 11:03 +0800, Dongxiao Xu wrote:
>> Add new API in command.py to get the CPU core and threads information
>> in order to set the appropriate BB_NUMBER_THREADS and PARALLEL_MAKE
>> variables.
>>
>> Signed-off-by: Shane Wang <shane.wang@intel.com>
>> ---
>> bitbake/lib/bb/helper.py | 39
>> +++++++++++++++++++++++++++++++++++++++ lib/bb/command.py |
>> 12 ++++++++++++ 2 files changed, 51 insertions(+), 0 deletions(-)
>> create mode 100644 bitbake/lib/bb/helper.py
>> diff --git a/bitbake/lib/bb/helper.py b/bitbake/lib/bb/helper.py new
>> file mode 100644 index 0000000..291158b --- /dev/null +++
>> b/bitbake/lib/bb/helper.py @@ -0,0 +1,39 @@ +# +# Helper for BitBake
>> Graphical GTK User Interface +# +# Copyright (C) 2011 Intel
>> Corporation +# +# Authored by Shane Wang <shane.wang@intel.com> +# +#
>> This program is free software; you can redistribute it and/or modify +#
>> it under the terms of the GNU General Public License version 2 as +#
>> published by the Free Software Foundation. +# +# This program is
>> distributed in the hope that it will be useful, +# but WITHOUT ANY
>> WARRANTY; without even the implied warranty of +# MERCHANTABILITY or
>> FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public
>> License for more details. +# +# You should have received a copy of the
>> GNU General Public License along +# with this program; if not, write to
>> the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor,
>> Boston, MA 02110-1301 USA. + +import os + +class CpuInfo(object): + +
>> coefficient = 4
>
> This is setting off alarm bells...
>
>> + @classmethod
>> + def getNumOfCpus(cls):
>> + pfile = os.popen('cat /proc/cpuinfo | grep cpu\ cores')
>> + num = len(pfile.readlines())
>> + return num
>
> You can get this number with something like:
>
> import multiprocessing
> multiprocessing.cpu_count()
Good to see this works, will change that.
>
>> + @classmethod + def getNumOfCpuCores(cls): + pfile =
>> os.popen('cat /proc/cpuinfo | grep cpu\ cores | cut -d: -f2') +
>> contents = pfile.readlines() + num = int(contents[0]) +
>> return num
>
> I'm curious what you're using the number of cores to do? It doesn't seem
> used by your code?
>
>
>> diff --git a/lib/bb/command.py b/lib/bb/command.py
>> index 05555c5..eaf8236 100644
>> --- a/lib/bb/command.py
>> +++ b/lib/bb/command.py
>> @@ -30,6 +30,7 @@ Commands are queued in a CommandQueue
>>
>> import bb.event
>> import bb.cooker
>> +import bb.helper
>>
>> class CommandCompleted(bb.event.Event):
>> pass
>> @@ -173,6 +174,17 @@ class CommandsSync:
>> """
>> command.cooker.reset()
>> + def getDefaultNumOfThreads(self, command, params): + """ +
>> Get the default number of threads on the server = number of CPUs
>> + """ + return bb.helper.CpuInfo.getNumOfCpus() + +
>> def getMaxNumOfThreads(self, command, params): + """ +
>> Get the max number of threads that the server can tolerate + """
>> + return bb.helper.CpuInfo.getNumOfCpus() *
>> bb.helper.CpuInfo.coefficient
>>
>
> I can understand needing to query the number of cpus but this last
> function seems rather arbitrary. If you want to do bounds checking, I'd
> suggest just adding the factor of 4 into the UI.
OK
> I'm still not convinced
> we should be setting any value for this though, or doing any bounds
> checking on the value.
Which value? We hope when the UI starts up, the UI can take advantage of cpus for build and set it as default.
But users can change between 0 ~ max num of threads.
>
>
> My only other comment would be to use bb.utils instead of creating a
> bb.helper.
>
> Cheers,
>
> Richard
>
>
>
> _______________________________________________
> bitbake-devel mailing list
> bitbake-devel@lists.openembedded.org
> http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/bitbake-devel
^ permalink raw reply [flat|nested] 16+ messages in thread* Re: [PATCH 7/8] command.py: add new command to get the CPU info
2012-01-11 11:52 ` Wang, Shane
@ 2012-01-11 12:02 ` Wang, Shane
2012-01-11 13:24 ` Richard Purdie
1 sibling, 0 replies; 16+ messages in thread
From: Wang, Shane @ 2012-01-11 12:02 UTC (permalink / raw)
To: Wang, Shane, Richard Purdie, Xu, Dongxiao
Cc: bitbake-devel@lists.openembedded.org
Wang, Shane wrote on 2012-01-11:
>> I'm still not convinced
>> we should be setting any value for this though, or doing any bounds
>> checking on the value.
> Which value? We hope when the UI starts up, the UI can take advantage of
> cpus for build and set it as default.
> But users can change between 0 ~ max num of threads.
Do you mean the max is not necessary and we can't get accurate max actually?
--
Shane
^ permalink raw reply [flat|nested] 16+ messages in thread
* Re: [PATCH 7/8] command.py: add new command to get the CPU info
2012-01-11 11:52 ` Wang, Shane
2012-01-11 12:02 ` Wang, Shane
@ 2012-01-11 13:24 ` Richard Purdie
1 sibling, 0 replies; 16+ messages in thread
From: Richard Purdie @ 2012-01-11 13:24 UTC (permalink / raw)
To: Wang, Shane; +Cc: bitbake-devel@lists.openembedded.org
On Wed, 2012-01-11 at 11:52 +0000, Wang, Shane wrote:
> Richard Purdie wrote on 2012-01-11:
>
> > On Wed, 2012-01-11 at 11:03 +0800, Dongxiao Xu wrote:
> >> Add new API in command.py to get the CPU core and threads information
> >> in order to set the appropriate BB_NUMBER_THREADS and PARALLEL_MAKE
> >> variables.
> >>
> >> Signed-off-by: Shane Wang <shane.wang@intel.com>
> >> ---
> >> bitbake/lib/bb/helper.py | 39
> >> +++++++++++++++++++++++++++++++++++++++ lib/bb/command.py |
> >> 12 ++++++++++++ 2 files changed, 51 insertions(+), 0 deletions(-)
> >> create mode 100644 bitbake/lib/bb/helper.py
> >> diff --git a/bitbake/lib/bb/helper.py b/bitbake/lib/bb/helper.py new
> >> file mode 100644 index 0000000..291158b --- /dev/null +++
> >> b/bitbake/lib/bb/helper.py @@ -0,0 +1,39 @@ +# +# Helper for BitBake
> >> Graphical GTK User Interface +# +# Copyright (C) 2011 Intel
> >> Corporation +# +# Authored by Shane Wang <shane.wang@intel.com> +# +#
> >> This program is free software; you can redistribute it and/or modify +#
> >> it under the terms of the GNU General Public License version 2 as +#
> >> published by the Free Software Foundation. +# +# This program is
> >> distributed in the hope that it will be useful, +# but WITHOUT ANY
> >> WARRANTY; without even the implied warranty of +# MERCHANTABILITY or
> >> FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public
> >> License for more details. +# +# You should have received a copy of the
> >> GNU General Public License along +# with this program; if not, write to
> >> the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor,
> >> Boston, MA 02110-1301 USA. + +import os + +class CpuInfo(object): + +
> >> coefficient = 4
> >
> > This is setting off alarm bells...
> >
> >> + @classmethod
> >> + def getNumOfCpus(cls):
> >> + pfile = os.popen('cat /proc/cpuinfo | grep cpu\ cores')
> >> + num = len(pfile.readlines())
> >> + return num
> >
> > You can get this number with something like:
> >
> > import multiprocessing
> > multiprocessing.cpu_count()
> Good to see this works, will change that.
>
> >
> >> + @classmethod + def getNumOfCpuCores(cls): + pfile =
> >> os.popen('cat /proc/cpuinfo | grep cpu\ cores | cut -d: -f2') +
> >> contents = pfile.readlines() + num = int(contents[0]) +
> >> return num
> >
> > I'm curious what you're using the number of cores to do? It doesn't seem
> > used by your code?
> >
> >
> >> diff --git a/lib/bb/command.py b/lib/bb/command.py
> >> index 05555c5..eaf8236 100644
> >> --- a/lib/bb/command.py
> >> +++ b/lib/bb/command.py
> >> @@ -30,6 +30,7 @@ Commands are queued in a CommandQueue
> >>
> >> import bb.event
> >> import bb.cooker
> >> +import bb.helper
> >>
> >> class CommandCompleted(bb.event.Event):
> >> pass
> >> @@ -173,6 +174,17 @@ class CommandsSync:
> >> """
> >> command.cooker.reset()
> >> + def getDefaultNumOfThreads(self, command, params): + """ +
> >> Get the default number of threads on the server = number of CPUs
> >> + """ + return bb.helper.CpuInfo.getNumOfCpus() + +
> >> def getMaxNumOfThreads(self, command, params): + """ +
> >> Get the max number of threads that the server can tolerate + """
> >> + return bb.helper.CpuInfo.getNumOfCpus() *
> >> bb.helper.CpuInfo.coefficient
> >>
> >
> > I can understand needing to query the number of cpus but this last
> > function seems rather arbitrary. If you want to do bounds checking, I'd
> > suggest just adding the factor of 4 into the UI.
> OK
>
> > I'm still not convinced
> > we should be setting any value for this though, or doing any bounds
> > checking on the value.
> Which value? We hope when the UI starts up, the UI can take advantage of cpus for build and set it as default.
> But users can change between 0 ~ max num of threads.
You can either:
a) Make it a text input (and then attempt to convert to an int, error if
its not an int). This means you don't need a maximum value.
b) In the UI, just show options up to 4 * cpu count.
I'd probably prefer a) but I can see why b) is easier. I don't see any
value in encoding "4" in the bitbake server though.
Cheers,
Richard
^ permalink raw reply [flat|nested] 16+ messages in thread
* [PATCH 8/8] runqueue: fire sceneQueueTaskStarted event when a setscene queue starts
2012-01-11 3:03 [PATCH 0/8 v2][PULL] Hob: bitbake related changes Dongxiao Xu
` (6 preceding siblings ...)
2012-01-11 3:03 ` [PATCH 7/8] command.py: add new command to get the CPU info Dongxiao Xu
@ 2012-01-11 3:03 ` Dongxiao Xu
7 siblings, 0 replies; 16+ messages in thread
From: Dongxiao Xu @ 2012-01-11 3:03 UTC (permalink / raw)
To: bitbake-devel
The current code prints a log when a setscene task starts, therefore
the progressbar in hob will not receive it. Use a sceneQueueTaskStarted
event instead.
Signed-off-by: Shane Wang <shane.wang@intel.com>
Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
---
lib/bb/event.py | 2 +-
lib/bb/runqueue.py | 10 ++++++++++
2 files changed, 11 insertions(+), 1 deletions(-)
diff --git a/lib/bb/event.py b/lib/bb/event.py
index 5e8f3db..5f71ea7 100644
--- a/lib/bb/event.py
+++ b/lib/bb/event.py
@@ -223,7 +223,7 @@ class OperationProgress(Event):
Event.__init__(self)
self.current = current
self.total = total
- self.msg = msg + ": %s/%s (%.0f%%)" % (current, total, (current*1.0/total)*100);
+ self.msg = msg + ": %s/%s" % (current, total);
class ConfigParsed(Event):
"""Configuration Parsing Complete"""
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 0e83d05..0ba414f 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -1619,6 +1619,9 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
self.task_skip(task)
return True
+ startevent = sceneQueueTaskStarted(task, self.stats, self.rq)
+ bb.event.fire(startevent, self.cfgData)
+
pid, pipein, pipeout = self.fork_off_task(fn, realtask, taskname)
self.build_pids[pid] = task
@@ -1686,6 +1689,13 @@ class runQueueTaskStarted(runQueueEvent):
runQueueEvent.__init__(self, task, stats, rq)
self.noexec = noexec
+class sceneQueueTaskStarted(runQueueTaskStarted):
+ """
+ Event notifing a setscene task was started
+ """
+ def __init__(self, task, stats, rq, noexec=False):
+ runQueueTaskStarted.__init__(self, task, stats, rq, noexec)
+
class runQueueTaskFailed(runQueueEvent):
"""
Event notifing a task failed
--
1.7.0.4
^ permalink raw reply related [flat|nested] 16+ messages in thread