* [PATCH 01/16] data_smart: Defer append/prepend handling
2015-07-08 14:58 [PATCH 00/16] update_data() removal patchset Richard Purdie
@ 2015-07-08 14:58 ` Richard Purdie
2015-07-08 14:58 ` [PATCH 02/16] data_smart: Remove need for update_data calls Richard Purdie
` (14 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2015-07-08 14:58 UTC (permalink / raw)
To: bitbake-devel
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
lib/bb/data_smart.py | 27 ++++++++++++++++++++++-----
1 file changed, 22 insertions(+), 5 deletions(-)
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index 7bb7b4a..b433489 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -429,12 +429,13 @@ class DataSmart(MutableMapping):
continue
if op == "_append":
- sval = self.getVar(append, False) or ""
- sval += a
- self.setVar(append, sval)
+ apps = self.getVarFlag(append, "_appendactive", False) or []
+ apps.extend([a])
+ self.setVarFlag(append, "_appendactive", apps, ignore=True)
elif op == "_prepend":
- sval = a + (self.getVar(append, False) or "")
- self.setVar(append, sval)
+ prepends = self.getVarFlag(append, "_prependactive", False) or []
+ prepends.extend([a])
+ self.setVarFlag(append, "_prependactive", prepends, ignore=True)
elif op == "_remove":
removes = self.getVarFlag(append, "_removeactive", False) or []
removes.extend(a.split())
@@ -507,6 +508,11 @@ class DataSmart(MutableMapping):
if not var in self.dict:
self._makeShadowCopy(var)
+ if "_appendactive" in self.dict[var]:
+ del self.dict[var]["_appendactive"]
+ if "_prependactive" in self.dict[var]:
+ del self.dict[var]["_prependactive"]
+
# more cookies for the cookie monster
if '_' in var:
self._setvar_update_overrides(var)
@@ -612,6 +618,17 @@ class DataSmart(MutableMapping):
value = copy.copy(local_var[flag])
elif flag == "_content" and "_defaultval" in local_var and not noweakdefault:
value = copy.copy(local_var["_defaultval"])
+
+ if flag == "_content" and local_var is not None and "_appendactive" in local_var:
+ if not value:
+ value = ""
+ for r in local_var["_appendactive"]:
+ value = value + r
+ if flag == "_content" and local_var is not None and "_prependactive" in local_var:
+ if not value:
+ value = ""
+ for r in local_var["_prependactive"]:
+ value = r + value
if expand and value:
# Only getvar (flag == _content) hits the expand cache
cachename = None
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 02/16] data_smart: Remove need for update_data calls
2015-07-08 14:58 [PATCH 00/16] update_data() removal patchset Richard Purdie
2015-07-08 14:58 ` [PATCH 01/16] data_smart: Defer append/prepend handling Richard Purdie
@ 2015-07-08 14:58 ` Richard Purdie
2015-07-08 14:58 ` [PATCH 03/16] data_smart: Improve override handling Richard Purdie
` (13 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2015-07-08 14:58 UTC (permalink / raw)
To: bitbake-devel
Move the update_data functionality into internal data store operations
so the main finalize (update_data) call is a nop.
To make this work we need to call the internal finalization function
whenever OVERRIDES is changed to ensure values get updated correctly.
This has performance issues but the subsequant patches look into this.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
lib/bb/data_smart.py | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index b433489..77d0c61 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -298,6 +298,7 @@ class VariableHistory(object):
class DataSmart(MutableMapping):
def __init__(self, special = None, seen = None ):
self.dict = {}
+ self.overrides = []
if special is None:
special = COWDictBase.copy()
@@ -354,11 +355,13 @@ class DataSmart(MutableMapping):
def expand(self, s, varname = None):
return self.expandWithRefs(s, varname).value
-
def finalize(self, parent = False):
+ return
+
+ def internal_finalize(self, parent = False):
"""Performs final steps upon the datastore, including application of overrides"""
- overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
+ self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
finalize_caller = {
'op': 'finalize',
}
@@ -383,7 +386,7 @@ class DataSmart(MutableMapping):
# We only want to report finalization once per variable overridden.
finalizes_reported = {}
- for o in overrides:
+ for o in self.overrides:
# calculate '_'+override
l = len(o) + 1
@@ -417,12 +420,15 @@ class DataSmart(MutableMapping):
if op in self._special_values:
appends = self._special_values[op] or []
for append in appends:
+ self.handle_special_values(append, op)
+
+ def handle_special_values(self, append, op):
keep = []
for (a, o) in self.getVarFlag(append, op) or []:
match = True
if o:
for o2 in o.split("_"):
- if not o2 in overrides:
+ if not o2 in self.overrides:
match = False
if not match:
keep.append((a ,o))
@@ -502,6 +508,7 @@ class DataSmart(MutableMapping):
except KeyError:
self._special_values[keyword] = set()
self._special_values[keyword].add(base)
+ self.handle_special_values(base, keyword)
return
@@ -521,6 +528,9 @@ class DataSmart(MutableMapping):
self.dict[var]["_content"] = value
self.varhistory.record(**loginfo)
+ if var == "OVERRIDES":
+ self.internal_finalize(True)
+
def _setvar_update_overrides(self, var):
# aka pay the cookie monster
override = var[var.rfind('_')+1:]
@@ -733,6 +743,7 @@ class DataSmart(MutableMapping):
# we really want this to be a DataSmart...
data = DataSmart(seen=self._seen_overrides.copy(), special=self._special_values.copy())
data.dict["_data"] = self.dict
+ data.overrides = copy.copy(self.overrides)
data.varhistory = self.varhistory.copy()
data.varhistory.datasmart = data
data.inchistory = self.inchistory.copy()
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 03/16] data_smart: Improve override handling
2015-07-08 14:58 [PATCH 00/16] update_data() removal patchset Richard Purdie
2015-07-08 14:58 ` [PATCH 01/16] data_smart: Defer append/prepend handling Richard Purdie
2015-07-08 14:58 ` [PATCH 02/16] data_smart: Remove need for update_data calls Richard Purdie
@ 2015-07-08 14:58 ` Richard Purdie
2015-07-08 14:58 ` [PATCH 04/16] data_smart: Fix cache clearance problems Richard Purdie
` (12 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2015-07-08 14:58 UTC (permalink / raw)
To: bitbake-devel
Rather than the heavy lifting in internal_finalize, move the bulk of
the functionality to getVar and rely on a new internal replaces variable
to keep track of any mappings that are needed. This removes the need
for the COW _special_values cache.
This change in functionality also implies we need to track any changes
not only to OVERRIDES but also any variable which OVERIDES depends upon.
Add code to handle this.
Explicitly list FILE as a value OVERRIDES depends upon since it doesn't
automatically get detected and is of key importance to OVERRIDES,
otherwise PN doesn't update when FILE changes.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
lib/bb/data_smart.py | 156 ++++++++++++++++++++-------------------------------
1 file changed, 60 insertions(+), 96 deletions(-)
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index 77d0c61..aefc0b9 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -296,12 +296,9 @@ class VariableHistory(object):
self.variables[var] = []
class DataSmart(MutableMapping):
- def __init__(self, special = None, seen = None ):
+ def __init__(self, seen = None):
self.dict = {}
- self.overrides = []
- if special is None:
- special = COWDictBase.copy()
if seen is None:
seen = COWDictBase.copy()
@@ -310,11 +307,13 @@ class DataSmart(MutableMapping):
self._tracking = False
# cookie monster tribute
- self._special_values = special
self._seen_overrides = seen
self.expand_cache = {}
+ self.overridevars = set(["OVERRIDES", "FILE"])
+ self.replaces = {}
+
def enableTracking(self):
self._tracking = True
@@ -361,11 +360,7 @@ class DataSmart(MutableMapping):
def internal_finalize(self, parent = False):
"""Performs final steps upon the datastore, including application of overrides"""
- self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
- finalize_caller = {
- 'op': 'finalize',
- }
- infer_caller_details(finalize_caller, parent = parent, varval = False)
+ overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
#
# Well let us see what breaks here. We used to iterate
@@ -383,10 +378,9 @@ class DataSmart(MutableMapping):
# information for later.
#
- # We only want to report finalization once per variable overridden.
- finalizes_reported = {}
+ self.replaces = {}
- for o in self.overrides:
+ for o in overrides:
# calculate '_'+override
l = len(o) + 1
@@ -397,61 +391,7 @@ class DataSmart(MutableMapping):
vars = self._seen_overrides[o].copy()
for var in vars:
name = var[:-l]
- try:
- # Report only once, even if multiple changes.
- if name not in finalizes_reported:
- finalizes_reported[name] = True
- finalize_caller['variable'] = name
- finalize_caller['detail'] = 'was: ' + str(self.getVar(name, False))
- self.varhistory.record(**finalize_caller)
- # Copy history of the override over.
- for event in self.varhistory.variable(var):
- loginfo = event.copy()
- loginfo['variable'] = name
- loginfo['op'] = 'override[%s]:%s' % (o, loginfo['op'])
- self.varhistory.record(**loginfo)
- self.setVar(name, self.getVar(var, False), op = 'finalize', file = 'override[%s]' % o, line = '')
- self.delVar(var)
- except Exception:
- logger.info("Untracked delVar")
-
- # now on to the appends and prepends, and stashing the removes
- for op in __setvar_keyword__:
- if op in self._special_values:
- appends = self._special_values[op] or []
- for append in appends:
- self.handle_special_values(append, op)
-
- def handle_special_values(self, append, op):
- keep = []
- for (a, o) in self.getVarFlag(append, op) or []:
- match = True
- if o:
- for o2 in o.split("_"):
- if not o2 in self.overrides:
- match = False
- if not match:
- keep.append((a ,o))
- continue
-
- if op == "_append":
- apps = self.getVarFlag(append, "_appendactive", False) or []
- apps.extend([a])
- self.setVarFlag(append, "_appendactive", apps, ignore=True)
- elif op == "_prepend":
- prepends = self.getVarFlag(append, "_prependactive", False) or []
- prepends.extend([a])
- self.setVarFlag(append, "_prependactive", prepends, ignore=True)
- elif op == "_remove":
- removes = self.getVarFlag(append, "_removeactive", False) or []
- removes.extend(a.split())
- self.setVarFlag(append, "_removeactive", removes, ignore=True)
-
- # We save overrides that may be applied at some later stage
- if keep:
- self.setVarFlag(append, op, keep, ignore=True)
- else:
- self.delVarFlag(append, op, ignore=True)
+ self.replaces[name] = var
def initVar(self, var):
self.expand_cache = {}
@@ -503,22 +443,19 @@ class DataSmart(MutableMapping):
self.varhistory.record(**loginfo)
# todo make sure keyword is not __doc__ or __module__
# pay the cookie monster
- try:
- self._special_values[keyword].add(base)
- except KeyError:
- self._special_values[keyword] = set()
- self._special_values[keyword].add(base)
- self.handle_special_values(base, keyword)
+ if base in self.overridevars:
+ self.overridevars.update(self.expandWithRefs(value, var).references)
+ self.internal_finalize(True)
return
if not var in self.dict:
self._makeShadowCopy(var)
- if "_appendactive" in self.dict[var]:
- del self.dict[var]["_appendactive"]
- if "_prependactive" in self.dict[var]:
- del self.dict[var]["_prependactive"]
+ if "_append" in self.dict[var]:
+ del self.dict[var]["_append"]
+ if "_prepend" in self.dict[var]:
+ del self.dict[var]["_prepend"]
# more cookies for the cookie monster
if '_' in var:
@@ -528,7 +465,8 @@ class DataSmart(MutableMapping):
self.dict[var]["_content"] = value
self.varhistory.record(**loginfo)
- if var == "OVERRIDES":
+ if var in self.overridevars:
+ self.overridevars.update(self.expandWithRefs(value, var).references)
self.internal_finalize(True)
def _setvar_update_overrides(self, var):
@@ -568,10 +506,6 @@ class DataSmart(MutableMapping):
dest.extend(src)
self.setVarFlag(newkey, i, dest, ignore=True)
- if i in self._special_values and key in self._special_values[i]:
- self._special_values[i].remove(key)
- self._special_values[i].add(newkey)
-
loginfo['variable'] = key
loginfo['op'] = 'rename (to)'
loginfo['detail'] = newkey
@@ -623,22 +557,42 @@ class DataSmart(MutableMapping):
def getVarFlag(self, var, flag, expand=False, noweakdefault=False):
local_var = self._findVar(var)
value = None
- if local_var is not None:
+
+ if flag == "_content" and var in self.replaces:
+ value = self.getVar(self.replaces[var])
+
+ if local_var is not None and value is None:
if flag in local_var:
value = copy.copy(local_var[flag])
elif flag == "_content" and "_defaultval" in local_var and not noweakdefault:
value = copy.copy(local_var["_defaultval"])
- if flag == "_content" and local_var is not None and "_appendactive" in local_var:
+ if flag == "_content" and local_var is not None and "_append" in local_var:
if not value:
value = ""
- for r in local_var["_appendactive"]:
- value = value + r
- if flag == "_content" and local_var is not None and "_prependactive" in local_var:
+ for (r, o) in local_var["_append"]:
+ match = True
+ if o:
+ overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
+ for o2 in o.split("_"):
+ if not o2 in overrides:
+ match = False
+ if match:
+ value = value + r
+
+ if flag == "_content" and local_var is not None and "_prepend" in local_var:
if not value:
value = ""
- for r in local_var["_prependactive"]:
- value = r + value
+ for (r, o) in local_var["_prepend"]:
+ match = True
+ if o:
+ overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
+ for o2 in o.split("_"):
+ if not o2 in overrides:
+ match = False
+ if match:
+ value = r + value
+
if expand and value:
# Only getvar (flag == _content) hits the expand cache
cachename = None
@@ -647,9 +601,19 @@ class DataSmart(MutableMapping):
else:
cachename = var + "[" + flag + "]"
value = self.expand(value, cachename)
- if value and flag == "_content" and local_var is not None and "_removeactive" in local_var:
- removes = [self.expand(r).split() for r in local_var["_removeactive"]]
- removes = reduce(lambda a, b: a+b, removes, [])
+
+ if value and flag == "_content" and local_var is not None and "_remove" in local_var:
+ removes = []
+ for (r, o) in local_var["_remove"]:
+ match = True
+ if o:
+ overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
+ for o2 in o.split("_"):
+ if not o2 in overrides:
+ match = False
+ if match:
+ removes.extend(self.expand(r).split())
+
filtered = filter(lambda v: v not in removes,
value.split())
value = " ".join(filtered)
@@ -735,21 +699,21 @@ class DataSmart(MutableMapping):
else:
del self.dict[var]
-
def createCopy(self):
"""
Create a copy of self by setting _data to self
"""
# we really want this to be a DataSmart...
- data = DataSmart(seen=self._seen_overrides.copy(), special=self._special_values.copy())
+ data = DataSmart(seen=self._seen_overrides.copy())
data.dict["_data"] = self.dict
- data.overrides = copy.copy(self.overrides)
data.varhistory = self.varhistory.copy()
data.varhistory.datasmart = data
data.inchistory = self.inchistory.copy()
data._tracking = self._tracking
+ data.overridevars = copy.copy(self.overridevars)
+
return data
def expandVarref(self, variable, parents=False):
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 04/16] data_smart: Fix cache clearance problems
2015-07-08 14:58 [PATCH 00/16] update_data() removal patchset Richard Purdie
` (2 preceding siblings ...)
2015-07-08 14:58 ` [PATCH 03/16] data_smart: Improve override handling Richard Purdie
@ 2015-07-08 14:58 ` Richard Purdie
2015-07-08 14:58 ` [PATCH 05/16] data_smart: Cache overrides and fix data store Richard Purdie
` (11 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2015-07-08 14:58 UTC (permalink / raw)
To: bitbake-devel
These write operations should clear the expand cache since they can
influence returned variable values but currently do not. Fix this.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
lib/bb/data_smart.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index aefc0b9..eee5827 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -536,6 +536,7 @@ class DataSmart(MutableMapping):
self._seen_overrides[override].remove(var)
def setVarFlag(self, var, flag, value, **loginfo):
+ self.expand_cache = {}
if 'op' not in loginfo:
loginfo['op'] = "set"
loginfo['flag'] = flag
@@ -624,6 +625,7 @@ class DataSmart(MutableMapping):
return value
def delVarFlag(self, var, flag, **loginfo):
+ self.expand_cache = {}
local_var = self._findVar(var)
if not local_var:
return
@@ -653,6 +655,7 @@ class DataSmart(MutableMapping):
self.setVarFlag(var, flag, newvalue, ignore=True)
def setVarFlags(self, var, flags, **loginfo):
+ self.expand_cache = {}
infer_caller_details(loginfo)
if not var in self.dict:
self._makeShadowCopy(var)
@@ -682,6 +685,7 @@ class DataSmart(MutableMapping):
def delVarFlags(self, var, **loginfo):
+ self.expand_cache = {}
if not var in self.dict:
self._makeShadowCopy(var)
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 05/16] data_smart: Cache overrides and fix data store
2015-07-08 14:58 [PATCH 00/16] update_data() removal patchset Richard Purdie
` (3 preceding siblings ...)
2015-07-08 14:58 ` [PATCH 04/16] data_smart: Fix cache clearance problems Richard Purdie
@ 2015-07-08 14:58 ` Richard Purdie
2015-07-08 14:59 ` [PATCH 06/16] data_smart: Move override handling to getVar Richard Purdie
` (10 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2015-07-08 14:58 UTC (permalink / raw)
To: bitbake-devel
Rather than repeatedly expanding the value of OVERRIDES, cache
the value and update it when things change.
There were also some bugs introduced in the replaces functionality
which this approach fixes by ensuring the replaces data is updated
at the correct points.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
lib/bb/data_smart.py | 35 +++++++++++++++++++++++++++--------
1 file changed, 27 insertions(+), 8 deletions(-)
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index eee5827..884ca19 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -311,6 +311,7 @@ class DataSmart(MutableMapping):
self.expand_cache = {}
+ self.overrides = []
self.overridevars = set(["OVERRIDES", "FILE"])
self.replaces = {}
@@ -360,7 +361,7 @@ class DataSmart(MutableMapping):
def internal_finalize(self, parent = False):
"""Performs final steps upon the datastore, including application of overrides"""
- overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
+ self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
#
# Well let us see what breaks here. We used to iterate
@@ -380,7 +381,7 @@ class DataSmart(MutableMapping):
self.replaces = {}
- for o in overrides:
+ for o in self.overrides:
# calculate '_'+override
l = len(o) + 1
@@ -393,6 +394,18 @@ class DataSmart(MutableMapping):
name = var[:-l]
self.replaces[name] = var
+ def internal_finalize2(self, var, o):
+ """Performs final steps upon the datastore, including application of overrides"""
+
+ l = len(o) + 1
+ name = var[:-l]
+
+ if name in self.replaces:
+ del self.replaces[name]
+ for o in self.overrides:
+ if o in self._seen_overrides and name + "_" + o in self._seen_overrides[o]:
+ self.replaces[name] = name + "_" + o
+
def initVar(self, var):
self.expand_cache = {}
if not var in self.dict:
@@ -457,6 +470,12 @@ class DataSmart(MutableMapping):
if "_prepend" in self.dict[var]:
del self.dict[var]["_prepend"]
+ if var in self.replaces:
+ del self.replaces[var]
+ for o in self.overrides:
+ if o in self._seen_overrides and var + "_" + o in self._seen_overrides[o]:
+ self.delVar(var + "_" + o)
+
# more cookies for the cookie monster
if '_' in var:
self._setvar_update_overrides(var)
@@ -477,6 +496,7 @@ class DataSmart(MutableMapping):
if override not in self._seen_overrides:
self._seen_overrides[override] = set()
self._seen_overrides[override].add( var )
+ self.internal_finalize2(var, override)
override = None
if "_" in shortvar:
override = var[shortvar.rfind('_')+1:]
@@ -534,6 +554,7 @@ class DataSmart(MutableMapping):
override = var[var.rfind('_')+1:]
if override and override in self._seen_overrides and var in self._seen_overrides[override]:
self._seen_overrides[override].remove(var)
+ self.internal_finalize2(var, override)
def setVarFlag(self, var, flag, value, **loginfo):
self.expand_cache = {}
@@ -574,9 +595,8 @@ class DataSmart(MutableMapping):
for (r, o) in local_var["_append"]:
match = True
if o:
- overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
for o2 in o.split("_"):
- if not o2 in overrides:
+ if not o2 in self.overrides:
match = False
if match:
value = value + r
@@ -587,9 +607,8 @@ class DataSmart(MutableMapping):
for (r, o) in local_var["_prepend"]:
match = True
if o:
- overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
for o2 in o.split("_"):
- if not o2 in overrides:
+ if not o2 in self.overrides:
match = False
if match:
value = r + value
@@ -608,9 +627,8 @@ class DataSmart(MutableMapping):
for (r, o) in local_var["_remove"]:
match = True
if o:
- overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
for o2 in o.split("_"):
- if not o2 in overrides:
+ if not o2 in self.overrides:
match = False
if match:
removes.extend(self.expand(r).split())
@@ -716,6 +734,7 @@ class DataSmart(MutableMapping):
data._tracking = self._tracking
+ data.overrides = copy.copy(self.overrides)
data.overridevars = copy.copy(self.overridevars)
return data
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 06/16] data_smart: Move override handling to getVar
2015-07-08 14:58 [PATCH 00/16] update_data() removal patchset Richard Purdie
` (4 preceding siblings ...)
2015-07-08 14:58 ` [PATCH 05/16] data_smart: Cache overrides and fix data store Richard Purdie
@ 2015-07-08 14:59 ` Richard Purdie
2015-07-08 14:59 ` [PATCH 07/16] parse/ast/data_smart: Add parsing flag to getVar/setVar Richard Purdie
` (9 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2015-07-08 14:59 UTC (permalink / raw)
To: bitbake-devel
Compeltely remove the replaces functionality and move all overrides
handling to getVar time. We can move the cookie cache into a hidden
flag within the variables themselves.
This removes the need for any of the internal_finalize steps.
This obsolete the need for the _seen_overrides COW cache.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
lib/bb/data_smart.py | 131 ++++++++++++++++++++++++---------------------------
1 file changed, 61 insertions(+), 70 deletions(-)
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index 884ca19..28ea2ae 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -296,24 +296,18 @@ class VariableHistory(object):
self.variables[var] = []
class DataSmart(MutableMapping):
- def __init__(self, seen = None):
+ def __init__(self):
self.dict = {}
- if seen is None:
- seen = COWDictBase.copy()
-
self.inchistory = IncludeHistory()
self.varhistory = VariableHistory(self)
self._tracking = False
- # cookie monster tribute
- self._seen_overrides = seen
-
self.expand_cache = {}
+ # cookie monster tribute
self.overrides = []
self.overridevars = set(["OVERRIDES", "FILE"])
- self.replaces = {}
def enableTracking(self):
self._tracking = True
@@ -363,49 +357,6 @@ class DataSmart(MutableMapping):
self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
- #
- # Well let us see what breaks here. We used to iterate
- # over each variable and apply the override and then
- # do the line expanding.
- # If we have bad luck - which we will have - the keys
- # where in some order that is so important for this
- # method which we don't have anymore.
- # Anyway we will fix that and write test cases this
- # time.
-
- #
- # First we apply all overrides
- # Then we will handle _append and _prepend and store the _remove
- # information for later.
- #
-
- self.replaces = {}
-
- for o in self.overrides:
- # calculate '_'+override
- l = len(o) + 1
-
- # see if one should even try
- if o not in self._seen_overrides:
- continue
-
- vars = self._seen_overrides[o].copy()
- for var in vars:
- name = var[:-l]
- self.replaces[name] = var
-
- def internal_finalize2(self, var, o):
- """Performs final steps upon the datastore, including application of overrides"""
-
- l = len(o) + 1
- name = var[:-l]
-
- if name in self.replaces:
- del self.replaces[name]
- for o in self.overrides:
- if o in self._seen_overrides and name + "_" + o in self._seen_overrides[o]:
- self.replaces[name] = name + "_" + o
-
def initVar(self, var):
self.expand_cache = {}
if not var in self.dict:
@@ -457,6 +408,11 @@ class DataSmart(MutableMapping):
# todo make sure keyword is not __doc__ or __module__
# pay the cookie monster
+ # more cookies for the cookie monster
+ if '_' in var:
+ self._setvar_update_overrides(base, **loginfo)
+
+
if base in self.overridevars:
self.overridevars.update(self.expandWithRefs(value, var).references)
self.internal_finalize(True)
@@ -469,16 +425,12 @@ class DataSmart(MutableMapping):
del self.dict[var]["_append"]
if "_prepend" in self.dict[var]:
del self.dict[var]["_prepend"]
-
- if var in self.replaces:
- del self.replaces[var]
- for o in self.overrides:
- if o in self._seen_overrides and var + "_" + o in self._seen_overrides[o]:
- self.delVar(var + "_" + o)
+ if "_overrides" in self.dict[var]:
+ del self.dict[var]["_overrides"]
# more cookies for the cookie monster
if '_' in var:
- self._setvar_update_overrides(var)
+ self._setvar_update_overrides(var, **loginfo)
# setting var
self.dict[var]["_content"] = value
@@ -488,19 +440,29 @@ class DataSmart(MutableMapping):
self.overridevars.update(self.expandWithRefs(value, var).references)
self.internal_finalize(True)
- def _setvar_update_overrides(self, var):
+ def _setvar_update_overrides(self, var, **loginfo):
# aka pay the cookie monster
override = var[var.rfind('_')+1:]
shortvar = var[:var.rfind('_')]
while override:
- if override not in self._seen_overrides:
- self._seen_overrides[override] = set()
- self._seen_overrides[override].add( var )
- self.internal_finalize2(var, override)
+ l = self.getVarFlag(shortvar, "_overrides") or []
+ if [var, override] not in l:
+ l.append([var, override])
+ self.setVarFlag(shortvar, "_overrides", l, ignore=True)
+ for event in self.varhistory.variable(var):
+ if 'flag' in loginfo and not loginfo['flag'].startswith("_"):
+ continue
+ loginfo = event.copy()
+ loginfo['variable'] = shortvar
+ loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op'])
+ self.varhistory.record(**loginfo)
+
override = None
if "_" in shortvar:
override = var[shortvar.rfind('_')+1:]
shortvar = var[:shortvar.rfind('_')]
+ if len(shortvar) == 0:
+ override = None
def getVar(self, var, expand=False, noweakdefault=False):
return self.getVarFlag(var, "_content", expand, noweakdefault)
@@ -526,6 +488,9 @@ class DataSmart(MutableMapping):
dest.extend(src)
self.setVarFlag(newkey, i, dest, ignore=True)
+ if '_' in newkey and val is None:
+ self._setvar_update_overrides(newkey, **loginfo)
+
loginfo['variable'] = key
loginfo['op'] = 'rename (to)'
loginfo['detail'] = newkey
@@ -552,9 +517,13 @@ class DataSmart(MutableMapping):
self.dict[var] = {}
if '_' in var:
override = var[var.rfind('_')+1:]
- if override and override in self._seen_overrides and var in self._seen_overrides[override]:
- self._seen_overrides[override].remove(var)
- self.internal_finalize2(var, override)
+ shortvar = var[:var.rfind('_')]
+ l = self.getVarFlag(shortvar, "_overrides") or []
+ try:
+ l.remove([var, override])
+ except ValueError as e:
+ pass
+ self.setVarFlag(shortvar, "_overrides", l, ignore=True)
def setVarFlag(self, var, flag, value, **loginfo):
self.expand_cache = {}
@@ -567,7 +536,7 @@ class DataSmart(MutableMapping):
self.dict[var][flag] = value
if flag == "_defaultval" and '_' in var:
- self._setvar_update_overrides(var)
+ self._setvar_update_overrides(var, **loginfo)
if flag == "unexport" or flag == "export":
if not "__exportlist" in self.dict:
@@ -580,8 +549,30 @@ class DataSmart(MutableMapping):
local_var = self._findVar(var)
value = None
- if flag == "_content" and var in self.replaces:
- value = self.getVar(self.replaces[var])
+ if flag == "_content" and local_var is not None and "_overrides" in local_var:
+ match = False
+ active = {}
+ for (r, o) in local_var["_overrides"]:
+ # What about double overrides both with "_" in the name?
+ if o in self.overrides:
+ active[o] = r
+ elif set(o.split("_")).issubset(set(self.overrides)):
+ active[o] = r
+ mod = True
+ while mod:
+ mod = False
+ for o in self.overrides:
+ for a in active.copy():
+ if a.endswith("_" + o):
+ t = active[a]
+ del active[a]
+ active[a.replace("_" + o, "")] = t
+ mod = True
+ elif a == o:
+ match = active[a]
+ del active[a]
+ if match:
+ value = self.getVar(match)
if local_var is not None and value is None:
if flag in local_var:
@@ -726,7 +717,7 @@ class DataSmart(MutableMapping):
Create a copy of self by setting _data to self
"""
# we really want this to be a DataSmart...
- data = DataSmart(seen=self._seen_overrides.copy())
+ data = DataSmart()
data.dict["_data"] = self.dict
data.varhistory = self.varhistory.copy()
data.varhistory.datasmart = data
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 07/16] parse/ast/data_smart: Add parsing flag to getVar/setVar
2015-07-08 14:58 [PATCH 00/16] update_data() removal patchset Richard Purdie
` (5 preceding siblings ...)
2015-07-08 14:59 ` [PATCH 06/16] data_smart: Move override handling to getVar Richard Purdie
@ 2015-07-08 14:59 ` Richard Purdie
2015-07-08 14:59 ` [PATCH 08/16] data_smart: VariableHistory: Ignore override duplicates and overlap with CoW functions Richard Purdie
` (8 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2015-07-08 14:59 UTC (permalink / raw)
To: bitbake-devel
When parsing we find problems if we clear prepends/appends when
setting variables during the initial parsing phases. Later, we actively
want to do this (in what would be post finalisation previously).
To handle this, pass a parsing flag to the operations to control
the correct behaviour for the context.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
lib/bb/data_smart.py | 36 +++++++++++++++++++++---------------
lib/bb/parse/ast.py | 14 +++++++-------
2 files changed, 28 insertions(+), 22 deletions(-)
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index 28ea2ae..68efc7b 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -386,6 +386,10 @@ class DataSmart(MutableMapping):
def setVar(self, var, value, **loginfo):
#print("var=" + str(var) + " val=" + str(value))
+ parsing=False
+ if 'parsing' in loginfo:
+ parsing=True
+
if 'op' not in loginfo:
loginfo['op'] = "set"
self.expand_cache = {}
@@ -421,12 +425,13 @@ class DataSmart(MutableMapping):
if not var in self.dict:
self._makeShadowCopy(var)
- if "_append" in self.dict[var]:
- del self.dict[var]["_append"]
- if "_prepend" in self.dict[var]:
- del self.dict[var]["_prepend"]
- if "_overrides" in self.dict[var]:
- del self.dict[var]["_overrides"]
+ if not parsing:
+ if "_append" in self.dict[var]:
+ del self.dict[var]["_append"]
+ if "_prepend" in self.dict[var]:
+ del self.dict[var]["_prepend"]
+ if "_overrides" in self.dict[var]:
+ del self.dict[var]["_overrides"]
# more cookies for the cookie monster
if '_' in var:
@@ -464,20 +469,20 @@ class DataSmart(MutableMapping):
if len(shortvar) == 0:
override = None
- def getVar(self, var, expand=False, noweakdefault=False):
- return self.getVarFlag(var, "_content", expand, noweakdefault)
+ def getVar(self, var, expand=False, noweakdefault=False, parsing=False):
+ return self.getVarFlag(var, "_content", expand, noweakdefault, parsing)
def renameVar(self, key, newkey, **loginfo):
"""
Rename the variable key to newkey
"""
- val = self.getVar(key, 0)
+ val = self.getVar(key, 0, parsing=True)
if val is not None:
loginfo['variable'] = newkey
loginfo['op'] = 'rename from %s' % key
loginfo['detail'] = val
self.varhistory.record(**loginfo)
- self.setVar(newkey, val, ignore=True)
+ self.setVar(newkey, val, ignore=True, parsing=True)
for i in (__setvar_keyword__):
src = self.getVarFlag(key, i)
@@ -545,11 +550,10 @@ class DataSmart(MutableMapping):
self.dict["__exportlist"]["_content"] = set()
self.dict["__exportlist"]["_content"].add(var)
- def getVarFlag(self, var, flag, expand=False, noweakdefault=False):
+ def getVarFlag(self, var, flag, expand=False, noweakdefault=False, parsing=False):
local_var = self._findVar(var)
value = None
-
- if flag == "_content" and local_var is not None and "_overrides" in local_var:
+ if flag == "_content" and local_var is not None and "_overrides" in local_var and not parsing:
match = False
active = {}
for (r, o) in local_var["_overrides"]:
@@ -580,7 +584,8 @@ class DataSmart(MutableMapping):
elif flag == "_content" and "_defaultval" in local_var and not noweakdefault:
value = copy.copy(local_var["_defaultval"])
- if flag == "_content" and local_var is not None and "_append" in local_var:
+
+ if flag == "_content" and local_var is not None and "_append" in local_var and not parsing:
if not value:
value = ""
for (r, o) in local_var["_append"]:
@@ -592,10 +597,11 @@ class DataSmart(MutableMapping):
if match:
value = value + r
- if flag == "_content" and local_var is not None and "_prepend" in local_var:
+ if flag == "_content" and local_var is not None and "_prepend" in local_var and not parsing:
if not value:
value = ""
for (r, o) in local_var["_prepend"]:
+
match = True
if o:
for o2 in o.split("_"):
diff --git a/lib/bb/parse/ast.py b/lib/bb/parse/ast.py
index 1130b14..bd42bd3 100644
--- a/lib/bb/parse/ast.py
+++ b/lib/bb/parse/ast.py
@@ -85,7 +85,7 @@ class DataNode(AstNode):
if 'flag' in self.groupd and self.groupd['flag'] != None:
return data.getVarFlag(key, self.groupd['flag'], noweakdefault=True)
else:
- return data.getVar(key, False, noweakdefault=True)
+ return data.getVar(key, False, noweakdefault=True, parsing=True)
def eval(self, data):
groupd = self.groupd
@@ -136,7 +136,7 @@ class DataNode(AstNode):
if flag:
data.setVarFlag(key, flag, val, **loginfo)
else:
- data.setVar(key, val, **loginfo)
+ data.setVar(key, val, parsing=True, **loginfo)
class MethodNode(AstNode):
tr_tbl = string.maketrans('/.+-@%&', '_______')
@@ -155,10 +155,10 @@ class MethodNode(AstNode):
anonfuncs = data.getVar('__BBANONFUNCS', False) or []
anonfuncs.append(funcname)
data.setVar('__BBANONFUNCS', anonfuncs)
- data.setVar(funcname, text)
+ data.setVar(funcname, text, parsing=True)
else:
data.setVarFlag(self.func_name, "func", 1)
- data.setVar(self.func_name, text)
+ data.setVar(self.func_name, text, parsing=True)
class PythonMethodNode(AstNode):
def __init__(self, filename, lineno, function, modulename, body):
@@ -175,7 +175,7 @@ class PythonMethodNode(AstNode):
bb.methodpool.insert_method(self.modulename, text, self.filename)
data.setVarFlag(self.function, "func", 1)
data.setVarFlag(self.function, "python", 1)
- data.setVar(self.function, text)
+ data.setVar(self.function, text, parsing=True)
class MethodFlagsNode(AstNode):
def __init__(self, filename, lineno, key, m):
@@ -224,11 +224,11 @@ class ExportFuncsNode(AstNode):
data.setVarFlag(calledfunc, flag, data.getVarFlag(func, flag))
if data.getVarFlag(calledfunc, "python"):
- data.setVar(func, " bb.build.exec_func('" + calledfunc + "', d)\n")
+ data.setVar(func, " bb.build.exec_func('" + calledfunc + "', d)\n", parsing=True)
else:
if "-" in self.classname:
bb.fatal("The classname %s contains a dash character and is calling an sh function %s using EXPORT_FUNCTIONS. Since a dash is illegal in sh function names, this cannot work, please rename the class or don't use EXPORT_FUNCTIONS." % (self.classname, calledfunc))
- data.setVar(func, " " + calledfunc + "\n")
+ data.setVar(func, " " + calledfunc + "\n", parsing=True)
data.setVarFlag(func, 'export_func', '1')
class AddTaskNode(AstNode):
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 08/16] data_smart: VariableHistory: Ignore override duplicates and overlap with CoW functions
2015-07-08 14:58 [PATCH 00/16] update_data() removal patchset Richard Purdie
` (6 preceding siblings ...)
2015-07-08 14:59 ` [PATCH 07/16] parse/ast/data_smart: Add parsing flag to getVar/setVar Richard Purdie
@ 2015-07-08 14:59 ` Richard Purdie
2015-07-08 14:59 ` [PATCH 09/16] data_smart: Seperate out the override cache Richard Purdie
` (7 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2015-07-08 14:59 UTC (permalink / raw)
To: bitbake-devel
Trying to look up a variable called 'copy' in COW is problematic due
to internal implmentation details, at least avoid tracebacks from this.
Also don't duplicate override history (which is an atrifact of changed
override behviour) as otherwise the bitbake -e output is convoluted.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
lib/bb/data_smart.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index 68efc7b..d414bf4 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -231,6 +231,10 @@ class VariableHistory(object):
if var not in self.variables:
self.variables[var] = []
+ if not isinstance(self.variables[var], list):
+ return
+ if 'nodups' in loginfo and loginfo in self.variables[var]:
+ return
self.variables[var].append(loginfo.copy())
def variable(self, var):
@@ -460,6 +464,7 @@ class DataSmart(MutableMapping):
loginfo = event.copy()
loginfo['variable'] = shortvar
loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op'])
+ loginfo['nodups'] = True
self.varhistory.record(**loginfo)
override = None
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 09/16] data_smart: Seperate out the override cache
2015-07-08 14:58 [PATCH 00/16] update_data() removal patchset Richard Purdie
` (7 preceding siblings ...)
2015-07-08 14:59 ` [PATCH 08/16] data_smart: VariableHistory: Ignore override duplicates and overlap with CoW functions Richard Purdie
@ 2015-07-08 14:59 ` Richard Purdie
2015-07-08 14:59 ` [PATCH 10/16] data_smart: Tweak OVERRIDES value cache for performance Richard Purdie
` (6 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2015-07-08 14:59 UTC (permalink / raw)
To: bitbake-devel
Using an internal flag for override mapping turns out to be slower
than is optimal, not least as we don't want the keys list to list
variables that have no value other than a potential override expansion.
Maintinaing a seperate cache is therefore more optimal from a speed
perspective.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
lib/bb/data_smart.py | 55 +++++++++++++++++++++++++++++++++++++++-------------
1 file changed, 41 insertions(+), 14 deletions(-)
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index d414bf4..4ccbedb 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -310,6 +310,8 @@ class DataSmart(MutableMapping):
self.expand_cache = {}
# cookie monster tribute
+ self.overridedata = {}
+
self.overrides = []
self.overridevars = set(["OVERRIDES", "FILE"])
@@ -434,8 +436,18 @@ class DataSmart(MutableMapping):
del self.dict[var]["_append"]
if "_prepend" in self.dict[var]:
del self.dict[var]["_prepend"]
- if "_overrides" in self.dict[var]:
- del self.dict[var]["_overrides"]
+ if var in self.overridedata:
+ active = []
+ self.need_overrides()
+ for (r, o) in self.overridedata[var]:
+ if o in self.overrides:
+ active.append(r)
+ elif "_" in o:
+ if set(o.split("_")).issubset(set(self.overrides)):
+ active.append(r)
+ for a in active:
+ self.delVar(a)
+ del self.overridedata[var]
# more cookies for the cookie monster
if '_' in var:
@@ -454,10 +466,10 @@ class DataSmart(MutableMapping):
override = var[var.rfind('_')+1:]
shortvar = var[:var.rfind('_')]
while override:
- l = self.getVarFlag(shortvar, "_overrides") or []
- if [var, override] not in l:
- l.append([var, override])
- self.setVarFlag(shortvar, "_overrides", l, ignore=True)
+ if shortvar not in self.overridedata:
+ self.overridedata[shortvar] = []
+ if [var, override] not in self.overridedata[shortvar]:
+ self.overridedata[shortvar].append([var, override])
for event in self.varhistory.variable(var):
if 'flag' in loginfo and not loginfo['flag'].startswith("_"):
continue
@@ -498,6 +510,12 @@ class DataSmart(MutableMapping):
dest.extend(src)
self.setVarFlag(newkey, i, dest, ignore=True)
+ if key in self.overridedata:
+ self.overridedata[newkey] = []
+ for (v, o) in self.overridedata[key]:
+ self.overridedata[newkey].append([v.replace(key, newkey), o])
+ self.renameVar(v, v.replace(key, newkey))
+
if '_' in newkey and val is None:
self._setvar_update_overrides(newkey, **loginfo)
@@ -525,15 +543,23 @@ class DataSmart(MutableMapping):
self.varhistory.record(**loginfo)
self.expand_cache = {}
self.dict[var] = {}
+ if var in self.overridedata:
+ del self.overridedata[var]
if '_' in var:
override = var[var.rfind('_')+1:]
shortvar = var[:var.rfind('_')]
- l = self.getVarFlag(shortvar, "_overrides") or []
- try:
- l.remove([var, override])
- except ValueError as e:
- pass
- self.setVarFlag(shortvar, "_overrides", l, ignore=True)
+ while override:
+ try:
+ if shortvar in self.overridedata:
+ self.overridedata[shortvar].remove([var, override])
+ except ValueError as e:
+ pass
+ override = None
+ if "_" in shortvar:
+ override = var[shortvar.rfind('_')+1:]
+ shortvar = var[:shortvar.rfind('_')]
+ if len(shortvar) == 0:
+ override = None
def setVarFlag(self, var, flag, value, **loginfo):
self.expand_cache = {}
@@ -558,10 +584,10 @@ class DataSmart(MutableMapping):
def getVarFlag(self, var, flag, expand=False, noweakdefault=False, parsing=False):
local_var = self._findVar(var)
value = None
- if flag == "_content" and local_var is not None and "_overrides" in local_var and not parsing:
+ if flag == "_content" and var in self.overridedata and not parsing:
match = False
active = {}
- for (r, o) in local_var["_overrides"]:
+ for (r, o) in self.overridedata[var]:
# What about double overrides both with "_" in the name?
if o in self.overrides:
active[o] = r
@@ -738,6 +764,7 @@ class DataSmart(MutableMapping):
data.overrides = copy.copy(self.overrides)
data.overridevars = copy.copy(self.overridevars)
+ data.overridedata = copy.copy(self.overridedata)
return data
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 10/16] data_smart: Tweak OVERRIDES value cache for performance
2015-07-08 14:58 [PATCH 00/16] update_data() removal patchset Richard Purdie
` (8 preceding siblings ...)
2015-07-08 14:59 ` [PATCH 09/16] data_smart: Seperate out the override cache Richard Purdie
@ 2015-07-08 14:59 ` Richard Purdie
2015-07-08 14:59 ` [PATCH 11/16] data_smart: Improve override recursion handling Richard Purdie
` (5 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2015-07-08 14:59 UTC (permalink / raw)
To: bitbake-devel
Updating the value of OVERRIDES whenever it changes turns out to be
extremely expensve/pointless. Instead, clear its value and re-establish
the value when we're going to use it.
This gives significant speed back.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
lib/bb/data_smart.py | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index 4ccbedb..c800a9a 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -311,8 +311,7 @@ class DataSmart(MutableMapping):
# cookie monster tribute
self.overridedata = {}
-
- self.overrides = []
+ self.overrides = None
self.overridevars = set(["OVERRIDES", "FILE"])
def enableTracking(self):
@@ -360,8 +359,13 @@ class DataSmart(MutableMapping):
def internal_finalize(self, parent = False):
"""Performs final steps upon the datastore, including application of overrides"""
+ self.overrides = None
- self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
+ def need_overrides(self):
+ if self.overrides is None:
+ self.overrides = []
+ self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
+ self.expand_cache = {}
def initVar(self, var):
self.expand_cache = {}
@@ -587,6 +591,7 @@ class DataSmart(MutableMapping):
if flag == "_content" and var in self.overridedata and not parsing:
match = False
active = {}
+ self.need_overrides()
for (r, o) in self.overridedata[var]:
# What about double overrides both with "_" in the name?
if o in self.overrides:
@@ -619,6 +624,7 @@ class DataSmart(MutableMapping):
if flag == "_content" and local_var is not None and "_append" in local_var and not parsing:
if not value:
value = ""
+ self.need_overrides()
for (r, o) in local_var["_append"]:
match = True
if o:
@@ -631,6 +637,7 @@ class DataSmart(MutableMapping):
if flag == "_content" and local_var is not None and "_prepend" in local_var and not parsing:
if not value:
value = ""
+ self.need_overrides()
for (r, o) in local_var["_prepend"]:
match = True
@@ -652,6 +659,7 @@ class DataSmart(MutableMapping):
if value and flag == "_content" and local_var is not None and "_remove" in local_var:
removes = []
+ self.need_overrides()
for (r, o) in local_var["_remove"]:
match = True
if o:
@@ -762,7 +770,7 @@ class DataSmart(MutableMapping):
data._tracking = self._tracking
- data.overrides = copy.copy(self.overrides)
+ data.overrides = None
data.overridevars = copy.copy(self.overridevars)
data.overridedata = copy.copy(self.overridedata)
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 11/16] data_smart: Improve override recursion handling
2015-07-08 14:58 [PATCH 00/16] update_data() removal patchset Richard Purdie
` (9 preceding siblings ...)
2015-07-08 14:59 ` [PATCH 10/16] data_smart: Tweak OVERRIDES value cache for performance Richard Purdie
@ 2015-07-08 14:59 ` Richard Purdie
2015-07-08 14:59 ` [PATCH 12/16] data_smart: Cache set(self.overrides) Richard Purdie
` (4 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2015-07-08 14:59 UTC (permalink / raw)
To: bitbake-devel
When expanding OVERRIDES, its possible someone might try and override a variable
that is used in OVERRIDES. This could lead to infinite recursion. Add in
guards against this.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
lib/bb/data_smart.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index c800a9a..c91b51f 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -313,6 +313,7 @@ class DataSmart(MutableMapping):
self.overridedata = {}
self.overrides = None
self.overridevars = set(["OVERRIDES", "FILE"])
+ self.inoverride = False
def enableTracking(self):
self._tracking = True
@@ -363,8 +364,13 @@ class DataSmart(MutableMapping):
def need_overrides(self):
if self.overrides is None:
+ if self.inoverride:
+ return
+ self.inoverride = True
+ # Can end up here recursively so setup dummy values
self.overrides = []
self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
+ self.inoverride = False
self.expand_cache = {}
def initVar(self, var):
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 12/16] data_smart: Cache set(self.overrides)
2015-07-08 14:58 [PATCH 00/16] update_data() removal patchset Richard Purdie
` (10 preceding siblings ...)
2015-07-08 14:59 ` [PATCH 11/16] data_smart: Improve override recursion handling Richard Purdie
@ 2015-07-08 14:59 ` Richard Purdie
2015-07-08 14:59 ` [PATCH 13/16] data_smart: Fix keys() missing overrides Richard Purdie
` (3 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2015-07-08 14:59 UTC (permalink / raw)
To: bitbake-devel
This performs better than continually regeneratiing the set().
Also only use set comparisions when its necessary to save some overhead
as issubset() is slower in the single item case.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
lib/bb/data_smart.py | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index c91b51f..941c158 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -369,7 +369,9 @@ class DataSmart(MutableMapping):
self.inoverride = True
# Can end up here recursively so setup dummy values
self.overrides = []
+ self.overridesset = set()
self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
+ self.overridesset = set(self.overrides)
self.inoverride = False
self.expand_cache = {}
@@ -450,10 +452,10 @@ class DataSmart(MutableMapping):
active = []
self.need_overrides()
for (r, o) in self.overridedata[var]:
- if o in self.overrides:
+ if o in self.overridesset:
active.append(r)
elif "_" in o:
- if set(o.split("_")).issubset(set(self.overrides)):
+ if set(o.split("_")).issubset(self.overridesset):
active.append(r)
for a in active:
self.delVar(a)
@@ -600,10 +602,12 @@ class DataSmart(MutableMapping):
self.need_overrides()
for (r, o) in self.overridedata[var]:
# What about double overrides both with "_" in the name?
- if o in self.overrides:
- active[o] = r
- elif set(o.split("_")).issubset(set(self.overrides)):
+ if o in self.overridesset:
active[o] = r
+ elif "_" in o:
+ if set(o.split("_")).issubset(self.overridesset):
+ active[o] = r
+
mod = True
while mod:
mod = False
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 13/16] data_smart: Fix keys() missing overrides
2015-07-08 14:58 [PATCH 00/16] update_data() removal patchset Richard Purdie
` (11 preceding siblings ...)
2015-07-08 14:59 ` [PATCH 12/16] data_smart: Cache set(self.overrides) Richard Purdie
@ 2015-07-08 14:59 ` Richard Purdie
2015-07-08 14:59 ` [PATCH 14/16] data_smart: Fix data expansion cache issues Richard Purdie
` (2 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2015-07-08 14:59 UTC (permalink / raw)
To: bitbake-devel
d.keys() was not returning items which had no base content but the
variable came into existance through overrides. We have to process
self.overridedata to find these other variables.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
lib/bb/data_smart.py | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index 941c158..b9c5d1e 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -809,6 +809,7 @@ class DataSmart(MutableMapping):
def __iter__(self):
deleted = set()
+ overrides = set()
def keylist(d):
klist = set()
for key in d:
@@ -816,6 +817,8 @@ class DataSmart(MutableMapping):
continue
if key in deleted:
continue
+ if key in overrides:
+ continue
if not d[key]:
deleted.add(key)
continue
@@ -826,9 +829,21 @@ class DataSmart(MutableMapping):
return klist
+ self.need_overrides()
+ for var in self.overridedata:
+ for (r, o) in self.overridedata[var]:
+ if o in self.overridesset:
+ overrides.add(var)
+ elif "_" in o:
+ if set(o.split("_")).issubset(self.overridesset):
+ overrides.add(var)
+
for k in keylist(self.dict):
yield k
+ for k in overrides:
+ yield k
+
def __len__(self):
return len(frozenset(self))
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 14/16] data_smart: Fix data expansion cache issues
2015-07-08 14:58 [PATCH 00/16] update_data() removal patchset Richard Purdie
` (12 preceding siblings ...)
2015-07-08 14:59 ` [PATCH 13/16] data_smart: Fix keys() missing overrides Richard Purdie
@ 2015-07-08 14:59 ` Richard Purdie
2015-07-08 14:59 ` [PATCH 15/16] data_smart: Fix appendVar/prependVar Richard Purdie
2015-07-08 14:59 ` [PATCH 16/16] tests/data: Add new data tests Richard Purdie
15 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2015-07-08 14:59 UTC (permalink / raw)
To: bitbake-devel
The expand cache can be cleared in the middle of getVar now
due to the use of operations like delVar. We therefore need
to check if variables are in the cache before accessing in
case it was cleared.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
lib/bb/data_smart.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index b9c5d1e..b7ccab7 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -682,7 +682,7 @@ class DataSmart(MutableMapping):
filtered = filter(lambda v: v not in removes,
value.split())
value = " ".join(filtered)
- if expand:
+ if expand and var in self.expand_cache:
# We need to ensure the expand cache has the correct value
# flag == "_content" here
self.expand_cache[var].value = value
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 15/16] data_smart: Fix appendVar/prependVar
2015-07-08 14:58 [PATCH 00/16] update_data() removal patchset Richard Purdie
` (13 preceding siblings ...)
2015-07-08 14:59 ` [PATCH 14/16] data_smart: Fix data expansion cache issues Richard Purdie
@ 2015-07-08 14:59 ` Richard Purdie
2015-07-08 14:59 ` [PATCH 16/16] tests/data: Add new data tests Richard Purdie
15 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2015-07-08 14:59 UTC (permalink / raw)
To: bitbake-devel
Now that overrides get expanded 'on the fly', change appendVar
and prependVar to work using _append and _prepend, else we'd have
to re-implement pieces of getVar and the timing of expansions
becomes problematic.
Using _append/_prepend equivalence gives the behaviour users likley
expect from these functions.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
lib/bb/data_smart.py | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index b7ccab7..7755f1a 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -540,14 +540,12 @@ class DataSmart(MutableMapping):
def appendVar(self, var, value, **loginfo):
loginfo['op'] = 'append'
self.varhistory.record(**loginfo)
- newvalue = (self.getVar(var, False) or "") + value
- self.setVar(var, newvalue, ignore=True)
+ self.setVar(var + "_append", value, ignore=True, parsing=True)
def prependVar(self, var, value, **loginfo):
loginfo['op'] = 'prepend'
self.varhistory.record(**loginfo)
- newvalue = value + (self.getVar(var, False) or "")
- self.setVar(var, newvalue, ignore=True)
+ self.setVar(var + "_prepend", value, ignore=True, parsing=True)
def delVar(self, var, **loginfo):
loginfo['detail'] = ""
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PATCH 16/16] tests/data: Add new data tests
2015-07-08 14:58 [PATCH 00/16] update_data() removal patchset Richard Purdie
` (14 preceding siblings ...)
2015-07-08 14:59 ` [PATCH 15/16] data_smart: Fix appendVar/prependVar Richard Purdie
@ 2015-07-08 14:59 ` Richard Purdie
15 siblings, 0 replies; 17+ messages in thread
From: Richard Purdie @ 2015-07-08 14:59 UTC (permalink / raw)
To: bitbake-devel
Add a variety of tests which were found to be useful when working
on the data store recently.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
lib/bb/tests/data.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++
lib/bb/tests/parse.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 93 insertions(+)
diff --git a/lib/bb/tests/data.py b/lib/bb/tests/data.py
index 2c2e7ae..e9aab57 100644
--- a/lib/bb/tests/data.py
+++ b/lib/bb/tests/data.py
@@ -270,6 +270,13 @@ class TestConcatOverride(unittest.TestCase):
bb.data.update_data(self.d)
self.assertEqual(self.d.getVar("TEST", True), "foo:val:val2:bar")
+ def test_append_unset(self):
+ self.d.setVar("TEST_prepend", "${FOO}:")
+ self.d.setVar("TEST_append", ":val2")
+ self.d.setVar("TEST_append", ":${BAR}")
+ bb.data.update_data(self.d)
+ self.assertEqual(self.d.getVar("TEST", True), "foo::val2:bar")
+
def test_remove(self):
self.d.setVar("TEST", "${VAL} ${BAR}")
self.d.setVar("TEST_remove", "val")
@@ -318,12 +325,52 @@ class TestOverrides(unittest.TestCase):
bb.data.update_data(self.d)
self.assertEqual(self.d.getVar("TEST", True), "testvalue2")
+ def test_one_override_unset(self):
+ self.d.setVar("TEST2_bar", "testvalue2")
+ bb.data.update_data(self.d)
+ self.assertEqual(self.d.getVar("TEST2", True), "testvalue2")
+ self.assertItemsEqual(self.d.keys(), ['TEST', 'TEST2', 'OVERRIDES', 'TEST2_bar'])
+
def test_multiple_override(self):
self.d.setVar("TEST_bar", "testvalue2")
self.d.setVar("TEST_local", "testvalue3")
self.d.setVar("TEST_foo", "testvalue4")
bb.data.update_data(self.d)
self.assertEqual(self.d.getVar("TEST", True), "testvalue3")
+ self.assertItemsEqual(self.d.keys(), ['TEST', 'TEST_foo', 'OVERRIDES', 'TEST_bar', 'TEST_local'])
+
+ def test_multiple_combined_overrides(self):
+ self.d.setVar("TEST_local_foo_bar", "testvalue3")
+ bb.data.update_data(self.d)
+ self.assertEqual(self.d.getVar("TEST", True), "testvalue3")
+
+ def test_multiple_overrides_unset(self):
+ self.d.setVar("TEST2_local_foo_bar", "testvalue3")
+ bb.data.update_data(self.d)
+ self.assertEqual(self.d.getVar("TEST2", True), "testvalue3")
+
+ def test_keyexpansion_override(self):
+ self.d.setVar("LOCAL", "local")
+ self.d.setVar("TEST_bar", "testvalue2")
+ self.d.setVar("TEST_${LOCAL}", "testvalue3")
+ self.d.setVar("TEST_foo", "testvalue4")
+ bb.data.update_data(self.d)
+ bb.data.expandKeys(self.d)
+ self.assertEqual(self.d.getVar("TEST", True), "testvalue3")
+
+ def test_rename_override(self):
+ self.d.setVar("ALTERNATIVE_ncurses-tools_class-target", "a")
+ self.d.setVar("OVERRIDES", "class-target")
+ bb.data.update_data(self.d)
+ self.d.renameVar("ALTERNATIVE_ncurses-tools", "ALTERNATIVE_lib32-ncurses-tools")
+ self.assertEqual(self.d.getVar("ALTERNATIVE_lib32-ncurses-tools", True), "a")
+
+ def test_underscore_override(self):
+ self.d.setVar("TEST_bar", "testvalue2")
+ self.d.setVar("TEST_some_val", "testvalue3")
+ self.d.setVar("TEST_foo", "testvalue4")
+ self.d.setVar("OVERRIDES", "foo:bar:some_val")
+ self.assertEqual(self.d.getVar("TEST", True), "testvalue3")
class TestKeyExpansion(unittest.TestCase):
def setUp(self):
diff --git a/lib/bb/tests/parse.py b/lib/bb/tests/parse.py
index fa40327..21fd78a 100644
--- a/lib/bb/tests/parse.py
+++ b/lib/bb/tests/parse.py
@@ -67,3 +67,49 @@ C = "3"
f = self.parsehelper(testfileB)
with self.assertRaises(bb.parse.ParseError):
d = bb.parse.handle(f.name, self.d)['']
+
+ overridetest = """
+RRECOMMENDS_${PN} = "a"
+RRECOMMENDS_${PN}_libc = "b"
+OVERRIDES = "libc:${PN}"
+PN = "gtk+"
+"""
+
+ def test_parse_overrides(self):
+ f = self.parsehelper(self.overridetest)
+ d = bb.parse.handle(f.name, self.d)['']
+ self.assertEqual(d.getVar("RRECOMMENDS", True), "b")
+ bb.data.expandKeys(d)
+ self.assertEqual(d.getVar("RRECOMMENDS", True), "b")
+ d.setVar("RRECOMMENDS_gtk+", "c")
+ self.assertEqual(d.getVar("RRECOMMENDS", True), "c")
+
+ overridetest2 = """
+EXTRA_OECONF = ""
+EXTRA_OECONF_class-target = "b"
+EXTRA_OECONF_append = " c"
+"""
+
+ def test_parse_overrides(self):
+ f = self.parsehelper(self.overridetest2)
+ d = bb.parse.handle(f.name, self.d)['']
+ d.appendVar("EXTRA_OECONF", " d")
+ d.setVar("OVERRIDES", "class-target")
+ self.assertEqual(d.getVar("EXTRA_OECONF", True), "b c d")
+
+ overridetest3 = """
+DESCRIPTION = "A"
+DESCRIPTION_${PN}-dev = "${DESCRIPTION} B"
+PN = "bc"
+"""
+
+ def test_parse_combinations(self):
+ f = self.parsehelper(self.overridetest3)
+ d = bb.parse.handle(f.name, self.d)['']
+ bb.data.expandKeys(d)
+ self.assertEqual(d.getVar("DESCRIPTION_bc-dev", True), "A B")
+ d.setVar("DESCRIPTION", "E")
+ d.setVar("DESCRIPTION_bc-dev", "C D")
+ d.setVar("OVERRIDES", "bc-dev")
+ self.assertEqual(d.getVar("DESCRIPTION", True), "C D")
+
--
2.1.0
^ permalink raw reply related [flat|nested] 17+ messages in thread