[PATCH 0/2] Fixes for some tinfoil2 omissions

[PATCH 0/2] Fixes for some tinfoil2 omissions

[Paul Eggleton]

The following changes since commit 1ff9b3c669fa187f152de7b8b57d14c2468d926c:

  tests/data: Add inactive remove override test (2017-03-16 12:54:59 +0000)

are available in the git repository at:

  git://git.yoctoproject.org/poky-contrib paule/tinfoil-datastore-fixes
  http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/tinfoil-datastore-fixes

Paul Eggleton (2):
  tinfoil: fix override handling in remote datastores
  data_smart: implement missing remote datastore operations

 lib/bb/command.py    |  37 +++++++++++++--
 lib/bb/data_smart.py |  44 +++++++++++++----
 lib/bb/tests/data.py | 132 ++++++++++++++++++++++++++++++++++++++++++++-------
 lib/bb/tinfoil.py    |  37 +++++++++++++--
 4 files changed, 218 insertions(+), 32 deletions(-)

-- 
2.9.3

[PATCH 1/2] tinfoil: fix override handling in remote datastores

[Paul Eggleton]

There was a huge gap in the remote datastore code introduced in the
tinfoil2 rework - we weren't handling overrides at all, since these are
stored separately from the actual data in the DataSmart object. Thus,
when a datastore actually represents a remote datastore we need to go
back to that remote datastore to get the override data as well, so
introduce code to do that.

To avoid a second round-trip I had to modify the _findVar() function to
return the override data as well. This will increase the overhead a
little when that data is superfluous, but without making the function
even uglier I don't think there's a way to avoid that.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 lib/bb/command.py    | 12 +++++++++---
 lib/bb/data_smart.py | 15 ++++++++-------
 lib/bb/tests/data.py |  4 ++++
 lib/bb/tinfoil.py    |  7 +++++--
 4 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/lib/bb/command.py b/lib/bb/command.py
index db20f3f..222a76e 100644
--- a/lib/bb/command.py
+++ b/lib/bb/command.py
@@ -457,16 +457,22 @@ class CommandsSync:
         dsindex = params[0]
         name = params[1]
         datastore = command.remotedatastores[dsindex]
-        value = datastore._findVar(name)
+        value, overridedata = datastore._findVar(name)
 
         if value:
             content = value.get('_content', None)
             if isinstance(content, bb.data_smart.DataSmart):
                 # Value is a datastore (e.g. BB_ORIGENV) - need to handle this carefully
                 idx = command.remotedatastores.check_store(content, True)
-                return {'_content': DataStoreConnectionHandle(idx), '_connector_origtype': 'DataStoreConnectionHandle'}
+                return {'_content': DataStoreConnectionHandle(idx),
+                        '_connector_origtype': 'DataStoreConnectionHandle',
+                        '_connector_overrides': overridedata}
             elif isinstance(content, set):
-                return {'_content': list(content), '_connector_origtype': 'set'}
+                return {'_content': list(content),
+                        '_connector_origtype': 'set',
+                        '_connector_overrides': overridedata}
+            else:
+                value['_connector_overrides'] = overridedata
         return value
     dataStoreConnectorFindVar.readonly = True
 
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index 4d56081..5777d54 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -473,7 +473,7 @@ class DataSmart(MutableMapping):
         dest = self.dict
         while dest:
             if var in dest:
-                return dest[var]
+                return dest[var], self.overridedata.get(var, None)
 
             if "_remote_data" in dest:
                 connector = dest["_remote_data"]["_content"]
@@ -482,12 +482,13 @@ class DataSmart(MutableMapping):
             if "_data" not in dest:
                 break
             dest = dest["_data"]
+        return None, self.overridedata.get(var, None)
 
     def _makeShadowCopy(self, var):
         if var in self.dict:
             return
 
-        local_var = self._findVar(var)
+        local_var, _ = self._findVar(var)
 
         if local_var:
             self.dict[var] = copy.copy(local_var)
@@ -699,13 +700,13 @@ class DataSmart(MutableMapping):
             self.dict["__exportlist"]["_content"].add(var)
 
     def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False):
-        local_var = self._findVar(var)
+        local_var, overridedata = self._findVar(var)
         value = None
-        if flag == "_content" and var in self.overridedata and not parsing:
+        if flag == "_content" and overridedata is not None and not parsing:
             match = False
             active = {}
             self.need_overrides()
-            for (r, o) in self.overridedata[var]:
+            for (r, o) in overridedata:
                 # What about double overrides both with "_" in the name?
                 if o in self.overridesset:
                     active[o] = r
@@ -796,7 +797,7 @@ class DataSmart(MutableMapping):
 
     def delVarFlag(self, var, flag, **loginfo):
         self.expand_cache = {}
-        local_var = self._findVar(var)
+        local_var, _ = self._findVar(var)
         if not local_var:
             return
         if not var in self.dict:
@@ -839,7 +840,7 @@ class DataSmart(MutableMapping):
             self.dict[var][i] = flags[i]
 
     def getVarFlags(self, var, expand = False, internalflags=False):
-        local_var = self._findVar(var)
+        local_var, _ = self._findVar(var)
         flags = {}
 
         if local_var:
diff --git a/lib/bb/tests/data.py b/lib/bb/tests/data.py
index 895489e..da3f4e6 100644
--- a/lib/bb/tests/data.py
+++ b/lib/bb/tests/data.py
@@ -503,3 +503,7 @@ class Remote(unittest.TestCase):
         # Test client side data is incorporated in python expansion (which is done on server)
         d2.setVar('FOO', 'bar')
         self.assertEqual(d2.expand('${@d.getVar("FOO")}'), 'bar')
+        # Test overrides work
+        d1.setVar('FOO_test', 'baz')
+        d1.appendVar('OVERRIDES', ':test')
+        self.assertEqual(d2.getVar('FOO'), 'baz')
diff --git a/lib/bb/tinfoil.py b/lib/bb/tinfoil.py
index 940f9ab..00cec59 100644
--- a/lib/bb/tinfoil.py
+++ b/lib/bb/tinfoil.py
@@ -60,12 +60,15 @@ class TinfoilDataStoreConnector:
         self.dsindex = dsindex
     def getVar(self, name):
         value = self.tinfoil.run_command('dataStoreConnectorFindVar', self.dsindex, name)
+        overrides = None
         if isinstance(value, dict):
             if '_connector_origtype' in value:
                 value['_content'] = self.tinfoil._reconvert_type(value['_content'], value['_connector_origtype'])
                 del value['_connector_origtype']
-
-        return value
+            if '_connector_overrides' in value:
+                overrides = value['_connector_overrides']
+                del value['_connector_overrides']
+        return value, overrides
     def getKeys(self):
         return set(self.tinfoil.run_command('dataStoreConnectorGetKeys', self.dsindex))
     def getVarHistory(self, name):
-- 
2.9.3

[PATCH 2/2] data_smart: implement missing remote datastore operations

[Paul Eggleton]

Enable the following operations from a remote datastore to affect the
other end:

* setVarFlag()
* delVar()
* delVarFlag()
* renameVar()

In practice I don't expect these to be used much, but they should be
present so that the implementation is at least reasonably filled out
and that the tests pass.

Also add tests for the interface, mostly by subclassing the existing
local test classes so that they are using a remote datastore. (These
don't actually test remote usage via tinfoil, just that the
datastore's interface can be used.)

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 lib/bb/command.py    |  25 ++++++++++
 lib/bb/data_smart.py |  29 +++++++++++-
 lib/bb/tests/data.py | 128 +++++++++++++++++++++++++++++++++++++++++++--------
 lib/bb/tinfoil.py    |  30 +++++++++++-
 4 files changed, 192 insertions(+), 20 deletions(-)

diff --git a/lib/bb/command.py b/lib/bb/command.py
index 222a76e..78a86ac 100644
--- a/lib/bb/command.py
+++ b/lib/bb/command.py
@@ -505,6 +505,31 @@ class CommandsSync:
             raise CommandError('dataStoreConnectorRelease: invalid index %d' % dsindex)
         command.remotedatastores.release(dsindex)
 
+    def dataStoreConnectorSetVarFlag(self, command, params):
+        dsindex = params[0]
+        name = params[1]
+        flag = params[2]
+        value = params[3]
+        datastore = command.remotedatastores[dsindex]
+        datastore.setVarFlag(name, flag, value)
+
+    def dataStoreConnectorDelVar(self, command, params):
+        dsindex = params[0]
+        name = params[1]
+        datastore = command.remotedatastores[dsindex]
+        if len(params) > 2:
+            flag = params[2]
+            datastore.delVarFlag(name, flag)
+        else:
+            datastore.delVar(name)
+
+    def dataStoreConnectorRenameVar(self, command, params):
+        dsindex = params[0]
+        name = params[1]
+        newname = params[2]
+        datastore = command.remotedatastores[dsindex]
+        datastore.renameVar(name, newname)
+
     def parseRecipeFile(self, command, params):
         """
         Parse the specified recipe file (with or without bbappends)
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index 5777d54..d6dd698 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -609,6 +609,12 @@ class DataSmart(MutableMapping):
         """
         Rename the variable key to newkey
         """
+        if '_remote_data' in self.dict:
+            connector = self.dict["_remote_data"]["_content"]
+            res = connector.renameVar(key, newkey)
+            if not res:
+                return
+
         val = self.getVar(key, 0, parsing=True)
         if val is not None:
             loginfo['variable'] = newkey
@@ -652,6 +658,12 @@ class DataSmart(MutableMapping):
         self.setVar(var + "_prepend", value, ignore=True, parsing=True)
 
     def delVar(self, var, **loginfo):
+        if '_remote_data' in self.dict:
+            connector = self.dict["_remote_data"]["_content"]
+            res = connector.delVar(var)
+            if not res:
+                return
+
         loginfo['detail'] = ""
         loginfo['op'] = 'del'
         self.varhistory.record(**loginfo)
@@ -678,6 +690,12 @@ class DataSmart(MutableMapping):
                          override = None
 
     def setVarFlag(self, var, flag, value, **loginfo):
+        if '_remote_data' in self.dict:
+            connector = self.dict["_remote_data"]["_content"]
+            res = connector.setVarFlag(var, flag, value)
+            if not res:
+                return
+
         self.expand_cache = {}
         if 'op' not in loginfo:
             loginfo['op'] = "set"
@@ -796,6 +814,12 @@ class DataSmart(MutableMapping):
         return value
 
     def delVarFlag(self, var, flag, **loginfo):
+        if '_remote_data' in self.dict:
+            connector = self.dict["_remote_data"]["_content"]
+            res = connector.delVarFlag(var, flag)
+            if not res:
+                return
+
         self.expand_cache = {}
         local_var, _ = self._findVar(var)
         if not local_var:
@@ -938,7 +962,10 @@ class DataSmart(MutableMapping):
 
             if "_remote_data" in d:
                 connector = d["_remote_data"]["_content"]
-                klist |= connector.getKeys()
+                for key in connector.getKeys():
+                    if key in deleted:
+                        continue
+                    klist.add(key)
 
             return klist
 
diff --git a/lib/bb/tests/data.py b/lib/bb/tests/data.py
index da3f4e6..fe947f5 100644
--- a/lib/bb/tests/data.py
+++ b/lib/bb/tests/data.py
@@ -460,26 +460,43 @@ class Serialize(unittest.TestCase):
         self.assertEqual(newd.getVarFlag('HELLO', 'other'), 'planet')
 
 
+# Remote datastore tests
+# These really only test the interface, since in actual usage we have a
+# tinfoil connector that does everything over RPC, and this doesn't test
+# that.
+
+class TestConnector:
+    d = None
+    def __init__(self, d):
+        self.d = d
+    def getVar(self, name):
+        return self.d._findVar(name)
+    def getKeys(self):
+        return set(self.d.keys())
+    def getVarHistory(self, name):
+        return self.d.varhistory.variable(name)
+    def expandPythonRef(self, varname, expr, d):
+        localdata = self.d.createCopy()
+        for key in d.localkeys():
+            localdata.setVar(d.getVar(key))
+        varparse = bb.data_smart.VariableParse(varname, localdata)
+        return varparse.python_sub(expr)
+    def setVar(self, name, value):
+        self.d.setVar(name, value)
+    def setVarFlag(self, name, flag, value):
+        self.d.setVarFlag(name, flag, value)
+    def delVar(self, name):
+        self.d.delVar(name)
+        return False
+    def delVarFlag(self, name, flag):
+        self.d.delVarFlag(name, flag)
+        return False
+    def renameVar(self, name, newname):
+        self.d.renameVar(name, newname)
+        return False
+
 class Remote(unittest.TestCase):
     def test_remote(self):
-        class TestConnector:
-            d = None
-            def __init__(self, d):
-                self.d = d
-            def getVar(self, name):
-                return self.d._findVar(name)
-            def getKeys(self):
-                return self.d.localkeys()
-            def getVarHistory(self, name):
-                return self.d.varhistory.variable(name)
-            def expandPythonRef(self, varname, expr, d):
-                localdata = self.d.createCopy()
-                for key in d.localkeys():
-                    localdata.setVar(d.getVar(key))
-                varparse = bb.data_smart.VariableParse(varname, localdata)
-                return varparse.python_sub(expr)
-            def setVar(self, name, value):
-                self.d.setVar(name, value)
 
         d1 = bb.data.init()
         d1.enableTracking()
@@ -500,6 +517,9 @@ class Remote(unittest.TestCase):
         # Test setVar on client side affects server
         d2.setVar('HELLO', 'other-world')
         self.assertEqual(d1.getVar('HELLO'), 'other-world')
+        # Test setVarFlag on client side affects server
+        d2.setVarFlag('HELLO', 'flagname', 'flagvalue')
+        self.assertEqual(d1.getVarFlag('HELLO', 'flagname'), 'flagvalue')
         # Test client side data is incorporated in python expansion (which is done on server)
         d2.setVar('FOO', 'bar')
         self.assertEqual(d2.expand('${@d.getVar("FOO")}'), 'bar')
@@ -507,3 +527,75 @@ class Remote(unittest.TestCase):
         d1.setVar('FOO_test', 'baz')
         d1.appendVar('OVERRIDES', ':test')
         self.assertEqual(d2.getVar('FOO'), 'baz')
+
+
+# Remote equivalents of local test classes
+# Note that these aren't perfect since we only test in one direction
+
+class RemoteDataExpansions(DataExpansions):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1["foo"] = "value_of_foo"
+        self.d1["bar"] = "value_of_bar"
+        self.d1["value_of_foo"] = "value_of_'value_of_foo'"
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
+
+class TestRemoteNestedExpansions(TestNestedExpansions):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1["foo"] = "foo"
+        self.d1["bar"] = "bar"
+        self.d1["value_of_foobar"] = "187"
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
+
+class TestRemoteConcat(TestConcat):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1.setVar("FOO", "foo")
+        self.d1.setVar("VAL", "val")
+        self.d1.setVar("BAR", "bar")
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
+
+class TestRemoteConcatOverride(TestConcatOverride):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1.setVar("FOO", "foo")
+        self.d1.setVar("VAL", "val")
+        self.d1.setVar("BAR", "bar")
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
+
+class TestRemoteOverrides(TestOverrides):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1.setVar("OVERRIDES", "foo:bar:local")
+        self.d1.setVar("TEST", "testvalue")
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
+
+class TestRemoteKeyExpansion(TestKeyExpansion):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1.setVar("FOO", "foo")
+        self.d1.setVar("BAR", "foo")
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
+
+class TestRemoteFlags(TestFlags):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1.setVar("foo", "value of foo")
+        self.d1.setVarFlag("foo", "flag1", "value of flag1")
+        self.d1.setVarFlag("foo", "flag2", "value of flag2")
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
diff --git a/lib/bb/tinfoil.py b/lib/bb/tinfoil.py
index 00cec59..19b41be 100644
--- a/lib/bb/tinfoil.py
+++ b/lib/bb/tinfoil.py
@@ -1,6 +1,6 @@
 # tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities
 #
-# Copyright (C) 2012-2016 Intel Corporation
+# Copyright (C) 2012-2017 Intel Corporation
 # Copyright (C) 2011 Mentor Graphics Corporation
 #
 # This program is free software; you can redistribute it and/or modify
@@ -84,6 +84,34 @@ class TinfoilDataStoreConnector:
             # Not currently implemented - indicate that setting should
             # be redirected to local side
             return True
+    def setVarFlag(self, varname, flagname, value):
+        if self.dsindex is None:
+            self.tinfoil.run_command('dataStoreConnectorSetVarFlag', self.dsindex, varname, flagname, value)
+        else:
+            # Not currently implemented - indicate that setting should
+            # be redirected to local side
+            return True
+    def delVar(self, varname):
+        if self.dsindex is None:
+            self.tinfoil.run_command('dataStoreConnectorDelVar', self.dsindex, varname)
+        else:
+            # Not currently implemented - indicate that setting should
+            # be redirected to local side
+            return True
+    def delVarFlag(self, varname, flagname):
+        if self.dsindex is None:
+            self.tinfoil.run_command('dataStoreConnectorDelVar', self.dsindex, varname, flagname)
+        else:
+            # Not currently implemented - indicate that setting should
+            # be redirected to local side
+            return True
+    def renameVar(self, name, newname):
+        if self.dsindex is None:
+            self.tinfoil.run_command('dataStoreConnectorRenameVar', self.dsindex, name, newname)
+        else:
+            # Not currently implemented - indicate that setting should
+            # be redirected to local side
+            return True
 
 class TinfoilCookerAdapter:
     """
-- 
2.9.3