[PATCH 2/2] base/bitbake.conf: Rework DISTRO_FEATURES and MACHINE_FEATURES default handling

[Richard Purdie <richard.purdie@linuxfoundation.org>]

The existing code for handling defaults in distro/machine features
triggers from anonymous python in base.bbclass.

Anonymous python is executed after all inherits including deferred inherits
are processed. This means conditional inherits can't use DISTRO_FEATURES
or MACHINE_FEATURES which is contra to user expectations. This leads to
a hard to understand failure mode.

Whilst it isn't as performant, we can handle the features using inline
python function calls. This patch switches to execute the code that way.

These changes depend on a new function in bb.utils (filter_string).
filter_default_features is changed to return a string rather than
set it and the tests for it are updated to match.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes-global/base.bbclass            |  5 -----
 meta/classes-recipe/crosssdk.bbclass        |  9 ++-------
 meta/classes-recipe/native.bbclass          |  9 ++-------
 meta/classes-recipe/nativesdk.bbclass       |  9 ++-------
 meta/conf/bitbake.conf                      |  4 ++--
 meta/lib/oe/utils.py                        |  7 ++++++-
 meta/lib/oeqa/selftest/cases/oelib/utils.py | 16 ++++++++--------
 7 files changed, 22 insertions(+), 37 deletions(-)

diff --git a/meta/classes-global/base.bbclass b/meta/classes-global/base.bbclass
index 76fd0ac046a..f2e8326e39a 100644
--- a/meta/classes-global/base.bbclass
+++ b/meta/classes-global/base.bbclass
@@ -454,11 +454,6 @@ def set_packagetriplet(d):
 python () {
     import string, re
 
-    # Filter default features to allow users to opt out of features they don't
-    # want.
-    oe.utils.filter_default_features("DISTRO_FEATURES", d)
-    oe.utils.filter_default_features("MACHINE_FEATURES", d)
-
     # To add a recipe to the skip list , set:
     #   SKIP_RECIPE[pn] = "message"
     pn = d.getVar('PN')
diff --git a/meta/classes-recipe/crosssdk.bbclass b/meta/classes-recipe/crosssdk.bbclass
index dac18b39edb..f7be3a23bf6 100644
--- a/meta/classes-recipe/crosssdk.bbclass
+++ b/meta/classes-recipe/crosssdk.bbclass
@@ -16,13 +16,8 @@ PACKAGE_ARCH = "${SDK_ARCH}"
 python () {
     # set TUNE_PKGARCH to SDK_ARCH
     d.setVar('TUNE_PKGARCH', d.getVar('SDK_ARCH'))
-    # Set features here to prevent DISTRO_FEATURES modifications from affecting
-    # crosssdk distro features
-    features = set(d.getVar("DISTRO_FEATURES_NATIVESDK").split())
-    oe.utils.filter_default_features("DISTRO_FEATURES", d)
-    filtered = set(bb.utils.filter("DISTRO_FEATURES", d.getVar("DISTRO_FEATURES_FILTER_NATIVESDK"), d).split())
-    d.setVar("DISTRO_FEATURES", " ".join(sorted(features | filtered)))
-    d.setVar("DISTRO_FEATURES_DEFAULTS", "")
+    defaults = d.getVar("DISTRO_FEATURES")
+    d.setVar("DISTRO_FEATURES", '${@oe.utils.class_filter_features("' + defaults + '", "DISTRO_FEATURES_NATIVESDK", "DISTRO_FEATURES_FILTER_NATIVESDK", d)}')
 }
 
 STAGING_BINDIR_TOOLCHAIN = "${STAGING_DIR_NATIVE}${bindir_native}/${TARGET_ARCH}${TARGET_VENDOR}-${TARGET_OS}"
diff --git a/meta/classes-recipe/native.bbclass b/meta/classes-recipe/native.bbclass
index 9f4ca170f76..89d862eb55a 100644
--- a/meta/classes-recipe/native.bbclass
+++ b/meta/classes-recipe/native.bbclass
@@ -126,13 +126,8 @@ python native_virtclass_handler () {
         return
     bpn = d.getVar("BPN")
 
-    # Set features here to prevent DISTRO_FEATURES modifications from affecting
-    # native distro features
-    features = set(d.getVar("DISTRO_FEATURES_NATIVE").split())
-    oe.utils.filter_default_features("DISTRO_FEATURES", d)
-    filtered = set(bb.utils.filter("DISTRO_FEATURES", d.getVar("DISTRO_FEATURES_FILTER_NATIVE"), d).split())
-    d.setVar("DISTRO_FEATURES", " ".join(sorted(features | filtered)))
-    d.setVar("DISTRO_FEATURES_DEFAULTS", "")
+    defaults = d.getVar("DISTRO_FEATURES")
+    d.setVar("DISTRO_FEATURES", '${@oe.utils.class_filter_features("' + defaults + '", "DISTRO_FEATURES_NATIVE", "DISTRO_FEATURES_FILTER_NATIVE", d)}')
 
     classextend = d.getVar('BBCLASSEXTEND') or ""
     if "native" not in classextend:
diff --git a/meta/classes-recipe/nativesdk.bbclass b/meta/classes-recipe/nativesdk.bbclass
index 80f716fc3c3..25fd463ac78 100644
--- a/meta/classes-recipe/nativesdk.bbclass
+++ b/meta/classes-recipe/nativesdk.bbclass
@@ -77,13 +77,8 @@ python nativesdk_virtclass_handler () {
     if not (pn.endswith("-nativesdk") or pn.startswith("nativesdk-")):
         return
 
-    # Set features here to prevent DISTRO_FEATURES modifications from affecting
-    # nativesdk distro features
-    features = set(d.getVar("DISTRO_FEATURES_NATIVESDK").split())
-    oe.utils.filter_default_features("DISTRO_FEATURES", d)
-    filtered = set(bb.utils.filter("DISTRO_FEATURES", d.getVar("DISTRO_FEATURES_FILTER_NATIVESDK"), d).split())
-    d.setVar("DISTRO_FEATURES", " ".join(sorted(features | filtered)))
-    d.setVar("DISTRO_FEATURES_DEFAULTS", "")
+    defaults = d.getVar("DISTRO_FEATURES")
+    d.setVar("DISTRO_FEATURES", '${@oe.utils.class_filter_features("' + defaults + '", "DISTRO_FEATURES_NATIVESDK", "DISTRO_FEATURES_FILTER_NATIVESDK", d)}')
 
     e.data.setVar("MLPREFIX", "nativesdk-")
     e.data.setVar("PN", "nativesdk-" + e.data.getVar("PN").replace("-nativesdk", "").replace("nativesdk-", ""))
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index 9fcd16615db..24e095632e9 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -893,10 +893,10 @@ OES_BITBAKE_CONF = "1"
 # Machine properties and packagegroup-base stuff
 ##################################################################
 
-MACHINE_FEATURES ?= ""
+MACHINE_FEATURES:append = " ${@oe.utils.filter_default_features('MACHINE_FEATURES', d)}"
 SDK_MACHINE_FEATURES ?= ""
 
-DISTRO_FEATURES ?= ""
+DISTRO_FEATURES:append = " ${@oe.utils.filter_default_features('DISTRO_FEATURES', d)}"
 
 DISTRO_EXTRA_RDEPENDS ?= ""
 DISTRO_EXTRA_RRECOMMENDS ?= ""
diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
index ed5dd5f4504..23f4b639c69 100644
--- a/meta/lib/oe/utils.py
+++ b/meta/lib/oe/utils.py
@@ -154,7 +154,12 @@ def filter_default_features(varname, d):
     default_features = set_difference(varname + "_DEFAULTS",
                                       varname + "_OPTED_OUT",
                                       d)
-    d.appendVar(varname, " " + default_features)
+    return default_features
+
+def class_filter_features(defaults, features_var, filter_var, d):
+    features = set(d.getVar(features_var).split())
+    filtered = set(bb.utils.filter_string(defaults, d.getVar(filter_var)).split())
+    return " ".join(sorted(features | filtered))
 
 def all_distro_features(d, features, truevalue="1", falsevalue=""):
     """
diff --git a/meta/lib/oeqa/selftest/cases/oelib/utils.py b/meta/lib/oeqa/selftest/cases/oelib/utils.py
index a72a0f5983f..75c69789a5b 100644
--- a/meta/lib/oeqa/selftest/cases/oelib/utils.py
+++ b/meta/lib/oeqa/selftest/cases/oelib/utils.py
@@ -115,26 +115,26 @@ class TestDefaultFeatures(TestCase):
         # Test with nothing opted out
         d.setVar("DISTRO_FEATURES", "")
         d.setVar("DISTRO_FEATURES_DEFAULTS", "alpha beta gamma")
-        filter_default_features("DISTRO_FEATURES", d)
-        self.assertEqual(d.getVar("DISTRO_FEATURES").strip(), "alpha beta gamma")
+        filtered = filter_default_features("DISTRO_FEATURES", d)
+        self.assertEqual(filtered.strip(), "alpha beta gamma")
 
         # opt out of a single feature
         d.setVar("DISTRO_FEATURES", "")
         d.setVar("DISTRO_FEATURES_DEFAULTS", "alpha beta gamma")
         d.setVar("DISTRO_FEATURES_OPTED_OUT", "beta")
-        filter_default_features("DISTRO_FEATURES", d)
-        self.assertEqual(d.getVar("DISTRO_FEATURES").strip(), "alpha gamma")
+        filtered = filter_default_features("DISTRO_FEATURES", d)
+        self.assertEqual(filtered.strip(), "alpha gamma")
 
         # opt out of everything
         d.setVar("DISTRO_FEATURES", "")
         d.setVar("DISTRO_FEATURES_DEFAULTS", "alpha beta gamma")
         d.setVar("DISTRO_FEATURES_OPTED_OUT", "gamma alpha beta")
-        filter_default_features("DISTRO_FEATURES", d)
-        self.assertEqual(d.getVar("DISTRO_FEATURES").strip(), "")
+        filtered = filter_default_features("DISTRO_FEATURES", d)
+        self.assertEqual(filtered.strip(), "")
 
         # opt out of something that isn't in our defaults
         d.setVar("DISTRO_FEATURES", "")
         d.setVar("DISTRO_FEATURES_DEFAULTS", "alpha beta gamma")
         d.setVar("DISTRO_FEATURES_OPTED_OUT", "omega")
-        filter_default_features("DISTRO_FEATURES", d)
-        self.assertEqual(d.getVar("DISTRO_FEATURES").strip(), "alpha beta gamma")
+        filtered = filter_default_features("DISTRO_FEATURES", d)
+        self.assertEqual(filtered.strip(), "alpha beta gamma")