Openembedded Core Discussions
 help / color / mirror / Atom feed
* [PATCH v3] rootfs-postcommands: Set vardeps for do_write_test_data
@ 2023-06-29 15:26 Peter Hoyes
  2023-07-02 21:40 ` [OE-core] " Alexandre Belloni
  0 siblings, 1 reply; 2+ messages in thread
From: Peter Hoyes @ 2023-06-29 15:26 UTC (permalink / raw)
  To: openembedded-core; +Cc: luca.ceresoli, Ross.Burton, richard.purdie, Peter Hoyes

From: Peter Hoyes <Peter.Hoyes@arm.com>

The testdata.json file generated by testdata.bbclass currently contains
almost all Bitbake variables. However only a small number of variables
are actually used and the testdata.json is not automatically updated
when the variables are updated.

Introduce the testdata_vars bbclass which defines TESTDATA_EXPORT_VARS
to explicitly set the variables to be collected for testdata.json and
populate with all variables used by runtime and sdk tests in OE-core.
Use this variable to set the vardeps of do_write_test_data.

Modify export2json to take an explicit list of keys to export, instead
of exporting everything in the datastore.

Adapt write_sdk_test_data and meta-ide-support to use the same list of
testdata variables.

Add a demonstrative OE selftest to rootfspostcommandstests.

Signed-off-by: Peter Hoyes <Peter.Hoyes@arm.com>
---

Changes from v2:
 * Move TESTDATA_EXPORT_VARS to common testdata_vars.bbclass
 * Update other export2json call sites (eSDK and meta-ide-support)

Tested with:
 * oe-selftest -r rootfspostcommandstests
 * oe-selftest -r sstatetests.SStateHashSameSigs
 * bitbake core-image-sato -c testimage
 * bitbake core-image-sato -c testsdkext
 * bitbake buildtools-tarball buildtools-extended-tarball
 * bitbake meta-ide-support

 meta/classes-recipe/populate_sdk_base.bbclass |  5 +++-
 .../rootfs-postcommands.bbclass               |  6 +++-
 meta/classes-recipe/testdata_vars.bbclass     | 30 +++++++++++++++++++
 meta/lib/oe/data.py                           | 17 +----------
 .../selftest/cases/rootfspostcommandstests.py | 19 ++++++++++++
 meta/recipes-core/meta/meta-ide-support.bb    |  5 +++-
 6 files changed, 63 insertions(+), 19 deletions(-)
 create mode 100644 meta/classes-recipe/testdata_vars.bbclass

diff --git a/meta/classes-recipe/populate_sdk_base.bbclass b/meta/classes-recipe/populate_sdk_base.bbclass
index b23ea26d40..ffb06f2062 100644
--- a/meta/classes-recipe/populate_sdk_base.bbclass
+++ b/meta/classes-recipe/populate_sdk_base.bbclass
@@ -134,11 +134,14 @@ sdk_prune_dirs () {
     done
 }
 
+inherit testdata_vars
+
 python write_sdk_test_data() {
     from oe.data import export2json
     testdata = "%s/%s.testdata.json" % (d.getVar('SDKDEPLOYDIR'), d.getVar('TOOLCHAIN_OUTPUTNAME'))
     bb.utils.mkdirhier(os.path.dirname(testdata))
-    export2json(d, testdata)
+    keys = d.getVar('TESTDATA_EXPORT_VARS').split()
+    export2json(d, keys, testdata)
 }
 
 python write_host_sdk_manifest () {
diff --git a/meta/classes-recipe/rootfs-postcommands.bbclass b/meta/classes-recipe/rootfs-postcommands.bbclass
index 652601b95f..26eddc8c35 100644
--- a/meta/classes-recipe/rootfs-postcommands.bbclass
+++ b/meta/classes-recipe/rootfs-postcommands.bbclass
@@ -454,6 +454,8 @@ rootfs_sysroot_relativelinks () {
 }
 
 # Generated test data json file
+inherit testdata_vars
+
 python write_image_test_data() {
     from oe.data import export2json
 
@@ -461,8 +463,9 @@ python write_image_test_data() {
     link_name = d.getVar('IMAGE_LINK_NAME')
     testdata_name = os.path.join(deploy_dir, "%s.testdata.json" % d.getVar('IMAGE_NAME'))
 
+    keys = d.getVar('TESTDATA_EXPORT_VARS').split()
     searchString = "%s/"%(d.getVar("TOPDIR")).replace("//","/")
-    export2json(d, testdata_name, searchString=searchString, replaceString="")
+    export2json(d, keys, testdata_name, searchString=searchString, replaceString="")
 
     if os.path.exists(testdata_name) and link_name:
         testdata_link = os.path.join(deploy_dir, "%s.testdata.json" % link_name)
@@ -471,6 +474,7 @@ python write_image_test_data() {
                 os.remove(testdata_link)
             os.symlink(os.path.basename(testdata_name), testdata_link)
 }
+write_image_test_data[vardeps] += "${TESTDATA_EXPORT_VARS}"
 write_image_test_data[vardepsexclude] += "TOPDIR"
 
 # Check for unsatisfied recommendations (RRECOMMENDS)
diff --git a/meta/classes-recipe/testdata_vars.bbclass b/meta/classes-recipe/testdata_vars.bbclass
new file mode 100644
index 0000000000..cecbc6d7ca
--- /dev/null
+++ b/meta/classes-recipe/testdata_vars.bbclass
@@ -0,0 +1,30 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
+# The variables exported to testdata.json for testimage and other similar tasks
+TESTDATA_EXPORT_VARS ?= "\
+    ARCH \
+    DEPLOY_DIR \
+    DEPLOY_DIR_DEB \
+    DISTRO_FEATURES \
+    DL_DIR \
+    IMAGE \
+    IMAGE_FEATURES \
+    libdir \
+    MACHINE \
+    MULTILIB_VARIANTS \
+    PACKAGE_FEED_GPG_NAME \
+    PTEST_EXPECT_FAILURE \
+    QEMU_USE_KVM \
+    SDK_DEPLOY \
+    SDKPATH \
+    T \
+    TEST_LOG_DIR \
+    TOOLCHAINEXT_OUTPUTNAME \
+    TUNE_PKGARCH \
+    VIRTUAL-RUNTIME_init_manager \
+    WORKDIR \
+"
diff --git a/meta/lib/oe/data.py b/meta/lib/oe/data.py
index 37121cfad2..731f2b5b46 100644
--- a/meta/lib/oe/data.py
+++ b/meta/lib/oe/data.py
@@ -23,23 +23,8 @@ def typed_value(key, d):
     except (TypeError, ValueError) as exc:
         bb.msg.fatal("Data", "%s: %s" % (key, str(exc)))
 
-def export2json(d, json_file, expand=True, searchString="",replaceString=""):
+def export2json(d, keys2export, json_file, expand=True, searchString="",replaceString=""):
     data2export = {}
-    keys2export = []
-
-    for key in d.keys():
-        if key.startswith("_"):
-            continue
-        elif key.startswith("BB"):
-            continue
-        elif key.startswith("B_pn"):
-            continue
-        elif key.startswith("do_"):
-            continue
-        elif d.getVarFlag(key, "func"):
-            continue
-
-        keys2export.append(key)
 
     for key in keys2export:
         try:
diff --git a/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py b/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py
index 44e2c09a6f..fec13ce213 100644
--- a/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py
+++ b/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py
@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MIT
 
+import json
 import os
 import oe
 import unittest
@@ -95,3 +96,21 @@ class ShadowUtilsTidyFiles(OESelftestTestCase):
                 unsorted.append(file)
         if (unsorted):
             raise Exception("The following files were not sorted by ID as expected: %s" % unsorted)
+
+
+class TestDataTests(OESelftestTestCase):
+    def test_vardeps(self):
+        """
+        Test that variables changes are reflected in testdata.json
+        """
+        test_image = "core-image-minimal"
+        self.write_config('TESTDATA_EXPORT_VARS:append = " TEST_VARIABLE"\nTEST_VARIABLE = "VALUE1"')
+        bitbake(test_image)
+        self.append_config('TEST_VARIABLE = "VALUE2"')
+        bitbake(test_image)
+
+        vars = get_bb_vars(('DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME'), test_image)
+        testdata_json = "%s/%s.testdata.json" % (vars['DEPLOY_DIR_IMAGE'], vars['IMAGE_LINK_NAME'])
+        with open(testdata_json, 'r') as tf:
+            testdata_vars = json.load(tf)
+        self.assertEqual(testdata_vars['TEST_VARIABLE'], 'VALUE2')
diff --git a/meta/recipes-core/meta/meta-ide-support.bb b/meta/recipes-core/meta/meta-ide-support.bb
index 7f349f673d..3ee796d124 100644
--- a/meta/recipes-core/meta/meta-ide-support.bb
+++ b/meta/recipes-core/meta/meta-ide-support.bb
@@ -22,13 +22,16 @@ python () {
 
 addtask populate_ide_support before do_deploy after do_install
 
+inherit testdata_vars
+
 python do_write_test_data() {
     from oe.data import export2json
 
     out_dir = d.getVar('B')
     testdata_name = os.path.join(out_dir, "%s.testdata.json" % d.getVar('PN'))
 
-    export2json(d, testdata_name)
+    keys = d.getVar('TESTDATA_EXPORT_VARS').split()
+    export2json(d, keys, testdata_name)
 }
 addtask write_test_data before do_deploy after do_install
 
-- 
2.34.1



^ permalink raw reply related	[flat|nested] 2+ messages in thread
end of thread, other threads:[~2023-07-02 21:40 UTC | newest]

Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2023-06-29 15:26 [PATCH v3] rootfs-postcommands: Set vardeps for do_write_test_data Peter Hoyes
2023-07-02 21:40 ` [OE-core] " Alexandre Belloni

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox