[PATCH 0/1] Support tests in other layers

[PATCH 0/1] Support tests in other layers

[Paul Eggleton]

Completion of an earlier work-in-progress change by Stefan. I had
earlier suggested using a special "tests/runtime" directory in other
layers that wish to add their own tests, but when I looked into it I
figured it was actually a lot simpler if we kept the same structure as
the tests in OE-Core i.e. lib/oeqa/runtime, since sys.path is already
extended in base.bbclass to make that work for all layers; we just
needed to tweak __init__.py in the oeqa/runtime directory to enable
unittest to import units from other layers in that same directory.


The following changes since commit e473e60d5572f36829068f6d3db9ce9ba9633d71:

  mkfontscale: This no longer needs a full libx11, xproto suffices (2013-08-22 18:29:50 +0100)

are available in the git repository at:

  git://git.openembedded.org/openembedded-core-contrib paule/testimage
  http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/testimage

Stefan Stanacar (1):
  classes/testimage: add support for finding tests in other layers

 meta/classes/testimage.bbclass    | 55 ++++++++++++++++++++++++++++++---------
 meta/lib/oeqa/runtime/__init__.py |  3 +++
 2 files changed, 46 insertions(+), 12 deletions(-)

-- 
1.8.1.2

[PATCH 1/1] classes/testimage: add support for finding tests in other layers

[Paul Eggleton]

From: Stefan Stanacar <stefanx.stanacar@intel.com>

A layer can add tests in lib/oeqa/runtime (provided it extends BBPATH as
normal) and enable them with TEST_SUITES_append = " testname". Test
module names shouldn't collide though.

Signed-off-by: Stefan Stanacar <stefanx.stanacar@intel.com>
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 meta/classes/testimage.bbclass    | 55 ++++++++++++++++++++++++++++++---------
 meta/lib/oeqa/runtime/__init__.py |  3 +++
 2 files changed, 46 insertions(+), 12 deletions(-)

diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 86121e4..f914e57 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -17,6 +17,7 @@
 # Each name in TEST_SUITES represents a required test for the image. (no skipping allowed)
 # Appending "auto" means that it will try to run all tests that are suitable for the image (each test decides that on it's own).
 # Note that order in TEST_SUITES is important (it's the order tests run) and it influences tests dependencies.
+# A layer can add its own tests in lib/oeqa/runtime, provided it extends BBPATH as normal in its layer.conf.
 
 # TEST_LOG_DIR contains a ssh log (what command is running, output and return codes) and a qemu boot log till login
 # Booting is handled by this class, and it's not a test in itself.
@@ -41,6 +42,43 @@ do_testimage[nostamp] = "1"
 do_testimage[depends] += "qemu-native:do_populate_sysroot"
 do_testimage[depends] += "qemu-helper-native:do_populate_sysroot"
 
+
+def get_tests_list(d):
+    testsuites = d.getVar("TEST_SUITES", True).split()
+    bbpath = d.getVar("BBPATH", True).split(':')
+
+    # This relies on lib/ under each directory in BBPATH being added to sys.path
+    # (as done by default in base.bbclass)
+    testslist = []
+    for testname in testsuites:
+        if testname != "auto":
+            found = False
+            for p in bbpath:
+                if os.path.exists(os.path.join(p, 'lib', 'oeqa', 'runtime', testname + '.py')):
+                    testslist.append("oeqa.runtime." + testname)
+                    found = True
+                    break
+            if not found:
+                bb.error('Test %s specified in TEST_SUITES could not be found in lib/oeqa/runtime under BBPATH' % testname)
+
+    if "auto" in testsuites:
+        def add_auto_list(path):
+            if not os.path.exists(os.path.join(path, '__init__.py')):
+                bb.fatal('Tests directory %s exists but is missing __init__.py' % path)
+            files = sorted([f for f in os.listdir(path) if f.endswith('.py') and not f.startswith('_')])
+            for f in files:
+                module = 'oeqa.runtime.' + f[:-3]
+                if module not in testslist:
+                    testslist.append(module)
+
+        for p in bbpath:
+            testpath = os.path.join(p, 'lib', 'oeqa', 'runtime')
+            bb.debug(2, 'Searching for tests in %s' % testpath)
+            if os.path.exists(testpath):
+                add_auto_list(testpath)
+
+    return testslist
+
 def testimage_main(d):
     import unittest
     import os
@@ -55,23 +93,16 @@ def testimage_main(d):
     bb.utils.mkdirhier(testdir)
 
     # tests in TEST_SUITES become required tests
-    # they won't be skipped even if they aren't suitable for a default image (like xorg for minimal)
-    testsuites = d.getVar("TEST_SUITES", True)
-    # testslist is what we'll run and order matters
-    testslist = [ x for x in testsuites.split() if x != "auto" ]
-    # if we have auto search for other modules
-    if "auto" in testsuites:
-        for f in os.listdir(os.path.dirname(os.path.abspath(oeqa.runtime.__file__))):
-            if f.endswith('.py') and not f.startswith('_')  and f[:-3] not in testslist:
-                testslist.append(f[:-3])
-
-    testslist = [ "oeqa.runtime." + x for x in testslist ]
+    # they won't be skipped even if they aren't suitable for a image (like xorg for minimal)
+    # testslist is what we'll actually pass to the unittest loader
+    testslist = get_tests_list(d)
+    testsrequired = [t for t in d.getVar("TEST_SUITES", True).split() if t != "auto"]
 
     class TestContext:
         def __init__(self):
             self.d = d
             self.testslist = testslist
-            self.testsrequired = testsuites.split()
+            self.testsrequired = testsrequired
             self.filesdir = os.path.join(os.path.dirname(os.path.abspath(oeqa.runtime.__file__)),"files")
 
     # test context
diff --git a/meta/lib/oeqa/runtime/__init__.py b/meta/lib/oeqa/runtime/__init__.py
index e69de29..4cf3fa7 100644
--- a/meta/lib/oeqa/runtime/__init__.py
+++ b/meta/lib/oeqa/runtime/__init__.py
@@ -0,0 +1,3 @@
+# Enable other layers to have tests in the same named directory
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
-- 
1.8.1.2