[PATCH 1/4] oeqa/runner: write testresult to json files

[PATCH 1/4] oeqa/runner: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA need to output testresult into json files, where these json
testresult files will be stored in git repository by the future
test-case-management tools.

Both the testresult (eg. PASSED, FAILED, ERROR) and  the test log
(eg. message from unit test assertion) will be created for storing.

Also the library class inside this patch will be reused by the future
test-case-management tools to write json testresult for manual test
case executed.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/lib/oeqa/core/runner.py | 137 +++++++++++++++++++++++++++++++++++++++----
 1 file changed, 126 insertions(+), 11 deletions(-)

diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py
index eeb625b..54efdd0 100644
--- a/meta/lib/oeqa/core/runner.py
+++ b/meta/lib/oeqa/core/runner.py
@@ -6,6 +6,8 @@ import time
 import unittest
 import logging
 import re
+import json
+import pathlib
 
 from unittest import TextTestResult as _TestResult
 from unittest import TextTestRunner as _TestRunner
@@ -44,6 +46,9 @@ class OETestResult(_TestResult):
 
         self.tc = tc
 
+        self.result_types = ['failures', 'errors', 'skipped', 'expectedFailures', 'successes']
+        self.result_desc = ['FAILED', 'ERROR', 'SKIPPED', 'EXPECTEDFAIL', 'PASSED']
+
     def startTest(self, test):
         # May have been set by concurrencytest
         if test.id() not in self.starttime:
@@ -80,7 +85,7 @@ class OETestResult(_TestResult):
             msg += " (skipped=%d)" % skipped
         self.tc.logger.info(msg)
 
-    def _getDetailsNotPassed(self, case, type, desc):
+    def _isTestResultContainTestCaseWithResultTypeProvided(self, case, type):
         found = False
 
         for (scase, msg) in getattr(self, type):
@@ -121,16 +126,12 @@ class OETestResult(_TestResult):
         for case_name in self.tc._registry['cases']:
             case = self.tc._registry['cases'][case_name]
 
-            result_types = ['failures', 'errors', 'skipped', 'expectedFailures', 'successes']
-            result_desc = ['FAILED', 'ERROR', 'SKIPPED', 'EXPECTEDFAIL', 'PASSED']
-
-            fail = False
+            found = False
             desc = None
-            for idx, name in enumerate(result_types):
-                (fail, msg) = self._getDetailsNotPassed(case, result_types[idx],
-                        result_desc[idx])
-                if fail:
-                    desc = result_desc[idx]
+            for idx, name in enumerate(self.result_types):
+                (found, msg) = self._isTestResultContainTestCaseWithResultTypeProvided(case, self.result_types[idx])
+                if found:
+                    desc = self.result_desc[idx]
                     break
 
             oeid = -1
@@ -143,13 +144,43 @@ class OETestResult(_TestResult):
             if case.id() in self.starttime and case.id() in self.endtime:
                 t = " (" + "{0:.2f}".format(self.endtime[case.id()] - self.starttime[case.id()]) + "s)"
 
-            if fail:
+            if found:
                 self.tc.logger.info("RESULTS - %s - Testcase %s: %s%s" % (case.id(),
                     oeid, desc, t))
             else:
                 self.tc.logger.info("RESULTS - %s - Testcase %s: %s%s" % (case.id(),
                     oeid, 'UNKNOWN', t))
 
+    def _get_testcase_result_and_testmessage_dict(self):
+        testcase_result_dict = {}
+        testcase_testmessage_dict = {}
+        for case_name in self.tc._registry['cases']:
+            case = self.tc._registry['cases'][case_name]
+
+            found = False
+            desc = None
+            test_msg = ''
+            for idx, name in enumerate(self.result_types):
+                (found, msg) = self._isTestResultContainTestCaseWithResultTypeProvided(case, self.result_types[idx])
+                if found:
+                    desc = self.result_desc[idx]
+                    test_msg = msg
+                    break
+
+            if found:
+                testcase_result_dict[case.id()] = desc
+                testcase_testmessage_dict[case.id()] = test_msg
+            else:
+                testcase_result_dict[case.id()] = "UNKNOWN"
+        return testcase_result_dict, testcase_testmessage_dict
+
+    def logDetailsInJson(self, file_dir):
+        (testcase_result_dict, testcase_testmessage_dict) = self._get_testcase_result_and_testmessage_dict()
+        if len(testcase_result_dict) > 0 and len(testcase_testmessage_dict) > 0:
+            jsontresulthelper = OEJSONTestResultHelper(testcase_result_dict, testcase_testmessage_dict)
+            jsontresulthelper.write_json_testresult_files(file_dir)
+            jsontresulthelper.write_testcase_log_files(os.path.join(file_dir, 'logs'))
+
 class OEListTestsResult(object):
     def wasSuccessful(self):
         return True
@@ -261,3 +292,87 @@ class OETestRunner(_TestRunner):
             self._list_tests_module(suite)
 
         return OEListTestsResult()
+
+class OEJSONTestResultHelper(object):
+    def __init__(self, testcase_result_dict, testcase_log_dict):
+        self.testcase_result_dict = testcase_result_dict
+        self.testcase_log_dict = testcase_log_dict
+
+    def get_testcase_list(self):
+        return self.testcase_result_dict.keys()
+
+    def get_testsuite_from_testcase(self, testcase):
+        testsuite = testcase[0:testcase.rfind(".")]
+        return testsuite
+
+    def get_testmodule_from_testsuite(self, testsuite):
+        testmodule = testsuite[0:testsuite.find(".")]
+        return testmodule
+
+    def get_testsuite_testcase_dictionary(self):
+        testsuite_testcase_dict = {}
+        for testcase in self.get_testcase_list():
+            testsuite = self.get_testsuite_from_testcase(testcase)
+            if testsuite in testsuite_testcase_dict:
+                testsuite_testcase_dict[testsuite].append(testcase)
+            else:
+                testsuite_testcase_dict[testsuite] = [testcase]
+        return testsuite_testcase_dict
+
+    def get_testmodule_testsuite_dictionary(self, testsuite_testcase_dict):
+        testsuite_list = testsuite_testcase_dict.keys()
+        testmodule_testsuite_dict = {}
+        for testsuite in testsuite_list:
+            testmodule = self.get_testmodule_from_testsuite(testsuite)
+            if testmodule in testmodule_testsuite_dict:
+                testmodule_testsuite_dict[testmodule].append(testsuite)
+            else:
+                testmodule_testsuite_dict[testmodule] = [testsuite]
+        return testmodule_testsuite_dict
+
+    def _get_testcase_result(self, testcase, testcase_status_dict):
+        if testcase in testcase_status_dict:
+            return testcase_status_dict[testcase]
+        return ""
+
+    def _create_testcase_testresult_object(self, testcase_list, testcase_result_dict):
+        testcase_dict = {}
+        for testcase in sorted(testcase_list):
+            result = self._get_testcase_result(testcase, testcase_result_dict)
+            testcase_dict[testcase] = {"testresult": result}
+        return testcase_dict
+
+    def _create_json_testsuite_string(self, testsuite_list, testsuite_testcase_dict, testcase_result_dict):
+        testsuite_object = {'testsuite': {}}
+        testsuite_dict = testsuite_object['testsuite']
+        for testsuite in sorted(testsuite_list):
+            testsuite_dict[testsuite] = {'testcase': {}}
+            testsuite_dict[testsuite]['testcase'] = self._create_testcase_testresult_object(
+                testsuite_testcase_dict[testsuite],
+                testcase_result_dict)
+        return json.dumps(testsuite_object, sort_keys=True, indent=4)
+
+    def write_json_testresult_files(self, write_dir):
+        if not os.path.exists(write_dir):
+            pathlib.Path(write_dir).mkdir(parents=True, exist_ok=True)
+        testsuite_testcase_dict = self.get_testsuite_testcase_dictionary()
+        testmodule_testsuite_dict = self.get_testmodule_testsuite_dictionary(testsuite_testcase_dict)
+        for testmodule in testmodule_testsuite_dict.keys():
+            testsuite_list = testmodule_testsuite_dict[testmodule]
+            json_testsuite = self._create_json_testsuite_string(testsuite_list, testsuite_testcase_dict,
+                                                                self.testcase_result_dict)
+            file_name = '%s.json' % testmodule
+            file_path = os.path.join(write_dir, file_name)
+            with open(file_path, 'w') as the_file:
+                the_file.write(json_testsuite)
+
+    def write_testcase_log_files(self, write_dir):
+        if not os.path.exists(write_dir):
+            pathlib.Path(write_dir).mkdir(parents=True, exist_ok=True)
+        for testcase in self.testcase_log_dict.keys():
+            test_log = self.testcase_log_dict[testcase]
+            if test_log is not None:
+                file_name = '%s.log' % testcase
+                file_path = os.path.join(write_dir, file_name)
+                with open(file_path, 'w') as the_file:
+                    the_file.write(test_log)
-- 
2.7.4

[PATCH 2/4] selftest/context: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA selftest need to output testresult into json files, where
these json testresult files will be stored into git repository
by the future test-case-management tools.

By default, oe-selftest will write json testresult into files.
To disable this, provide '-s' argument to oe-selftest execution.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/lib/oeqa/selftest/context.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/meta/lib/oeqa/selftest/context.py b/meta/lib/oeqa/selftest/context.py
index c78947e..61b4afb 100644
--- a/meta/lib/oeqa/selftest/context.py
+++ b/meta/lib/oeqa/selftest/context.py
@@ -73,6 +73,9 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
 
         parser.add_argument('--machine', required=False, choices=['random', 'all'],
                             help='Run tests on different machines (random/all).')
+
+        parser.add_argument('-s', '--skip-export-json', action='store_true',
+                            help='Skip the output test result in json format to files.')
         
         parser.set_defaults(func=self.run)
 
@@ -99,8 +102,8 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
         return cases_paths
 
     def _process_args(self, logger, args):
-        args.output_log = '%s-results-%s.log' % (self.name,
-                time.strftime("%Y%m%d%H%M%S"))
+        args.test_start_time = time.strftime("%Y%m%d%H%M%S")
+        args.output_log = '%s-results-%s.log' % (self.name, args.test_start_time)
         args.test_data_file = None
         args.CASES_PATHS = None
 
@@ -222,6 +225,11 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
             rc = self.tc.runTests(**self.tc_kwargs['run'])
             rc.logDetails()
             rc.logSummary(self.name)
+            if not args.skip_export_json:
+                json_result_dir = os.path.join(os.path.dirname(os.path.abspath(args.output_log)),
+                                               'json_testresults-%s' % args.test_start_time,
+                                               'oe-selftest')
+                rc.logDetailsInJson(json_result_dir)
 
         return rc
 
-- 
2.7.4

[PATCH 3/4] testimage.bbclass: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA testimage need to output testresult into json files, where
these json testresult files will be stored into git repository
by the future test-case-management tools.

By default, testimage will write json testresult, to disable
this, specify OEQA_SKIP_OUTPUT_JSON="1" as configuration.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/classes/testimage.bbclass | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 39de191..0adaf60 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -306,6 +306,18 @@ def testimage_main(d):
         bb.fatal('%s - FAILED - tests were interrupted during execution' % pn, forcelog=True)
     results.logDetails()
     results.logSummary(pn)
+    if (d.getVar('OEQA_SKIP_OUTPUT_JSON')) == '1':
+        bb.debug(2, 'Skip the OEQA output json testresult as OEQA_SKIP_OUTPUT_JSON=1')
+    else:
+        workdir = d.getVar("WORKDIR")
+        image_basename = d.getVar("IMAGE_BASENAME")
+        json_result_dir = os.path.join(workdir,
+                                       'temp',
+                                       'json_testresults-%s' % os.getpid(),
+                                       'runtime',
+                                       machine,
+                                       image_basename)
+        results.logDetailsInJson(json_result_dir)
     if not results.wasSuccessful():
         bb.fatal('%s - FAILED - check the task log and the ssh log' % pn, forcelog=True)
 
-- 
2.7.4

[PATCH 4/4] testsdk.bbclass: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA sdk and sdkext need to output testresult into json files, where
these json testresult files will be stored into git repository
by the future test-case-management tools.

By default, sdk and sdkext will write json testresult, to disable
this, specify OEQA_SKIP_OUTPUT_JSON="1" as configuration.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/classes/testsdk.bbclass | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/meta/classes/testsdk.bbclass b/meta/classes/testsdk.bbclass
index d3f475d..26c4789 100644
--- a/meta/classes/testsdk.bbclass
+++ b/meta/classes/testsdk.bbclass
@@ -83,7 +83,17 @@ def testsdk_main(d):
 
         result.logDetails()
         result.logSummary(component, context_msg)
-
+        if (d.getVar('OEQA_SKIP_OUTPUT_JSON')) == '1':
+            bb.debug(2, 'Skip the OEQA output json testresult as OEQA_SKIP_OUTPUT_JSON=1')
+        else:
+            workdir = d.getVar("WORKDIR")
+            image_basename = d.getVar("IMAGE_BASENAME")
+            json_result_dir = os.path.join(workdir,
+                                           'temp',
+                                           'json_testresults-%s' % os.getpid(),
+                                           'sdk',
+                                           image_basename)
+            result.logDetailsInJson(json_result_dir)
         if not result.wasSuccessful():
             fail = True
 
@@ -187,7 +197,17 @@ def testsdkext_main(d):
 
         result.logDetails()
         result.logSummary(component, context_msg)
-
+        if (d.getVar('OEQA_SKIP_OUTPUT_JSON')) == '1':
+            bb.debug(2, 'Skip the OEQA output json testresult as OEQA_SKIP_OUTPUT_JSON=1')
+        else:
+            workdir = d.getVar("WORKDIR")
+            image_basename = d.getVar("IMAGE_BASENAME")
+            json_result_dir = os.path.join(workdir,
+                                           'temp',
+                                           'json_testresults-%s' % os.getpid(),
+                                           'sdkext',
+                                           image_basename)
+            result.logDetailsInJson(json_result_dir)
         if not result.wasSuccessful():
             fail = True
 
-- 
2.7.4

[PATCH 3/4] testimage.bbclass: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA testimage need to output testresult into json files, where
these json testresult files will be stored into git repository
by the future test-case-management tools.

By default, testimage will write json testresult, to disable
this, specify OEQA_SKIP_OUTPUT_JSON="1" as configuration.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/classes/testimage.bbclass | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 0e07afa..2901e10 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -306,6 +306,18 @@ def testimage_main(d):
         bb.fatal('%s - FAILED - tests were interrupted during execution' % pn, forcelog=True)
     results.logDetails()
     results.logSummary(pn)
+    if (d.getVar('OEQA_SKIP_OUTPUT_JSON')) == '1':
+        bb.debug(2, 'Skip the OEQA output json testresult as OEQA_SKIP_OUTPUT_JSON=1')
+    else:
+        workdir = d.getVar("WORKDIR")
+        image_basename = d.getVar("IMAGE_BASENAME")
+        json_result_dir = os.path.join(workdir,
+                                       'temp',
+                                       'json_testresults-%s' % os.getpid(),
+                                       'runtime',
+                                       machine,
+                                       image_basename)
+        results.logDetailsInJson(json_result_dir)
     if not results.wasSuccessful():
         bb.fatal('%s - FAILED - check the task log and the ssh log' % pn, forcelog=True)
 
-- 
2.7.4

Re: [PATCH 3/4] testimage.bbclass: write testresult to json files

[Richard Purdie]

On Fri, 2018-10-12 at 14:33 +0800, Yeoh Ee Peng wrote:
> As part of the solution to replace Testopia to store testresult,
> OEQA testimage need to output testresult into json files, where
> these json testresult files will be stored into git repository
> by the future test-case-management tools.
> 
> By default, testimage will write json testresult, to disable
> this, specify OEQA_SKIP_OUTPUT_JSON="1" as configuration.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
> ---
>  meta/classes/testimage.bbclass | 12 ++++++++++++
>  1 file changed, 12 insertions(+)
> 
> diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
> index 0e07afa..2901e10 100644
> --- a/meta/classes/testimage.bbclass
> +++ b/meta/classes/testimage.bbclass
> @@ -306,6 +306,18 @@ def testimage_main(d):
>          bb.fatal('%s - FAILED - tests were interrupted during execution' % pn, forcelog=True)
>      results.logDetails()
>      results.logSummary(pn)
> +    if (d.getVar('OEQA_SKIP_OUTPUT_JSON')) == '1':
> +        bb.debug(2, 'Skip the OEQA output json testresult as OEQA_SKIP_OUTPUT_JSON=1')
> +    else:

Please don't add OEQA_SKIP_OUTPUT_JSON, lets just write these files out
in all cases. They're small and useful and I having too many codepaths
without good reason just means there is more to test and more to break.

> +        workdir = d.getVar("WORKDIR")
> +        image_basename = d.getVar("IMAGE_BASENAME")
> +        json_result_dir = os.path.join(workdir,

Just use d.getVar("WORKDIR") instead of the intermediate variable here,
it only makes sense to use the intermediate one if there are multiple
uses of it.

Cheers,

Richard

[PATCH 3/4] testimage.bbclass: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA testimage need to output testresult into json files, where
these json testresult files will be stored into git repository
by the future test-case-management tools.

To configure multiple instances of bitbake to write json testresult
to a single testresult file, user will define the variable
"OEQA_JSON_RESULT_COMMON_DIR" with the common directory for writing
json testresult.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/classes/testimage.bbclass | 34 ++++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 2642a72..112ba71 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -2,7 +2,7 @@
 #
 # Released under the MIT license (see COPYING.MIT)
 
-
+inherit metadata_scm
 # testimage.bbclass enables testing of qemu images using python unittests.
 # Most of the tests are commands run on target image over ssh.
 # To use it add testimage to global inherit and call your target image with -c testimage
@@ -141,6 +141,33 @@ def testimage_sanity(d):
         bb.fatal('When TEST_TARGET is set to "simpleremote" '
                  'TEST_TARGET_IP and TEST_SERVER_IP are needed too.')
 
+def _get_configuration(d, test_type, pid, machine):
+    import platform
+    bb_core_dir = d.getVar("BBFILE_PATTERN_core=")
+    return {'TEST_TYPE': test_type,
+            'BRANCH': base_get_metadata_git_branch(bb_core_dir, None).strip(),
+            'COMMIT': base_get_metadata_git_revision(bb_core_dir, None),
+            'PROCESS_ID': pid,
+            'MACHINE': machine,
+            'IMAGE_BASENAME': d.getVar("IMAGE_BASENAME"),
+            'IMAGE_PKGTYPE': d.getVar("IMAGE_PKGTYPE"),
+            'HOST_DISTRO': platform.linux_distribution()}
+
+def _get_json_result_dir(d, configuration):
+    json_result_dir = os.path.join(d.getVar("WORKDIR"),
+                                   'temp',
+                                   'json_testresults-%s' % configuration['PROCESS_ID'],
+                                   configuration['TEST_TYPE'],
+                                   configuration['MACHINE'],
+                                   configuration['IMAGE_BASENAME'])
+    oeqa_json_result_common_dir = d.getVar("OEQA_JSON_RESULT_COMMON_DIR")
+    if oeqa_json_result_common_dir:
+        json_result_dir = oeqa_json_result_common_dir
+    return json_result_dir
+
+def _get_result_id(configuration):
+    return '%s-%s-%s' % (configuration['TEST_TYPE'], configuration['IMAGE_BASENAME'], configuration['MACHINE'])
+
 def testimage_main(d):
     import os
     import json
@@ -308,7 +335,10 @@ def testimage_main(d):
     # Show results (if we have them)
     if not results:
         bb.fatal('%s - FAILED - tests were interrupted during execution' % pn, forcelog=True)
-    results.logDetails()
+    configuration = _get_configuration(d, 'runtime', os.getpid(), machine)
+    results.logDetails(_get_json_result_dir(d, configuration),
+                       configuration,
+                       _get_result_id(configuration))
     results.logSummary(pn)
     if not results.wasSuccessful():
         bb.fatal('%s - FAILED - check the task log and the ssh log' % pn, forcelog=True)
-- 
2.7.4

Re: [PATCH 3/4] testimage.bbclass: write testresult to json files

[Richard Purdie]

On Mon, 2018-10-22 at 14:54 +0800, Yeoh Ee Peng wrote:
> As part of the solution to replace Testopia to store testresult,
> OEQA testimage need to output testresult into json files, where
> these json testresult files will be stored into git repository
> by the future test-case-management tools.
> 
> To configure multiple instances of bitbake to write json testresult
> to a single testresult file, user will define the variable
> "OEQA_JSON_RESULT_COMMON_DIR" with the common directory for writing
> json testresult.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
> ---
>  meta/classes/testimage.bbclass | 34 ++++++++++++++++++++++++++++++++--
>  1 file changed, 32 insertions(+), 2 deletions(-)
> 
> diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
> index 2642a72..112ba71 100644
> --- a/meta/classes/testimage.bbclass
> +++ b/meta/classes/testimage.bbclass
> @@ -2,7 +2,7 @@
>  #
>  # Released under the MIT license (see COPYING.MIT)
>  
> -
> +inherit metadata_scm
>  # testimage.bbclass enables testing of qemu images using python unittests.
>  # Most of the tests are commands run on target image over ssh.
>  # To use it add testimage to global inherit and call your target image with -c testimage
> @@ -141,6 +141,33 @@ def testimage_sanity(d):
>          bb.fatal('When TEST_TARGET is set to "simpleremote" '
>                   'TEST_TARGET_IP and TEST_SERVER_IP are needed too.')
>  
> +def _get_configuration(d, test_type, pid, machine):
> +    import platform
> +    bb_core_dir = d.getVar("BBFILE_PATTERN_core=")
> +    return {'TEST_TYPE': test_type,
> +            'BRANCH': base_get_metadata_git_branch(bb_core_dir, None).strip(),
> +            'COMMIT': base_get_metadata_git_revision(bb_core_dir, None),

We need to record all the revisions/branches that are being used. Could
we just inject the contents of metadata['layers'] directly into the
json file here?

> +            'PROCESS_ID': pid,
> +            'MACHINE': machine,
> +            'IMAGE_BASENAME': d.getVar("IMAGE_BASENAME"),
> +            'IMAGE_PKGTYPE': d.getVar("IMAGE_PKGTYPE"),
> +            'HOST_DISTRO': platform.linux_distribution()}
> +
> +def _get_json_result_dir(d, configuration):
> +    json_result_dir = os.path.join(d.getVar("WORKDIR"),
> +                                   'temp',
> +                                   'json_testresults-%s' % configuration['PROCESS_ID'],
> +                                   configuration['TEST_TYPE'],
> +                                   configuration['MACHINE'],
> +                                   configuration['IMAGE_BASENAME'])
> +    oeqa_json_result_common_dir = d.getVar("OEQA_JSON_RESULT_COMMON_DIR")
> +    if oeqa_json_result_common_dir:
> +        json_result_dir = oeqa_json_result_common_dir

Do we need both codepaths here or can we just place things in
OEQA_JSON_RESULT_COMMON_DIR unconditionally here?

(we could rename it to OEQA_JSON_RESULT_DIR?)

Cheers,

Richard

Re: [PATCH 3/4] testimage.bbclass: write testresult to json files

[Yeoh, Ee Peng]

Richard,

Noted, let's us made the changes to record all the revisions/branches that are being used.

The reason to provide the default result_dir here was to prevent the case where no json testresult being write out when user does not provide the specific OEQA_JSON_RESULT_DIR, otherwise user might need to rerun test to get the json testresult.  Probably, it will be better that bitbake will prompt user to provide result_dir variable next round if it was not avaiable to write testresult to a specify location (eg. common location to write all testresult). Please let me know your inputs.
> +def _get_json_result_dir(d, configuration):
> +    json_result_dir = os.path.join(d.getVar("WORKDIR"),
> +                                   'temp',
> +                                   'json_testresults-%s' % configuration['PROCESS_ID'],
> +                                   configuration['TEST_TYPE'],
> +                                   configuration['MACHINE'],
> +                                   configuration['IMAGE_BASENAME'])
> +    oeqa_json_result_common_dir = d.getVar("OEQA_JSON_RESULT_COMMON_DIR")
> +    if oeqa_json_result_common_dir:
> +        json_result_dir = oeqa_json_result_common_dir

Best regards,
Yeoh Ee Peng 

-----Original Message-----
From: richard.purdie@linuxfoundation.org [mailto:richard.purdie@linuxfoundation.org] 
Sent: Monday, October 22, 2018 4:38 PM
To: Yeoh, Ee Peng <ee.peng.yeoh@intel.com>; openembedded-core@lists.openembedded.org
Subject: Re: [OE-core] [PATCH 3/4] testimage.bbclass: write testresult to json files

On Mon, 2018-10-22 at 14:54 +0800, Yeoh Ee Peng wrote:
> As part of the solution to replace Testopia to store testresult, OEQA 
> testimage need to output testresult into json files, where these json 
> testresult files will be stored into git repository by the future 
> test-case-management tools.
> 
> To configure multiple instances of bitbake to write json testresult to 
> a single testresult file, user will define the variable 
> "OEQA_JSON_RESULT_COMMON_DIR" with the common directory for writing 
> json testresult.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
> ---
>  meta/classes/testimage.bbclass | 34 
> ++++++++++++++++++++++++++++++++--
>  1 file changed, 32 insertions(+), 2 deletions(-)
> 
> diff --git a/meta/classes/testimage.bbclass 
> b/meta/classes/testimage.bbclass index 2642a72..112ba71 100644
> --- a/meta/classes/testimage.bbclass
> +++ b/meta/classes/testimage.bbclass
> @@ -2,7 +2,7 @@
>  #
>  # Released under the MIT license (see COPYING.MIT)
>  
> -
> +inherit metadata_scm
>  # testimage.bbclass enables testing of qemu images using python unittests.
>  # Most of the tests are commands run on target image over ssh.
>  # To use it add testimage to global inherit and call your target 
> image with -c testimage @@ -141,6 +141,33 @@ def testimage_sanity(d):
>          bb.fatal('When TEST_TARGET is set to "simpleremote" '
>                   'TEST_TARGET_IP and TEST_SERVER_IP are needed too.')
>  
> +def _get_configuration(d, test_type, pid, machine):
> +    import platform
> +    bb_core_dir = d.getVar("BBFILE_PATTERN_core=")
> +    return {'TEST_TYPE': test_type,
> +            'BRANCH': base_get_metadata_git_branch(bb_core_dir, None).strip(),
> +            'COMMIT': base_get_metadata_git_revision(bb_core_dir, 
> +None),

We need to record all the revisions/branches that are being used. Could we just inject the contents of metadata['layers'] directly into the json file here?

> +            'PROCESS_ID': pid,
> +            'MACHINE': machine,
> +            'IMAGE_BASENAME': d.getVar("IMAGE_BASENAME"),
> +            'IMAGE_PKGTYPE': d.getVar("IMAGE_PKGTYPE"),
> +            'HOST_DISTRO': platform.linux_distribution()}
> +
> +def _get_json_result_dir(d, configuration):
> +    json_result_dir = os.path.join(d.getVar("WORKDIR"),
> +                                   'temp',
> +                                   'json_testresults-%s' % configuration['PROCESS_ID'],
> +                                   configuration['TEST_TYPE'],
> +                                   configuration['MACHINE'],
> +                                   configuration['IMAGE_BASENAME'])
> +    oeqa_json_result_common_dir = d.getVar("OEQA_JSON_RESULT_COMMON_DIR")
> +    if oeqa_json_result_common_dir:
> +        json_result_dir = oeqa_json_result_common_dir

Do we need both codepaths here or can we just place things in OEQA_JSON_RESULT_COMMON_DIR unconditionally here?

(we could rename it to OEQA_JSON_RESULT_DIR?)

Cheers,

Richard

[PATCH 3/4] testimage.bbclass: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA testimage need to output testresult into json files, where
these json testresult files will be stored into git repository
by the future test-case-management tools.

To configure multiple instances of bitbake to write json testresult
to a single testresult file at custom direcotry, user will define
the variable "OEQA_JSON_RESULT_DIR" with the custom directory for writing
json testresult.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/classes/testimage.bbclass | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 2642a72..df91d90 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -2,7 +2,7 @@
 #
 # Released under the MIT license (see COPYING.MIT)
 
-
+inherit metadata_scm
 # testimage.bbclass enables testing of qemu images using python unittests.
 # Most of the tests are commands run on target image over ssh.
 # To use it add testimage to global inherit and call your target image with -c testimage
@@ -141,6 +141,30 @@ def testimage_sanity(d):
         bb.fatal('When TEST_TARGET is set to "simpleremote" '
                  'TEST_TARGET_IP and TEST_SERVER_IP are needed too.')
 
+def _get_testimage_configuration(d, test_type, pid, machine):
+    import platform
+    configuration = {'TEST_TYPE': test_type,
+                    'PROCESS_ID': pid,
+                    'MACHINE': machine,
+                    'IMAGE_BASENAME': d.getVar("IMAGE_BASENAME"),
+                    'IMAGE_PKGTYPE': d.getVar("IMAGE_PKGTYPE"),
+                    'HOST_DISTRO': platform.linux_distribution()}
+    layers = (d.getVar("BBLAYERS") or "").split()
+    for l in layers:
+        configuration['%s_BRANCH_REV' % os.path.basename(l)] = '%s:%s' % (base_get_metadata_git_branch(l, None).strip(),
+                                                                          base_get_metadata_git_revision(l, None))
+    return configuration
+
+def _get_testimage_json_result_dir(d, configuration):
+    json_result_dir = os.path.join(d.getVar("WORKDIR"), 'oeqa')
+    oeqa_json_result_common_dir = d.getVar("OEQA_JSON_RESULT_DIR")
+    if oeqa_json_result_common_dir:
+        json_result_dir = oeqa_json_result_common_dir
+    return json_result_dir
+
+def _get_testimage_result_id(configuration):
+    return '%s-%s-%s' % (configuration['TEST_TYPE'], configuration['IMAGE_BASENAME'], configuration['MACHINE'])
+
 def testimage_main(d):
     import os
     import json
@@ -308,7 +332,10 @@ def testimage_main(d):
     # Show results (if we have them)
     if not results:
         bb.fatal('%s - FAILED - tests were interrupted during execution' % pn, forcelog=True)
-    results.logDetails()
+    configuration = _get_testimage_configuration(d, 'runtime', os.getpid(), machine)
+    results.logDetails(_get_testimage_json_result_dir(d, configuration),
+                       configuration,
+                       _get_testimage_result_id(configuration))
     results.logSummary(pn)
     if not results.wasSuccessful():
         bb.fatal('%s - FAILED - check the task log and the ssh log' % pn, forcelog=True)
-- 
2.7.4

[PATCH 3/4] testimage.bbclass: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA testimage need to output testresult into json files, where
these json testresult files will be stored into git repository
by the future test-case-management tools.

By default, json testresult file will be written to "oeqa"
directory under the "WORKDIR" directory.

To configure multiple instances of bitbake to write json testresult
to a single testresult file at custom directory, user will define
the variable "OEQA_JSON_RESULT_DIR" with the custom directory for
json testresult.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/classes/testimage.bbclass | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 2642a72..109eecc 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -2,7 +2,7 @@
 #
 # Released under the MIT license (see COPYING.MIT)
 
-
+inherit metadata_scm
 # testimage.bbclass enables testing of qemu images using python unittests.
 # Most of the tests are commands run on target image over ssh.
 # To use it add testimage to global inherit and call your target image with -c testimage
@@ -141,6 +141,30 @@ def testimage_sanity(d):
         bb.fatal('When TEST_TARGET is set to "simpleremote" '
                  'TEST_TARGET_IP and TEST_SERVER_IP are needed too.')
 
+def _get_testimage_configuration(d, test_type, pid, machine):
+    import platform
+    configuration = {'TEST_TYPE': test_type,
+                    'PROCESS_ID': pid,
+                    'MACHINE': machine,
+                    'IMAGE_BASENAME': d.getVar("IMAGE_BASENAME"),
+                    'IMAGE_PKGTYPE': d.getVar("IMAGE_PKGTYPE"),
+                    'HOST_DISTRO': ('-'.join(platform.linux_distribution())).replace(' ', '-')}
+    layers = (d.getVar("BBLAYERS") or "").split()
+    for l in layers:
+        configuration['%s_BRANCH_REV' % os.path.basename(l)] = '%s:%s' % (base_get_metadata_git_branch(l, None).strip(),
+                                                                          base_get_metadata_git_revision(l, None))
+    return configuration
+
+def _get_testimage_json_result_dir(d):
+    json_result_dir = os.path.join(d.getVar("WORKDIR"), 'oeqa')
+    custom_json_result_dir = d.getVar("OEQA_JSON_RESULT_DIR")
+    if custom_json_result_dir:
+        json_result_dir = custom_json_result_dir
+    return json_result_dir
+
+def _get_testimage_result_id(configuration):
+    return '%s_%s_%s' % (configuration['TEST_TYPE'], configuration['IMAGE_BASENAME'], configuration['MACHINE'])
+
 def testimage_main(d):
     import os
     import json
@@ -308,7 +332,10 @@ def testimage_main(d):
     # Show results (if we have them)
     if not results:
         bb.fatal('%s - FAILED - tests were interrupted during execution' % pn, forcelog=True)
-    results.logDetails()
+    configuration = _get_testimage_configuration(d, 'runtime', os.getpid(), machine)
+    results.logDetails(_get_testimage_json_result_dir(d),
+                       configuration,
+                       _get_testimage_result_id(configuration))
     results.logSummary(pn)
     if not results.wasSuccessful():
         bb.fatal('%s - FAILED - check the task log and the ssh log' % pn, forcelog=True)
-- 
2.7.4