* [RFC PATCH 1/5] oeqa/targetcontrol.py: simplify checking for qemu_use_kvm
2017-07-19 8:16 [RFC PATCH 0/5] revive runtime/cases/_ptest.py Robert Yang
@ 2017-07-19 8:16 ` Robert Yang
2017-07-19 8:16 ` [RFC PATCH 2/5] runtime/cases/_ptest.py: revive it Robert Yang
` (4 subsequent siblings)
5 siblings, 0 replies; 9+ messages in thread
From: Robert Yang @ 2017-07-19 8:16 UTC (permalink / raw)
To: openembedded-core, richard.purdie, paul.eggleton, ross.burton
The "if qemu_use_kvm" is not needed.
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
meta/lib/oeqa/targetcontrol.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/meta/lib/oeqa/targetcontrol.py b/meta/lib/oeqa/targetcontrol.py
index 3255e3a5c63..11e6c820e85 100644
--- a/meta/lib/oeqa/targetcontrol.py
+++ b/meta/lib/oeqa/targetcontrol.py
@@ -132,9 +132,8 @@ class QemuTarget(BaseTarget):
dump_host_cmds = d.getVar("testimage_dump_host")
dump_dir = d.getVar("TESTIMAGE_DUMP_DIR")
qemu_use_kvm = d.getVar("QEMU_USE_KVM")
- if qemu_use_kvm and \
- (qemu_use_kvm == "True" and "x86" in d.getVar("MACHINE") or \
- d.getVar("MACHINE") in qemu_use_kvm.split()):
+ if qemu_use_kvm == "True" and "x86" in d.getVar("MACHINE") or \
+ d.getVar("MACHINE") in qemu_use_kvm.split():
use_kvm = True
else:
use_kvm = False
--
2.11.0
^ permalink raw reply related [flat|nested] 9+ messages in thread* [RFC PATCH 2/5] runtime/cases/_ptest.py: revive it
2017-07-19 8:16 [RFC PATCH 0/5] revive runtime/cases/_ptest.py Robert Yang
2017-07-19 8:16 ` [RFC PATCH 1/5] oeqa/targetcontrol.py: simplify checking for qemu_use_kvm Robert Yang
@ 2017-07-19 8:16 ` Robert Yang
2017-07-19 8:16 ` [RFC PATCH 3/5] oeqa/utils/logparser.py: add skip status Robert Yang
` (3 subsequent siblings)
5 siblings, 0 replies; 9+ messages in thread
From: Robert Yang @ 2017-07-19 8:16 UTC (permalink / raw)
To: openembedded-core, richard.purdie, paul.eggleton, ross.burton
* Make it work with current oeqa
* Skip the test if ptest is not in DISTRO_FEATURES
* Skip the test if ptest-pkgs is not in IMAGE_FEATURES
* The logs are saved to:
testimage/ptest_log -> testimage/ptest_log.<datetime>
* Will write a tool to make regression check
[YOCTO #11547]
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
meta/lib/oeqa/runtime/cases/_ptest.py | 91 ++++++++++++-----------------------
1 file changed, 30 insertions(+), 61 deletions(-)
diff --git a/meta/lib/oeqa/runtime/cases/_ptest.py b/meta/lib/oeqa/runtime/cases/_ptest.py
index aaed9a5352f..034bfe98959 100644
--- a/meta/lib/oeqa/runtime/cases/_ptest.py
+++ b/meta/lib/oeqa/runtime/cases/_ptest.py
@@ -1,28 +1,10 @@
-import os
-import shutil
-import subprocess
-
from oeqa.runtime.case import OERuntimeTestCase
from oeqa.core.decorator.depends import OETestDepends
from oeqa.core.decorator.oeid import OETestID
-from oeqa.core.decorator.data import skipIfNotDataVar, skipIfNotFeature
-from oeqa.runtime.decorator.package import OEHasPackage
-
-from oeqa.runtime.cases.dnf import DnfTest
-from oeqa.utils.logparser import *
-from oeqa.utils.httpserver import HTTPService
-
-class PtestRunnerTest(DnfTest):
+from oeqa.core.decorator.data import skipIfNotFeature
+from oeqa.utils.logparser import Lparser, Result
- @classmethod
- def setUpClass(cls):
- rpm_deploy = os.path.join(cls.tc.td['DEPLOY_DIR'], 'rpm')
- cls.repo_server = HTTPService(rpm_deploy, cls.tc.target.server_ip)
- cls.repo_server.start()
-
- @classmethod
- def tearDownClass(cls):
- cls.repo_server.stop()
+class PtestRunnerTest(OERuntimeTestCase):
# a ptest log parser
def parse_ptest(self, logfile):
@@ -59,45 +41,32 @@ class PtestRunnerTest(DnfTest):
result.sort_tests()
return result
- def _install_ptest_packages(self):
- # Get ptest packages that can be installed in the image.
- packages_dir = os.path.join(self.tc.td['DEPLOY_DIR'], 'rpm')
- ptest_pkgs = [pkg[:pkg.find('-ptest')+6]
- for _, _, filenames in os.walk(packages_dir)
- for pkg in filenames
- if 'ptest' in pkg
- and pkg[:pkg.find('-ptest')] in self.tc.image_packages]
-
- repo_url = 'http://%s:%s' % (self.target.server_ip,
- self.repo_server.port)
- dnf_options = ('--repofrompath=oe-ptest-repo,%s '
- '--nogpgcheck '
- 'install -y' % repo_url)
- self.dnf('%s %s ptest-runner' % (dnf_options, ' '.join(ptest_pkgs)))
-
- @skipIfNotFeature('package-management',
- 'Test requires package-management to be in DISTRO_FEATURES')
- @skipIfNotFeature('ptest',
- 'Test requires package-management to be in DISTRO_FEATURES')
- @skipIfNotDataVar('IMAGE_PKGTYPE', 'rpm',
- 'RPM is not the primary package manager')
- @OEHasPackage(['dnf'])
+ @OETestID(1600)
+ @skipIfNotFeature('ptest', 'Test requires ptest to be in DISTRO_FEATURES')
+ @skipIfNotFeature('ptest-pkgs', 'Test requires ptest-pkgs to be in IMAGE_FEATURES')
@OETestDepends(['ssh.SSHTest.test_ssh'])
def test_ptestrunner(self):
- self.ptest_log = os.path.join(self.tc.td['TEST_LOG_DIR'],
- 'ptest-%s.log' % self.tc.td['DATETIME'])
- self._install_ptest_packages()
-
- (runnerstatus, result) = self.target.run('/usr/bin/ptest-runner > /tmp/ptest.log 2>&1', 0)
- #exit code is !=0 even if ptest-runner executes because some ptest tests fail.
- self.assertTrue(runnerstatus != 127, msg="Cannot execute ptest-runner!")
- self.target.copyFrom('/tmp/ptest.log', self.ptest_log)
- shutil.copyfile(self.ptest_log, "ptest.log")
-
- result = self.parse_ptest("ptest.log")
- log_results_to_location = "./results"
- if os.path.exists(log_results_to_location):
- shutil.rmtree(log_results_to_location)
- os.makedirs(log_results_to_location)
-
- result.log_as_files(log_results_to_location, test_status = ['pass','fail'])
+ import datetime
+
+ test_log_dir = self.td['TEST_LOG_DIR']
+ # Don't use self.td['DATETIME'], it's a cached value, not up-to-date,
+ # and may cause "File exists" when re-reun.
+ datetime = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
+ ptest_log_dir_link = os.path.join(test_log_dir, 'ptest_log')
+ ptest_log_dir = '%s.%s' % (ptest_log_dir_link, datetime)
+ ptest_runner_log = os.path.join(ptest_log_dir, 'ptest-runner.log')
+
+ status, output = self.target.run('ptest-runner', 0)
+ os.makedirs(ptest_log_dir)
+ with open(ptest_runner_log, 'w') as f:
+ f.write(output)
+
+ # status !=0 is OK since some ptest tests may fail
+ self.assertTrue(status != 127, msg="Cannot execute ptest-runner!")
+
+ # Parse and save results
+ parse_result = self.parse_ptest(ptest_runner_log)
+ parse_result.log_as_files(ptest_log_dir, test_status = ['pass','fail'])
+ if os.path.exists(ptest_log_dir_link):
+ os.remove(ptest_log_dir_link)
+ os.symlink(os.path.basename(ptest_log_dir), ptest_log_dir_link)
--
2.11.0
^ permalink raw reply related [flat|nested] 9+ messages in thread* [RFC PATCH 3/5] oeqa/utils/logparser.py: add skip status
2017-07-19 8:16 [RFC PATCH 0/5] revive runtime/cases/_ptest.py Robert Yang
2017-07-19 8:16 ` [RFC PATCH 1/5] oeqa/targetcontrol.py: simplify checking for qemu_use_kvm Robert Yang
2017-07-19 8:16 ` [RFC PATCH 2/5] runtime/cases/_ptest.py: revive it Robert Yang
@ 2017-07-19 8:16 ` Robert Yang
2017-07-19 8:16 ` [RFC PATCH 4/5] runtime/cases/_ptest.py: " Robert Yang
` (2 subsequent siblings)
5 siblings, 0 replies; 9+ messages in thread
From: Robert Yang @ 2017-07-19 8:16 UTC (permalink / raw)
To: openembedded-core, richard.purdie, paul.eggleton, ross.burton
Some test cases maybe skipped, let's parse it.
[YOCTO #11547]
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
meta/lib/oeqa/utils/logparser.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/meta/lib/oeqa/utils/logparser.py b/meta/lib/oeqa/utils/logparser.py
index b377dcd2716..4355ce0b767 100644
--- a/meta/lib/oeqa/utils/logparser.py
+++ b/meta/lib/oeqa/utils/logparser.py
@@ -9,7 +9,7 @@ from . import ftools
# A parser that can be used to identify weather a line is a test result or a section statement.
class Lparser(object):
- def __init__(self, test_0_pass_regex, test_0_fail_regex, section_0_begin_regex=None, section_0_end_regex=None, **kwargs):
+ def __init__(self, test_0_pass_regex, test_0_fail_regex, test_0_skip_regex, section_0_begin_regex=None, section_0_end_regex=None, **kwargs):
# Initialize the arguments dictionary
if kwargs:
self.args = kwargs
@@ -19,12 +19,13 @@ class Lparser(object):
# Add the default args to the dictionary
self.args['test_0_pass_regex'] = test_0_pass_regex
self.args['test_0_fail_regex'] = test_0_fail_regex
+ self.args['test_0_skip_regex'] = test_0_skip_regex
if section_0_begin_regex:
self.args['section_0_begin_regex'] = section_0_begin_regex
if section_0_end_regex:
self.args['section_0_end_regex'] = section_0_end_regex
- self.test_possible_status = ['pass', 'fail', 'error']
+ self.test_possible_status = ['pass', 'fail', 'error', 'skip']
self.section_possible_status = ['begin', 'end']
self.initialized = False
--
2.11.0
^ permalink raw reply related [flat|nested] 9+ messages in thread* [RFC PATCH 4/5] runtime/cases/_ptest.py: add skip status
2017-07-19 8:16 [RFC PATCH 0/5] revive runtime/cases/_ptest.py Robert Yang
` (2 preceding siblings ...)
2017-07-19 8:16 ` [RFC PATCH 3/5] oeqa/utils/logparser.py: add skip status Robert Yang
@ 2017-07-19 8:16 ` Robert Yang
2017-07-19 8:16 ` [RFC PATCH 5/5] runtime/cases/_ptest.py: rename it to ptest.py Robert Yang
2017-07-20 2:09 ` [RFC PATCH 0/5] revive runtime/cases/_ptest.py Robert Yang
5 siblings, 0 replies; 9+ messages in thread
From: Robert Yang @ 2017-07-19 8:16 UTC (permalink / raw)
To: openembedded-core, richard.purdie, paul.eggleton, ross.burton
The packages' test cases maybe skipped, check and save them.
[YOCTO #11547]
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
meta/lib/oeqa/runtime/cases/_ptest.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/meta/lib/oeqa/runtime/cases/_ptest.py b/meta/lib/oeqa/runtime/cases/_ptest.py
index 034bfe98959..9810bd15178 100644
--- a/meta/lib/oeqa/runtime/cases/_ptest.py
+++ b/meta/lib/oeqa/runtime/cases/_ptest.py
@@ -10,6 +10,7 @@ class PtestRunnerTest(OERuntimeTestCase):
def parse_ptest(self, logfile):
parser = Lparser(test_0_pass_regex="^PASS:(.+)",
test_0_fail_regex="^FAIL:(.+)",
+ test_0_skip_regex="^SKIP:(.+)",
section_0_begin_regex="^BEGIN: .*/(.+)/ptest",
section_0_end_regex="^END: .*/(.+)/ptest")
parser.init()
@@ -38,6 +39,10 @@ class PtestRunnerTest(OERuntimeTestCase):
result.store(current_section, name, status)
continue
+ if line_type == 'test' and status == 'skip':
+ result.store(current_section, name, status)
+ continue
+
result.sort_tests()
return result
@@ -66,7 +71,7 @@ class PtestRunnerTest(OERuntimeTestCase):
# Parse and save results
parse_result = self.parse_ptest(ptest_runner_log)
- parse_result.log_as_files(ptest_log_dir, test_status = ['pass','fail'])
+ parse_result.log_as_files(ptest_log_dir, test_status = ['pass','fail', 'skip'])
if os.path.exists(ptest_log_dir_link):
os.remove(ptest_log_dir_link)
os.symlink(os.path.basename(ptest_log_dir), ptest_log_dir_link)
--
2.11.0
^ permalink raw reply related [flat|nested] 9+ messages in thread* [RFC PATCH 5/5] runtime/cases/_ptest.py: rename it to ptest.py
2017-07-19 8:16 [RFC PATCH 0/5] revive runtime/cases/_ptest.py Robert Yang
` (3 preceding siblings ...)
2017-07-19 8:16 ` [RFC PATCH 4/5] runtime/cases/_ptest.py: " Robert Yang
@ 2017-07-19 8:16 ` Robert Yang
2017-07-20 2:09 ` [RFC PATCH 0/5] revive runtime/cases/_ptest.py Robert Yang
5 siblings, 0 replies; 9+ messages in thread
From: Robert Yang @ 2017-07-19 8:16 UTC (permalink / raw)
To: openembedded-core, richard.purdie, paul.eggleton, ross.burton
It works now.
[YOCTO #11547]
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
meta/lib/oeqa/runtime/cases/{_ptest.py => ptest.py} | 0
1 file changed, 0 insertions(+), 0 deletions(-)
rename meta/lib/oeqa/runtime/cases/{_ptest.py => ptest.py} (100%)
diff --git a/meta/lib/oeqa/runtime/cases/_ptest.py b/meta/lib/oeqa/runtime/cases/ptest.py
similarity index 100%
rename from meta/lib/oeqa/runtime/cases/_ptest.py
rename to meta/lib/oeqa/runtime/cases/ptest.py
--
2.11.0
^ permalink raw reply [flat|nested] 9+ messages in thread* Re: [RFC PATCH 0/5] revive runtime/cases/_ptest.py
2017-07-19 8:16 [RFC PATCH 0/5] revive runtime/cases/_ptest.py Robert Yang
` (4 preceding siblings ...)
2017-07-19 8:16 ` [RFC PATCH 5/5] runtime/cases/_ptest.py: rename it to ptest.py Robert Yang
@ 2017-07-20 2:09 ` Robert Yang
2017-07-21 12:32 ` Richard Purdie
5 siblings, 1 reply; 9+ messages in thread
From: Robert Yang @ 2017-07-20 2:09 UTC (permalink / raw)
To: openembedded-core, richard.purdie, paul.eggleton, ross.burton
On 07/19/2017 04:16 PM, Robert Yang wrote:
> Hello,
>
> These patches can make ptest test case work, RP has suggested we write a tool to
> do the regression check on ptest result, I think that the use case is like:
>
> $ bitbake <image> -ctestiamge # Suppose we add ptest to default test cases in the future
> # Upgrade a recipe form V1.0 to V1.1
> $ bitbake <image> -ctestiamge # Run the test again
>
> Then the regression check tool can report what's different (passed, failed,
> skipped) between V1.0 and V1.1.
>
> Currently, I'm not sure about where to save the ptest results, I saved it to
> ${WORKDIR}/testimage/ptest_log atm, e.g.:
> tmp/work/qemux86-poky-linux/core-image-minimal/1.0-r0/testimage/ptest_log
I still have to think about more about where to put the test result.
>
> But it will be removed after -cclean, then no regression check can be made, so
> I'd like to save the ptest results to DEPLOY_DIR_IMAGE if no objections, and
> make -cclean not remove them. (Or only keep the latest 2 results).
>
> And I'm not sure where to add the regression check/tool, maybe one of:
> 1) Add a separate tool in oe-core/scripts, this can make it easy to do regression
> check among different build directories, and runtime/cases/_ptest.py can invoke it.
I'm leaning to add a script called ptest-regression-check which is more
flexible, the cases/ptest.py or buildhistory (also the user) can run it
when needed.
// Robert
> 2) Add it to runtime/cases/_ptest.py directly
> 3) Add it to buildhistory
>
> I prefer the first one, please feel free to give your comments.
>
> // Robert
>
> The following changes since commit ef68005a8c527e9b1d05b7769f0ec8ebe9ec3f91:
>
> webkitgtk: Upgrade to 2.16.5 (2017-07-17 13:49:04 +0100)
>
> are available in the git repository at:
>
> git://git.openembedded.org/openembedded-core-contrib rbt/ptest
> http://cgit.openembedded.org/openembedded-core-contrib/log/?h=rbt/ptest
>
> Robert Yang (5):
> oeqa/targetcontrol.py: simplify checking for qemu_use_kvm
> runtime/cases/_ptest.py: revive it
> oeqa/utils/logparser.py: add skip status
> runtime/cases/_ptest.py: add skip status
> runtime/cases/_ptest.py: rename it to ptest.py
>
> meta/lib/oeqa/runtime/cases/_ptest.py | 103 ----------------------------------
> meta/lib/oeqa/runtime/cases/ptest.py | 77 +++++++++++++++++++++++++
> meta/lib/oeqa/targetcontrol.py | 5 +-
> meta/lib/oeqa/utils/logparser.py | 5 +-
> 4 files changed, 82 insertions(+), 108 deletions(-)
> delete mode 100644 meta/lib/oeqa/runtime/cases/_ptest.py
> create mode 100644 meta/lib/oeqa/runtime/cases/ptest.py
>
^ permalink raw reply [flat|nested] 9+ messages in thread* Re: [RFC PATCH 0/5] revive runtime/cases/_ptest.py
2017-07-20 2:09 ` [RFC PATCH 0/5] revive runtime/cases/_ptest.py Robert Yang
@ 2017-07-21 12:32 ` Richard Purdie
2017-07-24 2:11 ` Robert Yang
0 siblings, 1 reply; 9+ messages in thread
From: Richard Purdie @ 2017-07-21 12:32 UTC (permalink / raw)
To: Robert Yang, openembedded-core, paul.eggleton, ross.burton
On Thu, 2017-07-20 at 10:09 +0800, Robert Yang wrote:
>
> On 07/19/2017 04:16 PM, Robert Yang wrote:
> >
> > Hello,
> >
> > These patches can make ptest test case work, RP has suggested we
> > write a tool to
> > do the regression check on ptest result, I think that the use case
> > is like:
> >
> > $ bitbake <image> -ctestiamge # Suppose we add ptest to default
> > test cases in the future
> > # Upgrade a recipe form V1.0 to V1.1
> > $ bitbake <image> -ctestiamge # Run the test again
> >
> > Then the regression check tool can report what's different (passed,
> > failed,
> > skipped) between V1.0 and V1.1.
> >
> > Currently, I'm not sure about where to save the ptest results, I
> > saved it to
> > ${WORKDIR}/testimage/ptest_log atm, e.g.:
> > tmp/work/qemux86-poky-linux/core-image-minimal/1.0-
> > r0/testimage/ptest_log
> I still have to think about more about where to put the test result.
Would buildhistory make sense for these since we already have a
mechanism to store results there between builds?
> > But it will be removed after -cclean, then no regression check can
> > be made, so
> > I'd like to save the ptest results to DEPLOY_DIR_IMAGE if no
> > objections, and
> > make -cclean not remove them. (Or only keep the latest 2 results).
> >
> > And I'm not sure where to add the regression check/tool, maybe one
> > of:
> > 1) Add a separate tool in oe-core/scripts, this can make it easy to
> > do regression
> > check among different build directories, and
> > runtime/cases/_ptest.py can invoke it.
> I'm leaning to add a script called ptest-regression-check which is
> more flexible, the cases/ptest.py or buildhistory (also the user) can
> run it when needed.
I'd either add a script, or perhaps an option to the buildhistory-diff?
Paul might have some adivce on that. I would like to understand if
there is a good reason we can't use buildhistory to at least help with
some of this?
Cheers,
Richard
^ permalink raw reply [flat|nested] 9+ messages in thread* Re: [RFC PATCH 0/5] revive runtime/cases/_ptest.py
2017-07-21 12:32 ` Richard Purdie
@ 2017-07-24 2:11 ` Robert Yang
0 siblings, 0 replies; 9+ messages in thread
From: Robert Yang @ 2017-07-24 2:11 UTC (permalink / raw)
To: Richard Purdie, openembedded-core, paul.eggleton, ross.burton
On 07/21/2017 08:32 PM, Richard Purdie wrote:
> On Thu, 2017-07-20 at 10:09 +0800, Robert Yang wrote:
>>
>> On 07/19/2017 04:16 PM, Robert Yang wrote:
>>>
>>> Hello,
>>>
>>> These patches can make ptest test case work, RP has suggested we
>>> write a tool to
>>> do the regression check on ptest result, I think that the use case
>>> is like:
>>>
>>> $ bitbake <image> -ctestiamge # Suppose we add ptest to default
>>> test cases in the future
>>> # Upgrade a recipe form V1.0 to V1.1
>>> $ bitbake <image> -ctestiamge # Run the test again
>>>
>>> Then the regression check tool can report what's different (passed,
>>> failed,
>>> skipped) between V1.0 and V1.1.
>>>
>>> Currently, I'm not sure about where to save the ptest results, I
>>> saved it to
>>> ${WORKDIR}/testimage/ptest_log atm, e.g.:
>>> tmp/work/qemux86-poky-linux/core-image-minimal/1.0-
>>> r0/testimage/ptest_log
>> I still have to think about more about where to put the test result.
>
> Would buildhistory make sense for these since we already have a
> mechanism to store results there between builds?
Thanks, I will add it to buildhistory.
>
>>> But it will be removed after -cclean, then no regression check can
>>> be made, so
>>> I'd like to save the ptest results to DEPLOY_DIR_IMAGE if no
>>> objections, and
>>> make -cclean not remove them. (Or only keep the latest 2 results).
>>>
>>> And I'm not sure where to add the regression check/tool, maybe one
>>> of:
>>> 1) Add a separate tool in oe-core/scripts, this can make it easy to
>>> do regression
>>> check among different build directories, and
>>> runtime/cases/_ptest.py can invoke it.
>> I'm leaning to add a script called ptest-regression-check which is
>> more flexible, the cases/ptest.py or buildhistory (also the user) can
>> run it when needed.
>
> I'd either add a script, or perhaps an option to the buildhistory-diff?
> Paul might have some adivce on that. I would like to understand if
> there is a good reason we can't use buildhistory to at least help with
> some of this?
Thanks, the script (I named it prc -- Ptest Result Compare or Ptest Regression
Compile) already done, I will investigate buildhistory to see how to merge it
into buildhistory.
// Robert
>
> Cheers,
>
> Richard
>
^ permalink raw reply [flat|nested] 9+ messages in thread