* [PATCH 2/2] oeqa/ltp: rewrote LTP testcase and parser
2023-07-20 15:51 [PATCH 1/2] ltp: add RDEPENDS on findutils ross.burton
@ 2023-07-20 15:51 ` ross.burton
0 siblings, 0 replies; 2+ messages in thread
From: ross.burton @ 2023-07-20 15:51 UTC (permalink / raw)
To: openembedded-core; +Cc: nd
From: Ross Burton <ross.burton@arm.com>
The LTP test reporting appears to be a little fragile so I tried to make
it more reliable.
Primarily this is done by not passing -p to runltp, which results in
machine-readable logfiles instead of human-readable. These are easier
to parse and have more context in, so we can also report correctly
skipped tests.
Signed-off-by: Ross Burton <ross.burton@arm.com>
---
meta/lib/oeqa/runtime/cases/ltp.py | 17 +++++---
meta/lib/oeqa/utils/logparser.py | 62 +++++++++++++++++++-----------
2 files changed, 51 insertions(+), 28 deletions(-)
diff --git a/meta/lib/oeqa/runtime/cases/ltp.py b/meta/lib/oeqa/runtime/cases/ltp.py
index a66d5d13d75..29c26d7d324 100644
--- a/meta/lib/oeqa/runtime/cases/ltp.py
+++ b/meta/lib/oeqa/runtime/cases/ltp.py
@@ -65,29 +65,34 @@ class LtpTest(LtpTestBase):
ltp_groups += ltp_fs
def runltp(self, ltp_group):
- cmd = '/opt/ltp/runltp -f %s -p -q -r /opt/ltp -l /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group)
+ # LTP appends to log files, so ensure we start with a clean log
+ self.target.deleteFiles("/opt/ltp/results/", ltp_group)
+
+ cmd = '/opt/ltp/runltp -f %s -q -r /opt/ltp -l /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group)
+
starttime = time.time()
(status, output) = self.target.run(cmd)
endtime = time.time()
+ # Write the console log to disk for convenience
with open(os.path.join(self.ltptest_log_dir, "%s-raw.log" % ltp_group), 'w') as f:
f.write(output)
+ # Also put the console log into the test result JSON
self.extras['ltpresult.rawlogs']['log'] = self.extras['ltpresult.rawlogs']['log'] + output
- # copy nice log from DUT
- dst = os.path.join(self.ltptest_log_dir, "%s" % ltp_group )
+ # Copy the machine-readable test results locally so we can parse it
+ dst = os.path.join(self.ltptest_log_dir, ltp_group)
remote_src = "/opt/ltp/results/%s" % ltp_group
(status, output) = self.target.copyFrom(remote_src, dst, True)
- msg = 'File could not be copied. Output: %s' % output
if status:
+ msg = 'File could not be copied. Output: %s' % output
self.target.logger.warning(msg)
parser = LtpParser()
results, sections = parser.parse(dst)
- runtime = int(endtime-starttime)
- sections['duration'] = runtime
+ sections['duration'] = int(endtime-starttime)
self.sections[ltp_group] = sections
failed_tests = {}
diff --git a/meta/lib/oeqa/utils/logparser.py b/meta/lib/oeqa/utils/logparser.py
index 8054acc853b..496d9e0c903 100644
--- a/meta/lib/oeqa/utils/logparser.py
+++ b/meta/lib/oeqa/utils/logparser.py
@@ -4,7 +4,7 @@
# SPDX-License-Identifier: MIT
#
-import sys
+import enum
import os
import re
@@ -106,30 +106,48 @@ class PtestParser(object):
f.write(status + ": " + test_name + "\n")
-# ltp log parsing
-class LtpParser(object):
- def __init__(self):
- self.results = {}
- self.section = {'duration': "", 'log': ""}
-
+class LtpParser:
+ """
+ Parse the machine-readable LTP log output into a ptest-friendly data structure.
+ """
def parse(self, logfile):
- test_regex = {}
- test_regex['PASSED'] = re.compile(r"PASS")
- test_regex['FAILED'] = re.compile(r"FAIL")
- test_regex['SKIPPED'] = re.compile(r"SKIP")
-
- with open(logfile, errors='replace') as f:
+ results = {}
+ # Aaccumulate the duration here but as the log rounds quick tests down
+ # to 0 seconds this is very much a lower bound. The caller can replace
+ # the value.
+ section = {"duration": 0, "log": ""}
+
+ class LtpExitCode(enum.IntEnum):
+ # Exit codes as defined in ltp/include/tst_res_flags.h
+ TPASS = 0 # Test passed flag
+ TFAIL = 1 # Test failed flag
+ TBROK = 2 # Test broken flag
+ TWARN = 4 # Test warning flag
+ TINFO = 16 # Test information flag
+ TCONF = 32 # Test not appropriate for configuration flag
+
+ with open(logfile, errors="replace") as f:
+ # Lines look like this:
+ # tag=cfs_bandwidth01 stime=1689762564 dur=0 exit=exited stat=32 core=no cu=0 cs=0
for line in f:
- for t in test_regex:
- result = test_regex[t].search(line)
- if result:
- self.results[line.split()[0].strip()] = t
-
- for test in self.results:
- result = self.results[test]
- self.section['log'] = self.section['log'] + ("%s: %s\n" % (result.strip()[:-2], test.strip()))
+ if not line.startswith("tag="):
+ continue
- return self.results, self.section
+ values = dict(s.split("=") for s in line.strip().split())
+
+ section["duration"] += int(values["dur"])
+ exitcode = int(values["stat"])
+ if values["exit"] == "exited" and exitcode == LtpExitCode.TCONF:
+ # Exited normally with the "invalid configuration" code
+ results[values["tag"]] = "SKIPPED"
+ elif exitcode == LtpExitCode.TPASS:
+ # Successful exit
+ results[values["tag"]] = "PASSED"
+ else:
+ # Other exit
+ results[values["tag"]] = "FAILED"
+
+ return results, section
# ltp Compliance log parsing
--
2.34.1
^ permalink raw reply related [flat|nested] 2+ messages in thread