From: alexis.lothore@bootlin.com
To: openembedded-core@lists.openembedded.org
Cc: alexandre.belloni@bootlin.com, thomas.petazzoni@bootlin.com
Subject: [PATCH v3 4/6] oeqa/selftest/resulttool: add test for metadata filtering on regression
Date: Fri, 24 Feb 2023 17:45:53 +0100 [thread overview]
Message-ID: <20230224164555.67634-5-alexis.lothore@bootlin.com> (raw)
In-Reply-To: <20230224164555.67634-1-alexis.lothore@bootlin.com>
From: Alexis Lothoré <alexis.lothore@bootlin.com>
Introduce new tests for the metadata-based filtering added for oeselftest
results
Signed-off-by: Alexis Lothoré <alexis.lothore@bootlin.com>
---
.../oeqa/selftest/cases/resulttooltests.py | 137 ++++++++++++++++++
1 file changed, 137 insertions(+)
diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py
index efdfd98af3c..75d406c122d 100644
--- a/meta/lib/oeqa/selftest/cases/resulttooltests.py
+++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py
@@ -98,3 +98,140 @@ class ResultToolTests(OESelftestTestCase):
resultutils.append_resultsdata(results, ResultToolTests.target_results_data, configmap=resultutils.flatten_map)
self.assertEqual(len(results[''].keys()), 5, msg="Flattened results not correct %s" % str(results))
+ def test_results_without_metadata_can_be_compared(self):
+ base_configuration = {"TEST_TYPE": "oeselftest",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86"}
+ target_configuration = {"TEST_TYPE": "oeselftest",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86"}
+ self.assertTrue(regression.can_be_compared(base_configuration, target_configuration),
+ msg="incorrect metadata filtering, tests without metadata should be compared")
+
+ def test_target_result_with_missing_metadata_can_not_be_compared(self):
+ base_configuration = {"TEST_TYPE": "oeselftest",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86",
+ "OESELFTEST_METADATA": {"run_all_tests": True,
+ "run_tests": None,
+ "skips": None,
+ "machine": None,
+ "select_tags": ["toolchain-user", "toolchain-system"],
+ "exclude_tags": None}}
+ target_configuration = {"TEST_TYPE": "oeselftest",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86"}
+ self.assertFalse(regression.can_be_compared(base_configuration, target_configuration),
+ msg="incorrect metadata filtering, tests should not be compared")
+
+ def test_results_with_matching_metadata_can_be_compared(self):
+ base_configuration = {"TEST_TYPE": "oeselftest",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86",
+ "OESELFTEST_METADATA": {"run_all_tests": True,
+ "run_tests": None,
+ "skips": None,
+ "machine": None,
+ "select_tags": ["toolchain-user", "toolchain-system"],
+ "exclude_tags": None}}
+ target_configuration = {"TEST_TYPE": "oeselftest",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86",
+ "OESELFTEST_METADATA": {"run_all_tests": True,
+ "run_tests": None,
+ "skips": None,
+ "machine": None,
+ "select_tags": ["toolchain-user", "toolchain-system"],
+ "exclude_tags": None}}
+ self.assertTrue(regression.can_be_compared(base_configuration, target_configuration),
+ msg="incorrect metadata filtering, tests with matching metadata should be compared")
+
+ def test_results_with_mismatching_metadata_can_not_be_compared(self):
+ base_configuration = {"TEST_TYPE": "oeselftest",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86",
+ "OESELFTEST_METADATA": {"run_all_tests": True,
+ "run_tests": None,
+ "skips": None,
+ "machine": None,
+ "select_tags": ["toolchain-user", "toolchain-system"],
+ "exclude_tags": None}}
+ target_configuration = {"TEST_TYPE": "oeselftest",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86",
+ "OESELFTEST_METADATA": {"run_all_tests": True,
+ "run_tests": None,
+ "skips": None,
+ "machine": None,
+ "select_tags": ["machine"],
+ "exclude_tags": None}}
+ self.assertFalse(regression.can_be_compared(base_configuration, target_configuration),
+ msg="incorrect metadata filtering, tests with mismatching metadata should not be compared")
+
+ def test_metadata_matching_is_only_checked_for_relevant_test_type(self):
+ base_configuration = {"TEST_TYPE": "runtime",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86",
+ "OESELFTEST_METADATA": {"run_all_tests": True,
+ "run_tests": None,
+ "skips": None,
+ "machine": None,
+ "select_tags": ["toolchain-user", "toolchain-system"],
+ "exclude_tags": None}}
+ target_configuration = {"TEST_TYPE": "runtime",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86",
+ "OESELFTEST_METADATA": {"run_all_tests": True,
+ "run_tests": None,
+ "skips": None,
+ "machine": None,
+ "select_tags": ["machine"],
+ "exclude_tags": None}}
+ self.assertTrue(regression.can_be_compared(base_configuration, target_configuration),
+ msg="incorrect metadata filtering, %s tests should be compared" % base_configuration['TEST_TYPE'])
+
+ def test_machine_matches(self):
+ base_configuration = {"TEST_TYPE": "runtime",
+ "MACHINE": "qemux86"}
+ target_configuration = {"TEST_TYPE": "runtime",
+ "MACHINE": "qemux86"}
+ self.assertTrue(regression.can_be_compared(base_configuration, target_configuration),
+ msg="incorrect machine filtering, identical machine tests should be compared")
+
+ def test_machine_mismatches(self):
+ base_configuration = {"TEST_TYPE": "runtime",
+ "MACHINE": "qemux86"}
+ target_configuration = {"TEST_TYPE": "runtime",
+ "MACHINE": "qemux86_64"}
+ self.assertFalse(regression.can_be_compared(base_configuration, target_configuration),
+ msg="incorrect machine filtering, mismatching machine tests should not be compared")
--
2.39.1
next prev parent reply other threads:[~2023-02-24 16:46 UTC|newest]
Thread overview: 17+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-02-24 16:45 [PATCH v3 0/6] scripts/resulttool/regression: add metadata filtering alexis.lothore
2023-02-24 16:45 ` [PATCH v3 1/6] scripts/oe-selftest: append metadata to tests results alexis.lothore
2023-02-24 16:45 ` [PATCH v3 2/6] scripts/resulttool/regression: remove unused import alexis.lothore
2023-02-24 16:45 ` [PATCH v3 3/6] scripts/resulttool/regression: add metadata filtering for oeselftest alexis.lothore
2023-02-24 16:45 ` alexis.lothore [this message]
2023-02-26 0:03 ` [OE-core] [PATCH v3 4/6] oeqa/selftest/resulttool: add test for metadata filtering on regression Richard Purdie
2023-02-24 16:45 ` [PATCH v3 5/6] scripts: add new helper for regression report generation alexis.lothore
2023-02-24 16:45 ` [PATCH v3 6/6] oeqa/selftest: add test for yocto_testresults_query.py alexis.lothore
2023-02-24 18:06 ` [OE-core] [PATCH v3 0/6] scripts/resulttool/regression: add metadata filtering Richard Purdie
[not found] ` <1746D4E8592324E9.29542@lists.openembedded.org>
2023-02-25 9:15 ` Richard Purdie
[not found] ` <1747067DAE80068A.29542@lists.openembedded.org>
2023-02-25 12:32 ` Richard Purdie
2023-02-25 15:59 ` Alexis Lothoré
2023-02-26 12:15 ` Richard Purdie
2023-02-26 15:42 ` Alexis Lothoré
2023-02-27 13:41 ` Richard Purdie
[not found] ` <1747113C8A4DBAD6.29542@lists.openembedded.org>
2023-02-25 12:44 ` Richard Purdie
2023-02-27 13:14 ` Ross Burton
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230224164555.67634-5-alexis.lothore@bootlin.com \
--to=alexis.lothore@bootlin.com \
--cc=alexandre.belloni@bootlin.com \
--cc=openembedded-core@lists.openembedded.org \
--cc=thomas.petazzoni@bootlin.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.