* [igt-dev] [PATCH i-g-t] scripts/test_list.py: make filtering logic more generic
@ 2023-04-04 17:17 Mauro Carvalho Chehab
2023-04-04 18:49 ` [igt-dev] ✗ Fi.CI.BAT: failure for " Patchwork
0 siblings, 1 reply; 2+ messages in thread
From: Mauro Carvalho Chehab @ 2023-04-04 17:17 UTC (permalink / raw)
To: igt-dev
From: Mauro Carvalho Chehab <mchehab@kernel.org>
Allow multiple filter filters and filter also print outputs,
except for the json one.
After the patch, applying multiple filters and display per-test
documentation, for instance, could be done with commands like:
./scripts/igt_doc.py --config tests/xe/*.json --filter-field \
Sub-category=~execbuf run\ type=~bat --per-test
The filter will now be applied to all possible outputs of the
script.
Requested-by: Jari Tahvanainen <jari.tahvanainen@intel.com>
Signed-off-by: Mauro Carvalho Chehab <mchehab@kernel.org>
---
scripts/igt_doc.py | 12 ++++--
scripts/test_list.py | 89 +++++++++++++++++++++++++++++++-------------
2 files changed, 72 insertions(+), 29 deletions(-)
diff --git a/scripts/igt_doc.py b/scripts/igt_doc.py
index 547cb81bce02..4af0f045d13d 100755
--- a/scripts/igt_doc.py
+++ b/scripts/igt_doc.py
@@ -32,8 +32,8 @@ parser.add_argument("--show-subtests", action="store_true",
help="Shows the name of the documented subtests in alphabetical order.")
parser.add_argument("--sort-field",
help="modify --show-subtests to sort output based on SORT_FIELD value")
-parser.add_argument("--filter-field",
- help="modify --show-subtests to filter output based a regex given by FILTER_FIELD=~'regex'")
+parser.add_argument("--filter-field", nargs='*',
+ help="filter subtests based on regular expressions given by FILTER_FIELD=~'regex'")
parser.add_argument("--check-testlist", action="store_true",
help="Compare documentation against IGT runner testlist.")
parser.add_argument("--include-plan", action="store_true",
@@ -54,10 +54,14 @@ parse_args = parser.parse_args()
tests = TestList(parse_args.config, parse_args.include_plan, parse_args.files,
parse_args.igt_build_path, parse_args.igt_runner)
+if parse_args.filter_field:
+ for filter_expr in parse_args.filter_field:
+ tests.add_filter(filter_expr)
+
RUN = 0
if parse_args.show_subtests:
RUN = 1
- tests.show_subtests(parse_args.sort_field, parse_args.filter_field)
+ tests.show_subtests(parse_args.sort_field)
if parse_args.check_testlist:
RUN = 1
@@ -67,7 +71,7 @@ if parse_args.gen_testlist:
RUN = 1
if not parse_args.sort_field:
sys.exit("Need a field to split the testlists")
- tests.gen_testlist(parse_args.gen_testlist, parse_args.sort_field, parse_args.filter_field)
+ tests.gen_testlist(parse_args.gen_testlist, parse_args.sort_field)
if parse_args.to_json:
RUN = 1
diff --git a/scripts/test_list.py b/scripts/test_list.py
index 075f44d5e604..cddbd333acba 100755
--- a/scripts/test_list.py
+++ b/scripts/test_list.py
@@ -258,6 +258,7 @@ class TestList:
self.level_count = 0
self.field_list = {}
self.title = None
+ self.filters = {}
driver_name = re.sub(r'(.*/)?([^\/]+)/.*', r'\2', config_fname).capitalize()
@@ -385,6 +386,23 @@ class TestList:
self.__add_field(key, sublevel, hierarchy_level, field[key])
+ def __filter_subtest(self, test, subtest, field_not_found_value):
+
+ """ Apply filter criteria to subtests """
+
+ for filter_field, regex in self.filters.items():
+ if filter_field in subtest:
+ if not re.match(regex, subtest[filter_field]):
+ return True
+ elif filter_field in test:
+ if not re.match(regex, test[filter_field]):
+ return True
+ else:
+ return field_not_found_value
+
+ # None of the filtering rules were applied
+ return False
+
def expand_subtest(self, fname, test_name, test, allow_inherit):
"""Expand subtest wildcards providing an array with subtests"""
@@ -394,7 +412,7 @@ class TestList:
for subtest in self.doc[test]["subtest"].keys():
summary = test_name
if self.doc[test]["subtest"][subtest]["Summary"] != '':
- summary += '@' + self.doc[test]["subtest"][subtest]["Summary"]
+ summary += '@' + self.doc[test]["subtest"][subtest]["Summary"]
if not summary:
continue
@@ -533,6 +551,9 @@ class TestList:
subtest_array = self.expand_subtest(fname, name, test, subtest_only)
for subtest in subtest_array:
+ if self.__filter_subtest(test, subtest, True):
+ continue
+
summary = subtest["Summary"]
dic[summary] = {}
@@ -571,6 +592,19 @@ class TestList:
name = re.sub(r'\.[ch]', '', name)
name = "igt@" + name
+ tmp_subtest = self.expand_subtest(fname, name, test, False)
+
+ # Get subtests first, to avoid displaying tests with
+ # all filtered subtests
+ subtest_array = []
+ for subtest in tmp_subtest:
+ if self.__filter_subtest(self.doc[test], subtest, True):
+ continue
+ subtest_array.append(subtest)
+
+ if not subtest_array:
+ continue
+
print(len(name) * '=')
print(name)
print(len(name) * '=')
@@ -584,9 +618,8 @@ class TestList:
print(f":{field}: {self.doc[test][field]}")
- subtest_array = self.expand_subtest(fname, name, test, False)
-
for subtest in subtest_array:
+
print()
print(subtest["Summary"])
print(len(subtest["Summary"]) * '=')
@@ -709,11 +742,28 @@ class TestList:
with open(out_fname, "w", encoding='utf8') as write_file:
json.dump(test_dict, write_file, indent = 4)
+ #
+ # Add filters
+ #
+ def add_filter(self, filter_field_expr):
+
+ """ Add a filter criteria for output data """
+
+ match = re.match(r"(.*)=~\s*(.*)", filter_field_expr)
+ if not match:
+ sys.exit(f"Filter field {filter_field_expr} is not at <field> =~ <regex> syntax")
+ field = match.group(1).strip().lower()
+ if field not in self.field_list:
+ sys.exit(f"Field '{field}' is not defined")
+ filter_field = self.field_list[field]
+ regex = re.compile("{0}".format(match.group(2).strip()), re.I) # pylint: disable=C0209
+ self.filters[filter_field] = regex
+
#
# Subtest list methods
#
- def get_subtests(self, sort_field = None, filter_field_expr = None):
+ def get_subtests(self, sort_field = None):
"""Return an array with all subtests"""
@@ -725,17 +775,6 @@ class TestList:
sys.exit(f"Field '{sort_field}' is not defined")
sort_field = self.field_list[sort_field.lower()]
- if filter_field_expr:
- match = re.match(r"(.*)=~\s*(.*)", filter_field_expr)
- if not match:
- sys.exit(f"Filter field {filter_field_expr} is not at <field> =~ <regex> syntax")
-
- field = match.group(1).strip().lower()
- if field not in self.field_list:
- sys.exit(f"Field '{field}' is not defined")
- filter_field = self.field_list[field]
- regex = re.compile("{0}".format(match.group(2).strip()), re.I) # pylint: disable=C0209
-
for test in sorted(self.doc.keys()):
fname = self.doc[test]["File"]
@@ -746,11 +785,8 @@ class TestList:
subtest_array = self.expand_subtest(fname, test_name, test, True)
for subtest in subtest_array:
- if filter_field_expr:
- if filter_field not in subtest:
- continue
- if not re.match(regex, subtest[filter_field]):
- continue
+ if self.__filter_subtest(test, subtest, True):
+ continue
if sort_field:
if sort_field in subtest:
@@ -776,6 +812,9 @@ class TestList:
if not self.igt_build_path or not self.igt_runner:
sys.exit("Need the IGT build path and igt_runner executable file name")
+ if self.filters:
+ print("NOTE: test checks are affected by filters")
+
doc_subtests = sorted(self.get_subtests()[""])
for i in range(0, len(doc_subtests)): # pylint: disable=C0200
@@ -1035,12 +1074,12 @@ class TestList:
sys.exit(f"{fname}:{file_ln + 1}: Error: unrecognized line. Need to add field at %s?\n\t==> %s" %
(self.config_fname, file_line))
- def show_subtests(self, sort_field, filter_field):
+ def show_subtests(self, sort_field):
"""Show subtests, allowing sort and filter a field """
if sort_field:
- test_subtests = self.get_subtests(sort_field, filter_field)
+ test_subtests = self.get_subtests(sort_field)
for val_key in sorted(test_subtests.keys()):
if not test_subtests[val_key]:
continue
@@ -1051,17 +1090,17 @@ class TestList:
for sub in test_subtests[val_key]:
print (f" {sub}")
else:
- for sub in self.get_subtests(sort_field, filter_field)[""]:
+ for sub in self.get_subtests(sort_field)[""]:
print (sub)
- def gen_testlist(self, directory, sort_field, filter_field):
+ def gen_testlist(self, directory, sort_field):
"""Generate testlists from the test documentation"""
test_prefix = os.path.commonprefix(self.get_subtests()[""])
test_prefix = re.sub(r'^igt@', '', test_prefix)
- test_subtests = self.get_subtests(sort_field, filter_field)
+ test_subtests = self.get_subtests(sort_field)
for test in test_subtests.keys(): # pylint: disable=C0201,C0206
if not test_subtests[test]:
--
2.39.2
^ permalink raw reply related [flat|nested] 2+ messages in thread
* [igt-dev] ✗ Fi.CI.BAT: failure for scripts/test_list.py: make filtering logic more generic
2023-04-04 17:17 [igt-dev] [PATCH i-g-t] scripts/test_list.py: make filtering logic more generic Mauro Carvalho Chehab
@ 2023-04-04 18:49 ` Patchwork
0 siblings, 0 replies; 2+ messages in thread
From: Patchwork @ 2023-04-04 18:49 UTC (permalink / raw)
To: Mauro Carvalho Chehab; +Cc: igt-dev
[-- Attachment #1: Type: text/plain, Size: 5158 bytes --]
== Series Details ==
Series: scripts/test_list.py: make filtering logic more generic
URL : https://patchwork.freedesktop.org/series/116098/
State : failure
== Summary ==
CI Bug Log - changes from CI_DRM_12965 -> IGTPW_8753
====================================================
Summary
-------
**FAILURE**
Serious unknown changes coming with IGTPW_8753 absolutely need to be
verified manually.
If you think the reported changes have nothing to do with the changes
introduced in IGTPW_8753, please notify your bug team to allow them
to document this new failure mode, which will reduce false positives in CI.
External URL: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8753/index.html
Participating hosts (38 -> 36)
------------------------------
Missing (2): fi-kbl-soraka fi-snb-2520m
Possible new issues
-------------------
Here are the unknown changes that may have been introduced in IGTPW_8753:
### IGT changes ###
#### Possible regressions ####
* igt@i915_module_load@reload:
- bat-jsl-3: [PASS][1] -> [INCOMPLETE][2]
[1]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12965/bat-jsl-3/igt@i915_module_load@reload.html
[2]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8753/bat-jsl-3/igt@i915_module_load@reload.html
Known issues
------------
Here are the changes found in IGTPW_8753 that come from known issues:
### IGT changes ###
#### Issues hit ####
* igt@i915_selftest@live@slpc:
- bat-rpls-2: NOTRUN -> [DMESG-FAIL][3] ([i915#6997] / [i915#7913])
[3]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8753/bat-rpls-2/igt@i915_selftest@live@slpc.html
* igt@kms_chamelium_hpd@common-hpd-after-suspend:
- bat-rpls-2: NOTRUN -> [SKIP][4] ([i915#7828])
[4]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8753/bat-rpls-2/igt@kms_chamelium_hpd@common-hpd-after-suspend.html
- bat-dg2-11: NOTRUN -> [SKIP][5] ([i915#7828])
[5]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8753/bat-dg2-11/igt@kms_chamelium_hpd@common-hpd-after-suspend.html
* igt@kms_pipe_crc_basic@nonblocking-crc-frame-sequence:
- bat-dg2-11: NOTRUN -> [SKIP][6] ([i915#5354])
[6]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8753/bat-dg2-11/igt@kms_pipe_crc_basic@nonblocking-crc-frame-sequence.html
* igt@kms_pipe_crc_basic@suspend-read-crc:
- bat-rpls-2: NOTRUN -> [SKIP][7] ([i915#1845])
[7]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8753/bat-rpls-2/igt@kms_pipe_crc_basic@suspend-read-crc.html
#### Possible fixes ####
* igt@i915_selftest@live@gt_heartbeat:
- fi-apl-guc: [DMESG-FAIL][8] ([i915#5334]) -> [PASS][9]
[8]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12965/fi-apl-guc/igt@i915_selftest@live@gt_heartbeat.html
[9]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8753/fi-apl-guc/igt@i915_selftest@live@gt_heartbeat.html
* igt@i915_selftest@live@gt_lrc:
- bat-dg2-11: [INCOMPLETE][10] ([i915#7609] / [i915#7913]) -> [PASS][11]
[10]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12965/bat-dg2-11/igt@i915_selftest@live@gt_lrc.html
[11]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8753/bat-dg2-11/igt@i915_selftest@live@gt_lrc.html
* igt@i915_selftest@live@requests:
- bat-rpls-2: [ABORT][12] ([i915#4983] / [i915#7913]) -> [PASS][13]
[12]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12965/bat-rpls-2/igt@i915_selftest@live@requests.html
[13]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8753/bat-rpls-2/igt@i915_selftest@live@requests.html
#### Warnings ####
* igt@i915_selftest@live@requests:
- bat-rpls-1: [ABORT][14] ([i915#4983] / [i915#7911]) -> [ABORT][15] ([i915#7911] / [i915#7982])
[14]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12965/bat-rpls-1/igt@i915_selftest@live@requests.html
[15]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8753/bat-rpls-1/igt@i915_selftest@live@requests.html
[i915#1845]: https://gitlab.freedesktop.org/drm/intel/issues/1845
[i915#4983]: https://gitlab.freedesktop.org/drm/intel/issues/4983
[i915#5334]: https://gitlab.freedesktop.org/drm/intel/issues/5334
[i915#5354]: https://gitlab.freedesktop.org/drm/intel/issues/5354
[i915#6997]: https://gitlab.freedesktop.org/drm/intel/issues/6997
[i915#7609]: https://gitlab.freedesktop.org/drm/intel/issues/7609
[i915#7828]: https://gitlab.freedesktop.org/drm/intel/issues/7828
[i915#7911]: https://gitlab.freedesktop.org/drm/intel/issues/7911
[i915#7913]: https://gitlab.freedesktop.org/drm/intel/issues/7913
[i915#7982]: https://gitlab.freedesktop.org/drm/intel/issues/7982
Build changes
-------------
* CI: CI-20190529 -> None
* IGT: IGT_7236 -> IGTPW_8753
CI-20190529: 20190529
CI_DRM_12965: 53e37ca502cfe620396e498fae8430c293fb2c83 @ git://anongit.freedesktop.org/gfx-ci/linux
IGTPW_8753: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8753/index.html
IGT_7236: bac5a4cc31b3212a205219a6cbc45a173d30d04b @ https://gitlab.freedesktop.org/drm/igt-gpu-tools.git
== Logs ==
For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8753/index.html
[-- Attachment #2: Type: text/html, Size: 6328 bytes --]
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2023-04-04 18:49 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2023-04-04 17:17 [igt-dev] [PATCH i-g-t] scripts/test_list.py: make filtering logic more generic Mauro Carvalho Chehab
2023-04-04 18:49 ` [igt-dev] ✗ Fi.CI.BAT: failure for " Patchwork
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox