Re: [igt-dev] [PATCH i-g-t 3/3] runner: Add --dmesg-warn-level switch

Petri Latvala
2019-04-01T11:10:56Z

On Mon, Apr 01, 2019 at 09:46:57AM +0300, Arkadiusz Hiler wrote:
> This switch allows users to select which dmesg log level is treated as
> warning resulting in overriding the test results to
> dmesg-fail/dmesg-warn.
>
> Signed-off-by: Arkadiusz Hiler <arkadiusz.hiler@intel.com>
> ---
> .../0/dmesg.txt | 7 ++
> .../0/err.txt | 3 +
> .../0/journal.txt | 2 +
> .../0/out.txt | 3 +
> .../README.txt | 8 ++
> .../endtime.txt | 1 +
> .../joblist.txt | 1 +
> .../metadata.txt | 13 +++
> .../reference.json | 73 +++++++++++++++
> .../starttime.txt | 1 +
> .../uname.txt | 1 +
> .../dmesg-warn-level-piglit-style/0/dmesg.txt | 6 ++
> .../dmesg-warn-level-piglit-style/0/err.txt | 3 +
> .../0/journal.txt | 2 +
> .../dmesg-warn-level-piglit-style/0/out.txt | 3 +
> .../dmesg-warn-level-piglit-style/README.txt | 7 ++
> .../dmesg-warn-level-piglit-style/endtime.txt | 1 +
> .../dmesg-warn-level-piglit-style/joblist.txt | 1 +
> .../metadata.txt | 13 +++
> .../reference.json | 73 +++++++++++++++
> .../starttime.txt | 1 +
> .../dmesg-warn-level-piglit-style/uname.txt | 1 +
> .../dmesg-warn-level/0/dmesg.txt | 6 ++
> .../dmesg-warn-level/0/err.txt | 3 +
> .../dmesg-warn-level/0/journal.txt | 2 +
> .../dmesg-warn-level/0/out.txt | 3 +
> .../dmesg-warn-level/README.txt | 4 +
> .../dmesg-warn-level/endtime.txt | 1 +
> .../dmesg-warn-level/joblist.txt | 1 +
> .../dmesg-warn-level/metadata.txt | 13 +++
> .../dmesg-warn-level/reference.json | 73 +++++++++++++++
> .../dmesg-warn-level/starttime.txt | 1 +
> .../dmesg-warn-level/uname.txt | 1 +
> runner/resultgen.c | 4 +-
> runner/runner_json_tests.c | 3 +
> runner/runner_tests.c | 90 +++++++++++++++++++
> runner/settings.c | 30 ++++++-
> runner/settings.h | 1 +
> 38 files changed, 457 insertions(+), 3 deletions(-)
> create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/dmesg.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/err.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/journal.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/out.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/README.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/endtime.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/joblist.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/metadata.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json
> create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/starttime.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/uname.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/0/dmesg.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/0/err.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/0/journal.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/0/out.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/README.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/endtime.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/joblist.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/metadata.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json
> create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/starttime.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/uname.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level/0/dmesg.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level/0/err.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level/0/journal.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level/0/out.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level/README.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level/endtime.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level/joblist.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level/metadata.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level/reference.json
> create mode 100644 runner/json_tests_data/dmesg-warn-level/starttime.txt
> create mode 100644 runner/json_tests_data/dmesg-warn-level/uname.txt
>
> diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/dmesg.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/dmesg.txt
> new file mode 100644
> index 00000000..e44a24fc
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/dmesg.txt
> @@ -0,0 +1,7 @@
> +15,952,3216186095097,-;[IGT] successtest: executing
> +15,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
> +6,951,3216186095083,-;drm_ KERN_INFO, disregerd

ermagerd disregerd

> +7,952,3216186095097,-;drm_ KERN_DEBUG disregard
> +4,953,3216186101159,-;drm_ KERN_WARNING disregard
> +1,953,3216186101159,-;drm_ KERN_ALERT should trip a warning
> +15,955,3216186101160,-;[IGT] successtest: exiting, ret=0
> diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/err.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/err.txt
> new file mode 100644
> index 00000000..e18c00e9
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/err.txt
> @@ -0,0 +1,3 @@
> +Starting subtest: first-subtest
> +This is a warning line
> +Subtest first-subtest: SUCCESS (0.000s)

Why mix in stderr handling with these?

> diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/journal.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/journal.txt
> new file mode 100644
> index 00000000..86a30e07
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/journal.txt
> @@ -0,0 +1,2 @@
> +first-subtest
> +exit:0 (0.014s)
> diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/out.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/out.txt
> new file mode 100644
> index 00000000..5946bf31
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/out.txt
> @@ -0,0 +1,3 @@
> +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
> +Starting subtest: first-subtest
> +Subtest first-subtest: SUCCESS (0.000s)
> diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/README.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/README.txt
> new file mode 100644
> index 00000000..d3b29f3e
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/README.txt
> @@ -0,0 +1,8 @@
> +When --demsg-warn-level is set to 6 also KERN_INFO level messages should be

s/demsg/dmesg/

Same error is copied to all three new tests.

> +treated as warnings triggering a result change to dmesg-warn/dmesg-fail.
> +
> +This makes sure that the piglit-style-dmesg does not clash with
> +dmesg-warn-level. All the messages are prefixed with drm_ to fit the filters.
> +
> +We set the --dmesg-warn-level to 1 to make sure that the setting works in this
> +direction too.

So do we set it to 6 or 1 in this test?

> diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/endtime.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/endtime.txt
> new file mode 100644
> index 00000000..635f6ae9
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/endtime.txt
> @@ -0,0 +1 @@
> +1539953735.172373
> diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/joblist.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/joblist.txt
> new file mode 100644
> index 00000000..81f914a7
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/joblist.txt
> @@ -0,0 +1 @@
> +successtest first-subtest
> diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/metadata.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/metadata.txt
> new file mode 100644
> index 00000000..172c0708
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/metadata.txt
> @@ -0,0 +1,13 @@
> +abort_mask : 0
> +name : normal-run
> +dry_run : 0
> +sync : 0
> +log_level : 0
> +overwrite : 0
> +multiple_mode : 0
> +inactivity_timeout : 0
> +use_watchdog : 0
> +piglit_style_dmesg : 1
> +dmesg_warn_level : 1
> +test_root : /path/does/not/exist
> +results_path : /path/does/not/exist
> diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json
> new file mode 100644
> index 00000000..011024db
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json
> @@ -0,0 +1,73 @@
> +{
> + "__type__":"TestrunResult",
> + "results_version":10,
> + "name":"normal-run",
> + "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
> + "time_elapsed":{
> + "__type__":"TimeAttribute",
> + "start":1539953735.1110389,
> + "end":1539953735.1723731
> + },
> + "tests":{
> + "igt@successtest@first-subtest":{
> + "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
> + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
> + "result":"dmesg-warn",
> + "time":{
> + "__type__":"TimeAttribute",
> + "start":0.0,
> + "end":0.0
> + },
> + "err":"Starting subtest: first-subtest\nThis is a warning line\nSubtest first-subtest: SUCCESS (0.000s)\n",
> + "dmesg":"<7> [3216186.095097] [IGT] successtest: executing\n<7> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.095083] drm_ KERN_INFO, disregerd\n<7> [3216186.095097] drm_ KERN_DEBUG disregard\n<4> [3216186.101159] drm_ KERN_WARNING disregard\n<1> [3216186.101159] drm_ KERN_ALERT should trip a warning\n<7> [3216186.101160] [IGT] successtest: exiting, ret=0\n",
> + "dmesg-warnings":"<1> [3216186.101159] drm_ KERN_ALERT should trip a warning\n"
> + }
> + },
> + "totals":{
> + "":{
> + "crash":0,
> + "pass":0,
> + "dmesg-fail":0,
> + "dmesg-warn":1,
> + "skip":0,
> + "incomplete":0,
> + "timeout":0,
> + "notrun":0,
> + "fail":0,
> + "warn":0
> + },
> + "root":{
> + "crash":0,
> + "pass":0,
> + "dmesg-fail":0,
> + "dmesg-warn":1,
> + "skip":0,
> + "incomplete":0,
> + "timeout":0,
> + "notrun":0,
> + "fail":0,
> + "warn":0
> + },
> + "igt@successtest":{
> + "crash":0,
> + "pass":0,
> + "dmesg-fail":0,
> + "dmesg-warn":1,
> + "skip":0,
> + "incomplete":0,
> + "timeout":0,
> + "notrun":0,
> + "fail":0,
> + "warn":0
> + }
> + },
> + "runtimes":{
> + "igt@successtest":{
> + "time":{
> + "__type__":"TimeAttribute",
> + "start":0.0,
> + "end":0.014
> + }
> + }
> + }
> +}
> \ No newline at end of file
> diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/starttime.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/starttime.txt
> new file mode 100644
> index 00000000..ae038f18
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/starttime.txt
> @@ -0,0 +1 @@
> +1539953735.111039
> diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/uname.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/uname.txt
> new file mode 100644
> index 00000000..a7aef6f7
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/uname.txt
> @@ -0,0 +1 @@
> +Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
> diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/0/dmesg.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/dmesg.txt
> new file mode 100644
> index 00000000..546a29ff
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/dmesg.txt
> @@ -0,0 +1,6 @@
> +15,952,3216186095097,-;[IGT] successtest: executing
> +15,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
> +6,951,3216186095083,-;drm_ KERN_INFO, should be a warning here too
> +7,952,3216186095097,-;drm_ KERN_DEBUG, should not be a warning
> +4,953,3216186101159,-;drm_ A proper KERN_WARNING
> +15,955,3216186101160,-;[IGT] successtest: exiting, ret=0
> diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/0/err.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/err.txt
> new file mode 100644
> index 00000000..e18c00e9
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/err.txt
> @@ -0,0 +1,3 @@
> +Starting subtest: first-subtest
> +This is a warning line
> +Subtest first-subtest: SUCCESS (0.000s)
> diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/0/journal.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/journal.txt
> new file mode 100644
> index 00000000..86a30e07
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/journal.txt
> @@ -0,0 +1,2 @@
> +first-subtest
> +exit:0 (0.014s)
> diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/0/out.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/out.txt
> new file mode 100644
> index 00000000..5946bf31
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/out.txt
> @@ -0,0 +1,3 @@
> +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
> +Starting subtest: first-subtest
> +Subtest first-subtest: SUCCESS (0.000s)
> diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/README.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/README.txt
> new file mode 100644
> index 00000000..f8420d76
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/README.txt
> @@ -0,0 +1,7 @@
> +When --demsg-warn-level is set to 6 also KERN_INFO level messages should be
> +treated as warnings triggering a result change to dmesg-warn/dmesg-fail.
> +
> +This makes sure that the piglit-style-dmesg does not clash with
> +dmesg-warn-level. All the messages are prefixed with drm_ to fit the filters.
> +
> +IGT messages were artifically bumped to KERN_DEBUG to not pollute the warnings.
> diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/endtime.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/endtime.txt
> new file mode 100644
> index 00000000..635f6ae9
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/endtime.txt
> @@ -0,0 +1 @@
> +1539953735.172373
> diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/joblist.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/joblist.txt
> new file mode 100644
> index 00000000..81f914a7
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/joblist.txt
> @@ -0,0 +1 @@
> +successtest first-subtest
> diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/metadata.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/metadata.txt
> new file mode 100644
> index 00000000..b05377a9
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/metadata.txt
> @@ -0,0 +1,13 @@
> +abort_mask : 0
> +name : normal-run
> +dry_run : 0
> +sync : 0
> +log_level : 0
> +overwrite : 0
> +multiple_mode : 0
> +inactivity_timeout : 0
> +use_watchdog : 0
> +piglit_style_dmesg : 1
> +dmesg_warn_level : 6
> +test_root : /path/does/not/exist
> +results_path : /path/does/not/exist
> diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json b/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json
> new file mode 100644
> index 00000000..5d1c00ea
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json
> @@ -0,0 +1,73 @@
> +{
> + "__type__":"TestrunResult",
> + "results_version":10,
> + "name":"normal-run",
> + "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
> + "time_elapsed":{
> + "__type__":"TimeAttribute",
> + "start":1539953735.1110389,
> + "end":1539953735.1723731
> + },
> + "tests":{
> + "igt@successtest@first-subtest":{
> + "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
> + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
> + "result":"dmesg-warn",
> + "time":{
> + "__type__":"TimeAttribute",
> + "start":0.0,
> + "end":0.0
> + },
> + "err":"Starting subtest: first-subtest\nThis is a warning line\nSubtest first-subtest: SUCCESS (0.000s)\n",
> + "dmesg":"<7> [3216186.095097] [IGT] successtest: executing\n<7> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.095083] drm_ KERN_INFO, should be a warning here too\n<7> [3216186.095097] drm_ KERN_DEBUG, should not be a warning\n<4> [3216186.101159] drm_ A proper KERN_WARNING\n<7> [3216186.101160] [IGT] successtest: exiting, ret=0\n",
> + "dmesg-warnings":"<6> [3216186.095083] drm_ KERN_INFO, should be a warning here too\n<4> [3216186.101159] drm_ A proper KERN_WARNING\n"
> + }
> + },
> + "totals":{
> + "":{
> + "crash":0,
> + "pass":0,
> + "dmesg-fail":0,
> + "dmesg-warn":1,
> + "skip":0,
> + "incomplete":0,
> + "timeout":0,
> + "notrun":0,
> + "fail":0,
> + "warn":0
> + },
> + "root":{
> + "crash":0,
> + "pass":0,
> + "dmesg-fail":0,
> + "dmesg-warn":1,
> + "skip":0,
> + "incomplete":0,
> + "timeout":0,
> + "notrun":0,
> + "fail":0,
> + "warn":0
> + },
> + "igt@successtest":{
> + "crash":0,
> + "pass":0,
> + "dmesg-fail":0,
> + "dmesg-warn":1,
> + "skip":0,
> + "incomplete":0,
> + "timeout":0,
> + "notrun":0,
> + "fail":0,
> + "warn":0
> + }
> + },
> + "runtimes":{
> + "igt@successtest":{
> + "time":{
> + "__type__":"TimeAttribute",
> + "start":0.0,
> + "end":0.014
> + }
> + }
> + }
> +}
> \ No newline at end of file
> diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/starttime.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/starttime.txt
> new file mode 100644
> index 00000000..ae038f18
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/starttime.txt
> @@ -0,0 +1 @@
> +1539953735.111039
> diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/uname.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/uname.txt
> new file mode 100644
> index 00000000..a7aef6f7
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/uname.txt
> @@ -0,0 +1 @@
> +Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
> diff --git a/runner/json_tests_data/dmesg-warn-level/0/dmesg.txt b/runner/json_tests_data/dmesg-warn-level/0/dmesg.txt
> new file mode 100644
> index 00000000..16ef1210
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level/0/dmesg.txt
> @@ -0,0 +1,6 @@
> +15,952,3216186095097,-;[IGT] successtest: executing
> +15,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
> +6,951,3216186095083,-;KERN_INFO, should be a warning here too
> +7,952,3216186095097,-;KERN_DEBUG, should not be a warning
> +4,953,3216186101159,-;A proper KERN_WARNING
> +15,955,3216186101160,-;[IGT] successtest: exiting, ret=0
> diff --git a/runner/json_tests_data/dmesg-warn-level/0/err.txt b/runner/json_tests_data/dmesg-warn-level/0/err.txt
> new file mode 100644
> index 00000000..e18c00e9
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level/0/err.txt
> @@ -0,0 +1,3 @@
> +Starting subtest: first-subtest
> +This is a warning line
> +Subtest first-subtest: SUCCESS (0.000s)
> diff --git a/runner/json_tests_data/dmesg-warn-level/0/journal.txt b/runner/json_tests_data/dmesg-warn-level/0/journal.txt
> new file mode 100644
> index 00000000..86a30e07
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level/0/journal.txt
> @@ -0,0 +1,2 @@
> +first-subtest
> +exit:0 (0.014s)
> diff --git a/runner/json_tests_data/dmesg-warn-level/0/out.txt b/runner/json_tests_data/dmesg-warn-level/0/out.txt
> new file mode 100644
> index 00000000..5946bf31
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level/0/out.txt
> @@ -0,0 +1,3 @@
> +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
> +Starting subtest: first-subtest
> +Subtest first-subtest: SUCCESS (0.000s)
> diff --git a/runner/json_tests_data/dmesg-warn-level/README.txt b/runner/json_tests_data/dmesg-warn-level/README.txt
> new file mode 100644
> index 00000000..f481130d
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level/README.txt
> @@ -0,0 +1,4 @@
> +When --demsg-warn-level is set to 6 also KERN_INFO level messages should be
> +treated as warnings triggering a result change to dmesg-warn/dmesg-fail.
> +
> +IGT messages were artifically bumped to KERN_DEBUG to not pollute the warnings.
> diff --git a/runner/json_tests_data/dmesg-warn-level/endtime.txt b/runner/json_tests_data/dmesg-warn-level/endtime.txt
> new file mode 100644
> index 00000000..635f6ae9
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level/endtime.txt
> @@ -0,0 +1 @@
> +1539953735.172373
> diff --git a/runner/json_tests_data/dmesg-warn-level/joblist.txt b/runner/json_tests_data/dmesg-warn-level/joblist.txt
> new file mode 100644
> index 00000000..81f914a7
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level/joblist.txt
> @@ -0,0 +1 @@
> +successtest first-subtest
> diff --git a/runner/json_tests_data/dmesg-warn-level/metadata.txt b/runner/json_tests_data/dmesg-warn-level/metadata.txt
> new file mode 100644
> index 00000000..7597a4eb
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level/metadata.txt
> @@ -0,0 +1,13 @@
> +abort_mask : 0
> +name : normal-run
> +dry_run : 0
> +sync : 0
> +log_level : 0
> +overwrite : 0
> +multiple_mode : 0
> +inactivity_timeout : 0
> +use_watchdog : 0
> +piglit_style_dmesg : 0
> +dmesg_warn_level : 6
> +test_root : /path/does/not/exist
> +results_path : /path/does/not/exist
> diff --git a/runner/json_tests_data/dmesg-warn-level/reference.json b/runner/json_tests_data/dmesg-warn-level/reference.json
> new file mode 100644
> index 00000000..32fb571b
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level/reference.json
> @@ -0,0 +1,73 @@
> +{
> + "__type__":"TestrunResult",
> + "results_version":10,
> + "name":"normal-run",
> + "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
> + "time_elapsed":{
> + "__type__":"TimeAttribute",
> + "start":1539953735.1110389,
> + "end":1539953735.1723731
> + },
> + "tests":{
> + "igt@successtest@first-subtest":{
> + "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
> + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
> + "result":"dmesg-warn",
> + "time":{
> + "__type__":"TimeAttribute",
> + "start":0.0,
> + "end":0.0
> + },
> + "err":"Starting subtest: first-subtest\nThis is a warning line\nSubtest first-subtest: SUCCESS (0.000s)\n",
> + "dmesg":"<7> [3216186.095097] [IGT] successtest: executing\n<7> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.095083] KERN_INFO, should be a warning here too\n<7> [3216186.095097] KERN_DEBUG, should not be a warning\n<4> [3216186.101159] A proper KERN_WARNING\n<7> [3216186.101160] [IGT] successtest: exiting, ret=0\n",
> + "dmesg-warnings":"<6> [3216186.095083] KERN_INFO, should be a warning here too\n<4> [3216186.101159] A proper KERN_WARNING\n"
> + }
> + },
> + "totals":{
> + "":{
> + "crash":0,
> + "pass":0,
> + "dmesg-fail":0,
> + "dmesg-warn":1,
> + "skip":0,
> + "incomplete":0,
> + "timeout":0,
> + "notrun":0,
> + "fail":0,
> + "warn":0
> + },
> + "root":{
> + "crash":0,
> + "pass":0,
> + "dmesg-fail":0,
> + "dmesg-warn":1,
> + "skip":0,
> + "incomplete":0,
> + "timeout":0,
> + "notrun":0,
> + "fail":0,
> + "warn":0
> + },
> + "igt@successtest":{
> + "crash":0,
> + "pass":0,
> + "dmesg-fail":0,
> + "dmesg-warn":1,
> + "skip":0,
> + "incomplete":0,
> + "timeout":0,
> + "notrun":0,
> + "fail":0,
> + "warn":0
> + }
> + },
> + "runtimes":{
> + "igt@successtest":{
> + "time":{
> + "__type__":"TimeAttribute",
> + "start":0.0,
> + "end":0.014
> + }
> + }
> + }
> +}
> \ No newline at end of file
> diff --git a/runner/json_tests_data/dmesg-warn-level/starttime.txt b/runner/json_tests_data/dmesg-warn-level/starttime.txt
> new file mode 100644
> index 00000000..ae038f18
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level/starttime.txt
> @@ -0,0 +1 @@
> +1539953735.111039
> diff --git a/runner/json_tests_data/dmesg-warn-level/uname.txt b/runner/json_tests_data/dmesg-warn-level/uname.txt
> new file mode 100644
> index 00000000..a7aef6f7
> --- /dev/null
> +++ b/runner/json_tests_data/dmesg-warn-level/uname.txt
> @@ -0,0 +1 @@
> +Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
> diff --git a/runner/resultgen.c b/runner/resultgen.c
> index 40d1cfcc..2d21cba7 100644
> --- a/runner/resultgen.c
> +++ b/runner/resultgen.c
> @@ -670,12 +670,12 @@ static bool fill_from_dmesg(int fd,
> }
>
> if (settings->piglit_style_dmesg) {
> - if ((flags & 0x07) <= 5 && continuation != 'c' &&
> + if ((flags & 0x07) <= settings->dmesg_warn_level && continuation != 'c' &&
> regexec(&re, message, (size_t)0, NULL, 0) != REG_NOMATCH) {
> append_line(&warnings, &warningslen, formatted);
> }
> } else {
> - if ((flags & 0x07) <= 4 && continuation != 'c' &&
> + if ((flags & 0x07) <= settings->dmesg_warn_level && continuation != 'c' &&
> regexec(&re, message, (size_t)0, NULL, 0) == REG_NOMATCH) {
> append_line(&warnings, &warningslen, formatted);
> }
> diff --git a/runner/runner_json_tests.c b/runner/runner_json_tests.c
> index 09a93189..7c9540d1 100644
> --- a/runner/runner_json_tests.c
> +++ b/runner/runner_json_tests.c
> @@ -159,6 +159,9 @@ static const char *dirnames[] = {
> "dmesg-escapes",
> "notrun-results",
> "notrun-results-multiple-mode",
> + "dmesg-warn-level",
> + "dmesg-warn-level-piglit-style",
> + "dmesg-warn-level-one-piglit-style"
> };
>
> igt_main
> diff --git a/runner/runner_tests.c b/runner/runner_tests.c
> index 06725536..f2b5ffa7 100644
> --- a/runner/runner_tests.c
> +++ b/runner/runner_tests.c
> @@ -155,6 +155,7 @@ static void assert_settings_equal(struct settings *one, struct settings *two)
> igt_assert_eqstr(one->test_root, two->test_root);
> igt_assert_eqstr(one->results_path, two->results_path);
> igt_assert_eq(one->piglit_style_dmesg, two->piglit_style_dmesg);
> + igt_assert_eq(one->dmesg_warn_level, two->dmesg_warn_level);
> }
>
> static void assert_job_list_equal(struct job_list *one, struct job_list *two)
> @@ -223,7 +224,9 @@ igt_main
> igt_assert(!settings.use_watchdog);
> igt_assert(strstr(settings.test_root, "test-root-dir") != NULL);
> igt_assert(strstr(settings.results_path, "path-to-results") != NULL);
> +
> igt_assert(!settings.piglit_style_dmesg);
> + igt_assert_eq(settings.dmesg_warn_level, 4);
> }
>
> igt_subtest_group {
> @@ -389,7 +392,36 @@ igt_main
> igt_assert(settings.use_watchdog);
> igt_assert(strstr(settings.test_root, "test-root-dir") != NULL);
> igt_assert(strstr(settings.results_path, "path-to-results") != NULL);
> +
> igt_assert(settings.piglit_style_dmesg);
> + igt_assert_eq(settings.dmesg_warn_level, 5);
> + }

This is the parse-all-settings subtest? Add --dmesg-warn-level=5 to the command line for it.

> +
> + igt_subtest("dmesg-warn-level-overwrittable-with-piglit-style") {

overrideable

> + const char *argv[] = { "runner",
> + "--piglit-style-dmesg",
> + "--dmesg-warn-level=3",
> + "test-root-dir",
> + "path-to-results",
> + };
> +
> + igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, &settings));
> +
> + igt_assert(settings.piglit_style_dmesg);
> + igt_assert_eq(settings.dmesg_warn_level, 3);
> + }
> +
> + igt_subtest("dmesg-warn-level-overwrittable") {

overrideable again

> + const char *argv[] = { "runner",
> + "--dmesg-warn-level=3",
> + "test-root-dir",
> + "path-to-results",
> + };
> +
> + igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, &settings));
> +
> + igt_assert(!settings.piglit_style_dmesg);
> + igt_assert_eq(settings.dmesg_warn_level, 3);
> }
>
> igt_subtest("invalid-option") {
> @@ -1092,6 +1124,64 @@ igt_main
> }
> }
>
> + igt_subtest_group {
> + igt_subtest("metadata-read-old-style-infer-dmesg-warn-piglit-style")
> + {

Starting brace goes on the end of the prev line.

> + char metadata[] = "piglit_style_dmesg : 1\n";
> + FILE *f = fmemopen(metadata, strlen(metadata), "r");
> + igt_assert(f);
> +
> + igt_assert(read_settings_from_file(&settings, f));
> +
> + igt_assert(settings.piglit_style_dmesg);
> + igt_assert_eq(settings.dmesg_warn_level, 5);
> +
> + fclose(f);
> + }
> +
> + igt_subtest("metadata-read-overrides-dmesg-warn-piglit-style")
> + {
> + char metadata[] = "piglit_style_dmesg : 1\ndmesg_warn_level : 3";
> + FILE *f = fmemopen(metadata, strlen(metadata), "r");
> + igt_assert(f);
> +
> + igt_assert(read_settings_from_file(&settings, f));
> +
> + igt_assert(settings.piglit_style_dmesg);
> + igt_assert_eq(settings.dmesg_warn_level, 3);
> +
> + fclose(f);
> + }
> +
> + igt_subtest("metadata-read-old-style-infer-dmesg-warn-default")
> + {
> + char metadata[] = "piglit_style_dmesg : 0\n";
> + FILE *f = fmemopen(metadata, strlen(metadata), "r");
> + igt_assert(f);
> +
> + igt_assert(read_settings_from_file(&settings, f));
> +
> + igt_assert(!settings.piglit_style_dmesg);
> + igt_assert_eq(settings.dmesg_warn_level, 4);
> +
> + fclose(f);
> + }
> +
> + igt_subtest("metadata-read-overrides-dmesg-warn-default")
> + {
> + char metadata[] = "piglit_style_dmesg : 0\ndmesg_warn_level : 3";
> + FILE *f = fmemopen(metadata, strlen(metadata), "r");
> + igt_assert(f);
> +
> + igt_assert(read_settings_from_file(&settings, f));
> +
> + igt_assert(!settings.piglit_style_dmesg);
> + igt_assert_eq(settings.dmesg_warn_level, 3);
> +
> + fclose(f);
> + }
> + }
> +
> igt_subtest_group {
> struct job_list list;
> int dirfd = -1, subdirfd = -1, fd = -1;
> diff --git a/runner/settings.c b/runner/settings.c
> index 20b21378..82e564a7 100644
> --- a/runner/settings.c
> +++ b/runner/settings.c
> @@ -17,6 +17,7 @@ enum {
> OPT_TEST_LIST,
> OPT_IGNORE_MISSING,
> OPT_PIGLIT_DMESG,
> + OPT_DMESG_WARN_LEVEL,
> OPT_OVERALL_TIMEOUT,
> OPT_HELP = 'h',
> OPT_NAME = 'n',
> @@ -159,12 +160,18 @@ static const char *usage_str =
> " --use-watchdog Use hardware watchdog for lethal enforcement of the\n"
> " above timeout. Killing the test process is still\n"
> " attempted at timeout trigger.\n"
> + " --demsg-warn-level <level>\n"

s/demsg/dmesg/

> + " Messages with log level equal or lower (more serious)\n"
> + " to the given one will override the test result to\n"
> + " dmesg-warn/dmesg-fail, assuming they go through filtering.\n"
> + " Defaults to 4 (KERN_WARNING).\n"
> " --piglit-style-dmesg Filter dmesg like piglit does. Piglit considers matches\n"
> " against a short filter list to mean the test result\n"
> " should be changed to dmesg-warn/dmesg-fail. Without\n"
> " this option everything except matches against a\n"
> " (longer) filter list means the test result should\n"
> - " change.\n"
> + " change. KERN_NOTICE dmesg level is treated as warn,\n"
> + " unless overridden with --dmesg-wan-level.\n"

dmesg-warn-level

With those fixed,
Reviewed-by: Petri Latvala <petri.latvala@intel.com>