Re: [LTP] [PATCH] doc: add tests catalog page

[Andrea Cervesato via ltp <ltp@lists.linux.it>]

I also have a question: what test configuration keys should be seen in 
the documentation tests list?
I.e. needs_forks, needs_kconfigs, etc.

Andrea

On 2/3/25 13:14, Andrea Cervesato wrote:
> Hi Cyril,
>
> On 2/3/25 13:07, Cyril Hrubis wrote:
>> Hi!
>>> Add a new section in the LTP documentation website, where we list all
>>> tests which are available in LTP and supporting new API.
>>>
>>> Signed-off-by: Andrea Cervesato <andrea.cervesato@suse.com>
>>> ---
>>> This patch-set is meant to introduce a new page in the LTP
>>> documentation, showing tests which are currently available with their
>>> description and information.
>>> ---
>>>   doc/.gitignore             |   1 +
>>>   doc/conf.py                | 101 
>>> +++++++++++++++++++++++++++++++++++++++++++--
>>>   doc/index.rst              |   4 ++
>>>   doc/users/test_catalog.rst |   7 ++++
>>>   4 files changed, 109 insertions(+), 4 deletions(-)
>>>
>>> diff --git a/doc/.gitignore b/doc/.gitignore
>>> index 
>>> 173179852070f25acb6a729975df9d52d171b422..2b05a1ec368573778cfe7ee6a1cb5d6c5ecb0b5a 
>>> 100644
>>> --- a/doc/.gitignore
>>> +++ b/doc/.gitignore
>>> @@ -1,4 +1,5 @@
>>>   html/
>>>   build/
>>>   _static/syscalls.rst
>>> +_static/tests.rst
>>>   syscalls.tbl
>>> diff --git a/doc/conf.py b/doc/conf.py
>>> index 
>>> c6a84ea5810424ce6e1c21d81946c1819f10a3cc..2ecfeb80ece1e14f94b757f26fa5e08fb79f1c69 
>>> 100644
>>> --- a/doc/conf.py
>>> +++ b/doc/conf.py
>>> @@ -5,6 +5,7 @@
>>>     import os
>>>   import re
>>> +import json
>>>   import socket
>>>   import urllib.request
>>>   import sphinx
>>> @@ -17,6 +18,7 @@ copyright = '2024, Linux Test Project'
>>>   author = 'Linux Test Project'
>>>   release = '1.0'
>>>   ltp_repo = 'https://github.com/linux-test-project/ltp'
>>> +ltp_repo_base_url = f"{ltp_repo}/tree/master"
>>>     # -- General configuration 
>>> ---------------------------------------------------
>>>   # 
>>> https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
>>> @@ -25,7 +27,7 @@ extensions = [
>>>       'linuxdoc.rstKernelDoc',
>>>       'sphinxcontrib.spelling',
>>>       'sphinx.ext.autosectionlabel',
>>> -    'sphinx.ext.extlinks'
>>> +    'sphinx.ext.extlinks',
>>>   ]
>>>     exclude_patterns = ["html*", '_static*']
>>> @@ -138,7 +140,6 @@ def generate_syscalls_stats(_):
>>>       if error:
>>>           return
>>>   -    syscalls_base_url = f"{ltp_repo}/tree/master"
>>>       text = [
>>>           'Syscalls\n',
>>>           '--------\n\n',
>>> @@ -176,7 +177,7 @@ def generate_syscalls_stats(_):
>>>               path = dirpath.replace('../', '')
>>>               name = match.group('name')
>>>   -            ltp_syscalls[name] = f'{syscalls_base_url}/{path}'
>>> +            ltp_syscalls[name] = f'{ltp_repo_base_url}/{path}'
>>>         # compare kernel syscalls with LTP tested syscalls
>>>       syscalls = {}
>>> @@ -186,7 +187,7 @@ def generate_syscalls_stats(_):
>>>             if kersc not in syscalls:
>>>               if kersc in white_list:
>>> -                syscalls[kersc] = 
>>> f'{syscalls_base_url}/{white_list[kersc]}'
>>> +                syscalls[kersc] = 
>>> f'{ltp_repo_base_url}/{white_list[kersc]}'
>>>                   continue
>>>                 syscalls[kersc] = None
>>> @@ -256,6 +257,97 @@ def generate_syscalls_stats(_):
>>>           stats.writelines(text)
>>>     +def generate_test_catalog(_):
>>> +    """
>>> +    Generate the test catalog from ltp.json metadata file.
>>> +    """
>>> +    output = '_static/tests.rst'
>>> +    metadata_file = '../metadata/ltp.json'
>>> +    cve_url = "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-"
>>> +    commit_url = 
>>> "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id="
>>> +    timeout_def = 0
>>> +    text = []
>>> +
>>> +    metadata = None
>>> +    with open(metadata_file, 'r', encoding='utf-8') as data:
>>> +        metadata = json.load(data)
>>> +
>>> +    timeout_def = metadata['defaults']['timeout']
>>> +
>>> +    for test_name, conf in metadata['tests'].items():
>>> +        text.extend([
>>> +            f'{test_name}\n',
>>> +            len(test_name) * '-' + '\n'
>>> +        ])
>>> +
>>> +        # source url location
>>> +        test_fname = conf.get('fname', None)
>>> +        if test_fname:
>>> +            text.append(f"\n`source 
>>> <{ltp_repo_base_url}/{test_fname}>`_\n\n")
>>> +
>>> +        # test description
>>> +        desc = conf.get('doc', None)
>>> +        if desc:
>>> +            desc_text = []
>>> +            for line in desc:
>>> +                if line.startswith("[Descr"):
>>> +                    desc_text.append("**Description**")
>>> +                elif line.startswith("[Algo"):
>> Huh, why not the whole [Description] and [Algorithm]?
> Typos in the code :-) I just overlay it fixing the result. But yes, in 
> the final version we need to use the full name. Maybe a comment there 
> would have helped.
>>
>>> + desc_text.append("**Algorithm**")
>>> +                else:
>>> +                    desc_text.append(line)
>>> +
>>> +            text.extend([
>>> +                '\n'.join(desc_text),
>>> +                '\n'
>>> +            ])
>>> +
>>> +        timeout = conf.get('timeout', None)
>>> +        if timeout:
>>> +            text.append(f'\nTest timeout to {timeout} seconds.')
>>> +        else:
>>> +            text.append(f'\nTest timeout defaults to {timeout_def} 
>>> seconds.')
>>> +
>>> +        text.append('\n\n')
>>> +
>>> +        # tags information
>>> +        tags = conf.get('tags', None)
>>> +        if tags:
>>> +            text.extend([
>>> +                '\n.. list-table::\n',
>>> +                '   :widths: 50 50\n'
>>> +                '   :header-rows: 1\n\n',
>>> +                '   * - Tag\n',
>>> +                '     - Info\n',
>>> +            ])
>>> +
>>> +            for tag in tags:
>>> +                tag_key = tag[0]
>>> +                tag_val = tag[1]
>>> +
>>> +                if tag_key == 'CVE':
>>> +                    tag_val = f'`{tag_val} <{cve_url}{tag_val}>`_'
>>> +                elif tag_key == 'linux-git':
>>> +                    tag_val = f'`{tag_val} <{commit_url}{tag_val}>`_'
>> We also have glibc-git and musl-git, see print_test_tags() in
>> lib/tst_test.c
> Thanks for pointing it out
>>> +                text.extend([
>>> +                    f'   * - {tag_key}\n',
>>> +                    f'     - {tag_val}\n',
>>> +                ])
>>> +
>>> +            text.append('\n')
>>> +
>>> +        # small separator between tests
>>> +        text.extend([
>>> +            '\n',
>>> +            '.. raw:: html\n\n',
>>> +            '    <hr>\n\n',
>>> +        ])
>>> +
>>> +    with open(output, 'w+', encoding='utf-8') as new_tests:
>>> +        new_tests.writelines(text)
>>> +
>>> +
>>>   def setup(app):
>>>       """
>>>       Setup the current documentation, using self generated data and 
>>> graphics
>>> @@ -263,3 +355,4 @@ def setup(app):
>>>       """
>>>       app.add_css_file('custom.css')
>>>       app.connect('builder-inited', generate_syscalls_stats)
>>> +    app.connect('builder-inited', generate_test_catalog)
>>> diff --git a/doc/index.rst b/doc/index.rst
>>> index 
>>> b907ac36f0c9328c576d25dee5777d808c2e5119..c00a59d31345142e78deb74eacc9da2941291d76 
>>> 100644
>>> --- a/doc/index.rst
>>> +++ b/doc/index.rst
>>> @@ -11,6 +11,7 @@
>>>      users/setup_tests
>>>      users/supported_systems
>>>      users/stats
>>> +   users/test_catalog
>>>     .. toctree::
>>>      :maxdepth: 3
>>> @@ -54,6 +55,9 @@ For users
>>>   :doc:`users/stats`
>>>      Some LTP statistics
>>>   +:doc:`users/test_catalog`
>>> +   The LTP test catalog
>>> +
>>>   For developers
>>>   --------------
>>>   diff --git a/doc/users/test_catalog.rst b/doc/users/test_catalog.rst
>>> new file mode 100644
>>> index 
>>> 0000000000000000000000000000000000000000..b1674f9dc614ea04a89cf084e92b72c6862a5f48
>>> --- /dev/null
>>> +++ b/doc/users/test_catalog.rst
>>> @@ -0,0 +1,7 @@
>>> +.. SPDX-License-Identifier: GPL-2.0-or-later
>>> +
>>> +Test catalog
>>> +============
>>> +
>>> +.. include:: ../_static/tests.rst
>>> +
>>>
>>> ---
>>> base-commit: 728759506cbe08612183275b3543007d1c47f7f4
>>> change-id: 20250131-doc_tests_list-1b82f51e43fd
>>>
>>> Best regards,
>>> -- 
>>> Andrea Cervesato <andrea.cervesato@suse.com>
>>>
>>>
>>> -- 
>>> Mailing list info: https://lists.linux.it/listinfo/ltp
> Andrea

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp