[PATCH v2 08/28] unittests: test_private: modify it to use CTokenizer directly

Mauro Carvalho Chehab
2026-03-12T14:54:58Z

Change the logic to use the tokenizer directly. This allows
adding more unit tests to check the validty of the tokenizer
itself.

Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
Message-ID: <2672257233ff73a9464c09b50924be51e25d4f59.1773074166.git.mchehab+huawei@kernel.org>
---
.../{test_private.py => test_tokenizer.py} | 76 +++++++++++++------
1 file changed, 52 insertions(+), 24 deletions(-)
rename tools/unittests/{test_private.py => test_tokenizer.py} (85%)

diff --git a/tools/unittests/test_private.py b/tools/unittests/test_tokenizer.py
similarity index 85%
rename from tools/unittests/test_private.py
rename to tools/unittests/test_tokenizer.py
index eae245ae8a12..da0f2c4c9e21 100755
--- a/tools/unittests/test_private.py
+++ b/tools/unittests/test_tokenizer.py
@@ -15,20 +15,44 @@ from unittest.mock import MagicMock
SRC_DIR = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, os.path.join(SRC_DIR, "../lib/python"))

-from kdoc.kdoc_parser import trim_private_members
+from kdoc.kdoc_re import CTokenizer
from unittest_helper import run_unittest

+
+
#
# List of tests.
#
# The code will dynamically generate one test for each key on this dictionary.
#

+def make_private_test(name, data):
+ """
+ Create a test named ``name`` using parameters given by ``data`` dict.
+ """
+
+ def test(self):
+ """In-lined lambda-like function to run the test"""
+ tokens = CTokenizer(data["source"])
+ result = str(tokens)
+
+ #
+ # Avoid whitespace false positives
+ #
+ result = re.sub(r"\s++", " ", result).strip()
+ expected = re.sub(r"\s++", " ", data["trimmed"]).strip()
+
+ msg = f"failed when parsing this source:\n{data['source']}"
+ self.assertEqual(result, expected, msg=msg)
+
+ return test
+
#: Tests to check if CTokenizer is handling properly public/private comments.
TESTS_PRIVATE = {
#
# Simplest case: no private. Ensure that trimming won't affect struct
#
+ "__run__": make_private_test,
"no private": {
"source": """
struct foo {
@@ -288,41 +312,45 @@ TESTS_PRIVATE = {
},
}

+#: Dict containing all test groups fror CTokenizer
+TESTS = {
+ "TestPublicPrivate": TESTS_PRIVATE,
+}

-class TestPublicPrivate(unittest.TestCase):
- """
- Main test class. Populated dynamically at runtime.
- """
+def setUp(self):
+ self.maxDiff = None

- def setUp(self):
- self.maxDiff = None
+def build_test_class(group_name, table):
+ """
+ Dynamically creates a class instance using type() as a generator
+ for a new class derivated from unittest.TestCase.

- def add_test(cls, name, source, trimmed):
- """
- Dynamically add a test to the class
- """
- def test(cls):
- result = trim_private_members(source)
+ We're opting to do it inside a function to avoid the risk of
+ changing the globals() dictionary.
+ """

- result = re.sub(r"\s++", " ", result).strip()
- expected = re.sub(r"\s++", " ", trimmed).strip()
+ class_dict = {
+ "setUp": setUp
+ }

- msg = f"failed when parsing this source:\n" + source
+ run = table["__run__"]

- cls.assertEqual(result, expected, msg=msg)
+ for test_name, data in table.items():
+ if test_name == "__run__":
+ continue

- test.__name__ = f'test {name}'
+ class_dict[f"test_{test_name}"] = run(test_name, data)

- setattr(TestPublicPrivate, test.__name__, test)
+ cls = type(group_name, (unittest.TestCase,), class_dict)

+ return cls.__name__, cls

#
-# Populate TestPublicPrivate class
+# Create classes and add them to the global dictionary
#
-test_class = TestPublicPrivate()
-for name, test in TESTS_PRIVATE.items():
- test_class.add_test(name, test["source"], test["trimmed"])
-
+for group, table in TESTS.items():
+ t = build_test_class(group, table)
+ globals()[t[0]] = t[1]

#
# main
--
2.52.0