[PATCH v2 09/28] unittests: test_tokenizer: check if the tokenizer works

Mauro Carvalho Chehab
2026-03-12T14:54:58Z

Add extra tests to check if the tokenizer is working properly.

Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
---
tools/lib/python/kdoc/c_lex.py | 4 +-
tools/unittests/test_tokenizer.py | 109 +++++++++++++++++++++++++++++-
2 files changed, 108 insertions(+), 5 deletions(-)

diff --git a/tools/lib/python/kdoc/c_lex.py b/tools/lib/python/kdoc/c_lex.py
index a104c29b63fb..38f70e836eb8 100644
--- a/tools/lib/python/kdoc/c_lex.py
+++ b/tools/lib/python/kdoc/c_lex.py
@@ -58,8 +58,8 @@ class CToken():

return CToken.MISMATCH

- def __init__(self, kind, value, pos,
- brace_level, paren_level, bracket_level):
+ def __init__(self, kind, value=None, pos=0,
+ brace_level=0, paren_level=0, bracket_level=0):
self.kind = kind
self.value = value
self.pos = pos
diff --git a/tools/unittests/test_tokenizer.py b/tools/unittests/test_tokenizer.py
index da0f2c4c9e21..efb1d1687811 100755
--- a/tools/unittests/test_tokenizer.py
+++ b/tools/unittests/test_tokenizer.py
@@ -15,16 +15,118 @@ from unittest.mock import MagicMock
SRC_DIR = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, os.path.join(SRC_DIR, "../lib/python"))

-from kdoc.kdoc_re import CTokenizer
+from kdoc.c_lex import CToken, CTokenizer
from unittest_helper import run_unittest

-
-
#
# List of tests.
#
# The code will dynamically generate one test for each key on this dictionary.
#
+def tokens_to_list(tokens):
+ tuples = []
+
+ for tok in tokens:
+ if tok.kind == CToken.SPACE:
+ continue
+
+ tuples += [(tok.kind, tok.value,
+ tok.brace_level, tok.paren_level, tok.bracket_level)]
+
+ return tuples
+
+
+def make_tokenizer_test(name, data):
+ """
+ Create a test named ``name`` using parameters given by ``data`` dict.
+ """
+
+ def test(self):
+ """In-lined lambda-like function to run the test"""
+
+ #
+ # Check if exceptions are properly handled
+ #
+ if "raises" in data:
+ with self.assertRaises(data["raises"]):
+ CTokenizer(data["source"])
+ return
+
+ #
+ # Check if tokenizer is producing expected results
+ #
+ tokens = CTokenizer(data["source"]).tokens
+
+ result = tokens_to_list(tokens)
+ expected = tokens_to_list(data["expected"])
+
+ self.assertEqual(result, expected, msg=f"{name}")
+
+ return test
+
+#: Tokenizer tests.
+TESTS_TOKENIZER = {
+ "__run__": make_tokenizer_test,
+
+ "basic_tokens": {
+ "source": """
+ int a; // comment
+ float b = 1.23;
+ """,
+ "expected": [
+ CToken(CToken.NAME, "int"),
+ CToken(CToken.NAME, "a"),
+ CToken(CToken.PUNC, ";"),
+ CToken(CToken.COMMENT, "// comment"),
+ CToken(CToken.NAME, "float"),
+ CToken(CToken.NAME, "b"),
+ CToken(CToken.OP, "="),
+ CToken(CToken.NUMBER, "1.23"),
+ CToken(CToken.PUNC, ";"),
+ ],
+ },
+
+ "depth_counters": {
+ "source": """
+ struct X {
+ int arr[10];
+ func(a[0], (b + c));
+ }
+ """,
+ "expected": [
+ CToken(CToken.STRUCT, "struct"),
+ CToken(CToken.NAME, "X"),
+ CToken(CToken.BEGIN, "{", brace_level=1),
+
+ CToken(CToken.NAME, "int", brace_level=1),
+ CToken(CToken.NAME, "arr", brace_level=1),
+ CToken(CToken.BEGIN, "[", brace_level=1, bracket_level=1),
+ CToken(CToken.NUMBER, "10", brace_level=1, bracket_level=1),
+ CToken(CToken.END, "]", brace_level=1),
+ CToken(CToken.PUNC, ";", brace_level=1),
+ CToken(CToken.NAME, "func", brace_level=1),
+ CToken(CToken.BEGIN, "(", brace_level=1, paren_level=1),
+ CToken(CToken.NAME, "a", brace_level=1, paren_level=1),
+ CToken(CToken.BEGIN, "[", brace_level=1, paren_level=1, bracket_level=1),
+ CToken(CToken.NUMBER, "0", brace_level=1, paren_level=1, bracket_level=1),
+ CToken(CToken.END, "]", brace_level=1, paren_level=1),
+ CToken(CToken.PUNC, ",", brace_level=1, paren_level=1),
+ CToken(CToken.BEGIN, "(", brace_level=1, paren_level=2),
+ CToken(CToken.NAME, "b", brace_level=1, paren_level=2),
+ CToken(CToken.OP, "+", brace_level=1, paren_level=2),
+ CToken(CToken.NAME, "c", brace_level=1, paren_level=2),
+ CToken(CToken.END, ")", brace_level=1, paren_level=1),
+ CToken(CToken.END, ")", brace_level=1),
+ CToken(CToken.PUNC, ";", brace_level=1),
+ CToken(CToken.END, "}"),
+ ],
+ },
+
+ "mismatch_error": {
+ "source": "int a$ = 5;", # $ is illegal
+ "raises": RuntimeError,
+ },
+}

def make_private_test(name, data):
"""
@@ -315,6 +417,7 @@ TESTS_PRIVATE = {
#: Dict containing all test groups fror CTokenizer
TESTS = {
"TestPublicPrivate": TESTS_PRIVATE,
+ "TestTokenizer": TESTS_TOKENIZER,
}

def setUp(self):
--
2.52.0