[PATCH 04/11] vt: introduce gen_ucs_width.py to create ucs_width.c

Nicolas Pitre
2025-04-10T01:19:03Z

From: Nicolas Pitre <npitre@baylibre.com>

The table in the current ucs_width.c is terribly out of date and
incomplete. We also need a second table to store zero-width code points.
Properly maintaining those tables manually is impossible. So here's a
script to automatically generate them.

Signed-off-by: Nicolas Pitre <npitre@baylibre.com>
---
drivers/tty/vt/gen_ucs_width.py | 264 ++++++++++++++++++++++++++++++++
1 file changed, 264 insertions(+)
create mode 100755 drivers/tty/vt/gen_ucs_width.py

diff --git a/drivers/tty/vt/gen_ucs_width.py b/drivers/tty/vt/gen_ucs_width.py
new file mode 100755
index 0000000000..41997fe001
--- /dev/null
+++ b/drivers/tty/vt/gen_ucs_width.py
@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+#
+# This script uses Python's unicodedata module to generate ucs_width.c
+
+import unicodedata
+import sys
+
+def generate_ucs_width():
+ # Output file name
+ c_file = "ucs_width.c"
+
+ # Width data mapping
+ width_map = {} # Maps code points to width (0, 1, 2)
+
+ # Define emoji modifiers and components that should have zero width
+ emoji_zero_width = [
+ # Skin tone modifiers
+ (0x1F3FB, 0x1F3FF), # Emoji modifiers (skin tones)
+
+ # Variation selectors (note: VS16 is treated specially in vt.c)
+ (0xFE00, 0xFE0F), # Variation Selectors 1-16
+
+ # Gender and hair style modifiers
+ (0x2640, 0x2640), # Female sign
+ (0x2642, 0x2642), # Male sign
+ (0x26A7, 0x26A7), # Transgender symbol
+ (0x1F9B0, 0x1F9B3), # Hair components (red, curly, white, bald)
+
+ # Tag characters
+ (0xE0020, 0xE007E), # Tags
+ ]
+
+ # Mark these emoji modifiers as zero-width
+ for start, end in emoji_zero_width:
+ for cp in range(start, end + 1):
+ try:
+ width_map[cp] = 0
+ except (ValueError, OverflowError):
+ continue
+
+ # Mark all regional indicators as single-width as they are usually paired
+ # providing a combined with of 2.
+ regional_indicators = (0x1F1E6, 0x1F1FF) # Regional indicator symbols A-Z
+ start, end = regional_indicators
+ for cp in range(start, end + 1):
+ try:
+ width_map[cp] = 1
+ except (ValueError, OverflowError):
+ continue
+
+ # Process all assigned Unicode code points (Basic Multilingual Plane + Supplementary Planes)
+ # Range 0x0 to 0x10FFFF (the full Unicode range)
+ for block_start in range(0, 0x110000, 0x1000):
+ block_end = block_start + 0x1000
+ for cp in range(block_start, block_end):
+ try:
+ char = chr(cp)
+
+ # Skip if already processed
+ if cp in width_map:
+ continue
+
+ # Check if the character is a combining mark
+ category = unicodedata.category(char)
+
+ # Combining marks, format characters, zero-width characters
+ if (category.startswith('M') or # Mark (combining)
+ (category == 'Cf' and cp not in (0x061C, 0x06DD, 0x070F, 0x180E, 0x200F, 0x202E, 0x2066, 0x2067, 0x2068, 0x2069)) or
+ cp in (0x200B, 0x200C, 0x200D, 0x2060, 0xFEFF)): # Known zero-width characters
+ width_map[cp] = 0
+ continue
+
+ # Use East Asian Width property
+ eaw = unicodedata.east_asian_width(char)
+
+ if eaw in ('F', 'W'): # Fullwidth or Wide
+ width_map[cp] = 2
+ elif eaw in ('Na', 'H', 'N', 'A'): # Narrow, Halfwidth, Neutral, Ambiguous
+ width_map[cp] = 1
+ else:
+ # Default to single-width for unknown
+ width_map[cp] = 1
+
+ except (ValueError, OverflowError):
+ # Skip invalid code points
+ continue
+
+ # Process Emoji - generally double-width
+ # Ranges according to Unicode Emoji standard
+ emoji_ranges = [
+ (0x1F000, 0x1F02F), # Mahjong Tiles
+ (0x1F0A0, 0x1F0FF), # Playing Cards
+ (0x1F300, 0x1F5FF), # Miscellaneous Symbols and Pictographs
+ (0x1F600, 0x1F64F), # Emoticons
+ (0x1F680, 0x1F6FF), # Transport and Map Symbols
+ (0x1F700, 0x1F77F), # Alchemical Symbols
+ (0x1F780, 0x1F7FF), # Geometric Shapes Extended
+ (0x1F800, 0x1F8FF), # Supplemental Arrows-C
+ (0x1F900, 0x1F9FF), # Supplemental Symbols and Pictographs
+ (0x1FA00, 0x1FA6F), # Chess Symbols
+ (0x1FA70, 0x1FAFF), # Symbols and Pictographs Extended-A
+ ]
+
+ for start, end in emoji_ranges:
+ for cp in range(start, end + 1):
+ if cp not in width_map or width_map[cp] != 0: # Don't override zero-width
+ try:
+ char = chr(cp)
+ width_map[cp] = 2
+ except (ValueError, OverflowError):
+ continue
+
+ # Optimize to create range tables
+ def ranges_optimize(width_data, target_width):
+ points = sorted([cp for cp, width in width_data.items() if width == target_width])
+ if not points:
+ return []
+
+ # Group consecutive code points into ranges
+ ranges = []
+ start = points[0]
+ prev = start
+
+ for cp in points[1:]:
+ if cp > prev + 1:
+ ranges.append((start, prev))
+ start = cp
+ prev = cp
+
+ # Add the last range
+ ranges.append((start, prev))
+ return ranges
+
+ # Extract ranges for each width
+ zero_width_ranges = ranges_optimize(width_map, 0)
+ double_width_ranges = ranges_optimize(width_map, 2)
+
+ # Get Unicode version information
+ unicode_version = unicodedata.unidata_version
+
+ # Generate C implementation file
+ with open(c_file, 'w') as f:
+ f.write(f"""\
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ucs_width.c - Unicode character width lookup
+ *
+ * Auto-generated by gen_ucs_width.py
+ *
+ * Unicode Version: {unicode_version}
+ */
+
+#include <linux/types.h>
+#include <linux/array_size.h>
+#include <linux/bsearch.h>
+#include <linux/consolemap.h>
+
+struct interval {{
+ uint32_t first;
+ uint32_t last;
+}};
+
+/* Zero-width character ranges */
+static const struct interval zero_width_ranges[] = {{
+""")
+
+ for start, end in zero_width_ranges:
+ try:
+ start_char_desc = unicodedata.name(chr(start)) if start < 0x10000 else f"U+{start:05X}"
+ if start == end:
+ comment = f"/* {start_char_desc} */"
+ else:
+ end_char_desc = unicodedata.name(chr(end)) if end < 0x10000 else f"U+{end:05X}"
+ comment = f"/* {start_char_desc} - {end_char_desc} */"
+ except:
+ if start == end:
+ comment = f"/* U+{start:05X} */"
+ else:
+ comment = f"/* U+{start:05X} - U+{end:05X} */"
+
+ f.write(f"\t{{ 0x{start:05X}, 0x{end:05X} }}, {comment}\n")
+
+ f.write("""\
+};
+
+/* Double-width character ranges */
+static const struct interval double_width_ranges[] = {
+""")
+
+ for start, end in double_width_ranges:
+ try:
+ start_char_desc = unicodedata.name(chr(start)) if start < 0x10000 else f"U+{start:05X}"
+ if start == end:
+ comment = f"/* {start_char_desc} */"
+ else:
+ end_char_desc = unicodedata.name(chr(end)) if end < 0x10000 else f"U+{end:05X}"
+ comment = f"/* {start_char_desc} - {end_char_desc} */"
+ except:
+ if start == end:
+ comment = f"/* U+{start:05X} */"
+ else:
+ comment = f"/* U+{start:05X} - U+{end:05X} */"
+
+ f.write(f"\t{{ 0x{start:05X}, 0x{end:05X} }}, {comment}\n")
+
+ f.write("""\
+};
+
+
+static int ucs_cmp(const void *key, const void *element)
+{
+ uint32_t cp = *(uint32_t *)key;
+ const struct interval *e = element;
+
+ if (cp > e->last)
+ return 1;
+ if (cp < e->first)
+ return -1;
+ return 0;
+}
+
+static bool is_in_interval(uint32_t cp, const struct interval *intervals, size_t count)
+{
+ if (cp < intervals[0].first || cp > intervals[count - 1].last)
+ return false;
+
+ return __inline_bsearch(&cp, intervals, count,
+ sizeof(*intervals), ucs_cmp) != NULL;
+}
+
+/**
+ * Determine if a Unicode code point is zero-width.
+ *
+ * @param ucs: Unicode code point (UCS-4)
+ * Return: true if the character is zero-width, false otherwise
+ */
+bool ucs_is_zero_width(uint32_t cp)
+{
+ return is_in_interval(cp, zero_width_ranges, ARRAY_SIZE(zero_width_ranges));
+}
+
+/**
+ * Determine if a Unicode code point is double-width.
+ *
+ * @param ucs: Unicode code point (UCS-4)
+ * Return: true if the character is double-width, false otherwise
+ */
+bool ucs_is_double_width(uint32_t cp)
+{
+ return is_in_interval(cp, double_width_ranges, ARRAY_SIZE(double_width_ranges));
+}
+""")
+
+ # Print summary
+ zero_width_count = sum(end - start + 1 for start, end in zero_width_ranges)
+ double_width_count = sum(end - start + 1 for start, end in double_width_ranges)
+
+ print(f"Generated {c_file} with:")
+ print(f"- {len(zero_width_ranges)} zero-width ranges covering ~{zero_width_count} code points")
+ print(f"- {len(double_width_ranges)} double-width ranges covering ~{double_width_count} code points")
+
+if __name__ == "__main__":
+ generate_ucs_width()
--
2.49.0