[PATCH 0/8] Fix Python string escapes

Benjamin Gray
2023-08-14T06:08:39Z

This is upgraded to a SyntaxWarning in Python 3.12 (3.12.0rc1), and is
intended to eventually be a syntax error.

This series aims to fix these now to get ahead of it before it's an error.

Most of the patches are generated using the below Python script. It
uses the builtin ast module to parse a Python file, locate all strings,
find the corresponding source code, and check for bad escape sequences.

If it finds things to fix then it applies the fixes and reparses the
file into a fixed AST. It finally compares the original and fixed ASTs
to ensure no semantic difference has been introduced (dumping is done to
remove node location information, which is expected to be different).

There are some limitations of the ast module, in particular it throws
away a lot of information about the string source. f-strings especially
interact poorly here (the slices between formats are presented as
separate strings, but the node range of each is the entire f-string),
so are skipped. f-strings are handled manually in the final patch.

A lot of the fixes are for regex patterns, which could be changed to use
r-strings instead. But that is less easy to automate, so I avoided doing
so in this series. AST verification should still be possible though,
because being a plain or r-string is stripped away in the AST.

---
#!/usr/bin/env python3

"""
Fix all bad escape characters in strings
"""

import ast
from pathlib import Path

def get_offset(source: str, row: int, col: int) -> int:
"""
Turn a row + column pair into a byte offset
"""
offset = 0

cur_row = 1 # 1-indexed rows
cur_col = 0 # 0-indexed columns

for c in source:
if cur_row == row and cur_col == col:
return offset

offset += 1

if c == "\n":
cur_row += 1
cur_col = 0
else:
cur_col += 1

raise Exception("Failed to get offset")

parse_failures: list[Path] = []
fix_failures = 0
bad_escapes = 0
fstrings: set[Path] = set()

for pyfile in Path(".").glob("**/*.py"):
content = pyfile.read_text("utf-8")

try:
syntax = ast.parse(content, filename=pyfile)
except:
print(f"{pyfile}: ERROR Failed to parse, is it Python3?")
parse_failures.append(pyfile)
continue

fixes: list[tuple[int, int, str]] = []

for node in ast.walk(syntax):
if not isinstance(node, ast.Constant):
continue

if not isinstance(node.value, str):
continue

if node.value.count("\\") == 0:
continue

assert(isinstance(node.end_lineno, int))
assert(isinstance(node.end_col_offset, int))

start = get_offset(content, node.lineno, node.col_offset)
end = get_offset(content, node.end_lineno, node.end_col_offset)
raw = content[start:end]

# backslashes in r-strings are already literal
if raw.startswith("r"):
continue

# f-strings are difficult to handle correctly
if raw.startswith("f"):
fstrings.add(pyfile)
continue

fixed = "" # The fixed representation of the string
escaped = False # If the current character is escaped by a previous backslash
allowed = '\n\\\'"abfnrtv01234567xNuU' # characters allowed after a backslash

for c in raw:
if escaped:
if c not in allowed:
fixed += '\\'

fixed += c
escaped = False
continue

fixed += c

if c == '\\':
escaped = True

if fixed != raw:
print(f"{pyfile}:{node.lineno}:{node.col_offset}: FOUND {raw}")
fixes.append((start, end, fixed))

if len(fixes) == 0:
continue

bad_escapes += len(fixes)

# Apply fixes in reverse order to keep offsets valid
for start, end, fix in reversed(sorted(fixes, key=lambda k: k[0])):
print(f"{pyfile}:[{start}-{end}]: APPLY {fix}")
content = content[:start] + fix + content[end:]

fixed_syntax = ast.parse(content, filename=f"{pyfile}+fixed")

if ast.dump(syntax) != ast.dump(fixed_syntax):
print(f"{pyfile}: ERROR Fixed syntax tree yields different AST")
fix_failures += 1
continue

pyfile.write_text(content)

print(f"---------------------------------")
print(f"Parse failures: {len(parse_failures)}")
for f in sorted(parse_failures):
print(f" - {f}")

print(f"Bad escapes fixed: {bad_escapes}")
print(f"Fixes that broke the AST: {fix_failures}")
print(f"Files with skipped f-strings: {len(fstrings)}")
for f in sorted(fstrings):
print(f" - {f}")

---

Benjamin Gray (8):
ia64: fix Python string escapes
Documentation/sphinx: fix Python string escapes
drivers/comedi: fix Python string escapes
scripts: fix Python string escapes
tools/perf: fix Python string escapes
tools/power: fix Python string escapes
selftests/bpf: fix Python string escapes
selftests/bpf: fix Python string escapes in f-strings

Documentation/sphinx/cdomain.py | 2 +-
Documentation/sphinx/kernel_abi.py | 2 +-
Documentation/sphinx/kernel_feat.py | 2 +-
Documentation/sphinx/kerneldoc.py | 2 +-
Documentation/sphinx/maintainers_include.py | 8 +--
arch/ia64/scripts/unwcheck.py | 2 +-
.../ni_routing/tools/convert_csv_to_c.py | 2 +-
scripts/bpf_doc.py | 56 +++++++++----------
scripts/clang-tools/gen_compile_commands.py | 2 +-
scripts/gdb/linux/symbols.py | 2 +-
tools/perf/pmu-events/jevents.py | 2 +-
.../scripts/python/arm-cs-trace-disasm.py | 4 +-
tools/perf/scripts/python/compaction-times.py | 2 +-
.../scripts/python/exported-sql-viewer.py | 4 +-
tools/power/pm-graph/bootgraph.py | 12 ++--
.../selftests/bpf/test_bpftool_synctypes.py | 26 ++++-----
tools/testing/selftests/bpf/test_offload.py | 2 +-
17 files changed, 66 insertions(+), 66 deletions(-)

--
2.41.0