[PATCH 45/73] c23: Add support for digit separators in numeric literals

[Bart Van Assche <bvanassche@acm.org>]

Allow single quotes (') between digits in preprocessing numbers and
strip them before parsing integer and floating-point values in
expression.c to support C23 digit separators (WG14 N2626) and emit a
pedantic warning in pre-C23 modes.

Signed-off-by: Bart Van Assche <bvanassche@acm.org>
---
 expression.c                               | 165 +++++++++++++++++++++
 tokenize.c                                 |  35 ++++-
 validation/c23-digit-separators-error.c    |  36 +++++
 validation/c23-digit-separators-pedantic.c |  12 ++
 validation/c23-digit-separators.c          |  29 ++++
 5 files changed, 275 insertions(+), 2 deletions(-)
 create mode 100644 validation/c23-digit-separators-error.c
 create mode 100644 validation/c23-digit-separators-pedantic.c
 create mode 100644 validation/c23-digit-separators.c

diff --git a/expression.c b/expression.c
index da2c14b458db..1b04d977a2d5 100644
--- a/expression.c
+++ b/expression.c
@@ -258,6 +258,157 @@ static unsigned long long parse_num(const char *nptr, char **end)
 	return strtoull(nptr, end, 0);
 }
 
+static int is_digit_for_base(int c, int base)
+{
+	if (base == 2)
+		return c == '0' || c == '1';
+	if (base == 8)
+		return c >= '0' && c <= '7';
+	if (base == 16)
+		return isxdigit((unsigned char)c);
+	return isdigit((unsigned char)c);
+}
+
+static int validate_digit_separators(struct expression *expr, const char *str)
+{
+	enum {
+		NUM_INT,
+		NUM_FRAC,
+		NUM_EXP_SIGN,
+		NUM_EXP,
+		NUM_SUFFIX,
+	} state = NUM_INT;
+	int is_hex = (str[0] == '0' && (str[1] == 'x' || str[1] == 'X'));
+	int is_bin = (str[0] == '0' && (str[1] == 'b' || str[1] == 'B'));
+	int is_oct = (str[0] == '0' && !is_hex && !is_bin &&
+		      strchr(str, '.') == NULL && strchr(str, 'e') == NULL &&
+		      strchr(str, 'E') == NULL);
+	int base = is_hex ? 16 : is_bin ? 2 : is_oct ? 8 : 10;
+	int in_digits = 0;
+	int i = (is_hex || is_bin) ? 2 : 0;
+
+	for (; str[i]; i++) {
+		char c = str[i];
+
+		if (c == '\'') {
+			char prev = i > 0 ? str[i - 1] : 0;
+			char next = str[i + 1];
+
+			if (prev == '\'' || next == '\'') {
+				sparse_error(expr->pos,
+					     "adjacent digit separators");
+				return 0;
+			}
+			if (i == 2 && (is_hex || is_bin)) {
+				sparse_error(
+					expr->pos,
+					"digit separator after base indicator");
+				return 0;
+			}
+			if (prev == '.' || next == '.') {
+				sparse_error(
+					expr->pos,
+					"digit separator adjacent to decimal point");
+				return 0;
+			}
+			if (state == NUM_EXP_SIGN ||
+			    (state != NUM_SUFFIX &&
+			     ((is_hex && (next == 'p' || next == 'P')) ||
+			      (!is_hex && (next == 'e' || next == 'E')))) ||
+			    (!in_digits && state == NUM_EXP)) {
+				sparse_error(
+					expr->pos,
+					"digit separator adjacent to exponent");
+				return 0;
+			}
+			if (state == NUM_SUFFIX || !in_digits ||
+			    !is_digit_for_base(next, base)) {
+				sparse_error(
+					expr->pos,
+					"digit separator outside digit sequence");
+				return 0;
+			}
+			in_digits = 0;
+			continue;
+		}
+
+		switch (state) {
+		case NUM_INT:
+			if (is_digit_for_base(c, base)) {
+				in_digits = 1;
+			} else if (c == '.' && (base == 10 || base == 16)) {
+				state = NUM_FRAC;
+				in_digits = 0;
+			} else if ((base == 10 && (c == 'e' || c == 'E')) ||
+				   (base == 16 && (c == 'p' || c == 'P'))) {
+				state = NUM_EXP_SIGN;
+				base = 10;
+				in_digits = 0;
+			} else {
+				state = NUM_SUFFIX;
+				in_digits = 0;
+			}
+			break;
+		case NUM_FRAC:
+			if (is_digit_for_base(c, base)) {
+				in_digits = 1;
+			} else if ((base == 10 && (c == 'e' || c == 'E')) ||
+				   (base == 16 && (c == 'p' || c == 'P'))) {
+				state = NUM_EXP_SIGN;
+				base = 10;
+				in_digits = 0;
+			} else {
+				state = NUM_SUFFIX;
+				in_digits = 0;
+			}
+			break;
+		case NUM_EXP_SIGN:
+			state = NUM_EXP;
+			if (c == '+' || c == '-') {
+				in_digits = 0;
+			} else if (isdigit((unsigned char)c)) {
+				in_digits = 1;
+			} else {
+				state = NUM_SUFFIX;
+				in_digits = 0;
+			}
+			break;
+		case NUM_EXP:
+			if (isdigit((unsigned char)c)) {
+				in_digits = 1;
+			} else {
+				state = NUM_SUFFIX;
+				in_digits = 0;
+			}
+			break;
+		case NUM_SUFFIX:
+			break;
+		}
+	}
+	return 1;
+}
+
+static const char *strip_digit_separators(const char *str)
+{
+	size_t len;
+	char *buf, *dst;
+
+	if (!strchr(str, '\''))
+		return str;
+
+	len = strlen(str);
+	buf = __alloc_bytes(len + 1);
+	dst = buf;
+
+	while (*str) {
+		if (*str != '\'')
+			*dst++ = *str;
+		str++;
+	}
+	*dst = '\0';
+	return buf;
+}
+
 static void get_number_value(struct expression *expr, struct token *token)
 {
 	const char *str = token->number;
@@ -268,6 +419,19 @@ static void get_number_value(struct expression *expr, struct token *token)
 	int try_unsigned = 1;
 	int bits;
 
+	if (strchr(str, '\'')) {
+		if (!validate_digit_separators(expr, str)) {
+			expr->type = EXPR_VALUE;
+			expr->flags = CEF_SET_ICE;
+			expr->ctype = &bad_ctype;
+			return;
+		}
+		if (standard < STANDARD_C23 && Wpedantic)
+			warning(expr->pos,
+				"ISO C does not support digit separators before C23");
+		str = strip_digit_separators(str);
+	}
+
 	errno = 0;
 	value = parse_num(str, &end);
 	if (end == str)
@@ -385,6 +549,7 @@ Float:
 Enoint:
 	sparse_error(expr->pos, "constant %s is not a valid number", show_token(token));
 	expr->type = EXPR_VALUE;
+	expr->flags = CEF_SET_ICE;
 	expr->value = 0;
 	expr->ctype = &int_ctype;
 }
diff --git a/tokenize.c b/tokenize.c
index 76b0278925ed..b0d4059c8782 100644
--- a/tokenize.c
+++ b/tokenize.c
@@ -50,6 +50,7 @@ typedef struct {
 	int fd, offset, size;
 	int pos, line, nr;
 	int newline, whitespace;
+	int pushed_char;
 	struct token **tokenlist;
 	struct token *token;
 	unsigned char *buffer;
@@ -446,8 +447,14 @@ got_eof:
  */
 static inline int nextchar(stream_t *stream)
 {
-	int offset = stream->offset;
+	int offset;
 
+	if (stream->pushed_char != -2) {
+		int c = stream->pushed_char;
+		stream->pushed_char = -2;
+		return c;
+	}
+	offset = stream->offset;
 	if (offset < stream->size) {
 		int c = stream->buffer[offset++];
 		static const char special[256] = {
@@ -462,6 +469,19 @@ static inline int nextchar(stream_t *stream)
 	return nextchar_slow(stream);
 }
 
+static int peekchar(stream_t *stream)
+{
+	if (stream->pushed_char != -2)
+		return stream->pushed_char;
+	if (stream->offset < stream->size) {
+		int c = stream->buffer[stream->offset];
+		if (c != '\r' && c != '\\')
+			return c;
+	}
+	stream->pushed_char = nextchar_slow(stream);
+	return stream->pushed_char;
+}
+
 struct token eof_token_entry;
 
 static struct token *mark_eof(stream_t *stream)
@@ -558,8 +578,18 @@ static int get_one_number(int c, int next, stream_t *stream)
 	*p++ = c;
 	for (;;) {
 		long class =  cclass[next + 1];
-		if (!(class & (Dot | Digit | Letter)))
+		if (!(class & (Dot | Digit | Letter))) {
+			if (next == '\'') {
+				int after = peekchar(stream);
+				if (cclass[after + 1] & (Digit | Letter)) {
+					if (p != buffer_end)
+						*p++ = next;
+					next = nextchar(stream);
+					continue;
+				}
+			}
 			break;
+		}
 		if (p != buffer_end)
 			*p++ = next;
 		next = nextchar(stream);
@@ -987,6 +1017,7 @@ static struct token *setup_stream(stream_t *stream, int idx, int fd,
 	stream->offset = 0;
 	stream->size = buf_size;
 	stream->buffer = buf;
+	stream->pushed_char = -2;
 
 	begin = alloc_token(stream);
 	token_type(begin) = TOKEN_STREAMBEGIN;
diff --git a/validation/c23-digit-separators-error.c b/validation/c23-digit-separators-error.c
new file mode 100644
index 000000000000..10ddeffcd99c
--- /dev/null
+++ b/validation/c23-digit-separators-error.c
@@ -0,0 +1,36 @@
+int a = 1''0;
+int b = 0x'1;
+int b2 = 0b'1;
+int c = 1'u;
+int c2 = 123'u;
+double d = 1.'0;
+double e = 1'.';
+double f = 1e'0;
+double g = 1'e0;
+double h = 1e+'0;
+float i = 0x1p'0f;
+float j = 0x1.0p1'f;
+float k = 1.0'f;
+
+/*
+ * check-name: c23-digit-separators-error
+ * check-command: sparse -std=c23 -Wno-decl $file
+ *
+ * check-error-start
+c23-digit-separators-error.c:1:10: error: Expected ; at end of declaration
+c23-digit-separators-error.c:1:10: error: got ''
+c23-digit-separators-error.c:2:9: error: digit separator after base indicator
+c23-digit-separators-error.c:3:10: error: digit separator after base indicator
+c23-digit-separators-error.c:4:9: error: digit separator outside digit sequence
+c23-digit-separators-error.c:5:10: error: digit separator outside digit sequence
+c23-digit-separators-error.c:6:12: error: digit separator adjacent to decimal point
+c23-digit-separators-error.c:7:13: error: Expected ; at end of declaration
+c23-digit-separators-error.c:7:13: error: got '.'
+c23-digit-separators-error.c:8:12: error: digit separator adjacent to exponent
+c23-digit-separators-error.c:9:12: error: digit separator adjacent to exponent
+c23-digit-separators-error.c:10:12: error: digit separator adjacent to exponent
+c23-digit-separators-error.c:11:11: error: digit separator adjacent to exponent
+c23-digit-separators-error.c:12:11: error: digit separator outside digit sequence
+c23-digit-separators-error.c:13:11: error: digit separator outside digit sequence
+ * check-error-end
+ */
diff --git a/validation/c23-digit-separators-pedantic.c b/validation/c23-digit-separators-pedantic.c
new file mode 100644
index 000000000000..d1f374d805e1
--- /dev/null
+++ b/validation/c23-digit-separators-pedantic.c
@@ -0,0 +1,12 @@
+int a = 1'000;
+double b = 1'0.5;
+
+/*
+ * check-name: c23-digit-separators-pedantic
+ * check-command: sparse -std=c17 -Wpedantic -Wno-decl $file
+ *
+ * check-error-start
+c23-digit-separators-pedantic.c:1:9: warning: ISO C does not support digit separators before C23
+c23-digit-separators-pedantic.c:2:12: warning: ISO C does not support digit separators before C23
+ * check-error-end
+ */
diff --git a/validation/c23-digit-separators.c b/validation/c23-digit-separators.c
new file mode 100644
index 000000000000..65af74822cb6
--- /dev/null
+++ b/validation/c23-digit-separators.c
@@ -0,0 +1,29 @@
+#if 0
+0' '
+1'+'2
+#endif
+
+#if 1'000 + 2'000 == 3'000
+int pp = 1;
+#endif
+
+_Static_assert(1'000'000 == 1000000, "dec");
+_Static_assert(0xdead'beef == 0xdeadbeef, "hex");
+_Static_assert(0XCAFE'BABE == 0xCAFEBABE, "hex upper");
+_Static_assert(0b1010'0101 == 0xa5, "bin");
+_Static_assert(0B1111'0000 == 0xf0, "bin upper");
+_Static_assert(0'123'456 == 0123456, "oct");
+_Static_assert(1'000ULL == 1000ULL, "ull");
+
+static int a = 1'000'000;
+static unsigned int b = 0xdead'beef;
+static int c = 0b1010'0101;
+static int d = 0'123'456;
+static double e = 1'000.000'1;
+static double f = 1'2.3'4e1'0;
+static float g = 0x1'a.b'cp1'0f;
+
+/*
+ * check-name: c23-digit-separators
+ * check-command: sparse -std=c23 -Wno-decl $file
+ */