[PATCH 07/73] pre-process: Fix macro expansion and include directive handling

[Bart Van Assche <bvanassche@acm.org>]

Fix buffer bounds, unterminated macro arguments, and include path checks
in pre-process.c:
- Handle TOKEN_STREAMEND and TOKEN_STREAMBEGIN on unterminated macro
  argument lists and bound token-pasting buffers in combine().
- Widen token_name_sequence() to PATH_MAX and check remaining capacity
  with pointer subtraction to prevent buffer and pointer overflows.
- Reject directory paths in try_include() via fstat() and diagnose
  incomplete defined() expressions in #if directives.
- Add regression tests in
  validation/preprocessor/preprocess-crash-fixes.c and
  validation/preprocessor/preprocess-errors-and-limits.c.

Signed-off-by: Bart Van Assche <bvanassche@acm.org>
---
 pre-process.c                                 | 83 +++++++++++++++----
 .../preprocessor/preprocess-crash-fixes.c     | 25 ++++++
 .../preprocess-errors-and-limits.c            | 33 ++++++++
 3 files changed, 123 insertions(+), 18 deletions(-)
 create mode 100644 validation/preprocessor/preprocess-crash-fixes.c
 create mode 100644 validation/preprocessor/preprocess-errors-and-limits.c

diff --git a/pre-process.c b/pre-process.c
index 7b6555677570..ca019d704a10 100644
--- a/pre-process.c
+++ b/pre-process.c
@@ -35,6 +35,7 @@
 #include <fcntl.h>
 #include <limits.h>
 #include <time.h>
+#include <sys/stat.h>
 
 #include "lib.h"
 #include "allocate.h"
@@ -107,8 +108,12 @@ static int expand(struct token **, struct symbol *);
 
 static void replace_with_string(struct token *token, const char *str)
 {
-	int size = strlen(str) + 1;
-	struct string *s = __alloc_string(size);
+	size_t size = strlen(str) + 1;
+	struct string *s;
+
+	if (size > INT_MAX - sizeof(struct string))
+		die("string too large");
+	s = __alloc_string(size);
 
 	s->length = size;
 	memcpy(s->data, str, size);
@@ -312,6 +317,13 @@ struct arg {
 	int n_str;
 };
 
+static inline bool stream_end_token(const struct token *t)
+{
+	return eof_token(t) ||
+	       token_type(t) == TOKEN_STREAMEND ||
+	       token_type(t) == TOKEN_STREAMBEGIN;
+}
+
 static int collect_arguments(struct token *start, struct token *arglist, struct arg *args, struct token *what)
 {
 	int wanted = arglist->count.normal;
@@ -322,7 +334,7 @@ static int collect_arguments(struct token *start, struct token *arglist, struct
 
 	if (!wanted) {
 		next = collect_arg(start, 0, &what->pos);
-		if (eof_token(next))
+		if (stream_end_token(next))
 			goto Eclosing;
 		if (!eof_token(start->next) || !match_op(next, ')')) {
 			count++;
@@ -332,7 +344,7 @@ static int collect_arguments(struct token *start, struct token *arglist, struct
 		for (count = 0; count < wanted; count++) {
 			struct argcount *p = &arglist->next->count;
 			next = collect_arg(start, p->vararg, &what->pos);
-			if (eof_token(next))
+			if (stream_end_token(next))
 				goto Eclosing;
 			if (p->vararg && wanted == 1 && eof_token(start->next))
 				break;
@@ -373,7 +385,7 @@ Emany:
 		next = collect_arg(next, 0, &what->pos);
 		count++;
 	}
-	if (eof_token(next))
+	if (stream_end_token(next))
 		goto Eclosing;
 	sparse_error(what->pos, "macro \"%s\" passed %d arguments, but takes just %d",
 		show_token(what), count, wanted);
@@ -382,7 +394,10 @@ Eclosing:
 	sparse_error(what->pos, "unterminated argument list invoking macro \"%s\"",
 		show_token(what));
 out:
-	what->next = next->next;
+	if (stream_end_token(next))
+		what->next = next;
+	else
+		what->next = next->next;
 	return 0;
 }
 
@@ -499,12 +514,22 @@ static enum token_type combine(struct token *left, struct token *right, char *p)
 	if (t2 != TOKEN_IDENT && t2 != TOKEN_NUMBER && t2 != TOKEN_SPECIAL)
 		return TOKEN_ERROR;
 
-	strcpy(p, show_token(left));
-	strcat(p, show_token(right));
-	len = strlen(p);
+	{
+		const char *s1 = show_token(left);
+		int l1 = strlen(s1);
+		const char *s2;
+		int l2;
 
-	if (len >= 256)
-		return TOKEN_ERROR;
+		if (l1 > 4096)
+			return TOKEN_ERROR;
+		memcpy(p, s1, l1);
+		s2 = show_token(right);
+		l2 = strlen(s2);
+		if (l1 + l2 > 4096)
+			return TOKEN_ERROR;
+		memcpy(p + l1, s2, l2 + 1);
+		len = l1 + l2;
+	}
 
 	if (t1 == TOKEN_IDENT) {
 		if (t2 == TOKEN_SPECIAL)
@@ -537,7 +562,7 @@ static enum token_type combine(struct token *left, struct token *right, char *p)
 
 static int merge(struct token *left, struct token *right)
 {
-	static char buffer[512];
+	static char buffer[4097];
 	enum token_type res = combine(left, right, buffer);
 	int n;
 
@@ -782,17 +807,22 @@ static int expand(struct token **list, struct symbol *sym)
 
 static const char *token_name_sequence(struct token *token, int endop, struct token *start)
 {
-	static char buffer[256];
+	static char buffer[PATH_MAX];
 	char *ptr = buffer;
 
 	while (!eof_token(token) && !match_op(token, endop)) {
-		int len;
+		size_t len, rem;
 		const char *val;
 		if (token_type(token) == TOKEN_STRING)
 			val = token->string->data;
 		else
 			val = show_token(token);
 		len = strlen(val);
+		rem = sizeof(buffer) - (size_t)(ptr - buffer);
+		if (len >= rem) {
+			sparse_error(token->pos, "header name too long");
+			return NULL;
+		}
 		memcpy(ptr, val, len);
 		ptr += len;
 		token = token->next;
@@ -896,23 +926,30 @@ static void set_stream_include_path(struct stream *stream)
 static int try_include(struct position pos, const char *path, const char *filename, int flen, struct token **where, const char **next_path)
 {
 	int fd;
+	struct stat st;
 	int plen = strlen(path);
-	static char fullname[PATH_MAX];
+	char fullname[PATH_MAX];
 
+	if (!flen || plen + flen + 2 > sizeof(fullname))
+		return 0;
 	memcpy(fullname, path, plen);
 	if (plen && path[plen-1] != '/') {
 		fullname[plen] = '/';
 		plen++;
 	}
 	memcpy(fullname+plen, filename, flen);
+	fullname[plen + flen] = '\0';
 	if (already_tokenized(fullname))
 		return 1;
 	fd = open(fullname, O_RDONLY);
 	if (fd >= 0) {
-		char *streamname = xmemdup(fullname, plen + flen);
-		*where = tokenize(&pos, streamname, fd, *where, next_path);
+		if (fstat(fd, &st) == 0 && !S_ISDIR(st.st_mode)) {
+			char *streamname = xmemdup(fullname, plen + flen + 1);
+			*where = tokenize(&pos, streamname, fd, *where, next_path);
+			close(fd);
+			return 1;
+		}
 		close(fd);
-		return 1;
 	}
 	return 0;
 }
@@ -961,6 +998,8 @@ static int handle_include_path(struct stream *stream, struct token **list, struc
 
 	token = next->next;
 	filename = token_name_sequence(token, expect, token);
+	if (!filename)
+		return 1;
 	flen = strlen(filename) + 1;
 
 	/* Absolute path? */
@@ -1649,6 +1688,14 @@ static int expression_value(struct token **where)
 		list = &p->next;
 	}
 
+	if (state == 1 || state == 2) {
+		sparse_error((*beginning)->pos, "expected identifier after \"defined\"");
+		replace_with_bool(*beginning, 0);
+		(*beginning)->next = &eof_token_entry;
+	} else if (state == 3) {
+		sparse_error((*beginning)->pos, "missing ')' after \"defined\"");
+	}
+
 	p = constant_expression(*where, &expr);
 	if (!eof_token(p))
 		sparse_error(p->pos, "garbage at end: %s", show_token_sequence(p, 0));
diff --git a/validation/preprocessor/preprocess-crash-fixes.c b/validation/preprocessor/preprocess-crash-fixes.c
new file mode 100644
index 000000000000..92ea1ecde717
--- /dev/null
+++ b/validation/preprocessor/preprocess-crash-fixes.c
@@ -0,0 +1,25 @@
+#define REC_A(...) REC_B(__VA_ARGS__)
+#define REC_B(...) REC_A(__VA_ARGS__)
+extern int REC_A(int, ...);
+int rec_nonempty = sizeof(REC_A(1));
+
+/* Recursive macro with empty __VA_ARGS__ must terminate without crash */
+#define REC_E(...) REC_F(__VA_ARGS__)
+#define REC_F(...) REC_E(__VA_ARGS__)
+extern int REC_E(void);
+int rec_empty = sizeof(REC_E());
+
+#define ID(x) x
+int a = ID(1);
+
+/* Octal escape > 0377 */
+int oct_overflow = '\400';
+
+/*
+ * check-name: preprocess-crash-fixes
+ * check-command: sparse -Wno-decl $file
+ *
+ * check-error-start
+preprocessor/preprocess-crash-fixes.c:16:20: warning: octal escape sequence out of range
+ * check-error-end
+ */
diff --git a/validation/preprocessor/preprocess-errors-and-limits.c b/validation/preprocessor/preprocess-errors-and-limits.c
new file mode 100644
index 000000000000..7ad1265ba9c0
--- /dev/null
+++ b/validation/preprocessor/preprocess-errors-and-limits.c
@@ -0,0 +1,33 @@
+#if defined()
+#endif
+#if defined(FOO
+#endif
+#if defined
+#endif
+#if defined(
+#endif
+#include "."
+static const char *s = "\x";
+
+/* 300-character identifier */
+static int aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = 42;
+
+static int get_val(void)
+{
+	return aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa;
+}
+
+/*
+ * check-name: preprocess-errors-and-limits
+ * check-command: sparse $file
+ * check-exit-value: 1
+ *
+ * check-error-start
+preprocessor/preprocess-errors-and-limits.c:10:27: warning: \x used with no following hex digits
+preprocessor/preprocess-errors-and-limits.c:1:13: error: expected preprocessor identifier
+preprocessor/preprocess-errors-and-limits.c:3:13: error: missing ')' after "defined"
+preprocessor/preprocess-errors-and-limits.c:5:5: error: expected identifier after "defined"
+preprocessor/preprocess-errors-and-limits.c:7:5: error: expected identifier after "defined"
+preprocessor/preprocess-errors-and-limits.c:9:10: error: unable to open '.'
+ * check-error-end
+ */