[PATCH 06/73] tokenize: Support 4096-byte identifiers and fix escape bounds

[Bart Van Assche <bvanassche@acm.org>]

Support long identifiers and harden character escape and token handling
in char.c, token.h, and tokenize.c:
- Widen struct ident's len field to unsigned short and enlarge identifier
  buffers in get_one_identifier() and show_ident() to 4096 bytes so
  identifiers longer than 255 bytes are not truncated.
- Guard parse_escape() against reading past the end of the buffer on a
  trailing backslash and cast input bytes to unsigned char.
- Format TOKEN_GNU_KLUDGE in show_token() and quote_token(), and use
  logical && when checking stream->pos && Wnewline_eof.

Signed-off-by: Bart Van Assche <bvanassche@acm.org>
---
 char.c     |  5 +++--
 token.h    |  2 +-
 tokenize.c | 29 +++++++++++++++++------------
 3 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/char.c b/char.c
index 730ae3f5d8ca..1f6eb1f6be83 100644
--- a/char.c
+++ b/char.c
@@ -8,14 +8,15 @@
 
 static const char *parse_escape(const char *p, unsigned *val, const char *end, int bits, struct position pos)
 {
-	unsigned c = *p++;
+	unsigned c = (unsigned char)*p++;
 	unsigned d;
 	if (c != '\\') {
 		*val = c;
 		return p;
 	}
 
-	c = *p++;
+	if (p < end)
+		c = (unsigned char)*p++;
 	switch (c) {
 	case 'a': c = '\a'; break;
 	case 'b': c = '\b'; break;
diff --git a/token.h b/token.h
index 9000e0cb6574..8c8a496bee89 100644
--- a/token.h
+++ b/token.h
@@ -70,7 +70,7 @@ extern int *hash_stream(const char *name);
 struct ident {
 	struct ident *next;	/* Hash chain of identifiers */
 	struct symbol *symbols;	/* Pointer to semantic meaning list */
-	unsigned char len;	/* Length of identifier name */
+	unsigned short len;	/* Length of identifier name */
 	unsigned char tainted:1,
 	              reserved:1,
 		      keyword:1;
diff --git a/tokenize.c b/tokenize.c
index 54ea348cab91..76b0278925ed 100644
--- a/tokenize.c
+++ b/tokenize.c
@@ -98,14 +98,14 @@ const char *show_special(int val)
 
 const char *show_ident(const struct ident *ident)
 {
-	static char buff[4][256];
+	static char buff[4][4097];
 	static int n;
 	char *buffer;
 
 	if (!ident)
 		return "<noident>";
 	buffer = buff[3 & ++n];
-	sprintf(buffer, "%.*s", ident->len, ident->name);
+	snprintf(buffer, sizeof(buff[0]), "%.*s", ident->len, ident->name);
 	return buffer;
 }
 
@@ -230,11 +230,11 @@ const char *show_token(const struct token *token)
 			token->string->length - 1, 'L', '"');
 
 	case TOKEN_STREAMBEGIN:
-		sprintf(buffer, "<beginning of '%s'>", stream_name(token->pos.stream));
+		snprintf(buffer, sizeof(buffer), "<beginning of '%s'>", stream_name(token->pos.stream));
 		return buffer;
 
 	case TOKEN_STREAMEND:
-		sprintf(buffer, "<end of '%s'>", stream_name(token->pos.stream));
+		snprintf(buffer, sizeof(buffer), "<end of '%s'>", stream_name(token->pos.stream));
 		return buffer;
 
 	case TOKEN_UNTAINT:
@@ -245,6 +245,9 @@ const char *show_token(const struct token *token)
 		sprintf(buffer, "<argcnt>");
 		return buffer;
 
+	case TOKEN_GNU_KLUDGE:
+		return ",";
+
 	default:
 		sprintf(buffer, "unhandled token type '%d' ", token_type(token));
 		return buffer;
@@ -287,6 +290,8 @@ const char *quote_token(const struct token *token)
 	case TOKEN_WIDE_STRING: 
 		return quote_char(token->string->data,
 			token->string->length - 1, 'L', '"');
+	case TOKEN_GNU_KLUDGE:
+		return ",";
 	default:
 		sprintf(buffer, "unhandled token type '%d' ", token_type(token));
 		return buffer;
@@ -427,7 +432,7 @@ got_eof:
 		c = '\\';
 		goto out;
 	}
-	if (stream->pos & Wnewline_eof)
+	if (stream->pos && Wnewline_eof)
 		warning(stream_pos(stream), "no newline at end of file");
 	else if (spliced)
 		warning(stream_pos(stream), "backslash-newline at end of file");
@@ -860,7 +865,7 @@ static struct ident *create_hashed_ident(const char *name, int len, unsigned lon
 
 	p = &hash_table[hash];
 	while ((ident = *p) != NULL) {
-		if (ident->len == (unsigned char) len) {
+		if (ident->len == (unsigned short) len) {
 			if (memcmp(name, ident->name, len) != 0)
 				goto next;
 
@@ -919,7 +924,7 @@ static int get_one_identifier(int c, stream_t *stream)
 	struct token *token;
 	struct ident *ident;
 	unsigned long hash;
-	char buf[256];
+	char buf[4096];
 	int len = 1;
 	int next;
 
@@ -929,11 +934,11 @@ static int get_one_identifier(int c, stream_t *stream)
 		next = nextchar(stream);
 		if (!(cclass[next + 1] & (Letter | Digit)))
 			break;
-		if (len >= sizeof(buf))
-			break;
-		hash = ident_hash_add(hash, next);
-		buf[len] = next;
-		len++;
+		if (len < sizeof(buf)) {
+			hash = ident_hash_add(hash, next);
+			buf[len] = next;
+			len++;
+		}
 	};
 	if (cclass[next + 1] & Quote) {
 		if (len == 1 && buf[0] == 'L') {