[PATCH 4/6] TOKEN_DIRECTIVE: recognize directive-introducing # in tokenizer

[Al Viro <viro@zeniv.linux.org.uk>]

'#' acts as start of directive it had been the first token in its line
at the beginning of phase 4.  In other words, it must have come from
the input stream rather than from macro expansion (# may be present in
the body of object-like macro) and there must've been no token between
it and beginning of the line *before* any macro expansion had been
done (i.e. in
	#define EMPTY
	EMPTY #include <stdlib.h>
the # in front of include is *not* a beginning of directive).

Currently we handle that by hijacking 'noexpand' flag to mark the instances
of # coming from macro expansion and checking if the token we'd run into
might be a directive-starting # in do_preprocess() and in collect_arg().

It's actually simpler and faster to recognize those in tokenizer.  Just
have get_one_special() check if it's about to return a TOKEN_SPECIAL[#]
and if it's the first token in its line, return a token of new type
(TOKEN_DIRECTIVE).  Checks in the preprocessor get simpler and we get
rid of 'noexpand' abuse - there's no way for TOKEN_DIRECTIVE to end up
in the body of any macro.

Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
---
 pre-process.c | 22 ++++++++--------------
 token.h       |  1 +
 tokenize.c    |  2 ++
 3 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/pre-process.c b/pre-process.c
index ea199a9a..e62d6379 100644
--- a/pre-process.c
+++ b/pre-process.c
@@ -263,12 +263,10 @@ static struct token *collect_arg(struct token *prev, bool vararg, const struct p
 	int nesting = 0;
 
 	while (!eof_token(next = scan_next(p))) {
-		if (next->pos.newline && match_op(next, '#')) {
-			if (!next->pos.noexpand) {
-				preprocessor_line(stream, p);
-				__free_token(next);	/* Free the '#' token */
-				continue;
-			}
+		if (token_type(next) == TOKEN_DIRECTIVE) {
+			preprocessor_line(stream, p);
+			__free_token(next);	/* Free the '#' token */
+			continue;
 		}
 		switch (token_type(next)) {
 		case TOKEN_STREAMEND:
@@ -1402,8 +1400,6 @@ static struct token *handle_hash(struct token **p, struct arg_state args[])
 			goto Equote;
 		__free_token(token);
 		token = *p = next;
-	} else {
-		token->pos.noexpand = 1;
 	}
 	return token;
 
@@ -2427,12 +2423,10 @@ static void do_preprocess(struct token **list)
 	while (!eof_token(next = scan_next(list))) {
 		struct stream *stream = input_streams + next->pos.stream;
 
-		if (next->pos.newline && match_op(next, '#')) {
-			if (!next->pos.noexpand) {
-				preprocessor_line(stream, list);
-				__free_token(next);	/* Free the '#' token */
-				continue;
-			}
+		if (token_type(next) == TOKEN_DIRECTIVE) {
+			preprocessor_line(stream, list);
+			__free_token(next);	/* Free the '#' token */
+			continue;
 		}
 
 		switch (token_type(next)) {
diff --git a/token.h b/token.h
index 5915d6a4..7265bae5 100644
--- a/token.h
+++ b/token.h
@@ -109,6 +109,7 @@ enum token_type {
 	TOKEN_IF,
 	TOKEN_SKIP_GROUPS,
 	TOKEN_ELSE,
+	TOKEN_DIRECTIVE,
 };
 
 /* Combination tokens */
diff --git a/tokenize.c b/tokenize.c
index e6e071ae..9f7912c4 100644
--- a/tokenize.c
+++ b/tokenize.c
@@ -790,6 +790,8 @@ static int get_one_special(int c, stream_t *stream)
 	/* Pass it on.. */
 	token = stream->token;
 	token_type(token) = TOKEN_SPECIAL;
+	if (value == '#' && token->pos.newline)
+		token_type(token) = TOKEN_DIRECTIVE;
 	token->special = value;
 	add_token(stream);
 	return next;
-- 
2.47.3