[PATCH 1/4] more transparent dealing with tokens/position

[PATCH 1/4] more transparent dealing with tokens/position

[Jan Pokorný]

Unify usage of `token_type'.

Signed-off-by: Jan Pokorny <pokorny_jan@seznam.cz>
---
 expression.h |    2 +-
 token.h      |    4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/expression.h b/expression.h
index 9778de8..480079c 100644
--- a/expression.h
+++ b/expression.h
@@ -200,7 +200,7 @@ struct token *typename(struct token *, struct symbol **, int *);
 
 static inline int lookup_type(struct token *token)
 {
-	if (token->pos.type == TOKEN_IDENT) {
+	if (token_type(token) == TOKEN_IDENT) {
 		struct symbol *sym = lookup_symbol(token->ident, NS_SYMBOL | NS_TYPEDEF);
 		return sym && (sym->namespace & NS_TYPEDEF);
 	}
diff --git a/token.h b/token.h
index a7ec77e..a3c194d 100644
--- a/token.h
+++ b/token.h
@@ -205,12 +205,12 @@ extern struct token *preprocess(struct token *);
 
 static inline int match_op(struct token *token, int op)
 {
-	return token->pos.type == TOKEN_SPECIAL && token->special == op;
+	return token_type(token) == TOKEN_SPECIAL && token->special == op;
 }
 
 static inline int match_ident(struct token *token, struct ident *id)
 {
-	return token->pos.type == TOKEN_IDENT && token->ident == id;
+	return token_type(token) == TOKEN_IDENT && token->ident == id;
 }
 
 #endif
-- 
1.7.1

[PATCH 2/4] more transparent dealing with tokens/position

[Jan Pokorný]

Unify usage of `eof_token'.

Version with respective assertions added will be provided,
mainly for convenient testing.  I used it to proceeded sparse itself
and linux-2.6.35.12 sources with no assertion violation encountered.

Signed-off-by: Jan Pokorny <pokorny_jan@seznam.cz>
---
 pre-process.c |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/pre-process.c b/pre-process.c
index 603cc00..c0fc89d 100644
--- a/pre-process.c
+++ b/pre-process.c
@@ -794,7 +794,7 @@ out:
 
 static int free_preprocessor_line(struct token *token)
 {
-	while (token_type(token) != TOKEN_EOF) {
+	while (!eof_token(token)) {
 		struct token *free = token;
 		token = token->next;
 		__free_token(free);
-- 
1.7.1

[PATCH 3/4] more transparent dealing with tokens/position

[Jan Pokorný]

Treat `enum token_type' as enumeration type rather than int.
For instance, avoid (a bit magical) zero assignments.

I admit this is quite a wild change, especially wrt the move of
`struct position'.  This was probably the easiest way to compile
sparse without additional dependecies mangling.  Still, another
solution can be found.

Note that after applying patch 2/4 (i.e., if `eof_token' and
`eof_token_entry' are used consistently), TOKEN_EOF should no longer
be necessary (this patch also modifies `show_token' to accommodate this).
But it might be a good idea to keep it for backward compatibility (it's
usage/meaning stays untouched with this patch series), only add
a comment that it should not be used (for maintainability reasons).

Another solution for TOKEN_EOF then making it "deprecated" would be to
make `eof_token_entry' initialized with ".pos.type = TOKEN_EOF" to make
this correspondence explicit (so far, this has silently applied because
of implicit zero initialization of `eof_token_entry' global).
This would kept duality of how to test the single case, though,
and I consider the only and consistent way better.

Also, add `POSITION_NOT_FROM_TOKEN' to the `enum token_type' to avoid
magical zero assignments and add some explaining comments.
It is the best name I was able to come up with, so that it still
describes the meaning quite clearly.  Another suggestions welcome.

Signed-off-by: Jan Pokorny <pokorny_jan@seznam.cz>
---
 c2xml.c      |    2 +-
 expression.c |    2 +-
 lib.h        |   10 +---------
 symbol.c     |    2 +-
 token.h      |   19 ++++++++++++++++++-
 tokenize.c   |    8 ++++----
 6 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/c2xml.c b/c2xml.c
index 37f29cf..2f76865 100644
--- a/c2xml.c
+++ b/c2xml.c
@@ -68,7 +68,7 @@ static xmlNodePtr new_sym_node(struct symbol *sym, const char *name, xmlNodePtr
 	newNumProp(node, "start-line", sym->pos.line);
 	newNumProp(node, "start-col", sym->pos.pos);
 
-	if (sym->endpos.type) {
+	if (sym->endpos.type != POSITION_NOT_FROM_TOKEN) {
 		newNumProp(node, "end-line", sym->endpos.line);
 		newNumProp(node, "end-col", sym->endpos.pos);
 		if (sym->pos.stream != sym->endpos.stream)
diff --git a/expression.c b/expression.c
index 7e06e60..43a3606 100644
--- a/expression.c
+++ b/expression.c
@@ -224,7 +224,7 @@ static struct token *string_expression(struct token *token, struct expression *e
 {
 	struct string *string = token->string;
 	struct token *next = token->next;
-	int stringtype = token_type(token);
+	enum token_type stringtype = token_type(token);
 
 	convert_function(token);
 
diff --git a/lib.h b/lib.h
index 2cea252..6b17280 100644
--- a/lib.h
+++ b/lib.h
@@ -31,15 +31,7 @@ extern int gcc_major, gcc_minor, gcc_patchlevel;
 
 extern unsigned int hexval(unsigned int c);
 
-struct position {
-	unsigned int type:6,
-		     stream:14,
-		     newline:1,
-		     whitespace:1,
-		     pos:10;
-	unsigned int line:31,
-		     noexpand:1;
-};
+struct position;
 
 struct cmdline_include {
 	char *filename;
diff --git a/symbol.c b/symbol.c
index 96dfbfa..34d4f06 100644
--- a/symbol.c
+++ b/symbol.c
@@ -62,7 +62,7 @@ struct symbol *alloc_symbol(struct position pos, int type)
 	struct symbol *sym = __alloc_symbol(0);
 	sym->type = type;
 	sym->pos = pos;
-	sym->endpos.type = 0;
+	sym->endpos.type = POSITION_NOT_FROM_TOKEN;
 	return sym;
 }
 
diff --git a/token.h b/token.h
index a3c194d..057b40a 100644
--- a/token.h
+++ b/token.h
@@ -60,8 +60,15 @@ struct ident {
 	char name[];		/* Actual identifier */
 };
 
+/*
+ * Tightly connected with "struct token", but used also outside them as a part
+ * of symbol etc. position, which is in turn borrowed from underlying tokens.
+ */
 enum token_type {
-	TOKEN_EOF,
+	/* Used outside tokens to mark a position not initialized from a token */
+	POSITION_NOT_FROM_TOKEN,
+	/* Token type (outside them, ~particular position was initialized from) */
+	TOKEN_EOF = POSITION_NOT_FROM_TOKEN,	/* use "eof_token()" instead */
 	TOKEN_ERROR,
 	TOKEN_IDENT,
 	TOKEN_ZERO_IDENT,
@@ -85,6 +92,16 @@ enum token_type {
 	TOKEN_ELSE,
 };
 
+struct position {
+	enum token_type	type:6;
+	unsigned int stream:14,
+			newline:1,
+			whitespace:1,
+			pos:10;
+	unsigned int line:31,
+			noexpand:1;
+};
+
 /* Combination tokens */
 #define COMBINATION_STRINGS {	\
 	"+=", "++",		\
diff --git a/tokenize.c b/tokenize.c
index 272974b..6fa8714 100644
--- a/tokenize.c
+++ b/tokenize.c
@@ -48,7 +48,7 @@ const char *stream_name(int stream)
 static struct position stream_pos(stream_t *stream)
 {
 	struct position pos;
-	pos.type = 0;
+	pos.type = POSITION_NOT_FROM_TOKEN;
 	pos.stream = stream->nr;
 	pos.newline = stream->newline;
 	pos.whitespace = stream->whitespace;
@@ -126,13 +126,13 @@ const char *show_token(const struct token *token)
 
 	if (!token)
 		return "<no token>";
+	if (eof_token(token))
+		return "end-of-input";
+
 	switch (token_type(token)) {
 	case TOKEN_ERROR:
 		return "syntax error";
 
-	case TOKEN_EOF:
-		return "end-of-input";

[PATCH 4/4] more transparent dealing with tokens/position

[Jan Pokorný]

Unify usage of `token_type' and `eof_token' also with gdbhelpers.
When 3/4 is applied, no need to do explicit cast to `enum token_type'.

Tested to work as it used to*).

*) testing commands sequence:
$ gdb --args ./sparse compat/mmap-blob.c
(gdb) source gdbhelpers
(gdb) b lib.c:882
(gdb) r
(gdb) gdb_show_tokens token

Signed-off-by: Jan Pokorny <pokorny_jan@seznam.cz>
---
 gdbhelpers |   10 +++++-----
 1 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/gdbhelpers b/gdbhelpers
index 8634786..9a3630d 100644
--- a/gdbhelpers
+++ b/gdbhelpers
@@ -259,7 +259,7 @@ end
 
 define gdb_show_token
 	printf "%p: '%s', type = ", $arg0, show_token($arg0)
-	output (enum token_type) ($arg0)->pos.type
+	output token_type($arg0)
 	printf "\n"
 
 	if (! $showing_token)
@@ -268,18 +268,18 @@ define gdb_show_token
 
 		set $token = $arg0
 
-		if ($token->pos.type == TOKEN_IDENT)
+		if (token_type($token) == TOKEN_IDENT)
 			gdb_tabs
 			printf "ident = "
 			gdb_show_ident $token.ident
 		end
 
-		if ($token->pos.type == TOKEN_MACRO_ARGUMENT)
+		if (token_type($token) == TOKEN_MACRO_ARGUMENT)
 			gdb_tabs
 			printf "argnum = %d\n", $token->argnum
 		end
 
-		if ($token->pos.type == TOKEN_SPECIAL)
+		if (token_type($token) == TOKEN_SPECIAL)
 			gdb_tabs
 			printf "special = \"%s\"\n", show_special($token.special)
 		end
@@ -294,7 +294,7 @@ define gdb_show_tokens
 	set $t = $arg0
 	printf "{\n"
 	set $ntabs++
-	while ($t != &eof_token_entry)
+	while (!eof_token($t))
 		gdb_tabs
 		printf "token = "
 		gdb_show_token($t)
-- 
1.7.1

Re: [PATCH 2/4] more transparent dealing with tokens/position

[Jan Pokorný]

Unify usage of `eof_token', version with respective asserts,
mainly intended for convenient testing.

Signed-off-by: Jan Pokorny <pokorny_jan@seznam.cz>
---
 pre-process.c |    5 ++++-
 1 files changed, 4 insertions(+), 1 deletions(-)

diff --git a/pre-process.c b/pre-process.c
index 603cc00..cee3d29 100644
--- a/pre-process.c
+++ b/pre-process.c
@@ -19,6 +19,7 @@
 #include <fcntl.h>
 #include <limits.h>
 #include <time.h>
+#include <assert.h>
 
 #include "lib.h"
 #include "allocate.h"
@@ -794,11 +795,13 @@ out:
 
 static int free_preprocessor_line(struct token *token)
 {
-	while (token_type(token) != TOKEN_EOF) {
+	while (!eof_token(token)) {
+		assert(token_type(token) != TOKEN_EOF);
 		struct token *free = token;
 		token = token->next;
 		__free_token(free);
 	};
+	assert(token_type(token) == TOKEN_EOF);
 	return 1;
 }
 
-- 
1.7.1