[PATCH 30/73] cse, canon: Cast pointers to uintptr_t before relational comparison

[Bart Van Assche <bvanassche@acm.org>]

Cast pointer operands to uintptr_t before comparing them with relational
operators (<) in cse.c, simplify.c, and storage.c.

Avoid undefined behavior under ISO C when comparing pointers that do not
point to elements of the same object, and prevent AddressSanitizer
invalid-pointer-pair aborts under detect_invalid_pointer_pairs=2.

Fixes: fe8df476cce2 ("Add initial CSE pass")
Fixes: 37a29afc774f ("Start splitting out generic "storage"...")
Fixes: 84f84262a122 ("canon: simplify calculation of canonical order")
Signed-off-by: Bart Van Assche <bvanassche@acm.org>
---
 cse.c      | 17 +++++++++--------
 simplify.c |  3 ++-
 storage.c  |  5 +++--
 3 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/cse.c b/cse.c
index b59581814ab7..b7d0314a0400 100644
--- a/cse.c
+++ b/cse.c
@@ -10,6 +10,7 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <stddef.h>
+#include <stdint.h>
 #include <assert.h>
 
 #include "parse.h"
@@ -28,9 +29,9 @@ static int phi_compare(pseudo_t phi1, pseudo_t phi2)
 	const struct instruction *def2 = phi2->def;
 
 	if (def1->src1 != def2->src1)
-		return def1->src1 < def2->src1 ? -1 : 1;
+		return (uintptr_t)def1->src1 < (uintptr_t)def2->src1 ? -1 : 1;
 	if (def1->bb != def2->bb)
-		return def1->bb < def2->bb ? -1 : 1;
+		return (uintptr_t)def1->bb < (uintptr_t)def2->bb ? -1 : 1;
 	return 0;
 }
 
@@ -184,7 +185,7 @@ static int insn_compare(const void *_i1, const void *_i2)
 
 	case OP_SEL:
 		if (i1->src3 != i2->src3)
-			return i1->src3 < i2->src3 ? -1 : 1;
+			return (uintptr_t)i1->src3 < (uintptr_t)i2->src3 ? -1 : 1;
 		/* Fall-through to binops */
 
 	/* Binary arithmetic */
@@ -208,7 +209,7 @@ static int insn_compare(const void *_i1, const void *_i2)
 	case OP_FDIV:
 	case_binops:
 		if (i1->src2 != i2->src2)
-			return i1->src2 < i2->src2 ? -1 : 1;
+			return (uintptr_t)i1->src2 < (uintptr_t)i2->src2 ? -1 : 1;
 		/* Fall through to unops */
 
 	/* Unary */
@@ -216,17 +217,17 @@ static int insn_compare(const void *_i1, const void *_i2)
 	case OP_FNEG:
 	case OP_SYMADDR:
 		if (i1->src1 != i2->src1)
-			return i1->src1 < i2->src1 ? -1 : 1;
+			return (uintptr_t)i1->src1 < (uintptr_t)i2->src1 ? -1 : 1;
 		break;
 
 	case OP_LABEL:
 		if (i1->bb_true != i2->bb_true)
-			return i1->bb_true < i2->bb_true ? -1 : 1;
+			return (uintptr_t)i1->bb_true < (uintptr_t)i2->bb_true ? -1 : 1;
 		break;
 
 	case OP_SETVAL:
 		if (i1->val != i2->val)
-			return i1->val < i2->val ? -1 : 1;
+			return (uintptr_t)i1->val < (uintptr_t)i2->val ? -1 : 1;
 		break;
 
 	case OP_SETFVAL:
@@ -244,7 +245,7 @@ static int insn_compare(const void *_i1, const void *_i2)
 	case OP_PTRCAST:
 	case OP_UTPTR: case OP_PTRTU:
 		if (i1->src != i2->src)
-			return i1->src < i2->src ? -1 : 1;
+			return (uintptr_t)i1->src < (uintptr_t)i2->src ? -1 : 1;
 
 		// Note: if it can be guaranted that identical ->src
 		// implies identical orig_type->bit_size, then this
diff --git a/simplify.c b/simplify.c
index 2ddaf1c619a8..88f261da297e 100644
--- a/simplify.c
+++ b/simplify.c
@@ -40,6 +40,7 @@
 // * `SHIFT(x, S)` is used for `LSR(x, S)` or `SHL(x, S)`.
 
 #include <assert.h>
+#include <stdint.h>
 
 #include "parse.h"
 #include "expression.h"
@@ -1814,7 +1815,7 @@ static int canonical_order(pseudo_t p1, pseudo_t p2)
 		return 0;
 	switch (t1) {
 	case PSEUDO_SYM:
-		return p1->sym <= p2->sym;
+		return (uintptr_t)p1->sym <= (uintptr_t)p2->sym;
 	case PSEUDO_REG:
 	case PSEUDO_ARG:
 		return p1->nr <= p2->nr;
diff --git a/storage.c b/storage.c
index 160d424ee1f6..1dafa08dccf0 100644
--- a/storage.c
+++ b/storage.c
@@ -8,6 +8,7 @@
  */
 #include <stdio.h>
 #include <stdlib.h>
+#include <stdint.h>
 #include <assert.h>
 
 #include "symbol.h"
@@ -33,7 +34,7 @@ static int hash_list_cmp(const void *_a, const void *_b)
 	const struct storage_hash *a = _a;
 	const struct storage_hash *b = _b;
 	if (a->pseudo != b->pseudo)
-		return a->pseudo < b->pseudo ? -1 : 1;
+		return (uintptr_t)a->pseudo < (uintptr_t)b->pseudo ? -1 : 1;
 	return 0;
 }
 
@@ -118,7 +119,7 @@ static int storage_hash_cmp(const void *_a, const void *_b)
 	struct storage *bb = b->storage;
 
 	if (a->bb != b->bb)
-		return a->bb < b->bb ? -1 : 1;
+		return (uintptr_t)a->bb < (uintptr_t)b->bb ? -1 : 1;
 	if (a->inout != b->inout)
 		return a->inout < b->inout ? -1 : 1;
 	if (aa->type != bb->type)