Re: [PATCH 2/4] tools/nolibc: stdlib: avoid signed overflow in abs() and friends

[David Laight <david.laight.linux@gmail.com>]

On Sun, 26 Jul 2026 17:13:03 +0700
Ammar Faizi <ammarfaizi2@openresty.com> wrote:

> Negating the smallest negative value of a signed type overflows, which
> is undefined behavior. The selftests are built with:
> 
>     -fsanitize=undefined -fsanitize-trap=all
> 
> so a caller passing INT_MIN does not merely get an unspecified answer,
> it dies (on both x86-64 and i386):
> 
>     A simple test program:
> 
>       printf("x = %d\n", abs(INT_MIN));
> 
>     $ ./ab
>     Illegal instruction (core dumped)
> 
>     (gdb) bt
>     #0  0x0000000000401009 in main ()
>     (gdb) x/6i main
>        0x401000 <main>:     mov    $0x80000000,%eax
>        0x401005 <main+5>:   neg    %eax
>        0x401007 <main+7>:   jno    0x40100b <main+11>
>     => 0x401009 <main+9>:   ud2  
>        0x40100b <main+11>:  push   %rax
>        0x40100c <main+12>:  mov    $0x80000000,%esi
> 
> Negate in the corresponding unsigned type instead. The value still
> cannot be represented in the result type, so the minimum is returned
> unchanged.
> 
> Cc: Yichun Zhang <yichun@openresty.com>
> Cc: Alviro Iskandar Setiawan <alviro.iskandar@gnuweeb.org>
> Fixes: bf5e8a78bede ("tools/nolibc: add abs() and friends")
> Signed-off-by: Ammar Faizi <ammarfaizi2@openresty.com>
> ---
>  tools/include/nolibc/stdlib.h | 12 +++++++++---
>  1 file changed, 9 insertions(+), 3 deletions(-)
> 
> diff --git a/tools/include/nolibc/stdlib.h b/tools/include/nolibc/stdlib.h
> index 1816c2368b68..8d86044f759f 100644
> --- a/tools/include/nolibc/stdlib.h
> +++ b/tools/include/nolibc/stdlib.h
> @@ -32,22 +32,28 @@ static __attribute__((unused)) char itoa_buffer[21];
>   * As much as possible, please keep functions alphabetically sorted.
>   */
>  
> +/*
> + * The absolute value of the smallest negative value is not representable in
> + * the result type. Negate in the unsigned type so that the overflow is
> + * defined and return it unchanged, like the other libcs do.
> + */
> +
>  static __inline__
>  int abs(int j)
>  {
> -	return j >= 0 ? j : -j;
> +	return j >= 0 ? j : (int)-(unsigned int)j;

An alternative expression is -(j + 1) - 1
gcc (and I think clang) optimise it to just -j.

	David


>  }
>  
>  static __inline__
>  long labs(long j)
>  {
> -	return j >= 0 ? j : -j;
> +	return j >= 0 ? j : (long)-(unsigned long)j;
>  }
>  
>  static __inline__
>  long long llabs(long long j)
>  {
> -	return j >= 0 ? j : -j;
> +	return j >= 0 ? j : (long long)-(unsigned long long)j;
>  }
>  
>  /* must be exported, as it's used by libgcc for various divide functions */