linux-user: array overflow in pselect6 emulation

[Andreas Schwab <schwab@suse.de>]

$ cat select.c
#include <stdlib.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/resource.h>
#include <sys/select.h>
#include <sys/syscall.h>

int
main (int argc, char **argv)
{
  int nfds = (argc > 1 ? atoi (argv[1]) : 1031);
  fd_set *fds = calloc ((nfds + (sizeof (fd_mask) * 8) - 1)
                        / (sizeof (fd_mask) * 8), sizeof (fd_mask));
  setrlimit (RLIMIT_NOFILE,
             &(struct rlimit){ .rlim_cur = nfds, .rlim_max = nfds });
  dup2 (open ("/dev/null", O_RDONLY), nfds - 1);
  FD_SET (nfds - 1, fds);
  syscall (SYS_pselect6, nfds, fds, 0, 0, 0, 0);
}
$ ./select
$ qemu-x86_64 -strace select
25005 brk(NULL) = 0x0000000000403000
25005 mmap(NULL,8192,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0) = 0x00007fe4293b6000
25005 uname(0x7fe429bba380) = 0
25005 access("/etc/ld.so.preload",R_OK) = -1 errno=2 (No such file or directory)
25005 openat(-100,"/etc/ld.so.cache",O_RDONLY|O_CLOEXEC) = 3
25005 fstat(3,0x00007fe429bb9950) = 0
25005 mmap(NULL,249267,PROT_READ,MAP_PRIVATE,3,0) = 0x00007fe429379000
25005 close(3) = 0
25005 openat(-100,"/lib64/libc.so.6",O_RDONLY|O_CLOEXEC) = 3
25005 read(3,0x29bb9b18,832) = 832
25005 fstat(3,0x00007fe429bb99b0) = 0
25005 mmap(NULL,2058296,PROT_EXEC|PROT_READ,MAP_PRIVATE|MAP_DENYWRITE,3,0) = 0x00007fe429182000
25005 mmap(0x00007fe42936a000,45056,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_DENYWRITE|MAP_FIXED,3,0x1e7000) = 0x00007fe42936a000
25005 mmap(0x00007fe429375000,14392,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED,-1,0) = 0x00007fe429375000
25005 close(3) = 0
25005 mmap(NULL,12288,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0) = 0x00007fe42917f000
25005 arch_prctl(4098,140617918707520,140617918709920,34,4294967295,0) = 0
25005 mprotect(0x00007fe42936a000,12288,PROT_READ) = 0
25005 mprotect(0x0000000000401000,4096,PROT_READ) = 0
25005 mprotect(0x000000000002a000,4096,PROT_READ) = 0
25005 munmap(0x00007fe429379000,249267) = 0
25005 brk(NULL) = 0x0000000000403000
25005 brk(0x0000000000424000) = 0x0000000000424000
25005 prlimit64(0,RLIMIT_NOFILE,{rlim_cur=1031,rlim_max=1031},NULL) = 0
25005 openat(-100,"/dev/null",O_RDONLY) = 3
25005 dup2(3,1030) = 1030
25005 pselect6(1031,4207264,0,0,0,0)*** bit out of range 0 - FD_SETSIZE on fd_set ***: terminated
Aborted (core dumped)

-- 
Andreas Schwab, SUSE Labs, schwab@suse.de
GPG Key fingerprint = 0196 BAD8 1CE9 1970 F4BE  1748 E4D4 88E3 0EEA B9D7
"And now for something completely different."