Regression related to ipc shmctl compat

[Kyle Huey <me@kylehuey.com>]

Beginning with 553f770ef71b, the following program fails when compiled
for 32 bit and executed on a 64 bit kernel and succeeds when compiled
for and executed on a 64 bit.  It continues to fail even after
58aff0af7573.  When compiled as 32 bit, an shmctl call fails with
EBADR (see the XXX comment).

The test program is adapted from rr's shm test[0].

#define _GNU_SOURCE 1

#include <assert.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/shm.h>
#include <sys/wait.h>

static uint64_t GUARD_VALUE = 0xdeadbeeff00dbaad;

inline static size_t ceil_page_size(size_t size) {
  size_t page_size = sysconf(_SC_PAGESIZE);
  return (size + page_size - 1) & ~(page_size - 1);
}

/**
 * Allocate 'size' bytes, fill with 'value', place canary value before
 * the allocated block, and put guard pages before and after. Ensure
 * there's a guard page immediately after `size`.
 * This lets us catch cases where too much data is being recorded --- which can
 * cause errors if the recorder tries to read invalid memory.
 */
inline static void* allocate_guard(size_t size, char value) {
  size_t page_size = sysconf(_SC_PAGESIZE);
  size_t map_size = ceil_page_size(size + sizeof(GUARD_VALUE)) + 2 * page_size;
  char* cp = (char*)mmap(NULL, map_size, PROT_READ | PROT_WRITE,
                         MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
  assert(cp != MAP_FAILED);
  /* create guard pages */
  assert(munmap(cp, page_size) == 0);
  assert(munmap(cp + map_size - page_size, page_size) == 0);
  cp = cp + map_size - page_size - size;
  memcpy(cp - sizeof(GUARD_VALUE), &GUARD_VALUE, sizeof(GUARD_VALUE));
  memset(cp, value, size);
  return cp;
}

/**
 * Verify that canary value before the block allocated at 'p'
 * (of size 'size') is still valid.
 */
inline static void verify_guard(__attribute__((unused)) size_t size, void* p) {
  char* cp = (char*)p;
  assert(memcmp(cp - sizeof(GUARD_VALUE), &GUARD_VALUE,
sizeof(GUARD_VALUE)) == 0);
}

/**
 * Verify that canary value before the block allocated at 'p'
 * (of size 'size') is still valid, and free the block.
 */
inline static void free_guard(size_t size, void* p) {
  verify_guard(size, p);
  size_t page_size = sysconf(_SC_PAGESIZE);
  size_t map_size = ceil_page_size(size + sizeof(GUARD_VALUE)) + 2 * page_size;
  char* cp = (char*)p + size + page_size - map_size;
  assert(0 == munmap(cp, map_size - 2 * page_size));
}

#define ALLOCATE_GUARD(p, v) p = allocate_guard(sizeof(*p), v)
#define VERIFY_GUARD(p) verify_guard(sizeof(*p), p)
#define FREE_GUARD(p) free_guard(sizeof(*p), p)

/* Make SIZE not a multiple of the page size, to ensure we handle that case.
   But make sure it's even, since we divide it by two. */
#define SIZE ((int)(16 * page_size) - 10)

static int shmid;

static void before_writing(void) {}
static void after_writing(void) {}

static int run_child(void) {
  int i;
  char* p;
  char* p2;
  pid_t child2;
  int status;
  struct shmid_ds* ds;
  struct shminfo* info;
  struct shm_info* info2;

  size_t page_size = sysconf(_SC_PAGESIZE);

  ALLOCATE_GUARD(ds, 'd');
  assert(0 == shmctl(shmid, IPC_STAT, ds));
  VERIFY_GUARD(ds);
  assert((int)ds->shm_segsz == SIZE);
  assert(ds->shm_cpid == getppid());
  assert(ds->shm_nattch == 0);

  ds->shm_perm.mode = 0660;
  assert(0 == shmctl(shmid, IPC_SET, ds));

  ALLOCATE_GUARD(info, 'i');
  assert(0 <= shmctl(shmid, IPC_INFO, (struct shmid_ds*)info));
  VERIFY_GUARD(info);
  assert(info->shmmin == 1);

  ALLOCATE_GUARD(info2, 'j');
  // XXX: This shmctl call fails with EBADR when compiled with -m32
  assert(0 <= shmctl(shmid, SHM_INFO, (struct shmid_ds*)info2));
  VERIFY_GUARD(info2);
  assert(info2->used_ids > 0);
  assert(info2->used_ids < 1000000);

  p = shmat(shmid, NULL, 0);
  assert(p != (char*)-1);

  before_writing();

  for (i = 0; i < SIZE; ++i) {
    assert(p[i] == 0);
  }
  memset(p, 'r', SIZE / 2);

  after_writing();

  p2 = shmat(shmid, NULL, 0);
  assert(p2 != (char*)-1);
  memset(p + SIZE / 2, 'r', SIZE / 2);
  assert(0 == shmdt(p));
  assert(0 == shmdt(p2));

  assert(p == mmap(p, SIZE, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
  assert(p[0] == 0);

  p = shmat(shmid, p, SHM_REMAP);
  assert(p != (char*)-1);
  for (i = 0; i < SIZE; ++i) {
    assert(p[i] == 'r');
  }

  if ((child2 = fork()) == 0) {
    memset(p, 's', SIZE);
    return 0;
  }
  assert(child2 == waitpid(child2, &status, __WALL));
  assert(0 == status);
  for (i = 0; i < SIZE; ++i) {
    assert(p[i] == 's');
  }

  return 0;
}

int main(void) {
  pid_t child;
  int status;

  size_t page_size = sysconf(_SC_PAGESIZE);

  shmid = shmget(IPC_PRIVATE, SIZE, 0666);
  assert(shmid >= 0);

  if ((child = fork()) == 0) {
    return run_child();
  }

  printf("child %d\n", child);

  assert(child == waitpid(child, &status, __WALL));
  /* delete the shm before testing status, because we want to ensure the
     segment is deleted even if the test failed. */
  assert(0 == shmctl(shmid, IPC_RMID, NULL));
  assert(status == 0);

  printf("EXIT-SUCCESS\n");

  return 0;
}

- Kyle

[0] https://github.com/mozilla/rr/blob/master/src/test/shm.c