Re: NFS problems with Linux-2.4

[Jim Nance <jlnance@us54.synopsys.com>]

[-- Attachment #1: Type: text/plain, Size: 977 bytes --]

On Sun, May 18, 2003 at 05:00:24PM +0200, Trond Myklebust wrote:
> 
> Sorry. stat doesn't obey close-to-open. It relies on standard
> attribute caching. close-to-open means "open()" (and only "open()")
> checks data cache consistency...

Hi Trond,
    I rewrote my test program so that it uses open() instead of stat().
I also changed it so that it does not rename the file after it writing
it.  This should only leave close, open, and unlink calls.  The program
still fails for me after running for a minute or so:

  cayman> ./p1 s
  Failed to find #0 which client wrote
  Failed on file number 10202

Again, this is with 2.4.20 kernel.  It fails much faster with a 2.4.7.

Thanks,

Jim

-- 
----------------------------------------------------------------------------
Jim Nance                                                           Synopsys
(919) 425-7219  Do you have sweet iced tea?        jlnance@us54.synopsys.com
                No, but there's sugar on the table.

[-- Attachment #2: p1.c --]
[-- Type: text/plain, Size: 4809 bytes --]

/* This program demonstrates a problem with the close/open consistency
 * of NFS file systems under Linux.  It fails very rapidy with Red Hats
 * 2.4.7-10smp kernel.  This kernel was known to have bugs.  It also fails
 * with Red Hats 2.4.20-13.7bigmem kernel, which was thought to have this
 * bug fixed.  For my testcase both linux machines were talking to a
 * network applicance file server and mounted like this:
 *
 * na1-rtp:/vol/vol0/home/jlnance /home/jlnance nfs rw,v3,rsize=4096,\
 * wsize=4096,hard,intr,udp,lock,addr=na1-rtp 0 0
 *
 * This program needs to be run on 2 machines, assume hostnames A & B.
 * A and B need to share an NFS mounted file system.
 *
 * On machine A:
 *   cd /some/nfs/path/common/to/both
 *   ./p1 s
 *
 * On machine B:
 *   cd /some/nfs/path/common/to/both
 *   ./p1 c A
 *
 * After a while you may see output similar to:
 *   cayman> ./p1 s
 *   Failed to find #0 which client wrote
 *   Failed on file number 483
 */


#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <netdb.h>

#define PORT 12387
#define FLEN 16

void die()
{
  perror("");
  exit(-1);
}

void Write(int fd, char *buff, size_t len)
{
  for(;;) {
    int nsent=write(fd, buff, len);
    if(nsent==0)
      exit(0);
    if(nsent==-1) {
      if(errno!=EINTR)
        die();
    } else {
      buff += nsent;
      len  -= nsent;
      if(len==0) {
        return;
      }
    }
  }
}

void Read(int fd, char *buff, size_t len)
{
  for(;;) {
    int nread=read(fd, buff, len);
    if(nread==0)
      exit(0);
    if(nread==-1) {
      if(errno!=EINTR)
        die();
    } else {
      buff += nread;
      len  -= nread;
      if(len==0) {
        return;
      }
    }
  }
}

int server()
{
  int sock = socket(AF_INET, SOCK_STREAM, 0);
  if(sock==-1) die(); else {
    struct sockaddr_in name;
    int                on = 1;
    name.sin_family       = AF_INET;
    name.sin_addr.s_addr  = htonl(INADDR_ANY);
    name.sin_port         = htons(PORT);

    setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &on, sizeof on);
    if(bind(sock, (struct sockaddr*)&name, sizeof(name))==-1) die(); else {
      if(listen(sock, 1)==-1) die(); else {
        int tsock = accept(sock, 0, 0);
        if(tsock!=-1) {
          int cnt;

          for(cnt=0; cnt<100000; cnt++) {
            int  fd;
            char dummy;
            char number[FLEN];
            struct stat sbuf;
            /*sprintf(number, "#%d", cnt);*/
            sprintf(number, "#%d", 0);
            Write(tsock, number, sizeof(number));
            Read(tsock, &dummy, 1);
#if 0
            if(stat(number, &sbuf)) {
              fprintf(stderr, "Failed to find %s which client wrote\n", number);
              fprintf(stderr, "Failed on file number %d\n", cnt);
              exit(-2);
            }
#else
            fd = open(number, O_RDONLY, 0);
            if(fd<0) {
              fprintf(stderr, "Failed to find %s which client wrote\n", number);
              fprintf(stderr, "Failed on file number %d\n", cnt);
              exit(-2);
            }
#endif
            close(fd);
            unlink(number);
          }
        }
      }
    }
  }

  return 0;
}

int client(char *server)
{
  struct hostent *info = gethostbyname(server);
  if(!info) die(); else {
    int rsocket = socket(AF_INET, SOCK_STREAM, 0);
    if(rsocket==-1) die(); else {
      struct sockaddr_in name;
      name.sin_family = AF_INET;
      name.sin_port   = htons(PORT);
      memcpy(&name.sin_addr, info->h_addr_list[0], sizeof(struct in_addr));
      if(connect(rsocket, (struct sockaddr*)&name, sizeof(name))==-1)
        die();
      else {
        for(;;) {
          int  fd;
          char fname[FLEN];
          char tname[FLEN+8];

          Read(rsocket, fname, sizeof(fname));
          strcpy(tname, fname);
          /*strcat(tname, ".tmp");*/

          fd = open(tname, O_WRONLY|O_CREAT, 0600);
          if(fd==-1) die();

          Write(fd, fname, sizeof(fname)); /* Junk data */
          close(fd);

          /*rename(tname, fname);*/

          Write(rsocket, fname, 1); /* Tells the server we are done */
        }
      }
    }
  }

  return 0;
}

void usage(char *prog)
{
  fprintf(stderr, "Usage:\n");
  fprintf(stderr, " %s s\n", prog);
  fprintf(stderr, " %s c servername\n", prog);
  fprintf(stderr, " Run 1 of each in the same NFS directory on 2 different "
        "machines\n Two processes total\n");
  exit(-1);
}

int main(int ac, char **av)
{
  if(ac<2) {
    usage(av[0]);
  } if(av[1][0]=='s') {
    return server();
  }else if(ac<3) {
    usage(av[0]);
  } else if(av[1][0]=='c') {
    return client(av[2]);
  } else {
    usage(av[0]);
  }

  return -1;
}