Re: epoll design problems with common fork/exec patterns

Mark Lord
2007-10-29T22:36:34Z

Willy Tarreau wrote:
>> On Sat, 27 Oct 2007, Marc Lehmann wrote:
>>
>>>> Please provide some code to illustrate one exact problem you have.
>>> // assume there is an open epoll set that listens for events on fd 5
>>> if (fork () = 0)
>>> {
>>> close (5);
>>> // fd 5 is now removed from the epoll set of the parent.
>>> _exit (0);
>>> }
..
> from what I understand, Marc is not asking for the code above to remove
> the fd from the epoll set, but he's in fact complaining that he *observed*
> that the fd was removed from the epoll set in the *parent* process when
> the child closes it, which is of course not expected at all. As strange
> as it looks like, this might need investigation. It is possible that there
> is some strange bug somewhere in some kernel versions.
>
> Marc, I think that if you indicate the last kernel version on which you
> observed this and provide a very short and easy reproducer, it would
> help everyone investigating this. Basically something which reports "OK"
> or "KO".

That's how I read it, too.
So basically, a program like this, perhaps.
Except that, here running 2.6.23.1, it works just fine (no removal bug).

#include <sys/epoll.h>
#include <stdio.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/socket.h>

static int del_from_epoll_set (int efd, int fd, const char *msg)
{
struct epoll_event e;

memset(&e, 0, sizeof(e));
e.data.fd = fd;
if (epoll_ctl(efd, EPOLL_CTL_DEL, fd, &e) == -1) {
int err = errno;
fprintf(stderr, "epoll_ctl(DEL) failed (%s): %s\n", msg, strerror(err));
return -1;
}
return 0;
}

static int add_to_epoll_set (int efd, int fd, __uint32_t events, const char *msg)
{
struct epoll_event e;

memset(&e, 0, sizeof(e));
e.events = events;
e.data.fd = fd;
if (epoll_ctl(efd, EPOLL_CTL_ADD, fd, &e) == -1) {
int err = errno;
fprintf(stderr, "epoll_ctl(ADD) failed (%s): %s\n", msg, strerror(err));
return -1;
}
return 0;
}

int main (int argc, char **argv)
{
int efd, sd, fds[2];
pid_t cpid;

if (pipe(fds) == -1) {
perror("pipe()");
exit(1);
}
sd = socket(PF_INET, SOCK_STREAM, 0);
if (sd == -1) {
perror("socket");
exit(1);
}

efd = epoll_create(5);
if (efd == -1) {
perror("epoll_create");
exit(1);
}

if (add_to_epoll_set(efd, fileno(stdin), EPOLLIN, "stdin"))
exit(1);
if (add_to_epoll_set(efd, fileno(stdout), EPOLLOUT, "stdout"))
exit(1);
if (add_to_epoll_set(efd, fds[0], EPOLLIN, "pipe_read"))
exit(1);
if (add_to_epoll_set(efd, fds[1], EPOLLOUT, "pipe_write"))
exit(1);
if (add_to_epoll_set(efd, sd, EPOLLIN|EPOLLOUT, "socket"))
exit(1);

// assume there is an open epoll set that listens for events on fd 5
cpid = fork();
if (cpid == 0) {
close(fileno(stdin));
close(fileno(stdout));
close(fds[0]);
close(fds[1]);
close(sd);
exit(0);
}
waitpid(cpid, NULL, 0);

// now test whether the fd's are still in the epoll set:
add_to_epoll_set(efd, sd, EPOLLIN|EPOLLOUT, "sd");
add_to_epoll_set(efd, fds[0], EPOLLIN, "fds[0]");
add_to_epoll_set(efd, fds[1], EPOLLOUT, "fds[1]");
add_to_epoll_set(efd, fileno(stdin), EPOLLIN, "fileno(stdin)");
add_to_epoll_set(efd, fileno(stdout), EPOLLOUT, "fileno(stdout)");

del_from_epoll_set(efd, sd, "sd");
del_from_epoll_set(efd, fds[0], "fds[0]");
del_from_epoll_set(efd, fds[1], "fds[1]");
del_from_epoll_set(efd, fileno(stdin), "fileno(stdin)");
del_from_epoll_set(efd, fileno(stdout), "fileno(stdout)");

printf("Done.\n");
exit(0);
}