Expected behavior of shutdown() in multi-threaded socket programming

Expected behavior of shutdown() in multi-threaded socket programming

[Nick Pelly]

[-- Attachment #1: Type: text/plain, Size: 2046 bytes --]

Hi,

I am interested in the expected behavior of shutdown() on a socket
that is also blocked on connect(), accept(), read(), write(), poll()
or select() in another thread.

For example:

THREAD 1
fd = socket()
listen(fd)
bind(fd)
accept(fd)  <--- blocks

THREAD 2
shutdown(fd)  <--- what is meant to happen to accept() in thread 1?

If thread 2 is run after thread 1, what should happen to the blocked
accept() call when shutdown() is called in thread 2?


My observations are that
TCP sockets: accept() immediately returns with errno EINVAL
unix domain sockets: accept() immediately returns with return errno EINVAL
RFCOMM sockets: accept() continues to block
L2CAP sockets: accept() continues to block

I tested on 2.6.18, 2.6.28 and 2.6.29 and results were the same.

Included is a sample program sock_shutdown_test.c that can easily
exhibit this behavior. For example
sock_shutdown_test unix accept_shutdown  # accept() returns
sock_shutdown_test tcp accept_shutdown   # accept() returns
sock_shutdown_test rfcomm accept_shutdown   # accept() blocks forever
sock_shutdown_test l2cap accept_shutdown   # accept() blocks forever

I also have similar results for other blocking syscalls such as
connect(), read(), write(), poll() etc, but the test program is not as
simple.

So my question is: What is the correct behavior for sockets here?


It is desirable for Android that shutdown() to force other threads
blocked on that socket to return. In fact, if they don't it makes
multi-threaded socket programming very hard since there is no other
simple way to abort a blocked I/O operation. We have to resort to
using poll() in combination with a selfpipe that we can write a byte
to in order to abort the poll(). But this is quite inefficient as it
triples the number of fd's needed for every socket not to mention the
4k buffer space needed in the kernel for the selfpipe. A global
selfpipe per process is an improvement but it is quite messy getting
this correct without race conditions.


Input as to the correct behavior is appreciated,
Nick

[-- Attachment #2: sock_shutdown_test.c --]
[-- Type: text/x-csrc, Size: 6870 bytes --]

/** testing behavior of shutdown() */

#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <sys/uio.h>
#include <unistd.h>

#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <unistd.h>
#include <sys/socket.h>
#include <sys/ioctl.h>
#include <sys/poll.h>
#include <sys/un.h>
#include <netinet/in.h>

#include <bluetooth/bluetooth.h>
#include <bluetooth/rfcomm.h>
#include <bluetooth/sco.h>
#include <bluetooth/l2cap.h>

enum sock_type {
    UNIX = 0,
    RFCOMM,
    SCO,
    L2CAP,
    TCP,
};

struct thread_args {
    int fd;
    int type;
    int delay;
};

struct sockaddr_un  local_addr_un  = {AF_UNIX, "/tmp/foo"};
struct sockaddr_rc  local_addr_rc  = {AF_BLUETOOTH, *BDADDR_ANY, 4};
struct sockaddr_sco local_addr_sco = {AF_BLUETOOTH, *BDADDR_LOCAL};
struct sockaddr_l2  local_addr_l2  = {AF_BLUETOOTH, htobs(0x1001), *BDADDR_ANY, 0};
struct sockaddr_in  local_addr_in  = {AF_INET, 9999, {0}, {0}};

struct sockaddr_un  remote_addr_un  ;
struct sockaddr_rc  remote_addr_rc  ;
struct sockaddr_sco remote_addr_sco ;
struct sockaddr_l2  remote_addr_l2  ;
struct sockaddr_in  remote_addr_in  ;

static int _socket(int type) {
    int ret;
    int family = -1;
    int typ = -1;
    int protocol = -1;

    switch (type) {
    case UNIX:
        family = PF_UNIX;
        typ = SOCK_STREAM;
        protocol = 0;
        break;
    case RFCOMM:
        family = PF_BLUETOOTH;
        typ = SOCK_STREAM;
        protocol = BTPROTO_RFCOMM;
        break;
    case SCO:
        family = PF_BLUETOOTH;
        typ = SOCK_SEQPACKET;
        protocol = BTPROTO_SCO;
        break;
    case L2CAP:
        family = PF_BLUETOOTH;
        typ = SOCK_SEQPACKET;
        protocol = BTPROTO_L2CAP;
        break;
    case TCP:
        family = PF_INET;
        typ = SOCK_STREAM;
        protocol = 0;
        break;
    }

    printf("%d: socket()\n", gettid());
    ret = socket(family, typ, protocol);
    printf("%d: socket() = %d\n", gettid(), ret);
    if (ret < 0) printf("\terr %d (%s)\n", errno, strerror(errno));

    return ret;
}

static int _close(int fd) {
    int ret;

    printf("%d: close(%d)\n", gettid(), fd);
    ret = close(fd);
    printf("%d: close(%d) = %d\n", gettid(), fd, ret);
    if (ret < 0) printf("\terr %d (%s)\n", errno, strerror(errno));

    return ret;
}

static int _bind(int fd, int type) {
    int len = 0;
    int ret;
    struct sockaddr *addr = NULL;

    switch (type) {
    case UNIX:
        unlink(local_addr_un.sun_path);
        addr = (struct sockaddr *) &local_addr_un;
        len = sizeof(local_addr_un);
        break;
    case RFCOMM:
        addr = (struct sockaddr *) &local_addr_rc;
        len = sizeof(local_addr_rc);
        break;
    case SCO:
        addr = (struct sockaddr *) &local_addr_sco;
        len = sizeof(local_addr_sco);
        break;
    case L2CAP:
        addr = (struct sockaddr *) &local_addr_l2;
        len = sizeof(local_addr_l2);
        break;
    case TCP:
        addr = (struct sockaddr *) &local_addr_in;
        len = sizeof(local_addr_in);
        break;
    }

    printf("%d: bind(%d)\n", gettid(), fd);
    ret = bind(fd, addr, len);
    printf("%d: bind(%d) = %d\n", gettid(), fd, ret);
    if (ret < 0) printf("\terr %d (%s)\n", errno, strerror(errno));

    return ret;
}

static int _listen(int fd, int type) {
    int ret;

    printf("%d: listen(%d)\n", gettid(), fd);
    ret = listen(fd, 1);
    printf("%d: listen(%d) = %d\n", gettid(), fd, ret);
    if (ret < 0) printf("\terr %d (%s)\n", errno, strerror(errno));

    return ret;
}

static int _accept(int fd, int type) {
    int ret;
    int len;
    struct sockaddr *addr = NULL;

    switch (type) {
    case UNIX:
        addr = (struct sockaddr *) &remote_addr_un;
        len = sizeof(remote_addr_un);
        break;
    case RFCOMM:
        addr = (struct sockaddr *) &remote_addr_rc;
        len = sizeof(remote_addr_rc);
        break;
    case SCO:
        addr = (struct sockaddr *) &remote_addr_sco;
        len = sizeof(remote_addr_sco);
        break;
    case L2CAP:
        addr = (struct sockaddr *) &remote_addr_l2;
        len = sizeof(remote_addr_l2);
        break;
    case TCP:
        addr = (struct sockaddr *) &remote_addr_in;
        len = sizeof(remote_addr_in);
        break;
    }

    printf("%d: accept(%d)\n", gettid(), fd);
    ret = accept(fd, addr, &len);
    printf("%d: accept(%d) = %d\n", gettid(), fd, ret);
    if (ret < 0) printf("\terr %d (%s)\n", errno, strerror(errno));
    else {
        printf("\tlen = %d\n", len);
    }

    return ret;
}

static int _shutdown(int fd, int how) {
    int ret;

    printf("%d: shutdown(%d)\n", gettid(), fd);
    ret = shutdown(fd, how);
    printf("%d: shutdown(%d) = %d\n", gettid(), fd, ret);
    if (ret < 0) printf("\terr %d (%s)\n", errno, strerror(errno));

    return ret;
}

static void thread_accept(struct thread_args *args) {
    printf("%d: START\n", gettid());
    sleep(args->delay);
    _accept(args->fd, args->type);
    printf("%d: END\n", gettid());
}

static int do_accept_shutdown(int type) {
    int fd;
    pthread_t thread;
    struct thread_args args = {-1, type, 0};

    fd = _socket(type);
    if (fd < 0) goto error;

    if (_bind(fd, type) < 0) goto error;

    if (_listen(fd, type) < 0) goto error;

    args.fd = fd;
    pthread_create(&thread, NULL, (void *)thread_accept, (void *)&args);

    sleep(2);
    _shutdown(fd, SHUT_RDWR);

    pthread_join(thread, NULL);

    _close(fd);

    return 0;

error:
    return -1;
}

struct {
    char *name;
    int (*ptr)(int);
} action_table[]  = {
    {"accept_shutdown", do_accept_shutdown},
    {NULL, NULL},
};

struct {
    char *name;
    enum sock_type type;
} type_table[]  = {
    {"unix", UNIX},
    {"rfcomm", RFCOMM},
    {"sco", SCO},
    {"l2cap", L2CAP},
    {"tcp", TCP},
    {NULL, -1},
};

static void usage() {
    int i;

    printf("sock_shutdown_test TYPE ACTION\n");
    printf("\nTYPE:\n");
    for (i = 0; type_table[i].name; i++) {
        printf("\t%s\n", type_table[i].name);
    }
    printf("\nACTION:\n");
    for (i = 0; action_table[i].name; i++) {
        printf("\t%s\n", action_table[i].name);
    }
}

int main(int argc, char **argv) {
    int i;
    int type = -1;

    if (argc != 3) {
        usage();
        return -1;
    }
    for (i = 0; type_table[i].name; i++) {
        if (!strcmp(argv[1], type_table[i].name)) {
            type = type_table[i].type;
            break;
        }
    }
    if (type == -1) {
        usage();
        return -1;
    }
    for (i = 0; action_table[i].name; i++) {
        if (!strcmp(argv[2], action_table[i].name)) {
            printf("TYPE = %s ACTION = %s\n", type_table[type].name,
                    action_table[i].name);
            return (*action_table[i].ptr)(type);
        }
    }
    usage();
    return -1;
}

Re: Expected behavior of shutdown() in multi-threaded socket programming

[Nick Pelly]

Any comments on this one? I would like to correct the behavior of
shutdown() on AF_BLUETOOTH sockets, but I have been advised by Marcel
Holtmann that we need to agree on the correct behavior first.

How should shutdown() behave when other threads are blocked on the same socket?

More detail in the original mail below.

Thanks,
Nick

On Tue, May 19, 2009 at 2:29 PM, Nick Pelly<npelly@google.com> wrote:
> Hi,
>
> I am interested in the expected behavior of shutdown() on a socket
> that is also blocked on connect(), accept(), read(), write(), poll()
> or select() in another thread.
>
> For example:
>
> THREAD 1
> fd = socket()
> listen(fd)
> bind(fd)
> accept(fd)  <--- blocks
>
> THREAD 2
> shutdown(fd)  <--- what is meant to happen to accept() in thread 1?
>
> If thread 2 is run after thread 1, what should happen to the blocked
> accept() call when shutdown() is called in thread 2?
>
>
> My observations are that
> TCP sockets: accept() immediately returns with errno EINVAL
> unix domain sockets: accept() immediately returns with return errno EINVAL
> RFCOMM sockets: accept() continues to block
> L2CAP sockets: accept() continues to block
>
> I tested on 2.6.18, 2.6.28 and 2.6.29 and results were the same.
>
> Included is a sample program sock_shutdown_test.c that can easily
> exhibit this behavior. For example
> sock_shutdown_test unix accept_shutdown  # accept() returns
> sock_shutdown_test tcp accept_shutdown   # accept() returns
> sock_shutdown_test rfcomm accept_shutdown   # accept() blocks forever
> sock_shutdown_test l2cap accept_shutdown   # accept() blocks forever
>
> I also have similar results for other blocking syscalls such as
> connect(), read(), write(), poll() etc, but the test program is not as
> simple.
>
> So my question is: What is the correct behavior for sockets here?
>
>
> It is desirable for Android that shutdown() to force other threads
> blocked on that socket to return. In fact, if they don't it makes
> multi-threaded socket programming very hard since there is no other
> simple way to abort a blocked I/O operation. We have to resort to
> using poll() in combination with a selfpipe that we can write a byte
> to in order to abort the poll(). But this is quite inefficient as it
> triples the number of fd's needed for every socket not to mention the
> 4k buffer space needed in the kernel for the selfpipe. A global
> selfpipe per process is an improvement but it is quite messy getting
> this correct without race conditions.
>
>
> Input as to the correct behavior is appreciated,
> Nick
>

Re: Expected behavior of shutdown() in multi-threaded socket programming

[Iain Hibbert]

On Thu, 11 Jun 2009, Nick Pelly wrote:

> Any comments on this one? I would like to correct the behavior of
> shutdown() on AF_BLUETOOTH sockets, but I have been advised by Marcel
> Holtmann that we need to agree on the correct behavior first.
>
> How should shutdown() behave when other threads are blocked on the same socket?

IMHO consistency should apply.

The opengroup specification for shutdown() says

  "The shutdown() function shall cause all or part of a full-duplex
  connection on the socket associated with the file descriptor socket to
  be shut down."

and while that does not really cover the case when the socket is blocked
in accept(), if all the other socket types abort the block then that is
what the PF_BLUETOOTH sockets should do too.

The opengroup specification for accept() suggests EINVAL would be returned
if the socket was not accepting connections and arguably that is the case
after a shutdown(), though ECONNABORTED could be used too (your program
displays ECONNABORTED on NetBSD for instance)

> I also have similar results for other blocking syscalls such as
> connect(), read(), write(), poll() etc, but the test program is not as
> simple.

They should all handle the shutdown().

iain

Re: Expected behavior of shutdown() in multi-threaded socket programming

[Nick Pelly]

On Fri, Jun 12, 2009 at 1:46 AM, Iain Hibbert<plunky@rya-online.net> wrote:
> On Thu, 11 Jun 2009, Nick Pelly wrote:
>
>> Any comments on this one? I would like to correct the behavior of
>> shutdown() on AF_BLUETOOTH sockets, but I have been advised by Marcel
>> Holtmann that we need to agree on the correct behavior first.
>>
>> How should shutdown() behave when other threads are blocked on the same =
socket?
>
> IMHO consistency should apply.
>
> The opengroup specification for shutdown() says
>
> =A0"The shutdown() function shall cause all or part of a full-duplex
> =A0connection on the socket associated with the file descriptor socket to
> =A0be shut down."
>
> and while that does not really cover the case when the socket is blocked
> in accept(), if all the other socket types abort the block then that is
> what the PF_BLUETOOTH sockets should do too.
>
> The opengroup specification for accept() suggests EINVAL would be returne=
d
> if the socket was not accepting connections and arguably that is the case
> after a shutdown(), though ECONNABORTED could be used too (your program
> displays ECONNABORTED on NetBSD for instance)
>
>> I also have similar results for other blocking syscalls such as
>> connect(), read(), write(), poll() etc, but the test program is not as
>> simple.
>
> They should all handle the shutdown().

Sounds good to me.

Thanks for the input.

Nick