Re: PROBLEM: Linux kernel 2.6.31 IPv4 TCP fails to open huge amount of outgoing connections (unable to bind ... )

[Eric Dumazet <eric.dumazet@gmail.com>]

Le mercredi 21 avril 2010 à 12:25 +0400, Evgeniy Polyakov a écrit :

> I believe this is a useful patch, but it addresses a different issue.
> This path should not fire up when we bind to single address.

Well, the real problem is that following sequence can happen :

socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 5
setsockopt(5, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
bind(5, {sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("127.0.0.2")}, 16) = 0
getsockname(5, {sa_family=AF_INET, sin_port=htons(34000), sin_addr=inet_addr("127.0.0.2")}, [16]) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 6
setsockopt(6, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
bind(6, {sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("127.0.0.2")}, 16) = 0
getsockname(6, {sa_family=AF_INET, sin_port=htons(34002), sin_addr=inet_addr("127.0.0.2")}, [16]) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 7
setsockopt(7, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
bind(7, {sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("127.0.0.2")}, 16) = 0
getsockname(7, {sa_family=AF_INET, sin_port=htons(34001), sin_addr=inet_addr("127.0.0.2")}, [16]) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 8
setsockopt(8, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
bind(8, {sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("127.0.0.2")}, 16) = 0
getsockname(8, {sa_family=AF_INET, sin_port=htons(34002), sin_addr=inet_addr("127.0.0.2")}, [16]) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 9
setsockopt(9, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
bind(9, {sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("127.0.0.2")}, 16) = 0
getsockname(9, {sa_family=AF_INET, sin_port=htons(34000), sin_addr=inet_addr("127.0.0.2")}, [16]) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 10
setsockopt(10, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
bind(10, {sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("127.0.0.2")}, 16) = 0
getsockname(10, {sa_family=AF_INET, sin_port=htons(34002), sin_addr=inet_addr("127.0.0.2")}, [16]) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 11
setsockopt(11, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
bind(11, {sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("127.0.0.2")}, 16) = 0
getsockname(11, {sa_family=AF_INET, sin_port=htons(34001), sin_addr=inet_addr("127.0.0.2")}, [16]) = 0


Note ports are given several times for different sockets.

So several sockets are 'bound' to same IP:port values

At connect() time, we refuse and say address is not available.



Following program to demonstrate the problem.

First time, launch it with an extra agument to setup ip aliases and ip_local_port_range




#include <stdio.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <string.h>
#include <errno.h>

int listenfd;
int on = 1;

void listener()
{
	if (fork())
		return;

	while (1) {
		struct sockaddr_in addr;
		socklen_t len = sizeof(addr);
		int fd = accept(listenfd, (struct sockaddr *)&addr, &len);
	}
}

int main(int argc, char *argv[])
{
	int i, port, total = 0;
	char cmd[128];
	struct sockaddr_in addr;
	socklen_t len;

	if (argc > 1) {
		for (i = 2; i < 8; i++) {
			sprintf(cmd, "ip addr add 127.0.0.%d/8 dev lo 2>/dev/null", i);
			system(cmd);
		}
		system("echo '34000 34002' >/proc/sys/net/ipv4/ip_local_port_range");
	}
	listenfd = socket(AF_INET, SOCK_STREAM, 0);
	setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
	memset(&addr, 0, sizeof(addr));
	addr.sin_family = AF_INET;
	addr.sin_port = htons(4444);
	if (bind(listenfd, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
		perror("bind");
		return 1;
	}
	listen(listenfd, 10);
	listener();

	for (i = 2; i < 8; i++) {
		for (port = 34000; port < 34010; port++) {
			int fd = socket(AF_INET, SOCK_STREAM, 0);
			if (fd == -1) {
				fprintf(stderr, "Could not open socket, errno=%d\n", errno);
				goto end;
			}
			setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
			addr.sin_addr.s_addr = htonl(0x7f000000 + i);
//			addr.sin_port = htons(port);
			addr.sin_port = 0;
			if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
				fprintf(stderr, "Could not bind()\n");
				goto end;
			}
			len = sizeof(addr);
			getsockname(fd, (struct sockaddr *)&addr, &len);
#if 0
			addr.sin_addr.s_addr = htonl(0x7f000001);
			addr.sin_port = htons(4444);
			if ((total < 10) && (connect(fd, (struct sockaddr *)&addr, sizeof(addr)) == -1)) {
				len = sizeof(addr);
				getsockname(fd, (struct sockaddr *)&addr, &len);
				fprintf(stderr, "Could not connect()\n");
				goto end;
			}
#endif
			total++;
		}
	}
end:
	printf("i=127.0.0.%d port=%d (total=%d)\n", i, port, total);
	pause();
}