* [PATCH pynfs 1/2] rpc: skip secure ports in TIME_WAIT and report connect errors
2026-11-19 21:40 [PATCH pynfs 0/2] rpc: fix two client connection failures Benjamin Coddington
@ 2026-11-19 21:40 ` Benjamin Coddington
2026-11-19 21:41 ` [PATCH pynfs 2/2] rpc: fail pending calls when a connection closes Benjamin Coddington
1 sibling, 0 replies; 3+ messages in thread
From: Benjamin Coddington @ 2026-11-19 21:40 UTC (permalink / raw)
To: calum.mackay; +Cc: linux-nfs
connect() set SO_REUSEADDR on client sockets, so bindsocket() could
bind a secure port whose previous connection to the same server was
still in TIME_WAIT. connect() then failed with EADDRNOTAVAIL, and the
bare except closed the socket and fell through to setblocking() on it,
replacing the real error with EBADF.
Drop SO_REUSEADDR. Linux lets bind() take a port held by a TIME_WAIT
socket only when both sockets set SO_REUSEADDR, so without it bind()
fails with EADDRINUSE and bindsocket() moves on to the next port. If
connect() still fails with EADDRNOTAVAIL on a secure port, retry from
the next port. Raise the last connect error when no attempt succeeds.
Signed-off-by: Benjamin Coddington <bcodding@hammerspace.com>
---
rpc/rpc.py | 38 ++++++++++++++++++++++++--------------
1 file changed, 24 insertions(+), 14 deletions(-)
diff --git a/rpc/rpc.py b/rpc/rpc.py
index 84088f6651a6..4d0cc2d01a20 100644
--- a/rpc/rpc.py
+++ b/rpc/rpc.py
@@ -6,7 +6,7 @@ import struct
import threading
import logging
from collections import deque as Deque
-from errno import EINPROGRESS, EWOULDBLOCK, EADDRINUSE
+from errno import EINPROGRESS, EWOULDBLOCK, EADDRINUSE, EADDRNOTAVAIL
from . import rpc_pack
from .rpc_const import *
@@ -812,21 +812,31 @@ class ConnectionHandler(object):
"""
log_t.info("Called connect(%r)" % (address,))
host, port = address
+ s = None
err = None
for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
af, socktype, proto, cannonname, sa = res
- sock = None
- try:
- s = socket.socket(af, socktype, proto)
- s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- if secure:
- self.bindsocket(s)
- s.connect(sa)
- err = None
+ lport = 1
+ while s is None:
+ try:
+ s = socket.socket(af, socktype, proto)
+ if secure:
+ lport = self.bindsocket(s, lport)
+ s.connect(sa)
+ except OSError as e:
+ err = e
+ if s is not None:
+ s.close()
+ s = None
+ # A local port in TIME_WAIT with this peer can still bind
+ if secure and e.errno == EADDRNOTAVAIL and lport < 1023:
+ lport += 1
+ continue
+ break
+ if s is not None:
break
- except:
- if s is not None:
- s.close()
+ if s is None:
+ raise err or OSError("no addresses for %r" % (host,))
s.setblocking(0)
pipe = RpcPipe(s, self._alarm)
@@ -838,13 +848,13 @@ class ConnectionHandler(object):
return pipe
def bindsocket(self, s, port=1):
- """Scan up through ports, looking for one we can bind to"""
+ """Scan up through ports, returning the first one we can bind to"""
# This is necessary when we need to use a 'secure' port
using = port
while 1:
try:
s.bind(('', using))
- return
+ return using
except OSError as why:
if why.errno == EADDRINUSE:
using += 1
--
2.53.0
^ permalink raw reply related [flat|nested] 3+ messages in thread* [PATCH pynfs 2/2] rpc: fail pending calls when a connection closes
2026-11-19 21:40 [PATCH pynfs 0/2] rpc: fix two client connection failures Benjamin Coddington
2026-11-19 21:40 ` [PATCH pynfs 1/2] rpc: skip secure ports in TIME_WAIT and report connect errors Benjamin Coddington
@ 2026-11-19 21:41 ` Benjamin Coddington
1 sibling, 0 replies; 3+ messages in thread
From: Benjamin Coddington @ 2026-11-19 21:41 UTC (permalink / raw)
To: calum.mackay; +Cc: linux-nfs
When the server closed a connection, _event_close() closed the pipe
but left callers waiting on replies until their timeout. A later call
on that pipe made _buzz_write_ready() add the closed socket's fileno()
of -1 to the write list, and select() raised ValueError, killing the
polling thread. Every call and connect after that hung until timeout.
Add RPCConnectionClosed. On close, mark the pipe inactive and fail its
pending calls with it. send_call() raises it on an inactive pipe, and
_buzz_write_ready() drops records for one. DeferredData.fill() now
ignores fills after the first, so a reply that arrives during close is
not overwritten.
In the polling loop, skip fds closed earlier in the same pass, and log
alarm command errors instead of closing the alarm socket.
Signed-off-by: Benjamin Coddington <bcodding@hammerspace.com>
---
rpc/rpc.py | 42 +++++++++++++++++++++++++++++++++++-------
1 file changed, 35 insertions(+), 7 deletions(-)
diff --git a/rpc/rpc.py b/rpc/rpc.py
index 4d0cc2d01a20..a7a053262d06 100644
--- a/rpc/rpc.py
+++ b/rpc/rpc.py
@@ -34,6 +34,9 @@ class RPCError(Exception):
class RPCTimeout(RPCError):
pass
+class RPCConnectionClosed(RPCError):
+ pass
+
class RPCAcceptError(RPCError):
def __init__(self, a):
self.verf = a.verf
@@ -141,6 +144,7 @@ class DeferredData(object):
"""
def __init__(self, msg=None):
self._filled = threading.Event()
+ self._lock = threading.Lock()
self.data = None
self._exception = None
self.msg = msg # Data that thread calling fill might need
@@ -157,10 +161,14 @@ class DeferredData(object):
"""Fill with data, and flag that this has been done.
Caller should no longer reference the object afterwards.
+ Only the first fill takes effect.
"""
- self._exception = exception
- self.data = data
- self._filled.set()
+ with self._lock:
+ if self._filled.is_set():
+ return
+ self._exception = exception
+ self.data = data
+ self._filled.set()
class Alarm(object):
"""A method of notifying select loop that there is data waiting"""
@@ -370,6 +378,14 @@ class RpcPipe(Pipe):
def is_active(self):
return self._active
+ def fail_pending(self, exc):
+ """Mark pipe closed and fail every call still waiting on a reply."""
+ with self._lock:
+ self._active = False
+ pending = list(self._pending.values())
+ for defer in pending:
+ defer.fill(exception=exc)
+
def listen(self, xid, timeout=None):
"""Wait for a reply to a CALL."""
self._pending[xid].wait(timeout)
@@ -400,7 +416,10 @@ class RpcPipe(Pipe):
msg = rpc_msg(xid, rpc_msg_body(CALL, body))
data = sec.secure_data(cred, data)
# Store info needed be receiving thread to match and verify reply
- self._pending[xid] = DeferredData((cred, sec))
+ with self._lock:
+ if not self._active:
+ raise RPCConnectionClosed("connection closed")
+ self._pending[xid] = DeferredData((cred, sec))
self.rpc_send(msg, data)
return xid
@@ -477,6 +496,9 @@ class ConnectionHandler(object):
def _buzz_write_ready(self, pipe):
"""Pipe has data ready to be sent out"""
+ if not pipe.is_active():
+ # Closed by _event_close(); its fileno() is now -1
+ return
pipe.pop_record(self.wsize)
self.writelist.add(pipe.fileno())
@@ -512,11 +534,16 @@ class ConnectionHandler(object):
log_p.warn(1, "polling error from %i" % fd)
# STUB - now what?
for fd in w:
+ if fd not in self.sockets:
+ continue
try:
self._event_write(fd)
except socket.error as e:
self._event_close(fd)
for fd in r:
+ if fd not in self.sockets:
+ # Closed earlier in this pass
+ continue
if fd in self.listeners:
try:
self._event_connect_incoming(fd)
@@ -529,7 +556,7 @@ class ConnectionHandler(object):
try:
switch[c](data)
except socket.error as e:
- self._event_close(fd)
+ log_p.error("alarm command %i failed: %s" % (c, e))
else:
try:
data = self.sockets[fd].recv_records(self.rsize)
@@ -578,8 +605,9 @@ class ConnectionHandler(object):
self.writelist -= temp
self.readlist -= temp
self.errlist -= temp
- self.sockets[fd].clear_active()
- self.sockets[fd].close()
+ pipe = self.sockets[fd]
+ pipe.fail_pending(RPCConnectionClosed("connection closed"))
+ pipe.close()
del self.sockets[fd]
def _event_write(self, fd):
--
2.53.0
^ permalink raw reply related [flat|nested] 3+ messages in thread