[PATCH] RPC: parametrize various transport connect timeouts

Each transport implementation can now set unique bind, connect,
reestablishment, and idle timeout values. These are variables,
allowing the values to be modified dynamically. This permits
exponential backoff of any of these values, for instance.

As an example, we implement exponential backoff for the connection
reestablishment timeout.

Test-plan:
Destructive testing (unplugging the network temporarily). Connectathon
with UDP and TCP.

Signed-off-by: Chuck Lever <[e-mail]>
Signed-off-by: Trond Myklebust <[e-mail]>

fs/nfs/inode.c

···
 	case IPPROTO_TCP:
 		if (!to->to_initval)
 			to->to_initval = 60 * HZ;
-		if (to->to_initval > RPC_MAX_TCP_TIMEOUT)
-			to->to_initval = RPC_MAX_TCP_TIMEOUT;
+		if (to->to_initval > NFS_MAX_TCP_TIMEOUT)
+			to->to_initval = NFS_MAX_TCP_TIMEOUT;
 		to->to_increment = to->to_initval;
 		to->to_maxval = to->to_initval + (to->to_increment * to->to_retries);
 		to->to_exponential = 0;
···
 	default:
 		if (!to->to_initval)
 			to->to_initval = 11 * HZ / 10;
-		if (to->to_initval > RPC_MAX_UDP_TIMEOUT)
-			to->to_initval = RPC_MAX_UDP_TIMEOUT;
-		to->to_maxval = RPC_MAX_UDP_TIMEOUT;
+		if (to->to_initval > NFS_MAX_UDP_TIMEOUT)
+			to->to_initval = NFS_MAX_UDP_TIMEOUT;
+		to->to_maxval = NFS_MAX_UDP_TIMEOUT;
 		to->to_exponential = 1;
 		break;
 	}

include/linux/nfs_fs.h

···
 #define NFS_MAX_FILE_IO_BUFFER_SIZE	32768
 #define NFS_DEF_FILE_IO_BUFFER_SIZE	4096
 
+/* Default timeout values */
+#define NFS_MAX_UDP_TIMEOUT	(60*HZ)
+#define NFS_MAX_TCP_TIMEOUT	(600*HZ)
+
 /*
  * superblock magic number for NFS
  */

include/linux/sunrpc/xprt.h

···
 #define RPC_DEF_SLOT_TABLE	(16U)
 #define RPC_MAX_SLOT_TABLE	(128U)
 
-/* Default timeout values */
-#define RPC_MAX_UDP_TIMEOUT	(60*HZ)
-#define RPC_MAX_TCP_TIMEOUT	(600*HZ)
-
-/*
- * Wait duration for an RPC TCP connection to be established.  Solaris
- * NFS over TCP uses 60 seconds, for example, which is in line with how
- * long a server takes to reboot.
- */
-#define RPC_CONNECT_TIMEOUT	(60*HZ)
-
-/*
- * Delay an arbitrary number of seconds before attempting to reconnect
- * after an error.
- */
-#define RPC_REESTABLISH_TIMEOUT	(15*HZ)
-
-/*
- * RPC transport idle timeout.
- */
-#define RPC_IDLE_DISCONNECT_TIMEOUT	(5*60*HZ)
-
 /*
  * RPC call and reply header size as number of 32bit words (verifier
  * size computed separately)
···
 	/*
 	 * Connection of transports
 	 */
+	unsigned long		connect_timeout,
+				bind_timeout,
+				reestablish_timeout;
 	struct work_struct	connect_worker;
 	unsigned short		port;
+
 	/*
 	 * Disconnection of idle transports
 	 */
 	struct work_struct	task_cleanup;
 	struct timer_list	timer;
-	unsigned long		last_used;
+	unsigned long		last_used,
+				idle_timeout;
 
 	/*
 	 * Send stuff

net/sunrpc/clnt.c

···
 	task->tk_action = call_connect;
 	if (!clnt->cl_port) {
 		task->tk_action = call_bind_status;
-		task->tk_timeout = RPC_CONNECT_TIMEOUT;
+		task->tk_timeout = task->tk_xprt->bind_timeout;
 		rpc_getport(task, clnt);
 	}
 }

net/sunrpc/xprt.c

···
 		if (task->tk_rqstp)
 			task->tk_rqstp->rq_bytes_sent = 0;
 
-		task->tk_timeout = RPC_CONNECT_TIMEOUT;
+		task->tk_timeout = xprt->connect_timeout;
 		rpc_sleep_on(&xprt->pending, task, xprt_connect_status, NULL);
 		xprt->ops->connect(task);
 	}
···
 
 	switch (status) {
 	case -ECONNREFUSED:
-		task->tk_timeout = RPC_REESTABLISH_TIMEOUT;
 		rpc_sleep_on(&xprt->sending, task, NULL, NULL);
 	case -EAGAIN:
 	case -ENOTCONN:
···
 	xprt->last_used = jiffies;
 	if (list_empty(&xprt->recv) && !xprt->shutdown)
 		mod_timer(&xprt->timer,
-				xprt->last_used + RPC_IDLE_DISCONNECT_TIMEOUT);
+				xprt->last_used + xprt->idle_timeout);
 	spin_unlock_bh(&xprt->transport_lock);
 	task->tk_rqstp = NULL;
 	memset(req, 0, sizeof(*req));	/* mark unused */

net/sunrpc/xprtsock.c

···
  */
 #define XS_SENDMSG_RETRY	(10U)
 
+/*
+ * Time out for an RPC UDP socket connect.  UDP socket connects are
+ * synchronous, but we set a timeout anyway in case of resource
+ * exhaustion on the local host.
+ */
+#define XS_UDP_CONN_TO		(5U * HZ)
+
+/*
+ * Wait duration for an RPC TCP connection to be established.  Solaris
+ * NFS over TCP uses 60 seconds, for example, which is in line with how
+ * long a server takes to reboot.
+ */
+#define XS_TCP_CONN_TO		(60U * HZ)
+
+/*
+ * Wait duration for a reply from the RPC portmapper.
+ */
+#define XS_BIND_TO		(60U * HZ)
+
+/*
+ * Delay if a UDP socket connect error occurs.  This is most likely some
+ * kind of resource problem on the local host.
+ */
+#define XS_UDP_REEST_TO		(2U * HZ)
+
+/*
+ * The reestablish timeout allows clients to delay for a bit before attempting
+ * to reconnect to a server that just dropped our connection.
+ *
+ * We implement an exponential backoff when trying to reestablish a TCP
+ * transport connection with the server.  Some servers like to drop a TCP
+ * connection when they are overworked, so we start with a short timeout and
+ * increase over time if the server is down or not responding.
+ */
+#define XS_TCP_INIT_REEST_TO	(3U * HZ)
+#define XS_TCP_MAX_REEST_TO	(5U * 60 * HZ)
+
+/*
+ * TCP idle timeout; client drops the transport socket if it is idle
+ * for this long.  Note that we also timeout UDP sockets to prevent
+ * holding port numbers when there is no RPC traffic.
+ */
+#define XS_IDLE_DISC_TO		(5U * 60 * HZ)
+
 #ifdef RPC_DEBUG
 # undef  RPC_DEBUG_DATA
 # define RPCDBG_FACILITY	RPCDBG_TRANS
···
 			xprt->tcp_reclen = 0;
 			xprt->tcp_copied = 0;
 			xprt->tcp_flags = XPRT_COPY_RECM | XPRT_COPY_XID;
+			xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
 			xprt_wake_pending_tasks(xprt, 0);
 		}
 		spin_unlock_bh(&xprt->transport_lock);
···
  * @task: address of RPC task that manages state of connect request
  *
  * TCP: If the remote end dropped the connection, delay reconnecting.
+ *
+ * UDP socket connects are synchronous, but we use a work queue anyway
+ * to guarantee that even unprivileged user processes can set up a
+ * socket on a privileged port.
+ *
+ * If a UDP socket connect fails, the delay behavior here prevents
+ * retry floods (hard mounts).
  */
 static void xs_connect(struct rpc_task *task)
 {
···
 		return;
 
 	if (xprt->sock != NULL) {
-		dprintk("RPC:      xs_connect delayed xprt %p\n", xprt);
+		dprintk("RPC:      xs_connect delayed xprt %p for %lu seconds\n",
+				xprt, xprt->reestablish_timeout / HZ);
 		schedule_delayed_work(&xprt->connect_worker,
-					RPC_REESTABLISH_TIMEOUT);
+					xprt->reestablish_timeout);
+		xprt->reestablish_timeout <<= 1;
+		if (xprt->reestablish_timeout > XS_TCP_MAX_REEST_TO)
+			xprt->reestablish_timeout = XS_TCP_MAX_REEST_TO;
 	} else {
 		dprintk("RPC:      xs_connect scheduled xprt %p\n", xprt);
 		schedule_work(&xprt->connect_worker);
···
 	xprt->max_payload = (1U << 16) - (MAX_HEADER << 3);
 
 	INIT_WORK(&xprt->connect_worker, xs_udp_connect_worker, xprt);
+	xprt->bind_timeout = XS_BIND_TO;
+	xprt->connect_timeout = XS_UDP_CONN_TO;
+	xprt->reestablish_timeout = XS_UDP_REEST_TO;
+	xprt->idle_timeout = XS_IDLE_DISC_TO;
 
 	xprt->ops = &xs_udp_ops;
 
···
 	xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
 
 	INIT_WORK(&xprt->connect_worker, xs_tcp_connect_worker, xprt);
+	xprt->bind_timeout = XS_BIND_TO;
+	xprt->connect_timeout = XS_TCP_CONN_TO;
+	xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
+	xprt->idle_timeout = XS_IDLE_DISC_TO;
 
 	xprt->ops = &xs_tcp_ops;