[PATCH v1 0/4] SunRPC fault injection

[PATCH v1 0/4] SunRPC fault injection

[Chuck Lever]

The following series (re)implements SunRPC disconnect injection
using the kernel's generic fault injection infrastructure under
the debugfs. It's partially a clean-up and partially a fresh
implementation of server-side disconnect injection, while also
enabling the straightforward addition of further types of
fault injection in the future.

---

Chuck Lever (4):
      SUNRPC: Add a /sys/kernel/debug/fail_sunrpc/ directory
      SUNRPC: Server-side disconnect injection
      SUNRPC: Move client-side disconnect injection
      SUNRPC: Add documentation for the fail_sunrpc/ directory


 .../fault-injection/fault-injection.rst       | 18 +++++
 include/linux/sunrpc/xprt.h                   | 18 -----
 net/sunrpc/debugfs.c                          | 71 +++++--------------
 net/sunrpc/fail.h                             | 20 ++++++
 net/sunrpc/svc.c                              |  6 ++
 net/sunrpc/xprt.c                             |  8 +++
 6 files changed, 70 insertions(+), 71 deletions(-)
 create mode 100644 net/sunrpc/fail.h

--
Chuck Lever

[PATCH v1 1/4] SUNRPC: Add a /sys/kernel/debug/fail_sunrpc/ directory

[Chuck Lever]

This directory will contain a set of administrative controls for
enabling error injection for kernel RPC consumers.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
 net/sunrpc/debugfs.c |   20 ++++++++++++++++++++
 net/sunrpc/fail.h    |   17 +++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 net/sunrpc/fail.h

diff --git a/net/sunrpc/debugfs.c b/net/sunrpc/debugfs.c
index 56029e3af6ff..7a9065b6a058 100644
--- a/net/sunrpc/debugfs.c
+++ b/net/sunrpc/debugfs.c
@@ -8,7 +8,9 @@
 #include <linux/debugfs.h>
 #include <linux/sunrpc/sched.h>
 #include <linux/sunrpc/clnt.h>
+
 #include "netns.h"
+#include "fail.h"
 
 static struct dentry *topdir;
 static struct dentry *rpc_clnt_dir;
@@ -297,6 +299,22 @@ static const struct file_operations fault_disconnect_fops = {
 	.release	= fault_release,
 };
 
+struct fail_sunrpc_attr fail_sunrpc = {
+	.attr			= FAULT_ATTR_INITIALIZER,
+};
+
+#if IS_ENABLED(CONFIG_FAULT_INJECTION_DEBUG_FS)
+static void fail_sunrpc_init(void)
+{
+	fault_create_debugfs_attr("fail_sunrpc", NULL,
+				  &fail_sunrpc.attr);
+}
+#else
+static inline void fail_sunrpc_init(void)
+{
+}
+#endif
+
 void __exit
 sunrpc_debugfs_exit(void)
 {
@@ -321,4 +339,6 @@ sunrpc_debugfs_init(void)
 
 	debugfs_create_file("disconnect", S_IFREG | 0400, rpc_fault_dir, NULL,
 			    &fault_disconnect_fops);
+
+	fail_sunrpc_init();
 }
diff --git a/net/sunrpc/fail.h b/net/sunrpc/fail.h
new file mode 100644
index 000000000000..96a46eff94e4
--- /dev/null
+++ b/net/sunrpc/fail.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2021, Oracle. All rights reserved.
+ */
+
+#ifndef _NET_SUNRPC_FAIL_H_
+#define _NET_SUNRPC_FAIL_H_
+
+#include <linux/fault-inject.h>
+
+struct fail_sunrpc_attr {
+	struct fault_attr	attr;
+};
+
+extern struct fail_sunrpc_attr fail_sunrpc;
+
+#endif /* _NET_SUNRPC_FAIL_H_ */

Re: [PATCH v1 1/4] SUNRPC: Add a /sys/kernel/debug/fail_sunrpc/ directory

[kernel test robot]

[-- Attachment #1: Type: text/plain, Size: 2140 bytes --]

Hi Chuck,

I love your patch! Yet something to improve:

[auto build test ERROR on nfs/linux-next]
[also build test ERROR on nfsd/nfsd-next linus/master v5.14-rc5 next-20210809]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Chuck-Lever/SunRPC-fault-injection/20210810-021150
base:   git://git.linux-nfs.org/projects/trondmy/linux-nfs.git linux-next
config: arc-randconfig-r006-20210809 (attached as .config)
compiler: arceb-elf-gcc (GCC) 10.3.0
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # https://github.com/0day-ci/linux/commit/f6324480d4b27a24f4df68d663ec3b56aeca60c0
        git remote add linux-review https://github.com/0day-ci/linux
        git fetch --no-tags linux-review Chuck-Lever/SunRPC-fault-injection/20210810-021150
        git checkout f6324480d4b27a24f4df68d663ec3b56aeca60c0
        # save the attached .config to linux build tree
        mkdir build_dir
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-10.3.0 make.cross O=build_dir ARCH=arc SHELL=/bin/bash net/sunrpc/

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>

All errors (new ones prefixed by >>):

   In file included from net/sunrpc/debugfs.c:13:
   net/sunrpc/fail.h:12:20: error: field 'attr' has incomplete type
      12 |  struct fault_attr attr;
         |                    ^~~~
>> net/sunrpc/debugfs.c:303:12: error: 'FAULT_ATTR_INITIALIZER' undeclared here (not in a function)
     303 |  .attr   = FAULT_ATTR_INITIALIZER,
         |            ^~~~~~~~~~~~~~~~~~~~~~


vim +/FAULT_ATTR_INITIALIZER +303 net/sunrpc/debugfs.c

   301	
   302	struct fail_sunrpc_attr fail_sunrpc = {
 > 303		.attr			= FAULT_ATTR_INITIALIZER,
   304	};
   305	

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 32906 bytes --]

[PATCH v1 2/4] SUNRPC: Server-side disconnect injection

[Chuck Lever]

Disconnect injection stress-tests the ability for both client and
server implementations to behave resiliently in the face of network
instability.

A file called /sys/kernel/debug/fail_sunrpc/ignore-server-disconnect
enables administrators to turn off server-side disconnect injection
while allowing other types of sunrpc errors to be injected. So far
there are no others. The default setting is that server-side
disconnect injection is enabled (ignore=false).

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
 net/sunrpc/debugfs.c |    9 +++++++--
 net/sunrpc/fail.h    |    2 ++
 net/sunrpc/svc.c     |    6 ++++++
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/net/sunrpc/debugfs.c b/net/sunrpc/debugfs.c
index 7a9065b6a058..654e513afff7 100644
--- a/net/sunrpc/debugfs.c
+++ b/net/sunrpc/debugfs.c
@@ -306,8 +306,13 @@ struct fail_sunrpc_attr fail_sunrpc = {
 #if IS_ENABLED(CONFIG_FAULT_INJECTION_DEBUG_FS)
 static void fail_sunrpc_init(void)
 {
-	fault_create_debugfs_attr("fail_sunrpc", NULL,
-				  &fail_sunrpc.attr);
+	struct dentry *dir;
+
+	dir = fault_create_debugfs_attr("fail_sunrpc", NULL,
+					&fail_sunrpc.attr);
+
+	debugfs_create_bool("ignore-server-disconnect", S_IFREG | 0600, dir,
+			    &fail_sunrpc.ignore_server_disconnect);
 }
 #else
 static inline void fail_sunrpc_init(void)
diff --git a/net/sunrpc/fail.h b/net/sunrpc/fail.h
index 96a46eff94e4..302daa1fea8b 100644
--- a/net/sunrpc/fail.h
+++ b/net/sunrpc/fail.h
@@ -10,6 +10,8 @@
 
 struct fail_sunrpc_attr {
 	struct fault_attr	attr;
+
+	bool			ignore_server_disconnect;
 };
 
 extern struct fail_sunrpc_attr fail_sunrpc;
diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
index 35b549c147a2..332628da0372 100644
--- a/net/sunrpc/svc.c
+++ b/net/sunrpc/svc.c
@@ -31,6 +31,8 @@
 
 #include <trace/events/sunrpc.h>
 
+#include "fail.h"
+
 #define RPCDBG_FACILITY	RPCDBG_SVCDSP
 
 static void svc_unregister(const struct svc_serv *serv, struct net *net);
@@ -1507,6 +1509,10 @@ svc_process(struct svc_rqst *rqstp)
 	struct svc_serv		*serv = rqstp->rq_server;
 	u32			dir;
 
+	if (!fail_sunrpc.ignore_server_disconnect &&
+	    should_fail(&fail_sunrpc.attr, 1))
+		svc_xprt_deferred_close(rqstp->rq_xprt);
+
 	/*
 	 * Setup response xdr_buf.
 	 * Initially it has just one page

Re: [PATCH v1 2/4] SUNRPC: Server-side disconnect injection

[kernel test robot]

[-- Attachment #1: Type: text/plain, Size: 1795 bytes --]

Hi Chuck,

I love your patch! Yet something to improve:

[auto build test ERROR on nfs/linux-next]
[also build test ERROR on nfsd/nfsd-next linus/master v5.14-rc5 next-20210809]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Chuck-Lever/SunRPC-fault-injection/20210810-021150
base:   git://git.linux-nfs.org/projects/trondmy/linux-nfs.git linux-next
config: h8300-buildonly-randconfig-r001-20210809 (attached as .config)
compiler: h8300-linux-gcc (GCC) 10.3.0
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # https://github.com/0day-ci/linux/commit/561bf84b4d936dc5ae88b55b133def1b7e50d263
        git remote add linux-review https://github.com/0day-ci/linux
        git fetch --no-tags linux-review Chuck-Lever/SunRPC-fault-injection/20210810-021150
        git checkout 561bf84b4d936dc5ae88b55b133def1b7e50d263
        # save the attached .config to linux build tree
        mkdir build_dir
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-10.3.0 make.cross O=build_dir ARCH=h8300 SHELL=/bin/bash

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>

All errors (new ones prefixed by >>):

   h8300-linux-ld: net/sunrpc/svc.o: in function `svc_process':
>> svc.c:(.text+0x23a0): undefined reference to `fail_sunrpc'
   h8300-linux-ld: svc.c:(.text+0x23ac): undefined reference to `fail_sunrpc'

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 39804 bytes --]

Re: [PATCH v1 2/4] SUNRPC: Server-side disconnect injection

[kernel test robot]

[-- Attachment #1: Type: text/plain, Size: 2370 bytes --]

Hi Chuck,

I love your patch! Yet something to improve:

[auto build test ERROR on nfs/linux-next]
[also build test ERROR on nfsd/nfsd-next linus/master v5.14-rc5 next-20210809]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Chuck-Lever/SunRPC-fault-injection/20210810-021150
base:   git://git.linux-nfs.org/projects/trondmy/linux-nfs.git linux-next
config: i386-randconfig-r033-20210809 (attached as .config)
compiler: gcc-9 (Debian 9.3.0-22) 9.3.0
reproduce (this is a W=1 build):
        # https://github.com/0day-ci/linux/commit/561bf84b4d936dc5ae88b55b133def1b7e50d263
        git remote add linux-review https://github.com/0day-ci/linux
        git fetch --no-tags linux-review Chuck-Lever/SunRPC-fault-injection/20210810-021150
        git checkout 561bf84b4d936dc5ae88b55b133def1b7e50d263
        # save the attached .config to linux build tree
        mkdir build_dir
        make W=1 O=build_dir ARCH=i386 SHELL=/bin/bash net/

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>

All errors (new ones prefixed by >>):

   In file included from net/sunrpc/svc.c:34:
>> net/sunrpc/fail.h:12:20: error: field 'attr' has incomplete type
      12 |  struct fault_attr attr;
         |                    ^~~~
   net/sunrpc/svc.c: In function 'svc_process':
>> net/sunrpc/svc.c:1509:6: error: implicit declaration of function 'should_fail'; did you mean 'should_failslab'? [-Werror=implicit-function-declaration]
    1509 |      should_fail(&fail_sunrpc.attr, 1))
         |      ^~~~~~~~~~~
         |      should_failslab
   cc1: some warnings being treated as errors


vim +/attr +12 net/sunrpc/fail.h

f6324480d4b27a2 Chuck Lever 2021-08-09  10  
f6324480d4b27a2 Chuck Lever 2021-08-09  11  struct fail_sunrpc_attr {
f6324480d4b27a2 Chuck Lever 2021-08-09 @12  	struct fault_attr	attr;
561bf84b4d936dc Chuck Lever 2021-08-09  13  
561bf84b4d936dc Chuck Lever 2021-08-09  14  	bool			ignore_server_disconnect;
f6324480d4b27a2 Chuck Lever 2021-08-09  15  };
f6324480d4b27a2 Chuck Lever 2021-08-09  16  

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 36621 bytes --]

[PATCH v1 3/4] SUNRPC: Move client-side disconnect injection

[Chuck Lever]

Disconnect injection stress-tests the ability for both client and
server implementations to behave resiliently in the face of network
instability.

Convert the existing client-side disconnect injection infrastructure
to use the kernel's generic error injection facility. The generic
facility has a richer set of injection criteria.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
 include/linux/sunrpc/xprt.h |   18 ------------
 net/sunrpc/debugfs.c        |   64 +------------------------------------------
 net/sunrpc/fail.h           |    1 +
 net/sunrpc/xprt.c           |    8 +++++
 4 files changed, 11 insertions(+), 80 deletions(-)

diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h
index c8c39f22d3b1..b15c1f07162d 100644
--- a/include/linux/sunrpc/xprt.h
+++ b/include/linux/sunrpc/xprt.h
@@ -288,7 +288,6 @@ struct rpc_xprt {
 	const char		*address_strings[RPC_DISPLAY_MAX];
 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
 	struct dentry		*debugfs;		/* debugfs directory */
-	atomic_t		inject_disconnect;
 #endif
 	struct rcu_head		rcu;
 	const struct xprt_class	*xprt_class;
@@ -502,21 +501,4 @@ static inline int xprt_test_and_set_binding(struct rpc_xprt *xprt)
 	return test_and_set_bit(XPRT_BINDING, &xprt->state);
 }
 
-#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
-extern unsigned int rpc_inject_disconnect;
-static inline void xprt_inject_disconnect(struct rpc_xprt *xprt)
-{
-	if (!rpc_inject_disconnect)
-		return;
-	if (atomic_dec_return(&xprt->inject_disconnect))
-		return;
-	atomic_set(&xprt->inject_disconnect, rpc_inject_disconnect);
-	xprt->ops->inject_disconnect(xprt);
-}
-#else
-static inline void xprt_inject_disconnect(struct rpc_xprt *xprt)
-{
-}
-#endif
-
 #endif /* _LINUX_SUNRPC_XPRT_H */
diff --git a/net/sunrpc/debugfs.c b/net/sunrpc/debugfs.c
index 654e513afff7..8c3d52df982f 100644
--- a/net/sunrpc/debugfs.c
+++ b/net/sunrpc/debugfs.c
@@ -16,8 +16,6 @@ static struct dentry *topdir;
 static struct dentry *rpc_clnt_dir;
 static struct dentry *rpc_xprt_dir;
 
-unsigned int rpc_inject_disconnect;
-
 static int
 tasks_show(struct seq_file *f, void *v)
 {
@@ -237,8 +235,6 @@ rpc_xprt_debugfs_register(struct rpc_xprt *xprt)
 	/* make tasks file */
 	debugfs_create_file("info", S_IFREG | 0400, xprt->debugfs, xprt,
 			    &xprt_info_fops);
-
-	atomic_set(&xprt->inject_disconnect, rpc_inject_disconnect);
 }
 
 void
@@ -248,57 +244,6 @@ rpc_xprt_debugfs_unregister(struct rpc_xprt *xprt)
 	xprt->debugfs = NULL;
 }
 
-static int
-fault_open(struct inode *inode, struct file *filp)
-{
-	filp->private_data = kmalloc(128, GFP_KERNEL);
-	if (!filp->private_data)
-		return -ENOMEM;
-	return 0;
-}
-
-static int
-fault_release(struct inode *inode, struct file *filp)
-{
-	kfree(filp->private_data);
-	return 0;
-}
-
-static ssize_t
-fault_disconnect_read(struct file *filp, char __user *user_buf,
-		      size_t len, loff_t *offset)
-{
-	char *buffer = (char *)filp->private_data;
-	size_t size;
-
-	size = sprintf(buffer, "%u\n", rpc_inject_disconnect);
-	return simple_read_from_buffer(user_buf, len, offset, buffer, size);
-}
-
-static ssize_t
-fault_disconnect_write(struct file *filp, const char __user *user_buf,
-		       size_t len, loff_t *offset)
-{
-	char buffer[16];
-
-	if (len >= sizeof(buffer))
-		len = sizeof(buffer) - 1;
-	if (copy_from_user(buffer, user_buf, len))
-		return -EFAULT;
-	buffer[len] = '\0';
-	if (kstrtouint(buffer, 10, &rpc_inject_disconnect))
-		return -EINVAL;
-	return len;
-}
-
-static const struct file_operations fault_disconnect_fops = {
-	.owner		= THIS_MODULE,
-	.open		= fault_open,
-	.read		= fault_disconnect_read,
-	.write		= fault_disconnect_write,
-	.release	= fault_release,
-};
-
 struct fail_sunrpc_attr fail_sunrpc = {
 	.attr			= FAULT_ATTR_INITIALIZER,
 };
@@ -313,6 +258,8 @@ static void fail_sunrpc_init(void)
 
 	debugfs_create_bool("ignore-server-disconnect", S_IFREG | 0600, dir,
 			    &fail_sunrpc.ignore_server_disconnect);
+	debugfs_create_bool("ignore-client-disconnect", S_IFREG | 0600, dir,
+			    &fail_sunrpc.ignore_client_disconnect);
 }
 #else
 static inline void fail_sunrpc_init(void)
@@ -332,18 +279,11 @@ sunrpc_debugfs_exit(void)
 void __init
 sunrpc_debugfs_init(void)
 {
-	struct dentry *rpc_fault_dir;
-
 	topdir = debugfs_create_dir("sunrpc", NULL);
 
 	rpc_clnt_dir = debugfs_create_dir("rpc_clnt", topdir);
 
 	rpc_xprt_dir = debugfs_create_dir("rpc_xprt", topdir);
 
-	rpc_fault_dir = debugfs_create_dir("inject_fault", topdir);
-
-	debugfs_create_file("disconnect", S_IFREG | 0400, rpc_fault_dir, NULL,
-			    &fault_disconnect_fops);
-
 	fail_sunrpc_init();
 }
diff --git a/net/sunrpc/fail.h b/net/sunrpc/fail.h
index 302daa1fea8b..e5d3e80fac0c 100644
--- a/net/sunrpc/fail.h
+++ b/net/sunrpc/fail.h
@@ -12,6 +12,7 @@ struct fail_sunrpc_attr {
 	struct fault_attr	attr;
 
 	bool			ignore_server_disconnect;
+	bool			ignore_client_disconnect;
 };
 
 extern struct fail_sunrpc_attr fail_sunrpc;
diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
index fb6db09725c7..fbbc38771c65 100644
--- a/net/sunrpc/xprt.c
+++ b/net/sunrpc/xprt.c
@@ -56,6 +56,7 @@
 
 #include "sunrpc.h"
 #include "sysfs.h"
+#include "fail.h"
 
 /*
  * Local variables
@@ -855,6 +856,13 @@ xprt_init_autodisconnect(struct timer_list *t)
 	queue_work(xprtiod_workqueue, &xprt->task_cleanup);
 }
 
+static void xprt_inject_disconnect(struct rpc_xprt *xprt)
+{
+	if (!fail_sunrpc.ignore_client_disconnect &&
+	    should_fail(&fail_sunrpc.attr, 1))
+		xprt->ops->inject_disconnect(xprt);
+}
+
 bool xprt_lock_connect(struct rpc_xprt *xprt,
 		struct rpc_task *task,
 		void *cookie)

[PATCH v1 4/4] SUNRPC: Add documentation for the fail_sunrpc/ directory

[Chuck Lever]

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
 Documentation/fault-injection/fault-injection.rst |   18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/Documentation/fault-injection/fault-injection.rst b/Documentation/fault-injection/fault-injection.rst
index f47d05ed0d94..4a25c5eb6f07 100644
--- a/Documentation/fault-injection/fault-injection.rst
+++ b/Documentation/fault-injection/fault-injection.rst
@@ -24,6 +24,10 @@ Available fault injection capabilities
 
   injects futex deadlock and uaddr fault errors.
 
+- fail_sunrpc
+
+  injects kernel RPC client and server failures.
+
 - fail_make_request
 
   injects disk IO errors on devices permitted by setting
@@ -151,6 +155,20 @@ configuration of fault-injection capabilities.
 	default is 'N', setting it to 'Y' will disable failure injections
 	when dealing with private (address space) futexes.
 
+- /sys/kernel/debug/fail_sunrpc/ignore-client-disconnect:
+
+	Format: { 'Y' | 'N' }
+
+	default is 'N', setting it to 'Y' will disable disconnect
+	injection on the RPC client.
+
+- /sys/kernel/debug/fail_sunrpc/ignore-server-disconnect:
+
+	Format: { 'Y' | 'N' }
+
+	default is 'N', setting it to 'Y' will disable disconnect
+	injection on the RPC server.
+
 - /sys/kernel/debug/fail_function/inject:
 
 	Format: { 'function-name' | '!function-name' | '' }