[PATCH v3 00/10] NFSD: Improve fault injection

[PATCH v3 00/10] NFSD: Improve fault injection

[bjschuma]

From: Bryan Schumaker <bjschuma@netapp.com>

While working on p2p-nfs, I discovered that I sometimes need to clear
state for a specific client to test all possible error recovery conditions.
The current fault injection code deletes state as it find it, so it can
be difficult to guess which state will be forgotten.  In addition, I
currently print out the amount of state forgotten but I don't give details
like "Forgot 3 locks from client w.x.y.z".  These patches set out to
improve that.

The first 7 patches clean up the current code and prepare it for specific
client state removal.  Patch 8 adds prints information to the server's logs
when a fault injection file is read (such as "Client w.x.y.z has 3 open
files").  Patch 9 adds in a custom file operations structure so users can
write strings to fault injection files in addition to u64s.  Finally, patch
10 allows users to remove state by writing a client's IP address to one of
the files.

Changes in v3:
- Patches apply cleanly to the most recent nfsd-next branch.
- Introduce non-static functions directly, rather than introducing them
  statically and then changing the definition a few patches later.
- Share more code between printing and releasing lockowners, openowners and
  delegations.

Questions, comments and suggestions are appreciated!

- Bryan

Bryan Schumaker (10):
  NFSD: Fold fault_inject.h into state.h
  NFSD: Lock state before calling fault injection function
  NFSD: Clean up forgetting clients
  NFSD: Clean up forgetting locks
  NFSD: Clean up forgetting openowners
  NFSD: Clean up forgetting and recalling delegations
  NFSD: Fault injection operations take a per-client forget function
  NFSD: Reading a fault injection file prints a state count
  NFSD: Add a custom file operations structure for fault injection
  NFSD: Forget state for a specific client

 fs/nfsd/fault_inject.c | 114 ++++++++++++++++++++++++++----
 fs/nfsd/fault_inject.h |  28 --------
 fs/nfsd/nfs4state.c    | 188 ++++++++++++++++++++++++++++++-------------------
 fs/nfsd/nfsctl.c       |   2 +-
 fs/nfsd/state.h        |  23 ++++++
 5 files changed, 239 insertions(+), 116 deletions(-)
 delete mode 100644 fs/nfsd/fault_inject.h

-- 
1.8.0.1

[PATCH v3 01/10] NFSD: Fold fault_inject.h into state.h

[bjschuma]

From: Bryan Schumaker <bjschuma@netapp.com>

There were only a small number of functions in this file and since they
all affect stored state I think it makes sense to put them in state.h
instead.  I also dropped most static inline declarations since there are
no callers when fault injection is not enabled.

Signed-off-by: Bryan Schumaker <bjschuma@netapp.com>
---
 fs/nfsd/fault_inject.c |  1 -
 fs/nfsd/fault_inject.h | 28 ----------------------------
 fs/nfsd/nfs4state.c    |  1 -
 fs/nfsd/nfsctl.c       |  2 +-
 fs/nfsd/state.h        | 15 +++++++++++++++
 5 files changed, 16 insertions(+), 31 deletions(-)
 delete mode 100644 fs/nfsd/fault_inject.h

diff --git a/fs/nfsd/fault_inject.c b/fs/nfsd/fault_inject.c
index e6c3815..0278112 100644
--- a/fs/nfsd/fault_inject.c
+++ b/fs/nfsd/fault_inject.c
@@ -10,7 +10,6 @@
 #include <linux/module.h>
 
 #include "state.h"
-#include "fault_inject.h"
 
 struct nfsd_fault_inject_op {
 	char *file;
diff --git a/fs/nfsd/fault_inject.h b/fs/nfsd/fault_inject.h
deleted file mode 100644
index 90bd057..0000000
--- a/fs/nfsd/fault_inject.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright (c) 2011 Bryan Schumaker <bjschuma@netapp.com>
- *
- * Function definitions for fault injection
- */
-
-#ifndef LINUX_NFSD_FAULT_INJECT_H
-#define LINUX_NFSD_FAULT_INJECT_H
-
-#ifdef CONFIG_NFSD_FAULT_INJECTION
-int nfsd_fault_inject_init(void);
-void nfsd_fault_inject_cleanup(void);
-void nfsd_forget_clients(u64);
-void nfsd_forget_locks(u64);
-void nfsd_forget_openowners(u64);
-void nfsd_forget_delegations(u64);
-void nfsd_recall_delegations(u64);
-#else /* CONFIG_NFSD_FAULT_INJECTION */
-static inline int nfsd_fault_inject_init(void) { return 0; }
-static inline void nfsd_fault_inject_cleanup(void) {}
-static inline void nfsd_forget_clients(u64 num) {}
-static inline void nfsd_forget_locks(u64 num) {}
-static inline void nfsd_forget_openowners(u64 num) {}
-static inline void nfsd_forget_delegations(u64 num) {}
-static inline void nfsd_recall_delegations(u64 num) {}
-#endif /* CONFIG_NFSD_FAULT_INJECTION */
-
-#endif /* LINUX_NFSD_FAULT_INJECT_H */
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 7e5dfa1..2d6386f 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -44,7 +44,6 @@
 #include "xdr4.h"
 #include "vfs.h"
 #include "current_stateid.h"
-#include "fault_inject.h"
 
 #include "netns.h"
 
diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
index dab350d..7e0d6db 100644
--- a/fs/nfsd/nfsctl.c
+++ b/fs/nfsd/nfsctl.c
@@ -19,7 +19,7 @@
 #include "idmap.h"
 #include "nfsd.h"
 #include "cache.h"
-#include "fault_inject.h"
+#include "state.h"
 #include "netns.h"
 
 /*
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index 2deb6a8..b542bf2 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -496,4 +496,19 @@ extern void nfsd4_client_record_create(struct nfs4_client *clp);
 extern void nfsd4_client_record_remove(struct nfs4_client *clp);
 extern int nfsd4_client_record_check(struct nfs4_client *clp);
 extern void nfsd4_record_grace_done(struct nfsd_net *nn, time_t boot_time);
+
+/* nfs fault injection functions */
+#ifdef CONFIG_NFSD_FAULT_INJECTION
+int nfsd_fault_inject_init(void);
+void nfsd_fault_inject_cleanup(void);
+void nfsd_forget_clients(u64);
+void nfsd_forget_locks(u64);
+void nfsd_forget_openowners(u64);
+void nfsd_forget_delegations(u64);
+void nfsd_recall_delegations(u64);
+#else /* CONFIG_NFSD_FAULT_INJECTION */
+static inline int nfsd_fault_inject_init(void) { return 0; }
+static inline void nfsd_fault_inject_cleanup(void) {}
+#endif /* CONFIG_NFSD_FAULT_INJECTION */
+
 #endif   /* NFSD4_STATE_H */
-- 
1.8.0.1

Re: [PATCH v3 01/10] NFSD: Fold fault_inject.h into state.h

[J. Bruce Fields]

On Tue, Nov 27, 2012 at 09:35:10AM -0500, bjschuma@netapp.com wrote:
> From: Bryan Schumaker <bjschuma@netapp.com>
> 
> There were only a small number of functions in this file and since they
> all affect stored state I think it makes sense to put them in state.h
> instead.  I also dropped most static inline declarations since there are
> no callers when fault injection is not enabled.

Went ahead and applied this one, just for fun.

--b.

> 
> Signed-off-by: Bryan Schumaker <bjschuma@netapp.com>
> ---
>  fs/nfsd/fault_inject.c |  1 -
>  fs/nfsd/fault_inject.h | 28 ----------------------------
>  fs/nfsd/nfs4state.c    |  1 -
>  fs/nfsd/nfsctl.c       |  2 +-
>  fs/nfsd/state.h        | 15 +++++++++++++++
>  5 files changed, 16 insertions(+), 31 deletions(-)
>  delete mode 100644 fs/nfsd/fault_inject.h
> 
> diff --git a/fs/nfsd/fault_inject.c b/fs/nfsd/fault_inject.c
> index e6c3815..0278112 100644
> --- a/fs/nfsd/fault_inject.c
> +++ b/fs/nfsd/fault_inject.c
> @@ -10,7 +10,6 @@
>  #include <linux/module.h>
>  
>  #include "state.h"
> -#include "fault_inject.h"
>  
>  struct nfsd_fault_inject_op {
>  	char *file;
> diff --git a/fs/nfsd/fault_inject.h b/fs/nfsd/fault_inject.h
> deleted file mode 100644
> index 90bd057..0000000
> --- a/fs/nfsd/fault_inject.h
> +++ /dev/null
> @@ -1,28 +0,0 @@
> -/*
> - * Copyright (c) 2011 Bryan Schumaker <bjschuma@netapp.com>
> - *
> - * Function definitions for fault injection
> - */
> -
> -#ifndef LINUX_NFSD_FAULT_INJECT_H
> -#define LINUX_NFSD_FAULT_INJECT_H
> -
> -#ifdef CONFIG_NFSD_FAULT_INJECTION
> -int nfsd_fault_inject_init(void);
> -void nfsd_fault_inject_cleanup(void);
> -void nfsd_forget_clients(u64);
> -void nfsd_forget_locks(u64);
> -void nfsd_forget_openowners(u64);
> -void nfsd_forget_delegations(u64);
> -void nfsd_recall_delegations(u64);
> -#else /* CONFIG_NFSD_FAULT_INJECTION */
> -static inline int nfsd_fault_inject_init(void) { return 0; }
> -static inline void nfsd_fault_inject_cleanup(void) {}
> -static inline void nfsd_forget_clients(u64 num) {}
> -static inline void nfsd_forget_locks(u64 num) {}
> -static inline void nfsd_forget_openowners(u64 num) {}
> -static inline void nfsd_forget_delegations(u64 num) {}
> -static inline void nfsd_recall_delegations(u64 num) {}
> -#endif /* CONFIG_NFSD_FAULT_INJECTION */
> -
> -#endif /* LINUX_NFSD_FAULT_INJECT_H */
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index 7e5dfa1..2d6386f 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -44,7 +44,6 @@
>  #include "xdr4.h"
>  #include "vfs.h"
>  #include "current_stateid.h"
> -#include "fault_inject.h"
>  
>  #include "netns.h"
>  
> diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
> index dab350d..7e0d6db 100644
> --- a/fs/nfsd/nfsctl.c
> +++ b/fs/nfsd/nfsctl.c
> @@ -19,7 +19,7 @@
>  #include "idmap.h"
>  #include "nfsd.h"
>  #include "cache.h"
> -#include "fault_inject.h"
> +#include "state.h"
>  #include "netns.h"
>  
>  /*
> diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
> index 2deb6a8..b542bf2 100644
> --- a/fs/nfsd/state.h
> +++ b/fs/nfsd/state.h
> @@ -496,4 +496,19 @@ extern void nfsd4_client_record_create(struct nfs4_client *clp);
>  extern void nfsd4_client_record_remove(struct nfs4_client *clp);
>  extern int nfsd4_client_record_check(struct nfs4_client *clp);
>  extern void nfsd4_record_grace_done(struct nfsd_net *nn, time_t boot_time);
> +
> +/* nfs fault injection functions */
> +#ifdef CONFIG_NFSD_FAULT_INJECTION
> +int nfsd_fault_inject_init(void);
> +void nfsd_fault_inject_cleanup(void);
> +void nfsd_forget_clients(u64);
> +void nfsd_forget_locks(u64);
> +void nfsd_forget_openowners(u64);
> +void nfsd_forget_delegations(u64);
> +void nfsd_recall_delegations(u64);
> +#else /* CONFIG_NFSD_FAULT_INJECTION */
> +static inline int nfsd_fault_inject_init(void) { return 0; }
> +static inline void nfsd_fault_inject_cleanup(void) {}
> +#endif /* CONFIG_NFSD_FAULT_INJECTION */
> +
>  #endif   /* NFSD4_STATE_H */
> -- 
> 1.8.0.1
> 

[PATCH v3 02/10] NFSD: Lock state before calling fault injection function

[bjschuma]

From: Bryan Schumaker <bjschuma@netapp.com>

Each function touches state in some way, so getting the lock earlier
can help simplify code.

Signed-off-by: Bryan Schumaker <bjschuma@netapp.com>
---
 fs/nfsd/fault_inject.c |  2 ++
 fs/nfsd/nfs4state.c    | 18 ++----------------
 2 files changed, 4 insertions(+), 16 deletions(-)

diff --git a/fs/nfsd/fault_inject.c b/fs/nfsd/fault_inject.c
index 0278112..4b385a1 100644
--- a/fs/nfsd/fault_inject.c
+++ b/fs/nfsd/fault_inject.c
@@ -51,7 +51,9 @@ static int nfsd_inject_set(void *op_ptr, u64 val)
 	else
 		printk(KERN_INFO "NFSD Fault Injection: %s (n = %llu)", op->file, val);
 
+	nfs4_lock_state();
 	op->func(val);
+	nfs4_unlock_state();
 	return 0;
 }
 
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 2d6386f..050a35e 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -4597,13 +4597,11 @@ void nfsd_forget_clients(u64 num)
 	int count = 0;
 	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns, nfsd_net_id);
 
-	nfs4_lock_state();
 	list_for_each_entry_safe(clp, next, &nn->client_lru, cl_lru) {
 		expire_client(clp);
 		if (++count == num)
 			break;
 	}
-	nfs4_unlock_state();
 
 	printk(KERN_INFO "NFSD: Forgot %d clients", count);
 }
@@ -4639,25 +4637,15 @@ static int nfsd_release_n_owners(u64 num, bool is_open_owner,
 
 void nfsd_forget_locks(u64 num)
 {
-	int count;
 	struct nfsd_net *nn = net_generic(&init_net, nfsd_net_id);
-
-	nfs4_lock_state();
-	count = nfsd_release_n_owners(num, false, release_lockowner_sop, nn);
-	nfs4_unlock_state();
-
+	int count = nfsd_release_n_owners(num, false, release_lockowner_sop, nn);
 	printk(KERN_INFO "NFSD: Forgot %d locks", count);
 }
 
 void nfsd_forget_openowners(u64 num)
 {
-	int count;
 	struct nfsd_net *nn = net_generic(&init_net, nfsd_net_id);
-
-	nfs4_lock_state();
-	count = nfsd_release_n_owners(num, true, release_openowner_sop, nn);
-	nfs4_unlock_state();
-
+	int count = nfsd_release_n_owners(num, true, release_openowner_sop, nn);
 	printk(KERN_INFO "NFSD: Forgot %d open owners", count);
 }
 
@@ -4690,10 +4678,8 @@ void nfsd_forget_delegations(u64 num)
 	count = nfsd_process_n_delegations(num, &victims);
 	spin_unlock(&recall_lock);
 
-	nfs4_lock_state();
 	list_for_each_entry_safe(dp, dnext, &victims, dl_recall_lru)
 		unhash_delegation(dp);
-	nfs4_unlock_state();
 
 	printk(KERN_INFO "NFSD: Forgot %d delegations", count);
 }
-- 
1.8.0.1

[PATCH v3 03/10] NFSD: Clean up forgetting clients

[bjschuma]

From: Bryan Schumaker <bjschuma@netapp.com>

I added in a generic for-each loop that takes a pass over the client_lru
list and calls some function.  The next few patches will update other
operations to use this function as well.  A value of 0 still means "forget
everything that is found".

Signed-off-by: Bryan Schumaker <bjschuma@netapp.com>
---
 fs/nfsd/nfs4state.c | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 050a35e..07abca5 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -4591,19 +4591,32 @@ nfs4_check_open_reclaim(clientid_t *clid, bool sessions, struct nfsd_net *nn)
 
 #ifdef CONFIG_NFSD_FAULT_INJECTION
 
-void nfsd_forget_clients(u64 num)
+u64 nfsd_forget_client(struct nfsd_net *nn, struct nfs4_client *clp, u64 max)
+{
+	nfsd4_client_record_remove(clp);
+	expire_client(clp);
+	return 1;
+}
+
+static u64 nfsd_for_n_state(u64 max, u64 (*func)(struct nfsd_net *, struct nfs4_client *, u64))
 {
 	struct nfs4_client *clp, *next;
-	int count = 0;
+	u64 count = 0;
 	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns, nfsd_net_id);
 
 	list_for_each_entry_safe(clp, next, &nn->client_lru, cl_lru) {
-		expire_client(clp);
-		if (++count == num)
+		count += func(nn, clp, max - count);
+		if ((max != 0) && (count >= max))
 			break;
 	}
 
-	printk(KERN_INFO "NFSD: Forgot %d clients", count);
+	return count;
+}
+
+void nfsd_forget_clients(u64 num)
+{
+	u64 count = nfsd_for_n_state(num, nfsd_forget_client);
+	printk(KERN_INFO "NFSD: Forgot %llu clients", count);
 }
 
 static void release_lockowner_sop(struct nfs4_stateowner *sop)
-- 
1.8.0.1

Re: [PATCH v3 03/10] NFSD: Clean up forgetting clients

[J. Bruce Fields]

On Tue, Nov 27, 2012 at 09:35:12AM -0500, bjschuma@netapp.com wrote:
> From: Bryan Schumaker <bjschuma@netapp.com>
> 
> I added in a generic for-each loop that takes a pass over the client_lru
> list and calls some function.  The next few patches will update other
> operations to use this function as well.  A value of 0 still means "forget
> everything that is found".
> 
> Signed-off-by: Bryan Schumaker <bjschuma@netapp.com>
> ---
>  fs/nfsd/nfs4state.c | 23 ++++++++++++++++++-----
>  1 file changed, 18 insertions(+), 5 deletions(-)
> 
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index 050a35e..07abca5 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -4591,19 +4591,32 @@ nfs4_check_open_reclaim(clientid_t *clid, bool sessions, struct nfsd_net *nn)
>  
>  #ifdef CONFIG_NFSD_FAULT_INJECTION
>  
> -void nfsd_forget_clients(u64 num)
> +u64 nfsd_forget_client(struct nfsd_net *nn, struct nfs4_client *clp, u64 max)

It doesn't look like you really need nfsd_net?

--b.

> +{
> +	nfsd4_client_record_remove(clp);
> +	expire_client(clp);
> +	return 1;
> +}
> +
> +static u64 nfsd_for_n_state(u64 max, u64 (*func)(struct nfsd_net *, struct nfs4_client *, u64))
>  {
>  	struct nfs4_client *clp, *next;
> -	int count = 0;
> +	u64 count = 0;
>  	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns, nfsd_net_id);
>  
>  	list_for_each_entry_safe(clp, next, &nn->client_lru, cl_lru) {
> -		expire_client(clp);
> -		if (++count == num)
> +		count += func(nn, clp, max - count);
> +		if ((max != 0) && (count >= max))
>  			break;
>  	}
>  
> -	printk(KERN_INFO "NFSD: Forgot %d clients", count);
> +	return count;
> +}
> +
> +void nfsd_forget_clients(u64 num)
> +{
> +	u64 count = nfsd_for_n_state(num, nfsd_forget_client);
> +	printk(KERN_INFO "NFSD: Forgot %llu clients", count);
>  }
>  
>  static void release_lockowner_sop(struct nfs4_stateowner *sop)
> -- 
> 1.8.0.1
> 

Re: [PATCH v3 03/10] NFSD: Clean up forgetting clients

[Bryan Schumaker]

On 11/28/2012 11:29 AM, J. Bruce Fields wrote:
> On Tue, Nov 27, 2012 at 09:35:12AM -0500, bjschuma@netapp.com wrote:
>> From: Bryan Schumaker <bjschuma@netapp.com>
>>
>> I added in a generic for-each loop that takes a pass over the client_lru
>> list and calls some function.  The next few patches will update other
>> operations to use this function as well.  A value of 0 still means "forget
>> everything that is found".
>>
>> Signed-off-by: Bryan Schumaker <bjschuma@netapp.com>
>> ---
>>  fs/nfsd/nfs4state.c | 23 ++++++++++++++++++-----
>>  1 file changed, 18 insertions(+), 5 deletions(-)
>>
>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
>> index 050a35e..07abca5 100644
>> --- a/fs/nfsd/nfs4state.c
>> +++ b/fs/nfsd/nfs4state.c
>> @@ -4591,19 +4591,32 @@ nfs4_check_open_reclaim(clientid_t *clid, bool sessions, struct nfsd_net *nn)
>>  
>>  #ifdef CONFIG_NFSD_FAULT_INJECTION
>>  
>> -void nfsd_forget_clients(u64 num)
>> +u64 nfsd_forget_client(struct nfsd_net *nn, struct nfs4_client *clp, u64 max)
> 
> It doesn't look like you really need nfsd_net?

Not for this, but I find locks through the nn->ownerstr_hashtable since the nfs4_client doesn't have a "cl_locks" list similar to "cl_openowners" (unless there is something I'm missing...?).  I figured I would introduce the function with the nfsd_net pointer early, rather than change it in the next patch.

I suppose I could have also put the locks patch first...

- Bryan

> 
> --b.
> 
>> +{
>> +	nfsd4_client_record_remove(clp);
>> +	expire_client(clp);
>> +	return 1;
>> +}
>> +
>> +static u64 nfsd_for_n_state(u64 max, u64 (*func)(struct nfsd_net *, struct nfs4_client *, u64))
>>  {
>>  	struct nfs4_client *clp, *next;
>> -	int count = 0;
>> +	u64 count = 0;
>>  	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns, nfsd_net_id);
>>  
>>  	list_for_each_entry_safe(clp, next, &nn->client_lru, cl_lru) {
>> -		expire_client(clp);
>> -		if (++count == num)
>> +		count += func(nn, clp, max - count);
>> +		if ((max != 0) && (count >= max))
>>  			break;
>>  	}
>>  
>> -	printk(KERN_INFO "NFSD: Forgot %d clients", count);
>> +	return count;
>> +}
>> +
>> +void nfsd_forget_clients(u64 num)
>> +{
>> +	u64 count = nfsd_for_n_state(num, nfsd_forget_client);
>> +	printk(KERN_INFO "NFSD: Forgot %llu clients", count);
>>  }
>>  
>>  static void release_lockowner_sop(struct nfs4_stateowner *sop)
>> -- 
>> 1.8.0.1
>>

Re: [PATCH v3 03/10] NFSD: Clean up forgetting clients

[J. Bruce Fields]

On Wed, Nov 28, 2012 at 11:34:40AM -0500, Bryan Schumaker wrote:
> On 11/28/2012 11:29 AM, J. Bruce Fields wrote:
> > On Tue, Nov 27, 2012 at 09:35:12AM -0500, bjschuma@netapp.com wrote:
> >> From: Bryan Schumaker <bjschuma@netapp.com>
> >>
> >> I added in a generic for-each loop that takes a pass over the
> >> client_lru list and calls some function.  The next few patches will
> >> update other operations to use this function as well.  A value of 0
> >> still means "forget everything that is found".
> >>
> >> Signed-off-by: Bryan Schumaker <bjschuma@netapp.com> ---
> >> fs/nfsd/nfs4state.c | 23 ++++++++++++++++++----- 1 file changed, 18
> >> insertions(+), 5 deletions(-)
> >>
> >> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index
> >> 050a35e..07abca5 100644 --- a/fs/nfsd/nfs4state.c +++
> >> b/fs/nfsd/nfs4state.c @@ -4591,19 +4591,32 @@
> >> nfs4_check_open_reclaim(clientid_t *clid, bool sessions, struct
> >> nfsd_net *nn)
> >>  
> >>  #ifdef CONFIG_NFSD_FAULT_INJECTION
> >>  
> >> -void nfsd_forget_clients(u64 num) +u64 nfsd_forget_client(struct
> >> nfsd_net *nn, struct nfs4_client *clp, u64 max)
> > 
> > It doesn't look like you really need nfsd_net?
> 
> Not for this, but I find locks through the nn->ownerstr_hashtable
> since the nfs4_client doesn't have a "cl_locks" list similar to
> "cl_openowners" (unless there is something I'm missing...?).

That's because lockowners are all reachable from openowners (e.g. see
how it's done in
release_openowner->unhash_openowner->release_open_stateid->unhash_open_stateid->release_stateid_lockowners.
Yeah, it's a little tangled.).

Alternatively you can get to the network namespace from the client
(clp->net).

--b.

> I
> figured I would introduce the function with the nfsd_net pointer
> early, rather than change it in the next patch.
> 
> I suppose I could have also put the locks patch first...
> 
> - Bryan
> 
> > 
> > --b.
> > 
> >> +{ +	nfsd4_client_record_remove(clp); +
> >> expire_client(clp); +	return 1; +} + +static u64
> >> nfsd_for_n_state(u64 max, u64 (*func)(struct nfsd_net *, struct
> >> nfs4_client *, u64)) { struct nfs4_client *clp, *next; -	int
> >> count = 0; +	u64 count = 0; struct nfsd_net *nn =
> >> net_generic(current->nsproxy->net_ns, nfsd_net_id);
> >>  
> >>  	list_for_each_entry_safe(clp, next, &nn->client_lru, cl_lru) { -
> >>  	expire_client(clp); -		if (++count == num) +
> >>  	count += func(nn, clp, max - count); +		if ((max != 0)
> >>  	&& (count >= max)) break; }
> >>  
> >> -	printk(KERN_INFO "NFSD: Forgot %d clients", count); +	return
> >> count; +} + +void nfsd_forget_clients(u64 num) +{ +	u64
> >> count = nfsd_for_n_state(num, nfsd_forget_client); +
> >> printk(KERN_INFO "NFSD: Forgot %llu clients", count); }
> >>  
> >>  static void release_lockowner_sop(struct nfs4_stateowner *sop) --
> >>  1.8.0.1
> >>
> 
> -- To unsubscribe from this list: send the line "unsubscribe
> linux-nfs" in the body of a message to majordomo@vger.kernel.org More
> majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v3 03/10] NFSD: Clean up forgetting clients

[Bryan Schumaker]

On 11/28/2012 11:47 AM, J. Bruce Fields wrote:
> On Wed, Nov 28, 2012 at 11:34:40AM -0500, Bryan Schumaker wrote:
>> On 11/28/2012 11:29 AM, J. Bruce Fields wrote:
>>> On Tue, Nov 27, 2012 at 09:35:12AM -0500, bjschuma@netapp.com wrote:
>>>> From: Bryan Schumaker <bjschuma@netapp.com>
>>>>
>>>> I added in a generic for-each loop that takes a pass over the
>>>> client_lru list and calls some function.  The next few patches will
>>>> update other operations to use this function as well.  A value of 0
>>>> still means "forget everything that is found".
>>>>
>>>> Signed-off-by: Bryan Schumaker <bjschuma@netapp.com> ---
>>>> fs/nfsd/nfs4state.c | 23 ++++++++++++++++++----- 1 file changed, 18
>>>> insertions(+), 5 deletions(-)
>>>>
>>>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index
>>>> 050a35e..07abca5 100644 --- a/fs/nfsd/nfs4state.c +++
>>>> b/fs/nfsd/nfs4state.c @@ -4591,19 +4591,32 @@
>>>> nfs4_check_open_reclaim(clientid_t *clid, bool sessions, struct
>>>> nfsd_net *nn)
>>>>  
>>>>  #ifdef CONFIG_NFSD_FAULT_INJECTION
>>>>  
>>>> -void nfsd_forget_clients(u64 num) +u64 nfsd_forget_client(struct
>>>> nfsd_net *nn, struct nfs4_client *clp, u64 max)
>>>
>>> It doesn't look like you really need nfsd_net?
>>
>> Not for this, but I find locks through the nn->ownerstr_hashtable
>> since the nfs4_client doesn't have a "cl_locks" list similar to
>> "cl_openowners" (unless there is something I'm missing...?).
> 
> That's because lockowners are all reachable from openowners (e.g. see
> how it's done in
> release_openowner->unhash_openowner->release_open_stateid->unhash_open_stateid->release_stateid_lockowners.
> Yeah, it's a little tangled.).
> 
> Alternatively you can get to the network namespace from the client
> (clp->net).

Both are good to know!  I'll look at getting lockowners from openowners first.

- Bryan

> 
> --b.
> 
>> I
>> figured I would introduce the function with the nfsd_net pointer
>> early, rather than change it in the next patch.
>>
>> I suppose I could have also put the locks patch first...
>>
>> - Bryan
>>
>>>
>>> --b.
>>>
>>>> +{ +	nfsd4_client_record_remove(clp); +
>>>> expire_client(clp); +	return 1; +} + +static u64
>>>> nfsd_for_n_state(u64 max, u64 (*func)(struct nfsd_net *, struct
>>>> nfs4_client *, u64)) { struct nfs4_client *clp, *next; -	int
>>>> count = 0; +	u64 count = 0; struct nfsd_net *nn =
>>>> net_generic(current->nsproxy->net_ns, nfsd_net_id);
>>>>  
>>>>  	list_for_each_entry_safe(clp, next, &nn->client_lru, cl_lru) { -
>>>>  	expire_client(clp); -		if (++count == num) +
>>>>  	count += func(nn, clp, max - count); +		if ((max != 0)
>>>>  	&& (count >= max)) break; }
>>>>  
>>>> -	printk(KERN_INFO "NFSD: Forgot %d clients", count); +	return
>>>> count; +} + +void nfsd_forget_clients(u64 num) +{ +	u64
>>>> count = nfsd_for_n_state(num, nfsd_forget_client); +
>>>> printk(KERN_INFO "NFSD: Forgot %llu clients", count); }
>>>>  
>>>>  static void release_lockowner_sop(struct nfs4_stateowner *sop) --
>>>>  1.8.0.1
>>>>
>>
>> -- To unsubscribe from this list: send the line "unsubscribe
>> linux-nfs" in the body of a message to majordomo@vger.kernel.org More
>> majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v3 03/10] NFSD: Clean up forgetting clients

[J. Bruce Fields]

On Wed, Nov 28, 2012 at 11:49:53AM -0500, Bryan Schumaker wrote:
> On 11/28/2012 11:47 AM, J. Bruce Fields wrote:
> > On Wed, Nov 28, 2012 at 11:34:40AM -0500, Bryan Schumaker wrote:
> >> On 11/28/2012 11:29 AM, J. Bruce Fields wrote:
> >>> On Tue, Nov 27, 2012 at 09:35:12AM -0500, bjschuma@netapp.com wrote:
> >>>> From: Bryan Schumaker <bjschuma@netapp.com>
> >>>>
> >>>> I added in a generic for-each loop that takes a pass over the
> >>>> client_lru list and calls some function.  The next few patches will
> >>>> update other operations to use this function as well.  A value of 0
> >>>> still means "forget everything that is found".
> >>>>
> >>>> Signed-off-by: Bryan Schumaker <bjschuma@netapp.com> ---
> >>>> fs/nfsd/nfs4state.c | 23 ++++++++++++++++++----- 1 file changed, 18
> >>>> insertions(+), 5 deletions(-)
> >>>>
> >>>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index
> >>>> 050a35e..07abca5 100644 --- a/fs/nfsd/nfs4state.c +++
> >>>> b/fs/nfsd/nfs4state.c @@ -4591,19 +4591,32 @@
> >>>> nfs4_check_open_reclaim(clientid_t *clid, bool sessions, struct
> >>>> nfsd_net *nn)
> >>>>  
> >>>>  #ifdef CONFIG_NFSD_FAULT_INJECTION
> >>>>  
> >>>> -void nfsd_forget_clients(u64 num) +u64 nfsd_forget_client(struct
> >>>> nfsd_net *nn, struct nfs4_client *clp, u64 max)
> >>>
> >>> It doesn't look like you really need nfsd_net?
> >>
> >> Not for this, but I find locks through the nn->ownerstr_hashtable
> >> since the nfs4_client doesn't have a "cl_locks" list similar to
> >> "cl_openowners" (unless there is something I'm missing...?).
> > 
> > That's because lockowners are all reachable from openowners (e.g. see
> > how it's done in
> > release_openowner->unhash_openowner->release_open_stateid->unhash_open_stateid->release_stateid_lockowners.
> > Yeah, it's a little tangled.).
> > 
> > Alternatively you can get to the network namespace from the client
> > (clp->net).
> 
> Both are good to know!  I'll look at getting lockowners from openowners first.

OK.  I've also updated my for-3.8 with more of Stanislav's
patches--doesn't look like there should be any interesting new
conflicts, but please check.

--b.

Re: [PATCH v3 03/10] NFSD: Clean up forgetting clients

[J. Bruce Fields]

On Wed, Nov 28, 2012 at 11:56:42AM -0500, J. Bruce Fields wrote:
> On Wed, Nov 28, 2012 at 11:49:53AM -0500, Bryan Schumaker wrote:
> > On 11/28/2012 11:47 AM, J. Bruce Fields wrote:
> > > On Wed, Nov 28, 2012 at 11:34:40AM -0500, Bryan Schumaker wrote:
> > >> On 11/28/2012 11:29 AM, J. Bruce Fields wrote:
> > >>> On Tue, Nov 27, 2012 at 09:35:12AM -0500, bjschuma@netapp.com wrote:
> > >>>> From: Bryan Schumaker <bjschuma@netapp.com>
> > >>>>
> > >>>> I added in a generic for-each loop that takes a pass over the
> > >>>> client_lru list and calls some function.  The next few patches will
> > >>>> update other operations to use this function as well.  A value of 0
> > >>>> still means "forget everything that is found".
> > >>>>
> > >>>> Signed-off-by: Bryan Schumaker <bjschuma@netapp.com> ---
> > >>>> fs/nfsd/nfs4state.c | 23 ++++++++++++++++++----- 1 file changed, 18
> > >>>> insertions(+), 5 deletions(-)
> > >>>>
> > >>>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index
> > >>>> 050a35e..07abca5 100644 --- a/fs/nfsd/nfs4state.c +++
> > >>>> b/fs/nfsd/nfs4state.c @@ -4591,19 +4591,32 @@
> > >>>> nfs4_check_open_reclaim(clientid_t *clid, bool sessions, struct
> > >>>> nfsd_net *nn)
> > >>>>  
> > >>>>  #ifdef CONFIG_NFSD_FAULT_INJECTION
> > >>>>  
> > >>>> -void nfsd_forget_clients(u64 num) +u64 nfsd_forget_client(struct
> > >>>> nfsd_net *nn, struct nfs4_client *clp, u64 max)
> > >>>
> > >>> It doesn't look like you really need nfsd_net?
> > >>
> > >> Not for this, but I find locks through the nn->ownerstr_hashtable
> > >> since the nfs4_client doesn't have a "cl_locks" list similar to
> > >> "cl_openowners" (unless there is something I'm missing...?).
> > > 
> > > That's because lockowners are all reachable from openowners (e.g. see
> > > how it's done in
> > > release_openowner->unhash_openowner->release_open_stateid->unhash_open_stateid->release_stateid_lockowners.
> > > Yeah, it's a little tangled.).
> > > 
> > > Alternatively you can get to the network namespace from the client
> > > (clp->net).
> > 
> > Both are good to know!  I'll look at getting lockowners from openowners first.
> 
> OK.  I've also updated my for-3.8 with more of Stanislav's
> patches--doesn't look like there should be any interesting new
> conflicts, but please check.

Oh, and maybe you can respin with that last bugfix folded in to the
right place, if that makes sense?

--b.

Re: [PATCH v3 03/10] NFSD: Clean up forgetting clients

[Bryan Schumaker]

On 11/28/2012 11:57 AM, J. Bruce Fields wrote:
> On Wed, Nov 28, 2012 at 11:56:42AM -0500, J. Bruce Fields wrote:
>> On Wed, Nov 28, 2012 at 11:49:53AM -0500, Bryan Schumaker wrote:
>>> On 11/28/2012 11:47 AM, J. Bruce Fields wrote:
>>>> On Wed, Nov 28, 2012 at 11:34:40AM -0500, Bryan Schumaker wrote:
>>>>> On 11/28/2012 11:29 AM, J. Bruce Fields wrote:
>>>>>> On Tue, Nov 27, 2012 at 09:35:12AM -0500, bjschuma@netapp.com wrote:
>>>>>>> From: Bryan Schumaker <bjschuma@netapp.com>
>>>>>>>
>>>>>>> I added in a generic for-each loop that takes a pass over the
>>>>>>> client_lru list and calls some function.  The next few patches will
>>>>>>> update other operations to use this function as well.  A value of 0
>>>>>>> still means "forget everything that is found".
>>>>>>>
>>>>>>> Signed-off-by: Bryan Schumaker <bjschuma@netapp.com> ---
>>>>>>> fs/nfsd/nfs4state.c | 23 ++++++++++++++++++----- 1 file changed, 18
>>>>>>> insertions(+), 5 deletions(-)
>>>>>>>
>>>>>>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index
>>>>>>> 050a35e..07abca5 100644 --- a/fs/nfsd/nfs4state.c +++
>>>>>>> b/fs/nfsd/nfs4state.c @@ -4591,19 +4591,32 @@
>>>>>>> nfs4_check_open_reclaim(clientid_t *clid, bool sessions, struct
>>>>>>> nfsd_net *nn)
>>>>>>>  
>>>>>>>  #ifdef CONFIG_NFSD_FAULT_INJECTION
>>>>>>>  
>>>>>>> -void nfsd_forget_clients(u64 num) +u64 nfsd_forget_client(struct
>>>>>>> nfsd_net *nn, struct nfs4_client *clp, u64 max)
>>>>>>
>>>>>> It doesn't look like you really need nfsd_net?
>>>>>
>>>>> Not for this, but I find locks through the nn->ownerstr_hashtable
>>>>> since the nfs4_client doesn't have a "cl_locks" list similar to
>>>>> "cl_openowners" (unless there is something I'm missing...?).
>>>>
>>>> That's because lockowners are all reachable from openowners (e.g. see
>>>> how it's done in
>>>> release_openowner->unhash_openowner->release_open_stateid->unhash_open_stateid->release_stateid_lockowners.
>>>> Yeah, it's a little tangled.).
>>>>
>>>> Alternatively you can get to the network namespace from the client
>>>> (clp->net).
>>>
>>> Both are good to know!  I'll look at getting lockowners from openowners first.
>>
>> OK.  I've also updated my for-3.8 with more of Stanislav's
>> patches--doesn't look like there should be any interesting new
>> conflicts, but please check.

I've got something with multiple levels of list_for_each() loops that I'm attempting to unroll.  I just noticed that I've been releasing lock owners and open owners this whole time.  I wonder if it would be better to release lock and open stateids instead?

- Bryan 

> 
> Oh, and maybe you can respin with that last bugfix folded in to the
> right place, if that makes sense?
> 
> --b.
> 

Re: [PATCH v3 03/10] NFSD: Clean up forgetting clients

[J. Bruce Fields]

On Wed, Nov 28, 2012 at 03:53:33PM -0500, Bryan Schumaker wrote:
> On 11/28/2012 11:57 AM, J. Bruce Fields wrote:
> > On Wed, Nov 28, 2012 at 11:56:42AM -0500, J. Bruce Fields wrote:
> >> On Wed, Nov 28, 2012 at 11:49:53AM -0500, Bryan Schumaker wrote:
> >>> On 11/28/2012 11:47 AM, J. Bruce Fields wrote:
> >>>> On Wed, Nov 28, 2012 at 11:34:40AM -0500, Bryan Schumaker wrote:
> >>>>> On 11/28/2012 11:29 AM, J. Bruce Fields wrote:
> >>>>>> On Tue, Nov 27, 2012 at 09:35:12AM -0500, bjschuma@netapp.com wrote:
> >>>>>>> From: Bryan Schumaker <bjschuma@netapp.com>
> >>>>>>>
> >>>>>>> I added in a generic for-each loop that takes a pass over the
> >>>>>>> client_lru list and calls some function.  The next few patches will
> >>>>>>> update other operations to use this function as well.  A value of 0
> >>>>>>> still means "forget everything that is found".
> >>>>>>>
> >>>>>>> Signed-off-by: Bryan Schumaker <bjschuma@netapp.com> ---
> >>>>>>> fs/nfsd/nfs4state.c | 23 ++++++++++++++++++----- 1 file changed, 18
> >>>>>>> insertions(+), 5 deletions(-)
> >>>>>>>
> >>>>>>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index
> >>>>>>> 050a35e..07abca5 100644 --- a/fs/nfsd/nfs4state.c +++
> >>>>>>> b/fs/nfsd/nfs4state.c @@ -4591,19 +4591,32 @@
> >>>>>>> nfs4_check_open_reclaim(clientid_t *clid, bool sessions, struct
> >>>>>>> nfsd_net *nn)
> >>>>>>>  
> >>>>>>>  #ifdef CONFIG_NFSD_FAULT_INJECTION
> >>>>>>>  
> >>>>>>> -void nfsd_forget_clients(u64 num) +u64 nfsd_forget_client(struct
> >>>>>>> nfsd_net *nn, struct nfs4_client *clp, u64 max)
> >>>>>>
> >>>>>> It doesn't look like you really need nfsd_net?
> >>>>>
> >>>>> Not for this, but I find locks through the nn->ownerstr_hashtable
> >>>>> since the nfs4_client doesn't have a "cl_locks" list similar to
> >>>>> "cl_openowners" (unless there is something I'm missing...?).
> >>>>
> >>>> That's because lockowners are all reachable from openowners (e.g. see
> >>>> how it's done in
> >>>> release_openowner->unhash_openowner->release_open_stateid->unhash_open_stateid->release_stateid_lockowners.
> >>>> Yeah, it's a little tangled.).
> >>>>
> >>>> Alternatively you can get to the network namespace from the client
> >>>> (clp->net).
> >>>
> >>> Both are good to know!  I'll look at getting lockowners from openowners first.
> >>
> >> OK.  I've also updated my for-3.8 with more of Stanislav's
> >> patches--doesn't look like there should be any interesting new
> >> conflicts, but please check.
> 
> I've got something with multiple levels of list_for_each() loops that I'm attempting to unroll.  I just noticed that I've been releasing lock owners and open owners this whole time.  I wonder if it would be better to release lock and open stateids instead?

Could be, I don't know.  I think stateid's always have well-defined
lifetimes, don't they?  Whereas (at least in the 4.0 case?), it may be
up to the server to decide when exactly an open owner goes away.  If
that's right, counting stateid's may be more likely to give you
reproduceable test results.

--b.

Re: [PATCH v3 03/10] NFSD: Clean up forgetting clients

[Bryan Schumaker]

On 11/28/2012 11:56 AM, J. Bruce Fields wrote:
> On Wed, Nov 28, 2012 at 11:49:53AM -0500, Bryan Schumaker wrote:
>> On 11/28/2012 11:47 AM, J. Bruce Fields wrote:
>>> On Wed, Nov 28, 2012 at 11:34:40AM -0500, Bryan Schumaker wrote:
>>>> On 11/28/2012 11:29 AM, J. Bruce Fields wrote:
>>>>> On Tue, Nov 27, 2012 at 09:35:12AM -0500, bjschuma@netapp.com wrote:
>>>>>> From: Bryan Schumaker <bjschuma@netapp.com>
>>>>>>
>>>>>> I added in a generic for-each loop that takes a pass over the
>>>>>> client_lru list and calls some function.  The next few patches will
>>>>>> update other operations to use this function as well.  A value of 0
>>>>>> still means "forget everything that is found".
>>>>>>
>>>>>> Signed-off-by: Bryan Schumaker <bjschuma@netapp.com> ---
>>>>>> fs/nfsd/nfs4state.c | 23 ++++++++++++++++++----- 1 file changed, 18
>>>>>> insertions(+), 5 deletions(-)
>>>>>>
>>>>>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index
>>>>>> 050a35e..07abca5 100644 --- a/fs/nfsd/nfs4state.c +++
>>>>>> b/fs/nfsd/nfs4state.c @@ -4591,19 +4591,32 @@
>>>>>> nfs4_check_open_reclaim(clientid_t *clid, bool sessions, struct
>>>>>> nfsd_net *nn)
>>>>>>  
>>>>>>  #ifdef CONFIG_NFSD_FAULT_INJECTION
>>>>>>  
>>>>>> -void nfsd_forget_clients(u64 num) +u64 nfsd_forget_client(struct
>>>>>> nfsd_net *nn, struct nfs4_client *clp, u64 max)
>>>>>
>>>>> It doesn't look like you really need nfsd_net?
>>>>
>>>> Not for this, but I find locks through the nn->ownerstr_hashtable
>>>> since the nfs4_client doesn't have a "cl_locks" list similar to
>>>> "cl_openowners" (unless there is something I'm missing...?).
>>>
>>> That's because lockowners are all reachable from openowners (e.g. see
>>> how it's done in
>>> release_openowner->unhash_openowner->release_open_stateid->unhash_open_stateid->release_stateid_lockowners.
>>> Yeah, it's a little tangled.).
>>>
>>> Alternatively you can get to the network namespace from the client
>>> (clp->net).
>>
>> Both are good to know!  I'll look at getting lockowners from openowners first.
> 
> OK.  I've also updated my for-3.8 with more of Stanislav's
> patches--doesn't look like there should be any interesting new
> conflicts, but please check.

Will do.  I'll also work in Sasha Levin's patch from yesterday while I'm updating things.

- Bryan
> 
> --b.
> 

[PATCH v3 04/10] NFSD: Clean up forgetting locks

[bjschuma]

From: Bryan Schumaker <bjschuma@netapp.com>

I use the new "forget_n_state()" function to iterate through each client
first when searching for locks.  This may slow down forgetting locks a
little bit, but it implements most of the code needed to forget a
specified client's locks.

Signed-off-by: Bryan Schumaker <bjschuma@netapp.com>
---
 fs/nfsd/nfs4state.c | 36 ++++++++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 8 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 07abca5..c931e65 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -4598,6 +4598,32 @@ u64 nfsd_forget_client(struct nfsd_net *nn, struct nfs4_client *clp, u64 max)
 	return 1;
 }
 
+static u64 nfsd_foreach_client_lock(struct nfsd_net *nn, struct nfs4_client *clp,
+				    u64 max, void (*func)(struct nfs4_lockowner *))
+{
+	struct nfs4_stateowner *sop, *next;
+	u64 count = 0;
+	int i;
+
+	for (i = 0; i < OWNER_HASH_SIZE; i++) {
+		list_for_each_entry_safe(sop, next, &nn->ownerstr_hashtbl[i], so_strhash) {
+			if (!sop->so_is_open_owner && (sop->so_client == clp)) {
+				if (func)
+					func(lockowner(sop));
+				if (++count == max)
+					break;
+			}
+		}
+	}
+
+	return count;
+}
+
+u64 nfsd_forget_client_locks(struct nfsd_net *nn, struct nfs4_client *clp, u64 max)
+{
+	return nfsd_foreach_client_lock(nn, clp, max, release_lockowner);
+}
+
 static u64 nfsd_for_n_state(u64 max, u64 (*func)(struct nfsd_net *, struct nfs4_client *, u64))
 {
 	struct nfs4_client *clp, *next;
@@ -4619,11 +4645,6 @@ void nfsd_forget_clients(u64 num)
 	printk(KERN_INFO "NFSD: Forgot %llu clients", count);
 }
 
-static void release_lockowner_sop(struct nfs4_stateowner *sop)
-{
-	release_lockowner(lockowner(sop));
-}
-
 static void release_openowner_sop(struct nfs4_stateowner *sop)
 {
 	release_openowner(openowner(sop));
@@ -4650,9 +4671,8 @@ static int nfsd_release_n_owners(u64 num, bool is_open_owner,
 
 void nfsd_forget_locks(u64 num)
 {
-	struct nfsd_net *nn = net_generic(&init_net, nfsd_net_id);
-	int count = nfsd_release_n_owners(num, false, release_lockowner_sop, nn);
-	printk(KERN_INFO "NFSD: Forgot %d locks", count);
+	u64 count = nfsd_for_n_state(num, nfsd_forget_client_locks);
+	printk(KERN_INFO "NFSD: Forgot %llu locks", count);
 }
 
 void nfsd_forget_openowners(u64 num)
-- 
1.8.0.1

[PATCH v3 05/10] NFSD: Clean up forgetting openowners

[bjschuma]

From: Bryan Schumaker <bjschuma@netapp.com>

Using "forget_n_state()" forces me to implement the code needed to
forget a specific client's openowners.

Signed-off-by: Bryan Schumaker <bjschuma@netapp.com>
---
 fs/nfsd/nfs4state.c | 50 +++++++++++++++++++++++---------------------------
 1 file changed, 23 insertions(+), 27 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index c931e65..94c009e 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -4624,6 +4624,27 @@ u64 nfsd_forget_client_locks(struct nfsd_net *nn, struct nfs4_client *clp, u64 m
 	return nfsd_foreach_client_lock(nn, clp, max, release_lockowner);
 }
 
+static u64 nfsd_foreach_client_open(struct nfsd_net *nn, struct nfs4_client *clp,
+				    u64 max, void (*func)(struct nfs4_openowner *))
+{
+	struct nfs4_openowner *oop, *next;
+	u64 count = 0;
+
+	list_for_each_entry_safe(oop, next, &clp->cl_openowners, oo_perclient) {
+		if (func)
+			func(oop);
+		if (++count == max)
+			break;
+	}
+
+	return count;
+}
+
+u64 nfsd_forget_client_openowners(struct nfsd_net *nn, struct nfs4_client *clp, u64 max)
+{
+	return nfsd_foreach_client_open(nn, clp, max, release_openowner);
+}
+
 static u64 nfsd_for_n_state(u64 max, u64 (*func)(struct nfsd_net *, struct nfs4_client *, u64))
 {
 	struct nfs4_client *clp, *next;
@@ -4645,30 +4666,6 @@ void nfsd_forget_clients(u64 num)
 	printk(KERN_INFO "NFSD: Forgot %llu clients", count);
 }
 
-static void release_openowner_sop(struct nfs4_stateowner *sop)
-{
-	release_openowner(openowner(sop));
-}
-
-static int nfsd_release_n_owners(u64 num, bool is_open_owner,
-				void (*release_sop)(struct nfs4_stateowner *),
-				struct nfsd_net *nn)
-{
-	int i, count = 0;
-	struct nfs4_stateowner *sop, *next;
-
-	for (i = 0; i < OWNER_HASH_SIZE; i++) {
-		list_for_each_entry_safe(sop, next, &nn->ownerstr_hashtbl[i], so_strhash) {
-			if (sop->so_is_open_owner != is_open_owner)
-				continue;
-			release_sop(sop);
-			if (++count == num)
-				return count;
-		}
-	}
-	return count;
-}
-
 void nfsd_forget_locks(u64 num)
 {
 	u64 count = nfsd_for_n_state(num, nfsd_forget_client_locks);
@@ -4677,9 +4674,8 @@ void nfsd_forget_locks(u64 num)
 
 void nfsd_forget_openowners(u64 num)
 {
-	struct nfsd_net *nn = net_generic(&init_net, nfsd_net_id);
-	int count = nfsd_release_n_owners(num, true, release_openowner_sop, nn);
-	printk(KERN_INFO "NFSD: Forgot %d open owners", count);
+	u64 count = nfsd_for_n_state(num, nfsd_forget_client_openowners);
+	printk(KERN_INFO "NFSD: Forgot %llu open owners", count);
 }
 
 static int nfsd_process_n_delegations(u64 num, struct list_head *list)
-- 
1.8.0.1

[PATCH v3 06/10] NFSD: Clean up forgetting and recalling delegations

[bjschuma]

From: Bryan Schumaker <bjschuma@netapp.com>

Once I have a client, I can easily use its delegation list rather than
searching the file hash table for delegations to remove.

Signed-off-by: Bryan Schumaker <bjschuma@netapp.com>
---
 fs/nfsd/nfs4state.c | 94 ++++++++++++++++++++++++++++-------------------------
 1 file changed, 50 insertions(+), 44 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 94c009e..84bd4c4 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -4645,6 +4645,52 @@ u64 nfsd_forget_client_openowners(struct nfsd_net *nn, struct nfs4_client *clp,
 	return nfsd_foreach_client_open(nn, clp, max, release_openowner);
 }
 
+static u64 nfsd_find_all_delegations(struct nfs4_client *clp, u64 max,
+				     struct list_head *victims)
+{
+	struct nfs4_delegation *dp, *next;
+	u64 count = 0;
+
+	list_for_each_entry_safe(dp, next, &clp->cl_delegations, dl_perclnt) {
+		if (victims)
+			list_move(&dp->dl_recall_lru, victims);
+		if (++count == max)
+			break;
+	}
+	return count;
+}
+
+u64 nfsd_forget_client_delegations(struct nfsd_net *nn, struct nfs4_client *clp, u64 max)
+{
+	struct nfs4_delegation *dp, *next;
+	LIST_HEAD(victims);
+	u64 count;
+
+	spin_lock(&recall_lock);
+	count = nfsd_find_all_delegations(clp, max, &victims);
+	spin_unlock(&recall_lock);
+
+	list_for_each_entry_safe(dp, next, &victims, dl_recall_lru)
+		unhash_delegation(dp);
+
+	return count;
+}
+
+u64 nfsd_recall_client_delegations(struct nfsd_net *nn, struct nfs4_client *clp, u64 max)
+{
+	struct nfs4_delegation *dp, *next;
+	LIST_HEAD(victims);
+	u64 count;
+
+	spin_lock(&recall_lock);
+	count = nfsd_find_all_delegations(clp, max, &victims);
+	list_for_each_entry_safe(dp, next, &victims, dl_recall_lru)
+		nfsd_break_one_deleg(dp);
+	spin_unlock(&recall_lock);
+
+	return count;
+}
+
 static u64 nfsd_for_n_state(u64 max, u64 (*func)(struct nfsd_net *, struct nfs4_client *, u64))
 {
 	struct nfs4_client *clp, *next;
@@ -4678,56 +4724,16 @@ void nfsd_forget_openowners(u64 num)
 	printk(KERN_INFO "NFSD: Forgot %llu open owners", count);
 }
 
-static int nfsd_process_n_delegations(u64 num, struct list_head *list)
-{
-	int i, count = 0;
-	struct nfs4_file *fp, *fnext;
-	struct nfs4_delegation *dp, *dnext;
-
-	for (i = 0; i < FILE_HASH_SIZE; i++) {
-		list_for_each_entry_safe(fp, fnext, &file_hashtbl[i], fi_hash) {
-			list_for_each_entry_safe(dp, dnext, &fp->fi_delegations, dl_perfile) {
-				list_move(&dp->dl_recall_lru, list);
-				if (++count == num)
-					return count;
-			}
-		}
-	}
-
-	return count;
-}
-
 void nfsd_forget_delegations(u64 num)
 {
-	unsigned int count;
-	LIST_HEAD(victims);
-	struct nfs4_delegation *dp, *dnext;
-
-	spin_lock(&recall_lock);
-	count = nfsd_process_n_delegations(num, &victims);
-	spin_unlock(&recall_lock);
-
-	list_for_each_entry_safe(dp, dnext, &victims, dl_recall_lru)
-		unhash_delegation(dp);
-
-	printk(KERN_INFO "NFSD: Forgot %d delegations", count);
+	u64 count = nfsd_for_n_state(num, nfsd_forget_client_delegations);
+	printk(KERN_INFO "NFSD: Forgot %llu delegations", count);
 }
 
 void nfsd_recall_delegations(u64 num)
 {
-	unsigned int count;
-	LIST_HEAD(victims);
-	struct nfs4_delegation *dp, *dnext;
-
-	spin_lock(&recall_lock);
-	count = nfsd_process_n_delegations(num, &victims);
-	list_for_each_entry_safe(dp, dnext, &victims, dl_recall_lru) {
-		list_del(&dp->dl_recall_lru);
-		nfsd_break_one_deleg(dp);
-	}
-	spin_unlock(&recall_lock);
-
-	printk(KERN_INFO "NFSD: Recalled %d delegations", count);
+	u64 count = nfsd_for_n_state(num, nfsd_recall_client_delegations);
+	printk(KERN_INFO "NFSD: Recalled %llu delegations", count);
 }
 
 #endif /* CONFIG_NFSD_FAULT_INJECTION */
-- 
1.8.0.1

[PATCH v3 07/10] NFSD: Fault injection operations take a per-client forget function

[bjschuma]

From: Bryan Schumaker <bjschuma@netapp.com>

The eventual goal is to forget state based on ip address, so it makes
sense to call this function in a for-each-client loop until the correct
amount of state is forgotten.  I also use this patch as an opportunity
to rename the forget function from "func()" to "forget()".

Signed-off-by: Bryan Schumaker <bjschuma@netapp.com>
---
 fs/nfsd/fault_inject.c | 16 +++++++++-------
 fs/nfsd/nfs4state.c    | 32 +-------------------------------
 fs/nfsd/state.h        | 12 +++++++-----
 3 files changed, 17 insertions(+), 43 deletions(-)

diff --git a/fs/nfsd/fault_inject.c b/fs/nfsd/fault_inject.c
index 4b385a1..bdef6a8 100644
--- a/fs/nfsd/fault_inject.c
+++ b/fs/nfsd/fault_inject.c
@@ -13,29 +13,29 @@
 
 struct nfsd_fault_inject_op {
 	char *file;
-	void (*func)(u64);
+	u64 (*forget)(struct nfsd_net *, struct nfs4_client *, u64);
 };
 
 static struct nfsd_fault_inject_op inject_ops[] = {
 	{
 		.file   = "forget_clients",
-		.func   = nfsd_forget_clients,
+		.forget = nfsd_forget_client,
 	},
 	{
 		.file   = "forget_locks",
-		.func   = nfsd_forget_locks,
+		.forget = nfsd_forget_client_locks,
 	},
 	{
 		.file   = "forget_openowners",
-		.func   = nfsd_forget_openowners,
+		.forget = nfsd_forget_client_openowners,
 	},
 	{
 		.file   = "forget_delegations",
-		.func   = nfsd_forget_delegations,
+		.forget = nfsd_forget_client_delegations,
 	},
 	{
 		.file   = "recall_delegations",
-		.func   = nfsd_recall_delegations,
+		.forget = nfsd_recall_client_delegations,
 	},
 };
 
@@ -44,6 +44,7 @@ static struct dentry *debug_dir;
 
 static int nfsd_inject_set(void *op_ptr, u64 val)
 {
+	u64 count = 0;
 	struct nfsd_fault_inject_op *op = op_ptr;
 
 	if (val == 0)
@@ -52,8 +53,9 @@ static int nfsd_inject_set(void *op_ptr, u64 val)
 		printk(KERN_INFO "NFSD Fault Injection: %s (n = %llu)", op->file, val);
 
 	nfs4_lock_state();
-	op->func(val);
+	count = nfsd_for_n_state(val, op->forget);
 	nfs4_unlock_state();
+	printk(KERN_INFO "NFSD: %s: found %llu", op->file, count);
 	return 0;
 }
 
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 84bd4c4..ba41006 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -4691,7 +4691,7 @@ u64 nfsd_recall_client_delegations(struct nfsd_net *nn, struct nfs4_client *clp,
 	return count;
 }
 
-static u64 nfsd_for_n_state(u64 max, u64 (*func)(struct nfsd_net *, struct nfs4_client *, u64))
+u64 nfsd_for_n_state(u64 max, u64 (*func)(struct nfsd_net *, struct nfs4_client *, u64))
 {
 	struct nfs4_client *clp, *next;
 	u64 count = 0;
@@ -4706,36 +4706,6 @@ static u64 nfsd_for_n_state(u64 max, u64 (*func)(struct nfsd_net *, struct nfs4_
 	return count;
 }
 
-void nfsd_forget_clients(u64 num)
-{
-	u64 count = nfsd_for_n_state(num, nfsd_forget_client);
-	printk(KERN_INFO "NFSD: Forgot %llu clients", count);
-}
-
-void nfsd_forget_locks(u64 num)
-{
-	u64 count = nfsd_for_n_state(num, nfsd_forget_client_locks);
-	printk(KERN_INFO "NFSD: Forgot %llu locks", count);
-}
-
-void nfsd_forget_openowners(u64 num)
-{
-	u64 count = nfsd_for_n_state(num, nfsd_forget_client_openowners);
-	printk(KERN_INFO "NFSD: Forgot %llu open owners", count);
-}
-
-void nfsd_forget_delegations(u64 num)
-{
-	u64 count = nfsd_for_n_state(num, nfsd_forget_client_delegations);
-	printk(KERN_INFO "NFSD: Forgot %llu delegations", count);
-}
-
-void nfsd_recall_delegations(u64 num)
-{
-	u64 count = nfsd_for_n_state(num, nfsd_recall_client_delegations);
-	printk(KERN_INFO "NFSD: Recalled %llu delegations", count);
-}
-
 #endif /* CONFIG_NFSD_FAULT_INJECTION */
 
 /* initialization to perform at module load time: */
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index b542bf2..4d836ed 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -501,11 +501,13 @@ extern void nfsd4_record_grace_done(struct nfsd_net *nn, time_t boot_time);
 #ifdef CONFIG_NFSD_FAULT_INJECTION
 int nfsd_fault_inject_init(void);
 void nfsd_fault_inject_cleanup(void);
-void nfsd_forget_clients(u64);
-void nfsd_forget_locks(u64);
-void nfsd_forget_openowners(u64);
-void nfsd_forget_delegations(u64);
-void nfsd_recall_delegations(u64);
+u64 nfsd_for_n_state(u64, u64 (*)(struct nfsd_net *, struct nfs4_client *, u64));
+
+u64 nfsd_forget_client(struct nfsd_net *, struct nfs4_client *, u64);
+u64 nfsd_forget_client_locks(struct nfsd_net *, struct nfs4_client*, u64);
+u64 nfsd_forget_client_openowners(struct nfsd_net *, struct nfs4_client *, u64);
+u64 nfsd_forget_client_delegations(struct nfsd_net *, struct nfs4_client *, u64);
+u64 nfsd_recall_client_delegations(struct nfsd_net *, struct nfs4_client *, u64);
 #else /* CONFIG_NFSD_FAULT_INJECTION */
 static inline int nfsd_fault_inject_init(void) { return 0; }
 static inline void nfsd_fault_inject_cleanup(void) {}
-- 
1.8.0.1

[PATCH v3 08/10] NFSD: Reading a fault injection file prints a state count

[bjschuma]

From: Bryan Schumaker <bjschuma@netapp.com>

I also log basic information that I can figure out about the type of
state (such as number of locks for each client IP address).  This can be
useful for checking that state was actually dropped and later for
checking if the client was able to recover.

Signed-off-by: Bryan Schumaker <bjschuma@netapp.com>
---
 fs/nfsd/fault_inject.c | 13 +++++++++++--
 fs/nfsd/nfs4state.c    | 42 ++++++++++++++++++++++++++++++++++++++++++
 fs/nfsd/state.h        |  5 +++++
 3 files changed, 58 insertions(+), 2 deletions(-)

diff --git a/fs/nfsd/fault_inject.c b/fs/nfsd/fault_inject.c
index bdef6a8..43a591b 100644
--- a/fs/nfsd/fault_inject.c
+++ b/fs/nfsd/fault_inject.c
@@ -14,28 +14,34 @@
 struct nfsd_fault_inject_op {
 	char *file;
 	u64 (*forget)(struct nfsd_net *, struct nfs4_client *, u64);
+	u64 (*print)(struct nfsd_net *, struct nfs4_client *, u64);
 };
 
 static struct nfsd_fault_inject_op inject_ops[] = {
 	{
 		.file   = "forget_clients",
 		.forget = nfsd_forget_client,
+		.print  = nfsd_print_client,
 	},
 	{
 		.file   = "forget_locks",
 		.forget = nfsd_forget_client_locks,
+		.print  = nfsd_print_client_locks,
 	},
 	{
 		.file   = "forget_openowners",
 		.forget = nfsd_forget_client_openowners,
+		.print  = nfsd_print_client_openowners,
 	},
 	{
 		.file   = "forget_delegations",
 		.forget = nfsd_forget_client_delegations,
+		.print  = nfsd_print_client_delegations,
 	},
 	{
 		.file   = "recall_delegations",
 		.forget = nfsd_recall_client_delegations,
+		.print  = nfsd_print_client_delegations,
 	},
 };
 
@@ -59,9 +65,12 @@ static int nfsd_inject_set(void *op_ptr, u64 val)
 	return 0;
 }
 
-static int nfsd_inject_get(void *data, u64 *val)
+static int nfsd_inject_get(void *op_ptr, u64 *val)
 {
-	*val = 0;
+	struct nfsd_fault_inject_op *op = op_ptr;
+	nfs4_lock_state();
+	*val = nfsd_for_n_state(0, op->print);
+	nfs4_unlock_state();
 	return 0;
 }
 
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index ba41006..cf2082d 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -4591,6 +4591,22 @@ nfs4_check_open_reclaim(clientid_t *clid, bool sessions, struct nfsd_net *nn)
 
 #ifdef CONFIG_NFSD_FAULT_INJECTION
 
+u64 nfsd_print_client(struct nfsd_net *nn, struct nfs4_client *clp, u64 num)
+{
+	char buf[INET6_ADDRSTRLEN];
+	rpc_ntop((struct sockaddr *)&clp->cl_addr, buf, 129);
+	printk(KERN_INFO "NFS Client: %s\n", buf);
+	return 1;
+}
+
+static void nfsd_print_count(struct nfs4_client *clp, unsigned int count,
+			     const char *type)
+{
+	char buf[INET6_ADDRSTRLEN];
+	rpc_ntop((struct sockaddr *)&clp->cl_addr, buf, 129);
+	printk(KERN_INFO "NFS Client: %s has %u %s\n", buf, count, type);
+}
+
 u64 nfsd_forget_client(struct nfsd_net *nn, struct nfs4_client *clp, u64 max)
 {
 	nfsd4_client_record_remove(clp);
@@ -4624,6 +4640,13 @@ u64 nfsd_forget_client_locks(struct nfsd_net *nn, struct nfs4_client *clp, u64 m
 	return nfsd_foreach_client_lock(nn, clp, max, release_lockowner);
 }
 
+u64 nfsd_print_client_locks(struct nfsd_net *nn, struct nfs4_client *clp, u64 max)
+{
+	u64 count = nfsd_foreach_client_lock(nn, clp, max, NULL);
+	nfsd_print_count(clp, count, "locked files");
+	return count;
+}
+
 static u64 nfsd_foreach_client_open(struct nfsd_net *nn, struct nfs4_client *clp,
 				    u64 max, void (*func)(struct nfs4_openowner *))
 {
@@ -4645,6 +4668,13 @@ u64 nfsd_forget_client_openowners(struct nfsd_net *nn, struct nfs4_client *clp,
 	return nfsd_foreach_client_open(nn, clp, max, release_openowner);
 }
 
+u64 nfsd_print_client_openowners(struct nfsd_net *nn, struct nfs4_client *clp, u64 max)
+{
+	u64 count = nfsd_foreach_client_open(nn, clp, max, NULL);
+	nfsd_print_count(clp, count, "open files");
+	return count;
+}
+
 static u64 nfsd_find_all_delegations(struct nfs4_client *clp, u64 max,
 				     struct list_head *victims)
 {
@@ -4691,6 +4721,18 @@ u64 nfsd_recall_client_delegations(struct nfsd_net *nn, struct nfs4_client *clp,
 	return count;
 }
 
+u64 nfsd_print_client_delegations(struct nfsd_net *nn, struct nfs4_client *clp, u64 max)
+{
+	u64 count = 0;
+
+	spin_lock(&recall_lock);
+	count = nfsd_find_all_delegations(clp, max, NULL);
+	spin_unlock(&recall_lock);
+
+	nfsd_print_count(clp, count, "delegations");
+	return count;
+}
+
 u64 nfsd_for_n_state(u64 max, u64 (*func)(struct nfsd_net *, struct nfs4_client *, u64))
 {
 	struct nfs4_client *clp, *next;
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index 4d836ed..1878234 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -508,6 +508,11 @@ u64 nfsd_forget_client_locks(struct nfsd_net *, struct nfs4_client*, u64);
 u64 nfsd_forget_client_openowners(struct nfsd_net *, struct nfs4_client *, u64);
 u64 nfsd_forget_client_delegations(struct nfsd_net *, struct nfs4_client *, u64);
 u64 nfsd_recall_client_delegations(struct nfsd_net *, struct nfs4_client *, u64);
+
+u64 nfsd_print_client(struct nfsd_net *, struct nfs4_client *, u64);
+u64 nfsd_print_client_locks(struct nfsd_net *, struct nfs4_client *, u64);
+u64 nfsd_print_client_openowners(struct nfsd_net *, struct nfs4_client *, u64);
+u64 nfsd_print_client_delegations(struct nfsd_net *, struct nfs4_client *, u64);
 #else /* CONFIG_NFSD_FAULT_INJECTION */
 static inline int nfsd_fault_inject_init(void) { return 0; }
 static inline void nfsd_fault_inject_cleanup(void) {}
-- 
1.8.0.1

[PATCH v3 09/10] NFSD: Add a custom file operations structure for fault injection

[bjschuma]

From: Bryan Schumaker <bjschuma@netapp.com>

Controlling the read and write functions allows me to add in "forget
client w.x.y.z", since we won't be limited to reading and writing only
u64 values.

Signed-off-by: Bryan Schumaker <bjschuma@netapp.com>
---
 fs/nfsd/fault_inject.c | 56 +++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 49 insertions(+), 7 deletions(-)

diff --git a/fs/nfsd/fault_inject.c b/fs/nfsd/fault_inject.c
index 43a591b..bf3b869 100644
--- a/fs/nfsd/fault_inject.c
+++ b/fs/nfsd/fault_inject.c
@@ -8,6 +8,7 @@
 #include <linux/fs.h>
 #include <linux/debugfs.h>
 #include <linux/module.h>
+#include <asm/uaccess.h>
 
 #include "state.h"
 
@@ -48,10 +49,9 @@ static struct nfsd_fault_inject_op inject_ops[] = {
 static long int NUM_INJECT_OPS = sizeof(inject_ops) / sizeof(struct nfsd_fault_inject_op);
 static struct dentry *debug_dir;
 
-static int nfsd_inject_set(void *op_ptr, u64 val)
+static void nfsd_inject_set(struct nfsd_fault_inject_op *op, u64 val)
 {
 	u64 count = 0;
-	struct nfsd_fault_inject_op *op = op_ptr;
 
 	if (val == 0)
 		printk(KERN_INFO "NFSD Fault Injection: %s (all)", op->file);
@@ -62,19 +62,61 @@ static int nfsd_inject_set(void *op_ptr, u64 val)
 	count = nfsd_for_n_state(val, op->forget);
 	nfs4_unlock_state();
 	printk(KERN_INFO "NFSD: %s: found %llu", op->file, count);
-	return 0;
 }
 
-static int nfsd_inject_get(void *op_ptr, u64 *val)
+static void nfsd_inject_get(struct nfsd_fault_inject_op *op, u64 *val)
 {
-	struct nfsd_fault_inject_op *op = op_ptr;
 	nfs4_lock_state();
 	*val = nfsd_for_n_state(0, op->print);
 	nfs4_unlock_state();
-	return 0;
 }
 
-DEFINE_SIMPLE_ATTRIBUTE(fops_nfsd, nfsd_inject_get, nfsd_inject_set, "%llu\n");
+static ssize_t fault_inject_read(struct file *file, char __user *buf,
+				 size_t len, loff_t *ppos)
+{
+	static u64 val;
+	char read_buf[25];
+	size_t size, ret;
+	loff_t pos = *ppos;
+
+	if (!pos)
+		nfsd_inject_get(file->f_dentry->d_inode->i_private, &val);
+	size = scnprintf(read_buf, sizeof(read_buf), "%llu\n", val);
+
+	if (pos < 0)
+		return -EINVAL;
+	if (pos >= size || !len)
+		return 0;
+	if (len > size - pos)
+		len = size - pos;
+	ret = copy_to_user(buf, read_buf + pos, len);
+	if (ret == len)
+		return -EFAULT;
+	len -= ret;
+	*ppos = pos + len;
+	return len;
+}
+
+static ssize_t fault_inject_write(struct file *file, const char __user *buf,
+				  size_t len, loff_t *ppos)
+{
+	char write_buf[24];
+	size_t size = min(sizeof(write_buf), len) - 1;
+	u64 val;
+
+	if (copy_from_user(write_buf, buf, size))
+		return -EFAULT;
+
+	val = simple_strtoll(write_buf, NULL, 0);
+	nfsd_inject_set(file->f_dentry->d_inode->i_private, val);
+	return len; /* on success, claim we got the whole input */
+}
+
+static const struct file_operations fops_nfsd = {
+	.owner   = THIS_MODULE,
+	.read    = fault_inject_read,
+	.write   = fault_inject_write,
+};
 
 void nfsd_fault_inject_cleanup(void)
 {
-- 
1.8.0.1

[PATCH v3 10/10] NFSD: Forget state for a specific client

[bjschuma]

From: Bryan Schumaker <bjschuma@netapp.com>

Write the client's ip address to any state file and all appropriate
state for that client will be forgotten.

Signed-off-by: Bryan Schumaker <bjschuma@netapp.com>
---
 fs/nfsd/fault_inject.c | 38 ++++++++++++++++++++++++++++++++++----
 fs/nfsd/nfs4state.c    | 12 ++++++++++++
 fs/nfsd/state.h        |  1 +
 3 files changed, 47 insertions(+), 4 deletions(-)

diff --git a/fs/nfsd/fault_inject.c b/fs/nfsd/fault_inject.c
index bf3b869..699de07 100644
--- a/fs/nfsd/fault_inject.c
+++ b/fs/nfsd/fault_inject.c
@@ -8,9 +8,12 @@
 #include <linux/fs.h>
 #include <linux/debugfs.h>
 #include <linux/module.h>
+#include <linux/nsproxy.h>
+#include <linux/sunrpc/clnt.h>
 #include <asm/uaccess.h>
 
 #include "state.h"
+#include "netns.h"
 
 struct nfsd_fault_inject_op {
 	char *file;
@@ -64,6 +67,25 @@ static void nfsd_inject_set(struct nfsd_fault_inject_op *op, u64 val)
 	printk(KERN_INFO "NFSD: %s: found %llu", op->file, count);
 }
 
+static void nfsd_inject_set_client(struct nfsd_fault_inject_op *op,
+				   struct sockaddr_storage *addr,
+				   size_t addr_size)
+{
+	char buf[INET6_ADDRSTRLEN];
+	struct nfs4_client *clp;
+	u64 count;
+	struct nfsd_net *nn = net_generic(current->nsproxy->net_ns, nfsd_net_id);
+
+	nfs4_lock_state();
+	clp = nfsd_find_client(nn, addr, addr_size);
+	if (clp) {
+		count = op->forget(nn, clp, 0);
+		rpc_ntop((struct sockaddr *)&clp->cl_addr, buf, 129);
+		printk(KERN_INFO "NFSD [%s]: Client %s had %llu state object(s)\n", op->file, buf, count);
+	}
+	nfs4_unlock_state();
+}
+
 static void nfsd_inject_get(struct nfsd_fault_inject_op *op, u64 *val)
 {
 	nfs4_lock_state();
@@ -100,15 +122,23 @@ static ssize_t fault_inject_read(struct file *file, char __user *buf,
 static ssize_t fault_inject_write(struct file *file, const char __user *buf,
 				  size_t len, loff_t *ppos)
 {
-	char write_buf[24];
+	char write_buf[INET6_ADDRSTRLEN];
 	size_t size = min(sizeof(write_buf), len) - 1;
+	struct net *net = current->nsproxy->net_ns;
+	struct sockaddr_storage sa;
 	u64 val;
 
 	if (copy_from_user(write_buf, buf, size))
 		return -EFAULT;
-
-	val = simple_strtoll(write_buf, NULL, 0);
-	nfsd_inject_set(file->f_dentry->d_inode->i_private, val);
+	write_buf[size] = '\0';
+
+	size = rpc_pton(net, write_buf, size, (struct sockaddr *)&sa, sizeof(sa));
+	if (size > 0)
+		nfsd_inject_set_client(file->f_dentry->d_inode->i_private, &sa, size);
+	else {
+		val = simple_strtoll(write_buf, NULL, 0);
+		nfsd_inject_set(file->f_dentry->d_inode->i_private, val);
+	}
 	return len; /* on success, claim we got the whole input */
 }
 
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index cf2082d..b32414e 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -4748,6 +4748,18 @@ u64 nfsd_for_n_state(u64 max, u64 (*func)(struct nfsd_net *, struct nfs4_client
 	return count;
 }
 
+struct nfs4_client *nfsd_find_client(struct nfsd_net *nn, struct sockaddr_storage *addr,
+				     size_t addr_size)
+{
+	struct nfs4_client *clp;
+
+	list_for_each_entry(clp, &nn->client_lru, cl_lru) {
+		if (memcmp(&clp->cl_addr, addr, addr_size) == 0)
+			return clp;
+	}
+	return NULL;
+}
+
 #endif /* CONFIG_NFSD_FAULT_INJECTION */
 
 /* initialization to perform at module load time: */
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index 1878234..996ce69 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -502,6 +502,7 @@ extern void nfsd4_record_grace_done(struct nfsd_net *nn, time_t boot_time);
 int nfsd_fault_inject_init(void);
 void nfsd_fault_inject_cleanup(void);
 u64 nfsd_for_n_state(u64, u64 (*)(struct nfsd_net *, struct nfs4_client *, u64));
+struct nfs4_client *nfsd_find_client(struct nfsd_net *, struct sockaddr_storage *, size_t);
 
 u64 nfsd_forget_client(struct nfsd_net *, struct nfs4_client *, u64);
 u64 nfsd_forget_client_locks(struct nfsd_net *, struct nfs4_client*, u64);
-- 
1.8.0.1