[PATCH 2/3] NFSD: Add infrastructure for tracking persistent SCSI registration keys

[Dai Ngo <dai.ngo@oracle.com>]

Introduce a per-net namespace hash table to maintain records of
NFSv4 clients that have been assigned persistent registration
keys for accessing SCSI block devices. Each entry in this table
consists of the client’s ID, the associated SCSI block device
(dev_t), and a fenced status flag.

This infrastructure is used to prevent duplicate fencing operations.

Implementation details:

When a client receives a persistent registration key for a SCSI
device the server creates a new record (with the ‘fenced’ flag set
to false) and inserts it into the hash table for the corresponding
network namespace.

When the server needs to perform a fencing operation, it consults
the client’s record to check the existing fenced status and avoids
issuing redundant fencing operations if the client is already
fenced.

Whenever a client issues a GETDEVINFO call to obtain a new layout,
the server resets the client’s fenced flag to false, allowing for
new fencing actions as appropriate.

Upon client unmount or session teardown, all records belonging to
that client are cleaned up in __destroy_client. The entire hash table
and all contained client records are freed when the corresponding
network namespace is shut down.

Signed-off-by: Dai Ngo <dai.ngo@oracle.com>
---
 fs/nfsd/blocklayout.c | 129 ++++++++++++++++++++++++++++++++++++++++++
 fs/nfsd/netns.h       |   2 +
 fs/nfsd/nfs4state.c   |   1 +
 fs/nfsd/nfsd.h        |  16 ++++++
 fs/nfsd/nfssvc.c      |   9 ++-
 fs/nfsd/pnfs.h        |  11 ++++
 6 files changed, 166 insertions(+), 2 deletions(-)

diff --git a/fs/nfsd/blocklayout.c b/fs/nfsd/blocklayout.c
index afa16d7a8013..f2ab264af88e 100644
--- a/fs/nfsd/blocklayout.c
+++ b/fs/nfsd/blocklayout.c
@@ -426,4 +426,133 @@ const struct nfsd4_layout_ops scsi_layout_ops = {
 	.proc_layoutcommit	= nfsd4_scsi_proc_layoutcommit,
 	.fence_client		= nfsd4_scsi_fence_client,
 };
+
+int
+nfsd4_scsi_pr_init_hashtbl(struct nfsd_net *nn)
+{
+	int ix;
+
+	nn->client_pr_record_hashtbl = kmalloc_array(CLIENT_HASH_SIZE,
+					sizeof(struct list_head),
+					GFP_KERNEL);
+	if (!nn->client_pr_record_hashtbl)
+		return -ENOMEM;
+	spin_lock_init(&nn->client_pr_record_hashtbl_lock);
+	for (ix = 0; ix < CLIENT_HASH_SIZE; ix++)
+		INIT_LIST_HEAD(&nn->client_pr_record_hashtbl[ix]);
+	return 0;
+}
+
+static struct scsi_pr_record *
+nfsd4_pr_find_client(struct nfs4_client *clp, struct block_device *blkdev)
+{
+	struct scsi_pr_record *pr_rec = NULL;
+	struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
+	unsigned int idhashval;
+	dev_t bdev = blkdev->bd_dev;
+
+	assert_spin_locked(&nn->client_pr_record_hashtbl_lock);
+	idhashval = clientid_hashval(clp->cl_clientid.cl_id);
+	list_for_each_entry(pr_rec, &nn->client_pr_record_hashtbl[idhashval],
+					spr_hash) {
+		if (same_clid(&pr_rec->spr_clid, &clp->cl_clientid) &&
+				pr_rec->spr_bdev == bdev) {
+			return pr_rec;
+		}
+	}
+	return NULL;
+}
+
+static int
+nfsd4_scsi_pr_add_client(struct nfs4_client *clp, struct block_device *blkdev)
+{
+	unsigned int idhashval;
+	struct scsi_pr_record *pr_rec = NULL;
+	struct scsi_pr_record *rec;
+	struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
+	dev_t bdev = blkdev->bd_dev;
+
+	spin_lock(&nn->client_pr_record_hashtbl_lock);
+	rec = nfsd4_pr_find_client(clp, blkdev);
+	if (rec) {
+		rec->spr_fenced = false;
+		spin_unlock(&nn->client_pr_record_hashtbl_lock);
+		return 0;
+	}
+	spin_unlock(&nn->client_pr_record_hashtbl_lock);
+
+	pr_rec = kzalloc(sizeof(struct scsi_pr_record), GFP_KERNEL);
+	if (!pr_rec)
+		return -ENOMEM;
+	pr_rec->spr_clid = clp->cl_clientid;
+	pr_rec->spr_bdev = bdev;
+	pr_rec->spr_fenced = false;
+
+	idhashval = clientid_hashval(clp->cl_clientid.cl_id);
+	spin_lock(&nn->client_pr_record_hashtbl_lock);
+	rec = nfsd4_pr_find_client(clp, blkdev);
+	if (rec) {
+		rec->spr_fenced = false;
+		spin_unlock(&nn->client_pr_record_hashtbl_lock);
+		kfree(pr_rec);
+		return 0;
+	}
+	list_add(&pr_rec->spr_hash, &nn->client_pr_record_hashtbl[idhashval]);
+	spin_unlock(&nn->client_pr_record_hashtbl_lock);
+	return 0;
+
+}
+
+bool
+nfsd4_scsi_pr_fence(struct nfs4_client *clp, struct block_device *blkdev)
+{
+	struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
+	struct scsi_pr_record *rec;
+
+	assert_spin_locked(&nn->client_pr_record_hashtbl_lock);
+	rec = nfsd4_pr_find_client(clp, blkdev);
+	if (rec && !rec->spr_fenced) {
+		rec->spr_fenced = true;
+		return true;
+	}
+	return false;
+}
+
+void
+nfsd4_scsi_pr_del_client(struct nfs4_client *clp)
+{
+	struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
+	struct scsi_pr_record *entry, *tmp;
+	unsigned int idhashval;
+
+	idhashval = clientid_hashval(clp->cl_clientid.cl_id);
+	spin_lock(&nn->client_pr_record_hashtbl_lock);
+	list_for_each_entry_safe(entry, tmp,
+			&nn->client_pr_record_hashtbl[idhashval], spr_hash) {
+		if (same_clid(&entry->spr_clid, &clp->cl_clientid)) {
+			list_del(&entry->spr_hash);
+			kfree(entry);
+		}
+	}
+	spin_unlock(&nn->client_pr_record_hashtbl_lock);
+}
+
+void
+nfsd4_scsi_pr_shutdown(struct nfsd_net *nn)
+{
+	int ix;
+	struct scsi_pr_record *entry;
+
+	spin_lock(&nn->client_pr_record_hashtbl_lock);
+	for (ix = 0; ix < CLIENT_HASH_SIZE; ix++) {
+		while (!list_empty(&nn->client_pr_record_hashtbl[ix])) {
+			entry = list_entry(nn->client_pr_record_hashtbl[ix].next,
+					struct scsi_pr_record, spr_hash);
+			list_del(&entry->spr_hash);
+			kfree(entry);
+		}
+	}
+	spin_unlock(&nn->client_pr_record_hashtbl_lock);
+	kfree(nn->client_pr_record_hashtbl);
+}
 #endif /* CONFIG_NFSD_SCSILAYOUT */
diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h
index 3e2d0fde80a7..466f8478d8f3 100644
--- a/fs/nfsd/netns.h
+++ b/fs/nfsd/netns.h
@@ -217,6 +217,8 @@ struct nfsd_net {
 	spinlock_t              local_clients_lock;
 	struct list_head	local_clients;
 #endif
+	spinlock_t		client_pr_record_hashtbl_lock;
+	struct list_head	*client_pr_record_hashtbl;
 };
 
 /* Simple check to find out if a given net was properly initialized */
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 13b4dc89b1e8..70009e882a4a 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -2528,6 +2528,7 @@ __destroy_client(struct nfs4_client *clp)
 		svc_xprt_put(clp->cl_cb_conn.cb_xprt);
 	atomic_add_unless(&nn->nfs4_client_count, -1, 0);
 	nfsd4_dec_courtesy_client_count(nn, clp);
+	nfsd4_scsi_pr_del_client(clp);
 	free_client(clp);
 	wake_up_all(&expiry_wq);
 }
diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h
index 369efe6ed665..4572daa94a79 100644
--- a/fs/nfsd/nfsd.h
+++ b/fs/nfsd/nfsd.h
@@ -570,6 +570,15 @@ extern void nfsd4_ssc_init_umount_work(struct nfsd_net *nn);
 
 extern void nfsd4_init_leases_net(struct nfsd_net *nn);
 
+extern int nfsd4_scsi_pr_init_hashtbl(struct nfsd_net *net);
+extern void nfsd4_scsi_pr_shutdown(struct nfsd_net *net);
+struct nfs4_client;
+extern void nfsd4_scsi_pr_del_client(struct nfs4_client *clp);
+extern int nfsd4_scsi_pr_add_client(struct nfs4_client *clp,
+		struct block_device *blkdev);
+extern bool nfsd4_scsi_pr_fence(struct nfs4_client *clp,
+		struct block_device *blkdev);
+
 #else /* CONFIG_NFSD_V4 */
 static inline int nfsd4_is_junction(struct dentry *dentry)
 {
@@ -578,6 +587,13 @@ static inline int nfsd4_is_junction(struct dentry *dentry)
 
 static inline void nfsd4_init_leases_net(struct nfsd_net *nn) { };
 
+extern inline int nfsd4_scsi_pr_init_hashtbl(struct nfsd_net *nn)
+{
+	return 0;
+}
+
+extern inline void nfsd4_scsi_pr_shutdown(struct nfsd_net *nn) {};
+
 #define register_cld_notifier() 0
 #define unregister_cld_notifier() do { } while(0)
 
diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c
index f6cae4430ba4..d4f96291b4b5 100644
--- a/fs/nfsd/nfssvc.c
+++ b/fs/nfsd/nfssvc.c
@@ -381,13 +381,17 @@ static int nfsd_startup_net(struct net *net, const struct cred *cred)
 #ifdef CONFIG_NFSD_V4_2_INTER_SSC
 	nfsd4_ssc_init_umount_work(nn);
 #endif
-	ret = nfs4_state_start_net(net);
+	ret = nfsd4_scsi_pr_init_hashtbl(nn);
 	if (ret)
 		goto out_reply_cache;
-
+	ret = nfs4_state_start_net(net);
+	if (ret)
+		goto out_pr_client;
 	nn->nfsd_net_up = true;
 	return 0;
 
+out_pr_client:
+	nfsd4_scsi_pr_shutdown(nn);
 out_reply_cache:
 	nfsd_reply_cache_shutdown(nn);
 out_filecache:
@@ -423,6 +427,7 @@ static void nfsd_shutdown_net(struct net *net)
 
 	wait_for_completion(&nn->nfsd_net_free_done);
 	percpu_ref_exit(&nn->nfsd_net_ref);
+	nfsd4_scsi_pr_shutdown(nn);
 
 	nn->nfsd_net_up = false;
 	nfsd_shutdown_generic();
diff --git a/fs/nfsd/pnfs.h b/fs/nfsd/pnfs.h
index db9af780438b..9ae02b6d922d 100644
--- a/fs/nfsd/pnfs.h
+++ b/fs/nfsd/pnfs.h
@@ -67,6 +67,17 @@ __be32 nfsd4_return_client_layouts(struct svc_rqst *rqstp,
 int nfsd4_set_deviceid(struct nfsd4_deviceid *id, const struct svc_fh *fhp,
 		u32 device_generation);
 struct nfsd4_deviceid_map *nfsd4_find_devid_map(int idx);
+
+int nfsd4_scsi_pr_init_hashtbl(struct nfsd_net *nn);
+void nfsd4_scsi_pr_shutdown(struct nfsd_net *nn);
+void nfsd4_scsi_pr_del_client(struct nfs4_client *clp);
+
+struct scsi_pr_record {
+	struct list_head spr_hash;
+	clientid_t spr_clid;
+	dev_t spr_bdev;
+	bool spr_fenced;
+};
 #endif /* CONFIG_NFSD_V4 */
 
 #ifdef CONFIG_NFSD_PNFS
-- 
2.47.3