[PATCH 3/5] scsi: target: Allow userspace to request direct submissions

[Mike Christie <michael.christie@oracle.com>]

This allows userspace to request the fabric drivers do direct submissions
if they support it.

When using a nvme drive and vhost-scsi we see around a 20% improvement
in 4K IOs:

fio jobs        1       2       4       8       10
--------------------------------------------------
defer           94K     190K    394K    770K    890K
direct          128K    252K    488K    950K    -

Signed-off-by: Mike Christie <michael.christie@oracle.com>
---
 drivers/target/loopback/tcm_loop.c     |  2 +-
 drivers/target/target_core_configfs.c  |  5 ++++
 drivers/target/target_core_device.c    |  1 +
 drivers/target/target_core_transport.c | 33 +++++++++++++++++++++++---
 drivers/vhost/scsi.c                   |  2 +-
 include/target/target_core_base.h      |  5 ++++
 6 files changed, 43 insertions(+), 5 deletions(-)

diff --git a/drivers/target/loopback/tcm_loop.c b/drivers/target/loopback/tcm_loop.c
index 75804303b175..fbbf80ae9766 100644
--- a/drivers/target/loopback/tcm_loop.c
+++ b/drivers/target/loopback/tcm_loop.c
@@ -154,7 +154,7 @@ static void tcm_loop_target_queue_cmd(struct tcm_loop_cmd *tl_cmd)
 			       GFP_ATOMIC))
 		return;
 
-	target_queue_submission(se_cmd);
+	target_submit(se_cmd);
 	return;
 
 out_done:
diff --git a/drivers/target/target_core_configfs.c b/drivers/target/target_core_configfs.c
index 936e5ff1b209..e4ddd8882f83 100644
--- a/drivers/target/target_core_configfs.c
+++ b/drivers/target/target_core_configfs.c
@@ -577,6 +577,7 @@ DEF_CONFIGFS_ATTRIB_SHOW(unmap_granularity_alignment);
 DEF_CONFIGFS_ATTRIB_SHOW(unmap_zeroes_data);
 DEF_CONFIGFS_ATTRIB_SHOW(max_write_same_len);
 DEF_CONFIGFS_ATTRIB_SHOW(emulate_rsoc);
+DEF_CONFIGFS_ATTRIB_SHOW(direct_submit);
 
 #define DEF_CONFIGFS_ATTRIB_STORE_U32(_name)				\
 static ssize_t _name##_store(struct config_item *item, const char *page,\
@@ -598,6 +599,7 @@ DEF_CONFIGFS_ATTRIB_STORE_U32(max_unmap_block_desc_count);
 DEF_CONFIGFS_ATTRIB_STORE_U32(unmap_granularity);
 DEF_CONFIGFS_ATTRIB_STORE_U32(unmap_granularity_alignment);
 DEF_CONFIGFS_ATTRIB_STORE_U32(max_write_same_len);
+DEF_CONFIGFS_ATTRIB_STORE_U32(direct_submit);
 
 #define DEF_CONFIGFS_ATTRIB_STORE_BOOL(_name)				\
 static ssize_t _name##_store(struct config_item *item, const char *page,	\
@@ -1266,6 +1268,7 @@ CONFIGFS_ATTR(, unmap_zeroes_data);
 CONFIGFS_ATTR(, max_write_same_len);
 CONFIGFS_ATTR(, alua_support);
 CONFIGFS_ATTR(, pgr_support);
+CONFIGFS_ATTR(, direct_submit);
 
 /*
  * dev_attrib attributes for devices using the target core SBC/SPC
@@ -1308,6 +1311,7 @@ struct configfs_attribute *sbc_attrib_attrs[] = {
 	&attr_alua_support,
 	&attr_pgr_support,
 	&attr_emulate_rsoc,
+	&attr_direct_submit,
 	NULL,
 };
 EXPORT_SYMBOL(sbc_attrib_attrs);
@@ -1325,6 +1329,7 @@ struct configfs_attribute *passthrough_attrib_attrs[] = {
 	&attr_emulate_pr,
 	&attr_alua_support,
 	&attr_pgr_support,
+	&attr_direct_submit,
 	NULL,
 };
 EXPORT_SYMBOL(passthrough_attrib_attrs);
diff --git a/drivers/target/target_core_device.c b/drivers/target/target_core_device.c
index b7ac60f4a219..b4c3e17ebfc9 100644
--- a/drivers/target/target_core_device.c
+++ b/drivers/target/target_core_device.c
@@ -779,6 +779,7 @@ struct se_device *target_alloc_device(struct se_hba *hba, const char *name)
 	dev->dev_attrib.unmap_zeroes_data =
 				DA_UNMAP_ZEROES_DATA_DEFAULT;
 	dev->dev_attrib.max_write_same_len = DA_MAX_WRITE_SAME_LEN;
+	dev->dev_attrib.direct_submit = DA_FABRIC_DEFAULT_SUBMIT;
 
 	xcopy_lun = &dev->xcopy_lun;
 	rcu_assign_pointer(xcopy_lun->lun_se_dev, dev);
diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c
index 687adc9e086c..5349a2dd8187 100644
--- a/drivers/target/target_core_transport.c
+++ b/drivers/target/target_core_transport.c
@@ -1781,13 +1781,13 @@ int target_submit_prep(struct se_cmd *se_cmd, unsigned char *cdb,
 EXPORT_SYMBOL_GPL(target_submit_prep);
 
 /**
- * target_submit - perform final initialization and submit cmd to LIO core
+ * __target_submit - perform final initialization and submit cmd to LIO core
  * @se_cmd: command descriptor to submit
  *
  * target_submit_prep must have been called on the cmd, and this must be
  * called from process context.
  */
-void target_submit(struct se_cmd *se_cmd)
+static void __target_submit(struct se_cmd *se_cmd)
 {
 	struct scatterlist *sgl = se_cmd->t_data_sg;
 	unsigned char *buf = NULL;
@@ -1825,6 +1825,33 @@ void target_submit(struct se_cmd *se_cmd)
 
 	transport_handle_cdb_direct(se_cmd);
 }
+
+/**
+ * target_submit - submit cmd to LIO core or queue it's submission
+ * @se_cmd: command descriptor to submit
+ */
+void target_submit(struct se_cmd *se_cmd)
+{
+	struct se_dev_attrib *da = &se_cmd->se_dev->dev_attrib;
+	u32 direct_submit;
+
+	if (da->direct_submit == DA_FABRIC_DEFAULT_SUBMIT) {
+		if (se_cmd->se_sess->se_tpg->se_tpg_tfo->default_direct_submit)
+			direct_submit = DA_DIRECT_SUBMIT;
+		else
+			direct_submit = DA_QUEUE_SUBMIT;
+	} else if (da->direct_submit == DA_DIRECT_SUBMIT &&
+		   se_cmd->se_sess->se_tpg->se_tpg_tfo->direct_submit_supp) {
+		direct_submit = DA_DIRECT_SUBMIT;
+	} else {
+		direct_submit = DA_QUEUE_SUBMIT;
+	}
+
+	if (direct_submit == DA_DIRECT_SUBMIT)
+		__target_submit(se_cmd);
+	else
+		target_queue_submission(se_cmd);
+}
 EXPORT_SYMBOL_GPL(target_submit);
 
 /**
@@ -1922,7 +1949,7 @@ void target_queued_submit_work(struct work_struct *work)
 			se_plug = target_plug_device(se_dev);
 		}
 
-		target_submit(se_cmd);
+		__target_submit(se_cmd);
 	}
 
 	if (se_plug)
diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c
index ed6cc84c0b85..e2603eb1b638 100644
--- a/drivers/vhost/scsi.c
+++ b/drivers/vhost/scsi.c
@@ -772,7 +772,7 @@ static void vhost_scsi_target_queue_cmd(struct vhost_scsi_cmd *cmd)
 			       cmd->tvc_prot_sgl_count, GFP_KERNEL))
 		return;
 
-	target_queue_submission(se_cmd);
+	target_submit(se_cmd);
 }
 
 static void
diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h
index 159567359bbb..4f37324ada5d 100644
--- a/include/target/target_core_base.h
+++ b/include/target/target_core_base.h
@@ -103,6 +103,10 @@
 #define DA_IS_NONROT				0
 /* Queue Algorithm Modifier default for restricted reordering in control mode page */
 #define DA_EMULATE_REST_REORD			0
+/* Command submission settings */
+#define DA_FABRIC_DEFAULT_SUBMIT		0
+#define DA_DIRECT_SUBMIT			1
+#define DA_QUEUE_SUBMIT				2
 
 #define SE_INQUIRY_BUF				1024
 #define SE_MODE_PAGE_BUF			512
@@ -717,6 +721,7 @@ struct se_dev_attrib {
 	u32		unmap_granularity;
 	u32		unmap_granularity_alignment;
 	u32		max_write_same_len;
+	u32		direct_submit;
 	struct se_device *da_dev;
 	struct config_group da_group;
 };
-- 
2.34.1