[PATCH 1/2] nvme/ioctl: split user request allocation from user command submission

[Tokunori Ikegami <ikegami.t@gmail.com>]

Delete unnecessary nvme command copy for the performance improvement.

Signed-off-by: Tokunori Ikegami <ikegami.t@gmail.com>
---
 drivers/nvme/host/core.c  |  10 +-
 drivers/nvme/host/ioctl.c | 213 +++++++++++++++++++++-----------------
 2 files changed, 125 insertions(+), 98 deletions(-)

diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index f69a232a000a..8736a9a8d74d 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -734,14 +734,16 @@ void nvme_init_request(struct request *req, struct nvme_command *cmd)
 	if (!logging_enabled)
 		req->rq_flags |= RQF_QUIET;
 
-	/* passthru commands should let the driver set the SGL flags */
-	cmd->common.flags &= ~NVME_CMD_SGL_ALL;
-
 	req->cmd_flags |= REQ_FAILFAST_DRIVER;
 	if (req->mq_hctx->type == HCTX_TYPE_POLL)
 		req->cmd_flags |= REQ_POLLED;
 	nvme_clear_nvme_request(req);
-	memcpy(nr->cmd, cmd, sizeof(*cmd));
+
+	if (likely(cmd)) {
+		/* passthru commands should let the driver set the SGL flags */
+		cmd->common.flags &= ~NVME_CMD_SGL_ALL;
+		memcpy(nr->cmd, cmd, sizeof(*cmd));
+	}
 }
 EXPORT_SYMBOL_GPL(nvme_init_request);
 
diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index ca86d3bf7ea4..02edb0900413 100644
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -14,7 +14,7 @@ enum {
 	NVME_IOCTL_PARTITION	= (1 << 1),
 };
 
-static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c,
+static bool nvme_cmd_allowed(struct nvme_ns *ns, __u8 opcode, __u32 cdw10,
 		unsigned int flags, bool open_for_write)
 {
 	u32 effects;
@@ -30,8 +30,7 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c,
 	 * Do not allow unprivileged processes to send vendor specific or fabrics
 	 * commands as we can't be sure about their effects.
 	 */
-	if (c->common.opcode >= nvme_cmd_vendor_start ||
-	    c->common.opcode == nvme_fabrics_command)
+	if (opcode >= nvme_cmd_vendor_start || opcode == nvme_fabrics_command)
 		goto admin;
 
 	/*
@@ -41,8 +40,8 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c,
 	 * potentially sensitive information.
 	 */
 	if (!ns) {
-		if (c->common.opcode == nvme_admin_identify) {
-			switch (c->identify.cns) {
+		if (opcode == nvme_admin_identify) {
+			switch (cdw10 & 0xff) {
 			case NVME_ID_CNS_NS:
 			case NVME_ID_CNS_CS_NS:
 			case NVME_ID_CNS_NS_CS_INDEP:
@@ -59,7 +58,7 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c,
 	 * and marks this command as supported.  If not reject unprivileged
 	 * passthrough.
 	 */
-	effects = nvme_command_effects(ns->ctrl, ns, c->common.opcode);
+	effects = nvme_command_effects(ns->ctrl, ns, opcode);
 	if (!(effects & NVME_CMD_EFFECTS_CSUPP))
 		goto admin;
 
@@ -77,7 +76,7 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c,
 	 * change the logical block contents if the file descriptor is open for
 	 * writing.
 	 */
-	if ((nvme_is_write(c) || (effects & NVME_CMD_EFFECTS_LBCC)) &&
+	if ((opcode & 1 || (effects & NVME_CMD_EFFECTS_LBCC)) &&
 	    !open_for_write)
 		goto admin;
 
@@ -99,15 +98,15 @@ static void __user *nvme_to_user_ptr(uintptr_t ptrval)
 }
 
 static struct request *nvme_alloc_user_request(struct request_queue *q,
-		struct nvme_command *cmd, blk_opf_t rq_flags,
-		blk_mq_req_flags_t blk_flags)
+		__u8 opcode, blk_opf_t rq_flags, blk_mq_req_flags_t blk_flags)
 {
+	enum req_op op = opcode & 1 ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN;
 	struct request *req;
 
-	req = blk_mq_alloc_request(q, nvme_req_op(cmd) | rq_flags, blk_flags);
+	req = blk_mq_alloc_request(q, op | rq_flags, blk_flags);
 	if (IS_ERR(req))
 		return req;
-	nvme_init_request(req, cmd);
+	nvme_init_request(req, NULL);
 	nvme_req(req)->flags |= NVME_REQ_USERCMD;
 	return req;
 }
@@ -165,21 +164,17 @@ static int nvme_map_user_request(struct request *req, u64 ubuffer,
 }
 
 static int nvme_submit_user_cmd(struct request_queue *q,
-		struct nvme_command *cmd, u64 ubuffer, unsigned bufflen,
+		struct request *req, u64 ubuffer, unsigned bufflen,
 		void __user *meta_buffer, unsigned meta_len,
 		u64 *result, unsigned timeout, unsigned int flags)
 {
+	struct nvme_command *cmd = nvme_req(req)->cmd;
 	struct nvme_ns *ns = q->queuedata;
 	struct nvme_ctrl *ctrl;
-	struct request *req;
 	struct bio *bio;
 	u32 effects;
 	int ret;
 
-	req = nvme_alloc_user_request(q, cmd, 0, 0);
-	if (IS_ERR(req))
-		return PTR_ERR(req);
-
 	req->timeout = timeout;
 	if (ubuffer && bufflen) {
 		ret = nvme_map_user_request(req, ubuffer, bufflen, meta_buffer,
@@ -211,9 +206,10 @@ static int nvme_submit_user_cmd(struct request_queue *q,
 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
 {
 	struct nvme_user_io io;
-	struct nvme_command c;
+	struct nvme_command *c;
 	unsigned length, meta_len;
 	void __user *metadata;
+	struct request *req;
 
 	if (copy_from_user(&io, uio, sizeof(io)))
 		return -EFAULT;
@@ -254,19 +250,25 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
 			return -EINVAL;
 	}
 
-	memset(&c, 0, sizeof(c));
-	c.rw.opcode = io.opcode;
-	c.rw.flags = io.flags;
-	c.rw.nsid = cpu_to_le32(ns->head->ns_id);
-	c.rw.slba = cpu_to_le64(io.slba);
-	c.rw.length = cpu_to_le16(io.nblocks);
-	c.rw.control = cpu_to_le16(io.control);
-	c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
-	c.rw.reftag = cpu_to_le32(io.reftag);
-	c.rw.lbat = cpu_to_le16(io.apptag);
-	c.rw.lbatm = cpu_to_le16(io.appmask);
-
-	return nvme_submit_user_cmd(ns->queue, &c, io.addr, length, metadata,
+	req = nvme_alloc_user_request(ns->queue, io.opcode, 0, 0);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
+
+	c = nvme_req(req)->cmd;
+	memset(c, 0, sizeof(*c));
+	c->rw.opcode = io.opcode;
+	/* passthru commands should let the driver set the SGL flags */
+	c->rw.flags = io.flags & ~NVME_CMD_SGL_ALL;
+	c->rw.nsid = cpu_to_le32(ns->head->ns_id);
+	c->rw.slba = cpu_to_le64(io.slba);
+	c->rw.length = cpu_to_le16(io.nblocks);
+	c->rw.control = cpu_to_le16(io.control);
+	c->rw.dsmgmt = cpu_to_le32(io.dsmgmt);
+	c->rw.reftag = cpu_to_le32(io.reftag);
+	c->rw.lbat = cpu_to_le16(io.apptag);
+	c->rw.lbatm = cpu_to_le16(io.appmask);
+
+	return nvme_submit_user_cmd(ns->queue, req, io.addr, length, metadata,
 			meta_len, NULL, 0, 0);
 }
 
@@ -287,9 +289,11 @@ static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 		struct nvme_passthru_cmd __user *ucmd, unsigned int flags,
 		bool open_for_write)
 {
+	struct request_queue *q = ns ? ns->queue : ctrl->admin_q;
 	struct nvme_passthru_cmd cmd;
-	struct nvme_command c;
+	struct nvme_command *c;
 	unsigned timeout = 0;
+	struct request *req;
 	u64 result;
 	int status;
 
@@ -300,28 +304,35 @@ static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 	if (!nvme_validate_passthru_nsid(ctrl, ns, cmd.nsid))
 		return -EINVAL;
 
-	memset(&c, 0, sizeof(c));
-	c.common.opcode = cmd.opcode;
-	c.common.flags = cmd.flags;
-	c.common.nsid = cpu_to_le32(cmd.nsid);
-	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
-	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
-	c.common.cdw10 = cpu_to_le32(cmd.cdw10);
-	c.common.cdw11 = cpu_to_le32(cmd.cdw11);
-	c.common.cdw12 = cpu_to_le32(cmd.cdw12);
-	c.common.cdw13 = cpu_to_le32(cmd.cdw13);
-	c.common.cdw14 = cpu_to_le32(cmd.cdw14);
-	c.common.cdw15 = cpu_to_le32(cmd.cdw15);
-
-	if (!nvme_cmd_allowed(ns, &c, 0, open_for_write))
+	if (!nvme_cmd_allowed(ns, cmd.opcode, cmd.cdw10, 0, open_for_write))
 		return -EACCES;
 
+	req = nvme_alloc_user_request(q, cmd.opcode, 0, 0);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
+
+	c = nvme_req(req)->cmd;
+	memset(c, 0, sizeof(*c));
+	c->common.opcode = cmd.opcode;
+
+	/* passthru commands should let the driver set the SGL flags */
+	c->common.flags = cmd.flags & ~NVME_CMD_SGL_ALL;
+	c->common.nsid = cpu_to_le32(cmd.nsid);
+	c->common.cdw2[0] = cpu_to_le32(cmd.cdw2);
+	c->common.cdw2[1] = cpu_to_le32(cmd.cdw3);
+	c->common.cdw10 = cpu_to_le32(cmd.cdw10);
+	c->common.cdw11 = cpu_to_le32(cmd.cdw11);
+	c->common.cdw12 = cpu_to_le32(cmd.cdw12);
+	c->common.cdw13 = cpu_to_le32(cmd.cdw13);
+	c->common.cdw14 = cpu_to_le32(cmd.cdw14);
+	c->common.cdw15 = cpu_to_le32(cmd.cdw15);
+
 	if (cmd.timeout_ms)
 		timeout = msecs_to_jiffies(cmd.timeout_ms);
 
-	status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
-			cmd.addr, cmd.data_len, nvme_to_user_ptr(cmd.metadata),
-			cmd.metadata_len, &result, timeout, 0);
+	status = nvme_submit_user_cmd(q, req, cmd.addr, cmd.data_len,
+			nvme_to_user_ptr(cmd.metadata), cmd.metadata_len,
+			&result, timeout, 0);
 
 	if (status >= 0) {
 		if (put_user(result, &ucmd->result))
@@ -335,9 +346,11 @@ static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 		struct nvme_passthru_cmd64 __user *ucmd, unsigned int flags,
 		bool open_for_write)
 {
+	struct request_queue *q = ns ? ns->queue : ctrl->admin_q;
 	struct nvme_passthru_cmd64 cmd;
-	struct nvme_command c;
+	struct nvme_command *c;
 	unsigned timeout = 0;
+	struct request *req;
 	int status;
 
 	if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
@@ -347,26 +360,32 @@ static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 	if (!nvme_validate_passthru_nsid(ctrl, ns, cmd.nsid))
 		return -EINVAL;
 
-	memset(&c, 0, sizeof(c));
-	c.common.opcode = cmd.opcode;
-	c.common.flags = cmd.flags;
-	c.common.nsid = cpu_to_le32(cmd.nsid);
-	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
-	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
-	c.common.cdw10 = cpu_to_le32(cmd.cdw10);
-	c.common.cdw11 = cpu_to_le32(cmd.cdw11);
-	c.common.cdw12 = cpu_to_le32(cmd.cdw12);
-	c.common.cdw13 = cpu_to_le32(cmd.cdw13);
-	c.common.cdw14 = cpu_to_le32(cmd.cdw14);
-	c.common.cdw15 = cpu_to_le32(cmd.cdw15);
-
-	if (!nvme_cmd_allowed(ns, &c, flags, open_for_write))
+	if (!nvme_cmd_allowed(ns, cmd.opcode, cmd.cdw10, flags, open_for_write))
 		return -EACCES;
 
+	req = nvme_alloc_user_request(q, cmd.opcode, 0, 0);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
+
+	c = nvme_req(req)->cmd;
+	memset(c, 0, sizeof(*c));
+	c->common.opcode = cmd.opcode;
+	/* passthru commands should let the driver set the SGL flags */
+	c->common.flags = cmd.flags & ~NVME_CMD_SGL_ALL;
+	c->common.nsid = cpu_to_le32(cmd.nsid);
+	c->common.cdw2[0] = cpu_to_le32(cmd.cdw2);
+	c->common.cdw2[1] = cpu_to_le32(cmd.cdw3);
+	c->common.cdw10 = cpu_to_le32(cmd.cdw10);
+	c->common.cdw11 = cpu_to_le32(cmd.cdw11);
+	c->common.cdw12 = cpu_to_le32(cmd.cdw12);
+	c->common.cdw13 = cpu_to_le32(cmd.cdw13);
+	c->common.cdw14 = cpu_to_le32(cmd.cdw14);
+	c->common.cdw15 = cpu_to_le32(cmd.cdw15);
+
 	if (cmd.timeout_ms)
 		timeout = msecs_to_jiffies(cmd.timeout_ms);
 
-	status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
+	status = nvme_submit_user_cmd(q, req,
 			cmd.addr, cmd.data_len, nvme_to_user_ptr(cmd.metadata),
 			cmd.metadata_len, &cmd.result, timeout, flags);
 
@@ -454,7 +473,7 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 	const struct nvme_uring_cmd *cmd = io_uring_sqe_cmd(ioucmd->sqe);
 	struct request_queue *q = ns ? ns->queue : ctrl->admin_q;
 	struct nvme_uring_data d;
-	struct nvme_command c;
+	struct nvme_command *c;
 	struct iov_iter iter;
 	struct iov_iter *map_iter = NULL;
 	struct request *req;
@@ -462,43 +481,22 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 	blk_mq_req_flags_t blk_flags = 0;
 	int ret;
 
-	c.common.opcode = READ_ONCE(cmd->opcode);
-	c.common.flags = READ_ONCE(cmd->flags);
-	if (c.common.flags)
+	if (READ_ONCE(cmd->flags))
 		return -EINVAL;
 
-	c.common.command_id = 0;
-	c.common.nsid = cpu_to_le32(cmd->nsid);
-	if (!nvme_validate_passthru_nsid(ctrl, ns, le32_to_cpu(c.common.nsid)))
+	if (!nvme_validate_passthru_nsid(ctrl, ns, cmd->nsid))
 		return -EINVAL;
 
-	c.common.cdw2[0] = cpu_to_le32(READ_ONCE(cmd->cdw2));
-	c.common.cdw2[1] = cpu_to_le32(READ_ONCE(cmd->cdw3));
-	c.common.metadata = 0;
-	c.common.dptr.prp1 = c.common.dptr.prp2 = 0;
-	c.common.cdw10 = cpu_to_le32(READ_ONCE(cmd->cdw10));
-	c.common.cdw11 = cpu_to_le32(READ_ONCE(cmd->cdw11));
-	c.common.cdw12 = cpu_to_le32(READ_ONCE(cmd->cdw12));
-	c.common.cdw13 = cpu_to_le32(READ_ONCE(cmd->cdw13));
-	c.common.cdw14 = cpu_to_le32(READ_ONCE(cmd->cdw14));
-	c.common.cdw15 = cpu_to_le32(READ_ONCE(cmd->cdw15));
-
-	if (!nvme_cmd_allowed(ns, &c, 0, ioucmd->file->f_mode & FMODE_WRITE))
+	if (!nvme_cmd_allowed(ns, cmd->opcode, cmd->cdw10, 0, ioucmd->file->f_mode & FMODE_WRITE))
 		return -EACCES;
 
-	d.metadata = READ_ONCE(cmd->metadata);
-	d.addr = READ_ONCE(cmd->addr);
-	d.data_len = READ_ONCE(cmd->data_len);
-	d.metadata_len = READ_ONCE(cmd->metadata_len);
-	d.timeout_ms = READ_ONCE(cmd->timeout_ms);
-
 	if (d.data_len && (ioucmd->flags & IORING_URING_CMD_FIXED)) {
 		/* fixedbufs is only for non-vectored io */
 		if (vec)
 			return -EINVAL;
 
 		ret = io_uring_cmd_import_fixed(d.addr, d.data_len,
-			nvme_is_write(&c) ? WRITE : READ, &iter, ioucmd,
+			cmd->opcode & 1 ? WRITE : READ, &iter, ioucmd,
 			issue_flags);
 		if (ret < 0)
 			return ret;
@@ -506,6 +504,36 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 		map_iter = &iter;
 	}
 
+	req = nvme_alloc_user_request(q, cmd->opcode, rq_flags, blk_flags);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
+
+	c = nvme_req(req)->cmd;
+	c->common.opcode = READ_ONCE(cmd->opcode);
+	/* passthru commands should let the driver set the SGL flags */
+	c->common.flags = READ_ONCE(cmd->flags) & ~NVME_CMD_SGL_ALL;
+
+	c->common.command_id = 0;
+	c->common.nsid = cpu_to_le32(cmd->nsid);
+
+	c->common.cdw2[0] = cpu_to_le32(READ_ONCE(cmd->cdw2));
+	c->common.cdw2[1] = cpu_to_le32(READ_ONCE(cmd->cdw3));
+	c->common.metadata = 0;
+	c->common.dptr.prp1 = c->common.dptr.prp2 = 0;
+	c->common.cdw10 = cpu_to_le32(READ_ONCE(cmd->cdw10));
+	c->common.cdw11 = cpu_to_le32(READ_ONCE(cmd->cdw11));
+	c->common.cdw12 = cpu_to_le32(READ_ONCE(cmd->cdw12));
+	c->common.cdw13 = cpu_to_le32(READ_ONCE(cmd->cdw13));
+	c->common.cdw14 = cpu_to_le32(READ_ONCE(cmd->cdw14));
+	c->common.cdw15 = cpu_to_le32(READ_ONCE(cmd->cdw15));
+
+	d.metadata = READ_ONCE(cmd->metadata);
+	d.addr = READ_ONCE(cmd->addr);
+	d.data_len = READ_ONCE(cmd->data_len);
+	d.metadata_len = READ_ONCE(cmd->metadata_len);
+	d.timeout_ms = READ_ONCE(cmd->timeout_ms);
+
+
 	if (issue_flags & IO_URING_F_NONBLOCK) {
 		rq_flags |= REQ_NOWAIT;
 		blk_flags = BLK_MQ_REQ_NOWAIT;
@@ -513,9 +541,6 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 	if (issue_flags & IO_URING_F_IOPOLL)
 		rq_flags |= REQ_POLLED;
 
-	req = nvme_alloc_user_request(q, &c, rq_flags, blk_flags);
-	if (IS_ERR(req))
-		return PTR_ERR(req);
 	req->timeout = d.timeout_ms ? msecs_to_jiffies(d.timeout_ms) : 0;
 
 	if (d.data_len) {
-- 
2.48.1