[lustre-devel] [PATCH 07/15] lustre: osc: Support RDMA only pages

[James Simmons <jsimmons@infradead.org>]

From: Amir Shehata <ashehata@whamcloud.com>

Some memory architectures and CPU-offload cards with
on-board memory do not map data pages into the CPU
address space. Allow RDMA of data directly into those
pages without accessing contents.

Therefore, made changes to prevent doing checksum on
these type of pages.

WC-bug-id: https://jira.whamcloud.com/browse/LU-14798
Lustre-commit: 29eabeb34c5ba2cffd ("LU-14798 lustre: Support RDMA only pages")
Signed-off-by: Wang Shilong <wshilong@ddn.com>
Signed-off-by: Amir Shehata <ashehata@whamcloud.com>
Lustre-change: https://review.whamcloud.com/37454
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Whamcloud-bug-id: EX-773
Reviewed-on: https://review.whamcloud.com/44111
Reviewed-by: Wang Shilong <wangshilong1991@gmail.com>
Reviewed-by: Patrick Farrell <pfarrell@whamcloud.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
 fs/lustre/include/lustre_osc.h |  6 ++++--
 fs/lustre/osc/osc_io.c         |  2 ++
 fs/lustre/osc/osc_request.c    | 15 +++++++++++----
 3 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/fs/lustre/include/lustre_osc.h b/fs/lustre/include/lustre_osc.h
index 09868ea..cdc9aae 100644
--- a/fs/lustre/include/lustre_osc.h
+++ b/fs/lustre/include/lustre_osc.h
@@ -956,8 +956,10 @@ struct osc_extent {
 				oe_ndelay:1,
 	/* direct IO pages */
 				oe_dio:1,
-	/* this extent consists of RDMA only pages */
-				oe_is_rdma_only;
+	/* this extent consists of pages that are not directly accessible
+	 *  from the CPU
+	 */
+				oe_is_rdma_only:1;
 	/* how many grants allocated for this extent.
 	 *  Grant allocated for this extent. There is no grant allocated
 	 *  for reading extents and sync write extents.
diff --git a/fs/lustre/osc/osc_io.c b/fs/lustre/osc/osc_io.c
index d828ae0..b867985 100644
--- a/fs/lustre/osc/osc_io.c
+++ b/fs/lustre/osc/osc_io.c
@@ -153,6 +153,8 @@ int osc_io_submit(const struct lu_env *env, const struct cl_io_slice *ios,
 	page = cl_page_list_first(qin);
 	if (page->cp_type == CPT_TRANSIENT)
 		brw_flags |= OBD_BRW_NOCACHE;
+	if (lnet_is_rdma_only_page(page->cp_vmpage))
+		brw_flags |= OBD_BRW_RDMA_ONLY;
 
 	/*
 	 * NOTE: here @page is a top-level page. This is done to avoid
diff --git a/fs/lustre/osc/osc_request.c b/fs/lustre/osc/osc_request.c
index 2ac0300..db73fce 100644
--- a/fs/lustre/osc/osc_request.c
+++ b/fs/lustre/osc/osc_request.c
@@ -1402,6 +1402,7 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,
 	const char *obd_name = cli->cl_import->imp_obd->obd_name;
 	struct inode *inode = NULL;
 	bool directio = false;
+	bool enable_checksum = true;
 
 	if (pga[0]->pg) {
 		inode = page2inode(pga[0]->pg);
@@ -1545,6 +1546,11 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,
 		}
 	}
 
+	if (lnet_is_rdma_only_page(pga[0]->pg)) {
+		enable_checksum = false;
+		short_io_size = 0;
+	}
+
 	/* Check if read/write is small enough to be a short io. */
 	if (short_io_size > cli->cl_max_short_io_bytes || niocount > 1 ||
 	    !imp_connect_shortio(cli->cl_import))
@@ -1700,10 +1706,12 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,
 	if (osc_should_shrink_grant(cli))
 		osc_shrink_grant_local(cli, &body->oa);
 
+	if (!cli->cl_checksum || sptlrpc_flavor_has_bulk(&req->rq_flvr))
+		enable_checksum = false;
+
 	/* size[REQ_REC_OFF] still sizeof (*body) */
 	if (opc == OST_WRITE) {
-		if (cli->cl_checksum &&
-		    !sptlrpc_flavor_has_bulk(&req->rq_flvr)) {
+		if (enable_checksum) {
 			/* store cl_cksum_type in a local variable since
 			 * it can be changed via lprocfs
 			 */
@@ -1743,8 +1751,7 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,
 		req_capsule_set_size(pill, &RMF_RCS, RCL_SERVER,
 				     sizeof(u32) * niocount);
 	} else {
-		if (cli->cl_checksum &&
-		    !sptlrpc_flavor_has_bulk(&req->rq_flvr)) {
+		if (enable_checksum) {
 			if ((body->oa.o_valid & OBD_MD_FLFLAGS) == 0)
 				body->oa.o_flags = 0;
 			body->oa.o_flags |= obd_cksum_type_pack(obd_name,
-- 
1.8.3.1

_______________________________________________
lustre-devel mailing list
lustre-devel@lists.lustre.org
http://lists.lustre.org/listinfo.cgi/lustre-devel-lustre.org