* [PATCH 1/4] libceph: distinguish page array and pagelist count
2013-02-25 23:40 [PATCH 0/4] libceph: abstract setting message data info Alex Elder
@ 2013-02-25 23:42 ` Alex Elder
2013-02-25 23:42 ` [PATCH 2/4] libceph: set page alignment in start_request() Alex Elder
` (3 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Alex Elder @ 2013-02-25 23:42 UTC (permalink / raw)
To: ceph-devel@vger.kernel.org
Use distinct fields for tracking the number of pages in a message's
page array and in a message's page list. Currently only one or the
other is used at a time, but that will be changing soon.
Signed-off-by: Alex Elder <elder@inktank.com>
---
fs/ceph/mds_client.c | 4 ++--
include/linux/ceph/messenger.h | 3 ++-
net/ceph/messenger.c | 14 ++++++++------
net/ceph/osd_client.c | 4 ++--
4 files changed, 14 insertions(+), 11 deletions(-)
diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
index d958420..44435c2 100644
--- a/fs/ceph/mds_client.c
+++ b/fs/ceph/mds_client.c
@@ -1719,7 +1719,7 @@ static struct ceph_msg
*create_request_message(struct ceph_mds_client *mdsc,
msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
msg->pages = req->r_pages;
- msg->nr_pages = req->r_num_pages;
+ msg->page_count = req->r_num_pages;
msg->hdr.data_len = cpu_to_le32(req->r_data_len);
msg->hdr.data_off = cpu_to_le16(0);
@@ -2600,10 +2600,10 @@ static void send_mds_reconnect(struct
ceph_mds_client *mdsc,
}
reply->pagelist = pagelist;
+ reply->pagelist_count = calc_pages_for(0, pagelist->length);
if (recon_state.flock)
reply->hdr.version = cpu_to_le16(2);
reply->hdr.data_len = cpu_to_le32(pagelist->length);
- reply->nr_pages = calc_pages_for(0, pagelist->length);
ceph_con_send(&session->s_con, reply);
mutex_unlock(&session->s_mutex);
diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h
index 8297288..1b08349 100644
--- a/include/linux/ceph/messenger.h
+++ b/include/linux/ceph/messenger.h
@@ -75,9 +75,10 @@ struct ceph_msg {
struct kvec front; /* unaligned blobs of message */
struct ceph_buffer *middle;
struct page **pages; /* data payload. NOT OWNER. */
- unsigned nr_pages; /* size of page array */
+ unsigned page_count; /* size of page array */
unsigned page_alignment; /* io offset in first page */
struct ceph_pagelist *pagelist; /* instead of pages */
+ unsigned int pagelist_count; /* number of pages in pagelist */
struct ceph_connection *con;
struct list_head list_head;
diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c
index c06f940..9d8abb0 100644
--- a/net/ceph/messenger.c
+++ b/net/ceph/messenger.c
@@ -813,7 +813,7 @@ static void prepare_write_message(struct
ceph_connection *con)
m, con->out_seq, le16_to_cpu(m->hdr.type),
le32_to_cpu(m->hdr.front_len), le32_to_cpu(m->hdr.middle_len),
le32_to_cpu(m->hdr.data_len),
- m->nr_pages);
+ m->page_count);
BUG_ON(le32_to_cpu(m->hdr.front_len) != m->front.iov_len);
/* tag + hdr + front + middle */
@@ -1072,7 +1072,7 @@ static int write_partial_msg_pages(struct
ceph_connection *con)
const size_t trail_off = data_len - trail_len;
dout("write_partial_msg_pages %p msg %p page %d/%d offset %d\n",
- con, msg, con->out_msg_pos.page, msg->nr_pages,
+ con, msg, con->out_msg_pos.page, msg->page_count,
con->out_msg_pos.page_pos);
/*
@@ -2715,9 +2715,10 @@ struct ceph_msg *ceph_msg_new(int type, int
front_len, gfp_t flags,
m->middle = NULL;
/* data */
- m->nr_pages = 0;
+ m->page_count = 0;
m->page_alignment = 0;
m->pages = NULL;
+ m->pagelist_count = 0;
m->pagelist = NULL;
#ifdef CONFIG_BLOCK
m->bio = NULL;
@@ -2890,13 +2891,14 @@ void ceph_msg_last_put(struct kref *kref)
ceph_buffer_put(m->middle);
m->middle = NULL;
}
- m->nr_pages = 0;
+ m->page_count = 0;
m->pages = NULL;
if (m->pagelist) {
ceph_pagelist_release(m->pagelist);
kfree(m->pagelist);
m->pagelist = NULL;
+ m->pagelist_count = 0;
}
m->trail = NULL;
@@ -2910,8 +2912,8 @@ EXPORT_SYMBOL(ceph_msg_last_put);
void ceph_msg_dump(struct ceph_msg *msg)
{
- pr_debug("msg_dump %p (front_max %d nr_pages %d)\n", msg,
- msg->front_max, msg->nr_pages);
+ pr_debug("msg_dump %p (front_max %d page_count %d)\n", msg,
+ msg->front_max, msg->page_count);
print_hex_dump(KERN_DEBUG, "header: ",
DUMP_PREFIX_OFFSET, 16, 1,
&msg->hdr, sizeof(msg->hdr), true);
diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
index 980911e..0a4d4a0 100644
--- a/net/ceph/osd_client.c
+++ b/net/ceph/osd_client.c
@@ -1631,7 +1631,7 @@ int ceph_osdc_start_request(struct ceph_osd_client
*osdc,
int rc = 0;
req->r_request->pages = req->r_pages;
- req->r_request->nr_pages = req->r_num_pages;
+ req->r_request->page_count = req->r_num_pages;
#ifdef CONFIG_BLOCK
req->r_request->bio = req->r_bio;
#endif
@@ -1982,7 +1982,7 @@ static struct ceph_msg *get_reply(struct
ceph_connection *con,
goto out;
}
m->pages = req->r_pages;
- m->nr_pages = req->r_num_pages;
+ m->page_count = req->r_num_pages;
m->page_alignment = req->r_page_alignment;
#ifdef CONFIG_BLOCK
m->bio = req->r_bio;
--
1.7.9.5
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 2/4] libceph: set page alignment in start_request()
2013-02-25 23:40 [PATCH 0/4] libceph: abstract setting message data info Alex Elder
2013-02-25 23:42 ` [PATCH 1/4] libceph: distinguish page array and pagelist count Alex Elder
@ 2013-02-25 23:42 ` Alex Elder
2013-02-25 23:42 ` [PATCH 3/4] libceph: isolate message page field manipulation Alex Elder
` (2 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Alex Elder @ 2013-02-25 23:42 UTC (permalink / raw)
To: ceph-devel@vger.kernel.org
The page alignment field for a request is currently set in
ceph_osdc_build_request(). It's not needed at that point
nor do either of its callers need that value assigned at
any point before they call ceph_osdc_start_request().
So move that assignment into ceph_osdc_start_request().
Signed-off-by: Alex Elder <elder@inktank.com>
---
net/ceph/osd_client.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
index 0a4d4a0..1bb2b59 100644
--- a/net/ceph/osd_client.c
+++ b/net/ceph/osd_client.c
@@ -369,7 +369,6 @@ void ceph_osdc_build_request(struct ceph_osd_request
*req,
data_len += len;
}
req->r_request->hdr.data_len = cpu_to_le32(data_len);
- req->r_request->page_alignment = req->r_page_alignment;
BUG_ON(p > msg->front.iov_base + msg->front.iov_len);
msg_size = p - msg->front.iov_base;
@@ -1632,6 +1631,7 @@ int ceph_osdc_start_request(struct ceph_osd_client
*osdc,
req->r_request->pages = req->r_pages;
req->r_request->page_count = req->r_num_pages;
+ req->r_request->page_alignment = req->r_page_alignment;
#ifdef CONFIG_BLOCK
req->r_request->bio = req->r_bio;
#endif
--
1.7.9.5
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 3/4] libceph: isolate message page field manipulation
2013-02-25 23:40 [PATCH 0/4] libceph: abstract setting message data info Alex Elder
2013-02-25 23:42 ` [PATCH 1/4] libceph: distinguish page array and pagelist count Alex Elder
2013-02-25 23:42 ` [PATCH 2/4] libceph: set page alignment in start_request() Alex Elder
@ 2013-02-25 23:42 ` Alex Elder
2013-02-25 23:42 ` [PATCH 4/4] libceph: isolate other message data fields Alex Elder
2013-02-26 18:39 ` [PATCH 0/4] libceph: abstract setting message data info Josh Durgin
4 siblings, 0 replies; 6+ messages in thread
From: Alex Elder @ 2013-02-25 23:42 UTC (permalink / raw)
To: ceph-devel@vger.kernel.org
Define a function ceph_msg_data_set_pages(), which more clearly
abstracts the assignment page-related fields for data in a ceph
message structure. These fields should never be set more than once
(add BUG_ON() calls to guarantee that). Use this new function in
the osd client and mds client.
Rearrange the field order in a ceph_msg structure to group those
that are used to define the possible data payloads.
Signed-off-by: Alex Elder <elder@inktank.com>
---
fs/ceph/mds_client.c | 4 ++--
include/linux/ceph/messenger.h | 22 +++++++++++++---------
net/ceph/messenger.c | 12 ++++++++++++
net/ceph/osd_client.c | 11 +++++------
4 files changed, 32 insertions(+), 17 deletions(-)
diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
index 44435c2..d8842a1 100644
--- a/fs/ceph/mds_client.c
+++ b/fs/ceph/mds_client.c
@@ -1718,8 +1718,8 @@ static struct ceph_msg
*create_request_message(struct ceph_mds_client *mdsc,
msg->front.iov_len = p - msg->front.iov_base;
msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
- msg->pages = req->r_pages;
- msg->page_count = req->r_num_pages;
+ ceph_msg_data_set_pages(msg, req->r_pages, req->r_num_pages, 0);
+
msg->hdr.data_len = cpu_to_le32(req->r_data_len);
msg->hdr.data_off = cpu_to_le16(0);
diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h
index 1b08349..eeb28a0 100644
--- a/include/linux/ceph/messenger.h
+++ b/include/linux/ceph/messenger.h
@@ -74,22 +74,23 @@ struct ceph_msg {
struct ceph_msg_footer footer; /* footer */
struct kvec front; /* unaligned blobs of message */
struct ceph_buffer *middle;
- struct page **pages; /* data payload. NOT OWNER. */
- unsigned page_count; /* size of page array */
- unsigned page_alignment; /* io offset in first page */
+
+ struct page **pages; /* data payload. NOT OWNER. */
+ unsigned int page_alignment; /* io offset in first page */
+ unsigned int page_count; /* # pages in array or list */
struct ceph_pagelist *pagelist; /* instead of pages */
unsigned int pagelist_count; /* number of pages in pagelist */
-
- struct ceph_connection *con;
- struct list_head list_head;
-
- struct kref kref;
#ifdef CONFIG_BLOCK
+ unsigned int bio_seg; /* current bio segment */
struct bio *bio; /* instead of pages/pagelist */
struct bio *bio_iter; /* bio iterator */
- unsigned int bio_seg; /* current bio segment */
#endif /* CONFIG_BLOCK */
struct ceph_pagelist *trail; /* the trailing part of the data */
+
+ struct ceph_connection *con;
+ struct list_head list_head;
+
+ struct kref kref;
bool front_is_vmalloc;
bool more_to_follow;
bool needs_out_seq;
@@ -219,6 +220,9 @@ extern void ceph_msg_revoke_incoming(struct ceph_msg
*msg);
extern void ceph_con_keepalive(struct ceph_connection *con);
+extern void ceph_msg_data_set_pages(struct ceph_msg *msg, struct page
**pages,
+ unsigned int page_count, size_t alignment);
+
extern struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags,
bool can_fail);
extern void ceph_msg_kfree(struct ceph_msg *m);
diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c
index 9d8abb0..7328e1b 100644
--- a/net/ceph/messenger.c
+++ b/net/ceph/messenger.c
@@ -2674,6 +2674,18 @@ void ceph_con_keepalive(struct ceph_connection *con)
}
EXPORT_SYMBOL(ceph_con_keepalive);
+void ceph_msg_data_set_pages(struct ceph_msg *msg, struct page **pages,
+ unsigned int page_count, size_t alignment)
+{
+ BUG_ON(msg->pages);
+ BUG_ON(msg->page_count);
+ BUG_ON(alignment >= PAGE_SIZE);
+
+ msg->pages = pages;
+ msg->page_count = page_count;
+ msg->page_alignment = alignment;
+}
+EXPORT_SYMBOL(ceph_msg_data_set_pages);
/*
* construct a new message with given type, size
diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
index 1bb2b59..0f8351d 100644
--- a/net/ceph/osd_client.c
+++ b/net/ceph/osd_client.c
@@ -1629,9 +1629,8 @@ int ceph_osdc_start_request(struct ceph_osd_client
*osdc,
{
int rc = 0;
- req->r_request->pages = req->r_pages;
- req->r_request->page_count = req->r_num_pages;
- req->r_request->page_alignment = req->r_page_alignment;
+ ceph_msg_data_set_pages(req->r_request, req->r_pages, req->r_num_pages,
+ req->r_page_alignment);
#ifdef CONFIG_BLOCK
req->r_request->bio = req->r_bio;
#endif
@@ -1981,9 +1980,9 @@ static struct ceph_msg *get_reply(struct
ceph_connection *con,
m = NULL;
goto out;
}
- m->pages = req->r_pages;
- m->page_count = req->r_num_pages;
- m->page_alignment = req->r_page_alignment;
+
+ ceph_msg_data_set_pages(m, req->r_pages, req->r_num_pages,
+ req->r_page_alignment);
#ifdef CONFIG_BLOCK
m->bio = req->r_bio;
#endif
--
1.7.9.5
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 4/4] libceph: isolate other message data fields
2013-02-25 23:40 [PATCH 0/4] libceph: abstract setting message data info Alex Elder
` (2 preceding siblings ...)
2013-02-25 23:42 ` [PATCH 3/4] libceph: isolate message page field manipulation Alex Elder
@ 2013-02-25 23:42 ` Alex Elder
2013-02-26 18:39 ` [PATCH 0/4] libceph: abstract setting message data info Josh Durgin
4 siblings, 0 replies; 6+ messages in thread
From: Alex Elder @ 2013-02-25 23:42 UTC (permalink / raw)
To: ceph-devel@vger.kernel.org
Define ceph_msg_data_set_pagelist(), ceph_msg_data_set_bio(), and
ceph_msg_data_set_trail() to clearly abstract the assignment the
remaining data-related fields in a ceph message structure. These
fields should never be used more than once; add BUG_ON() calls to
guarantee this. Use the new functions in the osd client and mds
client.
Signed-off-by: Alex Elder <elder@inktank.com>
---
fs/ceph/mds_client.c | 4 ++--
include/linux/ceph/messenger.h | 6 ++++++
net/ceph/messenger.c | 28 ++++++++++++++++++++++++++++
net/ceph/osd_client.c | 6 +++---
4 files changed, 39 insertions(+), 5 deletions(-)
diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
index d8842a1..c0eb44d 100644
--- a/fs/ceph/mds_client.c
+++ b/fs/ceph/mds_client.c
@@ -2599,8 +2599,8 @@ static void send_mds_reconnect(struct
ceph_mds_client *mdsc,
goto fail;
}
- reply->pagelist = pagelist;
- reply->pagelist_count = calc_pages_for(0, pagelist->length);
+ ceph_msg_data_set_pagelist(reply, pagelist,
+ calc_pages_for(0, pagelist->length));
if (recon_state.flock)
reply->hdr.version = cpu_to_le16(2);
reply->hdr.data_len = cpu_to_le32(pagelist->length);
diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h
index eeb28a0..3cb3e8c 100644
--- a/include/linux/ceph/messenger.h
+++ b/include/linux/ceph/messenger.h
@@ -222,6 +222,12 @@ extern void ceph_con_keepalive(struct
ceph_connection *con);
extern void ceph_msg_data_set_pages(struct ceph_msg *msg, struct page
**pages,
unsigned int page_count, size_t alignment);
+extern void ceph_msg_data_set_pagelist(struct ceph_msg *msg,
+ struct ceph_pagelist *pagelist,
+ unsigned int page_count);
+extern void ceph_msg_data_set_bio(struct ceph_msg *msg, struct bio *bio);
+extern void ceph_msg_data_set_trail(struct ceph_msg *msg,
+ struct ceph_pagelist *trail);
extern struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags,
bool can_fail);
diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c
index 7328e1b..82e3212 100644
--- a/net/ceph/messenger.c
+++ b/net/ceph/messenger.c
@@ -2687,6 +2687,34 @@ void ceph_msg_data_set_pages(struct ceph_msg
*msg, struct page **pages,
}
EXPORT_SYMBOL(ceph_msg_data_set_pages);
+void ceph_msg_data_set_pagelist(struct ceph_msg *msg,
+ struct ceph_pagelist *pagelist,
+ unsigned int pagelist_count)
+{
+ BUG_ON(msg->pagelist);
+ BUG_ON(msg->pagelist_count);
+
+ msg->pagelist = pagelist;
+ msg->pagelist_count = pagelist_count;
+}
+EXPORT_SYMBOL(ceph_msg_data_set_pagelist);
+
+void ceph_msg_data_set_bio(struct ceph_msg *msg, struct bio *bio)
+{
+ BUG_ON(msg->bio);
+
+ msg->bio = bio;
+}
+EXPORT_SYMBOL(ceph_msg_data_set_bio);
+
+void ceph_msg_data_set_trail(struct ceph_msg *msg, struct ceph_pagelist
*trail)
+{
+ BUG_ON(msg->trail);
+
+ msg->trail = trail;
+}
+EXPORT_SYMBOL(ceph_msg_data_set_trail);
+
/*
* construct a new message with given type, size
* the new msg has a ref count of 1.
diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
index 0f8351d..f984500 100644
--- a/net/ceph/osd_client.c
+++ b/net/ceph/osd_client.c
@@ -1632,9 +1632,9 @@ int ceph_osdc_start_request(struct ceph_osd_client
*osdc,
ceph_msg_data_set_pages(req->r_request, req->r_pages, req->r_num_pages,
req->r_page_alignment);
#ifdef CONFIG_BLOCK
- req->r_request->bio = req->r_bio;
+ ceph_msg_data_set_bio(req->r_request, req->r_bio);
#endif
- req->r_request->trail = &req->r_trail;
+ ceph_msg_data_set_trail(req->r_request, &req->r_trail);
register_request(osdc, req);
@@ -1984,7 +1984,7 @@ static struct ceph_msg *get_reply(struct
ceph_connection *con,
ceph_msg_data_set_pages(m, req->r_pages, req->r_num_pages,
req->r_page_alignment);
#ifdef CONFIG_BLOCK
- m->bio = req->r_bio;
+ ceph_msg_data_set_bio(m, req->r_bio);
#endif
}
*skip = 0;
--
1.7.9.5
^ permalink raw reply related [flat|nested] 6+ messages in thread* Re: [PATCH 0/4] libceph: abstract setting message data info
2013-02-25 23:40 [PATCH 0/4] libceph: abstract setting message data info Alex Elder
` (3 preceding siblings ...)
2013-02-25 23:42 ` [PATCH 4/4] libceph: isolate other message data fields Alex Elder
@ 2013-02-26 18:39 ` Josh Durgin
4 siblings, 0 replies; 6+ messages in thread
From: Josh Durgin @ 2013-02-26 18:39 UTC (permalink / raw)
To: Alex Elder; +Cc: ceph-devel@vger.kernel.org
On 02/25/2013 03:40 PM, Alex Elder wrote:
> This series makes the fields related to the data portion of
> a ceph message not get manipulated by code outside the ceph
> messenger. It implements some interface functions that can
> be used to assign data-related fields. Doing this will allow
> the way message data is managed to be changed independent
> of the users of the messenger module.
>
> -Alex
>
> [PATCH 1/4] libceph: distinguish page array and pagelist count
> [PATCH 2/4] libceph: set page alignment in start_request()
> [PATCH 3/4] libceph: isolate message page field manipulation
> [PATCH 4/4] libceph: isolate other message data fields
These all look good.
Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
^ permalink raw reply [flat|nested] 6+ messages in thread