[PATCH] rbd: use async write for copy and import

[PATCH] rbd: use async write for copy and import

[Christian Brunner]

Hi Yehuda,

copying images whith "rbd cp" can be quite slow. I did the
following change to speed up copy and import. Please decide
if you want to include it.

Christian

---
 src/rbd.cc |   28 ++++++++++++++++++++++++++--
 1 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/src/rbd.cc b/src/rbd.cc
index 07849ed..7c71b7d 100644
--- a/src/rbd.cc
+++ b/src/rbd.cc
@@ -37,6 +37,8 @@ struct pools {
 
 typedef struct pools pools_t;
 
+int64_t aiocnt = 0;
+
 static Rados rados;
 static string dir_oid = RBD_DIRECTORY;
 static string dir_info_oid = RBD_INFO;
@@ -722,6 +724,12 @@ done_img:
   update_snap_name(*new_img, snap);
 }
 
+static void aio_write_cb(Rados::AioCompletion comp, void *c) 
+{
+  comp.release();
+  aiocnt--;
+}
+
 static int do_import(pool_t pool, const char *imgname, int order, const char *path)
 {
   int fd = open(path, O_RDONLY);
@@ -730,6 +738,7 @@ static int do_import(pool_t pool, const char *imgname, int order, const char *pa
   uint64_t block_size;
   struct stat stat_buf;
   string md_oid;
+  Rados::AioCompletion *comp;
 
   if (fd < 0) {
     r = -errno;
@@ -784,7 +793,9 @@ static int do_import(pool_t pool, const char *imgname, int order, const char *pa
       bufferlist bl;
       bl.append(p);
       string oid = get_block_oid(&header, i);
-      r = rados.write(pool, oid, 0, bl, len);
+      comp = rados.aio_create_completion(NULL, NULL, (callback_t) aio_write_cb);
+      r = rados.aio_write(pool, oid, 0, bl, len, comp);
+      aiocnt++;
       if (r < 0) {
         cerr << "error writing to image block" << std::endl;
         return r;
@@ -796,6 +807,11 @@ static int do_import(pool_t pool, const char *imgname, int order, const char *pa
     seg_pos += seg_size;
   }
 
+  while(aiocnt > 0) {
+    cerr << "waiting for " << aiocnt << " async writes" << std::endl;
+    usleep(100000);
+  }
+
   return 0;
 }
 
@@ -805,6 +821,7 @@ static int do_copy(pools_t& pp, const char *imgname, const char *destname)
   int64_t ret;
   int r;
   string md_oid, dest_md_oid;
+  Rados::AioCompletion *comp;
 
   md_oid = imgname;
   md_oid += RBD_SUFFIX;
@@ -843,7 +860,9 @@ static int do_copy(pools_t& pp, const char *imgname, const char *destname)
 
     if (bl.length()) {
       string dest_oid = get_block_oid(&dest_header, i);
-      r = rados.write(pp.dest, dest_oid, 0, bl, bl.length());
+      comp = rados.aio_create_completion(NULL, NULL, (callback_t) aio_write_cb);
+      r = rados.aio_write(pp.dest, dest_oid, 0, bl, bl.length(), comp);
+      aiocnt++;
       if (r < 0) {
         cerr << "failed to write block " << dest_oid << std::endl;
         return r;
@@ -851,6 +870,11 @@ static int do_copy(pools_t& pp, const char *imgname, const char *destname)
     }
   }
 
+  while(aiocnt > 0) {
+    cerr << "waiting for " << aiocnt << " async writes" << std::endl;
+    usleep(100000);
+  }
+
   return 0;
 }
 
-- 
1.7.0.4

Re: [PATCH] rbd: use async write for copy and import

[Yehuda Sadeh Weinraub]

On Mon, Aug 2, 2010 at 1:12 PM, Christian Brunner <chb@muc.de> wrote:
> Hi Yehuda,
>
> copying images whith "rbd cp" can be quite slow. I did the
> following change to speed up copy and import. Please decide
> if you want to include it.

We'd certainly be happy to be able to speed up those operations. There
are some issues with this implementation, mainly not protecting
aiocnt, and using sleep. Also, it looks like it is unbounded, so with
large images you'd easily run out of memory.
It can be altered to be similar to the patch you supplied for
throttling the qemu-rbd messages, using a conditional variable.

Thanks,
Yehuda