[Qemu-devel] [PATCH v2 0/3] migration: add speed limit for multifd migration

[Qemu-devel] [PATCH v2 0/3] migration: add speed limit for multifd migration

[Ivan Ren]

From: Ivan Ren <ivanren@tencent.com>

Currently multifd migration has not been limited and it will consume
the whole bandwidth of Nic. These two patches add speed limitation to
it.

This is the v3 patches:

v3 VS v2:
Add Reviewed info and Suggested info.

v2 VS v1:
1. change qemu_file_update_rate_transfer interface name
   to qemu_file_update_transfer
2. add a new patch to update ram_counters for multifd sync packet

Ivan Ren (3):
  migration: add qemu_file_update_transfer interface
  migration: add speed limit for multifd migration
  migration: update ram_counters for multifd sync packet

 migration/qemu-file.c |  5 +++++
 migration/qemu-file.h |  1 +
 migration/ram.c       | 24 ++++++++++++++----------
 3 files changed, 20 insertions(+), 10 deletions(-)

-- 
2.17.2 (Apple Git-113)

[Qemu-devel] [PATCH v3 1/3] migration: add qemu_file_update_transfer interface

[Ivan Ren]

From: Ivan Ren <ivanren@tencent.com>

Add qemu_file_update_transfer for just update bytes_xfer for speed
limitation. This will be used for further migration feature such as
multifd migration.

Signed-off-by: Ivan Ren <ivanren@tencent.com>
Reviewed-by: Wei Yang <richardw.yang@linux.intel.com> 
---
 migration/qemu-file.c | 5 +++++
 migration/qemu-file.h | 1 +
 2 files changed, 6 insertions(+)

diff --git a/migration/qemu-file.c b/migration/qemu-file.c
index 0431585502..18f480529a 100644
--- a/migration/qemu-file.c
+++ b/migration/qemu-file.c
@@ -615,6 +615,11 @@ void qemu_file_reset_rate_limit(QEMUFile *f)
     f->bytes_xfer = 0;
 }
 
+void qemu_file_update_transfer(QEMUFile *f, int64_t len)
+{
+    f->bytes_xfer += len;
+}
+
 void qemu_put_be16(QEMUFile *f, unsigned int v)
 {
     qemu_put_byte(f, v >> 8);
diff --git a/migration/qemu-file.h b/migration/qemu-file.h
index 13baf896bd..5de9fa2e96 100644
--- a/migration/qemu-file.h
+++ b/migration/qemu-file.h
@@ -147,6 +147,7 @@ int qemu_peek_byte(QEMUFile *f, int offset);
 void qemu_file_skip(QEMUFile *f, int size);
 void qemu_update_position(QEMUFile *f, size_t size);
 void qemu_file_reset_rate_limit(QEMUFile *f);
+void qemu_file_update_transfer(QEMUFile *f, int64_t len);
 void qemu_file_set_rate_limit(QEMUFile *f, int64_t new_rate);
 int64_t qemu_file_get_rate_limit(QEMUFile *f);
 void qemu_file_set_error(QEMUFile *f, int ret);
-- 
2.17.2 (Apple Git-113)

Re: [Qemu-devel] [PATCH v3 1/3] migration: add qemu_file_update_transfer interface

[Juan Quintela]

Ivan Ren <renyime@gmail.com> wrote:
> From: Ivan Ren <ivanren@tencent.com>
>
> Add qemu_file_update_transfer for just update bytes_xfer for speed
> limitation. This will be used for further migration feature such as
> multifd migration.
>
> Signed-off-by: Ivan Ren <ivanren@tencent.com>
> Reviewed-by: Wei Yang <richardw.yang@linux.intel.com> 

Reviewed-by: Juan Quintela <quintela@redhat.com>

[Qemu-devel] [PATCH v3 2/3] migration: add speed limit for multifd migration

[Ivan Ren]

From: Ivan Ren <ivanren@tencent.com>

Limit the speed of multifd migration through common speed limitation
qemu file.

Signed-off-by: Ivan Ren <ivanren@tencent.com>
---
 migration/ram.c | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/migration/ram.c b/migration/ram.c
index 889148dd84..88ddd2bbe2 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -922,7 +922,7 @@ struct {
  * false.
  */
 
-static int multifd_send_pages(void)
+static int multifd_send_pages(RAMState *rs)
 {
     int i;
     static int next_channel;
@@ -954,6 +954,7 @@ static int multifd_send_pages(void)
     multifd_send_state->pages = p->pages;
     p->pages = pages;
     transferred = ((uint64_t) pages->used) * TARGET_PAGE_SIZE + p->packet_len;
+    qemu_file_update_transfer(rs->f, transferred);
     ram_counters.multifd_bytes += transferred;
     ram_counters.transferred += transferred;;
     qemu_mutex_unlock(&p->mutex);
@@ -962,7 +963,7 @@ static int multifd_send_pages(void)
     return 1;
 }
 
-static int multifd_queue_page(RAMBlock *block, ram_addr_t offset)
+static int multifd_queue_page(RAMState *rs, RAMBlock *block, ram_addr_t offset)
 {
     MultiFDPages_t *pages = multifd_send_state->pages;
 
@@ -981,12 +982,12 @@ static int multifd_queue_page(RAMBlock *block, ram_addr_t offset)
         }
     }
 
-    if (multifd_send_pages() < 0) {
+    if (multifd_send_pages(rs) < 0) {
         return -1;
     }
 
     if (pages->block != block) {
-        return  multifd_queue_page(block, offset);
+        return  multifd_queue_page(rs, block, offset);
     }
 
     return 1;
@@ -1054,7 +1055,7 @@ void multifd_save_cleanup(void)
     multifd_send_state = NULL;
 }
 
-static void multifd_send_sync_main(void)
+static void multifd_send_sync_main(RAMState *rs)
 {
     int i;
 
@@ -1062,7 +1063,7 @@ static void multifd_send_sync_main(void)
         return;
     }
     if (multifd_send_state->pages->used) {
-        if (multifd_send_pages() < 0) {
+        if (multifd_send_pages(rs) < 0) {
             error_report("%s: multifd_send_pages fail", __func__);
             return;
         }
@@ -1083,6 +1084,7 @@ static void multifd_send_sync_main(void)
         p->packet_num = multifd_send_state->packet_num++;
         p->flags |= MULTIFD_FLAG_SYNC;
         p->pending_job++;
+        qemu_file_update_transfer(rs->f, p->packet_len);
         qemu_mutex_unlock(&p->mutex);
         qemu_sem_post(&p->sem);
     }
@@ -2079,7 +2081,7 @@ static int ram_save_page(RAMState *rs, PageSearchStatus *pss, bool last_stage)
 static int ram_save_multifd_page(RAMState *rs, RAMBlock *block,
                                  ram_addr_t offset)
 {
-    if (multifd_queue_page(block, offset) < 0) {
+    if (multifd_queue_page(rs, block, offset) < 0) {
         return -1;
     }
     ram_counters.normal++;
@@ -3482,7 +3484,7 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
     ram_control_before_iterate(f, RAM_CONTROL_SETUP);
     ram_control_after_iterate(f, RAM_CONTROL_SETUP);
 
-    multifd_send_sync_main();
+    multifd_send_sync_main(*rsp);
     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
     qemu_fflush(f);
 
@@ -3570,7 +3572,7 @@ static int ram_save_iterate(QEMUFile *f, void *opaque)
     ram_control_after_iterate(f, RAM_CONTROL_ROUND);
 
 out:
-    multifd_send_sync_main();
+    multifd_send_sync_main(rs);
     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
     qemu_fflush(f);
     ram_counters.transferred += 8;
@@ -3629,7 +3631,7 @@ static int ram_save_complete(QEMUFile *f, void *opaque)
 
     rcu_read_unlock();
 
-    multifd_send_sync_main();
+    multifd_send_sync_main(rs);
     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
     qemu_fflush(f);
 
-- 
2.17.2 (Apple Git-113)

Re: [Qemu-devel] [PATCH v3 2/3] migration: add speed limit for multifd migration

[Wei Yang]

On Tue, Jul 30, 2019 at 01:33:35PM +0800, Ivan Ren wrote:
>From: Ivan Ren <ivanren@tencent.com>
>
>Limit the speed of multifd migration through common speed limitation
>qemu file.
>
>Signed-off-by: Ivan Ren <ivanren@tencent.com>

Reviewed-by: Wei Yang <richardw.yang@linux.intel.com>

>---
> migration/ram.c | 22 ++++++++++++----------
> 1 file changed, 12 insertions(+), 10 deletions(-)
>
>diff --git a/migration/ram.c b/migration/ram.c
>index 889148dd84..88ddd2bbe2 100644
>--- a/migration/ram.c
>+++ b/migration/ram.c
>@@ -922,7 +922,7 @@ struct {
>  * false.
>  */
> 
>-static int multifd_send_pages(void)
>+static int multifd_send_pages(RAMState *rs)
> {
>     int i;
>     static int next_channel;
>@@ -954,6 +954,7 @@ static int multifd_send_pages(void)
>     multifd_send_state->pages = p->pages;
>     p->pages = pages;
>     transferred = ((uint64_t) pages->used) * TARGET_PAGE_SIZE + p->packet_len;
>+    qemu_file_update_transfer(rs->f, transferred);
>     ram_counters.multifd_bytes += transferred;
>     ram_counters.transferred += transferred;;
>     qemu_mutex_unlock(&p->mutex);
>@@ -962,7 +963,7 @@ static int multifd_send_pages(void)
>     return 1;
> }
> 
>-static int multifd_queue_page(RAMBlock *block, ram_addr_t offset)
>+static int multifd_queue_page(RAMState *rs, RAMBlock *block, ram_addr_t offset)
> {
>     MultiFDPages_t *pages = multifd_send_state->pages;
> 
>@@ -981,12 +982,12 @@ static int multifd_queue_page(RAMBlock *block, ram_addr_t offset)
>         }
>     }
> 
>-    if (multifd_send_pages() < 0) {
>+    if (multifd_send_pages(rs) < 0) {
>         return -1;
>     }
> 
>     if (pages->block != block) {
>-        return  multifd_queue_page(block, offset);
>+        return  multifd_queue_page(rs, block, offset);
>     }
> 
>     return 1;
>@@ -1054,7 +1055,7 @@ void multifd_save_cleanup(void)
>     multifd_send_state = NULL;
> }
> 
>-static void multifd_send_sync_main(void)
>+static void multifd_send_sync_main(RAMState *rs)
> {
>     int i;
> 
>@@ -1062,7 +1063,7 @@ static void multifd_send_sync_main(void)
>         return;
>     }
>     if (multifd_send_state->pages->used) {
>-        if (multifd_send_pages() < 0) {
>+        if (multifd_send_pages(rs) < 0) {
>             error_report("%s: multifd_send_pages fail", __func__);
>             return;
>         }
>@@ -1083,6 +1084,7 @@ static void multifd_send_sync_main(void)
>         p->packet_num = multifd_send_state->packet_num++;
>         p->flags |= MULTIFD_FLAG_SYNC;
>         p->pending_job++;
>+        qemu_file_update_transfer(rs->f, p->packet_len);
>         qemu_mutex_unlock(&p->mutex);
>         qemu_sem_post(&p->sem);
>     }
>@@ -2079,7 +2081,7 @@ static int ram_save_page(RAMState *rs, PageSearchStatus *pss, bool last_stage)
> static int ram_save_multifd_page(RAMState *rs, RAMBlock *block,
>                                  ram_addr_t offset)
> {
>-    if (multifd_queue_page(block, offset) < 0) {
>+    if (multifd_queue_page(rs, block, offset) < 0) {
>         return -1;
>     }
>     ram_counters.normal++;
>@@ -3482,7 +3484,7 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
>     ram_control_before_iterate(f, RAM_CONTROL_SETUP);
>     ram_control_after_iterate(f, RAM_CONTROL_SETUP);
> 
>-    multifd_send_sync_main();
>+    multifd_send_sync_main(*rsp);
>     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
>     qemu_fflush(f);
> 
>@@ -3570,7 +3572,7 @@ static int ram_save_iterate(QEMUFile *f, void *opaque)
>     ram_control_after_iterate(f, RAM_CONTROL_ROUND);
> 
> out:
>-    multifd_send_sync_main();
>+    multifd_send_sync_main(rs);
>     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
>     qemu_fflush(f);
>     ram_counters.transferred += 8;
>@@ -3629,7 +3631,7 @@ static int ram_save_complete(QEMUFile *f, void *opaque)
> 
>     rcu_read_unlock();
> 
>-    multifd_send_sync_main();
>+    multifd_send_sync_main(rs);
>     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
>     qemu_fflush(f);
> 
>-- 
>2.17.2 (Apple Git-113)
>

-- 
Wei Yang
Help you, Help me

Re: [Qemu-devel] [PATCH v3 2/3] migration: add speed limit for multifd migration

[Juan Quintela]

Ivan Ren <renyime@gmail.com> wrote:
> From: Ivan Ren <ivanren@tencent.com>
>
> Limit the speed of multifd migration through common speed limitation
> qemu file.
>
> Signed-off-by: Ivan Ren <ivanren@tencent.com>

Reviewed-by: Juan Quintela <quintela@redhat.com>

[Qemu-devel] [PATCH v3 3/3] migration: update ram_counters for multifd sync packet

[Ivan Ren]

From: Ivan Ren <ivanren@tencent.com>

Multifd sync will send MULTIFD_FLAG_SYNC flag info to destination, add
these bytes to ram_counters record.

Signed-off-by: Ivan Ren <ivanren@tencent.com>
Suggested-by: Wei Yang <richardw.yang@linux.intel.com>
---
 migration/ram.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/migration/ram.c b/migration/ram.c
index 88ddd2bbe2..20b6eebb7c 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -1085,6 +1085,8 @@ static void multifd_send_sync_main(RAMState *rs)
         p->flags |= MULTIFD_FLAG_SYNC;
         p->pending_job++;
         qemu_file_update_transfer(rs->f, p->packet_len);
+        ram_counters.multifd_bytes += p->packet_len;
+        ram_counters.transferred += p->packet_len;
         qemu_mutex_unlock(&p->mutex);
         qemu_sem_post(&p->sem);
     }
-- 
2.17.2 (Apple Git-113)

Re: [Qemu-devel] [PATCH v3 3/3] migration: update ram_counters for multifd sync packet

[Juan Quintela]

Ivan Ren <renyime@gmail.com> wrote:
> From: Ivan Ren <ivanren@tencent.com>
>
> Multifd sync will send MULTIFD_FLAG_SYNC flag info to destination, add
> these bytes to ram_counters record.
>
> Signed-off-by: Ivan Ren <ivanren@tencent.com>
> Suggested-by: Wei Yang <richardw.yang@linux.intel.com>

Reviewed-by: Juan Quintela <quintela@redhat.com>

Re: [Qemu-devel] [PATCH v2 0/3] migration: add speed limit for multifd migration

[Dr. David Alan Gilbert]

* Ivan Ren (renyime@gmail.com) wrote:
> From: Ivan Ren <ivanren@tencent.com>
> 
> Currently multifd migration has not been limited and it will consume
> the whole bandwidth of Nic. These two patches add speed limitation to
> it.

Queued

> 
> This is the v3 patches:
> 
> v3 VS v2:
> Add Reviewed info and Suggested info.
> 
> v2 VS v1:
> 1. change qemu_file_update_rate_transfer interface name
>    to qemu_file_update_transfer
> 2. add a new patch to update ram_counters for multifd sync packet
> 
> Ivan Ren (3):
>   migration: add qemu_file_update_transfer interface
>   migration: add speed limit for multifd migration
>   migration: update ram_counters for multifd sync packet
> 
>  migration/qemu-file.c |  5 +++++
>  migration/qemu-file.h |  1 +
>  migration/ram.c       | 24 ++++++++++++++----------
>  3 files changed, 20 insertions(+), 10 deletions(-)
> 
> -- 
> 2.17.2 (Apple Git-113)
> 
> 
--
Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK