Re: [Qemu-devel] [PATCH COLO-Frame v12 18/38] COLO: Flush PVM's cached RAM into SVM's memory

[Hailiang Zhang <zhang.zhanghailiang@huawei.com>]

On 2015/12/15 19:07, Changlong Xie wrote:
> On 12/15/2015 04:22 PM, zhanghailiang wrote:
>> During the time of VM's running, PVM may dirty some pages, we will transfer
>> PVM's dirty pages to SVM and store them into SVM's RAM cache at next checkpoint
>> time. So, the content of SVM's RAM cache will always be some with PVM's memory
> "some" => "same"
>

Fixed, thanks.

> Thanks
>      -Xie
>> after checkpoint.
>>
>> Instead of flushing all content of PVM's RAM cache into SVM's MEMORY,
>> we do this in a more efficient way:
>> Only flush any page that dirtied by PVM since last checkpoint.
>> In this way, we can ensure SVM's memory same with PVM's.
>>
>> Besides, we must ensure flush RAM cache before load device state.
>>
>> Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
>> Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
>> Signed-off-by: Gonglei <arei.gonglei@huawei.com>
>> Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
>> ---
>> v12:
>> - Add a trace point in the end of colo_flush_ram_cache() (Dave's suggestion)
>> - Add Reviewed-by tag
>> v11:
>> - Move the place of 'need_flush' (Dave's suggestion)
>> - Remove unused 'DPRINTF("Flush ram_cache\n")'
>> v10:
>> - trace the number of dirty pages that be received.
>>
>> Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
>> ---
>>   include/migration/migration.h |  1 +
>>   migration/colo.c              |  2 --
>>   migration/ram.c               | 38 ++++++++++++++++++++++++++++++++++++++
>>   trace-events                  |  2 ++
>>   4 files changed, 41 insertions(+), 2 deletions(-)
>>
>> diff --git a/include/migration/migration.h b/include/migration/migration.h
>> index e41372d..221176b 100644
>> --- a/include/migration/migration.h
>> +++ b/include/migration/migration.h
>> @@ -336,4 +336,5 @@ PostcopyState postcopy_state_set(PostcopyState new_state);
>>   /* ram cache */
>>   int colo_init_ram_cache(void);
>>   void colo_release_ram_cache(void);
>> +void colo_flush_ram_cache(void);
>>   #endif
>> diff --git a/migration/colo.c b/migration/colo.c
>> index a4d49ff..e40cdb9 100644
>> --- a/migration/colo.c
>> +++ b/migration/colo.c
>> @@ -401,8 +401,6 @@ void *colo_process_incoming_thread(void *opaque)
>>           }
>>           qemu_mutex_unlock_iothread();
>>
>> -        /* TODO: flush vm state */
>> -
>>           ret = colo_put_cmd(mis->to_src_file, COLO_COMMAND_VMSTATE_LOADED);
>>           if (ret < 0) {
>>               goto out;
>> diff --git a/migration/ram.c b/migration/ram.c
>> index 3d5947b..8ff7f7c 100644
>> --- a/migration/ram.c
>> +++ b/migration/ram.c
>> @@ -2458,6 +2458,7 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
>>        * be atomic
>>        */
>>       bool postcopy_running = postcopy_state_get() >= POSTCOPY_INCOMING_LISTENING;
>> +    bool need_flush = false;
>>
>>       seq_iter++;
>>
>> @@ -2492,6 +2493,7 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
>>               /* After going into COLO, we should load the Page into colo_cache */
>>               if (ram_cache_enable) {
>>                   host = colo_cache_from_block_offset(block, addr);
>> +                need_flush = true;
>>               } else {
>>                   host = host_from_ram_block_offset(block, addr);
>>               }
>> @@ -2585,6 +2587,10 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
>>       }
>>
>>       rcu_read_unlock();
>> +
>> +    if (!ret  && ram_cache_enable && need_flush) {
>> +        colo_flush_ram_cache();
>> +    }
>>       DPRINTF("Completed load of VM with exit code %d seq iteration "
>>               "%" PRIu64 "\n", ret, seq_iter);
>>       return ret;
>> @@ -2657,6 +2663,38 @@ void colo_release_ram_cache(void)
>>       rcu_read_unlock();
>>   }
>>
>> +/*
>> + * Flush content of RAM cache into SVM's memory.
>> + * Only flush the pages that be dirtied by PVM or SVM or both.
>> + */
>> +void colo_flush_ram_cache(void)
>> +{
>> +    RAMBlock *block = NULL;
>> +    void *dst_host;
>> +    void *src_host;
>> +    ram_addr_t offset = 0;
>> +
>> +    trace_colo_flush_ram_cache_begin(migration_dirty_pages);
>> +    rcu_read_lock();
>> +    block = QLIST_FIRST_RCU(&ram_list.blocks);
>> +    while (block) {
>> +        ram_addr_t ram_addr_abs;
>> +        offset = migration_bitmap_find_dirty(block, offset, &ram_addr_abs);
>> +        migration_bitmap_clear_dirty(ram_addr_abs);
>> +        if (offset >= block->used_length) {
>> +            offset = 0;
>> +            block = QLIST_NEXT_RCU(block, next);
>> +        } else {
>> +            dst_host = block->host + offset;
>> +            src_host = block->colo_cache + offset;
>> +            memcpy(dst_host, src_host, TARGET_PAGE_SIZE);
>> +        }
>> +    }
>> +    rcu_read_unlock();
>> +    trace_colo_flush_ram_cache_end();
>> +    assert(migration_dirty_pages == 0);
>> +}
>> +
>>   static SaveVMHandlers savevm_ram_handlers = {
>>       .save_live_setup = ram_save_setup,
>>       .save_live_iterate = ram_save_iterate,
>> diff --git a/trace-events b/trace-events
>> index 39fdd8d..7f76029 100644
>> --- a/trace-events
>> +++ b/trace-events
>> @@ -1264,6 +1264,8 @@ migration_throttle(void) ""
>>   ram_load_postcopy_loop(uint64_t addr, int flags) "@%" PRIx64 " %x"
>>   ram_postcopy_send_discard_bitmap(void) ""
>>   ram_save_queue_pages(const char *rbname, size_t start, size_t len) "%s: start: %zx len: %zx"
>> +colo_flush_ram_cache_begin(uint64_t dirty_pages) "dirty_pages %" PRIu64
>> +colo_flush_ram_cache_end(void) ""
>>
>>   # hw/display/qxl.c
>>   disable qxl_interface_set_mm_time(int qid, uint32_t mm_time) "%d %d"
>>
>
>
>
> .
>