* [PATCH v2 1/6] hmp: Add "calc_dirty_rate" and "info dirty_rate" cmds
2021-06-06 19:44 [PATCH v2 0/6] support dirtyrate at the granualrity of vcpu huangy81
@ 2021-06-06 19:45 ` huangy81
2021-06-06 19:45 ` [PATCH v2 2/6] KVM: introduce dirty_pages and kvm_dirty_ring_enabled huangy81
` (4 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: huangy81 @ 2021-06-06 19:45 UTC (permalink / raw)
To: qemu-devel
Cc: Eduardo Habkost, Juan Quintela, Hyman, Dr. David Alan Gilbert,
Peter Xu, Paolo Bonzini
From: Peter Xu <peterx@redhat.com>
These two commands are missing when adding the QMP sister commands.
Add them, so developers can play with them easier.
Signed-off-by: Peter Xu <peterx@redhat.com>
Signed-off-by: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
---
hmp-commands-info.hx | 13 ++++++++++++
hmp-commands.hx | 14 +++++++++++++
include/monitor/hmp.h | 2 ++
migration/dirtyrate.c | 47 +++++++++++++++++++++++++++++++++++++++++++
4 files changed, 76 insertions(+)
diff --git a/hmp-commands-info.hx b/hmp-commands-info.hx
index b2347a6aea..fb59c27200 100644
--- a/hmp-commands-info.hx
+++ b/hmp-commands-info.hx
@@ -867,3 +867,16 @@ SRST
``info replay``
Display the record/replay information: mode and the current icount.
ERST
+
+ {
+ .name = "dirty_rate",
+ .args_type = "",
+ .params = "",
+ .help = "show dirty rate information",
+ .cmd = hmp_info_dirty_rate,
+ },
+
+SRST
+ ``info dirty_rate``
+ Display the vcpu dirty rate information.
+ERST
diff --git a/hmp-commands.hx b/hmp-commands.hx
index 2d21fe5ad4..84dcc3aae6 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -1727,3 +1727,17 @@ ERST
.flags = "p",
},
+SRST
+``calc_dirty_rate`` *second*
+ Start a round of dirty rate measurement with the period specified in *second*.
+ The result of the dirty rate measurement may be observed with ``info
+ dirty_rate`` command.
+ERST
+
+ {
+ .name = "calc_dirty_rate",
+ .args_type = "second:l,sample_pages_per_GB:l?",
+ .params = "second [sample_pages_per_GB]",
+ .help = "start a round of guest dirty rate measurement",
+ .cmd = hmp_calc_dirty_rate,
+ },
diff --git a/include/monitor/hmp.h b/include/monitor/hmp.h
index 605d57287a..3baa1058e2 100644
--- a/include/monitor/hmp.h
+++ b/include/monitor/hmp.h
@@ -129,5 +129,7 @@ void hmp_info_replay(Monitor *mon, const QDict *qdict);
void hmp_replay_break(Monitor *mon, const QDict *qdict);
void hmp_replay_delete_break(Monitor *mon, const QDict *qdict);
void hmp_replay_seek(Monitor *mon, const QDict *qdict);
+void hmp_info_dirty_rate(Monitor *mon, const QDict *qdict);
+void hmp_calc_dirty_rate(Monitor *mon, const QDict *qdict);
#endif
diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c
index 2ee3890721..320c56ba2c 100644
--- a/migration/dirtyrate.c
+++ b/migration/dirtyrate.c
@@ -20,6 +20,9 @@
#include "ram.h"
#include "trace.h"
#include "dirtyrate.h"
+#include "monitor/hmp.h"
+#include "monitor/monitor.h"
+#include "qapi/qmp/qdict.h"
static int CalculatingState = DIRTY_RATE_STATUS_UNSTARTED;
static struct DirtyRateStat DirtyStat;
@@ -447,3 +450,47 @@ struct DirtyRateInfo *qmp_query_dirty_rate(Error **errp)
{
return query_dirty_rate_info();
}
+
+void hmp_info_dirty_rate(Monitor *mon, const QDict *qdict)
+{
+ DirtyRateInfo *info = query_dirty_rate_info();
+
+ monitor_printf(mon, "Status: %s\n",
+ DirtyRateStatus_str(info->status));
+ monitor_printf(mon, "Start Time: %"PRIi64" (ms)\n",
+ info->start_time);
+ monitor_printf(mon, "Sample Pages: %"PRIu64" (per GB)\n",
+ info->sample_pages);
+ monitor_printf(mon, "Period: %"PRIi64" (sec)\n",
+ info->calc_time);
+ monitor_printf(mon, "Dirty rate: ");
+ if (info->has_dirty_rate) {
+ monitor_printf(mon, "%"PRIi64" (MB/s)\n", info->dirty_rate);
+ } else {
+ monitor_printf(mon, "(not ready)\n");
+ }
+ g_free(info);
+}
+
+void hmp_calc_dirty_rate(Monitor *mon, const QDict *qdict)
+{
+ int64_t sec = qdict_get_try_int(qdict, "second", 0);
+ int64_t sample_pages = qdict_get_try_int(qdict, "sample_pages_per_GB", -1);
+ bool has_sample_pages = (sample_pages != -1);
+ Error *err = NULL;
+
+ if (!sec) {
+ monitor_printf(mon, "Incorrect period length specified!\n");
+ return;
+ }
+
+ qmp_calc_dirty_rate(sec, has_sample_pages, sample_pages, &err);
+ if (err) {
+ hmp_handle_error(mon, err);
+ return;
+ }
+
+ monitor_printf(mon, "Starting dirty rate measurement with period %"PRIi64
+ " seconds\n", sec);
+ monitor_printf(mon, "[Please use 'info dirty_rate' to check results]\n");
+}
--
2.18.2
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH v2 2/6] KVM: introduce dirty_pages and kvm_dirty_ring_enabled
2021-06-06 19:44 [PATCH v2 0/6] support dirtyrate at the granualrity of vcpu huangy81
2021-06-06 19:45 ` [PATCH v2 1/6] hmp: Add "calc_dirty_rate" and "info dirty_rate" cmds huangy81
@ 2021-06-06 19:45 ` huangy81
2021-06-06 19:47 ` [PATCH v2 3/6] migration/dirtyrate: add per-vcpu option for calc-dirty-rate huangy81
` (3 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: huangy81 @ 2021-06-06 19:45 UTC (permalink / raw)
To: qemu-devel
Cc: Eduardo Habkost, Juan Quintela, Hyman, Dr. David Alan Gilbert,
Peter Xu, Paolo Bonzini
From: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
dirty_pages is used to calculate dirtyrate via dirty ring, when
enabled, kvm-reaper will increase the dirty pages after gfns
being dirtied.
kvm_dirty_ring_enabled shows if kvm-reaper is working. dirtyrate
thread could use it to check if measurement can base on dirty
ring feature.
Signed-off-by: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
---
accel/kvm/kvm-all.c | 7 +++++++
include/hw/core/cpu.h | 1 +
include/sysemu/kvm.h | 1 +
3 files changed, 9 insertions(+)
diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c
index c7ec538850..bc012f0bee 100644
--- a/accel/kvm/kvm-all.c
+++ b/accel/kvm/kvm-all.c
@@ -469,6 +469,7 @@ int kvm_init_vcpu(CPUState *cpu, Error **errp)
cpu->kvm_fd = ret;
cpu->kvm_state = s;
cpu->vcpu_dirty = true;
+ cpu->dirty_pages = 0;
mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
if (mmap_size < 0) {
@@ -743,6 +744,7 @@ static uint32_t kvm_dirty_ring_reap_one(KVMState *s, CPUState *cpu)
count++;
}
cpu->kvm_fetch_index = fetch;
+ cpu->dirty_pages += count;
return count;
}
@@ -2293,6 +2295,11 @@ bool kvm_vcpu_id_is_valid(int vcpu_id)
return vcpu_id >= 0 && vcpu_id < kvm_max_vcpu_id(s);
}
+bool kvm_dirty_ring_enabled(void)
+{
+ return kvm_state->kvm_dirty_ring_size ? true : false;
+}
+
static int kvm_init(MachineState *ms)
{
MachineClass *mc = MACHINE_GET_CLASS(ms);
diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h
index 4e0ea68efc..80fcb1d563 100644
--- a/include/hw/core/cpu.h
+++ b/include/hw/core/cpu.h
@@ -374,6 +374,7 @@ struct CPUState {
struct kvm_run *kvm_run;
struct kvm_dirty_gfn *kvm_dirty_gfns;
uint32_t kvm_fetch_index;
+ uint64_t dirty_pages;
/* Used for events with 'vcpu' and *without* the 'disabled' properties */
DECLARE_BITMAP(trace_dstate_delayed, CPU_TRACE_DSTATE_MAX_EVENTS);
diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
index a1ab1ee12d..7b22aeb6ae 100644
--- a/include/sysemu/kvm.h
+++ b/include/sysemu/kvm.h
@@ -547,4 +547,5 @@ bool kvm_cpu_check_are_resettable(void);
bool kvm_arch_cpu_check_are_resettable(void);
+bool kvm_dirty_ring_enabled(void);
#endif
--
2.18.2
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH v2 3/6] migration/dirtyrate: add per-vcpu option for calc-dirty-rate
2021-06-06 19:44 [PATCH v2 0/6] support dirtyrate at the granualrity of vcpu huangy81
2021-06-06 19:45 ` [PATCH v2 1/6] hmp: Add "calc_dirty_rate" and "info dirty_rate" cmds huangy81
2021-06-06 19:45 ` [PATCH v2 2/6] KVM: introduce dirty_pages and kvm_dirty_ring_enabled huangy81
@ 2021-06-06 19:47 ` huangy81
2021-06-06 19:47 ` [PATCH v2 4/6] migration/dirtyrate: adjust struct DirtyRateStat huangy81
` (2 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: huangy81 @ 2021-06-06 19:47 UTC (permalink / raw)
To: qemu-devel
Cc: Eduardo Habkost, Juan Quintela, Hyman, Dr. David Alan Gilbert,
Peter Xu, Paolo Bonzini
From: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
calculate dirtyrate for each vcpu if vcpu is true, add the
dirtyrate of each vcpu to the return value also.
Signed-off-by: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
---
migration/dirtyrate.c | 45 ++++++++++++++++++++++++++++++++++++++-----
migration/dirtyrate.h | 1 +
qapi/migration.json | 29 ++++++++++++++++++++++++++--
3 files changed, 68 insertions(+), 7 deletions(-)
diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c
index 320c56ba2c..5947c688f6 100644
--- a/migration/dirtyrate.c
+++ b/migration/dirtyrate.c
@@ -397,8 +397,11 @@ void *get_dirtyrate_thread(void *arg)
return NULL;
}
-void qmp_calc_dirty_rate(int64_t calc_time, bool has_sample_pages,
- int64_t sample_pages, Error **errp)
+void qmp_calc_dirty_rate(int64_t calc_time,
+ bool per_vcpu,
+ bool has_sample_pages,
+ int64_t sample_pages,
+ Error **errp)
{
static struct DirtyRateConfig config;
QemuThread thread;
@@ -419,6 +422,12 @@ void qmp_calc_dirty_rate(int64_t calc_time, bool has_sample_pages,
return;
}
+ if (has_sample_pages && per_vcpu) {
+ error_setg(errp, "per-vcpu and sample-pages are mutually exclusive, "
+ "only one of then can be specified!\n");
+ return;
+ }
+
if (has_sample_pages) {
if (!is_sample_pages_valid(sample_pages)) {
error_setg(errp, "sample-pages is out of range[%d, %d].",
@@ -430,6 +439,15 @@ void qmp_calc_dirty_rate(int64_t calc_time, bool has_sample_pages,
sample_pages = DIRTYRATE_DEFAULT_SAMPLE_PAGES;
}
+ /*
+ * Vcpu method only works when kvm dirty ring is enabled.
+ */
+ if (per_vcpu && !kvm_dirty_ring_enabled()) {
+ error_setg(errp, "dirty ring is disabled or conflict with migration"
+ "use sample method or remeasure later.");
+ return;
+ }
+
/*
* Init calculation state as unstarted.
*/
@@ -442,6 +460,7 @@ void qmp_calc_dirty_rate(int64_t calc_time, bool has_sample_pages,
config.sample_period_seconds = calc_time;
config.sample_pages_per_gigabytes = sample_pages;
+ config.per_vcpu = per_vcpu;
qemu_thread_create(&thread, "get_dirtyrate", get_dirtyrate_thread,
(void *)&config, QEMU_THREAD_DETACHED);
}
@@ -459,13 +478,22 @@ void hmp_info_dirty_rate(Monitor *mon, const QDict *qdict)
DirtyRateStatus_str(info->status));
monitor_printf(mon, "Start Time: %"PRIi64" (ms)\n",
info->start_time);
- monitor_printf(mon, "Sample Pages: %"PRIu64" (per GB)\n",
- info->sample_pages);
monitor_printf(mon, "Period: %"PRIi64" (sec)\n",
info->calc_time);
+ if (info->has_sample_pages) {
+ monitor_printf(mon, "Sample Pages: %"PRIu64" (per GB)\n",
+ info->sample_pages);
+ }
monitor_printf(mon, "Dirty rate: ");
if (info->has_dirty_rate) {
monitor_printf(mon, "%"PRIi64" (MB/s)\n", info->dirty_rate);
+ if (info->per_vcpu && info->has_vcpu_dirty_rate) {
+ DirtyRateVcpuList *rate, *head = info->vcpu_dirty_rate;
+ for (rate = head; rate != NULL; rate = rate->next) {
+ monitor_printf(mon, "vcpu[%"PRIi64"], Dirty rate: %"PRIi64"\n",
+ rate->value->id, rate->value->dirty_rate);
+ }
+ }
} else {
monitor_printf(mon, "(not ready)\n");
}
@@ -477,6 +505,7 @@ void hmp_calc_dirty_rate(Monitor *mon, const QDict *qdict)
int64_t sec = qdict_get_try_int(qdict, "second", 0);
int64_t sample_pages = qdict_get_try_int(qdict, "sample_pages_per_GB", -1);
bool has_sample_pages = (sample_pages != -1);
+ bool per_vcpu = qdict_get_try_bool(qdict, "per_vcpu", false);
Error *err = NULL;
if (!sec) {
@@ -484,7 +513,13 @@ void hmp_calc_dirty_rate(Monitor *mon, const QDict *qdict)
return;
}
- qmp_calc_dirty_rate(sec, has_sample_pages, sample_pages, &err);
+ if (has_sample_pages && per_vcpu) {
+ monitor_printf(mon, "per_vcpu and sample_pages are mutually exclusive, "
+ "only one of then can be specified!\n");
+ return;
+ }
+
+ qmp_calc_dirty_rate(sec, per_vcpu, has_sample_pages, sample_pages, &err);
if (err) {
hmp_handle_error(mon, err);
return;
diff --git a/migration/dirtyrate.h b/migration/dirtyrate.h
index e1fd29089e..ec82716b40 100644
--- a/migration/dirtyrate.h
+++ b/migration/dirtyrate.h
@@ -43,6 +43,7 @@
struct DirtyRateConfig {
uint64_t sample_pages_per_gigabytes; /* sample pages per GB */
int64_t sample_period_seconds; /* time duration between two sampling */
+ bool per_vcpu; /* calculate dirtyrate for each vcpu using dirty ring */
};
/*
diff --git a/qapi/migration.json b/qapi/migration.json
index 770ae54c17..7eef988182 100644
--- a/qapi/migration.json
+++ b/qapi/migration.json
@@ -1708,6 +1708,21 @@
{ 'event': 'UNPLUG_PRIMARY',
'data': { 'device-id': 'str' } }
+##
+# @DirtyRateVcpu:
+#
+# Dirty rate of vcpu.
+#
+# @id: vcpu index.
+#
+# @dirty-rate: dirty rate.
+#
+# Since: 6.1
+#
+##
+{ 'struct': 'DirtyRateVcpu',
+ 'data': { 'id': 'int', 'dirty-rate': 'int64' } }
+
##
# @DirtyRateStatus:
#
@@ -1743,6 +1758,10 @@
# @sample-pages: page count per GB for sample dirty pages
# the default value is 512 (since 6.1)
#
+# @per-vcpu: calculate dirtyrate for each vcpu (Since 6.1)
+#
+# @vcpu-dirty-rate: dirtyrate for each vcpu (Since 6.1)
+#
# Since: 5.2
#
##
@@ -1751,7 +1770,9 @@
'status': 'DirtyRateStatus',
'start-time': 'int64',
'calc-time': 'int64',
- 'sample-pages': 'uint64'} }
+ '*sample-pages': 'uint64',
+ 'per-vcpu': 'bool',
+ '*vcpu-dirty-rate': [ 'DirtyRateVcpu' ] } }
##
# @calc-dirty-rate:
@@ -1760,6 +1781,10 @@
#
# @calc-time: time in units of second for sample dirty pages
#
+# @per-vcpu: calculate vcpu dirty rate if true, the default value is
+# false, note that the per-vcpu and sample-pages are mutually
+# exclusive (since 6.1)
+#
# @sample-pages: page count per GB for sample dirty pages
# the default value is 512 (since 6.1)
#
@@ -1769,7 +1794,7 @@
# {"command": "calc-dirty-rate", "data": {"calc-time": 1, 'sample-pages': 512} }
#
##
-{ 'command': 'calc-dirty-rate', 'data': {'calc-time': 'int64', '*sample-pages': 'int'} }
+{ 'command': 'calc-dirty-rate', 'data': {'calc-time': 'int64', 'per-vcpu': 'bool', '*sample-pages': 'int'} }
##
# @query-dirty-rate:
--
2.18.2
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH v2 4/6] migration/dirtyrate: adjust struct DirtyRateStat
2021-06-06 19:44 [PATCH v2 0/6] support dirtyrate at the granualrity of vcpu huangy81
` (2 preceding siblings ...)
2021-06-06 19:47 ` [PATCH v2 3/6] migration/dirtyrate: add per-vcpu option for calc-dirty-rate huangy81
@ 2021-06-06 19:47 ` huangy81
2021-06-06 19:48 ` [PATCH v2 5/6] memory: make global_dirty_log a bitmask huangy81
2021-06-06 19:48 ` [PATCH v2 6/6] migration/dirtyrate: implement dirty-ring dirtyrate calculation huangy81
5 siblings, 0 replies; 7+ messages in thread
From: huangy81 @ 2021-06-06 19:47 UTC (permalink / raw)
To: qemu-devel
Cc: Eduardo Habkost, Juan Quintela, Hyman, Dr. David Alan Gilbert,
Peter Xu, Paolo Bonzini
From: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
use union to store stat data of two mutual exclusive methods.
Signed-off-by: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
---
migration/dirtyrate.c | 40 +++++++++++++++++++++-------------------
migration/dirtyrate.h | 18 +++++++++++++++---
2 files changed, 36 insertions(+), 22 deletions(-)
diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c
index 5947c688f6..055145c24c 100644
--- a/migration/dirtyrate.c
+++ b/migration/dirtyrate.c
@@ -88,33 +88,39 @@ static struct DirtyRateInfo *query_dirty_rate_info(void)
return info;
}
-static void init_dirtyrate_stat(int64_t start_time, int64_t calc_time,
- uint64_t sample_pages)
+static void init_dirtyrate_stat(int64_t start_time,
+ struct DirtyRateConfig config)
{
- DirtyStat.total_dirty_samples = 0;
- DirtyStat.total_sample_count = 0;
- DirtyStat.total_block_mem_MB = 0;
DirtyStat.dirty_rate = -1;
DirtyStat.start_time = start_time;
- DirtyStat.calc_time = calc_time;
- DirtyStat.sample_pages = sample_pages;
+ DirtyStat.calc_time = config.sample_period_seconds;
+ DirtyStat.sample_pages = config.sample_pages_per_gigabytes;
+
+ if (config.per_vcpu) {
+ DirtyStat.method.vcpu.nvcpu = -1;
+ DirtyStat.method.vcpu.rates = NULL;
+ } else {
+ DirtyStat.method.vm.total_dirty_samples = 0;
+ DirtyStat.method.vm.total_sample_count = 0;
+ DirtyStat.method.vm.total_block_mem_MB = 0;
+ }
}
static void update_dirtyrate_stat(struct RamblockDirtyInfo *info)
{
- DirtyStat.total_dirty_samples += info->sample_dirty_count;
- DirtyStat.total_sample_count += info->sample_pages_count;
+ DirtyStat.method.vm.total_dirty_samples += info->sample_dirty_count;
+ DirtyStat.method.vm.total_sample_count += info->sample_pages_count;
/* size of total pages in MB */
- DirtyStat.total_block_mem_MB += (info->ramblock_pages *
+ DirtyStat.method.vm.total_block_mem_MB += (info->ramblock_pages *
TARGET_PAGE_SIZE) >> 20;
}
static void update_dirtyrate(uint64_t msec)
{
uint64_t dirtyrate;
- uint64_t total_dirty_samples = DirtyStat.total_dirty_samples;
- uint64_t total_sample_count = DirtyStat.total_sample_count;
- uint64_t total_block_mem_MB = DirtyStat.total_block_mem_MB;
+ uint64_t total_dirty_samples = DirtyStat.method.vm.total_dirty_samples;
+ uint64_t total_sample_count = DirtyStat.method.vm.total_sample_count;
+ uint64_t total_block_mem_MB = DirtyStat.method.vm.total_block_mem_MB;
dirtyrate = total_dirty_samples * total_block_mem_MB *
1000 / (total_sample_count * msec);
@@ -327,7 +333,7 @@ static bool compare_page_hash_info(struct RamblockDirtyInfo *info,
update_dirtyrate_stat(block_dinfo);
}
- if (DirtyStat.total_sample_count == 0) {
+ if (DirtyStat.method.vm.total_sample_count == 0) {
return false;
}
@@ -372,8 +378,6 @@ void *get_dirtyrate_thread(void *arg)
struct DirtyRateConfig config = *(struct DirtyRateConfig *)arg;
int ret;
int64_t start_time;
- int64_t calc_time;
- uint64_t sample_pages;
ret = dirtyrate_set_state(&CalculatingState, DIRTY_RATE_STATUS_UNSTARTED,
DIRTY_RATE_STATUS_MEASURING);
@@ -383,9 +387,7 @@ void *get_dirtyrate_thread(void *arg)
}
start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) / 1000;
- calc_time = config.sample_period_seconds;
- sample_pages = config.sample_pages_per_gigabytes;
- init_dirtyrate_stat(start_time, calc_time, sample_pages);
+ init_dirtyrate_stat(start_time, config);
calculate_dirtyrate(config);
diff --git a/migration/dirtyrate.h b/migration/dirtyrate.h
index ec82716b40..af32e33d5d 100644
--- a/migration/dirtyrate.h
+++ b/migration/dirtyrate.h
@@ -59,17 +59,29 @@ struct RamblockDirtyInfo {
uint32_t *hash_result; /* array of hash result for sampled pages */
};
+typedef struct SampleVMStat {
+ uint64_t total_dirty_samples; /* total dirty sampled page */
+ uint64_t total_sample_count; /* total sampled pages */
+ uint64_t total_block_mem_MB; /* size of total sampled pages in MB */
+} SampleVMStat;
+
+typedef struct VcpuStat {
+ int nvcpu; /* number of vcpu */
+ DirtyRateVcpu *rates; /* array of dirty rate for each vcpu */
+} VcpuStat;
+
/*
* Store calculation statistics for each measure.
*/
struct DirtyRateStat {
- uint64_t total_dirty_samples; /* total dirty sampled page */
- uint64_t total_sample_count; /* total sampled pages */
- uint64_t total_block_mem_MB; /* size of total sampled pages in MB */
int64_t dirty_rate; /* dirty rate in MB/s */
int64_t start_time; /* calculation start time in units of second */
int64_t calc_time; /* time duration of two sampling in units of second */
uint64_t sample_pages; /* sample pages per GB */
+ union {
+ SampleVMStat vm;
+ VcpuStat vcpu;
+ } method;
};
void *get_dirtyrate_thread(void *arg);
--
2.18.2
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH v2 5/6] memory: make global_dirty_log a bitmask
2021-06-06 19:44 [PATCH v2 0/6] support dirtyrate at the granualrity of vcpu huangy81
` (3 preceding siblings ...)
2021-06-06 19:47 ` [PATCH v2 4/6] migration/dirtyrate: adjust struct DirtyRateStat huangy81
@ 2021-06-06 19:48 ` huangy81
2021-06-06 19:48 ` [PATCH v2 6/6] migration/dirtyrate: implement dirty-ring dirtyrate calculation huangy81
5 siblings, 0 replies; 7+ messages in thread
From: huangy81 @ 2021-06-06 19:48 UTC (permalink / raw)
To: qemu-devel
Cc: Eduardo Habkost, Juan Quintela, Hyman, Dr. David Alan Gilbert,
Peter Xu, Paolo Bonzini
From: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
dirty rate measurement may start or stop dirty logging during
calculation. this conflict with migration because stop dirty
log make migration leave dirty pages out then that'll be a problem.
make global_dirty_log a bitmask can let both migration and dirty
rate measurement work fine. introduce GLOBAL_DIRTY_MIGRATION and
GLOBAL_DIRTY_DIRTY_RATE to distinguish what current dirty log aims
for, migration or dirty rate.
all references to global_dirty_log should be untouched because any bit
set there should justify that global dirty logging is enabled.
Signed-off-by: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
---
include/exec/memory.h | 13 ++++++++++---
migration/ram.c | 8 ++++----
softmmu/memory.c | 36 +++++++++++++++++++++++++++---------
3 files changed, 41 insertions(+), 16 deletions(-)
diff --git a/include/exec/memory.h b/include/exec/memory.h
index c158fd7084..94c7088299 100644
--- a/include/exec/memory.h
+++ b/include/exec/memory.h
@@ -55,7 +55,10 @@ static inline void fuzz_dma_read_cb(size_t addr,
}
#endif
-extern bool global_dirty_log;
+#define GLOBAL_DIRTY_MIGRATION (1U<<0)
+#define GLOBAL_DIRTY_DIRTY_RATE (1U<<1)
+
+extern int global_dirty_log;
typedef struct MemoryRegionOps MemoryRegionOps;
@@ -2099,13 +2102,17 @@ void memory_listener_unregister(MemoryListener *listener);
/**
* memory_global_dirty_log_start: begin dirty logging for all regions
+ *
+ * @flags: purpose of start dirty log, migration or dirty rate
*/
-void memory_global_dirty_log_start(void);
+void memory_global_dirty_log_start(int flags);
/**
* memory_global_dirty_log_stop: end dirty logging for all regions
+ *
+ * @flags: purpose of stop dirty log, migration or dirty rate
*/
-void memory_global_dirty_log_stop(void);
+void memory_global_dirty_log_stop(int flags);
void mtree_info(bool flatview, bool dispatch_tree, bool owner, bool disabled);
diff --git a/migration/ram.c b/migration/ram.c
index 60ea913c54..9ce31af9d1 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -2190,7 +2190,7 @@ static void ram_save_cleanup(void *opaque)
/* caller have hold iothread lock or is in a bh, so there is
* no writing race against the migration bitmap
*/
- memory_global_dirty_log_stop();
+ memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
}
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
@@ -2652,7 +2652,7 @@ static void ram_init_bitmaps(RAMState *rs)
ram_list_init_bitmaps();
/* We don't use dirty log with background snapshots */
if (!migrate_background_snapshot()) {
- memory_global_dirty_log_start();
+ memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION);
migration_bitmap_sync_precopy(rs);
}
}
@@ -3393,7 +3393,7 @@ void colo_incoming_start_dirty_log(void)
/* Discard this dirty bitmap record */
bitmap_zero(block->bmap, block->max_length >> TARGET_PAGE_BITS);
}
- memory_global_dirty_log_start();
+ memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION);
}
ram_state->migration_dirty_pages = 0;
qemu_mutex_unlock_ramlist();
@@ -3405,7 +3405,7 @@ void colo_release_ram_cache(void)
{
RAMBlock *block;
- memory_global_dirty_log_stop();
+ memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
g_free(block->bmap);
block->bmap = NULL;
diff --git a/softmmu/memory.c b/softmmu/memory.c
index c19b0be6b1..b93baba82d 100644
--- a/softmmu/memory.c
+++ b/softmmu/memory.c
@@ -39,7 +39,7 @@
static unsigned memory_region_transaction_depth;
static bool memory_region_update_pending;
static bool ioeventfd_update_pending;
-bool global_dirty_log;
+int global_dirty_log;
static QTAILQ_HEAD(, MemoryListener) memory_listeners
= QTAILQ_HEAD_INITIALIZER(memory_listeners);
@@ -2659,14 +2659,20 @@ void memory_global_after_dirty_log_sync(void)
static VMChangeStateEntry *vmstate_change;
-void memory_global_dirty_log_start(void)
+void memory_global_dirty_log_start(int flags)
{
if (vmstate_change) {
qemu_del_vm_change_state_handler(vmstate_change);
vmstate_change = NULL;
}
- global_dirty_log = true;
+ if (flags & GLOBAL_DIRTY_MIGRATION) {
+ global_dirty_log |= GLOBAL_DIRTY_MIGRATION;
+ }
+
+ if (flags & GLOBAL_DIRTY_DIRTY_RATE) {
+ global_dirty_log |= GLOBAL_DIRTY_DIRTY_RATE;
+ }
MEMORY_LISTENER_CALL_GLOBAL(log_global_start, Forward);
@@ -2676,9 +2682,15 @@ void memory_global_dirty_log_start(void)
memory_region_transaction_commit();
}
-static void memory_global_dirty_log_do_stop(void)
+static void memory_global_dirty_log_do_stop(int flags)
{
- global_dirty_log = false;
+ if (flags & GLOBAL_DIRTY_MIGRATION) {
+ global_dirty_log &= ~GLOBAL_DIRTY_MIGRATION;
+ }
+
+ if (flags & GLOBAL_DIRTY_DIRTY_RATE) {
+ global_dirty_log &= ~GLOBAL_DIRTY_DIRTY_RATE;
+ }
/* Refresh DIRTY_MEMORY_MIGRATION bit. */
memory_region_transaction_begin();
@@ -2691,8 +2703,10 @@ static void memory_global_dirty_log_do_stop(void)
static void memory_vm_change_state_handler(void *opaque, bool running,
RunState state)
{
+ int *flags = opaque;
if (running) {
- memory_global_dirty_log_do_stop();
+ memory_global_dirty_log_do_stop(*flags);
+ g_free(opaque);
if (vmstate_change) {
qemu_del_vm_change_state_handler(vmstate_change);
@@ -2701,18 +2715,22 @@ static void memory_vm_change_state_handler(void *opaque, bool running,
}
}
-void memory_global_dirty_log_stop(void)
+void memory_global_dirty_log_stop(int flags)
{
+ int *opaque = NULL;
if (!runstate_is_running()) {
if (vmstate_change) {
return;
}
+
+ opaque = g_malloc0(sizeof(opaque));
+ *opaque = flags;
vmstate_change = qemu_add_vm_change_state_handler(
- memory_vm_change_state_handler, NULL);
+ memory_vm_change_state_handler, opaque);
return;
}
- memory_global_dirty_log_do_stop();
+ memory_global_dirty_log_do_stop(flags);
}
static void listener_add_address_space(MemoryListener *listener,
--
2.18.2
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH v2 6/6] migration/dirtyrate: implement dirty-ring dirtyrate calculation
2021-06-06 19:44 [PATCH v2 0/6] support dirtyrate at the granualrity of vcpu huangy81
` (4 preceding siblings ...)
2021-06-06 19:48 ` [PATCH v2 5/6] memory: make global_dirty_log a bitmask huangy81
@ 2021-06-06 19:48 ` huangy81
5 siblings, 0 replies; 7+ messages in thread
From: huangy81 @ 2021-06-06 19:48 UTC (permalink / raw)
To: qemu-devel
Cc: Eduardo Habkost, Juan Quintela, Hyman, Dr. David Alan Gilbert,
Peter Xu, Paolo Bonzini
From: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
use dirty ring feature to implement dirtyrate calculation.
to enable it, set vcpu option as true in calc-dirty-rate.
add per_vcpu as mandatory option in calc_dirty_rate, to calculate
dirty rate for vcpu, and use hmp cmd:
(qemu) calc_dirty_rate 1 on
Signed-off-by: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
---
hmp-commands.hx | 7 +-
migration/dirtyrate.c | 226 ++++++++++++++++++++++++++++++++++++++---
migration/trace-events | 5 +
3 files changed, 220 insertions(+), 18 deletions(-)
diff --git a/hmp-commands.hx b/hmp-commands.hx
index 84dcc3aae6..cc24ab2ab1 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -1736,8 +1736,9 @@ ERST
{
.name = "calc_dirty_rate",
- .args_type = "second:l,sample_pages_per_GB:l?",
- .params = "second [sample_pages_per_GB]",
- .help = "start a round of guest dirty rate measurement",
+ .args_type = "second:l,per_vcpu:b,sample_pages_per_GB:l?",
+ .params = "second on|off [sample_pages_per_GB]",
+ .help = "start a round of guest dirty rate measurement, "
+ "calculate for vcpu use on|off",
.cmd = hmp_calc_dirty_rate,
},
diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c
index 055145c24c..e432118f49 100644
--- a/migration/dirtyrate.c
+++ b/migration/dirtyrate.c
@@ -16,6 +16,9 @@
#include "cpu.h"
#include "exec/ramblock.h"
#include "qemu/rcu_queue.h"
+#include "qemu/main-loop.h"
+#include "sysemu/kvm.h"
+#include "sysemu/runstate.h"
#include "qapi/qapi-commands-migration.h"
#include "ram.h"
#include "trace.h"
@@ -23,9 +26,38 @@
#include "monitor/hmp.h"
#include "monitor/monitor.h"
#include "qapi/qmp/qdict.h"
+#include "exec/memory.h"
+
+typedef enum {
+ CALC_NONE = 0,
+ CALC_DIRTY_RING,
+ CALC_SAMPLE_PAGES,
+} CalcMethod;
+
+typedef struct DirtyPageRecord {
+ int64_t start_pages;
+ int64_t end_pages;
+} DirtyPageRecord;
+
+static DirtyPageRecord *dirty_pages;
static int CalculatingState = DIRTY_RATE_STATUS_UNSTARTED;
static struct DirtyRateStat DirtyStat;
+static CalcMethod last_method = CALC_NONE;
+bool register_powerdown_callback = false;
+
+static void dirtyrate_powerdown_req(Notifier *n, void *opaque)
+{
+ if (last_method == CALC_DIRTY_RING) {
+ g_free(DirtyStat.method.vcpu.rates);
+ DirtyStat.method.vcpu.rates = NULL;
+ }
+ trace_dirtyrate_powerdown_callback();
+}
+
+static Notifier dirtyrate_powerdown_notifier = {
+ .notify = dirtyrate_powerdown_req
+};
static int64_t set_sample_page_period(int64_t msec, int64_t initial_time)
{
@@ -72,6 +104,7 @@ static struct DirtyRateInfo *query_dirty_rate_info(void)
{
int64_t dirty_rate = DirtyStat.dirty_rate;
struct DirtyRateInfo *info = g_malloc0(sizeof(DirtyRateInfo));
+ DirtyRateVcpuList *head = NULL, **tail = &head;
if (qatomic_read(&CalculatingState) == DIRTY_RATE_STATUS_MEASURED) {
info->has_dirty_rate = true;
@@ -81,7 +114,22 @@ static struct DirtyRateInfo *query_dirty_rate_info(void)
info->status = CalculatingState;
info->start_time = DirtyStat.start_time;
info->calc_time = DirtyStat.calc_time;
- info->sample_pages = DirtyStat.sample_pages;
+
+ if (last_method == CALC_DIRTY_RING) {
+ int i = 0;
+ info->per_vcpu = true;
+ info->has_vcpu_dirty_rate = true;
+ for (i = 0; i < DirtyStat.method.vcpu.nvcpu; i++) {
+ DirtyRateVcpu *rate = g_malloc0(sizeof(DirtyRateVcpu));
+ rate->id = DirtyStat.method.vcpu.rates[i].id;
+ rate->dirty_rate = DirtyStat.method.vcpu.rates[i].dirty_rate;
+ QAPI_LIST_APPEND(tail, rate);
+ }
+ info->vcpu_dirty_rate = head;
+ } else {
+ info->has_sample_pages = true;
+ info->sample_pages = DirtyStat.sample_pages;
+ }
trace_query_dirty_rate_info(DirtyRateStatus_str(CalculatingState));
@@ -94,15 +142,37 @@ static void init_dirtyrate_stat(int64_t start_time,
DirtyStat.dirty_rate = -1;
DirtyStat.start_time = start_time;
DirtyStat.calc_time = config.sample_period_seconds;
- DirtyStat.sample_pages = config.sample_pages_per_gigabytes;
-
- if (config.per_vcpu) {
- DirtyStat.method.vcpu.nvcpu = -1;
- DirtyStat.method.vcpu.rates = NULL;
- } else {
- DirtyStat.method.vm.total_dirty_samples = 0;
- DirtyStat.method.vm.total_sample_count = 0;
- DirtyStat.method.vm.total_block_mem_MB = 0;
+ DirtyStat.sample_pages =
+ config.per_vcpu ? -1 : config.sample_pages_per_gigabytes;
+
+ if (unlikely(!register_powerdown_callback)) {
+ qemu_register_powerdown_notifier(&dirtyrate_powerdown_notifier);
+ register_powerdown_callback = true;
+ }
+
+ switch (last_method) {
+ case CALC_NONE:
+ case CALC_SAMPLE_PAGES:
+ if (config.per_vcpu) {
+ DirtyStat.method.vcpu.nvcpu = -1;
+ DirtyStat.method.vcpu.rates = NULL;
+ } else {
+ DirtyStat.method.vm.total_dirty_samples = 0;
+ DirtyStat.method.vm.total_sample_count = 0;
+ DirtyStat.method.vm.total_block_mem_MB = 0;
+ }
+ break;
+ case CALC_DIRTY_RING:
+ if (!config.per_vcpu) {
+ g_free(DirtyStat.method.vcpu.rates);
+ DirtyStat.method.vcpu.rates = NULL;
+ DirtyStat.method.vm.total_dirty_samples = 0;
+ DirtyStat.method.vm.total_sample_count = 0;
+ DirtyStat.method.vm.total_block_mem_MB = 0;
+ }
+ break;
+ default:
+ break;
}
}
@@ -316,7 +386,7 @@ find_block_matched(RAMBlock *block, int count,
}
static bool compare_page_hash_info(struct RamblockDirtyInfo *info,
- int block_count)
+ int block_count)
{
struct RamblockDirtyInfo *block_dinfo = NULL;
RAMBlock *block = NULL;
@@ -340,14 +410,125 @@ static bool compare_page_hash_info(struct RamblockDirtyInfo *info,
return true;
}
-static void calculate_dirtyrate(struct DirtyRateConfig config)
+static void record_dirtypages(CPUState *cpu, bool start)
+{
+ if (start) {
+ dirty_pages[cpu->cpu_index].start_pages = cpu->dirty_pages;
+ } else {
+ dirty_pages[cpu->cpu_index].end_pages = cpu->dirty_pages;
+ }
+}
+
+static void dirtyrate_global_dirty_log_start(void)
+{
+ /* dirty logging is enabled already */
+ if (global_dirty_log) {
+ return;
+ }
+
+ qemu_mutex_lock_iothread();
+ memory_global_dirty_log_start(GLOBAL_DIRTY_DIRTY_RATE);
+ qemu_mutex_unlock_iothread();
+ trace_dirtyrate_dirty_log_start();
+}
+
+static void dirtyrate_global_dirty_log_stop(void)
+{
+ /* migration is in process, do not stop dirty logging,
+ * just clear the GLOBAL_DIRTY_DIRTY_RATE bit */
+ if (global_dirty_log & GLOBAL_DIRTY_MIGRATION) {
+ global_dirty_log &= ~(GLOBAL_DIRTY_DIRTY_RATE);
+ return;
+ }
+
+ qemu_mutex_lock_iothread();
+ memory_global_dirty_log_stop(GLOBAL_DIRTY_DIRTY_RATE);
+ qemu_mutex_unlock_iothread();
+ trace_dirtyrate_dirty_log_stop();
+}
+
+static int64_t do_calculate_dirtyrate_vcpu(int idx)
+{
+ uint64_t memory_size_MB;
+ int64_t time_s;
+ uint64_t start_pages = dirty_pages[idx].start_pages;
+ uint64_t end_pages = dirty_pages[idx].end_pages;
+ uint64_t dirty_pages = 0;
+
+ /* uint64_t over the INT64_MAX */
+ if (unlikely(end_pages < start_pages)) {
+ dirty_pages = INT64_MAX - start_pages + end_pages + 1;
+ } else {
+ dirty_pages = end_pages - start_pages;
+ }
+
+ memory_size_MB = (dirty_pages * TARGET_PAGE_SIZE) >> 20;
+ time_s = DirtyStat.calc_time;
+
+ trace_dirtyrate_do_calculate_vcpu(idx, dirty_pages, time_s);
+
+ return memory_size_MB / time_s;
+}
+
+static void calculate_dirtyrate_vcpu(struct DirtyRateConfig config)
+{
+ CPUState *cpu;
+ int64_t msec = 0;
+ int64_t start_time;
+ uint64_t dirtyrate = 0;
+ uint64_t dirtyrate_sum = 0;
+ int nvcpu = 0;
+ int i = 0;
+
+ CPU_FOREACH(cpu) {
+ nvcpu++;
+ }
+
+ dirty_pages = g_malloc0(sizeof(*dirty_pages) * nvcpu);
+
+ dirtyrate_global_dirty_log_start();
+
+ CPU_FOREACH(cpu) {
+ record_dirtypages(cpu, true);
+ }
+
+ DirtyStat.method.vcpu.nvcpu = nvcpu;
+ if (last_method != CALC_DIRTY_RING) {
+ DirtyStat.method.vcpu.rates =
+ g_malloc0(sizeof(DirtyRateVcpu) * nvcpu);
+ }
+
+ start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+ DirtyStat.start_time = start_time / 1000;
+
+ msec = config.sample_period_seconds * 1000;
+ msec = set_sample_page_period(msec, start_time);
+ DirtyStat.calc_time = msec / 1000;
+
+ CPU_FOREACH(cpu) {
+ record_dirtypages(cpu, false);
+ }
+
+ dirtyrate_global_dirty_log_stop();
+
+ for (i = 0; i < DirtyStat.method.vcpu.nvcpu; i++) {
+ dirtyrate = do_calculate_dirtyrate_vcpu(i);
+ DirtyStat.method.vcpu.rates[i].id = i;
+ DirtyStat.method.vcpu.rates[i].dirty_rate = dirtyrate;
+ dirtyrate_sum += dirtyrate;
+ }
+
+ DirtyStat.dirty_rate = dirtyrate_sum / DirtyStat.method.vcpu.nvcpu;
+ g_free(dirty_pages);
+}
+
+static void calculate_dirtyrate_sample_vm(struct DirtyRateConfig config)
{
struct RamblockDirtyInfo *block_dinfo = NULL;
int block_count = 0;
int64_t msec = 0;
int64_t initial_time;
- rcu_register_thread();
rcu_read_lock();
initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
if (!record_ramblock_hash_info(&block_dinfo, config, &block_count)) {
@@ -364,13 +545,24 @@ static void calculate_dirtyrate(struct DirtyRateConfig config)
if (!compare_page_hash_info(block_dinfo, block_count)) {
goto out;
}
-
update_dirtyrate(msec);
out:
rcu_read_unlock();
free_ramblock_dirty_info(block_dinfo, block_count);
- rcu_unregister_thread();
+}
+
+static void calculate_dirtyrate(struct DirtyRateConfig config)
+{
+ if (config.per_vcpu) {
+ calculate_dirtyrate_vcpu(config);
+ last_method = CALC_DIRTY_RING;
+ } else {
+ calculate_dirtyrate_sample_vm(config);
+ last_method = CALC_SAMPLE_PAGES;
+ }
+
+ trace_dirtyrate_calculate(DirtyStat.dirty_rate);
}
void *get_dirtyrate_thread(void *arg)
@@ -379,6 +571,8 @@ void *get_dirtyrate_thread(void *arg)
int ret;
int64_t start_time;
+ rcu_register_thread();
+
ret = dirtyrate_set_state(&CalculatingState, DIRTY_RATE_STATUS_UNSTARTED,
DIRTY_RATE_STATUS_MEASURING);
if (ret == -1) {
@@ -396,6 +590,8 @@ void *get_dirtyrate_thread(void *arg)
if (ret == -1) {
error_report("change dirtyrate state failed.");
}
+
+ rcu_unregister_thread();
return NULL;
}
diff --git a/migration/trace-events b/migration/trace-events
index 860c4f4025..4c5a658665 100644
--- a/migration/trace-events
+++ b/migration/trace-events
@@ -330,6 +330,11 @@ get_ramblock_vfn_hash(const char *idstr, uint64_t vfn, uint32_t crc) "ramblock n
calc_page_dirty_rate(const char *idstr, uint32_t new_crc, uint32_t old_crc) "ramblock name: %s, new crc: %" PRIu32 ", old crc: %" PRIu32
skip_sample_ramblock(const char *idstr, uint64_t ramblock_size) "ramblock name: %s, ramblock size: %" PRIu64
find_page_matched(const char *idstr) "ramblock %s addr or size changed"
+dirtyrate_calculate(int64_t dirtyrate) "dirty rate: %" PRIi64
+dirtyrate_do_calculate_vcpu(int idx, uint64_t pages, int64_t seconds) "vcpu[%d]: dirty %"PRIu64 " pages in %"PRIi64 " seconds"
+dirtyrate_powerdown_callback(void) ""
+dirtyrate_dirty_log_start(void) ""
+dirtyrate_dirty_log_stop(void) ""
# block.c
migration_block_init_shared(const char *blk_device_name) "Start migration for %s with shared base image"
--
2.18.2
^ permalink raw reply related [flat|nested] 7+ messages in thread