Re: [PATCH v5 3/3] cpus-common: implement dirty limit on vCPU

[Markus Armbruster <armbru@redhat.com>]

huangy81@chinatelecom.cn writes:

> From: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
>
> Implement dirtyrate calculation periodically basing on
> dirty-ring and throttle vCPU until it reachs the quota
> dirtyrate given by user.
>
> Introduce qmp commands set-dirty-limit/cancel-dirty-limit to
> set/cancel dirty limit on vCPU.
>
> Signed-off-by: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
> ---
>  cpus-common.c         | 41 +++++++++++++++++++++++++++++++++++++++++
>  include/hw/core/cpu.h |  9 +++++++++
>  qapi/migration.json   | 43 +++++++++++++++++++++++++++++++++++++++++++
>  softmmu/vl.c          |  1 +
>  4 files changed, 94 insertions(+)
>
> diff --git a/cpus-common.c b/cpus-common.c
> index 6e73d3e..43b0078 100644
> --- a/cpus-common.c
> +++ b/cpus-common.c
> @@ -23,6 +23,11 @@
>  #include "hw/core/cpu.h"
>  #include "sysemu/cpus.h"
>  #include "qemu/lockable.h"
> +#include "sysemu/dirtylimit.h"
> +#include "sysemu/cpu-throttle.h"
> +#include "sysemu/kvm.h"
> +#include "qapi/error.h"
> +#include "qapi/qapi-commands-migration.h"
>  
>  static QemuMutex qemu_cpu_list_lock;
>  static QemuCond exclusive_cond;
> @@ -352,3 +357,39 @@ void process_queued_cpu_work(CPUState *cpu)
>      qemu_mutex_unlock(&cpu->work_mutex);
>      qemu_cond_broadcast(&qemu_work_cond);
>  }
> +
> +void qmp_set_dirty_limit(int64_t idx,
> +                         uint64_t dirtyrate,
> +                         Error **errp)
> +{
> +    if (!kvm_dirty_ring_enabled()) {
> +        error_setg(errp, "dirty ring not enable, needed by dirty restraint!");

"not enabled"

What is a "dirty restraint"?

Drop the exclamation point, please.  See error.h:

 * The resulting message should be a single phrase, with no newline or
 * trailing punctuation.

What about "setting a dirty page limit requires ...".

> +        return;
> +    }
> +
> +    dirtylimit_calc();
> +    dirtylimit_vcpu(idx, dirtyrate);
> +}
> +
> +void qmp_cancel_dirty_limit(int64_t idx,
> +                            Error **errp)
> +{
> +    if (!kvm_dirty_ring_enabled()) {
> +        error_setg(errp, "dirty ring not enable, needed by dirty restraint!");
> +        return;
> +    }
> +
> +    if (unlikely(!dirtylimit_cancel_vcpu(idx))) {
> +        dirtylimit_calc_quit();
> +    }
> +}
> +
> +void dirtylimit_setup(int max_cpus)
> +{
> +    if (!kvm_dirty_ring_enabled()) {

This crashes unless the accelerator is kvm.  Reproducer:

    $ qemu-system-x86_64 -display none -accel tcg
    Segmentation fault (core dumped)

> +        return;
> +    }
> +
> +    dirtylimit_calc_state_init(max_cpus);
> +    dirtylimit_state_init(max_cpus);
> +}
> diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h
> index e948e81..11df012 100644
> --- a/include/hw/core/cpu.h
> +++ b/include/hw/core/cpu.h
> @@ -881,6 +881,15 @@ void end_exclusive(void);
>   */
>  void qemu_init_vcpu(CPUState *cpu);
>  
> +/**
> + * dirtylimit_setup:
> + *
> + * Initializes the global state of dirtylimit calculation and
> + * dirtylimit itself. This is prepared for vCPU dirtylimit which
> + * could be triggered during vm lifecycle.
> + */
> +void dirtylimit_setup(int max_cpus);
> +
>  #define SSTEP_ENABLE  0x1  /* Enable simulated HW single stepping */
>  #define SSTEP_NOIRQ   0x2  /* Do not use IRQ while single stepping */
>  #define SSTEP_NOTIMER 0x4  /* Do not Timers while single stepping */
> diff --git a/qapi/migration.json b/qapi/migration.json
> index bbfd48c..42b260e 100644
> --- a/qapi/migration.json
> +++ b/qapi/migration.json
> @@ -1850,6 +1850,49 @@
>  { 'command': 'query-dirty-rate', 'returns': 'DirtyRateInfo' }
>  
>  ##
> +# @set-dirty-limit:
> +#
> +# Set the upper limit of dirty page rate for the interested vCPU.

"for a vCPU"

> +#
> +# This command could be used to cap the vCPU memory load, which is also
> +# refered as "dirty page rate". Users can use set-dirty-limit unconditionally,
> +# but if one want to know which vCPU is in high memory load and which vCPU
> +# should be limited, "calc-dirty-rate" with "dirty-ring" mode maybe an
> +# availiable method.

I think you should mention that the command fails unless dirty ring is
enabled, and a pointer to its documentation.

> +#
> +# @idx: vCPU index to set dirtylimit.

Please rename to @cpu-index for consistency with query-cpus-fast.  Same
for cancel-dirty-limit below.

> +#
> +# @dirtyrate: upper limit for the specified vCPU's dirty page rate (MB/s)

In QMP, we separate words with hyphens, like @dirty-rate.  Please
rename.

> +#
> +# Since: 6.3

7.0

> +#
> +# Example:
> +#   {"execute": "set-dirty-limit"}
> +#    "arguments": { "idx": 0,
> +#                   "dirtyrate": 200 } }
> +#
> +##
> +{ 'command': 'set-dirty-limit',
> +  'data': { 'idx': 'int', 'dirtyrate': 'uint64' } }
> +
> +##
> +# @cancel-dirty-limit:
> +#
> +# Cancel the dirtylimit for the vCPU which has been set with set-dirty-limit.

"the dirty page limit"

> +#
> +# @idx: vCPU index to canceled the dirtylimit
> +#
> +# Since: 6.3

7.0

> +#
> +# Example:
> +#   {"execute": "cancel-dirty-limit"}
> +#    "arguments": { "idx": 0 } }
> +#
> +##
> +{ 'command': 'cancel-dirty-limit',
> +  'data': { 'idx': 'int' } }
> +
> +##
>  # @snapshot-save:
>  #
>  # Save a VM snapshot
> diff --git a/softmmu/vl.c b/softmmu/vl.c
> index 620a1f1..0f83ce3 100644
> --- a/softmmu/vl.c
> +++ b/softmmu/vl.c
> @@ -3777,5 +3777,6 @@ void qemu_init(int argc, char **argv, char **envp)
>      qemu_init_displays();
>      accel_setup_post(current_machine);
>      os_setup_post();
> +    dirtylimit_setup(current_machine->smp.max_cpus);
>      resume_mux_open();
>  }