[RFC v3 0/2] target/loongarch: Add loongson binary translation feature

[RFC v3 0/2] target/loongarch: Add loongson binary translation feature

[Bibo Mao]

Loongson Binary Translation (LBT) is used to accelerate binary
translation. LBT feature is added in kvm mode, not supported in TCG
mode since it is not emulated.

Here lbt=on/off property is added to parse command line to
enable/disable lbt feature. Also fix registers relative lbt are saved
and restored during migration.

It depends LBT support on kvm side, and it is listed at
https://lore.kernel.org/kvm/20240527074644.836699-1-maobibo@loongson.cn/

---
v2 ... v3:
  1. Property lbt is added only if kvm is enabled
  2. Use feature variable lbt with OnOffAuto type, rather than feature
bitmap flags default_features and forced_features

v1 ... v2:
  1. Add LBT register saving and restoring in vmstate
  2. Add two pseudo feature flags: default_features and forced_features
---
Bibo Mao (2):
  target/loongarch: Add loongson binary translation feature
  target/loongarch: Implement lbt registers save/restore function

 target/loongarch/cpu.c                | 53 ++++++++++++++++++
 target/loongarch/cpu.h                | 18 +++++++
 target/loongarch/kvm/kvm.c            | 78 +++++++++++++++++++++++++++
 target/loongarch/kvm/kvm_loongarch.h  | 16 ++++++
 target/loongarch/loongarch-qmp-cmds.c |  2 +-
 target/loongarch/machine.c            | 24 +++++++++
 6 files changed, 190 insertions(+), 1 deletion(-)


base-commit: 3b2fe44bb7f605f179e5e7feb2c13c2eb3abbb80
-- 
2.39.3

[RFC v3 1/2] target/loongarch: Add loongson binary translation feature

[Bibo Mao]

Loongson Binary Translation (LBT) is used to accelerate binary
translation, which contains 4 scratch registers (scr0 to scr3), x86/ARM
eflags (eflags) and x87 fpu stack pointer (ftop).

Now LBT feature is added in kvm mode, not supported in TCG mode since
it is not emulated. Feature variable lbt is added with OnOffAuto type,
If lbt feature is not supported with KVM host, it reports error if there
is lbt=on command line.

If there is no any command line about lbt parameter, it checks whether
KVM host supports lbt feature and set the corresponding value in cpucfg.

Signed-off-by: Bibo Mao <maobibo@loongson.cn>
---
 target/loongarch/cpu.c                | 53 +++++++++++++++++++++++++++
 target/loongarch/cpu.h                |  6 +++
 target/loongarch/kvm/kvm.c            | 26 +++++++++++++
 target/loongarch/kvm/kvm_loongarch.h  | 16 ++++++++
 target/loongarch/loongarch-qmp-cmds.c |  2 +-
 5 files changed, 102 insertions(+), 1 deletion(-)

diff --git a/target/loongarch/cpu.c b/target/loongarch/cpu.c
index b5c1ec94af..14265b6667 100644
--- a/target/loongarch/cpu.c
+++ b/target/loongarch/cpu.c
@@ -571,6 +571,30 @@ static void loongarch_cpu_disas_set_info(CPUState *s, disassemble_info *info)
     info->print_insn = print_insn_loongarch;
 }
 
+static void loongarch_cpu_check_lbt(CPUState *cs, Error **errp)
+{
+    CPULoongArchState *env = cpu_env(cs);
+    LoongArchCPU *cpu = LOONGARCH_CPU(cs);
+    bool kvm_supported;
+
+    kvm_supported = kvm_feature_supported(cs, LOONGARCH_FEATURE_LBT);
+    if (cpu->lbt == ON_OFF_AUTO_ON) {
+        if (kvm_supported) {
+            env->cpucfg[2] = FIELD_DP32(env->cpucfg[2], CPUCFG2, LBT_ALL, 7);
+        } else {
+            error_setg(errp, "'lbt' feature not supported by KVM on this host");
+            return;
+        }
+    } else if ((cpu->lbt == ON_OFF_AUTO_AUTO) && kvm_supported) {
+        env->cpucfg[2] = FIELD_DP32(env->cpucfg[2], CPUCFG2, LBT_ALL, 7);
+    }
+}
+
+static void loongarch_cpu_feature_realize(CPUState *cs, Error **errp)
+{
+    loongarch_cpu_check_lbt(cs, errp);
+}
+
 static void loongarch_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
@@ -584,6 +608,11 @@ static void loongarch_cpu_realizefn(DeviceState *dev, Error **errp)
     }
 
     loongarch_cpu_register_gdb_regs_for_features(cs);
+    loongarch_cpu_feature_realize(cs, &local_err);
+    if (local_err != NULL) {
+        error_propagate(errp, local_err);
+        return;
+    }
 
     cpu_reset(cs);
     qemu_init_vcpu(cs);
@@ -643,12 +672,36 @@ static void loongarch_set_lasx(Object *obj, bool value, Error **errp)
     }
 }
 
+static bool loongarch_get_lbt(Object *obj, Error **errp)
+{
+    return LOONGARCH_CPU(obj)->lbt != ON_OFF_AUTO_OFF;
+}
+
+static void loongarch_set_lbt(Object *obj, bool value, Error **errp)
+{
+    LoongArchCPU *cpu = LOONGARCH_CPU(obj);
+
+    cpu->lbt = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
+}
+
 void loongarch_cpu_post_init(Object *obj)
 {
+    LoongArchCPU *cpu = LOONGARCH_CPU(obj);
+
     object_property_add_bool(obj, "lsx", loongarch_get_lsx,
                              loongarch_set_lsx);
     object_property_add_bool(obj, "lasx", loongarch_get_lasx,
                              loongarch_set_lasx);
+    /* lbt is enabled only in kvm mode, not supported in tcg mode */
+    if (kvm_enabled()) {
+        cpu->lbt = ON_OFF_AUTO_AUTO;
+        object_property_add_bool(obj, "lbt", loongarch_get_lbt,
+                                 loongarch_set_lbt);
+        object_property_set_description(obj, "lbt",
+                                   "Set off to disable Binary Tranlation.");
+    } else {
+        cpu->lbt = ON_OFF_AUTO_OFF;
+    }
 }
 
 static void loongarch_cpu_init(Object *obj)
diff --git a/target/loongarch/cpu.h b/target/loongarch/cpu.h
index 41b8e6d96d..623b96b184 100644
--- a/target/loongarch/cpu.h
+++ b/target/loongarch/cpu.h
@@ -152,6 +152,7 @@ FIELD(CPUCFG2, LLFTP_VER, 15, 3)
 FIELD(CPUCFG2, LBT_X86, 18, 1)
 FIELD(CPUCFG2, LBT_ARM, 19, 1)
 FIELD(CPUCFG2, LBT_MIPS, 20, 1)
+FIELD(CPUCFG2, LBT_ALL, 18, 3)
 FIELD(CPUCFG2, LSPW, 21, 1)
 FIELD(CPUCFG2, LAM, 22, 1)
 
@@ -280,6 +281,10 @@ struct LoongArchTLB {
 typedef struct LoongArchTLB LoongArchTLB;
 #endif
 
+enum loongarch_features {
+    LOONGARCH_FEATURE_LBT, /* loongson binary translation extension */
+};
+
 typedef struct CPUArchState {
     uint64_t gpr[32];
     uint64_t pc;
@@ -380,6 +385,7 @@ struct ArchCPU {
     CPULoongArchState env;
     QEMUTimer timer;
     uint32_t  phy_id;
+    OnOffAuto lbt;
 
     /* 'compatible' string for this CPU for Linux device trees */
     const char *dtb_compatible;
diff --git a/target/loongarch/kvm/kvm.c b/target/loongarch/kvm/kvm.c
index 8e6e27c8bf..55e85eff15 100644
--- a/target/loongarch/kvm/kvm.c
+++ b/target/loongarch/kvm/kvm.c
@@ -780,6 +780,32 @@ int kvm_loongarch_set_interrupt(LoongArchCPU *cpu, int irq, int level)
     return kvm_vcpu_ioctl(cs, KVM_INTERRUPT, &intr);
 }
 
+bool kvm_feature_supported(CPUState *cs, enum loongarch_features feature)
+{
+    int ret;
+    struct kvm_device_attr attr;
+
+    switch (feature) {
+    case LOONGARCH_FEATURE_LBT:
+        /*
+         * Return all if all the LBT features are supported such as:
+         *  KVM_LOONGARCH_VM_FEAT_X86BT
+         *  KVM_LOONGARCH_VM_FEAT_ARMBT
+         *  KVM_LOONGARCH_VM_FEAT_MIPSBT
+         */
+        attr.group = KVM_LOONGARCH_VM_FEAT_CTRL;
+        attr.attr = KVM_LOONGARCH_VM_FEAT_X86BT;
+        ret = kvm_vm_ioctl(kvm_state, KVM_HAS_DEVICE_ATTR, &attr);
+        attr.attr = KVM_LOONGARCH_VM_FEAT_ARMBT;
+        ret |= kvm_vm_ioctl(kvm_state, KVM_HAS_DEVICE_ATTR, &attr);
+        attr.attr = KVM_LOONGARCH_VM_FEAT_MIPSBT;
+        ret |= kvm_vm_ioctl(kvm_state, KVM_HAS_DEVICE_ATTR, &attr);
+        return (ret == 0);
+    default:
+        return false;
+    }
+}
+
 void kvm_arch_accel_class_init(ObjectClass *oc)
 {
 }
diff --git a/target/loongarch/kvm/kvm_loongarch.h b/target/loongarch/kvm/kvm_loongarch.h
index d945b6bb82..bdb4f180eb 100644
--- a/target/loongarch/kvm/kvm_loongarch.h
+++ b/target/loongarch/kvm/kvm_loongarch.h
@@ -13,4 +13,20 @@
 int  kvm_loongarch_set_interrupt(LoongArchCPU *cpu, int irq, int level);
 void kvm_arch_reset_vcpu(CPULoongArchState *env);
 
+#ifdef CONFIG_KVM
+/*
+ * kvm_feature_supported:
+ *
+ * Returns: true if KVM supports specified feature
+ * and false otherwise.
+ */
+bool kvm_feature_supported(CPUState *cs, enum loongarch_features feature);
+#else
+static inline bool kvm_feature_supported(CPUState *cs,
+                                         enum loongarch_features feature)
+{
+    return false;
+}
+#endif
+
 #endif
diff --git a/target/loongarch/loongarch-qmp-cmds.c b/target/loongarch/loongarch-qmp-cmds.c
index 8721a5eb13..c6f6e1ef85 100644
--- a/target/loongarch/loongarch-qmp-cmds.c
+++ b/target/loongarch/loongarch-qmp-cmds.c
@@ -40,7 +40,7 @@ CpuDefinitionInfoList *qmp_query_cpu_definitions(Error **errp)
 }
 
 static const char *cpu_model_advertised_features[] = {
-    "lsx", "lasx", NULL
+    "lsx", "lasx", "lbt", NULL
 };
 
 CpuModelExpansionInfo *qmp_query_cpu_model_expansion(CpuModelExpansionType type,
-- 
2.39.3

Re: [RFC v3 1/2] target/loongarch: Add loongson binary translation feature

[Jiaxun Yang]

在2024年5月30日五月 上午7:49，Bibo Mao写道：
> Loongson Binary Translation (LBT) is used to accelerate binary
> translation, which contains 4 scratch registers (scr0 to scr3), x86/ARM
> eflags (eflags) and x87 fpu stack pointer (ftop).
>
> Now LBT feature is added in kvm mode, not supported in TCG mode since
> it is not emulated. Feature variable lbt is added with OnOffAuto type,
> If lbt feature is not supported with KVM host, it reports error if there
> is lbt=on command line.
>
> If there is no any command line about lbt parameter, it checks whether
> KVM host supports lbt feature and set the corresponding value in cpucfg.
>
> Signed-off-by: Bibo Mao <maobibo@loongson.cn>
Hi Bibo,

I was going across recent LoongArch changes and this comes into my attention: 

> ---
>  target/loongarch/cpu.c                | 53 +++++++++++++++++++++++++++
>  target/loongarch/cpu.h                |  6 +++
>  target/loongarch/kvm/kvm.c            | 26 +++++++++++++
>  target/loongarch/kvm/kvm_loongarch.h  | 16 ++++++++
>  target/loongarch/loongarch-qmp-cmds.c |  2 +-
>  5 files changed, 102 insertions(+), 1 deletion(-)
>
> diff --git a/target/loongarch/cpu.c b/target/loongarch/cpu.c
> index b5c1ec94af..14265b6667 100644
> --- a/target/loongarch/cpu.c
> +++ b/target/loongarch/cpu.c
> @@ -571,6 +571,30 @@ static void loongarch_cpu_disas_set_info(CPUState 
> *s, disassemble_info *info)
>      info->print_insn = print_insn_loongarch;
>  }
> 
> +static void loongarch_cpu_check_lbt(CPUState *cs, Error **errp)
> +{
> +    CPULoongArchState *env = cpu_env(cs);
> +    LoongArchCPU *cpu = LOONGARCH_CPU(cs);
> +    bool kvm_supported;
> +
> +    kvm_supported = kvm_feature_supported(cs, LOONGARCH_FEATURE_LBT);

IMHO if there is no global states that should be saved/restored VM wise,
this should be handled at per CPU level, preferably with CPUCFG flags hint.

We should minimize non-privilege KVM feature bits to prevent hindering
asymmetry ISA system.

Thanks
- Jiaxun

-- 
- Jiaxun

Re: [RFC v3 1/2] target/loongarch: Add loongson binary translation feature

[maobibo]

On 2024/7/1 下午2:57, Jiaxun Yang wrote:
> 
> 
> 在2024年5月30日五月 上午7:49，Bibo Mao写道：
>> Loongson Binary Translation (LBT) is used to accelerate binary
>> translation, which contains 4 scratch registers (scr0 to scr3), x86/ARM
>> eflags (eflags) and x87 fpu stack pointer (ftop).
>>
>> Now LBT feature is added in kvm mode, not supported in TCG mode since
>> it is not emulated. Feature variable lbt is added with OnOffAuto type,
>> If lbt feature is not supported with KVM host, it reports error if there
>> is lbt=on command line.
>>
>> If there is no any command line about lbt parameter, it checks whether
>> KVM host supports lbt feature and set the corresponding value in cpucfg.
>>
>> Signed-off-by: Bibo Mao <maobibo@loongson.cn>
> Hi Bibo,
> 
> I was going across recent LoongArch changes and this comes into my attention:
> 
>> ---
>>   target/loongarch/cpu.c                | 53 +++++++++++++++++++++++++++
>>   target/loongarch/cpu.h                |  6 +++
>>   target/loongarch/kvm/kvm.c            | 26 +++++++++++++
>>   target/loongarch/kvm/kvm_loongarch.h  | 16 ++++++++
>>   target/loongarch/loongarch-qmp-cmds.c |  2 +-
>>   5 files changed, 102 insertions(+), 1 deletion(-)
>>
>> diff --git a/target/loongarch/cpu.c b/target/loongarch/cpu.c
>> index b5c1ec94af..14265b6667 100644
>> --- a/target/loongarch/cpu.c
>> +++ b/target/loongarch/cpu.c
>> @@ -571,6 +571,30 @@ static void loongarch_cpu_disas_set_info(CPUState
>> *s, disassemble_info *info)
>>       info->print_insn = print_insn_loongarch;
>>   }
>>
>> +static void loongarch_cpu_check_lbt(CPUState *cs, Error **errp)
>> +{
>> +    CPULoongArchState *env = cpu_env(cs);
>> +    LoongArchCPU *cpu = LOONGARCH_CPU(cs);
>> +    bool kvm_supported;
>> +
>> +    kvm_supported = kvm_feature_supported(cs, LOONGARCH_FEATURE_LBT);
> 
> IMHO if there is no global states that should be saved/restored VM wise,
> this should be handled at per CPU level, preferably with CPUCFG flags hint.
> 
> We should minimize non-privilege KVM feature bits to prevent hindering
> asymmetry ISA system.
For "asymmetry ISA system", do you meaning some vcpus have LBT feature, 
however some vcpus does have LBT features? That does not exists at all.

It will be big disaster for such products, how does application use this?

Regards
Bibo
> 
> Thanks
> - Jiaxun
> 

Re: [RFC v3 1/2] target/loongarch: Add loongson binary translation feature

[Jiaxun Yang]

在2024年7月1日七月 上午8:32，maobibo写道：
[...]
>>>
>>> +static void loongarch_cpu_check_lbt(CPUState *cs, Error **errp)
>>> +{
>>> +    CPULoongArchState *env = cpu_env(cs);
>>> +    LoongArchCPU *cpu = LOONGARCH_CPU(cs);
>>> +    bool kvm_supported;
>>> +
>>> +    kvm_supported = kvm_feature_supported(cs, LOONGARCH_FEATURE_LBT);
>> 
>> IMHO if there is no global states that should be saved/restored VM wise,
>> this should be handled at per CPU level, preferably with CPUCFG flags hint.
>> 
>> We should minimize non-privilege KVM feature bits to prevent hindering
>> asymmetry ISA system.
> For "asymmetry ISA system", do you meaning some vcpus have LBT feature, 
> however some vcpus does have LBT features? That does not exists at all.

Yes, we should always prepare for the future :-)

From Weiwu's presentations, I believe LASX asymmetry product is already on the
roadmap. So for LBT it's also a possibility.

Even if such product won't land in near future, we should still try our best
to bound to vCPU creation, not to the system.

>
> It will be big disaster for such products, how does application use this?

Affinity placement etc, there are many possibilities.

On Arm side, there are already systems with Big.Little asymmetry CPU that
some of the processor equipped 32 bit EL0 mode while others doesn't. They
managed that well with affinity.

See: arm64: Allow mismatched 32-bit EL0 support

Thanks
>
> Regards
> Bibo
>> 
>> Thanks
>> - Jiaxun
>>

-- 
- Jiaxun

Re: [RFC v3 1/2] target/loongarch: Add loongson binary translation feature

[maobibo]

On 2024/7/1 下午4:42, Jiaxun Yang wrote:
> 
> 
> 在2024年7月1日七月 上午8:32，maobibo写道：
> [...]
>>>>
>>>> +static void loongarch_cpu_check_lbt(CPUState *cs, Error **errp)
>>>> +{
>>>> +    CPULoongArchState *env = cpu_env(cs);
>>>> +    LoongArchCPU *cpu = LOONGARCH_CPU(cs);
>>>> +    bool kvm_supported;
>>>> +
>>>> +    kvm_supported = kvm_feature_supported(cs, LOONGARCH_FEATURE_LBT);
>>>
>>> IMHO if there is no global states that should be saved/restored VM wise,
>>> this should be handled at per CPU level, preferably with CPUCFG flags hint.
>>>
>>> We should minimize non-privilege KVM feature bits to prevent hindering
>>> asymmetry ISA system.
>> For "asymmetry ISA system", do you meaning some vcpus have LBT feature,
>> however some vcpus does have LBT features? That does not exists at all.
> 
> Yes, we should always prepare for the future :-)
> 
>  From Weiwu's presentations, I believe LASX asymmetry product is already on the
> roadmap. So for LBT it's also a possibility.
I never hear such product plan with different LASX/LSX capability even 
for the future big-little product.

> 
> Even if such product won't land in near future, we should still try our best
> to bound to vCPU creation, not to the system.
> 
>>
>> It will be big disaster for such products, how does application use this?
> 
> Affinity placement etc, there are many possibilities.
> 
> On Arm side, there are already systems with Big.Little asymmetry CPU that
> some of the processor equipped 32 bit EL0 mode while others doesn't. They
> managed that well with affinity.
> 
> See: arm64: Allow mismatched 32-bit EL0 support
Is it only 32 bit EL0 mode or 32bit compatible mode and 64 bit also?

What is the detailed product name and how does Linux/KVM support this 
product?

Regards
Bibo Mao
> 
> Thanks
>>
>> Regards
>> Bibo
>>>
>>> Thanks
>>> - Jiaxun
>>>
> 

Re: [RFC v3 1/2] target/loongarch: Add loongson binary translation feature

[Jiaxun Yang]

在2024年7月1日七月 下午2:57，Jiaxun Yang写道：
> 在2024年5月30日五月 上午7:49，Bibo Mao写道：
>> Loongson Binary Translation (LBT) is used to accelerate binary
>> translation, which contains 4 scratch registers (scr0 to scr3), x86/ARM
>> eflags (eflags) and x87 fpu stack pointer (ftop).
>>
>> Now LBT feature is added in kvm mode, not supported in TCG mode since
>> it is not emulated. Feature variable lbt is added with OnOffAuto type,
>> If lbt feature is not supported with KVM host, it reports error if there
>> is lbt=on command line.
>>
>> If there is no any command line about lbt parameter, it checks whether
>> KVM host supports lbt feature and set the corresponding value in cpucfg.
>>
>> Signed-off-by: Bibo Mao <maobibo@loongson.cn>
> Hi Bibo,
>
> I was going across recent LoongArch changes and this comes into my attention: 
>
>> ---
>>  target/loongarch/cpu.c                | 53 +++++++++++++++++++++++++++
>>  target/loongarch/cpu.h                |  6 +++
>>  target/loongarch/kvm/kvm.c            | 26 +++++++++++++
>>  target/loongarch/kvm/kvm_loongarch.h  | 16 ++++++++
>>  target/loongarch/loongarch-qmp-cmds.c |  2 +-
>>  5 files changed, 102 insertions(+), 1 deletion(-)
>>
>> diff --git a/target/loongarch/cpu.c b/target/loongarch/cpu.c
>> index b5c1ec94af..14265b6667 100644
>> --- a/target/loongarch/cpu.c
>> +++ b/target/loongarch/cpu.c
>> @@ -571,6 +571,30 @@ static void loongarch_cpu_disas_set_info(CPUState 
>> *s, disassemble_info *info)
>>      info->print_insn = print_insn_loongarch;
>>  }
>> 
>> +static void loongarch_cpu_check_lbt(CPUState *cs, Error **errp)
>> +{
>> +    CPULoongArchState *env = cpu_env(cs);
>> +    LoongArchCPU *cpu = LOONGARCH_CPU(cs);
>> +    bool kvm_supported;
>> +
>> +    kvm_supported = kvm_feature_supported(cs, LOONGARCH_FEATURE_LBT);
>
> IMHO if there is no global states that should be saved/restored VM wise,
> this should be handled at per CPU level, preferably with CPUCFG flags hint.
>
> We should minimize non-privilege KVM feature bits to prevent hindering
> asymmetry ISA system.

+ Huacai for further discussion

Hi Bibo, Huacai,

I investigated the topic further and went through the thread on kernel side.

I think Huacai and me are all on the same page that we should unify the interface for per-CPU
level feature probing and setting interface. Huacai purposed converting all features to VM feature
but I still believe CPUCFG is the best interface.

To probe LBT before actual vcpu creation, we can borrow the approach used by other architectures
(kvm_arm_create_scratch_host_vcpu() & kvm_riscv_create_scratch_vcpu()).

Kernel will reject setting unknown CPUCFG bits with -EINVAL, so to probe LBT we just need to perform
KVM_SET_REGS to scratch vcpu with LBT set to see if it's valid for kernel. There is no need for any other
probing interface.

I do think scratch CPU interface is also necessary if we are going to implement cpu = host.

Huacai, would you agree with me?

Thanks
- Jiaxun

>
> Thanks
> - Jiaxun
>
> -- 
> - Jiaxun

-- 
- Jiaxun

Re: [RFC v3 1/2] target/loongarch: Add loongson binary translation feature

[Huacai Chen]

On Wed, Jul 3, 2024 at 3:51 PM Jiaxun Yang <jiaxun.yang@flygoat.com> wrote:
>
>
>
> 在2024年7月1日七月 下午2:57，Jiaxun Yang写道：
> > 在2024年5月30日五月 上午7:49，Bibo Mao写道：
> >> Loongson Binary Translation (LBT) is used to accelerate binary
> >> translation, which contains 4 scratch registers (scr0 to scr3), x86/ARM
> >> eflags (eflags) and x87 fpu stack pointer (ftop).
> >>
> >> Now LBT feature is added in kvm mode, not supported in TCG mode since
> >> it is not emulated. Feature variable lbt is added with OnOffAuto type,
> >> If lbt feature is not supported with KVM host, it reports error if there
> >> is lbt=on command line.
> >>
> >> If there is no any command line about lbt parameter, it checks whether
> >> KVM host supports lbt feature and set the corresponding value in cpucfg.
> >>
> >> Signed-off-by: Bibo Mao <maobibo@loongson.cn>
> > Hi Bibo,
> >
> > I was going across recent LoongArch changes and this comes into my attention:
> >
> >> ---
> >>  target/loongarch/cpu.c                | 53 +++++++++++++++++++++++++++
> >>  target/loongarch/cpu.h                |  6 +++
> >>  target/loongarch/kvm/kvm.c            | 26 +++++++++++++
> >>  target/loongarch/kvm/kvm_loongarch.h  | 16 ++++++++
> >>  target/loongarch/loongarch-qmp-cmds.c |  2 +-
> >>  5 files changed, 102 insertions(+), 1 deletion(-)
> >>
> >> diff --git a/target/loongarch/cpu.c b/target/loongarch/cpu.c
> >> index b5c1ec94af..14265b6667 100644
> >> --- a/target/loongarch/cpu.c
> >> +++ b/target/loongarch/cpu.c
> >> @@ -571,6 +571,30 @@ static void loongarch_cpu_disas_set_info(CPUState
> >> *s, disassemble_info *info)
> >>      info->print_insn = print_insn_loongarch;
> >>  }
> >>
> >> +static void loongarch_cpu_check_lbt(CPUState *cs, Error **errp)
> >> +{
> >> +    CPULoongArchState *env = cpu_env(cs);
> >> +    LoongArchCPU *cpu = LOONGARCH_CPU(cs);
> >> +    bool kvm_supported;
> >> +
> >> +    kvm_supported = kvm_feature_supported(cs, LOONGARCH_FEATURE_LBT);
> >
> > IMHO if there is no global states that should be saved/restored VM wise,
> > this should be handled at per CPU level, preferably with CPUCFG flags hint.
> >
> > We should minimize non-privilege KVM feature bits to prevent hindering
> > asymmetry ISA system.
>
> + Huacai for further discussion
>
> Hi Bibo, Huacai,
>
> I investigated the topic further and went through the thread on kernel side.
>
> I think Huacai and me are all on the same page that we should unify the interface for per-CPU
> level feature probing and setting interface. Huacai purposed converting all features to VM feature
> but I still believe CPUCFG is the best interface.
>
> To probe LBT before actual vcpu creation, we can borrow the approach used by other architectures
> (kvm_arm_create_scratch_host_vcpu() & kvm_riscv_create_scratch_vcpu()).
>
> Kernel will reject setting unknown CPUCFG bits with -EINVAL, so to probe LBT we just need to perform
> KVM_SET_REGS to scratch vcpu with LBT set to see if it's valid for kernel. There is no need for any other
> probing interface.
>
> I do think scratch CPU interface is also necessary if we are going to implement cpu = host.
>
> Huacai, would you agree with me?
For me the important thing is consistency, all vm-features or all
vcpu-features are both accepted.

Huacai

>
> Thanks
> - Jiaxun
>
> >
> > Thanks
> > - Jiaxun
> >
> > --
> > - Jiaxun
>
> --
> - Jiaxun

Re: [RFC v3 1/2] target/loongarch: Add loongson binary translation feature

[maobibo]

On 2024/7/3 下午5:43, Huacai Chen wrote:
> On Wed, Jul 3, 2024 at 3:51 PM Jiaxun Yang <jiaxun.yang@flygoat.com> wrote:
>>
>>
>>
>> 在2024年7月1日七月 下午2:57，Jiaxun Yang写道：
>>> 在2024年5月30日五月 上午7:49，Bibo Mao写道：
>>>> Loongson Binary Translation (LBT) is used to accelerate binary
>>>> translation, which contains 4 scratch registers (scr0 to scr3), x86/ARM
>>>> eflags (eflags) and x87 fpu stack pointer (ftop).
>>>>
>>>> Now LBT feature is added in kvm mode, not supported in TCG mode since
>>>> it is not emulated. Feature variable lbt is added with OnOffAuto type,
>>>> If lbt feature is not supported with KVM host, it reports error if there
>>>> is lbt=on command line.
>>>>
>>>> If there is no any command line about lbt parameter, it checks whether
>>>> KVM host supports lbt feature and set the corresponding value in cpucfg.
>>>>
>>>> Signed-off-by: Bibo Mao <maobibo@loongson.cn>
>>> Hi Bibo,
>>>
>>> I was going across recent LoongArch changes and this comes into my attention:
>>>
>>>> ---
>>>>   target/loongarch/cpu.c                | 53 +++++++++++++++++++++++++++
>>>>   target/loongarch/cpu.h                |  6 +++
>>>>   target/loongarch/kvm/kvm.c            | 26 +++++++++++++
>>>>   target/loongarch/kvm/kvm_loongarch.h  | 16 ++++++++
>>>>   target/loongarch/loongarch-qmp-cmds.c |  2 +-
>>>>   5 files changed, 102 insertions(+), 1 deletion(-)
>>>>
>>>> diff --git a/target/loongarch/cpu.c b/target/loongarch/cpu.c
>>>> index b5c1ec94af..14265b6667 100644
>>>> --- a/target/loongarch/cpu.c
>>>> +++ b/target/loongarch/cpu.c
>>>> @@ -571,6 +571,30 @@ static void loongarch_cpu_disas_set_info(CPUState
>>>> *s, disassemble_info *info)
>>>>       info->print_insn = print_insn_loongarch;
>>>>   }
>>>>
>>>> +static void loongarch_cpu_check_lbt(CPUState *cs, Error **errp)
>>>> +{
>>>> +    CPULoongArchState *env = cpu_env(cs);
>>>> +    LoongArchCPU *cpu = LOONGARCH_CPU(cs);
>>>> +    bool kvm_supported;
>>>> +
>>>> +    kvm_supported = kvm_feature_supported(cs, LOONGARCH_FEATURE_LBT);
>>>
>>> IMHO if there is no global states that should be saved/restored VM wise,
>>> this should be handled at per CPU level, preferably with CPUCFG flags hint.
>>>
>>> We should minimize non-privilege KVM feature bits to prevent hindering
>>> asymmetry ISA system.
>>
>> + Huacai for further discussion
>>
>> Hi Bibo, Huacai,
>>
>> I investigated the topic further and went through the thread on kernel side.
>>
>> I think Huacai and me are all on the same page that we should unify the interface for per-CPU
>> level feature probing and setting interface. Huacai purposed converting all features to VM feature
>> but I still believe CPUCFG is the best interface.
>>
>> To probe LBT before actual vcpu creation, we can borrow the approach used by other architectures
>> (kvm_arm_create_scratch_host_vcpu() & kvm_riscv_create_scratch_vcpu()).
>>
>> Kernel will reject setting unknown CPUCFG bits with -EINVAL, so to probe LBT we just need to perform
>> KVM_SET_REGS to scratch vcpu with LBT set to see if it's valid for kernel. There is no need for any other
>> probing interface.
>>
>> I do think scratch CPU interface is also necessary if we are going to implement cpu = host.
>>
>> Huacai, would you agree with me?
> For me the important thing is consistency, all vm-features or all
> vcpu-features are both accepted.
To understand features immediately is difficult job for me. There is 
supported features/used features usages etc, overall feature detection 
should be VM relative by my knowledge.

Maybe after host machine type and migration feature detection and 
checking is finished, there will be further upstanding -:(

Regards
Bibo Mao

> 
> Huacai
> 
>>
>> Thanks
>> - Jiaxun
>>
>>>
>>> Thanks
>>> - Jiaxun
>>>
>>> --
>>> - Jiaxun
>>
>> --
>> - Jiaxun

Re: [RFC v3 1/2] target/loongarch: Add loongson binary translation feature

[Jiaxun Yang]

在2024年7月3日七月 下午6:10，maobibo写道：
[...]
>>>
>>> Huacai, would you agree with me?
>> For me the important thing is consistency, all vm-features or all
>> vcpu-features are both accepted.
> To understand features immediately is difficult job for me. There is 
> supported features/used features usages etc, overall feature detection 
> should be VM relative by my knowledge.
>
> Maybe after host machine type and migration feature detection and 
> checking is finished, there will be further upstanding -:(

Do you agree with the scratch_vcpu approach? This seems to be the only
straight forward way to make some progress here.

Thanks
-- 
- Jiaxun

[RFC v3 2/2] target/loongarch: Implement lbt registers save/restore function

[Bibo Mao]

Six registers scr0 - scr3, eflags and ftop are added in percpu vmstate.
And two functions kvm_loongarch_get_lbt/kvm_loongarch_put_lbt are added
to save/restore lbt registers.

Signed-off-by: Bibo Mao <maobibo@loongson.cn>
---
 target/loongarch/cpu.h     | 12 +++++++++
 target/loongarch/kvm/kvm.c | 52 ++++++++++++++++++++++++++++++++++++++
 target/loongarch/machine.c | 24 ++++++++++++++++++
 3 files changed, 88 insertions(+)

diff --git a/target/loongarch/cpu.h b/target/loongarch/cpu.h
index 623b96b184..93c97d5e23 100644
--- a/target/loongarch/cpu.h
+++ b/target/loongarch/cpu.h
@@ -285,6 +285,17 @@ enum loongarch_features {
     LOONGARCH_FEATURE_LBT, /* loongson binary translation extension */
 };
 
+typedef struct  LoongArchBT {
+    /* scratch registers */
+    uint64_t scr0;
+    uint64_t scr1;
+    uint64_t scr2;
+    uint64_t scr3;
+    /* loongarch eflags */
+    uint64_t eflags;
+    uint64_t ftop;
+} lbt_t;
+
 typedef struct CPUArchState {
     uint64_t gpr[32];
     uint64_t pc;
@@ -292,6 +303,7 @@ typedef struct CPUArchState {
     fpr_t fpr[32];
     bool cf[8];
     uint32_t fcsr0;
+    lbt_t  lbt;
 
     uint32_t cpucfg[21];
 
diff --git a/target/loongarch/kvm/kvm.c b/target/loongarch/kvm/kvm.c
index 55e85eff15..c9c240a573 100644
--- a/target/loongarch/kvm/kvm.c
+++ b/target/loongarch/kvm/kvm.c
@@ -475,6 +475,48 @@ static int kvm_loongarch_put_regs_fp(CPUState *cs)
     return ret;
 }
 
+static int kvm_loongarch_put_lbt(CPUState *cs)
+{
+    CPULoongArchState *env = cpu_env(cs);
+    int ret;
+
+    /* check whether vm support LBT firstly */
+    if (FIELD_EX32(env->cpucfg[2], CPUCFG2, LBT_ALL) != 7) {
+        return 0;
+    }
+
+    /* set six LBT registers including scr0-scr3, eflags, ftop */
+    ret = kvm_set_one_reg(cs, KVM_REG_LOONGARCH_LBT_SCR0, &env->lbt.scr0);
+    ret |= kvm_set_one_reg(cs, KVM_REG_LOONGARCH_LBT_SCR1, &env->lbt.scr1);
+    ret |= kvm_set_one_reg(cs, KVM_REG_LOONGARCH_LBT_SCR2, &env->lbt.scr2);
+    ret |= kvm_set_one_reg(cs, KVM_REG_LOONGARCH_LBT_SCR3, &env->lbt.scr3);
+    ret |= kvm_set_one_reg(cs, KVM_REG_LOONGARCH_LBT_EFLAGS, &env->lbt.eflags);
+    ret |= kvm_set_one_reg(cs, KVM_REG_LOONGARCH_LBT_FTOP, &env->lbt.ftop);
+
+    return ret;
+}
+
+static int kvm_loongarch_get_lbt(CPUState *cs)
+{
+    CPULoongArchState *env = cpu_env(cs);
+    int ret;
+
+    /* check whether vm support LBT firstly */
+    if (FIELD_EX32(env->cpucfg[2], CPUCFG2, LBT_ALL) != 7) {
+        return 0;
+    }
+
+    /* get six LBT registers including scr0-scr3, eflags, ftop */
+    ret = kvm_get_one_reg(cs, KVM_REG_LOONGARCH_LBT_SCR0, &env->lbt.scr0);
+    ret |= kvm_get_one_reg(cs, KVM_REG_LOONGARCH_LBT_SCR1, &env->lbt.scr1);
+    ret |= kvm_get_one_reg(cs, KVM_REG_LOONGARCH_LBT_SCR2, &env->lbt.scr2);
+    ret |= kvm_get_one_reg(cs, KVM_REG_LOONGARCH_LBT_SCR3, &env->lbt.scr3);
+    ret |= kvm_get_one_reg(cs, KVM_REG_LOONGARCH_LBT_EFLAGS, &env->lbt.eflags);
+    ret |= kvm_get_one_reg(cs, KVM_REG_LOONGARCH_LBT_FTOP, &env->lbt.ftop);
+
+    return ret;
+}
+
 void kvm_arch_reset_vcpu(CPULoongArchState *env)
 {
     env->mp_state = KVM_MP_STATE_RUNNABLE;
@@ -608,6 +650,11 @@ int kvm_arch_get_registers(CPUState *cs)
         return ret;
     }
 
+    ret = kvm_loongarch_get_lbt(cs);
+    if (ret) {
+        return ret;
+    }
+
     ret = kvm_loongarch_get_mpstate(cs);
     return ret;
 }
@@ -636,6 +683,11 @@ int kvm_arch_put_registers(CPUState *cs, int level)
         return ret;
     }
 
+    ret = kvm_loongarch_put_lbt(cs);
+    if (ret) {
+        return ret;
+    }
+
     ret = kvm_loongarch_put_mpstate(cs);
     return ret;
 }
diff --git a/target/loongarch/machine.c b/target/loongarch/machine.c
index 08a7fa5370..a371d5a2ab 100644
--- a/target/loongarch/machine.c
+++ b/target/loongarch/machine.c
@@ -110,6 +110,29 @@ static const VMStateDescription vmstate_lasx = {
     },
 };
 
+static bool lbt_needed(void *opaque)
+{
+    LoongArchCPU *cpu = opaque;
+
+    return !!FIELD_EX64(cpu->env.cpucfg[2], CPUCFG2, LBT_ALL);
+}
+
+static const VMStateDescription vmstate_lbt = {
+    .name = "cpu/lbt",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .needed = lbt_needed,
+    .fields = (const VMStateField[]) {
+        VMSTATE_UINT64(env.lbt.scr0,   LoongArchCPU),
+        VMSTATE_UINT64(env.lbt.scr1,   LoongArchCPU),
+        VMSTATE_UINT64(env.lbt.scr2,   LoongArchCPU),
+        VMSTATE_UINT64(env.lbt.scr3,   LoongArchCPU),
+        VMSTATE_UINT64(env.lbt.eflags, LoongArchCPU),
+        VMSTATE_UINT64(env.lbt.ftop,   LoongArchCPU),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
 #if defined(CONFIG_TCG) && !defined(CONFIG_USER_ONLY)
 static bool tlb_needed(void *opaque)
 {
@@ -219,6 +242,7 @@ const VMStateDescription vmstate_loongarch_cpu = {
 #if defined(CONFIG_TCG) && !defined(CONFIG_USER_ONLY)
         &vmstate_tlb,
 #endif
+        &vmstate_lbt,
         NULL
     }
 };
-- 
2.39.3