[PATCH RFC 2/4] cpu-target: support emulation from non-TCG accels

[Joelle van Dyne <j@getutm.app>]

We create a toggle to allow TCG emulation to be used dynamically when
running other accelerators. Tracking dirty code can be expensive so we
need to flush the TLBs and TBs every time we toggle emulation mode. Plugin
support is currently disabled when running in this mode.

Signed-off-by: Joelle van Dyne <j@getutm.app>
---
 include/hw/core/cpu.h     | 10 ++++++++++
 accel/tcg/plugin-gen.c    |  4 ++++
 accel/tcg/tb-maint.c      |  2 +-
 accel/tcg/tcg-accel-ops.c |  3 ++-
 cpu-target.c              | 13 +++++++++++++
 plugins/core.c            | 12 ++++++++++++
 system/physmem.c          |  5 +++--
 7 files changed, 45 insertions(+), 4 deletions(-)

diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h
index e3c8450f8f..dbbaca06ee 100644
--- a/include/hw/core/cpu.h
+++ b/include/hw/core/cpu.h
@@ -569,6 +569,9 @@ struct CPUState {
     /* track IOMMUs whose translations we've cached in the TCG TLB */
     GArray *iommu_notifiers;
 
+    /* doing emulation when not in TCG backend */
+    bool emulation_enabled;
+
     /*
      * MUST BE LAST in order to minimize the displacement to CPUArchState.
      */
@@ -1083,6 +1086,13 @@ void qemu_init_vcpu(CPUState *cpu);
  */
 void cpu_single_step(CPUState *cpu, int enabled);
 
+/**
+ * cpu_emulate:
+ * @cpu: CPU to set to emulation mode
+ * @enabled: enable emulation mode
+ */
+void cpu_emulate(CPUState *cpu, bool enabled);
+
 /* Breakpoint/watchpoint flags */
 #define BP_MEM_READ           0x01
 #define BP_MEM_WRITE          0x02
diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c
index 7e5f040bf7..e07dffeb00 100644
--- a/accel/tcg/plugin-gen.c
+++ b/accel/tcg/plugin-gen.c
@@ -388,6 +388,10 @@ bool plugin_gen_tb_start(CPUState *cpu, const DisasContextBase *db)
 {
     struct qemu_plugin_tb *ptb;
 
+    if (cpu->emulation_enabled) {
+        return false;
+    }
+
     if (!test_bit(QEMU_PLUGIN_EV_VCPU_TB_TRANS,
                   cpu->plugin_state->event_mask)) {
         return false;
diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c
index 3f1bebf6ab..14d4bed347 100644
--- a/accel/tcg/tb-maint.c
+++ b/accel/tcg/tb-maint.c
@@ -791,7 +791,7 @@ done:
 
 void tb_flush(CPUState *cpu)
 {
-    if (tcg_enabled()) {
+    if (tcg_enabled() || unlikely(cpu->emulation_enabled)) {
         unsigned tb_flush_count = qatomic_read(&tb_ctx.tb_flush_count);
 
         if (cpu_in_serial_context(cpu)) {
diff --git a/accel/tcg/tcg-accel-ops.c b/accel/tcg/tcg-accel-ops.c
index 6e3f1fa92b..3c07407ccf 100644
--- a/accel/tcg/tcg-accel-ops.c
+++ b/accel/tcg/tcg-accel-ops.c
@@ -32,6 +32,7 @@
 #include "qemu/main-loop.h"
 #include "qemu/guest-random.h"
 #include "qemu/timer.h"
+#include "exec/cpu-common.h"
 #include "exec/exec-all.h"
 #include "exec/hwaddr.h"
 #include "exec/tb-flush.h"
@@ -74,7 +75,7 @@ void tcg_cpu_destroy(CPUState *cpu)
 int tcg_cpu_exec(CPUState *cpu)
 {
     int ret;
-    assert(tcg_enabled());
+    assert(tcg_enabled() || cpu->emulation_enabled);
     cpu_exec_start(cpu);
     ret = cpu_exec(cpu);
     cpu_exec_end(cpu);
diff --git a/cpu-target.c b/cpu-target.c
index 6293477ed9..8df75e915a 100644
--- a/cpu-target.c
+++ b/cpu-target.c
@@ -339,6 +339,19 @@ void cpu_single_step(CPUState *cpu, int enabled)
     }
 }
 
+void cpu_emulate(CPUState *cpu, bool enabled)
+{
+    if (cpu->emulation_enabled != enabled) {
+        cpu->emulation_enabled = enabled;
+
+        if (enabled) {
+            /* FIXME: track dirty code to improve performance */
+            tb_flush(cpu);
+            tlb_flush(cpu);
+        }
+    }
+}
+
 void cpu_abort(CPUState *cpu, const char *fmt, ...)
 {
     va_list ap;
diff --git a/plugins/core.c b/plugins/core.c
index bb105e8e68..dee6ffd722 100644
--- a/plugins/core.c
+++ b/plugins/core.c
@@ -55,6 +55,10 @@ struct qemu_plugin_ctx *plugin_id_to_ctx_locked(qemu_plugin_id_t id)
 
 static void plugin_cpu_update__async(CPUState *cpu, run_on_cpu_data data)
 {
+    if (cpu->emulation_enabled) {
+        return;
+    }
+
     bitmap_copy(cpu->plugin_state->event_mask,
                 &data.host_ulong, QEMU_PLUGIN_EV_MAX);
     tcg_flush_jmp_cache(cpu);
@@ -499,6 +503,10 @@ qemu_plugin_vcpu_syscall(CPUState *cpu, int64_t num, uint64_t a1, uint64_t a2,
     struct qemu_plugin_cb *cb, *next;
     enum qemu_plugin_event ev = QEMU_PLUGIN_EV_VCPU_SYSCALL;
 
+    if (cpu->emulation_enabled) {
+        return;
+    }
+
     if (!test_bit(ev, cpu->plugin_state->event_mask)) {
         return;
     }
@@ -521,6 +529,10 @@ void qemu_plugin_vcpu_syscall_ret(CPUState *cpu, int64_t num, int64_t ret)
     struct qemu_plugin_cb *cb, *next;
     enum qemu_plugin_event ev = QEMU_PLUGIN_EV_VCPU_SYSCALL_RET;
 
+    if (cpu->emulation_enabled) {
+        return;
+    }
+
     if (!test_bit(ev, cpu->plugin_state->event_mask)) {
         return;
     }
diff --git a/system/physmem.c b/system/physmem.c
index 67c9db9daa..4bb2976646 100644
--- a/system/physmem.c
+++ b/system/physmem.c
@@ -2696,7 +2696,9 @@ static void tcg_commit_cpu(CPUState *cpu, run_on_cpu_data data)
     CPUAddressSpace *cpuas = data.host_ptr;
 
     cpuas->memory_dispatch = address_space_to_dispatch(cpuas->as);
-    tlb_flush(cpu);
+    if (tcg_enabled() || cpu->emulation_enabled) {
+        tlb_flush(cpu);
+    }
 }
 
 static void tcg_commit(MemoryListener *listener)
@@ -2704,7 +2706,6 @@ static void tcg_commit(MemoryListener *listener)
     CPUAddressSpace *cpuas;
     CPUState *cpu;
 
-    assert(tcg_enabled());
     /* since each CPU stores ram addresses in its TLB cache, we must
        reset the modified entries */
     cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
-- 
2.41.0