* [PATCH 1/8] accel/tcg: cache the result of curr_cflags()
2026-08-18 17:42 [RFC PATCH 0/8] accel/tcg: cut per-block dispatch overhead Matt Turner
@ 2026-08-18 17:42 ` Matt Turner
2026-08-18 17:42 ` [PATCH 2/8] accel/tcg: enlarge the TB jump cache to 64K entries Matt Turner
` (6 subsequent siblings)
7 siblings, 0 replies; 13+ messages in thread
From: Matt Turner @ 2026-08-18 17:42 UTC (permalink / raw)
To: qemu-devel
Cc: richard.henderson, pbonzini, philmd, zhao1.liu, laurent, deller,
pierrick.bouvier, Matt Turner
curr_cflags() is called once per TB dispatch, from helper_lookup_tb_ptr()
and from the cpu_exec() loop. It recomputes the same value every time:
uint32_t cflags = cpu->tcg_cflags;
if (unlikely(cpu_single_stepping(cpu))) { ... }
else if (qatomic_read(&one_insn_per_tb)) { ... }
else if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { ... }
That is three loads and three branches on the hottest path in the
interpreter, for state that changes only when gdb enables single-step,
when one-insn-per-tb is toggled, or when the log mask changes.
Compute the value once into CPUState::tcg_curr_cflags and recompute it
from the four places that can change an input: tcg_cflags_set(),
cpu_single_step(), tcg_set_one_insn_per_tb() and qemu_set_log_internal().
curr_cflags() becomes a single load.
Measured with qemu-alpha running an emulated alpha gcc 16.2.0 compiling
the SQLite 3.45.1 amalgamation (255k lines, -O2) on an x86-64 host, in a
build configured with --enable-lto:
before: 1,647,901,588,726 instructions
after: 1,563,829,403,943 instructions -5.10%
That workload issues 8.4 billion dispatches, so the per-call saving is
small but the aggregate is not. The emulated compiler produces
byte-identical output before and after.
Wall clock does not move: 133.57s to 133.13s, a 0.33% difference against a
run-to-run spread of the same size. The removed work is a few predictable
loads and branches that the host executes largely in parallel with the
surrounding dispatch, so this patch is worth taking for the instruction
count and for what it enables, not for a time saving that can be measured
on its own.
Note that tcg_set_one_insn_per_tb() does not tb_flush(), so the cache
cannot piggyback on TB flushing and needs its own update call.
Caching makes the pre-computed cflags one half of a pair that has to be
kept in step, and nothing in C enforces that. The cost of getting it
wrong is not a crash but silently wrong code generation: a stale cache
that is missing CF_PARALLEL makes TCG emit the non-atomic form of guest
atomics, and the guest then corrupts its own mutexes. linux-user's
cpu_copy() is exactly such a trap. do_fork() calls
begin_parallel_context() on the parent before cpu_copy(), so the child
inherits CF_PARALLEL and never calls tcg_cflags_set() itself; assigning
the field directly would leave every cloned thread dispatching with a
cflags of zero.
Close the hole from both ends.
Name the field tcg_cflags_priv and add tcg_cflags_get(), so that
tcg_cflags_has()/get()/set() are the only ways to reach it. C cannot
really make a struct member private, but an open-coded access now fails
to compile rather than quietly going stale, which is enough to force a
rebased or newly written user to look at the accessors. target/alpha's
CF_PCREL setup is converted along with it: it would be benign either way
today, because tcg_cpu_init_cflags() refreshes the cache afterwards in
system mode, but it is the same pattern and only ordering saved it.
Then have curr_cflags() recompute the value and compare, under
CONFIG_DEBUG_TCG. That is the check that catches a missed update on the
first dispatch, rather than days later by way of corrupted guest mutexes.
It cannot be unconditional, as recomputing on every dispatch is the very
cost the cache exists to avoid.
Verified by dropping the tcg_cflags_set() call from cpu_copy() against a
debug-tcg build: the assert fires immediately, reporting the cached and
recomputed values and the bits that differ.
Signed-off-by: Matt Turner <mattst88@gmail.com>
---
accel/tcg/cpu-exec-common.c | 48 +++++++++++++++++++++++++++++---
accel/tcg/internal-common.h | 19 ++++++++++++-
accel/tcg/tcg-all.c | 1 +
cpu-target.c | 3 ++
include/exec/translation-block.h | 6 ++++
include/hw/core/cpu.h | 13 +++++++--
include/system/tcg.h | 9 ++++++
linux-user/main.c | 2 +-
stubs/meson.build | 1 +
stubs/tcg-cflags.c | 16 +++++++++++
target/alpha/cpu.c | 2 +-
util/log.c | 4 +++
12 files changed, 114 insertions(+), 10 deletions(-)
create mode 100644 stubs/tcg-cflags.c
diff --git ./accel/tcg/cpu-exec-common.c ./accel/tcg/cpu-exec-common.c
index 44e84344f3..c75fbd344d 100644
--- ./accel/tcg/cpu-exec-common.c
+++ ./accel/tcg/cpu-exec-common.c
@@ -28,17 +28,23 @@ bool tcg_allowed;
bool tcg_cflags_has(CPUState *cpu, uint32_t flags)
{
- return cpu->tcg_cflags & flags;
+ return cpu->tcg_cflags_priv & flags;
+}
+
+uint32_t tcg_cflags_get(CPUState *cpu)
+{
+ return cpu->tcg_cflags_priv;
}
void tcg_cflags_set(CPUState *cpu, uint32_t flags)
{
- cpu->tcg_cflags |= flags;
+ cpu->tcg_cflags_priv |= flags;
+ tcg_update_curr_cflags(cpu);
}
-uint32_t curr_cflags(CPUState *cpu)
+static uint32_t compute_curr_cflags(CPUState *cpu)
{
- uint32_t cflags = cpu->tcg_cflags;
+ uint32_t cflags = cpu->tcg_cflags_priv;
/*
* Record gdb single-step. We should be exiting the TB by raising
@@ -58,6 +64,40 @@ uint32_t curr_cflags(CPUState *cpu)
return cflags;
}
+void tcg_update_curr_cflags(CPUState *cpu)
+{
+ cpu->tcg_curr_cflags = compute_curr_cflags(cpu);
+}
+
+void tcg_update_all_curr_cflags(void)
+{
+ CPUState *cpu;
+
+ CPU_FOREACH(cpu) {
+ tcg_update_curr_cflags(cpu);
+ }
+}
+
+#ifdef CONFIG_DEBUG_TCG
+/*
+ * Catch a cached value that has gone stale because an input changed without
+ * a matching tcg_update_curr_cflags(). Called from curr_cflags() on the
+ * dispatch path, so it exists only in debug-tcg builds.
+ */
+void tcg_assert_curr_cflags(CPUState *cpu)
+{
+ uint32_t cached = cpu->tcg_curr_cflags;
+ uint32_t fresh = compute_curr_cflags(cpu);
+
+ if (unlikely(cached != fresh)) {
+ fprintf(stderr, "stale tcg_curr_cflags on CPU %d: "
+ "cached 0x%08x, recomputed 0x%08x (differ in 0x%08x)\n",
+ cpu->cpu_index, cached, fresh, cached ^ fresh);
+ g_assert_not_reached();
+ }
+}
+#endif
+
/* exit the current TB, but without causing any exception to be raised */
void cpu_loop_exit_noexc(CPUState *cpu)
{
diff --git ./accel/tcg/internal-common.h ./accel/tcg/internal-common.h
index 9e7be2d78d..dc713a6e1a 100644
--- ./accel/tcg/internal-common.h
+++ ./accel/tcg/internal-common.h
@@ -70,7 +70,24 @@ bool tcg_exec_realizefn(CPUState *cpu, Error **errp);
void tcg_exec_unrealizefn(CPUState *cpu);
/* current cflags for hashing/comparison */
-uint32_t curr_cflags(CPUState *cpu);
+/*
+ * Cached by tcg_update_curr_cflags(). This is on the hot TB dispatch
+ * path, so it must stay a single load; see commit message. A debug-tcg
+ * build pays for a recompute here to prove the cache is still in step,
+ * which turns a missed update into a loud failure rather than subtly
+ * wrong code generation.
+ */
+#ifdef CONFIG_DEBUG_TCG
+void tcg_assert_curr_cflags(CPUState *cpu);
+#endif
+
+static inline uint32_t curr_cflags(CPUState *cpu)
+{
+#ifdef CONFIG_DEBUG_TCG
+ tcg_assert_curr_cflags(cpu);
+#endif
+ return cpu->tcg_curr_cflags;
+}
void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr);
diff --git ./accel/tcg/tcg-all.c ./accel/tcg/tcg-all.c
index 7186c10cf0..8f892f580f 100644
--- ./accel/tcg/tcg-all.c
+++ ./accel/tcg/tcg-all.c
@@ -254,6 +254,7 @@ static void tcg_set_one_insn_per_tb(Object *obj, bool value, Error **errp)
s->one_insn_per_tb = value;
/* Set the global also: this changes the behaviour */
qatomic_set(&one_insn_per_tb, value);
+ tcg_update_all_curr_cflags();
}
static void tcg_accel_class_init(ObjectClass *oc, const void *data)
diff --git ./cpu-target.c ./cpu-target.c
index 4783845c9b..9affbcd9c5 100644
--- ./cpu-target.c
+++ ./cpu-target.c
@@ -24,6 +24,7 @@
#include "exec/replay-core.h"
#include "exec/log.h"
#include "hw/core/cpu.h"
+#include "system/tcg.h"
#include "trace/trace-root.h"
/* enable or disable single step mode. EXCP_DEBUG is returned by the
@@ -35,6 +36,8 @@ void cpu_single_step(CPUState *cpu, unsigned flags)
cpu->singlestep_flags, flags);
cpu->singlestep_flags = flags;
+ tcg_update_curr_cflags(cpu);
+
#if !defined(CONFIG_USER_ONLY)
const AccelOpsClass *ops = cpus_get_accel();
if (ops->update_guest_debug) {
diff --git ./include/exec/translation-block.h ./include/exec/translation-block.h
index 40cc699031..ed2ce87503 100644
--- ./include/exec/translation-block.h
+++ ./include/exec/translation-block.h
@@ -158,7 +158,13 @@ static inline uint32_t tb_cflags(const TranslationBlock *tb)
return qatomic_read(&tb->cflags);
}
+/*
+ * CPUState::tcg_cflags_priv is reached only through these. The setter keeps
+ * the derived CPUState::tcg_curr_cflags in step, and assigning the field
+ * directly would silently leave that cache stale.
+ */
bool tcg_cflags_has(CPUState *cpu, uint32_t flags);
+uint32_t tcg_cflags_get(CPUState *cpu);
void tcg_cflags_set(CPUState *cpu, uint32_t flags);
static inline tb_page_addr_t tb_page_addr0(const TranslationBlock *tb)
diff --git ./include/hw/core/cpu.h ./include/hw/core/cpu.h
index 81af7b9ee1..399ac7bd57 100644
--- ./include/hw/core/cpu.h
+++ ./include/hw/core/cpu.h
@@ -411,10 +411,16 @@ struct qemu_work_item;
* to a cluster this will be UNASSIGNED_CLUSTER_INDEX; otherwise it will
* be the same as the cluster-id property of the CPU object's TYPE_CPU_CLUSTER
* QOM parent.
- * Under TCG this value is propagated to @tcg_cflags.
+ * Under TCG this value is propagated to @tcg_cflags_priv.
* See TranslationBlock::TCG CF_CLUSTER_MASK.
* @start_powered_off: Indicates whether the CPU starts in powered-off state.
- * @tcg_cflags: Pre-computed cflags for this cpu.
+ * @tcg_cflags_priv: Pre-computed cflags for this cpu. Private to
+ * tcg_cflags_has() and tcg_cflags_set(): @tcg_curr_cflags is derived from
+ * it and is refreshed by the setter, so a direct assignment here would
+ * leave the two out of step. The name is deliberately awkward to make an
+ * open-coded access fail to compile rather than silently go stale.
+ * @tcg_curr_cflags: Cached result of curr_cflags(), recomputed by
+ * tcg_update_curr_cflags() whenever any of its inputs change.
* @nr_threads: Number of threads within this CPU core.
* @thread: Host thread details, only live once @created is #true
* @sem: WIN32 only semaphore used only for qtest
@@ -557,7 +563,8 @@ struct CPUState {
/* TODO Move common fields from CPUArchState here. */
int cpu_index;
int cluster_index;
- uint32_t tcg_cflags;
+ uint32_t tcg_cflags_priv;
+ uint32_t tcg_curr_cflags;
uint32_t halted;
int32_t exception_index;
diff --git ./include/system/tcg.h ./include/system/tcg.h
index 7622dcea30..f41e6b3219 100644
--- ./include/system/tcg.h
+++ ./include/system/tcg.h
@@ -17,6 +17,15 @@ extern bool tcg_allowed;
#define tcg_enabled() 0
#endif
+/*
+ * Recompute CPUState::tcg_curr_cflags. Must be called whenever any input
+ * to the computation changes: CPUState::tcg_cflags_priv, gdb single-step
+ * state, one-insn-per-tb, or the CPU_LOG_TB_NOCHAIN log flag. The first of
+ * those is covered already, tcg_cflags_set() being the only way to change it.
+ */
+void tcg_update_curr_cflags(CPUState *cpu);
+void tcg_update_all_curr_cflags(void);
+
/**
* qemu_tcg_mttcg_enabled:
* Check whether we are running MultiThread TCG or not.
diff --git ./linux-user/main.c ./linux-user/main.c
index 60a695b7ca..ba04773398 100644
--- ./linux-user/main.c
+++ ./linux-user/main.c
@@ -242,7 +242,7 @@ CPUArchState *cpu_copy(CPUArchState *env)
/* Reset non arch specific state */
cpu_reset(new_cpu);
- new_cpu->tcg_cflags = cpu->tcg_cflags;
+ tcg_cflags_set(new_cpu, tcg_cflags_get(cpu));
memcpy(new_env, env, sizeof(CPUArchState));
#if defined(TARGET_I386) || defined(TARGET_X86_64)
new_env->gdt.base = target_mmap(0, sizeof(uint64_t) * TARGET_GDT_ENTRIES,
diff --git ./stubs/meson.build ./stubs/meson.build
index 3b2f2680b1..0025e79226 100644
--- ./stubs/meson.build
+++ ./stubs/meson.build
@@ -3,6 +3,7 @@
# below, so that it is clear who needs the stubbed functionality.
stub_ss.add(files('cpu-get-clock.c'))
+stub_ss.add(files('tcg-cflags.c'))
stub_ss.add(files('fdset.c'))
stub_ss.add(files('iothread-lock.c'))
stub_ss.add(files('is-daemonized.c'))
diff --git ./stubs/tcg-cflags.c ./stubs/tcg-cflags.c
new file mode 100644
index 0000000000..bd74fabf0e
--- /dev/null
+++ ./stubs/tcg-cflags.c
@@ -0,0 +1,16 @@
+/*
+ * Stubs for the cached cflags update hooks, for binaries that link
+ * util/log.c or cpu-target.c without linking TCG.
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+#include "qemu/osdep.h"
+#include "system/tcg.h"
+
+void tcg_update_curr_cflags(CPUState *cpu)
+{
+}
+
+void tcg_update_all_curr_cflags(void)
+{
+}
diff --git ./target/alpha/cpu.c ./target/alpha/cpu.c
index 12e8602166..7a768eae9f 100644
--- ./target/alpha/cpu.c
+++ ./target/alpha/cpu.c
@@ -114,7 +114,7 @@ static void alpha_cpu_realizefn(DeviceState *dev, Error **errp)
#ifndef CONFIG_USER_ONLY
/* Use pc-relative instructions in system-mode */
- cs->tcg_cflags |= CF_PCREL;
+ tcg_cflags_set(cs, CF_PCREL);
#endif
cpu_common_realize(cs, &local_err);
diff --git ./util/log.c ./util/log.c
index 7cffbc1bf8..62c7f09609 100644
--- ./util/log.c
+++ ./util/log.c
@@ -27,6 +27,7 @@
#include "qemu/thread.h"
#include "qemu/lockable.h"
#include "qemu/rcu.h"
+#include "system/tcg.h"
#ifdef CONFIG_LINUX
#include <sys/syscall.h>
#endif
@@ -301,6 +302,9 @@ static bool qemu_set_log_internal(const char *filename, bool changed_name,
#endif
qemu_loglevel = log_flags;
+ /* CPU_LOG_TB_NOCHAIN feeds into the per-CPU cached cflags. */
+ tcg_update_all_curr_cflags();
+
daemonized = is_daemonized();
need_to_open_file = false;
if (!daemonized) {
--
2.54.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH 2/8] accel/tcg: enlarge the TB jump cache to 64K entries
2026-08-18 17:42 [RFC PATCH 0/8] accel/tcg: cut per-block dispatch overhead Matt Turner
2026-08-18 17:42 ` [PATCH 1/8] accel/tcg: cache the result of curr_cflags() Matt Turner
@ 2026-08-18 17:42 ` Matt Turner
2026-08-18 17:42 ` [PATCH 3/8] accel/tcg: skip the can_do_io stores in user-only builds Matt Turner
` (5 subsequent siblings)
7 siblings, 0 replies; 13+ messages in thread
From: Matt Turner @ 2026-08-18 17:42 UTC (permalink / raw)
To: qemu-devel
Cc: richard.henderson, pbonzini, philmd, zhao1.liu, laurent, deller,
pierrick.bouvier, Matt Turner
The per-CPU TB jump cache has held 4096 entries since it was introduced.
That is too small for guests running large programs: an emulated compiler
misses often enough that the fallback qht lookup shows up prominently in
a profile.
Measured with qemu-alpha running an emulated alpha gcc 16.2.0 compiling
the SQLite 3.45.1 amalgamation (255k lines, -O2) on an x86-64 host. The
compile performs 34.2 billion TB executions, of which 8.4 billion take
the indirect dispatch path.
Sizing curve, on top of the preceding patch, instructions retired and
wall clock:
12 bits ( 64 KiB): 1,563,829,403,943 133.13s
14 bits ( 256 KiB): 1,493,865,985,972 -4.47% 124.89s -6.19%
16 bits ( 1 MiB): 1,469,729,281,442 -6.02% 120.97s -9.13%
18 bits ( 4 MiB): 1,462,262,363,257 -6.49% 120.16s -9.74%
16 bits is the knee. 18 buys another 0.47% of instructions for four times
the memory, and since instructions retired does not account for the data
cache pressure of a 4 MiB table, that 0.47% is probably not real: the
wall clock difference between 16 and 18 bits is 0.67%, against a
run-to-run spread of the same order.
In a perf profile the mechanism is visible directly: tb_htable_lookup(),
which is where qht_lookup_custom() lands once it is inlined in an LTO
build, falls from 5.73% of samples to 1.52%.
The cost is memory: the cache grows from 64 KiB to 1 MiB per vCPU. That
is easy to justify for a single-vCPU linux-user process and less obvious
for system emulation with many vCPUs, so this may want to be sized by
target or made tunable rather than raised unconditionally.
Signed-off-by: Matt Turner <mattst88@gmail.com>
---
accel/tcg/tb-jmp-cache.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git ./accel/tcg/tb-jmp-cache.h ./accel/tcg/tb-jmp-cache.h
index c3a505e394..268dacd7ba 100644
--- ./accel/tcg/tb-jmp-cache.h
+++ ./accel/tcg/tb-jmp-cache.h
@@ -12,7 +12,7 @@
#include "qemu/rcu.h"
#include "exec/cpu-common.h"
-#define TB_JMP_CACHE_BITS 12
+#define TB_JMP_CACHE_BITS 16
#define TB_JMP_CACHE_SIZE (1 << TB_JMP_CACHE_BITS)
/*
--
2.54.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH 3/8] accel/tcg: skip the can_do_io stores in user-only builds
2026-08-18 17:42 [RFC PATCH 0/8] accel/tcg: cut per-block dispatch overhead Matt Turner
2026-08-18 17:42 ` [PATCH 1/8] accel/tcg: cache the result of curr_cflags() Matt Turner
2026-08-18 17:42 ` [PATCH 2/8] accel/tcg: enlarge the TB jump cache to 64K entries Matt Turner
@ 2026-08-18 17:42 ` Matt Turner
2026-08-18 17:42 ` [PATCH 4/8] RFC: tcg: probe the TB jump cache inline instead of calling a helper Matt Turner
` (4 subsequent siblings)
7 siblings, 0 replies; 13+ messages in thread
From: Matt Turner @ 2026-08-18 17:42 UTC (permalink / raw)
To: qemu-devel
Cc: richard.henderson, pbonzini, philmd, zhao1.liu, laurent, deller,
pierrick.bouvier, Matt Turner
Every translation block stores to cpu->neg.can_do_io twice: false before
the first instruction, true before the last one. Nothing reads it in a
user-only build. There is no memory-mapped I/O in linux-user, and every
reader is in system_ss: cputlb.c, watchpoint.c, icount-common.c and
tcg-accel-ops-icount.c.
Two stores per TB is not much on its own, but TBs are short. An emulated
alpha gcc 16.2.0 compiling the SQLite 3.45.1 amalgamation (255k lines,
-O2) executes 34.2 billion TBs at 6.04 guest instructions each, so this is
68 billion stores for nothing.
Measured on an x86-64 host, LTO build, on top of the preceding two
patches:
before: 1,469,729,281,442 instructions
after: 1,402,816,253,499 instructions -4.55%
before: 120.97s wall clock
after: 115.75s wall clock -4.32%
The emulated compiler produces byte-identical output.
Signed-off-by: Matt Turner <mattst88@gmail.com>
---
accel/tcg/translator.c | 7 +++++++
1 file changed, 7 insertions(+)
diff --git ./accel/tcg/translator.c ./accel/tcg/translator.c
index cd7d079fe0..85bb21e911 100644
--- ./accel/tcg/translator.c
+++ ./accel/tcg/translator.c
@@ -23,6 +23,9 @@
static void set_can_do_io(DisasContextBase *db, bool val)
{
+ if (IS_ENABLED(CONFIG_USER_ONLY)) {
+ return;
+ }
QEMU_BUILD_BUG_ON(sizeof_field(CPUState, neg.can_do_io) != 1);
tcg_gen_st8_i32(tcg_constant_i32(val), tcg_env,
offsetof(CPUState, neg.can_do_io) - sizeof(CPUState));
@@ -210,6 +213,10 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int *max_insns,
/*
* Manage can_do_io for the translation block: set to false before
* the first insn and set to true before the last insn.
+ *
+ * Nothing reads can_do_io in user-only builds. There is no MMIO
+ * there, and every reader (cputlb.c, watchpoint.c, icount) is in
+ * system_ss, so skip the two stores per TB entirely.
*/
if (db->num_insns == 1) {
tcg_debug_assert(first_insn_start == db->insn_start);
--
2.54.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH 4/8] RFC: tcg: probe the TB jump cache inline instead of calling a helper
2026-08-18 17:42 [RFC PATCH 0/8] accel/tcg: cut per-block dispatch overhead Matt Turner
` (2 preceding siblings ...)
2026-08-18 17:42 ` [PATCH 3/8] accel/tcg: skip the can_do_io stores in user-only builds Matt Turner
@ 2026-08-18 17:42 ` Matt Turner
2026-08-18 17:42 ` [PATCH 5/8] RFC: accel/tcg: allow cross-page goto_tb chaining in user-only builds Matt Turner
` (3 subsequent siblings)
7 siblings, 0 replies; 13+ messages in thread
From: Matt Turner @ 2026-08-18 17:42 UTC (permalink / raw)
To: qemu-devel
Cc: richard.henderson, pbonzini, philmd, zhao1.liu, laurent, deller,
pierrick.bouvier, Matt Turner
Every indirect branch that cannot use goto_tb ends in
tcg_gen_lookup_and_goto_ptr(), which calls helper_lookup_tb_ptr(). For an
emulated compiler that is 8.4 billion helper calls in a single translation
unit: 24.6% of all TB exits take this path, because jsr/ret/jmp have a
register destination and because goto_tb is restricted to same-page
targets.
The helper itself is already tight, but each call pays for a call frame,
the can_do_io store, the get_tb_cpu_state() indirect call through
TCGCPUOps, curr_cflags(), and a breakpoint check, before it gets to the
jump cache probe that almost always hits (95.8% for this workload).
Emit the probe inline instead. The destination PC is already in a TCG
temp, and the flags and cflags the destination must match are constants at
translation time, so the fast path is a hash, three guarded loads and a
goto_ptr. Only a miss calls the helper, which still owns filling the cache.
Two details matter for the generated code. The flags and cflags guards are
folded into a single aligned 64-bit load and compare, since the fields are
adjacent. And each path emits its own goto_ptr rather than branching to a
shared one: a temp live across the label is spilled and reloaded on every
dispatch, which cost 6.3% on its own.
Measured with qemu-alpha running an emulated alpha gcc 16.2.0 compiling
the SQLite 3.45.1 amalgamation (255k lines, -O2) on an x86-64 host, LTO
build, on top of the preceding three patches:
before: 1,402,816,253,499 instructions
after: 890,713,633,237 instructions -36.51%
before: 115.75s wall clock
after: 84.44s wall clock -27.05%
The gap between the two is the point at which this stops being a
straight-line win: the helper call was highly predictable work that the
host pipelined well, so removing it retires far fewer instructions than it
saves time. IPC falls from 2.48 to 2.15 across this patch for that reason.
Despite emitting more code, this also reduces instruction cache pressure,
because a dispatch no longer jumps into qemu's .text and evicts translated
code:
before: 11,476,318,964 L1-icache-load-misses
after: 6,990,186,701 L1-icache-load-misses -39.1%
The mechanism is visible directly in a profile: helper_lookup_tb_ptr()
falls from 30.97% of samples to 0.42%, and qemu's own .text falls from
38.9% to 5.4%, with the balance moving into generated code.
Combined with the three preceding patches, against an unmodified LTO
build, 1,647,901,588,726 instructions fall to 890,713,633,237, or -45.95%.
The emulated compiler produces byte-identical output throughout.
Open issues, hence RFC:
- The flags/cflags guards use the *current* TB's values as constants. That
assumes the CPU flags feeding get_tb_cpu_state() cannot change within a
TB, and that curr_cflags() cannot change under a running TB (gdb
attaching to enable single-step would). Both need to be established or
the values need to be loaded at runtime.
- tcg/tcg-op.c has no business including accel/tcg/tb-jmp-cache.h or
knowing the CPUJumpCache layout. The probe likely belongs in accel/tcg
with a small emit helper exported from tcg/.
- The jump cache entry is read without qatomic_read(); entries are
invalidated concurrently by setting tb to NULL.
- Only wired up for alpha so far, and only for 64-bit guest PCs.
Signed-off-by: Matt Turner <mattst88@gmail.com>
---
include/tcg/tcg-op-common.h | 2 +
target/alpha/translate.c | 4 +-
tcg/tcg-op.c | 79 +++++++++++++++++++++++++++++++++++++
3 files changed, 83 insertions(+), 2 deletions(-)
diff --git ./include/tcg/tcg-op-common.h ./include/tcg/tcg-op-common.h
index 1fe342db0d..65084c38b9 100644
--- ./include/tcg/tcg-op-common.h
+++ ./include/tcg/tcg-op-common.h
@@ -84,6 +84,8 @@ void tcg_gen_goto_tb(unsigned idx);
* this op is equivalent to calling tcg_gen_exit_tb() with 0 as the argument.
*/
void tcg_gen_lookup_and_goto_ptr(void);
+void tcg_gen_lookup_and_goto_ptr_inline(TCGv_i64 pc,
+ const TranslationBlock *tb);
void tcg_gen_plugin_cb(unsigned from);
void tcg_gen_plugin_mem_cb(TCGv_i64 addr, unsigned meminfo);
diff --git ./target/alpha/translate.c ./target/alpha/translate.c
index c66e3f9c14..939170a28b 100644
--- ./target/alpha/translate.c
+++ ./target/alpha/translate.c
@@ -449,7 +449,7 @@ static void gen_goto_tb(DisasContext *ctx, unsigned tb_slot_idx, int32_t disp)
tcg_gen_exit_tb(ctx->base.tb, tb_slot_idx);
} else {
gen_pc_disp(ctx, cpu_pc, disp);
- tcg_gen_lookup_and_goto_ptr();
+ tcg_gen_lookup_and_goto_ptr_inline(cpu_pc, ctx->base.tb);
}
}
@@ -2917,7 +2917,7 @@ static void alpha_tr_tb_stop(DisasContextBase *dcbase, CPUState *cpu)
gen_pc_disp(ctx, cpu_pc, 0);
/* FALLTHRU */
case DISAS_PC_UPDATED:
- tcg_gen_lookup_and_goto_ptr();
+ tcg_gen_lookup_and_goto_ptr_inline(cpu_pc, ctx->base.tb);
break;
case DISAS_PC_UPDATED_NOCHAIN:
tcg_gen_exit_tb(NULL, 0);
diff --git ./tcg/tcg-op.c ./tcg/tcg-op.c
index bbcb510c76..ab8d101871 100644
--- ./tcg/tcg-op.c
+++ ./tcg/tcg-op.c
@@ -28,6 +28,8 @@
#include "tcg/tcg-op-common.h"
#include "exec/translation-block.h"
#include "exec/plugin-gen.h"
+#include "hw/core/cpu.h"
+#include "../accel/tcg/tb-jmp-cache.h"
#include "tcg-internal.h"
#include "tcg-has.h"
@@ -2620,3 +2622,80 @@ void tcg_gen_lookup_and_goto_ptr(void)
tcg_gen_op1i(INDEX_op_goto_ptr, TCG_TYPE_PTR, tcgv_ptr_arg(ptr));
tcg_temp_free_ptr(ptr);
}
+
+/*
+ * As tcg_gen_lookup_and_goto_ptr(), but probe the TB jump cache inline
+ * instead of calling helper_lookup_tb_ptr() unconditionally. @pc must
+ * hold the destination guest PC; @flags and @cflags are the values the
+ * destination TB must have been translated with.
+ */
+void tcg_gen_lookup_and_goto_ptr_inline(TCGv_i64 pc,
+ const TranslationBlock *tb)
+{
+ uint32_t flags = tb->flags;
+ uint32_t cflags = tb->cflags;
+ TCGv_ptr jc, ent, tbp, ptr;
+ TCGv_i64 h, tmp;
+ TCGLabel *slow;
+ uint64_t fpair;
+
+ if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) {
+ tcg_gen_exit_tb(NULL, 0);
+ return;
+ }
+
+ plugin_gen_disable_mem_helpers();
+
+ QEMU_BUILD_BUG_ON(sizeof(((CPUJumpCache *)0)->array[0]) != 16);
+ QEMU_BUILD_BUG_ON(offsetof(TranslationBlock, cflags) !=
+ offsetof(TranslationBlock, flags) + 4);
+
+ jc = tcg_temp_ebb_new_ptr();
+ ent = tcg_temp_ebb_new_ptr();
+ tbp = tcg_temp_ebb_new_ptr();
+ ptr = tcg_temp_ebb_new_ptr();
+ h = tcg_temp_ebb_new_i64();
+ tmp = tcg_temp_ebb_new_i64();
+ slow = gen_new_label();
+
+ /* h = tb_jmp_cache_hash_func(pc) * sizeof(array[0]) */
+ tcg_gen_shri_i64(h, pc, TB_JMP_CACHE_BITS);
+ tcg_gen_xor_i64(h, h, pc);
+ tcg_gen_andi_i64(h, h, TB_JMP_CACHE_SIZE - 1);
+ tcg_gen_shli_i64(h, h, 4);
+
+ tcg_gen_ld_ptr(jc, tcg_env,
+ offsetof(CPUState, tb_jmp_cache) - sizeof(CPUState));
+ tcg_gen_trunc_i64_ptr(ent, h);
+ tcg_gen_add_ptr(ent, jc, ent);
+
+ tcg_gen_ld_ptr(tbp, ent, offsetof(CPUJumpCache, array[0].tb));
+ tcg_gen_brcondi_ptr(TCG_COND_EQ, tbp, 0, slow);
+
+ tcg_gen_ld_i64(tmp, ent, offsetof(CPUJumpCache, array[0].pc));
+ tcg_gen_brcond_i64(TCG_COND_NE, tmp, pc, slow);
+
+ /*
+ * flags and cflags are adjacent uint32_t, so one aligned 64-bit load
+ * and compare covers both.
+ */
+#if HOST_BIG_ENDIAN
+ fpair = ((uint64_t)flags << 32) | cflags;
+#else
+ fpair = ((uint64_t)cflags << 32) | flags;
+#endif
+ tcg_gen_ld_i64(tmp, tbp, offsetof(TranslationBlock, flags));
+ tcg_gen_brcondi_i64(TCG_COND_NE, tmp, fpair, slow);
+
+ tcg_gen_ld_ptr(ptr, tbp, offsetof(TranslationBlock, tc.ptr));
+ tcg_gen_op1i(INDEX_op_goto_ptr, TCG_TYPE_PTR, tcgv_ptr_arg(ptr));
+
+ /*
+ * Emit a second goto_ptr rather than branching to a shared one: a temp
+ * live across the label would be spilled and reloaded on every dispatch.
+ */
+ gen_set_label(slow);
+ ptr = tcg_temp_ebb_new_ptr();
+ gen_helper_lookup_tb_ptr(ptr, tcg_env);
+ tcg_gen_op1i(INDEX_op_goto_ptr, TCG_TYPE_PTR, tcgv_ptr_arg(ptr));
+}
--
2.54.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH 5/8] RFC: accel/tcg: allow cross-page goto_tb chaining in user-only builds
2026-08-18 17:42 [RFC PATCH 0/8] accel/tcg: cut per-block dispatch overhead Matt Turner
` (3 preceding siblings ...)
2026-08-18 17:42 ` [PATCH 4/8] RFC: tcg: probe the TB jump cache inline instead of calling a helper Matt Turner
@ 2026-08-18 17:42 ` Matt Turner
2026-08-18 17:42 ` [PATCH 6/8] RFC: accel/tcg: only poll for interrupts in blocks that can close a cycle Matt Turner
` (2 subsequent siblings)
7 siblings, 0 replies; 13+ messages in thread
From: Matt Turner @ 2026-08-18 17:42 UTC (permalink / raw)
To: qemu-devel
Cc: richard.henderson, pbonzini, philmd, zhao1.liu, laurent, deller,
pierrick.bouvier, Matt Turner
translator_use_goto_tb() refuses to chain unless the destination is on the
same page as the start of the TB. For guests whose text is much larger than
a page this is expensive: an emulated alpha gcc compiling a 255k line
translation unit takes the indirect dispatch path for 8.4 billion of its
34.2 billion TB exits, and a large share of those are ordinary direct
branches that simply crossed an 8 KiB page boundary.
The restriction was made unconditional by d3a2a1d803 ("accel/tcg:
Introduce translator_use_goto_tb"), whose rationale was:
Various targets avoid the page crossing test for CONFIG_USER_ONLY,
but that is wrong: mmap and mprotect can change page permissions.
That is true, but in user-only builds the invalidation path already covers
it. There are no page tables: every mmap, mprotect and munmap reaches
page_set_flags(), which calls tb_invalidate_phys_range() whenever the flags
actually change, and tb_phys_invalidate() calls tb_jmp_unlink() to reset
incoming jumps. A chained cross-page jump is therefore broken whenever the
destination page's permissions change. This is not true in system mode,
where TBs are keyed by physical address and a page table change invalidates
nothing, so the restriction is kept there.
Add tests/tcg/alpha/test-xpage-chain.c to cover the hazard directly. It
places a direct branch near the end of one page targeting the next page,
runs it 200000 times so the chain is established, then checks that
mprotect(PROT_NONE) makes the next call fault, and that remapping the page
with different code runs the new code rather than a stale translation.
The test detects the hazard it is meant to detect: with the
tb_invalidate_phys_range() call in page_set_flags() commented out, it fails
both phases, executing page B after PROT_NONE and returning the stale
result.
Measured with qemu-alpha running an emulated alpha gcc 16.2.0 compiling the
SQLite 3.45.1 amalgamation on an x86-64 host, LTO build, on top of the
preceding patches:
before: 890,713,633,237 instructions
after: 869,178,598,378 instructions -2.42%
before: 84.44s wall clock
after: 80.49s wall clock -4.68%
Note that this is worth more in time than in instructions, the reverse of
the preceding patch: a chained jump replaces a cache probe whose loads can
miss, so the instructions it removes are more expensive than average.
Measured before the inline jump cache probe, when a missed chain cost a
helper call rather than an inline probe, the same change was worth -7.9%.
RFC because this reverses a deliberate decision and the reasoning above
wants review from someone who knows the invalidation paths better than I
do.
Signed-off-by: Matt Turner <mattst88@gmail.com>
---
accel/tcg/translator.c | 12 +++-
tests/tcg/alpha/Makefile.target | 2 +-
tests/tcg/alpha/test-xpage-chain.c | 111 +++++++++++++++++++++++++++++
3 files changed, 123 insertions(+), 2 deletions(-)
create mode 100644 tests/tcg/alpha/test-xpage-chain.c
diff --git ./accel/tcg/translator.c ./accel/tcg/translator.c
index 85bb21e911..3eab9f570d 100644
--- ./accel/tcg/translator.c
+++ ./accel/tcg/translator.c
@@ -108,6 +108,17 @@ static void gen_tb_end(const TranslationBlock *tb, uint32_t cflags,
bool translator_is_same_page(const DisasContextBase *db, vaddr addr)
{
+ /*
+ * In user-only mode there are no page tables. Every mmap, mprotect and
+ * munmap goes through page_set_flags(), which calls
+ * tb_invalidate_phys_range() whenever the flags actually change, and
+ * tb_phys_invalidate() unlinks incoming jumps. A cross-page link is
+ * therefore broken whenever the destination page's permissions change,
+ * so the same-page restriction is not needed.
+ */
+ if (IS_ENABLED(CONFIG_USER_ONLY)) {
+ return true;
+ }
return ((addr ^ db->pc_first) & TARGET_PAGE_MASK) == 0;
}
@@ -118,7 +129,6 @@ bool translator_use_goto_tb(DisasContextBase *db, vaddr dest)
return false;
}
- /* Check for the dest on the same page as the start of the TB. */
return translator_is_same_page(db, dest);
}
diff --git ./tests/tcg/alpha/Makefile.target ./tests/tcg/alpha/Makefile.target
index 36d8ed1eae..eee986bab6 100644
--- ./tests/tcg/alpha/Makefile.target
+++ ./tests/tcg/alpha/Makefile.target
@@ -5,7 +5,7 @@
ALPHA_SRC=$(SRC_PATH)/tests/tcg/alpha
VPATH+=$(ALPHA_SRC)
-ALPHA_TESTS=hello-alpha test-cond test-cmov test-ovf test-cvttq
+ALPHA_TESTS=hello-alpha test-cond test-cmov test-ovf test-cvttq test-xpage-chain
TESTS+=$(ALPHA_TESTS)
test-cmov: EXTRA_CFLAGS=-DTEST_CMOV
diff --git ./tests/tcg/alpha/test-xpage-chain.c ./tests/tcg/alpha/test-xpage-chain.c
new file mode 100644
index 0000000000..7916d544af
--- /dev/null
+++ ./tests/tcg/alpha/test-xpage-chain.c
@@ -0,0 +1,111 @@
+/*
+ * Cross-page TB chaining hazard test.
+ *
+ * Phase 1: a direct branch (br) near the end of page A targets page B.
+ * Run it enough times that QEMU chains TB_A -> TB_B.
+ * Phase 2: mprotect page B away. Re-running must fault.
+ * Phase 3: remap page B with different code. Re-running must execute the
+ * NEW code, not a stale chained translation of the old code.
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <setjmp.h>
+#include <signal.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#define PS 8192
+
+static sigjmp_buf jb;
+/*
+ * Written by the SIGSEGV handler and read by main(), so it must not be
+ * cached in a register across the faulting call.
+ */
+static volatile sig_atomic_t caught;
+
+static void segv(int sig)
+{
+ caught = 1;
+ siglongjmp(jb, 1);
+}
+
+/* lda $0, imm($31) -> v0 = imm */
+static unsigned int lda_v0(int imm)
+{
+ return 0x201F0000u | (unsigned short)imm;
+}
+
+int main(void)
+{
+ struct sigaction sa;
+ int rc = 0;
+ unsigned char *m = mmap(NULL, 2 * PS, PROT_READ | PROT_WRITE | PROT_EXEC,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (m == MAP_FAILED) {
+ perror("mmap");
+ return 2;
+ }
+
+ unsigned char *pa = m, *pb = m + PS;
+ unsigned int *entry = (unsigned int *)(pa + PS - 64);
+ unsigned int *tgt = (unsigned int *)(pb + 16);
+
+ entry[0] = lda_v0(1);
+ long disp = ((long)tgt - ((long)&entry[1] + 4)) / 4;
+ entry[1] = 0xC3E00000u | (unsigned int)(disp & 0x1FFFFF); /* br $31,tgt */
+ tgt[0] = 0x6BFA8001u; /* ret */
+ __builtin___clear_cache((char *)m, (char *)m + 2 * PS);
+
+ long (*fn)(void) = (long (*)(void))entry;
+
+ for (int i = 0; i < 200000; i++) {
+ if (fn() != 1) {
+ printf("FAIL: phase 1 wrong result\n");
+ return 1;
+ }
+ }
+ printf("phase 1 ok (chained)\n");
+
+ memset(&sa, 0, sizeof(sa));
+ sa.sa_handler = segv;
+ sigemptyset(&sa.sa_mask);
+ if (sigaction(SIGSEGV, &sa, NULL) != 0) {
+ perror("sigaction");
+ return 2;
+ }
+ if (mprotect(pb, PS, PROT_NONE) != 0) {
+ perror("mprotect");
+ return 2;
+ }
+ if (sigsetjmp(jb, 1) == 0) {
+ fn();
+ printf("FAIL: phase 2 executed page B after mprotect(PROT_NONE)\n");
+ rc = 1;
+ } else if (!caught) {
+ printf("FAIL: phase 2 longjmp without entering the handler\n");
+ rc = 1;
+ } else {
+ printf("phase 2 ok (faulted)\n");
+ }
+
+ /* Phase 3: remap with different code, expect the new code to run. */
+ if (mprotect(pb, PS, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) {
+ perror("mprotect back");
+ return 2;
+ }
+ tgt[0] = lda_v0(2);
+ tgt[1] = 0x6BFA8001u;
+ __builtin___clear_cache((char *)pb, (char *)pb + PS);
+
+ long r = fn();
+ if (r != 2) {
+ printf("FAIL: phase 3 returned %ld, expected 2 (stale chain)\n", r);
+ rc = 1;
+ } else {
+ printf("phase 3 ok (new code ran)\n");
+ }
+ return rc;
+}
--
2.54.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH 6/8] RFC: accel/tcg: only poll for interrupts in blocks that can close a cycle
2026-08-18 17:42 [RFC PATCH 0/8] accel/tcg: cut per-block dispatch overhead Matt Turner
` (4 preceding siblings ...)
2026-08-18 17:42 ` [PATCH 5/8] RFC: accel/tcg: allow cross-page goto_tb chaining in user-only builds Matt Turner
@ 2026-08-18 17:42 ` Matt Turner
2026-08-18 17:42 ` [PATCH 7/8] RFC: accel/tcg: poison the jump cache instead of polling for indirect exits Matt Turner
2026-08-18 17:42 ` [PATCH 8/8] RFC: tcg: fold a guest displacement into the host addressing mode Matt Turner
7 siblings, 0 replies; 13+ messages in thread
From: Matt Turner @ 2026-08-18 17:42 UTC (permalink / raw)
To: qemu-devel
Cc: richard.henderson, pbonzini, philmd, zhao1.liu, laurent, deller,
pierrick.bouvier, Matt Turner
Every translation block begins by loading cpu->neg.icount_decr.u32,
testing it and branching to the exit path. That is three host instructions
at the top of every TB. Blocks are short, so this is expensive: an emulated
alpha gcc 16.2.0 compiling a 255k line translation unit executes 34.2
billion TBs at 6.04 guest instructions each. Forcing CF_NOIRQ on for the
whole run, which is not correct but bounds the prize, is worth 12.0% of all
instructions retired.
The check does not have to be in every block. Interrupt latency is bounded
as long as every cycle in the guest control flow graph passes through at
least one block that polls. Any such cycle must contain either an edge
whose destination is at or below the start of the block it leaves from, or
an edge whose destination is not known at translation time: take the block
with the lowest start address in the cycle, and the edge entering it comes
from a block at or above it.
So record, during translation, whether this TB has such an edge.
translator_use_goto_tb() already sees every statically known destination,
and every target that emits goto_tb reaches it, so a backward edge sets
DisasContextBase::needs_exit_check there. Indirect destinations are flagged
by tcg_gen_lookup_and_goto_ptr(). Blocks with neither cannot close a cycle
on their own and can skip the poll.
The check is therefore emitted retroactively in gen_tb_end(), using the
same emit_before_op mechanism the can_do_io stores use, and only when one
of the two flags is set. icount opts out and keeps the unconditional
counter.
Measured on an x86-64 host, LTO build, on top of the preceding patches:
before: 869,178,598,378 instructions
after: 809,988,851,304 instructions -6.81%
before: 80.49s wall clock
after: 78.00s wall clock -3.10%
That is 57% of the 12.0% ceiling, which is about right: roughly a quarter of
TB exits are indirect and are still polled, plus every loop back edge.
For the series as a whole, against an unmodified LTO build, instructions
retired fall from 1,647,901,588,726 to 809,988,851,304 (-50.85%) and wall
clock from 133.57s to 78.00s (-41.61%). The two do not match because what
the series removes is mostly cheap, well-predicted dispatch overhead: IPC
falls from 2.53 to 2.12 as the remaining work gets less regular.
tests/tcg/alpha/test-xpage-chain.c still passes, the emulated compiler still
produces byte-identical output, and a tight loop under alarm(1) is still
interrupted, after 897 million iterations.
RFC because:
- The soundness argument depends on every goto_tb destination passing
through translator_use_goto_tb(). No target in the tree bypasses it
today, but nothing enforces that.
- System mode interrupt latency now depends on guest control flow rather
than on block count. The bound is one straight-line run between cycles,
which should be fine, but timer-driven guests deserve a closer look than
I can give them.
Signed-off-by: Matt Turner <mattst88@gmail.com>
---
accel/tcg/translator.c | 57 ++++++++++++++++++++++++++++++++++++---
include/exec/translator.h | 2 ++
include/tcg/tcg.h | 2 ++
tcg/tcg-op.c | 2 ++
4 files changed, 60 insertions(+), 3 deletions(-)
diff --git ./accel/tcg/translator.c ./accel/tcg/translator.c
index 3eab9f570d..048ad8bad2 100644
--- ./accel/tcg/translator.c
+++ ./accel/tcg/translator.c
@@ -43,12 +43,29 @@ bool translator_io_start(DisasContextBase *db)
return true;
}
+/*
+ * Any cycle in the guest control flow graph must contain an edge whose
+ * destination is at or below the start of the block it leaves from, or an
+ * edge whose destination is not known at translation time. Only blocks with
+ * such an edge need the interrupt check, so defer the decision until the end
+ * of translation, when we know which edges this TB has.
+ *
+ * icount needs the counter unconditionally, so it opts out.
+ */
+static bool defer_exit_check(uint32_t cflags)
+{
+ return !(cflags & CF_USE_ICOUNT);
+}
+
static TCGOp *gen_tb_start(DisasContextBase *db, uint32_t cflags)
{
TCGv_i32 count = NULL;
TCGOp *icount_start_insn = NULL;
- if ((cflags & CF_USE_ICOUNT) || !(cflags & CF_NOIRQ)) {
+ tcg_ctx->exit_check_needed = false;
+
+ if ((cflags & CF_USE_ICOUNT) ||
+ (!(cflags & CF_NOIRQ) && !defer_exit_check(cflags))) {
count = tcg_temp_new_i32();
tcg_gen_ld_i32(count, tcg_env,
offsetof(CPUState, neg.icount_decr.u32) -
@@ -74,6 +91,12 @@ static TCGOp *gen_tb_start(DisasContextBase *db, uint32_t cflags)
*/
if (cflags & CF_NOIRQ) {
tcg_ctx->exitreq_label = NULL;
+ } else if (defer_exit_check(cflags)) {
+ /*
+ * Emitted retroactively by gen_tb_end(), but only if this TB can be
+ * part of a control flow cycle.
+ */
+ tcg_ctx->exitreq_label = gen_new_label();
} else {
tcg_ctx->exitreq_label = gen_new_label();
tcg_gen_brcondi_i32(TCG_COND_LT, count, 0, tcg_ctx->exitreq_label);
@@ -89,7 +112,8 @@ static TCGOp *gen_tb_start(DisasContextBase *db, uint32_t cflags)
}
static void gen_tb_end(const TranslationBlock *tb, uint32_t cflags,
- TCGOp *icount_start_insn, int num_insns)
+ TCGOp *icount_start_insn, int num_insns,
+ DisasContextBase *db, TCGOp *first_insn_start)
{
if (cflags & CF_USE_ICOUNT) {
/*
@@ -100,6 +124,23 @@ static void gen_tb_end(const TranslationBlock *tb, uint32_t cflags,
tcgv_i32_arg(tcg_constant_i32(num_insns)));
}
+ if (tcg_ctx->exitreq_label && defer_exit_check(cflags) &&
+ !(cflags & CF_NOIRQ)) {
+ if (db->needs_exit_check || tcg_ctx->exit_check_needed) {
+ TCGv_i32 count = tcg_temp_new_i32();
+ TCGOp *save = tcg_ctx->emit_before_op;
+
+ tcg_ctx->emit_before_op = first_insn_start;
+ tcg_gen_ld_i32(count, tcg_env,
+ offsetof(CPUState, neg.icount_decr.u32) -
+ sizeof(CPUState));
+ tcg_gen_brcondi_i32(TCG_COND_LT, count, 0, tcg_ctx->exitreq_label);
+ tcg_ctx->emit_before_op = save;
+ } else {
+ tcg_ctx->exitreq_label = NULL;
+ }
+ }
+
if (tcg_ctx->exitreq_label) {
gen_set_label(tcg_ctx->exitreq_label);
tcg_gen_exit_tb(tb, TB_EXIT_REQUESTED);
@@ -129,6 +170,14 @@ bool translator_use_goto_tb(DisasContextBase *db, vaddr dest)
return false;
}
+ /*
+ * A destination at or below the start of this TB can close a cycle, so
+ * this TB must poll for interrupts. See defer_exit_check().
+ */
+ if (dest <= db->pc_first) {
+ db->needs_exit_check = true;
+ }
+
return translator_is_same_page(db, dest);
}
@@ -152,6 +201,7 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int *max_insns,
db->max_insns = *max_insns;
db->insn_start = NULL;
db->fake_insn = false;
+ db->needs_exit_check = false;
db->host_addr[0] = host_pc;
db->host_addr[1] = NULL;
db->record_start = 0;
@@ -218,7 +268,8 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int *max_insns,
/* Emit code to exit the TB, as indicated by db->is_jmp. */
ops->tb_stop(db, cpu);
- gen_tb_end(tb, cflags, icount_start_insn, db->num_insns);
+ gen_tb_end(tb, cflags, icount_start_insn, db->num_insns, db,
+ first_insn_start);
/*
* Manage can_do_io for the translation block: set to false before
diff --git ./include/exec/translator.h ./include/exec/translator.h
index 978dee25ad..003926c7f0 100644
--- ./include/exec/translator.h
+++ ./include/exec/translator.h
@@ -74,6 +74,8 @@ struct DisasContextBase {
int max_insns;
bool plugin_enabled;
bool fake_insn;
+ /* Set when this TB can be part of a control flow cycle. */
+ bool needs_exit_check;
uint8_t code_mmuidx;
struct TCGOp *insn_start;
void *host_addr[2];
diff --git ./include/tcg/tcg.h ./include/tcg/tcg.h
index 7669dc1c2d..be9ce7a0e2 100644
--- ./include/tcg/tcg.h
+++ ./include/tcg/tcg.h
@@ -389,6 +389,8 @@ struct TCGContext {
struct TCGLabelPoolData *pool_labels;
TCGLabel *exitreq_label;
+ /* Set by goto_ptr emission: destination is not known statically. */
+ bool exit_check_needed;
#ifdef CONFIG_PLUGIN
/*
diff --git ./tcg/tcg-op.c ./tcg/tcg-op.c
index ab8d101871..8282afa61a 100644
--- ./tcg/tcg-op.c
+++ ./tcg/tcg-op.c
@@ -2616,6 +2616,7 @@ void tcg_gen_lookup_and_goto_ptr(void)
return;
}
+ tcg_ctx->exit_check_needed = true;
plugin_gen_disable_mem_helpers();
ptr = tcg_temp_ebb_new_ptr();
gen_helper_lookup_tb_ptr(ptr, tcg_env);
@@ -2644,6 +2645,7 @@ void tcg_gen_lookup_and_goto_ptr_inline(TCGv_i64 pc,
return;
}
+ tcg_ctx->exit_check_needed = true;
plugin_gen_disable_mem_helpers();
QEMU_BUILD_BUG_ON(sizeof(((CPUJumpCache *)0)->array[0]) != 16);
--
2.54.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH 7/8] RFC: accel/tcg: poison the jump cache instead of polling for indirect exits
2026-08-18 17:42 [RFC PATCH 0/8] accel/tcg: cut per-block dispatch overhead Matt Turner
` (5 preceding siblings ...)
2026-08-18 17:42 ` [PATCH 6/8] RFC: accel/tcg: only poll for interrupts in blocks that can close a cycle Matt Turner
@ 2026-08-18 17:42 ` Matt Turner
2026-08-18 17:42 ` [PATCH 8/8] RFC: tcg: fold a guest displacement into the host addressing mode Matt Turner
7 siblings, 0 replies; 13+ messages in thread
From: Matt Turner @ 2026-08-18 17:42 UTC (permalink / raw)
To: qemu-devel
Cc: richard.henderson, pbonzini, philmd, zhao1.liu, laurent, deller,
pierrick.bouvier, Matt Turner
f799aeecff dropped the icount_decr poll from blocks that cannot close a
control flow cycle. What is left is dominated by blocks that end in an
indirect branch: tcg_gen_lookup_and_goto_ptr{,_inline}() set
exit_check_needed, because the destination is unknown at translation time
and so the block might be part of a cycle. For the emulated compiler that
is still 55.5% of translated blocks and 51.2% of generated-code cycles, and
the two leading instructions of those blocks
mov -0x10(%rbp),%ebx
test %ebx,%ebx
carry 5.01% of all cycles spent in generated code, measured with cycles:pp
so that the figure is not just skid from the dispatch that jumped there.
A block dispatching indirectly does not need to poll, because the dispatch
itself can be made to notice. The out-of-line path already calls
helper_lookup_tb_ptr() every time, so it only needs the helper to return the
epilogue while an exit is pending. The inline probe added by 14c3e5a1f3 is
the interesting case: it already loads the jump cache base from
CPUState and already branches to the slow path when the entry it finds has
a NULL tb. Pointing that base at a page of zeroes therefore turns every
indirect dispatch into a miss, and a miss lands in the same helper. The poll
becomes a pointer swap on the exit request path and costs the fast path
nothing.
So give the probe its own base pointer, tb_jmp_cache_probe, that nothing
else reads. The two places that set icount_decr.u16.high poison it; the
place that clears the flag restores it. The real tb_jmp_cache is untouched
throughout, so no cache contents are lost and the recovery is one store.
Blocks with a backward goto_tb edge still poll. Interrupt latency is
unchanged in kind: an exit is noticed at the next cycle-closing edge, which
is now either a poll or a dispatch, rather than only a poll.
Measured on an x86-64 host, LTO build, on top of the preceding patches. The
control was measured in the same session, because the host's all-core turbo
varies by ~3% between sessions and swamps the effect otherwise:
before: 810,079,619,264 instructions, 79.563s
after: 787,483,360,681 instructions, 78.019s
-2.79% instructions, -1.94% wall
Forcing the check off entirely, which is incorrect but is the ceiling, gives
-5.64% instructions and -2.55% wall. So this takes half the instructions and
three quarters of the time: what it removes sits directly after an indirect
branch, where the poll's dependent load was the most expensive place a poll
could be.
In a jitdump profile of the same workload, blocks opening with the poll fall
from 55.5% to 25.9%, cycles in blocks that poll from 51.2% to 18.5%, and
cycles on the two poll instructions from 5.01% to 1.50%.
tests/tcg/alpha/test-indirect-irq.c is added for this: a loop whose only back
edge is an indirect branch, under alarm(1). It passes before and after, and
hangs if the check is simply deleted, which is what makes it a test of the
new mechanism rather than of the old poll. The other alpha tests still pass
and the emulated compiler still produces byte-identical output.
RFC because:
- The restore in cpu_handle_interrupt() races a concurrent poison from
another thread. The existing barrier around icount_decr.u16.high covers
it -- a poison that lands after the restore also re-set the flag, and
exit_request was stored before it -- but this deserves more eyes than the
single-threaded user-mode testing I have given it.
- Only the inline probe needs the poison, and only alpha uses the inline
probe today. Targets on the out-of-line path are covered by the helper
check alone, but that has not been measured.
- The shared zero-filled CPUJumpCache is a 1MB allocation that is never
written. A read-only mapping would express that better.
Signed-off-by: Matt Turner <mattst88@gmail.com>
---
accel/tcg/cpu-exec.c | 54 +++++++++++++++++++++++++++++
accel/tcg/internal-common.h | 3 ++
accel/tcg/tcg-accel-ops.c | 2 ++
accel/tcg/translator.c | 4 +--
include/hw/core/cpu.h | 10 ++++++
include/tcg/tcg.h | 2 --
tcg/tcg-op.c | 13 +++++--
tests/tcg/alpha/Makefile.target | 3 +-
tests/tcg/alpha/test-indirect-irq.c | 53 ++++++++++++++++++++++++++++
9 files changed, 135 insertions(+), 9 deletions(-)
create mode 100644 tests/tcg/alpha/test-indirect-irq.c
diff --git ./accel/tcg/cpu-exec.c ./accel/tcg/cpu-exec.c
index 257211235d..af466ca14e 100644
--- ./accel/tcg/cpu-exec.c
+++ ./accel/tcg/cpu-exec.c
@@ -388,6 +388,16 @@ const void *HELPER(lookup_tb_ptr)(CPUArchState *env)
*/
cpu->neg.can_do_io = true;
+ /*
+ * A block that dispatches indirectly does not emit the icount_decr poll,
+ * so this is where a pending exit is noticed for that path: either the
+ * probe was poisoned and every dispatch arrives here, or the target uses
+ * the out-of-line lookup and always did.
+ */
+ if (unlikely(cpu_loop_exit_requested(cpu))) {
+ return tcg_code_gen_epilogue;
+ }
+
TCGTBCPUState s = cpu->cc->tcg_ops->get_tb_cpu_state(cpu);
s.cflags = curr_cflags(cpu);
@@ -752,6 +762,44 @@ static inline bool cpu_handle_exception(CPUState *cpu, int *ret)
return false;
}
+/*
+ * The inline jump cache probe reads cpu->tb_jmp_cache_probe and takes the
+ * slow path when the entry it finds has a NULL tb. Pointing the probe at a
+ * region that is all zeroes therefore forces every indirect dispatch into
+ * helper_lookup_tb_ptr(), which returns to the main loop while an exit is
+ * pending. That is what lets a block ending in an indirect branch skip the
+ * icount_decr poll: the poll's job is done by a pointer swap that costs the
+ * fast path nothing.
+ *
+ * Only ever read from, and only the tb field of one entry per dispatch, so
+ * one shared zero-filled cache is enough for every CPU.
+ */
+static const CPUJumpCache *tb_jmp_cache_poison(void)
+{
+ static CPUJumpCache *poison;
+
+ if (unlikely(poison == NULL)) {
+ /* Raced allocations are harmless: both are all zeroes. */
+ qatomic_cmpxchg(&poison, NULL, g_new0(CPUJumpCache, 1));
+ }
+ return poison;
+}
+
+void tcg_cpu_poison_jmp_cache(CPUState *cpu)
+{
+ if (qatomic_read(&cpu->tb_jmp_cache_probe) != NULL) {
+ qatomic_set(&cpu->tb_jmp_cache_probe,
+ (CPUJumpCache *)tb_jmp_cache_poison());
+ }
+}
+
+void tcg_cpu_restore_jmp_cache(CPUState *cpu)
+{
+ if (qatomic_read(&cpu->tb_jmp_cache_probe) != NULL) {
+ qatomic_set(&cpu->tb_jmp_cache_probe, cpu->tb_jmp_cache);
+ }
+}
+
void tcg_kick_vcpu_thread(CPUState *cpu)
{
/*
@@ -764,6 +812,9 @@ void tcg_kick_vcpu_thread(CPUState *cpu)
/* Ensure cpu_exec will see the exit request after TCG has exited. */
qatomic_store_release(&cpu->neg.icount_decr.u16.high, -1);
+
+ /* Blocks that only dispatch indirectly do not poll; stop them chaining. */
+ tcg_cpu_poison_jmp_cache(cpu);
}
static inline bool icount_exit_request(CPUState *cpu)
@@ -796,6 +847,7 @@ static inline bool cpu_handle_interrupt(CPUState *cpu,
* tcg_kick_vcpu_thread())
*/
qatomic_set_mb(&cpu->neg.icount_decr.u16.high, 0);
+ tcg_cpu_restore_jmp_cache(cpu);
#ifdef CONFIG_USER_ONLY
assert(!cpu_test_interrupt(cpu, ~0));
@@ -1069,6 +1121,7 @@ bool tcg_exec_realizefn(CPUState *cpu, Error **errp)
}
cpu->tb_jmp_cache = g_new0(CPUJumpCache, 1);
+ qatomic_set(&cpu->tb_jmp_cache_probe, cpu->tb_jmp_cache);
tlb_init(cpu);
#ifndef CONFIG_USER_ONLY
tcg_iommu_init_notifier_list(cpu);
@@ -1086,5 +1139,6 @@ void tcg_exec_unrealizefn(CPUState *cpu)
#endif /* !CONFIG_USER_ONLY */
tlb_destroy(cpu);
+ qatomic_set(&cpu->tb_jmp_cache_probe, NULL);
g_free_rcu(cpu->tb_jmp_cache, rcu);
}
diff --git ./accel/tcg/internal-common.h ./accel/tcg/internal-common.h
index dc713a6e1a..6007223285 100644
--- ./accel/tcg/internal-common.h
+++ ./accel/tcg/internal-common.h
@@ -154,6 +154,9 @@ void page_table_config_init(void);
G_NORETURN void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr);
#endif /* CONFIG_USER_ONLY */
+void tcg_cpu_poison_jmp_cache(CPUState *cpu);
+void tcg_cpu_restore_jmp_cache(CPUState *cpu);
+
void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr);
void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr);
diff --git ./accel/tcg/tcg-accel-ops.c ./accel/tcg/tcg-accel-ops.c
index 560fe2554b..fc134c48d9 100644
--- ./accel/tcg/tcg-accel-ops.c
+++ ./accel/tcg/tcg-accel-ops.c
@@ -44,6 +44,7 @@
#include "hw/core/cpu.h"
+#include "internal-common.h"
#include "tcg-accel-ops.h"
#include "tcg-accel-ops-mttcg.h"
#include "tcg-accel-ops-rr.h"
@@ -106,6 +107,7 @@ void tcg_handle_interrupt(CPUState *cpu, int mask)
qemu_cpu_kick(cpu);
} else {
qatomic_set(&cpu->neg.icount_decr.u16.high, -1);
+ tcg_cpu_poison_jmp_cache(cpu);
}
}
diff --git ./accel/tcg/translator.c ./accel/tcg/translator.c
index 048ad8bad2..5f98d6446b 100644
--- ./accel/tcg/translator.c
+++ ./accel/tcg/translator.c
@@ -62,8 +62,6 @@ static TCGOp *gen_tb_start(DisasContextBase *db, uint32_t cflags)
TCGv_i32 count = NULL;
TCGOp *icount_start_insn = NULL;
- tcg_ctx->exit_check_needed = false;
-
if ((cflags & CF_USE_ICOUNT) ||
(!(cflags & CF_NOIRQ) && !defer_exit_check(cflags))) {
count = tcg_temp_new_i32();
@@ -126,7 +124,7 @@ static void gen_tb_end(const TranslationBlock *tb, uint32_t cflags,
if (tcg_ctx->exitreq_label && defer_exit_check(cflags) &&
!(cflags & CF_NOIRQ)) {
- if (db->needs_exit_check || tcg_ctx->exit_check_needed) {
+ if (db->needs_exit_check) {
TCGv_i32 count = tcg_temp_new_i32();
TCGOp *save = tcg_ctx->emit_before_op;
diff --git ./include/hw/core/cpu.h ./include/hw/core/cpu.h
index 399ac7bd57..cecf9d0583 100644
--- ./include/hw/core/cpu.h
+++ ./include/hw/core/cpu.h
@@ -526,6 +526,16 @@ struct CPUState {
struct CPUJumpCache *tb_jmp_cache;
+ /*
+ * What the inline jump cache probe emitted by
+ * tcg_gen_lookup_and_goto_ptr_inline() reads. Normally equal to
+ * tb_jmp_cache; pointed at a page of zeroes while an exit is pending, so
+ * that every indirect dispatch misses and lands in the helper, which
+ * returns to the main loop. Only generated code and the two accessors in
+ * cpu-exec.c may touch it.
+ */
+ struct CPUJumpCache *tb_jmp_cache_probe;
+
GArray *gdb_regs;
int gdb_num_regs;
int gdb_num_g_regs;
diff --git ./include/tcg/tcg.h ./include/tcg/tcg.h
index be9ce7a0e2..7669dc1c2d 100644
--- ./include/tcg/tcg.h
+++ ./include/tcg/tcg.h
@@ -389,8 +389,6 @@ struct TCGContext {
struct TCGLabelPoolData *pool_labels;
TCGLabel *exitreq_label;
- /* Set by goto_ptr emission: destination is not known statically. */
- bool exit_check_needed;
#ifdef CONFIG_PLUGIN
/*
diff --git ./tcg/tcg-op.c ./tcg/tcg-op.c
index 8282afa61a..ff63f16ccd 100644
--- ./tcg/tcg-op.c
+++ ./tcg/tcg-op.c
@@ -2616,7 +2616,10 @@ void tcg_gen_lookup_and_goto_ptr(void)
return;
}
- tcg_ctx->exit_check_needed = true;
+ /*
+ * No icount_decr poll is needed for this exit: the helper is called on
+ * every dispatch and returns to the main loop while an exit is pending.
+ */
plugin_gen_disable_mem_helpers();
ptr = tcg_temp_ebb_new_ptr();
gen_helper_lookup_tb_ptr(ptr, tcg_env);
@@ -2645,7 +2648,11 @@ void tcg_gen_lookup_and_goto_ptr_inline(TCGv_i64 pc,
return;
}
- tcg_ctx->exit_check_needed = true;
+ /*
+ * No icount_decr poll is needed for this exit either. A pending exit
+ * poisons tb_jmp_cache_probe, so the guarded load below finds a NULL tb,
+ * takes the slow path, and the helper returns to the main loop.
+ */
plugin_gen_disable_mem_helpers();
QEMU_BUILD_BUG_ON(sizeof(((CPUJumpCache *)0)->array[0]) != 16);
@@ -2667,7 +2674,7 @@ void tcg_gen_lookup_and_goto_ptr_inline(TCGv_i64 pc,
tcg_gen_shli_i64(h, h, 4);
tcg_gen_ld_ptr(jc, tcg_env,
- offsetof(CPUState, tb_jmp_cache) - sizeof(CPUState));
+ offsetof(CPUState, tb_jmp_cache_probe) - sizeof(CPUState));
tcg_gen_trunc_i64_ptr(ent, h);
tcg_gen_add_ptr(ent, jc, ent);
diff --git ./tests/tcg/alpha/Makefile.target ./tests/tcg/alpha/Makefile.target
index eee986bab6..f9f135fc2f 100644
--- ./tests/tcg/alpha/Makefile.target
+++ ./tests/tcg/alpha/Makefile.target
@@ -5,7 +5,8 @@
ALPHA_SRC=$(SRC_PATH)/tests/tcg/alpha
VPATH+=$(ALPHA_SRC)
-ALPHA_TESTS=hello-alpha test-cond test-cmov test-ovf test-cvttq test-xpage-chain
+ALPHA_TESTS=hello-alpha test-cond test-cmov test-ovf test-cvttq test-xpage-chain \
+ test-indirect-irq
TESTS+=$(ALPHA_TESTS)
test-cmov: EXTRA_CFLAGS=-DTEST_CMOV
diff --git ./tests/tcg/alpha/test-indirect-irq.c ./tests/tcg/alpha/test-indirect-irq.c
new file mode 100644
index 0000000000..bef2844fd9
--- /dev/null
+++ ./tests/tcg/alpha/test-indirect-irq.c
@@ -0,0 +1,53 @@
+/*
+ * A loop whose only back edge is an indirect branch must still be
+ * interruptible.
+ *
+ * Blocks that dispatch indirectly do not emit the icount_decr poll; a pending
+ * exit instead poisons the inline jump cache probe so that the dispatch falls
+ * into helper_lookup_tb_ptr(), which returns to the main loop. If that
+ * mechanism breaks, this program never leaves the loop and the test times
+ * out rather than failing an assertion.
+ *
+ * A computed goto is used deliberately: a plain while(1) closes the cycle
+ * with a direct backward branch, which is still polled, and so would not
+ * exercise the path under test.
+ */
+#include <assert.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+static volatile sig_atomic_t fired;
+static volatile unsigned long iterations;
+
+static void handler(int sig)
+{
+ fired = 1;
+}
+
+int main(void)
+{
+ /*
+ * Indexing a table with a volatile index, rather than jumping through a
+ * volatile pointer: gcc happily proves a single-valued pointer constant
+ * and emits a direct branch, which is the case this test is not about.
+ */
+ void *target[2];
+ volatile int idx = 0;
+
+ assert(signal(SIGALRM, handler) != SIG_ERR);
+ alarm(1);
+
+ target[0] = &&spin;
+ target[1] = &&out;
+spin:
+ iterations++;
+ if (!fired) {
+ goto *target[idx];
+ }
+out:
+
+ printf("interrupted after %lu iterations\n", iterations);
+ return 0;
+}
--
2.54.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH 8/8] RFC: tcg: fold a guest displacement into the host addressing mode
2026-08-18 17:42 [RFC PATCH 0/8] accel/tcg: cut per-block dispatch overhead Matt Turner
` (6 preceding siblings ...)
2026-08-18 17:42 ` [PATCH 7/8] RFC: accel/tcg: poison the jump cache instead of polling for indirect exits Matt Turner
@ 2026-08-18 17:42 ` Matt Turner
7 siblings, 0 replies; 13+ messages in thread
From: Matt Turner @ 2026-08-18 17:42 UTC (permalink / raw)
To: qemu-devel
Cc: richard.henderson, pbonzini, philmd, zhao1.liu, laurent, deller,
pierrick.bouvier, Matt Turner
Nothing in the TCG frontend interface can express a based memory access.
tcg_gen_qemu_ld/st take an address and nothing else, so a target with a
displacement in its load and store encodings -- which is most of them --
has to materialise the address first:
ldq a1,8(a0) -> mov 0x80(%rbp),%rbx reload a0
lea 0x8(%rbx),%r12 address
mov (%r12),%r12 the load
mov %r12,0x88(%rbp) spill a1
The lea is pure loss on a host whose addressing mode has a displacement
field sitting empty. It also needs a register, at the point in a block
where pressure is highest.
Fold it. After optimisation, look for an add of a constant immediately
before a guest access, defining that access's address operand, and move the
constant into a new second constant argument on the op. The add is left for
liveness to remove, so nothing breaks if its result has another use. Only
the immediately preceding op is examined: that is what the frontends emit,
and a window of one op means the pass does not have to reason about what
could have happened in between. The one thing it does check is that the add
did not clobber the base it read, since the access now reads that base
directly.
Targets opt in with TCG_TARGET_HAS_ldst_disp and an out_disp member on
TCGOutOpQemuLdSt. Without it the pass does not run, the displacement stays
zero and the existing out member is called exactly as before, so no other
backend changes behaviour or needs touching.
For x86_64 the displacement goes in the disp32 that prepare_host_addr()
already fills in for guest_base. The fold is refused unless there is no
slow path at all, which means user-only -- softmmu compares the unadjusted
address against the TLB -- and an access needing no alignment test, since
the slow path hands addr_reg to the helper and that register no longer
holds the full guest address. It is also refused if guest_base plus the
displacement leaves disp32.
Measured with qemu-alpha running an emulated alpha gcc 16.2.0 compiling the
SQLite 3.45.1 amalgamation (255k lines, -O2) on an x86-64 host, LTO build,
on top of the preceding patches, against a control measured in the same
session:
before: 787,483,360,681 instructions, 78.250s
after: 738,020,161,466 instructions, 75.679s
-6.29% instructions, -3.29% wall
Emitted code shrinks from 46.29MB to 44.53MB over the run, 153.3 to 147.5
bytes per block. Per Alpha opcode, the host bytes emitted for an access
fall as expected and nothing else moves:
ldq 17.2 -> 14.3 ldah 16.9 -> 16.9
ldl 14.1 -> 11.6 lda 12.6 -> 12.6
stq 12.7 -> 9.7 mov 9.7 -> 9.7
The emulated compiler produces byte-identical output and the alpha tests
still pass, including with a non-zero guest_base forced via -B.
RFC because:
- Only wired up for x86_64, and only for qemu_ld and qemu_st; the i128
qemu_ld2 and qemu_st2 pairs are left alone.
- Requiring that no slow path exists is stricter than necessary. Recording
the displacement in TCGLabelQemuLdst and emitting one lea on the slow
path would cover alignment-checked accesses too, at no fast path cost.
- Softmmu wants the displacement folded into the TLB comparison as well,
which is a bigger change than this one.
- A one op window catches everything the frontends emit today but is
trivially defeated by anything scheduled in between.
Signed-off-by: Matt Turner <mattst88@gmail.com>
---
include/tcg/tcg-opc.h | 9 +++-
tcg/tcg-op-ldst.c | 3 +-
tcg/tcg.c | 86 ++++++++++++++++++++++++++++++++++++-
tcg/x86_64/tcg-target.c.inc | 61 ++++++++++++++++++++++++++
tcg/x86_64/tcg-target.h | 3 ++
5 files changed, 158 insertions(+), 4 deletions(-)
diff --git ./include/tcg/tcg-opc.h ./include/tcg/tcg-opc.h
index 61f1c28858..0a3b4330f1 100644
--- ./include/tcg/tcg-opc.h
+++ ./include/tcg/tcg-opc.h
@@ -118,8 +118,13 @@ DEF(goto_ptr, 0, 1, 0, TCG_OPF_BB_EXIT | TCG_OPF_BB_END)
DEF(plugin_cb, 0, 0, 1, TCG_OPF_NOT_PRESENT)
DEF(plugin_mem_cb, 0, 1, 1, TCG_OPF_NOT_PRESENT)
-DEF(qemu_ld, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_INT)
-DEF(qemu_st, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_INT)
+/*
+ * The second constant argument is a displacement to add to the address,
+ * zero unless a target advertises TCG_TARGET_HAS_ldst_disp and the fold in
+ * fold_ldst_disp() applied.
+ */
+DEF(qemu_ld, 1, 1, 2, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_INT)
+DEF(qemu_st, 0, 2, 2, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_INT)
DEF(qemu_ld2, 2, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_INT)
DEF(qemu_st2, 0, 3, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_INT)
diff --git ./tcg/tcg-op-ldst.c ./tcg/tcg-op-ldst.c
index 22211ccb45..ffc5e651a6 100644
--- ./tcg/tcg-op-ldst.c
+++ ./tcg/tcg-op-ldst.c
@@ -92,7 +92,8 @@ static MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st)
static void gen_ldst1(TCGOpcode opc, TCGType type, TCGTemp *v,
TCGTemp *addr, MemOpIdx oi)
{
- TCGOp *op = tcg_gen_op3(opc, type, temp_arg(v), temp_arg(addr), oi);
+ /* The trailing zero is the address displacement; see fold_ldst_disp(). */
+ TCGOp *op = tcg_gen_op4(opc, type, temp_arg(v), temp_arg(addr), oi, 0);
TCGOP_FLAGS(op) = get_memop(oi) & MO_SIZE;
}
diff --git ./tcg/tcg.c ./tcg/tcg.c
index 1e77f2365a..bdd95304ea 100644
--- ./tcg/tcg.c
+++ ./tcg/tcg.c
@@ -1058,6 +1058,13 @@ typedef struct TCGOutOpQemuLdSt {
TCGOutOp base;
void (*out)(TCGContext *s, TCGType type, TCGReg dest,
TCGReg addr, MemOpIdx oi);
+ /*
+ * As out(), for an access at addr + disp. Only required of targets that
+ * define TCG_TARGET_HAS_ldst_disp; for everyone else fold_ldst_disp()
+ * never runs and the displacement is always zero.
+ */
+ void (*out_disp)(TCGContext *s, TCGType type, TCGReg dest,
+ TCGReg addr, MemOpIdx oi, int32_t disp);
} TCGOutOpQemuLdSt;
typedef struct TCGOutOpQemuLdSt2 {
@@ -3560,6 +3567,77 @@ static void move_label_uses(TCGLabel *to, TCGLabel *from)
QSIMPLEQ_CONCAT(&to->branches, &from->branches);
}
+#ifndef TCG_TARGET_HAS_ldst_disp
+#define TCG_TARGET_HAS_ldst_disp 0
+static bool tcg_target_ldst_disp_ok(TCGContext *s, MemOpIdx oi, int64_t disp)
+{
+ return false;
+}
+#endif
+
+/*
+ * Fold "add addr, base, $disp" into the guest access that follows it, so
+ * that the displacement becomes part of the host addressing mode instead of
+ * a separate instruction. Frontends have no way to express this: there is
+ * no displacement operand on tcg_gen_qemu_ld/st, so a based access always
+ * costs an extra add, and an extra register to hold its result.
+ *
+ * Only an add in the op immediately before the access is recognised. That
+ * is what the frontends emit, and a window of one op means no analysis is
+ * needed of what might have happened in between. The add is left in place;
+ * liveness removes it if its result has no other use.
+ */
+static void __attribute__((noinline))
+fold_ldst_disp(TCGContext *s)
+{
+ TCGOp *op;
+
+ if (!TCG_TARGET_HAS_ldst_disp) {
+ return;
+ }
+
+ QTAILQ_FOREACH(op, &s->ops, link) {
+ TCGOp *prev;
+ TCGTemp *cts;
+ int64_t disp;
+
+ switch (op->opc) {
+ case INDEX_op_qemu_ld:
+ case INDEX_op_qemu_st:
+ break;
+ default:
+ continue;
+ }
+
+ prev = QTAILQ_PREV(op, link);
+ if (prev == NULL || prev->opc != INDEX_op_add ||
+ TCGOP_TYPE(prev) != s->addr_type) {
+ continue;
+ }
+
+ /*
+ * The add must define the address operand, and must not have
+ * clobbered the base it read: after the fold the access reads the
+ * base directly, so the base has to still hold its original value.
+ */
+ if (prev->args[0] != op->args[1] || prev->args[0] == prev->args[1]) {
+ continue;
+ }
+
+ cts = arg_temp(prev->args[2]);
+ if (cts->kind != TEMP_CONST) {
+ continue;
+ }
+ disp = cts->val;
+ if (disp == 0 || !tcg_target_ldst_disp_ok(s, op->args[2], disp)) {
+ continue;
+ }
+
+ op->args[1] = prev->args[1];
+ op->args[3] = disp;
+ }
+}
+
/* Reachable analysis : remove unreachable code. */
static void __attribute__((noinline))
reachable_code_pass(TCGContext *s)
@@ -5707,7 +5785,12 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
const TCGOutOpQemuLdSt *out =
container_of(all_outop[op->opc], TCGOutOpQemuLdSt, base);
- out->out(s, type, new_args[0], new_args[1], new_args[2]);
+ if (new_args[3]) {
+ out->out_disp(s, type, new_args[0], new_args[1],
+ new_args[2], new_args[3]);
+ } else {
+ out->out(s, type, new_args[0], new_args[1], new_args[2]);
+ }
}
break;
@@ -6590,6 +6673,7 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start)
tcg_temp_ebb_reset_freed(s);
tcg_optimize(s);
+ fold_ldst_disp(s);
reachable_code_pass(s);
liveness_pass_0(s);
diff --git ./tcg/x86_64/tcg-target.c.inc ./tcg/x86_64/tcg-target.c.inc
index 1fc45e4ec6..d3bde91816 100644
--- ./tcg/x86_64/tcg-target.c.inc
+++ ./tcg/x86_64/tcg-target.c.inc
@@ -2015,6 +2015,39 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h,
return ldst;
}
+/*
+ * Whether the displacement of a guest access can be folded into the host
+ * addressing mode rather than materialised by a separate lea.
+ *
+ * Folding rewrites the access to use base + disp, so nothing may need a
+ * register holding the complete guest address. The softmmu TLB comparison
+ * does, and so does any slow path, which hands addr_reg to the helper. In
+ * user-only mode prepare_host_addr() creates a slow path only for an
+ * alignment test, so requiring that none is needed rules it out. What is
+ * left to check is guest_base, which shares the disp32 field.
+ */
+static bool tcg_target_ldst_disp_ok(TCGContext *s, MemOpIdx oi, int64_t disp)
+{
+#ifdef CONFIG_USER_ONLY
+ MemOp opc = get_memop(oi);
+ TCGAtomAlign aa;
+ int64_t ofs;
+
+ if (tcg_use_softmmu || s->addr_type != TCG_TYPE_I64) {
+ return false;
+ }
+ aa = atom_and_align_for_opc(s, opc, MO_ATOM_WITHIN16,
+ (opc & MO_SIZE) == MO_128);
+ if (aa.align) {
+ return false;
+ }
+ ofs = (int64_t)x86_guest_base.ofs + disp;
+ return ofs == (int32_t)ofs;
+#else
+ return false;
+#endif
+}
+
static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
HostAddress h, TCGType type, MemOp memop)
{
@@ -2171,9 +2204,23 @@ static void tgen_qemu_ld(TCGContext *s, TCGType type, TCGReg data,
}
}
+static void tgen_qemu_ld_disp(TCGContext *s, TCGType type, TCGReg data,
+ TCGReg addr, MemOpIdx oi, int32_t disp)
+{
+ TCGLabelQemuLdst *ldst;
+ HostAddress h;
+
+ ldst = prepare_host_addr(s, &h, addr, oi, true);
+ /* tcg_target_ldst_disp_ok() has ruled out every slow path. */
+ tcg_debug_assert(ldst == NULL);
+ h.ofs += disp;
+ tcg_out_qemu_ld_direct(s, data, -1, h, type, get_memop(oi));
+}
+
static const TCGOutOpQemuLdSt outop_qemu_ld = {
.base.static_constraint = C_O1_I1(r, L),
.out = tgen_qemu_ld,
+ .out_disp = tgen_qemu_ld_disp,
};
static void tgen_qemu_ld2(TCGContext *s, TCGType type, TCGReg datalo,
@@ -2309,9 +2356,23 @@ static void tgen_qemu_st(TCGContext *s, TCGType type, TCGReg data,
}
}
+static void tgen_qemu_st_disp(TCGContext *s, TCGType type, TCGReg data,
+ TCGReg addr, MemOpIdx oi, int32_t disp)
+{
+ TCGLabelQemuLdst *ldst;
+ HostAddress h;
+
+ ldst = prepare_host_addr(s, &h, addr, oi, false);
+ /* tcg_target_ldst_disp_ok() has ruled out every slow path. */
+ tcg_debug_assert(ldst == NULL);
+ h.ofs += disp;
+ tcg_out_qemu_st_direct(s, data, -1, h, get_memop(oi));
+}
+
static const TCGOutOpQemuLdSt outop_qemu_st = {
.base.static_constraint = C_O0_I2(L, L),
.out = tgen_qemu_st,
+ .out_disp = tgen_qemu_st_disp,
};
static void tgen_qemu_st2(TCGContext *s, TCGType type, TCGReg datalo,
diff --git ./tcg/x86_64/tcg-target.h ./tcg/x86_64/tcg-target.h
index 7ebae56a7d..8f2315c15e 100644
--- ./tcg/x86_64/tcg-target.h
+++ ./tcg/x86_64/tcg-target.h
@@ -30,6 +30,9 @@
#define TCG_TARGET_NB_REGS 32
#define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB)
+/* A guest displacement can go in the disp32 of the addressing mode. */
+#define TCG_TARGET_HAS_ldst_disp 1
+
typedef enum {
TCG_REG_EAX = 0,
TCG_REG_ECX,
--
2.54.0
^ permalink raw reply related [flat|nested] 13+ messages in thread