diff --git a/accel/tcg/tb-flush.c b/accel/tcg/tb-flush.c index 8103d14969a..bf91b389521 100644 --- a/accel/tcg/tb-flush.c +++ b/accel/tcg/tb-flush.c @@ -89,6 +89,9 @@ void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) #endif cpu_tb_jmp_cache_clear(cpu); } +#ifdef CONFIG_LATX_FAST_JMPCACHE + latx_shadow_jmp_cache_clear_all(); +#endif qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE); tb_flush_remove_all(); diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index f4d515e257a..61878e386e5 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -1736,6 +1736,9 @@ static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list) if (!qht_remove(&tb_ctx.htable, tb, h)) { return; } +#ifdef CONFIG_LATX_FAST_JMPCACHE + latx_shadow_jmp_cache_remove(tb); +#endif /* remove the TB from the page list */ if (rm_from_page_list) { @@ -1867,6 +1870,10 @@ tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, if (unlikely(existing_tb)) { tb_remove(tb); tb = existing_tb; +#ifdef CONFIG_LATX_FAST_JMPCACHE + } else { + latx_shadow_jmp_cache_add(tb); +#endif } if (p2 && p2 != p) { diff --git a/configure b/configure index e4c41093ac9..49a6033fa3a 100755 --- a/configure +++ b/configure @@ -5685,6 +5685,11 @@ if test "$latx" = "yes" ; then echo "CONFIG_LATX_KZT=y" >> $config_host_mak fi echo "CONFIG_LATX=y" >> $config_host_mak + case " $target_list " in + *" x86_64-linux-user "*) + echo "CONFIG_LATX_X86_64=y" >> $config_host_mak + ;; + esac if test "$debug" = "yes" ; then echo "CONFIG_LATX_DEBUG=y" >> $config_host_mak else diff --git a/include/exec/fasttb.h b/include/exec/fasttb.h index 1d839f914e6..4e9d9ac1ea4 100644 --- a/include/exec/fasttb.h +++ b/include/exec/fasttb.h @@ -10,9 +10,27 @@ struct FastTB { #define FASTTB_ILLINST_MAGIC 0x88888888 +#ifndef LATX_SHADOW_JMP_BITS +#define LATX_SHADOW_JMP_BITS 20 +#endif +#define LATX_SHADOW_JMP_SIZE (1U << LATX_SHADOW_JMP_BITS) +#define LATX_SHADOW_JMP_HASH_MULT UINT64_C(11400714819323198485) +#define LATX_SHADOW_JMP_TOMBSTONE ((struct TranslationBlock *)1) + +typedef struct LatxShadowJmpEntry { + struct TranslationBlock *tb; +} LatxShadowJmpEntry; + +extern LatxShadowJmpEntry latx_shadow_jmp_entries[]; + void latx_fast_jmp_cache_add(CPUState *cs, int h, struct TranslationBlock *tb); void latx_fast_jmp_cache_clear(CPUState *cs, int h); void latx_fast_jmp_cache_clear_all(CPUState *cs); bool latx_fast_jmp_cache_init(void *env); void latx_fast_jmp_cache_free_rcu(void *ptr); +void latx_shadow_jmp_cache_add(struct TranslationBlock *tb); +void latx_shadow_jmp_cache_remove(struct TranslationBlock *tb); +void latx_shadow_jmp_cache_clear_all(void); +struct TranslationBlock *latx_shadow_jmp_cache_lookup( + uint64_t pc, uint32_t flags, uint32_t cflags); #endif diff --git a/include/exec/tb-hash.h b/include/exec/tb-hash.h index 0a273d9605a..7442181b97d 100644 --- a/include/exec/tb-hash.h +++ b/include/exec/tb-hash.h @@ -54,9 +54,13 @@ static inline unsigned int tb_jmp_cache_hash_func(target_ulong pc) /* In user-mode we can get better hashing because we do not have a TLB */ static inline unsigned int tb_jmp_cache_hash_func(target_ulong pc) { - return (pc ^ (pc >> TB_JMP_CACHE_BITS)) & (TB_JMP_CACHE_SIZE - 1); + return pc & (TB_JMP_CACHE_SIZE - 1); } +/* Instruction offsets from tb->jmp_indirect to the FastTB hit-path JIRL. */ +#define TB_JMP_CACHE_FAST_JIRL_OFFSET 5 +#define TB_JMP_CACHE_FAST_MASK_JIRL_OFFSET 9 + #endif /* CONFIG_SOFTMMU */ static inline diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h index 667e97d8228..17a90004d55 100644 --- a/include/hw/core/cpu.h +++ b/include/hw/core/cpu.h @@ -246,10 +246,15 @@ struct kvm_run; struct hax_vcpu_state; -#ifdef CONFIG_LATX +#if defined(CONFIG_LATX) && defined(CONFIG_LATX_X86_64) +#define TB_JMP_CACHE_BITS 19 +#define TB_JMP_CACHE_HASH_SHIFT 32 +#elif defined(CONFIG_LATX) #define TB_JMP_CACHE_BITS 16 +#define TB_JMP_CACHE_HASH_SHIFT TB_JMP_CACHE_BITS #else #define TB_JMP_CACHE_BITS 12 +#define TB_JMP_CACHE_HASH_SHIFT TB_JMP_CACHE_BITS #endif #define TB_JMP_CACHE_SIZE (1 << TB_JMP_CACHE_BITS) diff --git a/meson.build b/meson.build index f8b5d5980eb..6020839d364 100644 --- a/meson.build +++ b/meson.build @@ -676,6 +676,23 @@ foreach target : target_dirs suite: 'lat-pr-fast', ) + shadow_jmp_cache_test = executable( + 'test-latx-shadow-jmp-cache', + files( + 'target/i386/latx/sbt/tests/shadow-jmp-cache-test.c', + 'target/i386/latx/shadow-jmp-cache.c' + ) + genh + tcg_trace_genh + [syscall_nr_generated], + c_args: smc_reload_test_c_args + ['-DLATX_SHADOW_JMP_BITS=3'], + dependencies: [glib], + include_directories: target_inc, + link_args: smc_reload_test_link_args + ) + test( + 'latx-shadow-jmp-cache', + shadow_jmp_cache_test, + suite: 'lat-pr-fast', + ) + aot_file_publish_test = executable( 'test-latx-aot-file-publish', files( diff --git a/target/i386/latx/include/aot.h b/target/i386/latx/include/aot.h index f72613a814d..08ecbffad72 100644 --- a/target/i386/latx/include/aot.h +++ b/target/i386/latx/include/aot.h @@ -264,6 +264,8 @@ typedef enum aot_rel_kind { LOAD_HELPER_END, LOAD_TUNNEL_ADDR_BEGIN, + /* Keep this outside the dense tunnel relocation range. */ + B_SHADOW_JMP_GLUE = 0x10000, } aot_rel_kind; typedef struct aot_rel { diff --git a/target/i386/latx/include/translate.h b/target/i386/latx/include/translate.h index 38200bc4298..435997ed71e 100644 --- a/target/i386/latx/include/translate.h +++ b/target/i386/latx/include/translate.h @@ -1706,6 +1706,7 @@ void tr_generate_goto_tb(void); /* TODO */ extern ADDR native_rotate_fpu_by; extern ADDR indirect_jmp_glue; extern ADDR parallel_indirect_jmp_glue; +extern ADDR shadow_jmp_glue; void rotate_fpu_to_top(int top); void rotate_fpu_by(int step); void rotate_fpu_to_bias(int bias); diff --git a/target/i386/latx/latx-signal.c b/target/i386/latx/latx-signal.c index 2072ce47098..cbd63cef162 100644 --- a/target/i386/latx/latx-signal.c +++ b/target/i386/latx/latx-signal.c @@ -160,13 +160,13 @@ void unlink_indirect_jmp(CPUArchState *env, TranslationBlock *tb, ucontext_t *uc #ifdef CONFIG_LATX_FAST_JMPCACHE #ifdef CONFIG_LATX_GLUE_MASK - /* Glue-masked fast jmpcache lands in jirl after 12 instructions. */ - insn = qatomic_read((uint32_t *)(jmp_rx + INS_SIZE * 11)); - jmp_rw += INS_SIZE * 11; + insn = qatomic_read((uint32_t *)(jmp_rx + + INS_SIZE * TB_JMP_CACHE_FAST_MASK_JIRL_OFFSET)); + jmp_rw += INS_SIZE * TB_JMP_CACHE_FAST_MASK_JIRL_OFFSET; #else - /* Unmasked fast jmpcache reaches jirl after the eight-instruction hit path. */ - insn = qatomic_read((uint32_t *)(jmp_rx + INS_SIZE * 7)); - jmp_rw += INS_SIZE * 7; + insn = qatomic_read((uint32_t *)(jmp_rx + + INS_SIZE * TB_JMP_CACHE_FAST_JIRL_OFFSET)); + jmp_rw += INS_SIZE * TB_JMP_CACHE_FAST_JIRL_OFFSET; #endif #else CPUState *cpu = env_cpu(env); diff --git a/target/i386/latx/meson.build b/target/i386/latx/meson.build index 6c6d585c42f..b3d0ce2fa61 100644 --- a/target/i386/latx/meson.build +++ b/target/i386/latx/meson.build @@ -68,6 +68,7 @@ i386_ss.add(when: 'CONFIG_LATX', if_true: files( 'error.c', 'mem.c', 'latx-config.c', + 'shadow-jmp-cache.c', 'latx-options.c', 'latx-perf.c', 'latx-name-demangling.cpp', diff --git a/target/i386/latx/sbt/aot.c b/target/i386/latx/sbt/aot.c index 3aa6292cfac..a3f286ef20a 100644 --- a/target/i386/latx/sbt/aot.c +++ b/target/i386/latx/sbt/aot.c @@ -1310,6 +1310,7 @@ void aot_do_tb_reloc(TranslationBlock *tb, struct aot_tb *stb, uintptr_t helper_address; int lib_method_index; ADDR loacl_indirect_jmp_glue = indirect_jmp_glue; + ADDR local_shadow_jmp_glue = shadow_jmp_glue; aot_rel_table = aot_buffer + ((aot_header *)aot_buffer)->rel_table_offset; @@ -1376,6 +1377,13 @@ void aot_do_tb_reloc(TranslationBlock *tb, struct aot_tb *stb, light_tb_target_set_jmp_target((uintptr_t)tb->tc.ptr, (uintptr_t)pinsn, (uintptr_t)pinsn, loacl_indirect_jmp_glue); break; + case B_SHADOW_JMP_GLUE: + lsassert(((*pinsn) & 0xfc000000) == 0x50000000 || + (((*pinsn) & 0xfe000000) == 0x1e000000 && + (*(pinsn + 1) & 0xfc000000) == 0x4c000000)); + light_tb_target_set_jmp_target((uintptr_t)tb->tc.ptr, + (uintptr_t)pinsn, (uintptr_t)pinsn, local_shadow_jmp_glue); + break; case LOAD_TB_ADDR: lsassert(aot_rel_table[i].rel_slots_num == 3); lsassert((*pinsn & 0xfe000000) == 0x14000000); /* lu12i.w */ diff --git a/target/i386/latx/sbt/aot_merge.c b/target/i386/latx/sbt/aot_merge.c index 16ce5cdf721..91e70429961 100644 --- a/target/i386/latx/sbt/aot_merge.c +++ b/target/i386/latx/sbt/aot_merge.c @@ -215,6 +215,7 @@ static gboolean merge_dump_rel_table_tree_node(gpointer key, gpointer val, break; case B_NATIVE_JMP_GLUE2: + case B_SHADOW_JMP_GLUE: lsassert(((*pinsn) & 0xfc000000) == 0x50000000 || (((*pinsn) & 0xfe000000) == 0x1e000000 &&/* pcaddu18i */ (*(pinsn + 1) & 0xfc000000) == 0x4c000000)); diff --git a/target/i386/latx/sbt/tests/shadow-jmp-cache-test.c b/target/i386/latx/sbt/tests/shadow-jmp-cache-test.c new file mode 100644 index 00000000000..6164860ba42 --- /dev/null +++ b/target/i386/latx/sbt/tests/shadow-jmp-cache-test.c @@ -0,0 +1,118 @@ +#include "qemu/osdep.h" + +#include "exec/exec-all.h" +#include "exec/fasttb.h" + +static void init_tb(TranslationBlock *tb, target_ulong pc, + uint32_t flags, uint32_t cflags, uintptr_t code) +{ + memset(tb, 0, sizeof(*tb)); + tb->pc = pc; + tb->flags = flags; + tb->cflags = cflags; + tb->tc.ptr = (void *)code; +} + +static unsigned int test_hash(target_ulong pc) +{ + return (pc * LATX_SHADOW_JMP_HASH_MULT) >> + (64 - LATX_SHADOW_JMP_BITS); +} + +static target_ulong find_collision(target_ulong pc, target_ulong start) +{ + unsigned int wanted = test_hash(pc); + + for (target_ulong candidate = start; ; candidate++) { + if (candidate != pc && test_hash(candidate) == wanted) { + return candidate; + } + } +} + +static void test_distinct_keys_and_tombstone(void) +{ + TranslationBlock a, b, c, replacement, collided; + target_ulong collision_pc = find_collision(0x1000, 0x2000); + + init_tb(&a, 0x1000, 0x11, 0x21, 0x100000); + init_tb(&b, 0x1000, 0x12, 0x21, 0x200000); + init_tb(&c, 0x1000, 0x11, 0x22, 0x300000); + init_tb(&replacement, 0x1000, 0x11, 0x22, 0x400000); + init_tb(&collided, collision_pc, 0x31, 0x41, 0x500000); + + latx_shadow_jmp_cache_clear_all(); + latx_shadow_jmp_cache_add(&a); + latx_shadow_jmp_cache_add(&b); + latx_shadow_jmp_cache_add(&c); + + g_assert_true(latx_shadow_jmp_cache_lookup(a.pc, a.flags, + tb_cflags(&a)) == &a); + g_assert_true(latx_shadow_jmp_cache_lookup(b.pc, b.flags, + tb_cflags(&b)) == &b); + g_assert_true(latx_shadow_jmp_cache_lookup(c.pc, c.flags, + tb_cflags(&c)) == &c); + g_assert_null(latx_shadow_jmp_cache_lookup(a.pc, 0xff, tb_cflags(&a))); + + c.cflags |= CF_INVALID; + g_assert_null(latx_shadow_jmp_cache_lookup(c.pc, c.flags, 0x22)); + c.cflags &= ~CF_INVALID; + + latx_shadow_jmp_cache_remove(&b); + g_assert_null(latx_shadow_jmp_cache_lookup(b.pc, b.flags, tb_cflags(&b))); + g_assert_true(latx_shadow_jmp_cache_lookup(c.pc, c.flags, + tb_cflags(&c)) == &c); + + latx_shadow_jmp_cache_add(&collided); + g_assert_true(latx_shadow_jmp_cache_lookup( + collided.pc, collided.flags, + tb_cflags(&collided)) == &collided); + + latx_shadow_jmp_cache_add(&replacement); + g_assert_true(latx_shadow_jmp_cache_lookup( + replacement.pc, replacement.flags, + tb_cflags(&replacement)) == &replacement); + latx_shadow_jmp_cache_remove(&c); + g_assert_true(latx_shadow_jmp_cache_lookup( + replacement.pc, replacement.flags, + tb_cflags(&replacement)) == &replacement); +} + +static void test_full_table(void) +{ + TranslationBlock tb[LATX_SHADOW_JMP_SIZE + 1]; + + latx_shadow_jmp_cache_clear_all(); + for (unsigned int i = 0; i < LATX_SHADOW_JMP_SIZE; i++) { + init_tb(&tb[i], 0x10000 + i, i, 0x80 + i, 0x100000 + i * 16); + latx_shadow_jmp_cache_add(&tb[i]); + } + for (unsigned int i = 0; i < LATX_SHADOW_JMP_SIZE; i++) { + g_assert_true(latx_shadow_jmp_cache_lookup( + tb[i].pc, tb[i].flags, + tb_cflags(&tb[i])) == &tb[i]); + } + + init_tb(&tb[LATX_SHADOW_JMP_SIZE], 0x20000, 0xaa, 0xbb, 0x200000); + g_assert_null(latx_shadow_jmp_cache_lookup(0x30000, 0xcc, 0xdd)); + latx_shadow_jmp_cache_add(&tb[LATX_SHADOW_JMP_SIZE]); + g_assert_null(latx_shadow_jmp_cache_lookup( + tb[LATX_SHADOW_JMP_SIZE].pc, + tb[LATX_SHADOW_JMP_SIZE].flags, + tb_cflags(&tb[LATX_SHADOW_JMP_SIZE]))); + + latx_shadow_jmp_cache_remove(&tb[3]); + latx_shadow_jmp_cache_add(&tb[LATX_SHADOW_JMP_SIZE]); + g_assert_true(latx_shadow_jmp_cache_lookup( + tb[LATX_SHADOW_JMP_SIZE].pc, + tb[LATX_SHADOW_JMP_SIZE].flags, + tb_cflags(&tb[LATX_SHADOW_JMP_SIZE])) == + &tb[LATX_SHADOW_JMP_SIZE]); +} + +int main(void) +{ + test_distinct_keys_and_tombstone(); + test_full_table(); + return 0; +} diff --git a/target/i386/latx/sbt/tests/tb-flush-smc-reload-async-test.c b/target/i386/latx/sbt/tests/tb-flush-smc-reload-async-test.c index 16a6a740cc5..6337434e738 100644 --- a/target/i386/latx/sbt/tests/tb-flush-smc-reload-async-test.c +++ b/target/i386/latx/sbt/tests/tb-flush-smc-reload-async-test.c @@ -97,6 +97,10 @@ void latx_fast_jmp_cache_clear_all(CPUState *cpu G_GNUC_UNUSED) { } +void latx_shadow_jmp_cache_clear_all(void) +{ +} + static SMCReloadInfo *new_reload_node(target_ulong page_addr) { SMCReloadInfo *node = g_new0(SMCReloadInfo, 1); @@ -116,22 +120,22 @@ static void run_queued_flush(CPUState *cpu) int main(void) { - CPUState cpu = { 0 }; + g_autofree CPUState *cpu = g_new0(CPUState, 1); option_smc_reload = true; tb_ctx.tb_flush_count = 1; - tb_flush(&cpu); + tb_flush(cpu); g_assert(queued_func != NULL); smc_reload_tree_insert(new_reload_node(0x1000)); - run_queued_flush(&cpu); + run_queued_flush(cpu); g_assert(smc_reload_tree_get_node_count() == 0); smc_reload_tree_insert(new_reload_node(0x2000)); - tb_flush(&cpu); + tb_flush(cpu); g_assert(queued_func != NULL); tb_ctx.tb_flush_count++; - run_queued_flush(&cpu); + run_queued_flush(cpu); g_assert(smc_reload_tree_get_node_count() == 1); smc_reload_tree_clear(); return 0; diff --git a/target/i386/latx/shadow-jmp-cache.c b/target/i386/latx/shadow-jmp-cache.c new file mode 100644 index 00000000000..f1a49e5422e --- /dev/null +++ b/target/i386/latx/shadow-jmp-cache.c @@ -0,0 +1,102 @@ +#include "qemu/osdep.h" + +#include "exec/exec-all.h" +#include "exec/fasttb.h" + +LatxShadowJmpEntry latx_shadow_jmp_entries[LATX_SHADOW_JMP_SIZE]; + +static inline unsigned int latx_shadow_jmp_hash(uint64_t pc) +{ + return (pc * LATX_SHADOW_JMP_HASH_MULT) >> + (64 - LATX_SHADOW_JMP_BITS); +} + +static bool latx_shadow_jmp_key_equal(const TranslationBlock *tb, + uint64_t pc, uint32_t flags, + uint32_t cflags) +{ + return tb->pc == pc && tb->flags == flags && tb_cflags(tb) == cflags; +} + +TranslationBlock *latx_shadow_jmp_cache_lookup(uint64_t pc, + uint32_t flags, + uint32_t cflags) +{ + unsigned int h = latx_shadow_jmp_hash(pc); + + for (unsigned int i = 0; i < LATX_SHADOW_JMP_SIZE; i++) { + LatxShadowJmpEntry *entry = + &latx_shadow_jmp_entries[(h + i) & + (LATX_SHADOW_JMP_SIZE - 1)]; + TranslationBlock *tb = qatomic_read(&entry->tb); + + if (!tb) { + return NULL; + } + if (tb != LATX_SHADOW_JMP_TOMBSTONE && + latx_shadow_jmp_key_equal(tb, pc, flags, cflags)) { + return tb; + } + } + return NULL; +} + +void latx_shadow_jmp_cache_add(TranslationBlock *tb) +{ + unsigned int h = latx_shadow_jmp_hash(tb->pc); + LatxShadowJmpEntry *first_tombstone = NULL; + uint32_t cflags = tb_cflags(tb); + + for (unsigned int i = 0; i < LATX_SHADOW_JMP_SIZE; i++) { + LatxShadowJmpEntry *entry = + &latx_shadow_jmp_entries[(h + i) & + (LATX_SHADOW_JMP_SIZE - 1)]; + TranslationBlock *cached = qatomic_read(&entry->tb); + + if (cached == LATX_SHADOW_JMP_TOMBSTONE) { + if (!first_tombstone) { + first_tombstone = entry; + } + continue; + } + if (!cached) { + entry = first_tombstone ? first_tombstone : entry; + qatomic_set(&entry->tb, tb); + return; + } + if (cached == tb || + latx_shadow_jmp_key_equal(cached, tb->pc, tb->flags, cflags)) { + qatomic_set(&entry->tb, tb); + return; + } + } + + if (first_tombstone) { + qatomic_set(&first_tombstone->tb, tb); + } +} + +void latx_shadow_jmp_cache_remove(TranslationBlock *tb) +{ + unsigned int h = latx_shadow_jmp_hash(tb->pc); + + for (unsigned int i = 0; i < LATX_SHADOW_JMP_SIZE; i++) { + LatxShadowJmpEntry *entry = + &latx_shadow_jmp_entries[(h + i) & + (LATX_SHADOW_JMP_SIZE - 1)]; + TranslationBlock *cached = qatomic_read(&entry->tb); + + if (!cached) { + return; + } + if (cached == tb) { + qatomic_set(&entry->tb, LATX_SHADOW_JMP_TOMBSTONE); + return; + } + } +} + +void latx_shadow_jmp_cache_clear_all(void) +{ + memset(latx_shadow_jmp_entries, 0, sizeof(latx_shadow_jmp_entries)); +} diff --git a/target/i386/latx/translator/translate.c b/target/i386/latx/translator/translate.c index 38266d7bbcb..e92d0317199 100644 --- a/target/i386/latx/translator/translate.c +++ b/target/i386/latx/translator/translate.c @@ -83,6 +83,7 @@ void *interpret_glue; ADDR native_rotate_fpu_by; /* native_rotate_fpu_by(step, return_address) */ ADDR indirect_jmp_glue; ADDR parallel_indirect_jmp_glue; +ADDR shadow_jmp_glue; #ifndef TARGET_X86_64 int GPR_USEDEF_TO_SAVE = 0x7; @@ -2649,7 +2650,9 @@ int tr_translate_tb(struct TranslationBlock *tb) * ra_alloc_dbt_arg2: next x86 ip */ -static void generate_indirect_goto(void *code_buf) +static void generate_indirect_goto(void *code_buf, ADDR miss_target, + enum aot_rel_kind miss_rel_kind, + TranslationBlock *source_tb) { /* * WARNING!!! @@ -2668,16 +2671,14 @@ static void generate_indirect_goto(void *code_buf) IR2_OPND next_tb = V0_RENAME_OPND; /* * lookup HASH_JMP_CACHE - * Step 1: calculate HASH = (x86_addr >> 12) ^ (x86_addr & 0xfff) + * Step 1: calculate HASH from the low address bits * Step 2: load &HASH_JMP_CACHE[0] * Step 3: load tb = HASH_JMP_CACHE[HASH] * Step 4: if (tb == 0) {goto labal_miss} * Step 5: if (tb.pc == x86_addr) {goto fpu_rotate} * else {goto labal_miss} */ - la_srli_d(next_tb, next_x86_addr, TB_JMP_CACHE_BITS); - la_xor(next_tb, next_x86_addr, next_tb); - la_bstrpick_d(next_tb, next_tb, TB_JMP_CACHE_BITS - 1, 0); + la_bstrpick_d(next_tb, next_x86_addr, TB_JMP_CACHE_BITS - 1, 0); #ifdef CONFIG_LATX_FAST_JMPCACHE #ifdef CONFIG_LATX_GLUE_MASK @@ -2752,8 +2753,11 @@ static void generate_indirect_goto(void *code_buf) la_label(label_skip); } #endif - la_data_li(target, context_switch_native_to_bt_ret_0); - aot_la_append_ir2_jmp_far(target, base, B_EPILOGUE_RET_0, 0); + if (source_tb) { + aot_load_host_addr(V0_RENAME_OPND, (ADDR)source_tb, LOAD_TB_ADDR, 0); + } + la_data_li(target, miss_target); + aot_la_append_ir2_jmp_far(target, base, miss_rel_kind, 0); return; } @@ -3012,7 +3016,8 @@ void tr_generate_exit_tb(IR1_INST *branch, int succ_id) IR2_OPND old_jmp_label = ra_alloc_label(); la_label(old_jmp_label); tb->jmp_indirect = ir2_opnd_label_id(&old_jmp_label); - generate_indirect_goto((void *)tb->tc.ptr); + generate_indirect_goto((void *)tb->tc.ptr, shadow_jmp_glue, + B_SHADOW_JMP_GLUE, tb); } else { la_data_li(target, context_switch_native_to_bt_ret_0); aot_la_append_ir2_jmp_far(target, base, B_EPILOGUE_RET_0, 0); @@ -3161,7 +3166,80 @@ static int generate_indirect_jmp_glue(void *code_buf) int ins_num; tr_init(NULL); - generate_indirect_goto(code_buf); + generate_indirect_goto(code_buf, context_switch_native_to_bt_ret_0, + B_EPILOGUE_RET_0, NULL); + + TRANSLATION_DATA *lat_ctx = lsenv->tr_data; + label_dispose(NULL, lat_ctx); + ins_num = tr_ir2_assemble(code_buf, lat_ctx->first_ir2) + 1; + tr_fini(false); + + return ins_num; +} + +static int generate_shadow_jmp_glue(void *code_buf) +{ + int ins_num; + tr_init(NULL); + IR2_OPND current_tb = V0_RENAME_OPND; + IR2_OPND next_x86_addr = ra_alloc_dbt_arg2(); + IR2_OPND jmp_cache_addr = ra_alloc_static0(); + IR2_OPND index = ra_alloc_itemp(); + IR2_OPND first_index = ra_alloc_itemp(); + IR2_OPND base = ra_alloc_itemp(); + IR2_OPND entry = ra_alloc_itemp(); + IR2_OPND native_target = ra_alloc_itemp(); + IR2_OPND desired = ra_alloc_itemp(); + IR2_OPND exit_target = ra_alloc_data(); + IR2_OPND code_base = ra_alloc_data(); + IR2_OPND label_loop = ra_alloc_label(); + IR2_OPND label_next = ra_alloc_label(); + IR2_OPND label_miss = ra_alloc_label(); + + li_d(base, (ADDR)latx_shadow_jmp_entries); + li_d(native_target, LATX_SHADOW_JMP_HASH_MULT); + la_mul_d(index, next_x86_addr, native_target); + la_srli_d(index, index, 64 - LATX_SHADOW_JMP_BITS); + la_or(first_index, index, zero_ir2_opnd); + + la_label(label_loop); + la_alsl_d(entry, index, base, 2); + la_ld_d(native_target, entry, offsetof(LatxShadowJmpEntry, tb)); + la_beq(native_target, zero_ir2_opnd, label_miss); + la_addi_d(entry, native_target, -1); + la_beq(entry, zero_ir2_opnd, label_next); + la_ld_d(entry, native_target, offsetof(TranslationBlock, pc)); + la_bne(entry, next_x86_addr, label_next); + /* + * An indirect branch normally preserves the translation mode. If it + * changes flags or cflags, rejecting the candidate is conservative and + * lets the regular dispatcher calculate the exact next-TB key. + */ + la_ld_wu(desired, current_tb, offsetof(TranslationBlock, flags)); + la_ld_wu(entry, native_target, offsetof(TranslationBlock, flags)); + la_bne(entry, desired, label_next); + la_ld_wu(desired, current_tb, offsetof(TranslationBlock, cflags)); + la_ld_wu(entry, native_target, offsetof(TranslationBlock, cflags)); + la_bne(entry, desired, label_next); + la_ld_d(native_target, native_target, + offsetof(TranslationBlock, tc) + offsetof(struct tb_tc, ptr)); + /* Promote an FSHT hit so repeated jumps use the primary FastTB path. */ + la_bstrpick_d(entry, next_x86_addr, TB_JMP_CACHE_BITS - 1, 0); + la_alsl_d(entry, entry, jmp_cache_addr, 3); + la_st_d(native_target, entry, offsetof(FastTB, ptr)); + la_st_d(next_x86_addr, entry, offsetof(FastTB, pc)); + la_jirl(zero_ir2_opnd, native_target, 0); + + la_label(label_next); + la_addi_d(index, index, 1); + la_bstrpick_d(index, index, LATX_SHADOW_JMP_BITS - 1, 0); + la_beq(index, first_index, label_miss); + la_b(label_loop); + + la_label(label_miss); + la_data_li(code_base, (ADDR)code_buf); + la_data_li(exit_target, context_switch_native_to_bt_ret_0); + aot_la_append_ir2_jmp_far(exit_target, code_base, B_EPILOGUE_RET_0, 0); TRANSLATION_DATA *lat_ctx = lsenv->tr_data; label_dispose(NULL, lat_ctx); @@ -3392,6 +3470,15 @@ int generate_native_rotate_fpu_by(void *code_buf_addr) total_insts_num += insts_num; code_buf += insts_num * 4; + shadow_jmp_glue = (ADDR)code_buf; + insts_num = generate_shadow_jmp_glue(code_buf); + if (option_dump) { + qemu_log("[glue] shadow jump dispatch at %p. size = %d\n", + code_buf, insts_num); + } + total_insts_num += insts_num; + code_buf += insts_num * 4; + ss_match_fail_native = (ADDR)code_buf; insts_num = ss_generate_match_fail_native_code(code_buf); total_insts_num += insts_num; diff --git a/tests/unit/test-exclusive-timeout.c b/tests/unit/test-exclusive-timeout.c index 383c8ac5406..1bcdf784d68 100644 --- a/tests/unit/test-exclusive-timeout.c +++ b/tests/unit/test-exclusive-timeout.c @@ -16,30 +16,34 @@ bool qemu_cpu_is_self(CPUState *cpu) static void test_timeout_cancellation_allows_a_retry(void) { - CPUState current = { .cpu_index = 0 }; - CPUState blocked = { .cpu_index = 1, .running = true }; + g_autofree CPUState *current = g_new0(CPUState, 1); + g_autofree CPUState *blocked = g_new0(CPUState, 1); + + current->cpu_index = 0; + blocked->cpu_index = 1; + blocked->running = true; qemu_init_cpu_list(); - cpu_list_add(¤t); - cpu_list_add(&blocked); - current_cpu = ¤t; + cpu_list_add(current); + cpu_list_add(blocked); + current_cpu = current; g_assert_false(start_exclusive_timeout(0)); - g_assert_cmpuint(current.exclusive_context_count, ==, 0); - g_assert_false(blocked.has_waiter); + g_assert_cmpuint(current->exclusive_context_count, ==, 0); + g_assert_false(blocked->has_waiter); g_assert_false(start_exclusive_timeout(1)); - g_assert_cmpuint(current.exclusive_context_count, ==, 0); - g_assert_false(blocked.has_waiter); + g_assert_cmpuint(current->exclusive_context_count, ==, 0); + g_assert_false(blocked->has_waiter); - blocked.running = false; + blocked->running = false; g_assert_true(start_exclusive_timeout(100)); - g_assert_cmpuint(current.exclusive_context_count, ==, 1); + g_assert_cmpuint(current->exclusive_context_count, ==, 1); end_exclusive(); - g_assert_cmpuint(current.exclusive_context_count, ==, 0); - cpu_list_remove(&blocked); - cpu_list_remove(¤t); + g_assert_cmpuint(current->exclusive_context_count, ==, 0); + cpu_list_remove(blocked); + cpu_list_remove(current); } int main(int argc, char **argv)