LATX: reduce indirect jump lookup overhead

accel/tcg/tb-flush.c

@@ -89,6 +89,9 @@ void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count)
 #endif
         cpu_tb_jmp_cache_clear(cpu);
     }
+#ifdef CONFIG_LATX_FAST_JMPCACHE
+    latx_shadow_jmp_cache_clear_all();
+#endif
 
     qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE);
     tb_flush_remove_all();

accel/tcg/translate-all.c

@@ -1736,6 +1736,9 @@ static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list)
     if (!qht_remove(&tb_ctx.htable, tb, h)) {
         return;
     }
+#ifdef CONFIG_LATX_FAST_JMPCACHE
+    latx_shadow_jmp_cache_remove(tb);
+#endif
 
     /* remove the TB from the page list */
     if (rm_from_page_list) {
@@ -1867,6 +1870,10 @@ tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
     if (unlikely(existing_tb)) {
         tb_remove(tb);
         tb = existing_tb;
+#ifdef CONFIG_LATX_FAST_JMPCACHE
+    } else {
+        latx_shadow_jmp_cache_add(tb);
+#endif
     }
 
     if (p2 && p2 != p) {

configure

@@ -5685,6 +5685,11 @@ if test "$latx" = "yes" ; then
     echo "CONFIG_LATX_KZT=y" >> $config_host_mak
   fi
   echo "CONFIG_LATX=y" >> $config_host_mak
+  case " $target_list " in
+    *" x86_64-linux-user "*)
+      echo "CONFIG_LATX_X86_64=y" >> $config_host_mak
+      ;;
+  esac
   if test "$debug" = "yes" ; then
     echo "CONFIG_LATX_DEBUG=y" >> $config_host_mak
   else

include/exec/fasttb.h

@@ -10,9 +10,27 @@ struct FastTB {
 
 #define FASTTB_ILLINST_MAGIC 0x88888888
 
+#ifndef LATX_SHADOW_JMP_BITS
+#define LATX_SHADOW_JMP_BITS 20
+#endif
+#define LATX_SHADOW_JMP_SIZE (1U << LATX_SHADOW_JMP_BITS)
+#define LATX_SHADOW_JMP_HASH_MULT UINT64_C(11400714819323198485)
+#define LATX_SHADOW_JMP_TOMBSTONE ((struct TranslationBlock *)1)
+
+typedef struct LatxShadowJmpEntry {
+    struct TranslationBlock *tb;
+} LatxShadowJmpEntry;
+
+extern LatxShadowJmpEntry latx_shadow_jmp_entries[];
+
 void latx_fast_jmp_cache_add(CPUState *cs, int h, struct TranslationBlock *tb);
 void latx_fast_jmp_cache_clear(CPUState *cs, int h);
 void latx_fast_jmp_cache_clear_all(CPUState *cs);
 bool latx_fast_jmp_cache_init(void *env);
 void latx_fast_jmp_cache_free_rcu(void *ptr);
+void latx_shadow_jmp_cache_add(struct TranslationBlock *tb);
+void latx_shadow_jmp_cache_remove(struct TranslationBlock *tb);
+void latx_shadow_jmp_cache_clear_all(void);
+struct TranslationBlock *latx_shadow_jmp_cache_lookup(
+    uint64_t pc, uint32_t flags, uint32_t cflags);
 #endif

include/exec/tb-hash.h

@@ -54,9 +54,13 @@ static inline unsigned int tb_jmp_cache_hash_func(target_ulong pc)
 /* In user-mode we can get better hashing because we do not have a TLB */
 static inline unsigned int tb_jmp_cache_hash_func(target_ulong pc)
 {
-    return (pc ^ (pc >> TB_JMP_CACHE_BITS)) & (TB_JMP_CACHE_SIZE - 1);
+    return pc & (TB_JMP_CACHE_SIZE - 1);
 }
 
+/* Instruction offsets from tb->jmp_indirect to the FastTB hit-path JIRL. */
+#define TB_JMP_CACHE_FAST_JIRL_OFFSET 5
+#define TB_JMP_CACHE_FAST_MASK_JIRL_OFFSET 9
+
 #endif /* CONFIG_SOFTMMU */
 
 static inline

include/hw/core/cpu.h

@@ -246,10 +246,15 @@ struct kvm_run;
 
 struct hax_vcpu_state;
 
-#ifdef CONFIG_LATX
+#if defined(CONFIG_LATX) && defined(CONFIG_LATX_X86_64)
+#define TB_JMP_CACHE_BITS 19
+#define TB_JMP_CACHE_HASH_SHIFT 32
+#elif defined(CONFIG_LATX)
 #define TB_JMP_CACHE_BITS 16
+#define TB_JMP_CACHE_HASH_SHIFT TB_JMP_CACHE_BITS
 #else
 #define TB_JMP_CACHE_BITS 12
+#define TB_JMP_CACHE_HASH_SHIFT TB_JMP_CACHE_BITS
 #endif
 #define TB_JMP_CACHE_SIZE (1 << TB_JMP_CACHE_BITS)
 

meson.build

@@ -676,6 +676,23 @@ foreach target : target_dirs
       suite: 'lat-pr-fast',
     )
 
+    shadow_jmp_cache_test = executable(
+      'test-latx-shadow-jmp-cache',
+      files(
+        'target/i386/latx/sbt/tests/shadow-jmp-cache-test.c',
+        'target/i386/latx/shadow-jmp-cache.c'
+      ) + genh + tcg_trace_genh + [syscall_nr_generated],
+      c_args: smc_reload_test_c_args + ['-DLATX_SHADOW_JMP_BITS=3'],
+      dependencies: [glib],
+      include_directories: target_inc,
+      link_args: smc_reload_test_link_args
+    )
+    test(
+      'latx-shadow-jmp-cache',
+      shadow_jmp_cache_test,
+      suite: 'lat-pr-fast',
+    )
+
     aot_file_publish_test = executable(
       'test-latx-aot-file-publish',
       files(

target/i386/latx/include/aot.h

@@ -264,6 +264,8 @@ typedef enum aot_rel_kind {
     LOAD_HELPER_END,
 
     LOAD_TUNNEL_ADDR_BEGIN,
+    /* Keep this outside the dense tunnel relocation range. */
+    B_SHADOW_JMP_GLUE = 0x10000,
 } aot_rel_kind;
 
 typedef struct aot_rel {

target/i386/latx/include/translate.h

@@ -1706,6 +1706,7 @@ void tr_generate_goto_tb(void);                          /* TODO */
 extern ADDR native_rotate_fpu_by;
 extern ADDR indirect_jmp_glue;
 extern ADDR parallel_indirect_jmp_glue;
+extern ADDR shadow_jmp_glue;
 void rotate_fpu_to_top(int top);
 void rotate_fpu_by(int step);
 void rotate_fpu_to_bias(int bias);

target/i386/latx/latx-signal.c

@@ -160,13 +160,13 @@ void unlink_indirect_jmp(CPUArchState *env, TranslationBlock *tb, ucontext_t *uc
 
 #ifdef CONFIG_LATX_FAST_JMPCACHE
 #ifdef CONFIG_LATX_GLUE_MASK
-    /* Glue-masked fast jmpcache lands in jirl after 12 instructions. */
-    insn = qatomic_read((uint32_t *)(jmp_rx + INS_SIZE * 11));
-    jmp_rw += INS_SIZE * 11;
+    insn = qatomic_read((uint32_t *)(jmp_rx +
+                INS_SIZE * TB_JMP_CACHE_FAST_MASK_JIRL_OFFSET));
+    jmp_rw += INS_SIZE * TB_JMP_CACHE_FAST_MASK_JIRL_OFFSET;
 #else
-    /* Unmasked fast jmpcache reaches jirl after the eight-instruction hit path. */
-    insn = qatomic_read((uint32_t *)(jmp_rx + INS_SIZE * 7));
-    jmp_rw += INS_SIZE * 7;
+    insn = qatomic_read((uint32_t *)(jmp_rx +
+                INS_SIZE * TB_JMP_CACHE_FAST_JIRL_OFFSET));
+    jmp_rw += INS_SIZE * TB_JMP_CACHE_FAST_JIRL_OFFSET;
 #endif
 #else
     CPUState *cpu = env_cpu(env);

target/i386/latx/meson.build

@@ -68,6 +68,7 @@ i386_ss.add(when: 'CONFIG_LATX', if_true: files(
   'error.c',
   'mem.c',
   'latx-config.c',
+  'shadow-jmp-cache.c',
   'latx-options.c',
   'latx-perf.c',
   'latx-name-demangling.cpp',

target/i386/latx/sbt/aot.c

@@ -1310,6 +1310,7 @@ void aot_do_tb_reloc(TranslationBlock *tb, struct aot_tb *stb,
     uintptr_t helper_address;
     int lib_method_index;
     ADDR loacl_indirect_jmp_glue = indirect_jmp_glue;
+    ADDR local_shadow_jmp_glue = shadow_jmp_glue;
 
     aot_rel_table = aot_buffer +
         ((aot_header *)aot_buffer)->rel_table_offset;
@@ -1376,6 +1377,13 @@ void aot_do_tb_reloc(TranslationBlock *tb, struct aot_tb *stb,
             light_tb_target_set_jmp_target((uintptr_t)tb->tc.ptr,
             (uintptr_t)pinsn, (uintptr_t)pinsn, loacl_indirect_jmp_glue);
             break;
+        case B_SHADOW_JMP_GLUE:
+            lsassert(((*pinsn) & 0xfc000000) == 0x50000000 ||
+            (((*pinsn) & 0xfe000000) == 0x1e000000 &&
+            (*(pinsn + 1) & 0xfc000000) == 0x4c000000));
+            light_tb_target_set_jmp_target((uintptr_t)tb->tc.ptr,
+            (uintptr_t)pinsn, (uintptr_t)pinsn, local_shadow_jmp_glue);
+            break;
         case LOAD_TB_ADDR:
             lsassert(aot_rel_table[i].rel_slots_num == 3);
             lsassert((*pinsn & 0xfe000000) == 0x14000000); /* lu12i.w */

target/i386/latx/sbt/aot_merge.c

@@ -215,6 +215,7 @@ static gboolean merge_dump_rel_table_tree_node(gpointer key, gpointer val,
             break;
 
         case B_NATIVE_JMP_GLUE2:
+        case B_SHADOW_JMP_GLUE:
             lsassert(((*pinsn) & 0xfc000000) == 0x50000000 ||
               (((*pinsn) & 0xfe000000) == 0x1e000000 &&/* pcaddu18i */
               (*(pinsn + 1) & 0xfc000000) == 0x4c000000));

target/i386/latx/sbt/tests/shadow-jmp-cache-test.c

@@ -0,0 +1,118 @@
+#include "qemu/osdep.h"
+
+#include "exec/exec-all.h"
+#include "exec/fasttb.h"
+
+static void init_tb(TranslationBlock *tb, target_ulong pc,
+                    uint32_t flags, uint32_t cflags, uintptr_t code)
+{
+    memset(tb, 0, sizeof(*tb));
+    tb->pc = pc;
+    tb->flags = flags;
+    tb->cflags = cflags;
+    tb->tc.ptr = (void *)code;
+}
+
+static unsigned int test_hash(target_ulong pc)
+{
+    return (pc * LATX_SHADOW_JMP_HASH_MULT) >>
+           (64 - LATX_SHADOW_JMP_BITS);
+}
+
+static target_ulong find_collision(target_ulong pc, target_ulong start)
+{
+    unsigned int wanted = test_hash(pc);
+
+    for (target_ulong candidate = start; ; candidate++) {
+        if (candidate != pc && test_hash(candidate) == wanted) {
+            return candidate;
+        }
+    }
+}
+
+static void test_distinct_keys_and_tombstone(void)
+{
+    TranslationBlock a, b, c, replacement, collided;
+    target_ulong collision_pc = find_collision(0x1000, 0x2000);
+
+    init_tb(&a, 0x1000, 0x11, 0x21, 0x100000);
+    init_tb(&b, 0x1000, 0x12, 0x21, 0x200000);
+    init_tb(&c, 0x1000, 0x11, 0x22, 0x300000);
+    init_tb(&replacement, 0x1000, 0x11, 0x22, 0x400000);
+    init_tb(&collided, collision_pc, 0x31, 0x41, 0x500000);
+
+    latx_shadow_jmp_cache_clear_all();
+    latx_shadow_jmp_cache_add(&a);
+    latx_shadow_jmp_cache_add(&b);
+    latx_shadow_jmp_cache_add(&c);
+
+    g_assert_true(latx_shadow_jmp_cache_lookup(a.pc, a.flags,
+                                               tb_cflags(&a)) == &a);
+    g_assert_true(latx_shadow_jmp_cache_lookup(b.pc, b.flags,
+                                               tb_cflags(&b)) == &b);
+    g_assert_true(latx_shadow_jmp_cache_lookup(c.pc, c.flags,
+                                               tb_cflags(&c)) == &c);
+    g_assert_null(latx_shadow_jmp_cache_lookup(a.pc, 0xff, tb_cflags(&a)));
+
+    c.cflags |= CF_INVALID;
+    g_assert_null(latx_shadow_jmp_cache_lookup(c.pc, c.flags, 0x22));
+    c.cflags &= ~CF_INVALID;
+
+    latx_shadow_jmp_cache_remove(&b);
+    g_assert_null(latx_shadow_jmp_cache_lookup(b.pc, b.flags, tb_cflags(&b)));
+    g_assert_true(latx_shadow_jmp_cache_lookup(c.pc, c.flags,
+                                               tb_cflags(&c)) == &c);
+
+    latx_shadow_jmp_cache_add(&collided);
+    g_assert_true(latx_shadow_jmp_cache_lookup(
+                      collided.pc, collided.flags,
+                      tb_cflags(&collided)) == &collided);
+
+    latx_shadow_jmp_cache_add(&replacement);
+    g_assert_true(latx_shadow_jmp_cache_lookup(
+                      replacement.pc, replacement.flags,
+                      tb_cflags(&replacement)) == &replacement);
+    latx_shadow_jmp_cache_remove(&c);
+    g_assert_true(latx_shadow_jmp_cache_lookup(
+                      replacement.pc, replacement.flags,
+                      tb_cflags(&replacement)) == &replacement);
+}
+
+static void test_full_table(void)
+{
+    TranslationBlock tb[LATX_SHADOW_JMP_SIZE + 1];
+
+    latx_shadow_jmp_cache_clear_all();
+    for (unsigned int i = 0; i < LATX_SHADOW_JMP_SIZE; i++) {
+        init_tb(&tb[i], 0x10000 + i, i, 0x80 + i, 0x100000 + i * 16);
+        latx_shadow_jmp_cache_add(&tb[i]);
+    }
+    for (unsigned int i = 0; i < LATX_SHADOW_JMP_SIZE; i++) {
+        g_assert_true(latx_shadow_jmp_cache_lookup(
+                          tb[i].pc, tb[i].flags,
+                          tb_cflags(&tb[i])) == &tb[i]);
+    }
+
+    init_tb(&tb[LATX_SHADOW_JMP_SIZE], 0x20000, 0xaa, 0xbb, 0x200000);
+    g_assert_null(latx_shadow_jmp_cache_lookup(0x30000, 0xcc, 0xdd));
+    latx_shadow_jmp_cache_add(&tb[LATX_SHADOW_JMP_SIZE]);
+    g_assert_null(latx_shadow_jmp_cache_lookup(
+        tb[LATX_SHADOW_JMP_SIZE].pc,
+        tb[LATX_SHADOW_JMP_SIZE].flags,
+        tb_cflags(&tb[LATX_SHADOW_JMP_SIZE])));
+
+    latx_shadow_jmp_cache_remove(&tb[3]);
+    latx_shadow_jmp_cache_add(&tb[LATX_SHADOW_JMP_SIZE]);
+    g_assert_true(latx_shadow_jmp_cache_lookup(
+                      tb[LATX_SHADOW_JMP_SIZE].pc,
+                      tb[LATX_SHADOW_JMP_SIZE].flags,
+                      tb_cflags(&tb[LATX_SHADOW_JMP_SIZE])) ==
+                  &tb[LATX_SHADOW_JMP_SIZE]);
+}
+
+int main(void)
+{
+    test_distinct_keys_and_tombstone();
+    test_full_table();
+    return 0;
+}

target/i386/latx/sbt/tests/tb-flush-smc-reload-async-test.c

@@ -97,6 +97,10 @@ void latx_fast_jmp_cache_clear_all(CPUState *cpu G_GNUC_UNUSED)
 {
 }
 
+void latx_shadow_jmp_cache_clear_all(void)
+{
+}
+
 static SMCReloadInfo *new_reload_node(target_ulong page_addr)
 {
     SMCReloadInfo *node = g_new0(SMCReloadInfo, 1);
@@ -116,22 +120,22 @@ static void run_queued_flush(CPUState *cpu)
 
 int main(void)
 {
-    CPUState cpu = { 0 };
+    g_autofree CPUState *cpu = g_new0(CPUState, 1);
 
     option_smc_reload = true;
     tb_ctx.tb_flush_count = 1;
 
-    tb_flush(&cpu);
+    tb_flush(cpu);
     g_assert(queued_func != NULL);
     smc_reload_tree_insert(new_reload_node(0x1000));
-    run_queued_flush(&cpu);
+    run_queued_flush(cpu);
     g_assert(smc_reload_tree_get_node_count() == 0);
 
     smc_reload_tree_insert(new_reload_node(0x2000));
-    tb_flush(&cpu);
+    tb_flush(cpu);
     g_assert(queued_func != NULL);
     tb_ctx.tb_flush_count++;
-    run_queued_flush(&cpu);
+    run_queued_flush(cpu);
     g_assert(smc_reload_tree_get_node_count() == 1);
     smc_reload_tree_clear();
     return 0;