From 5d2f2d778673f8547b8c344a3998b646493ec932 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Fri, 28 Aug 2026 17:24:52 +0800 Subject: [PATCH 01/17] LATX, AVX: defer redundant VEX.128 upper-half clears Track VEX.128 destinations whose architectural YMM high halves are known to be zero, and materialize those clears only when a 256-bit operation can observe them or before leaving the TB. This removes repeated LASX clear instructions while preserving signal, JIT, TU, and AOT-visible state. Add a standalone JIT, cold-AOT, and hot-AOT semantic test for the deferred state. Signed-off-by: Lu Zeng --- target/i386/latx/include/env.h | 6 + target/i386/latx/include/translate.h | 4 + target/i386/latx/translator/tr-avx-cmp.c | 192 +++++++++--------- target/i386/latx/translator/tr-avx-cvt.c | 22 +- target/i386/latx/translator/tr-avx-mov.c | 16 +- target/i386/latx/translator/tr-avx.c | 76 +++---- target/i386/latx/translator/tr-misc.c | 1 + target/i386/latx/translator/tr-opnd-process.c | 106 +++++++++- target/i386/latx/translator/translate.c | 30 +++ tests/integration/meson.build | 1 + .../registrations/simd/meson.build | 11 + tests/integration/test-avx-ymmh-zero.sh | 41 ++++ tests/integration/vex128-ymmh-zero.S | 67 ++++++ 13 files changed, 414 insertions(+), 159 deletions(-) create mode 100644 tests/integration/registrations/simd/meson.build create mode 100755 tests/integration/test-avx-ymmh-zero.sh create mode 100644 tests/integration/vex128-ymmh-zero.S diff --git a/target/i386/latx/include/env.h b/target/i386/latx/include/env.h index 7ff6c0b7ce..dcc1da08e2 100644 --- a/target/i386/latx/include/env.h +++ b/target/i386/latx/include/env.h @@ -58,6 +58,12 @@ typedef struct TRANSLATION_DATA { int curr_top; /* top value (changes when translating) */ + /* + * VEX.128 writes make the architectural YMM high half zero. With LASX, + * defer that zeroing until the high half is observed or the TB exits. + */ + uint16_t ymmh_zero_pending; + uint8_t ymmh_zero_defer_disabled; /* TODO : support static translation */ uint8 curr_ir1_skipped_eflags; /* these eflag calculation can be skipped */ /* (because of flag pattern, etc) */ diff --git a/target/i386/latx/include/translate.h b/target/i386/latx/include/translate.h index 38200bc429..0fd4dd2704 100644 --- a/target/i386/latx/include/translate.h +++ b/target/i386/latx/include/translate.h @@ -1607,6 +1607,10 @@ IR2_OPND load_freg128_from_ir1(IR1_OPND *); void load_imm_to_ir1_opnd_gpr(IR1_OPND *opnd0, uint64_t imm); IR2_OPND load_freg256_from_ir1(IR1_OPND *opnd1); void set_high128_xreg_to_zero(IR2_OPND opnd); +void materialize_deferred_ymmh_zero(IR2_OPND opnd); +void materialize_deferred_ymmh_zeros_now(void); +void materialize_deferred_ymmh_zeros_for_exit(void); +void disable_deferred_ymmh_zero_for_tb(void); IR2_OPND load_ymm_high128_shadow(int index); void store_ymm_high128_shadow(IR2_OPND src, int index); void clear_ymm_high128_shadow(int index); diff --git a/target/i386/latx/translator/tr-avx-cmp.c b/target/i386/latx/translator/tr-avx-cmp.c index 5461b36dbd..eae7273da3 100644 --- a/target/i386/latx/translator/tr-avx-cmp.c +++ b/target/i386/latx/translator/tr-avx-cmp.c @@ -200,8 +200,8 @@ bool translate_vcmpeqpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_EQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -226,8 +226,8 @@ bool translate_vcmpltpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_LT); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -252,8 +252,8 @@ bool translate_vcmplepd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_LE); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -278,8 +278,8 @@ bool translate_vcmpunordpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_UNORD); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -304,8 +304,8 @@ bool translate_vcmpneqpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NEQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -331,8 +331,8 @@ bool translate_vcmpnltpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_NLT); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -357,8 +357,8 @@ bool translate_vcmpnlepd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_NLE); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -383,8 +383,8 @@ bool translate_vcmpordpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_ORD); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -409,8 +409,8 @@ bool translate_vcmpeq_uqpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_EQ_UQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -435,8 +435,8 @@ bool translate_vcmpngepd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NGE); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -461,8 +461,8 @@ bool translate_vcmpngtpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NGT); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -487,8 +487,8 @@ bool translate_vcmpfalsepd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_FALSE); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -513,8 +513,8 @@ bool translate_vcmpneq_oqpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NEQ_OQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -539,8 +539,8 @@ bool translate_vcmpgepd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_GE); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -565,8 +565,8 @@ bool translate_vcmpgtpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_GT); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -594,8 +594,8 @@ bool translate_vcmptruepd(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_TRUE); la_xvori_b(temp, temp, 0xff); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -620,8 +620,8 @@ bool translate_vcmpeq_ospd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_EQ_OS); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -646,8 +646,8 @@ bool translate_vcmplt_oqpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_LT_OQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -672,8 +672,8 @@ bool translate_vcmple_oqpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_LE_OQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -698,8 +698,8 @@ bool translate_vcmpunord_spd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_UNORD_S); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -724,8 +724,8 @@ bool translate_vcmpneq_uspd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NEQ_US); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -750,8 +750,8 @@ bool translate_vcmpnlt_uqpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_NLT_UQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -776,8 +776,8 @@ bool translate_vcmpnle_uqpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_NLE_UQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -802,8 +802,8 @@ bool translate_vcmpord_spd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_ORD_S); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -828,8 +828,8 @@ bool translate_vcmpeq_uspd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_EQ_US); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -854,8 +854,8 @@ bool translate_vcmpnge_uqpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NGE_UQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -880,8 +880,8 @@ bool translate_vcmpngt_uqpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NGT_UQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -906,8 +906,8 @@ bool translate_vcmpfalse_ospd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_FALSE_OS); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -932,8 +932,8 @@ bool translate_vcmpneq_ospd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_NEQ_OS); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -958,8 +958,8 @@ bool translate_vcmpge_oqpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_GE_OQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -984,8 +984,8 @@ bool translate_vcmpgt_oqpd(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src2, src1, X86_FCMP_COND_GT_OQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1013,8 +1013,8 @@ bool translate_vcmptrue_uspd(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_d(temp, src1, src2, X86_FCMP_COND_TRUE_US); la_xvori_b(temp, temp, 0xff); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1122,8 +1122,8 @@ bool translate_vcmpeqps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_EQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1148,8 +1148,8 @@ bool translate_vcmpltps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_LT); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1174,8 +1174,8 @@ bool translate_vcmpleps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_LE); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1200,8 +1200,8 @@ bool translate_vcmpunordps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_UNORD); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1226,8 +1226,8 @@ bool translate_vcmpneqps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NEQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1253,8 +1253,8 @@ bool translate_vcmpnltps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_NLT); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1279,8 +1279,8 @@ bool translate_vcmpnleps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_NLE); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1305,8 +1305,8 @@ bool translate_vcmpordps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_ORD); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1331,8 +1331,8 @@ bool translate_vcmpeq_uqps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_EQ_UQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1357,8 +1357,8 @@ bool translate_vcmpngeps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NGE); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1383,8 +1383,8 @@ bool translate_vcmpngtps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NGT); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1409,8 +1409,8 @@ bool translate_vcmpfalseps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_FALSE); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1435,8 +1435,8 @@ bool translate_vcmpneq_oqps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NEQ_OQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1461,8 +1461,8 @@ bool translate_vcmpgeps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_GE); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1487,8 +1487,8 @@ bool translate_vcmpgtps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_GT); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1516,8 +1516,8 @@ bool translate_vcmptrueps(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_TRUE); la_xvori_b(temp, temp, 0xff); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1542,8 +1542,8 @@ bool translate_vcmpeq_osps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_EQ_OS); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1568,8 +1568,8 @@ bool translate_vcmplt_oqps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_LT_OQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1594,8 +1594,8 @@ bool translate_vcmple_oqps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_LE_OQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1620,8 +1620,8 @@ bool translate_vcmpunord_sps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_UNORD_S); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1646,8 +1646,8 @@ bool translate_vcmpneq_usps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NEQ_US); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1672,8 +1672,8 @@ bool translate_vcmpnlt_uqps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_NLT_UQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1698,8 +1698,8 @@ bool translate_vcmpnle_uqps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_NLE_UQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1724,8 +1724,8 @@ bool translate_vcmpord_sps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_ORD_S); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1750,8 +1750,8 @@ bool translate_vcmpeq_usps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_EQ_US); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1776,8 +1776,8 @@ bool translate_vcmpnge_uqps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NGE_UQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1802,8 +1802,8 @@ bool translate_vcmpngt_uqps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NGT_UQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1828,8 +1828,8 @@ bool translate_vcmpfalse_osps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_FALSE_OS); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1854,8 +1854,8 @@ bool translate_vcmpneq_osps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_NEQ_OS); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1880,8 +1880,8 @@ bool translate_vcmpge_oqps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_GE_OQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1906,8 +1906,8 @@ bool translate_vcmpgt_oqps(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src2, src1, X86_FCMP_COND_GT_OQ); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1935,8 +1935,8 @@ bool translate_vcmptrue_usps(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_TRUE_US); la_xvori_b(temp, temp, 0xff); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -2041,8 +2041,8 @@ bool translate_vcmpeqsd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_EQ); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2067,8 +2067,8 @@ bool translate_vcmpltsd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_LT); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2093,8 +2093,8 @@ bool translate_vcmplesd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_LE); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2119,8 +2119,8 @@ bool translate_vcmpunordsd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_UNORD); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2145,8 +2145,8 @@ bool translate_vcmpneqsd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NEQ); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2171,8 +2171,8 @@ bool translate_vcmpnltsd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_NLT); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2197,8 +2197,8 @@ bool translate_vcmpnlesd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_NLE); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2223,8 +2223,8 @@ bool translate_vcmpordsd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_ORD); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2249,8 +2249,8 @@ bool translate_vcmpeq_uqsd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_EQ_UQ); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2275,8 +2275,8 @@ bool translate_vcmpngesd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NGE); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2301,8 +2301,8 @@ bool translate_vcmpngtsd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NGT); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2327,8 +2327,8 @@ bool translate_vcmpfalsesd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_FALSE); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2353,8 +2353,8 @@ bool translate_vcmpneq_oqsd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NEQ_OQ); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2379,8 +2379,8 @@ bool translate_vcmpgesd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_GE); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2405,8 +2405,8 @@ bool translate_vcmpgtsd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_GT); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2432,8 +2432,8 @@ bool translate_vcmptruesd(IR1_INST * pir1) { la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_TRUE); la_xvori_b(dest_temp, dest_temp, 0xff); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2458,8 +2458,8 @@ bool translate_vcmpeq_ossd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_EQ_OS); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2484,8 +2484,8 @@ bool translate_vcmplt_oqsd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_LT_OQ); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2510,8 +2510,8 @@ bool translate_vcmple_oqsd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_LE_OQ); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2536,8 +2536,8 @@ bool translate_vcmpunord_ssd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_UNORD_S); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2562,8 +2562,8 @@ bool translate_vcmpneq_ussd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NEQ_US); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2588,8 +2588,8 @@ bool translate_vcmpnlt_uqsd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_NLT_UQ); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2614,8 +2614,8 @@ bool translate_vcmpnle_uqsd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_NLE_UQ); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2640,8 +2640,8 @@ bool translate_vcmpord_ssd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_ORD_S); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2666,8 +2666,8 @@ bool translate_vcmpeq_ussd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_EQ_US); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2692,8 +2692,8 @@ bool translate_vcmpnge_uqsd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NGE_UQ); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2718,8 +2718,8 @@ bool translate_vcmpngt_uqsd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NGT_UQ); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2744,8 +2744,8 @@ bool translate_vcmpfalse_ossd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_FALSE_OS); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2770,8 +2770,8 @@ bool translate_vcmpneq_ossd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_NEQ_OS); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2796,8 +2796,8 @@ bool translate_vcmpge_oqsd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_GE_OQ); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2822,8 +2822,8 @@ bool translate_vcmpgt_oqsd(IR1_INST * pir1) { la_vreplve_d(temp2, src2, zero_ir2_opnd); la_vfcmp_cond_d(dest_temp, temp2, temp1, X86_FCMP_COND_GT_OQ); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); @@ -2849,8 +2849,8 @@ bool translate_vcmptrue_ussd(IR1_INST * pir1) { la_vfcmp_cond_d(dest_temp, temp1, temp2, X86_FCMP_COND_TRUE_US); la_xvori_b(dest_temp, dest_temp, 0xff); la_vshuf4i_d(dest_temp, src1, 0xc); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp1); ra_free_temp(temp2); diff --git a/target/i386/latx/translator/tr-avx-cvt.c b/target/i386/latx/translator/tr-avx-cvt.c index e903ceeddd..3d2ea81845 100644 --- a/target/i386/latx/translator/tr-avx-cvt.c +++ b/target/i386/latx/translator/tr-avx-cvt.c @@ -34,8 +34,8 @@ bool translate_vcvtps2pd(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_vfcvtl_d_s(temp, src); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -60,8 +60,8 @@ bool translate_vcvtpd2ps(IR1_INST * pir1) { la_xvpermi_q(temp, src, XVPERMI_Q_4_0(1, 1)); la_vfcvt_s_d(temp, temp, src); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } else { lsassert(0); } @@ -87,8 +87,8 @@ bool translate_vcvtdq2ps(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_vffint_s_w(temp, src); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -324,8 +324,8 @@ bool translate_vcvtps2dq(IR1_INST * pir1) { la_vinsgr2vr_w(temp, temp_int, 3); la_label(label_over); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp_fcsr); ra_free_temp(temp_int); ra_free_temp(temp_operand_count); @@ -359,8 +359,8 @@ bool translate_vcvtdq2pd(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_vffintl_d_w(temp, src); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } set_fpu_rounding_mode(fcsr_opnd); return true; @@ -499,8 +499,8 @@ bool translate_vcvtpd2dq(IR1_INST * pir1) { la_vinsgr2vr_w(temp, temp_int, 3); la_label(label_over); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp_fcsr); ra_free_temp(temp_int); ra_free_temp(ftemp_src_temp1); @@ -572,8 +572,8 @@ bool translate_vcvtsd2ss(IR1_INST * pir1) { la_fcvt_s_d(temp, src2); la_vori_b(dest_temp, src1, 0); la_xvinsve0_w(dest_temp, temp, 0); - set_high128_xreg_to_zero(dest_temp); la_xvori_b(dest, dest_temp, 0); + set_high128_xreg_to_zero(dest); set_fpu_rounding_mode(fcsr_opnd); return true; } @@ -598,8 +598,8 @@ bool translate_vcvtsi2sd(IR1_INST * pir1) { lsassert(0); } la_vshuf4i_d(temp, src1, 0xc); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); return true; } @@ -616,8 +616,8 @@ bool translate_vcvtss2sd(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_fcvt_d_s(temp, src2); la_vshuf4i_d(temp, src1, 0xc); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); return true; } @@ -762,8 +762,8 @@ bool translate_vcvttpd2dq(IR1_INST * pir1) { la_vinsgr2vr_w(temp, temp_int, 3); la_label(label_over); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp_fcsr); ra_free_temp(temp_int); ra_free_temp(ftemp_src_temp1); @@ -1052,8 +1052,8 @@ bool translate_vcvttps2dq(IR1_INST * pir1) { la_vinsgr2vr_w(temp, temp_int, 3); la_label(label_over); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp(temp_fcsr); ra_free_temp(temp_int); ra_free_temp(temp_operand_count); diff --git a/target/i386/latx/translator/tr-avx-mov.c b/target/i386/latx/translator/tr-avx-mov.c index a85e1f7ed8..aa05da9991 100644 --- a/target/i386/latx/translator/tr-avx-mov.c +++ b/target/i386/latx/translator/tr-avx-mov.c @@ -81,8 +81,8 @@ bool translate_vmovaps(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); load_freg128_from_ir1_mem(temp, src); - set_high128_xreg_to_zero(temp); la_xvori_b(ra_alloc_xmm(ir1_opnd_base_reg_num(dest)), temp, 0); + set_high128_xreg_to_zero(ra_alloc_xmm(ir1_opnd_base_reg_num(dest))); } else if (ir1_opnd_is_mem(dest) && ir1_opnd_is_xmm(src)) { store_freg128_to_ir1_mem(ra_alloc_xmm(ir1_opnd_base_reg_num(src)), dest); @@ -90,8 +90,8 @@ bool translate_vmovaps(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_vori_b(temp, ra_alloc_xmm(ir1_opnd_base_reg_num(src)), 0); - set_high128_xreg_to_zero(temp); la_xvori_b(ra_alloc_xmm(ir1_opnd_base_reg_num(dest)), temp, 0); + set_high128_xreg_to_zero(ra_alloc_xmm(ir1_opnd_base_reg_num(dest))); } else { #ifdef CONFIG_LATX_TS return false; @@ -212,8 +212,8 @@ bool translate_vmovshdup(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_xvpackod_w(temp, src, src); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -235,8 +235,8 @@ bool translate_vmovsldup(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_xvpackev_w(temp, src, src); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -259,8 +259,8 @@ bool translate_vmovddup(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_xvrepl128vei_d(temp, src, 0); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -319,8 +319,8 @@ bool translate_vmovss(IR1_INST *pir1) IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); la_xvori_b(temp, src1, 0); la_xvinsve0_w(temp, src2, 0); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } else if (ir1_opnd_is_xmm(opnd0) && ir1_opnd_is_mem(opnd1)) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); la_xvpickve_w(dest, src1, 0); @@ -464,8 +464,8 @@ bool translate_vmaskmovpx(IR1_INST * pir1) { tr_slt(mask, src1, 0); la_xvbitsel_v(temp, temp, src2, mask); if (ir1_opnd_is_xmm(opnd0)) - set_high128_xreg_to_zero(temp); - la_xvori_b(dest, temp, 0); + la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } else if (ir1_opnd_is_mem(opnd0)) { temp = load_freg256_from_ir1(opnd0); tr_slt(mask, src1, 0); diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 7c809154e5..713415bd7b 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -37,8 +37,8 @@ bool translate_vaddpd(IR1_INST * pir1) { src2 = load_freg128_from_ir1(opnd2); la_vfadd_d(temp, src1, src2); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -67,8 +67,8 @@ bool translate_vaddps(IR1_INST * pir1) { src2 = load_freg128_from_ir1(opnd2); la_vfadd_s(temp, src1, src2); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -87,8 +87,8 @@ bool translate_vaddsd(IR1_INST * pir1) { la_fadd_d(temp, src1, src2); la_vshuf4i_d(temp, src1, 0xc); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); return true; } @@ -138,8 +138,8 @@ bool translate_vsubpd(IR1_INST * pir1) { src2 = load_freg128_from_ir1(opnd2); la_vfsub_d(temp, src1, src2); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -168,8 +168,8 @@ bool translate_vsubps(IR1_INST * pir1) { src2 = load_freg128_from_ir1(opnd2); la_vfsub_s(temp, src1, src2); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -240,8 +240,8 @@ bool translate_vmulpd(IR1_INST * pir1) { src2 = load_freg128_from_ir1(opnd2); la_vfmul_d(temp, src1, src2); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -270,8 +270,8 @@ bool translate_vmulps(IR1_INST * pir1) { src2 = load_freg128_from_ir1(opnd2); la_vfmul_s(temp, src1, src2); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -289,8 +289,8 @@ bool translate_vmulsd(IR1_INST * pir1) { la_fmul_d(temp, src1, src2); la_vshuf4i_d(temp, src1, 0xc); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); return true; } @@ -340,8 +340,8 @@ bool translate_vdivpd(IR1_INST * pir1) { src2 = load_freg128_from_ir1(opnd2); la_vfdiv_d(temp, src1, src2); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -370,8 +370,8 @@ bool translate_vdivps(IR1_INST * pir1) { src2 = load_freg128_from_ir1(opnd2); la_vfdiv_s(temp, src1, src2); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -388,8 +388,8 @@ bool translate_vdivsd(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_fdiv_d(temp, src1, src2); la_vshuf4i_d(temp, src1, 0xc); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); return true; } @@ -434,8 +434,8 @@ bool translate_vsqrtpd(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_vfsqrt_d(temp, src); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -459,8 +459,8 @@ bool translate_vsqrtps(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_vfsqrt_s(temp, src); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -614,8 +614,8 @@ bool translate_vhaddpd(IR1_INST * pir1) { la_vpickev_d(temp1, src2, src1); la_vpickod_d(temp2, src2, src1); la_vfadd_d(temp, temp1, temp2); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -652,8 +652,8 @@ bool translate_vhaddps(IR1_INST * pir1) { la_vpickev_w(temp1, src2, src1); la_vpickod_w(temp2, src2, src1); la_vfadd_s(temp, temp1, temp2); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -690,8 +690,8 @@ bool translate_vhsubpd(IR1_INST * pir1) { la_vpickev_d(temp1, src2, src1); la_vpickod_d(temp2, src2, src1); la_vfsub_d(temp, temp1, temp2); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -728,8 +728,8 @@ bool translate_vhsubps(IR1_INST * pir1) { la_vpickev_w(temp1, src2, src1); la_vpickod_w(temp2, src2, src1); la_vfsub_s(temp, temp1, temp2); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -757,8 +757,8 @@ bool translate_vandnpd(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_vandn_v(temp, src1, src2); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -795,8 +795,8 @@ bool translate_vandpd(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_vand_v(temp, src1, src2); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -833,8 +833,8 @@ bool translate_vorps(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_vor_v(temp, src1, src2); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -871,8 +871,8 @@ bool translate_vxorps(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_vxor_v(temp, src1, src2); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1090,8 +1090,8 @@ bool translate_vblendvpd(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_vslti_d(temp, src3, 0); la_vbitsel_v(temp, src1, src2, temp); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1229,8 +1229,8 @@ bool translate_vbroadcastss(IR1_INST * pir1) { IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND temp = ra_alloc_ftemp(); la_xvreplve0_w(temp, src); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1278,8 +1278,8 @@ bool translate_vextractf128(IR1_INST * pir1) { } else { la_xvpermi_q(temp, src, VEXTRINS_IMM_4_0(3, 1)); } - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } else { if (!imm) { store_freg128_to_ir1_mem(src, ir1_get_opnd(pir1, 0)); @@ -1478,8 +1478,8 @@ bool translate_vshufpd(IR1_INST * pir1) { } la_xvori_b(temp, src1, 0); la_vshuf4i_d(temp, src2, imm8); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1513,8 +1513,8 @@ bool translate_vshufps(IR1_INST * pir1) { la_vshuf4i_w(temp1, src1, imm8); la_vshuf4i_w(temp2, src2, imm8 >> 4); la_vpickev_d(temp1, temp2, temp1); - set_high128_xreg_to_zero(temp1); la_xvori_b(dest, temp1, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1546,8 +1546,8 @@ bool translate_vunpckhpd(IR1_INST * pir1) { la_vori_b(temp, src1, 0); la_vshuf4i_d(temp, src2, 0xd); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1575,8 +1575,8 @@ bool translate_vunpckhps(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_vilvh_w(temp, src2, src1); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1608,8 +1608,8 @@ bool translate_vunpcklpd(IR1_INST * pir1) { la_vori_b(temp, src1, 0); la_vshuf4i_d(temp, src2, 0x8); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1637,8 +1637,8 @@ bool translate_vunpcklps(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_vilvl_w(temp, src2, src1); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); } return true; } @@ -1821,8 +1821,8 @@ bool translate_vpand(IR1_INST * pir1) { } IR2_OPND temp = ra_alloc_ftemp(); la_xvand_v(temp, src1, src2); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); return true; } @@ -1846,8 +1846,8 @@ bool translate_vpandn(IR1_INST * pir1) { } IR2_OPND temp = ra_alloc_ftemp(); la_xvandn_v(temp, src1, src2); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); return true; } @@ -1936,8 +1936,8 @@ bool translate_vpblendvb(IR1_INST * pir1) { la_xvslti_b(temp, src3, 0); la_xvbitsel_v(temp, src1, src2, temp); if (ir1_opnd_is_xmm(opnd0)) - set_high128_xreg_to_zero(temp); - la_xvori_b(dest, temp, 0); + la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); return true; } @@ -2017,8 +2017,8 @@ bool translate_vpblendw(IR1_INST * pir1) { la_xvextrins_h(temp, src2, VEXTRINS_IMM_4_0(7, 7)); if (ir1_opnd_is_xmm(opnd0)) - set_high128_xreg_to_zero(temp); - la_xvori_b(dest, temp, 0); + la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); return true; } @@ -5075,8 +5075,8 @@ bool translate_vmpsadbw(IR1_INST *pir1) la_vhaddw_wu_hu(temp_dest, temp3, temp3); la_vsrlni_h_w(temp_dest, temp2, 0); if(ir1_opnd_is_xmm(opnd0)){ - set_high128_xreg_to_zero(temp_dest); la_xvori_b(dest, temp_dest, 0); + set_high128_xreg_to_zero(dest); return true; } imm = imm>>3; @@ -5743,8 +5743,8 @@ bool translate_vpclmulqdq(IR1_INST * pir1) { la_vxor_v(temp, temp, temp); la_vinsgr2vr_d(temp, res_lo, 0); la_vinsgr2vr_d(temp, res_hi, 1); - set_high128_xreg_to_zero(temp); la_xvori_b(dest, temp, 0); + set_high128_xreg_to_zero(dest); ra_free_temp_auto(src2); ra_free_temp(temp); ra_free_temp(ftemp); diff --git a/target/i386/latx/translator/tr-misc.c b/target/i386/latx/translator/tr-misc.c index 12c9e8b663..3507488c0a 100644 --- a/target/i386/latx/translator/tr-misc.c +++ b/target/i386/latx/translator/tr-misc.c @@ -71,6 +71,7 @@ static void tr_generate_exit_tb_for_bridge(void) TranslationBlock *tb = lsenv->tr_data->curr_tb; IR2_OPND base = ra_alloc_data(); IR2_OPND target = ra_alloc_data(); + materialize_deferred_ymmh_zeros_for_exit(); la_data_li(base, (ADDR)tb->tc.ptr); if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { la_data_li(target, indirect_jmp_glue); diff --git a/target/i386/latx/translator/tr-opnd-process.c b/target/i386/latx/translator/tr-opnd-process.c index 3777fcf2d6..9e9d6391a1 100644 --- a/target/i386/latx/translator/tr-opnd-process.c +++ b/target/i386/latx/translator/tr-opnd-process.c @@ -1345,6 +1345,7 @@ void store_freg256_to_ir1_mem(IR2_OPND opnd2, IR1_OPND * opnd1) { int little_disp; lsassert(ir1_opnd_is_mem(opnd1)); lsassert(ir2_opnd_is_freg( & opnd2)); + materialize_deferred_ymmh_zero(opnd2); IR2_OPND mem_opnd = convert_mem(opnd1, & little_disp); gen_test_page_flag(mem_opnd, little_disp, PAGE_WRITE | PAGE_WRITE_ORG); la_xvst(opnd2, mem_opnd, little_disp); @@ -1362,7 +1363,9 @@ void load_freg256_from_ir1_mem(IR2_OPND opnd2, IR1_OPND * opnd1) { IR2_OPND load_freg256_from_ir1(IR1_OPND * opnd1) { if (ir1_opnd_is_ymm(opnd1)) { - return ra_alloc_xmm(ir1_opnd_base_reg_num(opnd1)); + IR2_OPND ret_opnd = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd1)); + materialize_deferred_ymmh_zero(ret_opnd); + return ret_opnd; } else if (ir1_opnd_is_mem(opnd1) || ir1_opnd_is_xmm(opnd1)) { IR2_OPND ret_opnd; if (ir1_opnd_size(opnd1) == 256) { @@ -1376,12 +1379,103 @@ IR2_OPND load_freg256_from_ir1(IR1_OPND * opnd1) { g_assert_not_reached(); } -void set_high128_xreg_to_zero(IR2_OPND opnd2) { - lsassert(ir2_opnd_is_freg( & opnd2)); - if (option_enable_lasx) { - la_xvinsgr2vr_d(opnd2, zero_ir2_opnd, 2); - la_xvinsgr2vr_d(opnd2, zero_ir2_opnd, 3); +static int mapped_ymm_index(IR2_OPND opnd) +{ + int reg = ir2_opnd_base_reg_num(&opnd); + + if (reg >= XMM0_MAPS && reg <= XMM15_MAPS) { + return reg - XMM0_MAPS; + } + return -1; +} + +static void clear_high128_xreg_now(IR2_OPND opnd) +{ + la_xvinsgr2vr_d(opnd, zero_ir2_opnd, 2); + la_xvinsgr2vr_d(opnd, zero_ir2_opnd, 3); +} + +void materialize_deferred_ymmh_zero(IR2_OPND opnd) +{ + int index; + + if (!option_enable_lasx) { + return; + } + index = mapped_ymm_index(opnd); + if (index >= 0 && + (lsenv->tr_data->ymmh_zero_pending & (UINT16_C(1) << index))) { + clear_high128_xreg_now(opnd); + } +} + +static void emit_deferred_ymmh_zeros(void) +{ + uint16_t pending; + + if (!option_enable_lasx) { + return; + } + + pending = lsenv->tr_data->ymmh_zero_pending; + for (int index = 0; index < CPU_NB_REGS; ++index) { + IR2_OPND opnd; + + if (!(pending & (UINT16_C(1) << index))) { + continue; + } + ir2_opnd_build(&opnd, IR2_OPND_FPR, reg_xmm_map[index]); + clear_high128_xreg_now(opnd); + } +} + +void materialize_deferred_ymmh_zeros_for_exit(void) +{ + /* + * Do not consume the translation-time mask here. Conditional branches + * emit two separate exit paths, and both paths must perform the clears. + */ + emit_deferred_ymmh_zeros(); +} + +void materialize_deferred_ymmh_zeros_now(void) +{ + emit_deferred_ymmh_zeros(); + lsenv->tr_data->ymmh_zero_pending = 0; +} + +void disable_deferred_ymmh_zero_for_tb(void) +{ + if (!option_enable_lasx || lsenv->tr_data->ymmh_zero_defer_disabled) { + return; + } + + materialize_deferred_ymmh_zeros_now(); + lsenv->tr_data->ymmh_zero_defer_disabled = 1; +} + +void set_high128_xreg_to_zero(IR2_OPND opnd) +{ + int index; + + lsassert(ir2_opnd_is_freg(&opnd)); + if (!option_enable_lasx) { + return; } + + if (lsenv->tr_data->ymmh_zero_defer_disabled) { + clear_high128_xreg_now(opnd); + return; + } + + index = mapped_ymm_index(opnd); + if (index >= 0) { + lsenv->tr_data->ymmh_zero_pending |= UINT16_C(1) << index; + return; + } + + /* Floating-point temporaries can be consumed as 256-bit values now. */ + clear_high128_xreg_now(opnd); } static IR2_OPND get_ymm_high128_shadow_addr(int index) diff --git a/target/i386/latx/translator/translate.c b/target/i386/latx/translator/translate.c index 38266d7bbc..71a38d08b2 100644 --- a/target/i386/latx/translator/translate.c +++ b/target/i386/latx/translator/translate.c @@ -107,6 +107,8 @@ void tr_init(void *tb) t->curr_tb); t->curr_tb = tb; t->curr_ir1_inst = NULL; + t->ymmh_zero_pending = 0; + t->ymmh_zero_defer_disabled = 0; /* register allocation init */ ra_free_all(); @@ -1867,6 +1869,28 @@ static bool (*translate_functions[])(IR1_INST *) = { bool ir1_translate(IR1_INST *ir1) { + TRANSLATION_DATA *t = lsenv->tr_data; + + /* Keep the clear in the TB body; TU/AOT may replace generated exits. */ + if (ir1_is_tb_ending(ir1)) { + materialize_deferred_ymmh_zeros_now(); + } + + if (option_enable_lasx && !t->ymmh_zero_defer_disabled) { + int opnd_num = ir1_get_opnd_num(ir1); + + for (int i = 0; i < opnd_num; ++i) { + if (ir1_opnd_size(ir1_get_opnd(ir1, i)) == 256) { + /* + * A 256-bit operation can observe or replace a physical + * high half. Keep mixed XMM/YMM TBs conservative. + */ + disable_deferred_ymmh_zero_for_tb(); + break; + } + } + } + #ifdef CONFIG_LATX_INSTS_PATTERN if (try_translate_instptn(ir1)) { ra_free_all(); @@ -2804,6 +2828,7 @@ static void tr_generate_single_step_exit(IR2_OPND next_pc) { IR2_OPND helper_addr = ra_alloc_itemp(); + materialize_deferred_ymmh_zeros_for_exit(); la_store_addrx(next_pc, env_ir2_opnd, lsenv_offset_of_eip(lsenv)); tr_save_registers_to_env(0xff, 0xff, option_save_xmm, options_to_save()); @@ -2822,6 +2847,7 @@ void tr_generate_exit_tb_to_next(IR1_INST *ir1) IR2_OPND next_pc = ra_alloc_dbt_arg2(); target_ulong call_offset = aot_get_call_offset(ir1_addr_next(ir1)); + materialize_deferred_ymmh_zeros_for_exit(); aot_load_guest_addr(next_pc, ir1_addr_next(ir1), LOAD_CALL_TARGET, call_offset); @@ -2850,6 +2876,7 @@ void tr_generate_exit_stub_tb(IR1_INST *branch, int succ_id, void *func, IR1_INS void tr_generate_exit_tb(IR1_INST *branch, int succ_id) #endif { + materialize_deferred_ymmh_zeros_for_exit(); TRANSLATION_DATA *t_data = lsenv->tr_data; (void)t_data; TranslationBlock *tb = lsenv->tr_data->curr_tb; @@ -3776,6 +3803,7 @@ void tr_save_xmm_to_env(uint8 xmm_to_save) #ifndef TARGET_X86_64 for (int i = 0; i < 8; i++) { if (BITS_ARE_SET(xmm_to_save, 1 << i)) { + materialize_deferred_ymmh_zero(ra_alloc_xmm(i)); if (option_enable_lasx) { la_xvst(ra_alloc_xmm(i), env_ir2_opnd, lsenv_offset_of_xmm(lsenv, i)); @@ -3791,6 +3819,7 @@ void tr_save_xmm_to_env(uint8 xmm_to_save) la_addi_d(tmp_env_opnd, env_ir2_opnd, 0x7f0); for (int i = 0; i < 8; i++) { if (BITS_ARE_SET(xmm_to_save, 1 << i)) { + materialize_deferred_ymmh_zero(ra_alloc_xmm(i)); if (option_enable_lasx) { la_xvst(ra_alloc_xmm(i), tmp_env_opnd, lsenv_offset_of_xmm(lsenv, i) - 0x7f0); @@ -3842,6 +3871,7 @@ void tr_save_xmm64_to_env(uint8 xmm_to_save) la_addi_d(tmp_env_opnd, env_ir2_opnd, 0x7f0); for (int i = 0; i < 8; i++) { if (BITS_ARE_SET(xmm_to_save, 1 << i)) { + materialize_deferred_ymmh_zero(ra_alloc_xmm(i + 8)); if (option_enable_lasx) { la_xvst(ra_alloc_xmm(i + 8), tmp_env_opnd, lsenv_offset_of_xmm(lsenv, i + 8) - 0x7f0); diff --git a/tests/integration/meson.build b/tests/integration/meson.build index be47597268..152f02e3f2 100644 --- a/tests/integration/meson.build +++ b/tests/integration/meson.build @@ -1,6 +1,7 @@ latx_integration_tests = [] if host_machine.cpu_family() == 'loongarch64' + subdir('registrations/simd') subdir('registrations/x11-kzt') subdir('registrations/sandbox') subdir('registrations/process') diff --git a/tests/integration/registrations/simd/meson.build b/tests/integration/registrations/simd/meson.build new file mode 100644 index 0000000000..3a5fb2ea88 --- /dev/null +++ b/tests/integration/registrations/simd/meson.build @@ -0,0 +1,11 @@ +if 'x86_64-linux-user' in target_dirs + latx_integration_tests += [{ + 'name': 'test-avx-ymmh-zero', + 'runner': find_program('../../test-avx-ymmh-zero.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../vex128-ymmh-zero.S'), + ], + 'timeout': 120, + }] +endif diff --git a/tests/integration/test-avx-ymmh-zero.sh b/tests/integration/test-avx-ymmh-zero.sh new file mode 100755 index 0000000000..55ff7e6f2b --- /dev/null +++ b/tests/integration/test-avx-ymmh-zero.sh @@ -0,0 +1,41 @@ +#!/bin/sh +set -eu + +emulator=$1 +source_file=$2 +workdir=$(mktemp -d) +trap 'rm -rf "$workdir"' EXIT HUP INT TERM + +if command -v clang-19 >/dev/null 2>&1; then + clang=clang-19 +elif command -v clang >/dev/null 2>&1; then + clang=clang +else + echo "SKIP: clang is required to build the x86_64 guest" + exit 77 +fi + +guest="$workdir/vex128-ymmh-zero" +"$clang" --target=x86_64-linux-gnu -fuse-ld=lld -nostdlib -static \ + -Wl,--build-id=none "$source_file" -o "$guest" + +LATX_AOT=0 LATX_TU=0 "$emulator" "$guest" + +aot_home="$workdir/aot" +mkdir -p "$aot_home" +HOME="$aot_home" LATX_AOT=1 LATX_TU=1 "$emulator" "$guest" + +aot_file= +for _ in $(seq 1 100); do + aot_file=$(find "$aot_home/.cache/latx" -type f -name '*.aot2' \ + -size +0c -print -quit 2>/dev/null || true) + [ -n "$aot_file" ] && break + sleep 0.1 +done +if [ -z "$aot_file" ]; then + echo "FAIL: no non-empty AOT file generated" >&2 + exit 1 +fi + +HOME="$aot_home" LATX_AOT=1 LATX_TU=1 "$emulator" "$guest" +echo "PASS: VEX.128 YMM-high zeroing JIT/cold-AOT/hot-AOT" diff --git a/tests/integration/vex128-ymmh-zero.S b/tests/integration/vex128-ymmh-zero.S new file mode 100644 index 0000000000..72c1b6e76e --- /dev/null +++ b/tests/integration/vex128-ymmh-zero.S @@ -0,0 +1,67 @@ +.intel_syntax noprefix +.global _start + +.macro CHECK_YMM_HIGH_ZERO reg, bit + vmovdqu YMMWORD PTR [rip + actual], \reg + mov rax, QWORD PTR [rip + actual + 16] + or rax, QWORD PTR [rip + actual + 24] + jz .Lcheck_done\@ + or DWORD PTR [rip + fail_count], \bit +.Lcheck_done\@: +.endm + +.section .data +fail_count: .long 0 + +.section .bss +.balign 32 +actual: .zero 32 + +.section .text +_start: + vpcmpeqd ymm0, ymm0, ymm0 + vxorps xmm1, xmm1, xmm1 + vaddps xmm0, xmm1, xmm1 + CHECK_YMM_HIGH_ZERO ymm0, 1 + + vpcmpeqd ymm0, ymm0, ymm0 + vaddps xmm0, xmm1, xmm1 + jmp .Ldirect_exit +.Ldirect_exit: + CHECK_YMM_HIGH_ZERO ymm0, 2 + + vpcmpeqd ymm0, ymm0, ymm0 + vaddps xmm0, xmm1, xmm1 + xor ecx, ecx + test ecx, ecx + jnz .Ltaken_fail + CHECK_YMM_HIGH_ZERO ymm0, 4 + jmp .Lconditional_taken +.Ltaken_fail: + inc DWORD PTR [rip + fail_count] + +.Lconditional_taken: + vpcmpeqd ymm0, ymm0, ymm0 + vaddps xmm0, xmm1, xmm1 + mov ecx, 1 + test ecx, ecx + jnz .Ltaken_ok + inc DWORD PTR [rip + fail_count] +.Ltaken_ok: + CHECK_YMM_HIGH_ZERO ymm0, 8 + + vpcmpeqd ymm0, ymm0, ymm0 + mov ecx, 500000 +.Laot_loop: + vaddps xmm0, xmm0, xmm1 + vshufps xmm0, xmm0, xmm0, 0xc9 + dec ecx + jne .Laot_loop + CHECK_YMM_HIGH_ZERO ymm0, 16 + + vzeroupper + mov edi, DWORD PTR [rip + fail_count] + mov eax, 60 + syscall + +.section .note.GNU-stack,"",@progbits From 26cd7c19156bba7a3f0c1d116acdd36fb29ce271 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Mon, 31 Aug 2026 14:10:37 +0800 Subject: [PATCH 02/17] LATX: keep mixed XMM/YMM TBs out of SHBR Signed-off-by: Lu Zeng --- target/i386/latx/optimization/hbr.c | 31 +++++++++++++++++++ tests/integration/test-avx-ymmh-zero.sh | 6 ++-- tests/integration/vex128-ymmh-zero.S | 40 +++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 2 deletions(-) diff --git a/target/i386/latx/optimization/hbr.c b/target/i386/latx/optimization/hbr.c index dcf0c0d773..cd2f640e2b 100644 --- a/target/i386/latx/optimization/hbr.c +++ b/target/i386/latx/optimization/hbr.c @@ -746,6 +746,20 @@ static void init_xmm_state(uint32_t xmm[XMM_NUM]) #include "tu.h" typedef bool (*xmm_analyse_func)(TranslationBlock *, IR1_INST *, uint32_t *); +static bool tb_has_ymm_operand(TranslationBlock *tb) +{ + for (int i = 0; i < tb_ir1_num(tb); ++i) { + IR1_INST *ir1 = tb_ir1_inst(tb, i); + + for (int j = 0; j < ir1_get_opnd_num(ir1); ++j) { + if (ir1_opnd_is_ymm(ir1_get_opnd(ir1, j))) { + return true; + } + } + } + return false; +} + /* static int xcount, scount, ncount; */ static void tb_xmm_analyse(TranslationBlock *tb, xmm_analyse_func analyse_func, uint32_t *xmm) @@ -755,6 +769,23 @@ static void tb_xmm_analyse(TranslationBlock *tb, tb->s_data->xmm_use = 0; tb->s_data->xmm_def = 0; IR1_INST *ir1 = NULL; + + /* + * SHBR models the low 128-bit XMM value only. Optimizing an XMM + * instruction in a TB that also uses YMM can leave the upper half in a + * different physical register state across TU-internal entries. Keep all + * vector registers live and skip SHBR for these mixed-width TBs. + */ + if (tb_has_ymm_operand(tb)) { + tb->s_data->xmm_use = SHBR_XMM_ALL; + for (int i = 0; i < tb_ir1_num(tb); ++i) { + ir1 = tb_ir1_inst(tb, i); + ir1->xmm_def = 0; + ir1->xmm_use = 0; + } + return; + } + for (int i = 0; i < tb_ir1_num(tb); ++i) { ir1 = tb_ir1_inst(tb, i); ir1->xmm_def = 0; diff --git a/tests/integration/test-avx-ymmh-zero.sh b/tests/integration/test-avx-ymmh-zero.sh index 55ff7e6f2b..c384f5dc54 100755 --- a/tests/integration/test-avx-ymmh-zero.sh +++ b/tests/integration/test-avx-ymmh-zero.sh @@ -37,5 +37,7 @@ if [ -z "$aot_file" ]; then exit 1 fi -HOME="$aot_home" LATX_AOT=1 LATX_TU=1 "$emulator" "$guest" -echo "PASS: VEX.128 YMM-high zeroing JIT/cold-AOT/hot-AOT" +for _ in $(seq 1 10); do + HOME="$aot_home" LATX_AOT=1 LATX_TU=1 "$emulator" "$guest" +done +echo "PASS: VEX.128 YMM-high zeroing and mixed-width HBR JIT/cold-AOT/hot-AOT" diff --git a/tests/integration/vex128-ymmh-zero.S b/tests/integration/vex128-ymmh-zero.S index 72c1b6e76e..d06b261c39 100644 --- a/tests/integration/vex128-ymmh-zero.S +++ b/tests/integration/vex128-ymmh-zero.S @@ -59,9 +59,49 @@ _start: jne .Laot_loop CHECK_YMM_HIGH_ZERO ymm0, 16 + # HBR only models XMM values. Keep a full YMM value live across the TB + # boundary created by DIV, then mix XMM and YMM operations in the next TB. + # The four qwords must survive unchanged on every TU-internal entry. + vmovdqu ymm10, YMMWORD PTR [rip + live_ymm] + vpxor ymm11, ymm11, ymm11 + mov r15d, 200000 + mov r14d, 7 +.Lmixed_width_loop: + mov eax, 123 + xor edx, edx + div r14d + vmovd xmm0, r9d + vpxor xmm13, xmm13, xmm13 + vpcmpeqd ymm14, ymm11, ymm11 + vextracti128 xmm15, ymm14, 1 + vmovq rax, xmm10 + vpextrq rcx, xmm10, 1 + vextracti128 xmm0, ymm10, 1 + vmovq rdx, xmm0 + vpextrq rsi, xmm0, 1 + cmp rax, 1 + jne .Lmixed_width_fail + cmp rcx, 2 + jne .Lmixed_width_fail + cmp rdx, 3 + jne .Lmixed_width_fail + cmp rsi, 4 + jne .Lmixed_width_fail + dec r15d + jne .Lmixed_width_loop + jmp .Lmixed_width_done +.Lmixed_width_fail: + or DWORD PTR [rip + fail_count], 32 +.Lmixed_width_done: + vzeroupper mov edi, DWORD PTR [rip + fail_count] mov eax, 60 syscall +.section .rodata +.balign 32 +live_ymm: + .quad 1, 2, 3, 4 + .section .note.GNU-stack,"",@progbits From 7c3667eb780255d0826c029be8e3a737a44604bf Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Thu, 3 Sep 2026 11:36:18 +0800 Subject: [PATCH 03/17] LATX, AVX: restore YMM mask and blend writeback Signed-off-by: Lu Zeng --- target/i386/latx/translator/tr-avx-mov.c | 7 +- target/i386/latx/translator/tr-avx.c | 14 ++- tests/integration/avx-ymm-writeback.S | 117 ++++++++++++++++++ .../registrations/simd/meson.build | 9 ++ tests/integration/test-avx-ymm-writeback.sh | 41 ++++++ 5 files changed, 179 insertions(+), 9 deletions(-) create mode 100644 tests/integration/avx-ymm-writeback.S create mode 100755 tests/integration/test-avx-ymm-writeback.sh diff --git a/target/i386/latx/translator/tr-avx-mov.c b/target/i386/latx/translator/tr-avx-mov.c index aa05da9991..a2c36c3906 100644 --- a/target/i386/latx/translator/tr-avx-mov.c +++ b/target/i386/latx/translator/tr-avx-mov.c @@ -463,9 +463,10 @@ bool translate_vmaskmovpx(IR1_INST * pir1) { la_xvandi_b(temp, temp, 0); tr_slt(mask, src1, 0); la_xvbitsel_v(temp, temp, src2, mask); - if (ir1_opnd_is_xmm(opnd0)) - la_xvori_b(dest, temp, 0); - set_high128_xreg_to_zero(dest); + la_xvori_b(dest, temp, 0); + if (ir1_opnd_is_xmm(opnd0)) { + set_high128_xreg_to_zero(dest); + } } else if (ir1_opnd_is_mem(opnd0)) { temp = load_freg256_from_ir1(opnd0); tr_slt(mask, src1, 0); diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 713415bd7b..6abd69e306 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -1935,9 +1935,10 @@ bool translate_vpblendvb(IR1_INST * pir1) { src3 = load_freg256_from_ir1(opnd3); la_xvslti_b(temp, src3, 0); la_xvbitsel_v(temp, src1, src2, temp); - if (ir1_opnd_is_xmm(opnd0)) - la_xvori_b(dest, temp, 0); - set_high128_xreg_to_zero(dest); + la_xvori_b(dest, temp, 0); + if (ir1_opnd_is_xmm(opnd0)) { + set_high128_xreg_to_zero(dest); + } return true; } @@ -2016,9 +2017,10 @@ bool translate_vpblendw(IR1_INST * pir1) { if (imm & 0x80) la_xvextrins_h(temp, src2, VEXTRINS_IMM_4_0(7, 7)); - if (ir1_opnd_is_xmm(opnd0)) - la_xvori_b(dest, temp, 0); - set_high128_xreg_to_zero(dest); + la_xvori_b(dest, temp, 0); + if (ir1_opnd_is_xmm(opnd0)) { + set_high128_xreg_to_zero(dest); + } return true; } diff --git a/tests/integration/avx-ymm-writeback.S b/tests/integration/avx-ymm-writeback.S new file mode 100644 index 0000000000..4d12491f03 --- /dev/null +++ b/tests/integration/avx-ymm-writeback.S @@ -0,0 +1,117 @@ +.intel_syntax noprefix +.global _start + +.section .text +_start: + vpcmpeqd ymm0, ymm0, ymm0 + vmovdqu ymm1, ymmword ptr [rip + .Lmask_dwords] + vmaskmovps ymm0, ymm1, ymmword ptr [rip + .Lmask_source] + vmovdqu ymmword ptr [rip + .Lactual], ymm0 + lea rsi, [rip + .Lactual] + lea rdi, [rip + .Lmask_expected] + call .Lcompare_32 + test eax, eax + jne .Lfail_maskmov + + vpcmpeqd ymm0, ymm0, ymm0 + vmovdqu ymm1, ymmword ptr [rip + .Lblend_source1] + vmovdqu ymm2, ymmword ptr [rip + .Lblend_source2] + vmovdqu ymm3, ymmword ptr [rip + .Lblend_mask] + vpblendvb ymm0, ymm1, ymm2, ymm3 + vmovdqu ymmword ptr [rip + .Lactual], ymm0 + lea rsi, [rip + .Lactual] + lea rdi, [rip + .Lblend_expected] + call .Lcompare_32 + test eax, eax + jne .Lfail_blendvb + + vpcmpeqd ymm0, ymm0, ymm0 + vmovdqu ymm1, ymmword ptr [rip + .Lword_source1] + vmovdqu ymm2, ymmword ptr [rip + .Lword_source2] + vpblendw ymm0, ymm1, ymm2, 0x5a + vmovdqu ymmword ptr [rip + .Lactual], ymm0 + lea rsi, [rip + .Lactual] + lea rdi, [rip + .Lword_expected] + call .Lcompare_32 + test eax, eax + jne .Lfail_blendw + + xor edi, edi + jmp .Lexit + +.Lcompare_32: + xor eax, eax + mov ecx, 4 +.Lcompare_loop: + mov r8, qword ptr [rsi] + cmp r8, qword ptr [rdi] + jne .Lcompare_fail + add rsi, 8 + add rdi, 8 + dec ecx + jne .Lcompare_loop + ret +.Lcompare_fail: + mov eax, 1 + ret + +.Lfail_maskmov: + mov edi, 1 + jmp .Lexit +.Lfail_blendvb: + mov edi, 2 + jmp .Lexit +.Lfail_blendw: + mov edi, 3 +.Lexit: + vzeroupper + mov eax, 60 + syscall + +.section .rodata +.p2align 5 +.Lmask_source: + .long 10, 11, 12, 13, 14, 15, 16, 17 +.Lmask_dwords: + .long 0, 0x80000000, 0, 0x80000000 + .long 0x80000000, 0, 0x80000000, 0 +.Lmask_expected: + .long 0, 11, 0, 13, 14, 0, 16, 0 + +.Lblend_source1: + .byte 0, 1, 2, 3, 4, 5, 6, 7 + .byte 8, 9, 10, 11, 12, 13, 14, 15 + .byte 16, 17, 18, 19, 20, 21, 22, 23 + .byte 24, 25, 26, 27, 28, 29, 30, 31 +.Lblend_source2: + .byte 128, 129, 130, 131, 132, 133, 134, 135 + .byte 136, 137, 138, 139, 140, 141, 142, 143 + .byte 144, 145, 146, 147, 148, 149, 150, 151 + .byte 152, 153, 154, 155, 156, 157, 158, 159 +.Lblend_mask: + .byte 0, 128, 0, 128, 0, 128, 0, 128 + .byte 0, 128, 0, 128, 0, 128, 0, 128 + .byte 0, 128, 0, 128, 0, 128, 0, 128 + .byte 0, 128, 0, 128, 0, 128, 0, 128 +.Lblend_expected: + .byte 0, 129, 2, 131, 4, 133, 6, 135 + .byte 8, 137, 10, 139, 12, 141, 14, 143 + .byte 16, 145, 18, 147, 20, 149, 22, 151 + .byte 24, 153, 26, 155, 28, 157, 30, 159 + +.Lword_source1: + .word 0, 1, 2, 3, 4, 5, 6, 7 + .word 8, 9, 10, 11, 12, 13, 14, 15 +.Lword_source2: + .word 100, 101, 102, 103, 104, 105, 106, 107 + .word 108, 109, 110, 111, 112, 113, 114, 115 +.Lword_expected: + .word 0, 101, 2, 103, 104, 5, 106, 7 + .word 8, 109, 10, 111, 112, 13, 114, 15 + +.section .bss +.p2align 5 +.Lactual: + .skip 32 + +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/registrations/simd/meson.build b/tests/integration/registrations/simd/meson.build index 3a5fb2ea88..82d17828b0 100644 --- a/tests/integration/registrations/simd/meson.build +++ b/tests/integration/registrations/simd/meson.build @@ -8,4 +8,13 @@ if 'x86_64-linux-user' in target_dirs ], 'timeout': 120, }] + latx_integration_tests += [{ + 'name': 'test-avx-ymm-writeback', + 'runner': find_program('../../test-avx-ymm-writeback.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../avx-ymm-writeback.S'), + ], + 'timeout': 120, + }] endif diff --git a/tests/integration/test-avx-ymm-writeback.sh b/tests/integration/test-avx-ymm-writeback.sh new file mode 100755 index 0000000000..4ae990f5ab --- /dev/null +++ b/tests/integration/test-avx-ymm-writeback.sh @@ -0,0 +1,41 @@ +#!/bin/sh +set -eu + +emulator=$1 +source_file=$2 +workdir=$(mktemp -d) +trap 'rm -rf "$workdir"' EXIT HUP INT TERM + +if command -v clang-19 >/dev/null 2>&1; then + clang=clang-19 +elif command -v clang >/dev/null 2>&1; then + clang=clang +else + echo "SKIP: clang is required to build the x86_64 guest" + exit 77 +fi + +guest="$workdir/avx-ymm-writeback" +"$clang" --target=x86_64-linux-gnu -fuse-ld=lld -nostdlib -static \ + -Wl,--build-id=none "$source_file" -o "$guest" + +LATX_AOT=0 LATX_TU=0 "$emulator" "$guest" + +aot_home="$workdir/aot" +mkdir -p "$aot_home" +HOME="$aot_home" LATX_AOT=1 LATX_TU=1 "$emulator" "$guest" + +aot_file= +for unused in $(seq 1 100); do + aot_file=$(find "$aot_home/.cache/latx" -type f -name '*.aot2' \ + -size +0c -print -quit 2>/dev/null || true) + test -n "$aot_file" && break + sleep 0.1 +done +if test -z "$aot_file"; then + echo "FAIL: no non-empty AOT file generated" >&2 + exit 1 +fi + +HOME="$aot_home" LATX_AOT=1 LATX_TU=1 "$emulator" "$guest" +echo "PASS: YMM mask and blend writeback in JIT, cold AOT, and hot AOT" From 70d46cf23a0e2d9f16fc2afb416e8198faf1c567 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Thu, 3 Sep 2026 13:13:11 +0800 Subject: [PATCH 04/17] LATX, AVX: cover blend source-destination aliases Signed-off-by: Lu Zeng --- tests/integration/avx-ymm-writeback.S | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/integration/avx-ymm-writeback.S b/tests/integration/avx-ymm-writeback.S index 4d12491f03..cb0da3c51f 100644 --- a/tests/integration/avx-ymm-writeback.S +++ b/tests/integration/avx-ymm-writeback.S @@ -36,6 +36,26 @@ _start: test eax, eax jne .Lfail_blendw + vmovdqu ymm0, ymmword ptr [rip + .Lword_source1] + vmovdqu ymm2, ymmword ptr [rip + .Lword_source2] + vpblendw ymm0, ymm0, ymm2, 0x55 + vmovdqu ymmword ptr [rip + .Lactual], ymm0 + lea rsi, [rip + .Lactual] + lea rdi, [rip + .Lblend55_expected] + call .Lcompare_32 + test eax, eax + jne .Lfail_blend55 + + vmovdqu ymm1, ymmword ptr [rip + .Lword_source1] + vmovdqu ymm0, ymmword ptr [rip + .Lword_source2] + vpblendw ymm0, ymm1, ymm0, 0xaa + vmovdqu ymmword ptr [rip + .Lactual], ymm0 + lea rsi, [rip + .Lactual] + lea rdi, [rip + .Lblendaa_expected] + call .Lcompare_32 + test eax, eax + jne .Lfail_blendaa + xor edi, edi jmp .Lexit @@ -63,6 +83,12 @@ _start: jmp .Lexit .Lfail_blendw: mov edi, 3 + jmp .Lexit +.Lfail_blend55: + mov edi, 4 + jmp .Lexit +.Lfail_blendaa: + mov edi, 5 .Lexit: vzeroupper mov eax, 60 @@ -108,6 +134,12 @@ _start: .Lword_expected: .word 0, 101, 2, 103, 104, 5, 106, 7 .word 8, 109, 10, 111, 112, 13, 114, 15 +.Lblend55_expected: + .word 100, 1, 102, 3, 104, 5, 106, 7 + .word 108, 9, 110, 11, 112, 13, 114, 15 +.Lblendaa_expected: + .word 0, 101, 2, 103, 4, 105, 6, 107 + .word 8, 109, 10, 111, 12, 113, 14, 115 .section .bss .p2align 5 From 90cac723cc044cfdafe5ed8009f41c3f3bfe51f8 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Thu, 3 Sep 2026 17:54:22 +0800 Subject: [PATCH 05/17] LATX, AVX: avoid temporary YMM mask and blend copies Signed-off-by: Lu Zeng --- target/i386/latx/translator/tr-avx-mov.c | 5 +-- target/i386/latx/translator/tr-avx.c | 48 ++++++++++++------------ 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/target/i386/latx/translator/tr-avx-mov.c b/target/i386/latx/translator/tr-avx-mov.c index a2c36c3906..58a5bfb09d 100644 --- a/target/i386/latx/translator/tr-avx-mov.c +++ b/target/i386/latx/translator/tr-avx-mov.c @@ -459,11 +459,8 @@ bool translate_vmaskmovpx(IR1_INST * pir1) { } if (ir1_opnd_is_xmm(opnd0) || ir1_opnd_is_ymm(opnd0)) { IR2_OPND dest = load_freg256_from_ir1(opnd0); - temp = ra_alloc_ftemp(); - la_xvandi_b(temp, temp, 0); tr_slt(mask, src1, 0); - la_xvbitsel_v(temp, temp, src2, mask); - la_xvori_b(dest, temp, 0); + la_xvand_v(dest, src2, mask); if (ir1_opnd_is_xmm(opnd0)) { set_high128_xreg_to_zero(dest); } diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 6abd69e306..46e2b422f2 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -1969,55 +1969,57 @@ bool translate_vpblendw(IR1_INST * pir1) { set_high128_xreg_to_zero(dest); return true; } - IR2_OPND temp = ra_alloc_ftemp(); - la_xvori_b(temp, src1, 0); - /* 64 bit fast path */ + bool dest_is_src1 = ir1_opnd_is_same_reg(opnd0, opnd1); + bool dest_is_src2 = ir1_opnd_is_same_reg(opnd0, opnd2); + IR2_OPND insert_src = src2; + + if (dest_is_src2) { + insert_src = src1; + imm = ~imm; + } else if (!dest_is_src1) { + la_xvori_b(dest, src1, 0); + } + if ((imm & 0xf) == 0xf) { - la_xvextrins_d(temp, src2, VEXTRINS_IMM_4_0(0, 0)); + la_xvextrins_d(dest, insert_src, VEXTRINS_IMM_4_0(0, 0)); imm &= ~0xf; } if ((imm & 0xf0) == 0xf0) { - la_xvextrins_d(temp, src2, VEXTRINS_IMM_4_0(1, 1)); + la_xvextrins_d(dest, insert_src, VEXTRINS_IMM_4_0(1, 1)); imm &= ~0xf0; } - - /* 32 bit fast path */ if ((imm & 0x3) == 0x3) { - la_xvextrins_w(temp, src2, VEXTRINS_IMM_4_0(0, 0)); + la_xvextrins_w(dest, insert_src, VEXTRINS_IMM_4_0(0, 0)); imm &= ~0x3; } if ((imm & 0xc) == 0xc) { - la_xvextrins_w(temp, src2, VEXTRINS_IMM_4_0(1, 1)); + la_xvextrins_w(dest, insert_src, VEXTRINS_IMM_4_0(1, 1)); imm &= ~0xc; } if ((imm & 0x30) == 0x30) { - la_xvextrins_w(temp, src2, VEXTRINS_IMM_4_0(2, 2)); + la_xvextrins_w(dest, insert_src, VEXTRINS_IMM_4_0(2, 2)); imm &= ~0x30; } if ((imm & 0xc0) == 0xc0) { - la_xvextrins_w(temp, src2, VEXTRINS_IMM_4_0(3, 3)); + la_xvextrins_w(dest, insert_src, VEXTRINS_IMM_4_0(3, 3)); imm &= ~0xc0; } - - /* 16 bit slow path */ if (imm & 0x1) - la_xvextrins_h(temp, src2, VEXTRINS_IMM_4_0(0, 0)); + la_xvextrins_h(dest, insert_src, VEXTRINS_IMM_4_0(0, 0)); if (imm & 0x2) - la_xvextrins_h(temp, src2, VEXTRINS_IMM_4_0(1, 1)); + la_xvextrins_h(dest, insert_src, VEXTRINS_IMM_4_0(1, 1)); if (imm & 0x4) - la_xvextrins_h(temp, src2, VEXTRINS_IMM_4_0(2, 2)); + la_xvextrins_h(dest, insert_src, VEXTRINS_IMM_4_0(2, 2)); if (imm & 0x8) - la_xvextrins_h(temp, src2, VEXTRINS_IMM_4_0(3, 3)); + la_xvextrins_h(dest, insert_src, VEXTRINS_IMM_4_0(3, 3)); if (imm & 0x10) - la_xvextrins_h(temp, src2, VEXTRINS_IMM_4_0(4, 4)); + la_xvextrins_h(dest, insert_src, VEXTRINS_IMM_4_0(4, 4)); if (imm & 0x20) - la_xvextrins_h(temp, src2, VEXTRINS_IMM_4_0(5, 5)); + la_xvextrins_h(dest, insert_src, VEXTRINS_IMM_4_0(5, 5)); if (imm & 0x40) - la_xvextrins_h(temp, src2, VEXTRINS_IMM_4_0(6, 6)); + la_xvextrins_h(dest, insert_src, VEXTRINS_IMM_4_0(6, 6)); if (imm & 0x80) - la_xvextrins_h(temp, src2, VEXTRINS_IMM_4_0(7, 7)); - - la_xvori_b(dest, temp, 0); + la_xvextrins_h(dest, insert_src, VEXTRINS_IMM_4_0(7, 7)); if (ir1_opnd_is_xmm(opnd0)) { set_high128_xreg_to_zero(dest); } From ff9a284f5cb831720adeb80bd8d2b69d6eb57621 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Thu, 3 Sep 2026 18:01:19 +0800 Subject: [PATCH 06/17] LATX: optimize AVX and scalar instruction sequences Combine independently tested AVX lowering, scalar FMA forwarding, VCOMIS flag handling, pixel updates, and VROUNDPS truncation. Preserve aliased VINSERTPS source values and use instruction-based test names. Signed-off-by: Lu Zeng --- target/i386/latx/include/insts-pattern.h | 7 + target/i386/latx/include/translate.h | 1 + target/i386/latx/optimization/flag-lbt.c | 4 + .../i386/latx/optimization/flag-reduction.c | 3 + target/i386/latx/optimization/hbr.c | 32 + target/i386/latx/optimization/insts-pattern.c | 594 +++++++++++++++++- .../i386/latx/optimization/ir1-optimization.c | 14 + .../i386/latx/optimization/ir2-optimization.c | 70 +++ target/i386/latx/translator/tr-avx-cmp.c | 8 +- target/i386/latx/translator/tr-avx-mov.c | 41 +- target/i386/latx/translator/tr-avx.c | 381 ++++++++--- .../i386/latx/translator/tr-eflag-process.c | 4 + target/i386/latx/translator/tr-opnd-process.c | 44 +- target/i386/latx/translator/tr-pattern.c | 240 ++++++- tests/integration/avx-sum3-pattern.S | 105 ++++ tests/integration/avx128-shuffle.S | 136 ++++ .../registrations/simd/meson.build | 32 + tests/integration/repeat-add.S | 23 + tests/integration/scalar-fma-forward.S | 64 ++ tests/integration/scalar-hdr-pattern.S | 146 +++++ tests/integration/test-avx-patterns.sh | 46 ++ tests/integration/vcomis-jcc.S | 140 +++++ tests/integration/vpshufd-distinct-regs.S | 59 ++ tests/integration/vroundps-imm0b.S | 76 +++ 24 files changed, 2119 insertions(+), 151 deletions(-) create mode 100644 tests/integration/avx-sum3-pattern.S create mode 100644 tests/integration/avx128-shuffle.S create mode 100644 tests/integration/repeat-add.S create mode 100644 tests/integration/scalar-fma-forward.S create mode 100644 tests/integration/scalar-hdr-pattern.S create mode 100755 tests/integration/test-avx-patterns.sh create mode 100644 tests/integration/vcomis-jcc.S create mode 100644 tests/integration/vpshufd-distinct-regs.S create mode 100644 tests/integration/vroundps-imm0b.S diff --git a/target/i386/latx/include/insts-pattern.h b/target/i386/latx/include/insts-pattern.h index 781a6677ca..2c67f3d34a 100644 --- a/target/i386/latx/include/insts-pattern.h +++ b/target/i386/latx/include/insts-pattern.h @@ -51,6 +51,13 @@ #define INSTPTN_OPC_SHR_JCC 0x10000000 #define INSTPTN_OPC_AND_JCC 0x20000000 +#define INSTPTN_OPC_REPEAT_ADD 0x40000000 +#define INSTPTN_OPC_AVX_SUM3 (-0x7fffffff - 1) +#define INSTPTN_OPC_SCALAR_HDR (-0x7fffffff) + +void insts_pattern_repeat_add(TranslationBlock *tb); +void insts_pattern_avx_sum3(TranslationBlock *tb); +void insts_pattern_scalar_hdr(TranslationBlock *tb); typedef int scan_elem_t; diff --git a/target/i386/latx/include/translate.h b/target/i386/latx/include/translate.h index 0fd4dd2704..7c8e1f693b 100644 --- a/target/i386/latx/include/translate.h +++ b/target/i386/latx/include/translate.h @@ -1607,6 +1607,7 @@ IR2_OPND load_freg128_from_ir1(IR1_OPND *); void load_imm_to_ir1_opnd_gpr(IR1_OPND *opnd0, uint64_t imm); IR2_OPND load_freg256_from_ir1(IR1_OPND *opnd1); void set_high128_xreg_to_zero(IR2_OPND opnd); +void mark_high128_xreg_zeroed(IR2_OPND opnd); void materialize_deferred_ymmh_zero(IR2_OPND opnd); void materialize_deferred_ymmh_zeros_now(void); void materialize_deferred_ymmh_zeros_for_exit(void); diff --git a/target/i386/latx/optimization/flag-lbt.c b/target/i386/latx/optimization/flag-lbt.c index 039c562036..85928589c5 100644 --- a/target/i386/latx/optimization/flag-lbt.c +++ b/target/i386/latx/optimization/flag-lbt.c @@ -121,6 +121,10 @@ bool generate_eflag_by_lbt(IR2_OPND dest, IR2_OPND src0, IR2_OPND src1, case dt_X86_INS_COMISD: case dt_X86_INS_UCOMISS: case dt_X86_INS_UCOMISD: + case dt_X86_INS_VCOMISS: + case dt_X86_INS_VCOMISD: + case dt_X86_INS_VUCOMISS: + case dt_X86_INS_VUCOMISD: #endif case dt_X86_INS_AAM: case dt_X86_INS_AAD: diff --git a/target/i386/latx/optimization/flag-reduction.c b/target/i386/latx/optimization/flag-reduction.c index c695f1be65..c8ea963909 100644 --- a/target/i386/latx/optimization/flag-reduction.c +++ b/target/i386/latx/optimization/flag-reduction.c @@ -112,7 +112,10 @@ static const IR1_EFLAG_USEDEF ir1_opcode_eflag_usedef[] = { FLAG_DEFINE(INVLPG, __NONE, __NONE, __NONE), FLAG_DEFINE(UCOMISD, __NONE, __ALL_EFLAGS, __NONE), FLAG_DEFINE(UCOMISS, __NONE, __ALL_EFLAGS, __NONE), + FLAG_DEFINE(VCOMISD, __NONE, __ALL_EFLAGS, __NONE), + FLAG_DEFINE(VCOMISS, __NONE, __ALL_EFLAGS, __NONE), FLAG_DEFINE(VUCOMISD, __NONE, __ALL_EFLAGS, __NONE), + FLAG_DEFINE(VUCOMISS, __NONE, __ALL_EFLAGS, __NONE), FLAG_DEFINE(IRET, __NONE, __INVALID, __NONE), /* Jcc */ diff --git a/target/i386/latx/optimization/hbr.c b/target/i386/latx/optimization/hbr.c index cd2f640e2b..f8160f3566 100644 --- a/target/i386/latx/optimization/hbr.c +++ b/target/i386/latx/optimization/hbr.c @@ -125,6 +125,21 @@ static bool deal_xmm_common(TranslationBlock *tb, IR1_INST *ir1, uint32_t *xmm) } switch (ir1_opcode(ir1)) { + /* VEX scalar arithmetic copies bits 127:32 from its first source. */ + case WRAP(VADDSS): + case WRAP(VSUBSS): + case WRAP(VMULSS): + case WRAP(VDIVSS): + src_update_des(ir1, xmm); + return true; + /* Scalar FMA leaves bits 127:32 of the old destination unchanged. */ + case WRAP(VFMADD132SS): + case WRAP(VFMADD213SS): + case WRAP(VFMADD231SS): + case WRAP(VFMSUB132SS): + case WRAP(VFMSUB213SS): + case WRAP(VFMSUB231SS): + return true; case WRAP(ADDPD): case WRAP(ADDPS): case WRAP(ADDSUBPD): @@ -204,6 +219,12 @@ static bool deal_xmm_common(TranslationBlock *tb, IR1_INST *ir1, uint32_t *xmm) case WRAP(MOVDQU): case WRAP(MOVNTPS): case WRAP(MOVNTPD): + case WRAP(VMOVUPS): + case WRAP(VMOVUPD): + case WRAP(VMOVAPS): + case WRAP(VMOVAPD): + case WRAP(VMOVDQA): + case WRAP(VMOVDQU): if (!ir1_opnd_is_xmm(des_opnd)) { /* src will write to mm. */ src_no_opt(ir1, xmm); @@ -225,6 +246,17 @@ static bool deal_xmm_common(TranslationBlock *tb, IR1_INST *ir1, uint32_t *xmm) zero_update_des(ir1, xmm); } return true; + /* VMOVSS xmm, m32 clears bits 127:32; stores only read bits 31:0. */ + case WRAP(VMOVSS): + if (!ir1_opnd_is_xmm(des_opnd)) { + return true; + } + if (ir1_opnd_is_mem(src_opnd)) { + zero_update_des(ir1, xmm); + } else { + src_update_des(ir1, xmm); + } + return true; /* need 0 ~ 127 every high bit. */ case WRAP(PMOVMSKB): if (ir1_opnd_is_xmm(src_opnd)) { diff --git a/target/i386/latx/optimization/insts-pattern.c b/target/i386/latx/optimization/insts-pattern.c index 4bac7d65c2..2106357d9c 100644 --- a/target/i386/latx/optimization/insts-pattern.c +++ b/target/i386/latx/optimization/insts-pattern.c @@ -604,6 +604,7 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i } #ifdef CONFIG_LATX_XCOMISX_OPT case WRAP(COMISD): + case WRAP(VCOMISD): SCAN_CHECK(scan, 0); ir1_jcc = SCAN_IR1(tb, scan, 0); switch (ir1_opcode(ir1_jcc)) { @@ -623,7 +624,7 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_NOP; // ir1_jcc->instptn.next = NULL; - } else { + } else if (ir1_opcode(pir1) == WRAP(COMISD)) { instptn_check_comisd_xx_jcc_0(); pir1->instptn.opc = INSTPTN_OPC_COMISD_XX_JCC; pir1->instptn.next = ir1_jcc; @@ -636,6 +637,7 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i return false; } case WRAP(COMISS): + case WRAP(VCOMISS): SCAN_CHECK(scan, 0); ir1_jcc = SCAN_IR1(tb, scan, 0); switch (ir1_opcode(ir1_jcc)) { @@ -655,7 +657,7 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_NOP; // ir1_jcc->instptn.next = NULL; - } else { + } else if (ir1_opcode(pir1) == WRAP(COMISS)) { instptn_check_comiss_xx_jcc_0(); pir1->instptn.opc = INSTPTN_OPC_COMISS_XX_JCC; pir1->instptn.next = ir1_jcc; @@ -668,6 +670,7 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i return false; } case WRAP(UCOMISD): + case WRAP(VUCOMISD): SCAN_CHECK(scan, 0); ir1_jcc = SCAN_IR1(tb, scan, 0); switch (ir1_opcode(ir1_jcc)) { @@ -687,7 +690,7 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_NOP; // ir1_jcc->instptn.next = NULL; - } else { + } else if (ir1_opcode(pir1) == WRAP(UCOMISD)) { instptn_check_ucomisd_xx_jcc_0(); pir1->instptn.opc = INSTPTN_OPC_UCOMISD_XX_JCC; pir1->instptn.next = ir1_jcc; @@ -700,6 +703,7 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i return false; } case WRAP(UCOMISS): + case WRAP(VUCOMISS): SCAN_CHECK(scan, 0); ir1_jcc = SCAN_IR1(tb, scan, 0); switch (ir1_opcode(ir1_jcc)) { @@ -719,7 +723,7 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i pir1->instptn.next = ir1_jcc; ir1_jcc->instptn.opc = INSTPTN_OPC_NOP; ir1_jcc->instptn.next = NULL; - } else { + } else if (ir1_opcode(pir1) == WRAP(UCOMISS)) { instptn_check_ucomiss_xx_jcc_0(); pir1->instptn.opc = INSTPTN_OPC_UCOMISS_XX_JCC; pir1->instptn.next = ir1_jcc; @@ -765,6 +769,588 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_i } } +static bool repeat_add_same_operands(IR1_INST *first, IR1_INST *next) +{ + IR1_OPND *first_dest; + IR1_OPND *first_src; + IR1_OPND *next_dest; + IR1_OPND *next_src; + + if (ir1_opcode(next) != WRAP(ADD) || ir1_get_opnd_num(next) != 2 || + next->instptn.opc != INSTPTN_OPC_NONE || + ir1_get_eflag_def(next) != 0 || ir1_is_prefix_lock(next)) { + return false; + } + first_dest = ir1_get_opnd(first, 0); + first_src = ir1_get_opnd(first, 1); + next_dest = ir1_get_opnd(next, 0); + next_src = ir1_get_opnd(next, 1); + + return + ir1_opnd_is_gpr(next_dest) && ir1_opnd_is_gpr(next_src) && + ir1_opnd_size(next_dest) == 64 && + ir1_opnd_size(next_src) == 64 && + ir1_opnd_base_reg_num(next_dest) == + ir1_opnd_base_reg_num(first_dest) && + ir1_opnd_base_reg_num(next_src) == + ir1_opnd_base_reg_num(first_src); +} + +static int avx_sum3_xmm(IR1_INST *ir1, int opnd_index) +{ + if (opnd_index >= ir1_get_opnd_num(ir1) || + !ir1_opnd_is_xmm(ir1_get_opnd(ir1, opnd_index))) { + return -1; + } + return ir1_opnd_base_reg_num(ir1_get_opnd(ir1, opnd_index)); +} + +static bool avx_sum3_imm(IR1_INST *ir1, int opnd_index, ulongx value) +{ + return opnd_index < ir1_get_opnd_num(ir1) && + ir1_opnd_is_imm(ir1_get_opnd(ir1, opnd_index)) && + ir1_opnd_uimm(ir1_get_opnd(ir1, opnd_index)) == value; +} + +static bool avx_sum3_match_group(TranslationBlock *tb, int pos, + int *product, int *zero) +{ + IR1_INST *mul = tb_ir1_inst(tb, pos); + IR1_INST *blend = tb_ir1_inst(tb, pos + 1); + IR1_INST *insert = tb_ir1_inst(tb, pos + 2); + IR1_INST *add = tb_ir1_inst(tb, pos + 3); + int temp; + + if (ir1_opcode(mul) != WRAP(VMULPS) || + ir1_opcode(blend) != WRAP(VBLENDPS) || + ir1_opcode(insert) != WRAP(VINSERTPS) || + ir1_opcode(add) != WRAP(VADDPS)) { + return false; + } + *product = avx_sum3_xmm(mul, 0); + temp = avx_sum3_xmm(blend, 0); + if (*product < 0 || temp < 0 || temp == *product || + avx_sum3_xmm(blend, 1) != *product || + avx_sum3_xmm(blend, 2) < 0 || + !avx_sum3_imm(blend, 3, 0x08) || + avx_sum3_xmm(insert, 0) != *product || + avx_sum3_xmm(insert, 1) != *product || + avx_sum3_xmm(insert, 2) != *product || + !avx_sum3_imm(insert, 3, 0x4c) || + avx_sum3_xmm(add, 0) != *product || + avx_sum3_xmm(add, 1) != temp || + avx_sum3_xmm(add, 2) != *product) { + return false; + } + if (*zero < 0) { + *zero = avx_sum3_xmm(blend, 2); + } + return avx_sum3_xmm(blend, 2) == *zero; +} + +static bool avx_sum3_match_insert(IR1_INST *ir1, int dest, int src1, + int src2, ulongx imm) +{ + return ir1_opcode(ir1) == WRAP(VINSERTPS) && + avx_sum3_xmm(ir1, 0) == dest && + avx_sum3_xmm(ir1, 1) == src1 && + avx_sum3_xmm(ir1, 2) == src2 && + avx_sum3_imm(ir1, 3, imm); +} + +static bool avx_sum3_match_add(IR1_INST *ir1, IR1_OPCODE opcode, + int dest, int src1, int src2) +{ + return ir1_opcode(ir1) == opcode && + avx_sum3_xmm(ir1, 0) == dest && + avx_sum3_xmm(ir1, 1) == src1 && + avx_sum3_xmm(ir1, 2) == src2; +} + +static bool avx_sum3_match_shuffle(IR1_INST *ir1, int dest, int src) +{ + return ir1_opcode(ir1) == WRAP(VSHUFPD) && + avx_sum3_xmm(ir1, 0) == dest && + avx_sum3_xmm(ir1, 1) == src && + avx_sum3_xmm(ir1, 2) == src && + avx_sum3_imm(ir1, 3, 1); +} + +static bool avx_sum3_safe_overwrite(IR1_INST *ir1, int reg) +{ + if (avx_sum3_xmm(ir1, 0) != reg) { + return false; + } + for (int i = 1; i < ir1_get_opnd_num(ir1); ++i) { + if (avx_sum3_xmm(ir1, i) == reg) { + return false; + } + } + switch (ir1_opcode(ir1)) { + case WRAP(VMOVSD): + return ir1_opnd_is_mem(ir1_get_opnd(ir1, 1)); + case WRAP(VINSERTPS): + case WRAP(VSQRTSS): + return true; + default: + return false; + } +} + +static bool avx_sum3_overwritten_before_use(TranslationBlock *tb, int pos, + int reg) +{ + for (int i = pos; i < tb_ir1_num(tb); ++i) { + IR1_INST *ir1 = tb_ir1_inst(tb, i); + + if (avx_sum3_safe_overwrite(ir1, reg)) { + return true; + } + for (int j = 0; j < ir1_get_opnd_num(ir1); ++j) { + if (avx_sum3_xmm(ir1, j) == reg) { + return false; + } + } + } + return false; +} + +void insts_pattern_avx_sum3(TranslationBlock *tb) +{ + static const int group_offset[] = {0, 4, 9, 13, 17, 21}; + static const int skipped_offset[] = { + 1, 2, 3, 5, 6, 7, 10, 11, 12, 14, 15, 16, + 18, 19, 20, 23, 24, 25, 26, 27, 28, 29, 30, + 31, 32, 33, 34, 35, 36, 37, 38, + }; + int count = tb_ir1_num(tb); + + for (int start = 0; start + 39 <= count; ++start) { + int product[6]; + int group_temp[6]; + int zero = -1; + int temp1; + int temp2; + int temp3; + bool match = true; + + for (int i = 0; i < 39; ++i) { + if (tb_ir1_inst(tb, start + i)->instptn.opc != + INSTPTN_OPC_NONE) { + match = false; + } + } + if (!match) { + continue; + } + for (int i = 0; i < 6; ++i) { + if (!avx_sum3_match_group(tb, start + group_offset[i], + &product[i], &zero)) { + match = false; + break; + } + group_temp[i] = avx_sum3_xmm( + tb_ir1_inst(tb, start + group_offset[i] + 1), 0); + for (int j = 0; j < i; ++j) { + if (product[i] == product[j] || + group_temp[i] == product[j]) { + match = false; + } + } + } + if (!match || zero < 0 || + ir1_opcode(tb_ir1_inst(tb, start + 8)) != WRAP(VMOVAPS) || + !ir1_opnd_is_mem(ir1_get_opnd(tb_ir1_inst(tb, start + 8), 1)) || + avx_sum3_xmm(tb_ir1_inst(tb, start + 8), 0) == product[0] || + avx_sum3_xmm(tb_ir1_inst(tb, start + 8), 0) == product[1]) { + continue; + } + for (int i = 0; i < 6; ++i) { + if (zero == product[i]) { + match = false; + } + for (int opnd = 1; + opnd < ir1_get_opnd_num( + tb_ir1_inst(tb, start + group_offset[i])); ++opnd) { + int src = avx_sum3_xmm( + tb_ir1_inst(tb, start + group_offset[i]), opnd); + + for (int previous = 0; previous < i; ++previous) { + if (src == product[previous]) { + match = false; + } + } + } + } + if (!match) { + continue; + } + temp1 = avx_sum3_xmm(tb_ir1_inst(tb, start + 25), 0); + temp2 = avx_sum3_xmm(tb_ir1_inst(tb, start + 29), 0); + temp3 = avx_sum3_xmm(tb_ir1_inst(tb, start + 34), 0); + if (temp1 < 0 || temp2 < 0 || temp3 < 0 || + group_temp[0] != product[1] || + group_temp[1] != avx_sum3_xmm( + tb_ir1_inst(tb, start + 8), 0) || + group_temp[2] != temp1 || group_temp[3] != temp1 || + group_temp[4] != temp1 || group_temp[5] != temp1 || + temp2 != product[1] || temp3 != product[4] || + zero == temp1 || zero == group_temp[1] || + !avx_sum3_match_insert(tb_ir1_inst(tb, start + 25), + temp1, product[1], product[3], 0x1c) || + !avx_sum3_match_shuffle(tb_ir1_inst(tb, start + 26), + product[1], product[1]) || + !avx_sum3_match_insert(tb_ir1_inst(tb, start + 27), + product[3], product[1], product[3], 0x9c) || + !avx_sum3_match_add(tb_ir1_inst(tb, start + 28), WRAP(VADDPS), + product[3], temp1, product[3]) || + !avx_sum3_match_insert(tb_ir1_inst(tb, start + 29), + temp2, product[0], product[4], 0x1c) || + !avx_sum3_match_shuffle(tb_ir1_inst(tb, start + 30), + product[0], product[0]) || + !avx_sum3_match_insert(tb_ir1_inst(tb, start + 31), + product[4], product[0], product[4], 0x9c) || + !avx_sum3_match_add(tb_ir1_inst(tb, start + 32), WRAP(VADDPS), + product[4], temp2, product[4]) || + !avx_sum3_match_add(tb_ir1_inst(tb, start + 33), WRAP(VADDPS), + product[3], product[4], product[3]) || + !avx_sum3_match_insert(tb_ir1_inst(tb, start + 34), + temp3, product[2], product[5], 0x1c) || + !avx_sum3_match_shuffle(tb_ir1_inst(tb, start + 35), + product[0], product[2]) || + !avx_sum3_match_insert(tb_ir1_inst(tb, start + 36), + product[0], product[0], product[5], 0x9c) || + !avx_sum3_match_add(tb_ir1_inst(tb, start + 37), WRAP(VADDPS), + product[4], temp3, product[0]) || + !avx_sum3_match_add(tb_ir1_inst(tb, start + 38), WRAP(VSUBPS), + product[3], product[3], product[4])) { + continue; + } + if (!avx_sum3_overwritten_before_use(tb, start + 39, product[0]) || + !avx_sum3_overwritten_before_use(tb, start + 39, product[1]) || + !avx_sum3_overwritten_before_use(tb, start + 39, product[2]) || + !avx_sum3_overwritten_before_use(tb, start + 39, product[4])) { + continue; + } + + for (size_t i = 0; i < ARRAY_SIZE(skipped_offset); ++i) { + tb_ir1_inst(tb, start + skipped_offset[i])->instptn.opc = + INSTPTN_OPC_NOP; + } + tb_ir1_inst(tb, start + 22)->instptn.opc = INSTPTN_OPC_AVX_SUM3; + start += 38; + } +} + +void insts_pattern_repeat_add(TranslationBlock *tb) +{ + int count = tb_ir1_num(tb); + + for (int i = 0; i < count; ++i) { + IR1_INST *first = tb_ir1_inst(tb, i); + IR1_OPND *dest; + IR1_OPND *src; + int end; + + if (ir1_opcode(first) != WRAP(ADD) || ir1_get_opnd_num(first) != 2 || + first->instptn.opc != INSTPTN_OPC_NONE || + ir1_get_eflag_def(first) != 0 || ir1_is_prefix_lock(first)) { + continue; + } + dest = ir1_get_opnd(first, 0); + src = ir1_get_opnd(first, 1); + if (!ir1_opnd_is_gpr(dest) || !ir1_opnd_is_gpr(src) || + ir1_opnd_size(dest) != 64 || ir1_opnd_size(src) != 64 || + ir1_opnd_base_reg_num(dest) == ir1_opnd_base_reg_num(src)) { + continue; + } + + for (end = i + 1; end < count; ++end) { + if (!repeat_add_same_operands(first, tb_ir1_inst(tb, end))) { + break; + } + } + if (end - i < 3) { + continue; + } + + first->instptn.opc = INSTPTN_OPC_REPEAT_ADD; + first->instptn.next = tb_ir1_inst(tb, end - 1); + for (int j = i + 1; j < end; ++j) { + tb_ir1_inst(tb, j)->instptn.opc = INSTPTN_OPC_NOP; + } + i = end - 1; + } +} + +static bool scalar_hdr_same_reg(IR1_OPND *a, IR1_OPND *b) +{ + return ir1_opnd_is_gpr(a) && ir1_opnd_is_gpr(b) && + ir1_opnd_base_reg_num(a) == ir1_opnd_base_reg_num(b); +} + +static bool scalar_hdr_same_xmm(IR1_OPND *a, IR1_OPND *b) +{ + return ir1_opnd_is_xmm(a) && ir1_opnd_is_xmm(b) && + ir1_opnd_base_reg_num(a) == ir1_opnd_base_reg_num(b); +} + +static bool scalar_hdr_same_mem(IR1_OPND *a, IR1_OPND *b, int64_t delta) +{ + return ir1_opnd_is_mem(a) && ir1_opnd_is_mem(b) && + a->mem.segment == b->mem.segment && + a->mem.default_segment == b->mem.default_segment && + a->mem.base == b->mem.base && a->mem.index == b->mem.index && + a->mem.scale == b->mem.scale && + b->mem.disp == a->mem.disp + delta; +} + +static bool scalar_hdr_mem_uses_reg(IR1_OPND *mem, int reg) +{ + return (ir1_opnd_has_base(mem) && ir1_opnd_base_reg_num(mem) == reg) || + (ir1_opnd_has_index(mem) && ir1_opnd_index_reg_num(mem) == reg); +} + +static bool scalar_hdr_match_channel(TranslationBlock *tb, int pos, + IR1_OPND *first_input, + IR1_OPND *first_table, + IR1_OPND *first_sub, + IR1_OPND *first_output, + IR1_OPND *index_reg, + IR1_OPND *value_xmm, + IR1_OPND *factor_xmm, + int channel) +{ + IR1_INST *movzx = tb_ir1_inst(tb, pos); + IR1_INST *lea = tb_ir1_inst(tb, pos + 1); + IR1_INST *load = tb_ir1_inst(tb, pos + 2); + IR1_INST *sub = tb_ir1_inst(tb, pos + 3); + IR1_INST *fma = tb_ir1_inst(tb, pos + 4); + IR1_INST *store = tb_ir1_inst(tb, pos + 5); + IR1_OPND *input; + IR1_OPND *lea_mem; + IR1_OPND *table; + IR1_OPND *sub_mem; + IR1_OPND *output; + + if (ir1_opcode(movzx) != WRAP(MOVZX) || + ir1_opcode(lea) != WRAP(LEA) || + ir1_opcode(load) != WRAP(VMOVSS) || + ir1_opcode(sub) != WRAP(VSUBSS) || + ir1_opcode(fma) != WRAP(VFMADD213SS) || + ir1_opcode(store) != WRAP(VMOVSS) || + ir1_get_opnd_num(movzx) != 2 || ir1_get_opnd_num(lea) != 2 || + ir1_get_opnd_num(load) != 2 || ir1_get_opnd_num(sub) != 3 || + ir1_get_opnd_num(fma) != 3 || ir1_get_opnd_num(store) != 2) { + return false; + } + + input = ir1_get_opnd(movzx, 1); + lea_mem = ir1_get_opnd(lea, 1); + table = ir1_get_opnd(load, 1); + sub_mem = ir1_get_opnd(sub, 2); + output = ir1_get_opnd(fma, 2); + if (!scalar_hdr_same_reg(ir1_get_opnd(movzx, 0), index_reg) || + !scalar_hdr_same_reg(ir1_get_opnd(lea, 0), index_reg) || + !ir1_opnd_is_mem(input) || ir1_opnd_size(input) != 8 || + !scalar_hdr_same_mem(first_input, input, channel) || + !ir1_opnd_is_mem(lea_mem) || lea_mem->mem.disp != 0 || + lea_mem->mem.scale != 2 || + ir1_opnd_base_reg_num(lea_mem) != + ir1_opnd_base_reg_num(index_reg) || + ir1_opnd_index_reg_num(lea_mem) != + ir1_opnd_base_reg_num(index_reg) || + !scalar_hdr_same_xmm(ir1_get_opnd(load, 0), value_xmm) || + !scalar_hdr_same_mem(first_table, table, channel * 4) || + !scalar_hdr_mem_uses_reg(table, + ir1_opnd_base_reg_num(index_reg)) || + table->mem.scale != 4 || + ir1_opnd_size(table) != 32 || + !scalar_hdr_same_xmm(ir1_get_opnd(sub, 0), value_xmm) || + !scalar_hdr_same_xmm(ir1_get_opnd(sub, 1), value_xmm) || + !scalar_hdr_same_mem(first_sub, sub_mem, 0) || + !scalar_hdr_same_xmm(ir1_get_opnd(fma, 0), value_xmm) || + !scalar_hdr_same_xmm(ir1_get_opnd(fma, 1), factor_xmm) || + !scalar_hdr_same_mem(first_output, output, channel * 4) || + !scalar_hdr_same_mem(output, ir1_get_opnd(store, 0), 0) || + !scalar_hdr_same_xmm(ir1_get_opnd(store, 1), value_xmm)) { + return false; + } + return true; +} + +void insts_pattern_scalar_hdr(TranslationBlock *tb) +{ + int count = tb_ir1_num(tb); + + if (!option_enable_lasx) { + return; + } + for (int start = 0; start + 36 <= count; ++start) { + IR1_INST *first = tb_ir1_inst(tb, start); + IR1_INST *factor_load = tb_ir1_inst(tb, start + 10); + IR1_INST *sign_extend = tb_ir1_inst(tb, start + 11); + IR1_INST *accumulate = tb_ir1_inst(tb, start + 12); + IR1_INST *acc_store = tb_ir1_inst(tb, start + 13); + IR1_OPND *weighted0; + IR1_OPND *weighted1; + IR1_OPND *weighted_index; + IR1_OPND *factor_xmm; + IR1_OPND *value_xmm; + IR1_OPND *first_input; + IR1_OPND *first_table; + IR1_OPND *first_sub; + IR1_OPND *first_output; + IR1_INST *imul0 = tb_ir1_inst(tb, start + 1); + IR1_INST *movzx1 = tb_ir1_inst(tb, start + 2); + IR1_INST *imul1 = tb_ir1_inst(tb, start + 3); + IR1_INST *add_weights = tb_ir1_inst(tb, start + 4); + IR1_INST *movzx2 = tb_ir1_inst(tb, start + 5); + IR1_INST *lea9 = tb_ir1_inst(tb, start + 6); + IR1_INST *lea19 = tb_ir1_inst(tb, start + 7); + IR1_INST *add_index = tb_ir1_inst(tb, start + 8); + IR1_INST *shift_index = tb_ir1_inst(tb, start + 9); + int modified[3]; + bool match = true; + + for (int i = 0; i < 36; ++i) { + if (tb_ir1_inst(tb, start + i)->instptn.opc != + INSTPTN_OPC_NONE) { + match = false; + break; + } + } + if (!match || ir1_opcode(first) != WRAP(MOVZX) || + ir1_opcode(imul0) != WRAP(IMUL) || + ir1_opcode(movzx1) != WRAP(MOVZX) || + ir1_opcode(imul1) != WRAP(IMUL) || + ir1_opcode(add_weights) != WRAP(ADD) || + ir1_opcode(movzx2) != WRAP(MOVZX) || + ir1_opcode(lea9) != WRAP(LEA) || + ir1_opcode(lea19) != WRAP(LEA) || + ir1_opcode(add_index) != WRAP(ADD) || + ir1_opcode(shift_index) != WRAP(SHR) || + ir1_opcode(factor_load) != WRAP(VMOVSS) || + ir1_opcode(sign_extend) != WRAP(MOVSXD) || + ir1_opcode(accumulate) != WRAP(VADDSS) || + ir1_opcode(acc_store) != WRAP(VMOVSS) || + ir1_opcode(tb_ir1_inst(tb, start + 32)) != WRAP(INC) || + ir1_opcode(tb_ir1_inst(tb, start + 33)) != WRAP(ADD) || + ir1_opcode(tb_ir1_inst(tb, start + 34)) != WRAP(DEC) || + ir1_opcode(tb_ir1_inst(tb, start + 35)) != WRAP(JNE)) { + continue; + } + + weighted0 = ir1_get_opnd(first, 0); + weighted1 = ir1_get_opnd(movzx1, 0); + weighted_index = ir1_get_opnd(lea9, 0); + factor_xmm = ir1_get_opnd(factor_load, 0); + value_xmm = ir1_get_opnd(accumulate, 0); + first_input = ir1_get_opnd(first, 1); + first_table = ir1_get_opnd(tb_ir1_inst(tb, start + 16), 1); + first_sub = ir1_get_opnd(tb_ir1_inst(tb, start + 17), 2); + first_output = ir1_get_opnd(tb_ir1_inst(tb, start + 18), 2); + modified[0] = ir1_opnd_base_reg_num(weighted0); + modified[1] = ir1_opnd_base_reg_num(weighted1); + modified[2] = ir1_opnd_base_reg_num(weighted_index); + + if (!ir1_opnd_is_gpr(weighted0) || !ir1_opnd_is_gpr(weighted1) || + !ir1_opnd_is_gpr(weighted_index) || + modified[0] == modified[1] || modified[0] == modified[2] || + modified[1] == modified[2] || + !ir1_opnd_is_xmm(factor_xmm) || !ir1_opnd_is_xmm(value_xmm) || + scalar_hdr_same_xmm(factor_xmm, value_xmm) || + ir1_addr_size(first) != 64 || + ir1_get_opnd_num(first) != 2 || ir1_get_opnd_num(imul0) != 3 || + ir1_get_opnd_num(movzx1) != 2 || + ir1_get_opnd_num(imul1) != 3 || + ir1_get_opnd_num(add_weights) != 2 || + ir1_get_opnd_num(movzx2) != 2 || + ir1_get_opnd_num(lea9) != 2 || ir1_get_opnd_num(lea19) != 2 || + ir1_get_opnd_num(add_index) != 2 || + ir1_get_opnd_num(shift_index) != 2 || + !scalar_hdr_same_reg(ir1_get_opnd(imul0, 0), weighted0) || + !scalar_hdr_same_reg(ir1_get_opnd(imul0, 1), weighted0) || + !ir1_opnd_is_imm(ir1_get_opnd(imul0, 2)) || + ir1_get_opnd(imul0, 2)->imm != 0x36 || + !scalar_hdr_same_mem(first_input, ir1_get_opnd(movzx1, 1), 1) || + !scalar_hdr_same_reg(ir1_get_opnd(imul1, 0), weighted1) || + !scalar_hdr_same_reg(ir1_get_opnd(imul1, 1), weighted1) || + !ir1_opnd_is_imm(ir1_get_opnd(imul1, 2)) || + ir1_get_opnd(imul1, 2)->imm != 0xb7 || + !scalar_hdr_same_reg(ir1_get_opnd(add_weights, 0), weighted1) || + !scalar_hdr_same_reg(ir1_get_opnd(add_weights, 1), weighted0) || + !scalar_hdr_same_reg(ir1_get_opnd(movzx2, 0), weighted0) || + !scalar_hdr_same_mem(first_input, ir1_get_opnd(movzx2, 1), 2) || + !ir1_opnd_is_mem(ir1_get_opnd(lea9, 1)) || + ir1_get_opnd(lea9, 1)->mem.disp != 0 || + ir1_get_opnd(lea9, 1)->mem.scale != 8 || + ir1_opnd_base_reg_num(ir1_get_opnd(lea9, 1)) != modified[0] || + ir1_opnd_index_reg_num(ir1_get_opnd(lea9, 1)) != modified[0] || + !scalar_hdr_same_reg(ir1_get_opnd(lea19, 0), weighted_index) || + !ir1_opnd_is_mem(ir1_get_opnd(lea19, 1)) || + ir1_get_opnd(lea19, 1)->mem.disp != 0 || + ir1_get_opnd(lea19, 1)->mem.scale != 2 || + ir1_opnd_base_reg_num(ir1_get_opnd(lea19, 1)) != modified[0] || + ir1_opnd_index_reg_num(ir1_get_opnd(lea19, 1)) != modified[2] || + !scalar_hdr_same_reg(ir1_get_opnd(add_index, 0), weighted_index) || + !scalar_hdr_same_reg(ir1_get_opnd(add_index, 1), weighted1) || + !scalar_hdr_same_reg(ir1_get_opnd(shift_index, 0), weighted_index) || + !ir1_opnd_is_imm(ir1_get_opnd(shift_index, 1)) || + ir1_get_opnd(shift_index, 1)->imm != 8) { + continue; + } + + if (ir1_get_opnd_num(factor_load) != 2 || + !ir1_opnd_is_mem(ir1_get_opnd(factor_load, 1)) || + ir1_opnd_size(ir1_get_opnd(factor_load, 1)) != 32 || + ir1_opnd_index_reg_num(ir1_get_opnd(factor_load, 1)) != + modified[2] || + ir1_get_opnd(factor_load, 1)->mem.scale != 4 || + ir1_get_opnd_num(sign_extend) != 2 || + !scalar_hdr_same_reg(ir1_get_opnd(sign_extend, 0), + ir1_get_opnd(sign_extend, 1)) || + ir1_get_opnd_num(accumulate) != 3 || + !scalar_hdr_same_xmm(ir1_get_opnd(accumulate, 1), factor_xmm) || + !ir1_opnd_is_mem(ir1_get_opnd(accumulate, 2)) || + ir1_get_opnd_num(acc_store) != 2 || + !scalar_hdr_same_mem(ir1_get_opnd(accumulate, 2), + ir1_get_opnd(acc_store, 0), 0) || + !scalar_hdr_same_xmm(ir1_get_opnd(acc_store, 1), value_xmm) || + !ir1_opnd_is_gpr(ir1_get_opnd(tb_ir1_inst(tb, start + 32), 0)) || + !ir1_opnd_is_gpr(ir1_get_opnd(tb_ir1_inst(tb, start + 33), 0)) || + !ir1_opnd_is_gpr(ir1_get_opnd(tb_ir1_inst(tb, start + 34), 0))) { + continue; + } + + for (int i = 0; i < 3 && match; ++i) { + if (scalar_hdr_mem_uses_reg(first_input, modified[i]) || + scalar_hdr_mem_uses_reg(first_sub, modified[i]) || + scalar_hdr_mem_uses_reg(first_output, modified[i])) { + match = false; + } + } + if (!match || + !scalar_hdr_match_channel(tb, start + 14, first_input, + first_table, first_sub, first_output, + weighted_index, value_xmm, factor_xmm, 0) || + !scalar_hdr_match_channel(tb, start + 20, first_input, + first_table, first_sub, first_output, + weighted_index, value_xmm, factor_xmm, 1) || + !scalar_hdr_match_channel(tb, start + 26, first_input, + first_table, first_sub, first_output, + weighted_index, value_xmm, factor_xmm, 2)) { + continue; + } + + first->instptn.opc = INSTPTN_OPC_SCALAR_HDR; + for (int i = 1; i < 32; ++i) { + tb_ir1_inst(tb, start + i)->instptn.opc = INSTPTN_OPC_NOP; + } + start += 31; + } +} + #undef WRAP #endif diff --git a/target/i386/latx/optimization/ir1-optimization.c b/target/i386/latx/optimization/ir1-optimization.c index 99b32fcc13..6242afcd0b 100644 --- a/target/i386/latx/optimization/ir1-optimization.c +++ b/target/i386/latx/optimization/ir1-optimization.c @@ -53,6 +53,13 @@ static void ir1_optimization_over_tb(TranslationBlock *tb) OPT_INSTS_PTN(tb, ir1, i, ptn); } SAVE_FLAG_TO_TB(rdtn, tb); +#ifdef CONFIG_LATX_INSTS_PATTERN + if (option_instptn) { + insts_pattern_avx_sum3(tb); + insts_pattern_repeat_add(tb); + insts_pattern_scalar_hdr(tb); + } +#endif } static void get_eflag_out(TranslationBlock *tb) @@ -167,4 +174,11 @@ void ir1_optimization(TranslationBlock *tb) OPT_INSTS_PTN(tb, ir1, i, ptn); } SAVE_FLAG_TO_TB(rdtn, tb); +#ifdef CONFIG_LATX_INSTS_PATTERN + if (option_instptn) { + insts_pattern_avx_sum3(tb); + insts_pattern_repeat_add(tb); + insts_pattern_scalar_hdr(tb); + } +#endif } diff --git a/target/i386/latx/optimization/ir2-optimization.c b/target/i386/latx/optimization/ir2-optimization.c index 58288b5a9b..8e97959f2d 100644 --- a/target/i386/latx/optimization/ir2-optimization.c +++ b/target/i386/latx/optimization/ir2-optimization.c @@ -215,6 +215,75 @@ void tri_avoid_leading_label(void) #endif } +static bool ir2_is_scalar_arith_s(const IR2_INST *ir2) +{ + IR2_OPCODE opc = ir2_opcode(ir2); + + return opc == LISA_FADD_S || opc == LISA_FSUB_S || opc == LISA_FMUL_S; +} + +static bool ir2_is_scalar_fma_s(const IR2_INST *ir2) +{ + IR2_OPCODE opc = ir2_opcode(ir2); + + return opc == LISA_FMADD_S || opc == LISA_FMSUB_S; +} + +static void ir2_opt_scalar_fma_forward(void) +{ + IR2_INST *arith = lsenv->tr_data->first_ir2; + + while (arith) { + IR2_INST *copy = NULL; + IR2_INST *insert; + IR2_INST *fma; + IR2_INST *final_insert; + IR2_INST *next; + + if (!ir2_is_scalar_arith_s(arith)) { + arith = ir2_next(arith); + continue; + } + insert = ir2_next(arith); + if (insert && ir2_opcode(insert) == LISA_VORI_B) { + copy = insert; + insert = ir2_next(insert); + } + if (!insert || ir2_opcode(insert) != LISA_VEXTRINS_W || + !ir2_opnd_cmp(&insert->_opnd[1], &arith->_opnd[0]) || + insert->_opnd[2]._imm32 != 0 || + (copy && + (!ir2_opnd_cmp(©->_opnd[0], &insert->_opnd[0]) || + copy->_opnd[2]._imm32 != 0))) { + arith = ir2_next(arith); + continue; + } + fma = ir2_next(insert); + while (fma && la_ir2_opcode_is_x86_inst(ir2_opcode(fma))) { + fma = ir2_next(fma); + } + if (!fma || !ir2_is_scalar_fma_s(fma) || + !ir2_opnd_cmp(&fma->_opnd[3], &insert->_opnd[0])) { + arith = ir2_next(arith); + continue; + } + final_insert = ir2_next(fma); + if (!final_insert || ir2_opcode(final_insert) != LISA_VEXTRINS_W || + !ir2_opnd_cmp(&final_insert->_opnd[0], &insert->_opnd[0]) || + !ir2_opnd_cmp(&final_insert->_opnd[1], &fma->_opnd[0]) || + final_insert->_opnd[2]._imm32 != 0) { + arith = ir2_next(arith); + continue; + } + + /* Keep the scalar result in the temporary until the fused operation. */ + fma->_opnd[3] = arith->_opnd[0]; + next = ir2_next(insert); + ir2_remove(ir2_get_id(insert)); + arith = next; + } +} + static int ir2_get_addi_rsp_offs(IR2_INST *ir2) { int rsp = reg_gpr_map[esp_index]; @@ -528,6 +597,7 @@ static void ir2_opt_push_pop(TranslationBlock *tb) void tr_ir2_optimize(TranslationBlock *tb) { + ir2_opt_scalar_fma_forward(); #ifdef CONFIG_LATX_OPT_PUSH_POP CPUX86State *env = (CPUX86State*)lsenv->cpu_state; CPUState *cpu = env_cpu(env); diff --git a/target/i386/latx/translator/tr-avx-cmp.c b/target/i386/latx/translator/tr-avx-cmp.c index eae7273da3..5fb062c777 100644 --- a/target/i386/latx/translator/tr-avx-cmp.c +++ b/target/i386/latx/translator/tr-avx-cmp.c @@ -1120,9 +1120,7 @@ bool translate_vcmpeqps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_EQ); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_EQ); set_high128_xreg_to_zero(dest); } return true; @@ -1146,9 +1144,7 @@ bool translate_vcmpltps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - la_vfcmp_cond_s(temp, src1, src2, X86_FCMP_COND_LT); - la_xvori_b(dest, temp, 0); + la_vfcmp_cond_s(dest, src1, src2, X86_FCMP_COND_LT); set_high128_xreg_to_zero(dest); } return true; diff --git a/target/i386/latx/translator/tr-avx-mov.c b/target/i386/latx/translator/tr-avx-mov.c index 58a5bfb09d..09747db289 100644 --- a/target/i386/latx/translator/tr-avx-mov.c +++ b/target/i386/latx/translator/tr-avx-mov.c @@ -78,20 +78,18 @@ bool translate_vmovaps(IR1_INST * pir1) { la_xvori_b(ra_alloc_xmm(ir1_opnd_base_reg_num(dest)), ra_alloc_xmm(ir1_opnd_base_reg_num(src)), 0); } else if (ir1_opnd_is_xmm(dest) && ir1_opnd_is_mem(src)) { - IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND dest_reg = ra_alloc_xmm(ir1_opnd_base_reg_num(dest)); - load_freg128_from_ir1_mem(temp, src); - la_xvori_b(ra_alloc_xmm(ir1_opnd_base_reg_num(dest)), temp, 0); - set_high128_xreg_to_zero(ra_alloc_xmm(ir1_opnd_base_reg_num(dest))); + load_freg128_from_ir1_mem(dest_reg, src); + set_high128_xreg_to_zero(dest_reg); } else if (ir1_opnd_is_mem(dest) && ir1_opnd_is_xmm(src)) { store_freg128_to_ir1_mem(ra_alloc_xmm(ir1_opnd_base_reg_num(src)), dest); } else if (ir1_opnd_is_xmm(dest) && ir1_opnd_is_xmm(src)) { - IR2_OPND temp = ra_alloc_ftemp(); + IR2_OPND dest_reg = ra_alloc_xmm(ir1_opnd_base_reg_num(dest)); - la_vori_b(temp, ra_alloc_xmm(ir1_opnd_base_reg_num(src)), 0); - la_xvori_b(ra_alloc_xmm(ir1_opnd_base_reg_num(dest)), temp, 0); - set_high128_xreg_to_zero(ra_alloc_xmm(ir1_opnd_base_reg_num(dest))); + la_vori_b(dest_reg, ra_alloc_xmm(ir1_opnd_base_reg_num(src)), 0); + set_high128_xreg_to_zero(dest_reg); } else { #ifdef CONFIG_LATX_TS return false; @@ -209,10 +207,7 @@ bool translate_vmovshdup(IR1_INST * pir1) { } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp = ra_alloc_ftemp(); - - la_xvpackod_w(temp, src, src); - la_xvori_b(dest, temp, 0); + la_vpackod_w(dest, src, src); set_high128_xreg_to_zero(dest); } return true; @@ -312,9 +307,9 @@ bool translate_vmovss(IR1_INST *pir1) } IR1_OPND *opnd0 = ir1_get_opnd(pir1, 0); IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1); - IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); if (ir1_opnd_is_xmm(opnd0) && ir1_opnd_is_xmm(opnd1)) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); + IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); la_xvori_b(temp, src1, 0); @@ -323,9 +318,20 @@ bool translate_vmovss(IR1_INST *pir1) set_high128_xreg_to_zero(dest); } else if (ir1_opnd_is_xmm(opnd0) && ir1_opnd_is_mem(opnd1)) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); - la_xvpickve_w(dest, src1, 0); - set_high128_xreg_to_zero(dest); + if (SHBR_ON_32(pir1)) { + int little_disp; + IR2_OPND mem = convert_mem(opnd1, &little_disp); + + la_fld_s(dest, mem, little_disp); + set_high128_xreg_to_zero(dest); + } else { + IR2_OPND src1 = load_freg128_from_ir1(opnd1); + + la_xvpickve_w(dest, src1, 0); + mark_high128_xreg_zeroed(dest); + } } else if (ir1_opnd_is_mem(opnd0) && ir1_opnd_is_xmm(opnd1)) { + IR2_OPND src1 = load_freg128_from_ir1(opnd1); int little_disp; IR2_OPND mem_opnd = convert_mem(opnd0, &little_disp); la_fst_s(src1, mem_opnd, little_disp); @@ -658,7 +664,10 @@ bool translate_vmovsd(IR1_INST * pir1) { return true; } if (ir1_opnd_is_xmm(dest) && ir1_opnd_is_mem(src)) { - la_xvpickve_d(ra_alloc_xmm(ir1_opnd_base_reg_num(dest)), load_freg128_from_ir1(src), 0); + IR2_OPND dest_reg = ra_alloc_xmm(ir1_opnd_base_reg_num(dest)); + + la_xvpickve_d(dest_reg, load_freg128_from_ir1(src), 0); + mark_high128_xreg_zeroed(dest_reg); return true; } if (ir1_opnd_is_mem(dest) && ir1_opnd_is_xmm(src)) { diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 46e2b422f2..6f4de9b0d2 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -63,11 +63,8 @@ bool translate_vaddps(IR1_INST * pir1) { src2 = load_freg256_from_ir1(opnd2); la_xvfadd_s(dest, src1, src2); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); - la_vfadd_s(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vfadd_s(dest, src1, src2); set_high128_xreg_to_zero(dest); } return true; @@ -102,14 +99,18 @@ bool translate_vaddss(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); + if (SHBR_ON_32(pir1)) { + la_fadd_s(dest, src1, src2); + } else { + IR2_OPND temp = ra_alloc_ftemp(); - la_fadd_s(temp, src1, src2); - if (ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) { - la_xvori_b(dest, src1, 0); + la_fadd_s(temp, src1, src2); + if (ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) { + la_vori_b(dest, src1, 0); + } + la_vextrins_w(dest, temp, VEXTRINS_IMM_4_0(0, 0)); } - la_xvinsve0_w(dest, temp, 0); set_high128_xreg_to_zero(dest); return true; } @@ -164,11 +165,8 @@ bool translate_vsubps(IR1_INST * pir1) { src2 = load_freg256_from_ir1(opnd2); la_xvfsub_s(dest, src1, src2); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); - la_vfsub_s(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vfsub_s(dest, src1, src2); set_high128_xreg_to_zero(dest); } return true; @@ -204,14 +202,18 @@ bool translate_vsubss(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); + if (SHBR_ON_32(pir1)) { + la_fsub_s(dest, src1, src2); + } else { + IR2_OPND temp = ra_alloc_ftemp(); - la_fsub_s(temp, src1, src2); - if (ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) { - la_xvori_b(dest, src1, 0); + la_fsub_s(temp, src1, src2); + if (ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) { + la_vori_b(dest, src1, 0); + } + la_vextrins_w(dest, temp, VEXTRINS_IMM_4_0(0, 0)); } - la_xvinsve0_w(dest, temp, 0); set_high128_xreg_to_zero(dest); return true; } @@ -266,11 +268,8 @@ bool translate_vmulps(IR1_INST * pir1) { src2 = load_freg256_from_ir1(opnd2); la_xvfmul_s(dest, src1, src2); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); - la_vfmul_s(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vfmul_s(dest, src1, src2); set_high128_xreg_to_zero(dest); } return true; @@ -304,14 +303,18 @@ bool translate_vmulss(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); + if (SHBR_ON_32(pir1)) { + la_fmul_s(dest, src1, src2); + } else { + IR2_OPND temp = ra_alloc_ftemp(); - la_fmul_s(temp, src1, src2); - if (ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != - ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) { - la_xvori_b(dest, src1, 0); + la_fmul_s(temp, src1, src2); + if (ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) { + la_vori_b(dest, src1, 0); + } + la_vextrins_w(dest, temp, VEXTRINS_IMM_4_0(0, 0)); } - la_xvinsve0_w(dest, temp, 0); set_high128_xreg_to_zero(dest); return true; } @@ -366,11 +369,8 @@ bool translate_vdivps(IR1_INST * pir1) { src2 = load_freg256_from_ir1(opnd2); la_xvfdiv_s(dest, src1, src2); } else if (ir1_opnd_is_xmm(opnd0)) { - IR2_OPND temp = ra_alloc_ftemp(); - src2 = load_freg128_from_ir1(opnd2); - la_vfdiv_s(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vfdiv_s(dest, src1, src2); set_high128_xreg_to_zero(dest); } return true; @@ -405,12 +405,17 @@ bool translate_vdivss(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); - la_fdiv_s(temp, src1, src2); - if (ir1_opnd_base_reg_num(opnd0) != ir1_opnd_base_reg_num(opnd1)) { - la_xvori_b(dest, src1, 0); + if (SHBR_ON_32(pir1)) { + la_fdiv_s(dest, src1, src2); + } else { + IR2_OPND temp = ra_alloc_ftemp(); + + la_fdiv_s(temp, src1, src2); + if (ir1_opnd_base_reg_num(opnd0) != ir1_opnd_base_reg_num(opnd1)) { + la_vori_b(dest, src1, 0); + } + la_vextrins_w(dest, temp, VEXTRINS_IMM_4_0(0, 0)); } - la_xvinsve0_w(dest, temp, 0); set_high128_xreg_to_zero(dest); return true; } @@ -502,9 +507,9 @@ bool translate_vsqrtss(IR1_INST * pir1) { IR2_OPND temp = ra_alloc_ftemp(); la_fsqrt_s(temp, src2); if (ir1_opnd_base_reg_num(opnd0) != ir1_opnd_base_reg_num(opnd1)) { - la_xvori_b(dest, src1, 0); + la_vori_b(dest, src1, 0); } - la_xvinsve0_w(dest, temp, 0); + la_vextrins_w(dest, temp, VEXTRINS_IMM_4_0(0, 0)); set_high128_xreg_to_zero(dest); return true; } @@ -792,10 +797,7 @@ bool translate_vandpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); - - la_vand_v(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vand_v(dest, src1, src2); set_high128_xreg_to_zero(dest); } return true; @@ -942,7 +944,6 @@ bool translate_vminpx(IR1_INST * pir1) { IR2_OPND src2 = load_freg256_from_ir1(opnd2); IR2_INST * ( * tr_inst_128)(IR2_OPND, IR2_OPND, IR2_OPND, int); IR2_INST * ( * tr_inst_256)(IR2_OPND, IR2_OPND, IR2_OPND, int); - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND mask = ra_alloc_ftemp(); IR1_OPCODE op = ir1_opcode(pir1); switch (op) { @@ -964,9 +965,11 @@ bool translate_vminpx(IR1_INST * pir1) { tr_inst_256(mask, src1, src2, 0x3); else tr_inst_128(mask, src1, src2, 0x3); - la_xvand_v(temp, src1, mask); - la_xvandn_v(mask, mask, src2); - la_xvor_v(dest, temp, mask); + if (ir1_opnd_is_ymm(opnd0)) { + la_xvbitsel_v(dest, src2, src1, mask); + } else { + la_vbitsel_v(dest, src2, src1, mask); + } if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); return true; @@ -1028,7 +1031,6 @@ bool translate_vmaxpx(IR1_INST * pir1) { IR2_OPND src2 = load_freg256_from_ir1(opnd2); IR2_INST * ( * tr_inst_128)(IR2_OPND, IR2_OPND, IR2_OPND, int); IR2_INST * ( * tr_inst_256)(IR2_OPND, IR2_OPND, IR2_OPND, int); - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND mask = ra_alloc_ftemp(); IR1_OPCODE op = ir1_opcode(pir1); switch (op) { @@ -1050,9 +1052,11 @@ bool translate_vmaxpx(IR1_INST * pir1) { tr_inst_256(mask, src2, src1, 0x3); else tr_inst_128(mask, src2, src1, 0x3); - la_xvand_v(temp, src1, mask); - la_xvandn_v(mask, mask, src2); - la_xvor_v(dest, temp, mask); + if (ir1_opnd_is_ymm(opnd0)) { + la_xvbitsel_v(dest, src2, src1, mask); + } else { + la_vbitsel_v(dest, src2, src1, mask); + } if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); return true; @@ -1147,6 +1151,24 @@ bool translate_vblendps(IR1_INST * pir1) { IR2_OPND src1 = load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); uint8 imm = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)); + + if (ir1_opnd_is_xmm(ir1_get_opnd(pir1, 0)) && + ir1_opnd_is_xmm(ir1_get_opnd(pir1, 2)) && + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 2))) { + if (ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) != + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))) { + la_vori_b(dest, src1, 0); + } + for (int i = 0; i < 4; ++i) { + if (imm & (1 << i)) { + la_vextrins_w(dest, src2, VEXTRINS_IMM_4_0(i, i)); + } + } + set_high128_xreg_to_zero(dest); + return true; + } + IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND rmask = ra_alloc_itemp(); @@ -1227,9 +1249,7 @@ bool translate_vbroadcastss(IR1_INST * pir1) { } else { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND temp = ra_alloc_ftemp(); - la_xvreplve0_w(temp, src); - la_xvori_b(dest, temp, 0); + la_vreplvei_w(dest, src, 0); set_high128_xreg_to_zero(dest); } return true; @@ -1337,7 +1357,13 @@ bool translate_vinsertps(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); + bool dest_is_src1 = ir1_opnd_base_reg_num(opnd0) == + ir1_opnd_base_reg_num(opnd1); + bool dest_is_src2 = ir1_opnd_is_xmm(opnd2) && + ir1_opnd_base_reg_num(opnd0) == + ir1_opnd_base_reg_num(opnd2); + bool write_dest = dest_is_src1 || !dest_is_src2; + IR2_OPND temp = write_dest ? dest : ra_alloc_ftemp(); uint8_t imm = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)); uint8_t count_s = (imm >> 6) & 0x3; uint8_t count_d = (imm >> 4) & 0x3; @@ -1345,8 +1371,44 @@ bool translate_vinsertps(IR1_INST * pir1) { if (ir1_opnd_is_mem(opnd2)) { count_s = 0; } - la_vori_b(temp, src1, 0); + if (ir1_opnd_is_xmm(opnd2) && + ir1_opnd_base_reg_num(opnd1) == ir1_opnd_base_reg_num(opnd2) && + count_s == 1 && count_d == 0 && zmask == 0xc) { + /* {src1.w[1], src1.w[1], 0, 0} */ + la_vreplvei_w(dest, src1, 1); + la_vinsgr2vr_d(dest, zero_ir2_opnd, 1); + set_high128_xreg_to_zero(dest); + return true; + } + if (ir1_opnd_is_xmm(opnd2) && count_s == 0 && count_d == 1 && + zmask == 0xc) { + /* {src1.w[0], src2.w[0], 0, 0} */ + la_vilvl_w(dest, src2, src1); + la_vinsgr2vr_d(dest, zero_ir2_opnd, 1); + set_high128_xreg_to_zero(dest); + return true; + } + if (dest_is_src2 && !dest_is_src1 && count_s == 2 && count_d == 1 && + zmask == 0xc) { + /* {src1.w[0], src2.w[2], 0, 0}; preserve src2 before dest overwrite. */ + la_vreplvei_w(dest, src2, 2); + la_vilvl_w(dest, dest, src1); + la_vinsgr2vr_d(dest, zero_ir2_opnd, 1); + set_high128_xreg_to_zero(dest); + return true; + } + if (!dest_is_src1) { + la_vori_b(temp, src1, 0); + } la_vextrins_w(temp, src2, VEXTRINS_IMM_4_0(count_d, count_s)); + if ((zmask & 0x3) == 0x3) { + la_vinsgr2vr_d(temp, zero_ir2_opnd, 0); + zmask &= ~0x3; + } + if ((zmask & 0xc) == 0xc) { + la_vinsgr2vr_d(temp, zero_ir2_opnd, 1); + zmask &= ~0xc; + } if (zmask & 0x1) la_vinsgr2vr_w(temp, zero_ir2_opnd, 0); if (zmask & 0x2) @@ -1355,7 +1417,9 @@ bool translate_vinsertps(IR1_INST * pir1) { la_vinsgr2vr_w(temp, zero_ir2_opnd, 2); if (zmask & 0x8) la_vinsgr2vr_w(temp, zero_ir2_opnd, 3); - la_vori_b(dest, temp, 0); + if (!write_dest) { + la_vori_b(dest, temp, 0); + } set_high128_xreg_to_zero(dest); return true; } @@ -1462,7 +1526,16 @@ bool translate_vshufpd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); + bool same_sources = ir1_opnd_is_xmm(ir1_get_opnd(pir1, 2)) && + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1)) == + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 2)); + bool dest_is_src1 = ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) == + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1)); + bool dest_is_src2 = ir1_opnd_is_xmm(ir1_get_opnd(pir1, 2)) && + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) == + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 2)); + bool write_dest = dest_is_src1 || !dest_is_src2; + IR2_OPND temp = write_dest ? dest : ra_alloc_ftemp(); uint8_t imm8 = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)); imm8 = imm8 & 3; if (imm8 == 0) { @@ -1476,9 +1549,21 @@ bool translate_vshufpd(IR1_INST * pir1) { } else { lsassert(0); } - la_xvori_b(temp, src1, 0); + if (same_sources) { + static const uint8_t permi[4] = {0x44, 0x4e, 0xe4, 0xee}; + + la_vshuf4i_w(dest, src1, permi[ + ir1_opnd_uimm(ir1_get_opnd(pir1, 3)) & 3]); + set_high128_xreg_to_zero(dest); + return true; + } + if (!dest_is_src1) { + la_vori_b(temp, src1, 0); + } la_vshuf4i_d(temp, src2, imm8); - la_xvori_b(dest, temp, 0); + if (!write_dest) { + la_vori_b(dest, temp, 0); + } set_high128_xreg_to_zero(dest); } return true; @@ -1493,21 +1578,35 @@ bool translate_vshufps(IR1_INST * pir1) { ir1_opnd_is_xmm(ir1_get_opnd(pir1, 1))) || (ir1_opnd_is_ymm(ir1_get_opnd(pir1, 0)) && ir1_opnd_is_ymm(ir1_get_opnd(pir1, 1)))); - if (ir1_opnd_is_ymm(ir1_get_opnd(pir1, 0))) { - IR2_OPND dest = load_freg256_from_ir1(ir1_get_opnd(pir1, 0)); - IR2_OPND src1 = load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); - uint64_t imm8 = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)); + IR1_OPND *opnd0 = ir1_get_opnd(pir1, 0); + IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1); + IR1_OPND *opnd2 = ir1_get_opnd(pir1, 2); + uint64_t imm8 = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)); + bool same_sources = ir1_opnd_is_xmm(opnd2) && + ir1_opnd_base_reg_num(opnd1) == ir1_opnd_base_reg_num(opnd2); + + if (ir1_opnd_is_ymm(opnd0)) { + IR2_OPND dest = load_freg256_from_ir1(opnd0); + IR2_OPND src1 = load_freg256_from_ir1(opnd1); + if (same_sources) { + la_xvshuf4i_w(dest, src1, imm8); + return true; + } + IR2_OPND src2 = load_freg256_from_ir1(opnd2); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); la_xvshuf4i_w(temp1, src1, imm8); la_xvshuf4i_w(temp2, src2, imm8 >> 4); la_xvpickev_d(dest, temp2, temp1); } else { - IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); - IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); - IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - uint64_t imm8 = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)); + IR2_OPND dest = load_freg128_from_ir1(opnd0); + IR2_OPND src1 = load_freg128_from_ir1(opnd1); + if (same_sources) { + la_vshuf4i_w(dest, src1, imm8); + set_high128_xreg_to_zero(dest); + return true; + } + IR2_OPND src2 = load_freg128_from_ir1(opnd2); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); la_vshuf4i_w(temp1, src1, imm8); @@ -1819,9 +1918,7 @@ bool translate_vpand(IR1_INST * pir1) { la_xvand_v(dest, src1, src2); return true; } - IR2_OPND temp = ra_alloc_ftemp(); - la_xvand_v(temp, src1, src2); - la_xvori_b(dest, temp, 0); + la_vand_v(dest, src1, src2); set_high128_xreg_to_zero(dest); return true; } @@ -2560,7 +2657,6 @@ bool translate_vfmaddxxxss(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND temp1, temp2, temp3; IR1_OPCODE op = ir1_opcode(pir1); switch (op) { @@ -2578,8 +2674,14 @@ bool translate_vfmaddxxxss(IR1_INST * pir1) { break; } - la_fmadd_s(temp, temp1, temp2, temp3); - la_xvinsve0_w(dest, temp, 0); + if (SHBR_ON_32(pir1)) { + la_fmadd_s(dest, temp1, temp2, temp3); + } else { + IR2_OPND temp = ra_alloc_ftemp(); + + la_fmadd_s(temp, temp1, temp2, temp3); + la_vextrins_w(dest, temp, VEXTRINS_IMM_4_0(0, 0)); + } set_high128_xreg_to_zero(dest); return true; } @@ -2613,7 +2715,7 @@ bool translate_vfmaddxxxsd(IR1_INST * pir1) { } la_fmadd_d(temp, temp1, temp2, temp3); - la_xvinsve0_d(dest, temp, 0); + la_vextrins_d(dest, temp, VEXTRINS_IMM_4_0(0, 0)); set_high128_xreg_to_zero(dest); return true; } @@ -2632,7 +2734,6 @@ bool translate_vfmaddxxxpd(IR1_INST * pir1) { IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); IR2_INST * ( * tr_inst)(IR2_OPND, IR2_OPND, IR2_OPND, IR2_OPND); - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND temp1, temp2, temp3; IR1_OPCODE op = ir1_opcode(pir1); switch (op) { @@ -2651,8 +2752,7 @@ bool translate_vfmaddxxxpd(IR1_INST * pir1) { } tr_inst = ir1_opnd_is_xmm(opnd0) ? la_vfmadd_d : la_xvfmadd_d; - tr_inst(temp, temp1, temp2, temp3); - la_xvori_b(dest, temp, 0); + tr_inst(dest, temp1, temp2, temp3); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); return true; @@ -2672,7 +2772,6 @@ bool translate_vfmaddxxxps(IR1_INST * pir1) { IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); IR2_INST * ( * tr_inst)(IR2_OPND, IR2_OPND, IR2_OPND, IR2_OPND); - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND temp1, temp2, temp3; IR1_OPCODE op = ir1_opcode(pir1); switch (op) { @@ -2691,8 +2790,7 @@ bool translate_vfmaddxxxps(IR1_INST * pir1) { } tr_inst = ir1_opnd_is_xmm(opnd0) ? la_vfmadd_s : la_xvfmadd_s; - tr_inst(temp, temp1, temp2, temp3); - la_xvori_b(dest, temp, 0); + tr_inst(dest, temp1, temp2, temp3); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); return true; @@ -2708,7 +2806,6 @@ bool translate_vfmsubxxxss(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); IR2_OPND src1 = load_freg128_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND temp1, temp2, temp3; IR1_OPCODE op = ir1_opcode(pir1); switch (op) { @@ -2726,8 +2823,14 @@ bool translate_vfmsubxxxss(IR1_INST * pir1) { break; } - la_fmsub_s(temp, temp1, temp2, temp3); - la_xvinsve0_w(dest, temp, 0); + if (SHBR_ON_32(pir1)) { + la_fmsub_s(dest, temp1, temp2, temp3); + } else { + IR2_OPND temp = ra_alloc_ftemp(); + + la_fmsub_s(temp, temp1, temp2, temp3); + la_vextrins_w(dest, temp, VEXTRINS_IMM_4_0(0, 0)); + } set_high128_xreg_to_zero(dest); return true; } @@ -2761,7 +2864,7 @@ bool translate_vfmsubxxxsd(IR1_INST * pir1) { } la_fmsub_d(temp, temp1, temp2, temp3); - la_xvinsve0_d(dest, temp, 0); + la_vextrins_d(dest, temp, VEXTRINS_IMM_4_0(0, 0)); set_high128_xreg_to_zero(dest); return true; } @@ -2780,7 +2883,6 @@ bool translate_vfmsubxxxpd(IR1_INST * pir1) { IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); IR2_INST * ( * tr_inst)(IR2_OPND, IR2_OPND, IR2_OPND, IR2_OPND); - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND temp1, temp2, temp3; IR1_OPCODE op = ir1_opcode(pir1); switch (op) { @@ -2799,8 +2901,7 @@ bool translate_vfmsubxxxpd(IR1_INST * pir1) { } tr_inst = ir1_opnd_is_xmm(opnd0) ? la_vfmsub_d : la_xvfmsub_d; - tr_inst(temp, temp1, temp2, temp3); - la_xvori_b(dest, temp, 0); + tr_inst(dest, temp1, temp2, temp3); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); return true; @@ -2820,7 +2921,6 @@ bool translate_vfmsubxxxps(IR1_INST * pir1) { IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(ir1_get_opnd(pir1, 2)); IR2_INST * ( * tr_inst)(IR2_OPND, IR2_OPND, IR2_OPND, IR2_OPND); - IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND temp1, temp2, temp3; IR1_OPCODE op = ir1_opcode(pir1); switch (op) { @@ -2839,8 +2939,7 @@ bool translate_vfmsubxxxps(IR1_INST * pir1) { } tr_inst = ir1_opnd_is_xmm(opnd0) ? la_vfmsub_s : la_xvfmsub_s; - tr_inst(temp, temp1, temp2, temp3); - la_xvori_b(dest, temp, 0); + tr_inst(dest, temp1, temp2, temp3); if (ir1_opnd_is_xmm(opnd0)) set_high128_xreg_to_zero(dest); return true; @@ -2888,7 +2987,7 @@ bool translate_vfnmaddxxxss(IR1_INST * pir1) { la_fmsub_s(temp, src1, src2, dest); la_label(label_over); - la_xvinsve0_w(dest, temp, 0); + la_vextrins_w(dest, temp, VEXTRINS_IMM_4_0(0, 0)); set_high128_xreg_to_zero(dest); return true; } @@ -2935,7 +3034,7 @@ bool translate_vfnmaddxxxsd(IR1_INST * pir1) { la_fmsub_d(temp, src1, src2, dest); la_label(label_over); - la_xvinsve0_d(dest, temp, 0); + la_vextrins_d(dest, temp, VEXTRINS_IMM_4_0(0, 0)); set_high128_xreg_to_zero(dest); return true; } @@ -3117,7 +3216,7 @@ bool translate_vfnmsubxxxss(IR1_INST * pir1) { la_fmsub_s(temp, src1, src2, dest); la_label(label_over); - la_xvinsve0_w(dest, temp, 0); + la_vextrins_w(dest, temp, VEXTRINS_IMM_4_0(0, 0)); set_high128_xreg_to_zero(dest); return true; } @@ -3165,7 +3264,7 @@ bool translate_vfnmsubxxxsd(IR1_INST * pir1) { la_fmsub_d(temp, src1, src2, dest); la_label(label_over); - la_xvinsve0_d(dest, temp, 0); + la_vextrins_d(dest, temp, VEXTRINS_IMM_4_0(0, 0)); set_high128_xreg_to_zero(dest); return true; } @@ -3496,7 +3595,7 @@ bool translate_vpbroadcastd(IR1_INST *pir1) } if (ir1_opnd_is_xmm(opnd0)) { - la_xvreplve0_w(dest, src); + la_vreplvei_w(dest, src, 0); } else if (ir1_opnd_is_ymm(opnd0)) { la_xvreplve0_w(dest, src); la_xvinsve0_d(dest, dest, 2); @@ -5206,6 +5305,74 @@ bool translate_vpsadbw(IR1_INST *pir1) return true; } +static void translate_vroundps_trunc_sae(IR2_OPND dest, IR2_OPND src, + bool is_xmm) +{ + IR2_OPND nan_mask = ra_alloc_ftemp(); + IR2_OPND exponent = ra_alloc_ftemp(); + IR2_OPND shift = ra_alloc_ftemp(); + IR2_OPND bits_mask = ra_alloc_ftemp(); + IR2_OPND sign = ra_alloc_ftemp(); + IR2_OPND select_mask = ra_alloc_ftemp(); + IR2_OPND quiet_nan = ra_alloc_ftemp(); + + if (is_xmm) { + la_vfcmp_cond_s(nan_mask, src, src, FCMP_COND_CUN); + la_vslli_w(exponent, src, 1); + la_vsrli_w(exponent, exponent, 24); + la_vldi(shift, VLDI_IMM_TYPE0(2, 150)); + la_vsub_w(shift, shift, exponent); + la_vseq_w(bits_mask, exponent, exponent); + la_vsll_w(bits_mask, bits_mask, shift); + la_vand_v(dest, src, bits_mask); + + la_vldi(sign, 0b1001110000000); + la_vsrli_w(quiet_nan, sign, 9); + la_vand_v(sign, src, sign); + la_vldi(bits_mask, VLDI_IMM_TYPE0(2, 127)); + la_vslt_wu(select_mask, exponent, bits_mask); + la_vbitsel_v(dest, dest, sign, select_mask); + + la_vldi(bits_mask, VLDI_IMM_TYPE0(2, 150)); + la_vslt_wu(select_mask, exponent, bits_mask); + la_vbitsel_v(dest, src, dest, select_mask); + + la_vor_v(quiet_nan, src, quiet_nan); + la_vbitsel_v(dest, dest, quiet_nan, nan_mask); + } else { + la_xvfcmp_cond_s(nan_mask, src, src, FCMP_COND_CUN); + la_xvslli_w(exponent, src, 1); + la_xvsrli_w(exponent, exponent, 24); + la_xvldi(shift, VLDI_IMM_TYPE0(2, 150)); + la_xvsub_w(shift, shift, exponent); + la_xvseq_w(bits_mask, exponent, exponent); + la_xvsll_w(bits_mask, bits_mask, shift); + la_xvand_v(dest, src, bits_mask); + + la_xvldi(sign, 0b1001110000000); + la_xvsrli_w(quiet_nan, sign, 9); + la_xvand_v(sign, src, sign); + la_xvldi(bits_mask, VLDI_IMM_TYPE0(2, 127)); + la_xvslt_wu(select_mask, exponent, bits_mask); + la_xvbitsel_v(dest, dest, sign, select_mask); + + la_xvldi(bits_mask, VLDI_IMM_TYPE0(2, 150)); + la_xvslt_wu(select_mask, exponent, bits_mask); + la_xvbitsel_v(dest, src, dest, select_mask); + + la_xvor_v(quiet_nan, src, quiet_nan); + la_xvbitsel_v(dest, dest, quiet_nan, nan_mask); + } + + ra_free_temp(quiet_nan); + ra_free_temp(select_mask); + ra_free_temp(sign); + ra_free_temp(bits_mask); + ra_free_temp(shift); + ra_free_temp(exponent); + ra_free_temp(nan_mask); +} + bool translate_vroundps(IR1_INST *pir1) { if (!option_enable_lasx) { @@ -5220,11 +5387,25 @@ bool translate_vroundps(IR1_INST *pir1) IR2_OPND src = load_freg256_from_ir1(opnd1); uint8_t imm = ir1_opnd_uimm(opnd2); + bool is_xmm = ir1_opnd_is_xmm(opnd0); + + if (imm == 0xb) { + /* + * Integer masking avoids both FCSR serialization and the precision + * flag that imm[3] suppresses. CUN still raises invalid for SNaN. + */ + translate_vroundps_trunc_sae(dest, src, is_xmm); + if (is_xmm) { + set_high128_xreg_to_zero(dest); + } + return true; + } + IR2_OPND temp = ra_alloc_ftemp(); IR2_OPND fcsr = ra_alloc_itemp(); IR2_OPND fcsr_save = ra_alloc_itemp(); IR2_OPND mxcsr = ra_alloc_itemp(); - bool is_xmm = ir1_opnd_is_xmm(opnd0); + if(imm & 0x8){ IR2_INST * ( * tr_inst)(IR2_OPND, IR2_OPND, IR2_OPND, int); tr_inst = is_xmm ? la_vfcmp_cond_s : la_xvfcmp_cond_s; diff --git a/target/i386/latx/translator/tr-eflag-process.c b/target/i386/latx/translator/tr-eflag-process.c index cf40a80ea3..3e1c9e84c7 100644 --- a/target/i386/latx/translator/tr-eflag-process.c +++ b/target/i386/latx/translator/tr-eflag-process.c @@ -488,15 +488,19 @@ void generate_eflag_calculation(IR2_OPND dest, IR2_OPND src0, IR2_OPND src1, ir1_get_eflag_def(pir1) & (~(lsenv->tr_data->curr_ir1_skipped_eflags)); switch (ir1_opcode(pir1)) { case WRAP(COMISS): + case WRAP(VCOMISS): generate_xcomisx(src0, src1, false, true, use_flags); return; case WRAP(COMISD): + case WRAP(VCOMISD): generate_xcomisx(src0, src1, true, true, use_flags); return; case WRAP(UCOMISS): + case WRAP(VUCOMISS): generate_xcomisx(src0, src1, false, false, use_flags); return; case WRAP(UCOMISD): + case WRAP(VUCOMISD): generate_xcomisx(src0, src1, true, false, use_flags); return; default: diff --git a/target/i386/latx/translator/tr-opnd-process.c b/target/i386/latx/translator/tr-opnd-process.c index 9e9d6391a1..2006abae43 100644 --- a/target/i386/latx/translator/tr-opnd-process.c +++ b/target/i386/latx/translator/tr-opnd-process.c @@ -1389,23 +1389,50 @@ static int mapped_ymm_index(IR2_OPND opnd) return -1; } +static void clear_high128_xreg_with_zero(IR2_OPND opnd, IR2_OPND zero) +{ + la_xvpermi_q(opnd, zero, XVPERMI_Q_4_0(0, 2)); +} + static void clear_high128_xreg_now(IR2_OPND opnd) { - la_xvinsgr2vr_d(opnd, zero_ir2_opnd, 2); - la_xvinsgr2vr_d(opnd, zero_ir2_opnd, 3); + IR2_OPND zero = ra_alloc_ftemp(); + + la_vxor_v(zero, zero, zero); + clear_high128_xreg_with_zero(opnd, zero); + ra_free_temp(zero); } void materialize_deferred_ymmh_zero(IR2_OPND opnd) { int index; + uint16_t mask; if (!option_enable_lasx) { return; } index = mapped_ymm_index(opnd); - if (index >= 0 && - (lsenv->tr_data->ymmh_zero_pending & (UINT16_C(1) << index))) { + if (index < 0) { + return; + } + mask = UINT16_C(1) << index; + if (lsenv->tr_data->ymmh_zero_pending & mask) { clear_high128_xreg_now(opnd); + lsenv->tr_data->ymmh_zero_pending &= ~mask; + } +} + +void mark_high128_xreg_zeroed(IR2_OPND opnd) +{ + int index; + + if (!option_enable_lasx) { + return; + } + index = mapped_ymm_index(opnd); + if (index >= 0) { + lsenv->tr_data->ymmh_zero_pending &= + ~(UINT16_C(1) << index); } } @@ -1418,6 +1445,12 @@ static void emit_deferred_ymmh_zeros(void) } pending = lsenv->tr_data->ymmh_zero_pending; + if (!pending) { + return; + } + + IR2_OPND zero = ra_alloc_ftemp(); + la_vxor_v(zero, zero, zero); for (int index = 0; index < CPU_NB_REGS; ++index) { IR2_OPND opnd; @@ -1425,8 +1458,9 @@ static void emit_deferred_ymmh_zeros(void) continue; } ir2_opnd_build(&opnd, IR2_OPND_FPR, reg_xmm_map[index]); - clear_high128_xreg_now(opnd); + clear_high128_xreg_with_zero(opnd, zero); } + ra_free_temp(zero); } void materialize_deferred_ymmh_zeros_for_exit(void) diff --git a/target/i386/latx/translator/tr-pattern.c b/target/i386/latx/translator/tr-pattern.c index 2bddc00471..7dd29d5c04 100644 --- a/target/i386/latx/translator/tr-pattern.c +++ b/target/i386/latx/translator/tr-pattern.c @@ -432,26 +432,19 @@ static inline bool xcomisx_jcc(IR1_INST *ir1, bool is_double, bool qnan_exp) if (is_double) { la_fcmp = la_fcmp_cond_d; - if (qnan_exp) { - curr->info->id = WRAP(COMISD); - } else { - curr->info->id = WRAP(UCOMISD); - } } else { la_fcmp = la_fcmp_cond_s; - if (qnan_exp) { - curr->info->id = WRAP(COMISS); - } else { - curr->info->id = WRAP(UCOMISS); - } } IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(ir1, 0)); IR2_OPND src = load_freg128_from_ir1(ir1_get_opnd(ir1, 1)); IR2_OPND target_label_opnd = ra_alloc_label(); + IR2_OPND condition = ra_alloc_itemp(); #ifdef CONFIG_LATX_TU TranslationBlock *tb = lsenv->tr_data->curr_tb; #endif + /* Direct BCC linking would bypass the successor flag-recovery stub. */ + ((TranslationBlock *)lsenv->tr_data->curr_tb)->bool_flags &= ~OPT_BCC; switch (ir1_opcode(next)) { case WRAP(JA): @@ -493,6 +486,7 @@ static inline bool xcomisx_jcc(IR1_INST *ir1, bool is_double, bool qnan_exp) lsassert(0); break; } + la_movcf2gr(condition, fcc7_ir2_opnd); #ifdef CONFIG_LATX_TU if (judge_tu_eflag_gen(tb)) { @@ -507,7 +501,7 @@ static inline bool xcomisx_jcc(IR1_INST *ir1, bool is_double, bool qnan_exp) la_label(tu_reset_label_opnd); tb->tu_jmp[TU_TB_INDEX_TARGET] = tu_reset_label_opnd._label_id; if (ir1_opcode(next) != WRAP(JL)) { - la_bcnez(fcc7_ir2_opnd, target_label_opnd); + la_bne(condition, zero_ir2_opnd, target_label_opnd); tu_jcc_nop_gen(tb); } else { /* For unlink. */ @@ -535,7 +529,7 @@ static inline bool xcomisx_jcc(IR1_INST *ir1, bool is_double, bool qnan_exp) #endif if (ir1_opcode(next) != WRAP(JL)) - la_bcnez(fcc7_ir2_opnd, target_label_opnd); + la_bne(condition, zero_ir2_opnd, target_label_opnd); /* not taken */ tr_generate_exit_stub_tb(next, 0, trans, curr); @@ -544,6 +538,7 @@ static inline bool xcomisx_jcc(IR1_INST *ir1, bool is_double, bool qnan_exp) /* taken */ tr_generate_exit_stub_tb(next, 1, trans, curr); + ra_free_temp(condition); return true; } @@ -2211,27 +2206,27 @@ static inline bool xcomisx_xx_jcc(IR1_INST *pir1, bool is_jcc, bool is_double, b switch (ir1_opcode(next)) { case WRAP(JA): - la_fcmp(fcc0_ir2_opnd, src, dest, FCMP_COND_CLT + qnan_exp); + la_fcmp(fcc7_ir2_opnd, src, dest, FCMP_COND_CLT + qnan_exp); break; case WRAP(JAE): - la_fcmp(fcc0_ir2_opnd, src, dest, FCMP_COND_CLE + qnan_exp); + la_fcmp(fcc7_ir2_opnd, src, dest, FCMP_COND_CLE + qnan_exp); break; case WRAP(JB): - la_fcmp(fcc0_ir2_opnd, dest, src, FCMP_COND_CULT + qnan_exp); + la_fcmp(fcc7_ir2_opnd, dest, src, FCMP_COND_CULT + qnan_exp); /* below or NAN, x86 special define */ break; case WRAP(JBE): /* below or equal or NAN, x86 special define */ - la_fcmp(fcc0_ir2_opnd, dest, src, FCMP_COND_CULE + qnan_exp); + la_fcmp(fcc7_ir2_opnd, dest, src, FCMP_COND_CULE + qnan_exp); break; case WRAP(JE): case WRAP(JLE): - la_fcmp(fcc0_ir2_opnd, dest, src, FCMP_COND_CUEQ + qnan_exp); + la_fcmp(fcc7_ir2_opnd, dest, src, FCMP_COND_CUEQ + qnan_exp); /* equal or NAN, x86 special define */ break; case WRAP(JNE): case WRAP(JG): - la_fcmp(fcc0_ir2_opnd, dest, src, FCMP_COND_CNE + qnan_exp); + la_fcmp(fcc7_ir2_opnd, dest, src, FCMP_COND_CNE + qnan_exp); break; case WRAP(JL): case WRAP(JGE): @@ -2243,6 +2238,9 @@ static inline bool xcomisx_xx_jcc(IR1_INST *pir1, bool is_jcc, bool is_double, b } else { IR2_OPND target_label_opnd = ra_alloc_label(); + IR2_OPND condition = ra_alloc_itemp(); + + la_movcf2gr(condition, fcc7_ir2_opnd); #ifdef CONFIG_LATX_TU TranslationBlock *tb = lsenv->tr_data->curr_tb; @@ -2263,7 +2261,7 @@ static inline bool xcomisx_xx_jcc(IR1_INST *pir1, bool is_jcc, bool is_double, b /* Just for unlink. */ la_nop(); } else { - la_bcnez(fcc0_ir2_opnd, target_label_opnd); + la_bne(condition, zero_ir2_opnd, target_label_opnd); } tu_jcc_nop_gen(tb); @@ -2291,7 +2289,7 @@ static inline bool xcomisx_xx_jcc(IR1_INST *pir1, bool is_jcc, bool is_double, b la_b(target_label_opnd); } else if (ir1_opcode(curr) == WRAP(JL)) { } else { - la_bcnez(fcc0_ir2_opnd, target_label_opnd); + la_bne(condition, zero_ir2_opnd, target_label_opnd); } /* not taken */ @@ -2302,6 +2300,7 @@ static inline bool xcomisx_xx_jcc(IR1_INST *pir1, bool is_jcc, bool is_double, b /* taken */ tr_generate_exit_tb(curr, 1); + ra_free_temp(condition); } return true; } @@ -2384,6 +2383,201 @@ static bool translate_ucomiss_xx_jcc(IR1_INST *pir1) #endif +static void translate_avx_sum3_pair(IR2_OPND out, IR2_OPND first, + IR2_OPND second, IR2_OPND save_low, + bool save) +{ + IR2_OPND low = ra_alloc_ftemp(); + IR2_OPND high = ra_alloc_ftemp(); + + /* Keep the x86 order: (lane 0 + lane 1) + lane 2. */ + la_vilvl_w(low, second, first); + la_vshuf4i_w(high, low, 0x4e); + la_vfadd_s(low, low, high); + if (save) { + la_vori_b(save_low, low, 0); + la_vinsgr2vr_d(save_low, zero_ir2_opnd, 1); + } + la_vilvh_w(high, second, first); + la_vfadd_s(out, low, high); + la_vinsgr2vr_d(out, zero_ir2_opnd, 1); + ra_free_temp(high); + ra_free_temp(low); +} + +static bool translate_avx_sum3(IR1_INST *pir1) +{ + static const int product_offset[] = {-22, -18, -13, -9, -5, -1}; + IR2_OPND product[6]; + IR2_OPND zero_src = ra_alloc_xmm( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 2))); + IR2_OPND save_low = ra_alloc_xmm( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1 + 3, 0))); + IR2_OPND temp = ra_alloc_ftemp(); + + for (int i = 0; i < 6; ++i) { + product[i] = ra_alloc_xmm(ir1_opnd_base_reg_num( + ir1_get_opnd(pir1 + product_offset[i], 0))); + } + + translate_avx_sum3_pair(product[3], product[1], product[3], + save_low, true); + translate_avx_sum3_pair(product[4], product[0], product[4], + zero_ir2_opnd, false); + la_vfadd_s(product[3], product[3], product[4]); + translate_avx_sum3_pair(product[0], product[2], product[5], + zero_ir2_opnd, false); + la_vfsub_s(product[3], product[3], product[0]); + + /* VBLENDPS/VINSERTPS leave product[5] as [p0+p1, 2*p1, p2, z3]. */ + la_vshuf4i_w(temp, product[5], 0xe5); + la_vfadd_s(product[5], product[5], temp); + la_vextrins_w(temp, zero_src, VEXTRINS_IMM_4_0(3, 3)); + la_vextrins_d(product[5], temp, VEXTRINS_IMM_4_0(1, 1)); + ra_free_temp(temp); + + set_high128_xreg_to_zero(product[3]); + set_high128_xreg_to_zero(product[5]); + set_high128_xreg_to_zero(save_low); + return true; +} + +static bool translate_repeat_add(IR1_INST *pir1) +{ + IR2_OPND dest = load_ireg_from_ir1(ir1_get_opnd(pir1, 0), + UNKNOWN_EXTENSION, false); + IR2_OPND src = load_ireg_from_ir1(ir1_get_opnd(pir1, 1), + UNKNOWN_EXTENSION, false); + IR2_OPND count = ra_alloc_itemp(); + IR2_OPND product = ra_alloc_itemp(); + int repeats = pir1->instptn.next - pir1 + 1; + + lsassert(repeats >= 3); + li_d(count, repeats); + la_mul_d(product, src, count); + la_add_d(dest, dest, product); + ra_free_temp(product); + ra_free_temp(count); + return true; +} + +static bool translate_scalar_hdr(IR1_INST *pir1) +{ + IR1_OPND *input_mem = ir1_get_opnd(pir1, 1); + IR1_OPND *factor_mem = ir1_get_opnd(pir1 + 10, 1); + IR1_OPND *acc_mem = ir1_get_opnd(pir1 + 12, 2); + IR1_OPND *sub_mem = ir1_get_opnd(pir1 + 17, 2); + IR1_OPND *output_mem = ir1_get_opnd(pir1 + 18, 2); + IR2_OPND weighted0 = ra_alloc_gpr( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0))); + IR2_OPND weighted1 = ra_alloc_gpr( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1 + 2, 0))); + IR2_OPND index = ra_alloc_gpr( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1 + 6, 0))); + IR2_OPND acc_index = ra_alloc_gpr( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1 + 11, 0))); + IR2_OPND factor_xmm = ra_alloc_xmm( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1 + 10, 0))); + IR2_OPND value_xmm = ra_alloc_xmm( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1 + 12, 0))); + IR2_OPND multiplier = ra_alloc_itemp(); + IR2_OPND factor = ra_alloc_ftemp(); + IR2_OPND acc = ra_alloc_ftemp(); + IR2_OPND table = ra_alloc_ftemp(); + IR2_OPND sub = ra_alloc_ftemp(); + IR2_OPND old = ra_alloc_ftemp(); + IR2_OPND result = ra_alloc_ftemp(); + int input_disp; + int factor_disp; + int acc_disp; + int sub_disp; + int output_disp; + IR2_OPND input_addr = convert_mem(input_mem, &input_disp); + IR2_OPND factor_addr; + IR2_OPND acc_addr; + IR2_OPND sub_addr; + IR2_OPND output_addr; + + gen_test_page_flag(input_addr, input_disp, PAGE_READ); + la_ld_bu(weighted0, input_addr, input_disp); + li_d(multiplier, 0x36); + la_mul_w(weighted0, weighted0, multiplier); + + gen_test_page_flag(input_addr, input_disp + 1, PAGE_READ); + la_ld_bu(weighted1, input_addr, input_disp + 1); + li_d(multiplier, 0xb7); + la_mul_w(weighted1, weighted1, multiplier); + la_add_d(weighted1, weighted1, weighted0); + + gen_test_page_flag(input_addr, input_disp + 2, PAGE_READ); + la_ld_bu(weighted0, input_addr, input_disp + 2); + /* ALSL shifts by sa + 1: 8*x + x, then 2*(9*x) + x. */ + la_alsl_d(index, weighted0, weighted0, 2); + la_alsl_d(index, index, weighted0, 0); + la_add_d(index, index, weighted1); + la_srli_w(index, index, 8); + + factor_addr = convert_mem(factor_mem, &factor_disp); + gen_test_page_flag(factor_addr, factor_disp, PAGE_READ); + la_fld_s(factor, factor_addr, factor_disp); + la_xvpickve_w(factor_xmm, factor, 0); + mark_high128_xreg_zeroed(factor_xmm); + + la_slli_w(acc_index, acc_index, 0); + acc_addr = convert_mem(acc_mem, &acc_disp); + gen_test_page_flag(acc_addr, acc_disp, PAGE_READ); + la_fld_s(acc, acc_addr, acc_disp); + la_fadd_s(acc, factor, acc); + la_xvpickve_w(value_xmm, acc, 0); + mark_high128_xreg_zeroed(value_xmm); + gen_test_page_flag(acc_addr, acc_disp, PAGE_WRITE | PAGE_WRITE_ORG); + la_fst_s(acc, acc_addr, acc_disp); + + sub_addr = convert_mem(sub_mem, &sub_disp); + output_addr = convert_mem(output_mem, &output_disp); + for (int channel = 0; channel < 3; ++channel) { + IR1_OPND *table_mem = ir1_get_opnd(pir1 + 16 + channel * 6, 1); + IR2_OPND table_addr; + int table_disp; + + gen_test_page_flag(input_addr, input_disp + channel, PAGE_READ); + la_ld_bu(index, input_addr, input_disp + channel); + /* 2*x + x = 3*x. */ + la_alsl_d(index, index, index, 0); + + table_addr = convert_mem(table_mem, &table_disp); + gen_test_page_flag(table_addr, table_disp, PAGE_READ); + la_fld_s(table, table_addr, table_disp); + gen_test_page_flag(sub_addr, sub_disp, PAGE_READ); + la_fld_s(sub, sub_addr, sub_disp); + la_fsub_s(result, table, sub); + gen_test_page_flag(output_addr, output_disp + channel * 4, + PAGE_READ); + la_fld_s(old, output_addr, output_disp + channel * 4); + la_fmadd_s(result, factor, result, old); + la_xvpickve_w(value_xmm, result, 0); + mark_high128_xreg_zeroed(value_xmm); + gen_test_page_flag(output_addr, output_disp + channel * 4, + PAGE_WRITE | PAGE_WRITE_ORG); + la_fst_s(result, output_addr, output_disp + channel * 4); + ra_free_temp_auto(table_addr); + } + + ra_free_temp_auto(output_addr); + ra_free_temp_auto(sub_addr); + ra_free_temp_auto(acc_addr); + ra_free_temp_auto(factor_addr); + ra_free_temp_auto(input_addr); + ra_free_temp(result); + ra_free_temp(old); + ra_free_temp(sub); + ra_free_temp(table); + ra_free_temp(acc); + ra_free_temp(factor); + ra_free_temp(multiplier); + return true; +} + bool try_translate_instptn(IR1_INST *pir1) { instptn_check_false(); @@ -2464,6 +2658,12 @@ bool try_translate_instptn(IR1_INST *pir1) return translate_shr_jcc(pir1); case INSTPTN_OPC_AND_JCC: return translate_and_jcc(pir1); + case INSTPTN_OPC_REPEAT_ADD: + return translate_repeat_add(pir1); + case INSTPTN_OPC_AVX_SUM3: + return translate_avx_sum3(pir1); + case INSTPTN_OPC_SCALAR_HDR: + return translate_scalar_hdr(pir1); default: lsassert(0); break; diff --git a/tests/integration/avx-sum3-pattern.S b/tests/integration/avx-sum3-pattern.S new file mode 100644 index 0000000000..17e72df299 --- /dev/null +++ b/tests/integration/avx-sum3-pattern.S @@ -0,0 +1,105 @@ +.intel_syntax noprefix +.section .text +.global _start + +_start: + vxorps xmm1, xmm1, xmm1 + vmovaps xmm2, xmmword ptr [rip + .Lp1] + vmovaps xmm3, xmmword ptr [rip + .Lp4] + vmovaps xmm7, xmmword ptr [rip + .Lones] + vmovaps xmm8, xmmword ptr [rip + .Lones] + + vmulps xmm9, xmm2, xmmword ptr [rip + .Lones] + vblendps xmm10, xmm9, xmm1, 8 + vinsertps xmm9, xmm9, xmm9, 0x4c + vaddps xmm9, xmm10, xmm9 + + vmulps xmm10, xmm8, xmmword ptr [rip + .Lp2] + vblendps xmm11, xmm10, xmm1, 8 + vinsertps xmm10, xmm10, xmm10, 0x4c + vaddps xmm10, xmm11, xmm10 + + vmovaps xmm11, xmmword ptr [rip + .Lones] + vmulps xmm12, xmm11, xmmword ptr [rip + .Lp3] + vblendps xmm13, xmm12, xmm1, 8 + vinsertps xmm12, xmm12, xmm12, 0x4c + vaddps xmm12, xmm13, xmm12 + + vmulps xmm7, xmm3, xmm7 + vblendps xmm13, xmm7, xmm1, 8 + vinsertps xmm7, xmm7, xmm7, 0x4c + vaddps xmm7, xmm13, xmm7 + + vmulps xmm8, xmm8, xmmword ptr [rip + .Lp5] + vblendps xmm13, xmm8, xmm1, 8 + vinsertps xmm8, xmm8, xmm8, 0x4c + vaddps xmm8, xmm13, xmm8 + + vmulps xmm11, xmm11, xmmword ptr [rip + .Lp6] + vblendps xmm13, xmm11, xmm1, 8 + vinsertps xmm11, xmm11, xmm11, 0x4c + vaddps xmm11, xmm13, xmm11 + + vinsertps xmm13, xmm10, xmm7, 0x1c + vshufpd xmm10, xmm10, xmm10, 1 + vinsertps xmm7, xmm10, xmm7, 0x9c + vaddps xmm7, xmm13, xmm7 + vinsertps xmm10, xmm9, xmm8, 0x1c + vshufpd xmm9, xmm9, xmm9, 1 + vinsertps xmm8, xmm9, xmm8, 0x9c + vaddps xmm8, xmm10, xmm8 + vaddps xmm7, xmm8, xmm7 + vinsertps xmm8, xmm12, xmm11, 0x1c + vshufpd xmm9, xmm12, xmm12, 1 + vinsertps xmm9, xmm9, xmm11, 0x9c + vaddps xmm8, xmm8, xmm9 + vsubps xmm7, xmm7, xmm8 + + /* These writes prove that the registers replaced by the pattern are dead. */ + vmovsd xmm12, qword ptr [rip + .Lzero] + vmovsd xmm10, qword ptr [rip + .Lzero] + vinsertps xmm8, xmm10, xmm12, 0x4c + vsqrtss xmm9, xmm1, xmm1 + + vmovups xmmword ptr [rip + .Lout], xmm7 + vmovups xmmword ptr [rip + .Lout + 16], xmm11 + vmovups xmmword ptr [rip + .Lout + 32], xmm13 + + lea rsi, [rip + .Lout] + lea rdi, [rip + .Lexpected] + mov ecx, 12 +.Lcheck: + mov eax, dword ptr [rsi] + cmp eax, dword ptr [rdi] + jne .Lfail + add rsi, 4 + add rdi, 4 + dec ecx + jne .Lcheck + + xor edi, edi + jmp .Lexit +.Lfail: + mov edi, 1 +.Lexit: + mov eax, 60 + syscall + +.section .rodata +.p2align 4 +.Lones: .float 1.0, 1.0, 1.0, 1.0 +.Lp1: .float 1.0, 2.0, 3.0, 4.0 +.Lp2: .float 5.0, 6.0, 7.0, 8.0 +.Lp3: .float 9.0, 10.0, 11.0, 12.0 +.Lp4: .float 13.0, 14.0, 15.0, 16.0 +.Lp5: .float 17.0, 18.0, 19.0, 20.0 +.Lp6: .float 21.0, 22.0, 23.0, 24.0 +.Lzero: .quad 0 +.Lexpected: + .float -6.0, 30.0, 0.0, 0.0 + .float 43.0, 44.0, 23.0, 0.0 + .float 11.0, 27.0, 0.0, 0.0 + +.section .bss +.p2align 4 +.Lout: .skip 48 diff --git a/tests/integration/avx128-shuffle.S b/tests/integration/avx128-shuffle.S new file mode 100644 index 0000000000..f04d069042 --- /dev/null +++ b/tests/integration/avx128-shuffle.S @@ -0,0 +1,136 @@ +.intel_syntax noprefix +.global _start + +.section .rodata +.p2align 5 +.Lsrc1: + .long 1, 2, 3, 4 + .long 9, 10, 11, 12 +.Lsrc2: + .long 5, 6, 7, 8 +.Lqwords: + .quad 0x1111222233334444, 0x5555666677778888 + +.section .bss +.p2align 5 +.Lout: + .zero 32 + +.text +_start: + mov edi, 1 + vmovaps ymm3, ymmword ptr [rip + .Lsrc1] + vmovaps xmm1, xmmword ptr [rip + .Lsrc1] + vmovaps xmm2, xmmword ptr [rip + .Lsrc2] + vinsertps xmm3, xmm1, xmm2, 0x1c + vmovups ymmword ptr [rip + .Lout], ymm3 + cmp dword ptr [rip + .Lout], 1 + jne .Lfail + cmp dword ptr [rip + .Lout + 4], 5 + jne .Lfail + cmp qword ptr [rip + .Lout + 8], 0 + jne .Lfail + cmp qword ptr [rip + .Lout + 16], 0 + jne .Lfail + cmp qword ptr [rip + .Lout + 24], 0 + jne .Lfail + + mov edi, 2 + vmovaps ymm3, ymmword ptr [rip + .Lsrc1] + vinsertps xmm3, xmm1, xmm1, 0x4c + vmovups ymmword ptr [rip + .Lout], ymm3 + cmp dword ptr [rip + .Lout], 2 + jne .Lfail + cmp dword ptr [rip + .Lout + 4], 2 + jne .Lfail + cmp qword ptr [rip + .Lout + 8], 0 + jne .Lfail + cmp qword ptr [rip + .Lout + 16], 0 + jne .Lfail + cmp qword ptr [rip + .Lout + 24], 0 + jne .Lfail + + mov edi, 3 + vmovaps ymm3, ymmword ptr [rip + .Lsrc1] + movaps xmm3, xmmword ptr [rip + .Lsrc2] + vinsertps xmm3, xmm1, xmm3, 0x9c + vmovups ymmword ptr [rip + .Lout], ymm3 + cmp dword ptr [rip + .Lout], 1 + jne .Lfail + cmp dword ptr [rip + .Lout + 4], 7 + jne .Lfail + cmp qword ptr [rip + .Lout + 8], 0 + jne .Lfail + cmp qword ptr [rip + .Lout + 16], 0 + jne .Lfail + cmp qword ptr [rip + .Lout + 24], 0 + jne .Lfail + + mov edi, 4 + vmovaps ymm3, ymmword ptr [rip + .Lsrc1] + vmovss xmm3, dword ptr [rip + .Lsrc2] + vmovups ymmword ptr [rip + .Lout], ymm3 + cmp dword ptr [rip + .Lout], 5 + jne .Lfail + cmp qword ptr [rip + .Lout + 4], 0 + jne .Lfail + cmp dword ptr [rip + .Lout + 12], 0 + jne .Lfail + cmp qword ptr [rip + .Lout + 16], 0 + jne .Lfail + cmp qword ptr [rip + .Lout + 24], 0 + jne .Lfail + + mov edi, 5 + vmovaps ymm3, ymmword ptr [rip + .Lsrc1] + vmovsd xmm3, qword ptr [rip + .Lqwords] + vmovups ymmword ptr [rip + .Lout], ymm3 + mov rax, qword ptr [rip + .Lqwords] + cmp qword ptr [rip + .Lout], rax + jne .Lfail + cmp qword ptr [rip + .Lout + 8], 0 + jne .Lfail + cmp qword ptr [rip + .Lout + 16], 0 + jne .Lfail + cmp qword ptr [rip + .Lout + 24], 0 + jne .Lfail + + mov edi, 6 + vmovaps ymm3, ymmword ptr [rip + .Lsrc1] + vmovapd xmm1, xmmword ptr [rip + .Lqwords] + vshufpd xmm3, xmm1, xmm1, 1 + vmovups ymmword ptr [rip + .Lout], ymm3 + mov rax, qword ptr [rip + .Lqwords + 8] + cmp qword ptr [rip + .Lout], rax + jne .Lfail + mov rax, qword ptr [rip + .Lqwords] + cmp qword ptr [rip + .Lout + 8], rax + jne .Lfail + cmp qword ptr [rip + .Lout + 16], 0 + jne .Lfail + cmp qword ptr [rip + .Lout + 24], 0 + jne .Lfail + + mov edi, 7 + vmovaps ymm4, ymmword ptr [rip + .Lsrc1] + vinsertps xmm4, xmm4, xmm4, 0x9c + vmovups ymmword ptr [rip + .Lout], ymm4 + cmp dword ptr [rip + .Lout], 1 + jne .Lfail + cmp dword ptr [rip + .Lout + 4], 3 + jne .Lfail + cmp qword ptr [rip + .Lout + 8], 0 + jne .Lfail + cmp qword ptr [rip + .Lout + 16], 0 + jne .Lfail + cmp qword ptr [rip + .Lout + 24], 0 + jne .Lfail + + mov eax, 60 + xor edi, edi + syscall +.Lfail: + mov eax, 60 + syscall + +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/registrations/simd/meson.build b/tests/integration/registrations/simd/meson.build index 82d17828b0..a9bfeff955 100644 --- a/tests/integration/registrations/simd/meson.build +++ b/tests/integration/registrations/simd/meson.build @@ -17,4 +17,36 @@ if 'x86_64-linux-user' in target_dirs ], 'timeout': 120, }] + latx_integration_tests += [{ + 'name': 'test-avx-patterns', + 'runner': find_program('../../test-avx-patterns.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../avx128-shuffle.S'), + files('../../avx-sum3-pattern.S'), + files('../../repeat-add.S'), + files('../../vcomis-jcc.S'), + ], + 'timeout': 120, + }] + latx_integration_tests += [{ + 'name': 'test-scalar-fma-forward', + 'runner': find_program('../../test-avx-patterns.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../scalar-fma-forward.S'), + files('../../vpshufd-distinct-regs.S'), + ], + 'timeout': 120, + }] + latx_integration_tests += [{ + 'name': 'test-scalar-pixel-update', + 'runner': find_program('../../test-avx-patterns.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../scalar-hdr-pattern.S'), + files('../../vroundps-imm0b.S'), + ], + 'timeout': 120, + }] endif diff --git a/tests/integration/repeat-add.S b/tests/integration/repeat-add.S new file mode 100644 index 0000000000..541e7ab6a7 --- /dev/null +++ b/tests/integration/repeat-add.S @@ -0,0 +1,23 @@ +.intel_syntax noprefix +.global _start + +.text +_start: + mov rax, 5 + mov rdx, 7 + .rept 100 + add rax, rdx + .endr + cmp rax, 705 + jne .Lfail + + mov eax, 60 + xor edi, edi + syscall + +.Lfail: + mov eax, 60 + mov edi, 1 + syscall + +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/scalar-fma-forward.S b/tests/integration/scalar-fma-forward.S new file mode 100644 index 0000000000..39d8d57d3e --- /dev/null +++ b/tests/integration/scalar-fma-forward.S @@ -0,0 +1,64 @@ +.intel_syntax noprefix +.section .text +.global _start + +_start: + vmovaps xmm1, xmmword ptr [rip + .Lmul_lhs] + vmovaps xmm2, xmmword ptr [rip + .Lthree] + vmovaps xmm3, xmmword ptr [rip + .Lfour] + vmulss xmm4, xmm1, xmm2 + vfmadd231ss xmm4, xmm2, xmm3 + vmovaps xmmword ptr [rip + .Lout], xmm4 + + vmovaps xmm1, xmmword ptr [rip + .Lsub_lhs] + vmovaps xmm2, xmmword ptr [rip + .Lthree] + vmovaps xmm3, xmmword ptr [rip + .Ltwo] + vmovaps xmm4, xmmword ptr [rip + .Lfive] + vsubss xmm5, xmm1, xmm2 + vfmsub231ss xmm5, xmm3, xmm4 + vmovaps xmmword ptr [rip + .Lout + 16], xmm5 + + vmovaps xmm6, xmmword ptr [rip + .Lnan] + vmovaps xmm2, xmmword ptr [rip + .Lone] + vmulss xmm6, xmm6, xmm2 + vfmadd231ss xmm6, xmm2, xmm2 + vmovaps xmmword ptr [rip + .Lout + 32], xmm6 + + lea rsi, [rip + .Lout] + lea rdi, [rip + .Lexpected] + mov ecx, 12 +.Lcheck: + mov eax, dword ptr [rsi] + cmp eax, dword ptr [rdi] + jne .Lfail + add rsi, 4 + add rdi, 4 + dec ecx + jne .Lcheck + + xor edi, edi + jmp .Lexit +.Lfail: + mov edi, 1 +.Lexit: + mov eax, 60 + syscall + +.section .rodata +.p2align 4 +.Lmul_lhs: .long 0x40000000, 0x11223344, 0x55667788, 0x99aabbcc +.Lsub_lhs: .long 0x41200000, 0xa1b2c3d4, 0xe5f60718, 0x293a4b5c +.Lnan: .long 0x7fc12345, 0x76543210, 0xfedcba98, 0x89abcdef +.Lone: .float 1.0, 0.0, 0.0, 0.0 +.Ltwo: .float 2.0, 0.0, 0.0, 0.0 +.Lthree: .float 3.0, 0.0, 0.0, 0.0 +.Lfour: .float 4.0, 0.0, 0.0, 0.0 +.Lfive: .float 5.0, 0.0, 0.0, 0.0 +.Lexpected: + .long 0x41900000, 0x11223344, 0x55667788, 0x99aabbcc + .long 0x40400000, 0xa1b2c3d4, 0xe5f60718, 0x293a4b5c + .long 0x7fc12345, 0x76543210, 0xfedcba98, 0x89abcdef + +.section .bss +.p2align 4 +.Lout: .skip 48 diff --git a/tests/integration/scalar-hdr-pattern.S b/tests/integration/scalar-hdr-pattern.S new file mode 100644 index 0000000000..f064b9d1a2 --- /dev/null +++ b/tests/integration/scalar-hdr-pattern.S @@ -0,0 +1,146 @@ +.intel_syntax noprefix +.section .text +.global _start + +_start: + lea rsi, [rip + .Linput] + lea rdi, [rip + .Lfactor] + lea rbx, [rip + .Lacc] + lea r11, [rip + .Ltable] + lea r12, [rip + .Lsub] + lea r13, [rip + .Loutput] + xor eax, eax + xor r14d, r14d + xor r15d, r15d + mov ecx, 1 + vmovups xmm1, xmmword ptr [rip + .Lgarbage] + vmovups xmm2, xmmword ptr [rip + .Lgarbage] + +.Lloop: + movzx r8d, byte ptr [rsi] + imul r8d, r8d, 0x36 + movzx r9d, byte ptr [rsi + 1] + imul r9d, r9d, 0xb7 + add r9d, r8d + movzx r8d, byte ptr [rsi + 2] + lea r10d, [r8 + r8 * 8] + lea r10d, [r8 + r10 * 2] + add r10d, r9d + shr r10d, 8 + vmovss xmm1, dword ptr [rdi + r10 * 4] + movsxd rax, eax + vaddss xmm2, xmm1, dword ptr [rbx + rax * 4] + vmovss dword ptr [rbx + rax * 4], xmm2 + + movzx r10d, byte ptr [rsi] + lea r10, [r10 + r10 * 2] + vmovss xmm2, dword ptr [r11 + r10 * 4] + vsubss xmm2, xmm2, dword ptr [r12] + vfmadd213ss xmm2, xmm1, dword ptr [r13] + vmovss dword ptr [r13], xmm2 + + movzx r10d, byte ptr [rsi + 1] + lea r10, [r10 + r10 * 2] + vmovss xmm2, dword ptr [r11 + r10 * 4 + 4] + vsubss xmm2, xmm2, dword ptr [r12] + vfmadd213ss xmm2, xmm1, dword ptr [r13 + 4] + vmovss dword ptr [r13 + 4], xmm2 + + movzx r10d, byte ptr [rsi + 2] + lea r10, [r10 + r10 * 2] + vmovss xmm2, dword ptr [r11 + r10 * 4 + 8] + vsubss xmm2, xmm2, dword ptr [r12] + vfmadd213ss xmm2, xmm1, dword ptr [r13 + 8] + vmovss dword ptr [r13 + 8], xmm2 + + inc r14d + add r15d, 2 + dec ecx + jne .Lloop + + cmp r8d, 3 + jne .Lfail + cmp r9d, 420 + jne .Lfail + cmp r10d, 9 + jne .Lfail + cmp eax, 0 + jne .Lfail + cmp r14d, 1 + jne .Lfail + cmp r15d, 2 + jne .Lfail + + lea rsi, [rip + .Lacc] + lea rdi, [rip + .Lexpected_acc] + mov ecx, 1 + call .Lcheck_words + test eax, eax + jne .Lfail + lea rsi, [rip + .Loutput] + lea rdi, [rip + .Lexpected_output] + mov ecx, 3 + call .Lcheck_words + test eax, eax + jne .Lfail + + vmovdqu xmmword ptr [rip + .Lxmm_out], xmm1 + vmovdqu xmmword ptr [rip + .Lxmm_out + 16], xmm2 + lea rsi, [rip + .Lxmm_out] + lea rdi, [rip + .Lexpected_xmm] + mov ecx, 8 + call .Lcheck_words + test eax, eax + jne .Lfail + + xor edi, edi + jmp .Lexit + +.Lcheck_words: + xor eax, eax +.Lcheck_next: + mov edx, dword ptr [rsi] + cmp edx, dword ptr [rdi] + jne .Lcheck_bad + add rsi, 4 + add rdi, 4 + dec ecx + jne .Lcheck_next + ret +.Lcheck_bad: + mov eax, 1 + ret + +.Lfail: + mov edi, 1 +.Lexit: + mov eax, 60 + syscall + +.section .rodata +.p2align 5 +.Linput: .byte 1, 2, 3 +.p2align 4 +.Lfactor: .float 0.0, 2.0 +.Ltable: + .float 0.0, 0.0, 0.0, 10.0 + .float 0.0, 0.0, 0.0, 20.0 + .float 0.0, 0.0, 0.0, 30.0 +.Lsub: .float 1.0 +.Lgarbage: .long 0x3f000000, 0x11223344, 0x55667788, 0x99aabbcc +.Lexpected_acc: .float 6.0 +.Lexpected_output: .float 23.0, 44.0, 65.0 +.Lexpected_xmm: + .long 0x40000000, 0, 0, 0 + .long 0x42820000, 0, 0, 0 + +.section .data +.p2align 4 +.Lacc: .float 4.0 +.Loutput: .float 5.0, 6.0, 7.0 + +.section .bss +.p2align 4 +.Lxmm_out: .skip 32 + +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/test-avx-patterns.sh b/tests/integration/test-avx-patterns.sh new file mode 100755 index 0000000000..73dd8510db --- /dev/null +++ b/tests/integration/test-avx-patterns.sh @@ -0,0 +1,46 @@ +#!/bin/sh +set -eu + +emulator=$1 +shift +workdir=$(mktemp -d) +trap 'rm -rf "$workdir"' EXIT HUP INT TERM + +if command -v clang-19 >/dev/null 2>&1; then + clang=clang-19 +elif command -v clang >/dev/null 2>&1; then + clang=clang +else + echo "SKIP: clang is required to build the x86_64 guests" + exit 77 +fi + +for source_file in "$@"; do + name=$(basename "$source_file" .S) + "$clang" --target=x86_64-linux-gnu -fuse-ld=lld -nostdlib -static \ + -Wl,--build-id=none "$source_file" -o "$workdir/$name" + + LATX_AOT=0 LATX_TU=0 "$emulator" "$workdir/$name" + + aot_home="$workdir/aot-$name" + mkdir -p "$aot_home" + HOME="$aot_home" LATX_AOT=1 LATX_TU=1 \ + "$emulator" "$workdir/$name" + + aot_file= + for _ in $(seq 1 100); do + aot_file=$(find "$aot_home/.cache/latx" -type f -name '*.aot2' \ + -size +0c -print -quit 2>/dev/null || true) + [ -n "$aot_file" ] && break + sleep 0.1 + done + if [ -z "$aot_file" ]; then + echo "FAIL: no non-empty AOT file generated for $name" >&2 + exit 1 + fi + + HOME="$aot_home" LATX_AOT=1 LATX_TU=1 \ + "$emulator" "$workdir/$name" +done + +echo "PASS: LATX AVX integration JIT/cold-AOT/hot-AOT" diff --git a/tests/integration/vcomis-jcc.S b/tests/integration/vcomis-jcc.S new file mode 100644 index 0000000000..bccf888fa6 --- /dev/null +++ b/tests/integration/vcomis-jcc.S @@ -0,0 +1,140 @@ +.intel_syntax noprefix +.global _start + +.section .rodata +.p2align 4 +.Ltwo_s: .long 0x40000000 +.Lone_s: .long 0x3f800000 +.Lnan_s: .long 0x7fc00000 +.p2align 3 +.Ltwo_d: .quad 0x4000000000000000 +.Lone_d: .quad 0x3ff0000000000000 +.Lnan_d: .quad 0x7ff8000000000000 + +.text +.macro check_s inst, cc, lhs, rhs, expected, id + vmovss xmm0, dword ptr [rip + \lhs] + vmovss xmm1, dword ptr [rip + \rhs] + \inst xmm0, xmm1 + \cc .Ltaken\@ + mov eax, 0 + jmp .Ldone\@ +.Ltaken\@: + mov eax, 1 +.Ldone\@: + cmp eax, \expected + je .Lok\@ + mov edi, \id + jmp .Lfail +.Lok\@: +.endm + +.macro check_d inst, cc, lhs, rhs, expected, id + vmovsd xmm0, qword ptr [rip + \lhs] + vmovsd xmm1, qword ptr [rip + \rhs] + \inst xmm0, xmm1 + \cc .Ltaken\@ + mov eax, 0 + jmp .Ldone\@ +.Ltaken\@: + mov eax, 1 +.Ldone\@: + cmp eax, \expected + je .Lok\@ + mov edi, \id + jmp .Lfail +.Lok\@: +.endm + +.macro check_all_s inst, lhs, rhs, ja, jae, jb, jbe, je, jne, jle, jg, base + check_s \inst, ja, \lhs, \rhs, \ja, \base + check_s \inst, jae, \lhs, \rhs, \jae, \base + 1 + check_s \inst, jb, \lhs, \rhs, \jb, \base + 2 + check_s \inst, jbe, \lhs, \rhs, \jbe, \base + 3 + check_s \inst, je, \lhs, \rhs, \je, \base + 4 + check_s \inst, jne, \lhs, \rhs, \jne, \base + 5 + check_s \inst, jl, \lhs, \rhs, 0, \base + 6 + check_s \inst, jge, \lhs, \rhs, 1, \base + 7 + check_s \inst, jle, \lhs, \rhs, \jle, \base + 8 + check_s \inst, jg, \lhs, \rhs, \jg, \base + 9 +.endm + +.macro check_all_d inst, lhs, rhs, ja, jae, jb, jbe, je, jne, jle, jg, base + check_d \inst, ja, \lhs, \rhs, \ja, \base + check_d \inst, jae, \lhs, \rhs, \jae, \base + 1 + check_d \inst, jb, \lhs, \rhs, \jb, \base + 2 + check_d \inst, jbe, \lhs, \rhs, \jbe, \base + 3 + check_d \inst, je, \lhs, \rhs, \je, \base + 4 + check_d \inst, jne, \lhs, \rhs, \jne, \base + 5 + check_d \inst, jl, \lhs, \rhs, 0, \base + 6 + check_d \inst, jge, \lhs, \rhs, 1, \base + 7 + check_d \inst, jle, \lhs, \rhs, \jle, \base + 8 + check_d \inst, jg, \lhs, \rhs, \jg, \base + 9 +.endm + +_start: + /* greater: JA/JAE/JNE/JGE/JG */ + check_all_s vcomiss, .Ltwo_s, .Lone_s, 1, 1, 0, 0, 0, 1, 0, 1, 1 + check_all_s vucomiss, .Ltwo_s, .Lone_s, 1, 1, 0, 0, 0, 1, 0, 1, 11 + check_all_d vcomisd, .Ltwo_d, .Lone_d, 1, 1, 0, 0, 0, 1, 0, 1, 21 + check_all_d vucomisd, .Ltwo_d, .Lone_d, 1, 1, 0, 0, 0, 1, 0, 1, 31 + + /* less: JB/JBE/JNE/JGE/JG */ + check_all_s vcomiss, .Lone_s, .Ltwo_s, 0, 0, 1, 1, 0, 1, 0, 1, 41 + check_all_s vucomiss, .Lone_s, .Ltwo_s, 0, 0, 1, 1, 0, 1, 0, 1, 51 + check_all_d vcomisd, .Lone_d, .Ltwo_d, 0, 0, 1, 1, 0, 1, 0, 1, 61 + check_all_d vucomisd, .Lone_d, .Ltwo_d, 0, 0, 1, 1, 0, 1, 0, 1, 71 + + /* equal: JAE/JBE/JE/JGE/JLE */ + check_all_s vcomiss, .Lone_s, .Lone_s, 0, 1, 0, 1, 1, 0, 1, 0, 81 + check_all_s vucomiss, .Lone_s, .Lone_s, 0, 1, 0, 1, 1, 0, 1, 0, 91 + check_all_d vcomisd, .Lone_d, .Lone_d, 0, 1, 0, 1, 1, 0, 1, 0, 101 + check_all_d vucomisd, .Lone_d, .Lone_d, 0, 1, 0, 1, 1, 0, 1, 0, 111 + + /* unordered: JB/JBE/JE/JGE/JLE */ + check_all_s vcomiss, .Lnan_s, .Lone_s, 0, 0, 1, 1, 1, 0, 1, 0, 121 + check_all_s vucomiss, .Lnan_s, .Lone_s, 0, 0, 1, 1, 1, 0, 1, 0, 131 + check_all_d vcomisd, .Lnan_d, .Lone_d, 0, 0, 1, 1, 1, 0, 1, 0, 141 + check_all_d vucomisd, .Lnan_d, .Lone_d, 0, 0, 1, 1, 1, 0, 1, 0, 151 + + /* A second branch in the successor TB must see the compare flags. */ + mov ecx, 1000 +.Lfallthrough_flags: + vmovss xmm0, dword ptr [rip + .Ltwo_s] + vmovss xmm1, dword ptr [rip + .Lone_s] + vucomiss xmm0, xmm1 + jb .Lfail_201 + jne .Lfallthrough_ok + jmp .Lfail_201 +.Lfallthrough_ok: + dec ecx + jnz .Lfallthrough_flags + + mov ecx, 1000 +.Ltaken_flags: + vmovss xmm0, dword ptr [rip + .Lnan_s] + vmovss xmm1, dword ptr [rip + .Lone_s] + vucomiss xmm0, xmm1 + jb .Ltaken_path + jmp .Lfail_202 +.Ltaken_path: + jp .Ltaken_ok + jmp .Lfail_202 +.Ltaken_ok: + dec ecx + jnz .Ltaken_flags + + mov eax, 60 + xor edi, edi + syscall +.Lfail_201: + mov edi, 201 + jmp .Lfail +.Lfail_202: + mov edi, 202 + jmp .Lfail +.Lfail: + mov eax, 60 + syscall + +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/vpshufd-distinct-regs.S b/tests/integration/vpshufd-distinct-regs.S new file mode 100644 index 0000000000..0738798837 --- /dev/null +++ b/tests/integration/vpshufd-distinct-regs.S @@ -0,0 +1,59 @@ +.intel_syntax noprefix +.section .text +.global _start + +_start: + vmovdqu ymm0, ymmword ptr [rip + .Ljunk] + vmovdqu xmm1, xmmword ptr [rip + .Lsrc] + vpshufd xmm0, xmm1, 0x1b + vmovdqu ymmword ptr [rip + .Lout], ymm0 + + mov edi, 1 + lea rsi, [rip + .Lout] + lea rdx, [rip + .Lexpected_xmm] + mov ecx, 8 + call .Lcheck + + vmovdqu ymm0, ymmword ptr [rip + .Ljunk] + vmovdqu ymm2, ymmword ptr [rip + .Lsrc] + vpshufd ymm0, ymm2, 0x1b + vmovdqu ymmword ptr [rip + .Lout], ymm0 + + mov edi, 2 + lea rsi, [rip + .Lout] + lea rdx, [rip + .Lexpected_ymm] + mov ecx, 8 + call .Lcheck + + xor edi, edi +.Lexit: + mov eax, 60 + syscall + +.Lcheck: + mov eax, dword ptr [rsi] + cmp eax, dword ptr [rdx] + jne .Lexit + add rsi, 4 + add rdx, 4 + dec ecx + jne .Lcheck + ret + +.section .rodata +.p2align 5 +.Lsrc: + .long 1, 2, 3, 4, 5, 6, 7, 8 +.Ljunk: + .long 101, 102, 103, 104, 105, 106, 107, 108 +.Lexpected_xmm: + .long 4, 3, 2, 1, 0, 0, 0, 0 +.Lexpected_ymm: + .long 4, 3, 2, 1, 8, 7, 6, 5 + +.section .bss +.p2align 5 +.Lout: + .skip 32 + +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/vroundps-imm0b.S b/tests/integration/vroundps-imm0b.S new file mode 100644 index 0000000000..f00ac9a074 --- /dev/null +++ b/tests/integration/vroundps-imm0b.S @@ -0,0 +1,76 @@ +.intel_syntax noprefix +.section .text +.global _start + +_start: + lea rsi, [rip + .Linput] + lea rdi, [rip + .Lexpected] + mov edx, 3 +.Lcase: + vmovdqu ymm0, ymmword ptr [rsi] + vroundps ymm1, ymm0, 0xb + vmovdqu ymmword ptr [rip + .Lout], ymm1 + lea r8, [rip + .Lout] + mov ecx, 8 +.Lcheck: + mov eax, dword ptr [r8] + cmp eax, dword ptr [rdi] + jne .Lfail + add r8, 4 + add rdi, 4 + dec ecx + jne .Lcheck + add rsi, 32 + dec edx + jne .Lcase + + vmovdqu ymm1, ymmword ptr [rip + .Lall_ones] + vmovdqu xmm0, xmmword ptr [rip + .Linput + 32] + vroundps xmm1, xmm0, 0xb + vmovdqu ymmword ptr [rip + .Lout], ymm1 + lea r8, [rip + .Lout] + lea rdi, [rip + .Lxmm_expected] + mov ecx, 8 +.Lxmm_check: + mov eax, dword ptr [r8] + cmp eax, dword ptr [rdi] + jne .Lfail + add r8, 4 + add rdi, 4 + dec ecx + jne .Lxmm_check + + xor edi, edi + jmp .Lexit +.Lfail: + mov edi, 1 +.Lexit: + mov eax, 60 + syscall + +.section .rodata +.p2align 5 +.Linput: + .long 0x00000000, 0x80000000, 0x00000001, 0x80000001 + .long 0x3f000000, 0xbf000000, 0x3fc00000, 0xbfc00000 + .long 0x4affffff, 0xcaffffff, 0x4b000000, 0xcb000000 + .long 0x7f800000, 0xff800000, 0x7fc12345, 0x7f812345 + .long 0xff812345, 0x3f800000, 0xbf800000, 0x40200000 + .long 0xc0200000, 0x7f7fffff, 0xff7fffff, 0x00800000 +.Lexpected: + .long 0x00000000, 0x80000000, 0x00000000, 0x80000000 + .long 0x00000000, 0x80000000, 0x3f800000, 0xbf800000 + .long 0x4afffffe, 0xcafffffe, 0x4b000000, 0xcb000000 + .long 0x7f800000, 0xff800000, 0x7fc12345, 0x7fc12345 + .long 0xffc12345, 0x3f800000, 0xbf800000, 0x40000000 + .long 0xc0000000, 0x7f7fffff, 0xff7fffff, 0x00000000 +.Lall_ones: .long -1, -1, -1, -1, -1, -1, -1, -1 +.Lxmm_expected: + .long 0x4afffffe, 0xcafffffe, 0x4b000000, 0xcb000000 + .long 0, 0, 0, 0 + +.section .bss +.p2align 5 +.Lout: .skip 32 + +.section .note.GNU-stack,"",@progbits From 38e5f4817c1d59ac0be36f50479ed5cc2c2bdffb Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Wed, 2 Sep 2026 18:07:24 +0800 Subject: [PATCH 07/17] LATX, AVX: write packed conversions directly to the destination Signed-off-by: Lu Zeng --- target/i386/latx/translator/tr-avx-cvt.c | 76 ++++++++++++++++-------- 1 file changed, 50 insertions(+), 26 deletions(-) diff --git a/target/i386/latx/translator/tr-avx-cvt.c b/target/i386/latx/translator/tr-avx-cvt.c index 3d2ea81845..62d0904486 100644 --- a/target/i386/latx/translator/tr-avx-cvt.c +++ b/target/i386/latx/translator/tr-avx-cvt.c @@ -102,26 +102,38 @@ static bool translate_vcvtps2dq_opt(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src = load_freg256_from_ir1(opnd1); - IR2_OPND temp_f = ra_alloc_ftemp(); IR2_OPND sse_invalid = ra_alloc_ftemp(); IR2_OPND overflow = ra_alloc_ftemp(); IR2_OPND comp_mask = ra_alloc_ftemp(); + bool same_reg = ir1_opnd_is_same_reg(opnd0, opnd1); if (ir1_opnd_is_xmm(opnd0)) { - la_vftint_w_s(temp_f, src); - la_vldi(sse_invalid, 0b1001110000000); // broadcast 0x80000000 to all 0x1380 - la_vldi(overflow, (0b10011 << 8) | 0x4f); //0x134f - la_vfcmp_cond_s(comp_mask, overflow, src, 0xE); // get Nan mark 0xE=cULE - la_vbitsel_v(temp_f, temp_f, sse_invalid, comp_mask); - la_vand_v(dest, temp_f, temp_f); + if (same_reg) { + la_vldi(sse_invalid, 0b1001110000000); + la_vldi(overflow, (0b10011 << 8) | 0x4f); + la_vfcmp_cond_s(comp_mask, overflow, src, 0xE); + la_vftint_w_s(dest, src); + } else { + la_vftint_w_s(dest, src); + la_vldi(sse_invalid, 0b1001110000000); + la_vldi(overflow, (0b10011 << 8) | 0x4f); + la_vfcmp_cond_s(comp_mask, overflow, src, 0xE); + } + la_vbitsel_v(dest, dest, sse_invalid, comp_mask); set_high128_xreg_to_zero(dest); } else { - la_xvftint_w_s(temp_f, src); - la_xvldi(sse_invalid, 0b1001110000000); // broadcast 0x80000000 to all 0x1380 - la_xvldi(overflow, (0b10011 << 8) | 0x4f); //0x134f - la_xvfcmp_cond_s(comp_mask, overflow, src, 0xE); // get Nan mark 0xE=cULE - la_xvbitsel_v(temp_f, temp_f, sse_invalid, comp_mask); - la_xvand_v(dest, temp_f, temp_f); + if (same_reg) { + la_xvldi(sse_invalid, 0b1001110000000); + la_xvldi(overflow, (0b10011 << 8) | 0x4f); + la_xvfcmp_cond_s(comp_mask, overflow, src, 0xE); + la_xvftint_w_s(dest, src); + } else { + la_xvftint_w_s(dest, src); + la_xvldi(sse_invalid, 0b1001110000000); + la_xvldi(overflow, (0b10011 << 8) | 0x4f); + la_xvfcmp_cond_s(comp_mask, overflow, src, 0xE); + } + la_xvbitsel_v(dest, dest, sse_invalid, comp_mask); } return true; } @@ -829,26 +841,38 @@ static bool translate_vcvttps2dq_opt(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src = load_freg256_from_ir1(opnd1); - IR2_OPND temp_f = ra_alloc_ftemp(); IR2_OPND sse_invalid = ra_alloc_ftemp(); IR2_OPND overflow = ra_alloc_ftemp(); IR2_OPND comp_mask = ra_alloc_ftemp(); + bool same_reg = ir1_opnd_is_same_reg(opnd0, opnd1); if (ir1_opnd_is_xmm(opnd0)) { - la_vftintrz_w_s(temp_f, src); - la_vldi(sse_invalid, 0b1001110000000); // broadcast 0x80000000 to all 0x1380 - la_vldi(overflow, (0b10011 << 8) | 0x4f); //0x134f 2^31 - la_vfcmp_cond_s(comp_mask, overflow, src, 0xE); // get Nan mark 0xE=cULE - la_vbitsel_v(temp_f, temp_f, sse_invalid, comp_mask); - la_vand_v(dest, temp_f, temp_f); + if (same_reg) { + la_vldi(sse_invalid, 0b1001110000000); + la_vldi(overflow, (0b10011 << 8) | 0x4f); + la_vfcmp_cond_s(comp_mask, overflow, src, 0xE); + la_vftintrz_w_s(dest, src); + } else { + la_vftintrz_w_s(dest, src); + la_vldi(sse_invalid, 0b1001110000000); + la_vldi(overflow, (0b10011 << 8) | 0x4f); + la_vfcmp_cond_s(comp_mask, overflow, src, 0xE); + } + la_vbitsel_v(dest, dest, sse_invalid, comp_mask); set_high128_xreg_to_zero(dest); } else { - la_xvftintrz_w_s(temp_f, src); - la_xvldi(sse_invalid, 0b1001110000000); // broadcast 0x80000000 to all 0x1380 - la_xvldi(overflow, (0b10011 << 8) | 0x4f); //0x134f 2^31 - la_xvfcmp_cond_s(comp_mask, overflow, src, 0xE); // get Nan mark 0xE=cULE - la_xvbitsel_v(temp_f, temp_f, sse_invalid, comp_mask); - la_xvand_v(dest, temp_f, temp_f); + if (same_reg) { + la_xvldi(sse_invalid, 0b1001110000000); + la_xvldi(overflow, (0b10011 << 8) | 0x4f); + la_xvfcmp_cond_s(comp_mask, overflow, src, 0xE); + la_xvftintrz_w_s(dest, src); + } else { + la_xvftintrz_w_s(dest, src); + la_xvldi(sse_invalid, 0b1001110000000); + la_xvldi(overflow, (0b10011 << 8) | 0x4f); + la_xvfcmp_cond_s(comp_mask, overflow, src, 0xE); + } + la_xvbitsel_v(dest, dest, sse_invalid, comp_mask); } return true; } From a09520589af9b4d7db81c4625e2c7856bfbd3509 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Wed, 2 Sep 2026 18:08:24 +0800 Subject: [PATCH 08/17] LATX, AVX: streamline integer narrowing and rounded multiply Signed-off-by: Lu Zeng --- target/i386/latx/translator/tr-avx.c | 77 ++++++++++++---------------- 1 file changed, 34 insertions(+), 43 deletions(-) diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 6f4de9b0d2..7e511ca756 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -1801,41 +1801,42 @@ bool translate_vpackusxx(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2; + bool dest_is_src2 = (ir1_opnd_is_xmm(opnd2) || + ir1_opnd_is_ymm(opnd2)) && + ir1_opnd_base_reg_num(opnd0) == + ir1_opnd_base_reg_num(opnd2); IR1_OPCODE op = ir1_opcode(pir1); - IR2_INST * ( * cmp_inst)(IR2_OPND, IR2_OPND, int); IR2_INST * ( * cvt_inst)(IR2_OPND, IR2_OPND, int); switch (op) { case dt_X86_INS_VPACKUSDW: - cmp_inst = la_xvslti_w; cvt_inst = la_xvssrani_hu_w; break; case dt_X86_INS_VPACKUSWB: - cmp_inst = la_xvslti_h; cvt_inst = la_xvssrani_bu_h; break; default: - cmp_inst = NULL; cvt_inst = NULL; lsassert(0); break; } - cmp_inst(temp1, src1, 0); - la_xvandn_v(temp1, temp1, src1); - if ((ir1_opnd_is_xmm(opnd2) || ir1_opnd_is_ymm(opnd2)) && + if (dest_is_src2) { + temp2 = dest; + } else if ((ir1_opnd_is_xmm(opnd2) || ir1_opnd_is_ymm(opnd2)) && ir1_opnd_base_reg_num(opnd1) == ir1_opnd_base_reg_num(opnd2)) { - temp2 = temp1; + temp2 = ra_alloc_ftemp(); + la_xvori_b(temp2, src2, 0); } else { temp2 = ra_alloc_ftemp(); - cmp_inst(temp2, src2, 0); - la_xvandn_v(temp2, temp2, src2); + la_xvori_b(temp2, src2, 0); } - cvt_inst(temp2, temp1, 0); + cvt_inst(temp2, src1, 0); if (ir1_opnd_is_xmm(opnd0)) { set_high128_xreg_to_zero(temp2); } - la_xvori_b(dest, temp2, 0); + if (!dest_is_src2) { + la_xvori_b(dest, temp2, 0); + } return true; } @@ -4709,22 +4710,10 @@ bool translate_vpmulhrsw(IR1_INST * pir1) { IR2_OPND src2 = load_freg128_from_ir1(opnd2); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND temp3 = ra_alloc_ftemp(); la_vmulwev_w_h(temp1, src1, src2); la_vmulwod_w_h(temp2, src1, src2); - - la_vsrai_w(temp1, temp1, 0xe); - la_vsrai_w(temp2, temp2, 0xe); - - la_vandi_b(temp3, temp3, 0x0); - la_vbitseti_w(temp3, temp3, 0); - - la_vadd_w(temp1, temp1, temp3); - la_vadd_w(temp2, temp2, temp3); - - la_vsrai_w(temp1, temp1, 0x1); - la_vsrai_w(temp2, temp2, 0x1); - + la_vsrari_w(temp1, temp1, 15); + la_vsrari_w(temp2, temp2, 15); la_vpackev_h(dest, temp2, temp1); set_high128_xreg_to_zero(dest); @@ -4735,22 +4724,10 @@ bool translate_vpmulhrsw(IR1_INST * pir1) { IR2_OPND src2 = load_freg256_from_ir1(opnd2); IR2_OPND temp1 = ra_alloc_ftemp(); IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND temp3 = ra_alloc_ftemp(); la_xvmulwev_w_h(temp1, src1, src2); la_xvmulwod_w_h(temp2, src1, src2); - - la_xvsrai_w(temp1, temp1, 0xe); - la_xvsrai_w(temp2, temp2, 0xe); - - la_xvandi_b(temp3, temp3, 0x0); - la_xvbitseti_w(temp3, temp3, 0); - - la_xvadd_w(temp1, temp1, temp3); - la_xvadd_w(temp2, temp2, temp3); - - la_xvsrai_w(temp1, temp1, 0x1); - la_xvsrai_w(temp2, temp2, 0x1); - + la_xvsrari_w(temp1, temp1, 15); + la_xvsrari_w(temp2, temp2, 15); la_xvpackev_h(dest, temp2, temp1); } @@ -4908,6 +4885,12 @@ bool translate_vpackssxx(IR1_INST * pir1) { IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(opnd2); IR2_OPND temp; + bool dest_is_src1 = ir1_opnd_base_reg_num(opnd0) == + ir1_opnd_base_reg_num(opnd1); + bool dest_is_src2 = (ir1_opnd_is_xmm(opnd2) || + ir1_opnd_is_ymm(opnd2)) && + ir1_opnd_base_reg_num(opnd0) == + ir1_opnd_base_reg_num(opnd2); IR2_INST * ( * cvt_inst)(IR2_OPND, IR2_OPND, int); switch (ir1_opcode(pir1)) { case dt_X86_INS_VPACKSSDW: @@ -4921,9 +4904,15 @@ bool translate_vpackssxx(IR1_INST * pir1) { lsassert(0); break; } - if (ir1_opnd_is_xmm(opnd2) || ir1_opnd_is_ymm(opnd2)) { + if (dest_is_src2) { + temp = dest; + } else if ((ir1_opnd_is_xmm(opnd2) || ir1_opnd_is_ymm(opnd2)) && + dest_is_src1) { temp = ra_alloc_ftemp(); la_xvori_b(temp, src2, 0); + } else if (ir1_opnd_is_xmm(opnd2) || ir1_opnd_is_ymm(opnd2)) { + temp = dest; + la_xvori_b(temp, src2, 0); } else { temp = src2; } @@ -4931,7 +4920,9 @@ bool translate_vpackssxx(IR1_INST * pir1) { if (ir1_opnd_is_xmm(opnd0)) { set_high128_xreg_to_zero(temp); } - la_xvori_b(dest, temp, 0); + if (temp._reg_num != dest._reg_num) { + la_xvori_b(dest, temp, 0); + } return true; } From 66ff1d3cd9394b40e55dbf88e7253fc3db8625f9 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Wed, 2 Sep 2026 18:10:26 +0800 Subject: [PATCH 09/17] LATX, AVX: avoid redundant vector copies and lane inserts Signed-off-by: Lu Zeng --- target/i386/latx/translator/tr-avx.c | 33 +++++++++++++++------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 7e511ca756..15042973eb 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -1292,13 +1292,11 @@ bool translate_vextractf128(IR1_INST * pir1) { uint8 imm = ir1_opnd_uimm(ir1_get_opnd(pir1, 2)) & 0x1; if (ir1_opnd_is_xmm(ir1_get_opnd(pir1, 0))) { IR2_OPND dest = load_freg128_from_ir1(ir1_get_opnd(pir1, 0)); - IR2_OPND temp = ra_alloc_ftemp(); if (!imm) { - la_xvpermi_q(temp, src, VEXTRINS_IMM_4_0(3, 0)); + la_xvpermi_q(dest, src, VEXTRINS_IMM_4_0(3, 0)); } else { - la_xvpermi_q(temp, src, VEXTRINS_IMM_4_0(3, 1)); + la_xvpermi_q(dest, src, VEXTRINS_IMM_4_0(3, 1)); } - la_xvori_b(dest, temp, 0); set_high128_xreg_to_zero(dest); } else { if (!imm) { @@ -1438,14 +1436,25 @@ bool translate_vinserti128(IR1_INST * pir1) { IR2_OPND src1 = load_freg256_from_ir1(ir1_get_opnd(pir1, 1)); IR2_OPND src2 = load_freg128_from_ir1(ir1_get_opnd(pir1, 2)); uint8 imm = ir1_opnd_uimm(ir1_get_opnd(pir1, 3)) & 0x1; - IR2_OPND temp = ra_alloc_ftemp(); - la_xvori_b(temp, src1, 0); + bool dest_is_src1 = ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) == + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1)); + bool dest_is_src2 = ir1_opnd_is_xmm(ir1_get_opnd(pir1, 2)) && + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0)) == + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 2)); + IR2_OPND result = dest_is_src2 && !dest_is_src1 ? + ra_alloc_ftemp() : dest; + + if (!dest_is_src1) { + la_xvori_b(result, src1, 0); + } if (!imm) { - la_xvpermi_q(temp, src2, VEXTRINS_IMM_4_0(3, 0)); + la_xvpermi_q(result, src2, VEXTRINS_IMM_4_0(3, 0)); } else { - la_xvpermi_q(temp, src2, VEXTRINS_IMM_4_0(0, 2)); + la_xvpermi_q(result, src2, VEXTRINS_IMM_4_0(0, 2)); + } + if (dest_is_src2 && !dest_is_src1) { + la_xvori_b(dest, result, 0); } - la_xvori_b(dest, temp, 0); return true; } @@ -3599,8 +3608,6 @@ bool translate_vpbroadcastd(IR1_INST *pir1) la_vreplvei_w(dest, src, 0); } else if (ir1_opnd_is_ymm(opnd0)) { la_xvreplve0_w(dest, src); - la_xvinsve0_d(dest, dest, 2); - la_xvinsve0_d(dest, dest, 3); } else { lsassert(0); } @@ -6643,8 +6650,6 @@ bool translate_vpbroadcastb(IR1_INST *pir1) la_xvreplve0_b(dest, src); } else if (ir1_opnd_is_ymm(opnd0)) { la_xvreplve0_b(dest, src); - la_xvinsve0_d(dest, dest, 2); - la_xvinsve0_d(dest, dest, 3); } else { lsassert(0); } @@ -6679,8 +6684,6 @@ bool translate_vpbroadcastw(IR1_INST *pir1) la_xvreplve0_h(dest, src); } else if (ir1_opnd_is_ymm(opnd0)) { la_xvreplve0_h(dest, src); - la_xvinsve0_d(dest, dest, 2); - la_xvinsve0_d(dest, dest, 3); } else { lsassert(0); } From 85efe63cf265a8627fcd3f2dccc039aeb4a53892 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Wed, 2 Sep 2026 18:26:20 +0800 Subject: [PATCH 10/17] LATX, AVX: write VPBLENDVB directly to the destination Signed-off-by: Lu Zeng --- target/i386/latx/translator/tr-avx.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 15042973eb..d9a70fc041 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -2036,13 +2036,12 @@ bool translate_vpblendvb(IR1_INST * pir1) { (ir1_opnd_is_ymm(opnd0) && ir1_opnd_is_ymm(opnd1))); IR2_OPND dest, src1, src2, src3; IR2_OPND temp = ra_alloc_ftemp(); - dest = load_freg256_from_ir1(opnd0); + dest = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd0)); src1 = load_freg256_from_ir1(opnd1); src2 = load_freg256_from_ir1(opnd2); src3 = load_freg256_from_ir1(opnd3); la_xvslti_b(temp, src3, 0); - la_xvbitsel_v(temp, src1, src2, temp); - la_xvori_b(dest, temp, 0); + la_xvbitsel_v(dest, src1, src2, temp); if (ir1_opnd_is_xmm(opnd0)) { set_high128_xreg_to_zero(dest); } From a55596db789dcf47a65b65aa67daffe47df5e696 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Wed, 2 Sep 2026 18:36:16 +0800 Subject: [PATCH 11/17] LATX, AVX: use direct widening multiply-add sequences Signed-off-by: Lu Zeng --- target/i386/latx/translator/tr-avx-lsx.c | 53 ++++++-------- target/i386/latx/translator/tr-avx.c | 91 +++++++++--------------- 2 files changed, 56 insertions(+), 88 deletions(-) diff --git a/target/i386/latx/translator/tr-avx-lsx.c b/target/i386/latx/translator/tr-avx-lsx.c index 10d8342950..5ac247a882 100644 --- a/target/i386/latx/translator/tr-avx-lsx.c +++ b/target/i386/latx/translator/tr-avx-lsx.c @@ -605,46 +605,35 @@ static bool translate_avx_integer_3op_custom_lsx( static void translate_vpmaddwd_lane_lsx(IR2_OPND dest, IR2_OPND src1, IR2_OPND src2) { - IR2_OPND temp = ra_alloc_ftemp(); + if (dest._reg_num != src1._reg_num && + dest._reg_num != src2._reg_num) { + la_vmulwev_w_h(dest, src1, src2); + la_vmaddwod_w_h(dest, src1, src2); + return; + } - la_vxor_v(temp, temp, temp); - la_vmaddwev_w_h(temp, src1, src2); - la_vmaddwod_w_h(temp, src1, src2); - la_vbsll_v(dest, temp, 0); - ra_free_temp(temp); + IR2_OPND even = ra_alloc_ftemp(); + IR2_OPND odd = ra_alloc_ftemp(); + + la_vmulwev_w_h(even, src1, src2); + la_vmulwod_w_h(odd, src1, src2); + la_vadd_w(dest, even, odd); + ra_free_temp(odd); + ra_free_temp(even); } static void translate_vpmaddubsw_lane_lsx(IR2_OPND dest, IR2_OPND src1, IR2_OPND src2) { - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND temp3 = ra_alloc_ftemp(); - IR2_OPND temp4 = ra_alloc_ftemp(); - IR2_OPND temp5 = ra_alloc_ftemp(); - IR2_OPND one = ra_alloc_itemp(); + IR2_OPND even = ra_alloc_ftemp(); + IR2_OPND odd = ra_alloc_ftemp(); /* Unsigned src1 multiplied by signed src2, then saturated to halfwords. */ - la_vreplgr2vr_d(temp1, zero_ir2_opnd); - la_vabsd_b(temp3, src2, temp1); - la_vmaddwev_h_bu(temp1, src1, temp3); - la_vreplgr2vr_d(temp2, zero_ir2_opnd); - la_vmaddwod_h_bu(temp2, src1, temp3); - - la_ori(one, zero_ir2_opnd, 1); - la_vreplgr2vr_b(temp3, one); - la_vsigncov_b(temp4, src2, temp3); - la_vmulwev_h_b(temp5, temp4, temp3); - la_vmulwod_h_b(temp3, temp4, temp3); - la_vmul_h(temp1, temp1, temp5); - la_vmul_h(temp2, temp2, temp3); - la_vsadd_h(dest, temp2, temp1); - ra_free_temp(one); - ra_free_temp(temp5); - ra_free_temp(temp4); - ra_free_temp(temp3); - ra_free_temp(temp2); - ra_free_temp(temp1); + la_vmulwev_h_bu_b(even, src1, src2); + la_vmulwod_h_bu_b(odd, src1, src2); + la_vsadd_h(dest, even, odd); + ra_free_temp(odd); + ra_free_temp(even); } static void translate_vpmulhrsw_lane_lsx(IR2_OPND dest, IR2_OPND src1, diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index d9a70fc041..5bc0e6fb59 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -3864,23 +3864,35 @@ bool translate_vpmaddwd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); + if (dest._reg_num != src1._reg_num && + dest._reg_num != src2._reg_num) { + la_vmulwev_w_h(dest, src1, src2); + la_vmaddwod_w_h(dest, src1, src2); + } else { + IR2_OPND even = ra_alloc_ftemp(); + IR2_OPND odd = ra_alloc_ftemp(); - la_vxor_v(temp, temp, temp); - la_vmaddwev_w_h(temp, src1, src2); - la_vmaddwod_w_h(temp, src1, src2); - la_vbsll_v(dest, temp, 0); + la_vmulwev_w_h(even, src1, src2); + la_vmulwod_w_h(odd, src1, src2); + la_vadd_w(dest, even, odd); + } set_high128_xreg_to_zero(dest); } else { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND temp = ra_alloc_ftemp(); + if (dest._reg_num != src1._reg_num && + dest._reg_num != src2._reg_num) { + la_xvmulwev_w_h(dest, src1, src2); + la_xvmaddwod_w_h(dest, src1, src2); + } else { + IR2_OPND even = ra_alloc_ftemp(); + IR2_OPND odd = ra_alloc_ftemp(); - la_xvxor_v(temp, temp, temp); - la_xvmaddwev_w_h(temp, src1, src2); - la_xvmaddwod_w_h(temp, src1, src2); - la_xvbsll_v(dest, temp, 0); + la_xvmulwev_w_h(even, src1, src2); + la_xvmulwod_w_h(odd, src1, src2); + la_xvadd_w(dest, even, odd); + } } return true; } @@ -3898,29 +3910,13 @@ bool translate_vpmaddubsw(IR1_INST * pir1) { IR2_OPND src1 = load_freg128_from_ir1(opnd1); IR2_OPND src2 = load_freg128_from_ir1(opnd2); - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND temp3 = ra_alloc_ftemp(); - IR2_OPND temp4 = ra_alloc_ftemp(); - IR2_OPND temp5 = ra_alloc_ftemp(); - IR2_OPND itmp = ra_alloc_itemp(); - /* unsigned src1 * signed src2 */ - la_vreplgr2vr_d(temp1, zero_ir2_opnd); - la_vabsd_b(temp3, src2, temp1); - la_vmaddwev_h_bu(temp1, src1, temp3); - la_vreplgr2vr_d(temp2, zero_ir2_opnd); - la_vmaddwod_h_bu(temp2, src1, temp3); - - la_ori(itmp, zero_ir2_opnd, 0x1); - la_vreplgr2vr_b(temp3, itmp); - la_vsigncov_b(temp4, src2, temp3); - - la_vmulwev_h_b(temp5, temp4, temp3); - la_vmulwod_h_b(temp3, temp4, temp3); - - la_vmul_h(temp1, temp1, temp5); - la_vmul_h(temp2, temp2, temp3); - la_vsadd_h(dest, temp2, temp1); + IR2_OPND even = ra_alloc_ftemp(); + IR2_OPND odd = ra_alloc_ftemp(); + + /* Unsigned src1 bytes times signed src2 bytes, pairwise saturated. */ + la_vmulwev_h_bu_b(even, src1, src2); + la_vmulwod_h_bu_b(odd, src1, src2); + la_vsadd_h(dest, even, odd); set_high128_xreg_to_zero(dest); @@ -3928,29 +3924,12 @@ bool translate_vpmaddubsw(IR1_INST * pir1) { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND temp1 = ra_alloc_ftemp(); - IR2_OPND temp2 = ra_alloc_ftemp(); - IR2_OPND temp3 = ra_alloc_ftemp(); - IR2_OPND temp4 = ra_alloc_ftemp(); - IR2_OPND temp5 = ra_alloc_ftemp(); - IR2_OPND itmp = ra_alloc_itemp(); - /* unsigned src1 * signed src2 */ - la_xvreplgr2vr_d(temp1, zero_ir2_opnd); - la_xvabsd_b(temp3, src2, temp1); - la_xvmaddwev_h_bu(temp1, src1, temp3); - la_xvreplgr2vr_d(temp2, zero_ir2_opnd); - la_xvmaddwod_h_bu(temp2, src1, temp3); - - la_ori(itmp, zero_ir2_opnd, 0x1); - la_xvreplgr2vr_b(temp3, itmp); - la_xvsigncov_b(temp4, src2, temp3); - - la_xvmulwev_h_b(temp5, temp4, temp3); - la_xvmulwod_h_b(temp3, temp4, temp3); - - la_xvmul_h(temp1, temp1, temp5); - la_xvmul_h(temp2, temp2, temp3); - la_xvsadd_h(dest, temp2, temp1); + IR2_OPND even = ra_alloc_ftemp(); + IR2_OPND odd = ra_alloc_ftemp(); + + la_xvmulwev_h_bu_b(even, src1, src2); + la_xvmulwod_h_bu_b(odd, src1, src2); + la_xvsadd_h(dest, even, odd); } return true; } From 8abd7930ebdb1b198fa3226ed5b298b92cd91768 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Wed, 2 Sep 2026 18:36:17 +0800 Subject: [PATCH 12/17] LATX, AVX: write shuffle and byte-shift results directly Signed-off-by: Lu Zeng --- target/i386/latx/translator/tr-avx-shift.c | 4 +-- target/i386/latx/translator/tr-avx.c | 36 +++++++++++++--------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/target/i386/latx/translator/tr-avx-shift.c b/target/i386/latx/translator/tr-avx-shift.c index 6e75e3ece6..e89939163f 100644 --- a/target/i386/latx/translator/tr-avx-shift.c +++ b/target/i386/latx/translator/tr-avx-shift.c @@ -21,7 +21,7 @@ bool translate_vpslldq(IR1_INST * pir1) { lsassert((ir1_opnd_is_xmm(opnd0) && ir1_opnd_is_xmm(opnd1)) || (ir1_opnd_is_ymm(opnd0) && ir1_opnd_is_ymm(opnd1))); lsassert(ir1_opnd_is_imm(opnd2)); - IR2_OPND dest = load_freg256_from_ir1(opnd0); + IR2_OPND dest = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd0)); IR2_OPND src = load_freg256_from_ir1(opnd1); uint8_t imm = ir1_opnd_uimm(opnd2); if (imm > 15) { @@ -118,7 +118,7 @@ bool translate_vpsrldq(IR1_INST * pir1) { lsassert((ir1_opnd_is_xmm(opnd0) && ir1_opnd_is_xmm(opnd1)) || (ir1_opnd_is_ymm(opnd0) && ir1_opnd_is_ymm(opnd1))); lsassert(ir1_opnd_is_imm(opnd2)); - IR2_OPND dest = load_freg256_from_ir1(opnd0); + IR2_OPND dest = ra_alloc_xmm(ir1_opnd_base_reg_num(opnd0)); IR2_OPND src = load_freg256_from_ir1(opnd1); uint8_t imm = ir1_opnd_uimm(opnd2); if (imm > 15) { diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index 5bc0e6fb59..a06f3d02fa 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -4196,15 +4196,13 @@ bool translate_vpshufd(IR1_INST * pir1) { IR2_OPND dest = load_freg128_from_ir1(opnd0); IR2_OPND src1 = load_freg128_from_ir1(opnd1); uint8_t imm = ir1_opnd_uimm(opnd2); - la_xvori_b(dest, src1, 0x00); - la_vpermi_w(dest, src1, imm); + la_xvshuf4i_w(dest, src1, imm); set_high128_xreg_to_zero(dest); } else { IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src1 = load_freg256_from_ir1(opnd1); uint8_t imm = ir1_opnd_uimm(opnd2); - la_xvori_b(dest, src1, 0x00); - la_xvpermi_w(dest, src1, imm); + la_xvshuf4i_w(dest, src1, imm); } return true; } @@ -4923,13 +4921,18 @@ bool translate_vpshufhw(IR1_INST *pir1) IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1); IR1_OPND *opnd2 = ir1_get_opnd(pir1, 2); + uint64_t imm8 = ir1_opnd_uimm(opnd2); IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src = load_freg256_from_ir1(opnd1); - IR2_OPND temp = ra_alloc_ftemp(); - uint64_t imm8 = ir1_opnd_uimm(opnd2); - la_xvshuf4i_h(temp, src, imm8); - la_xvshuf4i_d(temp, src, 0x66); - la_xvori_b(dest, temp, 0); + if (dest._reg_num != src._reg_num) { + la_xvshuf4i_h(dest, src, imm8); + la_xvshuf4i_d(dest, src, 0x66); + } else { + IR2_OPND temp = ra_alloc_ftemp(); + la_xvshuf4i_h(temp, src, imm8); + la_xvshuf4i_d(temp, src, 0x66); + la_xvori_b(dest, temp, 0); + } if(ir1_opnd_is_xmm(opnd0)){ set_high128_xreg_to_zero(dest); } @@ -4948,13 +4951,18 @@ bool translate_vpshuflw(IR1_INST *pir1) IR1_OPND *opnd1 = ir1_get_opnd(pir1, 1); IR1_OPND *opnd2 = ir1_get_opnd(pir1, 2); + uint64_t imm8 = ir1_opnd_uimm(opnd2); IR2_OPND dest = load_freg256_from_ir1(opnd0); IR2_OPND src = load_freg256_from_ir1(opnd1); - IR2_OPND temp = ra_alloc_ftemp(); - uint64_t imm8 = ir1_opnd_uimm(opnd2); - la_xvshuf4i_h(temp, src, imm8); - la_xvshuf4i_d(temp, src, 0xcc); - la_xvori_b(dest, temp, 0); + if (dest._reg_num != src._reg_num) { + la_xvshuf4i_h(dest, src, imm8); + la_xvshuf4i_d(dest, src, 0xcc); + } else { + IR2_OPND temp = ra_alloc_ftemp(); + la_xvshuf4i_h(temp, src, imm8); + la_xvshuf4i_d(temp, src, 0xcc); + la_xvori_b(dest, temp, 0); + } if(ir1_opnd_is_xmm(opnd0)){ set_high128_xreg_to_zero(dest); } From 18d469b59ee8e64b15539559354c6624973b7736 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Wed, 2 Sep 2026 18:11:33 +0800 Subject: [PATCH 13/17] LATX: eliminate redundant LASX immediate loads Signed-off-by: Lu Zeng --- .../i386/latx/optimization/ir2-optimization.c | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/target/i386/latx/optimization/ir2-optimization.c b/target/i386/latx/optimization/ir2-optimization.c index 8e97959f2d..17205a1ba2 100644 --- a/target/i386/latx/optimization/ir2-optimization.c +++ b/target/i386/latx/optimization/ir2-optimization.c @@ -284,6 +284,54 @@ static void ir2_opt_scalar_fma_forward(void) } } +/* + * Temporary vector registers are released at each x86 instruction boundary. + * Consecutive translations can therefore allocate the same physical register + * and reload the same LASX immediate even though the earlier value is still + * live in that register. Keep the value across the boundary until an actual + * write or a control-flow join invalidates it. + */ +static void ir2_opt_redundant_xvldi(void) +{ + bool valid[32] = { false }; + int imm[32] = { 0 }; + IR2_INST *ir2 = lsenv->tr_data->first_ir2; + + while (ir2) { + IR2_INST *next = ir2_next(ir2); + IR2_OPCODE opc = ir2_opcode(ir2); + + if (opc == LISA_XVLDI) { + int reg = ir2->_opnd[0]._reg_num; + int value = ir2->_opnd[1]._imm32; + + if (reg >= 0 && reg < ARRAY_SIZE(valid)) { + if (valid[reg] && imm[reg] == value) { + ir2_remove(ir2_get_id(ir2)); + ir2 = next; + continue; + } + valid[reg] = true; + imm[reg] = value; + } + } else { + if (ir2_opcode_is_branch(opc) || ir2_opcode_is_jirl(opc) || + la_ir2_opcode_is_label(opc)) { + memset(valid, 0, sizeof(valid)); + } + if (!la_ir2_opcode_is_store(opc) && ir2->op_count > 0 && + ir2_opnd_is_freg(&ir2->_opnd[0])) { + int reg = ir2->_opnd[0]._reg_num; + + if (reg >= 0 && reg < ARRAY_SIZE(valid)) { + valid[reg] = false; + } + } + } + ir2 = next; + } +} + static int ir2_get_addi_rsp_offs(IR2_INST *ir2) { int rsp = reg_gpr_map[esp_index]; @@ -598,6 +646,7 @@ static void ir2_opt_push_pop(TranslationBlock *tb) void tr_ir2_optimize(TranslationBlock *tb) { ir2_opt_scalar_fma_forward(); + ir2_opt_redundant_xvldi(); #ifdef CONFIG_LATX_OPT_PUSH_POP CPUX86State *env = (CPUX86State*)lsenv->cpu_state; CPUState *cpu = env_cpu(env); From 2782c980e03662536a153e7b158bb8e4736fa228 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Wed, 2 Sep 2026 18:12:03 +0800 Subject: [PATCH 14/17] LATX: fold RIP-relative displacement into JIT loads Signed-off-by: Lu Zeng --- target/i386/latx/translator/mem-interface.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/target/i386/latx/translator/mem-interface.c b/target/i386/latx/translator/mem-interface.c index 5ad6bdf96c..0e5fcc6428 100644 --- a/target/i386/latx/translator/mem-interface.c +++ b/target/i386/latx/translator/mem-interface.c @@ -289,11 +289,24 @@ static IR2_OPND convert_mem_helper(IR1_OPND *opnd1, IR2_OPND *arg_dest_op, } } #endif + longx load_offset = 0; + target_ulong base_offset = offset; + +#ifdef CONFIG_LATX_AOT + bool aot_reloc = option_aot && in_pre_translate; +#else + bool aot_reloc = false; +#endif + if (host_off && !aot_reloc) { + load_offset = sextract64(offset, 0, 12); + base_offset -= load_offset; + } target_ulong call_offset __attribute__((unused)) = - aot_get_call_offset(offset); - aot_load_guest_addr(dest_op, offset, LOAD_CALL_TARGET, call_offset); + aot_get_call_offset(base_offset); + aot_load_guest_addr(dest_op, base_offset, LOAD_CALL_TARGET, + call_offset); if (host_off) { - *host_off = 0; + *host_off = load_offset; } return dest_op; } From 5672cc03095c0d19d69844c560fb55d08c812cfd Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Thu, 3 Sep 2026 19:28:03 +0800 Subject: [PATCH 15/17] tests: cover consolidated LATX AVX optimizations Signed-off-by: Lu Zeng --- tests/integration/meson.build | 2 +- .../registrations/{simd => avx}/meson.build | 16 ++- tests/integration/rip-relative-load.S | 40 ++++++++ tests/integration/test-avx-lowering.sh | 48 +++++++++ tests/integration/test-avx-ymmh-zero.sh | 49 +++++----- tests/integration/vcvtps2dq-alias.c | 62 ++++++++++++ tests/integration/vex128-ymmh-signal.S | 76 +++++++++++++++ tests/integration/vpackus-fast.c | 97 +++++++++++++++++++ tests/integration/vpmulhrsw-fast.c | 55 +++++++++++ tests/integration/vpshufb-fast.c | 47 +++++++++ 10 files changed, 467 insertions(+), 25 deletions(-) rename tests/integration/registrations/{simd => avx}/meson.build (76%) create mode 100644 tests/integration/rip-relative-load.S create mode 100755 tests/integration/test-avx-lowering.sh create mode 100644 tests/integration/vcvtps2dq-alias.c create mode 100644 tests/integration/vex128-ymmh-signal.S create mode 100644 tests/integration/vpackus-fast.c create mode 100644 tests/integration/vpmulhrsw-fast.c create mode 100644 tests/integration/vpshufb-fast.c diff --git a/tests/integration/meson.build b/tests/integration/meson.build index 152f02e3f2..7c451b0a15 100644 --- a/tests/integration/meson.build +++ b/tests/integration/meson.build @@ -1,7 +1,7 @@ latx_integration_tests = [] if host_machine.cpu_family() == 'loongarch64' - subdir('registrations/simd') + subdir('registrations/avx') subdir('registrations/x11-kzt') subdir('registrations/sandbox') subdir('registrations/process') diff --git a/tests/integration/registrations/simd/meson.build b/tests/integration/registrations/avx/meson.build similarity index 76% rename from tests/integration/registrations/simd/meson.build rename to tests/integration/registrations/avx/meson.build index a9bfeff955..de68fa1d9d 100644 --- a/tests/integration/registrations/simd/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -5,8 +5,9 @@ if 'x86_64-linux-user' in target_dirs 'args': [ emulators['latx-x86_64'], files('../../vex128-ymmh-zero.S'), + files('../../vex128-ymmh-signal.S'), ], - 'timeout': 120, + 'timeout': 180, }] latx_integration_tests += [{ 'name': 'test-avx-ymm-writeback', @@ -46,7 +47,20 @@ if 'x86_64-linux-user' in target_dirs emulators['latx-x86_64'], files('../../scalar-hdr-pattern.S'), files('../../vroundps-imm0b.S'), + files('../../rip-relative-load.S'), ], 'timeout': 120, }] + latx_integration_tests += [{ + 'name': 'test-avx-lowering', + 'runner': find_program('../../test-avx-lowering.sh'), + 'args': [ + emulators['latx-x86_64'], + files('../../vcvtps2dq-alias.c'), + files('../../vpackus-fast.c'), + files('../../vpmulhrsw-fast.c'), + files('../../vpshufb-fast.c'), + ], + 'timeout': 300, + }] endif diff --git a/tests/integration/rip-relative-load.S b/tests/integration/rip-relative-load.S new file mode 100644 index 0000000000..ee409cd0b4 --- /dev/null +++ b/tests/integration/rip-relative-load.S @@ -0,0 +1,40 @@ +.intel_syntax noprefix +.global _start + +.text +_start: + mov eax, dword ptr [rip + .Lvalue0] + cmp eax, 0x13579bdf + jne .Lfail + + mov rax, qword ptr [rip + .Lvalue1] + mov rdx, 0x0123456789abcdef + cmp rax, rdx + jne .Lfail + + vmovdqu xmm0, xmmword ptr [rip + .Lvector] + vpxor xmm0, xmm0, xmmword ptr [rip + .Lexpected] + vptest xmm0, xmm0 + jne .Lfail + + mov eax, 60 + xor edi, edi + syscall + +.Lfail: + mov eax, 60 + mov edi, 1 + syscall + +.section .rodata +.p2align 4 +.Lvalue0: + .long 0x13579bdf +.Lvalue1: + .quad 0x0123456789abcdef +.Lvector: + .quad 0x0011223344556677, 0x8899aabbccddeeff +.Lexpected: + .quad 0x0011223344556677, 0x8899aabbccddeeff + +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/test-avx-lowering.sh b/tests/integration/test-avx-lowering.sh new file mode 100755 index 0000000000..ff8bea906a --- /dev/null +++ b/tests/integration/test-avx-lowering.sh @@ -0,0 +1,48 @@ +#!/bin/sh +set -eu + +emulator=$1 +shift +workdir=$(mktemp -d) +trap 'rm -rf "$workdir"' EXIT HUP INT TERM + +if command -v clang-19 >/dev/null 2>&1; then + clang=clang-19 +elif command -v clang >/dev/null 2>&1; then + clang=clang +else + echo "SKIP: clang is required to build the x86_64 guests" + exit 77 +fi + +for source_file in "$@"; do + name=$(basename "$source_file" .c) + "$clang" --target=x86_64-linux-gnu -fuse-ld=lld -O2 -mavx2 \ + -nostdlib -static -ffreestanding -fno-builtin \ + -fno-stack-protector -Wl,--build-id=none \ + "$source_file" -o "$workdir/$name" + + LATX_AOT=0 LATX_TU=0 "$emulator" "$workdir/$name" + + aot_home="$workdir/aot-$name" + mkdir -p "$aot_home" + HOME="$aot_home" LATX_AOT=1 LATX_TU=1 \ + "$emulator" "$workdir/$name" + + aot_file= + for _ in $(seq 1 100); do + aot_file=$(find "$aot_home/.cache/latx" -type f -name '*.aot2' \ + -size +0c -print -quit 2>/dev/null || true) + [ -n "$aot_file" ] && break + sleep 0.1 + done + if [ -z "$aot_file" ]; then + echo "FAIL: no non-empty AOT file generated for $name" >&2 + exit 1 + fi + + HOME="$aot_home" LATX_AOT=1 LATX_TU=1 \ + "$emulator" "$workdir/$name" +done + +echo "PASS: AVX lowering JIT/cold-AOT/hot-AOT" diff --git a/tests/integration/test-avx-ymmh-zero.sh b/tests/integration/test-avx-ymmh-zero.sh index c384f5dc54..15b2db17bb 100755 --- a/tests/integration/test-avx-ymmh-zero.sh +++ b/tests/integration/test-avx-ymmh-zero.sh @@ -2,7 +2,7 @@ set -eu emulator=$1 -source_file=$2 +shift workdir=$(mktemp -d) trap 'rm -rf "$workdir"' EXIT HUP INT TERM @@ -11,33 +11,36 @@ if command -v clang-19 >/dev/null 2>&1; then elif command -v clang >/dev/null 2>&1; then clang=clang else - echo "SKIP: clang is required to build the x86_64 guest" + echo "SKIP: clang is required to build the x86_64 guests" exit 77 fi -guest="$workdir/vex128-ymmh-zero" -"$clang" --target=x86_64-linux-gnu -fuse-ld=lld -nostdlib -static \ - -Wl,--build-id=none "$source_file" -o "$guest" +for source_file in "$@"; do + name=$(basename "$source_file" .S) + guest="$workdir/$name" + "$clang" --target=x86_64-linux-gnu -fuse-ld=lld -nostdlib -static \ + -Wl,--build-id=none "$source_file" -o "$guest" -LATX_AOT=0 LATX_TU=0 "$emulator" "$guest" + LATX_AOT=0 LATX_TU=0 "$emulator" "$guest" -aot_home="$workdir/aot" -mkdir -p "$aot_home" -HOME="$aot_home" LATX_AOT=1 LATX_TU=1 "$emulator" "$guest" + aot_home="$workdir/aot-$name" + mkdir -p "$aot_home" + HOME="$aot_home" LATX_AOT=1 LATX_TU=1 "$emulator" "$guest" -aot_file= -for _ in $(seq 1 100); do - aot_file=$(find "$aot_home/.cache/latx" -type f -name '*.aot2' \ - -size +0c -print -quit 2>/dev/null || true) - [ -n "$aot_file" ] && break - sleep 0.1 -done -if [ -z "$aot_file" ]; then - echo "FAIL: no non-empty AOT file generated" >&2 - exit 1 -fi + aot_file= + for _ in $(seq 1 100); do + aot_file=$(find "$aot_home/.cache/latx" -type f -name '*.aot2' \ + -size +0c -print -quit 2>/dev/null || true) + [ -n "$aot_file" ] && break + sleep 0.1 + done + if [ -z "$aot_file" ]; then + echo "FAIL: no non-empty AOT file generated for $name" >&2 + exit 1 + fi -for _ in $(seq 1 10); do - HOME="$aot_home" LATX_AOT=1 LATX_TU=1 "$emulator" "$guest" + for _ in 1 2 3; do + HOME="$aot_home" LATX_AOT=1 LATX_TU=1 "$emulator" "$guest" + done done -echo "PASS: VEX.128 YMM-high zeroing and mixed-width HBR JIT/cold-AOT/hot-AOT" +echo "PASS: VEX.128 YMM-high zeroing, signal state, and JIT/cold-AOT/hot-AOT" diff --git a/tests/integration/vcvtps2dq-alias.c b/tests/integration/vcvtps2dq-alias.c new file mode 100644 index 0000000000..123c54482f --- /dev/null +++ b/tests/integration/vcvtps2dq-alias.c @@ -0,0 +1,62 @@ +#include +#include + +static __attribute__((noreturn)) void exit_group(int status) +{ + __asm__ volatile("syscall\n1:\n\tjmp 1b" + : : "a"(60), "D"(status) : "rcx", "r11"); + __builtin_unreachable(); +} + +void _start(void) +{ + static const uint32_t bits[8] __attribute__((aligned(32))) = { + 0x00000000, 0x3f000000, 0x3fc00000, 0x40200000, + 0xbf000000, 0xbfc00000, 0x4f000000, 0x7fc12345, + }; + static const int32_t rounded_expected[4][8] = { + { 0, 0, 2, 2, 0, -2, INT32_MIN, INT32_MIN }, + { 0, 0, 1, 2, -1, -2, INT32_MIN, INT32_MIN }, + { 0, 1, 2, 3, 0, -1, INT32_MIN, INT32_MIN }, + { 0, 0, 1, 2, 0, -1, INT32_MIN, INT32_MIN }, + }; + static const int32_t truncated_expected[8] = { + 0, 0, 1, 2, 0, -1, INT32_MIN, INT32_MIN, + }; + int32_t output[4][4][8] __attribute__((aligned(32))); + __m256 input = _mm256_castsi256_ps( + _mm256_load_si256((const __m256i *)bits)); + + for (unsigned int rounding = 0; rounding < 4; ++rounding) { + unsigned int mxcsr = _mm_getcsr(); + _mm_setcsr((mxcsr & ~(3u << 13)) | (rounding << 13)); + + __m256i rounded_alias = _mm256_castps_si256(input); + __asm__ volatile("vcvtps2dq %0, %0" : "+x"(rounded_alias)); + _mm256_store_si256((__m256i *)output[rounding][0], rounded_alias); + + __m256i rounded_distinct; + __asm__ volatile("vcvtps2dq %1, %0" + : "=&x"(rounded_distinct) : "x"(input)); + _mm256_store_si256((__m256i *)output[rounding][1], rounded_distinct); + + __m256i truncated_alias = _mm256_castps_si256(input); + __asm__ volatile("vcvttps2dq %0, %0" : "+x"(truncated_alias)); + _mm256_store_si256((__m256i *)output[rounding][2], truncated_alias); + + __m256i truncated_distinct; + __asm__ volatile("vcvttps2dq %1, %0" + : "=&x"(truncated_distinct) : "x"(input)); + _mm256_store_si256((__m256i *)output[rounding][3], truncated_distinct); + + for (int i = 0; i < 8; ++i) { + if (output[rounding][0][i] != rounded_expected[rounding][i] || + output[rounding][1][i] != rounded_expected[rounding][i] || + output[rounding][2][i] != truncated_expected[i] || + output[rounding][3][i] != truncated_expected[i]) { + exit_group(1); + } + } + } + exit_group(0); +} diff --git a/tests/integration/vex128-ymmh-signal.S b/tests/integration/vex128-ymmh-signal.S new file mode 100644 index 0000000000..efa08c0f1d --- /dev/null +++ b/tests/integration/vex128-ymmh-signal.S @@ -0,0 +1,76 @@ +.intel_syntax noprefix +.equ __NR_rt_sigaction, 13 +.equ __NR_rt_sigreturn, 15 +.equ __NR_alarm, 37 +.equ __NR_exit, 60 +.equ SIGALRM, 14 +.equ SA_RESTORER, 0x04000000 + +.section .data +.balign 8 +alarm_action: + .quad alarm_handler + .quad SA_RESTORER + .quad signal_restorer + .quad 0 + +.balign 32 +initial: + .quad -1, -1, -1, -1 + +.section .bss +.balign 4 +done: + .long 0 +failed: + .long 0 + +.section .text +.global _start +.type _start, @function +_start: + mov eax, __NR_rt_sigaction + mov edi, SIGALRM + lea rsi, [rip + alarm_action] + xor edx, edx + mov r10d, 8 + syscall + test rax, rax + js .Linstall_failed + + mov eax, __NR_alarm + mov edi, 1 + syscall + vmovdqu ymm0, YMMWORD PTR [rip + initial] +.Lwait: + vpxor xmm1, xmm1, xmm1 + vmovdqa xmm0, xmm1 + cmp DWORD PTR [rip + done], 0 + je .Lwait + + mov edi, DWORD PTR [rip + failed] + jmp .Lexit + +.Linstall_failed: + mov edi, 2 +.Lexit: + mov eax, __NR_exit + syscall +.size _start, .-_start + +.type alarm_handler, @function +alarm_handler: + vextracti128 xmm2, ymm0, 1 + vptest xmm2, xmm2 + setne BYTE PTR [rip + failed] + mov DWORD PTR [rip + done], 1 + ret +.size alarm_handler, .-alarm_handler + +.type signal_restorer, @function +signal_restorer: + mov eax, __NR_rt_sigreturn + syscall +.size signal_restorer, .-signal_restorer + +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/vpackus-fast.c b/tests/integration/vpackus-fast.c new file mode 100644 index 0000000000..7f1763984a --- /dev/null +++ b/tests/integration/vpackus-fast.c @@ -0,0 +1,97 @@ +#include +#include + +static __attribute__((noreturn)) void exit_group(int status) +{ + __asm__ volatile("syscall\n1:\n\tjmp 1b" + : : "a"(60), "D"(status) : "rcx", "r11"); + __builtin_unreachable(); +} + +static uint8_t sat_u8(int16_t value) +{ + if (value < 0) { + return 0; + } + if (value > UINT8_MAX) { + return UINT8_MAX; + } + return value; +} + +static uint16_t sat_u16(int32_t value) +{ + if (value < 0) { + return 0; + } + if (value > UINT16_MAX) { + return UINT16_MAX; + } + return value; +} + +void _start(void) +{ + uint32_t state = 1; + int16_t h1[16] __attribute__((aligned(32))); + int16_t h2[16] __attribute__((aligned(32))); + int32_t w1[8] __attribute__((aligned(32))); + int32_t w2[8] __attribute__((aligned(32))); + uint8_t got_b[32] __attribute__((aligned(32))); + uint16_t got_h[16] __attribute__((aligned(32))); + + for (int round = 0; round < 100000; ++round) { + for (int i = 0; i < 16; ++i) { + state = state * 1664525u + 1013904223u; + h1[i] = state; + state = state * 1664525u + 1013904223u; + h2[i] = state; + } + for (int i = 0; i < 8; ++i) { + state = state * 1664525u + 1013904223u; + w1[i] = state; + state = state * 1664525u + 1013904223u; + w2[i] = state; + } + if (round == 0) { + h1[0] = INT16_MIN; + h1[1] = -1; + h1[2] = 0; + h1[3] = UINT8_MAX; + h1[4] = UINT8_MAX + 1; + h1[5] = INT16_MAX; + w1[0] = INT32_MIN; + w1[1] = -1; + w1[2] = 0; + w1[3] = UINT16_MAX; + w1[4] = UINT16_MAX + 1; + w1[5] = INT32_MAX; + } + + __m256i bh1 = _mm256_load_si256((const __m256i *)h1); + __m256i bh2 = _mm256_load_si256((const __m256i *)h2); + __m256i bw1 = _mm256_load_si256((const __m256i *)w1); + __m256i bw2 = _mm256_load_si256((const __m256i *)w2); + + _mm256_store_si256((__m256i *)got_b, + _mm256_packus_epi16(bh1, bh2)); + _mm256_store_si256((__m256i *)got_h, + _mm256_packus_epi32(bw1, bw2)); + + for (int lane = 0; lane < 2; ++lane) { + for (int i = 0; i < 8; ++i) { + if (got_b[lane * 16 + i] != sat_u8(h1[lane * 8 + i]) || + got_b[lane * 16 + 8 + i] != sat_u8(h2[lane * 8 + i])) { + exit_group(1); + } + } + for (int i = 0; i < 4; ++i) { + if (got_h[lane * 8 + i] != sat_u16(w1[lane * 4 + i]) || + got_h[lane * 8 + 4 + i] != sat_u16(w2[lane * 4 + i])) { + exit_group(2); + } + } + } + } + exit_group(0); +} diff --git a/tests/integration/vpmulhrsw-fast.c b/tests/integration/vpmulhrsw-fast.c new file mode 100644 index 0000000000..43a5fc72ec --- /dev/null +++ b/tests/integration/vpmulhrsw-fast.c @@ -0,0 +1,55 @@ +#include +#include + +static int16_t reference(int16_t a, int16_t b) +{ + int32_t value = ((int32_t)a * b + 0x4000) >> 15; + + /* PMULHRSW returns 0x8000 for the sole +32768 result. */ + return (int16_t)value; +} + +static __attribute__((noreturn)) void exit_group(int status) +{ + __asm__ volatile("syscall\n1:\n\tjmp 1b" + : : "a"(60), "D"(status) : "rcx", "r11"); + __builtin_unreachable(); +} + +void _start(void) +{ + uint32_t state = 1; + int16_t a[16] __attribute__((aligned(32))); + int16_t b[16] __attribute__((aligned(32))); + int16_t got[16] __attribute__((aligned(32))); + + for (int round = 0; round < 100000; ++round) { + for (int i = 0; i < 16; ++i) { + state = state * 1664525u + 1013904223u; + a[i] = state; + state = state * 1664525u + 1013904223u; + b[i] = state; + } + if (round == 0) { + a[0] = b[0] = INT16_MIN; + a[1] = INT16_MIN; + b[1] = INT16_MAX; + a[2] = b[2] = INT16_MAX; + a[3] = 1; + b[3] = 0x4000; + a[4] = -1; + b[4] = 0x4000; + } + __m256i va = _mm256_load_si256((const __m256i *)a); + __m256i vb = _mm256_load_si256((const __m256i *)b); + _mm256_store_si256((__m256i *)got, _mm256_mulhrs_epi16(va, vb)); + for (int i = 0; i < 16; ++i) { + int16_t want = reference(a[i], b[i]); + + if (got[i] != want) { + exit_group(1); + } + } + } + exit_group(0); +} diff --git a/tests/integration/vpshufb-fast.c b/tests/integration/vpshufb-fast.c new file mode 100644 index 0000000000..493202ee7c --- /dev/null +++ b/tests/integration/vpshufb-fast.c @@ -0,0 +1,47 @@ +#include +#include + +static __attribute__((noreturn)) void exit_group(int status) +{ + __asm__ volatile("syscall\n1:\n\tjmp 1b" + : : "a"(60), "D"(status) : "rcx", "r11"); + __builtin_unreachable(); +} + +void _start(void) +{ + uint32_t state = 1; + uint8_t source[32] __attribute__((aligned(32))); + uint8_t control[32] __attribute__((aligned(32))); + uint8_t got[32] __attribute__((aligned(32))); + + for (int round = 0; round < 100000; ++round) { + for (int i = 0; i < 32; ++i) { + state = state * 1664525u + 1013904223u; + source[i] = state; + state = state * 1664525u + 1013904223u; + control[i] = state; + } + if (round == 0) { + for (int i = 0; i < 32; ++i) { + source[i] = i; + control[i] = i * 17; + } + } + + __m256i data = _mm256_load_si256((const __m256i *)source); + __m256i index = _mm256_load_si256((const __m256i *)control); + _mm256_store_si256((__m256i *)got, _mm256_shuffle_epi8(data, index)); + + for (int i = 0; i < 32; ++i) { + int lane = i & ~15; + uint8_t want = (control[i] & 0x80) ? 0 : + source[lane + (control[i] & 0x0f)]; + + if (got[i] != want) { + exit_group(1); + } + } + } + exit_group(0); +} From 2d7322cbf5fdbf383ab8508b8f8b3aaaae3c9ea6 Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Thu, 3 Sep 2026 21:20:21 +0800 Subject: [PATCH 16/17] LATX, AVX: defer YMM high-half clears per register Signed-off-by: Lu Zeng --- target/i386/latx/translator/tr-avx.c | 21 ++++++----- target/i386/latx/translator/translate.c | 50 ++++++++++++++++++++----- 2 files changed, 52 insertions(+), 19 deletions(-) diff --git a/target/i386/latx/translator/tr-avx.c b/target/i386/latx/translator/tr-avx.c index a06f3d02fa..490afc38ec 100644 --- a/target/i386/latx/translator/tr-avx.c +++ b/target/i386/latx/translator/tr-avx.c @@ -1840,12 +1840,12 @@ bool translate_vpackusxx(IR1_INST * pir1) { la_xvori_b(temp2, src2, 0); } cvt_inst(temp2, src1, 0); - if (ir1_opnd_is_xmm(opnd0)) { - set_high128_xreg_to_zero(temp2); - } if (!dest_is_src2) { la_xvori_b(dest, temp2, 0); } + if (ir1_opnd_is_xmm(opnd0)) { + set_high128_xreg_to_zero(dest); + } return true; } @@ -2443,11 +2443,14 @@ bool translate_vpshufb(IR1_INST * pir1) { IR2_OPND src1 = load_freg256_from_ir1(opnd1); IR2_OPND src2 = load_freg256_from_ir1(opnd2); - IR2_OPND index = ra_alloc_ftemp(); IR2_OPND mask = ra_alloc_ftemp(); - la_xvandi_b(index, src2, 0xf); la_xvslti_b(mask, src2, 0); - la_xvshuf_b(dest, src1, src1, index); + /* + * XVSHUF.B ignores control bits 7:5. Bit 4 only selects between its + * two identical data operands, so the original PSHUFB control is a + * valid index. Keep the explicit sign-bit mask for x86 zeroing. + */ + la_xvshuf_b(dest, src1, src1, src2); la_xvandn_v(dest, mask, dest); if (ir1_opnd_is_xmm(opnd0)) { set_high128_xreg_to_zero(dest); @@ -4900,12 +4903,12 @@ bool translate_vpackssxx(IR1_INST * pir1) { temp = src2; } cvt_inst(temp, src1, 0); - if (ir1_opnd_is_xmm(opnd0)) { - set_high128_xreg_to_zero(temp); - } if (temp._reg_num != dest._reg_num) { la_xvori_b(dest, temp, 0); } + if (ir1_opnd_is_xmm(opnd0)) { + set_high128_xreg_to_zero(dest); + } return true; } diff --git a/target/i386/latx/translator/translate.c b/target/i386/latx/translator/translate.c index 71a38d08b2..c4c09ff006 100644 --- a/target/i386/latx/translator/translate.c +++ b/target/i386/latx/translator/translate.c @@ -1869,30 +1869,46 @@ static bool (*translate_functions[])(IR1_INST *) = { bool ir1_translate(IR1_INST *ir1) { - TRANSLATION_DATA *t = lsenv->tr_data; + bool instptn_owns_ymmh = false; + +#ifdef CONFIG_LATX_INSTS_PATTERN + instptn_owns_ymmh = ir1->instptn.opc == INSTPTN_OPC_NOP || + ir1->instptn.opc == INSTPTN_OPC_NOP_DIV; +#endif /* Keep the clear in the TB body; TU/AOT may replace generated exits. */ if (ir1_is_tb_ending(ir1)) { materialize_deferred_ymmh_zeros_now(); } - if (option_enable_lasx && !t->ymmh_zero_defer_disabled) { + if (option_enable_lasx && !instptn_owns_ymmh) { int opnd_num = ir1_get_opnd_num(ir1); - for (int i = 0; i < opnd_num; ++i) { - if (ir1_opnd_size(ir1_get_opnd(ir1, i)) == 256) { - /* - * A 256-bit operation can observe or replace a physical - * high half. Keep mixed XMM/YMM TBs conservative. - */ - disable_deferred_ymmh_zero_for_tb(); - break; + /* Materialize only YMM registers that this instruction reads. */ + for (int i = 1; i < opnd_num; ++i) { + IR1_OPND *opnd = ir1_get_opnd(ir1, i); + + if (ir1_opnd_is_ymm(opnd)) { + materialize_deferred_ymmh_zero( + ra_alloc_xmm(ir1_opnd_base_reg_num(opnd))); } } } #ifdef CONFIG_LATX_INSTS_PATTERN if (try_translate_instptn(ir1)) { + if (option_enable_lasx && !instptn_owns_ymmh) { + int opnd_num = ir1_get_opnd_num(ir1); + + for (int i = 0; i < opnd_num; ++i) { + IR1_OPND *opnd = ir1_get_opnd(ir1, i); + + if (ir1_opnd_is_ymm(opnd)) { + mark_high128_xreg_zeroed( + ra_alloc_xmm(ir1_opnd_base_reg_num(opnd))); + } + } + } ra_free_all(); return true; } @@ -1982,6 +1998,20 @@ bool ir1_translate(IR1_INST *ir1) restore_h128_of_ymm(ir1, temp); } + if (option_enable_lasx) { + int opnd_num = ir1_get_opnd_num(ir1); + + /* A translated 256-bit operand no longer carries a deferred clear. */ + for (int i = 0; i < opnd_num; ++i) { + IR1_OPND *opnd = ir1_get_opnd(ir1, i); + + if (ir1_opnd_is_ymm(opnd)) { + mark_high128_xreg_zeroed( + ra_alloc_xmm(ir1_opnd_base_reg_num(opnd))); + } + } + } + #ifdef CONFIG_LATX_DEBUG if (unlikely(latx_trace_mem)) { From b382b0fa191ed993f61efbfbcb9ad38970c95d1e Mon Sep 17 00:00:00 2001 From: Lu Zeng Date: Fri, 4 Sep 2026 09:17:01 +0800 Subject: [PATCH 17/17] LATX: fold integer and YMM reduction sequences Signed-off-by: Lu Zeng --- target/i386/latx/include/insts-pattern.h | 8 + target/i386/latx/include/translate.h | 1 + target/i386/latx/optimization/insts-pattern.c | 282 ++++++++++++++++++ .../i386/latx/optimization/ir1-optimization.c | 8 + target/i386/latx/translator/tr-opnd-process.c | 7 + target/i386/latx/translator/tr-pattern.c | 95 ++++++ target/i386/latx/translator/translate.c | 3 +- tests/integration/avx-ymm-horizontal-sum.S | 37 +++ .../integration/registrations/avx/meson.build | 2 + tests/integration/scalar-integer-patterns.S | 137 +++++++++ 10 files changed, 579 insertions(+), 1 deletion(-) create mode 100644 tests/integration/avx-ymm-horizontal-sum.S create mode 100644 tests/integration/scalar-integer-patterns.S diff --git a/target/i386/latx/include/insts-pattern.h b/target/i386/latx/include/insts-pattern.h index 2c67f3d34a..ef9846f89c 100644 --- a/target/i386/latx/include/insts-pattern.h +++ b/target/i386/latx/include/insts-pattern.h @@ -54,10 +54,18 @@ #define INSTPTN_OPC_REPEAT_ADD 0x40000000 #define INSTPTN_OPC_AVX_SUM3 (-0x7fffffff - 1) #define INSTPTN_OPC_SCALAR_HDR (-0x7fffffff) +#define INSTPTN_OPC_BSR_NOT_ADD (-0x7ffffffe) +#define INSTPTN_OPC_CLAMP_U8 (-0x7ffffffd) +#define INSTPTN_OPC_DIFF_CMOV_U16 (-0x7ffffffc) +#define INSTPTN_OPC_YMM_HSUMQ (-0x7ffffffb) void insts_pattern_repeat_add(TranslationBlock *tb); void insts_pattern_avx_sum3(TranslationBlock *tb); void insts_pattern_scalar_hdr(TranslationBlock *tb); +void insts_pattern_bsr_not_add(TranslationBlock *tb); +void insts_pattern_clamp_u8(TranslationBlock *tb); +void insts_pattern_diff_cmov_u16(TranslationBlock *tb); +void insts_pattern_ymm_hsumq(TranslationBlock *tb); typedef int scan_elem_t; diff --git a/target/i386/latx/include/translate.h b/target/i386/latx/include/translate.h index 7c8e1f693b..140c90cad8 100644 --- a/target/i386/latx/include/translate.h +++ b/target/i386/latx/include/translate.h @@ -1607,6 +1607,7 @@ IR2_OPND load_freg128_from_ir1(IR1_OPND *); void load_imm_to_ir1_opnd_gpr(IR1_OPND *opnd0, uint64_t imm); IR2_OPND load_freg256_from_ir1(IR1_OPND *opnd1); void set_high128_xreg_to_zero(IR2_OPND opnd); +void set_all_high128_xregs_to_zero(void); void mark_high128_xreg_zeroed(IR2_OPND opnd); void materialize_deferred_ymmh_zero(IR2_OPND opnd); void materialize_deferred_ymmh_zeros_now(void); diff --git a/target/i386/latx/optimization/insts-pattern.c b/target/i386/latx/optimization/insts-pattern.c index 2106357d9c..880bc43c2c 100644 --- a/target/i386/latx/optimization/insts-pattern.c +++ b/target/i386/latx/optimization/insts-pattern.c @@ -412,6 +412,225 @@ void insts_pattern_scan_con(TranslationBlock *tb, IR1_INST *ir1, int index, scan } } +void insts_pattern_bsr_not_add(TranslationBlock *tb) +{ + for (int i = 0; i + 2 < tb_ir1_num(tb); ++i) { + IR1_INST *bsr = tb_ir1_inst(tb, i); + IR1_INST *not = tb_ir1_inst(tb, i + 1); + IR1_INST *add = tb_ir1_inst(tb, i + 2); + IR1_OPND *dest; + + if (ir1_opcode(bsr) != WRAP(BSR) || + ir1_opcode(not) != WRAP(NOT) || + ir1_opcode(add) != WRAP(ADD) || + bsr->instptn.opc != INSTPTN_OPC_NONE || + not->instptn.opc != INSTPTN_OPC_NONE || + add->instptn.opc != INSTPTN_OPC_NONE || + bsr->info->bytes[0] == 0xf3 || + ir1_get_eflag_def(bsr) != 0 || + ir1_get_eflag_def(add) != 0) { + continue; + } + + dest = ir1_get_opnd(bsr, 0); + if (!ir1_opnd_is_gpr(dest) || ir1_opnd_size(dest) != 32 || + !ir1_opnd_is_same_reg(dest, ir1_get_opnd(not, 0)) || + !ir1_opnd_is_same_reg(dest, ir1_get_opnd(add, 0)) || + !ir1_opnd_is_imm(ir1_get_opnd(add, 1)) || + ir1_opnd_simm(ir1_get_opnd(add, 1)) != 15) { + continue; + } + + bsr->instptn.opc = INSTPTN_OPC_BSR_NOT_ADD; + bsr->instptn.next = add; + not->instptn.opc = INSTPTN_OPC_NOP; + add->instptn.opc = INSTPTN_OPC_NOP; + i += 2; + } +} + +void insts_pattern_clamp_u8(TranslationBlock *tb) +{ + int count = tb_ir1_num(tb); + + for (int i = 0; i + 8 < count; ++i) { + IR1_INST *mov_limit = tb_ir1_inst(tb, i); + IR1_INST *sub = tb_ir1_inst(tb, i + 1); + IR1_INST *shr = tb_ir1_inst(tb, i + 2); + IR1_INST *xor = tb_ir1_inst(tb, i + 3); + IR1_INST *test = tb_ir1_inst(tb, i + 4); + IR1_INST *sete = tb_ir1_inst(tb, i + 5); + IR1_INST *or_value = tb_ir1_inst(tb, i + 6); + IR1_INST *or_zero = tb_ir1_inst(tb, i + 7); + IR1_INST *movzx = tb_ir1_inst(tb, i + 8); + IR1_OPND *limit; + IR1_OPND *value; + IR1_OPND *zero; + if (ir1_opcode(mov_limit) != WRAP(MOV) || + ir1_opcode(sub) != WRAP(SUB) || + ir1_opcode(shr) != WRAP(SHR) || + ir1_opcode(xor) != WRAP(XOR) || + ir1_opcode(test) != WRAP(TEST) || + ir1_opcode(sete) != WRAP(SETE) || + ir1_opcode(or_value) != WRAP(OR) || + ir1_opcode(or_zero) != WRAP(OR) || + ir1_opcode(movzx) != WRAP(MOVZX) || + ir1_get_opnd_num(mov_limit) != 2 || + ir1_get_opnd_num(sub) != 2 || + ir1_get_opnd_num(shr) != 2 || + ir1_get_opnd_num(xor) != 2 || + ir1_get_opnd_num(test) != 2 || + ir1_get_opnd_num(sete) != 1 || + ir1_get_opnd_num(or_value) != 2 || + ir1_get_opnd_num(or_zero) != 2 || + ir1_get_opnd_num(movzx) != 2 || + ir1_get_eflag_def(or_zero) != 0) { + continue; + } + + limit = ir1_get_opnd(mov_limit, 0); + value = ir1_get_opnd(sub, 1); + zero = ir1_get_opnd(xor, 0); + if (!ir1_opnd_is_gpr(limit) || ir1_opnd_size(limit) != 32 || + !ir1_opnd_is_imm(ir1_get_opnd(mov_limit, 1)) || + ir1_opnd_uimm(ir1_get_opnd(mov_limit, 1)) != 255 || + !ir1_opnd_is_same_reg(limit, ir1_get_opnd(sub, 0)) || + !ir1_opnd_is_gpr(value) || ir1_opnd_size(value) != 32 || + !ir1_opnd_is_same_reg(limit, ir1_get_opnd(shr, 0)) || + !ir1_opnd_is_imm(ir1_get_opnd(shr, 1)) || + ir1_opnd_uimm(ir1_get_opnd(shr, 1)) != 23 || + !ir1_opnd_is_gpr(zero) || ir1_opnd_size(zero) != 32 || + ir1_opnd_base_reg_num(limit) == ir1_opnd_base_reg_num(value) || + ir1_opnd_base_reg_num(limit) == ir1_opnd_base_reg_num(zero) || + ir1_opnd_base_reg_num(value) == ir1_opnd_base_reg_num(zero) || + !ir1_opnd_is_same_reg(zero, ir1_get_opnd(xor, 1)) || + !ir1_opnd_is_same_reg_without_width(value, + ir1_get_opnd(test, 0)) || + !ir1_opnd_is_same_reg_without_width(value, + ir1_get_opnd(test, 1)) || + ir1_opnd_size(ir1_get_opnd(test, 0)) != 64 || + !ir1_opnd_is_same_reg_without_width(zero, + ir1_get_opnd(sete, 0)) || + ir1_opnd_size(ir1_get_opnd(sete, 0)) != 8 || + !ir1_opnd_is_same_reg(limit, ir1_get_opnd(or_value, 0)) || + !ir1_opnd_is_same_reg(value, ir1_get_opnd(or_value, 1)) || + !ir1_opnd_is_same_reg(limit, ir1_get_opnd(or_zero, 0)) || + !ir1_opnd_is_same_reg(zero, ir1_get_opnd(or_zero, 1)) || + !ir1_opnd_is_gpr(ir1_get_opnd(movzx, 0)) || + ir1_opnd_size(ir1_get_opnd(movzx, 0)) != 32 || + !ir1_opnd_is_same_reg_without_width(limit, + ir1_get_opnd(movzx, 1)) || + ir1_opnd_size(ir1_get_opnd(movzx, 1)) != 8) { + continue; + } + + mov_limit->instptn.opc = INSTPTN_OPC_CLAMP_U8; + for (int j = 1; j < 9; ++j) { + tb_ir1_inst(tb, i + j)->instptn.opc = INSTPTN_OPC_NOP; + } + i += 8; + } +} + +void insts_pattern_diff_cmov_u16(TranslationBlock *tb) +{ + int count = tb_ir1_num(tb); + + for (int i = 0; i + 8 < count; ++i) { + IR1_INST *load = tb_ir1_inst(tb, i); + IR1_INST *mov_imm = tb_ir1_inst(tb, i + 1); + IR1_INST *sub_input = tb_ir1_inst(tb, i + 2); + IR1_INST *mov_diff = tb_ir1_inst(tb, i + 3); + IR1_INST *sub_prev = tb_ir1_inst(tb, i + 4); + IR1_INST *cmp = tb_ir1_inst(tb, i + 5); + IR1_INST *cmov = tb_ir1_inst(tb, i + 6); + IR1_INST *movzx = tb_ir1_inst(tb, i + 7); + IR1_INST *next_cmp = tb_ir1_inst(tb, i + 8); + IR1_OPND *input; + IR1_OPND *diff; + IR1_OPND *value; + IR1_OPND *previous; + IR1_OPND *alternate; + int regs[5]; + bool distinct = true; + + if (ir1_opcode(load) != WRAP(MOVZX) || + ir1_opcode(mov_imm) != WRAP(MOV) || + ir1_opcode(sub_input) != WRAP(SUB) || + ir1_opcode(mov_diff) != WRAP(MOV) || + ir1_opcode(sub_prev) != WRAP(SUB) || + ir1_opcode(cmp) != WRAP(CMP) || + ir1_opcode(cmov) != WRAP(CMOVB) || + ir1_opcode(movzx) != WRAP(MOVZX) || + ir1_opcode(next_cmp) != WRAP(CMP) || + ir1_get_opnd_num(load) != 2 || + ir1_get_opnd_num(mov_imm) != 2 || + ir1_get_opnd_num(sub_input) != 2 || + ir1_get_opnd_num(mov_diff) != 2 || + ir1_get_opnd_num(sub_prev) != 2 || + ir1_get_opnd_num(cmp) != 2 || + ir1_get_opnd_num(cmov) != 2 || + ir1_get_opnd_num(movzx) != 2 || + ir1_get_opnd_num(next_cmp) != 2) { + continue; + } + + input = ir1_get_opnd(load, 0); + diff = ir1_get_opnd(mov_imm, 0); + value = ir1_get_opnd(mov_diff, 0); + previous = ir1_get_opnd(sub_prev, 1); + alternate = ir1_get_opnd(cmov, 1); + if (!ir1_opnd_is_gpr(input) || ir1_opnd_size(input) != 32 || + !ir1_opnd_is_mem(ir1_get_opnd(load, 1)) || + ir1_opnd_size(ir1_get_opnd(load, 1)) != 16 || + !ir1_opnd_is_gpr(diff) || ir1_opnd_size(diff) != 32 || + !ir1_opnd_is_imm(ir1_get_opnd(mov_imm, 1)) || + (uint32_t)ir1_opnd_uimm(ir1_get_opnd(mov_imm, 1)) != + UINT32_C(0xffff8000) || + !ir1_opnd_is_same_reg(diff, ir1_get_opnd(sub_input, 0)) || + !ir1_opnd_is_same_reg(input, ir1_get_opnd(sub_input, 1)) || + !ir1_opnd_is_gpr(value) || ir1_opnd_size(value) != 32 || + !ir1_opnd_is_same_reg(diff, ir1_get_opnd(mov_diff, 1)) || + !ir1_opnd_is_same_reg(value, ir1_get_opnd(sub_prev, 0)) || + !ir1_opnd_is_gpr(previous) || ir1_opnd_size(previous) != 32 || + !ir1_opnd_is_same_reg_without_width(value, + ir1_get_opnd(cmp, 0)) || + ir1_opnd_size(ir1_get_opnd(cmp, 0)) != 16 || + !ir1_opnd_is_imm(ir1_get_opnd(cmp, 1)) || + ir1_opnd_uimm(ir1_get_opnd(cmp, 1)) != 5 || + !ir1_opnd_is_same_reg(value, ir1_get_opnd(cmov, 0)) || + !ir1_opnd_is_gpr(alternate) || ir1_opnd_size(alternate) != 32 || + !ir1_opnd_is_same_reg(previous, ir1_get_opnd(movzx, 0)) || + !ir1_opnd_is_same_reg_without_width(value, + ir1_get_opnd(movzx, 1)) || + ir1_opnd_size(ir1_get_opnd(movzx, 1)) != 16 || + !ir1_opnd_is_same_reg(previous, ir1_get_opnd(next_cmp, 0)) || + !ir1_opnd_is_imm(ir1_get_opnd(next_cmp, 1)) || + ir1_opnd_uimm(ir1_get_opnd(next_cmp, 1)) != 0x7fff) { + continue; + } + + regs[0] = ir1_opnd_base_reg_num(input); + regs[1] = ir1_opnd_base_reg_num(diff); + regs[2] = ir1_opnd_base_reg_num(value); + regs[3] = ir1_opnd_base_reg_num(previous); + regs[4] = ir1_opnd_base_reg_num(alternate); + for (int a = 0; a < 5; ++a) { + for (int b = a + 1; b < 5; ++b) { + distinct &= regs[a] != regs[b]; + } + } + if (!distinct) { + continue; + } + load->instptn.opc = INSTPTN_OPC_DIFF_CMOV_U16; + for (int j = 1; j < 8; ++j) { + tb_ir1_inst(tb, i + j)->instptn.opc = INSTPTN_OPC_NOP; + } + i += 7; + } +} + bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *pir1, int pir1_index, scan_elem_t *scan) { @@ -812,6 +1031,69 @@ static bool avx_sum3_imm(IR1_INST *ir1, int opnd_index, ulongx value) ir1_opnd_uimm(ir1_get_opnd(ir1, opnd_index)) == value; } +static int ymm_hsumq_reg(IR1_INST *ir1, int opnd_index, bool ymm) +{ + IR1_OPND *opnd; + + if (opnd_index >= ir1_get_opnd_num(ir1)) { + return -1; + } + opnd = ir1_get_opnd(ir1, opnd_index); + if (ymm ? !ir1_opnd_is_ymm(opnd) : !ir1_opnd_is_xmm(opnd)) { + return -1; + } + return ir1_opnd_base_reg_num(opnd); +} + +void insts_pattern_ymm_hsumq(TranslationBlock *tb) +{ + int count = tb_ir1_num(tb); + + for (int i = 0; i + 6 <= count; ++i) { + IR1_INST *shift = tb_ir1_inst(tb, i); + IR1_INST *add256 = tb_ir1_inst(tb, i + 1); + IR1_INST *extract = tb_ir1_inst(tb, i + 2); + IR1_INST *add128 = tb_ir1_inst(tb, i + 3); + IR1_INST *move = tb_ir1_inst(tb, i + 4); + IR1_INST *zero = tb_ir1_inst(tb, i + 5); + int src = ymm_hsumq_reg(shift, 1, true); + int temp = ymm_hsumq_reg(shift, 0, true); + bool match = src >= 0 && temp >= 0 && src != temp && + ir1_opcode(shift) == WRAP(VPSRLDQ) && + avx_sum3_imm(shift, 2, 8) && + ir1_opcode(add256) == WRAP(VPADDQ) && + ymm_hsumq_reg(add256, 0, true) == src && + ymm_hsumq_reg(add256, 1, true) == src && + ymm_hsumq_reg(add256, 2, true) == temp && + ir1_opcode(extract) == WRAP(VEXTRACTI128) && + ymm_hsumq_reg(extract, 0, false) == temp && + ymm_hsumq_reg(extract, 1, true) == src && + avx_sum3_imm(extract, 2, 1) && + ir1_opcode(add128) == WRAP(VPADDQ) && + ymm_hsumq_reg(add128, 0, false) == src && + ymm_hsumq_reg(add128, 1, false) == src && + ymm_hsumq_reg(add128, 2, false) == temp && + ir1_opcode(move) == WRAP(VMOVQ) && + ir1_get_opnd_num(move) == 2 && + ir1_opnd_is_gpr(ir1_get_opnd(move, 0)) && + ymm_hsumq_reg(move, 1, false) == src && + ir1_opcode(zero) == WRAP(VZEROUPPER); + + for (int j = 0; match && j < 6; ++j) { + match = tb_ir1_inst(tb, i + j)->instptn.opc == + INSTPTN_OPC_NONE; + } + if (!match) { + continue; + } + shift->instptn.opc = INSTPTN_OPC_YMM_HSUMQ; + for (int j = 1; j < 6; ++j) { + tb_ir1_inst(tb, i + j)->instptn.opc = INSTPTN_OPC_NOP; + } + i += 5; + } +} + static bool avx_sum3_match_group(TranslationBlock *tb, int pos, int *product, int *zero) { diff --git a/target/i386/latx/optimization/ir1-optimization.c b/target/i386/latx/optimization/ir1-optimization.c index 6242afcd0b..1265f96714 100644 --- a/target/i386/latx/optimization/ir1-optimization.c +++ b/target/i386/latx/optimization/ir1-optimization.c @@ -55,9 +55,13 @@ static void ir1_optimization_over_tb(TranslationBlock *tb) SAVE_FLAG_TO_TB(rdtn, tb); #ifdef CONFIG_LATX_INSTS_PATTERN if (option_instptn) { + insts_pattern_ymm_hsumq(tb); insts_pattern_avx_sum3(tb); insts_pattern_repeat_add(tb); insts_pattern_scalar_hdr(tb); + insts_pattern_bsr_not_add(tb); + insts_pattern_clamp_u8(tb); + insts_pattern_diff_cmov_u16(tb); } #endif } @@ -176,9 +180,13 @@ void ir1_optimization(TranslationBlock *tb) SAVE_FLAG_TO_TB(rdtn, tb); #ifdef CONFIG_LATX_INSTS_PATTERN if (option_instptn) { + insts_pattern_ymm_hsumq(tb); insts_pattern_avx_sum3(tb); insts_pattern_repeat_add(tb); insts_pattern_scalar_hdr(tb); + insts_pattern_bsr_not_add(tb); + insts_pattern_clamp_u8(tb); + insts_pattern_diff_cmov_u16(tb); } #endif } diff --git a/target/i386/latx/translator/tr-opnd-process.c b/target/i386/latx/translator/tr-opnd-process.c index 2006abae43..2a662abef4 100644 --- a/target/i386/latx/translator/tr-opnd-process.c +++ b/target/i386/latx/translator/tr-opnd-process.c @@ -1512,6 +1512,13 @@ void set_high128_xreg_to_zero(IR2_OPND opnd) clear_high128_xreg_now(opnd); } +void set_all_high128_xregs_to_zero(void) +{ + if (option_enable_lasx) { + lsenv->tr_data->ymmh_zero_pending = UINT16_MAX; + } +} + static IR2_OPND get_ymm_high128_shadow_addr(int index) { #ifdef TARGET_X86_64 diff --git a/target/i386/latx/translator/tr-pattern.c b/target/i386/latx/translator/tr-pattern.c index 7dd29d5c04..4829e5a155 100644 --- a/target/i386/latx/translator/tr-pattern.c +++ b/target/i386/latx/translator/tr-pattern.c @@ -2442,6 +2442,26 @@ static bool translate_avx_sum3(IR1_INST *pir1) return true; } +static bool translate_ymm_hsumq(IR1_INST *pir1) +{ + IR1_INST *move = pir1 + 4; + IR2_OPND temp = ra_alloc_xmm( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0))); + IR2_OPND src = ra_alloc_xmm( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 1))); + IR2_OPND dest = ra_alloc_gpr( + ir1_opnd_base_reg_num(ir1_get_opnd(move, 0))); + + materialize_deferred_ymmh_zero(src); + la_xvbsrl_v(temp, src, 8); + la_xvadd_d(src, src, temp); + la_xvpermi_q(temp, src, VEXTRINS_IMM_4_0(3, 1)); + la_vadd_d(src, src, temp); + la_vpickve2gr_du(dest, src, 0); + set_all_high128_xregs_to_zero(); + return true; +} + static bool translate_repeat_add(IR1_INST *pir1) { IR2_OPND dest = load_ireg_from_ir1(ir1_get_opnd(pir1, 0), @@ -2461,6 +2481,73 @@ static bool translate_repeat_add(IR1_INST *pir1) return true; } +static bool translate_bsr_not_add(IR1_INST *pir1) +{ + IR2_OPND src = load_ireg_from_ir1(ir1_get_opnd(pir1, 1), + ZERO_EXTENSION, false); + IR2_OPND dest = ra_alloc_gpr( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0))); + + /* BSR leaves its destination undefined for a zero source. */ + la_clz_d(dest, src); + la_addi_d(dest, dest, -49); + la_bstrpick_d(dest, dest, 31, 0); + return true; +} + +static bool translate_clamp_u8(IR1_INST *pir1) +{ + IR2_OPND limit = ra_alloc_gpr( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0))); + IR2_OPND value = ra_alloc_gpr( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1 + 1, 1))); + IR2_OPND zero = ra_alloc_gpr( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1 + 3, 0))); + IR2_OPND output = ra_alloc_gpr( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1 + 8, 0))); + + li_wu(limit, 255); + la_sub_w(limit, limit, value); + la_srli_w(limit, limit, 23); + la_sltui(zero, value, 1); + la_or(limit, limit, value); + la_or(limit, limit, zero); + la_andi(output, limit, 255); + return true; +} + +static bool translate_diff_cmov_u16(IR1_INST *pir1) +{ + IR1_OPND *load_mem = ir1_get_opnd(pir1, 1); + IR2_OPND input = ra_alloc_gpr( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1, 0))); + IR2_OPND diff = ra_alloc_gpr( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1 + 1, 0))); + IR2_OPND value = ra_alloc_gpr( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1 + 3, 0))); + IR2_OPND previous = ra_alloc_gpr( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1 + 4, 1))); + IR2_OPND alternate = ra_alloc_gpr( + ir1_opnd_base_reg_num(ir1_get_opnd(pir1 + 6, 1))); + IR2_OPND condition = ra_alloc_itemp(); + IR2_OPND keep = ra_alloc_itemp(); + + load_ireg_from_ir1_mem(input, load_mem, ZERO_EXTENSION, false); + li_wu(diff, UINT32_C(0x8000)); + la_sub_d(diff, diff, input); + la_sub_d(value, diff, previous); + la_bstrpick_d(value, value, 15, 0); + la_sltui(condition, value, 5); + la_masknez(keep, value, condition); + la_maskeqz(condition, alternate, condition); + la_or(value, keep, condition); + la_bstrpick_d(previous, value, 15, 0); + + ra_free_temp(keep); + ra_free_temp(condition); + return true; +} + static bool translate_scalar_hdr(IR1_INST *pir1) { IR1_OPND *input_mem = ir1_get_opnd(pir1, 1); @@ -2662,8 +2749,16 @@ bool try_translate_instptn(IR1_INST *pir1) return translate_repeat_add(pir1); case INSTPTN_OPC_AVX_SUM3: return translate_avx_sum3(pir1); + case INSTPTN_OPC_YMM_HSUMQ: + return translate_ymm_hsumq(pir1); case INSTPTN_OPC_SCALAR_HDR: return translate_scalar_hdr(pir1); + case INSTPTN_OPC_BSR_NOT_ADD: + return translate_bsr_not_add(pir1); + case INSTPTN_OPC_CLAMP_U8: + return translate_clamp_u8(pir1); + case INSTPTN_OPC_DIFF_CMOV_U16: + return translate_diff_cmov_u16(pir1); default: lsassert(0); break; diff --git a/target/i386/latx/translator/translate.c b/target/i386/latx/translator/translate.c index c4c09ff006..e775fb915a 100644 --- a/target/i386/latx/translator/translate.c +++ b/target/i386/latx/translator/translate.c @@ -1872,7 +1872,8 @@ bool ir1_translate(IR1_INST *ir1) bool instptn_owns_ymmh = false; #ifdef CONFIG_LATX_INSTS_PATTERN - instptn_owns_ymmh = ir1->instptn.opc == INSTPTN_OPC_NOP || + instptn_owns_ymmh = ir1->instptn.opc == INSTPTN_OPC_YMM_HSUMQ || + ir1->instptn.opc == INSTPTN_OPC_NOP || ir1->instptn.opc == INSTPTN_OPC_NOP_DIV; #endif diff --git a/tests/integration/avx-ymm-horizontal-sum.S b/tests/integration/avx-ymm-horizontal-sum.S new file mode 100644 index 0000000000..e6ec6dd138 --- /dev/null +++ b/tests/integration/avx-ymm-horizontal-sum.S @@ -0,0 +1,37 @@ +.intel_syntax noprefix +.section .text +.global _start + +_start: + vmovdqu ymm0, YMMWORD PTR [rip + input] + vmovdqu ymm2, YMMWORD PTR [rip + nonzero] + + vpsrldq ymm1, ymm0, 8 + vpaddq ymm0, ymm0, ymm1 + vextracti128 xmm1, ymm0, 1 + vpaddq xmm0, xmm0, xmm1 + vmovq rax, xmm0 + vzeroupper + + cmp rax, 10 + jne .Lfail + vextracti128 xmm3, ymm2, 1 + vptest xmm3, xmm3 + jne .Lfail + + xor edi, edi + jmp .Lexit +.Lfail: + mov edi, 1 +.Lexit: + mov eax, 60 + syscall + +.section .rodata +.balign 32 +input: + .quad 1, 2, 3, 4 +nonzero: + .quad -1, -1, -1, -1 + +.section .note.GNU-stack,"",@progbits diff --git a/tests/integration/registrations/avx/meson.build b/tests/integration/registrations/avx/meson.build index de68fa1d9d..0099e73123 100644 --- a/tests/integration/registrations/avx/meson.build +++ b/tests/integration/registrations/avx/meson.build @@ -27,6 +27,8 @@ if 'x86_64-linux-user' in target_dirs files('../../avx-sum3-pattern.S'), files('../../repeat-add.S'), files('../../vcomis-jcc.S'), + files('../../avx-ymm-horizontal-sum.S'), + files('../../scalar-integer-patterns.S'), ], 'timeout': 120, }] diff --git a/tests/integration/scalar-integer-patterns.S b/tests/integration/scalar-integer-patterns.S new file mode 100644 index 0000000000..8a832791c1 --- /dev/null +++ b/tests/integration/scalar-integer-patterns.S @@ -0,0 +1,137 @@ +.intel_syntax noprefix +.section .text +.global _start + +_start: + lea rdi, [rip + input_zero] + xor esi, esi + mov edx, 0x12345678 + call diff_cmov_u16 + cmp eax, 0x8000 + jne .Lfail + + lea rdi, [rip + input_7fff] + xor esi, esi + mov edx, 0x12345678 + call diff_cmov_u16 + cmp eax, 0x5678 + jne .Lfail + + lea rdi, [rip + input_one] + mov esi, 0x7ffc + mov edx, 0xffff0001 + call diff_cmov_u16 + cmp eax, 1 + jne .Lfail + + mov r11d, 1 + call bsr_not_add + cmp eax, 14 + jne .Lfail + mov r11d, 0x8000 + call bsr_not_add + cmp eax, -1 + jne .Lfail + mov r11d, 0x80000000 + call bsr_not_add + cmp eax, -17 + jne .Lfail + + xor edi, edi + call clamp_u8 + cmp eax, 1 + jne .Lfail + mov edi, 1 + call clamp_u8 + cmp eax, 1 + jne .Lfail + mov edi, 254 + call clamp_u8 + cmp eax, 254 + jne .Lfail + mov edi, 255 + call clamp_u8 + cmp eax, 255 + jne .Lfail + mov edi, 256 + call clamp_u8 + cmp eax, 255 + jne .Lfail + + /* The matcher rejects this source/scratch alias. */ + mov edi, 123 + call clamp_u8_alias + cmp eax, 1 + jne .Lfail + + xor edi, edi + jmp .Lexit +.Lfail: + mov edi, 1 +.Lexit: + mov eax, 60 + syscall + +.type diff_cmov_u16, @function +diff_cmov_u16: + mov r11d, esi + movzx r9d, WORD PTR [rdi] + mov r10d, 0xffff8000 + sub r10d, r9d + mov ecx, r10d + sub ecx, r11d + cmp cx, 5 + cmovb ecx, edx + movzx r11d, cx + cmp r11d, 0x7fff + mov eax, r11d + ret +.size diff_cmov_u16, .-diff_cmov_u16 + +.type bsr_not_add, @function +bsr_not_add: + bsr ecx, r11d + not ecx + add ecx, 15 + cmp ecx, ecx + mov eax, ecx + ret +.size bsr_not_add, .-bsr_not_add + +.type clamp_u8, @function +clamp_u8: + mov eax, 255 + sub eax, edi + shr eax, 23 + xor ecx, ecx + test rdi, rdi + sete cl + or eax, edi + or eax, ecx + movzx eax, al + cmp eax, eax + ret +.size clamp_u8, .-clamp_u8 + +.type clamp_u8_alias, @function +clamp_u8_alias: + mov eax, 255 + sub eax, edi + shr eax, 23 + xor edi, edi + test rdi, rdi + sete dil + or eax, edi + or eax, edi + movzx eax, al + cmp eax, eax + ret +.size clamp_u8_alias, .-clamp_u8_alias + +.section .rodata +.balign 2 +input_zero: .word 0 +input_one: .word 1 +input_7fff: .word 0x7fff + +.section .note.GNU-stack,"",@progbits