Vector registers in thread contexts; stop JIT pool code-buffer leaks (Persona 5 Royal reaches gameplay)

build/ntdll-unix/virtual_ios.c

@@ -3797,6 +3797,20 @@ static int ios_pool_preserve_code_carve( size_t available, size_t want, int low_
     return low_fits && want <= 0x10000 && available >= 0x100000 && available / 4 >= want;
 }
 
+/* Nor should any request take a free carve four or more times its size while the pool's
+ * budget still affords a fresh carve of the size asked: the carve stays with its taker for
+ * that process's life. Persona 5 Royal's crashpad_handler.exe took a 16 MB carve for its
+ * 16 KB dispatcher and a 128 MB one for a 16 MB code buffer, and the game itself was then
+ * capped at 16 MB. When the fresh carve fails anyway, the allocator retries with every
+ * carve allowed (reuse_all_carves). MADEIRA_POOL_CODE_KEEP_BIG=0 restores the old policy. */
+static int ios_pool_preserve_big_carve( size_t available, size_t want, size_t budget_cap )
+{
+    const char *env = getenv( "MADEIRA_POOL_CODE_KEEP_BIG" );   /* 0: a code buffer request takes the smallest free buffer that fits, however much larger, as before */
+
+    if (env && !strcmp( env, "0" )) return 0;
+    return want && available / 4 >= want && want <= budget_cap;
+}
+
 /* ml557 (#74 REGRESSION): is any thread's PC currently INSIDE this tail carve?
  *
  * ml438 made a MEM_RELEASE of a live tail EC-buffer carve mark it free for reuse,
@@ -33982,7 +33996,7 @@ NTSTATUS WINAPI NtAllocateVirtualMemoryEx( HANDLE process, PVOID *ret, SIZE_T *s
         int low_fits = low_ok && alloc_size <= IOS_POOL_LOW_CARVE_MAX &&
                        alloc_size <= ios_jit_low_size_global &&
                        ios_jit_low_reserved <= ios_jit_low_size_global - alloc_size;
-        int kept_large_carve = 0, reuse_all_carves = 0, retired_busy = 0;
+        int kept_large_carve = 0, kept_big_carve = 0, reuse_all_carves = 0, retired_busy = 0;
         size_t tail_cap = ~(size_t)0, budget_cap = ~(size_t)0;
         void *carve_peb = ios_jit_current_peb();   /* madeira-bcd: the carve's owner, see ios_tail_carve_ready */
         /* ml459 (#75): cap a single EC code buffer at 16MB. FEX asks for 32MB
@@ -34082,6 +34096,12 @@ NTSTATUS WINAPI NtAllocateVirtualMemoryEx( HANDLE process, PVOID *ret, SIZE_T *s
                 kept_large_carve = 1;
                 best = ~0u;
             }
+            else if (best != ~0u && !reuse_all_carves &&
+                     ios_pool_preserve_big_carve( ios_tail_carves[best].size, alloc_size, budget_cap ))
+            {
+                kept_big_carve = 1;
+                best = ~0u;
+            }
             /* madeira-bcd: a dead process's buffer, only when no thread runs in it;
              * a busy one starts its grace again and the next one is tried */
             if (best != ~0u && ios_tail_carves[best].retired &&
@@ -34288,6 +34308,12 @@ NTSTATUS WINAPI NtAllocateVirtualMemoryEx( HANDLE process, PVOID *ret, SIZE_T *s
              * tail reservation — the old path leaked it on every refusal. */
             ios_pool_tail_unreserve( &ios_jit_tail_reserved, reserve_offset + alloc_size, tail_added,
                                      ios_jit_hole_end > ios_jit_hole_off );
+            /* a free carve was kept back for a bigger request: use it after all */
+            if ((kept_large_carve || kept_big_carve) && !reuse_all_carves)
+            {
+                reuse_all_carves = 1;
+                goto retry_code_carve_reuse;
+            }
             ERR("NtAllocateVirtualMemoryEx iOS: JIT-pool tail exhausted for FEX EC_CODE %zu bytes\n",
                 (size_t)*size_ptr);
             /* ml459 (#75): dump the carve table on the FIRST refusal — the
@@ -35194,6 +35220,8 @@ NTSTATUS WINAPI NtFreeVirtualMemory( HANDLE process, PVOID *addr_ptr, SIZE_T *si
         size_t off = (size_t)((char *)base - (char *)ios_jit_rx_base_global);
         size_t carve_size = 0;
         void *caller_peb = ios_jit_current_peb();
+        const char *defer_env = getenv( "MADEIRA_POOL_CODE_DEFER_FREE" );   /* 0: a FEX code buffer released while a thread runs in it is refused and stays live, as before */
+        int defer_free = !(defer_env && !strcmp( defer_env, "0" ));
         pthread_mutex_lock( &ios_tail_carve_lock );
         for (i = 0; i < ios_tail_carve_n; i++)
         {
@@ -35210,16 +35238,30 @@ NTSTATUS WINAPI NtFreeVirtualMemory( HANDLE process, PVOID *addr_ptr, SIZE_T *si
                 carve_size = 0;
                 break;
             }
-            /* ml557: do NOT recycle a carve a thread is still executing in. */
+            /* ml557: do NOT recycle a carve a thread is still executing in. Refusing the
+             * release leaked it for good, though: FEX releases a buffer once and never
+             * again, so "freeable on the next release" never came. Persona 5 Royal lost a
+             * 128 MB buffer that way and ran its last 4 minutes on 16 MB ones. Free it as
+             * retired instead, so reuse waits out the grace and its strict scan, exactly
+             * as for a dead process's buffer. MADEIRA_POOL_CODE_DEFER_FREE=0 refuses it. */
             if (ios_tail_carve_occupied( base, ios_tail_carves[i].size ))
             {
                 static unsigned long occ_n;
+                if (!defer_free)
+                {
+                    if (++occ_n <= 16 || (occ_n % 256) == 0)
+                        dprintf(2, "[jit-pool] tail FREE REFUSED rx=%p size=0x%lx — a thread's "
+                                   "PC is inside it (occupied=%lu) rev=ml557\n",
+                                base, (unsigned long)ios_tail_carves[i].size, occ_n);
+                    carve_size = 0;
+                    break;
+                }
                 if (++occ_n <= 16 || (occ_n % 256) == 0)
-                    dprintf(2, "[jit-pool] tail FREE REFUSED rx=%p size=0x%lx — a thread's "
-                               "PC is inside it (occupied=%lu) rev=ml557\n",
+                    dprintf(2, "[jit-pool] tail FREE deferred rx=%p size=0x%lx — a thread's "
+                               "PC is inside it (occupied=%lu): retired until it leaves rev=ml557\n",
                             base, (unsigned long)ios_tail_carves[i].size, occ_n);
-                carve_size = 0;
-                break;
+                ios_tail_carves[i].retired = 1;
+                ios_tail_carves[i].peb = NULL;
             }
             ios_tail_carves[i].free = 1;
             ios_tail_carves[i].freed_at = time( NULL );

build/wineserver/mach_ios.c

@@ -443,6 +443,9 @@ int send_thread_signal( struct thread *thread, int sig )
 #define IOS_SCF_SP    0x0f8
 #define IOS_SCF_PC    0x100
 #define IOS_SCF_CPSR  0x108
+#define IOS_SCF_FPCR  0x128
+#define IOS_SCF_FPSR  0x12c
+#define IOS_SCF_V     0x130                    /* NEON128 v[32] */
 #define IOS_SCF_SIZE  0x330
 #define IOS_SCF_RESTORE_FLAGS 0x10c           /* syscall_frame.restore_flags */
 #define IOS_CTX_ARM64_CONTROL 0x00400001u     /* CONTEXT_ARM64 | CONTEXT_ARM64_CONTROL */
@@ -876,6 +879,17 @@ int ios_apply_resume_context( struct thread *thread, const struct context_data *
  * resuming in between. Snapshot-only callers leave it NULL. */
 struct thread *ios_pending_persistent_hold;
 
+/* MADEIRA_CTX_FP=0: a cross-thread GetThreadContext returns zeroed vector registers, as
+ * before. Read on every capture, madeira.cfg (and the game's own lines) first: this server
+ * starts before the app exports env.* lines (ml1236) and runs again for the next session. */
+static int ios_ctx_fp_enabled(void)
+{
+    char buf[16];
+    const char *e = madeira_cfg_get( "env.MADEIRA_CTX_FP", buf, sizeof(buf) ) ? buf
+                  : getenv( "MADEIRA_CTX_FP" );
+    return !(e && e[0] == '0');
+}
+
 int ios_fill_thread_context( struct thread *thread,
                              struct context_data *native,
                              struct context_data *wow )
@@ -909,6 +923,55 @@ int ios_fill_thread_context( struct thread *thread,
         have_native = 1;
     }
 
+    /* The vector registers too: without them every cross-thread floating-point context
+     * came back zeroed. FEX keeps guest XMM0-15 in V0-V15 under ARM64EC, the same mapping
+     * the ARM64EC ABI uses, so these are the guest's XMM registers. Persona 5 Royal moves
+     * code between threads this way: a thread loads it into XMM13, and the main thread
+     * reads its floating-point context into a CONTEXT in the image and jumps to the
+     * XMM13 slot. A thread blocked in a syscall is overridden below from the saved frame.
+     * MADEIRA_CTX_FP=0 restores the zeroed vector registers. */
+    if (have_native && ios_ctx_fp_enabled())
+    {
+        arm_neon_state64_t neon;
+        mach_msg_type_number_t ncount = ARM_NEON_STATE64_COUNT;
+
+        if (!thread_get_state( port, ARM_NEON_STATE64, (thread_state_t)&neon, &ncount ))
+        {
+            unsigned int i;
+            for (i = 0; i < 32; i++)
+            {
+                native->fp.arm64_regs.q[i].low  = (uint64_t)neon.__v[i];
+                native->fp.arm64_regs.q[i].high = (uint64_t)(neon.__v[i] >> 64);
+            }
+            native->fp.arm64_regs.fpcr = neon.__fpcr;
+            native->fp.arm64_regs.fpsr = neon.__fpsr;
+            native->flags |= SERVER_CTX_FLOATING_POINT;
+        }
+    }
+
+    /* Inside a syscall the live vector registers belong to Unix code, which may have
+     * clobbered them; the dispatcher saved the guest's at entry. Same validation as the
+     * ml716 frame substitution below, but not behind its opt-in: only FP is replaced.
+     * A Unix call saves only q8-q15 in that frame, so inside one, q0-q7 are the values of
+     * the thread's last syscall. */
+    if (have_native && thread->teb && (native->flags & SERVER_CTX_FLOATING_POINT))
+    {
+        uint64_t frame = 0, kstack = 0;
+        uint64_t sp = native->ctl.arm64_regs.sp;
+        if (ios_safe_read( (uint64_t)thread->teb + IOS_TEB_SYSCALL_FRAME_OFF, &frame, 8 ) &&
+            ios_safe_read( (uint64_t)thread->teb + IOS_TEB_KERNEL_STACK_OFF, &kstack, 8 ) &&
+            frame && kstack && !(frame & 7) && kstack < frame && sp >= kstack && sp <= frame)
+        {
+            unsigned char f[IOS_SCF_SIZE];
+            if (ios_safe_read( frame, f, sizeof(f) ))
+            {
+                memcpy( native->fp.arm64_regs.q, f + IOS_SCF_V, sizeof(native->fp.arm64_regs.q) );
+                native->fp.arm64_regs.fpcr = *(uint32_t *)(f + IOS_SCF_FPCR);
+                native->fp.arm64_regs.fpsr = *(uint32_t *)(f + IOS_SCF_FPSR);
+            }
+        }
+    }
+
     /* ml716: A THREAD PARKED INSIDE A SYSCALL MUST REPORT ITS SYSCALL FRAME.
      *
      * Wine's own suspend path already does this: usr1_handler checks

tests/host/check-ctx-fp.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""Vector registers in a cross-thread context (build/wineserver/mach_ios.c); no Wine runs.
+
+The server's Mach capture read only ARM_THREAD_STATE64, so every cross-thread GetThreadContext
+returned zeroed XMM registers. Persona 5 Royal reads another thread's floating-point context into
+a CONTEXT inside its image and jumps to the XMM13 slot, and executed zeros. Checks that the syscall
+frame offsets the capture reads match struct syscall_frame, that the syscall dispatcher saves every
+vector register there, that the capture is gated by MADEIRA_CTX_FP and runs while the thread is
+halted, and the switch itself: unset or 1 captures, 0 restores the zeroed registers, and the
+madeira.cfg line wins over the environment.
+"""
+from pathlib import Path
+import re, subprocess, tempfile
+
+root = Path(__file__).resolve().parents[2]
+src = (root / "build/wineserver/mach_ios.c").read_text()
+sig = (root / "build/ntdll-unix/signal_arm64_ios.c").read_text()
+
+def function(source, start):
+    a = source.index(start); b = source.index("{", a); depth = 1; c = b + 1
+    while depth:
+        depth += (source[c] == "{") - (source[c] == "}"); c += 1
+    return source[a:c]
+
+offs = dict(re.findall(r"#define (IOS_SCF_\w+)\s+(0x[0-9a-f]+)\b", src))
+frame = sig[sig.index("struct syscall_frame\n{"):]
+frame = frame[:frame.index("};") + 2]
+harness = r"""
+#include <stddef.h>
+#include <stdint.h>
+typedef uint64_t ULONG64; typedef uint32_t ULONG;
+typedef struct { uint64_t Low; int64_t High; } NEON128;
+""" + frame + "\n"
+for name, field in (("FP", "fp"), ("LR", "lr"), ("SP", "sp"), ("PC", "pc"), ("CPSR", "cpsr"),
+                    ("FPCR", "fpcr"), ("FPSR", "fpsr"), ("V", "v")):
+    harness += f"_Static_assert(offsetof(struct syscall_frame, {field}) == {offs['IOS_SCF_' + name]}, \"{name}\");\n"
+harness += f"_Static_assert(sizeof(struct syscall_frame) == {offs['IOS_SCF_SIZE']}, \"SIZE\");\n"
+harness += "int main(void) { return 0; }\n"
+with tempfile.TemporaryDirectory() as t:
+    c = Path(t) / "layout.c"; c.write_text(harness)
+    subprocess.run(["cc", "-std=gnu11", "-Wall", str(c), "-o", str(Path(t) / "layout")], check=True)
+print("PASS: the IOS_SCF offsets match struct syscall_frame, including fpcr, fpsr and v[32]")
+
+dispatcher = sig[sig.index("__ASM_GLOBAL_FUNC( __wine_syscall_dispatcher,"):]
+dispatcher = dispatcher[:dispatcher.index("__ASM_GLOBAL_FUNC(", 10)]
+for q in range(0, 32, 2):
+    assert re.search(rf'"stp q{q},\s+q{q + 1},\s+\[x10, #0x{0x130 + q * 16:x}\]', dispatcher), q
+assert '"str w9, [x10, #0x128]' in dispatcher and '"str w9, [x10, #0x12c]' in dispatcher
+print("PASS: the syscall dispatcher saves q0-q31, fpcr and fpsr where the capture reads them")
+
+fill = function(src, "int ios_fill_thread_context(")
+neon = fill.index("thread_get_state( port, ARM_NEON_STATE64")
+assert fill.index("thread_suspend( port )") < fill.index("thread_get_state( port, ARM_THREAD_STATE64") < neon
+assert neon < fill.index("f + IOS_SCF_V") < fill.rindex("thread_resume( port );")
+assert "if (have_native && ios_ctx_fp_enabled())" in fill[:neon]
+assert "(native->flags & SERVER_CTX_FLOATING_POINT)" in fill[neon:fill.index("f + IOS_SCF_V")]
+assert "native->flags |= SERVER_CTX_FLOATING_POINT;" in fill
+print("PASS: the vector capture is gated by MADEIRA_CTX_FP, runs while the thread is halted, and the "
+      "frame override only replaces a capture that happened")
+
+switch = function(src, "static int ios_ctx_fp_enabled(void)")
+harness = r"""
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+static const char *cfg_value;
+static int madeira_cfg_get(const char *key, char *buf, size_t cap)
+{
+    if (strcmp(key, "env.MADEIRA_CTX_FP") || !cfg_value) return 0;
+    snprintf(buf, cap, "%s", cfg_value);
+    return 1;
+}
+""" + switch + r"""
+#define CHECK(x) do { if (!(x)) { printf("failed line %d: %s\n", __LINE__, #x); return 1; } } while (0)
+int main(void)
+{
+    unsetenv("MADEIRA_CTX_FP");
+    CHECK(ios_ctx_fp_enabled());
+    setenv("MADEIRA_CTX_FP", "1", 1); CHECK(ios_ctx_fp_enabled());
+    setenv("MADEIRA_CTX_FP", "0", 1); CHECK(!ios_ctx_fp_enabled());
+    cfg_value = "1"; CHECK(ios_ctx_fp_enabled());
+    unsetenv("MADEIRA_CTX_FP"); cfg_value = "0"; CHECK(!ios_ctx_fp_enabled());
+    return 0;
+}
+"""
+with tempfile.TemporaryDirectory() as t:
+    c = Path(t) / "switch.c"; c.write_text(harness)
+    exe = Path(t) / "switch"
+    subprocess.run(["cc", "-std=c11", "-D_POSIX_C_SOURCE=200809L", "-Wall", "-Wextra", "-Werror",
+                    str(c), "-o", str(exe)], check=True)
+    run = subprocess.run([str(exe)], capture_output=True, text=True)
+    print(run.stdout, end="")
+    run.check_returncode()
+print("PASS: MADEIRA_CTX_FP unset or 1 captures, 0 restores zeroed registers, madeira.cfg wins over the environment")

tests/host/check-pool-code-exit.py

@@ -41,6 +41,7 @@ def function(signature):
     'static int ios_pool_low_take(',
     'static int ios_tail_carve_in_low(',
     'static int ios_pool_preserve_code_carve(',
+    'static int ios_pool_preserve_big_carve(',
     'static int ios_tail_carve_ready(',
     'static unsigned ios_tail_carves_retire_owner(',
     'static int ios_tail_carve_foreign_release(',
@@ -59,7 +60,7 @@ def function(signature):
 assert reclaim.count('ios_tail_carves_owner_died( peb );') == 1
 assert reclaim.index('ios_tail_carves_owner_died( peb );') > reclaim.rindex('pthread_mutex_unlock( &ios_pool_lock );'), \
     'the carve lock is never taken under the pool lock'
-assert source.count('.retired = 1;') == 1, 'only an exit retires a carve'
+assert source.count('.retired = 1;') == 2, 'an exit, or a release while a thread still runs in the carve, retires it'
 assert source.count('ios_tail_carves[ios_tail_carve_n].retired = 0;') == 4, 'every new carve starts unretired'
 assert source.count('ios_tail_carves[ios_tail_carve_n].peb = carve_peb;') == 3, 'region C, ml1039 and tail carves'
 assert source.count('ios_tail_carves[ios_tail_carve_n].peb = NULL;') == 1, 'a skipped piece has no owner'
@@ -111,6 +112,7 @@ def function(signature):
 static int allocate(size_t *size_ptr, void **ret)
 {
 ''' + ec + r'''
+    (void)kept_big_carve;  /* read by the production tail path after this prefix */
     return FALLBACK_TAIL;  /* the production tail path starts after this prefix */
 }
 static int release(void *base, unsigned type, void **addr_ptr, size_t *size_ptr)
@@ -361,15 +363,26 @@ def function(signature):
     owner = 8;
     CHECK(release(rx, MEM_RELEASE, &addr, &size) == FALL_THROUGH && !ios_tail_carves[0].free);
     owner = 0;                                                    /* a thread without a TEB: as before */
+    /* MADEIRA_POOL_CODE_DEFER_FREE=0: refused and left live, as before. */
     occupied = 1;
+    setenv("MADEIRA_POOL_CODE_DEFER_FREE", "0", 1);
     CHECK(release(rx, MEM_RELEASE, &addr, &size) == FALL_THROUGH && !ios_tail_carves[0].free);
+    unsetenv("MADEIRA_POOL_CODE_DEFER_FREE");
+    /* A release while a thread still runs in the buffer frees it as retired: no
+     * owner, and reuse waits out the grace and a strict scan, as for an exit. */
+    CHECK(release(rx, MEM_RELEASE, &addr, &size) == STATUS_SUCCESS && addr == rx && size == 64 * MB);
+    CHECK(ios_tail_carves[0].free && ios_tail_carves[0].retired && !ios_tail_carves[0].peb);
+    CHECK(ios_tail_carves[0].freed_at == fake_now && !ios_tail_carve_ready( &ios_tail_carves[0], fake_now ));
+    CHECK(ios_tail_carve_ready( &ios_tail_carves[0], fake_now + 3 ));
     occupied = 0;
+    ios_tail_carves[0].free = ios_tail_carves[0].retired = 0;   /* live again, owned by 9 */
+    ios_tail_carves[0].peb = (void *)9;
     owner = 9;
     CHECK(release(rx, MEM_RELEASE, &addr, &size) == STATUS_SUCCESS && addr == rx && size == 64 * MB);
     CHECK(ios_tail_carves[0].free && !ios_tail_carves[0].retired);
     owner = 8;
     CHECK(release(rx, MEM_RELEASE, &addr, &size) == FALL_THROUGH);  /* already free */
-    puts("PASS: a release from another process is refused; the owner's (or a TEB-less thread's) works as before");
+    puts("PASS: a release from another process is refused; an occupied one is retired until its thread leaves (refused with MADEIRA_POOL_CODE_DEFER_FREE=0); the owner's (or a TEB-less thread's) works as before");
 }
 
 int main(void)