Format: *
* [4 bytes: magic "CL4J"]
- * [4 bytes: version (2)]
+ * [4 bytes: version (3)]
* [modified UTF-8: target triple]
* [4 bytes: image length]
* [N bytes: image]
@@ -30,7 +30,7 @@
public final class NativeCodeSerializer {
private static final int MAGIC = 0x434C344A; // "CL4J"
- private static final int VERSION = 2;
+ private static final int VERSION = 3;
private NativeCodeSerializer() {}
diff --git a/redline/api/src/main/java/run/endive/redline/experimental/api/internal/CtxBuffer.java b/redline/api/src/main/java/run/endive/redline/experimental/api/internal/CtxBuffer.java
index 420362c07..9209feca8 100644
--- a/redline/api/src/main/java/run/endive/redline/experimental/api/internal/CtxBuffer.java
+++ b/redline/api/src/main/java/run/endive/redline/experimental/api/internal/CtxBuffer.java
@@ -19,6 +19,7 @@
* 56 i64 memmovePtr Pointer to libc memmove
* 64 i64 memsetPtr Pointer to libc memset
* 72 i64 interruptFlag Non-zero = interrupt requested
+ * 80 i64 memoryPagesPtr Shared memory: pointer to its current page count (i32)
* 200 i64 globalsPtr Pointer to globals buffer
* 208 i64 memGrowPtr Upcall stub for memory.grow
* 216 i32 memoryPages Current memory page count
@@ -52,6 +53,7 @@ private CtxBuffer() {}
public static final int MEMMOVE_PTR = 56;
public static final int MEMSET_PTR = 64;
public static final int INTERRUPT_FLAG = 72;
+ public static final int MEMORY_PAGES_PTR = 80;
public static final int GLOBALS_PTR = 200;
public static final int MEM_GROW_PTR = 208;
diff --git a/redline/api/src/main/java/run/endive/redline/experimental/api/internal/NativeAtomics.java b/redline/api/src/main/java/run/endive/redline/experimental/api/internal/NativeAtomics.java
new file mode 100644
index 000000000..e68cd4d6a
--- /dev/null
+++ b/redline/api/src/main/java/run/endive/redline/experimental/api/internal/NativeAtomics.java
@@ -0,0 +1,215 @@
+package run.endive.redline.experimental.api.internal;
+
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.VarHandle;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+/** Hardware atomics on a native memory's mapping, via a direct buffer; callers check bounds. */
+public final class NativeAtomics {
+
+ private static final VarHandle INT =
+ MethodHandles.byteBufferViewVarHandle(int[].class, ByteOrder.LITTLE_ENDIAN);
+ private static final VarHandle LONG =
+ MethodHandles.byteBufferViewVarHandle(long[].class, ByteOrder.LITTLE_ENDIAN);
+
+ private static final int ADD = 0;
+ private static final int AND = 1;
+ private static final int OR = 2;
+ private static final int XOR = 3;
+ private static final int XCHG = 4;
+
+ private final ByteBuffer buffer;
+
+ public NativeAtomics(ByteBuffer buffer) {
+ this.buffer = buffer;
+ }
+
+ public int readInt(int addr) {
+ return (int) INT.getVolatile(buffer, addr);
+ }
+
+ public long readLong(int addr) {
+ return (long) LONG.getVolatile(buffer, addr);
+ }
+
+ public short readShort(int addr) {
+ return (short) narrowGet(addr, 16);
+ }
+
+ public byte readByte(int addr) {
+ return (byte) narrowGet(addr, 8);
+ }
+
+ public void writeInt(int addr, int value) {
+ INT.setVolatile(buffer, addr, value);
+ }
+
+ public void writeLong(int addr, long value) {
+ LONG.setVolatile(buffer, addr, value);
+ }
+
+ public void writeShort(int addr, short value) {
+ narrowUpdate(addr, 16, XCHG, value);
+ }
+
+ public void writeByte(int addr, byte value) {
+ narrowUpdate(addr, 8, XCHG, value);
+ }
+
+ public int addInt(int addr, int delta) {
+ return (int) INT.getAndAdd(buffer, addr, delta);
+ }
+
+ public int andInt(int addr, int mask) {
+ return (int) INT.getAndBitwiseAnd(buffer, addr, mask);
+ }
+
+ public int orInt(int addr, int mask) {
+ return (int) INT.getAndBitwiseOr(buffer, addr, mask);
+ }
+
+ public int xorInt(int addr, int mask) {
+ return (int) INT.getAndBitwiseXor(buffer, addr, mask);
+ }
+
+ public int xchgInt(int addr, int value) {
+ return (int) INT.getAndSet(buffer, addr, value);
+ }
+
+ public int cmpxchgInt(int addr, int expected, int replacement) {
+ return (int) INT.compareAndExchange(buffer, addr, expected, replacement);
+ }
+
+ public long addLong(int addr, long delta) {
+ return (long) LONG.getAndAdd(buffer, addr, delta);
+ }
+
+ public long andLong(int addr, long mask) {
+ return (long) LONG.getAndBitwiseAnd(buffer, addr, mask);
+ }
+
+ public long orLong(int addr, long mask) {
+ return (long) LONG.getAndBitwiseOr(buffer, addr, mask);
+ }
+
+ public long xorLong(int addr, long mask) {
+ return (long) LONG.getAndBitwiseXor(buffer, addr, mask);
+ }
+
+ public long xchgLong(int addr, long value) {
+ return (long) LONG.getAndSet(buffer, addr, value);
+ }
+
+ public long cmpxchgLong(int addr, long expected, long replacement) {
+ return (long) LONG.compareAndExchange(buffer, addr, expected, replacement);
+ }
+
+ public short addShort(int addr, short delta) {
+ return (short) narrowUpdate(addr, 16, ADD, delta);
+ }
+
+ public short andShort(int addr, short mask) {
+ return (short) narrowUpdate(addr, 16, AND, mask);
+ }
+
+ public short orShort(int addr, short mask) {
+ return (short) narrowUpdate(addr, 16, OR, mask);
+ }
+
+ public short xorShort(int addr, short mask) {
+ return (short) narrowUpdate(addr, 16, XOR, mask);
+ }
+
+ public short xchgShort(int addr, short value) {
+ return (short) narrowUpdate(addr, 16, XCHG, value);
+ }
+
+ public short cmpxchgShort(int addr, short expected, short replacement) {
+ return (short) narrowCmpxchg(addr, 16, expected, replacement);
+ }
+
+ public byte addByte(int addr, byte delta) {
+ return (byte) narrowUpdate(addr, 8, ADD, delta);
+ }
+
+ public byte andByte(int addr, byte mask) {
+ return (byte) narrowUpdate(addr, 8, AND, mask);
+ }
+
+ public byte orByte(int addr, byte mask) {
+ return (byte) narrowUpdate(addr, 8, OR, mask);
+ }
+
+ public byte xorByte(int addr, byte mask) {
+ return (byte) narrowUpdate(addr, 8, XOR, mask);
+ }
+
+ public byte xchgByte(int addr, byte value) {
+ return (byte) narrowUpdate(addr, 8, XCHG, value);
+ }
+
+ public byte cmpxchgByte(int addr, byte expected, byte replacement) {
+ return (byte) narrowCmpxchg(addr, 8, expected, replacement);
+ }
+
+ private int narrowGet(int addr, int bits) {
+ int shift = shift(addr, bits);
+ return ((int) INT.getVolatile(buffer, addr & ~3) >>> shift) & ((1 << bits) - 1);
+ }
+
+ // Byte and short have no atomic VarHandle modes: CAS the enclosing aligned int.
+ private int narrowUpdate(int addr, int bits, int op, int operand) {
+ int base = addr & ~3;
+ int shift = shift(addr, bits);
+ int mask = ((1 << bits) - 1) << shift;
+ while (true) {
+ int word = (int) INT.getVolatile(buffer, base);
+ int old = (word & mask) >>> shift;
+ int updated = (word & ~mask) | ((apply(op, old, operand) << shift) & mask);
+ if (INT.compareAndSet(buffer, base, word, updated)) {
+ return old;
+ }
+ }
+ }
+
+ private int narrowCmpxchg(int addr, int bits, int expected, int replacement) {
+ int base = addr & ~3;
+ int shift = shift(addr, bits);
+ int valueMask = (1 << bits) - 1;
+ int mask = valueMask << shift;
+ while (true) {
+ int word = (int) INT.getVolatile(buffer, base);
+ int old = (word & mask) >>> shift;
+ if (old != (expected & valueMask)) {
+ return old;
+ }
+ int updated = (word & ~mask) | ((replacement & valueMask) << shift);
+ if (INT.compareAndSet(buffer, base, word, updated)) {
+ return old;
+ }
+ }
+ }
+
+ private static int shift(int addr, int bits) {
+ if ((addr & (bits / 8 - 1)) != 0) {
+ throw new IllegalStateException("Misaligned atomic access at address " + addr);
+ }
+ return (addr & 3) * 8;
+ }
+
+ private static int apply(int op, int old, int operand) {
+ switch (op) {
+ case ADD:
+ return old + operand;
+ case AND:
+ return old & operand;
+ case OR:
+ return old | operand;
+ case XOR:
+ return old ^ operand;
+ default:
+ return operand;
+ }
+ }
+}
diff --git a/redline/api/src/main/java/run/endive/redline/experimental/api/internal/NativeWaiters.java b/redline/api/src/main/java/run/endive/redline/experimental/api/internal/NativeWaiters.java
new file mode 100644
index 000000000..9539f8953
--- /dev/null
+++ b/redline/api/src/main/java/run/endive/redline/experimental/api/internal/NativeWaiters.java
@@ -0,0 +1,100 @@
+package run.endive.redline.experimental.api.internal;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.BooleanSupplier;
+import run.endive.runtime.WasmInterruptedException;
+import run.endive.wasm.WasmEngineException;
+
+/** memory.atomic.wait and notify for a native memory, as ByteArrayMemory implements them. */
+public final class NativeWaiters {
+
+ // Guarded by its own monitor; 0 <= pendingWakeups <= waiterCount
+ private static final class WaitState {
+ int waiterCount;
+ int pendingWakeups;
+ // a waiter only consumes a wakeup minted after it registered
+ long generation;
+ }
+
+ private final Map waitStates = new ConcurrentHashMap<>();
+ private final boolean shared;
+
+ public NativeWaiters(boolean shared) {
+ this.shared = shared;
+ }
+
+ public Object monitor(int address) {
+ return waitStates.computeIfAbsent(address, k -> new WaitState());
+ }
+
+ /** Waits while {@code condition} holds: 0 woken, 1 not-equal, 2 timed out. */
+ public int waitOn(int address, BooleanSupplier condition, long timeout) {
+ if (!shared) {
+ throw new WasmEngineException("Attempt to wait on a non-shared memory, not supported.");
+ }
+
+ long deadline = (timeout < 0) ? Long.MAX_VALUE : System.nanoTime() + timeout;
+ WaitState state = waitStates.computeIfAbsent(address, k -> new WaitState());
+
+ synchronized (state) {
+ if (!condition.getAsBoolean()) {
+ return 1;
+ }
+
+ state.waiterCount++;
+ final long arrivalGeneration = state.generation;
+ try {
+ while (state.pendingWakeups == 0 || state.generation == arrivalGeneration) {
+ long remaining = deadline - System.nanoTime();
+ if (remaining <= 0) {
+ return 2;
+ }
+ long millis = Math.max(remaining / 1_000_000L, 0);
+ int nanos = Math.max((int) (remaining % 1_000_000L), 0);
+ try {
+ state.wait(millis, nanos);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new WasmInterruptedException("Thread interrupted");
+ }
+ }
+ return 0;
+ } finally {
+ // Only a wakeup minted while this thread waited is its to consume.
+ if (state.pendingWakeups > 0 && state.generation != arrivalGeneration) {
+ state.pendingWakeups--;
+ }
+ state.waiterCount--;
+ assert (0 <= state.pendingWakeups);
+ assert (state.pendingWakeups <= state.waiterCount);
+ }
+ }
+ }
+
+ /** Wakes up to {@code maxThreads} waiters (all if negative); returns how many. */
+ public int notify(int address, int maxThreads) {
+ if (!shared) {
+ return 0;
+ }
+
+ WaitState state = waitStates.get(address);
+ if (state == null) {
+ return 0;
+ }
+
+ synchronized (state) {
+ int actualWaiters = state.waiterCount - state.pendingWakeups;
+ if (actualWaiters == 0) {
+ return 0;
+ }
+ int toWake = maxThreads < 0 ? actualWaiters : Math.min(actualWaiters, maxThreads);
+ // waiters arriving after this notify wait for the next one
+ state.pendingWakeups += toWake;
+ state.generation++;
+ assert (state.pendingWakeups <= state.waiterCount);
+ state.notifyAll();
+ return toWake;
+ }
+ }
+}
diff --git a/redline/compiler/src/main/java/run/endive/redline/experimental/compiler/internal/EmitContext.java b/redline/compiler/src/main/java/run/endive/redline/experimental/compiler/internal/EmitContext.java
index 3c10e0eac..48b524b04 100644
--- a/redline/compiler/src/main/java/run/endive/redline/experimental/compiler/internal/EmitContext.java
+++ b/redline/compiler/src/main/java/run/endive/redline/experimental/compiler/internal/EmitContext.java
@@ -25,6 +25,7 @@ final class EmitContext {
final Map sigRefCache;
final boolean multiReturn;
final int[] canonicalTypeMap;
+ final boolean sharedMemory;
EmitContext(
CraneliftBridge bridge,
@@ -37,7 +38,8 @@ final class EmitContext {
int ctxPtrVar,
Map sigRefCache,
boolean multiReturn,
- int[] canonicalTypeMap) {
+ int[] canonicalTypeMap,
+ boolean sharedMemory) {
this.bridge = bridge;
this.valueStack = valueStack;
this.module = module;
@@ -49,6 +51,19 @@ final class EmitContext {
this.sigRefCache = sigRefCache;
this.multiReturn = multiReturn;
this.canonicalTypeMap = canonicalTypeMap;
+ this.sharedMemory = sharedMemory;
+ }
+
+ /** The current page count; a shared memory's lives with the memory, as others may grow it. */
+ int emitLoadMemoryPages() {
+ var b = bridge.exports();
+ int zero = b.emitIconst32(0);
+ int ctxPtr = b.useVar(ctxPtrVar);
+ if (!sharedMemory) {
+ return b.emitLoadI32(ctxPtr, zero, CtxBuffer.MEMORY_PAGES);
+ }
+ int pagesPtr = b.emitLoadI64(ctxPtr, zero, CtxBuffer.MEMORY_PAGES_PTR);
+ return b.emitLoadI32(pagesPtr, zero, 0);
}
// --- Helpers used by emitters ---
diff --git a/redline/compiler/src/main/java/run/endive/redline/experimental/compiler/internal/NativeCompiler.java b/redline/compiler/src/main/java/run/endive/redline/experimental/compiler/internal/NativeCompiler.java
index 157f9324d..8c8a08267 100644
--- a/redline/compiler/src/main/java/run/endive/redline/experimental/compiler/internal/NativeCompiler.java
+++ b/redline/compiler/src/main/java/run/endive/redline/experimental/compiler/internal/NativeCompiler.java
@@ -24,6 +24,7 @@
import run.endive.wasm.types.ExternalType;
import run.endive.wasm.types.FunctionImport;
import run.endive.wasm.types.FunctionType;
+import run.endive.wasm.types.MemoryImport;
import run.endive.wasm.types.OpCode;
import run.endive.wasm.types.ValType;
@@ -41,6 +42,7 @@ public final class NativeCompiler {
private final WasmModule module;
private final int numImports;
private final int[] canonicalTypeMap;
+ private final boolean sharedMemory;
private NativeCompiler(CraneliftBridge bridge, String triple, WasmModule module) {
this.bridge = bridge;
@@ -52,6 +54,20 @@ private NativeCompiler(CraneliftBridge bridge, String triple, WasmModule module)
.filter(i -> i.importType() == ExternalType.FUNCTION)
.count();
this.canonicalTypeMap = TypeMapUtils.buildCanonicalTypeMap(module);
+ this.sharedMemory = isMemoryShared(module);
+ }
+
+ // Memory 0 is the first memory import, or else the first defined memory
+ private static boolean isMemoryShared(WasmModule module) {
+ for (int i = 0; i < module.importSection().importCount(); i++) {
+ var imp = module.importSection().getImport(i);
+ if (imp.importType() == ExternalType.MEMORY) {
+ return ((MemoryImport) imp).limits().shared();
+ }
+ }
+ return module.memorySection()
+ .map(ms -> ms.memoryCount() > 0 && ms.getMemory(0).limits().shared())
+ .orElse(false);
}
// --- Control frame ---
@@ -434,7 +450,8 @@ private byte[] compileFunction(int bodyIndex) {
ctxPtrVar,
new HashMap<>(),
multiReturn,
- canonicalTypeMap);
+ canonicalTypeMap,
+ sharedMemory);
// --- Emission loop ---
Deque controlStack = new ArrayDeque<>();
diff --git a/redline/compiler/src/main/java/run/endive/redline/experimental/compiler/internal/NativeEmitters.java b/redline/compiler/src/main/java/run/endive/redline/experimental/compiler/internal/NativeEmitters.java
index 9e336a93f..aadda55be 100644
--- a/redline/compiler/src/main/java/run/endive/redline/experimental/compiler/internal/NativeEmitters.java
+++ b/redline/compiler/src/main/java/run/endive/redline/experimental/compiler/internal/NativeEmitters.java
@@ -452,14 +452,7 @@ private static void emitBoundsCheck(EmitContext ctx, int addr, int offset, int a
(int) (endOffset >>> 32)));
// Load memory size in bytes: memPages * 65536
- int zero = ctx.bridge.exports().emitIconst32(0);
- int memPages =
- ctx.bridge
- .exports()
- .emitLoadI32(
- ctx.bridge.exports().useVar(ctx.ctxPtrVar),
- zero,
- CtxBuffer.MEMORY_PAGES);
+ int memPages = ctx.emitLoadMemoryPages();
int memPages64 = ctx.bridge.exports().emitUextendI64(memPages);
int memSize =
ctx.bridge.exports().emitIshl(memPages64, ctx.bridge.exports().emitIconst64(16, 0));
@@ -636,15 +629,7 @@ static void emitGlobalSet(EmitContext ctx, AnnotatedInstruction ins) {
// --- Memory operations ---
static void emitMemorySize(EmitContext ctx) {
- int zero = ctx.bridge.exports().emitIconst32(0);
- int pages =
- ctx.bridge
- .exports()
- .emitLoadI32(
- ctx.bridge.exports().useVar(ctx.ctxPtrVar),
- zero,
- CtxBuffer.MEMORY_PAGES);
- ctx.valueStack.push(pages);
+ ctx.valueStack.push(ctx.emitLoadMemoryPages());
}
static void emitMemoryGrow(EmitContext ctx) {
@@ -667,6 +652,7 @@ static void emitMemoryGrow(EmitContext ctx) {
int growSig = ctx.getOrCreateTrampolineSigRef();
ctx.bridge.exports().pushCallArg(ctx.bridge.exports().useVar(ctx.ctxPtrVar));
int rawResult = ctx.bridge.exports().emitCallIndirect(growSig, memGrowPtr);
+ emitTrapCheck(ctx);
int result = ctx.bridge.exports().emitIreduceI32(rawResult);
ctx.valueStack.push(result);
int newMemBase =
@@ -1257,6 +1243,7 @@ static void emitTableGrow(EmitContext ctx, AnnotatedInstruction ins) {
int trampolineSig = ctx.getOrCreateTrampolineSigRef();
b.pushCallArg(b.useVar(ctx.ctxPtrVar));
b.emitCallIndirect(trampolineSig, tableOpsPtr);
+ emitTrapCheck(ctx);
b.emitJumpWithArg(mergeBlock, oldSize);
@@ -1304,6 +1291,7 @@ static void emitTableFill(EmitContext ctx, AnnotatedInstruction ins) {
int trampolineSig = ctx.getOrCreateTrampolineSigRef();
b.pushCallArg(b.useVar(ctx.ctxPtrVar));
b.emitCallIndirect(trampolineSig, trampolinePtr);
+ emitTrapCheck(ctx);
}
static void emitTableCopy(EmitContext ctx, AnnotatedInstruction ins) {
@@ -1350,6 +1338,7 @@ static void emitTableCopy(EmitContext ctx, AnnotatedInstruction ins) {
int trampolineSig = ctx.getOrCreateTrampolineSigRef();
b.pushCallArg(b.useVar(ctx.ctxPtrVar));
b.emitCallIndirect(trampolineSig, trampolinePtr);
+ emitTrapCheck(ctx);
}
static void emitTableInit(EmitContext ctx, AnnotatedInstruction ins) {
@@ -1380,6 +1369,7 @@ static void emitTableInit(EmitContext ctx, AnnotatedInstruction ins) {
int trampolineSig = ctx.getOrCreateTrampolineSigRef();
b.pushCallArg(b.useVar(ctx.ctxPtrVar));
b.emitCallIndirect(trampolineSig, trampolinePtr);
+ emitTrapCheck(ctx);
}
static void emitElemDrop(EmitContext ctx, AnnotatedInstruction ins) {
@@ -1401,6 +1391,7 @@ static void emitElemDrop(EmitContext ctx, AnnotatedInstruction ins) {
int trampolineSig = ctx.getOrCreateTrampolineSigRef();
b.pushCallArg(b.useVar(ctx.ctxPtrVar));
b.emitCallIndirect(trampolineSig, trampolinePtr);
+ emitTrapCheck(ctx);
}
// --- Bulk memory operations (inline native memmove/memset) ---
@@ -1415,8 +1406,7 @@ private static void emitBulkBoundsCheck(EmitContext ctx, int addr, int size) {
int size64 = b.emitUextendI64(size);
int end = b.emitIadd(addr64, size64);
- int zero = b.emitIconst32(0);
- int memPages = b.emitLoadI32(b.useVar(ctx.ctxPtrVar), zero, CtxBuffer.MEMORY_PAGES);
+ int memPages = ctx.emitLoadMemoryPages();
int memPages64 = b.emitUextendI64(memPages);
int memSize = b.emitIshl(memPages64, b.emitIconst64(16, 0));
@@ -1504,6 +1494,7 @@ static void emitMemoryInit(EmitContext ctx, AnnotatedInstruction ins) {
int trampolineSig = ctx.getOrCreateTrampolineSigRef();
b.pushCallArg(b.useVar(ctx.ctxPtrVar));
b.emitCallIndirect(trampolineSig, trampolinePtr);
+ emitTrapCheck(ctx);
}
static void emitDataDrop(EmitContext ctx, AnnotatedInstruction ins) {
@@ -1524,6 +1515,7 @@ static void emitDataDrop(EmitContext ctx, AnnotatedInstruction ins) {
int trampolineSig = ctx.getOrCreateTrampolineSigRef();
b.pushCallArg(b.useVar(ctx.ctxPtrVar));
b.emitCallIndirect(trampolineSig, trampolinePtr);
+ emitTrapCheck(ctx);
}
// --- Atomic operations ---
@@ -1766,6 +1758,7 @@ static void emitAtomicWait32(EmitContext ctx, AnnotatedInstruction ins) {
int trampolineSig = ctx.getOrCreateTrampolineSigRef();
b.pushCallArg(b.useVar(ctx.ctxPtrVar));
int rawResult = b.emitCallIndirect(trampolineSig, trampolinePtr);
+ emitTrapCheck(ctx);
ctx.valueStack.push(b.emitIreduceI32(rawResult));
}
@@ -1796,6 +1789,7 @@ static void emitAtomicWait64(EmitContext ctx, AnnotatedInstruction ins) {
int trampolineSig = ctx.getOrCreateTrampolineSigRef();
b.pushCallArg(b.useVar(ctx.ctxPtrVar));
int rawResult = b.emitCallIndirect(trampolineSig, trampolinePtr);
+ emitTrapCheck(ctx);
ctx.valueStack.push(b.emitIreduceI32(rawResult));
}
@@ -1824,6 +1818,7 @@ static void emitAtomicNotify(EmitContext ctx, AnnotatedInstruction ins) {
int trampolineSig = ctx.getOrCreateTrampolineSigRef();
b.pushCallArg(b.useVar(ctx.ctxPtrVar));
int rawResult = b.emitCallIndirect(trampolineSig, trampolinePtr);
+ emitTrapCheck(ctx);
ctx.valueStack.push(b.emitIreduceI32(rawResult));
}
diff --git a/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/GuestSeesGrowthTest.java b/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/GuestSeesGrowthTest.java
new file mode 100644
index 000000000..78c4d06ed
--- /dev/null
+++ b/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/GuestSeesGrowthTest.java
@@ -0,0 +1,121 @@
+package run.endive.redline.experimental.runner.jffi.internal;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Test;
+import run.endive.corpus.CorpusResources;
+import run.endive.redline.experimental.api.NativeCode;
+import run.endive.redline.experimental.api.internal.RedlineTarget;
+import run.endive.redline.experimental.compiler.internal.NativeCompiler;
+import run.endive.redline.experimental.runner.jffi.JffiNativeMachineFactory;
+import run.endive.runtime.HostFunction;
+import run.endive.runtime.ImportMemory;
+import run.endive.runtime.ImportValues;
+import run.endive.runtime.Instance;
+import run.endive.runtime.Memory;
+import run.endive.wasm.Parser;
+import run.endive.wasm.WasmModule;
+import run.endive.wasm.types.FunctionType;
+import run.endive.wasm.types.MemoryLimits;
+
+/** Compiled code sees growth by another instance sharing the memory, or by a host import. */
+public class GuestSeesGrowthTest {
+
+ private static final int PAGE = 65536;
+
+ private static final WasmModule SHARED = parse("shared-memory-grow");
+ private static final NativeCode SHARED_CODE = compile(SHARED);
+ private static final WasmModule HOST_GROW = parse("host-grow");
+ private static final NativeCode HOST_GROW_CODE = compile(HOST_GROW);
+
+ private static final ExecutorService POOL = Executors.newCachedThreadPool();
+
+ @AfterAll
+ public static void shutdown() {
+ POOL.shutdownNow();
+ }
+
+ @Test
+ public void aRunningInstanceReadsThePageAnotherInstanceGrew() throws Exception {
+ var memory = JffiNativeMachineFactory.createMemory(new MemoryLimits(1, 2, true));
+ try (var waiter = shared(memory);
+ var grower = shared(memory)) {
+ var read = POOL.submit(() -> waiter.export("waitThenRead").apply(0, PAGE + 16)[0]);
+ Thread.sleep(50);
+ grower.export("growAndPublish").apply(0, PAGE + 16, 42);
+ assertEquals(42L, read.get(30, TimeUnit.SECONDS));
+ }
+ }
+
+ @Test
+ public void aRunningInstanceSeesTheSizeAnotherInstanceGrew() throws Exception {
+ var memory = JffiNativeMachineFactory.createMemory(new MemoryLimits(1, 2, true));
+ try (var waiter = shared(memory);
+ var grower = shared(memory)) {
+ var size = POOL.submit(() -> waiter.export("waitThenSize").apply(0)[0]);
+ Thread.sleep(50);
+ grower.export("growAndPublish").apply(0, PAGE + 16, 42);
+ assertEquals(2L, size.get(30, TimeUnit.SECONDS));
+ }
+ }
+
+ @Test
+ public void theGuestReadsThePageAHostImportGrew() {
+ var memory = JffiNativeMachineFactory.createMemory(new MemoryLimits(1, 2));
+ try (var instance = hostGrow(memory, 77)) {
+ assertEquals(77L, instance.export("growThenRead").apply(PAGE + 8)[0]);
+ }
+ }
+
+ @Test
+ public void theGuestSeesTheSizeAHostImportGrew() {
+ var memory = JffiNativeMachineFactory.createMemory(new MemoryLimits(1, 2));
+ try (var instance = hostGrow(memory, 77)) {
+ assertEquals(2L, instance.export("growThenSize").apply()[0]);
+ }
+ }
+
+ private static Instance shared(Memory memory) {
+ return JffiNativeMachineFactory.builder(SHARED)
+ .withPrecompiledCode(SHARED_CODE)
+ .withImportValues(
+ ImportValues.builder()
+ .addMemory(new ImportMemory("env", "memory", memory))
+ .build())
+ .build();
+ }
+
+ private static Instance hostGrow(Memory memory, int value) {
+ var grow =
+ new HostFunction(
+ "env",
+ "grow",
+ FunctionType.of(List.of(), List.of()),
+ (instance, args) -> {
+ memory.grow(1);
+ memory.writeI32(PAGE + 8, value);
+ return null;
+ });
+ return JffiNativeMachineFactory.builder(HOST_GROW)
+ .withPrecompiledCode(HOST_GROW_CODE)
+ .withImportValues(
+ ImportValues.builder()
+ .addMemory(new ImportMemory("env", "memory", memory))
+ .addFunction(grow)
+ .build())
+ .build();
+ }
+
+ private static WasmModule parse(String name) {
+ return Parser.parse(CorpusResources.getResource("compiled/" + name + ".wat.wasm"));
+ }
+
+ private static NativeCode compile(WasmModule module) {
+ return NativeCompiler.compile(RedlineTarget.detectHost().orElseThrow().triple(), module);
+ }
+}
diff --git a/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/MemoryBoundsTest.java b/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/MemoryBoundsTest.java
index b567d70aa..5c07372dc 100644
--- a/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/MemoryBoundsTest.java
+++ b/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/MemoryBoundsTest.java
@@ -1,10 +1,13 @@
package run.endive.redline.experimental.runner.jffi.internal;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import org.junit.jupiter.api.Test;
import run.endive.redline.experimental.runner.jffi.JffiNativeMachineFactory;
+import run.endive.runtime.Memory;
import run.endive.runtime.WasmRuntimeException;
+import run.endive.wasm.UninstantiableException;
import run.endive.wasm.types.MemoryLimits;
/**
@@ -41,4 +44,31 @@ public void insideTheMemoryIsUntouched() {
memory.writeI32(PAGE - 4, 0x11223344);
org.junit.jupiter.api.Assertions.assertEquals(0x11223344, memory.readInt(PAGE - 4));
}
+
+ @Test
+ public void wideAccessesAreLittleEndianAtOddOffsets() {
+ var memory = JffiNativeMachineFactory.createMemory(new MemoryLimits(1, 1));
+ for (int i = 0; i < 8; i++) {
+ memory.writeByte(101 + i, (byte) (i + 1));
+ }
+ assertEquals((short) 0x0201, memory.readShort(101));
+ assertEquals(0x0807060504030201L, memory.readLong(101));
+
+ memory.writeShort(201, (short) 0x0201);
+ memory.writeLong(301, 0x0807060504030201L);
+ assertEquals(1, memory.read(201));
+ assertEquals(2, memory.read(202));
+ for (int i = 0; i < 8; i++) {
+ assertEquals(i + 1, memory.read(301 + i));
+ }
+ }
+
+ @Test
+ public void anInitialSizePastTheRuntimeLimitIsRejected() {
+ assertThrows(
+ UninstantiableException.class,
+ () ->
+ JffiNativeMachineFactory.createMemory(
+ new MemoryLimits(Memory.RUNTIME_MAX_PAGES + 1)));
+ }
}
diff --git a/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/MemoryInitTrapTest.java b/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/MemoryInitTrapTest.java
new file mode 100644
index 000000000..feda1d8ff
--- /dev/null
+++ b/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/MemoryInitTrapTest.java
@@ -0,0 +1,54 @@
+package run.endive.redline.experimental.runner.jffi.internal;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import org.junit.jupiter.api.Test;
+import run.endive.corpus.CorpusResources;
+import run.endive.redline.experimental.api.NativeCode;
+import run.endive.redline.experimental.api.internal.RedlineTarget;
+import run.endive.redline.experimental.compiler.internal.NativeCompiler;
+import run.endive.redline.experimental.runner.jffi.JffiNativeMachineFactory;
+import run.endive.runtime.Instance;
+import run.endive.runtime.WasmRuntimeException;
+import run.endive.wasm.Parser;
+import run.endive.wasm.WasmModule;
+
+/** A trap raised while the runner handles memory.init has to stop the guest there. */
+public class MemoryInitTrapTest {
+
+ private static final WasmModule MODULE =
+ Parser.parse(CorpusResources.getResource("compiled/memory-init-trap.wat.wasm"));
+ private static final NativeCode CODE =
+ NativeCompiler.compile(RedlineTarget.detectHost().orElseThrow().triple(), MODULE);
+
+ @Test
+ public void anOutOfBoundsInitStopsBeforeTheNextInstruction() {
+ try (var instance = instance()) {
+ assertThrows(
+ WasmRuntimeException.class,
+ () -> instance.export("initOutOfBoundsThenStore").apply());
+ assertEquals(0L, instance.export("read").apply(100)[0]);
+ }
+ }
+
+ @Test
+ public void anEmptyInitPastTheEndOfADroppedSegmentTraps() {
+ try (var instance = instance()) {
+ assertThrows(
+ WasmRuntimeException.class,
+ () -> instance.export("dropThenInitPastEnd").apply());
+ }
+ }
+
+ @Test
+ public void anEmptyInitAtTheStartOfADroppedSegmentIsAllowed() {
+ try (var instance = instance()) {
+ instance.export("dropThenInitNothing").apply();
+ }
+ }
+
+ private static Instance instance() {
+ return JffiNativeMachineFactory.builder(MODULE).withPrecompiledCode(CODE).build();
+ }
+}
diff --git a/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/NativeAtomicsTest.java b/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/NativeAtomicsTest.java
new file mode 100644
index 000000000..24fa7682e
--- /dev/null
+++ b/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/NativeAtomicsTest.java
@@ -0,0 +1,145 @@
+package run.endive.redline.experimental.runner.jffi.internal;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Test;
+import run.endive.corpus.CorpusResources;
+import run.endive.redline.experimental.api.NativeCode;
+import run.endive.redline.experimental.api.internal.RedlineTarget;
+import run.endive.redline.experimental.compiler.internal.NativeCompiler;
+import run.endive.redline.experimental.runner.jffi.JffiNativeMachineFactory;
+import run.endive.runtime.ImportMemory;
+import run.endive.runtime.ImportValues;
+import run.endive.runtime.Instance;
+import run.endive.runtime.Memory;
+import run.endive.wasm.Parser;
+import run.endive.wasm.WasmModule;
+import run.endive.wasm.types.MemoryLimits;
+
+/** Host atomics on a native memory have to be atomic against the guest's own. */
+public class NativeAtomicsTest {
+
+ private static final int N = 1_000_000;
+
+ private static final WasmModule MODULE =
+ Parser.parse(CorpusResources.getResource("compiled/atomic-counter.wat.wasm"));
+ private static final NativeCode CODE =
+ NativeCompiler.compile(RedlineTarget.detectHost().orElseThrow().triple(), MODULE);
+
+ private static final ExecutorService POOL = Executors.newSingleThreadExecutor();
+
+ @AfterAll
+ public static void shutdown() {
+ POOL.shutdownNow();
+ }
+
+ @Test
+ public void hostAndGuestIntAddsDoNotLoseIncrements() throws Exception {
+ var memory = sharedMemory();
+ race(memory, "add32", () -> memory.atomicAddInt(0, 1));
+ assertEquals(2 * N, memory.readInt(0));
+ }
+
+ @Test
+ public void hostAndGuestLongAddsDoNotLoseIncrements() throws Exception {
+ var memory = sharedMemory();
+ race(memory, "add64", () -> memory.atomicAddLong(16, 1L));
+ assertEquals(2L * N, memory.readLong(16));
+ }
+
+ @Test
+ public void hostAndGuestShortAddsDoNotLoseIncrements() throws Exception {
+ var memory = sharedMemory();
+ race(memory, "add16", () -> memory.atomicAddShort(10, (short) 1));
+ assertEquals((short) (2 * N), memory.readShort(10));
+ assertEquals(0, memory.readShort(8), "the other half of the int is untouched");
+ }
+
+ @Test
+ public void hostAndGuestByteAddsDoNotLoseIncrements() throws Exception {
+ var memory = sharedMemory();
+ race(memory, "add8", () -> memory.atomicAddByte(13, (byte) 1));
+ assertEquals((byte) (2 * N), memory.read(13));
+ assertEquals(0, memory.read(12), "neighbouring bytes are untouched");
+ assertEquals(0, memory.read(14));
+ assertEquals(0, memory.read(15));
+ }
+
+ @Test
+ public void narrowOperationsOnlyTouchTheirBytes() {
+ var memory = sharedMemory();
+ memory.writeI32(0, 0x44332211);
+
+ assertEquals((byte) 0x22, memory.atomicXchgByte(1, (byte) 0x99));
+ assertEquals((byte) 0x33, memory.atomicCmpxchgByte(2, (byte) 0x00, (byte) 0x77));
+ assertEquals((byte) 0x33, memory.atomicCmpxchgByte(2, (byte) 0x33, (byte) 0x77));
+ assertEquals((short) 0x9911, memory.atomicOrShort(0, (short) 0x0002));
+ assertEquals(0x44779913, memory.atomicReadInt(0));
+
+ assertEquals((short) 0x4477, memory.atomicAndShort(2, (short) 0x00FF));
+ assertEquals((short) 0x0077, memory.atomicXorShort(2, (short) 0x0101));
+ assertEquals((short) 0x0176, memory.atomicReadShort(2));
+ memory.atomicWriteByte(3, (byte) 0xAB);
+ assertEquals((byte) 0xAB, memory.atomicReadByte(3));
+ assertEquals(0xAB769913, memory.readInt(0));
+ }
+
+ @Test
+ public void wideOperationsBehaveLikeTheirScalarForms() {
+ var memory = sharedMemory();
+ memory.atomicWriteLong(8, 0x0F0F0F0F0F0F0F0FL);
+ assertEquals(0x0F0F0F0F0F0F0F0FL, memory.atomicAndLong(8, 0x00FF00FF00FF00FFL));
+ assertEquals(0x000F000F000F000FL, memory.atomicOrLong(8, 0x1000000000000000L));
+ assertEquals(0x100F000F000F000FL, memory.atomicXorLong(8, 0x100F000F000F000FL));
+ assertEquals(0L, memory.atomicCmpxchgLong(8, 0L, 5L));
+ assertEquals(5L, memory.atomicXchgLong(8, 6L));
+ assertEquals(6L, memory.atomicReadLong(8));
+
+ memory.atomicWriteInt(4, 10);
+ assertEquals(10, memory.atomicCmpxchgInt(4, 11, 12));
+ assertEquals(10, memory.atomicCmpxchgInt(4, 10, 12));
+ assertEquals(12, memory.atomicXchgInt(4, 1));
+ assertEquals(1, memory.atomicOrInt(4, 6));
+ assertEquals(7, memory.atomicAndInt(4, 5));
+ assertEquals(5, memory.atomicXorInt(4, 1));
+ assertEquals(4, memory.atomicReadInt(4));
+ }
+
+ private static Memory sharedMemory() {
+ return JffiNativeMachineFactory.createMemory(new MemoryLimits(1, 1, true));
+ }
+
+ private static void race(Memory memory, String export, Runnable hostIncrement)
+ throws Exception {
+ try (var instance = instance(memory)) {
+ var start = new CyclicBarrier(2);
+ var guest =
+ POOL.submit(
+ () -> {
+ start.await();
+ instance.export(export).apply(N);
+ return null;
+ });
+ start.await();
+ for (int i = 0; i < N; i++) {
+ hostIncrement.run();
+ }
+ guest.get(60, TimeUnit.SECONDS);
+ }
+ }
+
+ private static Instance instance(Memory memory) {
+ return JffiNativeMachineFactory.builder(MODULE)
+ .withPrecompiledCode(CODE)
+ .withImportValues(
+ ImportValues.builder()
+ .addMemory(new ImportMemory("env", "memory", memory))
+ .build())
+ .build();
+ }
+}
diff --git a/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/NativeMemoryConcurrencyTest.java b/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/NativeMemoryConcurrencyTest.java
new file mode 100644
index 000000000..e31032598
--- /dev/null
+++ b/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/NativeMemoryConcurrencyTest.java
@@ -0,0 +1,83 @@
+package run.endive.redline.experimental.runner.jffi.internal;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import org.junit.jupiter.api.Test;
+import run.endive.redline.experimental.runner.jffi.JffiNativeMachineFactory;
+import run.endive.wasm.types.MemoryLimits;
+
+/** A shared memory is grown and read by several host threads at once. */
+public class NativeMemoryConcurrencyTest {
+
+ private static final int PAGE = 65536;
+
+ @Test
+ public void concurrentGrowsHandOutEveryPageOnce() throws Exception {
+ int threads = 8;
+ int growsPerThread = 200;
+ var memory =
+ JffiNativeMachineFactory.createMemory(
+ new MemoryLimits(0, threads * growsPerThread, true));
+ Set handedOut = ConcurrentHashMap.newKeySet();
+ var start = new CountDownLatch(1);
+ ExecutorService pool = Executors.newFixedThreadPool(threads);
+ try {
+ var futures = new java.util.ArrayList>();
+ for (int t = 0; t < threads; t++) {
+ futures.add(
+ pool.submit(
+ () -> {
+ start.await();
+ for (int i = 0; i < growsPerThread; i++) {
+ assertTrue(handedOut.add(memory.grow(1)));
+ }
+ return null;
+ }));
+ }
+ start.countDown();
+ for (var f : futures) {
+ f.get(30, TimeUnit.SECONDS);
+ }
+ } finally {
+ pool.shutdownNow();
+ }
+ assertEquals(threads * growsPerThread, memory.pages());
+ assertEquals(threads * growsPerThread, handedOut.size());
+ }
+
+ // The reader polls with plain reads, the pattern that let the JIT keep a stale view
+ @Test
+ public void aHostReaderCanReachPagesAnotherThreadGrew() throws Exception {
+ ExecutorService pool = Executors.newSingleThreadExecutor();
+ try {
+ for (int round = 0; round < 20; round++) {
+ var memory = JffiNativeMachineFactory.createMemory(new MemoryLimits(1, 2, true));
+ Future reader =
+ pool.submit(
+ () -> {
+ long deadline = System.nanoTime() + 10_000_000_000L;
+ while (memory.readInt(0) != 1) {
+ if (System.nanoTime() > deadline) {
+ throw new AssertionError("flag never seen");
+ }
+ }
+ return memory.readInt(PAGE + 16);
+ });
+ Thread.sleep(5);
+ memory.grow(1);
+ memory.atomicWriteInt(0, 1);
+ reader.get(30, TimeUnit.SECONDS);
+ }
+ } finally {
+ pool.shutdownNow();
+ }
+ }
+}
diff --git a/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/NativeWaitNotifyTest.java b/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/NativeWaitNotifyTest.java
new file mode 100644
index 000000000..ae690beea
--- /dev/null
+++ b/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/NativeWaitNotifyTest.java
@@ -0,0 +1,93 @@
+package run.endive.redline.experimental.runner.jffi.internal;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import org.junit.jupiter.api.Test;
+import run.endive.corpus.CorpusResources;
+import run.endive.redline.experimental.api.internal.RedlineTarget;
+import run.endive.redline.experimental.compiler.internal.NativeCompiler;
+import run.endive.redline.experimental.runner.jffi.JffiNativeMachineFactory;
+import run.endive.runtime.ImportMemory;
+import run.endive.runtime.ImportValues;
+import run.endive.runtime.WasmInterruptedException;
+import run.endive.wasm.Parser;
+import run.endive.wasm.types.MemoryLimits;
+
+public class NativeWaitNotifyTest {
+
+ // Ported from the runtime's MemoryTest (#107): a late wait can't take the parked one's wakeup
+ @Test
+ public void notifyWakesTheParkedThreadNotALateArrival() throws Exception {
+ final int addr = 0;
+ final int sentinel = 42;
+ var memory = JffiNativeMachineFactory.createMemory(new MemoryLimits(1, 1, true));
+
+ for (int round = 0; round < 200; round++) {
+ memory.atomicWriteInt(addr, sentinel);
+ var parkedReady = new CountDownLatch(1);
+ var parked =
+ new Thread(
+ () -> {
+ parkedReady.countDown();
+ memory.atomicWait(addr, sentinel, 100_000_000L);
+ });
+ parked.setDaemon(true);
+ parked.start();
+ parkedReady.await();
+ Thread.sleep(1);
+
+ if (memory.atomicNotify(addr, 1) == 0) {
+ continue;
+ }
+ assertEquals(2, memory.atomicWait(addr, sentinel, 0L), "round " + round);
+ }
+ }
+
+ @Test
+ public void interruptingAGuestBlockedInAWaitStopsIt() throws Exception {
+ var module = Parser.parse(CorpusResources.getResource("compiled/threads-example.wat.wasm"));
+ var memory = JffiNativeMachineFactory.createMemory(new MemoryLimits(1, 1, true));
+ try (var instance =
+ JffiNativeMachineFactory.builder(module)
+ .withPrecompiledCode(
+ NativeCompiler.compile(
+ RedlineTarget.detectHost().orElseThrow().triple(), module))
+ .withImportValues(
+ ImportValues.builder()
+ .addMemory(new ImportMemory("env", "memory", memory))
+ .build())
+ .build()) {
+ // locked, so lockMutex parks in memory.atomic.wait32
+ memory.atomicWriteInt(0, 1);
+ var failure = new AtomicReference();
+ var blocked =
+ new Thread(
+ () -> {
+ try {
+ instance.export("lockMutex").apply(0);
+ } catch (Throwable t) {
+ failure.set(t);
+ }
+ });
+ blocked.setDaemon(true);
+ blocked.start();
+
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30);
+ while (blocked.getState() != Thread.State.WAITING
+ && blocked.getState() != Thread.State.TIMED_WAITING) {
+ assertTrue(System.nanoTime() < deadline, "never parked");
+ Thread.sleep(1);
+ }
+ blocked.interrupt();
+ blocked.join(TimeUnit.SECONDS.toMillis(30));
+
+ assertTrue(!blocked.isAlive(), "the interrupted guest kept running");
+ assertInstanceOf(WasmInterruptedException.class, failure.get());
+ }
+ }
+}
diff --git a/redline/runner-jffi/src/main/java/run/endive/redline/experimental/runner/jffi/internal/JffiNativeMachine.java b/redline/runner-jffi/src/main/java/run/endive/redline/experimental/runner/jffi/internal/JffiNativeMachine.java
index 7b55a50cc..3ba3eceb3 100644
--- a/redline/runner-jffi/src/main/java/run/endive/redline/experimental/runner/jffi/internal/JffiNativeMachine.java
+++ b/redline/runner-jffi/src/main/java/run/endive/redline/experimental/runner/jffi/internal/JffiNativeMachine.java
@@ -415,6 +415,14 @@ private static void setClosureReturn(Closure.Buffer buf, long result, FunctionTy
}
}
+ // Host code may have grown the memory since compiled code last saw its size
+ private void refreshMemoryPages() {
+ var mem = instance.memory();
+ if (mem != null) {
+ MEM.putInt(ctxBufferAddr + CtxBuffer.MEMORY_PAGES, mem.pages());
+ }
+ }
+
private long importDispatchDirect(int funcId) {
try {
int argCount = MEM.getInt(ctxBufferAddr + CtxBuffer.ARG_COUNT);
@@ -425,6 +433,7 @@ private long importDispatchDirect(int funcId) {
if (funcId < numImports) {
var importFunc = instance.imports().function(funcId);
long[] result = importFunc.handle().apply(instance, args);
+ refreshMemoryPages();
if (result == null || result.length == 0) {
return 0L;
}
@@ -914,6 +923,9 @@ public long[] call(int funcId, long[] args) throws WasmEngineException {
if (mem instanceof JffiNativeMemory) {
cachedMemBase = ((JffiNativeMemory) mem).nativeAddress();
MEM.putLong(ctxBufferAddr + CtxBuffer.MEM_BASE_ADDR, cachedMemBase);
+ MEM.putLong(
+ ctxBufferAddr + CtxBuffer.MEMORY_PAGES_PTR,
+ ((JffiNativeMemory) mem).pagesAddress());
} else if (mem != null) {
throw new WasmEngineException(foreignImportMessage("memory", mem));
} else {
@@ -921,11 +933,7 @@ public long[] call(int funcId, long[] args) throws WasmEngineException {
}
memBaseInitialized = true;
}
- // MEMORY_PAGES can change via grow() — must refresh every call
- var mem = instance.memory();
- if (mem != null) {
- MEM.putInt(ctxBufferAddr + CtxBuffer.MEMORY_PAGES, mem.pages());
- }
+ refreshMemoryPages();
if (Thread.currentThread().isInterrupted()) {
throw new WasmInterruptedException("Thread interrupted");
diff --git a/redline/runner-jffi/src/main/java/run/endive/redline/experimental/runner/jffi/internal/JffiNativeMemory.java b/redline/runner-jffi/src/main/java/run/endive/redline/experimental/runner/jffi/internal/JffiNativeMemory.java
index 5db8bc26b..bd6c0e3ee 100644
--- a/redline/runner-jffi/src/main/java/run/endive/redline/experimental/runner/jffi/internal/JffiNativeMemory.java
+++ b/redline/runner-jffi/src/main/java/run/endive/redline/experimental/runner/jffi/internal/JffiNativeMemory.java
@@ -9,8 +9,10 @@
import com.kenai.jffi.MemoryIO;
import com.kenai.jffi.PageManager;
import com.kenai.jffi.Type;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import java.lang.invoke.VarHandle;
+import java.nio.ByteBuffer;
+import run.endive.redline.experimental.api.internal.NativeAtomics;
+import run.endive.redline.experimental.api.internal.NativeWaiters;
import run.endive.runtime.Instance;
import run.endive.runtime.Memory;
import run.endive.wasm.UninstantiableException;
@@ -88,23 +90,31 @@ private static int winVirtualFree(long addr, long size, int freeType) {
return (int) INV.invokeN3(VIRTUAL_FREE_CTX, VIRTUAL_FREE_ADDR, addr, size, freeType);
}
- private static final class WaitState {
- int waiterCount;
- int pendingWakeups;
- }
+ private static final byte[] NO_DATA = new byte[0];
- private final Map waitStates = new ConcurrentHashMap<>();
private final MemoryLimits limits;
private final long reservedAddress;
private final int reservedOsPages;
private final int osPerWasm; // OS pages per Wasm page
- private int nPages;
+ private final NativeWaiters waiters;
+ private final NativeAtomics atomics;
+ // Page count compiled code reads for a shared memory, which other instances may grow
+ private long pagesCell;
+ private volatile int nPages;
private DataSegment[] dataSegments;
public JffiNativeMemory(MemoryLimits limits) {
this.limits = limits;
- this.nPages = limits.initialPages();
int maxPages = Math.min(limits.maximumPages(), RUNTIME_MAX_PAGES);
+ if (limits.initialPages() > maxPages) {
+ throw new UninstantiableException(
+ "memory of "
+ + limits.initialPages()
+ + " pages exceeds the limit of "
+ + maxPages
+ + " pages");
+ }
+ this.nPages = limits.initialPages();
int osPageSize = (int) PM.pageSize();
this.osPerWasm = PAGE_SIZE / osPageSize;
this.reservedOsPages = maxPages * osPerWasm;
@@ -133,19 +143,37 @@ public JffiNativeMemory(MemoryLimits limits) {
throw new WasmEngineException("Failed to reserve memory pages");
}
if (nPages > 0) {
- PM.protectPages(
- reservedAddress,
- nPages * osPerWasm,
- PageManager.PROT_READ | PageManager.PROT_WRITE);
+ try {
+ PM.protectPages(
+ reservedAddress,
+ nPages * osPerWasm,
+ PageManager.PROT_READ | PageManager.PROT_WRITE);
+ } catch (Throwable t) {
+ PM.freePages(reservedAddress, reservedOsPages);
+ throw new WasmEngineException("Failed to commit initial memory pages", t);
+ }
}
}
} else {
this.reservedAddress = 0;
}
+
+ this.waiters = new NativeWaiters(limits.shared());
+ this.pagesCell = MEM.allocateMemory(Integer.BYTES, true);
+ if (pagesCell == 0) {
+ freeReservation();
+ throw new WasmEngineException("Failed to allocate the memory page count");
+ }
+ MEM.putInt(pagesCell, nPages);
+ this.atomics =
+ new NativeAtomics(
+ reservedOsPages > 0
+ ? MEM.newDirectByteBuffer(
+ reservedAddress, reservedOsPages * osPageSize)
+ : ByteBuffer.allocateDirect(0));
}
- @Override
- public void close() {
+ private void freeReservation() {
if (reservedOsPages > 0 && reservedAddress != 0) {
if (IS_WINDOWS) {
winVirtualFree(reservedAddress, 0, MEM_RELEASE);
@@ -155,18 +183,32 @@ public void close() {
}
}
+ @Override
+ public void close() {
+ freeReservation();
+ if (pagesCell != 0) {
+ MEM.freeMemory(pagesCell);
+ pagesCell = 0;
+ }
+ }
+
/** Get the native address of the memory buffer, for passing to native code. */
public long nativeAddress() {
return reservedAddress;
}
+ /** Where compiled code reads the current page count of a shared memory. */
+ public long pagesAddress() {
+ return pagesCell;
+ }
+
@Override
public int pages() {
return nPages;
}
@Override
- public int grow(int size) {
+ public synchronized int grow(int size) {
var prevPages = nPages;
var numPages = prevPages + size;
if (numPages > maximumPages() || numPages < prevPages) {
@@ -191,6 +233,8 @@ public int grow(int size) {
numPages * osPerWasm,
PageManager.PROT_READ | PageManager.PROT_WRITE);
}
+ VarHandle.releaseFence();
+ MEM.putInt(pagesCell, numPages);
this.nPages = numPages;
} catch (Throwable t) {
return -1;
@@ -220,87 +264,25 @@ public Object lock(int address) {
if (!shared()) {
return new Object();
}
- return waitStates.computeIfAbsent(address, k -> new WaitState());
- }
-
- private int waitOn(int address, java.util.function.BooleanSupplier condition, long timeout) {
- if (!shared()) {
- throw new WasmEngineException("Attempt to wait on a non-shared memory, not supported.");
- }
-
- long deadline = (timeout < 0) ? Long.MAX_VALUE : System.nanoTime() + timeout;
- WaitState state = waitStates.computeIfAbsent(address, k -> new WaitState());
-
- synchronized (state) {
- if (!condition.getAsBoolean()) {
- return 1; // not-equal
- }
-
- state.waiterCount++;
- try {
- while (state.pendingWakeups == 0) {
- long remaining = deadline - System.nanoTime();
- if (remaining <= 0) {
- return 2; // timeout
- }
- long millis = Math.max(remaining / 1_000_000L, 0);
- int nanos = Math.max((int) (remaining % 1_000_000L), 0);
- try {
- state.wait(millis, nanos);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- throw new WasmEngineException("Thread interrupted");
- }
- }
- return 0; // woken
- } finally {
- if (state.pendingWakeups > 0) {
- state.pendingWakeups--;
- }
- state.waiterCount--;
- }
- }
+ return waiters.monitor(address);
}
@SuppressWarnings("removal")
@Override
public int waitOn(int address, int expected, long timeout) {
- return waitOn(address, () -> readInt(address) == expected, timeout);
+ return waiters.waitOn(address, () -> atomicReadInt(address) == expected, timeout);
}
@SuppressWarnings("removal")
@Override
public int waitOn(int address, long expected, long timeout) {
- return waitOn(address, () -> readLong(address) == expected, timeout);
+ return waiters.waitOn(address, () -> atomicReadLong(address) == expected, timeout);
}
@SuppressWarnings("removal")
@Override
public int notify(int address, int maxThreads) {
- if (!shared()) {
- return 0;
- }
-
- WaitState state = waitStates.get(address);
- if (state == null) {
- return 0;
- }
-
- synchronized (state) {
- int actualWaiters = state.waiterCount - state.pendingWakeups;
- if (actualWaiters == 0) {
- return 0;
- }
- int toWake;
- if (maxThreads < 0) {
- toWake = actualWaiters;
- } else {
- toWake = Math.min(actualWaiters, maxThreads);
- }
- state.pendingWakeups += toWake;
- state.notifyAll();
- return toWake;
- }
+ return waiters.notify(address, maxThreads);
}
@Override
@@ -312,9 +294,9 @@ public void initialize(Instance instance, DataSegment[] dataSegments) {
for (var s : dataSegments) {
if (s instanceof ActiveDataSegment) {
var seg = (ActiveDataSegment) s;
- var data = seg.data();
+ var data = seg.bytes();
var offset = (int) computeConstantValue(instance, seg.offsetInstructions())[0];
- if (offset < 0 || offset + data.length > sizeInBytes()) {
+ if (offset < 0 || (long) offset + data.length > sizeInBytes()) {
throw new UninstantiableException(
"out of bounds memory access: offset="
+ offset
@@ -329,17 +311,13 @@ public void initialize(Instance instance, DataSegment[] dataSegments) {
@Override
public void initPassiveSegment(int segmentId, int dest, int offset, int size) {
var seg = dataSegments[segmentId];
- if (seg == null || seg == PassiveDataSegment.EMPTY) {
- if (size > 0) {
- throw new run.endive.runtime.WasmRuntimeException("out of bounds memory access");
- }
- return;
- }
- write(dest, seg.data(), offset, size);
+ // a dropped segment is empty, and still bounds-checks both offsets
+ var data = (seg == null || seg == PassiveDataSegment.EMPTY) ? NO_DATA : seg.bytes();
+ write(dest, data, offset, size);
}
- private int sizeInBytes() {
- return PAGE_SIZE * nPages;
+ private long sizeInBytes() {
+ return PAGE_SIZE * (long) nPages;
}
/**
@@ -490,4 +468,196 @@ public void drop(int segment) {
dataSegments[segment] = PassiveDataSegment.EMPTY;
}
}
+
+ @Override
+ public int atomicReadInt(int addr) {
+ checkBounds(addr, 4);
+ return atomics.readInt(addr);
+ }
+
+ @Override
+ public long atomicReadLong(int addr) {
+ checkBounds(addr, 8);
+ return atomics.readLong(addr);
+ }
+
+ @Override
+ public short atomicReadShort(int addr) {
+ checkBounds(addr, 2);
+ return atomics.readShort(addr);
+ }
+
+ @Override
+ public byte atomicReadByte(int addr) {
+ checkBounds(addr, 1);
+ return atomics.readByte(addr);
+ }
+
+ @Override
+ public void atomicWriteInt(int addr, int value) {
+ checkBounds(addr, 4);
+ atomics.writeInt(addr, value);
+ }
+
+ @Override
+ public void atomicWriteLong(int addr, long value) {
+ checkBounds(addr, 8);
+ atomics.writeLong(addr, value);
+ }
+
+ @Override
+ public void atomicWriteShort(int addr, short value) {
+ checkBounds(addr, 2);
+ atomics.writeShort(addr, value);
+ }
+
+ @Override
+ public void atomicWriteByte(int addr, byte value) {
+ checkBounds(addr, 1);
+ atomics.writeByte(addr, value);
+ }
+
+ @Override
+ public int atomicAddInt(int addr, int delta) {
+ checkBounds(addr, 4);
+ return atomics.addInt(addr, delta);
+ }
+
+ @Override
+ public int atomicAndInt(int addr, int mask) {
+ checkBounds(addr, 4);
+ return atomics.andInt(addr, mask);
+ }
+
+ @Override
+ public int atomicOrInt(int addr, int mask) {
+ checkBounds(addr, 4);
+ return atomics.orInt(addr, mask);
+ }
+
+ @Override
+ public int atomicXorInt(int addr, int mask) {
+ checkBounds(addr, 4);
+ return atomics.xorInt(addr, mask);
+ }
+
+ @Override
+ public int atomicXchgInt(int addr, int value) {
+ checkBounds(addr, 4);
+ return atomics.xchgInt(addr, value);
+ }
+
+ @Override
+ public int atomicCmpxchgInt(int addr, int expected, int replacement) {
+ checkBounds(addr, 4);
+ return atomics.cmpxchgInt(addr, expected, replacement);
+ }
+
+ @Override
+ public long atomicAddLong(int addr, long delta) {
+ checkBounds(addr, 8);
+ return atomics.addLong(addr, delta);
+ }
+
+ @Override
+ public long atomicAndLong(int addr, long mask) {
+ checkBounds(addr, 8);
+ return atomics.andLong(addr, mask);
+ }
+
+ @Override
+ public long atomicOrLong(int addr, long mask) {
+ checkBounds(addr, 8);
+ return atomics.orLong(addr, mask);
+ }
+
+ @Override
+ public long atomicXorLong(int addr, long mask) {
+ checkBounds(addr, 8);
+ return atomics.xorLong(addr, mask);
+ }
+
+ @Override
+ public long atomicXchgLong(int addr, long value) {
+ checkBounds(addr, 8);
+ return atomics.xchgLong(addr, value);
+ }
+
+ @Override
+ public long atomicCmpxchgLong(int addr, long expected, long replacement) {
+ checkBounds(addr, 8);
+ return atomics.cmpxchgLong(addr, expected, replacement);
+ }
+
+ @Override
+ public short atomicAddShort(int addr, short delta) {
+ checkBounds(addr, 2);
+ return atomics.addShort(addr, delta);
+ }
+
+ @Override
+ public short atomicAndShort(int addr, short mask) {
+ checkBounds(addr, 2);
+ return atomics.andShort(addr, mask);
+ }
+
+ @Override
+ public short atomicOrShort(int addr, short mask) {
+ checkBounds(addr, 2);
+ return atomics.orShort(addr, mask);
+ }
+
+ @Override
+ public short atomicXorShort(int addr, short mask) {
+ checkBounds(addr, 2);
+ return atomics.xorShort(addr, mask);
+ }
+
+ @Override
+ public short atomicXchgShort(int addr, short value) {
+ checkBounds(addr, 2);
+ return atomics.xchgShort(addr, value);
+ }
+
+ @Override
+ public short atomicCmpxchgShort(int addr, short expected, short replacement) {
+ checkBounds(addr, 2);
+ return atomics.cmpxchgShort(addr, expected, replacement);
+ }
+
+ @Override
+ public byte atomicAddByte(int addr, byte delta) {
+ checkBounds(addr, 1);
+ return atomics.addByte(addr, delta);
+ }
+
+ @Override
+ public byte atomicAndByte(int addr, byte mask) {
+ checkBounds(addr, 1);
+ return atomics.andByte(addr, mask);
+ }
+
+ @Override
+ public byte atomicOrByte(int addr, byte mask) {
+ checkBounds(addr, 1);
+ return atomics.orByte(addr, mask);
+ }
+
+ @Override
+ public byte atomicXorByte(int addr, byte mask) {
+ checkBounds(addr, 1);
+ return atomics.xorByte(addr, mask);
+ }
+
+ @Override
+ public byte atomicXchgByte(int addr, byte value) {
+ checkBounds(addr, 1);
+ return atomics.xchgByte(addr, value);
+ }
+
+ @Override
+ public byte atomicCmpxchgByte(int addr, byte expected, byte replacement) {
+ checkBounds(addr, 1);
+ return atomics.cmpxchgByte(addr, expected, replacement);
+ }
}
diff --git a/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/GuestSeesGrowthTest.java b/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/GuestSeesGrowthTest.java
new file mode 100644
index 000000000..b601dc7dc
--- /dev/null
+++ b/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/GuestSeesGrowthTest.java
@@ -0,0 +1,121 @@
+package run.endive.redline.experimental.runner.internal;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Test;
+import run.endive.corpus.CorpusResources;
+import run.endive.redline.experimental.api.NativeCode;
+import run.endive.redline.experimental.api.internal.RedlineTarget;
+import run.endive.redline.experimental.compiler.internal.NativeCompiler;
+import run.endive.redline.experimental.runner.NativeMachineFactory;
+import run.endive.runtime.HostFunction;
+import run.endive.runtime.ImportMemory;
+import run.endive.runtime.ImportValues;
+import run.endive.runtime.Instance;
+import run.endive.runtime.Memory;
+import run.endive.wasm.Parser;
+import run.endive.wasm.WasmModule;
+import run.endive.wasm.types.FunctionType;
+import run.endive.wasm.types.MemoryLimits;
+
+/** Compiled code sees growth by another instance sharing the memory, or by a host import. */
+public class GuestSeesGrowthTest {
+
+ private static final int PAGE = 65536;
+
+ private static final WasmModule SHARED = parse("shared-memory-grow");
+ private static final NativeCode SHARED_CODE = compile(SHARED);
+ private static final WasmModule HOST_GROW = parse("host-grow");
+ private static final NativeCode HOST_GROW_CODE = compile(HOST_GROW);
+
+ private static final ExecutorService POOL = Executors.newCachedThreadPool();
+
+ @AfterAll
+ public static void shutdown() {
+ POOL.shutdownNow();
+ }
+
+ @Test
+ public void aRunningInstanceReadsThePageAnotherInstanceGrew() throws Exception {
+ var memory = NativeMachineFactory.createMemory(new MemoryLimits(1, 2, true));
+ try (var waiter = shared(memory);
+ var grower = shared(memory)) {
+ var read = POOL.submit(() -> waiter.export("waitThenRead").apply(0, PAGE + 16)[0]);
+ Thread.sleep(50);
+ grower.export("growAndPublish").apply(0, PAGE + 16, 42);
+ assertEquals(42L, read.get(30, TimeUnit.SECONDS));
+ }
+ }
+
+ @Test
+ public void aRunningInstanceSeesTheSizeAnotherInstanceGrew() throws Exception {
+ var memory = NativeMachineFactory.createMemory(new MemoryLimits(1, 2, true));
+ try (var waiter = shared(memory);
+ var grower = shared(memory)) {
+ var size = POOL.submit(() -> waiter.export("waitThenSize").apply(0)[0]);
+ Thread.sleep(50);
+ grower.export("growAndPublish").apply(0, PAGE + 16, 42);
+ assertEquals(2L, size.get(30, TimeUnit.SECONDS));
+ }
+ }
+
+ @Test
+ public void theGuestReadsThePageAHostImportGrew() {
+ var memory = NativeMachineFactory.createMemory(new MemoryLimits(1, 2));
+ try (var instance = hostGrow(memory, 77)) {
+ assertEquals(77L, instance.export("growThenRead").apply(PAGE + 8)[0]);
+ }
+ }
+
+ @Test
+ public void theGuestSeesTheSizeAHostImportGrew() {
+ var memory = NativeMachineFactory.createMemory(new MemoryLimits(1, 2));
+ try (var instance = hostGrow(memory, 77)) {
+ assertEquals(2L, instance.export("growThenSize").apply()[0]);
+ }
+ }
+
+ private static Instance shared(Memory memory) {
+ return NativeMachineFactory.builder(SHARED)
+ .withPrecompiledCode(SHARED_CODE)
+ .withImportValues(
+ ImportValues.builder()
+ .addMemory(new ImportMemory("env", "memory", memory))
+ .build())
+ .build();
+ }
+
+ private static Instance hostGrow(Memory memory, int value) {
+ var grow =
+ new HostFunction(
+ "env",
+ "grow",
+ FunctionType.of(List.of(), List.of()),
+ (instance, args) -> {
+ memory.grow(1);
+ memory.writeI32(PAGE + 8, value);
+ return null;
+ });
+ return NativeMachineFactory.builder(HOST_GROW)
+ .withPrecompiledCode(HOST_GROW_CODE)
+ .withImportValues(
+ ImportValues.builder()
+ .addMemory(new ImportMemory("env", "memory", memory))
+ .addFunction(grow)
+ .build())
+ .build();
+ }
+
+ private static WasmModule parse(String name) {
+ return Parser.parse(CorpusResources.getResource("compiled/" + name + ".wat.wasm"));
+ }
+
+ private static NativeCode compile(WasmModule module) {
+ return NativeCompiler.compile(RedlineTarget.detectHost().orElseThrow().triple(), module);
+ }
+}
diff --git a/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/MemoryBoundsTest.java b/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/MemoryBoundsTest.java
index 855e18d25..6febccfe9 100644
--- a/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/MemoryBoundsTest.java
+++ b/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/MemoryBoundsTest.java
@@ -1,10 +1,13 @@
package run.endive.redline.experimental.runner.internal;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import org.junit.jupiter.api.Test;
import run.endive.redline.experimental.runner.NativeMachineFactory;
+import run.endive.runtime.Memory;
import run.endive.runtime.WasmRuntimeException;
+import run.endive.wasm.UninstantiableException;
import run.endive.wasm.types.MemoryLimits;
/**
@@ -41,4 +44,31 @@ public void insideTheMemoryIsUntouched() {
memory.writeI32(PAGE - 4, 0x11223344);
org.junit.jupiter.api.Assertions.assertEquals(0x11223344, memory.readInt(PAGE - 4));
}
+
+ @Test
+ public void wideAccessesAreLittleEndianAtOddOffsets() {
+ var memory = NativeMachineFactory.createMemory(new MemoryLimits(1, 1));
+ for (int i = 0; i < 8; i++) {
+ memory.writeByte(101 + i, (byte) (i + 1));
+ }
+ assertEquals((short) 0x0201, memory.readShort(101));
+ assertEquals(0x0807060504030201L, memory.readLong(101));
+
+ memory.writeShort(201, (short) 0x0201);
+ memory.writeLong(301, 0x0807060504030201L);
+ assertEquals(1, memory.read(201));
+ assertEquals(2, memory.read(202));
+ for (int i = 0; i < 8; i++) {
+ assertEquals(i + 1, memory.read(301 + i));
+ }
+ }
+
+ @Test
+ public void anInitialSizePastTheRuntimeLimitIsRejected() {
+ assertThrows(
+ UninstantiableException.class,
+ () ->
+ NativeMachineFactory.createMemory(
+ new MemoryLimits(Memory.RUNTIME_MAX_PAGES + 1)));
+ }
}
diff --git a/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/MemoryInitTrapTest.java b/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/MemoryInitTrapTest.java
new file mode 100644
index 000000000..363ceb059
--- /dev/null
+++ b/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/MemoryInitTrapTest.java
@@ -0,0 +1,54 @@
+package run.endive.redline.experimental.runner.internal;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import org.junit.jupiter.api.Test;
+import run.endive.corpus.CorpusResources;
+import run.endive.redline.experimental.api.NativeCode;
+import run.endive.redline.experimental.api.internal.RedlineTarget;
+import run.endive.redline.experimental.compiler.internal.NativeCompiler;
+import run.endive.redline.experimental.runner.NativeMachineFactory;
+import run.endive.runtime.Instance;
+import run.endive.runtime.WasmRuntimeException;
+import run.endive.wasm.Parser;
+import run.endive.wasm.WasmModule;
+
+/** A trap raised while the runner handles memory.init has to stop the guest there. */
+public class MemoryInitTrapTest {
+
+ private static final WasmModule MODULE =
+ Parser.parse(CorpusResources.getResource("compiled/memory-init-trap.wat.wasm"));
+ private static final NativeCode CODE =
+ NativeCompiler.compile(RedlineTarget.detectHost().orElseThrow().triple(), MODULE);
+
+ @Test
+ public void anOutOfBoundsInitStopsBeforeTheNextInstruction() {
+ try (var instance = instance()) {
+ assertThrows(
+ WasmRuntimeException.class,
+ () -> instance.export("initOutOfBoundsThenStore").apply());
+ assertEquals(0L, instance.export("read").apply(100)[0]);
+ }
+ }
+
+ @Test
+ public void anEmptyInitPastTheEndOfADroppedSegmentTraps() {
+ try (var instance = instance()) {
+ assertThrows(
+ WasmRuntimeException.class,
+ () -> instance.export("dropThenInitPastEnd").apply());
+ }
+ }
+
+ @Test
+ public void anEmptyInitAtTheStartOfADroppedSegmentIsAllowed() {
+ try (var instance = instance()) {
+ instance.export("dropThenInitNothing").apply();
+ }
+ }
+
+ private static Instance instance() {
+ return NativeMachineFactory.builder(MODULE).withPrecompiledCode(CODE).build();
+ }
+}
diff --git a/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/NativeAtomicsTest.java b/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/NativeAtomicsTest.java
new file mode 100644
index 000000000..0b8986d8a
--- /dev/null
+++ b/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/NativeAtomicsTest.java
@@ -0,0 +1,145 @@
+package run.endive.redline.experimental.runner.internal;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Test;
+import run.endive.corpus.CorpusResources;
+import run.endive.redline.experimental.api.NativeCode;
+import run.endive.redline.experimental.api.internal.RedlineTarget;
+import run.endive.redline.experimental.compiler.internal.NativeCompiler;
+import run.endive.redline.experimental.runner.NativeMachineFactory;
+import run.endive.runtime.ImportMemory;
+import run.endive.runtime.ImportValues;
+import run.endive.runtime.Instance;
+import run.endive.runtime.Memory;
+import run.endive.wasm.Parser;
+import run.endive.wasm.WasmModule;
+import run.endive.wasm.types.MemoryLimits;
+
+/** Host atomics on a native memory have to be atomic against the guest's own. */
+public class NativeAtomicsTest {
+
+ private static final int N = 1_000_000;
+
+ private static final WasmModule MODULE =
+ Parser.parse(CorpusResources.getResource("compiled/atomic-counter.wat.wasm"));
+ private static final NativeCode CODE =
+ NativeCompiler.compile(RedlineTarget.detectHost().orElseThrow().triple(), MODULE);
+
+ private static final ExecutorService POOL = Executors.newSingleThreadExecutor();
+
+ @AfterAll
+ public static void shutdown() {
+ POOL.shutdownNow();
+ }
+
+ @Test
+ public void hostAndGuestIntAddsDoNotLoseIncrements() throws Exception {
+ var memory = sharedMemory();
+ race(memory, "add32", () -> memory.atomicAddInt(0, 1));
+ assertEquals(2 * N, memory.readInt(0));
+ }
+
+ @Test
+ public void hostAndGuestLongAddsDoNotLoseIncrements() throws Exception {
+ var memory = sharedMemory();
+ race(memory, "add64", () -> memory.atomicAddLong(16, 1L));
+ assertEquals(2L * N, memory.readLong(16));
+ }
+
+ @Test
+ public void hostAndGuestShortAddsDoNotLoseIncrements() throws Exception {
+ var memory = sharedMemory();
+ race(memory, "add16", () -> memory.atomicAddShort(10, (short) 1));
+ assertEquals((short) (2 * N), memory.readShort(10));
+ assertEquals(0, memory.readShort(8), "the other half of the int is untouched");
+ }
+
+ @Test
+ public void hostAndGuestByteAddsDoNotLoseIncrements() throws Exception {
+ var memory = sharedMemory();
+ race(memory, "add8", () -> memory.atomicAddByte(13, (byte) 1));
+ assertEquals((byte) (2 * N), memory.read(13));
+ assertEquals(0, memory.read(12), "neighbouring bytes are untouched");
+ assertEquals(0, memory.read(14));
+ assertEquals(0, memory.read(15));
+ }
+
+ @Test
+ public void narrowOperationsOnlyTouchTheirBytes() {
+ var memory = sharedMemory();
+ memory.writeI32(0, 0x44332211);
+
+ assertEquals((byte) 0x22, memory.atomicXchgByte(1, (byte) 0x99));
+ assertEquals((byte) 0x33, memory.atomicCmpxchgByte(2, (byte) 0x00, (byte) 0x77));
+ assertEquals((byte) 0x33, memory.atomicCmpxchgByte(2, (byte) 0x33, (byte) 0x77));
+ assertEquals((short) 0x9911, memory.atomicOrShort(0, (short) 0x0002));
+ assertEquals(0x44779913, memory.atomicReadInt(0));
+
+ assertEquals((short) 0x4477, memory.atomicAndShort(2, (short) 0x00FF));
+ assertEquals((short) 0x0077, memory.atomicXorShort(2, (short) 0x0101));
+ assertEquals((short) 0x0176, memory.atomicReadShort(2));
+ memory.atomicWriteByte(3, (byte) 0xAB);
+ assertEquals((byte) 0xAB, memory.atomicReadByte(3));
+ assertEquals(0xAB769913, memory.readInt(0));
+ }
+
+ @Test
+ public void wideOperationsBehaveLikeTheirScalarForms() {
+ var memory = sharedMemory();
+ memory.atomicWriteLong(8, 0x0F0F0F0F0F0F0F0FL);
+ assertEquals(0x0F0F0F0F0F0F0F0FL, memory.atomicAndLong(8, 0x00FF00FF00FF00FFL));
+ assertEquals(0x000F000F000F000FL, memory.atomicOrLong(8, 0x1000000000000000L));
+ assertEquals(0x100F000F000F000FL, memory.atomicXorLong(8, 0x100F000F000F000FL));
+ assertEquals(0L, memory.atomicCmpxchgLong(8, 0L, 5L));
+ assertEquals(5L, memory.atomicXchgLong(8, 6L));
+ assertEquals(6L, memory.atomicReadLong(8));
+
+ memory.atomicWriteInt(4, 10);
+ assertEquals(10, memory.atomicCmpxchgInt(4, 11, 12));
+ assertEquals(10, memory.atomicCmpxchgInt(4, 10, 12));
+ assertEquals(12, memory.atomicXchgInt(4, 1));
+ assertEquals(1, memory.atomicOrInt(4, 6));
+ assertEquals(7, memory.atomicAndInt(4, 5));
+ assertEquals(5, memory.atomicXorInt(4, 1));
+ assertEquals(4, memory.atomicReadInt(4));
+ }
+
+ private static Memory sharedMemory() {
+ return NativeMachineFactory.createMemory(new MemoryLimits(1, 1, true));
+ }
+
+ private static void race(Memory memory, String export, Runnable hostIncrement)
+ throws Exception {
+ try (var instance = instance(memory)) {
+ var start = new CyclicBarrier(2);
+ var guest =
+ POOL.submit(
+ () -> {
+ start.await();
+ instance.export(export).apply(N);
+ return null;
+ });
+ start.await();
+ for (int i = 0; i < N; i++) {
+ hostIncrement.run();
+ }
+ guest.get(60, TimeUnit.SECONDS);
+ }
+ }
+
+ private static Instance instance(Memory memory) {
+ return NativeMachineFactory.builder(MODULE)
+ .withPrecompiledCode(CODE)
+ .withImportValues(
+ ImportValues.builder()
+ .addMemory(new ImportMemory("env", "memory", memory))
+ .build())
+ .build();
+ }
+}
diff --git a/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/NativeMemoryConcurrencyTest.java b/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/NativeMemoryConcurrencyTest.java
new file mode 100644
index 000000000..b1c151b0b
--- /dev/null
+++ b/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/NativeMemoryConcurrencyTest.java
@@ -0,0 +1,83 @@
+package run.endive.redline.experimental.runner.internal;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import org.junit.jupiter.api.Test;
+import run.endive.redline.experimental.runner.NativeMachineFactory;
+import run.endive.wasm.types.MemoryLimits;
+
+/** A shared memory is grown and read by several host threads at once. */
+public class NativeMemoryConcurrencyTest {
+
+ private static final int PAGE = 65536;
+
+ @Test
+ public void concurrentGrowsHandOutEveryPageOnce() throws Exception {
+ int threads = 8;
+ int growsPerThread = 200;
+ var memory =
+ NativeMachineFactory.createMemory(
+ new MemoryLimits(0, threads * growsPerThread, true));
+ Set handedOut = ConcurrentHashMap.newKeySet();
+ var start = new CountDownLatch(1);
+ ExecutorService pool = Executors.newFixedThreadPool(threads);
+ try {
+ var futures = new java.util.ArrayList>();
+ for (int t = 0; t < threads; t++) {
+ futures.add(
+ pool.submit(
+ () -> {
+ start.await();
+ for (int i = 0; i < growsPerThread; i++) {
+ assertTrue(handedOut.add(memory.grow(1)));
+ }
+ return null;
+ }));
+ }
+ start.countDown();
+ for (var f : futures) {
+ f.get(30, TimeUnit.SECONDS);
+ }
+ } finally {
+ pool.shutdownNow();
+ }
+ assertEquals(threads * growsPerThread, memory.pages());
+ assertEquals(threads * growsPerThread, handedOut.size());
+ }
+
+ // The reader polls with plain reads, the pattern that let the JIT keep a stale view
+ @Test
+ public void aHostReaderCanReachPagesAnotherThreadGrew() throws Exception {
+ ExecutorService pool = Executors.newSingleThreadExecutor();
+ try {
+ for (int round = 0; round < 20; round++) {
+ var memory = NativeMachineFactory.createMemory(new MemoryLimits(1, 2, true));
+ Future reader =
+ pool.submit(
+ () -> {
+ long deadline = System.nanoTime() + 10_000_000_000L;
+ while (memory.readInt(0) != 1) {
+ if (System.nanoTime() > deadline) {
+ throw new AssertionError("flag never seen");
+ }
+ }
+ return memory.readInt(PAGE + 16);
+ });
+ Thread.sleep(5);
+ memory.grow(1);
+ memory.atomicWriteInt(0, 1);
+ reader.get(30, TimeUnit.SECONDS);
+ }
+ } finally {
+ pool.shutdownNow();
+ }
+ }
+}
diff --git a/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/NativeWaitNotifyTest.java b/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/NativeWaitNotifyTest.java
new file mode 100644
index 000000000..973d98554
--- /dev/null
+++ b/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/NativeWaitNotifyTest.java
@@ -0,0 +1,93 @@
+package run.endive.redline.experimental.runner.internal;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import org.junit.jupiter.api.Test;
+import run.endive.corpus.CorpusResources;
+import run.endive.redline.experimental.api.internal.RedlineTarget;
+import run.endive.redline.experimental.compiler.internal.NativeCompiler;
+import run.endive.redline.experimental.runner.NativeMachineFactory;
+import run.endive.runtime.ImportMemory;
+import run.endive.runtime.ImportValues;
+import run.endive.runtime.WasmInterruptedException;
+import run.endive.wasm.Parser;
+import run.endive.wasm.types.MemoryLimits;
+
+public class NativeWaitNotifyTest {
+
+ // Ported from the runtime's MemoryTest (#107): a late wait can't take the parked one's wakeup
+ @Test
+ public void notifyWakesTheParkedThreadNotALateArrival() throws Exception {
+ final int addr = 0;
+ final int sentinel = 42;
+ var memory = NativeMachineFactory.createMemory(new MemoryLimits(1, 1, true));
+
+ for (int round = 0; round < 200; round++) {
+ memory.atomicWriteInt(addr, sentinel);
+ var parkedReady = new CountDownLatch(1);
+ var parked =
+ new Thread(
+ () -> {
+ parkedReady.countDown();
+ memory.atomicWait(addr, sentinel, 100_000_000L);
+ });
+ parked.setDaemon(true);
+ parked.start();
+ parkedReady.await();
+ Thread.sleep(1);
+
+ if (memory.atomicNotify(addr, 1) == 0) {
+ continue;
+ }
+ assertEquals(2, memory.atomicWait(addr, sentinel, 0L), "round " + round);
+ }
+ }
+
+ @Test
+ public void interruptingAGuestBlockedInAWaitStopsIt() throws Exception {
+ var module = Parser.parse(CorpusResources.getResource("compiled/threads-example.wat.wasm"));
+ var memory = NativeMachineFactory.createMemory(new MemoryLimits(1, 1, true));
+ try (var instance =
+ NativeMachineFactory.builder(module)
+ .withPrecompiledCode(
+ NativeCompiler.compile(
+ RedlineTarget.detectHost().orElseThrow().triple(), module))
+ .withImportValues(
+ ImportValues.builder()
+ .addMemory(new ImportMemory("env", "memory", memory))
+ .build())
+ .build()) {
+ // locked, so lockMutex parks in memory.atomic.wait32
+ memory.atomicWriteInt(0, 1);
+ var failure = new AtomicReference();
+ var blocked =
+ new Thread(
+ () -> {
+ try {
+ instance.export("lockMutex").apply(0);
+ } catch (Throwable t) {
+ failure.set(t);
+ }
+ });
+ blocked.setDaemon(true);
+ blocked.start();
+
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30);
+ while (blocked.getState() != Thread.State.WAITING
+ && blocked.getState() != Thread.State.TIMED_WAITING) {
+ assertTrue(System.nanoTime() < deadline, "never parked");
+ Thread.sleep(1);
+ }
+ blocked.interrupt();
+ blocked.join(TimeUnit.SECONDS.toMillis(30));
+
+ assertTrue(!blocked.isAlive(), "the interrupted guest kept running");
+ assertInstanceOf(WasmInterruptedException.class, failure.get());
+ }
+ }
+}
diff --git a/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/NativeMachine.java b/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/NativeMachine.java
index 0c396e4a9..25df60753 100644
--- a/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/NativeMachine.java
+++ b/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/NativeMachine.java
@@ -454,6 +454,14 @@ private MemorySegment createImportStub(int funcId, FunctionType funcType) {
* Dispatches a function call from native code. Reads args from ctxBuffer.
* Called by import upcall stubs and bridge stubs for non-native functions.
*/
+ // Host code may have grown the memory since compiled code last saw its size
+ private void refreshMemoryPages() {
+ var mem = instance.memory();
+ if (mem != null) {
+ ctxBuffer.set(ValueLayout.JAVA_INT, CtxBuffer.MEMORY_PAGES, mem.pages());
+ }
+ }
+
@SuppressWarnings("unused")
private long importDispatchDirect(int funcId) {
try {
@@ -465,6 +473,7 @@ private long importDispatchDirect(int funcId) {
if (funcId < numImports) {
var importFunc = instance.imports().function(funcId);
long[] result = importFunc.handle().apply(instance, args);
+ refreshMemoryPages();
if (result == null || result.length == 0) {
return 0L;
}
@@ -929,6 +938,10 @@ public long[] call(int funcId, long[] args) throws WasmEngineException {
ValueLayout.JAVA_LONG,
CtxBuffer.MEM_BASE_ADDR,
cachedMemBase.address());
+ ctxBuffer.set(
+ ValueLayout.JAVA_LONG,
+ CtxBuffer.MEMORY_PAGES_PTR,
+ nativeMemory.pagesAddress().address());
} else if (mem != null) {
throw new WasmEngineException(foreignImportMessage("memory", mem));
} else {
@@ -936,11 +949,7 @@ public long[] call(int funcId, long[] args) throws WasmEngineException {
}
memBaseInitialized = true;
}
- // MEMORY_PAGES can change via grow() — must refresh every call
- var mem = instance.memory();
- if (mem != null) {
- ctxBuffer.set(ValueLayout.JAVA_INT, CtxBuffer.MEMORY_PAGES, mem.pages());
- }
+ refreshMemoryPages();
if (Thread.currentThread().isInterrupted()) {
throw new WasmInterruptedException("Thread interrupted");
diff --git a/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/NativeMemory.java b/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/NativeMemory.java
index a76c0fb78..f8b67600b 100644
--- a/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/NativeMemory.java
+++ b/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/NativeMemory.java
@@ -4,9 +4,11 @@
import java.lang.foreign.MemorySegment;
import java.lang.foreign.ValueLayout;
+import java.lang.invoke.VarHandle;
+import java.nio.ByteBuffer;
import java.nio.ByteOrder;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import run.endive.redline.experimental.api.internal.NativeAtomics;
+import run.endive.redline.experimental.api.internal.NativeWaiters;
import run.endive.runtime.Instance;
import run.endive.runtime.Memory;
import run.endive.wasm.UninstantiableException;
@@ -25,44 +27,69 @@
*/
public final class NativeMemory implements Memory, AutoCloseable {
- private static final class WaitState {
- int waiterCount;
- int pendingWakeups;
- }
-
- private final Map waitStates = new ConcurrentHashMap<>();
- private final MemoryLimits limits;
- private final MemorySegment reserved;
- // Build the value layouts at compile time and use known JIT value for lookups instead of building object for each call.
- private static final ValueLayout.OfInt INT_LE =
+ // Static final so the JIT folds each layout's VarHandle; withOrder() allocates per call.
+ private static final ValueLayout.OfInt INT_UNALIGNED_LE =
ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
- private static final ValueLayout.OfLong LONG_LE =
+ private static final ValueLayout.OfLong LONG_UNALIGNED_LE =
ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
- private static final ValueLayout.OfShort SHORT_LE =
+ private static final ValueLayout.OfShort SHORT_UNALIGNED_LE =
ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
+ private static final VarHandle PAGES_CELL = ValueLayout.JAVA_INT.varHandle();
+ private static final byte[] NO_DATA = new byte[0];
- private MemorySegment segment;
- private int nPages;
+ private final MemoryLimits limits;
+ private final MemorySegment reserved;
private final long reservedSize;
+ private final NativeWaiters waiters;
+ private final NativeAtomics atomics;
+ // Page count compiled code reads for a shared memory, which other instances may grow
+ private MemorySegment pagesCell;
+ private volatile MemorySegment segment;
+ private volatile int nPages;
private DataSegment[] dataSegments;
public NativeMemory(MemoryLimits limits) {
this.limits = limits;
- this.nPages = limits.initialPages();
int maxPages = Math.min(limits.maximumPages(), RUNTIME_MAX_PAGES);
+ if (limits.initialPages() > maxPages) {
+ throw new UninstantiableException(
+ "memory of "
+ + limits.initialPages()
+ + " pages exceeds the limit of "
+ + maxPages
+ + " pages");
+ }
+ this.nPages = limits.initialPages();
this.reservedSize = PAGE_SIZE * (long) maxPages;
+ this.waiters = new NativeWaiters(limits.shared());
+ MemorySegment mapping = MemorySegment.NULL;
try {
- this.reserved = PanamaExecutor.mmapNoAccess(reservedSize);
-
+ mapping = PanamaExecutor.mmapNoAccess(reservedSize);
if (nPages > 0) {
- PanamaExecutor.mprotectReadWrite(reserved, PAGE_SIZE * (long) nPages);
+ PanamaExecutor.mprotectReadWrite(mapping, PAGE_SIZE * (long) nPages);
}
+ this.pagesCell = PanamaExecutor.malloc(Integer.BYTES);
} catch (Throwable t) {
- throw new WasmEngineException("Failed to mmap native memory", t);
+ var failure = new WasmEngineException("Failed to mmap native memory", t);
+ if (reservedSize > 0 && mapping.address() != 0) {
+ try {
+ PanamaExecutor.munmap(mapping, reservedSize);
+ } catch (Throwable e) {
+ failure.addSuppressed(e);
+ }
+ }
+ throw failure;
}
+ this.reserved = mapping;
+ PAGES_CELL.setVolatile(pagesCell, 0L, nPages);
this.segment = reserved.reinterpret(PAGE_SIZE * (long) nPages);
+ this.atomics =
+ new NativeAtomics(
+ reservedSize > 0
+ ? reserved.reinterpret(reservedSize).asByteBuffer()
+ : ByteBuffer.allocateDirect(0));
}
@Override
@@ -74,6 +101,14 @@ public void close() {
throw new WasmEngineException("Failed to unmap native memory", e);
}
}
+ if (pagesCell.address() != 0) {
+ try {
+ PanamaExecutor.free(pagesCell);
+ } catch (Throwable e) {
+ throw new WasmEngineException("Failed to free native memory page count", e);
+ }
+ pagesCell = MemorySegment.NULL;
+ }
}
/** Get the native address of the memory buffer, for passing to native code. */
@@ -81,13 +116,18 @@ public MemorySegment nativeAddress() {
return segment;
}
+ /** Where compiled code reads the current page count of a shared memory. */
+ public MemorySegment pagesAddress() {
+ return pagesCell;
+ }
+
@Override
public int pages() {
return nPages;
}
@Override
- public int grow(int size) {
+ public synchronized int grow(int size) {
var prevPages = nPages;
var numPages = prevPages + size;
if (numPages > maximumPages() || numPages < prevPages) {
@@ -97,6 +137,8 @@ public int grow(int size) {
try {
long newSize = PAGE_SIZE * (long) numPages;
PanamaExecutor.mprotectReadWrite(reserved, newSize);
+ PAGES_CELL.setVolatile(pagesCell, 0L, numPages);
+ // the segment first, so a host that sees the new page count can access it
this.segment = reserved.reinterpret(newSize);
this.nPages = numPages;
} catch (Throwable t) {
@@ -127,87 +169,25 @@ public Object lock(int address) {
if (!shared()) {
return new Object();
}
- return waitStates.computeIfAbsent(address, k -> new WaitState());
- }
-
- private int waitOn(int address, java.util.function.BooleanSupplier condition, long timeout) {
- if (!shared()) {
- throw new WasmEngineException("Attempt to wait on a non-shared memory, not supported.");
- }
-
- long deadline = (timeout < 0) ? Long.MAX_VALUE : System.nanoTime() + timeout;
- WaitState state = waitStates.computeIfAbsent(address, k -> new WaitState());
-
- synchronized (state) {
- if (!condition.getAsBoolean()) {
- return 1; // not-equal
- }
-
- state.waiterCount++;
- try {
- while (state.pendingWakeups == 0) {
- long remaining = deadline - System.nanoTime();
- if (remaining <= 0) {
- return 2; // timeout
- }
- long millis = Math.max(remaining / 1_000_000L, 0);
- int nanos = Math.max((int) (remaining % 1_000_000L), 0);
- try {
- state.wait(millis, nanos);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- throw new WasmEngineException("Thread interrupted");
- }
- }
- return 0; // woken
- } finally {
- if (state.pendingWakeups > 0) {
- state.pendingWakeups--;
- }
- state.waiterCount--;
- }
- }
+ return waiters.monitor(address);
}
@SuppressWarnings("removal")
@Override
public int waitOn(int address, int expected, long timeout) {
- return waitOn(address, () -> readInt(address) == expected, timeout);
+ return waiters.waitOn(address, () -> atomicReadInt(address) == expected, timeout);
}
@SuppressWarnings("removal")
@Override
public int waitOn(int address, long expected, long timeout) {
- return waitOn(address, () -> readLong(address) == expected, timeout);
+ return waiters.waitOn(address, () -> atomicReadLong(address) == expected, timeout);
}
@SuppressWarnings("removal")
@Override
public int notify(int address, int maxThreads) {
- if (!shared()) {
- return 0;
- }
-
- WaitState state = waitStates.get(address);
- if (state == null) {
- return 0;
- }
-
- synchronized (state) {
- int actualWaiters = state.waiterCount - state.pendingWakeups;
- if (actualWaiters == 0) {
- return 0;
- }
- int toWake;
- if (maxThreads < 0) {
- toWake = actualWaiters;
- } else {
- toWake = Math.min(actualWaiters, maxThreads);
- }
- state.pendingWakeups += toWake;
- state.notifyAll();
- return toWake;
- }
+ return waiters.notify(address, maxThreads);
}
@Override
@@ -218,9 +198,9 @@ public void initialize(Instance instance, DataSegment[] dataSegments) {
}
for (var s : dataSegments) {
if (s instanceof ActiveDataSegment seg) {
- var data = seg.data();
+ var data = seg.bytes();
var offset = (int) computeConstantValue(instance, seg.offsetInstructions())[0];
- if (offset < 0 || offset + data.length > sizeInBytes()) {
+ if (offset < 0 || (long) offset + data.length > sizeInBytes()) {
throw new UninstantiableException(
"out of bounds memory access: offset="
+ offset
@@ -235,27 +215,23 @@ public void initialize(Instance instance, DataSegment[] dataSegments) {
@Override
public void initPassiveSegment(int segmentId, int dest, int offset, int size) {
var seg = dataSegments[segmentId];
- if (seg == null || seg == run.endive.wasm.types.PassiveDataSegment.EMPTY) {
- if (size > 0) {
- throw new run.endive.runtime.WasmRuntimeException("out of bounds memory access");
- }
- return;
- }
- write(dest, seg.data(), offset, size);
+ // a dropped segment is empty, and still bounds-checks both offsets
+ var data = (seg == null || seg == PassiveDataSegment.EMPTY) ? NO_DATA : seg.bytes();
+ write(dest, data, offset, size);
}
- private int sizeInBytes() {
- return PAGE_SIZE * nPages;
+ private long sizeInBytes() {
+ return PAGE_SIZE * (long) nPages;
}
@Override
public void write(int addr, byte[] data, int offset, int size) {
- long limit = segment.byteSize();
+ var view = segment;
if (Integer.toUnsignedLong(offset) + Integer.toUnsignedLong(size) > data.length
- || Integer.toUnsignedLong(addr) + Integer.toUnsignedLong(size) > limit) {
+ || Integer.toUnsignedLong(addr) + Integer.toUnsignedLong(size) > view.byteSize()) {
throw new run.endive.runtime.WasmRuntimeException("out of bounds memory access");
}
- MemorySegment.copy(MemorySegment.ofArray(data), offset, segment, addr, size);
+ MemorySegment.copy(MemorySegment.ofArray(data), offset, view, addr, size);
}
/**
@@ -268,6 +244,13 @@ private static run.endive.runtime.WasmRuntimeException outOfBounds(int addr) {
"out of bounds memory access: attempted to access address: " + addr);
}
+ private int checkAtomic(int addr, int size) {
+ if (Integer.toUnsignedLong(addr) + size > segment.byteSize()) {
+ throw outOfBounds(addr);
+ }
+ return addr;
+ }
+
@Override
public byte read(int addr) {
try {
@@ -279,17 +262,20 @@ public byte read(int addr) {
@Override
public byte[] readBytes(int addr, int len) {
- try {
- return segment.asSlice(addr, len).toArray(ValueLayout.JAVA_BYTE);
- } catch (IndexOutOfBoundsException e) {
+ var view = segment;
+ if (len < 0
+ || Integer.toUnsignedLong(addr) + Integer.toUnsignedLong(len) > view.byteSize()) {
throw outOfBounds(addr);
}
+ byte[] result = new byte[len];
+ MemorySegment.copy(view, ValueLayout.JAVA_BYTE, addr, result, 0, len);
+ return result;
}
@Override
public void writeI32(int addr, int data) {
try {
- segment.set(INT_LE, addr, data);
+ segment.set(INT_UNALIGNED_LE, addr, data);
} catch (IndexOutOfBoundsException e) {
throw outOfBounds(addr);
}
@@ -298,7 +284,7 @@ public void writeI32(int addr, int data) {
@Override
public int readInt(int addr) {
try {
- return segment.get(INT_LE, addr);
+ return segment.get(INT_UNALIGNED_LE, addr);
} catch (IndexOutOfBoundsException e) {
throw outOfBounds(addr);
}
@@ -307,7 +293,7 @@ public int readInt(int addr) {
@Override
public void writeLong(int addr, long data) {
try {
- segment.set(LONG_LE, addr, data);
+ segment.set(LONG_UNALIGNED_LE, addr, data);
} catch (IndexOutOfBoundsException e) {
throw outOfBounds(addr);
}
@@ -316,7 +302,7 @@ public void writeLong(int addr, long data) {
@Override
public long readLong(int addr) {
try {
- return segment.get(LONG_LE, addr);
+ return segment.get(LONG_UNALIGNED_LE, addr);
} catch (IndexOutOfBoundsException e) {
throw outOfBounds(addr);
}
@@ -325,7 +311,7 @@ public long readLong(int addr) {
@Override
public void writeShort(int addr, short data) {
try {
- segment.set(SHORT_LE, addr, data);
+ segment.set(SHORT_UNALIGNED_LE, addr, data);
} catch (IndexOutOfBoundsException e) {
throw outOfBounds(addr);
}
@@ -334,7 +320,7 @@ public void writeShort(int addr, short data) {
@Override
public short readShort(int addr) {
try {
- return segment.get(SHORT_LE, addr);
+ return segment.get(SHORT_UNALIGNED_LE, addr);
} catch (IndexOutOfBoundsException e) {
throw outOfBounds(addr);
}
@@ -391,23 +377,25 @@ public void zero() {
@Override
public void copy(int dest, int src, int size) {
- long limit = segment.byteSize();
+ var view = segment;
+ long limit = view.byteSize();
if (Integer.toUnsignedLong(src) + Integer.toUnsignedLong(size) > limit
|| Integer.toUnsignedLong(dest) + Integer.toUnsignedLong(size) > limit) {
throw new run.endive.runtime.WasmRuntimeException("out of bounds memory access");
}
- MemorySegment.copy(segment, src, segment, dest, size);
+ MemorySegment.copy(view, src, view, dest, size);
}
@Override
public void fill(byte value, int fromIndex, int toIndex) {
- long limit = segment.byteSize();
+ var view = segment;
+ long limit = view.byteSize();
if (Integer.toUnsignedLong(fromIndex) > limit
|| Integer.toUnsignedLong(toIndex) > limit
|| fromIndex > toIndex) {
throw new run.endive.runtime.WasmRuntimeException("out of bounds memory access");
}
- segment.asSlice(fromIndex, toIndex - fromIndex).fill(value);
+ view.asSlice(fromIndex, toIndex - fromIndex).fill(value);
}
@Override
@@ -416,4 +404,164 @@ public void drop(int segment) {
dataSegments[segment] = PassiveDataSegment.EMPTY;
}
}
+
+ @Override
+ public int atomicReadInt(int addr) {
+ return atomics.readInt(checkAtomic(addr, 4));
+ }
+
+ @Override
+ public long atomicReadLong(int addr) {
+ return atomics.readLong(checkAtomic(addr, 8));
+ }
+
+ @Override
+ public short atomicReadShort(int addr) {
+ return atomics.readShort(checkAtomic(addr, 2));
+ }
+
+ @Override
+ public byte atomicReadByte(int addr) {
+ return atomics.readByte(checkAtomic(addr, 1));
+ }
+
+ @Override
+ public void atomicWriteInt(int addr, int value) {
+ atomics.writeInt(checkAtomic(addr, 4), value);
+ }
+
+ @Override
+ public void atomicWriteLong(int addr, long value) {
+ atomics.writeLong(checkAtomic(addr, 8), value);
+ }
+
+ @Override
+ public void atomicWriteShort(int addr, short value) {
+ atomics.writeShort(checkAtomic(addr, 2), value);
+ }
+
+ @Override
+ public void atomicWriteByte(int addr, byte value) {
+ atomics.writeByte(checkAtomic(addr, 1), value);
+ }
+
+ @Override
+ public int atomicAddInt(int addr, int delta) {
+ return atomics.addInt(checkAtomic(addr, 4), delta);
+ }
+
+ @Override
+ public int atomicAndInt(int addr, int mask) {
+ return atomics.andInt(checkAtomic(addr, 4), mask);
+ }
+
+ @Override
+ public int atomicOrInt(int addr, int mask) {
+ return atomics.orInt(checkAtomic(addr, 4), mask);
+ }
+
+ @Override
+ public int atomicXorInt(int addr, int mask) {
+ return atomics.xorInt(checkAtomic(addr, 4), mask);
+ }
+
+ @Override
+ public int atomicXchgInt(int addr, int value) {
+ return atomics.xchgInt(checkAtomic(addr, 4), value);
+ }
+
+ @Override
+ public int atomicCmpxchgInt(int addr, int expected, int replacement) {
+ return atomics.cmpxchgInt(checkAtomic(addr, 4), expected, replacement);
+ }
+
+ @Override
+ public long atomicAddLong(int addr, long delta) {
+ return atomics.addLong(checkAtomic(addr, 8), delta);
+ }
+
+ @Override
+ public long atomicAndLong(int addr, long mask) {
+ return atomics.andLong(checkAtomic(addr, 8), mask);
+ }
+
+ @Override
+ public long atomicOrLong(int addr, long mask) {
+ return atomics.orLong(checkAtomic(addr, 8), mask);
+ }
+
+ @Override
+ public long atomicXorLong(int addr, long mask) {
+ return atomics.xorLong(checkAtomic(addr, 8), mask);
+ }
+
+ @Override
+ public long atomicXchgLong(int addr, long value) {
+ return atomics.xchgLong(checkAtomic(addr, 8), value);
+ }
+
+ @Override
+ public long atomicCmpxchgLong(int addr, long expected, long replacement) {
+ return atomics.cmpxchgLong(checkAtomic(addr, 8), expected, replacement);
+ }
+
+ @Override
+ public short atomicAddShort(int addr, short delta) {
+ return atomics.addShort(checkAtomic(addr, 2), delta);
+ }
+
+ @Override
+ public short atomicAndShort(int addr, short mask) {
+ return atomics.andShort(checkAtomic(addr, 2), mask);
+ }
+
+ @Override
+ public short atomicOrShort(int addr, short mask) {
+ return atomics.orShort(checkAtomic(addr, 2), mask);
+ }
+
+ @Override
+ public short atomicXorShort(int addr, short mask) {
+ return atomics.xorShort(checkAtomic(addr, 2), mask);
+ }
+
+ @Override
+ public short atomicXchgShort(int addr, short value) {
+ return atomics.xchgShort(checkAtomic(addr, 2), value);
+ }
+
+ @Override
+ public short atomicCmpxchgShort(int addr, short expected, short replacement) {
+ return atomics.cmpxchgShort(checkAtomic(addr, 2), expected, replacement);
+ }
+
+ @Override
+ public byte atomicAddByte(int addr, byte delta) {
+ return atomics.addByte(checkAtomic(addr, 1), delta);
+ }
+
+ @Override
+ public byte atomicAndByte(int addr, byte mask) {
+ return atomics.andByte(checkAtomic(addr, 1), mask);
+ }
+
+ @Override
+ public byte atomicOrByte(int addr, byte mask) {
+ return atomics.orByte(checkAtomic(addr, 1), mask);
+ }
+
+ @Override
+ public byte atomicXorByte(int addr, byte mask) {
+ return atomics.xorByte(checkAtomic(addr, 1), mask);
+ }
+
+ @Override
+ public byte atomicXchgByte(int addr, byte value) {
+ return atomics.xchgByte(checkAtomic(addr, 1), value);
+ }
+
+ @Override
+ public byte atomicCmpxchgByte(int addr, byte expected, byte replacement) {
+ return atomics.cmpxchgByte(checkAtomic(addr, 1), expected, replacement);
+ }
}
diff --git a/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/PanamaExecutor.java b/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/PanamaExecutor.java
index fca696c58..451aae742 100644
--- a/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/PanamaExecutor.java
+++ b/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/PanamaExecutor.java
@@ -29,6 +29,8 @@ private PanamaExecutor() {}
private static final java.lang.invoke.MethodHandle MMAP;
private static final java.lang.invoke.MethodHandle MPROTECT;
private static final java.lang.invoke.MethodHandle MUNMAP;
+ private static final java.lang.invoke.MethodHandle MALLOC;
+ private static final java.lang.invoke.MethodHandle FREE;
// POSIX constants
private static final int PROT_READ = 0x1;
@@ -60,6 +62,14 @@ private PanamaExecutor() {}
var lookup = LINKER.defaultLookup();
MEMMOVE_ADDR = lookup.find("memmove").orElseThrow().address();
MEMSET_ADDR = lookup.find("memset").orElseThrow().address();
+ MALLOC =
+ LINKER.downcallHandle(
+ lookup.find("malloc").orElseThrow(),
+ FunctionDescriptor.of(ValueLayout.ADDRESS, ValueLayout.JAVA_LONG));
+ FREE =
+ LINKER.downcallHandle(
+ lookup.find("free").orElseThrow(),
+ FunctionDescriptor.ofVoid(ValueLayout.ADDRESS));
try {
if (IS_WINDOWS) {
@@ -129,6 +139,18 @@ private PanamaExecutor() {}
}
}
+ static MemorySegment malloc(long size) throws Throwable {
+ MemorySegment addr = (MemorySegment) MALLOC.invokeExact(size);
+ if (addr.address() == 0) {
+ throw new OutOfMemoryError("malloc failed for " + size + " bytes");
+ }
+ return addr.reinterpret(size);
+ }
+
+ static void free(MemorySegment addr) throws Throwable {
+ FREE.invokeExact(addr);
+ }
+
/** Allocate a writable memory region. Must call mprotectExec before executing. */
static MemorySegment mmapCode(long size) throws Throwable {
if (IS_WINDOWS) {
diff --git a/wasm-corpus/src/main/resources/compiled/atomic-counter.wat.wasm b/wasm-corpus/src/main/resources/compiled/atomic-counter.wat.wasm
new file mode 100644
index 000000000..7e83ee415
Binary files /dev/null and b/wasm-corpus/src/main/resources/compiled/atomic-counter.wat.wasm differ
diff --git a/wasm-corpus/src/main/resources/compiled/host-grow.wat.wasm b/wasm-corpus/src/main/resources/compiled/host-grow.wat.wasm
new file mode 100644
index 000000000..cafeb7df2
Binary files /dev/null and b/wasm-corpus/src/main/resources/compiled/host-grow.wat.wasm differ
diff --git a/wasm-corpus/src/main/resources/compiled/memory-init-trap.wat.wasm b/wasm-corpus/src/main/resources/compiled/memory-init-trap.wat.wasm
new file mode 100644
index 000000000..c7dad3abc
Binary files /dev/null and b/wasm-corpus/src/main/resources/compiled/memory-init-trap.wat.wasm differ
diff --git a/wasm-corpus/src/main/resources/compiled/shared-memory-grow.wat.wasm b/wasm-corpus/src/main/resources/compiled/shared-memory-grow.wat.wasm
new file mode 100644
index 000000000..f431825fc
Binary files /dev/null and b/wasm-corpus/src/main/resources/compiled/shared-memory-grow.wat.wasm differ
diff --git a/wasm-corpus/src/main/resources/wat/atomic-counter.wat b/wasm-corpus/src/main/resources/wat/atomic-counter.wat
new file mode 100644
index 000000000..2cf74b478
--- /dev/null
+++ b/wasm-corpus/src/main/resources/wat/atomic-counter.wat
@@ -0,0 +1,25 @@
+;; Counters bumped with atomics, to race against host atomics on the same addresses.
+(module
+ (import "env" "memory" (memory 1 1 shared))
+
+ (func (export "add32") (param $n i32)
+ (loop $l
+ (drop (i32.atomic.rmw.add (i32.const 0) (i32.const 1)))
+ (br_if $l (local.tee $n (i32.sub (local.get $n) (i32.const 1))))))
+
+ ;; upper half of the int at 8
+ (func (export "add16") (param $n i32)
+ (loop $l
+ (drop (i32.atomic.rmw16.add_u (i32.const 10) (i32.const 1)))
+ (br_if $l (local.tee $n (i32.sub (local.get $n) (i32.const 1))))))
+
+ ;; second byte of the int at 12
+ (func (export "add8") (param $n i32)
+ (loop $l
+ (drop (i32.atomic.rmw8.add_u (i32.const 13) (i32.const 1)))
+ (br_if $l (local.tee $n (i32.sub (local.get $n) (i32.const 1))))))
+
+ (func (export "add64") (param $n i32)
+ (loop $l
+ (drop (i64.atomic.rmw.add (i32.const 16) (i64.const 1)))
+ (br_if $l (local.tee $n (i32.sub (local.get $n) (i32.const 1)))))))
diff --git a/wasm-corpus/src/main/resources/wat/host-grow.wat b/wasm-corpus/src/main/resources/wat/host-grow.wat
new file mode 100644
index 000000000..4c8c42c3d
--- /dev/null
+++ b/wasm-corpus/src/main/resources/wat/host-grow.wat
@@ -0,0 +1,12 @@
+;; A host import grows the memory, and the guest then uses the new page.
+(module
+ (import "env" "memory" (memory 1 2))
+ (import "env" "grow" (func $grow))
+
+ (func (export "growThenRead") (param $addr i32) (result i32)
+ (call $grow)
+ (i32.load (local.get $addr)))
+
+ (func (export "growThenSize") (result i32)
+ (call $grow)
+ (memory.size)))
diff --git a/wasm-corpus/src/main/resources/wat/memory-init-trap.wat b/wasm-corpus/src/main/resources/wat/memory-init-trap.wat
new file mode 100644
index 000000000..416ad60b9
--- /dev/null
+++ b/wasm-corpus/src/main/resources/wat/memory-init-trap.wat
@@ -0,0 +1,22 @@
+;; A memory.init that traps must stop execution.
+(module
+ (memory 1)
+ (data $d "hello")
+
+ ;; 6 bytes from a 5 byte segment: traps before the store
+ (func (export "initOutOfBoundsThenStore")
+ (memory.init $d (i32.const 0) (i32.const 0) (i32.const 6))
+ (i32.store (i32.const 100) (i32.const 1)))
+
+ ;; source offset 1 of a dropped (empty) segment: traps even with length 0
+ (func (export "dropThenInitPastEnd")
+ (data.drop $d)
+ (memory.init $d (i32.const 0) (i32.const 1) (i32.const 0)))
+
+ ;; offset 0 and length 0 of a dropped segment: allowed
+ (func (export "dropThenInitNothing")
+ (data.drop $d)
+ (memory.init $d (i32.const 0) (i32.const 0) (i32.const 0)))
+
+ (func (export "read") (param i32) (result i32)
+ (i32.load (local.get 0))))
diff --git a/wasm-corpus/src/main/resources/wat/shared-memory-grow.wat b/wasm-corpus/src/main/resources/wat/shared-memory-grow.wat
new file mode 100644
index 000000000..4f3be87dd
--- /dev/null
+++ b/wasm-corpus/src/main/resources/wat/shared-memory-grow.wat
@@ -0,0 +1,25 @@
+;; Two instances share this memory: one grows it while the other is already running.
+(module
+ (import "env" "memory" (memory 1 2 shared))
+
+ ;; Spins until the flag is set, then reads an address in the page grown meanwhile.
+ (func (export "waitThenRead") (param $flag i32) (param $addr i32) (result i32)
+ (block $done
+ (loop $spin
+ (br_if $done (i32.atomic.load (local.get $flag)))
+ (br $spin)))
+ (i32.load (local.get $addr)))
+
+ ;; Spins until the flag is set, then returns the memory size.
+ (func (export "waitThenSize") (param $flag i32) (result i32)
+ (block $done
+ (loop $spin
+ (br_if $done (i32.atomic.load (local.get $flag)))
+ (br $spin)))
+ (memory.size))
+
+ ;; Grows by one page, writes the value in it, then sets the flag.
+ (func (export "growAndPublish") (param $flag i32) (param $addr i32) (param $value i32)
+ (drop (memory.grow (i32.const 1)))
+ (i32.store (local.get $addr) (local.get $value))
+ (i32.atomic.store (local.get $flag) (i32.const 1))))